feat(knowledge): add FAQ Excel import export

This commit is contained in:
mlogclub
2026-06-01 13:57:49 +08:00
parent d8692b45cd
commit 8933e963f5
15 changed files with 1133 additions and 363 deletions
+370
View File
@@ -0,0 +1,370 @@
package services
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"log/slog"
"path/filepath"
"strings"
"time"
"agent-desk/internal/ai/rag"
"agent-desk/internal/models"
"agent-desk/internal/pkg/dto"
"agent-desk/internal/pkg/dto/request"
"agent-desk/internal/pkg/dto/response"
"agent-desk/internal/pkg/enums"
"agent-desk/internal/pkg/errorsx"
"agent-desk/internal/pkg/utils"
"agent-desk/internal/repositories"
"github.com/mlogclub/simple/sqls"
"github.com/xuri/excelize/v2"
)
const (
knowledgeFAQExcelSheetName = "FAQ"
knowledgeFAQExcelContentType = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
)
var indexKnowledgeFAQAfterImport = func(faqID int64) error {
return rag.Index.IndexFAQByID(context.Background(), faqID)
}
type knowledgeFAQImportRow struct {
RowNo int
Question string
Answer string
SimilarQuestions []string
Remark string
}
func (s *knowledgeFAQService) BuildKnowledgeFAQImportTemplate() (*response.KnowledgeFAQExportedFile, error) {
workbook := excelize.NewFile()
defer workbook.Close()
if err := writeKnowledgeFAQWorkbookHeader(workbook); err != nil {
return nil, err
}
_ = workbook.SetCellValue(knowledgeFAQExcelSheetName, "A2", "如何重置密码?")
_ = workbook.SetCellValue(knowledgeFAQExcelSheetName, "B2", "进入个人设置后点击重置密码。")
_ = workbook.SetCellValue(knowledgeFAQExcelSheetName, "C2", "忘记密码怎么办\n重置密码在哪里")
_ = workbook.SetCellValue(knowledgeFAQExcelSheetName, "D2", "账号")
return buildKnowledgeFAQExcelFile("knowledge-faq-import-template.xlsx", workbook)
}
func (s *knowledgeFAQService) ExportKnowledgeFAQs(knowledgeBaseID int64) (*response.KnowledgeFAQExportedFile, error) {
if _, err := s.requireFAQKnowledgeBase(knowledgeBaseID); err != nil {
return nil, err
}
list := repositories.KnowledgeFAQRepository.FindAllByKnowledgeBaseID(sqls.DB(), knowledgeBaseID)
workbook := excelize.NewFile()
defer workbook.Close()
if err := writeKnowledgeFAQWorkbookHeader(workbook); err != nil {
return nil, err
}
for index, item := range list {
rowNo := index + 2
_ = workbook.SetCellValue(knowledgeFAQExcelSheetName, fmt.Sprintf("A%d", rowNo), item.Question)
_ = workbook.SetCellValue(knowledgeFAQExcelSheetName, fmt.Sprintf("B%d", rowNo), item.Answer)
_ = workbook.SetCellValue(knowledgeFAQExcelSheetName, fmt.Sprintf("C%d", rowNo), strings.Join(decodeKnowledgeFAQSimilarQuestions(item.SimilarQuestions), "\n"))
_ = workbook.SetCellValue(knowledgeFAQExcelSheetName, fmt.Sprintf("D%d", rowNo), item.Remark)
}
filename := fmt.Sprintf("knowledge-faq-%d-%s.xlsx", knowledgeBaseID, time.Now().Format("20060102150405"))
return buildKnowledgeFAQExcelFile(filename, workbook)
}
func (s *knowledgeFAQService) ImportKnowledgeFAQs(req request.ImportKnowledgeFAQRequest, operator *dto.AuthPrincipal) (*response.KnowledgeFAQImportResult, error) {
if operator == nil {
return nil, errorsx.Unauthorized("未登录或登录已过期")
}
if req.Mode != request.KnowledgeFAQImportModeAppend && req.Mode != request.KnowledgeFAQImportModeOverwrite {
return nil, errorsx.InvalidParam("导入模式不合法")
}
if req.Reader == nil {
return nil, errorsx.InvalidParam("请选择导入文件")
}
if strings.ToLower(filepath.Ext(req.Filename)) != ".xlsx" {
return nil, errorsx.InvalidParam("仅支持.xlsx文件")
}
kb, err := s.requireFAQKnowledgeBase(req.KnowledgeBaseID)
if err != nil {
return nil, err
}
rows, result, err := parseKnowledgeFAQImportRows(req.Reader)
if err != nil {
return nil, err
}
if len(rows) == 0 {
return result, nil
}
questions := make([]string, 0, len(rows))
for _, row := range rows {
questions = append(questions, row.Question)
}
existingList := repositories.KnowledgeFAQRepository.FindByKnowledgeBaseIDAndQuestions(sqls.DB(), req.KnowledgeBaseID, questions)
existingMap := make(map[string]models.KnowledgeFAQ, len(existingList))
for _, item := range existingList {
if _, ok := existingMap[item.Question]; !ok {
existingMap[item.Question] = item
}
}
indexFAQIDs := make([]int64, 0, len(rows))
err = sqls.WithTransaction(func(ctx *sqls.TxContext) error {
for _, row := range rows {
existing, exists := existingMap[row.Question]
if exists && req.Mode == request.KnowledgeFAQImportModeAppend {
result.Skipped++
result.Errors = append(result.Errors, response.KnowledgeFAQImportError{Row: row.RowNo, Message: "标准问题已存在,已跳过"})
continue
}
similarQuestions, marshalErr := json.Marshal(row.SimilarQuestions)
if marshalErr != nil {
result.Failed++
result.Errors = append(result.Errors, response.KnowledgeFAQImportError{Row: row.RowNo, Message: "相似问格式不合法"})
continue
}
if exists {
if updateErr := repositories.KnowledgeFAQRepository.Updates(ctx.Tx, existing.ID, map[string]any{
"answer": row.Answer,
"similar_questions": string(similarQuestions),
"index_status": enums.KnowledgeDocumentIndexStatusPending,
"indexed_at": nil,
"index_error": "",
"remark": row.Remark,
"update_user_id": operator.UserID,
"update_user_name": operator.Username,
"updated_at": time.Now(),
}); updateErr != nil {
return updateErr
}
result.Updated++
indexFAQIDs = append(indexFAQIDs, existing.ID)
continue
}
item := &models.KnowledgeFAQ{
KnowledgeBaseID: req.KnowledgeBaseID,
Question: row.Question,
Answer: row.Answer,
SimilarQuestions: string(similarQuestions),
Status: kb.Status,
IndexStatus: enums.KnowledgeDocumentIndexStatusPending,
IndexError: "",
IndexedAt: nil,
Remark: row.Remark,
AuditFields: utils.BuildAuditFields(operator),
}
if createErr := repositories.KnowledgeFAQRepository.Create(ctx.Tx, item); createErr != nil {
return createErr
}
result.Created++
indexFAQIDs = append(indexFAQIDs, item.ID)
}
return nil
})
if err != nil {
return nil, err
}
for _, faqID := range indexFAQIDs {
if indexErr := indexKnowledgeFAQAfterImport(faqID); indexErr != nil {
slog.Error("failed to index imported knowledge faq", "faq_id", faqID, "error", indexErr)
}
}
return result, nil
}
func writeKnowledgeFAQWorkbookHeader(workbook *excelize.File) error {
sheet := workbook.GetSheetName(0)
if sheet == "" {
sheet = knowledgeFAQExcelSheetName
workbook.NewSheet(sheet)
} else if sheet != knowledgeFAQExcelSheetName {
if err := workbook.SetSheetName(sheet, knowledgeFAQExcelSheetName); err != nil {
return err
}
}
headers := []string{"标准问题", "答案", "相似问", "备注"}
for index, header := range headers {
cell, err := excelize.CoordinatesToCellName(index+1, 1)
if err != nil {
return err
}
if err := workbook.SetCellValue(knowledgeFAQExcelSheetName, cell, header); err != nil {
return err
}
}
style, err := workbook.NewStyle(&excelize.Style{
Font: &excelize.Font{Bold: true},
Alignment: &excelize.Alignment{WrapText: true, Vertical: "top"},
})
if err == nil {
_ = workbook.SetCellStyle(knowledgeFAQExcelSheetName, "A1", "D1", style)
}
_ = workbook.SetColWidth(knowledgeFAQExcelSheetName, "A", "A", 36)
_ = workbook.SetColWidth(knowledgeFAQExcelSheetName, "B", "B", 60)
_ = workbook.SetColWidth(knowledgeFAQExcelSheetName, "C", "C", 40)
_ = workbook.SetColWidth(knowledgeFAQExcelSheetName, "D", "D", 24)
return nil
}
func buildKnowledgeFAQExcelFile(filename string, workbook *excelize.File) (*response.KnowledgeFAQExportedFile, error) {
var buf bytes.Buffer
if err := workbook.Write(&buf); err != nil {
return nil, err
}
return &response.KnowledgeFAQExportedFile{
Filename: filename,
ContentType: knowledgeFAQExcelContentType,
Data: buf.Bytes(),
}, nil
}
func parseKnowledgeFAQImportRows(reader io.Reader) ([]knowledgeFAQImportRow, *response.KnowledgeFAQImportResult, error) {
workbook, err := excelize.OpenReader(reader)
if err != nil {
return nil, nil, errorsx.InvalidParam("Excel文件解析失败")
}
defer workbook.Close()
sheet := knowledgeFAQExcelSheetName
if index, _ := workbook.GetSheetIndex(sheet); index < 0 {
sheets := workbook.GetSheetList()
if len(sheets) == 0 {
return nil, nil, errorsx.InvalidParam("Excel文件为空")
}
sheet = sheets[0]
}
table, err := workbook.GetRows(sheet)
if err != nil {
return nil, nil, errorsx.InvalidParam("Excel文件读取失败")
}
if len(table) == 0 {
return nil, nil, errorsx.InvalidParam("Excel文件为空")
}
headerMap := buildKnowledgeFAQHeaderMap(table[0])
if _, ok := headerMap["question"]; !ok {
return nil, nil, errorsx.InvalidParam("缺少标准问题列")
}
if _, ok := headerMap["answer"]; !ok {
return nil, nil, errorsx.InvalidParam("缺少答案列")
}
result := &response.KnowledgeFAQImportResult{Errors: make([]response.KnowledgeFAQImportError, 0)}
rows := make([]knowledgeFAQImportRow, 0, len(table)-1)
seen := make(map[string]int)
for index := 1; index < len(table); index++ {
current := table[index]
if isKnowledgeFAQExcelRowEmpty(current) {
continue
}
result.Total++
rowNo := index + 1
row := knowledgeFAQImportRow{
RowNo: rowNo,
Question: getKnowledgeFAQExcelCell(current, headerMap["question"]),
Answer: getKnowledgeFAQExcelCell(current, headerMap["answer"]),
SimilarQuestions: parseKnowledgeFAQSimilarQuestions(getKnowledgeFAQExcelCell(current, headerMap["similarQuestions"])),
Remark: getKnowledgeFAQExcelCell(current, headerMap["remark"]),
}
if row.Question == "" {
result.Failed++
result.Errors = append(result.Errors, response.KnowledgeFAQImportError{Row: rowNo, Message: "问题不能为空"})
continue
}
if len([]rune(row.Question)) > 500 {
result.Failed++
result.Errors = append(result.Errors, response.KnowledgeFAQImportError{Row: rowNo, Message: "问题不能超过500字"})
continue
}
if row.Answer == "" {
result.Failed++
result.Errors = append(result.Errors, response.KnowledgeFAQImportError{Row: rowNo, Message: "答案不能为空"})
continue
}
if firstRow, exists := seen[row.Question]; exists {
result.Failed++
result.Errors = append(result.Errors, response.KnowledgeFAQImportError{Row: rowNo, Message: fmt.Sprintf("同一文件中标准问题重复,首次出现于第%d行", firstRow)})
continue
}
seen[row.Question] = rowNo
rows = append(rows, row)
}
return rows, result, nil
}
func buildKnowledgeFAQHeaderMap(headerRow []string) map[string]int {
accepted := map[string]string{
"标准问题": "question",
"问题": "question",
"question": "question",
"答案": "answer",
"answer": "answer",
"相似问": "similarQuestions",
"相似问题": "similarQuestions",
"similarquestions": "similarQuestions",
"备注": "remark",
"remark": "remark",
}
result := make(map[string]int)
for index, header := range headerRow {
key := strings.ToLower(strings.TrimSpace(strings.TrimPrefix(header, "\uFEFF")))
if field, ok := accepted[key]; ok {
if _, exists := result[field]; !exists {
result[field] = index
}
}
}
return result
}
func getKnowledgeFAQExcelCell(row []string, index int) string {
if index < 0 || index >= len(row) {
return ""
}
return strings.TrimSpace(row[index])
}
func isKnowledgeFAQExcelRowEmpty(row []string) bool {
for _, cell := range row {
if strings.TrimSpace(cell) != "" {
return false
}
}
return true
}
func parseKnowledgeFAQSimilarQuestions(value string) []string {
lines := strings.Split(strings.ReplaceAll(value, "\r\n", "\n"), "\n")
items := make([]string, 0, len(lines))
seen := make(map[string]struct{}, len(lines))
for _, line := range lines {
item := strings.TrimSpace(strings.ReplaceAll(line, "\r", ""))
if item == "" {
continue
}
if _, ok := seen[item]; ok {
continue
}
seen[item] = struct{}{}
items = append(items, item)
}
return items
}
func decodeKnowledgeFAQSimilarQuestions(value string) []string {
var items []string
if strings.TrimSpace(value) == "" {
return nil
}
if err := json.Unmarshal([]byte(value), &items); err != nil {
return nil
}
return items
}
@@ -0,0 +1,249 @@
package services
import (
"bytes"
"encoding/json"
"testing"
"agent-desk/internal/models"
"agent-desk/internal/pkg/dto"
"agent-desk/internal/pkg/dto/request"
"agent-desk/internal/pkg/enums"
"agent-desk/internal/repositories"
"github.com/glebarez/sqlite"
"github.com/mlogclub/simple/sqls"
"github.com/xuri/excelize/v2"
"gorm.io/gorm"
)
func TestKnowledgeFAQExcelTemplateContainsExpectedHeaders(t *testing.T) {
file, err := KnowledgeFAQService.BuildKnowledgeFAQImportTemplate()
if err != nil {
t.Fatalf("BuildKnowledgeFAQImportTemplate() error = %v", err)
}
workbook, err := excelize.OpenReader(bytes.NewReader(file.Data))
if err != nil {
t.Fatalf("open template workbook: %v", err)
}
defer workbook.Close()
rows, err := workbook.GetRows("FAQ")
if err != nil {
t.Fatalf("get FAQ rows: %v", err)
}
if len(rows) == 0 {
t.Fatal("template has no rows")
}
want := []string{"标准问题", "答案", "相似问", "备注"}
for i, value := range want {
if rows[0][i] != value {
t.Fatalf("header %d = %q, want %q", i, rows[0][i], value)
}
}
}
func TestKnowledgeFAQExcelExportWritesSimilarQuestionsWithLineBreaks(t *testing.T) {
setupKnowledgeFAQExcelTestDB(t)
kb := createKnowledgeFAQExcelTestBase(t, enums.StatusOk)
similarQuestions, _ := json.Marshal([]string{"怎么重置密码", "忘记密码怎么办"})
if err := repositories.KnowledgeFAQRepository.Create(sqls.DB(), &models.KnowledgeFAQ{
KnowledgeBaseID: kb.ID,
Question: "如何重置密码?",
Answer: "进入个人设置后点击重置密码。",
SimilarQuestions: string(similarQuestions),
Status: enums.StatusOk,
IndexStatus: enums.KnowledgeDocumentIndexStatusPending,
Remark: "账号",
}); err != nil {
t.Fatalf("create faq: %v", err)
}
file, err := KnowledgeFAQService.ExportKnowledgeFAQs(kb.ID)
if err != nil {
t.Fatalf("ExportKnowledgeFAQs() error = %v", err)
}
workbook, err := excelize.OpenReader(bytes.NewReader(file.Data))
if err != nil {
t.Fatalf("open export workbook: %v", err)
}
defer workbook.Close()
cell, err := workbook.GetCellValue("FAQ", "C2")
if err != nil {
t.Fatalf("get C2: %v", err)
}
if cell != "怎么重置密码\n忘记密码怎么办" {
t.Fatalf("similar questions cell = %q", cell)
}
}
func TestImportKnowledgeFAQsAppendSkipsExistingQuestion(t *testing.T) {
setupKnowledgeFAQExcelTestDB(t)
restore := stubKnowledgeFAQImportIndex(t)
defer restore()
kb := createKnowledgeFAQExcelTestBase(t, enums.StatusOk)
if err := repositories.KnowledgeFAQRepository.Create(sqls.DB(), &models.KnowledgeFAQ{
KnowledgeBaseID: kb.ID,
Question: "如何重置密码?",
Answer: "旧答案",
Status: enums.StatusOk,
IndexStatus: enums.KnowledgeDocumentIndexStatusIndexed,
}); err != nil {
t.Fatalf("create faq: %v", err)
}
result, err := KnowledgeFAQService.ImportKnowledgeFAQs(request.ImportKnowledgeFAQRequest{
KnowledgeBaseID: kb.ID,
Mode: request.KnowledgeFAQImportModeAppend,
Filename: "faq.xlsx",
Reader: bytes.NewReader(buildKnowledgeFAQExcelWorkbook(t, [][]string{
{"标准问题", "答案", "相似问", "备注"},
{"如何重置密码?", "新答案", "重置密码在哪里\n忘记密码", "账号"},
{"支持哪些渠道?", "支持网页和企业微信。", "", "渠道"},
})),
}, knowledgeFAQExcelTestOperator())
if err != nil {
t.Fatalf("ImportKnowledgeFAQs() error = %v", err)
}
if result.Created != 1 || result.Updated != 0 || result.Skipped != 1 || result.Failed != 0 {
t.Fatalf("result = %+v", result)
}
list := repositories.KnowledgeFAQRepository.Find(sqls.DB(), sqls.NewCnd().Eq("knowledge_base_id", kb.ID))
if len(list) != 2 {
t.Fatalf("faq count = %d, want 2", len(list))
}
existing := repositories.KnowledgeFAQRepository.Find(sqls.DB(), sqls.NewCnd().Eq("question", "如何重置密码?"))
if len(existing) != 1 || existing[0].Answer != "旧答案" {
t.Fatalf("existing faq changed: %+v", existing)
}
}
func TestImportKnowledgeFAQsOverwriteUpdatesExistingQuestion(t *testing.T) {
setupKnowledgeFAQExcelTestDB(t)
restore := stubKnowledgeFAQImportIndex(t)
defer restore()
kb := createKnowledgeFAQExcelTestBase(t, enums.StatusDisabled)
if err := repositories.KnowledgeFAQRepository.Create(sqls.DB(), &models.KnowledgeFAQ{
KnowledgeBaseID: kb.ID,
Question: "如何重置密码?",
Answer: "旧答案",
Status: enums.StatusOk,
IndexStatus: enums.KnowledgeDocumentIndexStatusIndexed,
}); err != nil {
t.Fatalf("create faq: %v", err)
}
result, err := KnowledgeFAQService.ImportKnowledgeFAQs(request.ImportKnowledgeFAQRequest{
KnowledgeBaseID: kb.ID,
Mode: request.KnowledgeFAQImportModeOverwrite,
Filename: "faq.xlsx",
Reader: bytes.NewReader(buildKnowledgeFAQExcelWorkbook(t, [][]string{
{"标准问题", "答案", "相似问", "备注"},
{"如何重置密码?", "新答案", "重置密码在哪里\n重置密码在哪里\n忘记密码", "账号"},
{"支持哪些渠道?", "支持网页和企业微信。", "", "渠道"},
})),
}, knowledgeFAQExcelTestOperator())
if err != nil {
t.Fatalf("ImportKnowledgeFAQs() error = %v", err)
}
if result.Created != 1 || result.Updated != 1 || result.Skipped != 0 || result.Failed != 0 {
t.Fatalf("result = %+v", result)
}
updated := repositories.KnowledgeFAQRepository.Find(sqls.DB(), sqls.NewCnd().Eq("question", "如何重置密码?"))
if len(updated) != 1 {
t.Fatalf("updated faq count = %d", len(updated))
}
if updated[0].Answer != "新答案" {
t.Fatalf("updated answer = %q", updated[0].Answer)
}
if updated[0].IndexStatus != enums.KnowledgeDocumentIndexStatusPending {
t.Fatalf("index status = %q", updated[0].IndexStatus)
}
var similar []string
if err := json.Unmarshal([]byte(updated[0].SimilarQuestions), &similar); err != nil {
t.Fatalf("unmarshal similar questions: %v", err)
}
if len(similar) != 2 || similar[0] != "重置密码在哪里" || similar[1] != "忘记密码" {
t.Fatalf("similar questions = %#v", similar)
}
created := repositories.KnowledgeFAQRepository.Find(sqls.DB(), sqls.NewCnd().Eq("question", "支持哪些渠道?"))
if len(created) != 1 || created[0].Status != enums.StatusDisabled {
t.Fatalf("created faq = %+v", created)
}
}
func setupKnowledgeFAQExcelTestDB(t *testing.T) {
t.Helper()
db, err := gorm.Open(sqlite.Open("file:"+t.Name()+"?mode=memory&cache=shared"), &gorm.Config{})
if err != nil {
t.Fatalf("open sqlite db: %v", err)
}
if err := db.AutoMigrate(&models.KnowledgeBase{}, &models.KnowledgeFAQ{}, &models.KnowledgeChunk{}); err != nil {
t.Fatalf("auto migrate: %v", err)
}
sqls.SetDB(db)
}
func createKnowledgeFAQExcelTestBase(t *testing.T, status enums.Status) *models.KnowledgeBase {
t.Helper()
item := &models.KnowledgeBase{
Name: "FAQ",
KnowledgeType: string(enums.KnowledgeBaseTypeFAQ),
Status: status,
}
if err := repositories.KnowledgeBaseRepository.Create(sqls.DB(), item); err != nil {
t.Fatalf("create knowledge base: %v", err)
}
return item
}
func buildKnowledgeFAQExcelWorkbook(t *testing.T, rows [][]string) []byte {
t.Helper()
workbook := excelize.NewFile()
defer workbook.Close()
sheet := workbook.GetSheetName(0)
if err := workbook.SetSheetName(sheet, "FAQ"); err != nil {
t.Fatalf("set sheet name: %v", err)
}
for rowIndex, row := range rows {
for colIndex, value := range row {
cell, err := excelize.CoordinatesToCellName(colIndex+1, rowIndex+1)
if err != nil {
t.Fatalf("cell name: %v", err)
}
if err := workbook.SetCellValue("FAQ", cell, value); err != nil {
t.Fatalf("set cell value: %v", err)
}
}
}
var buf bytes.Buffer
if err := workbook.Write(&buf); err != nil {
t.Fatalf("write workbook: %v", err)
}
return buf.Bytes()
}
func knowledgeFAQExcelTestOperator() *dto.AuthPrincipal {
return &dto.AuthPrincipal{UserID: 1, Username: "admin"}
}
func stubKnowledgeFAQImportIndex(t *testing.T) func() {
t.Helper()
original := indexKnowledgeFAQAfterImport
indexed := make([]int64, 0)
indexKnowledgeFAQAfterImport = func(faqID int64) error {
indexed = append(indexed, faqID)
return nil
}
return func() {
if len(indexed) == 0 {
t.Fatalf("expected imported FAQs to be indexed")
}
indexKnowledgeFAQAfterImport = original
}
}