refactor: 将客服后端重构为宿主可嵌入模块

- 注入数据库、运行时配置、统一响应、文件存储和平台 AI 能力，补充业务读写工具与客户快捷操作契约。 - 移除模块内重复的组织、客户、工单、标签、技能、旧工作流、MCP 和迁移实现，将身份权限与业务主体交由宿主管理。 - 使用 libSQL 重构向量存储，并完善图片消息、访客身份、排队调度、企业微信和支持聊天页面。 - 统一 HTTP、DTO 与 WebSocket 的 snake_case 协议，补齐模块初始化、业务动作和公共载荷等回归测试。
2026-08-28 22:23:13 +08:00
parent 6845c728f8
commit 18c9354095
377 changed files with 13199 additions and 22881 deletions
@@ -0,0 +1,414 @@
+package vectordb
+
+import (
+	"context"
+	"database/sql"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+	"regexp"
+	"strings"
+
+	"code.tczkiot.com/wlw/ai-agent/internal/pkg/config"
+
+	turso "turso.tech/database/tursogo"
+)
+
+const (
+	defaultLibSQLPath = "data/agent/vectors.db"
+	defaultSearchTopK = 10
+	busyTimeoutMillis = 5000
+)
+
+var collectionNamePattern = regexp.MustCompile(`^[A-Za-z][A-Za-z0-9_]*$`)
+
+type LibSQLProvider struct {
+	db *sql.DB
+}
+
+func NewLibSQLProvider(cfg *config.VectorDBConfig) (*LibSQLProvider, error) {
+	if cfg == nil {
+		return nil, fmt.Errorf("libsql vector database config is required")
+	}
+	path := strings.TrimSpace(cfg.Path)
+	if path == "" {
+		path = defaultLibSQLPath
+	}
+	absPath, err := filepath.Abs(path)
+	if err != nil {
+		return nil, fmt.Errorf("resolve libsql vector database path: %w", err)
+	}
+	if err := os.MkdirAll(filepath.Dir(absPath), 0o755); err != nil {
+		return nil, fmt.Errorf("create libsql vector database directory: %w", err)
+	}
+
+	connector, err := turso.NewConnector(absPath, turso.WithBusyTimeout(busyTimeoutMillis))
+	if err != nil {
+		return nil, fmt.Errorf("create libsql vector database connector: %w", err)
+	}
+	db := sql.OpenDB(connector)
+	db.SetMaxOpenConns(1)
+	db.SetMaxIdleConns(1)
+	provider := &LibSQLProvider{db: db}
+	if err := provider.initialize(context.Background()); err != nil {
+		_ = db.Close()
+		return nil, err
+	}
+	return provider, nil
+}
+
+func (p *LibSQLProvider) initialize(ctx context.Context) error {
+	if p == nil || p.db == nil {
+		return fmt.Errorf("libsql vector database is closed")
+	}
+	if err := p.db.PingContext(ctx); err != nil {
+		return fmt.Errorf("connect to libsql vector database: %w", err)
+	}
+	_, err := p.db.ExecContext(ctx, `CREATE TABLE IF NOT EXISTS "_agent_vector_collections" (
+		name TEXT PRIMARY KEY NOT NULL,
+		dimension INTEGER NOT NULL,
+		created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
+	)`)
+	if err != nil {
+		return fmt.Errorf("initialize libsql collection registry: %w", err)
+	}
+	return nil
+}
+
+func (p *LibSQLProvider) Close() error {
+	if p == nil || p.db == nil {
+		return nil
+	}
+	err := p.db.Close()
+	p.db = nil
+	return err
+}
+
+func (p *LibSQLProvider) CreateCollection(ctx context.Context, name string, dimension int) error {
+	tableName, err := collectionIdentifier(name)
+	if err != nil {
+		return err
+	}
+	if dimension <= 0 || dimension > 65536 {
+		return fmt.Errorf("invalid libsql vector dimension: %d", dimension)
+	}
+	if info, getErr := p.GetCollection(ctx, name); getErr == nil {
+		if info.Dimension != dimension {
+			return fmt.Errorf("collection %s already uses dimension %d, requested %d", name, info.Dimension, dimension)
+		}
+		return nil
+	} else if !errors.Is(getErr, sql.ErrNoRows) {
+		return getErr
+	}
+
+	tx, err := p.db.BeginTx(ctx, nil)
+	if err != nil {
+		return fmt.Errorf("begin libsql collection transaction: %w", err)
+	}
+	defer func() { _ = tx.Rollback() }()
+
+	createTable := fmt.Sprintf(`CREATE TABLE IF NOT EXISTS %s (
+		id TEXT PRIMARY KEY NOT NULL,
+		embedding BLOB NOT NULL,
+		knowledge_base_id INTEGER NOT NULL DEFAULT 0,
+		document_id INTEGER NOT NULL DEFAULT 0,
+		document_title TEXT NOT NULL DEFAULT '',
+		faq_id INTEGER NOT NULL DEFAULT 0,
+		faq_question TEXT NOT NULL DEFAULT '',
+		chunk_no INTEGER NOT NULL DEFAULT 0,
+		chunk_type TEXT NOT NULL DEFAULT '',
+		section_path TEXT NOT NULL DEFAULT '',
+		title TEXT NOT NULL DEFAULT '',
+		content TEXT NOT NULL DEFAULT '',
+		provider TEXT NOT NULL DEFAULT ''
+	)`, tableName)
+	if _, err := tx.ExecContext(ctx, createTable); err != nil {
+		return fmt.Errorf("create libsql collection %s: %w", name, err)
+	}
+	if _, err := tx.ExecContext(ctx, fmt.Sprintf(
+		`CREATE INDEX IF NOT EXISTS %s ON %s (knowledge_base_id, document_id)`,
+		quoteIdentifier(name+"_payload_idx"), tableName,
+	)); err != nil {
+		return fmt.Errorf("create libsql payload index for %s: %w", name, err)
+	}
+	if _, err := tx.ExecContext(ctx,
+		`INSERT INTO "_agent_vector_collections" (name, dimension) VALUES (?, ?)`, name, dimension,
+	); err != nil {
+		return fmt.Errorf("register libsql collection %s: %w", name, err)
+	}
+	if err := tx.Commit(); err != nil {
+		return fmt.Errorf("commit libsql collection %s: %w", name, err)
+	}
+	return nil
+}
+
+func (p *LibSQLProvider) DeleteCollection(ctx context.Context, name string) error {
+	tableName, err := collectionIdentifier(name)
+	if err != nil {
+		return err
+	}
+	tx, err := p.db.BeginTx(ctx, nil)
+	if err != nil {
+		return fmt.Errorf("begin libsql collection transaction: %w", err)
+	}
+	defer func() { _ = tx.Rollback() }()
+	if _, err := tx.ExecContext(ctx, "DROP TABLE IF EXISTS "+tableName); err != nil {
+		return fmt.Errorf("drop libsql collection %s: %w", name, err)
+	}
+	if _, err := tx.ExecContext(ctx, `DELETE FROM "_agent_vector_collections" WHERE name = ?`, name); err != nil {
+		return fmt.Errorf("unregister libsql collection %s: %w", name, err)
+	}
+	if err := tx.Commit(); err != nil {
+		return fmt.Errorf("commit libsql collection deletion %s: %w", name, err)
+	}
+	return nil
+}
+
+func (p *LibSQLProvider) GetCollection(ctx context.Context, name string) (*CollectionInfo, error) {
+	tableName, err := collectionIdentifier(name)
+	if err != nil {
+		return nil, err
+	}
+	var dimension int
+	if err := p.db.QueryRowContext(ctx,
+		`SELECT dimension FROM "_agent_vector_collections" WHERE name = ?`, name,
+	).Scan(&dimension); err != nil {
+		return nil, err
+	}
+	var count int
+	if err := p.db.QueryRowContext(ctx, "SELECT COUNT(*) FROM "+tableName).Scan(&count); err != nil {
+		return nil, fmt.Errorf("count libsql collection %s: %w", name, err)
+	}
+	return &CollectionInfo{Name: name, Dimension: dimension, PointCount: count, Status: "ready"}, nil
+}
+
+func (p *LibSQLProvider) ListCollections(ctx context.Context) ([]string, error) {
+	rows, err := p.db.QueryContext(ctx, `SELECT name FROM "_agent_vector_collections" ORDER BY name`)
+	if err != nil {
+		return nil, fmt.Errorf("list libsql collections: %w", err)
+	}
+	defer rows.Close()
+	collections := make([]string, 0)
+	for rows.Next() {
+		var name string
+		if err := rows.Scan(&name); err != nil {
+			return nil, fmt.Errorf("scan libsql collection: %w", err)
+		}
+		collections = append(collections, name)
+	}
+	if err := rows.Err(); err != nil {
+		return nil, fmt.Errorf("iterate libsql collections: %w", err)
+	}
+	return collections, nil
+}
+
+func (p *LibSQLProvider) UpsertVectors(ctx context.Context, collectionName string, vectors []Vector) error {
+	if len(vectors) == 0 {
+		return nil
+	}
+	tableName, err := collectionIdentifier(collectionName)
+	if err != nil {
+		return err
+	}
+	info, err := p.GetCollection(ctx, collectionName)
+	if err != nil {
+		return fmt.Errorf("get libsql collection %s: %w", collectionName, err)
+	}
+
+	tx, err := p.db.BeginTx(ctx, nil)
+	if err != nil {
+		return fmt.Errorf("begin libsql vector upsert: %w", err)
+	}
+	defer func() { _ = tx.Rollback() }()
+	statement := fmt.Sprintf(`INSERT INTO %s (
+		id, embedding, knowledge_base_id, document_id, document_title,
+		faq_id, faq_question, chunk_no, chunk_type, section_path, title, content, provider
+	) VALUES (?, vector32(?), ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+	ON CONFLICT(id) DO UPDATE SET
+		embedding=excluded.embedding,
+		knowledge_base_id=excluded.knowledge_base_id,
+		document_id=excluded.document_id,
+		document_title=excluded.document_title,
+		faq_id=excluded.faq_id,
+		faq_question=excluded.faq_question,
+		chunk_no=excluded.chunk_no,
+		chunk_type=excluded.chunk_type,
+		section_path=excluded.section_path,
+		title=excluded.title,
+		content=excluded.content,
+		provider=excluded.provider`, tableName)
+	stmt, err := tx.PrepareContext(ctx, statement)
+	if err != nil {
+		return fmt.Errorf("prepare libsql vector upsert: %w", err)
+	}
+	defer stmt.Close()
+	for _, item := range vectors {
+		if strings.TrimSpace(item.ID) == "" {
+			return fmt.Errorf("libsql vector id is required")
+		}
+		if len(item.Vector) != info.Dimension {
+			return fmt.Errorf("invalid vector dimension for %s: got %d, want %d", item.ID, len(item.Vector), info.Dimension)
+		}
+		encoded, err := json.Marshal(item.Vector)
+		if err != nil {
+			return fmt.Errorf("encode vector %s: %w", item.ID, err)
+		}
+		payload := item.Payload
+		if _, err := stmt.ExecContext(ctx,
+			item.ID, string(encoded), payload.KnowledgeBaseID, payload.DocumentID, payload.DocumentTitle,
+			payload.FaqID, payload.FaqQuestion, payload.ChunkNo, payload.ChunkType,
+			payload.SectionPath, payload.Title, payload.Content, payload.Provider,
+		); err != nil {
+			return fmt.Errorf("upsert libsql vector %s: %w", item.ID, err)
+		}
+	}
+	if err := tx.Commit(); err != nil {
+		return fmt.Errorf("commit libsql vector upsert: %w", err)
+	}
+	return nil
+}
+
+func (p *LibSQLProvider) DeleteVectors(ctx context.Context, collectionName string, ids []string) error {
+	if len(ids) == 0 {
+		return nil
+	}
+	tableName, err := collectionIdentifier(collectionName)
+	if err != nil {
+		return err
+	}
+	tx, err := p.db.BeginTx(ctx, nil)
+	if err != nil {
+		return fmt.Errorf("begin libsql vector deletion: %w", err)
+	}
+	defer func() { _ = tx.Rollback() }()
+	stmt, err := tx.PrepareContext(ctx, "DELETE FROM "+tableName+" WHERE id = ?")
+	if err != nil {
+		return fmt.Errorf("prepare libsql vector deletion: %w", err)
+	}
+	defer stmt.Close()
+	for _, id := range ids {
+		if _, err := stmt.ExecContext(ctx, id); err != nil {
+			return fmt.Errorf("delete libsql vector %s: %w", id, err)
+		}
+	}
+	if err := tx.Commit(); err != nil {
+		return fmt.Errorf("commit libsql vector deletion: %w", err)
+	}
+	return nil
+}
+
+func (p *LibSQLProvider) Search(ctx context.Context, req *SearchRequest) ([]SearchResult, error) {
+	if req == nil {
+		return nil, fmt.Errorf("libsql search request is required")
+	}
+	tableName, err := collectionIdentifier(req.CollectionName)
+	if err != nil {
+		return nil, err
+	}
+	info, err := p.GetCollection(ctx, req.CollectionName)
+	if err != nil {
+		return nil, fmt.Errorf("get libsql collection %s: %w", req.CollectionName, err)
+	}
+	if len(req.Vector) != info.Dimension {
+		return nil, fmt.Errorf("invalid search vector dimension: got %d, want %d", len(req.Vector), info.Dimension)
+	}
+	topK := req.TopK
+	if topK <= 0 {
+		topK = defaultSearchTopK
+	}
+	encoded, err := json.Marshal(req.Vector)
+	if err != nil {
+		return nil, fmt.Errorf("encode search vector: %w", err)
+	}
+	vectorJSON := string(encoded)
+
+	filterSQL, filterArgs := buildSearchFilter(req.Filter)
+	innerWhere := filterSQL
+	innerArgs := []any{vectorJSON}
+	if filterSQL != "" {
+		innerArgs = append(innerArgs, filterArgs...)
+	}
+
+	query := fmt.Sprintf(`SELECT id, score, knowledge_base_id, document_id, document_title,
+		faq_id, faq_question, chunk_no, chunk_type, section_path, title, content, provider
+	FROM (
+		SELECT id, 1.0 - vector_distance_cos(embedding, vector32(?)) AS score,
+			knowledge_base_id, document_id, document_title, faq_id, faq_question,
+			chunk_no, chunk_type, section_path, title, content, provider
+		FROM %s%s
+	) ranked
+	WHERE score >= ?
+	ORDER BY score DESC
+	LIMIT ?`, tableName, innerWhere)
+	innerArgs = append(innerArgs, req.ScoreThreshold, topK)
+	rows, err := p.db.QueryContext(ctx, query, innerArgs...)
+	if err != nil {
+		return nil, fmt.Errorf("search libsql collection %s: %w", req.CollectionName, err)
+	}
+	defer rows.Close()
+
+	results := make([]SearchResult, 0, topK)
+	for rows.Next() {
+		var result SearchResult
+		if err := rows.Scan(
+			&result.ID, &result.Score,
+			&result.Payload.KnowledgeBaseID, &result.Payload.DocumentID, &result.Payload.DocumentTitle,
+			&result.Payload.FaqID, &result.Payload.FaqQuestion, &result.Payload.ChunkNo,
+			&result.Payload.ChunkType, &result.Payload.SectionPath, &result.Payload.Title,
+			&result.Payload.Content, &result.Payload.Provider,
+		); err != nil {
+			return nil, fmt.Errorf("scan libsql search result: %w", err)
+		}
+		results = append(results, result)
+	}
+	if err := rows.Err(); err != nil {
+		return nil, fmt.Errorf("iterate libsql search results: %w", err)
+	}
+	return results, nil
+}
+
+func collectionIdentifier(name string) (string, error) {
+	name = strings.TrimSpace(name)
+	if !collectionNamePattern.MatchString(name) {
+		return "", fmt.Errorf("invalid libsql collection name %q", name)
+	}
+	return quoteIdentifier(name), nil
+}
+
+func quoteIdentifier(value string) string {
+	return `"` + value + `"`
+}
+
+func buildSearchFilter(filter *SearchFilter) (string, []any) {
+	if filter == nil {
+		return "", nil
+	}
+	clauses := make([]string, 0, 2)
+	args := make([]any, 0, len(filter.KnowledgeBaseIDs)+len(filter.DocumentIDs))
+	if len(filter.KnowledgeBaseIDs) > 0 {
+		clauses = append(clauses, "knowledge_base_id IN ("+placeholders(len(filter.KnowledgeBaseIDs))+")")
+		for _, id := range filter.KnowledgeBaseIDs {
+			args = append(args, id)
+		}
+	}
+	if len(filter.DocumentIDs) > 0 {
+		clauses = append(clauses, "document_id IN ("+placeholders(len(filter.DocumentIDs))+")")
+		for _, id := range filter.DocumentIDs {
+			args = append(args, id)
+		}
+	}
+	if len(clauses) == 0 {
+		return "", nil
+	}
+	return " WHERE " + strings.Join(clauses, " AND "), args
+}
+
+func placeholders(count int) string {
+	values := make([]string, count)
+	for i := range values {
+		values[i] = "?"
+	}
+	return strings.Join(values, ",")
+}