feat: refactor indexing logic by introducing runDocumentIndex and runFAQIndex helper methods

2026-04-13 20:10:50 +08:00
parent 4cd31ce454
commit 2a5629b9f1
2 changed files with 99 additions and 80 deletions
@@ -0,0 +1,96 @@
+package rag
+
+import (
+	"context"
+	"fmt"
+
+	"cs-agent/internal/ai"
+	"cs-agent/internal/ai/rag/vectordb"
+	"cs-agent/internal/models"
+	"cs-agent/internal/repositories"
+
+	"github.com/mlogclub/simple/common/strs"
+	"github.com/mlogclub/simple/sqls"
+)
+
+func (s *index) runDocumentIndex(ctx context.Context, document *models.KnowledgeDocument, knowledgeBase *models.KnowledgeBase) ([]vectordb.Vector, int, error) {
+	existingChunks := repositories.KnowledgeChunkRepository.FindByDocumentID(sqls.DB(), document.ID)
+	chunks, err := s.buildDocumentChunks(ctx, document, knowledgeBase)
+	if err != nil {
+		return nil, 0, err
+	}
+
+	collectionName := s.getCollectionName()
+	provider := vectordb.GetProvider()
+	if provider == nil {
+		return nil, 0, fmt.Errorf("vectordb provider not initialized")
+	}
+	if _, err := ai.Embedding.GetModel(ctx); err != nil {
+		return nil, 0, fmt.Errorf("failed to get embedding model: %w", err)
+	}
+
+	existingVectorIDs := collectExistingVectorIDs(existingChunks)
+	vectors, chunkModels, dimension, err := s.prepareDocumentVectors(ctx, knowledgeBase, document, chunks)
+	if err != nil {
+		return nil, 0, err
+	}
+	if err := s.ensureCollection(ctx, provider, collectionName, dimension); err != nil {
+		return nil, 0, err
+	}
+	if len(existingVectorIDs) > 0 {
+		if err := provider.DeleteVectors(ctx, collectionName, existingVectorIDs); err != nil {
+			return nil, 0, fmt.Errorf("failed to delete old vectors: %w", err)
+		}
+	}
+	if err := provider.UpsertVectors(ctx, collectionName, vectors); err != nil {
+		return nil, 0, fmt.Errorf("failed to upsert vectors: %w", err)
+	}
+	if err := s.replaceDocumentChunks(document.ID, chunkModels); err != nil {
+		return nil, 0, fmt.Errorf("failed to save chunks: %w", err)
+	}
+	return vectors, len(chunks), nil
+}
+
+func (s *index) runFAQIndex(ctx context.Context, faq *models.KnowledgeFAQ, knowledgeBase *models.KnowledgeBase) error {
+	existingChunks := repositories.KnowledgeChunkRepository.FindByFaqID(sqls.DB(), faq.ID)
+	content := buildFAQChunkContent(faq)
+	if content == "" {
+		return fmt.Errorf("faq content is empty")
+	}
+
+	provider := vectordb.GetProvider()
+	if provider == nil {
+		return fmt.Errorf("vectordb provider not initialized")
+	}
+	if _, err := ai.Embedding.GetModel(ctx); err != nil {
+		return fmt.Errorf("failed to get embedding model: %w", err)
+	}
+	vector, chunkModel, dimension, err := s.prepareFAQVector(ctx, knowledgeBase, faq, content)
+	if err != nil {
+		return err
+	}
+
+	collectionName := s.getCollectionName()
+	if err := s.ensureCollection(ctx, provider, collectionName, dimension); err != nil {
+		return err
+	}
+
+	existingVectorIDs := make([]string, 0, len(existingChunks))
+	for _, chunk := range existingChunks {
+		if strs.IsNotBlank(chunk.VectorID) {
+			existingVectorIDs = append(existingVectorIDs, chunk.VectorID)
+		}
+	}
+	if len(existingVectorIDs) > 0 {
+		if err := provider.DeleteVectors(ctx, collectionName, existingVectorIDs); err != nil {
+			return fmt.Errorf("failed to delete old vectors: %w", err)
+		}
+	}
+	if err := provider.UpsertVectors(ctx, collectionName, []vectordb.Vector{vector}); err != nil {
+		return fmt.Errorf("failed to upsert vectors: %w", err)
+	}
+	if err := s.replaceFAQChunk(faq.ID, &chunkModel); err != nil {
+		return fmt.Errorf("failed to save faq chunk: %w", err)
+	}
+	return nil
+}