ai-agent/internal/ai/rag/index_document_helpers.go

package rag

import (
	"context"
	"fmt"
	"log/slog"
	"time"

	ragchunk "code.tczkiot.com/wlw/ai-agent/internal/ai/rag/chunk"
	"code.tczkiot.com/wlw/ai-agent/internal/ai/rag/vectordb"
	"code.tczkiot.com/wlw/ai-agent/internal/models"
	"code.tczkiot.com/wlw/ai-agent/internal/pkg/enums"

	"code.tczkiot.com/wlw/ai-agent/internal/ai"
)

func (s *index) buildDocumentChunkRequest(document models.KnowledgeDocument, knowledgeBase models.KnowledgeBase) *ragchunk.ChunkRequest {
	return &ragchunk.ChunkRequest{
		KnowledgeBaseID: document.KnowledgeBaseID,
		DocumentID:      document.ID,
		DocumentTitle:   document.Title,
		ContentType:     document.ContentType,
		Content:         document.Content,
		PlainText:       ExtractPlainText(document.Content, document.ContentType),
		Options: ragchunk.ChunkOptions{
			Provider:       firstNonEmptyString(knowledgeBase.ChunkProvider, s.chunkConfig.Provider),
			TargetTokens:   firstPositiveInt(knowledgeBase.ChunkTargetTokens, s.chunkConfig.TargetTokens),
			MaxTokens:      firstPositiveInt(knowledgeBase.ChunkMaxTokens, s.chunkConfig.MaxTokens),
			OverlapTokens:  firstPositiveInt(knowledgeBase.ChunkOverlapTokens, s.chunkConfig.OverlapTokens),
			EnableFallback: s.chunkConfig.EnableFallback,
		},
	}
}

func (s *index) buildDocumentChunks(ctx context.Context, document models.KnowledgeDocument, knowledgeBase models.KnowledgeBase) ([]ragchunk.ChunkResult, error) {
	chunks, err := s.registry.Chunk(ctx, s.buildDocumentChunkRequest(document, knowledgeBase))
	if err != nil {
		return nil, fmt.Errorf("failed to chunk document: %w", err)
	}
	if len(chunks) == 0 {
		return nil, fmt.Errorf("no chunks generated from document")
	}
	return chunks, nil
}

func (s *index) prepareDocumentVectors(ctx context.Context, knowledgeBase models.KnowledgeBase, document models.KnowledgeDocument, chunks []ragchunk.ChunkResult) ([]vectordb.Vector, []models.KnowledgeChunk, int, error) {
	vectors := make([]vectordb.Vector, 0, len(chunks))
	chunkModels := make([]models.KnowledgeChunk, 0, len(chunks))
	dimension := 0
	directoryPath := loadKnowledgeDirectoryPath(document.DirectoryID)

	for i, chunk := range chunks {
		embeddingBase := fmt.Sprintf("knowledge-index:base:%d:document:%d:version:%d:chunk:%d", knowledgeBase.ID, document.ID, document.UpdatedAt.UnixNano(), chunk.ChunkNo)
		embeddingCtx := ai.WithPlatformAIRequestScope(ctx, embeddingBase)
		embeddingCtx = ai.WithPlatformAIRequestPurpose(embeddingCtx, "embedding.document-index")
		embeddingResult, err := ai.Embedding.GenerateEmbedding(embeddingCtx, chunk.Content)
		if err != nil {
			slog.Error("Failed to generate embedding for chunk", "document_id", document.ID, "chunk_index", i, "error", err)
			return nil, nil, 0, fmt.Errorf("failed to generate embedding for chunk %d: %w", i, err)
		}
		if dimension == 0 {
			dimension = embeddingResult.Dimension
		}

		chunkID := buildKnowledgeChunkVectorID(knowledgeBase.ID, document.ID, chunk.ChunkNo)
		providerName := ""
		if chunk.Metadata != nil {
			if value, ok := chunk.Metadata["provider"].(string); ok {
				providerName = value
			}
		}
		sectionPath := joinKnowledgeSectionPath(directoryPath, chunk.SectionPath)
		now := time.Now()
		chunkModels = append(chunkModels, models.KnowledgeChunk{
			KnowledgeBaseID: knowledgeBase.ID,
			DocumentID:      document.ID,
			ChunkNo:         chunk.ChunkNo,
			Title:           chunk.Title,
			Content:         chunk.Content,
			ContentHash:     buildChunkContentHash(chunk.Content),
			CharCount:       chunk.CharCount,
			TokenCount:      chunk.TokenCount,
			ChunkType:       string(chunk.ChunkType),
			SectionPath:     sectionPath,
			Provider:        providerName,
			VectorID:        chunkID,
			Status:          enums.StatusOk,
			CreatedAt:       now,
			UpdatedAt:       now,
		})

		vectors = append(vectors, vectordb.Vector{
			ID:     chunkID,
			Vector: embeddingResult.Vector,
			Payload: vectordb.ChunkPayload{
				KnowledgeBaseID: knowledgeBase.ID,
				DocumentID:      document.ID,
				DocumentTitle:   document.Title,
				ChunkNo:         chunk.ChunkNo,
				ChunkType:       string(chunk.ChunkType),
				SectionPath:     sectionPath,
				Content:         chunk.Content,
				Title:           chunk.Title,
				Provider:        providerName,
			},
		})
	}

	if len(vectors) == 0 {
		return nil, nil, 0, fmt.Errorf("no vectors generated")
	}
	return vectors, chunkModels, dimension, nil
}