From 2737f660f7c7d705e8cc2d0751180e93714f15bc Mon Sep 17 00:00:00 2001 From: mlogclub Date: Sat, 2 May 2026 14:40:16 +0800 Subject: [PATCH] fix: close answerability gate review gaps --- .../ai/runtime/executor/answerability_gate.go | 78 +++++++++++ .../executor/answerability_gate_test.go | 123 ++++++++++++++++++ .../ai/runtime/executor/context_builders.go | 13 ++ internal/ai/runtime/executor/service.go | 32 ++--- 4 files changed, 231 insertions(+), 15 deletions(-) diff --git a/internal/ai/runtime/executor/answerability_gate.go b/internal/ai/runtime/executor/answerability_gate.go index 1b5a865..cd2482e 100644 --- a/internal/ai/runtime/executor/answerability_gate.go +++ b/internal/ai/runtime/executor/answerability_gate.go @@ -4,9 +4,11 @@ import ( "context" "encoding/json" "fmt" + "strconv" "strings" "time" + "cs-agent/internal/ai/rag" "cs-agent/internal/ai/runtime/internal/impl/callbacks" "cs-agent/internal/ai/runtime/internal/impl/factory" "cs-agent/internal/ai/runtime/internal/impl/retrievers" @@ -283,6 +285,12 @@ func (g *KnowledgeAnswerabilityGate) gradeAnswerability(ctx context.Context, sta return state, nil } state.Grade = decision + if err := validateAnswerabilitySupport(decision, state.RetrieveResult); err != nil { + state.FallbackReply = resolveKnowledgeHumanSupportFallback(req.AIAgent) + state.ErrorMessage = err.Error() + state.recordAnswerabilityWithLatency(answerabilityStatusUnanswerable, "answerability supporting chunks invalid", err, started) + return state, nil + } if !decision.Answerable { state.FallbackReply = resolveKnowledgeHumanSupportFallback(req.AIAgent) state.recordAnswerabilityWithLatency(answerabilityStatusUnanswerable, decision.Reason, nil, started) @@ -352,6 +360,76 @@ func buildAnswerabilityContext(result *retrievers.KnowledgeRetrieveResult) strin return strings.TrimSpace(builder.String()) } +func validateAnswerabilitySupport(decision answerabilityDecision, result *retrievers.KnowledgeRetrieveResult) error { + if !decision.Answerable { + return nil + } + if len(decision.SupportingChunkIDs) == 0 { + return fmt.Errorf("answerable decision requires supportingChunkIds") + } + items := []ragRetrieveItem(nil) + if result != nil { + if len(result.ContextResults) > 0 { + items = appendRetrieveItems(items, result.ContextResults) + } else { + items = appendRetrieveItems(items, result.Hits) + } + } + if len(items) == 0 { + return fmt.Errorf("answerable decision has no retrieved chunks to support it") + } + allowed := make(map[string]struct{}) + for _, item := range items { + addAllowedSupportingChunkIDs(allowed, item) + } + for _, supportingChunkID := range decision.SupportingChunkIDs { + supportingChunkID = strings.TrimSpace(supportingChunkID) + if supportingChunkID == "" { + continue + } + if _, ok := allowed[supportingChunkID]; !ok { + return fmt.Errorf("supporting chunk id %q was not retrieved", supportingChunkID) + } + } + return nil +} + +type ragRetrieveItem struct { + KnowledgeBaseID int64 + DocumentID int64 + FaqID int64 + ChunkID int64 +} + +func appendRetrieveItems(dst []ragRetrieveItem, src []rag.RetrieveResult) []ragRetrieveItem { + for _, item := range src { + dst = append(dst, ragRetrieveItem{ + KnowledgeBaseID: item.KnowledgeBaseID, + DocumentID: item.DocumentID, + FaqID: item.FaqID, + ChunkID: item.ChunkID, + }) + } + return dst +} + +func addAllowedSupportingChunkIDs(allowed map[string]struct{}, item ragRetrieveItem) { + if item.ChunkID <= 0 { + return + } + chunkID := strconv.FormatInt(item.ChunkID, 10) + allowed[chunkID] = struct{}{} + allowed["chunk:"+chunkID] = struct{}{} + allowed["chunkId:"+chunkID] = struct{}{} + allowed["chunk-"+chunkID] = struct{}{} + if item.KnowledgeBaseID > 0 && item.DocumentID > 0 { + allowed[fmt.Sprintf("kb:%d:doc:%d:chunk:%d", item.KnowledgeBaseID, item.DocumentID, item.ChunkID)] = struct{}{} + } + if item.KnowledgeBaseID > 0 && item.FaqID > 0 { + allowed[fmt.Sprintf("kb:%d:faq:%d:chunk:%d", item.KnowledgeBaseID, item.FaqID, item.ChunkID)] = struct{}{} + } +} + func parseAnswerabilityDecision(raw string) (answerabilityDecision, error) { text := trimMarkdownFence(raw) if text == "" { diff --git a/internal/ai/runtime/executor/answerability_gate_test.go b/internal/ai/runtime/executor/answerability_gate_test.go index d795a60..8736268 100644 --- a/internal/ai/runtime/executor/answerability_gate_test.go +++ b/internal/ai/runtime/executor/answerability_gate_test.go @@ -2,6 +2,7 @@ package executor import ( "context" + "encoding/json" "errors" "strings" "testing" @@ -186,6 +187,128 @@ func TestKnowledgeAnswerabilityGateEvaluateAllowsAnswerableDecisionAndProducesKn } } +func TestKnowledgeAnswerabilityGateEvaluateFallsBackWhenSupportingChunkIDsAreInvalid(t *testing.T) { + collector := callbacks.NewRuntimeTraceCollector() + chatModel := &fakeAnswerabilityChatModel{ + response: `{"answerable": true, "reason": "refund condition is directly supported", "supportingChunkIds": ["not-real"]}`, + } + gate := newTestKnowledgeAnswerabilityGate(newAnswerabilityRetrieverWithHit(), chatModel) + + state, err := gate.Evaluate(context.Background(), answerabilityGateInput{ + Request: newAnswerabilityGateRunInput("满足什么条件可以退款？", "1"), + Collector: collector, + }) + if err != nil { + t.Fatalf("Evaluate returned error: %v", err) + } + + if !strings.Contains(state.FallbackReply, "建议你联系人工客服进一步确认。") { + t.Fatalf("expected human-support fallback, got %q", state.FallbackReply) + } + if len(state.Decision.Instructions) != 0 { + t.Fatalf("expected no knowledge instruction, got %d", len(state.Decision.Instructions)) + } + if collector.Data.Answerability.Status != answerabilityStatusUnanswerable { + t.Fatalf("unexpected answerability status: %q", collector.Data.Answerability.Status) + } + if collector.Data.Answerability.Reason != "answerability supporting chunks invalid" { + t.Fatalf("unexpected reason: %q", collector.Data.Answerability.Reason) + } + if strings.TrimSpace(collector.Data.Answerability.ErrorMessage) == "" { + t.Fatal("expected invalid supporting chunk error message") + } +} + +func TestValidateAnswerabilitySupportAcceptsSupportedIDForms(t *testing.T) { + result := &retrievers.KnowledgeRetrieveResult{ + ContextResults: []rag.RetrieveResult{ + {KnowledgeBaseID: 1, DocumentID: 10, FaqID: 20, ChunkID: 101}, + }, + Hits: []rag.RetrieveResult{ + {KnowledgeBaseID: 2, DocumentID: 30, ChunkID: 202}, + }, + } + supportedIDs := []string{ + "101", + "chunk:101", + "chunkId:101", + "chunk-101", + "kb:1:doc:10:chunk:101", + "kb:1:faq:20:chunk:101", + } + + for _, supportingChunkID := range supportedIDs { + t.Run(supportingChunkID, func(t *testing.T) { + err := validateAnswerabilitySupport(answerabilityDecision{ + Answerable: true, + SupportingChunkIDs: []string{supportingChunkID}, + }, result) + if err != nil { + t.Fatalf("expected %q to be accepted: %v", supportingChunkID, err) + } + }) + } +} + +func TestValidateAnswerabilitySupportUsesContextResultsBeforeHits(t *testing.T) { + result := &retrievers.KnowledgeRetrieveResult{ + ContextResults: []rag.RetrieveResult{ + {KnowledgeBaseID: 1, DocumentID: 10, ChunkID: 101}, + }, + Hits: []rag.RetrieveResult{ + {KnowledgeBaseID: 2, DocumentID: 20, ChunkID: 202}, + }, + } + + err := validateAnswerabilitySupport(answerabilityDecision{ + Answerable: true, + SupportingChunkIDs: []string{"202"}, + }, result) + + if err == nil { + t.Fatal("expected hit outside context results to be rejected") + } +} + +func TestExecuteRunReturnsKnowledgeFallbackBeforeInvalidMCPToolConfig(t *testing.T) { + gate := newTestKnowledgeAnswerabilityGate(&fakeKnowledgeContextRetriever{ + knowledgeBaseIDs: []int64{1}, + err: errors.New("vector store unavailable"), + }, nil) + service := &Service{ + answerabilityGate: gate, + } + + result, err := service.ExecuteRun(context.Background(), RunInput{ + UserMessage: models.Message{Content: "是否支持退款？"}, + AIAgent: models.AIAgent{ + KnowledgeIDs: "1", + AllowedMCPTools: "{invalid", + }, + AIConfig: models.AIConfig{ModelName: "fake-model"}, + }) + if err != nil { + t.Fatalf("ExecuteRun returned error before fallback: %v", err) + } + + if result.Status != "completed" { + t.Fatalf("expected completed fallback result, got %q", result.Status) + } + if !strings.Contains(result.ReplyText, "建议你联系人工客服进一步确认。") { + t.Fatalf("expected human-support fallback, got %q", result.ReplyText) + } + var trace callbacks.RuntimeTraceData + if err := json.Unmarshal([]byte(result.TraceData), &trace); err != nil { + t.Fatalf("unmarshal trace data: %v", err) + } + if trace.Answerability.Status != answerabilityStatusUnanswerable { + t.Fatalf("unexpected answerability status: %q", trace.Answerability.Status) + } + if trace.Model.Name != "fake-model" { + t.Fatalf("expected model name to be recorded before fallback, got %q", trace.Model.Name) + } +} + func TestBuildRunMessagesReturnsFallbackWhenGateRejectsGrayZoneQuestion(t *testing.T) { summary := &RunResult{} question := "满足什么条件可以退款？" diff --git a/internal/ai/runtime/executor/context_builders.go b/internal/ai/runtime/executor/context_builders.go index fc314e0..ba98efb 100644 --- a/internal/ai/runtime/executor/context_builders.go +++ b/internal/ai/runtime/executor/context_builders.go @@ -48,6 +48,19 @@ func appendRetrievedContext(ctx context.Context, req RunInput, summary *RunResul Messages: append([]*schema.Message(nil), (*messages)...), }) if err != nil || state == nil { + errorMessage := "" + if err != nil { + errorMessage = err.Error() + } else { + errorMessage = "answerability gate returned nil state" + } + if collector != nil { + collector.SetAnswerability(callbacks.AnswerabilityTraceData{ + Status: answerabilityStatusUnanswerable, + Reason: "answerability gate failed", + ErrorMessage: errorMessage, + }) + } decision := buildKnowledgeUnavailableDecision(req.AIAgent, utils.SplitInt64s(req.AIAgent.KnowledgeIDs)) if strings.TrimSpace(decision.FallbackReply) != "" { decision.FallbackReply = resolveKnowledgeHumanSupportFallback(req.AIAgent) diff --git a/internal/ai/runtime/executor/service.go b/internal/ai/runtime/executor/service.go index 9129724..c27ab67 100644 --- a/internal/ai/runtime/executor/service.go +++ b/internal/ai/runtime/executor/service.go @@ -35,6 +35,23 @@ func (s *Service) ExecuteRun(ctx context.Context, req RunInput) (*RunResult, err } collector := callbacks.NewRuntimeTraceCollector() collector.Data.RunID = summary.RunID + summary.ModelName = req.AIConfig.ModelName + collector.Data.Model.Provider = string(req.AIConfig.Provider) + collector.Data.Model.Name = req.AIConfig.ModelName + + checkPointID := resolveCheckPointID(req.CheckPointID, summary.RunID) + summary.CheckPointID = checkPointID + messages := buildRunMessages(ctx, req, summary, collector, s.answerabilityGate) + if strings.TrimSpace(summary.ReplyText) != "" { + summary.Status = "completed" + summary.ModelName = req.AIConfig.ModelName + collector.Data.Status = summary.Status + collector.Data.Output.ReplyText = summary.ReplyText + collector.Data.Output.FinishReason = summary.Status + summary.TraceData = collector.Marshal() + return summary, nil + } + toolDefs, err := factory.NewToolFactory().BuildMCPTools(req.AIAgent) if err != nil { summary.Status = "error" @@ -51,9 +68,6 @@ func (s *Service) ExecuteRun(ctx context.Context, req RunInput) (*RunResult, err collector.Data.Input.ToolCodes = append(collector.Data.Input.ToolCodes, summary.ToolCodes...) collector.SetTooling(tooling.staticToolCodes, definitionToolCodes(tooling.definitions), len(tooling.definitions) > 0) - collector.Data.Model.Provider = string(req.AIConfig.Provider) - collector.Data.Model.Name = req.AIConfig.ModelName - agent, err := s.agentFactory.BuildCustomerServiceAgent(ctx, factory.BuildCustomerServiceAgentInput{ AIAgent: req.AIAgent, AIConfig: req.AIConfig, @@ -74,8 +88,6 @@ func (s *Service) ExecuteRun(ctx context.Context, req RunInput) (*RunResult, err return summary, err } - checkPointID := resolveCheckPointID(req.CheckPointID, summary.RunID) - summary.CheckPointID = checkPointID runner := s.runnerFactory.Build(ctx, agent, false, true) if runner == nil { summary.Status = "error" @@ -86,16 +98,6 @@ func (s *Service) ExecuteRun(ctx context.Context, req RunInput) (*RunResult, err summary.TraceData = collector.Marshal() return summary, fmt.Errorf("%s", summary.ErrorMessage) } - messages := buildRunMessages(ctx, req, summary, collector, s.answerabilityGate) - if strings.TrimSpace(summary.ReplyText) != "" { - summary.Status = "completed" - summary.ModelName = req.AIConfig.ModelName - collector.Data.Status = summary.Status - collector.Data.Output.ReplyText = summary.ReplyText - collector.Data.Output.FinishReason = summary.Status - summary.TraceData = collector.Marshal() - return summary, nil - } collector.Data.Interrupt.CheckPointID = checkPointID consumeAgentEvents(runner.Run(ctx, messages, buildRunOptions(checkPointID)...), summary, collector, tooling.toolDefsByModelName) summary.ModelName = req.AIConfig.ModelName