package executor import ( "context" "encoding/json" "errors" "strings" "testing" "cs-agent/internal/ai/rag" "cs-agent/internal/ai/runtime/internal/impl/callbacks" "cs-agent/internal/ai/runtime/internal/impl/retrievers" "cs-agent/internal/models" "github.com/cloudwego/eino/components/model" "github.com/cloudwego/eino/schema" ) func TestParseAnswerabilityDecisionRejectsMalformedJSON(t *testing.T) { _, err := parseAnswerabilityDecision(`{"answerable": true`) if err == nil { t.Fatal("expected malformed JSON to be rejected") } } func TestParseAnswerabilityDecisionRejectsAnswerableWithoutSupportingChunkIDs(t *testing.T) { _, err := parseAnswerabilityDecision(`{"answerable": true, "reason": "directly supported"}`) if err == nil { t.Fatal("expected answerable decision without supporting chunks to be rejected") } } func TestParseAnswerabilityDecisionAcceptsAnswerableWithSupportingChunkIDs(t *testing.T) { got, err := parseAnswerabilityDecision("```json\n{\"answerable\": true, \"reason\": \"directly supported\", \"supportingChunkIds\": [\" chunk-1 \", \"chunk-2\"]}\n```") if err != nil { t.Fatalf("parse decision failed: %v", err) } if !got.Answerable { t.Fatal("expected answerable decision") } if got.SupportingChunkIDs[0] != "chunk-1" || got.SupportingChunkIDs[1] != "chunk-2" { t.Fatalf("unexpected supporting chunks: %#v", got.SupportingChunkIDs) } } func TestKnowledgeAnswerabilityGateEvaluateFallsBackOnRetrieverError(t *testing.T) { collector := callbacks.NewRuntimeTraceCollector() gate := newTestKnowledgeAnswerabilityGate(&fakeKnowledgeContextRetriever{ knowledgeBaseIDs: []int64{1}, err: errors.New("vector store unavailable"), }, nil) state, err := gate.Evaluate(context.Background(), answerabilityGateInput{ Request: newAnswerabilityGateRunInput("是否支持退款?", "1"), Summary: &RunResult{}, Collector: collector, }) if err != nil { t.Fatalf("Evaluate returned error: %v", err) } if !strings.Contains(state.FallbackReply, "建议你联系人工客服进一步确认。") { t.Fatalf("expected human-support fallback, got %q", state.FallbackReply) } if collector.Data.Answerability.Status != answerabilityStatusUnanswerable { t.Fatalf("unexpected answerability status: %q", collector.Data.Answerability.Status) } if collector.Data.Answerability.Reason != "knowledge retrieval failed" { t.Fatalf("unexpected reason: %q", collector.Data.Answerability.Reason) } } func TestKnowledgeAnswerabilityGateEvaluateSkipsWhenNoKnowledgeConfigured(t *testing.T) { collector := callbacks.NewRuntimeTraceCollector() gate := newTestKnowledgeAnswerabilityGate(&fakeKnowledgeContextRetriever{}, nil) state, err := gate.Evaluate(context.Background(), answerabilityGateInput{ Request: newAnswerabilityGateRunInput("是否支持退款?", ""), Collector: collector, }) if err != nil { t.Fatalf("Evaluate returned error: %v", err) } if !state.SkipGate { t.Fatal("expected gate to skip without knowledge") } if state.FallbackReply != "" { t.Fatalf("expected no fallback when gate skips, got %q", state.FallbackReply) } if collector.Data.Answerability.Status != answerabilityStatusSkipped { t.Fatalf("unexpected answerability status: %q", collector.Data.Answerability.Status) } } func TestKnowledgeAnswerabilityGateEvaluateFallsBackWhenConfiguredRetrieverUnavailable(t *testing.T) { collector := callbacks.NewRuntimeTraceCollector() gate := newTestKnowledgeAnswerabilityGate(nil, nil) state, err := gate.Evaluate(context.Background(), answerabilityGateInput{ Request: newAnswerabilityGateRunInput("是否支持退款?", "1"), Collector: collector, }) if err != nil { t.Fatalf("Evaluate returned error: %v", err) } if state.SkipGate { t.Fatal("expected configured knowledge to fail closed, not skip") } if !strings.Contains(state.FallbackReply, "建议你联系人工客服进一步确认。") { t.Fatalf("expected human-support fallback, got %q", state.FallbackReply) } if collector.Data.Answerability.Status != answerabilityStatusUnanswerable { t.Fatalf("unexpected answerability status: %q", collector.Data.Answerability.Status) } if collector.Data.Answerability.Reason != "knowledge retriever unavailable" { t.Fatalf("unexpected reason: %q", collector.Data.Answerability.Reason) } } func TestKnowledgeAnswerabilityGateEvaluateFallsBackOnGrayZoneUnanswerableDecision(t *testing.T) { collector := callbacks.NewRuntimeTraceCollector() gate := newTestKnowledgeAnswerabilityGate(newAnswerabilityRetrieverWithHit(), &fakeAnswerabilityChatModel{ response: `{"answerable": false, "reason": "retrieved snippets mention refunds but not the requested condition", "missingInfo": ["refund condition"]}`, }) state, err := gate.Evaluate(context.Background(), answerabilityGateInput{ Request: newAnswerabilityGateRunInput("满足什么条件可以退款?", "1"), Collector: collector, }) if err != nil { t.Fatalf("Evaluate returned error: %v", err) } if !strings.Contains(state.FallbackReply, "建议你联系人工客服进一步确认。") { t.Fatalf("expected human-support fallback, got %q", state.FallbackReply) } if collector.Data.Answerability.Status != answerabilityStatusUnanswerable { t.Fatalf("unexpected answerability status: %q", collector.Data.Answerability.Status) } if collector.Data.Answerability.MissingInfo[0] != "refund condition" { t.Fatalf("unexpected missing info: %#v", collector.Data.Answerability.MissingInfo) } } func TestKnowledgeAnswerabilityGateEvaluateAllowsAnswerableDecisionAndProducesKnowledgeInstruction(t *testing.T) { collector := callbacks.NewRuntimeTraceCollector() summary := &RunResult{} chatModel := &fakeAnswerabilityChatModel{ response: `{"answerable": true, "reason": "refund condition is directly supported", "supportingChunkIds": ["101"]}`, } gate := newTestKnowledgeAnswerabilityGate(newAnswerabilityRetrieverWithHit(), chatModel) state, err := gate.Evaluate(context.Background(), answerabilityGateInput{ Request: newAnswerabilityGateRunInput("满足什么条件可以退款?", "1"), Summary: summary, Collector: collector, }) if err != nil { t.Fatalf("Evaluate returned error: %v", err) } if state.FallbackReply != "" { t.Fatalf("expected answerable gate to allow, got fallback %q", state.FallbackReply) } if len(state.Decision.Instructions) != 1 { t.Fatalf("expected one knowledge instruction, got %d", len(state.Decision.Instructions)) } if !strings.Contains(state.Decision.Instructions[0].Content, "知识库回答约束") { t.Fatalf("unexpected instruction: %q", state.Decision.Instructions[0].Content) } if summary.RetrieverCount != 1 { t.Fatalf("expected retriever count 1, got %d", summary.RetrieverCount) } if collector.Data.Answerability.Status != answerabilityStatusAnswerable { t.Fatalf("unexpected answerability status: %q", collector.Data.Answerability.Status) } if collector.Data.Answerability.SupportingChunkIDs[0] != "101" { t.Fatalf("unexpected supporting chunks: %#v", collector.Data.Answerability.SupportingChunkIDs) } if len(chatModel.input) == 0 || !strings.Contains(chatModel.input[len(chatModel.input)-1].Content, "chunkId: 101") { t.Fatalf("expected grader prompt to include chunk id, got %#v", chatModel.input) } } func TestKnowledgeAnswerabilityGateEvaluateFallsBackWhenSupportingChunkIDsAreInvalid(t *testing.T) { collector := callbacks.NewRuntimeTraceCollector() chatModel := &fakeAnswerabilityChatModel{ response: `{"answerable": true, "reason": "refund condition is directly supported", "supportingChunkIds": ["not-real"]}`, } gate := newTestKnowledgeAnswerabilityGate(newAnswerabilityRetrieverWithHit(), chatModel) state, err := gate.Evaluate(context.Background(), answerabilityGateInput{ Request: newAnswerabilityGateRunInput("满足什么条件可以退款?", "1"), Collector: collector, }) if err != nil { t.Fatalf("Evaluate returned error: %v", err) } if !strings.Contains(state.FallbackReply, "建议你联系人工客服进一步确认。") { t.Fatalf("expected human-support fallback, got %q", state.FallbackReply) } if len(state.Decision.Instructions) != 0 { t.Fatalf("expected no knowledge instruction, got %d", len(state.Decision.Instructions)) } if collector.Data.Answerability.Status != answerabilityStatusUnanswerable { t.Fatalf("unexpected answerability status: %q", collector.Data.Answerability.Status) } if collector.Data.Answerability.Reason != "answerability supporting chunks invalid" { t.Fatalf("unexpected reason: %q", collector.Data.Answerability.Reason) } if strings.TrimSpace(collector.Data.Answerability.ErrorMessage) == "" { t.Fatal("expected invalid supporting chunk error message") } } func TestValidateAnswerabilitySupportAcceptsSupportedIDForms(t *testing.T) { result := &retrievers.KnowledgeRetrieveResult{ ContextResults: []rag.RetrieveResult{ {KnowledgeBaseID: 1, DocumentID: 10, FaqID: 20, ChunkID: 101}, }, Hits: []rag.RetrieveResult{ {KnowledgeBaseID: 2, DocumentID: 30, ChunkID: 202}, }, } supportedIDs := []string{ "101", "chunk:101", "chunkId:101", "chunk-101", "kb:1:doc:10:chunk:101", "kb:1:faq:20:chunk:101", } for _, supportingChunkID := range supportedIDs { t.Run(supportingChunkID, func(t *testing.T) { err := validateAnswerabilitySupport(answerabilityDecision{ Answerable: true, SupportingChunkIDs: []string{supportingChunkID}, }, result) if err != nil { t.Fatalf("expected %q to be accepted: %v", supportingChunkID, err) } }) } } func TestValidateAnswerabilitySupportUsesContextResultsBeforeHits(t *testing.T) { result := &retrievers.KnowledgeRetrieveResult{ ContextResults: []rag.RetrieveResult{ {KnowledgeBaseID: 1, DocumentID: 10, ChunkID: 101}, }, Hits: []rag.RetrieveResult{ {KnowledgeBaseID: 2, DocumentID: 20, ChunkID: 202}, }, } err := validateAnswerabilitySupport(answerabilityDecision{ Answerable: true, SupportingChunkIDs: []string{"202"}, }, result) if err == nil { t.Fatal("expected hit outside context results to be rejected") } } func TestExecuteRunReturnsKnowledgeFallbackBeforeInvalidMCPToolConfig(t *testing.T) { gate := newTestKnowledgeAnswerabilityGate(&fakeKnowledgeContextRetriever{ knowledgeBaseIDs: []int64{1}, err: errors.New("vector store unavailable"), }, nil) service := &Service{ answerabilityGate: gate, } result, err := service.ExecuteRun(context.Background(), RunInput{ UserMessage: models.Message{Content: "是否支持退款?"}, AIAgent: models.AIAgent{ KnowledgeIDs: "1", AllowedMCPTools: "{invalid", }, AIConfig: models.AIConfig{ModelName: "fake-model"}, }) if err != nil { t.Fatalf("ExecuteRun returned error before fallback: %v", err) } if result.Status != "completed" { t.Fatalf("expected completed fallback result, got %q", result.Status) } if !strings.Contains(result.ReplyText, "建议你联系人工客服进一步确认。") { t.Fatalf("expected human-support fallback, got %q", result.ReplyText) } var trace callbacks.RuntimeTraceData if err := json.Unmarshal([]byte(result.TraceData), &trace); err != nil { t.Fatalf("unmarshal trace data: %v", err) } if trace.Answerability.Status != answerabilityStatusUnanswerable { t.Fatalf("unexpected answerability status: %q", trace.Answerability.Status) } if trace.Model.Name != "fake-model" { t.Fatalf("expected model name to be recorded before fallback, got %q", trace.Model.Name) } } func TestBuildRunMessagesReturnsFallbackWhenGateRejectsGrayZoneQuestion(t *testing.T) { summary := &RunResult{} question := "满足什么条件可以退款?" gate := newTestKnowledgeAnswerabilityGate(newAnswerabilityRetrieverWithHit(), &fakeAnswerabilityChatModel{ response: `{"answerable": false, "reason": "retrieved snippets mention refunds but not the requested condition", "missingInfo": ["refund condition"]}`, }) messages := buildRunMessages(context.Background(), newAnswerabilityGateRunInput(question, "1"), summary, nil, gate) if !strings.Contains(summary.ReplyText, "建议你联系人工客服进一步确认。") { t.Fatalf("expected human-support fallback, got %q", summary.ReplyText) } if messagesContainContent(messages, question) { t.Fatalf("expected returned messages to omit current user message, got %#v", messages) } } func TestBuildRunMessagesInjectsKnowledgeWhenGateAllows(t *testing.T) { summary := &RunResult{} question := "满足什么条件可以退款?" gate := newTestKnowledgeAnswerabilityGate(newAnswerabilityRetrieverWithHit(), &fakeAnswerabilityChatModel{ response: `{"answerable": true, "reason": "refund condition is directly supported", "supportingChunkIds": ["101"]}`, }) messages := buildRunMessages(context.Background(), newAnswerabilityGateRunInput(question, "1"), summary, nil, gate) if summary.ReplyText != "" { t.Fatalf("expected no fallback, got %q", summary.ReplyText) } if !messagesContainContent(messages, "知识库回答约束") { t.Fatalf("expected knowledge instruction in messages, got %#v", messages) } if !messagesContainContent(messages, "购买后七天内且未使用可以退款。") { t.Fatalf("expected knowledge context in messages, got %#v", messages) } if !messagesContainContent(messages, question) { t.Fatalf("expected current user message in messages, got %#v", messages) } } func TestBuildRunMessagesSkipsAnswerabilityGateForExplicitHandoffIntent(t *testing.T) { summary := &RunResult{} retriever := &fakeKnowledgeContextRetriever{ knowledgeBaseIDs: []int64{1}, err: errors.New("retriever should not be called for handoff intent"), } gate := newTestKnowledgeAnswerabilityGate(retriever, nil) messages := buildRunMessages(context.Background(), newAnswerabilityGateRunInput("我要转人工", "1"), summary, nil, gate) if summary.ReplyText != "" { t.Fatalf("expected handoff intent to continue to agent/tool routing, got fallback %q", summary.ReplyText) } if retriever.lastQuery != "" { t.Fatalf("expected retriever to be skipped for handoff intent, got query %q", retriever.lastQuery) } if !messagesContainContent(messages, "我要转人工") { t.Fatalf("expected current user message in messages, got %#v", messages) } } func TestBuildRunMessagesSkipsAnswerabilityGateForExplicitTicketIntent(t *testing.T) { summary := &RunResult{} retriever := &fakeKnowledgeContextRetriever{ knowledgeBaseIDs: []int64{1}, err: errors.New("retriever should not be called for ticket intent"), } gate := newTestKnowledgeAnswerabilityGate(retriever, nil) messages := buildRunMessages(context.Background(), newAnswerabilityGateRunInput("帮我创建一个工单", "1"), summary, nil, gate) if summary.ReplyText != "" { t.Fatalf("expected ticket intent to continue to agent/tool routing, got fallback %q", summary.ReplyText) } if retriever.lastQuery != "" { t.Fatalf("expected retriever to be skipped for ticket intent, got query %q", retriever.lastQuery) } if !messagesContainContent(messages, "帮我创建一个工单") { t.Fatalf("expected current user message in messages, got %#v", messages) } } func TestRuntimeActionIntentDoesNotMatchKnowledgeQuestions(t *testing.T) { cases := []string{ "人工客服服务时间是什么?", "工单状态怎么查询?", "需要查询工单状态怎么操作?", } for _, tc := range cases { t.Run(tc, func(t *testing.T) { if isRuntimeActionIntent(tc) { t.Fatalf("expected %q to stay in knowledge answerability flow", tc) } }) } } func newTestKnowledgeAnswerabilityGate(retriever knowledgeContextRetriever, chatModel model.BaseChatModel) *KnowledgeAnswerabilityGate { return &KnowledgeAnswerabilityGate{ newRetriever: func(aiAgent models.AIAgent) knowledgeContextRetriever { return retriever }, newChatModel: func(ctx context.Context, aiConfig models.AIConfig) (model.BaseChatModel, error) { return chatModel, nil }, } } func newAnswerabilityGateRunInput(question string, knowledgeIDs string) RunInput { return RunInput{ UserMessage: models.Message{Content: question}, AIAgent: models.AIAgent{ KnowledgeIDs: knowledgeIDs, }, } } func newAnswerabilityRetrieverWithHit() *fakeKnowledgeContextRetriever { return &fakeKnowledgeContextRetriever{ knowledgeBaseIDs: []int64{1}, result: &retrievers.KnowledgeRetrieveResult{ KnowledgeBaseIDs: []int64{1}, Hits: []rag.RetrieveResult{ {KnowledgeBaseID: 1, DocumentID: 10, ChunkID: 101, Score: 0.93, Content: "购买后七天内且未使用可以退款。"}, }, ContextResults: []rag.RetrieveResult{ {KnowledgeBaseID: 1, DocumentID: 10, ChunkID: 101, Score: 0.93, Content: "购买后七天内且未使用可以退款。"}, }, ContextText: "购买后七天内且未使用可以退款。", }, } } type fakeKnowledgeContextRetriever struct { knowledgeBaseIDs []int64 result *retrievers.KnowledgeRetrieveResult err error lastOptions retrievers.KnowledgeRetrieveOptions lastQuery string } func (f *fakeKnowledgeContextRetriever) KnowledgeBaseIDs() []int64 { return append([]int64(nil), f.knowledgeBaseIDs...) } func (f *fakeKnowledgeContextRetriever) RetrieveContextByOptions(ctx context.Context, opts retrievers.KnowledgeRetrieveOptions, query string) (*retrievers.KnowledgeRetrieveResult, error) { f.lastOptions = opts f.lastQuery = query return f.result, f.err } type fakeAnswerabilityChatModel struct { response string err error input []*schema.Message } func (f *fakeAnswerabilityChatModel) Generate(ctx context.Context, input []*schema.Message, opts ...model.Option) (*schema.Message, error) { f.input = input if f.err != nil { return nil, f.err } return schema.AssistantMessage(f.response, nil), nil } func (f *fakeAnswerabilityChatModel) Stream(ctx context.Context, input []*schema.Message, opts ...model.Option) (*schema.StreamReader[*schema.Message], error) { return nil, errors.New("stream is not implemented in fakeAnswerabilityChatModel") } func messagesContainContent(messages []*schema.Message, text string) bool { for _, message := range messages { if message != nil && strings.Contains(message.Content, text) { return true } } return false }