fix: close answerability gate review gaps

internal/ai/runtime/executor/answerability_gate.go

@@ -4,9 +4,11 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"strconv"
 	"strings"
 	"time"
 
+	"cs-agent/internal/ai/rag"
 	"cs-agent/internal/ai/runtime/internal/impl/callbacks"
 	"cs-agent/internal/ai/runtime/internal/impl/factory"
 	"cs-agent/internal/ai/runtime/internal/impl/retrievers"
@@ -283,6 +285,12 @@ func (g *KnowledgeAnswerabilityGate) gradeAnswerability(ctx context.Context, sta
 		return state, nil
 	}
 	state.Grade = decision
+	if err := validateAnswerabilitySupport(decision, state.RetrieveResult); err != nil {
+		state.FallbackReply = resolveKnowledgeHumanSupportFallback(req.AIAgent)
+		state.ErrorMessage = err.Error()
+		state.recordAnswerabilityWithLatency(answerabilityStatusUnanswerable, "answerability supporting chunks invalid", err, started)
+		return state, nil
+	}
 	if !decision.Answerable {
 		state.FallbackReply = resolveKnowledgeHumanSupportFallback(req.AIAgent)
 		state.recordAnswerabilityWithLatency(answerabilityStatusUnanswerable, decision.Reason, nil, started)
@@ -352,6 +360,76 @@ func buildAnswerabilityContext(result *retrievers.KnowledgeRetrieveResult) strin
 	return strings.TrimSpace(builder.String())
 }
 
+func validateAnswerabilitySupport(decision answerabilityDecision, result *retrievers.KnowledgeRetrieveResult) error {
+	if !decision.Answerable {
+		return nil
+	}
+	if len(decision.SupportingChunkIDs) == 0 {
+		return fmt.Errorf("answerable decision requires supportingChunkIds")
+	}
+	items := []ragRetrieveItem(nil)
+	if result != nil {
+		if len(result.ContextResults) > 0 {
+			items = appendRetrieveItems(items, result.ContextResults)
+		} else {
+			items = appendRetrieveItems(items, result.Hits)
+		}
+	}
+	if len(items) == 0 {
+		return fmt.Errorf("answerable decision has no retrieved chunks to support it")
+	}
+	allowed := make(map[string]struct{})
+	for _, item := range items {
+		addAllowedSupportingChunkIDs(allowed, item)
+	}
+	for _, supportingChunkID := range decision.SupportingChunkIDs {
+		supportingChunkID = strings.TrimSpace(supportingChunkID)
+		if supportingChunkID == "" {
+			continue
+		}
+		if _, ok := allowed[supportingChunkID]; !ok {
+			return fmt.Errorf("supporting chunk id %q was not retrieved", supportingChunkID)
+		}
+	}
+	return nil
+}
+
+type ragRetrieveItem struct {
+	KnowledgeBaseID int64
+	DocumentID      int64
+	FaqID           int64
+	ChunkID         int64
+}
+
+func appendRetrieveItems(dst []ragRetrieveItem, src []rag.RetrieveResult) []ragRetrieveItem {
+	for _, item := range src {
+		dst = append(dst, ragRetrieveItem{
+			KnowledgeBaseID: item.KnowledgeBaseID,
+			DocumentID:      item.DocumentID,
+			FaqID:           item.FaqID,
+			ChunkID:         item.ChunkID,
+		})
+	}
+	return dst
+}
+
+func addAllowedSupportingChunkIDs(allowed map[string]struct{}, item ragRetrieveItem) {
+	if item.ChunkID <= 0 {
+		return
+	}
+	chunkID := strconv.FormatInt(item.ChunkID, 10)
+	allowed[chunkID] = struct{}{}
+	allowed["chunk:"+chunkID] = struct{}{}
+	allowed["chunkId:"+chunkID] = struct{}{}
+	allowed["chunk-"+chunkID] = struct{}{}
+	if item.KnowledgeBaseID > 0 && item.DocumentID > 0 {
+		allowed[fmt.Sprintf("kb:%d:doc:%d:chunk:%d", item.KnowledgeBaseID, item.DocumentID, item.ChunkID)] = struct{}{}
+	}
+	if item.KnowledgeBaseID > 0 && item.FaqID > 0 {
+		allowed[fmt.Sprintf("kb:%d:faq:%d:chunk:%d", item.KnowledgeBaseID, item.FaqID, item.ChunkID)] = struct{}{}
+	}
+}
+
 func parseAnswerabilityDecision(raw string) (answerabilityDecision, error) {
 	text := trimMarkdownFence(raw)
 	if text == "" {

internal/ai/runtime/executor/answerability_gate_test.go

@@ -2,6 +2,7 @@ package executor
 
 import (
 	"context"
+	"encoding/json"
 	"errors"
 	"strings"
 	"testing"
@@ -186,6 +187,128 @@ func TestKnowledgeAnswerabilityGateEvaluateAllowsAnswerableDecisionAndProducesKn
 	}
 }
 
+func TestKnowledgeAnswerabilityGateEvaluateFallsBackWhenSupportingChunkIDsAreInvalid(t *testing.T) {
+	collector := callbacks.NewRuntimeTraceCollector()
+	chatModel := &fakeAnswerabilityChatModel{
+		response: `{"answerable": true, "reason": "refund condition is directly supported", "supportingChunkIds": ["not-real"]}`,
+	}
+	gate := newTestKnowledgeAnswerabilityGate(newAnswerabilityRetrieverWithHit(), chatModel)
+
+	state, err := gate.Evaluate(context.Background(), answerabilityGateInput{
+		Request:   newAnswerabilityGateRunInput("满足什么条件可以退款？", "1"),
+		Collector: collector,
+	})
+	if err != nil {
+		t.Fatalf("Evaluate returned error: %v", err)
+	}
+
+	if !strings.Contains(state.FallbackReply, "建议你联系人工客服进一步确认。") {
+		t.Fatalf("expected human-support fallback, got %q", state.FallbackReply)
+	}
+	if len(state.Decision.Instructions) != 0 {
+		t.Fatalf("expected no knowledge instruction, got %d", len(state.Decision.Instructions))
+	}
+	if collector.Data.Answerability.Status != answerabilityStatusUnanswerable {
+		t.Fatalf("unexpected answerability status: %q", collector.Data.Answerability.Status)
+	}
+	if collector.Data.Answerability.Reason != "answerability supporting chunks invalid" {
+		t.Fatalf("unexpected reason: %q", collector.Data.Answerability.Reason)
+	}
+	if strings.TrimSpace(collector.Data.Answerability.ErrorMessage) == "" {
+		t.Fatal("expected invalid supporting chunk error message")
+	}
+}
+
+func TestValidateAnswerabilitySupportAcceptsSupportedIDForms(t *testing.T) {
+	result := &retrievers.KnowledgeRetrieveResult{
+		ContextResults: []rag.RetrieveResult{
+			{KnowledgeBaseID: 1, DocumentID: 10, FaqID: 20, ChunkID: 101},
+		},
+		Hits: []rag.RetrieveResult{
+			{KnowledgeBaseID: 2, DocumentID: 30, ChunkID: 202},
+		},
+	}
+	supportedIDs := []string{
+		"101",
+		"chunk:101",
+		"chunkId:101",
+		"chunk-101",
+		"kb:1:doc:10:chunk:101",
+		"kb:1:faq:20:chunk:101",
+	}
+
+	for _, supportingChunkID := range supportedIDs {
+		t.Run(supportingChunkID, func(t *testing.T) {
+			err := validateAnswerabilitySupport(answerabilityDecision{
+				Answerable:         true,
+				SupportingChunkIDs: []string{supportingChunkID},
+			}, result)
+			if err != nil {
+				t.Fatalf("expected %q to be accepted: %v", supportingChunkID, err)
+			}
+		})
+	}
+}
+
+func TestValidateAnswerabilitySupportUsesContextResultsBeforeHits(t *testing.T) {
+	result := &retrievers.KnowledgeRetrieveResult{
+		ContextResults: []rag.RetrieveResult{
+			{KnowledgeBaseID: 1, DocumentID: 10, ChunkID: 101},
+		},
+		Hits: []rag.RetrieveResult{
+			{KnowledgeBaseID: 2, DocumentID: 20, ChunkID: 202},
+		},
+	}
+
+	err := validateAnswerabilitySupport(answerabilityDecision{
+		Answerable:         true,
+		SupportingChunkIDs: []string{"202"},
+	}, result)
+
+	if err == nil {
+		t.Fatal("expected hit outside context results to be rejected")
+	}
+}
+
+func TestExecuteRunReturnsKnowledgeFallbackBeforeInvalidMCPToolConfig(t *testing.T) {
+	gate := newTestKnowledgeAnswerabilityGate(&fakeKnowledgeContextRetriever{
+		knowledgeBaseIDs: []int64{1},
+		err:              errors.New("vector store unavailable"),
+	}, nil)
+	service := &Service{
+		answerabilityGate: gate,
+	}
+
+	result, err := service.ExecuteRun(context.Background(), RunInput{
+		UserMessage: models.Message{Content: "是否支持退款？"},
+		AIAgent: models.AIAgent{
+			KnowledgeIDs:    "1",
+			AllowedMCPTools: "{invalid",
+		},
+		AIConfig: models.AIConfig{ModelName: "fake-model"},
+	})
+	if err != nil {
+		t.Fatalf("ExecuteRun returned error before fallback: %v", err)
+	}
+
+	if result.Status != "completed" {
+		t.Fatalf("expected completed fallback result, got %q", result.Status)
+	}
+	if !strings.Contains(result.ReplyText, "建议你联系人工客服进一步确认。") {
+		t.Fatalf("expected human-support fallback, got %q", result.ReplyText)
+	}
+	var trace callbacks.RuntimeTraceData
+	if err := json.Unmarshal([]byte(result.TraceData), &trace); err != nil {
+		t.Fatalf("unmarshal trace data: %v", err)
+	}
+	if trace.Answerability.Status != answerabilityStatusUnanswerable {
+		t.Fatalf("unexpected answerability status: %q", trace.Answerability.Status)
+	}
+	if trace.Model.Name != "fake-model" {
+		t.Fatalf("expected model name to be recorded before fallback, got %q", trace.Model.Name)
+	}
+}
+
 func TestBuildRunMessagesReturnsFallbackWhenGateRejectsGrayZoneQuestion(t *testing.T) {
 	summary := &RunResult{}
 	question := "满足什么条件可以退款？"

internal/ai/runtime/executor/context_builders.go

@@ -48,6 +48,19 @@ func appendRetrievedContext(ctx context.Context, req RunInput, summary *RunResul
 		Messages:  append([]*schema.Message(nil), (*messages)...),
 	})
 	if err != nil || state == nil {
+		errorMessage := ""
+		if err != nil {
+			errorMessage = err.Error()
+		} else {
+			errorMessage = "answerability gate returned nil state"
+		}
+		if collector != nil {
+			collector.SetAnswerability(callbacks.AnswerabilityTraceData{
+				Status:       answerabilityStatusUnanswerable,
+				Reason:       "answerability gate failed",
+				ErrorMessage: errorMessage,
+			})
+		}
 		decision := buildKnowledgeUnavailableDecision(req.AIAgent, utils.SplitInt64s(req.AIAgent.KnowledgeIDs))
 		if strings.TrimSpace(decision.FallbackReply) != "" {
 			decision.FallbackReply = resolveKnowledgeHumanSupportFallback(req.AIAgent)

internal/ai/runtime/executor/service.go

@@ -35,6 +35,23 @@ func (s *Service) ExecuteRun(ctx context.Context, req RunInput) (*RunResult, err
 	}
 	collector := callbacks.NewRuntimeTraceCollector()
 	collector.Data.RunID = summary.RunID
+	summary.ModelName = req.AIConfig.ModelName
+	collector.Data.Model.Provider = string(req.AIConfig.Provider)
+	collector.Data.Model.Name = req.AIConfig.ModelName
+
+	checkPointID := resolveCheckPointID(req.CheckPointID, summary.RunID)
+	summary.CheckPointID = checkPointID
+	messages := buildRunMessages(ctx, req, summary, collector, s.answerabilityGate)
+	if strings.TrimSpace(summary.ReplyText) != "" {
+		summary.Status = "completed"
+		summary.ModelName = req.AIConfig.ModelName
+		collector.Data.Status = summary.Status
+		collector.Data.Output.ReplyText = summary.ReplyText
+		collector.Data.Output.FinishReason = summary.Status
+		summary.TraceData = collector.Marshal()
+		return summary, nil
+	}
+
 	toolDefs, err := factory.NewToolFactory().BuildMCPTools(req.AIAgent)
 	if err != nil {
 		summary.Status = "error"
@@ -51,9 +68,6 @@ func (s *Service) ExecuteRun(ctx context.Context, req RunInput) (*RunResult, err
 	collector.Data.Input.ToolCodes = append(collector.Data.Input.ToolCodes, summary.ToolCodes...)
 	collector.SetTooling(tooling.staticToolCodes, definitionToolCodes(tooling.definitions), len(tooling.definitions) > 0)
 
-	collector.Data.Model.Provider = string(req.AIConfig.Provider)
-	collector.Data.Model.Name = req.AIConfig.ModelName
-
 	agent, err := s.agentFactory.BuildCustomerServiceAgent(ctx, factory.BuildCustomerServiceAgentInput{
 		AIAgent:                    req.AIAgent,
 		AIConfig:                   req.AIConfig,
@@ -74,8 +88,6 @@ func (s *Service) ExecuteRun(ctx context.Context, req RunInput) (*RunResult, err
 		return summary, err
 	}
 
-	checkPointID := resolveCheckPointID(req.CheckPointID, summary.RunID)
-	summary.CheckPointID = checkPointID
 	runner := s.runnerFactory.Build(ctx, agent, false, true)
 	if runner == nil {
 		summary.Status = "error"
@@ -86,16 +98,6 @@ func (s *Service) ExecuteRun(ctx context.Context, req RunInput) (*RunResult, err
 		summary.TraceData = collector.Marshal()
 		return summary, fmt.Errorf("%s", summary.ErrorMessage)
 	}
-	messages := buildRunMessages(ctx, req, summary, collector, s.answerabilityGate)
-	if strings.TrimSpace(summary.ReplyText) != "" {
-		summary.Status = "completed"
-		summary.ModelName = req.AIConfig.ModelName
-		collector.Data.Status = summary.Status
-		collector.Data.Output.ReplyText = summary.ReplyText
-		collector.Data.Output.FinishReason = summary.Status
-		summary.TraceData = collector.Marshal()
-		return summary, nil
-	}
 	collector.Data.Interrupt.CheckPointID = checkPointID
 	consumeAgentEvents(runner.Run(ctx, messages, buildRunOptions(checkPointID)...), summary, collector, tooling.toolDefsByModelName)
 	summary.ModelName = req.AIConfig.ModelName