wlw/ai-agent
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: close answerability gate review gaps

Browse Source
This commit is contained in:
mlogclub
2026-05-02 14:40:16 +08:00
parent 8f968fe3ca
commit 2737f660f7
4 changed files with 231 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/ai/runtime/executor/answerability_gate.go
+78
View File
@@ -4,9 +4,11 @@ import (
"context"
"encoding/json"
"fmt"
"strconv"
"strings"
"time"
"cs-agent/internal/ai/rag"
"cs-agent/internal/ai/runtime/internal/impl/callbacks"
"cs-agent/internal/ai/runtime/internal/impl/factory"
"cs-agent/internal/ai/runtime/internal/impl/retrievers"
@@ -283,6 +285,12 @@ func (g *KnowledgeAnswerabilityGate) gradeAnswerability(ctx context.Context, sta
return state, nil
}
state.Grade = decision
if err := validateAnswerabilitySupport(decision, state.RetrieveResult); err != nil {
state.FallbackReply = resolveKnowledgeHumanSupportFallback(req.AIAgent)
state.ErrorMessage = err.Error()
state.recordAnswerabilityWithLatency(answerabilityStatusUnanswerable, "answerability supporting chunks invalid", err, started)
return state, nil
}
if !decision.Answerable {
state.FallbackReply = resolveKnowledgeHumanSupportFallback(req.AIAgent)
state.recordAnswerabilityWithLatency(answerabilityStatusUnanswerable, decision.Reason, nil, started)
@@ -352,6 +360,76 @@ func buildAnswerabilityContext(result *retrievers.KnowledgeRetrieveResult) strin
return strings.TrimSpace(builder.String())
}
func validateAnswerabilitySupport(decision answerabilityDecision, result *retrievers.KnowledgeRetrieveResult) error {
if !decision.Answerable {
return nil
}
if len(decision.SupportingChunkIDs) == 0 {
return fmt.Errorf("answerable decision requires supportingChunkIds")
}
items := []ragRetrieveItem(nil)
if result != nil {
if len(result.ContextResults) > 0 {
items = appendRetrieveItems(items, result.ContextResults)
} else {
items = appendRetrieveItems(items, result.Hits)
}
}
if len(items) == 0 {
return fmt.Errorf("answerable decision has no retrieved chunks to support it")
}
allowed := make(map[string]struct{})
for _, item := range items {
addAllowedSupportingChunkIDs(allowed, item)
}
for _, supportingChunkID := range decision.SupportingChunkIDs {
supportingChunkID = strings.TrimSpace(supportingChunkID)
if supportingChunkID == "" {
continue
}
if _, ok := allowed[supportingChunkID]; !ok {
return fmt.Errorf("supporting chunk id %q was not retrieved", supportingChunkID)
}
}
return nil
}
type ragRetrieveItem struct {
KnowledgeBaseID int64
DocumentID int64
FaqID int64
ChunkID int64
}
func appendRetrieveItems(dst []ragRetrieveItem, src []rag.RetrieveResult) []ragRetrieveItem {
for _, item := range src {
dst = append(dst, ragRetrieveItem{
KnowledgeBaseID: item.KnowledgeBaseID,
DocumentID: item.DocumentID,
FaqID: item.FaqID,
ChunkID: item.ChunkID,
})
}
return dst
}
func addAllowedSupportingChunkIDs(allowed map[string]struct{}, item ragRetrieveItem) {
if item.ChunkID <= 0 {
return
}
chunkID := strconv.FormatInt(item.ChunkID, 10)
allowed[chunkID] = struct{}{}
allowed["chunk:"+chunkID] = struct{}{}
allowed["chunkId:"+chunkID] = struct{}{}
allowed["chunk-"+chunkID] = struct{}{}
if item.KnowledgeBaseID > 0 && item.DocumentID > 0 {
allowed[fmt.Sprintf("kb:%d:doc:%d:chunk:%d", item.KnowledgeBaseID, item.DocumentID, item.ChunkID)] = struct{}{}
}
if item.KnowledgeBaseID > 0 && item.FaqID > 0 {
allowed[fmt.Sprintf("kb:%d:faq:%d:chunk:%d", item.KnowledgeBaseID, item.FaqID, item.ChunkID)] = struct{}{}
}
}
func parseAnswerabilityDecision(raw string) (answerabilityDecision, error) {
text := trimMarkdownFence(raw)
if text == "" {
internal/ai/runtime/executor/answerability_gate_test.go
+123
View File
@@ -2,6 +2,7 @@ package executor
import (
"context"
"encoding/json"
"errors"
"strings"
"testing"
@@ -186,6 +187,128 @@ func TestKnowledgeAnswerabilityGateEvaluateAllowsAnswerableDecisionAndProducesKn
}
}
func TestKnowledgeAnswerabilityGateEvaluateFallsBackWhenSupportingChunkIDsAreInvalid(t *testing.T) {
collector := callbacks.NewRuntimeTraceCollector()
chatModel := &fakeAnswerabilityChatModel{
response: `{"answerable": true, "reason": "refund condition is directly supported", "supportingChunkIds": ["not-real"]}`,
}
gate := newTestKnowledgeAnswerabilityGate(newAnswerabilityRetrieverWithHit(), chatModel)
state, err := gate.Evaluate(context.Background(), answerabilityGateInput{
Request: newAnswerabilityGateRunInput("满足什么条件可以退款？", "1"),
Collector: collector,
})
if err != nil {
t.Fatalf("Evaluate returned error: %v", err)
}
if !strings.Contains(state.FallbackReply, "建议你联系人工客服进一步确认。") {
t.Fatalf("expected human-support fallback, got %q", state.FallbackReply)
}
if len(state.Decision.Instructions) != 0 {
t.Fatalf("expected no knowledge instruction, got %d", len(state.Decision.Instructions))
}
if collector.Data.Answerability.Status != answerabilityStatusUnanswerable {
t.Fatalf("unexpected answerability status: %q", collector.Data.Answerability.Status)
}
if collector.Data.Answerability.Reason != "answerability supporting chunks invalid" {
t.Fatalf("unexpected reason: %q", collector.Data.Answerability.Reason)
}
if strings.TrimSpace(collector.Data.Answerability.ErrorMessage) == "" {
t.Fatal("expected invalid supporting chunk error message")
}
}
func TestValidateAnswerabilitySupportAcceptsSupportedIDForms(t *testing.T) {
result := &retrievers.KnowledgeRetrieveResult{
ContextResults: []rag.RetrieveResult{
{KnowledgeBaseID: 1, DocumentID: 10, FaqID: 20, ChunkID: 101},
},
Hits: []rag.RetrieveResult{
{KnowledgeBaseID: 2, DocumentID: 30, ChunkID: 202},
},
}
supportedIDs := []string{
"101",
"chunk:101",
"chunkId:101",
"chunk-101",
"kb:1:doc:10:chunk:101",
"kb:1:faq:20:chunk:101",
}
for _, supportingChunkID := range supportedIDs {
t.Run(supportingChunkID, func(t *testing.T) {
err := validateAnswerabilitySupport(answerabilityDecision{
Answerable: true,
SupportingChunkIDs: []string{supportingChunkID},
}, result)
if err != nil {
t.Fatalf("expected %q to be accepted: %v", supportingChunkID, err)
}
})
}
}
func TestValidateAnswerabilitySupportUsesContextResultsBeforeHits(t *testing.T) {
result := &retrievers.KnowledgeRetrieveResult{
ContextResults: []rag.RetrieveResult{
{KnowledgeBaseID: 1, DocumentID: 10, ChunkID: 101},
},
Hits: []rag.RetrieveResult{
{KnowledgeBaseID: 2, DocumentID: 20, ChunkID: 202},
},
}
err := validateAnswerabilitySupport(answerabilityDecision{
Answerable: true,
SupportingChunkIDs: []string{"202"},
}, result)
if err == nil {
t.Fatal("expected hit outside context results to be rejected")
}
}
func TestExecuteRunReturnsKnowledgeFallbackBeforeInvalidMCPToolConfig(t *testing.T) {
gate := newTestKnowledgeAnswerabilityGate(&fakeKnowledgeContextRetriever{
knowledgeBaseIDs: []int64{1},
err: errors.New("vector store unavailable"),
}, nil)
service := &Service{
answerabilityGate: gate,
}
result, err := service.ExecuteRun(context.Background(), RunInput{
UserMessage: models.Message{Content: "是否支持退款？"},
AIAgent: models.AIAgent{
KnowledgeIDs: "1",
AllowedMCPTools: "{invalid",
},
AIConfig: models.AIConfig{ModelName: "fake-model"},
})
if err != nil {
t.Fatalf("ExecuteRun returned error before fallback: %v", err)
}
if result.Status != "completed" {
t.Fatalf("expected completed fallback result, got %q", result.Status)
}
if !strings.Contains(result.ReplyText, "建议你联系人工客服进一步确认。") {
t.Fatalf("expected human-support fallback, got %q", result.ReplyText)
}
var trace callbacks.RuntimeTraceData
if err := json.Unmarshal([]byte(result.TraceData), &trace); err != nil {
t.Fatalf("unmarshal trace data: %v", err)
}
if trace.Answerability.Status != answerabilityStatusUnanswerable {
t.Fatalf("unexpected answerability status: %q", trace.Answerability.Status)
}
if trace.Model.Name != "fake-model" {
t.Fatalf("expected model name to be recorded before fallback, got %q", trace.Model.Name)
}
}
func TestBuildRunMessagesReturnsFallbackWhenGateRejectsGrayZoneQuestion(t *testing.T) {
summary := &RunResult{}
question := "满足什么条件可以退款？"
internal/ai/runtime/executor/context_builders.go
+13
View File
@@ -48,6 +48,19 @@ func appendRetrievedContext(ctx context.Context, req RunInput, summary *RunResul
Messages: append([]*schema.Message(nil), (*messages)...),
})
if err != nil || state == nil {
errorMessage := ""
if err != nil {
errorMessage = err.Error()
} else {
errorMessage = "answerability gate returned nil state"
}
if collector != nil {
collector.SetAnswerability(callbacks.AnswerabilityTraceData{
Status: answerabilityStatusUnanswerable,
Reason: "answerability gate failed",
ErrorMessage: errorMessage,
})
}
decision := buildKnowledgeUnavailableDecision(req.AIAgent, utils.SplitInt64s(req.AIAgent.KnowledgeIDs))
if strings.TrimSpace(decision.FallbackReply) != "" {
decision.FallbackReply = resolveKnowledgeHumanSupportFallback(req.AIAgent)
internal/ai/runtime/executor/service.go
+17 -15
View File
@@ -35,6 +35,23 @@ func (s *Service) ExecuteRun(ctx context.Context, req RunInput) (*RunResult, err
}
collector := callbacks.NewRuntimeTraceCollector()
collector.Data.RunID = summary.RunID
summary.ModelName = req.AIConfig.ModelName
collector.Data.Model.Provider = string(req.AIConfig.Provider)
collector.Data.Model.Name = req.AIConfig.ModelName
checkPointID := resolveCheckPointID(req.CheckPointID, summary.RunID)
summary.CheckPointID = checkPointID
messages := buildRunMessages(ctx, req, summary, collector, s.answerabilityGate)
if strings.TrimSpace(summary.ReplyText) != "" {
summary.Status = "completed"
summary.ModelName = req.AIConfig.ModelName
collector.Data.Status = summary.Status
collector.Data.Output.ReplyText = summary.ReplyText
collector.Data.Output.FinishReason = summary.Status
summary.TraceData = collector.Marshal()
return summary, nil
}
toolDefs, err := factory.NewToolFactory().BuildMCPTools(req.AIAgent)
if err != nil {
summary.Status = "error"
@@ -51,9 +68,6 @@ func (s *Service) ExecuteRun(ctx context.Context, req RunInput) (*RunResult, err
collector.Data.Input.ToolCodes = append(collector.Data.Input.ToolCodes, summary.ToolCodes...)
collector.SetTooling(tooling.staticToolCodes, definitionToolCodes(tooling.definitions), len(tooling.definitions) > 0)
collector.Data.Model.Provider = string(req.AIConfig.Provider)
collector.Data.Model.Name = req.AIConfig.ModelName
agent, err := s.agentFactory.BuildCustomerServiceAgent(ctx, factory.BuildCustomerServiceAgentInput{
AIAgent: req.AIAgent,
AIConfig: req.AIConfig,
@@ -74,8 +88,6 @@ func (s *Service) ExecuteRun(ctx context.Context, req RunInput) (*RunResult, err
return summary, err
}
checkPointID := resolveCheckPointID(req.CheckPointID, summary.RunID)
summary.CheckPointID = checkPointID
runner := s.runnerFactory.Build(ctx, agent, false, true)
if runner == nil {
summary.Status = "error"
@@ -86,16 +98,6 @@ func (s *Service) ExecuteRun(ctx context.Context, req RunInput) (*RunResult, err
summary.TraceData = collector.Marshal()
return summary, fmt.Errorf("%s", summary.ErrorMessage)
}
messages := buildRunMessages(ctx, req, summary, collector, s.answerabilityGate)
if strings.TrimSpace(summary.ReplyText) != "" {
summary.Status = "completed"
summary.ModelName = req.AIConfig.ModelName
collector.Data.Status = summary.Status
collector.Data.Output.ReplyText = summary.ReplyText
collector.Data.Output.FinishReason = summary.Status
summary.TraceData = collector.Marshal()
return summary, nil
}
collector.Data.Interrupt.CheckPointID = checkPointID
consumeAgentEvents(runner.Run(ctx, messages, buildRunOptions(checkPointID)...), summary, collector, tooling.toolDefsByModelName)
summary.ModelName = req.AIConfig.ModelName