wlw/ai-agent
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: enhance response policies and update tests for knowledge retrieval handling

Browse Source
This commit is contained in:
mlogclub
2026-07-25 12:12:26 +08:00
parent 88649acaa6
commit 52bcc556d7
2 changed files with 87 additions and 59 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/ai/application/runtime/autonomous_engine.go
+23 -42
View File
@@ -132,14 +132,16 @@ func (e *AutonomousEngine) Run(ctx context.Context, req RunInput) (*RunResult, e
return nil, recordErr
}
trace, _ := json.Marshal(map[string]any{
"engine": EngineCodeAutonomous,
"mode": autonomousExecutionMode(allowedTools),
"historyMessageCount": historyCount,
"retrieverCount": retrieverCount,
"skillID": skillContext.SkillID(),
"skillRouteError": skillContext.ErrorMessage,
"responsePolicyAction": responsePolicy.Action,
"debug": req.Debug,
"engine": EngineCodeAutonomous,
"mode": autonomousExecutionMode(allowedTools),
"historyMessageCount": historyCount,
"retrieverCount": retrieverCount,
"skillID": skillContext.SkillID(),
"skillRouteError": skillContext.ErrorMessage,
"responsePolicyAction": responsePolicy.Action,
"responsePolicyReason": responsePolicy.Reason,
"responsePolicyEnforced": responsePolicy.Enforced,
"debug": req.Debug,
})
return &Summary{
Status: "completed",
@@ -258,22 +260,11 @@ func evaluateAutonomousResponsePolicy(agent models.AIAgent, knowledgeContext str
return autonomousResponsePolicy{}
}
if retrieveErr != nil {
return autonomousKnowledgeFallbackPolicy(agent, "knowledge_retrieve_error")
}
return autonomousKnowledgeFallbackPolicy(agent, "knowledge_evidence_missing")
}
func autonomousKnowledgeFallbackPolicy(agent models.AIAgent, reason string) autonomousResponsePolicy {
if agent.FallbackMode == enums.AIAgentFallbackModeHandoff {
return autonomousResponsePolicy{
Enforced: true, Action: "handoff", Reason: reason, RequestHandoff: true,
ReplyText: autonomousKnowledgeFallbackReply(agent),
}
}
return autonomousResponsePolicy{
Enforced: true, Action: "clarify", Reason: reason,
ReplyText: autonomousKnowledgeFallbackReply(agent),
return autonomousResponsePolicy{Action: "retrieval_unavailable", Reason: "knowledge_retrieve_error"}
}
// Knowledge retrieval is an evidence signal, not a replacement for the
// model's ability to handle greetings and other non-factual conversation.
return autonomousResponsePolicy{Action: "evidence_required", Reason: "knowledge_evidence_missing"}
}
func autonomousToolFailurePolicy(agent models.AIAgent, reason string) autonomousResponsePolicy {
@@ -313,19 +304,6 @@ func autonomousHasConsecutiveToolFailures(calls []svc.EngineToolCallInput, minim
return failures >= minimum
}
func autonomousKnowledgeFallbackReply(agent models.AIAgent) string {
if reply := strings.TrimSpace(agent.FallbackMessage); reply != "" {
return reply
}
if agent.FallbackMode == 0 || agent.FallbackMode == enums.AIAgentFallbackModeSuggestRetry {
return "当前知识库里没有找到足够明确的信息，你可以换个更具体的问法再试一次。"
}
if agent.FallbackMode == enums.AIAgentFallbackModeHandoff {
return "当前知识库没有足够明确的信息，正在为你转接人工客服。"
}
return "当前知识库暂无明确信息。"
}
func (c autonomousSkillContext) SkillID() int64 {
if c.Skill == nil {
return 0
@@ -598,11 +576,10 @@ func buildAutonomousSystemPrompt(agent models.AIAgent, hasKnowledgeBase bool, kn
if prompt == "" {
prompt = "You are a customer service assistant. Answer accurately, ask for clarification when evidence is insufficient, and do not invent facts."
}
if hasKnowledgeBase && strings.TrimSpace(knowledgeContext) == "" {
prompt += "\n\nNo supporting knowledge was retrieved. Do not invent an answer; ask a focused clarification question or offer human handoff."
}
if retrieveErr != nil {
prompt += "\n\nKnowledge retrieval is temporarily unavailable. Do not claim to have verified any policy or factual detail."
prompt += "\n\nKnowledge retrieval is temporarily unavailable for this message. You may answer greetings, acknowledgements, gratitude, farewells, and requests for clarification naturally. For product facts, policies, pricing, functions, procedures, timing, refunds, accounts, permissions, or after-sales questions, do not claim that any detail is verified. Explain that you cannot verify it now, ask one focused question when useful, or offer human handoff."
} else if hasKnowledgeBase && strings.TrimSpace(knowledgeContext) == "" {
prompt += "\n\nKnowledge retrieval found no supporting evidence for this message. You may answer greetings, acknowledgements, gratitude, farewells, and requests for clarification naturally. For product facts, policies, pricing, functions, procedures, timing, refunds, accounts, permissions, or after-sales questions, do not infer or invent an answer. State that the available information is insufficient, ask one focused question when useful, or offer human handoff."
}
return prompt
}
@@ -672,13 +649,17 @@ func autonomousAdditionalSteps(req Request, retrieverCount int, retrieveErr erro
InputPreview: strings.TrimSpace(req.UserMessage.Content), OutputPreview: "retrieved context items: " + strconv.Itoa(retrieverCount), ErrorMessage: errorMessage,
})
}
if responsePolicy.Enforced {
if responsePolicy.Enforced || responsePolicy.Reason != "" {
policyCode := "knowledge_evidence"
if strings.HasPrefix(responsePolicy.Reason, "tool_") {
policyCode = "tool_failure"
}
status := "completed"
if !responsePolicy.Enforced {
status = "advisory"
}
steps = append(steps, svc.EngineStepInput{
StepType: "policy", StepCode: policyCode, Status: "completed",
StepType: "policy", StepCode: policyCode, Status: status,
InputPreview: responsePolicy.Reason, OutputPreview: responsePolicy.Action,
})
}
internal/ai/application/runtime/engine_test.go
+64 -17
View File
@@ -210,7 +210,7 @@ func TestAutonomousEngineInjectsSelectedSkillAndRecordsRoute(t *testing.T) {
}
}
func TestAutonomousEngineEnforcesKnowledgeFallbackPolicy(t *testing.T) {
func TestAutonomousEngineLetsModelHandleGreetingWithoutKnowledgeEvidence(t *testing.T) {
db, err := gorm.Open(sqlite.Open("file:"+strings.ReplaceAll(t.Name(), "/", "_")+"?mode=memory&cache=shared"), &gorm.Config{NamingStrategy: schema.NamingStrategy{TablePrefix: "t_", SingularTable: true}})
if err != nil {
t.Fatalf("open sqlite: %v", err)
@@ -223,31 +223,71 @@ func TestAutonomousEngineEnforcesKnowledgeFallbackPolicy(t *testing.T) {
if err := db.Create(revision).Error; err != nil {
t.Fatalf("create revision: %v", err)
}
chatCalled := false
engine := newAutonomousEngineWithChat(func(context.Context, models.AIConfig, string, string) (*ai.ChatCompletionResult, error) {
chatCalled = true
return &ai.ChatCompletionResult{Content: "should not be used"}, nil
var systemPrompt string
engine := newAutonomousEngineWithChat(func(_ context.Context, _ models.AIConfig, system, _ string) (*ai.ChatCompletionResult, error) {
systemPrompt = system
return &ai.ChatCompletionResult{Content: "你好，有什么可以帮你？", ModelName: "test-model"}, nil
})
engine.retrieve = func(context.Context, models.AIAgent, string) (string, int, error) {
return "", 0, nil
}
summary, err := engine.Run(context.Background(), Request{
Conversation: models.Conversation{ID: 1}, UserMessage: models.Message{ID: 2, Content: "你好"},
AIAgent: models.AIAgent{ID: 10, PublishedRevisionID: revision.ID, KnowledgeIDs: "100"},
AIConfig: models.AIConfig{ModelName: "test-model"},
})
if err != nil {
t.Fatalf("Run returned error: %v", err)
}
if summary.ReplyText != "你好，有什么可以帮你？" {
t.Fatalf("unexpected model reply: %#v", summary)
}
if !strings.Contains(systemPrompt, "answer greetings") || !strings.Contains(systemPrompt, "Knowledge retrieval found no supporting evidence") {
t.Fatalf("missing no-evidence greeting instructions: %q", systemPrompt)
}
var steps []models.AgentStep
if err := db.Where("agent_run_id = ?", summary.AgentRunID).Find(&steps).Error; err != nil {
t.Fatalf("load steps: %v", err)
}
if len(steps) != 3 || steps[1].StepType != "knowledge" || steps[2].StepType != "policy" || steps[2].StepCode != "knowledge_evidence" || steps[2].Status != "advisory" || steps[2].OutputPreview != "evidence_required" {
t.Fatalf("expected model, knowledge and policy steps, got %#v", steps)
}
}
func TestAutonomousEngineInstructsModelNotToInventFactsWithoutKnowledgeEvidence(t *testing.T) {
db, err := gorm.Open(sqlite.Open("file:"+strings.ReplaceAll(t.Name(), "/", "_")+"?mode=memory&cache=shared"), &gorm.Config{NamingStrategy: schema.NamingStrategy{TablePrefix: "t_", SingularTable: true}})
if err != nil {
t.Fatalf("open sqlite: %v", err)
}
if err := db.AutoMigrate(&models.AgentRevision{}, &models.AgentRun{}, &models.AgentStep{}); err != nil {
t.Fatalf("auto migrate: %v", err)
}
sqls.SetDB(db)
revision := &models.AgentRevision{AgentID: 13, Revision: 1}
if err := db.Create(revision).Error; err != nil {
t.Fatalf("create revision: %v", err)
}
var systemPrompt string
engine := newAutonomousEngineWithChat(func(_ context.Context, _ models.AIConfig, system, _ string) (*ai.ChatCompletionResult, error) {
systemPrompt = system
return &ai.ChatCompletionResult{Content: "我暂时没有查到保修期限的准确依据。请提供产品型号，我再继续查询。", ModelName: "test-model"}, nil
})
engine.retrieve = func(context.Context, models.AIAgent, string) (string, int, error) {
return "", 0, nil
}
summary, err := engine.Run(context.Background(), Request{
Conversation: models.Conversation{ID: 1}, UserMessage: models.Message{ID: 2, Content: "保修多久"},
AIAgent: models.AIAgent{ID: 10, PublishedRevisionID: revision.ID, KnowledgeIDs: "100", FallbackMessage: "请提供产品型号，我再继续查询。"},
AIAgent: models.AIAgent{ID: 13, PublishedRevisionID: revision.ID, KnowledgeIDs: "100"},
AIConfig: models.AIConfig{ModelName: "test-model"},
})
if err != nil {
t.Fatalf("Run returned error: %v", err)
}
if chatCalled || summary.ReplyText != "请提供产品型号，我再继续查询。" {
t.Fatalf("knowledge fallback policy was not enforced: chatCalled=%t summary=%#v", chatCalled, summary)
if summary.ReplyText != "我暂时没有查到保修期限的准确依据。请提供产品型号，我再继续查询。" {
t.Fatalf("unexpected model reply: %#v", summary)
}
var steps []models.AgentStep
if err := db.Where("agent_run_id = ?", summary.AgentRunID).Find(&steps).Error; err != nil {
t.Fatalf("load steps: %v", err)
}
if len(steps) != 3 || steps[1].StepType != "knowledge" || steps[2].StepType != "policy" || steps[2].StepCode != "knowledge_evidence" {
t.Fatalf("expected model, knowledge and policy steps, got %#v", steps)
if !strings.Contains(systemPrompt, "product facts, policies, pricing") || !strings.Contains(systemPrompt, "do not infer or invent an answer") {
t.Fatalf("missing factual-answer evidence constraints: %q", systemPrompt)
}
}
@@ -415,12 +455,19 @@ func TestParseAutonomousToolPolicyAndPerToolCount(t *testing.T) {
}
}
func TestAutonomousResponsePolicyRequestsHandoffOnlyWhenConfigured(t *testing.T) {
handoff := evaluateAutonomousResponsePolicy(models.AIAgent{KnowledgeIDs: "1", FallbackMode: enums.AIAgentFallbackModeHandoff}, "", nil)
func TestAutonomousKnowledgeEvidencePolicyIsAdvisory(t *testing.T) {
policy := evaluateAutonomousResponsePolicy(models.AIAgent{KnowledgeIDs: "1", FallbackMode: enums.AIAgentFallbackModeHandoff}, "", nil)
if policy.Enforced || policy.RequestHandoff || policy.Action != "evidence_required" || policy.Reason != "knowledge_evidence_missing" {
t.Fatalf("unexpected knowledge evidence policy: %#v", policy)
}
}
func TestAutonomousToolFailurePolicyRequestsHandoffOnlyWhenConfigured(t *testing.T) {
handoff := autonomousToolFailurePolicy(models.AIAgent{FallbackMode: enums.AIAgentFallbackModeHandoff}, "tool_loop_error")
if !handoff.Enforced || !handoff.RequestHandoff || handoff.Action != "handoff" {
t.Fatalf("unexpected handoff policy: %#v", handoff)
}
clarify := evaluateAutonomousResponsePolicy(models.AIAgent{KnowledgeIDs: "1", FallbackMode: enums.AIAgentFallbackModeSuggestRetry}, "", nil)
clarify := autonomousToolFailurePolicy(models.AIAgent{FallbackMode: enums.AIAgentFallbackModeSuggestRetry}, "tool_loop_error")
if !clarify.Enforced || clarify.RequestHandoff || clarify.Action != "clarify" {
t.Fatalf("unexpected clarify policy: %#v", clarify)
}