refactor: use controlled RAG no-context policy
This commit is contained in:
+1
-1
Submodule docs updated: ff7b57f9a1...121a572919
@@ -2,33 +2,26 @@ package executor
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
|
||||||
"fmt"
|
|
||||||
"strconv"
|
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"cs-agent/internal/ai/rag"
|
|
||||||
"cs-agent/internal/ai/runtime/internal/impl/callbacks"
|
"cs-agent/internal/ai/runtime/internal/impl/callbacks"
|
||||||
"cs-agent/internal/ai/runtime/internal/impl/factory"
|
|
||||||
"cs-agent/internal/ai/runtime/internal/impl/retrievers"
|
"cs-agent/internal/ai/runtime/internal/impl/retrievers"
|
||||||
"cs-agent/internal/models"
|
"cs-agent/internal/models"
|
||||||
"cs-agent/internal/pkg/utils"
|
"cs-agent/internal/pkg/utils"
|
||||||
|
|
||||||
"github.com/cloudwego/eino/components/model"
|
|
||||||
"github.com/cloudwego/eino/components/prompt"
|
|
||||||
"github.com/cloudwego/eino/compose"
|
"github.com/cloudwego/eino/compose"
|
||||||
"github.com/cloudwego/eino/schema"
|
"github.com/cloudwego/eino/schema"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
answerabilityNodeRetrieve = "retrieve_knowledge"
|
answerabilityNodeRetrieve = "retrieve_knowledge"
|
||||||
answerabilityNodeGrade = "grade_answerability"
|
|
||||||
answerabilityNodeAllow = "allow_agent"
|
answerabilityNodeAllow = "allow_agent"
|
||||||
answerabilityNodeFallback = "fallback"
|
answerabilityNodeFallback = "fallback"
|
||||||
|
|
||||||
answerabilityStatusSkipped = "skipped"
|
answerabilityStatusSkipped = "skipped"
|
||||||
answerabilityStatusAnswerable = "answerable"
|
answerabilityStatusNoContext = "no_context"
|
||||||
|
answerabilityStatusHasContext = "has_context"
|
||||||
answerabilityStatusUnanswerable = "unanswerable"
|
answerabilityStatusUnanswerable = "unanswerable"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -39,12 +32,8 @@ type knowledgeContextRetriever interface {
|
|||||||
|
|
||||||
type answerabilityRetrieverFactory func(aiAgent models.AIAgent) knowledgeContextRetriever
|
type answerabilityRetrieverFactory func(aiAgent models.AIAgent) knowledgeContextRetriever
|
||||||
|
|
||||||
type answerabilityChatModelFactory func(ctx context.Context, aiConfig models.AIConfig) (model.BaseChatModel, error)
|
|
||||||
|
|
||||||
type KnowledgeAnswerabilityGate struct {
|
type KnowledgeAnswerabilityGate struct {
|
||||||
newRetriever answerabilityRetrieverFactory
|
newRetriever answerabilityRetrieverFactory
|
||||||
newChatModel answerabilityChatModelFactory
|
|
||||||
now func() time.Time
|
|
||||||
}
|
}
|
||||||
|
|
||||||
type answerabilityGateInput struct {
|
type answerabilityGateInput struct {
|
||||||
@@ -59,29 +48,16 @@ type answerabilityGateState struct {
|
|||||||
KnowledgeIDs []int64
|
KnowledgeIDs []int64
|
||||||
RetrieveResult *retrievers.KnowledgeRetrieveResult
|
RetrieveResult *retrievers.KnowledgeRetrieveResult
|
||||||
Decision knowledgeGuardDecision
|
Decision knowledgeGuardDecision
|
||||||
Grade answerabilityDecision
|
|
||||||
SkipGate bool
|
SkipGate bool
|
||||||
FallbackReply string
|
FallbackReply string
|
||||||
ErrorMessage string
|
ErrorMessage string
|
||||||
}
|
}
|
||||||
|
|
||||||
type answerabilityDecision struct {
|
|
||||||
Answerable bool `json:"answerable"`
|
|
||||||
Reason string `json:"reason"`
|
|
||||||
SupportingChunkIDs []string `json:"supportingChunkIds"`
|
|
||||||
MissingInfo []string `json:"missingInfo"`
|
|
||||||
}
|
|
||||||
|
|
||||||
func NewKnowledgeAnswerabilityGate() *KnowledgeAnswerabilityGate {
|
func NewKnowledgeAnswerabilityGate() *KnowledgeAnswerabilityGate {
|
||||||
chatModelFactory := factory.NewChatModelFactory()
|
|
||||||
return &KnowledgeAnswerabilityGate{
|
return &KnowledgeAnswerabilityGate{
|
||||||
newRetriever: func(aiAgent models.AIAgent) knowledgeContextRetriever {
|
newRetriever: func(aiAgent models.AIAgent) knowledgeContextRetriever {
|
||||||
return retrievers.NewKnowledgeRetriever(aiAgent)
|
return retrievers.NewKnowledgeRetriever(aiAgent)
|
||||||
},
|
},
|
||||||
newChatModel: func(ctx context.Context, aiConfig models.AIConfig) (model.BaseChatModel, error) {
|
|
||||||
return chatModelFactory.Build(ctx, aiConfig)
|
|
||||||
},
|
|
||||||
now: time.Now,
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -94,12 +70,6 @@ func (g *KnowledgeAnswerabilityGate) withDefaults() *KnowledgeAnswerabilityGate
|
|||||||
if ret.newRetriever == nil {
|
if ret.newRetriever == nil {
|
||||||
ret.newRetriever = defaults.newRetriever
|
ret.newRetriever = defaults.newRetriever
|
||||||
}
|
}
|
||||||
if ret.newChatModel == nil {
|
|
||||||
ret.newChatModel = defaults.newChatModel
|
|
||||||
}
|
|
||||||
if ret.now == nil {
|
|
||||||
ret.now = time.Now
|
|
||||||
}
|
|
||||||
return &ret
|
return &ret
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -109,9 +79,6 @@ func (g *KnowledgeAnswerabilityGate) Evaluate(ctx context.Context, input answera
|
|||||||
if err := graph.AddLambdaNode(answerabilityNodeRetrieve, compose.InvokableLambda(gate.retrieveKnowledge)); err != nil {
|
if err := graph.AddLambdaNode(answerabilityNodeRetrieve, compose.InvokableLambda(gate.retrieveKnowledge)); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
if err := graph.AddLambdaNode(answerabilityNodeGrade, compose.InvokableLambda(gate.gradeAnswerability)); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
if err := graph.AddLambdaNode(answerabilityNodeAllow, compose.InvokableLambda(allowAnswerabilityPassThrough)); err != nil {
|
if err := graph.AddLambdaNode(answerabilityNodeAllow, compose.InvokableLambda(allowAnswerabilityPassThrough)); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@@ -121,10 +88,7 @@ func (g *KnowledgeAnswerabilityGate) Evaluate(ctx context.Context, input answera
|
|||||||
if err := graph.AddEdge(compose.START, answerabilityNodeRetrieve); err != nil {
|
if err := graph.AddEdge(compose.START, answerabilityNodeRetrieve); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
if err := graph.AddEdge(answerabilityNodeRetrieve, answerabilityNodeGrade); err != nil {
|
if err := graph.AddBranch(answerabilityNodeRetrieve, compose.NewGraphBranch(routeAnswerabilityGate, map[string]bool{
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
if err := graph.AddBranch(answerabilityNodeGrade, compose.NewGraphBranch(routeAnswerabilityGate, map[string]bool{
|
|
||||||
answerabilityNodeAllow: true,
|
answerabilityNodeAllow: true,
|
||||||
answerabilityNodeFallback: true,
|
answerabilityNodeFallback: true,
|
||||||
})); err != nil {
|
})); err != nil {
|
||||||
@@ -187,14 +151,14 @@ func (g *KnowledgeAnswerabilityGate) retrieveKnowledge(ctx context.Context, stat
|
|||||||
return state, nil
|
return state, nil
|
||||||
}
|
}
|
||||||
configuredKnowledgeIDs := utils.SplitInt64s(req.AIAgent.KnowledgeIDs)
|
configuredKnowledgeIDs := utils.SplitInt64s(req.AIAgent.KnowledgeIDs)
|
||||||
|
if len(configuredKnowledgeIDs) == 0 {
|
||||||
|
state.SkipGate = true
|
||||||
|
state.recordAnswerability(answerabilityStatusSkipped, "no knowledge configured", nil)
|
||||||
|
return state, nil
|
||||||
|
}
|
||||||
retriever := gate.newRetriever(req.AIAgent)
|
retriever := gate.newRetriever(req.AIAgent)
|
||||||
|
state.KnowledgeIDs = append([]int64(nil), configuredKnowledgeIDs...)
|
||||||
if retriever == nil {
|
if retriever == nil {
|
||||||
state.KnowledgeIDs = append([]int64(nil), configuredKnowledgeIDs...)
|
|
||||||
if len(configuredKnowledgeIDs) == 0 {
|
|
||||||
state.SkipGate = true
|
|
||||||
state.recordAnswerability(answerabilityStatusSkipped, "no knowledge configured", nil)
|
|
||||||
return state, nil
|
|
||||||
}
|
|
||||||
state.FallbackReply = resolveKnowledgeHumanSupportFallback(req.AIAgent)
|
state.FallbackReply = resolveKnowledgeHumanSupportFallback(req.AIAgent)
|
||||||
state.recordAnswerability(answerabilityStatusUnanswerable, "knowledge retriever unavailable", nil)
|
state.recordAnswerability(answerabilityStatusUnanswerable, "knowledge retriever unavailable", nil)
|
||||||
return state, nil
|
return state, nil
|
||||||
@@ -208,8 +172,8 @@ func (g *KnowledgeAnswerabilityGate) retrieveKnowledge(ctx context.Context, stat
|
|||||||
}
|
}
|
||||||
query := strings.TrimSpace(req.UserMessage.Content)
|
query := strings.TrimSpace(req.UserMessage.Content)
|
||||||
if query == "" {
|
if query == "" {
|
||||||
state.FallbackReply = resolveKnowledgeHumanSupportFallback(req.AIAgent)
|
state.Decision = buildKnowledgeNoContextDecision(req.AIAgent, knowledgeIDs)
|
||||||
state.recordAnswerability(answerabilityStatusUnanswerable, "empty user question", nil)
|
state.recordAnswerability(answerabilityStatusNoContext, "empty user question", nil)
|
||||||
return state, nil
|
return state, nil
|
||||||
}
|
}
|
||||||
retrieveOptions := retrievers.DefaultKnowledgeRetrieveOptions()
|
retrieveOptions := retrievers.DefaultKnowledgeRetrieveOptions()
|
||||||
@@ -230,10 +194,12 @@ func (g *KnowledgeAnswerabilityGate) retrieveKnowledge(ctx context.Context, stat
|
|||||||
state.Input.Collector.AddRetrieverItems(result.TraceItems)
|
state.Input.Collector.AddRetrieverItems(result.TraceItems)
|
||||||
}
|
}
|
||||||
if result == nil || len(result.Hits) == 0 || strings.TrimSpace(result.ContextText) == "" {
|
if result == nil || len(result.Hits) == 0 || strings.TrimSpace(result.ContextText) == "" {
|
||||||
state.FallbackReply = resolveKnowledgeHumanSupportFallback(req.AIAgent)
|
state.Decision = buildKnowledgeNoContextDecision(req.AIAgent, knowledgeIDs)
|
||||||
state.recordAnswerability(answerabilityStatusUnanswerable, "no retrieved context", nil)
|
state.recordAnswerability(answerabilityStatusNoContext, "no retrieved context", nil)
|
||||||
return state, nil
|
return state, nil
|
||||||
}
|
}
|
||||||
|
state.Decision = buildKnowledgeGuardDecision(req.AIAgent, result)
|
||||||
|
state.recordAnswerability(answerabilityStatusHasContext, "retrieved context injected", nil)
|
||||||
return state, nil
|
return state, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -303,255 +269,6 @@ func isShortActionPhrase(text string) bool {
|
|||||||
return len([]rune(text)) <= 8
|
return len([]rune(text)) <= 8
|
||||||
}
|
}
|
||||||
|
|
||||||
func (g *KnowledgeAnswerabilityGate) gradeAnswerability(ctx context.Context, state *answerabilityGateState) (*answerabilityGateState, error) {
|
|
||||||
if state == nil {
|
|
||||||
return &answerabilityGateState{}, nil
|
|
||||||
}
|
|
||||||
if state.SkipGate || strings.TrimSpace(state.FallbackReply) != "" {
|
|
||||||
return state, nil
|
|
||||||
}
|
|
||||||
gate := g.withDefaults()
|
|
||||||
started := gate.now()
|
|
||||||
req := state.Input.Request
|
|
||||||
modelInstance, err := gate.newChatModel(ctx, req.AIConfig)
|
|
||||||
if err != nil {
|
|
||||||
state.FallbackReply = resolveKnowledgeHumanSupportFallback(req.AIAgent)
|
|
||||||
state.ErrorMessage = err.Error()
|
|
||||||
state.recordAnswerabilityWithLatency(answerabilityStatusUnanswerable, "answerability model factory failed", err, started)
|
|
||||||
return state, nil
|
|
||||||
}
|
|
||||||
if modelInstance == nil {
|
|
||||||
err = fmt.Errorf("answerability model is nil")
|
|
||||||
state.FallbackReply = resolveKnowledgeHumanSupportFallback(req.AIAgent)
|
|
||||||
state.ErrorMessage = err.Error()
|
|
||||||
state.recordAnswerabilityWithLatency(answerabilityStatusUnanswerable, "answerability model factory failed", err, started)
|
|
||||||
return state, nil
|
|
||||||
}
|
|
||||||
messages, err := buildAnswerabilityMessages(ctx, req.UserMessage.Content, state.RetrieveResult)
|
|
||||||
if err != nil {
|
|
||||||
state.FallbackReply = resolveKnowledgeHumanSupportFallback(req.AIAgent)
|
|
||||||
state.ErrorMessage = err.Error()
|
|
||||||
state.recordAnswerabilityWithLatency(answerabilityStatusUnanswerable, "answerability prompt failed", err, started)
|
|
||||||
return state, nil
|
|
||||||
}
|
|
||||||
response, err := modelInstance.Generate(ctx, messages)
|
|
||||||
if err != nil {
|
|
||||||
state.FallbackReply = resolveKnowledgeHumanSupportFallback(req.AIAgent)
|
|
||||||
state.ErrorMessage = err.Error()
|
|
||||||
state.recordAnswerabilityWithLatency(answerabilityStatusUnanswerable, "answerability model generate failed", err, started)
|
|
||||||
return state, nil
|
|
||||||
}
|
|
||||||
if response == nil {
|
|
||||||
err = fmt.Errorf("answerability model returned empty response")
|
|
||||||
state.FallbackReply = resolveKnowledgeHumanSupportFallback(req.AIAgent)
|
|
||||||
state.ErrorMessage = err.Error()
|
|
||||||
state.recordAnswerabilityWithLatency(answerabilityStatusUnanswerable, "answerability model generate failed", err, started)
|
|
||||||
return state, nil
|
|
||||||
}
|
|
||||||
decision, err := parseAnswerabilityDecision(response.Content)
|
|
||||||
if err != nil {
|
|
||||||
state.FallbackReply = resolveKnowledgeHumanSupportFallback(req.AIAgent)
|
|
||||||
state.ErrorMessage = err.Error()
|
|
||||||
state.recordAnswerabilityWithLatency(answerabilityStatusUnanswerable, "answerability decision parse failed", err, started)
|
|
||||||
return state, nil
|
|
||||||
}
|
|
||||||
state.Grade = decision
|
|
||||||
if err := validateAnswerabilitySupport(decision, state.RetrieveResult); err != nil {
|
|
||||||
state.FallbackReply = resolveKnowledgeHumanSupportFallback(req.AIAgent)
|
|
||||||
state.ErrorMessage = err.Error()
|
|
||||||
state.recordAnswerabilityWithLatency(answerabilityStatusUnanswerable, "answerability supporting chunks invalid", err, started)
|
|
||||||
return state, nil
|
|
||||||
}
|
|
||||||
if !decision.Answerable {
|
|
||||||
state.FallbackReply = resolveKnowledgeHumanSupportFallback(req.AIAgent)
|
|
||||||
state.recordAnswerabilityWithLatency(answerabilityStatusUnanswerable, decision.Reason, nil, started)
|
|
||||||
return state, nil
|
|
||||||
}
|
|
||||||
state.Decision = buildKnowledgeGuardDecision(req.AIAgent, state.RetrieveResult)
|
|
||||||
if strings.TrimSpace(state.Decision.FallbackReply) != "" {
|
|
||||||
state.FallbackReply = resolveKnowledgeHumanSupportFallback(req.AIAgent)
|
|
||||||
state.recordAnswerabilityWithLatency(answerabilityStatusUnanswerable, "knowledge guard rejected retrieved context", nil, started)
|
|
||||||
return state, nil
|
|
||||||
}
|
|
||||||
state.recordAnswerabilityWithLatency(answerabilityStatusAnswerable, decision.Reason, nil, started)
|
|
||||||
return state, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func buildAnswerabilityMessages(ctx context.Context, question string, result *retrievers.KnowledgeRetrieveResult) ([]*schema.Message, error) {
|
|
||||||
contextText := buildAnswerabilityContext(result)
|
|
||||||
template := prompt.FromMessages(schema.FString,
|
|
||||||
schema.SystemMessage(strings.TrimSpace(`你是一个知识库可回答性判定器。
|
|
||||||
你只判断“已召回知识片段”是否直接支持回答用户问题,不要使用模型常识补充。
|
|
||||||
如果问题中的具体对象、条件、步骤、承诺或限制不能被片段直接支持,判定为不可回答。
|
|
||||||
只输出 JSON,不要输出 Markdown、解释或多余文本。
|
|
||||||
JSON 字段必须包含:
|
|
||||||
- answerable: boolean
|
|
||||||
- reason: string
|
|
||||||
- supportingChunkIds: string array,answerable 为 true 时必须至少包含一个直接支持的 chunk id
|
|
||||||
- missingInfo: string array,answerable 为 false 时列出缺失信息`)),
|
|
||||||
schema.UserMessage(strings.TrimSpace(`用户问题:
|
|
||||||
{question}
|
|
||||||
|
|
||||||
已召回知识片段:
|
|
||||||
{context}
|
|
||||||
|
|
||||||
请基于上述片段判定是否可以直接回答用户问题。`)),
|
|
||||||
)
|
|
||||||
return template.Format(ctx, map[string]any{
|
|
||||||
"question": strings.TrimSpace(question),
|
|
||||||
"context": contextText,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func buildAnswerabilityContext(result *retrievers.KnowledgeRetrieveResult) string {
|
|
||||||
if result == nil {
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
items := result.ContextResults
|
|
||||||
if len(items) == 0 {
|
|
||||||
items = result.Hits
|
|
||||||
}
|
|
||||||
if len(items) == 0 {
|
|
||||||
return strings.TrimSpace(result.ContextText)
|
|
||||||
}
|
|
||||||
var builder strings.Builder
|
|
||||||
for idx, item := range items {
|
|
||||||
if idx > 0 {
|
|
||||||
builder.WriteString("\n\n")
|
|
||||||
}
|
|
||||||
builder.WriteString(fmt.Sprintf("snippet %d\nknowledgeBaseId: %d\ndocumentId: %d\nchunkId: %d\nscore: %.4f\ncontent:\n%s",
|
|
||||||
idx+1,
|
|
||||||
item.KnowledgeBaseID,
|
|
||||||
item.DocumentID,
|
|
||||||
item.ChunkID,
|
|
||||||
item.Score,
|
|
||||||
strings.TrimSpace(item.Content),
|
|
||||||
))
|
|
||||||
}
|
|
||||||
return strings.TrimSpace(builder.String())
|
|
||||||
}
|
|
||||||
|
|
||||||
func validateAnswerabilitySupport(decision answerabilityDecision, result *retrievers.KnowledgeRetrieveResult) error {
|
|
||||||
if !decision.Answerable {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
if len(decision.SupportingChunkIDs) == 0 {
|
|
||||||
return fmt.Errorf("answerable decision requires supportingChunkIds")
|
|
||||||
}
|
|
||||||
items := []ragRetrieveItem(nil)
|
|
||||||
if result != nil {
|
|
||||||
if len(result.ContextResults) > 0 {
|
|
||||||
items = appendRetrieveItems(items, result.ContextResults)
|
|
||||||
} else {
|
|
||||||
items = appendRetrieveItems(items, result.Hits)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if len(items) == 0 {
|
|
||||||
return fmt.Errorf("answerable decision has no retrieved chunks to support it")
|
|
||||||
}
|
|
||||||
allowed := make(map[string]struct{})
|
|
||||||
for _, item := range items {
|
|
||||||
addAllowedSupportingChunkIDs(allowed, item)
|
|
||||||
}
|
|
||||||
for _, supportingChunkID := range decision.SupportingChunkIDs {
|
|
||||||
supportingChunkID = strings.TrimSpace(supportingChunkID)
|
|
||||||
if supportingChunkID == "" {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if _, ok := allowed[supportingChunkID]; !ok {
|
|
||||||
return fmt.Errorf("supporting chunk id %q was not retrieved", supportingChunkID)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
type ragRetrieveItem struct {
|
|
||||||
KnowledgeBaseID int64
|
|
||||||
DocumentID int64
|
|
||||||
FaqID int64
|
|
||||||
ChunkID int64
|
|
||||||
}
|
|
||||||
|
|
||||||
func appendRetrieveItems(dst []ragRetrieveItem, src []rag.RetrieveResult) []ragRetrieveItem {
|
|
||||||
for _, item := range src {
|
|
||||||
dst = append(dst, ragRetrieveItem{
|
|
||||||
KnowledgeBaseID: item.KnowledgeBaseID,
|
|
||||||
DocumentID: item.DocumentID,
|
|
||||||
FaqID: item.FaqID,
|
|
||||||
ChunkID: item.ChunkID,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
return dst
|
|
||||||
}
|
|
||||||
|
|
||||||
func addAllowedSupportingChunkIDs(allowed map[string]struct{}, item ragRetrieveItem) {
|
|
||||||
if item.ChunkID <= 0 {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
chunkID := strconv.FormatInt(item.ChunkID, 10)
|
|
||||||
allowed[chunkID] = struct{}{}
|
|
||||||
allowed["chunk:"+chunkID] = struct{}{}
|
|
||||||
allowed["chunkId:"+chunkID] = struct{}{}
|
|
||||||
allowed["chunk-"+chunkID] = struct{}{}
|
|
||||||
if item.KnowledgeBaseID > 0 && item.DocumentID > 0 {
|
|
||||||
allowed[fmt.Sprintf("kb:%d:doc:%d:chunk:%d", item.KnowledgeBaseID, item.DocumentID, item.ChunkID)] = struct{}{}
|
|
||||||
}
|
|
||||||
if item.KnowledgeBaseID > 0 && item.FaqID > 0 {
|
|
||||||
allowed[fmt.Sprintf("kb:%d:faq:%d:chunk:%d", item.KnowledgeBaseID, item.FaqID, item.ChunkID)] = struct{}{}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func parseAnswerabilityDecision(raw string) (answerabilityDecision, error) {
|
|
||||||
text := trimMarkdownFence(raw)
|
|
||||||
if text == "" {
|
|
||||||
return answerabilityDecision{}, fmt.Errorf("answerability decision is empty")
|
|
||||||
}
|
|
||||||
var decision answerabilityDecision
|
|
||||||
if err := json.Unmarshal([]byte(text), &decision); err != nil {
|
|
||||||
return answerabilityDecision{}, fmt.Errorf("parse answerability decision: %w", err)
|
|
||||||
}
|
|
||||||
decision.Reason = strings.TrimSpace(decision.Reason)
|
|
||||||
decision.SupportingChunkIDs = trimStringSlice(decision.SupportingChunkIDs)
|
|
||||||
decision.MissingInfo = trimStringSlice(decision.MissingInfo)
|
|
||||||
if decision.Answerable && len(decision.SupportingChunkIDs) == 0 {
|
|
||||||
return answerabilityDecision{}, fmt.Errorf("answerable decision requires supportingChunkIds")
|
|
||||||
}
|
|
||||||
return decision, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func trimMarkdownFence(raw string) string {
|
|
||||||
text := strings.TrimSpace(raw)
|
|
||||||
if !strings.HasPrefix(text, "```") {
|
|
||||||
return text
|
|
||||||
}
|
|
||||||
lines := strings.Split(text, "\n")
|
|
||||||
if len(lines) == 0 {
|
|
||||||
return text
|
|
||||||
}
|
|
||||||
if strings.HasPrefix(strings.TrimSpace(lines[0]), "```") {
|
|
||||||
lines = lines[1:]
|
|
||||||
}
|
|
||||||
if len(lines) > 0 && strings.HasPrefix(strings.TrimSpace(lines[len(lines)-1]), "```") {
|
|
||||||
lines = lines[:len(lines)-1]
|
|
||||||
}
|
|
||||||
return strings.TrimSpace(strings.Join(lines, "\n"))
|
|
||||||
}
|
|
||||||
|
|
||||||
func trimStringSlice(items []string) []string {
|
|
||||||
if len(items) == 0 {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
ret := make([]string, 0, len(items))
|
|
||||||
for _, item := range items {
|
|
||||||
item = strings.TrimSpace(item)
|
|
||||||
if item == "" {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
ret = append(ret, item)
|
|
||||||
}
|
|
||||||
return ret
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *answerabilityGateState) recordAnswerability(status string, reason string, err error) {
|
func (s *answerabilityGateState) recordAnswerability(status string, reason string, err error) {
|
||||||
s.recordAnswerabilityWithLatency(status, reason, err, time.Time{})
|
s.recordAnswerabilityWithLatency(status, reason, err, time.Time{})
|
||||||
}
|
}
|
||||||
@@ -565,11 +282,9 @@ func (s *answerabilityGateState) recordAnswerabilityWithLatency(status string, r
|
|||||||
errorMessage = err.Error()
|
errorMessage = err.Error()
|
||||||
}
|
}
|
||||||
data := callbacks.AnswerabilityTraceData{
|
data := callbacks.AnswerabilityTraceData{
|
||||||
Status: status,
|
Status: status,
|
||||||
Reason: strings.TrimSpace(reason),
|
Reason: strings.TrimSpace(reason),
|
||||||
SupportingChunkIDs: append([]string(nil), s.Grade.SupportingChunkIDs...),
|
ErrorMessage: errorMessage,
|
||||||
MissingInfo: append([]string(nil), s.Grade.MissingInfo...),
|
|
||||||
ErrorMessage: errorMessage,
|
|
||||||
}
|
}
|
||||||
if !started.IsZero() {
|
if !started.IsZero() {
|
||||||
data.LatencyMs = time.Since(started).Milliseconds()
|
data.LatencyMs = time.Since(started).Milliseconds()
|
||||||
|
|||||||
@@ -0,0 +1,275 @@
|
|||||||
|
package executor
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"cs-agent/internal/ai/rag"
|
||||||
|
"cs-agent/internal/ai/runtime/internal/impl/callbacks"
|
||||||
|
"cs-agent/internal/ai/runtime/internal/impl/retrievers"
|
||||||
|
"cs-agent/internal/models"
|
||||||
|
"cs-agent/internal/pkg/enums"
|
||||||
|
|
||||||
|
"github.com/cloudwego/eino/schema"
|
||||||
|
)
|
||||||
|
|
||||||
|
type fakeKnowledgeContextRetriever struct {
|
||||||
|
knowledgeBaseIDs []int64
|
||||||
|
result *retrievers.KnowledgeRetrieveResult
|
||||||
|
err error
|
||||||
|
called bool
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *fakeKnowledgeContextRetriever) KnowledgeBaseIDs() []int64 {
|
||||||
|
return append([]int64(nil), r.knowledgeBaseIDs...)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *fakeKnowledgeContextRetriever) RetrieveContextByOptions(ctx context.Context, opts retrievers.KnowledgeRetrieveOptions, query string) (*retrievers.KnowledgeRetrieveResult, error) {
|
||||||
|
r.called = true
|
||||||
|
if r.err != nil {
|
||||||
|
return nil, r.err
|
||||||
|
}
|
||||||
|
if r.result != nil {
|
||||||
|
return r.result, nil
|
||||||
|
}
|
||||||
|
return &retrievers.KnowledgeRetrieveResult{
|
||||||
|
KnowledgeBaseIDs: append([]int64(nil), r.knowledgeBaseIDs...),
|
||||||
|
Query: query,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func newTestKnowledgePolicyGate(retriever knowledgeContextRetriever) *KnowledgeAnswerabilityGate {
|
||||||
|
return &KnowledgeAnswerabilityGate{
|
||||||
|
newRetriever: func(aiAgent models.AIAgent) knowledgeContextRetriever {
|
||||||
|
return retriever
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func newKnowledgePolicyRunInput(content string, knowledgeIDs string) RunInput {
|
||||||
|
return RunInput{
|
||||||
|
UserMessage: models.Message{Content: content},
|
||||||
|
AIAgent: models.AIAgent{
|
||||||
|
KnowledgeIDs: knowledgeIDs,
|
||||||
|
FallbackMode: enums.AIAgentFallbackModeSuggestRetry,
|
||||||
|
FallbackMessage: "我暂时没有找到足够准确的信息。你可以补充更具体的问题,我再继续帮你查。",
|
||||||
|
AllowedMCPTools: "[]",
|
||||||
|
},
|
||||||
|
AIConfig: models.AIConfig{ModelName: "fake-model"},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func messagesContainContent(messages []*schema.Message, needle string) bool {
|
||||||
|
for _, message := range messages {
|
||||||
|
if message != nil && strings.Contains(message.Content, needle) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestKnowledgePolicyEvaluateInjectsNoContextInstructionWithoutFallback(t *testing.T) {
|
||||||
|
collector := callbacks.NewRuntimeTraceCollector()
|
||||||
|
gate := newTestKnowledgePolicyGate(&fakeKnowledgeContextRetriever{
|
||||||
|
knowledgeBaseIDs: []int64{1},
|
||||||
|
result: &retrievers.KnowledgeRetrieveResult{
|
||||||
|
KnowledgeBaseIDs: []int64{1},
|
||||||
|
},
|
||||||
|
})
|
||||||
|
|
||||||
|
state, err := gate.Evaluate(context.Background(), answerabilityGateInput{
|
||||||
|
Request: newKnowledgePolicyRunInput("你好", "1"),
|
||||||
|
Summary: &RunResult{},
|
||||||
|
Collector: collector,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Evaluate returned error: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if state.FallbackReply != "" {
|
||||||
|
t.Fatalf("expected no direct fallback, got %q", state.FallbackReply)
|
||||||
|
}
|
||||||
|
if state.SkipGate {
|
||||||
|
t.Fatal("expected configured knowledge to inject policy, not skip")
|
||||||
|
}
|
||||||
|
if len(state.Decision.Instructions) != 1 {
|
||||||
|
t.Fatalf("expected one no-context instruction, got %d", len(state.Decision.Instructions))
|
||||||
|
}
|
||||||
|
if !strings.Contains(state.Decision.Instructions[0].Content, "当前没有从知识库检索到可用资料") {
|
||||||
|
t.Fatalf("unexpected no-context instruction: %q", state.Decision.Instructions[0].Content)
|
||||||
|
}
|
||||||
|
if !strings.Contains(state.Decision.Instructions[0].Content, "不得编造") {
|
||||||
|
t.Fatalf("expected anti-hallucination policy, got %q", state.Decision.Instructions[0].Content)
|
||||||
|
}
|
||||||
|
if collector.Data.Answerability.Status != answerabilityStatusNoContext {
|
||||||
|
t.Fatalf("unexpected policy status: %q", collector.Data.Answerability.Status)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBuildRunMessagesContinuesAgentFlowWhenNoContext(t *testing.T) {
|
||||||
|
summary := &RunResult{}
|
||||||
|
gate := newTestKnowledgePolicyGate(&fakeKnowledgeContextRetriever{
|
||||||
|
knowledgeBaseIDs: []int64{1},
|
||||||
|
result: &retrievers.KnowledgeRetrieveResult{
|
||||||
|
KnowledgeBaseIDs: []int64{1},
|
||||||
|
},
|
||||||
|
})
|
||||||
|
|
||||||
|
messages := buildRunMessages(context.Background(), newKnowledgePolicyRunInput("你好", "1"), summary, nil, gate)
|
||||||
|
|
||||||
|
if summary.ReplyText != "" {
|
||||||
|
t.Fatalf("expected no early fallback reply, got %q", summary.ReplyText)
|
||||||
|
}
|
||||||
|
if !messagesContainContent(messages, "当前没有从知识库检索到可用资料") {
|
||||||
|
t.Fatalf("expected no-context instruction in messages: %#v", messages)
|
||||||
|
}
|
||||||
|
if !messagesContainContent(messages, "你好") {
|
||||||
|
t.Fatalf("expected current user message to remain in messages: %#v", messages)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestKnowledgePolicyEvaluateInjectsGroundedInstructionAndContext(t *testing.T) {
|
||||||
|
collector := callbacks.NewRuntimeTraceCollector()
|
||||||
|
gate := newTestKnowledgePolicyGate(&fakeKnowledgeContextRetriever{
|
||||||
|
knowledgeBaseIDs: []int64{1},
|
||||||
|
result: &retrievers.KnowledgeRetrieveResult{
|
||||||
|
KnowledgeBaseIDs: []int64{1},
|
||||||
|
Hits: []rag.RetrieveResult{
|
||||||
|
{KnowledgeBaseID: 1, DocumentID: 10, ChunkID: 101, Content: "退款规则:订单发货前可以申请退款。", Score: 0.91},
|
||||||
|
},
|
||||||
|
ContextResults: []rag.RetrieveResult{
|
||||||
|
{KnowledgeBaseID: 1, DocumentID: 10, ChunkID: 101, Content: "退款规则:订单发货前可以申请退款。", Score: 0.91},
|
||||||
|
},
|
||||||
|
ContextText: "知识库片段:退款规则:订单发货前可以申请退款。",
|
||||||
|
AnswerMode: enums.KnowledgeAnswerModeStrict,
|
||||||
|
},
|
||||||
|
})
|
||||||
|
|
||||||
|
state, err := gate.Evaluate(context.Background(), answerabilityGateInput{
|
||||||
|
Request: newKnowledgePolicyRunInput("怎么退款", "1"),
|
||||||
|
Summary: &RunResult{},
|
||||||
|
Collector: collector,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Evaluate returned error: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if state.FallbackReply != "" {
|
||||||
|
t.Fatalf("expected no direct fallback, got %q", state.FallbackReply)
|
||||||
|
}
|
||||||
|
if len(state.Decision.Instructions) != 1 {
|
||||||
|
t.Fatalf("expected one grounded instruction, got %d", len(state.Decision.Instructions))
|
||||||
|
}
|
||||||
|
if !strings.Contains(state.Decision.Instructions[0].Content, "知识库回答约束") {
|
||||||
|
t.Fatalf("unexpected grounded instruction: %q", state.Decision.Instructions[0].Content)
|
||||||
|
}
|
||||||
|
if collector.Data.Answerability.Status != answerabilityStatusHasContext {
|
||||||
|
t.Fatalf("unexpected policy status: %q", collector.Data.Answerability.Status)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBuildRunMessagesInjectsRetrievedContextWhenHasContext(t *testing.T) {
|
||||||
|
summary := &RunResult{}
|
||||||
|
gate := newTestKnowledgePolicyGate(&fakeKnowledgeContextRetriever{
|
||||||
|
knowledgeBaseIDs: []int64{1},
|
||||||
|
result: &retrievers.KnowledgeRetrieveResult{
|
||||||
|
KnowledgeBaseIDs: []int64{1},
|
||||||
|
Hits: []rag.RetrieveResult{
|
||||||
|
{KnowledgeBaseID: 1, DocumentID: 10, ChunkID: 101, Content: "退款规则:订单发货前可以申请退款。", Score: 0.91},
|
||||||
|
},
|
||||||
|
ContextText: "知识库片段:退款规则:订单发货前可以申请退款。",
|
||||||
|
AnswerMode: enums.KnowledgeAnswerModeStrict,
|
||||||
|
},
|
||||||
|
})
|
||||||
|
|
||||||
|
messages := buildRunMessages(context.Background(), newKnowledgePolicyRunInput("怎么退款", "1"), summary, nil, gate)
|
||||||
|
|
||||||
|
if summary.ReplyText != "" {
|
||||||
|
t.Fatalf("expected no fallback, got %q", summary.ReplyText)
|
||||||
|
}
|
||||||
|
if !messagesContainContent(messages, "知识库回答约束") {
|
||||||
|
t.Fatalf("expected knowledge instruction in messages: %#v", messages)
|
||||||
|
}
|
||||||
|
if !messagesContainContent(messages, "退款规则") {
|
||||||
|
t.Fatalf("expected retrieved context in messages: %#v", messages)
|
||||||
|
}
|
||||||
|
if !messagesContainContent(messages, "怎么退款") {
|
||||||
|
t.Fatalf("expected current user message in messages: %#v", messages)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestKnowledgePolicyEvaluateSkipsWhenNoKnowledgeConfigured(t *testing.T) {
|
||||||
|
retriever := &fakeKnowledgeContextRetriever{}
|
||||||
|
collector := callbacks.NewRuntimeTraceCollector()
|
||||||
|
gate := newTestKnowledgePolicyGate(retriever)
|
||||||
|
|
||||||
|
state, err := gate.Evaluate(context.Background(), answerabilityGateInput{
|
||||||
|
Request: newKnowledgePolicyRunInput("你好", ""),
|
||||||
|
Collector: collector,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Evaluate returned error: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if !state.SkipGate {
|
||||||
|
t.Fatal("expected skip without knowledge")
|
||||||
|
}
|
||||||
|
if retriever.called {
|
||||||
|
t.Fatal("expected retriever not to run without configured knowledge")
|
||||||
|
}
|
||||||
|
if collector.Data.Answerability.Status != answerabilityStatusSkipped {
|
||||||
|
t.Fatalf("unexpected status: %q", collector.Data.Answerability.Status)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestKnowledgePolicyEvaluateSkipsRuntimeActionIntent(t *testing.T) {
|
||||||
|
retriever := &fakeKnowledgeContextRetriever{knowledgeBaseIDs: []int64{1}}
|
||||||
|
collector := callbacks.NewRuntimeTraceCollector()
|
||||||
|
gate := newTestKnowledgePolicyGate(retriever)
|
||||||
|
|
||||||
|
state, err := gate.Evaluate(context.Background(), answerabilityGateInput{
|
||||||
|
Request: newKnowledgePolicyRunInput("帮我转人工", "1"),
|
||||||
|
Collector: collector,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Evaluate returned error: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if !state.SkipGate {
|
||||||
|
t.Fatal("expected runtime action to skip knowledge policy")
|
||||||
|
}
|
||||||
|
if retriever.called {
|
||||||
|
t.Fatal("expected retriever not to run for runtime action")
|
||||||
|
}
|
||||||
|
if collector.Data.Answerability.Status != answerabilityStatusSkipped {
|
||||||
|
t.Fatalf("unexpected status: %q", collector.Data.Answerability.Status)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestKnowledgePolicyEvaluateFallsBackOnRetrieverError(t *testing.T) {
|
||||||
|
collector := callbacks.NewRuntimeTraceCollector()
|
||||||
|
gate := newTestKnowledgePolicyGate(&fakeKnowledgeContextRetriever{
|
||||||
|
knowledgeBaseIDs: []int64{1},
|
||||||
|
err: errors.New("vector store unavailable"),
|
||||||
|
})
|
||||||
|
|
||||||
|
state, err := gate.Evaluate(context.Background(), answerabilityGateInput{
|
||||||
|
Request: newKnowledgePolicyRunInput("怎么退款", "1"),
|
||||||
|
Collector: collector,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Evaluate returned error: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if !strings.Contains(state.FallbackReply, "我暂时没有找到足够准确的信息") {
|
||||||
|
t.Fatalf("expected configured fallback on retrieval error, got %q", state.FallbackReply)
|
||||||
|
}
|
||||||
|
if collector.Data.Answerability.Status != answerabilityStatusUnanswerable {
|
||||||
|
t.Fatalf("unexpected status: %q", collector.Data.Answerability.Status)
|
||||||
|
}
|
||||||
|
if collector.Data.Answerability.Reason != "knowledge retrieval failed" {
|
||||||
|
t.Fatalf("unexpected reason: %q", collector.Data.Answerability.Reason)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -40,6 +40,19 @@ func buildKnowledgeGuardDecision(aiAgent models.AIAgent, retrieveResult *retriev
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func buildKnowledgeNoContextDecision(aiAgent models.AIAgent, knowledgeBaseIDs []int64) knowledgeGuardDecision {
|
||||||
|
if len(knowledgeBaseIDs) == 0 {
|
||||||
|
return knowledgeGuardDecision{}
|
||||||
|
}
|
||||||
|
instruction := buildKnowledgeNoContextInstruction(resolveKnowledgeFallbackReply(aiAgent))
|
||||||
|
if instruction == "" {
|
||||||
|
return knowledgeGuardDecision{}
|
||||||
|
}
|
||||||
|
return knowledgeGuardDecision{
|
||||||
|
Instructions: []*schema.Message{schema.SystemMessage(instruction)},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func resolveKnowledgeFallbackReply(aiAgent models.AIAgent) string {
|
func resolveKnowledgeFallbackReply(aiAgent models.AIAgent) string {
|
||||||
if reply := strings.TrimSpace(aiAgent.FallbackMessage); reply != "" {
|
if reply := strings.TrimSpace(aiAgent.FallbackMessage); reply != "" {
|
||||||
return reply
|
return reply
|
||||||
@@ -66,7 +79,20 @@ func buildKnowledgeRuntimeInstruction(answerMode enums.KnowledgeAnswerMode, fall
|
|||||||
fallbackReply = "当前知识库暂无明确信息。"
|
fallbackReply = "当前知识库暂无明确信息。"
|
||||||
}
|
}
|
||||||
if answerMode == enums.KnowledgeAnswerModeAssist {
|
if answerMode == enums.KnowledgeAnswerModeAssist {
|
||||||
return "知识库回答约束:优先依据后续提供的知识片段回答,可以做轻度归纳,但不要编造片段中未提供的事实。回答中的具体事实、步骤、承诺必须能被知识片段直接支持;若知识片段不足以直接支持答案,必须明确回复:" + fallbackReply
|
return "知识库回答约束:优先依据后续提供的知识片段回答,可以做轻度归纳,但不要编造片段中未提供的事实。回答中的具体事实、步骤、承诺、价格、时效、政策必须能被知识片段直接支持;若知识片段不足以直接支持答案,必须明确回复:" + fallbackReply
|
||||||
}
|
}
|
||||||
return "知识库回答约束:本轮只能依据后续提供的知识片段回答,不得使用模型常识补充未提供的事实,不得输出知识片段外的具体事实、步骤、承诺或建议。若知识片段不足以支持回答,必须明确回复:" + fallbackReply
|
return "知识库回答约束:本轮只能依据后续提供的知识片段回答,不得使用模型常识补充未提供的事实,不得输出知识片段外的具体事实、步骤、承诺、建议、价格、时效或政策。若知识片段不足以支持回答,必须明确回复:" + fallbackReply
|
||||||
|
}
|
||||||
|
|
||||||
|
func buildKnowledgeNoContextInstruction(fallbackReply string) string {
|
||||||
|
fallbackReply = strings.TrimSpace(fallbackReply)
|
||||||
|
if fallbackReply == "" {
|
||||||
|
fallbackReply = "当前知识库暂无明确信息。"
|
||||||
|
}
|
||||||
|
return "知识库检索状态:当前没有从知识库检索到可用资料。\n" +
|
||||||
|
"回复策略:\n" +
|
||||||
|
"1. 如果用户只是寒暄、问候、感谢、确认或结束语,可以自然、简短地回复。\n" +
|
||||||
|
"2. 如果用户表达不清楚或缺少上下文,应追问具体场景、对象、报错信息或操作步骤。\n" +
|
||||||
|
"3. 如果用户询问业务事实、规则、价格、流程、配置、时效、承诺、售后、退款、权限或政策,不得编造答案,必须明确回复:" + fallbackReply + "\n" +
|
||||||
|
"4. 不得输出知识库未提供的具体事实、流程、承诺、价格、时效或政策。"
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user