package utils import ( "code.tczkiot.com/wlw/ai-agent/internal/pkg/enums" "strings" "golang.org/x/net/html" ) type ContentChunkType string const ( ContentChunkTypeText ContentChunkType = "text" ContentChunkTypeImage ContentChunkType = "image" ) type ContentChunk struct { Type ContentChunkType // Content string // text content or image url AssetID string // image asset id Provider enums.AssetProvider // image provider StorageKey string // image storage key } func SplitHTMLContentChunks(content string) ([]ContentChunk, error) { root, err := html.Parse(strings.NewReader("
" + strings.TrimSpace(content) + "
")) if err != nil { return nil, err } var ( chunks []ContentChunk buffer strings.Builder ) flushText := func() { text := normalizeContentChunkText(buffer.String()) buffer.Reset() if text == "" { return } chunks = append(chunks, ContentChunk{ Type: ContentChunkTypeText, Content: text, }) } var walk func(*html.Node) walk = func(node *html.Node) { if node == nil { return } switch node.Type { case html.TextNode: buffer.WriteString(node.Data) case html.ElementNode: switch node.Data { case "br": buffer.WriteString("\n") case "p", "div", "li", "blockquote": buffer.WriteString("\n") case "img": flushText() src := strings.TrimSpace(findContentChunkHTMLAttr(node, "src")) assetID := strings.TrimSpace(findContentChunkHTMLAttr(node, "data-asset-id")) provider := enums.AssetProvider(strings.TrimSpace(findContentChunkHTMLAttr(node, "data-provider"))) storageKey := strings.TrimSpace(findContentChunkHTMLAttr(node, "data-storage-key")) if src == "" && assetID == "" { return } chunks = append(chunks, ContentChunk{ Type: ContentChunkTypeImage, Content: src, AssetID: assetID, Provider: provider, StorageKey: storageKey, }) return } } for child := node.FirstChild; child != nil; child = child.NextSibling { walk(child) } if node.Type == html.ElementNode { switch node.Data { case "p", "div", "li", "blockquote": buffer.WriteString("\n") } } } walk(root) flushText() return chunks, nil } func normalizeContentChunkText(value string) string { value = strings.ReplaceAll(value, "\u00a0", " ") lines := strings.Split(value, "\n") normalizedLines := make([]string, 0, len(lines)) for _, line := range lines { line = strings.Join(strings.Fields(strings.TrimSpace(line)), " ") if line == "" { if len(normalizedLines) > 0 && normalizedLines[len(normalizedLines)-1] != "" { normalizedLines = append(normalizedLines, "") } continue } normalizedLines = append(normalizedLines, line) } for len(normalizedLines) > 0 && normalizedLines[0] == "" { normalizedLines = normalizedLines[1:] } for len(normalizedLines) > 0 && normalizedLines[len(normalizedLines)-1] == "" { normalizedLines = normalizedLines[:len(normalizedLines)-1] } return strings.Join(normalizedLines, "\n") } func findContentChunkHTMLAttr(node *html.Node, key string) string { for _, attr := range node.Attr { if strings.EqualFold(attr.Key, key) { return attr.Val } } return "" }