diff --git a/.gitignore b/.gitignore index 27b47b2..f059b7f 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,8 @@ data/ config/config.yaml +include/ +lib/ dist/ __debug_bin* @@ -12,4 +14,4 @@ node_modules/ .next/ *.tsbuildinfo -.superpowers \ No newline at end of file +.superpowers diff --git a/Makefile b/Makefile index 951aa5a..d95d177 100644 --- a/Makefile +++ b/Makefile @@ -8,12 +8,59 @@ PNPM ?= pnpm GOOS ?= $(shell $(GO) env GOOS) GOARCH ?= $(shell $(GO) env GOARCH) DEV_SERVER_URL ?= http://127.0.0.1:8083 +LANCEDB_VERSION ?= v0.1.2 +LANCEDB_DOWNLOAD_SCRIPT ?= https://raw.githubusercontent.com/lancedb/lancedb-go/main/scripts/download-artifacts.sh +LANCEDB_TEST_PKGS ?= ./internal/ai/rag/vectordb + +UNAME_S := $(shell uname -s) +UNAME_M := $(shell uname -m) + +ifeq ($(UNAME_M),x86_64) + LANCEDB_ARCH := amd64 +else ifeq ($(UNAME_M),amd64) + LANCEDB_ARCH := amd64 +else ifeq ($(UNAME_M),arm64) + LANCEDB_ARCH := arm64 +else ifeq ($(UNAME_M),aarch64) + LANCEDB_ARCH := arm64 +else + LANCEDB_ARCH := unsupported +endif + +ifeq ($(UNAME_S),Darwin) + LANCEDB_PLATFORM := darwin + LANCEDB_SYSTEM_LDFLAGS := -framework Security -framework CoreFoundation +else ifeq ($(UNAME_S),Linux) + LANCEDB_PLATFORM := linux + LANCEDB_SYSTEM_LDFLAGS := -lm -ldl -lpthread +else ifneq (,$(findstring MINGW,$(UNAME_S))) + LANCEDB_PLATFORM := windows + LANCEDB_ARCH := amd64 + LANCEDB_SYSTEM_LDFLAGS := +else ifneq (,$(findstring MSYS,$(UNAME_S))) + LANCEDB_PLATFORM := windows + LANCEDB_ARCH := amd64 + LANCEDB_SYSTEM_LDFLAGS := +else ifneq (,$(findstring CYGWIN,$(UNAME_S))) + LANCEDB_PLATFORM := windows + LANCEDB_ARCH := amd64 + LANCEDB_SYSTEM_LDFLAGS := +else + LANCEDB_PLATFORM := unsupported + LANCEDB_SYSTEM_LDFLAGS := +endif + +LANCEDB_PLATFORM_ARCH := $(LANCEDB_PLATFORM)_$(LANCEDB_ARCH) +LANCEDB_NATIVE_LIB := $(CURDIR)/lib/$(LANCEDB_PLATFORM_ARCH)/liblancedb_go.a +LANCEDB_CGO_CFLAGS := -I$(CURDIR)/include +LANCEDB_CGO_LDFLAGS := $(LANCEDB_NATIVE_LIB) $(LANCEDB_SYSTEM_LDFLAGS) .DEFAULT_GOAL := help .PHONY: all help build build-go build-linux release run run-go dev test check clean clean-web \ web-install web-dev web-build-spa ensure-spa build-spa web-build-ssr web-typecheck web-lint \ - generator enums migration testdata + generator enums migration testdata lancedb-platform-info lancedb-artifacts lancedb-check \ + build-lancedb test-lancedb clean-lancedb-artifacts all: build @@ -39,6 +86,9 @@ help: @echo " make enums Generate frontend enums" @echo " make migration Run migration command" @echo " make testdata Run testdata generator" + @echo " make lancedb-artifacts Download LanceDB native libraries for this platform" + @echo " make build-lancedb Build Go binary with LanceDB provider enabled" + @echo " make test-lancedb Run LanceDB provider tests with native libraries" build: web-build-spa @$(MAKE) build-go @@ -128,3 +178,47 @@ migration: testdata: @$(GO) run ./cmd/testdata -lang $(or $(TESTDATA_LANG),zh) + +lancedb-platform-info: + @echo "LanceDB platform information:" + @echo " OS/arch: $(UNAME_S)/$(UNAME_M)" + @echo " platform-arch: $(LANCEDB_PLATFORM_ARCH)" + @echo " version: $(LANCEDB_VERSION)" + @echo " CGO_CFLAGS: $(LANCEDB_CGO_CFLAGS)" + @echo " CGO_LDFLAGS: $(LANCEDB_CGO_LDFLAGS)" + @echo " native library: $(LANCEDB_NATIVE_LIB)" + +lancedb-artifacts: + @if [ "$(LANCEDB_PLATFORM)" = "unsupported" ] || [ "$(LANCEDB_ARCH)" = "unsupported" ]; then \ + echo "Unsupported LanceDB platform: $(UNAME_S)/$(UNAME_M)"; \ + exit 1; \ + fi + @if [ -f "$(LANCEDB_NATIVE_LIB)" ] && [ -f "$(CURDIR)/include/lancedb.h" ]; then \ + echo "LanceDB native artifacts already exist for $(LANCEDB_PLATFORM_ARCH)."; \ + else \ + echo "Downloading LanceDB native artifacts $(LANCEDB_VERSION) for $(LANCEDB_PLATFORM_ARCH)..."; \ + curl -sSL "$(LANCEDB_DOWNLOAD_SCRIPT)" | bash -s "$(LANCEDB_VERSION)"; \ + fi + +lancedb-check: lancedb-artifacts + @if [ ! -f "$(LANCEDB_NATIVE_LIB)" ]; then \ + echo "Missing LanceDB native library: $(LANCEDB_NATIVE_LIB)"; \ + exit 1; \ + fi + @if [ ! -f "$(CURDIR)/include/lancedb.h" ]; then \ + echo "Missing LanceDB header: $(CURDIR)/include/lancedb.h"; \ + exit 1; \ + fi + +build-lancedb: ensure-spa lancedb-check + @echo "Building $(APP) with LanceDB provider enabled..." + @CGO_ENABLED=1 CGO_CFLAGS="$(LANCEDB_CGO_CFLAGS)" CGO_LDFLAGS="$(LANCEDB_CGO_LDFLAGS)" \ + $(GO) build -tags lancedb -v -o $(APP) $(MAIN) + +test-lancedb: lancedb-check + @echo "Running LanceDB tests with native libraries..." + @CGO_ENABLED=1 CGO_CFLAGS="$(LANCEDB_CGO_CFLAGS)" CGO_LDFLAGS="$(LANCEDB_CGO_LDFLAGS)" \ + $(GO) test -tags lancedb $(LANCEDB_TEST_PKGS) + +clean-lancedb-artifacts: + @rm -rf lib include diff --git a/internal/ai/rag/vectordb/lancedb_test.go b/internal/ai/rag/vectordb/lancedb_test.go new file mode 100644 index 0000000..cdff34d --- /dev/null +++ b/internal/ai/rag/vectordb/lancedb_test.go @@ -0,0 +1,102 @@ +//go:build lancedb + +package vectordb + +import ( + "context" + "testing" + + "agent-desk/internal/pkg/config" +) + +func TestLanceDBProviderVectorLifecycle(t *testing.T) { + ctx := context.Background() + provider, err := NewLanceDBProvider(&config.LanceDBVectorDBConfig{Path: t.TempDir()}) + if err != nil { + t.Fatalf("NewLanceDBProvider() error = %v", err) + } + defer provider.Close() + + const collectionName = "knowledge_chunks" + if err := provider.CreateCollection(ctx, collectionName, 3); err != nil { + t.Fatalf("CreateCollection() error = %v", err) + } + + vectors := []Vector{ + { + ID: "a", + Vector: []float32{1, 0, 0}, + Payload: ChunkPayload{ + KnowledgeBaseID: 10, + DocumentID: 100, + Title: "A", + Content: "alpha", + }, + }, + { + ID: "b", + Vector: []float32{0, 1, 0}, + Payload: ChunkPayload{ + KnowledgeBaseID: 20, + DocumentID: 200, + Title: "B", + Content: "beta", + }, + }, + } + if err := provider.UpsertVectors(ctx, collectionName, vectors); err != nil { + t.Fatalf("UpsertVectors() error = %v", err) + } + + info, err := provider.GetCollection(ctx, collectionName) + if err != nil { + t.Fatalf("GetCollection() error = %v", err) + } + if info.Dimension != 3 { + t.Fatalf("CollectionInfo.Dimension = %d, want 3", info.Dimension) + } + if info.PointCount != 2 { + t.Fatalf("CollectionInfo.PointCount = %d, want 2", info.PointCount) + } + + results, err := provider.Search(ctx, &SearchRequest{ + CollectionName: collectionName, + Vector: []float32{1, 0, 0}, + TopK: 5, + ScoreThreshold: 0, + Filter: &SearchFilter{ + KnowledgeBaseIDs: []int64{10}, + }, + }) + if err != nil { + t.Fatalf("Search() error = %v", err) + } + if len(results) != 1 { + t.Fatalf("Search() returned %d results, want 1: %#v", len(results), results) + } + if results[0].ID != "a" { + t.Fatalf("Search()[0].ID = %q, want %q", results[0].ID, "a") + } + if results[0].Payload.KnowledgeBaseID != 10 { + t.Fatalf("Search()[0].Payload.KnowledgeBaseID = %d, want 10", results[0].Payload.KnowledgeBaseID) + } + + if err := provider.DeleteVectors(ctx, collectionName, []string{"a"}); err != nil { + t.Fatalf("DeleteVectors() error = %v", err) + } + results, err = provider.Search(ctx, &SearchRequest{ + CollectionName: collectionName, + Vector: []float32{1, 0, 0}, + TopK: 5, + ScoreThreshold: 0, + Filter: &SearchFilter{ + KnowledgeBaseIDs: []int64{10}, + }, + }) + if err != nil { + t.Fatalf("Search() after delete error = %v", err) + } + if len(results) != 0 { + t.Fatalf("Search() after delete returned %d results, want 0: %#v", len(results), results) + } +} diff --git a/internal/ai/rag/vectordb/provider_test.go b/internal/ai/rag/vectordb/provider_test.go index 66a223e..fd2a283 100644 --- a/internal/ai/rag/vectordb/provider_test.go +++ b/internal/ai/rag/vectordb/provider_test.go @@ -1,3 +1,5 @@ +//go:build !lancedb + package vectordb import (