feat: add LanceDB provider support with lifecycle tests and Makefile enhancements
This commit is contained in:
mlogclub
+3
-1
@@ -5,6 +5,8 @@
|
||||
|
||||
data/
|
||||
config/config.yaml
|
||||
include/
|
||||
lib/
|
||||
|
||||
dist/
|
||||
__debug_bin*
|
||||
@@ -12,4 +14,4 @@ node_modules/
|
||||
.next/
|
||||
*.tsbuildinfo
|
||||
|
||||
.superpowers
|
||||
.superpowers
|
||||
|
||||
@@ -8,12 +8,59 @@ PNPM ?= pnpm
|
||||
GOOS ?= $(shell $(GO) env GOOS)
|
||||
GOARCH ?= $(shell $(GO) env GOARCH)
|
||||
DEV_SERVER_URL ?= http://127.0.0.1:8083
|
||||
LANCEDB_VERSION ?= v0.1.2
|
||||
LANCEDB_DOWNLOAD_SCRIPT ?= https://raw.githubusercontent.com/lancedb/lancedb-go/main/scripts/download-artifacts.sh
|
||||
LANCEDB_TEST_PKGS ?= ./internal/ai/rag/vectordb
|
||||
|
||||
UNAME_S := $(shell uname -s)
|
||||
UNAME_M := $(shell uname -m)
|
||||
|
||||
ifeq ($(UNAME_M),x86_64)
|
||||
LANCEDB_ARCH := amd64
|
||||
else ifeq ($(UNAME_M),amd64)
|
||||
LANCEDB_ARCH := amd64
|
||||
else ifeq ($(UNAME_M),arm64)
|
||||
LANCEDB_ARCH := arm64
|
||||
else ifeq ($(UNAME_M),aarch64)
|
||||
LANCEDB_ARCH := arm64
|
||||
else
|
||||
LANCEDB_ARCH := unsupported
|
||||
endif
|
||||
|
||||
ifeq ($(UNAME_S),Darwin)
|
||||
LANCEDB_PLATFORM := darwin
|
||||
LANCEDB_SYSTEM_LDFLAGS := -framework Security -framework CoreFoundation
|
||||
else ifeq ($(UNAME_S),Linux)
|
||||
LANCEDB_PLATFORM := linux
|
||||
LANCEDB_SYSTEM_LDFLAGS := -lm -ldl -lpthread
|
||||
else ifneq (,$(findstring MINGW,$(UNAME_S)))
|
||||
LANCEDB_PLATFORM := windows
|
||||
LANCEDB_ARCH := amd64
|
||||
LANCEDB_SYSTEM_LDFLAGS :=
|
||||
else ifneq (,$(findstring MSYS,$(UNAME_S)))
|
||||
LANCEDB_PLATFORM := windows
|
||||
LANCEDB_ARCH := amd64
|
||||
LANCEDB_SYSTEM_LDFLAGS :=
|
||||
else ifneq (,$(findstring CYGWIN,$(UNAME_S)))
|
||||
LANCEDB_PLATFORM := windows
|
||||
LANCEDB_ARCH := amd64
|
||||
LANCEDB_SYSTEM_LDFLAGS :=
|
||||
else
|
||||
LANCEDB_PLATFORM := unsupported
|
||||
LANCEDB_SYSTEM_LDFLAGS :=
|
||||
endif
|
||||
|
||||
LANCEDB_PLATFORM_ARCH := $(LANCEDB_PLATFORM)_$(LANCEDB_ARCH)
|
||||
LANCEDB_NATIVE_LIB := $(CURDIR)/lib/$(LANCEDB_PLATFORM_ARCH)/liblancedb_go.a
|
||||
LANCEDB_CGO_CFLAGS := -I$(CURDIR)/include
|
||||
LANCEDB_CGO_LDFLAGS := $(LANCEDB_NATIVE_LIB) $(LANCEDB_SYSTEM_LDFLAGS)
|
||||
|
||||
.DEFAULT_GOAL := help
|
||||
|
||||
.PHONY: all help build build-go build-linux release run run-go dev test check clean clean-web \
|
||||
web-install web-dev web-build-spa ensure-spa build-spa web-build-ssr web-typecheck web-lint \
|
||||
generator enums migration testdata
|
||||
generator enums migration testdata lancedb-platform-info lancedb-artifacts lancedb-check \
|
||||
build-lancedb test-lancedb clean-lancedb-artifacts
|
||||
|
||||
all: build
|
||||
|
||||
@@ -39,6 +86,9 @@ help:
|
||||
@echo " make enums Generate frontend enums"
|
||||
@echo " make migration Run migration command"
|
||||
@echo " make testdata Run testdata generator"
|
||||
@echo " make lancedb-artifacts Download LanceDB native libraries for this platform"
|
||||
@echo " make build-lancedb Build Go binary with LanceDB provider enabled"
|
||||
@echo " make test-lancedb Run LanceDB provider tests with native libraries"
|
||||
|
||||
build: web-build-spa
|
||||
@$(MAKE) build-go
|
||||
@@ -128,3 +178,47 @@ migration:
|
||||
|
||||
testdata:
|
||||
@$(GO) run ./cmd/testdata -lang $(or $(TESTDATA_LANG),zh)
|
||||
|
||||
lancedb-platform-info:
|
||||
@echo "LanceDB platform information:"
|
||||
@echo " OS/arch: $(UNAME_S)/$(UNAME_M)"
|
||||
@echo " platform-arch: $(LANCEDB_PLATFORM_ARCH)"
|
||||
@echo " version: $(LANCEDB_VERSION)"
|
||||
@echo " CGO_CFLAGS: $(LANCEDB_CGO_CFLAGS)"
|
||||
@echo " CGO_LDFLAGS: $(LANCEDB_CGO_LDFLAGS)"
|
||||
@echo " native library: $(LANCEDB_NATIVE_LIB)"
|
||||
|
||||
lancedb-artifacts:
|
||||
@if [ "$(LANCEDB_PLATFORM)" = "unsupported" ] || [ "$(LANCEDB_ARCH)" = "unsupported" ]; then \
|
||||
echo "Unsupported LanceDB platform: $(UNAME_S)/$(UNAME_M)"; \
|
||||
exit 1; \
|
||||
fi
|
||||
@if [ -f "$(LANCEDB_NATIVE_LIB)" ] && [ -f "$(CURDIR)/include/lancedb.h" ]; then \
|
||||
echo "LanceDB native artifacts already exist for $(LANCEDB_PLATFORM_ARCH)."; \
|
||||
else \
|
||||
echo "Downloading LanceDB native artifacts $(LANCEDB_VERSION) for $(LANCEDB_PLATFORM_ARCH)..."; \
|
||||
curl -sSL "$(LANCEDB_DOWNLOAD_SCRIPT)" | bash -s "$(LANCEDB_VERSION)"; \
|
||||
fi
|
||||
|
||||
lancedb-check: lancedb-artifacts
|
||||
@if [ ! -f "$(LANCEDB_NATIVE_LIB)" ]; then \
|
||||
echo "Missing LanceDB native library: $(LANCEDB_NATIVE_LIB)"; \
|
||||
exit 1; \
|
||||
fi
|
||||
@if [ ! -f "$(CURDIR)/include/lancedb.h" ]; then \
|
||||
echo "Missing LanceDB header: $(CURDIR)/include/lancedb.h"; \
|
||||
exit 1; \
|
||||
fi
|
||||
|
||||
build-lancedb: ensure-spa lancedb-check
|
||||
@echo "Building $(APP) with LanceDB provider enabled..."
|
||||
@CGO_ENABLED=1 CGO_CFLAGS="$(LANCEDB_CGO_CFLAGS)" CGO_LDFLAGS="$(LANCEDB_CGO_LDFLAGS)" \
|
||||
$(GO) build -tags lancedb -v -o $(APP) $(MAIN)
|
||||
|
||||
test-lancedb: lancedb-check
|
||||
@echo "Running LanceDB tests with native libraries..."
|
||||
@CGO_ENABLED=1 CGO_CFLAGS="$(LANCEDB_CGO_CFLAGS)" CGO_LDFLAGS="$(LANCEDB_CGO_LDFLAGS)" \
|
||||
$(GO) test -tags lancedb $(LANCEDB_TEST_PKGS)
|
||||
|
||||
clean-lancedb-artifacts:
|
||||
@rm -rf lib include
|
||||
|
||||
@@ -0,0 +1,102 @@
|
||||
//go:build lancedb
|
||||
|
||||
package vectordb
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"agent-desk/internal/pkg/config"
|
||||
)
|
||||
|
||||
func TestLanceDBProviderVectorLifecycle(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
provider, err := NewLanceDBProvider(&config.LanceDBVectorDBConfig{Path: t.TempDir()})
|
||||
if err != nil {
|
||||
t.Fatalf("NewLanceDBProvider() error = %v", err)
|
||||
}
|
||||
defer provider.Close()
|
||||
|
||||
const collectionName = "knowledge_chunks"
|
||||
if err := provider.CreateCollection(ctx, collectionName, 3); err != nil {
|
||||
t.Fatalf("CreateCollection() error = %v", err)
|
||||
}
|
||||
|
||||
vectors := []Vector{
|
||||
{
|
||||
ID: "a",
|
||||
Vector: []float32{1, 0, 0},
|
||||
Payload: ChunkPayload{
|
||||
KnowledgeBaseID: 10,
|
||||
DocumentID: 100,
|
||||
Title: "A",
|
||||
Content: "alpha",
|
||||
},
|
||||
},
|
||||
{
|
||||
ID: "b",
|
||||
Vector: []float32{0, 1, 0},
|
||||
Payload: ChunkPayload{
|
||||
KnowledgeBaseID: 20,
|
||||
DocumentID: 200,
|
||||
Title: "B",
|
||||
Content: "beta",
|
||||
},
|
||||
},
|
||||
}
|
||||
if err := provider.UpsertVectors(ctx, collectionName, vectors); err != nil {
|
||||
t.Fatalf("UpsertVectors() error = %v", err)
|
||||
}
|
||||
|
||||
info, err := provider.GetCollection(ctx, collectionName)
|
||||
if err != nil {
|
||||
t.Fatalf("GetCollection() error = %v", err)
|
||||
}
|
||||
if info.Dimension != 3 {
|
||||
t.Fatalf("CollectionInfo.Dimension = %d, want 3", info.Dimension)
|
||||
}
|
||||
if info.PointCount != 2 {
|
||||
t.Fatalf("CollectionInfo.PointCount = %d, want 2", info.PointCount)
|
||||
}
|
||||
|
||||
results, err := provider.Search(ctx, &SearchRequest{
|
||||
CollectionName: collectionName,
|
||||
Vector: []float32{1, 0, 0},
|
||||
TopK: 5,
|
||||
ScoreThreshold: 0,
|
||||
Filter: &SearchFilter{
|
||||
KnowledgeBaseIDs: []int64{10},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Search() error = %v", err)
|
||||
}
|
||||
if len(results) != 1 {
|
||||
t.Fatalf("Search() returned %d results, want 1: %#v", len(results), results)
|
||||
}
|
||||
if results[0].ID != "a" {
|
||||
t.Fatalf("Search()[0].ID = %q, want %q", results[0].ID, "a")
|
||||
}
|
||||
if results[0].Payload.KnowledgeBaseID != 10 {
|
||||
t.Fatalf("Search()[0].Payload.KnowledgeBaseID = %d, want 10", results[0].Payload.KnowledgeBaseID)
|
||||
}
|
||||
|
||||
if err := provider.DeleteVectors(ctx, collectionName, []string{"a"}); err != nil {
|
||||
t.Fatalf("DeleteVectors() error = %v", err)
|
||||
}
|
||||
results, err = provider.Search(ctx, &SearchRequest{
|
||||
CollectionName: collectionName,
|
||||
Vector: []float32{1, 0, 0},
|
||||
TopK: 5,
|
||||
ScoreThreshold: 0,
|
||||
Filter: &SearchFilter{
|
||||
KnowledgeBaseIDs: []int64{10},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Search() after delete error = %v", err)
|
||||
}
|
||||
if len(results) != 0 {
|
||||
t.Fatalf("Search() after delete returned %d results, want 0: %#v", len(results), results)
|
||||
}
|
||||
}
|
||||
@@ -1,3 +1,5 @@
|
||||
//go:build !lancedb
|
||||
|
||||
package vectordb
|
||||
|
||||
import (
|
||||
|
||||
Reference in New Issue
Block a user