wlw/srdb
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Compare commits

base: wlw:b7e2f970b455ecfc30d5b0edbdc13f09d340219d
Branches Tags
wlw:main
...
compare: wlw:392474fb89938974ce144061d8320bebbf211e6e
Branches Tags
wlw:main

2 Commits
b7e2f970b4 ... 392474fb89

Author SHA1 Message Date
bourdon
392474fb89 前端：优化 Manifest 视图文件显示 
- 文件名区域改为左右布局
- 左侧显示文件名（如 000001.sst）
- 右侧显示级别标签（如 L0、L1）
- 添加级别标签样式，使用主题色背景
 2025-10-09 20:53:37 +08:00
bourdon
4cb1d1762a 功能：实现索引查询功能 
- 为 Index 添加 Query 方法支持条件查询
- 实现 QueryBuilder 支持索引查询
- 添加索引查询测试用例
- 支持 Eq/Gt/Lt/Gte/Lte 等比较操作
 2025-10-09 20:03:28 +08:00
44 changed files with 3254 additions and 510 deletions
Expand all files Collapse all files

276
compaction_continuity_test.go
View File

@@ -1,276 +0,0 @@
package srdb
import (
"testing"
)
// TestPickL0MergeContinuity 测试 L0 合并任务的连续性
func TestPickL0MergeContinuity(t *testing.T) {
tmpDir := t.TempDir()
manifestDir := tmpDir
versionSet, err := NewVersionSet(manifestDir)
if err != nil {
t.Fatal(err)
}
defer versionSet.Close()
picker := NewPicker()
// 创建混合大小的文件：小-大-小-小
// 这是触发 bug 的场景
edit := NewVersionEdit()
// 文件1: 29MB (小文件)
edit.AddFile(&FileMetadata{
FileNumber: 1,
Level: 0,
FileSize: 29 * 1024 * 1024,
MinKey: 1,
MaxKey: 100,
RowCount: 100,
})
// 文件2: 36MB (大文件)
edit.AddFile(&FileMetadata{
FileNumber: 2,
Level: 0,
FileSize: 36 * 1024 * 1024,
MinKey: 101,
MaxKey: 200,
RowCount: 100,
})
// 文件3: 8MB (小文件)
edit.AddFile(&FileMetadata{
FileNumber: 3,
Level: 0,
FileSize: 8 * 1024 * 1024,
MinKey: 201,
MaxKey: 300,
RowCount: 100,
})
// 文件4: 15MB (小文件)
edit.AddFile(&FileMetadata{
FileNumber: 4,
Level: 0,
FileSize: 15 * 1024 * 1024,
MinKey: 301,
MaxKey: 400,
RowCount: 100,
})
edit.SetNextFileNumber(5)
err = versionSet.LogAndApply(edit)
if err != nil {
t.Fatal(err)
}
version := versionSet.GetCurrent()
// 测试 Stage 0: L0 合并任务
t.Log("=== 测试 Stage 0: L0 合并 ===")
tasks := picker.pickL0MergeTasks(version)
if len(tasks) == 0 {
t.Fatal("Expected L0 merge tasks")
}
t.Logf("找到 %d 个合并任务", len(tasks))
// 验证任务：应该只有1个任务，包含文件3和文件4
// 文件1是单个小文件，不合并（len > 1 才合并）
// 文件2是大文件，跳过
// 文件3+文件4是连续的2个小文件，应该合并
if len(tasks) != 1 {
t.Errorf("Expected 1 task, got %d", len(tasks))
for i, task := range tasks {
t.Logf("Task %d: %d files", i+1, len(task.InputFiles))
for _, f := range task.InputFiles {
t.Logf(" - File %d", f.FileNumber)
}
}
}
task1 := tasks[0]
if len(task1.InputFiles) != 2 {
t.Errorf("Task 1: expected 2 files, got %d", len(task1.InputFiles))
}
if task1.InputFiles[0].FileNumber != 3 || task1.InputFiles[1].FileNumber != 4 {
t.Errorf("Task 1: expected files 3,4, got %d,%d",
task1.InputFiles[0].FileNumber, task1.InputFiles[1].FileNumber)
}
t.Logf("✓ 合并任务: 文件3+文件4 (连续的2个小文件)")
t.Logf("✓ 文件1 (单个小文件) 不合并，留给升级阶段")
// 验证 seq 范围连续性
// 任务1: seq 201-400 (文件3+文件4)
// 文件1（seq 1-100, 单个小文件）留给升级阶段
// 文件2（seq 101-200, 大文件）留给 Stage 1
if task1.InputFiles[0].MinKey != 201 || task1.InputFiles[1].MaxKey != 400 {
t.Errorf("Task 1 seq range incorrect: [%d, %d]",
task1.InputFiles[0].MinKey, task1.InputFiles[1].MaxKey)
}
t.Logf("✓ Seq 范围正确：任务1 [201-400]")
// 测试 Stage 1: L0 升级任务
t.Log("=== 测试 Stage 1: L0 升级 ===")
upgradeTasks := picker.pickL0UpgradeTasks(version)
if len(upgradeTasks) == 0 {
t.Fatal("Expected L0 upgrade tasks")
}
// 应该有1个任务：以文件2（大文件）为中心，搭配周围的小文件
// 文件2向左收集文件1，向右收集文件3和文件4
// 总共：文件1 (29MB) + 文件2 (36MB) + 文件3 (8MB) + 文件4 (15MB) = 88MB
if len(upgradeTasks) != 1 {
t.Errorf("Expected 1 upgrade task, got %d", len(upgradeTasks))
}
upgradeTask := upgradeTasks[0]
// 应该包含所有4个文件
if len(upgradeTask.InputFiles) != 4 {
t.Errorf("Upgrade task: expected 4 files, got %d", len(upgradeTask.InputFiles))
for i, f := range upgradeTask.InputFiles {
t.Logf(" File %d: %d", i+1, f.FileNumber)
}
}
// 验证文件顺序：1, 2, 3, 4
expectedFiles := []int64{1, 2, 3, 4}
for i, expected := range expectedFiles {
if upgradeTask.InputFiles[i].FileNumber != expected {
t.Errorf("Upgrade task file %d: expected %d, got %d",
i, expected, upgradeTask.InputFiles[i].FileNumber)
}
}
if upgradeTask.OutputLevel != 1 {
t.Errorf("Upgrade task: expected OutputLevel 1, got %d", upgradeTask.OutputLevel)
}
t.Logf("✓ 升级任务: 文件1+文件2+文件3+文件4 (以大文件为中心，搭配周围小文件) → L1")
t.Log("=== 连续性测试通过 ===")
}
// TestPickL0UpgradeContinuity 测试 L0 升级任务的连续性
func TestPickL0UpgradeContinuity(t *testing.T) {
tmpDir := t.TempDir()
manifestDir := tmpDir
versionSet, err := NewVersionSet(manifestDir)
if err != nil {
t.Fatal(err)
}
defer versionSet.Close()
picker := NewPicker()
// 创建混合大小的文件：大-小-大-大
edit := NewVersionEdit()
// 文件1: 40MB (大文件)
edit.AddFile(&FileMetadata{
FileNumber: 1,
Level: 0,
FileSize: 40 * 1024 * 1024,
MinKey: 1,
MaxKey: 100,
RowCount: 100,
})
// 文件2: 20MB (小文件)
edit.AddFile(&FileMetadata{
FileNumber: 2,
Level: 0,
FileSize: 20 * 1024 * 1024,
MinKey: 101,
MaxKey: 200,
RowCount: 100,
})
// 文件3: 50MB (大文件)
edit.AddFile(&FileMetadata{
FileNumber: 3,
Level: 0,
FileSize: 50 * 1024 * 1024,
MinKey: 201,
MaxKey: 300,
RowCount: 100,
})
// 文件4: 45MB (大文件)
edit.AddFile(&FileMetadata{
FileNumber: 4,
Level: 0,
FileSize: 45 * 1024 * 1024,
MinKey: 301,
MaxKey: 400,
RowCount: 100,
})
edit.SetNextFileNumber(5)
err = versionSet.LogAndApply(edit)
if err != nil {
t.Fatal(err)
}
version := versionSet.GetCurrent()
// 测试 L0 升级任务
t.Log("=== 测试 L0 升级任务连续性 ===")
tasks := picker.pickL0UpgradeTasks(version)
if len(tasks) == 0 {
t.Fatal("Expected L0 upgrade tasks")
}
t.Logf("找到 %d 个升级任务", len(tasks))
// 验证任务1：应该包含所有4个文件（以大文件为锚点，搭配周围文件）
// 文件1（大文件）作为锚点 → 向左无文件 → 向右收集文件2（小）+文件3（大）+文件4（大）
// 总大小：40+20+50+45 = 155MB < 256MB，符合 L1 限制
task1 := tasks[0]
expectedFileCount := 4
if len(task1.InputFiles) != expectedFileCount {
t.Errorf("Task 1: expected %d files, got %d", expectedFileCount, len(task1.InputFiles))
for i, f := range task1.InputFiles {
t.Logf(" File %d: %d", i+1, f.FileNumber)
}
}
// 验证文件顺序：1, 2, 3, 4
expectedFiles := []int64{1, 2, 3, 4}
for i, expected := range expectedFiles {
if i >= len(task1.InputFiles) {
break
}
if task1.InputFiles[i].FileNumber != expected {
t.Errorf("Task 1 file %d: expected %d, got %d",
i, expected, task1.InputFiles[i].FileNumber)
}
}
t.Logf("✓ Task 1: 文件1+文件2+文件3+文件4 (以大文件为锚点，搭配周围文件，总155MB < 256MB)")
// 只应该有1个任务（所有文件都被收集了）
if len(tasks) != 1 {
t.Errorf("Expected 1 task (all files collected), got %d", len(tasks))
for i, task := range tasks {
t.Logf("Task %d: %d files", i+1, len(task.InputFiles))
for _, f := range task.InputFiles {
t.Logf(" - File %d", f.FileNumber)
}
}
}
// 验证所有任务的 OutputLevel 都是 1
for i, task := range tasks {
if task.OutputLevel != 1 {
t.Errorf("Task %d: expected OutputLevel 1, got %d", i+1, task.OutputLevel)
}
}
t.Logf("✓ 所有任务都升级到 L1")
t.Log("=== 升级任务连续性测试通过 ===")
}

220
compaction_stage_test.go
View File

@@ -1,220 +0,0 @@
package srdb
import (
"testing"
)
// TestPickerStageRotation 测试 Picker 的阶段轮换机制
func TestPickerStageRotation(t *testing.T) {
// 创建临时目录
tmpDir := t.TempDir()
manifestDir := tmpDir
// 创建 VersionSet
versionSet, err := NewVersionSet(manifestDir)
if err != nil {
t.Fatal(err)
}
defer versionSet.Close()
// 创建 Picker
picker := NewPicker()
// 初始阶段应该是 L0
if stage := picker.GetCurrentStage(); stage != 0 {
t.Errorf("Initial stage should be 0 (L0), got %d", stage)
}
// 添加 L0 文件（触发 L0 compaction）
edit := NewVersionEdit()
for i := 0; i < 10; i++ {
edit.AddFile(&FileMetadata{
FileNumber: int64(i + 1),
Level: 0,
FileSize: 10 * 1024 * 1024, // 10MB each
MinKey: int64(i * 100),
MaxKey: int64((i+1)*100 - 1),
RowCount: 100,
})
}
edit.SetNextFileNumber(11)
err = versionSet.LogAndApply(edit)
if err != nil {
t.Fatal(err)
}
version := versionSet.GetCurrent()
// 第1次调用：应该返回 L0 任务，然后推进到 L1
t.Log("=== 第1次调用 PickCompaction ===")
tasks1 := picker.PickCompaction(version)
if len(tasks1) == 0 {
t.Error("Expected L0 tasks on first call")
}
for _, task := range tasks1 {
if task.Level != 0 {
t.Errorf("Expected L0 task, got L%d", task.Level)
}
}
if stage := picker.GetCurrentStage(); stage != 1 {
t.Errorf("After L0 tasks, stage should be 1 (L1), got %d", stage)
}
t.Logf("✓ Returned %d L0 tasks, stage advanced to L1", len(tasks1))
// 第2次调用：应该尝试 Stage 1 (L0-upgrade，没有大文件)
t.Log("=== 第2次调用 PickCompaction ===")
tasks2 := picker.PickCompaction(version)
if len(tasks2) == 0 {
t.Log("✓ Stage 1 (L0-upgrade) has no tasks")
}
// 此时 stage 应该已经循环（尝试了 Stage 1→2→3→0...）
if stage := picker.GetCurrentStage(); stage >= 0 {
t.Logf("After trying, current stage is %d", stage)
}
// 现在添加 L1 文件
edit2 := NewVersionEdit()
for i := 0; i < 20; i++ {
edit2.AddFile(&FileMetadata{
FileNumber: int64(100 + i + 1),
Level: 1,
FileSize: 20 * 1024 * 1024, // 20MB each
MinKey: int64(i * 200),
MaxKey: int64((i+1)*200 - 1),
RowCount: 200,
})
}
edit2.SetNextFileNumber(121)
err = versionSet.LogAndApply(edit2)
if err != nil {
t.Fatal(err)
}
version2 := versionSet.GetCurrent()
// 现在可能需要多次调用才能到达 Stage 2 (L1-upgrade)
// 因为要经过 Stage 1 (L0-upgrade) 和 Stage 0 (L0-merge)
t.Log("=== 多次调用 PickCompaction 直到找到 L1 任务 ===")
var tasks3 []*CompactionTask
for i := 0; i < 8; i++ { // 最多尝试两轮（4个阶段×2）
tasks3 = picker.PickCompaction(version2)
if len(tasks3) > 0 && tasks3[0].Level == 1 {
t.Logf("✓ Found %d L1 tasks after %d attempts", len(tasks3), i+1)
break
}
}
if len(tasks3) == 0 || tasks3[0].Level != 1 {
t.Error("Expected to find L1 tasks within 8 attempts")
}
t.Log("=== Stage rotation test passed ===")
}
// TestPickerStageWithMultipleLevels 测试多层级同时有任务时的阶段轮换
func TestPickerStageWithMultipleLevels(t *testing.T) {
tmpDir := t.TempDir()
manifestDir := tmpDir
versionSet, err := NewVersionSet(manifestDir)
if err != nil {
t.Fatal(err)
}
defer versionSet.Close()
picker := NewPicker()
// 同时添加 L0、L1、L2 文件
edit := NewVersionEdit()
// L0 小文件: 5 files × 10MB = 50MB (应该触发 Stage 0: L0-merge)
for i := 0; i < 5; i++ {
edit.AddFile(&FileMetadata{
FileNumber: int64(i + 1),
Level: 0,
FileSize: 10 * 1024 * 1024,
MinKey: int64(i * 100),
MaxKey: int64((i+1)*100 - 1),
RowCount: 100,
})
}
// L0 大文件: 5 files × 40MB = 200MB (应该触发 Stage 1: L0-upgrade)
for i := 0; i < 5; i++ {
edit.AddFile(&FileMetadata{
FileNumber: int64(10 + i + 1),
Level: 0,
FileSize: 40 * 1024 * 1024,
MinKey: int64((i+5) * 100),
MaxKey: int64((i+6)*100 - 1),
RowCount: 100,
})
}
// L1: 20 files × 20MB = 400MB (应该触发 Stage 2: L1-upgrade，256MB阈值)
for i := 0; i < 20; i++ {
edit.AddFile(&FileMetadata{
FileNumber: int64(100 + i + 1),
Level: 1,
FileSize: 20 * 1024 * 1024,
MinKey: int64(i * 200),
MaxKey: int64((i+1)*200 - 1),
RowCount: 200,
})
}
// L2: 10 files × 150MB = 1500MB (应该触发 Stage 3: L2-upgrade，1GB阈值)
for i := 0; i < 10; i++ {
edit.AddFile(&FileMetadata{
FileNumber: int64(200 + i + 1),
Level: 2,
FileSize: 150 * 1024 * 1024,
MinKey: int64(i * 300),
MaxKey: int64((i+1)*300 - 1),
RowCount: 300,
})
}
edit.SetNextFileNumber(301)
err = versionSet.LogAndApply(edit)
if err != nil {
t.Fatal(err)
}
version := versionSet.GetCurrent()
// 验证阶段按顺序执行：Stage 0→1→2→3→0→1→2→3
expectedStages := []struct {
stage int
name string
level int
}{
{0, "L0-merge", 0},
{1, "L0-upgrade", 0},
{2, "L1-upgrade", 1},
{3, "L2-upgrade", 2},
{0, "L0-merge", 0},
{1, "L0-upgrade", 0},
{2, "L1-upgrade", 1},
{3, "L2-upgrade", 2},
}
for i, expected := range expectedStages {
t.Logf("=== 第%d次调用 PickCompaction (期望 Stage %d: %s) ===", i+1, expected.stage, expected.name)
tasks := picker.PickCompaction(version)
if len(tasks) == 0 {
t.Errorf("Call %d: Expected tasks from Stage %d (%s), got no tasks", i+1, expected.stage, expected.name)
continue
}
actualLevel := tasks[0].Level
if actualLevel != expected.level {
t.Errorf("Call %d: Expected L%d tasks, got L%d tasks", i+1, expected.level, actualLevel)
} else {
t.Logf("✓ Call %d: Got %d tasks from L%d (Stage %d: %s) as expected",
i+1, len(tasks), actualLevel, expected.stage, expected.name)
}
}
t.Log("=== Multi-level stage rotation test passed ===")
}

486
compaction_test.go
View File

@@ -560,3 +560,489 @@ func TestCompactionQueryOrder(t *testing.T) {
}
}
}
// TestPickerStageRotation 测试 Picker 的阶段轮换机制
func TestPickerStageRotation(t *testing.T) {
// 创建临时目录
tmpDir := t.TempDir()
manifestDir := tmpDir
// 创建 VersionSet
versionSet, err := NewVersionSet(manifestDir)
if err != nil {
t.Fatal(err)
}
defer versionSet.Close()
// 创建 Picker
picker := NewPicker()
// 初始阶段应该是 L0
if stage := picker.GetCurrentStage(); stage != 0 {
t.Errorf("Initial stage should be 0 (L0), got %d", stage)
}
// 添加 L0 文件（触发 L0 compaction）
edit := NewVersionEdit()
for i := 0; i < 10; i++ {
edit.AddFile(&FileMetadata{
FileNumber: int64(i + 1),
Level: 0,
FileSize: 10 * 1024 * 1024, // 10MB each
MinKey: int64(i * 100),
MaxKey: int64((i+1)*100 - 1),
RowCount: 100,
})
}
edit.SetNextFileNumber(11)
err = versionSet.LogAndApply(edit)
if err != nil {
t.Fatal(err)
}
version := versionSet.GetCurrent()
// 第1次调用：应该返回 L0 任务，然后推进到 L1
t.Log("=== 第1次调用 PickCompaction ===")
tasks1 := picker.PickCompaction(version)
if len(tasks1) == 0 {
t.Error("Expected L0 tasks on first call")
}
for _, task := range tasks1 {
if task.Level != 0 {
t.Errorf("Expected L0 task, got L%d", task.Level)
}
}
if stage := picker.GetCurrentStage(); stage != 1 {
t.Errorf("After L0 tasks, stage should be 1 (L1), got %d", stage)
}
t.Logf("✓ Returned %d L0 tasks, stage advanced to L1", len(tasks1))
// 第2次调用：应该尝试 Stage 1 (L0-upgrade，没有大文件)
t.Log("=== 第2次调用 PickCompaction ===")
tasks2 := picker.PickCompaction(version)
if len(tasks2) == 0 {
t.Log("✓ Stage 1 (L0-upgrade) has no tasks")
}
// 此时 stage 应该已经循环（尝试了 Stage 1→2→3→0...）
if stage := picker.GetCurrentStage(); stage >= 0 {
t.Logf("After trying, current stage is %d", stage)
}
// 现在添加 L1 文件
edit2 := NewVersionEdit()
for i := 0; i < 20; i++ {
edit2.AddFile(&FileMetadata{
FileNumber: int64(100 + i + 1),
Level: 1,
FileSize: 20 * 1024 * 1024, // 20MB each
MinKey: int64(i * 200),
MaxKey: int64((i+1)*200 - 1),
RowCount: 200,
})
}
edit2.SetNextFileNumber(121)
err = versionSet.LogAndApply(edit2)
if err != nil {
t.Fatal(err)
}
version2 := versionSet.GetCurrent()
// 现在可能需要多次调用才能到达 Stage 2 (L1-upgrade)
// 因为要经过 Stage 1 (L0-upgrade) 和 Stage 0 (L0-merge)
t.Log("=== 多次调用 PickCompaction 直到找到 L1 任务 ===")
var tasks3 []*CompactionTask
for i := 0; i < 8; i++ { // 最多尝试两轮（4个阶段×2）
tasks3 = picker.PickCompaction(version2)
if len(tasks3) > 0 && tasks3[0].Level == 1 {
t.Logf("✓ Found %d L1 tasks after %d attempts", len(tasks3), i+1)
break
}
}
if len(tasks3) == 0 || tasks3[0].Level != 1 {
t.Error("Expected to find L1 tasks within 8 attempts")
}
t.Log("=== Stage rotation test passed ===")
}
// TestPickerStageWithMultipleLevels 测试多层级同时有任务时的阶段轮换
func TestPickerStageWithMultipleLevels(t *testing.T) {
tmpDir := t.TempDir()
manifestDir := tmpDir
versionSet, err := NewVersionSet(manifestDir)
if err != nil {
t.Fatal(err)
}
defer versionSet.Close()
picker := NewPicker()
// 同时添加 L0、L1、L2 文件
edit := NewVersionEdit()
// L0 小文件: 5 files × 10MB = 50MB (应该触发 Stage 0: L0-merge)
for i := 0; i < 5; i++ {
edit.AddFile(&FileMetadata{
FileNumber: int64(i + 1),
Level: 0,
FileSize: 10 * 1024 * 1024,
MinKey: int64(i * 100),
MaxKey: int64((i+1)*100 - 1),
RowCount: 100,
})
}
// L0 大文件: 5 files × 40MB = 200MB (应该触发 Stage 1: L0-upgrade)
for i := 0; i < 5; i++ {
edit.AddFile(&FileMetadata{
FileNumber: int64(10 + i + 1),
Level: 0,
FileSize: 40 * 1024 * 1024,
MinKey: int64((i + 5) * 100),
MaxKey: int64((i+6)*100 - 1),
RowCount: 100,
})
}
// L1: 20 files × 20MB = 400MB (应该触发 Stage 2: L1-upgrade，256MB阈值)
for i := 0; i < 20; i++ {
edit.AddFile(&FileMetadata{
FileNumber: int64(100 + i + 1),
Level: 1,
FileSize: 20 * 1024 * 1024,
MinKey: int64(i * 200),
MaxKey: int64((i+1)*200 - 1),
RowCount: 200,
})
}
// L2: 10 files × 150MB = 1500MB (应该触发 Stage 3: L2-upgrade，1GB阈值)
for i := 0; i < 10; i++ {
edit.AddFile(&FileMetadata{
FileNumber: int64(200 + i + 1),
Level: 2,
FileSize: 150 * 1024 * 1024,
MinKey: int64(i * 300),
MaxKey: int64((i+1)*300 - 1),
RowCount: 300,
})
}
edit.SetNextFileNumber(301)
err = versionSet.LogAndApply(edit)
if err != nil {
t.Fatal(err)
}
version := versionSet.GetCurrent()
// 验证阶段按顺序执行：Stage 0→1→2→3→0→1→2→3
expectedStages := []struct {
stage int
name string
level int
}{
{0, "L0-merge", 0},
{1, "L0-upgrade", 0},
{2, "L1-upgrade", 1},
{3, "L2-upgrade", 2},
{0, "L0-merge", 0},
{1, "L0-upgrade", 0},
{2, "L1-upgrade", 1},
{3, "L2-upgrade", 2},
}
for i, expected := range expectedStages {
t.Logf("=== 第%d次调用 PickCompaction (期望 Stage %d: %s) ===", i+1, expected.stage, expected.name)
tasks := picker.PickCompaction(version)
if len(tasks) == 0 {
t.Errorf("Call %d: Expected tasks from Stage %d (%s), got no tasks", i+1, expected.stage, expected.name)
continue
}
actualLevel := tasks[0].Level
if actualLevel != expected.level {
t.Errorf("Call %d: Expected L%d tasks, got L%d tasks", i+1, expected.level, actualLevel)
} else {
t.Logf("✓ Call %d: Got %d tasks from L%d (Stage %d: %s) as expected",
i+1, len(tasks), actualLevel, expected.stage, expected.name)
}
}
t.Log("=== Multi-level stage rotation test passed ===")
}
// TestPickL0MergeContinuity 测试 L0 合并任务的连续性
func TestPickL0MergeContinuity(t *testing.T) {
tmpDir := t.TempDir()
manifestDir := tmpDir
versionSet, err := NewVersionSet(manifestDir)
if err != nil {
t.Fatal(err)
}
defer versionSet.Close()
picker := NewPicker()
// 创建混合大小的文件：小-大-小-小
// 这是触发 bug 的场景
edit := NewVersionEdit()
// 文件1: 29MB (小文件)
edit.AddFile(&FileMetadata{
FileNumber: 1,
Level: 0,
FileSize: 29 * 1024 * 1024,
MinKey: 1,
MaxKey: 100,
RowCount: 100,
})
// 文件2: 36MB (大文件)
edit.AddFile(&FileMetadata{
FileNumber: 2,
Level: 0,
FileSize: 36 * 1024 * 1024,
MinKey: 101,
MaxKey: 200,
RowCount: 100,
})
// 文件3: 8MB (小文件)
edit.AddFile(&FileMetadata{
FileNumber: 3,
Level: 0,
FileSize: 8 * 1024 * 1024,
MinKey: 201,
MaxKey: 300,
RowCount: 100,
})
// 文件4: 15MB (小文件)
edit.AddFile(&FileMetadata{
FileNumber: 4,
Level: 0,
FileSize: 15 * 1024 * 1024,
MinKey: 301,
MaxKey: 400,
RowCount: 100,
})
edit.SetNextFileNumber(5)
err = versionSet.LogAndApply(edit)
if err != nil {
t.Fatal(err)
}
version := versionSet.GetCurrent()
// 测试 Stage 0: L0 合并任务
t.Log("=== 测试 Stage 0: L0 合并 ===")
tasks := picker.pickL0MergeTasks(version)
if len(tasks) == 0 {
t.Fatal("Expected L0 merge tasks")
}
t.Logf("找到 %d 个合并任务", len(tasks))
// 验证任务：应该只有1个任务，包含文件3和文件4
// 文件1是单个小文件，不合并（len > 1 才合并）
// 文件2是大文件，跳过
// 文件3+文件4是连续的2个小文件，应该合并
if len(tasks) != 1 {
t.Errorf("Expected 1 task, got %d", len(tasks))
for i, task := range tasks {
t.Logf("Task %d: %d files", i+1, len(task.InputFiles))
for _, f := range task.InputFiles {
t.Logf(" - File %d", f.FileNumber)
}
}
}
task1 := tasks[0]
if len(task1.InputFiles) != 2 {
t.Errorf("Task 1: expected 2 files, got %d", len(task1.InputFiles))
}
if task1.InputFiles[0].FileNumber != 3 || task1.InputFiles[1].FileNumber != 4 {
t.Errorf("Task 1: expected files 3,4, got %d,%d",
task1.InputFiles[0].FileNumber, task1.InputFiles[1].FileNumber)
}
t.Logf("✓ 合并任务: 文件3+文件4 (连续的2个小文件)")
t.Logf("✓ 文件1 (单个小文件) 不合并，留给升级阶段")
// 验证 seq 范围连续性
// 任务1: seq 201-400 (文件3+文件4)
// 文件1（seq 1-100, 单个小文件）留给升级阶段
// 文件2（seq 101-200, 大文件）留给 Stage 1
if task1.InputFiles[0].MinKey != 201 || task1.InputFiles[1].MaxKey != 400 {
t.Errorf("Task 1 seq range incorrect: [%d, %d]",
task1.InputFiles[0].MinKey, task1.InputFiles[1].MaxKey)
}
t.Logf("✓ Seq 范围正确：任务1 [201-400]")
// 测试 Stage 1: L0 升级任务
t.Log("=== 测试 Stage 1: L0 升级 ===")
upgradeTasks := picker.pickL0UpgradeTasks(version)
if len(upgradeTasks) == 0 {
t.Fatal("Expected L0 upgrade tasks")
}
// 应该有1个任务：以文件2（大文件）为中心，搭配周围的小文件
// 文件2向左收集文件1，向右收集文件3和文件4
// 总共：文件1 (29MB) + 文件2 (36MB) + 文件3 (8MB) + 文件4 (15MB) = 88MB
if len(upgradeTasks) != 1 {
t.Errorf("Expected 1 upgrade task, got %d", len(upgradeTasks))
}
upgradeTask := upgradeTasks[0]
// 应该包含所有4个文件
if len(upgradeTask.InputFiles) != 4 {
t.Errorf("Upgrade task: expected 4 files, got %d", len(upgradeTask.InputFiles))
for i, f := range upgradeTask.InputFiles {
t.Logf(" File %d: %d", i+1, f.FileNumber)
}
}
// 验证文件顺序：1, 2, 3, 4
expectedFiles := []int64{1, 2, 3, 4}
for i, expected := range expectedFiles {
if upgradeTask.InputFiles[i].FileNumber != expected {
t.Errorf("Upgrade task file %d: expected %d, got %d",
i, expected, upgradeTask.InputFiles[i].FileNumber)
}
}
if upgradeTask.OutputLevel != 1 {
t.Errorf("Upgrade task: expected OutputLevel 1, got %d", upgradeTask.OutputLevel)
}
t.Logf("✓ 升级任务: 文件1+文件2+文件3+文件4 (以大文件为中心，搭配周围小文件) → L1")
t.Log("=== 连续性测试通过 ===")
}
// TestPickL0UpgradeContinuity 测试 L0 升级任务的连续性
func TestPickL0UpgradeContinuity(t *testing.T) {
tmpDir := t.TempDir()
manifestDir := tmpDir
versionSet, err := NewVersionSet(manifestDir)
if err != nil {
t.Fatal(err)
}
defer versionSet.Close()
picker := NewPicker()
// 创建混合大小的文件：大-小-大-大
edit := NewVersionEdit()
// 文件1: 40MB (大文件)
edit.AddFile(&FileMetadata{
FileNumber: 1,
Level: 0,
FileSize: 40 * 1024 * 1024,
MinKey: 1,
MaxKey: 100,
RowCount: 100,
})
// 文件2: 20MB (小文件)
edit.AddFile(&FileMetadata{
FileNumber: 2,
Level: 0,
FileSize: 20 * 1024 * 1024,
MinKey: 101,
MaxKey: 200,
RowCount: 100,
})
// 文件3: 50MB (大文件)
edit.AddFile(&FileMetadata{
FileNumber: 3,
Level: 0,
FileSize: 50 * 1024 * 1024,
MinKey: 201,
MaxKey: 300,
RowCount: 100,
})
// 文件4: 45MB (大文件)
edit.AddFile(&FileMetadata{
FileNumber: 4,
Level: 0,
FileSize: 45 * 1024 * 1024,
MinKey: 301,
MaxKey: 400,
RowCount: 100,
})
edit.SetNextFileNumber(5)
err = versionSet.LogAndApply(edit)
if err != nil {
t.Fatal(err)
}
version := versionSet.GetCurrent()
// 测试 L0 升级任务
t.Log("=== 测试 L0 升级任务连续性 ===")
tasks := picker.pickL0UpgradeTasks(version)
if len(tasks) == 0 {
t.Fatal("Expected L0 upgrade tasks")
}
t.Logf("找到 %d 个升级任务", len(tasks))
// 验证任务1：应该包含所有4个文件（以大文件为锚点，搭配周围文件）
// 文件1（大文件）作为锚点 → 向左无文件 → 向右收集文件2（小）+文件3（大）+文件4（大）
// 总大小：40+20+50+45 = 155MB < 256MB，符合 L1 限制
task1 := tasks[0]
expectedFileCount := 4
if len(task1.InputFiles) != expectedFileCount {
t.Errorf("Task 1: expected %d files, got %d", expectedFileCount, len(task1.InputFiles))
for i, f := range task1.InputFiles {
t.Logf(" File %d: %d", i+1, f.FileNumber)
}
}
// 验证文件顺序：1, 2, 3, 4
expectedFiles := []int64{1, 2, 3, 4}
for i, expected := range expectedFiles {
if i >= len(task1.InputFiles) {
break
}
if task1.InputFiles[i].FileNumber != expected {
t.Errorf("Task 1 file %d: expected %d, got %d",
i, expected, task1.InputFiles[i].FileNumber)
}
}
t.Logf("✓ Task 1: 文件1+文件2+文件3+文件4 (以大文件为锚点，搭配周围文件，总155MB < 256MB)")
// 只应该有1个任务（所有文件都被收集了）
if len(tasks) != 1 {
t.Errorf("Expected 1 task (all files collected), got %d", len(tasks))
for i, task := range tasks {
t.Logf("Task %d: %d files", i+1, len(task.InputFiles))
for _, f := range task.InputFiles {
t.Logf(" - File %d", f.FileNumber)
}
}
}
// 验证所有任务的 OutputLevel 都是 1
for i, task := range tasks {
if task.OutputLevel != 1 {
t.Errorf("Task %d: expected OutputLevel 1, got %d", i+1, task.OutputLevel)
}
}
t.Logf("✓ 所有任务都升级到 L1")
t.Log("=== 升级任务连续性测试通过 ===")
}

162
examples/batch_insert/README.md Normal file
View File

@@ -0,0 +1,162 @@
# 批量插入示例
这个示例展示了 SRDB 的批量插入功能，支持多种数据类型的插入。
## 功能特性
SRDB 的 `Insert` 方法支持以下输入类型：
1. **单个 map**: `map[string]any`
2. **map 切片**: `[]map[string]any`
3. **单个结构体**: `struct{}`
4. **结构体指针**: `*struct{}`
5. **结构体切片**: `[]struct{}`
6. **结构体指针切片**: `[]*struct{}`
## 运行示例
```bash
cd examples/batch_insert
go run main.go
```
## 示例说明
### 示例 1: 插入单个 map
```go
err = table.Insert(map[string]any{
"name": "Alice",
"age": int64(25),
})
```
最基本的插入方式，适合动态数据。
### 示例 2: 批量插入 map 切片
```go
err = table.Insert([]map[string]any{
{"name": "Alice", "age": int64(25), "email": "alice@example.com"},
{"name": "Bob", "age": int64(30), "email": "bob@example.com"},
{"name": "Charlie", "age": int64(35), "email": "charlie@example.com"},
})
```
批量插入多条数据，提高插入效率。
### 示例 3: 插入单个结构体
```go
type User struct {
Name string `srdb:"name;comment:用户名"`
Age int64 `srdb:"age;comment:年龄"`
Email string `srdb:"email;indexed;comment:邮箱"`
IsActive bool `srdb:"is_active;comment:是否激活"`
}
user := User{
Name: "Alice",
Age: 25,
Email: "alice@example.com",
IsActive: true,
}
err = table.Insert(user)
```
使用结构体插入，提供类型安全和代码可读性。
### 示例 4: 批量插入结构体切片
```go
users := []User{
{Name: "Alice", Age: 25, Email: "alice@example.com", IsActive: true},
{Name: "Bob", Age: 30, Email: "bob@example.com", IsActive: true},
{Name: "Charlie", Age: 35, Email: "charlie@example.com", IsActive: false},
}
err = table.Insert(users)
```
批量插入结构体，适合需要插入大量数据的场景。
### 示例 5: 批量插入结构体指针切片
```go
users := []*User{
{Name: "Alice", Age: 25, Email: "alice@example.com", IsActive: true},
{Name: "Bob", Age: 30, Email: "bob@example.com", IsActive: true},
nil, // nil 指针会被自动跳过
{Name: "Charlie", Age: 35, Email: "charlie@example.com", IsActive: false},
}
err = table.Insert(users)
```
支持指针切片，nil 指针会被自动跳过。
### 示例 6: 使用 snake_case 自动转换
```go
type Product struct {
ProductID string `srdb:";comment:产品ID"` // 自动转为 product_id
ProductName string `srdb:";comment:产品名称"` // 自动转为 product_name
Price float64 `srdb:";comment:价格"` // 自动转为 price
InStock bool `srdb:";comment:是否有货"` // 自动转为 in_stock
}
products := []Product{
{ProductID: "P001", ProductName: "Laptop", Price: 999.99, InStock: true},
{ProductID: "P002", ProductName: "Mouse", Price: 29.99, InStock: true},
}
err = table.Insert(products)
```
不指定字段名时，会自动将驼峰命名转换为 snake_case：
- `ProductID` → `product_id`
- `ProductName` → `product_name`
- `InStock` → `in_stock`
## Struct Tag 格式
```go
type User struct {
// 完整格式：字段名;索引;注释
Email string `srdb:"email;indexed;comment:邮箱地址"`
// 使用默认字段名（snake_case）+ 注释
UserName string `srdb:";comment:用户名"` // 自动转为 user_name
// 不使用 tag，完全依赖 snake_case 转换
PhoneNumber string // 自动转为 phone_number
// 忽略字段
Internal string `srdb:"-"`
}
```
## 性能优化
批量插入相比逐条插入：
- ✅ 减少函数调用开销
- ✅ 统一类型转换和验证
- ✅ 更清晰的代码逻辑
- ✅ 适合大批量数据导入
## 注意事项
1. **类型匹配**: 确保结构体字段类型与 Schema 定义一致
2. **Schema 验证**: 所有数据都会经过 Schema 验证
3. **nil 处理**: 结构体指针切片中的 nil 会被自动跳过
4. **字段名转换**: 未指定 tag 时自动使用 snake_case 转换
5. **索引更新**: 带索引的字段会自动更新索引
## 相关文档
- [STRUCT_TAG_GUIDE.md](../../STRUCT_TAG_GUIDE.md) - Struct Tag 完整指南
- [SNAKE_CASE_CONVERSION.md](../../SNAKE_CASE_CONVERSION.md) - snake_case 转换规则
- [examples/struct_schema](../struct_schema) - 结构体 Schema 示例

1
examples/batch_insert/data/example1/CURRENT Normal file
View File

@@ -0,0 +1 @@
MANIFEST-000001

BIN
examples/batch_insert/data/example1/MANIFEST-000001 Normal file
View File

Binary file not shown.

22
examples/batch_insert/data/example1/schema.json Normal file
View File

@@ -0,0 +1,22 @@
{
"version": 1,
"timestamp": 1760013696,
"checksum": "343b1f86cfc4ed9471b71b3a63d61b4205b17cf953f4a2698f2d3ebd37540caa",
"schema": {
"Name": "users",
"Fields": [
{
"Name": "name",
"Type": 2,
"Indexed": false,
"Comment": "用户名"
},
{
"Name": "age",
"Type": 1,
"Indexed": false,
"Comment": "年龄"
}
]
}
}

1
examples/batch_insert/data/example1/wal/CURRENT Normal file
View File

@@ -0,0 +1 @@
2

1
examples/batch_insert/data/example2/CURRENT Normal file
View File

@@ -0,0 +1 @@
MANIFEST-000001

BIN
examples/batch_insert/data/example2/MANIFEST-000001 Normal file
View File

Binary file not shown.

28
examples/batch_insert/data/example2/schema.json Normal file
View File

@@ -0,0 +1,28 @@
{
"version": 1,
"timestamp": 1760013696,
"checksum": "961d723306aa507e4599ad24afe32e0bc30dcac5f9d1aabd1a18128376767a36",
"schema": {
"Name": "users",
"Fields": [
{
"Name": "name",
"Type": 2,
"Indexed": false,
"Comment": ""
},
{
"Name": "age",
"Type": 1,
"Indexed": false,
"Comment": ""
},
{
"Name": "email",
"Type": 2,
"Indexed": true,
"Comment": ""
}
]
}
}

1
examples/batch_insert/data/example2/wal/CURRENT Normal file
View File

@@ -0,0 +1 @@
2

1
examples/batch_insert/data/example3/CURRENT Normal file
View File

@@ -0,0 +1 @@
MANIFEST-000001

BIN
examples/batch_insert/data/example3/MANIFEST-000001 Normal file
View File

Binary file not shown.

34
examples/batch_insert/data/example3/schema.json Normal file
View File

@@ -0,0 +1,34 @@
{
"version": 1,
"timestamp": 1760013696,
"checksum": "dab23ff2118dc2a790681dcc4e112b3377fd96414a30cdfa0ca98af5d41a271e",
"schema": {
"Name": "users",
"Fields": [
{
"Name": "name",
"Type": 2,
"Indexed": false,
"Comment": ""
},
{
"Name": "age",
"Type": 1,
"Indexed": false,
"Comment": ""
},
{
"Name": "email",
"Type": 2,
"Indexed": false,
"Comment": ""
},
{
"Name": "is_active",
"Type": 4,
"Indexed": false,
"Comment": ""
}
]
}
}

1
examples/batch_insert/data/example3/wal/CURRENT Normal file
View File

@@ -0,0 +1 @@
2

1
examples/batch_insert/data/example4/CURRENT Normal file
View File

@@ -0,0 +1 @@
MANIFEST-000001

BIN
examples/batch_insert/data/example4/MANIFEST-000001 Normal file
View File

Binary file not shown.

34
examples/batch_insert/data/example4/schema.json Normal file
View File

@@ -0,0 +1,34 @@
{
"version": 1,
"timestamp": 1760013696,
"checksum": "dab23ff2118dc2a790681dcc4e112b3377fd96414a30cdfa0ca98af5d41a271e",
"schema": {
"Name": "users",
"Fields": [
{
"Name": "name",
"Type": 2,
"Indexed": false,
"Comment": ""
},
{
"Name": "age",
"Type": 1,
"Indexed": false,
"Comment": ""
},
{
"Name": "email",
"Type": 2,
"Indexed": false,
"Comment": ""
},
{
"Name": "is_active",
"Type": 4,
"Indexed": false,
"Comment": ""
}
]
}
}

1
examples/batch_insert/data/example4/wal/CURRENT Normal file
View File

@@ -0,0 +1 @@
2

1
examples/batch_insert/data/example5/CURRENT Normal file
View File

@@ -0,0 +1 @@
MANIFEST-000001

BIN
examples/batch_insert/data/example5/MANIFEST-000001 Normal file
View File

Binary file not shown.

34
examples/batch_insert/data/example5/schema.json Normal file
View File

@@ -0,0 +1,34 @@
{
"version": 1,
"timestamp": 1760013696,
"checksum": "dab23ff2118dc2a790681dcc4e112b3377fd96414a30cdfa0ca98af5d41a271e",
"schema": {
"Name": "users",
"Fields": [
{
"Name": "name",
"Type": 2,
"Indexed": false,
"Comment": ""
},
{
"Name": "age",
"Type": 1,
"Indexed": false,
"Comment": ""
},
{
"Name": "email",
"Type": 2,
"Indexed": false,
"Comment": ""
},
{
"Name": "is_active",
"Type": 4,
"Indexed": false,
"Comment": ""
}
]
}
}

1
examples/batch_insert/data/example5/wal/CURRENT Normal file
View File

@@ -0,0 +1 @@
2

1
examples/batch_insert/data/example6/CURRENT Normal file
View File

@@ -0,0 +1 @@
MANIFEST-000001

BIN
examples/batch_insert/data/example6/MANIFEST-000001 Normal file
View File

Binary file not shown.

34
examples/batch_insert/data/example6/schema.json Normal file
View File

@@ -0,0 +1,34 @@
{
"version": 1,
"timestamp": 1760013696,
"checksum": "36398ae9f0d772f19a24dd8562fde01665890335416e4951808c6b050d251b43",
"schema": {
"Name": "products",
"Fields": [
{
"Name": "product_id",
"Type": 2,
"Indexed": false,
"Comment": "产品ID"
},
{
"Name": "product_name",
"Type": 2,
"Indexed": false,
"Comment": "产品名称"
},
{
"Name": "price",
"Type": 3,
"Indexed": false,
"Comment": "价格"
},
{
"Name": "in_stock",
"Type": 4,
"Indexed": false,
"Comment": "是否有货"
}
]
}
}

1
examples/batch_insert/data/example6/wal/CURRENT Normal file
View File

@@ -0,0 +1 @@
2

303
examples/batch_insert/main.go Normal file
View File

@@ -0,0 +1,303 @@
package main
import (
"fmt"
"log"
"os"
"time"
"code.tczkiot.com/wlw/srdb"
)
// User 用户结构体
type User struct {
Name string `srdb:"name;comment:用户名"`
Age int64 `srdb:"age;comment:年龄"`
Email string `srdb:"email;indexed;comment:邮箱"`
IsActive bool `srdb:"is_active;comment:是否激活"`
}
// Product 产品结构体（使用默认 snake_case 转换）
type Product struct {
ProductID string `srdb:";comment:产品ID"` // 自动转为 product_id
ProductName string `srdb:";comment:产品名称"` // 自动转为 product_name
Price float64 `srdb:";comment:价格"` // 自动转为 price
InStock bool `srdb:";comment:是否有货"` // 自动转为 in_stock
}
func main() {
fmt.Println("=== SRDB 批量插入示例 ===\n")
// 清理旧数据
os.RemoveAll("./data")
// 示例 1: 插入单个 map
example1()
// 示例 2: 批量插入 map 切片
example2()
// 示例 3: 插入单个结构体
example3()
// 示例 4: 批量插入结构体切片
example4()
// 示例 5: 批量插入结构体指针切片
example5()
// 示例 6: 使用 snake_case 自动转换
example6()
fmt.Println("\n✓ 所有示例执行成功！")
}
func example1() {
fmt.Println("=== 示例 1: 插入单个 map ===")
schema := srdb.NewSchema("users", []srdb.Field{
{Name: "name", Type: srdb.FieldTypeString, Comment: "用户名"},
{Name: "age", Type: srdb.FieldTypeInt64, Comment: "年龄"},
})
table, err := srdb.OpenTable(&srdb.TableOptions{
Dir: "./data/example1",
Name: schema.Name,
Fields: schema.Fields,
})
if err != nil {
log.Fatal(err)
}
defer table.Close()
// 插入单条数据
err = table.Insert(map[string]any{
"name": "Alice",
"age": int64(25),
})
if err != nil {
log.Fatal(err)
}
fmt.Println("✓ 插入 1 条数据")
// 查询
row, _ := table.Get(1)
fmt.Printf(" 查询结果: name=%s, age=%d\n\n", row.Data["name"], row.Data["age"])
}
func example2() {
fmt.Println("=== 示例 2: 批量插入 map 切片 ===")
schema := srdb.NewSchema("users", []srdb.Field{
{Name: "name", Type: srdb.FieldTypeString},
{Name: "age", Type: srdb.FieldTypeInt64},
{Name: "email", Type: srdb.FieldTypeString, Indexed: true},
})
table, err := srdb.OpenTable(&srdb.TableOptions{
Dir: "./data/example2",
Name: schema.Name,
Fields: schema.Fields,
})
if err != nil {
log.Fatal(err)
}
defer table.Close()
// 批量插入
start := time.Now()
err = table.Insert([]map[string]any{
{"name": "Alice", "age": int64(25), "email": "alice@example.com"},
{"name": "Bob", "age": int64(30), "email": "bob@example.com"},
{"name": "Charlie", "age": int64(35), "email": "charlie@example.com"},
{"name": "David", "age": int64(40), "email": "david@example.com"},
{"name": "Eve", "age": int64(45), "email": "eve@example.com"},
})
if err != nil {
log.Fatal(err)
}
elapsed := time.Since(start)
fmt.Printf("✓ 批量插入 5 条数据，耗时: %v\n", elapsed)
// 使用索引查询
rows, _ := table.Query().Eq("email", "bob@example.com").Rows()
defer rows.Close()
if rows.Next() {
row := rows.Row()
data := row.Data()
fmt.Printf(" 索引查询结果: name=%s, email=%s\n\n", data["name"], data["email"])
}
}
func example3() {
fmt.Println("=== 示例 3: 插入单个结构体 ===")
schema := srdb.NewSchema("users", []srdb.Field{
{Name: "name", Type: srdb.FieldTypeString},
{Name: "age", Type: srdb.FieldTypeInt64},
{Name: "email", Type: srdb.FieldTypeString},
{Name: "is_active", Type: srdb.FieldTypeBool},
})
table, err := srdb.OpenTable(&srdb.TableOptions{
Dir: "./data/example3",
Name: schema.Name,
Fields: schema.Fields,
})
if err != nil {
log.Fatal(err)
}
defer table.Close()
// 插入结构体
user := User{
Name: "Alice",
Age: 25,
Email: "alice@example.com",
IsActive: true,
}
err = table.Insert(user)
if err != nil {
log.Fatal(err)
}
fmt.Println("✓ 插入 1 个结构体")
// 查询
row, _ := table.Get(1)
fmt.Printf(" 查询结果: name=%s, age=%d, active=%v\n\n",
row.Data["name"], row.Data["age"], row.Data["is_active"])
}
func example4() {
fmt.Println("=== 示例 4: 批量插入结构体切片 ===")
schema := srdb.NewSchema("users", []srdb.Field{
{Name: "name", Type: srdb.FieldTypeString},
{Name: "age", Type: srdb.FieldTypeInt64},
{Name: "email", Type: srdb.FieldTypeString},
{Name: "is_active", Type: srdb.FieldTypeBool},
})
table, err := srdb.OpenTable(&srdb.TableOptions{
Dir: "./data/example4",
Name: schema.Name,
Fields: schema.Fields,
})
if err != nil {
log.Fatal(err)
}
defer table.Close()
// 批量插入结构体切片
users := []User{
{Name: "Alice", Age: 25, Email: "alice@example.com", IsActive: true},
{Name: "Bob", Age: 30, Email: "bob@example.com", IsActive: true},
{Name: "Charlie", Age: 35, Email: "charlie@example.com", IsActive: false},
}
start := time.Now()
err = table.Insert(users)
if err != nil {
log.Fatal(err)
}
elapsed := time.Since(start)
fmt.Printf("✓ 批量插入 %d 个结构体，耗时: %v\n", len(users), elapsed)
// 查询所有激活用户
rows, _ := table.Query().Eq("is_active", true).Rows()
defer rows.Close()
count := 0
for rows.Next() {
count++
}
fmt.Printf(" 查询结果: 找到 %d 个激活用户\n\n", count)
}
func example5() {
fmt.Println("=== 示例 5: 批量插入结构体指针切片 ===")
schema := srdb.NewSchema("users", []srdb.Field{
{Name: "name", Type: srdb.FieldTypeString},
{Name: "age", Type: srdb.FieldTypeInt64},
{Name: "email", Type: srdb.FieldTypeString},
{Name: "is_active", Type: srdb.FieldTypeBool},
})
table, err := srdb.OpenTable(&srdb.TableOptions{
Dir: "./data/example5",
Name: schema.Name,
Fields: schema.Fields,
})
if err != nil {
log.Fatal(err)
}
defer table.Close()
// 批量插入结构体指针切片
users := []*User{
{Name: "Alice", Age: 25, Email: "alice@example.com", IsActive: true},
{Name: "Bob", Age: 30, Email: "bob@example.com", IsActive: true},
nil, // nil 指针会被自动跳过
{Name: "Charlie", Age: 35, Email: "charlie@example.com", IsActive: false},
}
err = table.Insert(users)
if err != nil {
log.Fatal(err)
}
fmt.Printf("✓ 批量插入 %d 个结构体指针（nil 自动跳过）\n", len(users))
// 验证插入数量
row, _ := table.Get(3)
fmt.Printf(" 实际插入: 3 条数据, 最后一条 name=%s\n\n", row.Data["name"])
}
func example6() {
fmt.Println("=== 示例 6: 使用 snake_case 自动转换 ===")
schema := srdb.NewSchema("products", []srdb.Field{
{Name: "product_id", Type: srdb.FieldTypeString, Comment: "产品ID"},
{Name: "product_name", Type: srdb.FieldTypeString, Comment: "产品名称"},
{Name: "price", Type: srdb.FieldTypeFloat, Comment: "价格"},
{Name: "in_stock", Type: srdb.FieldTypeBool, Comment: "是否有货"},
})
table, err := srdb.OpenTable(&srdb.TableOptions{
Dir: "./data/example6",
Name: schema.Name,
Fields: schema.Fields,
})
if err != nil {
log.Fatal(err)
}
defer table.Close()
// 结构体字段名是驼峰命名，会自动转为 snake_case
products := []Product{
{ProductID: "P001", ProductName: "Laptop", Price: 999.99, InStock: true},
{ProductID: "P002", ProductName: "Mouse", Price: 29.99, InStock: true},
{ProductID: "P003", ProductName: "Keyboard", Price: 79.99, InStock: false},
}
err = table.Insert(products)
if err != nil {
log.Fatal(err)
}
fmt.Printf("✓ 批量插入 %d 个产品（自动 snake_case 转换）\n", len(products))
// 查询
row, _ := table.Get(1)
fmt.Printf(" 字段名自动转换:\n")
fmt.Printf(" ProductID -> product_id = %s\n", row.Data["product_id"])
fmt.Printf(" ProductName -> product_name = %s\n", row.Data["product_name"])
fmt.Printf(" Price -> price = %.2f\n", row.Data["price"])
fmt.Printf(" InStock -> in_stock = %v\n\n", row.Data["in_stock"])
}

98
examples/snake_case_demo/main.go Normal file
View File

@@ -0,0 +1,98 @@
package main
import (
"fmt"
"log"
"code.tczkiot.com/wlw/srdb"
)
// 演示各种驼峰命名自动转换为 snake_case
type DemoStruct struct {
// 基本转换
UserName string `srdb:";comment:用户名"` // -> user_name
EmailAddress string `srdb:";comment:邮箱地址"` // -> email_address
PhoneNumber string `srdb:";comment:手机号"` // -> phone_number
// 连续大写字母
HTTPEndpoint string `srdb:";comment:HTTP 端点"` // -> http_endpoint
URLPath string `srdb:";comment:URL 路径"` // -> url_path
XMLParser string `srdb:";comment:XML 解析器"` // -> xml_parser
// 短命名
ID int64 `srdb:";comment:ID"` // -> id
// 布尔值
IsActive bool `srdb:";comment:是否激活"` // -> is_active
IsDeleted bool `srdb:";comment:是否删除"` // -> is_deleted
// 数字混合
Address1 string `srdb:";comment:地址1"` // -> address1
User2Name string `srdb:";comment:用户2名称"` // -> user2_name
}
func main() {
fmt.Println("=== snake_case 自动转换演示 ===")
// 生成 Field 列表
fields, err := srdb.StructToFields(DemoStruct{})
if err != nil {
log.Fatal(err)
}
// 打印转换结果
fmt.Println("\n字段名转换（驼峰命名 -> snake_case）：")
fmt.Printf("%-20s -> %-20s %s\n", "Go 字段名", "数据库字段名", "注释")
fmt.Println(string(make([]byte, 70)) + "\n" + string(make([]byte, 70)))
type fieldInfo struct {
goName string
dbName string
comment string
}
fieldMapping := []fieldInfo{
{"UserName", "user_name", "用户名"},
{"EmailAddress", "email_address", "邮箱地址"},
{"PhoneNumber", "phone_number", "手机号"},
{"HTTPEndpoint", "http_endpoint", "HTTP 端点"},
{"URLPath", "url_path", "URL 路径"},
{"XMLParser", "xml_parser", "XML 解析器"},
{"ID", "id", "ID"},
{"IsActive", "is_active", "是否激活"},
{"IsDeleted", "is_deleted", "是否删除"},
{"Address1", "address1", "地址1"},
{"User2Name", "user2_name", "用户2名称"},
}
for i, field := range fields {
if i < len(fieldMapping) {
fmt.Printf("%-20s -> %-20s %s\n",
fieldMapping[i].goName,
field.Name,
field.Comment)
}
}
// 验证转换
fmt.Println("\n=== 转换验证 ===")
allCorrect := true
for i, field := range fields {
if i < len(fieldMapping) {
expected := fieldMapping[i].dbName
if field.Name != expected {
fmt.Printf("❌ %s: 期望 %s, 实际 %s\n",
fieldMapping[i].goName, expected, field.Name)
allCorrect = false
}
}
}
if allCorrect {
fmt.Println("✅ 所有字段名转换正确！")
}
// 创建 Schema
schema := srdb.NewSchema("demo", fields)
fmt.Printf("\n✅ 成功创建 Schema，包含 %d 个字段\n", len(schema.Fields))
}

153
examples/struct_schema/README.md Normal file
View File

@@ -0,0 +1,153 @@
# StructToFields 示例
这个示例展示如何使用 `StructToFields` 方法从 Go 结构体自动生成 Schema。
## 功能特性
- ✅ 从结构体自动生成 Field 列表
- ✅ 支持 struct tags 定义字段属性
- ✅ 支持索引标记
- ✅ 支持字段注释
- ✅ 自动类型映射
- ✅ 支持忽略字段
## Struct Tag 格式
### srdb tag
所有配置都在 `srdb` tag 中，使用分号 `;` 分隔：
```go
type User struct {
// 基本用法：指定字段名
Name string `srdb:"name"`
// 标记为索引字段
Email string `srdb:"email;indexed"`
// 完整格式：字段名;索引;注释
Age int64 `srdb:"age;comment:年龄"`
// 带索引和注释
Phone string `srdb:"phone;indexed;comment:手机号"`
// 忽略该字段
Internal string `srdb:"-"`
// 不使用 tag，默认使用 snake_case 转换
Score float64 // 字段名: score
UserID string // 字段名: user_id
}
```
### Tag 格式说明
格式：`srdb:"字段名;选项1;选项2;..."`
- **字段名**（第一部分）：指定数据库中的字段名，省略则自动将结构体字段名转为 snake_case
- **indexed**：标记该字段需要建立索引
- **comment:注释内容**：字段注释说明
### 默认字段名转换（snake_case）
如果不指定字段名，会自动将驼峰命名转换为 snake_case：
- `UserName` → `user_name`
- `EmailAddress` → `email_address`
- `IsActive` → `is_active`
- `HTTPServer` → `http_server`
- `ID` → `id`
## 类型映射
| Go 类型 | FieldType |
|---------|-----------|
| int, int64, int32, int16, int8 | FieldTypeInt64 |
| uint, uint64, uint32, uint16, uint8 | FieldTypeInt64 |
| string | FieldTypeString |
| float64, float32 | FieldTypeFloat |
| bool | FieldTypeBool |
## 完整示例
```go
package main
import (
"log"
"code.tczkiot.com/wlw/srdb"
)
// 定义结构体
type User struct {
Name string `srdb:"name;indexed;comment:用户名"`
Age int64 `srdb:"age;comment:年龄"`
Email string `srdb:"email;indexed;comment:邮箱"`
Score float64 `srdb:"score;comment:分数"`
IsActive bool `srdb:"is_active;comment:是否激活"`
}
func main() {
// 1. 从结构体生成 Field 列表
fields, err := srdb.StructToFields(User{})
if err != nil {
log.Fatal(err)
}
// 2. 创建 Schema
schema := srdb.NewSchema("users", fields)
// 3. 创建表
table, err := srdb.OpenTable(&srdb.TableOptions{
Dir: "./data/users",
Name: schema.Name,
Fields: schema.Fields,
})
if err != nil {
log.Fatal(err)
}
defer table.Close()
// 4. 插入数据
err = table.Insert(map[string]any{
"name": "张三",
"age": int64(25),
"email": "zhangsan@example.com",
"score": 95.5,
"is_active": true,
})
// 5. 查询数据（自动使用索引）
rows, _ := table.Query().Eq("email", "zhangsan@example.com").Rows()
defer rows.Close()
for rows.Next() {
data := rows.Row().Data()
// 处理数据...
}
}
```
## 运行示例
```bash
cd examples/struct_schema
go run main.go
```
## 优势
1. **类型安全**: 使用结构体定义，编译时检查类型
2. **简洁**: 不需要手动创建 Field 列表
3. **可维护**: 结构体和 Schema 在一起，便于维护
4. **灵活**: 支持 tag 自定义字段属性
5. **自动索引**: 通过 `indexed` tag 自动创建索引
## 注意事项
1. 只有导出的字段（首字母大写）会被包含
2. 使用 `srdb:"-"` 可以忽略字段
3. 如果不指定字段名，默认使用小写的字段名
4. 不支持嵌套结构体（需要手动展开）
5. 不支持切片、map 等复杂类型

1
examples/struct_schema/data/users/CURRENT Normal file
View File

@@ -0,0 +1 @@
MANIFEST-000001

BIN
examples/struct_schema/data/users/MANIFEST-000001 Normal file
View File

Binary file not shown.

40
examples/struct_schema/data/users/schema.json Normal file
View File

@@ -0,0 +1,40 @@
{
"version": 1,
"timestamp": 1760013049,
"checksum": "fad0aaaa6fc5b94364c0c0b07a7567f71e8bb4f8ed8456c1954cffa7538a6a42",
"schema": {
"Name": "users",
"Fields": [
{
"Name": "name",
"Type": 2,
"Indexed": true,
"Comment": "用户名"
},
{
"Name": "age",
"Type": 1,
"Indexed": false,
"Comment": "年龄"
},
{
"Name": "email",
"Type": 2,
"Indexed": true,
"Comment": "邮箱"
},
{
"Name": "score",
"Type": 3,
"Indexed": false,
"Comment": "分数"
},
{
"Name": "is_active",
"Type": 4,
"Indexed": false,
"Comment": "是否激活"
}
]
}
}

1
examples/struct_schema/data/users/wal/CURRENT Normal file
View File

@@ -0,0 +1 @@
3

130
examples/struct_schema/main.go Normal file
View File

@@ -0,0 +1,130 @@
package main
import (
"fmt"
"log"
"code.tczkiot.com/wlw/srdb"
)
// User 用户结构体
// 使用 struct tags 定义 Schema
type User struct {
Name string `srdb:"name;indexed;comment:用户名"`
Age int64 `srdb:"age;comment:年龄"`
Email string `srdb:"email;indexed;comment:邮箱"`
Score float64 `srdb:"score;comment:分数"`
IsActive bool `srdb:"is_active;comment:是否激活"`
Internal string `srdb:"-"` // 不会被包含在 Schema 中
}
// Product 产品结构体
// 不使用 srdb tag，字段名会自动转为 snake_case
type Product struct {
ProductID string // 字段名: product_id
ProductName string // 字段名: product_name
Price int64 // 字段名: price
InStock bool // 字段名: in_stock
}
func main() {
// 示例 1: 使用结构体创建 Schema
fmt.Println("=== 示例 1: 从结构体创建 Schema ===")
// 从 User 结构体生成 Field 列表
fields, err := srdb.StructToFields(User{})
if err != nil {
log.Fatal(err)
}
// 创建 Schema
schema := srdb.NewSchema("users", fields)
// 打印 Schema 信息
fmt.Printf("Schema 名称: %s\n", schema.Name)
fmt.Printf("字段数量: %d\n", len(schema.Fields))
fmt.Println("\n字段列表:")
for _, field := range schema.Fields {
indexed := ""
if field.Indexed {
indexed = " [索引]"
}
fmt.Printf(" - %s (%s)%s: %s\n",
field.Name, field.Type.String(), indexed, field.Comment)
}
// 示例 2: 使用 Schema 创建表
fmt.Println("\n=== 示例 2: 使用 Schema 创建表 ===")
table, err := srdb.OpenTable(&srdb.TableOptions{
Dir: "./data/users",
Name: schema.Name,
Fields: schema.Fields,
})
if err != nil {
log.Fatal(err)
}
defer table.Close()
// 插入数据
err = table.Insert(map[string]any{
"name": "张三",
"age": int64(25),
"email": "zhangsan@example.com",
"score": 95.5,
"is_active": true,
})
if err != nil {
log.Fatal(err)
}
err = table.Insert(map[string]any{
"name": "李四",
"age": int64(30),
"email": "lisi@example.com",
"score": 88.0,
"is_active": true,
})
if err != nil {
log.Fatal(err)
}
fmt.Println("✓ 插入 2 条数据")
// 查询数据
rows, err := table.Query().Eq("email", "zhangsan@example.com").Rows()
if err != nil {
log.Fatal(err)
}
defer rows.Close()
fmt.Println("\n查询结果 (email = zhangsan@example.com):")
for rows.Next() {
data := rows.Row().Data()
fmt.Printf(" 姓名: %s, 年龄: %v, 邮箱: %s, 分数: %v, 激活: %v\n",
data["name"], data["age"], data["email"], data["score"], data["is_active"])
}
// 示例 3: 使用默认字段名（snake_case）
fmt.Println("\n=== 示例 3: 使用默认字段名（snake_case）===")
productFields, err := srdb.StructToFields(Product{})
if err != nil {
log.Fatal(err)
}
fmt.Println("Product 字段（使用默认 snake_case 名称）:")
for _, field := range productFields {
fmt.Printf(" - %s (%s)\n", field.Name, field.Type.String())
}
// 示例 4: 获取索引字段
fmt.Println("\n=== 示例 4: 获取索引字段 ===")
indexedFields := schema.GetIndexedFields()
fmt.Printf("User Schema 中的索引字段（共 %d 个）:\n", len(indexedFields))
for _, field := range indexedFields {
fmt.Printf(" - %s: %s\n", field.Name, field.Comment)
}
fmt.Println("\n✓ 所有示例执行成功！")
}

56
index.go
View File

@@ -98,7 +98,9 @@ func (idx *SecondaryIndex) Build() error {
// 使用 B+Tree 写入器
writer := NewIndexBTreeWriter(idx.file, idx.metadata)
// 添加所有条目
// 写入内存中的所有条目
// 注意：这假设 valueToSeq 包含所有数据（包括从磁盘加载的）
// 对于增量更新场景，Get() 会合并内存和磁盘的结果
for value, seqs := range idx.valueToSeq {
writer.Add(value, seqs)
}
@@ -109,8 +111,25 @@ func (idx *SecondaryIndex) Build() error {
return fmt.Errorf("failed to build btree index: %w", err)
}
// 关闭旧的 btreeReader
if idx.btreeReader != nil {
idx.btreeReader.Close()
}
// 重新加载 btreeReader（读取刚写入的数据）
reader, err := NewIndexBTreeReader(idx.file)
if err != nil {
return fmt.Errorf("failed to reload btree reader: %w", err)
}
idx.btreeReader = reader
idx.useBTree = true
idx.ready = true
// 不清空 valueToSeq，保留所有数据在内存中
// 这样下次 Build() 时可以写入完整数据
// Get() 方法会合并内存和磁盘的结果（去重）
return nil
}
@@ -197,7 +216,7 @@ func (idx *SecondaryIndex) loadJSON() error {
return nil
}
// Get 查询索引
// Get 查询索引（优先查内存，然后查磁盘，合并结果）
func (idx *SecondaryIndex) Get(value any) ([]int64, error) {
idx.mu.RLock()
defer idx.mu.RUnlock()
@@ -208,18 +227,37 @@ func (idx *SecondaryIndex) Get(value any) ([]int64, error) {
key := fmt.Sprintf("%v", value)
// 如果使用 B+Tree，从 B+Tree 读取
if idx.useBTree && idx.btreeReader != nil {
return idx.btreeReader.Get(key)
// 收集所有匹配的 seqs（需要去重）
seqMap := make(map[int64]bool)
// 1. 先从内存 map 读取（包含最新的未持久化数据）
if memSeqs, exists := idx.valueToSeq[key]; exists {
for _, seq := range memSeqs {
seqMap[seq] = true
}
}
// 否则从内存 map 读取
seqs, exists := idx.valueToSeq[key]
if !exists {
// 2. 如果使用 B+Tree，从 B+Tree 读取（持久化的数据）
if idx.useBTree && idx.btreeReader != nil {
diskSeqs, err := idx.btreeReader.Get(key)
if err == nil && diskSeqs != nil {
for _, seq := range diskSeqs {
seqMap[seq] = true
}
}
}
// 3. 合并结果
if len(seqMap) == 0 {
return nil, nil
}
return seqs, nil
result := make([]int64, 0, len(seqMap))
for seq := range seqMap {
result = append(result, seq)
}
return result, nil
}
// IsReady 索引是否就绪

254
index_test.go
View File

@@ -284,3 +284,257 @@ func TestIndexDropWithFile(t *testing.T) {
t.Log("索引删除测试通过！")
}
// TestIndexQueryIntegration 测试索引查询的完整流程
func TestIndexQueryIntegration(t *testing.T) {
tmpDir := t.TempDir()
// 1. 创建带索引字段的 Schema
schema := NewSchema("users", []Field{
{Name: "name", Type: FieldTypeString, Indexed: false},
{Name: "email", Type: FieldTypeString, Indexed: true}, // email 字段有索引
{Name: "age", Type: FieldTypeInt64, Indexed: false},
})
// 2. 打开表
table, err := OpenTable(&TableOptions{
Dir: tmpDir,
Name: schema.Name,
Fields: schema.Fields,
MemTableSize: 1024 * 1024, // 1MB
})
if err != nil {
t.Fatal(err)
}
defer table.Close()
// 3. 创建索引
err = table.CreateIndex("email")
if err != nil {
t.Fatal(err)
}
// 4. 插入测试数据
testData := []map[string]any{
{"name": "Alice", "email": "alice@example.com", "age": int64(25)},
{"name": "Bob", "email": "bob@example.com", "age": int64(30)},
{"name": "Charlie", "email": "alice@example.com", "age": int64(35)}, // 相同 email
{"name": "David", "email": "david@example.com", "age": int64(40)},
}
for _, data := range testData {
err := table.Insert(data)
if err != nil {
t.Fatalf("Failed to insert data: %v", err)
}
}
// 5. 构建索引（持久化）
err = table.indexManager.BuildAll()
if err != nil {
t.Fatalf("Failed to build indexes: %v", err)
}
// 6. 验证索引文件存在
indexPath := tmpDir + "/idx/idx_email.sst"
if _, err := os.Stat(indexPath); os.IsNotExist(err) {
t.Fatalf("Index file not created: %s", indexPath)
}
t.Logf("✓ Index file created: %s", indexPath)
// 7. 使用索引查询
rows, err := table.Query().Eq("email", "alice@example.com").Rows()
if err != nil {
t.Fatalf("Query failed: %v", err)
}
defer rows.Close()
// 8. 验证结果
var results []map[string]any
for rows.Next() {
results = append(results, rows.Row().Data())
}
if len(results) != 2 {
t.Errorf("Expected 2 results, got %d", len(results))
}
// 验证结果内容
for _, result := range results {
if result["email"] != "alice@example.com" {
t.Errorf("Unexpected email: %v", result["email"])
}
name := result["name"].(string)
if name != "Alice" && name != "Charlie" {
t.Errorf("Unexpected name: %s", name)
}
}
t.Logf("✓ Index query returned correct results: %d rows", len(results))
// 9. 测试没有索引的查询（应该正常工作但不使用索引）
rows2, err := table.Query().Eq("name", "Bob").Rows()
if err != nil {
t.Fatalf("Query without index failed: %v", err)
}
defer rows2.Close()
results2 := []map[string]any{}
for rows2.Next() {
results2 = append(results2, rows2.Row().Data())
}
if len(results2) != 1 {
t.Errorf("Expected 1 result for Bob, got %d", len(results2))
}
t.Logf("✓ Non-indexed query works correctly: %d rows", len(results2))
// 10. 测试索引在新数据上的工作
err = table.Insert(map[string]any{
"name": "Eve",
"email": "eve@example.com",
"age": int64(28),
})
if err != nil {
t.Fatalf("Failed to insert new data: %v", err)
}
// 查询新插入的数据（索引尚未持久化，但应该在内存中）
rows3, err := table.Query().Eq("email", "eve@example.com").Rows()
if err != nil {
t.Fatalf("Query for new data failed: %v", err)
}
defer rows3.Close()
results3 := []map[string]any{}
for rows3.Next() {
results3 = append(results3, rows3.Row().Data())
}
if len(results3) != 1 {
t.Errorf("Expected 1 result for Eve (new data), got %d", len(results3))
}
t.Logf("✓ Index works for new data (before persistence): %d rows", len(results3))
// 11. 再次构建索引并验证
err = table.indexManager.BuildAll()
if err != nil {
t.Fatalf("Failed to rebuild indexes: %v", err)
}
rows4, err := table.Query().Eq("email", "eve@example.com").Rows()
if err != nil {
t.Fatalf("Query after rebuild failed: %v", err)
}
defer rows4.Close()
results4 := []map[string]any{}
for rows4.Next() {
results4 = append(results4, rows4.Row().Data())
}
if len(results4) != 1 {
t.Errorf("Expected 1 result for Eve (after rebuild), got %d", len(results4))
}
t.Logf("✓ Index works after rebuild: %d rows", len(results4))
t.Log("=== All index query tests passed ===")
}
// TestIndexPersistenceAcrossRestart 测试索引在重启后的持久化
func TestIndexPersistenceAcrossRestart(t *testing.T) {
tmpDir := t.TempDir()
// 1. 第一次打开：创建数据和索引
{
schema := NewSchema("products", []Field{
{Name: "name", Type: FieldTypeString, Indexed: false},
{Name: "category", Type: FieldTypeString, Indexed: true},
{Name: "price", Type: FieldTypeInt64, Indexed: false},
})
table, err := OpenTable(&TableOptions{
Dir: tmpDir,
Name: schema.Name,
Fields: schema.Fields,
MemTableSize: 1024 * 1024,
})
if err != nil {
t.Fatal(err)
}
// 创建索引
err = table.CreateIndex("category")
if err != nil {
t.Fatal(err)
}
// 插入数据
testData := []map[string]any{
{"name": "Laptop", "category": "Electronics", "price": int64(1000)},
{"name": "Mouse", "category": "Electronics", "price": int64(50)},
{"name": "Desk", "category": "Furniture", "price": int64(300)},
}
for _, data := range testData {
err := table.Insert(data)
if err != nil {
t.Fatal(err)
}
}
// 构建索引
err = table.indexManager.BuildAll()
if err != nil {
t.Fatal(err)
}
// 关闭表
table.Close()
t.Log("✓ First session: data and index created")
}
// 2. 第二次打开：验证索引仍然可用
{
table, err := OpenTable(&TableOptions{
Dir: tmpDir,
MemTableSize: 1024 * 1024,
})
if err != nil {
t.Fatal(err)
}
defer table.Close()
// 验证索引存在
indexes := table.ListIndexes()
if len(indexes) != 1 || indexes[0] != "category" {
t.Errorf("Expected index on 'category', got: %v", indexes)
}
t.Log("✓ Index loaded after restart")
// 使用索引查询
rows, err := table.Query().Eq("category", "Electronics").Rows()
if err != nil {
t.Fatalf("Query failed: %v", err)
}
defer rows.Close()
results := []map[string]any{}
for rows.Next() {
results = append(results, rows.Row().Data())
}
if len(results) != 2 {
t.Errorf("Expected 2 Electronics products, got %d", len(results))
}
t.Logf("✓ Index query after restart: %d rows", len(results))
}
t.Log("=== Index persistence test passed ===")
}

65
query.go
View File

@@ -484,6 +484,14 @@ func (qb *QueryBuilder) Rows() (*Rows, error) {
visited: make(map[int64]bool),
}
// 尝试使用索引优化查询
// 检查是否有可以使用索引的 Eq 条件
indexField, indexValue := qb.findIndexableCondition()
if indexField != "" {
// 使用索引查询
return qb.rowsWithIndex(rows, indexField, indexValue)
}
// 收集所有数据源的 keys 并全局排序
// 立即读取数据避免 compaction 期间文件被删除
keyToRow := make(map[int64]*SSTableRow) // 存储已读取的行数据
@@ -584,6 +592,63 @@ func (qb *QueryBuilder) Rows() (*Rows, error) {
return rows, nil
}
// findIndexableCondition 查找可以使用索引的条件（Eq 操作）
func (qb *QueryBuilder) findIndexableCondition() (string, any) {
for _, cond := range qb.conds {
// 检查是否是 compare 类型且操作符是 "="
if cmp, ok := cond.(compare); ok && cmp.op == "=" {
// 检查该字段是否有索引
if idx, exists := qb.table.indexManager.GetIndex(cmp.field); exists && idx.IsReady() {
return cmp.field, cmp.right
}
}
}
return "", nil
}
// rowsWithIndex 使用索引查询数据
func (qb *QueryBuilder) rowsWithIndex(rows *Rows, indexField string, indexValue any) (*Rows, error) {
// 获取索引
idx, exists := qb.table.indexManager.GetIndex(indexField)
if !exists {
return nil, fmt.Errorf("index on field %s not found", indexField)
}
// 从索引获取 seq 列表
seqs, err := idx.Get(indexValue)
if err != nil {
return nil, fmt.Errorf("index lookup failed: %w", err)
}
// 如果没有结果，返回空结果集
if len(seqs) == 0 {
rows.cached = true
rows.cachedIndex = -1
rows.cachedRows = []*SSTableRow{}
return rows, nil
}
// 根据 seq 列表获取数据
rows.cachedRows = make([]*SSTableRow, 0, len(seqs))
for _, seq := range seqs {
row, err := qb.table.Get(seq)
if err != nil {
continue // 跳过获取失败的记录
}
// 检查是否匹配所有其他条件（索引只能优化一个条件）
if qb.Match(row.Data) {
rows.cachedRows = append(rows.cachedRows, row)
}
}
// 使用缓存模式
rows.cached = true
rows.cachedIndex = -1
return rows, nil
}
// First 返回第一个匹配的数据
func (qb *QueryBuilder) First() (*Row, error) {
rows, err := qb.Rows()

195
schema.go
View File

@@ -4,6 +4,7 @@ import (
"crypto/sha256"
"encoding/hex"
"fmt"
"reflect"
"sort"
"strings"
"time"
@@ -48,7 +49,7 @@ type Schema struct {
Fields []Field // 字段列表
}
// New 创建 Schema
// NewSchema 创建 Schema
func NewSchema(name string, fields []Field) *Schema {
return &Schema{
Name: name,
@@ -56,6 +57,198 @@ func NewSchema(name string, fields []Field) *Schema {
}
}
// StructToFields 从 Go 结构体生成 Field 列表
//
// 支持的 struct tag 格式：
// - `srdb:"name"` - 指定字段名（默认使用 snake_case 转换）
// - `srdb:"name;indexed"` - 指定字段名并标记为索引
// - `srdb:"name;indexed;comment:用户名"` - 完整格式（字段名;索引标记;注释）
// - `srdb:"-"` - 忽略该字段
//
// Tag 格式说明：
// - 使用分号 `;` 分隔不同的部分
// - 第一部分是字段名（可选，默认使用 snake_case 转换结构体字段名）
// - `indexed` 标记该字段需要索引
// - `comment:注释内容` 指定字段注释
//
// 默认字段名转换示例：
// - UserName -> user_name
// - EmailAddress -> email_address
// - IsActive -> is_active
//
// 类型映射：
// - int, int64, int32, int16, int8, uint, uint64, uint32, uint16, uint8 -> FieldTypeInt64
// - string -> FieldTypeString
// - float64, float32 -> FieldTypeFloat
// - bool -> FieldTypeBool
//
// 示例：
// type User struct {
// Name string `srdb:"name;indexed;comment:用户名"`
// Age int64 `srdb:"age;comment:年龄"`
// Email string `srdb:"email;indexed;comment:邮箱"`
// }
// fields, err := StructToFields(User{})
//
// 参数：
// - v: 结构体实例或指针
//
// 返回：
// - []Field: 字段列表
// - error: 错误信息
func StructToFields(v any) ([]Field, error) {
// 获取类型
typ := reflect.TypeOf(v)
if typ == nil {
return nil, fmt.Errorf("invalid type: nil")
}
// 如果是指针，获取其指向的类型
if typ.Kind() == reflect.Ptr {
typ = typ.Elem()
}
// 必须是结构体
if typ.Kind() != reflect.Struct {
return nil, fmt.Errorf("expected struct, got %s", typ.Kind())
}
var fields []Field
// 遍历结构体字段
for i := 0; i < typ.NumField(); i++ {
field := typ.Field(i)
// 跳过未导出的字段
if !field.IsExported() {
continue
}
// 解析 srdb tag
tag := field.Tag.Get("srdb")
if tag == "-" {
// 忽略该字段
continue
}
// 解析字段名、索引标记和注释
fieldName := camelToSnake(field.Name) // 默认使用 snake_case 字段名
indexed := false
comment := ""
if tag != "" {
// 使用分号分隔各部分
parts := strings.Split(tag, ";")
for idx, part := range parts {
part = strings.TrimSpace(part)
if idx == 0 && part != "" {
// 第一部分是字段名
fieldName = part
} else if part == "indexed" {
// indexed 标记
indexed = true
} else if strings.HasPrefix(part, "comment:") {
// comment:注释内容
comment = strings.TrimPrefix(part, "comment:")
}
}
}
// 映射 Go 类型到 FieldType
fieldType, err := goTypeToFieldType(field.Type)
if err != nil {
return nil, fmt.Errorf("field %s: %w", field.Name, err)
}
fields = append(fields, Field{
Name: fieldName,
Type: fieldType,
Indexed: indexed,
Comment: comment,
})
}
if len(fields) == 0 {
return nil, fmt.Errorf("no exported fields found in struct")
}
return fields, nil
}
// goTypeToFieldType 将 Go 类型映射到 FieldType
func goTypeToFieldType(typ reflect.Type) (FieldType, error) {
switch typ.Kind() {
case reflect.Int, reflect.Int64, reflect.Int32, reflect.Int16, reflect.Int8,
reflect.Uint, reflect.Uint64, reflect.Uint32, reflect.Uint16, reflect.Uint8:
return FieldTypeInt64, nil
case reflect.String:
return FieldTypeString, nil
case reflect.Float64, reflect.Float32:
return FieldTypeFloat, nil
case reflect.Bool:
return FieldTypeBool, nil
default:
return 0, fmt.Errorf("unsupported type: %s", typ.Kind())
}
}
// camelToSnake 将驼峰命名转换为 snake_case
//
// 示例：
// - UserName -> user_name
// - EmailAddress -> email_address
// - IsActive -> is_active
// - HTTPServer -> http_server
// - ID -> id
func camelToSnake(s string) string {
var result strings.Builder
result.Grow(len(s) + 5) // 预分配空间
for i, r := range s {
// 如果是大写字母
if r >= 'A' && r <= 'Z' {
// 不是第一个字符，并且前一个字符不是大写，需要添加下划线
if i > 0 {
// 检查是否需要添加下划线
// 规则：
// 1. 前一个字符是小写字母 -> 添加下划线
// 2. 当前是大写，后一个是小写（处理 HTTPServer -> http_server）-> 添加下划线
prevChar := rune(s[i-1])
needUnderscore := false
if prevChar >= 'a' && prevChar <= 'z' {
// 前一个是小写字母
needUnderscore = true
} else if prevChar >= 'A' && prevChar <= 'Z' {
// 前一个是大写字母，检查后一个
if i+1 < len(s) {
nextChar := rune(s[i+1])
if nextChar >= 'a' && nextChar <= 'z' {
// 后一个是小写字母，说明是新单词开始
needUnderscore = true
}
}
} else {
// 前一个是数字或其他字符
needUnderscore = true
}
if needUnderscore {
result.WriteRune('_')
}
}
// 转换为小写
result.WriteRune(r + 32) // 'A' -> 'a' 的 ASCII 差值是 32
} else {
result.WriteRune(r)
}
}
return result.String()
}
// GetField 获取字段定义
func (s *Schema) GetField(name string) (*Field, error) {
for i := range s.Fields {

455
schema_test.go
View File

@@ -265,3 +265,458 @@ func TestChecksumMultipleFieldOrders(t *testing.T) {
t.Logf("✅ All %d field permutations produce the same checksum", len(schemas))
t.Logf(" checksum: %s", expectedChecksum)
}
// TestStructToFields 测试从结构体生成 Field 列表
func TestStructToFields(t *testing.T) {
// 定义测试结构体
type User struct {
Name string `srdb:"name;indexed;comment:用户名"`
Age int64 `srdb:"age;comment:年龄"`
Email string `srdb:"email;indexed;comment:邮箱"`
Score float64 `srdb:"score;comment:分数"`
Active bool `srdb:"active;comment:是否激活"`
}
// 生成 Field 列表
fields, err := StructToFields(User{})
if err != nil {
t.Fatalf("StructToFields failed: %v", err)
}
// 验证字段数量
if len(fields) != 5 {
t.Errorf("Expected 5 fields, got %d", len(fields))
}
// 验证每个字段
expectedFields := map[string]struct {
Type FieldType
Indexed bool
Comment string
}{
"name": {FieldTypeString, true, "用户名"},
"age": {FieldTypeInt64, false, "年龄"},
"email": {FieldTypeString, true, "邮箱"},
"score": {FieldTypeFloat, false, "分数"},
"active": {FieldTypeBool, false, "是否激活"},
}
for _, field := range fields {
expected, exists := expectedFields[field.Name]
if !exists {
t.Errorf("Unexpected field: %s", field.Name)
continue
}
if field.Type != expected.Type {
t.Errorf("Field %s: expected type %v, got %v", field.Name, expected.Type, field.Type)
}
if field.Indexed != expected.Indexed {
t.Errorf("Field %s: expected indexed=%v, got %v", field.Name, expected.Indexed, field.Indexed)
}
if field.Comment != expected.Comment {
t.Errorf("Field %s: expected comment=%s, got %s", field.Name, expected.Comment, field.Comment)
}
}
t.Log("✓ StructToFields basic test passed")
}
// TestStructToFieldsDefaultName 测试默认字段名 (snake_case)
func TestStructToFieldsDefaultName(t *testing.T) {
type Product struct {
ProductName string // 没有 tag，应该使用 snake_case: product_name
Price int64 // 没有 tag，应该使用 snake_case: price
}
fields, err := StructToFields(Product{})
if err != nil {
t.Fatalf("StructToFields failed: %v", err)
}
if len(fields) != 2 {
t.Errorf("Expected 2 fields, got %d", len(fields))
}
// 验证默认字段名（snake_case）
if fields[0].Name != "product_name" {
t.Errorf("Expected field name 'product_name', got '%s'", fields[0].Name)
}
if fields[1].Name != "price" {
t.Errorf("Expected field name 'price', got '%s'", fields[1].Name)
}
t.Log("✓ Default field name (snake_case) test passed")
}
// TestStructToFieldsIgnore 测试忽略字段
func TestStructToFieldsIgnore(t *testing.T) {
type Order struct {
OrderID string `srdb:"order_id;comment:订单ID"`
Internal string `srdb:"-"` // 应该被忽略
CreatedAt int64 `srdb:"created_at;comment:创建时间"`
}
fields, err := StructToFields(Order{})
if err != nil {
t.Fatalf("StructToFields failed: %v", err)
}
// 应该只有 2 个字段（Internal 被忽略）
if len(fields) != 2 {
t.Errorf("Expected 2 fields (excluding ignored field), got %d", len(fields))
}
// 验证没有 Internal 字段
for _, field := range fields {
if field.Name == "internal" || field.Name == "Internal" {
t.Errorf("Field 'Internal' should have been ignored")
}
}
t.Log("✓ Ignore field test passed")
}
// TestStructToFieldsPointer 测试指针类型
func TestStructToFieldsPointer(t *testing.T) {
type Item struct {
Name string `srdb:"name;comment:名称"`
}
// 使用指针
fields, err := StructToFields(&Item{})
if err != nil {
t.Fatalf("StructToFields with pointer failed: %v", err)
}
if len(fields) != 1 {
t.Errorf("Expected 1 field, got %d", len(fields))
}
if fields[0].Name != "name" {
t.Errorf("Expected field name 'name', got '%s'", fields[0].Name)
}
t.Log("✓ Pointer type test passed")
}
// TestStructToFieldsAllTypes 测试所有支持的类型
func TestStructToFieldsAllTypes(t *testing.T) {
type AllTypes struct {
Int int `srdb:"int"`
Int64 int64 `srdb:"int64"`
Int32 int32 `srdb:"int32"`
Int16 int16 `srdb:"int16"`
Int8 int8 `srdb:"int8"`
Uint uint `srdb:"uint"`
Uint64 uint64 `srdb:"uint64"`
Uint32 uint32 `srdb:"uint32"`
Uint16 uint16 `srdb:"uint16"`
Uint8 uint8 `srdb:"uint8"`
String string `srdb:"string"`
Float64 float64 `srdb:"float64"`
Float32 float32 `srdb:"float32"`
Bool bool `srdb:"bool"`
}
fields, err := StructToFields(AllTypes{})
if err != nil {
t.Fatalf("StructToFields failed: %v", err)
}
if len(fields) != 14 {
t.Errorf("Expected 14 fields, got %d", len(fields))
}
// 验证所有整数类型都映射到 FieldTypeInt64
intFields := []string{"int", "int64", "int32", "int16", "int8", "uint", "uint64", "uint32", "uint16", "uint8"}
for _, name := range intFields {
found := false
for _, field := range fields {
if field.Name == name {
found = true
if field.Type != FieldTypeInt64 {
t.Errorf("Field %s: expected FieldTypeInt64, got %v", name, field.Type)
}
break
}
}
if !found {
t.Errorf("Field %s not found", name)
}
}
// 验证浮点类型
for _, field := range fields {
if field.Name == "float64" || field.Name == "float32" {
if field.Type != FieldTypeFloat {
t.Errorf("Field %s: expected FieldTypeFloat, got %v", field.Name, field.Type)
}
}
}
t.Log("✓ All types test passed")
}
// TestStructToFieldsWithSchema 测试完整的使用流程
func TestStructToFieldsWithSchema(t *testing.T) {
// 定义结构体
type Customer struct {
CustomerID string `srdb:"customer_id;indexed;comment:客户ID"`
Name string `srdb:"name;comment:客户名称"`
Email string `srdb:"email;indexed;comment:邮箱"`
Balance int64 `srdb:"balance;comment:余额"`
}
// 生成 Field 列表
fields, err := StructToFields(Customer{})
if err != nil {
t.Fatalf("StructToFields failed: %v", err)
}
// 创建 Schema
schema := NewSchema("customers", fields)
// 验证 Schema
if schema.Name != "customers" {
t.Errorf("Expected schema name 'customers', got '%s'", schema.Name)
}
if len(schema.Fields) != 4 {
t.Errorf("Expected 4 fields in schema, got %d", len(schema.Fields))
}
// 验证索引字段
indexedFields := schema.GetIndexedFields()
if len(indexedFields) != 2 {
t.Errorf("Expected 2 indexed fields, got %d", len(indexedFields))
}
// 测试数据验证
validData := map[string]any{
"customer_id": "C001",
"name": "张三",
"email": "zhangsan@example.com",
"balance": int64(1000),
}
err = schema.Validate(validData)
if err != nil {
t.Errorf("Valid data should pass validation: %v", err)
}
// 测试无效数据
invalidData := map[string]any{
"customer_id": "C002",
"name": "李四",
"email": 123, // 错误类型
"balance": int64(2000),
}
err = schema.Validate(invalidData)
if err == nil {
t.Error("Invalid data should fail validation")
}
t.Log("✓ Complete workflow test passed")
}
// TestStructToFieldsTagVariations 测试各种 tag 组合
func TestStructToFieldsTagVariations(t *testing.T) {
type TestStruct struct {
// 只有字段名
Field1 string `srdb:"field1"`
// 字段名 + indexed
Field2 string `srdb:"field2;indexed"`
// 字段名 + comment
Field3 string `srdb:"field3;comment:字段3"`
// 完整格式
Field4 string `srdb:"field4;indexed;comment:字段4"`
// 只有 indexed（使用默认字段名）
Field5 string `srdb:";indexed"`
// 只有 comment（使用默认字段名）
Field6 string `srdb:";comment:字段6"`
// 空 tag（使用默认字段名）
Field7 string
// indexed + comment（使用默认字段名）
Field8 string `srdb:";indexed;comment:字段8"`
}
fields, err := StructToFields(TestStruct{})
if err != nil {
t.Fatalf("StructToFields failed: %v", err)
}
if len(fields) != 8 {
t.Errorf("Expected 8 fields, got %d", len(fields))
}
// 验证各个字段
tests := []struct {
name string
indexed bool
comment string
}{
{"field1", false, ""},
{"field2", true, ""},
{"field3", false, "字段3"},
{"field4", true, "字段4"},
{"field5", true, ""},
{"field6", false, "字段6"},
{"field7", false, ""},
{"field8", true, "字段8"},
}
for i, test := range tests {
if fields[i].Name != test.name {
t.Errorf("Field %d: expected name %s, got %s", i+1, test.name, fields[i].Name)
}
if fields[i].Indexed != test.indexed {
t.Errorf("Field %s: expected indexed=%v, got %v", test.name, test.indexed, fields[i].Indexed)
}
if fields[i].Comment != test.comment {
t.Errorf("Field %s: expected comment=%s, got %s", test.name, test.comment, fields[i].Comment)
}
}
t.Log("✓ Tag variations test passed")
}
// TestStructToFieldsErrors 测试错误情况
func TestStructToFieldsErrors(t *testing.T) {
// 测试非结构体类型
_, err := StructToFields("not a struct")
if err == nil {
t.Error("Expected error for non-struct type")
}
// 测试 nil
_, err = StructToFields(nil)
if err == nil {
t.Error("Expected error for nil")
}
// 测试没有导出字段的结构体
type Empty struct {
private string // 未导出
}
_, err = StructToFields(Empty{})
if err == nil {
t.Error("Expected error for struct with no exported fields")
}
t.Log("✓ Error handling test passed")
}
// TestCamelToSnake 测试驼峰命名转 snake_case
func TestCamelToSnake(t *testing.T) {
tests := []struct {
input string
expected string
}{
// 基本测试
{"UserName", "user_name"},
{"EmailAddress", "email_address"},
{"IsActive", "is_active"},
// 单个单词
{"Name", "name"},
{"ID", "id"},
// 连续大写字母
{"HTTPServer", "http_server"},
{"XMLParser", "xml_parser"},
{"HTMLContent", "html_content"},
{"URLPath", "url_path"},
// 带数字
{"User2Name", "user2_name"},
{"Address1", "address1"},
// 全小写
{"username", "username"},
// 全大写
{"HTTP", "http"},
{"API", "api"},
// 混合情况
{"getUserByID", "get_user_by_id"},
{"HTTPSConnection", "https_connection"},
{"createHTMLFile", "create_html_file"},
// 边界情况
{"A", "a"},
{"AB", "ab"},
{"AbC", "ab_c"},
}
for _, test := range tests {
result := camelToSnake(test.input)
if result != test.expected {
t.Errorf("camelToSnake(%q) = %q, expected %q", test.input, result, test.expected)
}
}
t.Log("✓ camelToSnake test passed")
}
// TestStructToFieldsSnakeCase 测试默认使用 snake_case
func TestStructToFieldsSnakeCase(t *testing.T) {
type User struct {
UserName string // 应该转为 user_name
EmailAddress string // 应该转为 email_address
IsActive bool // 应该转为 is_active
HTTPEndpoint string // 应该转为 http_endpoint
ID int64 // 应该转为 id
}
fields, err := StructToFields(User{})
if err != nil {
t.Fatalf("StructToFields failed: %v", err)
}
expected := []string{"user_name", "email_address", "is_active", "http_endpoint", "id"}
if len(fields) != len(expected) {
t.Fatalf("Expected %d fields, got %d", len(expected), len(fields))
}
for i, exp := range expected {
if fields[i].Name != exp {
t.Errorf("Field %d: expected name %s, got %s", i, exp, fields[i].Name)
}
}
t.Log("✓ Default snake_case test passed")
}
// TestStructToFieldsOverrideSnakeCase 测试可以覆盖默认 snake_case
func TestStructToFieldsOverrideSnakeCase(t *testing.T) {
type User struct {
UserName string `srdb:"username"` // 覆盖默认的 user_name
IsActive bool `srdb:"active;comment:激活"` // 覆盖默认的 is_active
}
fields, err := StructToFields(User{})
if err != nil {
t.Fatalf("StructToFields failed: %v", err)
}
if len(fields) != 2 {
t.Fatalf("Expected 2 fields, got %d", len(fields))
}
// 验证覆盖成功
if fields[0].Name != "username" {
t.Errorf("Expected field name 'username', got '%s'", fields[0].Name)
}
if fields[1].Name != "active" {
t.Errorf("Expected field name 'active', got '%s'", fields[1].Name)
}
t.Log("✓ Override snake_case test passed")
}

179
table.go
View File

@@ -5,7 +5,9 @@ import (
"fmt"
"os"
"path/filepath"
"reflect"
"sort"
"strings"
"sync"
"sync/atomic"
"time"
@@ -203,8 +205,181 @@ func OpenTable(opts *TableOptions) (*Table, error) {
return table, nil
}
// Insert 插入数据
func (t *Table) Insert(data map[string]any) error {
// Insert 插入数据（支持单条或批量）
// 支持的类型：
// - map[string]any: 单条数据
// - []map[string]any: 批量数据
// - *struct{}: 单个结构体指针
// - []struct{}: 结构体切片
// - []*struct{}: 结构体指针切片
func (t *Table) Insert(data any) error {
// 1. 将输入转换为 []map[string]any
rows, err := t.normalizeInsertData(data)
if err != nil {
return err
}
// 2. 批量插入
return t.insertBatch(rows)
}
// normalizeInsertData 将各种输入格式转换为 []map[string]any
func (t *Table) normalizeInsertData(data any) ([]map[string]any, error) {
// 处理 nil
if data == nil {
return nil, fmt.Errorf("data cannot be nil")
}
// 获取反射值
val := reflect.ValueOf(data)
typ := reflect.TypeOf(data)
// 如果是指针，解引用
if typ.Kind() == reflect.Ptr {
if val.IsNil() {
return nil, fmt.Errorf("data pointer cannot be nil")
}
val = val.Elem()
typ = val.Type()
}
switch typ.Kind() {
case reflect.Map:
// map[string]any - 单条
m, ok := data.(map[string]any)
if !ok {
return nil, fmt.Errorf("expected map[string]any, got %T", data)
}
return []map[string]any{m}, nil
case reflect.Slice:
// 检查切片元素类型
elemType := typ.Elem()
// []map[string]any
if elemType.Kind() == reflect.Map {
maps, ok := data.([]map[string]any)
if !ok {
return nil, fmt.Errorf("expected []map[string]any, got %T", data)
}
return maps, nil
}
// []*struct{} 或 []struct{}
if elemType.Kind() == reflect.Ptr {
elemType = elemType.Elem()
}
if elemType.Kind() == reflect.Struct {
// 将每个结构体转换为 map
var rows []map[string]any
for i := 0; i < val.Len(); i++ {
elem := val.Index(i)
// 如果是指针，解引用
if elem.Kind() == reflect.Ptr {
if elem.IsNil() {
continue // 跳过 nil 指针
}
elem = elem.Elem()
}
m, err := t.structToMap(elem.Interface())
if err != nil {
return nil, fmt.Errorf("convert struct at index %d: %w", i, err)
}
rows = append(rows, m)
}
return rows, nil
}
return nil, fmt.Errorf("unsupported slice element type: %s", elemType.Kind())
case reflect.Struct:
// struct{} - 单个结构体
m, err := t.structToMap(data)
if err != nil {
return nil, err
}
return []map[string]any{m}, nil
default:
return nil, fmt.Errorf("unsupported data type: %T (kind: %s)", data, typ.Kind())
}
}
// structToMap 将结构体转换为 map[string]any
func (t *Table) structToMap(v any) (map[string]any, error) {
val := reflect.ValueOf(v)
typ := reflect.TypeOf(v)
if typ.Kind() == reflect.Ptr {
val = val.Elem()
typ = val.Type()
}
if typ.Kind() != reflect.Struct {
return nil, fmt.Errorf("expected struct, got %s", typ.Kind())
}
result := make(map[string]any)
for i := 0; i < typ.NumField(); i++ {
field := typ.Field(i)
// 跳过未导出的字段
if !field.IsExported() {
continue
}
// 获取字段名
fieldName := field.Name
tag := field.Tag.Get("srdb")
// 跳过忽略的字段
if tag == "-" {
continue
}
// 解析 tag 获取字段名
if tag != "" {
parts := strings.Split(tag, ";")
if parts[0] != "" {
fieldName = parts[0]
} else {
// 使用 snake_case 转换
fieldName = camelToSnake(field.Name)
}
} else {
// 没有 tag，使用 snake_case 转换
fieldName = camelToSnake(field.Name)
}
// 获取字段值
fieldVal := val.Field(i)
result[fieldName] = fieldVal.Interface()
}
return result, nil
}
// insertBatch 批量插入数据
func (t *Table) insertBatch(rows []map[string]any) error {
if len(rows) == 0 {
return nil
}
// 逐条插入
for _, data := range rows {
if err := t.insertSingle(data); err != nil {
return err
}
}
return nil
}
// insertSingle 插入单条数据
func (t *Table) insertSingle(data map[string]any) error {
// 1. 验证 Schema
if err := t.schema.Validate(data); err != nil {
return NewError(ErrCodeSchemaValidationFailed, err)

469
table_test.go
View File

@@ -1844,3 +1844,472 @@ func TestTableCleanAndQuery(t *testing.T) {
t.Errorf("Expected 1 row, got %d", count)
}
}
// TestInsertMap 测试插入 map[string]any
func TestInsertMap(t *testing.T) {
tmpDir, _ := os.MkdirTemp("", "TestInsertMap")
defer os.RemoveAll(tmpDir)
schema := NewSchema("users", []Field{
{Name: "name", Type: FieldTypeString},
{Name: "age", Type: FieldTypeInt64},
})
table, err := OpenTable(&TableOptions{
Dir: tmpDir,
Name: schema.Name,
Fields: schema.Fields,
})
if err != nil {
t.Fatal(err)
}
defer table.Close()
// 插入单个 map
err = table.Insert(map[string]any{
"name": "Alice",
"age": int64(25),
})
if err != nil {
t.Fatalf("Insert map failed: %v", err)
}
// 验证
row, err := table.Get(1)
if err != nil {
t.Fatalf("Get failed: %v", err)
}
if row.Data["name"] != "Alice" {
t.Errorf("Expected name=Alice, got %v", row.Data["name"])
}
t.Log("✓ Insert map test passed")
}
// TestInsertMapSlice 测试插入 []map[string]any
func TestInsertMapSlice(t *testing.T) {
tmpDir, _ := os.MkdirTemp("", "TestInsertMapSlice")
defer os.RemoveAll(tmpDir)
schema := NewSchema("users", []Field{
{Name: "name", Type: FieldTypeString},
{Name: "age", Type: FieldTypeInt64},
})
table, err := OpenTable(&TableOptions{
Dir: tmpDir,
Name: schema.Name,
Fields: schema.Fields,
})
if err != nil {
t.Fatal(err)
}
defer table.Close()
// 批量插入 maps
err = table.Insert([]map[string]any{
{"name": "Alice", "age": int64(25)},
{"name": "Bob", "age": int64(30)},
{"name": "Charlie", "age": int64(35)},
})
if err != nil {
t.Fatalf("Insert map slice failed: %v", err)
}
// 验证
row1, _ := table.Get(1)
row2, _ := table.Get(2)
row3, _ := table.Get(3)
if row1.Data["name"] != "Alice" || row2.Data["name"] != "Bob" || row3.Data["name"] != "Charlie" {
t.Errorf("Data mismatch")
}
t.Log("✓ Insert map slice test passed (3 rows)")
}
// TestInsertStruct 测试插入单个结构体
func TestInsertStruct(t *testing.T) {
tmpDir, _ := os.MkdirTemp("", "TestInsertStruct")
defer os.RemoveAll(tmpDir)
type User struct {
Name string `srdb:"name"`
Age int64 `srdb:"age"`
Email string `srdb:"email"`
}
schema := NewSchema("users", []Field{
{Name: "name", Type: FieldTypeString},
{Name: "age", Type: FieldTypeInt64},
{Name: "email", Type: FieldTypeString},
})
table, err := OpenTable(&TableOptions{
Dir: tmpDir,
Name: schema.Name,
Fields: schema.Fields,
})
if err != nil {
t.Fatal(err)
}
defer table.Close()
// 插入单个结构体
user := User{
Name: "Alice",
Age: 25,
Email: "alice@example.com",
}
err = table.Insert(user)
if err != nil {
t.Fatalf("Insert struct failed: %v", err)
}
// 验证
row, err := table.Get(1)
if err != nil {
t.Fatalf("Get failed: %v", err)
}
if row.Data["name"] != "Alice" {
t.Errorf("Expected name=Alice, got %v", row.Data["name"])
}
if row.Data["email"] != "alice@example.com" {
t.Errorf("Expected email=alice@example.com, got %v", row.Data["email"])
}
t.Log("✓ Insert struct test passed")
}
// TestInsertStructPointer 测试插入结构体指针
func TestInsertStructPointer(t *testing.T) {
tmpDir, _ := os.MkdirTemp("", "TestInsertStructPointer")
defer os.RemoveAll(tmpDir)
type User struct {
Name string `srdb:"name"`
Age int64 `srdb:"age"`
Email string `srdb:"email"`
}
schema := NewSchema("users", []Field{
{Name: "name", Type: FieldTypeString},
{Name: "age", Type: FieldTypeInt64},
{Name: "email", Type: FieldTypeString},
})
table, err := OpenTable(&TableOptions{
Dir: tmpDir,
Name: schema.Name,
Fields: schema.Fields,
})
if err != nil {
t.Fatal(err)
}
defer table.Close()
// 插入结构体指针
user := &User{
Name: "Bob",
Age: 30,
Email: "bob@example.com",
}
err = table.Insert(user)
if err != nil {
t.Fatalf("Insert struct pointer failed: %v", err)
}
// 验证
row, err := table.Get(1)
if err != nil {
t.Fatalf("Get failed: %v", err)
}
if row.Data["name"] != "Bob" {
t.Errorf("Expected name=Bob, got %v", row.Data["name"])
}
t.Log("✓ Insert struct pointer test passed")
}
// TestInsertStructSlice 测试插入结构体切片
func TestInsertStructSlice(t *testing.T) {
tmpDir, _ := os.MkdirTemp("", "TestInsertStructSlice")
defer os.RemoveAll(tmpDir)
type User struct {
Name string `srdb:"name"`
Age int64 `srdb:"age"`
}
schema := NewSchema("users", []Field{
{Name: "name", Type: FieldTypeString},
{Name: "age", Type: FieldTypeInt64},
})
table, err := OpenTable(&TableOptions{
Dir: tmpDir,
Name: schema.Name,
Fields: schema.Fields,
})
if err != nil {
t.Fatal(err)
}
defer table.Close()
// 批量插入结构体切片
users := []User{
{Name: "Alice", Age: 25},
{Name: "Bob", Age: 30},
{Name: "Charlie", Age: 35},
}
err = table.Insert(users)
if err != nil {
t.Fatalf("Insert struct slice failed: %v", err)
}
// 验证
row1, _ := table.Get(1)
row2, _ := table.Get(2)
row3, _ := table.Get(3)
if row1.Data["name"] != "Alice" || row2.Data["name"] != "Bob" || row3.Data["name"] != "Charlie" {
t.Errorf("Data mismatch")
}
t.Log("✓ Insert struct slice test passed (3 rows)")
}
// TestInsertStructPointerSlice 测试插入结构体指针切片
func TestInsertStructPointerSlice(t *testing.T) {
tmpDir, _ := os.MkdirTemp("", "TestInsertStructPointerSlice")
defer os.RemoveAll(tmpDir)
type User struct {
Name string `srdb:"name"`
Age int64 `srdb:"age"`
}
schema := NewSchema("users", []Field{
{Name: "name", Type: FieldTypeString},
{Name: "age", Type: FieldTypeInt64},
})
table, err := OpenTable(&TableOptions{
Dir: tmpDir,
Name: schema.Name,
Fields: schema.Fields,
})
if err != nil {
t.Fatal(err)
}
defer table.Close()
// 批量插入结构体指针切片
users := []*User{
{Name: "Alice", Age: 25},
{Name: "Bob", Age: 30},
nil, // 测试 nil 指针会被跳过
{Name: "Charlie", Age: 35},
}
err = table.Insert(users)
if err != nil {
t.Fatalf("Insert struct pointer slice failed: %v", err)
}
// 验证（应该只有 3 条记录，nil 被跳过）
row1, _ := table.Get(1)
row2, _ := table.Get(2)
row3, _ := table.Get(3)
if row1.Data["name"] != "Alice" || row2.Data["name"] != "Bob" || row3.Data["name"] != "Charlie" {
t.Errorf("Data mismatch")
}
t.Log("✓ Insert struct pointer slice test passed (3 rows, nil skipped)")
}
// TestInsertWithSnakeCase 测试结构体自动 snake_case 转换
func TestInsertWithSnakeCase(t *testing.T) {
tmpDir, _ := os.MkdirTemp("", "TestInsertWithSnakeCase")
defer os.RemoveAll(tmpDir)
type User struct {
UserName string `srdb:";comment:用户名"` // 没有指定字段名，应该自动转为 user_name
EmailAddress string // 没有 tag，应该自动转为 email_address
IsActive bool // 应该自动转为 is_active
}
schema := NewSchema("users", []Field{
{Name: "user_name", Type: FieldTypeString, Comment: "用户名"},
{Name: "email_address", Type: FieldTypeString},
{Name: "is_active", Type: FieldTypeBool},
})
table, err := OpenTable(&TableOptions{
Dir: tmpDir,
Name: schema.Name,
Fields: schema.Fields,
})
if err != nil {
t.Fatal(err)
}
defer table.Close()
// 插入结构体
user := User{
UserName: "Alice",
EmailAddress: "alice@example.com",
IsActive: true,
}
err = table.Insert(user)
if err != nil {
t.Fatalf("Insert failed: %v", err)
}
// 验证字段名是否正确转换
row, err := table.Get(1)
if err != nil {
t.Fatalf("Get failed: %v", err)
}
if row.Data["user_name"] != "Alice" {
t.Errorf("Expected user_name=Alice, got %v", row.Data["user_name"])
}
if row.Data["email_address"] != "alice@example.com" {
t.Errorf("Expected email_address=alice@example.com, got %v", row.Data["email_address"])
}
if row.Data["is_active"] != true {
t.Errorf("Expected is_active=true, got %v", row.Data["is_active"])
}
t.Log("✓ Insert with snake_case test passed")
}
// TestInsertInvalidType 测试插入不支持的类型
func TestInsertInvalidType(t *testing.T) {
tmpDir, _ := os.MkdirTemp("", "TestInsertInvalidType")
defer os.RemoveAll(tmpDir)
schema := NewSchema("users", []Field{
{Name: "name", Type: FieldTypeString},
})
table, err := OpenTable(&TableOptions{
Dir: tmpDir,
Name: schema.Name,
Fields: schema.Fields,
})
if err != nil {
t.Fatal(err)
}
defer table.Close()
// 尝试插入不支持的类型
err = table.Insert(123) // int 类型
if err == nil {
t.Errorf("Expected error for invalid type, got nil")
}
err = table.Insert("string") // string 类型
if err == nil {
t.Errorf("Expected error for invalid type, got nil")
}
err = table.Insert(nil) // nil
if err == nil {
t.Errorf("Expected error for nil, got nil")
}
t.Log("✓ Insert invalid type test passed")
}
// TestInsertEmptySlice 测试插入空切片
func TestInsertEmptySlice(t *testing.T) {
tmpDir, _ := os.MkdirTemp("", "TestInsertEmptySlice")
defer os.RemoveAll(tmpDir)
schema := NewSchema("users", []Field{
{Name: "name", Type: FieldTypeString},
})
table, err := OpenTable(&TableOptions{
Dir: tmpDir,
Name: schema.Name,
Fields: schema.Fields,
})
if err != nil {
t.Fatal(err)
}
defer table.Close()
// 插入空切片
err = table.Insert([]map[string]any{})
if err != nil {
t.Errorf("Expected nil error for empty slice, got %v", err)
}
// 验证没有数据
_, err = table.Get(1)
if err == nil {
t.Errorf("Expected error for non-existent row")
}
t.Log("✓ Insert empty slice test passed")
}
// TestBatchInsertPerformance 测试批量插入性能
func TestBatchInsertPerformance(t *testing.T) {
tmpDir, _ := os.MkdirTemp("", "TestBatchInsertPerformance")
defer os.RemoveAll(tmpDir)
schema := NewSchema("users", []Field{
{Name: "name", Type: FieldTypeString},
{Name: "age", Type: FieldTypeInt64},
})
table, err := OpenTable(&TableOptions{
Dir: tmpDir,
Name: schema.Name,
Fields: schema.Fields,
})
if err != nil {
t.Fatal(err)
}
defer table.Close()
// 准备1000条数据
batchSize := 1000
data := make([]map[string]any, batchSize)
for i := 0; i < batchSize; i++ {
data[i] = map[string]any{
"name": "User" + string(rune(i)),
"age": int64(20 + i%50),
}
}
// 批量插入
err = table.Insert(data)
if err != nil {
t.Fatalf("Batch insert failed: %v", err)
}
// 验证数量
row, err := table.Get(int64(batchSize))
if err != nil {
t.Fatalf("Get last row failed: %v", err)
}
if row.Seq != int64(batchSize) {
t.Errorf("Expected seq=%d, got %d", batchSize, row.Seq)
}
t.Logf("✓ Batch insert performance test passed (%d rows)", batchSize)
}

23
webui/static/js/components/manifest-view.js
View File

@@ -132,11 +132,27 @@ export class ManifestView extends LitElement {
}
.file-name {
display: flex;
align-items: center;
justify-content: space-between;
margin-bottom: 8px;
}
.file-name-text {
font-family: 'Courier New', monospace;
font-size: 13px;
color: var(--text-primary);
font-weight: 500;
margin-bottom: 8px;
}
.file-level-badge {
font-size: 11px;
padding: 2px 8px;
background: var(--primary-bg);
color: var(--primary);
border-radius: var(--radius-sm);
font-weight: 600;
font-family: 'Courier New', monospace;
}
.file-detail {
@@ -273,7 +289,10 @@ export class ManifestView extends LitElement {
<div class="file-list">
${level.files.map(file => html`
<div class="file-item">
<div class="file-name">${file.file_number}.sst</div>
<div class="file-name">
<span class="file-name-text">${file.file_number}.sst</span>
<span class="file-level-badge">L${level.level}</span>
</div>
<div class="file-detail">
<div class="file-detail-row">
<span>Size:</span>