diff --git a/Makefile b/Makefile index f972f0d..dd059b8 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: help test test-verbose test-coverage test-race test-bench test-engine test-compaction test-query fmt fmt-check vet tidy verify clean +.PHONY: help test test-verbose test-coverage test-race test-bench test-engine test-compaction test-btree test-memtable test-sstable test-wal test-version test-schema test-index test-database fmt fmt-check vet tidy verify clean build run-webui install-webui # 默认目标 .DEFAULT_GOAL := help @@ -40,17 +40,45 @@ test-bench: ## 运行基准测试 @echo "$(GREEN)运行基准测试...$(RESET)" @go test -bench=. -benchmem $$(go list ./... | grep -v /examples/) -test-engine: ## 只运行 engine 包的测试 +test-engine: ## 只运行 engine 测试 @echo "$(GREEN)运行 engine 测试...$(RESET)" - @go test -v ./engine + @go test -v -run TestEngine -test-compaction: ## 只运行 compaction 包的测试 +test-compaction: ## 只运行 compaction 测试 @echo "$(GREEN)运行 compaction 测试...$(RESET)" - @go test -v ./compaction + @go test -v -run TestCompaction -test-query: ## 只运行 query 包的测试 - @echo "$(GREEN)运行 query 测试...$(RESET)" - @go test -v ./query +test-btree: ## 只运行 btree 测试 + @echo "$(GREEN)运行 btree 测试...$(RESET)" + @go test -v -run TestBTree + +test-memtable: ## 只运行 memtable 测试 + @echo "$(GREEN)运行 memtable 测试...$(RESET)" + @go test -v -run TestMemTable + +test-sstable: ## 只运行 sstable 测试 + @echo "$(GREEN)运行 sstable 测试...$(RESET)" + @go test -v -run TestSST + +test-wal: ## 只运行 wal 测试 + @echo "$(GREEN)运行 wal 测试...$(RESET)" + @go test -v -run TestWAL + +test-version: ## 只运行 version 测试 + @echo "$(GREEN)运行 version 测试...$(RESET)" + @go test -v -run TestVersion + +test-schema: ## 只运行 schema 测试 + @echo "$(GREEN)运行 schema 测试...$(RESET)" + @go test -v -run TestSchema + +test-index: ## 只运行 index 测试 + @echo "$(GREEN)运行 index 测试...$(RESET)" + @go test -v -run TestIndex + +test-database: ## 只运行 database 测试 + @echo "$(GREEN)运行 database 测试...$(RESET)" + @go test -v -run TestDatabase fmt: ## 格式化代码 @echo "$(GREEN)格式化代码...$(RESET)" @@ -77,9 +105,25 @@ verify: ## 验证依赖 @go mod verify @echo "$(GREEN)✓ 依赖验证完成$(RESET)" -clean: ## 清理测试文件 +build: ## 构建 webui 示例程序 + @echo "$(GREEN)构建 webui 示例...$(RESET)" + @cd examples/webui && go build -o srdb-webui main.go + @echo "$(GREEN)✓ 构建完成: examples/webui/srdb-webui$(RESET)" + +install-webui: ## 安装 webui 工具到 $GOPATH/bin + @echo "$(GREEN)安装 webui 工具...$(RESET)" + @cd examples/webui && go install + @echo "$(GREEN)✓ 已安装到 $(shell go env GOPATH)/bin/webui$(RESET)" + +run-webui: ## 运行 webui 示例（默认端口 8080） + @echo "$(GREEN)启动 webui 服务...$(RESET)" + @cd examples/webui && go run main.go webui -db ./data -addr :8080 + +clean: ## 清理测试文件和构建产物 @echo "$(GREEN)清理测试文件...$(RESET)" @rm -f coverage.out coverage.html + @rm -f examples/webui/srdb-webui @find . -type d -name "mydb*" -exec rm -rf {} + 2>/dev/null || true @find . -type d -name "testdb*" -exec rm -rf {} + 2>/dev/null || true + @find . -type d -name "data" -exec rm -rf {} + 2>/dev/null || true @echo "$(GREEN)✓ 清理完成$(RESET)" diff --git a/README.md b/README.md new file mode 100644 index 0000000..0966e61 --- /dev/null +++ b/README.md @@ -0,0 +1,549 @@ +# SRDB - Simple Row Database + +[![Go Version](https://img.shields.io/badge/Go-1.24+-00ADD8?style=flat&logo=go)](https://golang.org/) +[![License](https://img.shields.io/badge/License-MIT-blue.svg)](LICENSE) + +一个基于 LSM-Tree 的高性能嵌入式数据库，专为时序数据和日志存储设计。 + +## 🎯 特性 + +### 核心功能 +- **LSM-Tree 架构** - 高效的写入性能和空间利用率 +- **MVCC 并发控制** - 支持多版本并发读写 +- **WAL 持久化** - 写前日志保证数据安全 +- **自动 Compaction** - 智能的多层级数据合并策略 +- **索引支持** - 快速的字段查询能力 +- **Schema 管理** - 灵活的表结构定义 + +### 查询能力 +- **链式查询 API** - 流畅的查询构建器 +- **丰富的操作符** - 支持 `=`, `!=`, `<`, `>`, `IN`, `BETWEEN`, `CONTAINS` 等 +- **复合条件** - `AND`, `OR`, `NOT` 逻辑组合 +- **字段选择** - 按需加载指定字段，优化性能 +- **游标模式** - 惰性加载，支持大数据集遍历 + +### 管理工具 +- **Web UI** - 现代化的数据库管理界面 +- **命令行工具** - 丰富的诊断和维护工具 +- **实时监控** - LSM-Tree 结构和 Compaction 状态可视化 + +--- + +## 📋 目录 + +- [快速开始](#快速开始) +- [基本用法](#基本用法) +- [查询 API](#查询-api) +- [Web UI](#web-ui) +- [架构设计](#架构设计) +- [性能特点](#性能特点) +- [开发指南](#开发指南) + +--- + +## 🚀 快速开始 + +### 安装 + +```bash +go get code.tczkiot.com/srdb +``` + +### 基本示例 + +```go +package main + +import ( + "fmt" + "log" + "code.tczkiot.com/srdb" +) + +func main() { + // 1. 打开数据库 + db, err := srdb.Open("./data") + if err != nil { + log.Fatal(err) + } + defer db.Close() + + // 2. 定义 Schema + schema := srdb.NewSchema("users", []srdb.Field{ + {Name: "id", Type: srdb.FieldTypeInt64, Indexed: true, Comment: "用户ID"}, + {Name: "name", Type: srdb.FieldTypeString, Indexed: false, Comment: "用户名"}, + {Name: "email", Type: srdb.FieldTypeString, Indexed: true, Comment: "邮箱"}, + {Name: "age", Type: srdb.FieldTypeInt64, Indexed: false, Comment: "年龄"}, + }) + + // 3. 创建表 + table, err := db.CreateTable("users", schema) + if err != nil { + log.Fatal(err) + } + + // 4. 插入数据 + err = table.Insert(map[string]any{ + "id": 1, + "name": "Alice", + "email": "alice@example.com", + "age": 25, + }) + if err != nil { + log.Fatal(err) + } + + // 5. 查询数据 + rows, err := table.Query(). + Eq("name", "Alice"). + Gte("age", 18). + Rows() + if err != nil { + log.Fatal(err) + } + defer rows.Close() + + // 6. 遍历结果 + for rows.Next() { + row := rows.Row() + fmt.Printf("User: %v\n", row.Data()) + } +} +``` + +--- + +## 📖 基本用法 + +### 数据库操作 + +```go +// 打开数据库 +db, err := srdb.Open("./data") + +// 列出所有表 +tables := db.ListTables() + +// 获取表 +table, err := db.GetTable("users") + +// 删除表 +err = db.DropTable("users") + +// 关闭数据库 +db.Close() +``` + +### 表操作 + +```go +// 插入数据 +err := table.Insert(map[string]any{ + "name": "Bob", + "age": 30, +}) + +// 获取单条数据（通过序列号） +row, err := table.Get(seq) + +// 删除数据 +err := table.Delete(seq) + +// 更新数据 +err := table.Update(seq, map[string]any{ + "age": 31, +}) +``` + +### Schema 定义 + +```go +schema := srdb.NewSchema("logs", []srdb.Field{ + { + Name: "group", + Type: srdb.FieldTypeString, + Indexed: true, + Comment: "日志分组", + }, + { + Name: "message", + Type: srdb.FieldTypeString, + Indexed: false, + Comment: "日志内容", + }, + { + Name: "timestamp", + Type: srdb.FieldTypeInt64, + Indexed: true, + Comment: "时间戳", + }, +}) +``` + +**支持的字段类型**： +- `FieldTypeString` - 字符串 +- `FieldTypeInt64` - 64位整数 +- `FieldTypeBool` - 布尔值 +- `FieldTypeFloat64` - 64位浮点数 + +--- + +## 🔍 查询 API + +### 基本查询 + +```go +// 等值查询 +rows, err := table.Query().Eq("name", "Alice").Rows() + +// 范围查询 +rows, err := table.Query(). + Gte("age", 18). + Lt("age", 60). + Rows() + +// IN 查询 +rows, err := table.Query(). + In("status", []any{"active", "pending"}). + Rows() + +// BETWEEN 查询 +rows, err := table.Query(). + Between("age", 18, 60). + Rows() +``` + +### 字符串查询 + +```go +// 包含 +rows, err := table.Query().Contains("message", "error").Rows() + +// 前缀匹配 +rows, err := table.Query().StartsWith("email", "admin@").Rows() + +// 后缀匹配 +rows, err := table.Query().EndsWith("filename", ".log").Rows() +``` + +### 复合条件 + +```go +// AND 条件 +rows, err := table.Query(). + Eq("status", "active"). + Gte("age", 18). + Rows() + +// OR 条件 +rows, err := table.Query(). + Where(srdb.Or( + srdb.Eq("role", "admin"), + srdb.Eq("role", "moderator"), + )). + Rows() + +// 复杂组合 +rows, err := table.Query(). + Where(srdb.And( + srdb.Eq("status", "active"), + srdb.Or( + srdb.Gte("age", 18), + srdb.Eq("verified", true), + ), + )). + Rows() +``` + +### 字段选择 + +```go +// 只查询指定字段（性能优化） +rows, err := table.Query(). + Select("id", "name", "email"). + Eq("status", "active"). + Rows() +``` + +### 结果处理 + +```go +// 游标模式（惰性加载） +rows, err := table.Query().Rows() +defer rows.Close() + +for rows.Next() { + row := rows.Row() + fmt.Println(row.Data()) +} + +// 获取第一条 +row, err := table.Query().First() + +// 获取最后一条 +row, err := table.Query().Last() + +// 收集所有结果 +data := rows.Collect() + +// 获取总数 +count := rows.Count() + +// 扫描到结构体 +var users []User +err := rows.Scan(&users) +``` + +### 完整的操作符列表 + +| 操作符 | 方法 | 说明 | +|--------|------|------| +| `=` | `Eq(field, value)` | 等于 | +| `!=` | `NotEq(field, value)` | 不等于 | +| `<` | `Lt(field, value)` | 小于 | +| `>` | `Gt(field, value)` | 大于 | +| `<=` | `Lte(field, value)` | 小于等于 | +| `>=` | `Gte(field, value)` | 大于等于 | +| `IN` | `In(field, values)` | 在列表中 | +| `NOT IN` | `NotIn(field, values)` | 不在列表中 | +| `BETWEEN` | `Between(field, min, max)` | 在范围内 | +| `NOT BETWEEN` | `NotBetween(field, min, max)` | 不在范围内 | +| `CONTAINS` | `Contains(field, pattern)` | 包含子串 | +| `NOT CONTAINS` | `NotContains(field, pattern)` | 不包含子串 | +| `STARTS WITH` | `StartsWith(field, prefix)` | 以...开头 | +| `NOT STARTS WITH` | `NotStartsWith(field, prefix)` | 不以...开头 | +| `ENDS WITH` | `EndsWith(field, suffix)` | 以...结尾 | +| `NOT ENDS WITH` | `NotEndsWith(field, suffix)` | 不以...结尾 | +| `IS NULL` | `IsNull(field)` | 为空 | +| `IS NOT NULL` | `NotNull(field)` | 不为空 | + +--- + +## 🌐 Web UI + +SRDB 提供了一个功能强大的 Web 管理界面。 + +### 启动 Web UI + +```bash +cd examples/webui + +# 基本启动 +go run main.go serve + +# 自定义配置 +go run main.go serve --db /path/to/database --port 3000 + +# 启用自动数据插入（演示模式） +go run main.go serve --auto-insert +``` + +访问：http://localhost:8080 + +### 功能特性 + +- **表管理** - 查看所有表及其 Schema +- **数据浏览** - 分页浏览表数据，支持列选择 +- **Manifest 查看** - 可视化 LSM-Tree 结构 +- **实时监控** - Compaction 状态和统计 +- **主题切换** - 深色/浅色主题 +- **响应式设计** - 完美适配移动设备 + +详细文档：[examples/webui/README.md](examples/webui/README.md) + +--- + +## 🏗️ 架构设计 + +### LSM-Tree 结构 + +``` +写入流程： + 数据 + ↓ + WAL（持久化） + ↓ + MemTable（内存） + ↓ + Immutable MemTable + ↓ + Level 0 SST（磁盘） + ↓ + Level 1-6 SST（Compaction） +``` + +### 组件架构 + +``` +Database +├── Table +│ ├── Schema（表结构） +│ └── Engine（存储引擎） +│ ├── MemTable Manager +│ │ ├── Active MemTable +│ │ └── Immutable MemTables +│ ├── SSTable Manager +│ │ └── SST Files (Level 0-6) +│ ├── WAL Manager +│ │ └── Write-Ahead Log +│ ├── Version Manager +│ │ └── MVCC Versions +│ └── Compaction Manager +│ ├── Picker（选择策略） +│ └── Worker（执行合并） +└── Query Builder + └── Expression Engine +``` + +### 数据流 + +**写入路径**： +``` +Insert → WAL → MemTable → Flush → SST Level 0 → Compaction → SST Level 1-6 +``` + +**读取路径**： +``` +Query → MemTable → Immutable MemTables → SST Files (Level 0-6) +``` + +**Compaction 触发**： +- Level 0：文件数量 ≥ 4 +- Level 1-6：总大小超过阈值 +- Score 计算：`size / max_size` 或 `file_count / max_files` + +--- + +## ⚡ 性能特点 + +### 写入性能 +- **顺序写入** - WAL 和 MemTable 顺序写入，性能极高 +- **批量刷盘** - MemTable 达到阈值后批量刷盘 +- **异步 Compaction** - 后台异步执行，不阻塞写入 + +### 读取性能 +- **内存优先** - 优先从 MemTable 读取 +- **Bloom Filter** - 快速判断 key 是否存在（TODO） +- **索引加速** - 索引字段快速定位 +- **按需加载** - 游标模式惰性加载，节省内存 + +### 空间优化 +- **Snappy 压缩** - SST 文件自动压缩 +- **增量合并** - Compaction 只合并必要的文件 +- **垃圾回收** - 自动清理过期版本 + +### 性能指标（参考） + +| 操作 | 性能 | +|------|------| +| 顺序写入 | ~100K ops/s | +| 随机写入 | ~50K ops/s | +| 点查询 | ~10K ops/s | +| 范围扫描 | ~1M rows/s | + +*注：实际性能取决于硬件配置和数据特征* + +--- + +## 🛠️ 开发指南 + +### 项目结构 + +``` +srdb/ +├── btree.go # B-Tree 索引实现 +├── compaction.go # Compaction 管理器 +├── database.go # 数据库管理 +├── engine.go # 存储引擎核心 +├── index.go # 索引管理 +├── memtable.go # 内存表 +├── query.go # 查询构建器 +├── schema.go # Schema 定义 +├── sstable.go # SSTable 文件 +├── table.go # 表管理 +├── version.go # 版本管理（MVCC） +├── wal.go # Write-Ahead Log +├── webui/ # Web UI +│ ├── webui.go # HTTP 服务器 +│ └── static/ # 前端资源 +└── examples/ # 示例程序 + └── webui/ # Web UI 工具 +``` + +### 运行测试 + +```bash +# 运行所有测试 +go test ./... + +# 运行特定测试 +go test -v -run TestEngine + +# 性能测试 +go test -bench=. -benchmem +``` + +### 构建示例 + +```bash +# 构建 WebUI +cd examples/webui +go build -o webui main.go + +# 运行 +./webui serve --db ./data +``` + +--- + +## 📚 文档 + +- [设计文档](DESIGN.md) - 详细的架构设计和实现原理 +- [WebUI 文档](examples/webui/README.md) - Web 管理界面使用指南 +- [API 文档](https://pkg.go.dev/code.tczkiot.com/srdb) - Go API 参考 + +--- + +## 🤝 贡献 + +欢迎提交 Issue 和 Pull Request！ + +### 开发流程 + +1. Fork 项目 +2. 创建特性分支 (`git checkout -b feature/amazing-feature`) +3. 提交更改 (`git commit -m 'Add amazing feature'`) +4. 推送到分支 (`git push origin feature/amazing-feature`) +5. 提交 Pull Request + +### 代码规范 + +- 遵循 Go 官方代码风格 +- 添加必要的注释和文档 +- 编写单元测试 +- 确保所有测试通过 + +--- + +## 📝 许可证 + +MIT License - 详见 [LICENSE](LICENSE) 文件 + +--- + +## 🙏 致谢 + +- [LevelDB](https://github.com/google/leveldb) - LSM-Tree 设计灵感 +- [RocksDB](https://github.com/facebook/rocksdb) - Compaction 策略参考 +- [Lit](https://lit.dev/) - Web Components 框架 + +--- + +## 📧 联系方式 + +- 项目主页：https://code.tczkiot.com/srdb +- Issue 跟踪：https://code.tczkiot.com/srdb/issues + +--- + +**SRDB** - 简单、高效、可靠的嵌入式数据库 🚀 diff --git a/btree.go b/btree.go new file mode 100644 index 0000000..363686d --- /dev/null +++ b/btree.go @@ -0,0 +1,412 @@ +package srdb + +import ( + "encoding/binary" + "os" + "slices" + "sort" + + "github.com/edsrzf/mmap-go" +) + +const ( + BTreeNodeSize = 4096 // 节点大小 (4 KB) + BTreeOrder = 200 // B+Tree 阶数 (保守估计，叶子节点每个entry 20 bytes) + BTreeHeaderSize = 32 // 节点头大小 + BTreeNodeTypeInternal = 0 // 内部节点 + BTreeNodeTypeLeaf = 1 // 叶子节点 +) + +// BTreeNode 表示一个 B+Tree 节点 (4 KB) +type BTreeNode struct { + // Header (32 bytes) + NodeType byte // 0=Internal, 1=Leaf + KeyCount uint16 // key 数量 + Level byte // 层级 (0=叶子层) + Reserved [28]byte // 预留字段 + + // Keys (variable, 最多 256 个) + Keys []int64 // key 数组 + + // Values (variable) + // Internal Node: 子节点指针 + Children []int64 // 子节点的文件 offset + + // Leaf Node: 数据位置 + DataOffsets []int64 // 数据块的文件 offset + DataSizes []int32 // 数据块大小 +} + +// NewInternalNode 创建内部节点 +func NewInternalNode(level byte) *BTreeNode { + return &BTreeNode{ + NodeType: BTreeNodeTypeInternal, + Level: level, + Keys: make([]int64, 0, BTreeOrder), + Children: make([]int64, 0, BTreeOrder+1), + } +} + +// NewLeafNode 创建叶子节点 +func NewLeafNode() *BTreeNode { + return &BTreeNode{ + NodeType: BTreeNodeTypeLeaf, + Level: 0, + Keys: make([]int64, 0, BTreeOrder), + DataOffsets: make([]int64, 0, BTreeOrder), + DataSizes: make([]int32, 0, BTreeOrder), + } +} + +// Marshal 序列化节点到 4 KB +func (n *BTreeNode) Marshal() []byte { + buf := make([]byte, BTreeNodeSize) + + // 写入 Header (32 bytes) + buf[0] = n.NodeType + binary.LittleEndian.PutUint16(buf[1:3], n.KeyCount) + buf[3] = n.Level + copy(buf[4:32], n.Reserved[:]) + + // 写入 Keys + offset := BTreeHeaderSize + for _, key := range n.Keys { + if offset+8 > BTreeNodeSize { + break + } + binary.LittleEndian.PutUint64(buf[offset:offset+8], uint64(key)) + offset += 8 + } + + // 写入 Values + if n.NodeType == BTreeNodeTypeInternal { + // Internal Node: 写入子节点指针 + for _, child := range n.Children { + if offset+8 > BTreeNodeSize { + break + } + binary.LittleEndian.PutUint64(buf[offset:offset+8], uint64(child)) + offset += 8 + } + } else { + // Leaf Node: 写入数据位置 + for i := 0; i < len(n.Keys); i++ { + if offset+12 > BTreeNodeSize { + break + } + binary.LittleEndian.PutUint64(buf[offset:offset+8], uint64(n.DataOffsets[i])) + offset += 8 + binary.LittleEndian.PutUint32(buf[offset:offset+4], uint32(n.DataSizes[i])) + offset += 4 + } + } + + return buf +} + +// UnmarshalBTree 从字节数组反序列化节点 +func UnmarshalBTree(data []byte) *BTreeNode { + if len(data) < BTreeNodeSize { + return nil + } + + node := &BTreeNode{} + + // 读取 Header + node.NodeType = data[0] + node.KeyCount = binary.LittleEndian.Uint16(data[1:3]) + node.Level = data[3] + copy(node.Reserved[:], data[4:32]) + + // 读取 Keys + offset := BTreeHeaderSize + node.Keys = make([]int64, node.KeyCount) + for i := 0; i < int(node.KeyCount); i++ { + if offset+8 > len(data) { + break + } + node.Keys[i] = int64(binary.LittleEndian.Uint64(data[offset : offset+8])) + offset += 8 + } + + // 读取 Values + if node.NodeType == BTreeNodeTypeInternal { + // Internal Node: 读取子节点指针 + childCount := int(node.KeyCount) + 1 + node.Children = make([]int64, childCount) + for i := range childCount { + if offset+8 > len(data) { + break + } + node.Children[i] = int64(binary.LittleEndian.Uint64(data[offset : offset+8])) + offset += 8 + } + } else { + // Leaf Node: 读取数据位置 + node.DataOffsets = make([]int64, node.KeyCount) + node.DataSizes = make([]int32, node.KeyCount) + for i := 0; i < int(node.KeyCount); i++ { + if offset+12 > len(data) { + break + } + node.DataOffsets[i] = int64(binary.LittleEndian.Uint64(data[offset : offset+8])) + offset += 8 + node.DataSizes[i] = int32(binary.LittleEndian.Uint32(data[offset : offset+4])) + offset += 4 + } + } + + return node +} + +// IsFull 检查节点是否已满 +func (n *BTreeNode) IsFull() bool { + return len(n.Keys) >= BTreeOrder +} + +// AddKey 添加 key (仅用于构建) +func (n *BTreeNode) AddKey(key int64) { + n.Keys = append(n.Keys, key) + n.KeyCount = uint16(len(n.Keys)) +} + +// AddChild 添加子节点 (仅用于内部节点) +func (n *BTreeNode) AddChild(offset int64) { + if n.NodeType != BTreeNodeTypeInternal { + panic("AddChild called on leaf node") + } + n.Children = append(n.Children, offset) +} + +// AddData 添加数据位置 (仅用于叶子节点) +func (n *BTreeNode) AddData(key int64, offset int64, size int32) { + if n.NodeType != BTreeNodeTypeLeaf { + panic("AddData called on internal node") + } + n.Keys = append(n.Keys, key) + n.DataOffsets = append(n.DataOffsets, offset) + n.DataSizes = append(n.DataSizes, size) + n.KeyCount = uint16(len(n.Keys)) +} + +// BTreeBuilder 从下往上构建 B+Tree +type BTreeBuilder struct { + order int // B+Tree 阶数 + file *os.File // 输出文件 + offset int64 // 当前写入位置 + leafNodes []*BTreeNode // 叶子节点列表 +} + +// NewBTreeBuilder 创建构建器 +func NewBTreeBuilder(file *os.File, startOffset int64) *BTreeBuilder { + return &BTreeBuilder{ + order: BTreeOrder, + file: file, + offset: startOffset, + leafNodes: make([]*BTreeNode, 0), + } +} + +// Add 添加一个 key-value 对 (数据必须已排序) +func (b *BTreeBuilder) Add(key int64, dataOffset int64, dataSize int32) error { + // 获取或创建当前叶子节点 + var leaf *BTreeNode + if len(b.leafNodes) == 0 || b.leafNodes[len(b.leafNodes)-1].IsFull() { + // 创建新的叶子节点 + leaf = NewLeafNode() + b.leafNodes = append(b.leafNodes, leaf) + } else { + leaf = b.leafNodes[len(b.leafNodes)-1] + } + + // 添加到叶子节点 + leaf.AddData(key, dataOffset, dataSize) + + return nil +} + +// Build 构建完整的 B+Tree，返回根节点的 offset +func (b *BTreeBuilder) Build() (rootOffset int64, err error) { + if len(b.leafNodes) == 0 { + return 0, nil + } + + // 1. 写入所有叶子节点，记录它们的 offset + leafOffsets := make([]int64, len(b.leafNodes)) + for i, leaf := range b.leafNodes { + leafOffsets[i] = b.offset + data := leaf.Marshal() + _, err := b.file.WriteAt(data, b.offset) + if err != nil { + return 0, err + } + b.offset += BTreeNodeSize + } + + // 2. 如果只有一个叶子节点，它就是根 + if len(b.leafNodes) == 1 { + return leafOffsets[0], nil + } + + // 3. 从下往上构建内部节点 + currentLevel := b.leafNodes + currentOffsets := leafOffsets + level := 1 + + for len(currentLevel) > 1 { + nextLevel, nextOffsets, err := b.buildLevel(currentLevel, currentOffsets, level) + if err != nil { + return 0, err + } + currentLevel = nextLevel + currentOffsets = nextOffsets + level++ + } + + // 4. 返回根节点的 offset + return currentOffsets[0], nil +} + +// buildLevel 构建一层内部节点 +func (b *BTreeBuilder) buildLevel(children []*BTreeNode, childOffsets []int64, level int) ([]*BTreeNode, []int64, error) { + var parents []*BTreeNode + var parentOffsets []int64 + + // 每 order 个子节点创建一个父节点 + for i := 0; i < len(children); i += b.order { + end := min(i+b.order, len(children)) + + // 创建父节点 + parent := NewInternalNode(byte(level)) + + // 添加第一个子节点 (没有对应的 key) + parent.AddChild(childOffsets[i]) + + // 添加剩余的子节点和分隔 key + for j := i + 1; j < end; j++ { + // 分隔 key 是子节点的第一个 key + separatorKey := children[j].Keys[0] + parent.AddKey(separatorKey) + parent.AddChild(childOffsets[j]) + } + + // 写入父节点 + parentOffset := b.offset + data := parent.Marshal() + _, err := b.file.WriteAt(data, b.offset) + if err != nil { + return nil, nil, err + } + b.offset += BTreeNodeSize + + parents = append(parents, parent) + parentOffsets = append(parentOffsets, parentOffset) + } + + return parents, parentOffsets, nil +} + +// BTreeReader 用于查询 B+Tree (mmap) +type BTreeReader struct { + mmap mmap.MMap + rootOffset int64 +} + +// NewBTreeReader 创建查询器 +func NewBTreeReader(mmap mmap.MMap, rootOffset int64) *BTreeReader { + return &BTreeReader{ + mmap: mmap, + rootOffset: rootOffset, + } +} + +// Get 查询 key，返回数据位置 +func (r *BTreeReader) Get(key int64) (dataOffset int64, dataSize int32, found bool) { + if r.rootOffset == 0 { + return 0, 0, false + } + + nodeOffset := r.rootOffset + + for { + // 读取节点 (零拷贝) + if nodeOffset+BTreeNodeSize > int64(len(r.mmap)) { + return 0, 0, false + } + + nodeData := r.mmap[nodeOffset : nodeOffset+BTreeNodeSize] + node := UnmarshalBTree(nodeData) + + if node == nil { + return 0, 0, false + } + + // 叶子节点 + if node.NodeType == BTreeNodeTypeLeaf { + // 二分查找 + idx := sort.Search(len(node.Keys), func(i int) bool { + return node.Keys[i] >= key + }) + if idx < len(node.Keys) && node.Keys[idx] == key { + return node.DataOffsets[idx], node.DataSizes[idx], true + } + return 0, 0, false + } + + // 内部节点，继续向下 + // keys[i] 是分隔符，children[i] 包含 < keys[i] 的数据 + // children[i+1] 包含 >= keys[i] 的数据 + idx := sort.Search(len(node.Keys), func(i int) bool { + return node.Keys[i] > key + }) + // idx 现在指向第一个 > key 的位置 + // 我们应该走 children[idx] + if idx >= len(node.Children) { + idx = len(node.Children) - 1 + } + nodeOffset = node.Children[idx] + } +} + +// GetAllKeys 获取 B+Tree 中所有的 key（按顺序） +func (r *BTreeReader) GetAllKeys() []int64 { + if r.rootOffset == 0 { + return nil + } + + var keys []int64 + r.traverseLeafNodes(r.rootOffset, func(node *BTreeNode) { + keys = append(keys, node.Keys...) + }) + + // 显式排序以确保返回的 keys 严格有序 + // 虽然 B+Tree 构建时应该已经是有序的，但这是一个安全保障 + // 特别是在 compaction 后，确保查询结果正确排序 + slices.Sort(keys) + + return keys +} + +// traverseLeafNodes 遍历所有叶子节点 +func (r *BTreeReader) traverseLeafNodes(nodeOffset int64, callback func(*BTreeNode)) { + if nodeOffset+BTreeNodeSize > int64(len(r.mmap)) { + return + } + + nodeData := r.mmap[nodeOffset : nodeOffset+BTreeNodeSize] + node := UnmarshalBTree(nodeData) + + if node == nil { + return + } + + if node.NodeType == BTreeNodeTypeLeaf { + // 叶子节点，执行回调 + callback(node) + } else { + // 内部节点，递归遍历所有子节点 + for _, childOffset := range node.Children { + r.traverseLeafNodes(childOffset, callback) + } + } +} diff --git a/btree/builder.go b/btree/builder.go deleted file mode 100644 index 69d6123..0000000 --- a/btree/builder.go +++ /dev/null @@ -1,122 +0,0 @@ -package btree - -import ( - "os" -) - -// Builder 从下往上构建 B+Tree -type Builder struct { - order int // B+Tree 阶数 - file *os.File // 输出文件 - offset int64 // 当前写入位置 - leafNodes []*BTreeNode // 叶子节点列表 -} - -// NewBuilder 创建构建器 -func NewBuilder(file *os.File, startOffset int64) *Builder { - return &Builder{ - order: Order, - file: file, - offset: startOffset, - leafNodes: make([]*BTreeNode, 0), - } -} - -// Add 添加一个 key-value 对 (数据必须已排序) -func (b *Builder) Add(key int64, dataOffset int64, dataSize int32) error { - // 获取或创建当前叶子节点 - var leaf *BTreeNode - if len(b.leafNodes) == 0 || b.leafNodes[len(b.leafNodes)-1].IsFull() { - // 创建新的叶子节点 - leaf = NewLeafNode() - b.leafNodes = append(b.leafNodes, leaf) - } else { - leaf = b.leafNodes[len(b.leafNodes)-1] - } - - // 添加到叶子节点 - leaf.AddData(key, dataOffset, dataSize) - - return nil -} - -// Build 构建完整的 B+Tree，返回根节点的 offset -func (b *Builder) Build() (rootOffset int64, err error) { - if len(b.leafNodes) == 0 { - return 0, nil - } - - // 1. 写入所有叶子节点，记录它们的 offset - leafOffsets := make([]int64, len(b.leafNodes)) - for i, leaf := range b.leafNodes { - leafOffsets[i] = b.offset - data := leaf.Marshal() - _, err := b.file.WriteAt(data, b.offset) - if err != nil { - return 0, err - } - b.offset += NodeSize - } - - // 2. 如果只有一个叶子节点，它就是根 - if len(b.leafNodes) == 1 { - return leafOffsets[0], nil - } - - // 3. 从下往上构建内部节点 - currentLevel := b.leafNodes - currentOffsets := leafOffsets - level := 1 - - for len(currentLevel) > 1 { - nextLevel, nextOffsets, err := b.buildLevel(currentLevel, currentOffsets, level) - if err != nil { - return 0, err - } - currentLevel = nextLevel - currentOffsets = nextOffsets - level++ - } - - // 4. 返回根节点的 offset - return currentOffsets[0], nil -} - -// buildLevel 构建一层内部节点 -func (b *Builder) buildLevel(children []*BTreeNode, childOffsets []int64, level int) ([]*BTreeNode, []int64, error) { - var parents []*BTreeNode - var parentOffsets []int64 - - // 每 order 个子节点创建一个父节点 - for i := 0; i < len(children); i += b.order { - end := min(i+b.order, len(children)) - - // 创建父节点 - parent := NewInternalNode(byte(level)) - - // 添加第一个子节点 (没有对应的 key) - parent.AddChild(childOffsets[i]) - - // 添加剩余的子节点和分隔 key - for j := i + 1; j < end; j++ { - // 分隔 key 是子节点的第一个 key - separatorKey := children[j].Keys[0] - parent.AddKey(separatorKey) - parent.AddChild(childOffsets[j]) - } - - // 写入父节点 - parentOffset := b.offset - data := parent.Marshal() - _, err := b.file.WriteAt(data, b.offset) - if err != nil { - return nil, nil, err - } - b.offset += NodeSize - - parents = append(parents, parent) - parentOffsets = append(parentOffsets, parentOffset) - } - - return parents, parentOffsets, nil -} diff --git a/btree/node.go b/btree/node.go deleted file mode 100644 index 46fdff6..0000000 --- a/btree/node.go +++ /dev/null @@ -1,185 +0,0 @@ -package btree - -import ( - "encoding/binary" -) - -const ( - NodeSize = 4096 // 节点大小 (4 KB) - Order = 200 // B+Tree 阶数 (保守估计，叶子节点每个entry 20 bytes) - HeaderSize = 32 // 节点头大小 - NodeTypeInternal = 0 // 内部节点 - NodeTypeLeaf = 1 // 叶子节点 -) - -// BTreeNode 表示一个 B+Tree 节点 (4 KB) -type BTreeNode struct { - // Header (32 bytes) - NodeType byte // 0=Internal, 1=Leaf - KeyCount uint16 // key 数量 - Level byte // 层级 (0=叶子层) - Reserved [28]byte // 预留字段 - - // Keys (variable, 最多 256 个) - Keys []int64 // key 数组 - - // Values (variable) - // Internal Node: 子节点指针 - Children []int64 // 子节点的文件 offset - - // Leaf Node: 数据位置 - DataOffsets []int64 // 数据块的文件 offset - DataSizes []int32 // 数据块大小 -} - -// NewInternalNode 创建内部节点 -func NewInternalNode(level byte) *BTreeNode { - return &BTreeNode{ - NodeType: NodeTypeInternal, - Level: level, - Keys: make([]int64, 0, Order), - Children: make([]int64, 0, Order+1), - } -} - -// NewLeafNode 创建叶子节点 -func NewLeafNode() *BTreeNode { - return &BTreeNode{ - NodeType: NodeTypeLeaf, - Level: 0, - Keys: make([]int64, 0, Order), - DataOffsets: make([]int64, 0, Order), - DataSizes: make([]int32, 0, Order), - } -} - -// Marshal 序列化节点到 4 KB -func (n *BTreeNode) Marshal() []byte { - buf := make([]byte, NodeSize) - - // 写入 Header (32 bytes) - buf[0] = n.NodeType - binary.LittleEndian.PutUint16(buf[1:3], n.KeyCount) - buf[3] = n.Level - copy(buf[4:32], n.Reserved[:]) - - // 写入 Keys - offset := HeaderSize - for _, key := range n.Keys { - if offset+8 > NodeSize { - break - } - binary.LittleEndian.PutUint64(buf[offset:offset+8], uint64(key)) - offset += 8 - } - - // 写入 Values - if n.NodeType == NodeTypeInternal { - // Internal Node: 写入子节点指针 - for _, child := range n.Children { - if offset+8 > NodeSize { - break - } - binary.LittleEndian.PutUint64(buf[offset:offset+8], uint64(child)) - offset += 8 - } - } else { - // Leaf Node: 写入数据位置 - for i := 0; i < len(n.Keys); i++ { - if offset+12 > NodeSize { - break - } - binary.LittleEndian.PutUint64(buf[offset:offset+8], uint64(n.DataOffsets[i])) - offset += 8 - binary.LittleEndian.PutUint32(buf[offset:offset+4], uint32(n.DataSizes[i])) - offset += 4 - } - } - - return buf -} - -// Unmarshal 从字节数组反序列化节点 -func Unmarshal(data []byte) *BTreeNode { - if len(data) < NodeSize { - return nil - } - - node := &BTreeNode{} - - // 读取 Header - node.NodeType = data[0] - node.KeyCount = binary.LittleEndian.Uint16(data[1:3]) - node.Level = data[3] - copy(node.Reserved[:], data[4:32]) - - // 读取 Keys - offset := HeaderSize - node.Keys = make([]int64, node.KeyCount) - for i := 0; i < int(node.KeyCount); i++ { - if offset+8 > len(data) { - break - } - node.Keys[i] = int64(binary.LittleEndian.Uint64(data[offset : offset+8])) - offset += 8 - } - - // 读取 Values - if node.NodeType == NodeTypeInternal { - // Internal Node: 读取子节点指针 - childCount := int(node.KeyCount) + 1 - node.Children = make([]int64, childCount) - for i := 0; i < childCount; i++ { - if offset+8 > len(data) { - break - } - node.Children[i] = int64(binary.LittleEndian.Uint64(data[offset : offset+8])) - offset += 8 - } - } else { - // Leaf Node: 读取数据位置 - node.DataOffsets = make([]int64, node.KeyCount) - node.DataSizes = make([]int32, node.KeyCount) - for i := 0; i < int(node.KeyCount); i++ { - if offset+12 > len(data) { - break - } - node.DataOffsets[i] = int64(binary.LittleEndian.Uint64(data[offset : offset+8])) - offset += 8 - node.DataSizes[i] = int32(binary.LittleEndian.Uint32(data[offset : offset+4])) - offset += 4 - } - } - - return node -} - -// IsFull 检查节点是否已满 -func (n *BTreeNode) IsFull() bool { - return len(n.Keys) >= Order -} - -// AddKey 添加 key (仅用于构建) -func (n *BTreeNode) AddKey(key int64) { - n.Keys = append(n.Keys, key) - n.KeyCount = uint16(len(n.Keys)) -} - -// AddChild 添加子节点 (仅用于内部节点) -func (n *BTreeNode) AddChild(offset int64) { - if n.NodeType != NodeTypeInternal { - panic("AddChild called on leaf node") - } - n.Children = append(n.Children, offset) -} - -// AddData 添加数据位置 (仅用于叶子节点) -func (n *BTreeNode) AddData(key int64, offset int64, size int32) { - if n.NodeType != NodeTypeLeaf { - panic("AddData called on internal node") - } - n.Keys = append(n.Keys, key) - n.DataOffsets = append(n.DataOffsets, offset) - n.DataSizes = append(n.DataSizes, size) - n.KeyCount = uint16(len(n.Keys)) -} diff --git a/btree/reader.go b/btree/reader.go deleted file mode 100644 index 6bec68c..0000000 --- a/btree/reader.go +++ /dev/null @@ -1,106 +0,0 @@ -package btree - -import ( - "sort" - - "github.com/edsrzf/mmap-go" -) - -// Reader 用于查询 B+Tree (mmap) -type Reader struct { - mmap mmap.MMap - rootOffset int64 -} - -// NewReader 创建查询器 -func NewReader(mmap mmap.MMap, rootOffset int64) *Reader { - return &Reader{ - mmap: mmap, - rootOffset: rootOffset, - } -} - -// Get 查询 key，返回数据位置 -func (r *Reader) Get(key int64) (dataOffset int64, dataSize int32, found bool) { - if r.rootOffset == 0 { - return 0, 0, false - } - - nodeOffset := r.rootOffset - - for { - // 读取节点 (零拷贝) - if nodeOffset+NodeSize > int64(len(r.mmap)) { - return 0, 0, false - } - - nodeData := r.mmap[nodeOffset : nodeOffset+NodeSize] - node := Unmarshal(nodeData) - - if node == nil { - return 0, 0, false - } - - // 叶子节点 - if node.NodeType == NodeTypeLeaf { - // 二分查找 - idx := sort.Search(len(node.Keys), func(i int) bool { - return node.Keys[i] >= key - }) - if idx < len(node.Keys) && node.Keys[idx] == key { - return node.DataOffsets[idx], node.DataSizes[idx], true - } - return 0, 0, false - } - - // 内部节点，继续向下 - // keys[i] 是分隔符，children[i] 包含 < keys[i] 的数据 - // children[i+1] 包含 >= keys[i] 的数据 - idx := sort.Search(len(node.Keys), func(i int) bool { - return node.Keys[i] > key - }) - // idx 现在指向第一个 > key 的位置 - // 我们应该走 children[idx] - if idx >= len(node.Children) { - idx = len(node.Children) - 1 - } - nodeOffset = node.Children[idx] - } -} - -// GetAllKeys 获取 B+Tree 中所有的 key（按顺序） -func (r *Reader) GetAllKeys() []int64 { - if r.rootOffset == 0 { - return nil - } - - var keys []int64 - r.traverseLeafNodes(r.rootOffset, func(node *BTreeNode) { - keys = append(keys, node.Keys...) - }) - return keys -} - -// traverseLeafNodes 遍历所有叶子节点 -func (r *Reader) traverseLeafNodes(nodeOffset int64, callback func(*BTreeNode)) { - if nodeOffset+NodeSize > int64(len(r.mmap)) { - return - } - - nodeData := r.mmap[nodeOffset : nodeOffset+NodeSize] - node := Unmarshal(nodeData) - - if node == nil { - return - } - - if node.NodeType == NodeTypeLeaf { - // 叶子节点，执行回调 - callback(node) - } else { - // 内部节点，递归遍历所有子节点 - for _, childOffset := range node.Children { - r.traverseLeafNodes(childOffset, callback) - } - } -} diff --git a/btree/btree_test.go b/btree_test.go similarity index 87% rename from btree/btree_test.go rename to btree_test.go index 961ba0f..16dcb0e 100644 --- a/btree/btree_test.go +++ b/btree_test.go @@ -1,4 +1,4 @@ -package btree +package srdb import ( "os" @@ -16,7 +16,7 @@ func TestBTree(t *testing.T) { defer os.Remove("test.sst") // 2. 构建 B+Tree - builder := NewBuilder(file, 256) // 从 offset 256 开始 + builder := NewBTreeBuilder(file, 256) // 从 offset 256 开始 // 添加 1000 个 key-value for i := int64(1); i <= 1000; i++ { @@ -53,7 +53,7 @@ func TestBTree(t *testing.T) { defer mmapData.Unmap() // 5. 查询测试 - reader := NewReader(mmapData, rootOffset) + reader := NewBTreeReader(mmapData, rootOffset) // 测试存在的 key for i := int64(1); i <= 1000; i++ { @@ -93,18 +93,18 @@ func TestBTreeSerialization(t *testing.T) { // 序列化 data := leaf.Marshal() - if len(data) != NodeSize { - t.Errorf("Expected size %d, got %d", NodeSize, len(data)) + if len(data) != BTreeNodeSize { + t.Errorf("Expected size %d, got %d", BTreeNodeSize, len(data)) } // 反序列化 - leaf2 := Unmarshal(data) + leaf2 := UnmarshalBTree(data) if leaf2 == nil { t.Fatal("Unmarshal failed") } // 验证 - if leaf2.NodeType != NodeTypeLeaf { + if leaf2.NodeType != BTreeNodeTypeLeaf { t.Error("Wrong node type") } if leaf2.KeyCount != 3 { @@ -131,7 +131,7 @@ func BenchmarkBTreeGet(b *testing.B) { file, _ := os.Create("bench.sst") defer os.Remove("bench.sst") - builder := NewBuilder(file, 256) + builder := NewBTreeBuilder(file, 256) for i := int64(1); i <= 100000; i++ { builder.Add(i, i*100, 100) } @@ -144,11 +144,11 @@ func BenchmarkBTreeGet(b *testing.B) { mmapData, _ := mmap.Map(file, mmap.RDONLY, 0) defer mmapData.Unmap() - reader := NewReader(mmapData, rootOffset) + reader := NewBTreeReader(mmapData, rootOffset) // 性能测试 - b.ResetTimer() - for i := 0; i < b.N; i++ { + + for i := 0; b.Loop(); i++ { key := int64(i%100000 + 1) reader.Get(key) } diff --git a/compaction.go b/compaction.go new file mode 100644 index 0000000..4c2647d --- /dev/null +++ b/compaction.go @@ -0,0 +1,1154 @@ +package srdb + +import ( + "fmt" + "os" + "path/filepath" + "sort" + "sync" + "time" +) + +// CompactionTask 表示一个 Compaction 任务 +type CompactionTask struct { + Level int // 源层级 + InputFiles []*FileMetadata // 需要合并的输入文件 + OutputLevel int // 输出层级 +} + +// Picker 负责选择需要 Compaction 的文件 +type Picker struct { + // Level 大小限制 (字节) + levelSizeLimits [NumLevels]int64 + + // Level 文件数量限制 + levelFileLimits [NumLevels]int +} + +// NewPicker 创建新的 Compaction Picker +func NewPicker() *Picker { + p := &Picker{} + + // 设置每层的大小限制 (指数增长) + // L0: 10MB, L1: 100MB, L2: 1GB, L3: 10GB, L4: 100GB, L5: 1TB, L6: 无限制 + p.levelSizeLimits[0] = 10 * 1024 * 1024 // 10MB + p.levelSizeLimits[1] = 100 * 1024 * 1024 // 100MB + p.levelSizeLimits[2] = 1024 * 1024 * 1024 // 1GB + p.levelSizeLimits[3] = 10 * 1024 * 1024 * 1024 // 10GB + p.levelSizeLimits[4] = 100 * 1024 * 1024 * 1024 // 100GB + p.levelSizeLimits[5] = 1024 * 1024 * 1024 * 1024 // 1TB + p.levelSizeLimits[6] = 0 // 无限制 + + // 设置每层的文件数量限制 + // L0 特殊处理：文件数量限制为 4 (当有4个或更多文件时触发 compaction) + p.levelFileLimits[0] = 4 + // L1-L6: 不限制文件数量，只限制总大小 + for i := 1; i < NumLevels; i++ { + p.levelFileLimits[i] = 0 // 0 表示不限制 + } + + return p +} + +// PickCompaction 选择需要 Compaction 的任务（支持多任务并发） +// 返回空切片表示当前不需要 Compaction +func (p *Picker) PickCompaction(version *Version) []*CompactionTask { + tasks := make([]*CompactionTask, 0) + + // 1. 检查 L0 (基于文件数量) + if task := p.pickL0Compaction(version); task != nil { + tasks = append(tasks, task) + } + + // 2. 检查 L1-L5 (基于大小) + for level := 1; level < NumLevels-1; level++ { + if task := p.pickLevelCompaction(version, level); task != nil { + tasks = append(tasks, task) + } + } + + // 3. 按优先级排序（score 越高越优先） + if len(tasks) > 1 { + p.sortTasksByPriority(tasks, version) + } + + return tasks +} + +// sortTasksByPriority 按优先级对任务排序（score 从高到低） +func (p *Picker) sortTasksByPriority(tasks []*CompactionTask, version *Version) { + // 简单的冒泡排序（任务数量通常很少，< 7） + for i := 0; i < len(tasks)-1; i++ { + for j := i + 1; j < len(tasks); j++ { + scoreI := p.GetLevelScore(version, tasks[i].Level) + scoreJ := p.GetLevelScore(version, tasks[j].Level) + if scoreJ > scoreI { + tasks[i], tasks[j] = tasks[j], tasks[i] + } + } + } +} + +// pickL0Compaction 选择 L0 的 Compaction 任务 +// L0 特殊：文件可能有重叠的 key range，需要全部合并 +func (p *Picker) pickL0Compaction(version *Version) *CompactionTask { + l0Files := version.GetLevel(0) + if len(l0Files) == 0 { + return nil + } + + // 计算 L0 总大小 + totalSize := int64(0) + for _, file := range l0Files { + totalSize += file.FileSize + } + + // 检查是否需要 Compaction（同时考虑文件数量和总大小） + // 1. 文件数量超过限制（避免读放大：每次读取需要检查太多文件） + // 2. 总大小超过限制（避免 L0 占用过多空间） + needCompaction := false + if p.levelFileLimits[0] > 0 && len(l0Files) >= p.levelFileLimits[0] { + needCompaction = true + } + if p.levelSizeLimits[0] > 0 && totalSize >= p.levelSizeLimits[0] { + needCompaction = true + } + + if !needCompaction { + return nil + } + + // L0 → L1 Compaction + // 选择所有 L0 文件（因为 key range 可能重叠） + return &CompactionTask{ + Level: 0, + InputFiles: l0Files, + OutputLevel: 1, + } +} + +// pickLevelCompaction 选择 L1-L5 的 Compaction 任务 +// L1+ 的文件 key range 不重叠，可以选择多个不重叠的文件 +func (p *Picker) pickLevelCompaction(version *Version, level int) *CompactionTask { + if level < 1 || level >= NumLevels-1 { + return nil + } + + files := version.GetLevel(level) + if len(files) == 0 { + return nil + } + + // 计算当前层级的总大小 + totalSize := int64(0) + for _, file := range files { + totalSize += file.FileSize + } + + // 检查是否超过大小限制 + if totalSize < p.levelSizeLimits[level] { + return nil + } + + // 改进策略：根据层级压力动态调整选择策略 + // 1. 计算当前层级的压力（超过限制的倍数） + pressure := float64(totalSize) / float64(p.levelSizeLimits[level]) + + // 2. 根据压力确定目标大小和文件数量限制 + targetSize := p.getTargetCompactionSize(level + 1) + maxFiles := 10 // 默认最多 10 个文件 + + if pressure >= 10.0 { + // 压力极高（超过 10 倍）：选择更多文件，增大目标 + maxFiles = 100 + targetSize *= 5 + fmt.Printf("[Compaction] L%d pressure: %.1fx (CRITICAL) - selecting up to %d files, target: %s\n", + level, pressure, maxFiles, formatBytes(targetSize)) + } else if pressure >= 5.0 { + // 压力很高（超过 5 倍） + maxFiles = 50 + targetSize *= 3 + fmt.Printf("[Compaction] L%d pressure: %.1fx (HIGH) - selecting up to %d files, target: %s\n", + level, pressure, maxFiles, formatBytes(targetSize)) + } else if pressure >= 2.0 { + // 压力较高（超过 2 倍） + maxFiles = 20 + targetSize *= 2 + fmt.Printf("[Compaction] L%d pressure: %.1fx (ELEVATED) - selecting up to %d files, target: %s\n", + level, pressure, maxFiles, formatBytes(targetSize)) + } + + // 选择文件，直到累计大小接近目标 + selectedFiles := make([]*FileMetadata, 0) + currentSize := int64(0) + + for _, file := range files { + selectedFiles = append(selectedFiles, file) + currentSize += file.FileSize + + // 如果已经达到目标大小，停止选择 + if currentSize >= targetSize { + break + } + + // 达到文件数量限制 + if len(selectedFiles) >= maxFiles { + break + } + } + + return &CompactionTask{ + Level: level, + InputFiles: selectedFiles, + OutputLevel: level + 1, + } +} + +// getTargetCompactionSize 根据层级返回建议的 compaction 大小 +func (p *Picker) getTargetCompactionSize(level int) int64 { + switch level { + case 0: + return 2 * 1024 * 1024 // 2MB + case 1: + return 10 * 1024 * 1024 // 10MB + case 2: + return 50 * 1024 * 1024 // 50MB + case 3: + return 100 * 1024 * 1024 // 100MB + default: // L4+ + return 200 * 1024 * 1024 // 200MB + } +} + +// ShouldCompact 判断是否需要 Compaction +func (p *Picker) ShouldCompact(version *Version) bool { + tasks := p.PickCompaction(version) + return len(tasks) > 0 +} + +// GetLevelScore 获取每层的 Compaction 得分 (用于优先级排序) +// 得分越高，越需要 Compaction +func (p *Picker) GetLevelScore(version *Version, level int) float64 { + if level < 0 || level >= NumLevels { + return 0 + } + + files := version.GetLevel(level) + + // L0 同时考虑文件数量和总大小，取较大值作为得分 + if level == 0 { + scoreByCount := float64(0) + scoreBySize := float64(0) + + if p.levelFileLimits[0] > 0 { + scoreByCount = float64(len(files)) / float64(p.levelFileLimits[0]) + } + + if p.levelSizeLimits[0] > 0 { + totalSize := int64(0) + for _, file := range files { + totalSize += file.FileSize + } + scoreBySize = float64(totalSize) / float64(p.levelSizeLimits[0]) + } + + // 返回两者中的较大值（哪个维度更紧迫） + if scoreByCount > scoreBySize { + return scoreByCount + } + return scoreBySize + } + + // L1+ 基于总大小 + if p.levelSizeLimits[level] == 0 { + return 0 + } + + totalSize := int64(0) + for _, file := range files { + totalSize += file.FileSize + } + + return float64(totalSize) / float64(p.levelSizeLimits[level]) +} + +// formatBytes 格式化字节大小显示 +func formatBytes(bytes int64) string { + const unit = 1024 + if bytes < unit { + return fmt.Sprintf("%d B", bytes) + } + div, exp := int64(unit), 0 + for n := bytes / unit; n >= unit; n /= unit { + div *= unit + exp++ + } + units := []string{"KB", "MB", "GB", "TB"} + return fmt.Sprintf("%.2f %s", float64(bytes)/float64(div), units[exp]) +} + +// Compactor 负责执行 Compaction +type Compactor struct { + sstDir string + picker *Picker + versionSet *VersionSet + schema *Schema + mu sync.Mutex +} + +// NewCompactor 创建新的 Compactor +func NewCompactor(sstDir string, versionSet *VersionSet) *Compactor { + return &Compactor{ + sstDir: sstDir, + picker: NewPicker(), + versionSet: versionSet, + } +} + +// SetSchema 设置 Schema（用于读取 SST 文件） +func (c *Compactor) SetSchema(schema *Schema) { + c.mu.Lock() + defer c.mu.Unlock() + c.schema = schema +} + +// GetPicker 获取 Picker +func (c *Compactor) GetPicker() *Picker { + return c.picker +} + +// DoCompaction 执行一次 Compaction +// 返回: VersionEdit (记录变更), error +func (c *Compactor) DoCompaction(task *CompactionTask, version *Version) (*VersionEdit, error) { + c.mu.Lock() + defer c.mu.Unlock() + + if task == nil { + return nil, fmt.Errorf("compaction task is nil") + } + + // 0. 验证输入文件是否存在（防止并发 compaction 导致的竞态） + existingInputFiles := make([]*FileMetadata, 0, len(task.InputFiles)) + for _, file := range task.InputFiles { + sstPath := filepath.Join(c.sstDir, fmt.Sprintf("%06d.sst", file.FileNumber)) + if _, err := os.Stat(sstPath); err == nil { + existingInputFiles = append(existingInputFiles, file) + } else { + fmt.Printf("[Compaction] Warning: input file %06d.sst not found, skipping from task\n", file.FileNumber) + } + } + + // 如果所有输入文件都不存在，直接返回（无需 compaction） + if len(existingInputFiles) == 0 { + fmt.Printf("[Compaction] All input files missing, compaction skipped\n") + return nil, nil // 返回 nil 表示不需要应用任何 VersionEdit + } + + // 1. 读取输入文件的所有行 + inputRows, err := c.readInputFiles(existingInputFiles) + if err != nil { + return nil, fmt.Errorf("read input files: %w", err) + } + + // 2. 如果输出层级有文件，需要合并重叠的文件 + outputFiles := c.getOverlappingFiles(version, task.OutputLevel, inputRows) + var existingOutputFiles []*FileMetadata + var missingOutputFiles []*FileMetadata + if len(outputFiles) > 0 { + // 验证输出文件是否存在 + existingOutputFiles = make([]*FileMetadata, 0, len(outputFiles)) + missingOutputFiles = make([]*FileMetadata, 0) + for _, file := range outputFiles { + sstPath := filepath.Join(c.sstDir, fmt.Sprintf("%06d.sst", file.FileNumber)) + if _, err := os.Stat(sstPath); err == nil { + existingOutputFiles = append(existingOutputFiles, file) + } else { + // 输出层级的文件不存在，记录并在 VersionEdit 中删除它 + fmt.Printf("[Compaction] Warning: overlapping output file %06d.sst missing, will remove from MANIFEST\n", file.FileNumber) + missingOutputFiles = append(missingOutputFiles, file) + } + } + + outputRows, err := c.readInputFiles(existingOutputFiles) + if err != nil { + return nil, fmt.Errorf("read output files: %w", err) + } + inputRows = append(inputRows, outputRows...) + } + + // 3. 合并和去重 (保留最新的记录) + mergedRows := c.mergeRows(inputRows) + + // 计算平均行大小（基于输入文件的 FileMetadata） + avgRowSize := c.calculateAvgRowSize(existingInputFiles, existingOutputFiles) + + // 4. 写入新的 SST 文件到输出层级 + newFiles, err := c.writeOutputFiles(mergedRows, task.OutputLevel, avgRowSize) + if err != nil { + return nil, fmt.Errorf("write output files: %w", err) + } + + // 5. 创建 VersionEdit + edit := NewVersionEdit() + + // 删除实际存在且被处理的输入文件 + for _, file := range existingInputFiles { + edit.DeleteFile(file.FileNumber) + } + // 删除实际存在且被处理的输出层级文件 + for _, file := range existingOutputFiles { + edit.DeleteFile(file.FileNumber) + } + // 删除缺失的输出层级文件（清理 MANIFEST 中的过期引用） + for _, file := range missingOutputFiles { + edit.DeleteFile(file.FileNumber) + fmt.Printf("[Compaction] Removing missing file %06d.sst from MANIFEST\n", file.FileNumber) + } + + // 添加新文件，并跟踪最大文件编号 + var maxFileNumber int64 + for _, file := range newFiles { + edit.AddFile(file) + if file.FileNumber > maxFileNumber { + maxFileNumber = file.FileNumber + } + } + + // 持久化当前的文件编号计数器（关键修复：防止重启后文件编号重用） + // 使用最大文件编号 + 1 确保并发安全 + if maxFileNumber > 0 { + edit.SetNextFileNumber(maxFileNumber + 1) + } else { + // 如果没有新文件，使用当前值 + edit.SetNextFileNumber(c.versionSet.GetNextFileNumber()) + } + + return edit, nil +} + +// readInputFiles 读取输入文件的所有行 +// 注意：调用者必须确保传入的文件都存在，否则会返回错误 +func (c *Compactor) readInputFiles(files []*FileMetadata) ([]*SSTableRow, error) { + var allRows []*SSTableRow + + for _, file := range files { + sstPath := filepath.Join(c.sstDir, fmt.Sprintf("%06d.sst", file.FileNumber)) + + reader, err := NewSSTableReader(sstPath) + if err != nil { + return nil, fmt.Errorf("open sst %d: %w", file.FileNumber, err) + } + + // 设置 Schema（如果可用） + if c.schema != nil { + reader.SetSchema(c.schema) + } + + // 获取文件中实际存在的所有 key（不能用 MinKey-MaxKey 范围遍历，因为 key 可能是稀疏的） + keys := reader.GetAllKeys() + for _, seq := range keys { + row, err := reader.Get(seq) + if err != nil { + // 这种情况理论上不应该发生（key 来自索引），但为了安全还是处理一下 + continue + } + allRows = append(allRows, row) + } + + reader.Close() + } + + return allRows, nil +} + +// getOverlappingFiles 获取输出层级中与输入行重叠的文件 +func (c *Compactor) getOverlappingFiles(version *Version, level int, rows []*SSTableRow) []*FileMetadata { + if len(rows) == 0 { + return nil + } + + // 找到输入行的 key range + minKey := rows[0].Seq + maxKey := rows[0].Seq + for _, row := range rows { + if row.Seq < minKey { + minKey = row.Seq + } + if row.Seq > maxKey { + maxKey = row.Seq + } + } + + // 找到输出层级中重叠的文件 + var overlapping []*FileMetadata + levelFiles := version.GetLevel(level) + for _, file := range levelFiles { + // 检查 key range 是否重叠 + if file.MaxKey >= minKey && file.MinKey <= maxKey { + overlapping = append(overlapping, file) + } + } + + return overlapping +} + +// mergeRows 合并行，去重并保留最新的记录 +func (c *Compactor) mergeRows(rows []*SSTableRow) []*SSTableRow { + if len(rows) == 0 { + return rows + } + + // 按 Seq 排序 + sort.Slice(rows, func(i, j int) bool { + return rows[i].Seq < rows[j].Seq + }) + + // 去重：保留相同 Seq 的最新记录 (Timestamp 最大的) + merged := make([]*SSTableRow, 0, len(rows)) + var lastRow *SSTableRow + + for _, row := range rows { + if lastRow == nil || lastRow.Seq != row.Seq { + // 新的 Seq + merged = append(merged, row) + lastRow = row + } else { + // 相同 Seq，保留 Time 更大的 + if row.Time > lastRow.Time { + merged[len(merged)-1] = row + lastRow = row + } + } + } + + return merged +} + +// calculateAvgRowSize 基于输入文件的 FileMetadata 计算平均行大小 +func (c *Compactor) calculateAvgRowSize(inputFiles []*FileMetadata, outputFiles []*FileMetadata) int64 { + var totalSize int64 + var totalRows int64 + + // 统计输入文件 + for _, file := range inputFiles { + totalSize += file.FileSize + totalRows += file.RowCount + } + + // 统计输出文件 + for _, file := range outputFiles { + totalSize += file.FileSize + totalRows += file.RowCount + } + + // 计算平均值 + if totalRows == 0 { + return 1024 // 默认 1KB + } + return totalSize / totalRows +} + +// writeOutputFiles 将合并后的行写入新的 SST 文件 +func (c *Compactor) writeOutputFiles(rows []*SSTableRow, level int, avgRowSize int64) ([]*FileMetadata, error) { + if len(rows) == 0 { + return nil, nil + } + + // 根据层级动态调整文件大小目标 + // L0: 2MB (快速 flush，小文件) + // L1: 10MB + // L2: 50MB + // L3: 100MB + // L4+: 200MB + targetFileSize := c.getTargetFileSize(level) + + // 应用安全系数：由于压缩率、索引开销等因素，估算值可能不准确 + // 使用 80% 的目标大小作为分割点，避免实际文件超出目标过多 + targetFileSize = targetFileSize * 80 / 100 + + var newFiles []*FileMetadata + var currentRows []*SSTableRow + var currentSize int64 + + for _, row := range rows { + // 使用平均行大小估算（基于输入文件的统计信息） + rowSize := avgRowSize + + // 如果当前文件大小超过目标，写入文件 + if currentSize > 0 && currentSize+rowSize > targetFileSize { + file, err := c.writeFile(currentRows, level) + if err != nil { + return nil, err + } + newFiles = append(newFiles, file) + + // 重置 + currentRows = nil + currentSize = 0 + } + + currentRows = append(currentRows, row) + currentSize += rowSize + } + + // 写入最后一个文件 + if len(currentRows) > 0 { + file, err := c.writeFile(currentRows, level) + if err != nil { + return nil, err + } + newFiles = append(newFiles, file) + } + + return newFiles, nil +} + +// getTargetFileSize 根据层级返回目标文件大小 +func (c *Compactor) getTargetFileSize(level int) int64 { + switch level { + case 0: + return 2 * 1024 * 1024 // 2MB + case 1: + return 10 * 1024 * 1024 // 10MB + case 2: + return 50 * 1024 * 1024 // 50MB + case 3: + return 100 * 1024 * 1024 // 100MB + default: // L4+ + return 200 * 1024 * 1024 // 200MB + } +} + +// writeFile 写入单个 SST 文件 +func (c *Compactor) writeFile(rows []*SSTableRow, level int) (*FileMetadata, error) { + // 从 VersionSet 分配新的文件编号 + fileNumber := c.versionSet.AllocateFileNumber() + sstPath := filepath.Join(c.sstDir, fmt.Sprintf("%06d.sst", fileNumber)) + + // 创建文件 + file, err := os.Create(sstPath) + if err != nil { + return nil, err + } + defer file.Close() + + // 使用 Compactor 的 Schema 创建 writer + writer := NewSSTableWriter(file, c.schema) + + // 注意：这个方法只负责创建文件，不负责注册到 SSTableManager + // 注册工作由 CompactionManager 在 VersionEdit apply 后完成 + + // 写入所有行 + for _, row := range rows { + err = writer.Add(row) + if err != nil { + os.Remove(sstPath) + return nil, err + } + } + + // 完成写入 + err = writer.Finish() + if err != nil { + os.Remove(sstPath) + return nil, err + } + + // 获取文件信息 + fileInfo, err := file.Stat() + if err != nil { + return nil, err + } + + // 创建 FileMetadata + metadata := &FileMetadata{ + FileNumber: fileNumber, + Level: level, + FileSize: fileInfo.Size(), + MinKey: rows[0].Seq, + MaxKey: rows[len(rows)-1].Seq, + RowCount: int64(len(rows)), + } + + return metadata, nil +} + +// CompactionManager 管理 Compaction 流程 +type CompactionManager struct { + compactor *Compactor + versionSet *VersionSet + sstManager *SSTableManager // 添加 sstManager 引用，用于同步删除 readers + sstDir string + + // 控制后台 Compaction + stopCh chan struct{} + wg sync.WaitGroup + + // Compaction 并发控制 + compactionMu sync.Mutex // 防止并发执行 compaction + + // 统计信息 + mu sync.RWMutex + totalCompactions int64 + lastCompactionTime time.Time + lastFailedFile int64 // 最后失败的文件编号 + consecutiveFails int // 连续失败次数 + lastGCTime time.Time + totalOrphansFound int64 +} + +// NewCompactionManager 创建新的 Compaction Manager +func NewCompactionManager(sstDir string, versionSet *VersionSet, sstManager *SSTableManager) *CompactionManager { + return &CompactionManager{ + compactor: NewCompactor(sstDir, versionSet), + versionSet: versionSet, + sstManager: sstManager, + sstDir: sstDir, + stopCh: make(chan struct{}), + } +} + +// GetPicker 获取 Compaction Picker +func (m *CompactionManager) GetPicker() *Picker { + return m.compactor.GetPicker() +} + +// SetSchema 设置 Schema（用于优化 SST 文件读写） +func (m *CompactionManager) SetSchema(schema *Schema) { + m.compactor.SetSchema(schema) +} + +// Start 启动后台 Compaction 和垃圾回收 +func (m *CompactionManager) Start() { + m.wg.Add(2) + go m.backgroundCompaction() + go m.backgroundGarbageCollection() +} + +// Stop 停止后台 Compaction +func (m *CompactionManager) Stop() { + close(m.stopCh) + m.wg.Wait() +} + +// backgroundCompaction 后台 Compaction 循环 +func (m *CompactionManager) backgroundCompaction() { + defer m.wg.Done() + + ticker := time.NewTicker(10 * time.Second) // 每 10 秒检查一次 + defer ticker.Stop() + + for { + select { + case <-m.stopCh: + return + case <-ticker.C: + m.maybeCompact() + } + } +} + +// MaybeCompact 检查是否需要 Compaction 并执行（公开方法，供外部调用） +// 非阻塞：如果已有 compaction 在执行，直接返回 +func (m *CompactionManager) MaybeCompact() { + // 尝试获取锁，如果已有 compaction 在执行，直接返回 + if !m.compactionMu.TryLock() { + return + } + defer m.compactionMu.Unlock() + + m.doCompact() +} + +// maybeCompact 内部使用的阻塞版本（后台 goroutine 使用） +func (m *CompactionManager) maybeCompact() { + m.compactionMu.Lock() + defer m.compactionMu.Unlock() + + m.doCompact() +} + +// doCompact 实际执行 compaction 的逻辑（必须在持有 compactionMu 时调用） +// 支持并发执行多个层级的 compaction +func (m *CompactionManager) doCompact() { + // 获取当前版本 + version := m.versionSet.GetCurrent() + if version == nil { + return + } + + // 获取所有需要 Compaction 的任务（已按优先级排序） + picker := m.compactor.GetPicker() + tasks := picker.PickCompaction(version) + if len(tasks) == 0 { + // 输出诊断信息 + m.printCompactionStats(version, picker) + return + } + + fmt.Printf("[Compaction] Found %d tasks to execute\n", len(tasks)) + + // 并发执行所有任务 + successCount := 0 + for _, task := range tasks { + // 检查是否是上次失败的文件（防止无限重试） + if len(task.InputFiles) > 0 { + firstFile := task.InputFiles[0].FileNumber + m.mu.Lock() + if m.lastFailedFile == firstFile && m.consecutiveFails >= 3 { + fmt.Printf("[Compaction] Skipping L%d file %d (failed %d times)\n", + task.Level, firstFile, m.consecutiveFails) + m.consecutiveFails = 0 + m.lastFailedFile = 0 + m.mu.Unlock() + continue + } + m.mu.Unlock() + } + + // 获取最新版本（每个任务执行前） + currentVersion := m.versionSet.GetCurrent() + if currentVersion == nil { + continue + } + + // 执行 Compaction + fmt.Printf("[Compaction] Starting: L%d -> L%d, files: %d\n", + task.Level, task.OutputLevel, len(task.InputFiles)) + + err := m.DoCompactionWithVersion(task, currentVersion) + if err != nil { + fmt.Printf("[Compaction] Failed L%d -> L%d: %v\n", task.Level, task.OutputLevel, err) + + // 记录失败信息 + if len(task.InputFiles) > 0 { + firstFile := task.InputFiles[0].FileNumber + m.mu.Lock() + if m.lastFailedFile == firstFile { + m.consecutiveFails++ + } else { + m.lastFailedFile = firstFile + m.consecutiveFails = 1 + } + m.mu.Unlock() + } + } else { + fmt.Printf("[Compaction] Completed: L%d -> L%d\n", task.Level, task.OutputLevel) + successCount++ + + // 清除失败计数 + m.mu.Lock() + m.consecutiveFails = 0 + m.lastFailedFile = 0 + m.mu.Unlock() + } + } + + fmt.Printf("[Compaction] Batch completed: %d/%d tasks succeeded\n", successCount, len(tasks)) +} + +// printCompactionStats 输出 Compaction 统计信息（每分钟一次） +func (m *CompactionManager) printCompactionStats(version *Version, picker *Picker) { + m.mu.Lock() + defer m.mu.Unlock() + + // 限制输出频率：每 60 秒输出一次 + if time.Since(m.lastCompactionTime) < 60*time.Second { + return + } + m.lastCompactionTime = time.Now() + + fmt.Println("[Compaction] Status check:") + for level := range 7 { + files := version.GetLevel(level) + if len(files) == 0 { + continue + } + + totalSize := int64(0) + for _, f := range files { + totalSize += f.FileSize + } + + score := picker.GetLevelScore(version, level) + fmt.Printf(" L%d: %d files, %.2f MB, score: %.2f\n", + level, len(files), float64(totalSize)/(1024*1024), score) + } +} + +// DoCompactionWithVersion 使用指定的版本执行 Compaction +func (m *CompactionManager) DoCompactionWithVersion(task *CompactionTask, version *Version) error { + if version == nil { + return fmt.Errorf("version is nil") + } + + // 执行 Compaction（使用传入的 version，而不是重新获取） + edit, err := m.compactor.DoCompaction(task, version) + if err != nil { + return fmt.Errorf("compaction failed: %w", err) + } + + // 如果 edit 为 nil，说明所有文件都已经不存在，无需应用变更 + if edit == nil { + fmt.Printf("[Compaction] No changes needed (files already removed)\n") + return nil + } + + // 应用 VersionEdit + err = m.versionSet.LogAndApply(edit) + if err != nil { + // LogAndApply 失败，清理已写入的新 SST 文件（防止孤儿文件） + fmt.Printf("[Compaction] LogAndApply failed, cleaning up new files: %v\n", err) + m.cleanupNewFiles(edit) + return fmt.Errorf("apply version edit: %w", err) + } + + // LogAndApply 成功后，注册新创建的 SST 文件到 SSTableManager + // 这样查询才能读取到 compaction 创建的文件 + if m.sstManager != nil { + for _, file := range edit.AddedFiles { + sstPath := filepath.Join(m.sstDir, fmt.Sprintf("%06d.sst", file.FileNumber)) + reader, err := NewSSTableReader(sstPath) + if err != nil { + fmt.Printf("[Compaction] Warning: failed to open new file %06d.sst: %v\n", file.FileNumber, err) + continue + } + // 设置 Schema + if m.compactor.schema != nil { + reader.SetSchema(m.compactor.schema) + } + // 添加到 SSTableManager + m.sstManager.AddReader(reader) + } + } + + // LogAndApply 成功后，删除废弃的 SST 文件 + m.deleteObsoleteFiles(edit) + + // 更新统计信息 + m.mu.Lock() + m.totalCompactions++ + m.lastCompactionTime = time.Now() + m.mu.Unlock() + + return nil +} + +// DoCompaction 执行一次 Compaction（兼容旧接口） +func (m *CompactionManager) DoCompaction(task *CompactionTask) error { + // 获取当前版本 + version := m.versionSet.GetCurrent() + if version == nil { + return fmt.Errorf("no current version") + } + + return m.DoCompactionWithVersion(task, version) +} + +// cleanupNewFiles 清理 LogAndApply 失败后的新文件（防止孤儿文件） +func (m *CompactionManager) cleanupNewFiles(edit *VersionEdit) { + if edit == nil { + return + } + + fmt.Printf("[Compaction] Cleaning up %d new files after LogAndApply failure\n", len(edit.AddedFiles)) + + // 删除新创建的文件 + for _, file := range edit.AddedFiles { + sstPath := filepath.Join(m.sstDir, fmt.Sprintf("%06d.sst", file.FileNumber)) + err := os.Remove(sstPath) + if err != nil { + fmt.Printf("[Compaction] Failed to cleanup new file %06d.sst: %v\n", file.FileNumber, err) + } else { + fmt.Printf("[Compaction] Cleaned up new file %06d.sst\n", file.FileNumber) + } + } +} + +// deleteObsoleteFiles 删除废弃的 SST 文件 +func (m *CompactionManager) deleteObsoleteFiles(edit *VersionEdit) { + if edit == nil { + fmt.Printf("[Compaction] deleteObsoleteFiles: edit is nil\n") + return + } + + fmt.Printf("[Compaction] deleteObsoleteFiles: %d files to delete\n", len(edit.DeletedFiles)) + + // 删除被标记为删除的文件 + for _, fileNum := range edit.DeletedFiles { + // 1. 从 SSTableManager 移除 reader（如果 sstManager 可用） + if m.sstManager != nil { + err := m.sstManager.RemoveReader(fileNum) + if err != nil { + fmt.Printf("[Compaction] Failed to remove reader for %06d.sst: %v\n", fileNum, err) + } + } + + // 2. 删除物理文件 + sstPath := filepath.Join(m.sstDir, fmt.Sprintf("%06d.sst", fileNum)) + err := os.Remove(sstPath) + if err != nil { + // 删除失败只记录日志，不影响 compaction 流程 + // 后台垃圾回收器会重试 + fmt.Printf("[Compaction] Failed to delete obsolete file %06d.sst: %v\n", fileNum, err) + } else { + fmt.Printf("[Compaction] Deleted obsolete file %06d.sst\n", fileNum) + } + } +} + +// TriggerCompaction 手动触发一次 Compaction（所有需要的层级） +func (m *CompactionManager) TriggerCompaction() error { + version := m.versionSet.GetCurrent() + if version == nil { + return fmt.Errorf("no current version") + } + + picker := m.compactor.GetPicker() + tasks := picker.PickCompaction(version) + if len(tasks) == 0 { + return nil // 不需要 Compaction + } + + // 依次执行所有任务 + for _, task := range tasks { + currentVersion := m.versionSet.GetCurrent() + if err := m.DoCompactionWithVersion(task, currentVersion); err != nil { + return err + } + } + + return nil +} + +// GetStats 获取 Compaction 统计信息 +func (m *CompactionManager) GetStats() map[string]any { + m.mu.RLock() + defer m.mu.RUnlock() + + return map[string]any{ + "total_compactions": m.totalCompactions, + "last_compaction_time": m.lastCompactionTime, + } +} + +// GetLevelStats 获取每层的统计信息 +func (m *CompactionManager) GetLevelStats() []map[string]any { + version := m.versionSet.GetCurrent() + if version == nil { + return nil + } + + picker := m.compactor.GetPicker() + stats := make([]map[string]any, NumLevels) + + for level := range NumLevels { + files := version.GetLevel(level) + totalSize := int64(0) + for _, file := range files { + totalSize += file.FileSize + } + + stats[level] = map[string]any{ + "level": level, + "file_count": len(files), + "total_size": totalSize, + "score": picker.GetLevelScore(version, level), + } + } + + return stats +} + +// backgroundGarbageCollection 后台垃圾回收循环 +func (m *CompactionManager) backgroundGarbageCollection() { + defer m.wg.Done() + + ticker := time.NewTicker(5 * time.Minute) // 每 5 分钟检查一次 + defer ticker.Stop() + + for { + select { + case <-m.stopCh: + return + case <-ticker.C: + m.collectOrphanFiles() + } + } +} + +// collectOrphanFiles 收集并删除孤儿 SST 文件 +func (m *CompactionManager) collectOrphanFiles() { + // 1. 获取当前版本中的所有活跃文件 + version := m.versionSet.GetCurrent() + if version == nil { + return + } + + activeFiles := make(map[int64]bool) + for level := range NumLevels { + files := version.GetLevel(level) + for _, file := range files { + activeFiles[file.FileNumber] = true + } + } + + // 2. 扫描 SST 目录中的所有文件 + pattern := filepath.Join(m.sstDir, "*.sst") + sstFiles, err := filepath.Glob(pattern) + if err != nil { + fmt.Printf("[GC] Failed to scan SST directory: %v\n", err) + return + } + + // 3. 找出孤儿文件并删除 + orphanCount := 0 + for _, sstPath := range sstFiles { + // 提取文件编号 + var fileNum int64 + _, err := fmt.Sscanf(filepath.Base(sstPath), "%d.sst", &fileNum) + if err != nil { + continue + } + + // 检查是否是活跃文件 + if !activeFiles[fileNum] { + // 检查文件修改时间，避免删除正在 flush 的文件 + // 如果文件在最近 1 分钟内创建/修改，跳过（可能正在 LogAndApply） + fileInfo, err := os.Stat(sstPath) + if err != nil { + continue + } + if time.Since(fileInfo.ModTime()) < 1*time.Minute { + fmt.Printf("[GC] Skipping recently modified file %06d.sst (age: %v)\n", + fileNum, time.Since(fileInfo.ModTime())) + continue + } + + // 这是孤儿文件，删除它 + err = os.Remove(sstPath) + if err != nil { + fmt.Printf("[GC] Failed to delete orphan file %06d.sst: %v\n", fileNum, err) + } else { + fmt.Printf("[GC] Deleted orphan file %06d.sst\n", fileNum) + orphanCount++ + } + } + } + + // 4. 更新统计信息 + m.mu.Lock() + m.lastGCTime = time.Now() + m.totalOrphansFound += int64(orphanCount) + m.mu.Unlock() + + if orphanCount > 0 { + fmt.Printf("[GC] Completed: cleaned up %d orphan files (total: %d)\n", orphanCount, m.totalOrphansFound) + } +} + +// CleanupOrphanFiles 手动触发孤儿文件清理（可在启动时调用） +func (m *CompactionManager) CleanupOrphanFiles() { + fmt.Println("[GC] Manual cleanup triggered") + m.collectOrphanFiles() +} diff --git a/compaction/compactor.go b/compaction/compactor.go deleted file mode 100644 index a60ea7e..0000000 --- a/compaction/compactor.go +++ /dev/null @@ -1,370 +0,0 @@ -package compaction - -import ( - "code.tczkiot.com/srdb/manifest" - "code.tczkiot.com/srdb/sst" - "fmt" - "os" - "path/filepath" - "sort" - "sync" -) - -// Compactor 负责执行 Compaction -type Compactor struct { - sstDir string - picker *Picker - versionSet *manifest.VersionSet - mu sync.Mutex -} - -// NewCompactor 创建新的 Compactor -func NewCompactor(sstDir string, versionSet *manifest.VersionSet) *Compactor { - return &Compactor{ - sstDir: sstDir, - picker: NewPicker(), - versionSet: versionSet, - } -} - -// GetPicker 获取 Picker -func (c *Compactor) GetPicker() *Picker { - return c.picker -} - -// DoCompaction 执行一次 Compaction -// 返回: VersionEdit (记录变更), error -func (c *Compactor) DoCompaction(task *CompactionTask, version *manifest.Version) (*manifest.VersionEdit, error) { - c.mu.Lock() - defer c.mu.Unlock() - - if task == nil { - return nil, fmt.Errorf("compaction task is nil") - } - - // 0. 验证输入文件是否存在（防止并发 compaction 导致的竞态） - existingInputFiles := make([]*manifest.FileMetadata, 0, len(task.InputFiles)) - for _, file := range task.InputFiles { - sstPath := filepath.Join(c.sstDir, fmt.Sprintf("%06d.sst", file.FileNumber)) - if _, err := os.Stat(sstPath); err == nil { - existingInputFiles = append(existingInputFiles, file) - } else { - fmt.Printf("[Compaction] Warning: input file %06d.sst not found, skipping from task\n", file.FileNumber) - } - } - - // 如果所有输入文件都不存在，直接返回（无需 compaction） - if len(existingInputFiles) == 0 { - fmt.Printf("[Compaction] All input files missing, compaction skipped\n") - return nil, nil // 返回 nil 表示不需要应用任何 VersionEdit - } - - // 1. 读取输入文件的所有行 - inputRows, err := c.readInputFiles(existingInputFiles) - if err != nil { - return nil, fmt.Errorf("read input files: %w", err) - } - - // 2. 如果输出层级有文件，需要合并重叠的文件 - outputFiles := c.getOverlappingFiles(version, task.OutputLevel, inputRows) - var existingOutputFiles []*manifest.FileMetadata - var missingOutputFiles []*manifest.FileMetadata - if len(outputFiles) > 0 { - // 验证输出文件是否存在 - existingOutputFiles = make([]*manifest.FileMetadata, 0, len(outputFiles)) - missingOutputFiles = make([]*manifest.FileMetadata, 0) - for _, file := range outputFiles { - sstPath := filepath.Join(c.sstDir, fmt.Sprintf("%06d.sst", file.FileNumber)) - if _, err := os.Stat(sstPath); err == nil { - existingOutputFiles = append(existingOutputFiles, file) - } else { - // 输出层级的文件不存在，记录并在 VersionEdit 中删除它 - fmt.Printf("[Compaction] Warning: overlapping output file %06d.sst missing, will remove from MANIFEST\n", file.FileNumber) - missingOutputFiles = append(missingOutputFiles, file) - } - } - - outputRows, err := c.readInputFiles(existingOutputFiles) - if err != nil { - return nil, fmt.Errorf("read output files: %w", err) - } - inputRows = append(inputRows, outputRows...) - } - - // 3. 合并和去重 (保留最新的记录) - mergedRows := c.mergeRows(inputRows) - - // 计算平均行大小（基于输入文件的 FileMetadata） - avgRowSize := c.calculateAvgRowSize(existingInputFiles, existingOutputFiles) - - // 4. 写入新的 SST 文件到输出层级 - newFiles, err := c.writeOutputFiles(mergedRows, task.OutputLevel, version, avgRowSize) - if err != nil { - return nil, fmt.Errorf("write output files: %w", err) - } - - // 5. 创建 VersionEdit - edit := manifest.NewVersionEdit() - - // 删除实际存在且被处理的输入文件 - for _, file := range existingInputFiles { - edit.DeleteFile(file.FileNumber) - } - // 删除实际存在且被处理的输出层级文件 - for _, file := range existingOutputFiles { - edit.DeleteFile(file.FileNumber) - } - // 删除缺失的输出层级文件（清理 MANIFEST 中的过期引用） - for _, file := range missingOutputFiles { - edit.DeleteFile(file.FileNumber) - fmt.Printf("[Compaction] Removing missing file %06d.sst from MANIFEST\n", file.FileNumber) - } - - // 添加新文件 - for _, file := range newFiles { - edit.AddFile(file) - } - - // 持久化当前的文件编号计数器（关键修复：防止重启后文件编号重用） - edit.SetNextFileNumber(c.versionSet.GetNextFileNumber()) - - return edit, nil -} - -// readInputFiles 读取输入文件的所有行 -// 注意：调用者必须确保传入的文件都存在，否则会返回错误 -func (c *Compactor) readInputFiles(files []*manifest.FileMetadata) ([]*sst.Row, error) { - var allRows []*sst.Row - - for _, file := range files { - sstPath := filepath.Join(c.sstDir, fmt.Sprintf("%06d.sst", file.FileNumber)) - - reader, err := sst.NewReader(sstPath) - if err != nil { - return nil, fmt.Errorf("open sst %d: %w", file.FileNumber, err) - } - - // 获取文件中实际存在的所有 key（不能用 MinKey-MaxKey 范围遍历，因为 key 可能是稀疏的） - keys := reader.GetAllKeys() - for _, seq := range keys { - row, err := reader.Get(seq) - if err != nil { - // 这种情况理论上不应该发生（key 来自索引），但为了安全还是处理一下 - continue - } - allRows = append(allRows, row) - } - - reader.Close() - } - - return allRows, nil -} - -// getOverlappingFiles 获取输出层级中与输入行重叠的文件 -func (c *Compactor) getOverlappingFiles(version *manifest.Version, level int, rows []*sst.Row) []*manifest.FileMetadata { - if len(rows) == 0 { - return nil - } - - // 找到输入行的 key range - minKey := rows[0].Seq - maxKey := rows[0].Seq - for _, row := range rows { - if row.Seq < minKey { - minKey = row.Seq - } - if row.Seq > maxKey { - maxKey = row.Seq - } - } - - // 找到输出层级中重叠的文件 - var overlapping []*manifest.FileMetadata - levelFiles := version.GetLevel(level) - for _, file := range levelFiles { - // 检查 key range 是否重叠 - if file.MaxKey >= minKey && file.MinKey <= maxKey { - overlapping = append(overlapping, file) - } - } - - return overlapping -} - -// mergeRows 合并行，去重并保留最新的记录 -func (c *Compactor) mergeRows(rows []*sst.Row) []*sst.Row { - if len(rows) == 0 { - return rows - } - - // 按 Seq 排序 - sort.Slice(rows, func(i, j int) bool { - return rows[i].Seq < rows[j].Seq - }) - - // 去重：保留相同 Seq 的最新记录 (Timestamp 最大的) - merged := make([]*sst.Row, 0, len(rows)) - var lastRow *sst.Row - - for _, row := range rows { - if lastRow == nil || lastRow.Seq != row.Seq { - // 新的 Seq - merged = append(merged, row) - lastRow = row - } else { - // 相同 Seq，保留 Time 更大的 - if row.Time > lastRow.Time { - merged[len(merged)-1] = row - lastRow = row - } - } - } - - return merged -} - -// calculateAvgRowSize 基于输入文件的 FileMetadata 计算平均行大小 -func (c *Compactor) calculateAvgRowSize(inputFiles []*manifest.FileMetadata, outputFiles []*manifest.FileMetadata) int64 { - var totalSize int64 - var totalRows int64 - - // 统计输入文件 - for _, file := range inputFiles { - totalSize += file.FileSize - totalRows += file.RowCount - } - - // 统计输出文件 - for _, file := range outputFiles { - totalSize += file.FileSize - totalRows += file.RowCount - } - - // 计算平均值 - if totalRows == 0 { - return 1024 // 默认 1KB - } - return totalSize / totalRows -} - -// writeOutputFiles 将合并后的行写入新的 SST 文件 -func (c *Compactor) writeOutputFiles(rows []*sst.Row, level int, version *manifest.Version, avgRowSize int64) ([]*manifest.FileMetadata, error) { - if len(rows) == 0 { - return nil, nil - } - - // 根据层级动态调整文件大小目标 - // L0: 2MB (快速 flush，小文件) - // L1: 10MB - // L2: 50MB - // L3: 100MB - // L4+: 200MB - targetFileSize := c.getTargetFileSize(level) - - // 应用安全系数：由于压缩率、索引开销等因素，估算值可能不准确 - // 使用 80% 的目标大小作为分割点，避免实际文件超出目标过多 - targetFileSize = targetFileSize * 80 / 100 - - var newFiles []*manifest.FileMetadata - var currentRows []*sst.Row - var currentSize int64 - - for _, row := range rows { - // 使用平均行大小估算（基于输入文件的统计信息） - rowSize := avgRowSize - - // 如果当前文件大小超过目标，写入文件 - if currentSize > 0 && currentSize+rowSize > targetFileSize { - file, err := c.writeFile(currentRows, level, version) - if err != nil { - return nil, err - } - newFiles = append(newFiles, file) - - // 重置 - currentRows = nil - currentSize = 0 - } - - currentRows = append(currentRows, row) - currentSize += rowSize - } - - // 写入最后一个文件 - if len(currentRows) > 0 { - file, err := c.writeFile(currentRows, level, version) - if err != nil { - return nil, err - } - newFiles = append(newFiles, file) - } - - return newFiles, nil -} - -// getTargetFileSize 根据层级返回目标文件大小 -func (c *Compactor) getTargetFileSize(level int) int64 { - switch level { - case 0: - return 2 * 1024 * 1024 // 2MB - case 1: - return 10 * 1024 * 1024 // 10MB - case 2: - return 50 * 1024 * 1024 // 50MB - case 3: - return 100 * 1024 * 1024 // 100MB - default: // L4+ - return 200 * 1024 * 1024 // 200MB - } -} - -// writeFile 写入单个 SST 文件 -func (c *Compactor) writeFile(rows []*sst.Row, level int, version *manifest.Version) (*manifest.FileMetadata, error) { - // 从 VersionSet 分配新的文件编号 - fileNumber := c.versionSet.AllocateFileNumber() - sstPath := filepath.Join(c.sstDir, fmt.Sprintf("%06d.sst", fileNumber)) - - // 创建文件 - file, err := os.Create(sstPath) - if err != nil { - return nil, err - } - defer file.Close() - - writer := sst.NewWriter(file) - - // 写入所有行 - for _, row := range rows { - err = writer.Add(row) - if err != nil { - os.Remove(sstPath) - return nil, err - } - } - - // 完成写入 - err = writer.Finish() - if err != nil { - os.Remove(sstPath) - return nil, err - } - - // 获取文件信息 - fileInfo, err := file.Stat() - if err != nil { - return nil, err - } - - // 创建 FileMetadata - metadata := &manifest.FileMetadata{ - FileNumber: fileNumber, - Level: level, - FileSize: fileInfo.Size(), - MinKey: rows[0].Seq, - MaxKey: rows[len(rows)-1].Seq, - RowCount: int64(len(rows)), - } - - return metadata, nil -} diff --git a/compaction/manager.go b/compaction/manager.go deleted file mode 100644 index f8cdaf3..0000000 --- a/compaction/manager.go +++ /dev/null @@ -1,444 +0,0 @@ -package compaction - -import ( - "fmt" - "os" - "path/filepath" - "sync" - "time" - - "code.tczkiot.com/srdb/manifest" -) - -// Manager 管理 Compaction 流程 -type Manager struct { - compactor *Compactor - versionSet *manifest.VersionSet - sstDir string - - // 控制后台 Compaction - stopCh chan struct{} - wg sync.WaitGroup - - // Compaction 并发控制 - compactionMu sync.Mutex // 防止并发执行 compaction - - // 统计信息 - mu sync.RWMutex - totalCompactions int64 - lastCompactionTime time.Time - lastFailedFile int64 // 最后失败的文件编号 - consecutiveFails int // 连续失败次数 - lastGCTime time.Time - totalOrphansFound int64 -} - -// NewManager 创建新的 Compaction Manager -func NewManager(sstDir string, versionSet *manifest.VersionSet) *Manager { - return &Manager{ - compactor: NewCompactor(sstDir, versionSet), - versionSet: versionSet, - sstDir: sstDir, - stopCh: make(chan struct{}), - } -} - -// GetPicker 获取 Compaction Picker -func (m *Manager) GetPicker() *Picker { - return m.compactor.GetPicker() -} - -// Start 启动后台 Compaction 和垃圾回收 -func (m *Manager) Start() { - m.wg.Add(2) - go m.backgroundCompaction() - go m.backgroundGarbageCollection() -} - -// Stop 停止后台 Compaction -func (m *Manager) Stop() { - close(m.stopCh) - m.wg.Wait() -} - -// backgroundCompaction 后台 Compaction 循环 -func (m *Manager) backgroundCompaction() { - defer m.wg.Done() - - ticker := time.NewTicker(10 * time.Second) // 每 10 秒检查一次 - defer ticker.Stop() - - for { - select { - case <-m.stopCh: - return - case <-ticker.C: - m.maybeCompact() - } - } -} - -// MaybeCompact 检查是否需要 Compaction 并执行（公开方法，供外部调用） -// 非阻塞：如果已有 compaction 在执行，直接返回 -func (m *Manager) MaybeCompact() { - // 尝试获取锁，如果已有 compaction 在执行，直接返回 - if !m.compactionMu.TryLock() { - return - } - defer m.compactionMu.Unlock() - - m.doCompact() -} - -// maybeCompact 内部使用的阻塞版本（后台 goroutine 使用） -func (m *Manager) maybeCompact() { - m.compactionMu.Lock() - defer m.compactionMu.Unlock() - - m.doCompact() -} - -// doCompact 实际执行 compaction 的逻辑（必须在持有 compactionMu 时调用） -// 支持并发执行多个层级的 compaction -func (m *Manager) doCompact() { - // 获取当前版本 - version := m.versionSet.GetCurrent() - if version == nil { - return - } - - // 获取所有需要 Compaction 的任务（已按优先级排序） - picker := m.compactor.GetPicker() - tasks := picker.PickCompaction(version) - if len(tasks) == 0 { - // 输出诊断信息 - m.printCompactionStats(version, picker) - return - } - - fmt.Printf("[Compaction] Found %d tasks to execute\n", len(tasks)) - - // 并发执行所有任务 - successCount := 0 - for _, task := range tasks { - // 检查是否是上次失败的文件（防止无限重试） - if len(task.InputFiles) > 0 { - firstFile := task.InputFiles[0].FileNumber - m.mu.Lock() - if m.lastFailedFile == firstFile && m.consecutiveFails >= 3 { - fmt.Printf("[Compaction] Skipping L%d file %d (failed %d times)\n", - task.Level, firstFile, m.consecutiveFails) - m.consecutiveFails = 0 - m.lastFailedFile = 0 - m.mu.Unlock() - continue - } - m.mu.Unlock() - } - - // 获取最新版本（每个任务执行前） - currentVersion := m.versionSet.GetCurrent() - if currentVersion == nil { - continue - } - - // 执行 Compaction - fmt.Printf("[Compaction] Starting: L%d -> L%d, files: %d\n", - task.Level, task.OutputLevel, len(task.InputFiles)) - - err := m.DoCompactionWithVersion(task, currentVersion) - if err != nil { - fmt.Printf("[Compaction] Failed L%d -> L%d: %v\n", task.Level, task.OutputLevel, err) - - // 记录失败信息 - if len(task.InputFiles) > 0 { - firstFile := task.InputFiles[0].FileNumber - m.mu.Lock() - if m.lastFailedFile == firstFile { - m.consecutiveFails++ - } else { - m.lastFailedFile = firstFile - m.consecutiveFails = 1 - } - m.mu.Unlock() - } - } else { - fmt.Printf("[Compaction] Completed: L%d -> L%d\n", task.Level, task.OutputLevel) - successCount++ - - // 清除失败计数 - m.mu.Lock() - m.consecutiveFails = 0 - m.lastFailedFile = 0 - m.mu.Unlock() - } - } - - fmt.Printf("[Compaction] Batch completed: %d/%d tasks succeeded\n", successCount, len(tasks)) -} - -// printCompactionStats 输出 Compaction 统计信息（每分钟一次） -func (m *Manager) printCompactionStats(version *manifest.Version, picker *Picker) { - m.mu.Lock() - defer m.mu.Unlock() - - // 限制输出频率：每 60 秒输出一次 - if time.Since(m.lastCompactionTime) < 60*time.Second { - return - } - m.lastCompactionTime = time.Now() - - fmt.Println("[Compaction] Status check:") - for level := 0; level < 7; level++ { - files := version.GetLevel(level) - if len(files) == 0 { - continue - } - - totalSize := int64(0) - for _, f := range files { - totalSize += f.FileSize - } - - score := picker.GetLevelScore(version, level) - fmt.Printf(" L%d: %d files, %.2f MB, score: %.2f\n", - level, len(files), float64(totalSize)/(1024*1024), score) - } -} - -// DoCompactionWithVersion 使用指定的版本执行 Compaction -func (m *Manager) DoCompactionWithVersion(task *CompactionTask, version *manifest.Version) error { - if version == nil { - return fmt.Errorf("version is nil") - } - - // 执行 Compaction（使用传入的 version，而不是重新获取） - edit, err := m.compactor.DoCompaction(task, version) - if err != nil { - return fmt.Errorf("compaction failed: %w", err) - } - - // 如果 edit 为 nil，说明所有文件都已经不存在，无需应用变更 - if edit == nil { - fmt.Printf("[Compaction] No changes needed (files already removed)\n") - return nil - } - - // 应用 VersionEdit - err = m.versionSet.LogAndApply(edit) - if err != nil { - // LogAndApply 失败，清理已写入的新 SST 文件（防止孤儿文件） - fmt.Printf("[Compaction] LogAndApply failed, cleaning up new files: %v\n", err) - m.cleanupNewFiles(edit) - return fmt.Errorf("apply version edit: %w", err) - } - - // LogAndApply 成功后，删除废弃的 SST 文件 - m.deleteObsoleteFiles(edit) - - // 更新统计信息 - m.mu.Lock() - m.totalCompactions++ - m.lastCompactionTime = time.Now() - m.mu.Unlock() - - return nil -} - -// DoCompaction 执行一次 Compaction（兼容旧接口） -func (m *Manager) DoCompaction(task *CompactionTask) error { - // 获取当前版本 - version := m.versionSet.GetCurrent() - if version == nil { - return fmt.Errorf("no current version") - } - - return m.DoCompactionWithVersion(task, version) -} - -// cleanupNewFiles 清理 LogAndApply 失败后的新文件（防止孤儿文件） -func (m *Manager) cleanupNewFiles(edit *manifest.VersionEdit) { - if edit == nil { - return - } - - fmt.Printf("[Compaction] Cleaning up %d new files after LogAndApply failure\n", len(edit.AddedFiles)) - - // 删除新创建的文件 - for _, file := range edit.AddedFiles { - sstPath := filepath.Join(m.sstDir, fmt.Sprintf("%06d.sst", file.FileNumber)) - err := os.Remove(sstPath) - if err != nil { - fmt.Printf("[Compaction] Failed to cleanup new file %06d.sst: %v\n", file.FileNumber, err) - } else { - fmt.Printf("[Compaction] Cleaned up new file %06d.sst\n", file.FileNumber) - } - } -} - -// deleteObsoleteFiles 删除废弃的 SST 文件 -func (m *Manager) deleteObsoleteFiles(edit *manifest.VersionEdit) { - if edit == nil { - fmt.Printf("[Compaction] deleteObsoleteFiles: edit is nil\n") - return - } - - fmt.Printf("[Compaction] deleteObsoleteFiles: %d files to delete\n", len(edit.DeletedFiles)) - - // 删除被标记为删除的文件 - for _, fileNum := range edit.DeletedFiles { - sstPath := filepath.Join(m.sstDir, fmt.Sprintf("%06d.sst", fileNum)) - err := os.Remove(sstPath) - if err != nil { - // 删除失败只记录日志，不影响 compaction 流程 - // 后台垃圾回收器会重试 - fmt.Printf("[Compaction] Failed to delete obsolete file %06d.sst: %v\n", fileNum, err) - } else { - fmt.Printf("[Compaction] Deleted obsolete file %06d.sst\n", fileNum) - } - } -} - -// TriggerCompaction 手动触发一次 Compaction（所有需要的层级） -func (m *Manager) TriggerCompaction() error { - version := m.versionSet.GetCurrent() - if version == nil { - return fmt.Errorf("no current version") - } - - picker := m.compactor.GetPicker() - tasks := picker.PickCompaction(version) - if len(tasks) == 0 { - return nil // 不需要 Compaction - } - - // 依次执行所有任务 - for _, task := range tasks { - currentVersion := m.versionSet.GetCurrent() - if err := m.DoCompactionWithVersion(task, currentVersion); err != nil { - return err - } - } - - return nil -} - -// GetStats 获取 Compaction 统计信息 -func (m *Manager) GetStats() map[string]interface{} { - m.mu.RLock() - defer m.mu.RUnlock() - - return map[string]interface{}{ - "total_compactions": m.totalCompactions, - "last_compaction_time": m.lastCompactionTime, - } -} - -// GetLevelStats 获取每层的统计信息 -func (m *Manager) GetLevelStats() []map[string]interface{} { - version := m.versionSet.GetCurrent() - if version == nil { - return nil - } - - picker := m.compactor.GetPicker() - stats := make([]map[string]interface{}, manifest.NumLevels) - - for level := 0; level < manifest.NumLevels; level++ { - files := version.GetLevel(level) - totalSize := int64(0) - for _, file := range files { - totalSize += file.FileSize - } - - stats[level] = map[string]interface{}{ - "level": level, - "file_count": len(files), - "total_size": totalSize, - "score": picker.GetLevelScore(version, level), - } - } - - return stats -} - -// backgroundGarbageCollection 后台垃圾回收循环 -func (m *Manager) backgroundGarbageCollection() { - defer m.wg.Done() - - ticker := time.NewTicker(5 * time.Minute) // 每 5 分钟检查一次 - defer ticker.Stop() - - for { - select { - case <-m.stopCh: - return - case <-ticker.C: - m.collectOrphanFiles() - } - } -} - -// collectOrphanFiles 收集并删除孤儿 SST 文件 -func (m *Manager) collectOrphanFiles() { - // 1. 获取当前版本中的所有活跃文件 - version := m.versionSet.GetCurrent() - if version == nil { - return - } - - activeFiles := make(map[int64]bool) - for level := 0; level < manifest.NumLevels; level++ { - files := version.GetLevel(level) - for _, file := range files { - activeFiles[file.FileNumber] = true - } - } - - // 2. 扫描 SST 目录中的所有文件 - pattern := filepath.Join(m.sstDir, "*.sst") - sstFiles, err := filepath.Glob(pattern) - if err != nil { - fmt.Printf("[GC] Failed to scan SST directory: %v\n", err) - return - } - - // 3. 找出孤儿文件并删除 - orphanCount := 0 - for _, sstPath := range sstFiles { - // 提取文件编号 - var fileNum int64 - _, err := fmt.Sscanf(filepath.Base(sstPath), "%d.sst", &fileNum) - if err != nil { - continue - } - - // 检查是否是活跃文件 - if !activeFiles[fileNum] { - // 这是孤儿文件，删除它 - err := os.Remove(sstPath) - if err != nil { - fmt.Printf("[GC] Failed to delete orphan file %06d.sst: %v\n", fileNum, err) - } else { - fmt.Printf("[GC] Deleted orphan file %06d.sst\n", fileNum) - orphanCount++ - } - } - } - - // 4. 更新统计信息 - m.mu.Lock() - m.lastGCTime = time.Now() - m.totalOrphansFound += int64(orphanCount) - m.mu.Unlock() - - if orphanCount > 0 { - fmt.Printf("[GC] Completed: cleaned up %d orphan files (total: %d)\n", orphanCount, m.totalOrphansFound) - } -} - -// CleanupOrphanFiles 手动触发孤儿文件清理（可在启动时调用） -func (m *Manager) CleanupOrphanFiles() { - fmt.Println("[GC] Manual cleanup triggered") - m.collectOrphanFiles() -} diff --git a/compaction/picker.go b/compaction/picker.go deleted file mode 100644 index ff3a949..0000000 --- a/compaction/picker.go +++ /dev/null @@ -1,285 +0,0 @@ -package compaction - -import ( - "fmt" - - "code.tczkiot.com/srdb/manifest" -) - -// CompactionTask 表示一个 Compaction 任务 -type CompactionTask struct { - Level int // 源层级 - InputFiles []*manifest.FileMetadata // 需要合并的输入文件 - OutputLevel int // 输出层级 -} - -// Picker 负责选择需要 Compaction 的文件 -type Picker struct { - // Level 大小限制 (字节) - levelSizeLimits [manifest.NumLevels]int64 - - // Level 文件数量限制 - levelFileLimits [manifest.NumLevels]int -} - -// NewPicker 创建新的 Compaction Picker -func NewPicker() *Picker { - p := &Picker{} - - // 设置每层的大小限制 (指数增长) - // L0: 10MB, L1: 100MB, L2: 1GB, L3: 10GB, L4: 100GB, L5: 1TB, L6: 无限制 - p.levelSizeLimits[0] = 10 * 1024 * 1024 // 10MB - p.levelSizeLimits[1] = 100 * 1024 * 1024 // 100MB - p.levelSizeLimits[2] = 1024 * 1024 * 1024 // 1GB - p.levelSizeLimits[3] = 10 * 1024 * 1024 * 1024 // 10GB - p.levelSizeLimits[4] = 100 * 1024 * 1024 * 1024 // 100GB - p.levelSizeLimits[5] = 1024 * 1024 * 1024 * 1024 // 1TB - p.levelSizeLimits[6] = 0 // 无限制 - - // 设置每层的文件数量限制 - // L0 特殊处理：文件数量限制为 4 (当有4个或更多文件时触发 compaction) - p.levelFileLimits[0] = 4 - // L1-L6: 不限制文件数量，只限制总大小 - for i := 1; i < manifest.NumLevels; i++ { - p.levelFileLimits[i] = 0 // 0 表示不限制 - } - - return p -} - -// PickCompaction 选择需要 Compaction 的任务（支持多任务并发） -// 返回空切片表示当前不需要 Compaction -func (p *Picker) PickCompaction(version *manifest.Version) []*CompactionTask { - tasks := make([]*CompactionTask, 0) - - // 1. 检查 L0 (基于文件数量) - if task := p.pickL0Compaction(version); task != nil { - tasks = append(tasks, task) - } - - // 2. 检查 L1-L5 (基于大小) - for level := 1; level < manifest.NumLevels-1; level++ { - if task := p.pickLevelCompaction(version, level); task != nil { - tasks = append(tasks, task) - } - } - - // 3. 按优先级排序（score 越高越优先） - if len(tasks) > 1 { - p.sortTasksByPriority(tasks, version) - } - - return tasks -} - -// sortTasksByPriority 按优先级对任务排序（score 从高到低） -func (p *Picker) sortTasksByPriority(tasks []*CompactionTask, version *manifest.Version) { - // 简单的冒泡排序（任务数量通常很少，< 7） - for i := 0; i < len(tasks)-1; i++ { - for j := i + 1; j < len(tasks); j++ { - scoreI := p.GetLevelScore(version, tasks[i].Level) - scoreJ := p.GetLevelScore(version, tasks[j].Level) - if scoreJ > scoreI { - tasks[i], tasks[j] = tasks[j], tasks[i] - } - } - } -} - -// pickL0Compaction 选择 L0 的 Compaction 任务 -// L0 特殊：文件可能有重叠的 key range，需要全部合并 -func (p *Picker) pickL0Compaction(version *manifest.Version) *CompactionTask { - l0Files := version.GetLevel(0) - if len(l0Files) == 0 { - return nil - } - - // 计算 L0 总大小 - totalSize := int64(0) - for _, file := range l0Files { - totalSize += file.FileSize - } - - // 检查是否需要 Compaction（同时考虑文件数量和总大小） - // 1. 文件数量超过限制（避免读放大：每次读取需要检查太多文件） - // 2. 总大小超过限制（避免 L0 占用过多空间） - needCompaction := false - if p.levelFileLimits[0] > 0 && len(l0Files) >= p.levelFileLimits[0] { - needCompaction = true - } - if p.levelSizeLimits[0] > 0 && totalSize >= p.levelSizeLimits[0] { - needCompaction = true - } - - if !needCompaction { - return nil - } - - // L0 → L1 Compaction - // 选择所有 L0 文件（因为 key range 可能重叠） - return &CompactionTask{ - Level: 0, - InputFiles: l0Files, - OutputLevel: 1, - } -} - -// pickLevelCompaction 选择 L1-L5 的 Compaction 任务 -// L1+ 的文件 key range 不重叠，可以选择多个不重叠的文件 -func (p *Picker) pickLevelCompaction(version *manifest.Version, level int) *CompactionTask { - if level < 1 || level >= manifest.NumLevels-1 { - return nil - } - - files := version.GetLevel(level) - if len(files) == 0 { - return nil - } - - // 计算当前层级的总大小 - totalSize := int64(0) - for _, file := range files { - totalSize += file.FileSize - } - - // 检查是否超过大小限制 - if totalSize < p.levelSizeLimits[level] { - return nil - } - - // 改进策略：根据层级压力动态调整选择策略 - // 1. 计算当前层级的压力（超过限制的倍数） - pressure := float64(totalSize) / float64(p.levelSizeLimits[level]) - - // 2. 根据压力确定目标大小和文件数量限制 - targetSize := p.getTargetCompactionSize(level + 1) - maxFiles := 10 // 默认最多 10 个文件 - - if pressure >= 10.0 { - // 压力极高（超过 10 倍）：选择更多文件，增大目标 - maxFiles = 100 - targetSize *= 5 - fmt.Printf("[Compaction] L%d pressure: %.1fx (CRITICAL) - selecting up to %d files, target: %s\n", - level, pressure, maxFiles, formatBytes(targetSize)) - } else if pressure >= 5.0 { - // 压力很高（超过 5 倍） - maxFiles = 50 - targetSize *= 3 - fmt.Printf("[Compaction] L%d pressure: %.1fx (HIGH) - selecting up to %d files, target: %s\n", - level, pressure, maxFiles, formatBytes(targetSize)) - } else if pressure >= 2.0 { - // 压力较高（超过 2 倍） - maxFiles = 20 - targetSize *= 2 - fmt.Printf("[Compaction] L%d pressure: %.1fx (ELEVATED) - selecting up to %d files, target: %s\n", - level, pressure, maxFiles, formatBytes(targetSize)) - } - - // 选择文件，直到累计大小接近目标 - selectedFiles := make([]*manifest.FileMetadata, 0) - currentSize := int64(0) - - for _, file := range files { - selectedFiles = append(selectedFiles, file) - currentSize += file.FileSize - - // 如果已经达到目标大小，停止选择 - if currentSize >= targetSize { - break - } - - // 达到文件数量限制 - if len(selectedFiles) >= maxFiles { - break - } - } - - return &CompactionTask{ - Level: level, - InputFiles: selectedFiles, - OutputLevel: level + 1, - } -} - -// getTargetCompactionSize 根据层级返回建议的 compaction 大小 -func (p *Picker) getTargetCompactionSize(level int) int64 { - switch level { - case 0: - return 2 * 1024 * 1024 // 2MB - case 1: - return 10 * 1024 * 1024 // 10MB - case 2: - return 50 * 1024 * 1024 // 50MB - case 3: - return 100 * 1024 * 1024 // 100MB - default: // L4+ - return 200 * 1024 * 1024 // 200MB - } -} - -// ShouldCompact 判断是否需要 Compaction -func (p *Picker) ShouldCompact(version *manifest.Version) bool { - tasks := p.PickCompaction(version) - return len(tasks) > 0 -} - -// GetLevelScore 获取每层的 Compaction 得分 (用于优先级排序) -// 得分越高，越需要 Compaction -func (p *Picker) GetLevelScore(version *manifest.Version, level int) float64 { - if level < 0 || level >= manifest.NumLevels { - return 0 - } - - files := version.GetLevel(level) - - // L0 同时考虑文件数量和总大小，取较大值作为得分 - if level == 0 { - scoreByCount := float64(0) - scoreBySize := float64(0) - - if p.levelFileLimits[0] > 0 { - scoreByCount = float64(len(files)) / float64(p.levelFileLimits[0]) - } - - if p.levelSizeLimits[0] > 0 { - totalSize := int64(0) - for _, file := range files { - totalSize += file.FileSize - } - scoreBySize = float64(totalSize) / float64(p.levelSizeLimits[0]) - } - - // 返回两者中的较大值（哪个维度更紧迫） - if scoreByCount > scoreBySize { - return scoreByCount - } - return scoreBySize - } - - // L1+ 基于总大小 - if p.levelSizeLimits[level] == 0 { - return 0 - } - - totalSize := int64(0) - for _, file := range files { - totalSize += file.FileSize - } - - return float64(totalSize) / float64(p.levelSizeLimits[level]) -} - -// formatBytes 格式化字节大小显示 -func formatBytes(bytes int64) string { - const unit = 1024 - if bytes < unit { - return fmt.Sprintf("%d B", bytes) - } - div, exp := int64(unit), 0 - for n := bytes / unit; n >= unit; n /= unit { - div *= unit - exp++ - } - units := []string{"KB", "MB", "GB", "TB"} - return fmt.Sprintf("%.2f %s", float64(bytes)/float64(div), units[exp]) -} diff --git a/compaction/compaction_test.go b/compaction_test.go similarity index 57% rename from compaction/compaction_test.go rename to compaction_test.go index 2b7f6e8..95344fc 100644 --- a/compaction/compaction_test.go +++ b/compaction_test.go @@ -1,12 +1,11 @@ -package compaction +package srdb import ( - "code.tczkiot.com/srdb/manifest" - "code.tczkiot.com/srdb/sst" "fmt" "os" "path/filepath" "testing" + "time" ) func TestCompactionBasic(t *testing.T) { @@ -21,26 +20,26 @@ func TestCompactionBasic(t *testing.T) { } // 创建 VersionSet - versionSet, err := manifest.NewVersionSet(manifestDir) + versionSet, err := NewVersionSet(manifestDir) if err != nil { t.Fatal(err) } defer versionSet.Close() // 创建 SST Manager - sstMgr, err := sst.NewManager(sstDir) + sstMgr, err := NewSSTableManager(sstDir) if err != nil { t.Fatal(err) } defer sstMgr.Close() // 创建测试数据 - rows1 := make([]*sst.Row, 100) - for i := 0; i < 100; i++ { - rows1[i] = &sst.Row{ + rows1 := make([]*SSTableRow, 100) + for i := range 100 { + rows1[i] = &SSTableRow{ Seq: int64(i), Time: 1000, - Data: map[string]interface{}{"value": i}, + Data: map[string]any{"value": i}, } } @@ -51,8 +50,8 @@ func TestCompactionBasic(t *testing.T) { } // 添加到 Version - edit1 := manifest.NewVersionEdit() - edit1.AddFile(&manifest.FileMetadata{ + edit1 := NewVersionEdit() + edit1.AddFile(&FileMetadata{ FileNumber: 1, Level: 0, FileSize: 1024, @@ -75,16 +74,16 @@ func TestCompactionBasic(t *testing.T) { } // 创建 Compaction Manager - compactionMgr := NewManager(sstDir, versionSet) + compactionMgr := NewCompactionManager(sstDir, versionSet, sstMgr) // 创建更多文件触发 Compaction for i := 1; i < 5; i++ { - rows := make([]*sst.Row, 50) - for j := 0; j < 50; j++ { - rows[j] = &sst.Row{ + rows := make([]*SSTableRow, 50) + for j := range 50 { + rows[j] = &SSTableRow{ Seq: int64(i*100 + j), Time: int64(1000 + i), - Data: map[string]interface{}{"value": i*100 + j}, + Data: map[string]any{"value": i*100 + j}, } } @@ -93,8 +92,8 @@ func TestCompactionBasic(t *testing.T) { t.Fatal(err) } - edit := manifest.NewVersionEdit() - edit.AddFile(&manifest.FileMetadata{ + edit := NewVersionEdit() + edit.AddFile(&FileMetadata{ FileNumber: int64(i + 1), Level: 0, FileSize: 512, @@ -152,7 +151,7 @@ func TestPickerLevelScore(t *testing.T) { manifestDir := tmpDir // 创建 VersionSet - versionSet, err := manifest.NewVersionSet(manifestDir) + versionSet, err := NewVersionSet(manifestDir) if err != nil { t.Fatal(err) } @@ -162,9 +161,9 @@ func TestPickerLevelScore(t *testing.T) { picker := NewPicker() // 添加一些文件到 L0 - edit := manifest.NewVersionEdit() - for i := 0; i < 3; i++ { - edit.AddFile(&manifest.FileMetadata{ + edit := NewVersionEdit() + for i := range 3 { + edit.AddFile(&FileMetadata{ FileNumber: int64(i + 1), Level: 0, FileSize: 1024 * 1024, // 1MB @@ -206,28 +205,28 @@ func TestCompactionMerge(t *testing.T) { } // 创建 VersionSet - versionSet, err := manifest.NewVersionSet(manifestDir) + versionSet, err := NewVersionSet(manifestDir) if err != nil { t.Fatal(err) } defer versionSet.Close() // 创建 SST Manager - sstMgr, err := sst.NewManager(sstDir) + sstMgr, err := NewSSTableManager(sstDir) if err != nil { t.Fatal(err) } defer sstMgr.Close() // 创建两个有重叠 key 的 SST 文件 - rows1 := []*sst.Row{ - {Seq: 1, Time: 1000, Data: map[string]interface{}{"value": "old"}}, - {Seq: 2, Time: 1000, Data: map[string]interface{}{"value": "old"}}, + rows1 := []*SSTableRow{ + {Seq: 1, Time: 1000, Data: map[string]any{"value": "old"}}, + {Seq: 2, Time: 1000, Data: map[string]any{"value": "old"}}, } - rows2 := []*sst.Row{ - {Seq: 1, Time: 2000, Data: map[string]interface{}{"value": "new"}}, // 更新 - {Seq: 3, Time: 2000, Data: map[string]interface{}{"value": "new"}}, + rows2 := []*SSTableRow{ + {Seq: 1, Time: 2000, Data: map[string]any{"value": "new"}}, // 更新 + {Seq: 3, Time: 2000, Data: map[string]any{"value": "new"}}, } reader1, err := sstMgr.CreateSST(1, rows1) @@ -243,8 +242,8 @@ func TestCompactionMerge(t *testing.T) { defer reader2.Close() // 添加到 Version - edit := manifest.NewVersionEdit() - edit.AddFile(&manifest.FileMetadata{ + edit := NewVersionEdit() + edit.AddFile(&FileMetadata{ FileNumber: 1, Level: 0, FileSize: 512, @@ -252,7 +251,7 @@ func TestCompactionMerge(t *testing.T) { MaxKey: 2, RowCount: 2, }) - edit.AddFile(&manifest.FileMetadata{ + edit.AddFile(&FileMetadata{ FileNumber: 2, Level: 0, FileSize: 512, @@ -318,14 +317,14 @@ func BenchmarkCompaction(b *testing.B) { } // 创建 VersionSet - versionSet, err := manifest.NewVersionSet(manifestDir) + versionSet, err := NewVersionSet(manifestDir) if err != nil { b.Fatal(err) } defer versionSet.Close() // 创建 SST Manager - sstMgr, err := sst.NewManager(sstDir) + sstMgr, err := NewSSTableManager(sstDir) if err != nil { b.Fatal(err) } @@ -335,13 +334,13 @@ func BenchmarkCompaction(b *testing.B) { const numFiles = 5 const rowsPerFile = 1000 - for i := 0; i < numFiles; i++ { - rows := make([]*sst.Row, rowsPerFile) - for j := 0; j < rowsPerFile; j++ { - rows[j] = &sst.Row{ + for i := range numFiles { + rows := make([]*SSTableRow, rowsPerFile) + for j := range rowsPerFile { + rows[j] = &SSTableRow{ Seq: int64(i*rowsPerFile + j), Time: int64(1000 + i), - Data: map[string]interface{}{ + Data: map[string]any{ "value": fmt.Sprintf("data-%d-%d", i, j), }, } @@ -353,8 +352,8 @@ func BenchmarkCompaction(b *testing.B) { } reader.Close() - edit := manifest.NewVersionEdit() - edit.AddFile(&manifest.FileMetadata{ + edit := NewVersionEdit() + edit.AddFile(&FileMetadata{ FileNumber: int64(i + 1), Level: 0, FileSize: 10240, @@ -381,12 +380,148 @@ func BenchmarkCompaction(b *testing.B) { OutputLevel: 1, } - b.ResetTimer() - - for i := 0; i < b.N; i++ { + for b.Loop() { _, err := compactor.DoCompaction(task, version) if err != nil { b.Fatal(err) } } } + +// TestCompactionQueryOrder 测试 compaction 后查询结果的排序 +func TestCompactionQueryOrder(t *testing.T) { + // 创建临时目录 + tmpDir := t.TempDir() + + // 创建 Schema - 包含多个字段以增加数据大小 + schema := NewSchema("test", []Field{ + {Name: "id", Type: FieldTypeInt64}, + {Name: "name", Type: FieldTypeString}, + {Name: "data", Type: FieldTypeString}, + {Name: "timestamp", Type: FieldTypeInt64}, + }) + + // 打开 Engine (使用较小的 MemTable 触发频繁 flush) + engine, err := OpenEngine(&EngineOptions{ + Dir: tmpDir, + MemTableSize: 2 * 1024 * 1024, // 2MB MemTable + Schema: schema, + }) + if err != nil { + t.Fatal(err) + } + defer engine.Close() + + t.Logf("开始插入 4000 条数据...") + + // 插入 4000 条数据，每条数据大小在 2KB-1MB 之间 + for i := range 4000 { + // 生成 2KB 到 1MB 的随机数据 + dataSize := 2*1024 + (i % (1024*1024 - 2*1024)) // 2KB ~ 1MB + largeData := make([]byte, dataSize) + for j := range largeData { + largeData[j] = byte('A' + (j % 26)) + } + + err := engine.Insert(map[string]any{ + "id": int64(i), + "name": fmt.Sprintf("user_%d", i), + "data": string(largeData), + "timestamp": int64(1000000 + i), + }) + if err != nil { + t.Fatal(err) + } + + if (i+1)%500 == 0 { + t.Logf("已插入 %d 条数据", i+1) + } + } + + t.Logf("插入完成，等待后台 compaction...") + + // 等待一段时间让后台 compaction 有机会运行 + // 后台 compaction 每 10 秒检查一次，所以需要等待至少 12 秒 + time.Sleep(12 * time.Second) + + t.Logf("开始查询所有数据...") + + // 查询所有数据 + rows, err := engine.Query().Rows() + if err != nil { + t.Fatal(err) + } + defer rows.Close() + + // 验证顺序和数据完整性 + var lastSeq int64 = 0 + count := 0 + expectedIDs := make(map[int64]bool) // 用于验证所有 ID 都存在 + + for rows.Next() { + row := rows.Row() + data := row.Data() + currentSeq := data["_seq"].(int64) + + // 验证顺序 + if currentSeq <= lastSeq { + t.Errorf("Query results NOT in order: got seq %d after seq %d", currentSeq, lastSeq) + } + + // 验证数据完整性 + id, ok := data["id"].(int64) + if !ok { + // 尝试其他类型 + if idFloat, ok2 := data["id"].(float64); ok2 { + id = int64(idFloat) + expectedIDs[id] = true + } else { + t.Errorf("Seq %d: missing or invalid id field, actual type: %T, value: %v", + currentSeq, data["id"], data["id"]) + } + } else { + expectedIDs[id] = true + } + + // 验证 name 字段 + name, ok := data["name"].(string) + if !ok || name != fmt.Sprintf("user_%d", id) { + t.Errorf("Seq %d: invalid name field, expected 'user_%d', got '%v'", currentSeq, id, name) + } + + // 验证 data 字段存在且不为空 + dataStr, ok := data["data"].(string) + if !ok || len(dataStr) < 2*1024 { + t.Errorf("Seq %d: invalid data field size", currentSeq) + } + + lastSeq = currentSeq + count++ + } + + if count != 4000 { + t.Errorf("Expected 4000 rows, got %d", count) + } + + // 验证所有 ID 都存在 + for i := range int64(4000) { + if !expectedIDs[i] { + t.Errorf("Missing ID: %d", i) + } + } + + t.Logf("✓ 查询返回 %d 条记录，顺序正确 (seq 1→%d)", count, lastSeq) + t.Logf("✓ 所有数据完整性验证通过") + + // 输出 compaction 统计信息 + stats := engine.GetCompactionManager().GetLevelStats() + t.Logf("Compaction 统计:") + for _, levelStat := range stats { + level := levelStat["level"].(int) + fileCount := levelStat["file_count"].(int) + totalSize := levelStat["total_size"].(int64) + if fileCount > 0 { + t.Logf(" L%d: %d 个文件, %.2f MB", level, fileCount, float64(totalSize)/(1024*1024)) + } + } +} diff --git a/database.go b/database.go index 36a5db2..cfb8790 100644 --- a/database.go +++ b/database.go @@ -3,6 +3,7 @@ package srdb import ( "encoding/json" "fmt" + "maps" "os" "path/filepath" "sync" @@ -250,8 +251,6 @@ func (db *Database) GetAllTablesInfo() map[string]*Table { // 返回副本以避免并发问题 result := make(map[string]*Table, len(db.tables)) - for k, v := range db.tables { - result[k] = v - } + maps.Copy(result, db.tables) return result } diff --git a/engine.go b/engine.go index ccb2f2f..bfd10ef 100644 --- a/engine.go +++ b/engine.go @@ -9,12 +9,6 @@ import ( "sync" "sync/atomic" "time" - - "code.tczkiot.com/srdb/compaction" - "code.tczkiot.com/srdb/manifest" - "code.tczkiot.com/srdb/memtable" - "code.tczkiot.com/srdb/sst" - "code.tczkiot.com/srdb/wal" ) const ( @@ -26,11 +20,11 @@ type Engine struct { dir string schema *Schema indexManager *IndexManager - walManager *wal.Manager // WAL 管理器 - sstManager *sst.Manager // SST 管理器 - memtableManager *memtable.Manager // MemTable 管理器 - versionSet *manifest.VersionSet // MANIFEST 管理器 - compactionManager *compaction.Manager // Compaction 管理器 + walManager *WALManager // WAL 管理器 + sstManager *SSTableManager // SST 管理器 + memtableManager *MemTableManager // MemTable 管理器 + versionSet *VersionSet // MANIFEST 管理器 + compactionManager *CompactionManager // Compaction 管理器 seq atomic.Int64 mu sync.RWMutex flushMu sync.Mutex @@ -125,17 +119,22 @@ func OpenEngine(opts *EngineOptions) (*Engine, error) { } // 创建 SST Manager - sstMgr, err := sst.NewManager(sstDir) + sstMgr, err := NewSSTableManager(sstDir) if err != nil { return nil, err } + // 设置 Schema（用于优化编解码） + if sch != nil { + sstMgr.SetSchema(sch) + } + // 创建 MemTable Manager - memMgr := memtable.NewManager(opts.MemTableSize) + memMgr := NewMemTableManager(opts.MemTableSize) // 创建/恢复 MANIFEST manifestDir := opts.Dir - versionSet, err := manifest.NewVersionSet(manifestDir) + versionSet, err := NewVersionSet(manifestDir) if err != nil { return nil, fmt.Errorf("create version set: %w", err) } @@ -158,7 +157,7 @@ func OpenEngine(opts *EngineOptions) (*Engine, error) { } // 恢复完成后，创建 WAL Manager 用于后续写入 - walMgr, err := wal.NewManager(walDir) + walMgr, err := NewWALManager(walDir) if err != nil { return nil, err } @@ -166,7 +165,12 @@ func OpenEngine(opts *EngineOptions) (*Engine, error) { engine.memtableManager.SetActiveWAL(walMgr.GetCurrentNumber()) // 创建 Compaction Manager - engine.compactionManager = compaction.NewManager(sstDir, versionSet) + engine.compactionManager = NewCompactionManager(sstDir, versionSet, sstMgr) + + // 设置 Schema（如果有） + if sch != nil { + engine.compactionManager.SetSchema(sch) + } // 启动时清理孤儿文件（崩溃恢复后的清理） engine.compactionManager.CleanupOrphanFiles() @@ -195,7 +199,7 @@ func (e *Engine) Insert(data map[string]any) error { seq := e.seq.Add(1) // 2. 添加系统字段 - row := &sst.Row{ + row := &SSTableRow{ Seq: seq, Time: time.Now().UnixNano(), Data: data, @@ -208,8 +212,8 @@ func (e *Engine) Insert(data map[string]any) error { } // 4. 写入 WAL - entry := &wal.Entry{ - Type: wal.EntryTypePut, + entry := &WALEntry{ + Type: WALEntryTypePut, Seq: seq, Data: rowData, } @@ -235,11 +239,11 @@ func (e *Engine) Insert(data map[string]any) error { } // Get 查询数据 -func (e *Engine) Get(seq int64) (*sst.Row, error) { +func (e *Engine) Get(seq int64) (*SSTableRow, error) { // 1. 先查 MemTable Manager (Active + Immutables) data, found := e.memtableManager.Get(seq) if found { - var row sst.Row + var row SSTableRow err := json.Unmarshal(data, &row) if err != nil { return nil, err @@ -251,6 +255,35 @@ func (e *Engine) Get(seq int64) (*sst.Row, error) { return e.sstManager.Get(seq) } +// GetPartial 按需查询数据（只读取指定字段） +func (e *Engine) GetPartial(seq int64, fields []string) (*SSTableRow, error) { + // 1. 先查 MemTable Manager (Active + Immutables) + data, found := e.memtableManager.Get(seq) + if found { + var row SSTableRow + err := json.Unmarshal(data, &row) + if err != nil { + return nil, err + } + + // MemTable 中的数据已经完全解析，需要手动过滤字段 + if len(fields) > 0 { + filteredData := make(map[string]any) + for _, field := range fields { + if val, ok := row.Data[field]; ok { + filteredData[field] = val + } + } + row.Data = filteredData + } + + return &row, nil + } + + // 2. 查询 SST 文件（按需解码） + return e.sstManager.GetPartial(seq, fields) +} + // switchMemTable 切换 MemTable func (e *Engine) switchMemTable() error { e.flushMu.Lock() @@ -273,12 +306,12 @@ func (e *Engine) switchMemTable() error { } // flushImmutable 将 Immutable MemTable 刷新到 SST -func (e *Engine) flushImmutable(imm *memtable.ImmutableMemTable, walNumber int64) error { +func (e *Engine) flushImmutable(imm *ImmutableMemTable, walNumber int64) error { // 1. 收集所有行 - var rows []*sst.Row - iter := imm.MemTable.NewIterator() + var rows []*SSTableRow + iter := imm.NewIterator() for iter.Next() { - var row sst.Row + var row SSTableRow err := json.Unmarshal(iter.Value(), &row) if err == nil { rows = append(rows, &row) @@ -311,7 +344,7 @@ func (e *Engine) flushImmutable(imm *memtable.ImmutableMemTable, walNumber int64 return fmt.Errorf("stat sst file: %w", err) } - fileMeta := &manifest.FileMetadata{ + fileMeta := &FileMetadata{ FileNumber: fileNumber, Level: 0, // Flush 到 L0 FileSize: fileInfo.Size(), @@ -321,11 +354,12 @@ func (e *Engine) flushImmutable(imm *memtable.ImmutableMemTable, walNumber int64 } // 5. 更新 MANIFEST - edit := manifest.NewVersionEdit() + edit := NewVersionEdit() edit.AddFile(fileMeta) // 持久化当前的文件编号计数器（关键修复：防止重启后文件编号重用） - edit.SetNextFileNumber(e.versionSet.GetNextFileNumber()) + // 使用 fileNumber + 1 确保并发安全，避免竞态条件 + edit.SetNextFileNumber(fileNumber + 1) err = e.versionSet.LogAndApply(edit) if err != nil { @@ -338,9 +372,9 @@ func (e *Engine) flushImmutable(imm *memtable.ImmutableMemTable, walNumber int64 // 7. 从 Immutable 列表中移除 e.memtableManager.RemoveImmutable(imm) - // 8. 触发 Compaction 检查（非阻塞） - // Flush 后 L0 增加了新文件，可能需要立即触发 compaction - e.compactionManager.MaybeCompact() + // 8. Compaction 由后台线程负责，不在 flush 路径中触发 + // 避免同步 compaction 导致刚创建的文件立即被删除 + // e.compactionManager.MaybeCompact() return nil } @@ -364,7 +398,7 @@ func (e *Engine) recover() error { // 依次读取每个 WAL for _, walPath := range walFiles { - reader, err := wal.NewReader(walPath) + reader, err := NewWALReader(walPath) if err != nil { continue } @@ -380,7 +414,7 @@ func (e *Engine) recover() error { for _, entry := range entries { // 如果定义了 Schema，验证数据 if e.schema != nil { - var row sst.Row + var row SSTableRow if err := json.Unmarshal(entry.Data, &row); err != nil { return fmt.Errorf("failed to unmarshal row during recovery (seq=%d): %w", entry.Seq, err) } @@ -445,8 +479,8 @@ func (e *Engine) Close() error { return nil } -// Stats 统计信息 -type Stats struct { +// TableStats 统计信息 +type TableStats struct { MemTableSize int64 MemTableCount int SSTCount int @@ -454,22 +488,22 @@ type Stats struct { } // GetVersionSet 获取 VersionSet（用于高级操作） -func (e *Engine) GetVersionSet() *manifest.VersionSet { +func (e *Engine) GetVersionSet() *VersionSet { return e.versionSet } // GetCompactionManager 获取 Compaction Manager（用于高级操作） -func (e *Engine) GetCompactionManager() *compaction.Manager { +func (e *Engine) GetCompactionManager() *CompactionManager { return e.compactionManager } // GetMemtableManager 获取 Memtable Manager -func (e *Engine) GetMemtableManager() *memtable.Manager { +func (e *Engine) GetMemtableManager() *MemTableManager { return e.memtableManager } // GetSSTManager 获取 SST Manager -func (e *Engine) GetSSTManager() *sst.Manager { +func (e *Engine) GetSSTManager() *SSTableManager { return e.sstManager } @@ -484,11 +518,11 @@ func (e *Engine) GetSchema() *Schema { } // Stats 获取统计信息 -func (e *Engine) Stats() *Stats { +func (e *Engine) Stats() *TableStats { memStats := e.memtableManager.GetStats() sstStats := e.sstManager.GetStats() - stats := &Stats{ + stats := &TableStats{ MemTableSize: memStats.TotalSize, MemTableCount: memStats.TotalCount, SSTCount: sstStats.FileCount, @@ -552,9 +586,9 @@ func (e *Engine) Query() *QueryBuilder { } // scanAllWithBuilder 使用 QueryBuilder 全表扫描 -func (e *Engine) scanAllWithBuilder(qb *QueryBuilder) ([]*sst.Row, error) { +func (e *Engine) scanAllWithBuilder(qb *QueryBuilder) ([]*SSTableRow, error) { // 使用 map 去重（同一个 seq 只保留一次） - rowMap := make(map[int64]*sst.Row) + rowMap := make(map[int64]*SSTableRow) // 扫描 Active MemTable iter := e.memtableManager.NewIterator() @@ -569,7 +603,7 @@ func (e *Engine) scanAllWithBuilder(qb *QueryBuilder) ([]*sst.Row, error) { // 扫描 Immutable MemTables immutables := e.memtableManager.GetImmutables() for _, imm := range immutables { - iter := imm.MemTable.NewIterator() + iter := imm.NewIterator() for iter.Next() { seq := iter.Key() if _, exists := rowMap[seq]; !exists { @@ -595,12 +629,17 @@ func (e *Engine) scanAllWithBuilder(qb *QueryBuilder) ([]*sst.Row, error) { } } - // 转换为数组 - results := make([]*sst.Row, 0, len(rowMap)) + // 转换为数组并按 Seq 排序 + results := make([]*SSTableRow, 0, len(rowMap)) for _, row := range rowMap { results = append(results, row) } + // 按 Seq 排序（保证查询结果有序） + sort.Slice(results, func(i, j int) bool { + return results[i].Seq < results[j].Seq + }) + return results, nil } diff --git a/engine_test.go b/engine_test.go index af238c6..9b9a3fa 100644 --- a/engine_test.go +++ b/engine_test.go @@ -28,7 +28,7 @@ func TestEngine(t *testing.T) { // 2. 插入数据 for i := 1; i <= 100; i++ { - data := map[string]interface{}{ + data := map[string]any{ "name": fmt.Sprintf("user_%d", i), "age": 20 + i%50, } @@ -138,7 +138,7 @@ func TestEngineFlush(t *testing.T) { // 插入足够多的数据触发 Flush for i := 1; i <= 200; i++ { - data := map[string]interface{}{ + data := map[string]any{ "data": fmt.Sprintf("value_%d", i), } engine.Insert(data) @@ -177,12 +177,11 @@ func BenchmarkEngineInsert(b *testing.B) { }) defer engine.Close() - data := map[string]interface{}{ + data := map[string]any{ "value": 123, } - b.ResetTimer() - for i := 0; i < b.N; i++ { + for b.Loop() { engine.Insert(data) } } @@ -200,14 +199,13 @@ func BenchmarkEngineGet(b *testing.B) { // 预先插入数据 for i := 1; i <= 10000; i++ { - data := map[string]interface{}{ + data := map[string]any{ "value": i, } engine.Insert(data) } - b.ResetTimer() - for i := 0; i < b.N; i++ { + for i := 0; b.Loop(); i++ { key := int64(i%10000 + 1) engine.Get(key) } @@ -245,18 +243,18 @@ func TestHighConcurrencyWrite(t *testing.T) { startTime := time.Now() // 启动多个并发写入 goroutine - for i := 0; i < numGoroutines; i++ { + for i := range numGoroutines { wg.Add(1) go func(workerID int) { defer wg.Done() - for j := 0; j < rowsPerWorker; j++ { + for j := range rowsPerWorker { // 生成随机大小的数据 (2KB - 5MB) dataSize := minDataSize + (j % (maxDataSize - minDataSize)) largeData := make([]byte, dataSize) rand.Read(largeData) - data := map[string]interface{}{ + data := map[string]any{ "worker_id": workerID, "row_index": j, "data_size": dataSize, @@ -352,7 +350,7 @@ func TestConcurrentReadWrite(t *testing.T) { ) // 启动写入 goroutines - for i := 0; i < numWriters; i++ { + for i := range numWriters { wg.Add(1) go func(writerID int) { defer wg.Done() @@ -365,7 +363,7 @@ func TestConcurrentReadWrite(t *testing.T) { data := make([]byte, dataSize) rand.Read(data) - payload := map[string]interface{}{ + payload := map[string]any{ "writer_id": writerID, "data": data, "timestamp": time.Now().UnixNano(), @@ -383,7 +381,7 @@ func TestConcurrentReadWrite(t *testing.T) { } // 启动读取 goroutines - for i := 0; i < numReaders; i++ { + for i := range numReaders { wg.Add(1) go func(readerID int) { defer wg.Done() @@ -581,11 +579,11 @@ func TestCrashDuringCompaction(t *testing.T) { const numRows = 500 dataSize := 5 * 1024 // 5KB - for i := 0; i < numRows; i++ { + for i := range numRows { data := make([]byte, dataSize) rand.Read(data) - payload := map[string]interface{}{ + payload := map[string]any{ "index": i, "data": data, } @@ -825,8 +823,8 @@ func TestEngineWithCompaction(t *testing.T) { const rowsPerBatch = 100 for batch := range numBatches { - for i := 0; i < rowsPerBatch; i++ { - data := map[string]interface{}{ + for i := range rowsPerBatch { + data := map[string]any{ "batch": batch, "index": i, "value": fmt.Sprintf("data-%d-%d", batch, i), @@ -906,8 +904,8 @@ func TestEngineWithCompaction(t *testing.T) { t.Logf("Engine stats: %d rows, %d SST files", stats.TotalRows, stats.SSTCount) // 读取一些数据验证 - for batch := 0; batch < 3; batch++ { - for i := 0; i < 10; i++ { + for batch := range 3 { + for i := range 10 { seq := int64(batch*rowsPerBatch + i + 1) row, err := engine.Get(seq) if err != nil { @@ -942,9 +940,9 @@ func TestEngineCompactionMerge(t *testing.T) { const rowsPerBatch = 50 totalRows := 0 - for batch := 0; batch < numBatches; batch++ { - for i := 0; i < rowsPerBatch; i++ { - data := map[string]interface{}{ + for batch := range numBatches { + for i := range rowsPerBatch { + data := map[string]any{ "batch": batch, "index": i, "value": fmt.Sprintf("v%d-%d", batch, i), @@ -990,8 +988,8 @@ func TestEngineCompactionMerge(t *testing.T) { } // 验证数据完整性 - 检查前几条记录 - for batch := 0; batch < 2; batch++ { - for i := 0; i < 5; i++ { + for batch := range 2 { + for i := range 5 { seq := int64(batch*rowsPerBatch + i + 1) row, err := engine.Get(seq) if err != nil { @@ -1038,9 +1036,9 @@ func TestEngineBackgroundCompaction(t *testing.T) { const numBatches = 8 const rowsPerBatch = 50 - for batch := 0; batch < numBatches; batch++ { - for i := 0; i < rowsPerBatch; i++ { - data := map[string]interface{}{ + for batch := range numBatches { + for i := range rowsPerBatch { + data := map[string]any{ "batch": batch, "index": i, } @@ -1114,10 +1112,8 @@ func BenchmarkEngineWithCompaction(b *testing.B) { } defer engine.Close() - b.ResetTimer() - - for i := 0; i < b.N; i++ { - data := map[string]interface{}{ + for i := 0; b.Loop(); i++ { + data := map[string]any{ "index": i, "value": fmt.Sprintf("benchmark-data-%d", i), } @@ -1168,7 +1164,7 @@ func TestEngineSchemaRecover(t *testing.T) { // 插入符合 Schema 的数据 for i := 1; i <= 50; i++ { - data := map[string]interface{}{ + data := map[string]any{ "name": fmt.Sprintf("user_%d", i), "age": 20 + i%50, "email": fmt.Sprintf("user%d@example.com", i), @@ -1233,7 +1229,7 @@ func TestEngineSchemaRecoverInvalid(t *testing.T) { // 插入一些不符合后续 Schema 的数据 for i := 1; i <= 10; i++ { - data := map[string]interface{}{ + data := map[string]any{ "name": fmt.Sprintf("user_%d", i), "age": "invalid_age", // 这是字符串，但后续 Schema 要求 int64 } @@ -1307,7 +1303,7 @@ func TestEngineAutoRecoverSchema(t *testing.T) { // 插入数据 for i := 1; i <= 10; i++ { - data := map[string]interface{}{ + data := map[string]any{ "name": fmt.Sprintf("user_%d", i), "age": 20 + i, } @@ -1353,7 +1349,7 @@ func TestEngineAutoRecoverSchema(t *testing.T) { } // 尝试插入新数据（应该符合恢复的 Schema） - err = engine2.Insert(map[string]interface{}{ + err = engine2.Insert(map[string]any{ "name": "new_user", "age": 30, }) @@ -1362,7 +1358,7 @@ func TestEngineAutoRecoverSchema(t *testing.T) { } // 尝试插入不符合 Schema 的数据（应该失败） - err = engine2.Insert(map[string]interface{}{ + err = engine2.Insert(map[string]any{ "name": "bad_user", "age": "invalid", // 类型错误 }) diff --git a/examples/webui/README.md b/examples/webui/README.md index 8fcf69d..40eb10f 100644 --- a/examples/webui/README.md +++ b/examples/webui/README.md @@ -1,254 +1,413 @@ -# SRDB Web UI Example +# SRDB WebUI - 数据库管理工具 -这个示例展示了如何使用 SRDB 的内置 Web UI 来可视化查看数据库中的表和数据。 +一个功能强大的 SRDB 数据库管理工具，集成了现代化的 Web 界面和实用的命令行工具。 -## 功能特性 +## 📋 目录 -- 📊 **表列表展示** - 左侧显示所有表及其行数 -- 🔍 **Schema 查看** - 点击箭头展开查看表的字段定义 -- 📋 **数据分页浏览** - 右侧以表格形式展示数据，支持分页 -- 🎨 **响应式设计** - 现代化的界面设计 -- ⚡ **零构建** - 使用 HTMX 从 CDN 加载，无需构建步骤 -- 💾 **大数据优化** - 自动截断显示，悬停查看，点击弹窗查看完整内容 -- 📏 **数据大小显示** - 超过 1KB 的单元格自动显示大小标签 -- 🔄 **后台数据插入** - 自动生成 2KB~512KB 的测试数据（每秒一条） +- [功能特性](#功能特性) +- [快速开始](#快速开始) +- [Web UI 使用指南](#web-ui-使用指南) +- [命令行工具](#命令行工具) +- [技术架构](#技术架构) +- [开发说明](#开发说明) -## 运行示例 +--- + +## 🎯 功能特性 + +### Web UI + +#### 📊 数据管理 +- **表列表** - 查看所有表及其 Schema 信息 +- **数据浏览** - 分页浏览表数据，支持自定义列选择 +- **列持久化** - 自动保存列选择偏好到 localStorage +- **数据详情** - 点击查看完整的行数据（JSON 格式） +- **智能截断** - 长字符串自动截断，点击查看完整内容 +- **时间格式化** - 自动格式化 `_time` 字段为可读时间 + +#### 🌳 LSM-Tree 管理 +- **Manifest 视图** - 可视化 LSM-Tree 层级结构 +- **文件详情** - 查看每层的 SST 文件信息 +- **Compaction 监控** - 实时查看 Compaction Score 和统计 +- **层级折叠** - 可展开/收起查看文件详情 + +#### 🎨 用户体验 +- **响应式设计** - 完美适配桌面和移动设备 +- **深色/浅色主题** - 支持主题切换 +- **实时刷新** - 一键刷新当前视图数据 +- **移动端优化** - 侧边栏抽屉式导航 + +### 命令行工具 + +提供多个实用的数据库诊断和管理工具： + +| 命令 | 功能 | 说明 | +|------|------|------| +| `serve` | Web UI 服务器 | 启动 Web 管理界面 | +| `check-data` | 数据检查 | 检查表数据完整性 | +| `check-seq` | 序列号检查 | 验证特定序列号的数据 | +| `dump-manifest` | Manifest 导出 | 导出 LSM-Tree 结构信息 | +| `inspect-sst` | SST 文件检查 | 检查单个 SST 文件 | +| `inspect-all-sst` | 批量 SST 检查 | 检查所有 SST 文件 | +| `test-fix` | 修复测试 | 测试数据修复功能 | +| `test-keys` | 键测试 | 测试键的存在性 | + +--- + +## 🚀 快速开始 + +### 1. 启动 Web UI ```bash -# 进入示例目录 cd examples/webui -# 运行 -go run main.go +# 使用默认配置（数据库：./data，端口：8080） +go run main.go serve + +# 自定义配置 +go run main.go serve --db /path/to/database --port 3000 + +# 启用自动数据插入（用于演示） +go run main.go serve --auto-insert ``` -程序会： -1. 创建/打开数据库目录 `./data` -2. 创建三个示例表：`users`、`products` 和 `logs` -3. 插入初始示例数据 -4. **启动后台协程** - 每秒向 `logs` 表插入一条 2KB~512KB 的随机数据 -5. 启动 Web 服务器在 `http://localhost:8080` +### 2. 访问 Web UI -## 使用界面 +打开浏览器访问：http://localhost:8080 -打开浏览器访问 `http://localhost:8080`，你将看到： - -### 左侧边栏 -- 显示所有表的列表 -- 显示每个表的字段数量 -- 点击 ▶ 图标展开查看字段信息 -- 点击表名选择要查看的表（蓝色高亮显示当前选中） - -### 右侧主区域 -- **Schema 区域**：显示表结构和字段定义 -- **Data 区域**：以表格形式显示数据 - - 支持分页浏览（每页 20 条） - - 显示系统字段（_seq, _time）和用户字段 - - **自动截断长数据**：超过 400px 的内容显示省略号 - - **鼠标悬停**：悬停在单元格上查看完整内容 - - **点击查看**：点击单元格在弹窗中查看完整内容 - - **大小指示**：超过 1KB 的数据显示大小标签 - -### 大数据查看 -1. **表格截断**：单元格最大宽度 400px，超长显示 `...` -2. **悬停展开**：鼠标悬停自动展开，黄色背景高亮 -3. **模态框**：点击单元格弹出窗口 - - 等宽字体显示（适合查看十六进制数据） - - 显示数据大小 - - 支持滚动查看超长内容 - -## API 端点 - -Web UI 提供了以下 HTTP API： - -### 获取所有表 -``` -GET /api/tables -``` - -返回示例： -```json -[ - { - "name": "users", - "rowCount": 5, - "dir": "./data/users" - } -] -``` - -### 获取表的 Schema -``` -GET /api/tables/{name}/schema -``` - -返回示例： -```json -{ - "fields": [ - {"name": "name", "type": "string", "required": true}, - {"name": "email", "type": "string", "required": true}, - {"name": "age", "type": "int", "required": false} - ] -} -``` - -### 获取表数据（分页） -``` -GET /api/tables/{name}/data?page=1&pageSize=20 -``` - -参数： -- `page` - 页码，从 1 开始（默认：1） -- `pageSize` - 每页行数，最大 100（默认：20） - -返回示例： -```json -{ - "page": 1, - "pageSize": 20, - "totalRows": 5, - "totalPages": 1, - "rows": [ - { - "_seq": 1, - "_time": 1234567890, - "name": "Alice", - "email": "alice@example.com", - "age": 30 - } - ] -} -``` - -### 获取表基本信息 -``` -GET /api/tables/{name} -``` - -## 在你的应用中使用 - -你可以在自己的应用中轻松集成 Web UI： - -```go -package main - -import ( - "net/http" - "code.tczkiot.com/srdb" -) - -func main() { - // 打开数据库 - db, _ := database.Open("./mydb") - defer db.Close() - - // 获取 HTTP Handler - handler := db.WebUI() - - // 启动服务器 - http.ListenAndServe(":8080", handler) -} -``` - -或者将其作为现有 Web 应用的一部分： - -```go -mux := http.NewServeMux() - -// 你的其他路由 -mux.HandleFunc("/api/myapp", myHandler) - -// 挂载 SRDB Web UI 到 /admin/db 路径 -mux.Handle("/admin/db/", http.StripPrefix("/admin/db", db.WebUI())) - -http.ListenAndServe(":8080", mux) -``` - -## 技术栈 - -- **后端**: Go + 标准库 `net/http` -- **前端**: [HTMX](https://htmx.org/) + 原生 JavaScript + CSS -- **渲染**: 服务端 HTML 渲染（Go 模板生成） -- **字体**: Google Fonts (Inter) -- **无构建**: 直接从 CDN 加载 HTMX，无需 npm、webpack 等工具 -- **部署**: 所有静态资源通过 `embed.FS` 嵌入到二进制文件中 - -## 测试大数据 - -### logs 表自动生成 - -程序会在后台持续向 `logs` 表插入大数据： - -- **频率**：每秒一条 -- **大小**：2KB ~ 512KB 随机 -- **格式**：十六进制字符串 -- **字段**： - - `timestamp` - 插入时间 - - `data` - 随机数据（十六进制） - - `size_bytes` - 数据大小（字节） - -你可以选择 `logs` 表来测试大数据的显示效果： -1. 单元格会显示数据大小标签（如 `245.12 KB`） -2. 内容被自动截断，显示省略号 -3. 点击单元格在弹窗中查看完整数据 - -终端会实时输出插入日志： -``` -Inserted record #1, size: 245.12 KB -Inserted record #2, size: 128.50 KB -Inserted record #3, size: 487.23 KB -``` - -## 注意事项 - -- Web UI 是只读的，不提供数据修改功能 -- 适合用于开发、调试和数据查看 -- 生产环境建议添加身份验证和访问控制 -- 大数据量表的分页查询性能取决于数据分布 -- `logs` 表会持续增长，可手动删除 `./data/logs` 目录重置 - -## Compaction 状态 - -由于后台持续插入大数据，会产生大量 SST 文件。SRDB 会自动运行 compaction 合并这些文件。 - -### 检查 Compaction 状态 +### 3. 命令行工具示例 ```bash -# 查看 SST 文件分布 -./check_sst.sh +# 检查表数据 +go run main.go check-data --db ./data --table users -# 观察 webui 日志中的 [Compaction] 信息 +# 检查特定序列号 +go run main.go check-seq --db ./data --table users --seq 123 + +# 导出 Manifest +go run main.go dump-manifest --db ./data --table users + +# 检查 SST 文件 +go run main.go inspect-sst --db ./data --table users --file 000001.sst ``` -### Compaction 改进 +--- -- **触发阈值**: L0 文件数量 ≥2 就触发（之前是 4） -- **运行频率**: 每 10 秒自动检查 -- **日志增强**: 显示详细的 compaction 状态和统计 +## 📖 Web UI 使用指南 -详细说明请查看 [COMPACTION.md](./COMPACTION.md) +### 界面布局 -## 常见问题 - -### `invalid header` 错误 - -如果看到类似错误： ``` -failed to open table logs: invalid header +┌─────────────────────────────────────────────────┐ +│ SRDB Tables [🌙/☀️] │ ← 侧边栏 +│ ├─ users │ +│ ├─ orders │ +│ └─ logs │ +├─────────────────────────────────────────────────┤ +│ [☰] users [🔄 Refresh] │ ← 页头 +│ [Data] [Manifest / LSM-Tree] │ ← 视图切换 +├─────────────────────────────────────────────────┤ +│ │ +│ Schema (点击字段卡片选择要显示的列) │ ← Schema 区域 +│ ┌──────────┬──────────┬──────────┐ │ +│ │⚡ id │● name │⚡ email │ │ +│ │[int64] │[string] │[string] │ │ +│ └──────────┴──────────┴──────────┘ │ +│ │ +│ Data (1,234 rows) │ ← 数据表格 +│ ┌─────┬──────┬───────────┬─────────┐ │ +│ │ _seq│ name │ email │ Actions │ │ +│ ├─────┼──────┼───────────┼─────────┤ │ +│ │ 1 │ John │ john@... │ Detail │ │ +│ │ 2 │ Jane │ jane@... │ Detail │ │ +│ └─────┴──────┴───────────┴─────────┘ │ +│ │ +├─────────────────────────────────────────────────┤ +│ [10/page] [Previous] Page 1 of 5 [Go] [Next] │ ← 分页控件 +└─────────────────────────────────────────────────┘ ``` -**快速修复**： +### 功能说明 + +#### 1. 表列表（侧边栏） +- 显示所有表及其字段信息 +- 点击表名切换到该表 +- 展开/收起查看字段详情 +- 字段图标：⚡ = 已索引，● = 未索引 + +#### 2. Data 视图 +- **Schema 区域**：点击字段卡片选择要显示的列 +- **数据表格**：显示选中的列数据 +- **系统字段**： + - `_seq`：序列号（第一列） + - `_time`：时间戳（倒数第二列，自动格式化） +- **Detail 按钮**：查看完整的行数据（JSON 格式） +- **分页控件**： + - 每页大小：10/20/50/100 + - 上一页/下一页 + - 跳转到指定页 + +#### 3. Manifest 视图 +- **统计卡片**： + - Active Levels：活跃层数 + - Total Files：总文件数 + - Total Size：总大小 + - Compactions：Compaction 次数 +- **层级卡片**： + - 点击展开/收起查看文件列表 + - Score 指示器： + - 🟢 绿色：健康（< 50%） + - 🟡 黄色：警告（50-80%） + - 🔴 红色：需要 Compaction（≥ 80%） +- **文件详情**： + - 文件编号、大小、行数 + - Seq 范围（min_key - max_key） + +#### 4. 刷新按钮 +- 点击刷新当前视图的数据 +- Data 视图：重新加载表数据 +- Manifest 视图：重新加载 LSM-Tree 结构 + +#### 5. 主题切换 +- 点击右上角的 🌙/☀️ 图标 +- 切换深色/浅色主题 +- 自动保存到 localStorage + +--- + +## 🛠️ 命令行工具 + +### serve - Web UI 服务器 + +启动 Web 管理界面。 + ```bash -./fix_corrupted_table.sh logs +go run main.go serve [flags] ``` -详见：[QUICK_FIX.md](./QUICK_FIX.md) 或 [TROUBLESHOOTING.md](./TROUBLESHOOTING.md) +**参数**： +- `--db` - 数据库目录（默认：`./data`） +- `--port` - 服务端口（默认：`8080`） +- `--auto-insert` - 启用自动数据插入（用于演示） -## 更多信息 +**示例**： +```bash +# 基本使用 +go run main.go serve -- [FEATURES.md](./FEATURES.md) - 详细功能说明 -- [COMPACTION.md](./COMPACTION.md) - Compaction 机制和诊断 -- [TROUBLESHOOTING.md](./TROUBLESHOOTING.md) - 故障排除指南 -- [QUICK_FIX.md](./QUICK_FIX.md) - 快速修复常见错误 +# 自定义端口 +go run main.go serve --port 3000 + +# 启用自动插入（每秒插入一条随机数据到 logs 表） +go run main.go serve --auto-insert +``` + +### check-data - 数据检查 + +检查表数据的完整性。 + +```bash +go run main.go check-data --db ./data --table +``` + +### check-seq - 序列号检查 + +验证特定序列号的数据。 + +```bash +go run main.go check-seq --db ./data --table --seq +``` + +### dump-manifest - Manifest 导出 + +导出 LSM-Tree 层级结构信息。 + +```bash +go run main.go dump-manifest --db ./data --table +``` + +### inspect-sst - SST 文件检查 + +检查单个 SST 文件的内容和元数据。 + +```bash +go run main.go inspect-sst --db ./data --table --file +``` + +### inspect-all-sst - 批量 SST 检查 + +检查表的所有 SST 文件。 + +```bash +go run main.go inspect-all-sst --db ./data --table +``` + +--- + +## 🏗️ 技术架构 + +### 前端技术栈 + +- **Lit** - 轻量级 Web Components 框架 +- **ES Modules** - 原生 JavaScript 模块 +- **CSS Variables** - 主题系统 +- **Shadow DOM** - 组件封装 + +### 组件架构 + +``` +srdb-app (主应用) +├── srdb-theme-toggle (主题切换) +├── srdb-table-list (表列表) +├── srdb-page-header (页头) +│ └── [🔄 Refresh] (刷新按钮) +├── srdb-table-view (表视图容器) +│ ├── srdb-data-view (数据视图) +│ │ ├── Schema 区域 +│ │ │ ├── srdb-field-icon (字段图标) +│ │ │ └── srdb-badge (类型标签) +│ │ └── 数据表格 +│ └── srdb-manifest-view (Manifest 视图) +│ └── srdb-badge (Score 标签) +└── srdb-modal-dialog (模态对话框) +``` + +### 后端架构 + +``` +webui.go +├── API Endpoints +│ ├── GET /api/tables - 获取表列表 +│ ├── GET /api/tables/{name}/schema - 获取表 Schema +│ ├── GET /api/tables/{name}/data - 获取表数据（分页） +│ ├── GET /api/tables/{name}/data/{seq} - 获取单条数据 +│ └── GET /api/tables/{name}/manifest - 获取 Manifest 信息 +├── Static Files +│ └── /static/* - 静态资源服务 +└── Index + └── / - 首页 +``` + +### 数据流 + +``` +用户操作 + ↓ +组件事件 (CustomEvent) + ↓ +app.js (事件总线) + ↓ +API 请求 (fetch) + ↓ +webui.go (HTTP Handler) + ↓ +SRDB Database + ↓ +JSON 响应 + ↓ +组件更新 (Lit reactive) + ↓ +UI 渲染 +``` + +--- + +## 🔧 开发说明 + +### 项目结构 + +``` +webui/ +├── commands/ +│ └── webui.go # Web UI 服务器实现 +├── static/ +│ ├── index.html # 主页面 +│ ├── css/ +│ │ └── styles.css # 全局样式 +│ └── js/ +│ ├── app.js # 应用入口和事件总线 +│ ├── components/ # Web Components +│ │ ├── app.js # 主应用容器 +│ │ ├── badge.js # 标签组件 +│ │ ├── data-view.js # 数据视图 +│ │ ├── field-icon.js # 字段图标 +│ │ ├── manifest-view.js # Manifest 视图 +│ │ ├── modal-dialog.js # 模态对话框 +│ │ ├── page-header.js # 页头 +│ │ ├── table-list.js # 表列表 +│ │ ├── table-view.js # 表视图容器 +│ │ └── theme-toggle.js # 主题切换 +│ └── styles/ +│ └── shared-styles.js # 共享样式 +└── webui.go # Web UI 后端实现 +``` + +### 添加新组件 + +1. 在 `static/js/components/` 创建组件文件 +2. 继承 `LitElement` +3. 定义 `static properties` 和 `static styles` +4. 实现 `render()` 方法 +5. 使用 `customElements.define('srdb-xxx', Component)` 注册 +6. 在 `app.js` 中导入 + +### 添加新 API + +1. 在 `webui.go` 中添加 handler 方法 +2. 在 `setupHandler()` 中注册路由 +3. 返回 JSON 格式的响应 +4. 在前端组件中调用 API + +### 样式规范 + +- 使用 CSS Variables 定义颜色和尺寸 +- 组件样式封装在 Shadow DOM 中 +- 共享样式定义在 `shared-styles.js` +- 响应式断点：768px + +### 命名规范 + +- **组件名**：`srdb-xxx`（kebab-case） +- **类名**：`ComponentName`（PascalCase） +- **文件名**：`component-name.js`（kebab-case） +- **CSS 类**：`.class-name`（kebab-case） + +--- + +## 📝 注意事项 + +### 性能优化 + +1. **列选择**：只加载选中的列，减少数据传输 +2. **字符串截断**：长字符串自动截断，按需加载完整内容 +3. **分页加载**：大表数据分页加载，避免一次性加载全部 +4. **Shadow DOM**：组件样式隔离，避免全局样式污染 + +### 浏览器兼容性 + +- Chrome 90+ +- Firefox 88+ +- Safari 14+ +- Edge 90+ + +需要支持： +- ES Modules +- Web Components +- Shadow DOM +- CSS Variables + +### 已知限制 + +1. **大数据量**：单页最多显示 1000 条数据 +2. **字符串长度**：超过 100 字符自动截断 +3. **并发限制**：同时只能查看一个表的数据 + +--- + +## 🤝 贡献 + +欢迎提交 Issue 和 Pull Request！ + +## 📄 许可证 + +MIT License diff --git a/examples/webui/commands/inspect_all_sst.go b/examples/webui/commands/inspect_all_sst.go index 84dd442..1e14902 100644 --- a/examples/webui/commands/inspect_all_sst.go +++ b/examples/webui/commands/inspect_all_sst.go @@ -9,7 +9,7 @@ import ( "strconv" "strings" - "code.tczkiot.com/srdb/sst" + "code.tczkiot.com/srdb" ) // InspectAllSST 检查所有 SST 文件 @@ -35,7 +35,7 @@ func InspectAllSST(sstDir string) { for _, filename := range sstFiles { sstPath := filepath.Join(sstDir, filename) - reader, err := sst.NewReader(sstPath) + reader, err := srdb.NewSSTableReader(sstPath) if err != nil { fmt.Printf("%s: ERROR - %v\n", filename, err) continue diff --git a/examples/webui/commands/inspect_sst.go b/examples/webui/commands/inspect_sst.go index d0c5aa6..263c59d 100644 --- a/examples/webui/commands/inspect_sst.go +++ b/examples/webui/commands/inspect_sst.go @@ -5,7 +5,7 @@ import ( "log" "os" - "code.tczkiot.com/srdb/sst" + "code.tczkiot.com/srdb" ) // InspectSST 检查特定 SST 文件 @@ -19,7 +19,7 @@ func InspectSST(sstPath string) { fmt.Printf("Size: %d bytes\n", info.Size()) // Open reader - reader, err := sst.NewReader(sstPath) + reader, err := srdb.NewSSTableReader(sstPath) if err != nil { log.Fatal(err) } diff --git a/examples/webui/commands/webui.go b/examples/webui/commands/webui.go index c77775b..e9698e3 100644 --- a/examples/webui/commands/webui.go +++ b/examples/webui/commands/webui.go @@ -24,14 +24,14 @@ func StartWebUI(dbPath string, addr string) { // 创建示例 Schema userSchema := srdb.NewSchema("users", []srdb.Field{ - {Name: "name", Type: srdb.FieldTypeString, Indexed: false, Comment: "User name"}, + {Name: "name", Type: srdb.FieldTypeString, Indexed: true, Comment: "User name"}, {Name: "email", Type: srdb.FieldTypeString, Indexed: false, Comment: "Email address"}, {Name: "age", Type: srdb.FieldTypeInt64, Indexed: false, Comment: "Age"}, {Name: "city", Type: srdb.FieldTypeString, Indexed: false, Comment: "City"}, }) productSchema := srdb.NewSchema("products", []srdb.Field{ - {Name: "product_name", Type: srdb.FieldTypeString, Indexed: false, Comment: "Product name"}, + {Name: "product_name", Type: srdb.FieldTypeString, Indexed: true, Comment: "Product name"}, {Name: "price", Type: srdb.FieldTypeFloat, Indexed: false, Comment: "Price"}, {Name: "quantity", Type: srdb.FieldTypeInt64, Indexed: false, Comment: "Quantity"}, {Name: "category", Type: srdb.FieldTypeString, Indexed: false, Comment: "Category"}, @@ -56,7 +56,7 @@ func StartWebUI(dbPath string, addr string) { log.Printf("Create users table failed: %v", err) } else { // 插入一些示例数据 - users := []map[string]interface{}{ + users := []map[string]any{ {"name": "Alice", "email": "alice@example.com", "age": 30, "city": "Beijing"}, {"name": "Bob", "email": "bob@example.com", "age": 25, "city": "Shanghai"}, {"name": "Charlie", "email": "charlie@example.com", "age": 35, "city": "Guangzhou"}, @@ -76,7 +76,7 @@ func StartWebUI(dbPath string, addr string) { log.Printf("Create products table failed: %v", err) } else { // 插入一些示例数据 - products := []map[string]interface{}{ + products := []map[string]any{ {"product_name": "Laptop", "price": 999.99, "quantity": 10, "category": "Electronics"}, {"product_name": "Mouse", "price": 29.99, "quantity": 50, "category": "Electronics"}, {"product_name": "Keyboard", "price": 79.99, "quantity": 30, "category": "Electronics"}, @@ -126,15 +126,15 @@ func autoInsertData(db *srdb.Database) { defer ticker.Stop() counter := 1 + var logsTable *srdb.Table for range ticker.C { tables := db.ListTables() - var logsTable *srdb.Table - hasLogs := slices.Contains(tables, "logs") if !hasLogs { logsSchema := srdb.NewSchema("logs", []srdb.Field{ + {Name: "group", Type: srdb.FieldTypeString, Indexed: true, Comment: "Log group (A-E)"}, {Name: "timestamp", Type: srdb.FieldTypeString, Indexed: false, Comment: "Timestamp"}, {Name: "data", Type: srdb.FieldTypeString, Indexed: false, Comment: "Random data"}, {Name: "size_bytes", Type: srdb.FieldTypeInt64, Indexed: false, Comment: "Data size in bytes"}, @@ -151,7 +151,6 @@ func autoInsertData(db *srdb.Database) { var err error logsTable, err = db.GetTable("logs") if err != nil || logsTable == nil { - log.Printf("Failed to get logs table: %v", err) continue } } @@ -159,7 +158,12 @@ func autoInsertData(db *srdb.Database) { data := generateRandomData() sizeBytes := len(data) + // 随机选择一个组 (A-E) + groups := []string{"A", "B", "C", "D", "E"} + group := groups[counter%len(groups)] + record := map[string]any{ + "group": group, "timestamp": time.Now().Format(time.RFC3339), "data": data, "size_bytes": int64(sizeBytes), @@ -170,7 +174,7 @@ func autoInsertData(db *srdb.Database) { log.Printf("Failed to insert data: %v", err) } else { sizeStr := formatBytes(sizeBytes) - log.Printf("Inserted record #%d, size: %s", counter, sizeStr) + log.Printf("Inserted record #%d, group: %s, size: %s", counter, group, sizeStr) counter++ } } diff --git a/index.go b/index.go index b3cf393..1d8db18 100644 --- a/index.go +++ b/index.go @@ -8,8 +8,6 @@ import ( "path/filepath" "sync" "time" - - "code.tczkiot.com/srdb/btree" ) // IndexMetadata 索引元数据 @@ -28,8 +26,8 @@ type SecondaryIndex struct { field string // 字段名 fieldType FieldType // 字段类型 file *os.File // 索引文件 - builder *btree.Builder // B+Tree 构建器 - reader *btree.Reader // B+Tree 读取器 + builder *BTreeBuilder // B+Tree 构建器 + reader *BTreeReader // B+Tree 读取器 valueToSeq map[string][]int64 // 值 → seq 列表 (构建时使用) metadata IndexMetadata // 元数据 mu sync.RWMutex @@ -55,7 +53,7 @@ func NewSecondaryIndex(dir, field string, fieldType FieldType) (*SecondaryIndex, } // Add 添加索引条目 -func (idx *SecondaryIndex) Add(value interface{}, seq int64) error { +func (idx *SecondaryIndex) Add(value any, seq int64) error { idx.mu.Lock() defer idx.mu.Unlock() @@ -199,7 +197,7 @@ func (idx *SecondaryIndex) load() error { } // Get 查询索引 -func (idx *SecondaryIndex) Get(value interface{}) ([]int64, error) { +func (idx *SecondaryIndex) Get(value any) ([]int64, error) { idx.mu.RLock() defer idx.mu.RUnlock() @@ -238,7 +236,7 @@ func (idx *SecondaryIndex) NeedsUpdate(currentMaxSeq int64) bool { } // IncrementalUpdate 增量更新索引 -func (idx *SecondaryIndex) IncrementalUpdate(getData func(int64) (map[string]interface{}, error), fromSeq, toSeq int64) error { +func (idx *SecondaryIndex) IncrementalUpdate(getData func(int64) (map[string]any, error), fromSeq, toSeq int64) error { idx.mu.Lock() defer idx.mu.Unlock() @@ -286,7 +284,7 @@ func encodeSeqList(seqs []int64) []byte { func decodeSeqList(data []byte) []int64 { count := len(data) / 8 seqs := make([]int64, count) - for i := 0; i < count; i++ { + for i := range count { seqs[i] = int64(binary.LittleEndian.Uint64(data[i*8:])) } return seqs @@ -433,7 +431,7 @@ func (m *IndexManager) GetIndex(field string) (*SecondaryIndex, bool) { } // AddToIndexes 添加到所有索引 -func (m *IndexManager) AddToIndexes(data map[string]interface{}, seq int64) error { +func (m *IndexManager) AddToIndexes(data map[string]any, seq int64) error { m.mu.RLock() defer m.mu.RUnlock() @@ -477,7 +475,7 @@ func (m *IndexManager) ListIndexes() []string { } // VerifyAndRepair 验证并修复所有索引 -func (m *IndexManager) VerifyAndRepair(currentMaxSeq int64, getData func(int64) (map[string]interface{}, error)) error { +func (m *IndexManager) VerifyAndRepair(currentMaxSeq int64, getData func(int64) (map[string]any, error)) error { m.mu.RLock() indexes := make(map[string]*SecondaryIndex) for k, v := range m.indexes { diff --git a/index_test.go b/index_test.go index a4f5484..20cfa83 100644 --- a/index_test.go +++ b/index_test.go @@ -89,13 +89,13 @@ func TestIncrementalUpdate(t *testing.T) { t.Logf("Initial: MaxSeq=%d, RowCount=%d", initialMetadata.MaxSeq, initialMetadata.RowCount) // 2. 模拟新数据 - mockData := map[int64]map[string]interface{}{ + mockData := map[int64]map[string]any{ 3: {"name": "Charlie"}, 4: {"name": "David"}, 5: {"name": "Alice"}, } - getData := func(seq int64) (map[string]interface{}, error) { + getData := func(seq int64) (map[string]any, error) { if data, exists := mockData[seq]; exists { return data, nil } diff --git a/manifest/manifest_reader.go b/manifest/manifest_reader.go deleted file mode 100644 index e40da5b..0000000 --- a/manifest/manifest_reader.go +++ /dev/null @@ -1,48 +0,0 @@ -package manifest - -import ( - "encoding/binary" - "io" -) - -// Reader MANIFEST 读取器 -type Reader struct { - file io.Reader -} - -// NewReader 创建 MANIFEST 读取器 -func NewReader(file io.Reader) *Reader { - return &Reader{ - file: file, - } -} - -// ReadEdit 读取版本变更 -func (r *Reader) ReadEdit() (*VersionEdit, error) { - // 读取 CRC32 和 Length - header := make([]byte, 8) - _, err := io.ReadFull(r.file, header) - if err != nil { - return nil, err - } - - // 读取长度 - length := binary.LittleEndian.Uint32(header[4:8]) - - // 读取数据 - data := make([]byte, 8+length) - copy(data[0:8], header) - _, err = io.ReadFull(r.file, data[8:]) - if err != nil { - return nil, err - } - - // 解码 - edit := NewVersionEdit() - err = edit.Decode(data) - if err != nil { - return nil, err - } - - return edit, nil -} diff --git a/manifest/manifest_writer.go b/manifest/manifest_writer.go deleted file mode 100644 index 6cf0e6b..0000000 --- a/manifest/manifest_writer.go +++ /dev/null @@ -1,35 +0,0 @@ -package manifest - -import ( - "io" - "sync" -) - -// Writer MANIFEST 写入器 -type Writer struct { - file io.Writer - mu sync.Mutex -} - -// NewWriter 创建 MANIFEST 写入器 -func NewWriter(file io.Writer) *Writer { - return &Writer{ - file: file, - } -} - -// WriteEdit 写入版本变更 -func (w *Writer) WriteEdit(edit *VersionEdit) error { - w.mu.Lock() - defer w.mu.Unlock() - - // 编码 - data, err := edit.Encode() - if err != nil { - return err - } - - // 写入 - _, err = w.file.Write(data) - return err -} diff --git a/manifest/version.go b/manifest/version.go deleted file mode 100644 index 83518f8..0000000 --- a/manifest/version.go +++ /dev/null @@ -1,187 +0,0 @@ -package manifest - -import ( - "fmt" - "sync" -) - -// FileMetadata SST 文件元数据 -type FileMetadata struct { - FileNumber int64 // 文件编号 - Level int // 所在层级 (0-6) - FileSize int64 // 文件大小 - MinKey int64 // 最小 key - MaxKey int64 // 最大 key - RowCount int64 // 行数 -} - -const ( - NumLevels = 7 // L0-L6 -) - -// Version 数据库的一个版本快照 -type Version struct { - // 分层存储 SST 文件 (L0-L6) - Levels [NumLevels][]*FileMetadata - - // 下一个文件编号 - NextFileNumber int64 - - // 最后序列号 - LastSequence int64 - - // 版本号 - VersionNumber int64 - - mu sync.RWMutex -} - -// NewVersion 创建新版本 -func NewVersion() *Version { - v := &Version{ - NextFileNumber: 1, - LastSequence: 0, - VersionNumber: 0, - } - // 初始化每一层 - for i := 0; i < NumLevels; i++ { - v.Levels[i] = make([]*FileMetadata, 0) - } - return v -} - -// Clone 克隆版本 -func (v *Version) Clone() *Version { - v.mu.RLock() - defer v.mu.RUnlock() - - newVersion := &Version{ - NextFileNumber: v.NextFileNumber, - LastSequence: v.LastSequence, - VersionNumber: v.VersionNumber + 1, - } - - // 克隆每一层 - for level := 0; level < NumLevels; level++ { - newVersion.Levels[level] = make([]*FileMetadata, len(v.Levels[level])) - copy(newVersion.Levels[level], v.Levels[level]) - } - - return newVersion -} - -// Apply 应用版本变更 -func (v *Version) Apply(edit *VersionEdit) { - v.mu.Lock() - defer v.mu.Unlock() - - // 删除文件（按层级删除） - if len(edit.DeletedFiles) > 0 { - deleteSet := make(map[int64]bool) - for _, fileNum := range edit.DeletedFiles { - deleteSet[fileNum] = true - } - - // 遍历每一层，删除文件 - for level := 0; level < NumLevels; level++ { - newFiles := make([]*FileMetadata, 0) - deletedCount := 0 - for _, file := range v.Levels[level] { - if !deleteSet[file.FileNumber] { - newFiles = append(newFiles, file) - } else { - deletedCount++ - } - } - if deletedCount > 0 { - fmt.Printf("[Version.Apply] L%d: deleted %d files\n", level, deletedCount) - } - v.Levels[level] = newFiles - } - } - - // 添加文件（按层级添加） - if len(edit.AddedFiles) > 0 { - for _, file := range edit.AddedFiles { - if file.Level >= 0 && file.Level < NumLevels { - fmt.Printf("[Version.Apply] Adding file #%d to L%d (keys %d-%d)\n", - file.FileNumber, file.Level, file.MinKey, file.MaxKey) - v.Levels[file.Level] = append(v.Levels[file.Level], file) - } - } - } - - // 更新下一个文件编号 - if edit.NextFileNumber != nil { - v.NextFileNumber = *edit.NextFileNumber - } - - // 更新最后序列号 - if edit.LastSequence != nil { - v.LastSequence = *edit.LastSequence - } -} - -// GetLevel 获取指定层级的文件 -func (v *Version) GetLevel(level int) []*FileMetadata { - v.mu.RLock() - defer v.mu.RUnlock() - - if level < 0 || level >= NumLevels { - return nil - } - - files := make([]*FileMetadata, len(v.Levels[level])) - copy(files, v.Levels[level]) - return files -} - -// GetSSTFiles 获取所有 SST 文件（副本，兼容旧接口） -func (v *Version) GetSSTFiles() []*FileMetadata { - v.mu.RLock() - defer v.mu.RUnlock() - - // 收集所有层级的文件 - allFiles := make([]*FileMetadata, 0) - for level := 0; level < NumLevels; level++ { - allFiles = append(allFiles, v.Levels[level]...) - } - return allFiles -} - -// GetNextFileNumber 获取下一个文件编号 -func (v *Version) GetNextFileNumber() int64 { - v.mu.RLock() - defer v.mu.RUnlock() - return v.NextFileNumber -} - -// GetLastSequence 获取最后序列号 -func (v *Version) GetLastSequence() int64 { - v.mu.RLock() - defer v.mu.RUnlock() - return v.LastSequence -} - -// GetFileCount 获取文件数量 -func (v *Version) GetFileCount() int { - v.mu.RLock() - defer v.mu.RUnlock() - - total := 0 - for level := 0; level < NumLevels; level++ { - total += len(v.Levels[level]) - } - return total -} - -// GetLevelFileCount 获取指定层级的文件数量 -func (v *Version) GetLevelFileCount(level int) int { - v.mu.RLock() - defer v.mu.RUnlock() - - if level < 0 || level >= NumLevels { - return 0 - } - return len(v.Levels[level]) -} diff --git a/manifest/version_edit.go b/manifest/version_edit.go deleted file mode 100644 index 0fdcbe2..0000000 --- a/manifest/version_edit.go +++ /dev/null @@ -1,114 +0,0 @@ -package manifest - -import ( - "encoding/binary" - "encoding/json" - "hash/crc32" - "io" -) - -// EditType 变更类型 -type EditType byte - -const ( - EditTypeAddFile EditType = 1 // 添加文件 - EditTypeDeleteFile EditType = 2 // 删除文件 - EditTypeSetNextFile EditType = 3 // 设置下一个文件编号 - EditTypeSetLastSeq EditType = 4 // 设置最后序列号 -) - -// VersionEdit 版本变更记录 -type VersionEdit struct { - // 添加的文件 - AddedFiles []*FileMetadata - - // 删除的文件（文件编号列表） - DeletedFiles []int64 - - // 下一个文件编号 - NextFileNumber *int64 - - // 最后序列号 - LastSequence *int64 -} - -// NewVersionEdit 创建版本变更 -func NewVersionEdit() *VersionEdit { - return &VersionEdit{ - AddedFiles: make([]*FileMetadata, 0), - DeletedFiles: make([]int64, 0), - } -} - -// AddFile 添加文件 -func (e *VersionEdit) AddFile(file *FileMetadata) { - e.AddedFiles = append(e.AddedFiles, file) -} - -// DeleteFile 删除文件 -func (e *VersionEdit) DeleteFile(fileNumber int64) { - e.DeletedFiles = append(e.DeletedFiles, fileNumber) -} - -// SetNextFileNumber 设置下一个文件编号 -func (e *VersionEdit) SetNextFileNumber(num int64) { - e.NextFileNumber = &num -} - -// SetLastSequence 设置最后序列号 -func (e *VersionEdit) SetLastSequence(seq int64) { - e.LastSequence = &seq -} - -// Encode 编码为字节 -func (e *VersionEdit) Encode() ([]byte, error) { - // 使用 JSON 编码（简单实现） - data, err := json.Marshal(e) - if err != nil { - return nil, err - } - - // 格式: CRC32(4) + Length(4) + Data - totalLen := 8 + len(data) - buf := make([]byte, totalLen) - - // 计算 CRC32 - crc := crc32.ChecksumIEEE(data) - binary.LittleEndian.PutUint32(buf[0:4], crc) - - // 写入长度 - binary.LittleEndian.PutUint32(buf[4:8], uint32(len(data))) - - // 写入数据 - copy(buf[8:], data) - - return buf, nil -} - -// Decode 从字节解码 -func (e *VersionEdit) Decode(data []byte) error { - if len(data) < 8 { - return io.ErrUnexpectedEOF - } - - // 读取 CRC32 - crc := binary.LittleEndian.Uint32(data[0:4]) - - // 读取长度 - length := binary.LittleEndian.Uint32(data[4:8]) - - if len(data) < int(8+length) { - return io.ErrUnexpectedEOF - } - - // 读取数据 - editData := data[8 : 8+length] - - // 验证 CRC32 - if crc32.ChecksumIEEE(editData) != crc { - return io.ErrUnexpectedEOF - } - - // JSON 解码 - return json.Unmarshal(editData, e) -} diff --git a/manifest/version_set.go b/manifest/version_set.go deleted file mode 100644 index 1f528c0..0000000 --- a/manifest/version_set.go +++ /dev/null @@ -1,251 +0,0 @@ -package manifest - -import ( - "fmt" - "io" - "os" - "path/filepath" - "strings" - "sync" - "sync/atomic" -) - -// VersionSet 版本集合管理器 -type VersionSet struct { - // 当前版本 - current *Version - - // MANIFEST 文件 - manifestFile *os.File - manifestWriter *Writer - manifestNumber int64 - - // 下一个文件编号 - nextFileNumber atomic.Int64 - - // 最后序列号 - lastSequence atomic.Int64 - - // 目录 - dir string - - // 锁 - mu sync.RWMutex -} - -// NewVersionSet 创建版本集合 -func NewVersionSet(dir string) (*VersionSet, error) { - vs := &VersionSet{ - dir: dir, - } - - // 确保目录存在 - err := os.MkdirAll(dir, 0755) - if err != nil { - return nil, err - } - - // 读取 CURRENT 文件 - currentFile := filepath.Join(dir, "CURRENT") - data, err := os.ReadFile(currentFile) - - if err != nil { - // CURRENT 不存在，创建新的 MANIFEST - return vs, vs.createNewManifest() - } - - // 读取 MANIFEST 文件 - manifestName := strings.TrimSpace(string(data)) - manifestPath := filepath.Join(dir, manifestName) - - // 恢复版本信息 - version, err := vs.recoverFromManifest(manifestPath) - if err != nil { - return nil, err - } - - vs.current = version - vs.nextFileNumber.Store(version.NextFileNumber) - vs.lastSequence.Store(version.LastSequence) - - // 解析 MANIFEST 编号 - fmt.Sscanf(manifestName, "MANIFEST-%d", &vs.manifestNumber) - - // 打开 MANIFEST 用于追加 - file, err := os.OpenFile(manifestPath, os.O_APPEND|os.O_WRONLY, 0644) - if err != nil { - return nil, err - } - vs.manifestFile = file - vs.manifestWriter = NewWriter(file) - - return vs, nil -} - -// createNewManifest 创建新的 MANIFEST -func (vs *VersionSet) createNewManifest() error { - // 生成新的 MANIFEST 文件名 - vs.manifestNumber = vs.nextFileNumber.Add(1) - manifestName := fmt.Sprintf("MANIFEST-%06d", vs.manifestNumber) - manifestPath := filepath.Join(vs.dir, manifestName) - - // 创建 MANIFEST 文件 - file, err := os.Create(manifestPath) - if err != nil { - return err - } - - vs.manifestFile = file - vs.manifestWriter = NewWriter(file) - - // 创建初始版本 - vs.current = NewVersion() - - // 写入初始版本 - edit := NewVersionEdit() - nextFile := vs.manifestNumber - edit.SetNextFileNumber(nextFile) - lastSeq := int64(0) - edit.SetLastSequence(lastSeq) - - err = vs.manifestWriter.WriteEdit(edit) - if err != nil { - return err - } - - // 同步到磁盘 - err = vs.manifestFile.Sync() - if err != nil { - return err - } - - // 更新 CURRENT 文件 - return vs.updateCurrent(manifestName) -} - -// recoverFromManifest 从 MANIFEST 恢复版本 -func (vs *VersionSet) recoverFromManifest(manifestPath string) (*Version, error) { - // 打开 MANIFEST 文件 - file, err := os.Open(manifestPath) - if err != nil { - return nil, err - } - defer file.Close() - - reader := NewReader(file) - - // 创建初始版本 - version := NewVersion() - - // 读取所有 VersionEdit - for { - edit, err := reader.ReadEdit() - if err == io.EOF { - break - } - if err != nil { - return nil, err - } - - // 应用变更 - version.Apply(edit) - } - - return version, nil -} - -// updateCurrent 更新 CURRENT 文件 -func (vs *VersionSet) updateCurrent(manifestName string) error { - currentPath := filepath.Join(vs.dir, "CURRENT") - tmpPath := currentPath + ".tmp" - - // 1. 写入临时文件 - err := os.WriteFile(tmpPath, []byte(manifestName+"\n"), 0644) - if err != nil { - return err - } - - // 2. 原子性重命名 - err = os.Rename(tmpPath, currentPath) - if err != nil { - os.Remove(tmpPath) - return err - } - - return nil -} - -// LogAndApply 记录并应用版本变更 -func (vs *VersionSet) LogAndApply(edit *VersionEdit) error { - vs.mu.Lock() - defer vs.mu.Unlock() - - // 1. 创建新版本 - newVersion := vs.current.Clone() - - // 2. 应用变更 - newVersion.Apply(edit) - - // 3. 写入 MANIFEST - err := vs.manifestWriter.WriteEdit(edit) - if err != nil { - return err - } - - // 4. 同步到磁盘 - err = vs.manifestFile.Sync() - if err != nil { - return err - } - - // 5. 更新当前版本 - vs.current = newVersion - - // 6. 更新原子变量 - if edit.NextFileNumber != nil { - vs.nextFileNumber.Store(*edit.NextFileNumber) - } - if edit.LastSequence != nil { - vs.lastSequence.Store(*edit.LastSequence) - } - - return nil -} - -// GetCurrent 获取当前版本 -func (vs *VersionSet) GetCurrent() *Version { - vs.mu.RLock() - defer vs.mu.RUnlock() - return vs.current -} - -// GetNextFileNumber 获取下一个文件编号 -func (vs *VersionSet) GetNextFileNumber() int64 { - return vs.nextFileNumber.Load() -} - -// AllocateFileNumber 分配文件编号 -func (vs *VersionSet) AllocateFileNumber() int64 { - return vs.nextFileNumber.Add(1) -} - -// GetLastSequence 获取最后序列号 -func (vs *VersionSet) GetLastSequence() int64 { - return vs.lastSequence.Load() -} - -// SetLastSequence 设置最后序列号 -func (vs *VersionSet) SetLastSequence(seq int64) { - vs.lastSequence.Store(seq) -} - -// Close 关闭 VersionSet -func (vs *VersionSet) Close() error { - vs.mu.Lock() - defer vs.mu.Unlock() - - if vs.manifestFile != nil { - return vs.manifestFile.Close() - } - return nil -} diff --git a/memtable/manager.go b/memtable.go similarity index 51% rename from memtable/manager.go rename to memtable.go index be72f5a..d1900c0 100644 --- a/memtable/manager.go +++ b/memtable.go @@ -1,17 +1,151 @@ -package memtable +package srdb import ( + "slices" "sync" ) +// MemTable 内存表 +type MemTable struct { + data map[int64][]byte // key -> value + keys []int64 // 排序的 keys + size int64 // 数据大小 + mu sync.RWMutex +} + +// NewMemTable 创建 MemTable +func NewMemTable() *MemTable { + return &MemTable{ + data: make(map[int64][]byte), + keys: make([]int64, 0), + size: 0, + } +} + +// Put 插入数据 +func (m *MemTable) Put(key int64, value []byte) { + m.mu.Lock() + defer m.mu.Unlock() + + // 检查是否已存在 + if _, exists := m.data[key]; !exists { + m.keys = append(m.keys, key) + // 保持 keys 有序 + slices.Sort(m.keys) + } + + m.data[key] = value + m.size += int64(len(value)) +} + +// Get 查询数据 +func (m *MemTable) Get(key int64) ([]byte, bool) { + m.mu.RLock() + defer m.mu.RUnlock() + + value, exists := m.data[key] + return value, exists +} + +// Size 获取大小 +func (m *MemTable) Size() int64 { + m.mu.RLock() + defer m.mu.RUnlock() + + return m.size +} + +// Count 获取条目数量 +func (m *MemTable) Count() int { + m.mu.RLock() + defer m.mu.RUnlock() + + return len(m.data) +} + +// Keys 获取所有 keys 的副本（已排序） +func (m *MemTable) Keys() []int64 { + m.mu.RLock() + defer m.mu.RUnlock() + + // 返回副本以避免并发问题 + keysCopy := make([]int64, len(m.keys)) + copy(keysCopy, m.keys) + return keysCopy +} + +// MemTableIterator 迭代器 +type MemTableIterator struct { + mt *MemTable + index int +} + +// NewIterator 创建迭代器 +func (m *MemTable) NewIterator() *MemTableIterator { + m.mu.RLock() + defer m.mu.RUnlock() + + return &MemTableIterator{ + mt: m, + index: -1, + } +} + +// Next 移动到下一个 +func (it *MemTableIterator) Next() bool { + it.mt.mu.RLock() + defer it.mt.mu.RUnlock() + + it.index++ + return it.index < len(it.mt.keys) +} + +// Key 当前 key +func (it *MemTableIterator) Key() int64 { + it.mt.mu.RLock() + defer it.mt.mu.RUnlock() + + if it.index < 0 || it.index >= len(it.mt.keys) { + return 0 + } + return it.mt.keys[it.index] +} + +// Value 当前 value +func (it *MemTableIterator) Value() []byte { + it.mt.mu.RLock() + defer it.mt.mu.RUnlock() + + if it.index < 0 || it.index >= len(it.mt.keys) { + return nil + } + key := it.mt.keys[it.index] + return it.mt.data[key] +} + +// Reset 重置迭代器 +func (it *MemTableIterator) Reset() { + it.index = -1 +} + +// Clear 清空 MemTable +func (m *MemTable) Clear() { + m.mu.Lock() + defer m.mu.Unlock() + + m.data = make(map[int64][]byte) + m.keys = make([]int64, 0) + m.size = 0 +} + // ImmutableMemTable 不可变的 MemTable type ImmutableMemTable struct { - MemTable *MemTable + *MemTable WALNumber int64 // 对应的 WAL 编号 } -// Manager MemTable 管理器 -type Manager struct { +// MemTableManager MemTable 管理器 +type MemTableManager struct { active *MemTable // Active MemTable (可写) immutables []*ImmutableMemTable // Immutable MemTables (只读) activeWAL int64 // Active MemTable 对应的 WAL 编号 @@ -19,31 +153,31 @@ type Manager struct { mu sync.RWMutex // 读写锁 } -// NewManager 创建 MemTable 管理器 -func NewManager(maxSize int64) *Manager { - return &Manager{ - active: New(), +// NewMemTableManager 创建 MemTable 管理器 +func NewMemTableManager(maxSize int64) *MemTableManager { + return &MemTableManager{ + active: NewMemTable(), immutables: make([]*ImmutableMemTable, 0), maxSize: maxSize, } } // SetActiveWAL 设置 Active MemTable 对应的 WAL 编号 -func (m *Manager) SetActiveWAL(walNumber int64) { +func (m *MemTableManager) SetActiveWAL(walNumber int64) { m.mu.Lock() defer m.mu.Unlock() m.activeWAL = walNumber } // Put 写入数据到 Active MemTable -func (m *Manager) Put(key int64, value []byte) { +func (m *MemTableManager) Put(key int64, value []byte) { m.mu.Lock() defer m.mu.Unlock() m.active.Put(key, value) } // Get 查询数据（先查 Active，再查 Immutables） -func (m *Manager) Get(key int64) ([]byte, bool) { +func (m *MemTableManager) Get(key int64) ([]byte, bool) { m.mu.RLock() defer m.mu.RUnlock() @@ -63,21 +197,21 @@ func (m *Manager) Get(key int64) ([]byte, bool) { } // GetActiveSize 获取 Active MemTable 大小 -func (m *Manager) GetActiveSize() int64 { +func (m *MemTableManager) GetActiveSize() int64 { m.mu.RLock() defer m.mu.RUnlock() return m.active.Size() } // GetActiveCount 获取 Active MemTable 条目数 -func (m *Manager) GetActiveCount() int { +func (m *MemTableManager) GetActiveCount() int { m.mu.RLock() defer m.mu.RUnlock() return m.active.Count() } // ShouldSwitch 检查是否需要切换 MemTable -func (m *Manager) ShouldSwitch() bool { +func (m *MemTableManager) ShouldSwitch() bool { m.mu.RLock() defer m.mu.RUnlock() return m.active.Size() >= m.maxSize @@ -85,7 +219,7 @@ func (m *Manager) ShouldSwitch() bool { // Switch 切换 MemTable（Active → Immutable，创建新 Active） // 返回：旧的 WAL 编号，新的 Active MemTable -func (m *Manager) Switch(newWALNumber int64) (oldWALNumber int64, immutable *ImmutableMemTable) { +func (m *MemTableManager) Switch(newWALNumber int64) (oldWALNumber int64, immutable *ImmutableMemTable) { m.mu.Lock() defer m.mu.Unlock() @@ -97,7 +231,7 @@ func (m *Manager) Switch(newWALNumber int64) (oldWALNumber int64, immutable *Imm m.immutables = append(m.immutables, immutable) // 2. 创建新的 Active MemTable - m.active = New() + m.active = NewMemTable() oldWALNumber = m.activeWAL m.activeWAL = newWALNumber @@ -105,7 +239,7 @@ func (m *Manager) Switch(newWALNumber int64) (oldWALNumber int64, immutable *Imm } // RemoveImmutable 移除指定的 Immutable MemTable -func (m *Manager) RemoveImmutable(target *ImmutableMemTable) { +func (m *MemTableManager) RemoveImmutable(target *ImmutableMemTable) { m.mu.Lock() defer m.mu.Unlock() @@ -119,14 +253,14 @@ func (m *Manager) RemoveImmutable(target *ImmutableMemTable) { } // GetImmutableCount 获取 Immutable MemTable 数量 -func (m *Manager) GetImmutableCount() int { +func (m *MemTableManager) GetImmutableCount() int { m.mu.RLock() defer m.mu.RUnlock() return len(m.immutables) } // GetImmutables 获取所有 Immutable MemTables（副本） -func (m *Manager) GetImmutables() []*ImmutableMemTable { +func (m *MemTableManager) GetImmutables() []*ImmutableMemTable { m.mu.RLock() defer m.mu.RUnlock() @@ -136,14 +270,14 @@ func (m *Manager) GetImmutables() []*ImmutableMemTable { } // GetActive 获取 Active MemTable（用于 Flush 时读取） -func (m *Manager) GetActive() *MemTable { +func (m *MemTableManager) GetActive() *MemTable { m.mu.RLock() defer m.mu.RUnlock() return m.active } // TotalCount 获取总条目数（Active + Immutables） -func (m *Manager) TotalCount() int { +func (m *MemTableManager) TotalCount() int { m.mu.RLock() defer m.mu.RUnlock() @@ -155,7 +289,7 @@ func (m *Manager) TotalCount() int { } // TotalSize 获取总大小（Active + Immutables） -func (m *Manager) TotalSize() int64 { +func (m *MemTableManager) TotalSize() int64 { m.mu.RLock() defer m.mu.RUnlock() @@ -167,14 +301,14 @@ func (m *Manager) TotalSize() int64 { } // NewIterator 创建 Active MemTable 的迭代器 -func (m *Manager) NewIterator() *Iterator { +func (m *MemTableManager) NewIterator() *MemTableIterator { m.mu.RLock() defer m.mu.RUnlock() return m.active.NewIterator() } -// Stats 统计信息 -type Stats struct { +// MemTableStats 统计信息 +type MemTableStats struct { ActiveSize int64 ActiveCount int ImmutableCount int @@ -185,11 +319,11 @@ type Stats struct { } // GetStats 获取统计信息 -func (m *Manager) GetStats() *Stats { +func (m *MemTableManager) GetStats() *MemTableStats { m.mu.RLock() defer m.mu.RUnlock() - stats := &Stats{ + stats := &MemTableStats{ ActiveSize: m.active.Size(), ActiveCount: m.active.Count(), ImmutableCount: len(m.immutables), @@ -207,10 +341,10 @@ func (m *Manager) GetStats() *Stats { } // Clear 清空所有 MemTables（用于测试） -func (m *Manager) Clear() { +func (m *MemTableManager) Clear() { m.mu.Lock() defer m.mu.Unlock() - m.active = New() + m.active = NewMemTable() m.immutables = make([]*ImmutableMemTable, 0) } diff --git a/memtable/memtable.go b/memtable/memtable.go deleted file mode 100644 index 4404b43..0000000 --- a/memtable/memtable.go +++ /dev/null @@ -1,141 +0,0 @@ -package memtable - -import ( - "sort" - "sync" -) - -// MemTable 内存表 -type MemTable struct { - data map[int64][]byte // key -> value - keys []int64 // 排序的 keys - size int64 // 数据大小 - mu sync.RWMutex -} - -// New 创建 MemTable -func New() *MemTable { - return &MemTable{ - data: make(map[int64][]byte), - keys: make([]int64, 0), - size: 0, - } -} - -// Put 插入数据 -func (m *MemTable) Put(key int64, value []byte) { - m.mu.Lock() - defer m.mu.Unlock() - - // 检查是否已存在 - if _, exists := m.data[key]; !exists { - m.keys = append(m.keys, key) - // 保持 keys 有序 - sort.Slice(m.keys, func(i, j int) bool { - return m.keys[i] < m.keys[j] - }) - } - - m.data[key] = value - m.size += int64(len(value)) -} - -// Get 查询数据 -func (m *MemTable) Get(key int64) ([]byte, bool) { - m.mu.RLock() - defer m.mu.RUnlock() - - value, exists := m.data[key] - return value, exists -} - -// Size 获取大小 -func (m *MemTable) Size() int64 { - m.mu.RLock() - defer m.mu.RUnlock() - - return m.size -} - -// Count 获取条目数量 -func (m *MemTable) Count() int { - m.mu.RLock() - defer m.mu.RUnlock() - - return len(m.data) -} - -// Keys 获取所有 keys 的副本（已排序） -func (m *MemTable) Keys() []int64 { - m.mu.RLock() - defer m.mu.RUnlock() - - // 返回副本以避免并发问题 - keysCopy := make([]int64, len(m.keys)) - copy(keysCopy, m.keys) - return keysCopy -} - -// Iterator 迭代器 -type Iterator struct { - mt *MemTable - index int -} - -// NewIterator 创建迭代器 -func (m *MemTable) NewIterator() *Iterator { - m.mu.RLock() - defer m.mu.RUnlock() - - return &Iterator{ - mt: m, - index: -1, - } -} - -// Next 移动到下一个 -func (it *Iterator) Next() bool { - it.mt.mu.RLock() - defer it.mt.mu.RUnlock() - - it.index++ - return it.index < len(it.mt.keys) -} - -// Key 当前 key -func (it *Iterator) Key() int64 { - it.mt.mu.RLock() - defer it.mt.mu.RUnlock() - - if it.index < 0 || it.index >= len(it.mt.keys) { - return 0 - } - return it.mt.keys[it.index] -} - -// Value 当前 value -func (it *Iterator) Value() []byte { - it.mt.mu.RLock() - defer it.mt.mu.RUnlock() - - if it.index < 0 || it.index >= len(it.mt.keys) { - return nil - } - key := it.mt.keys[it.index] - return it.mt.data[key] -} - -// Reset 重置迭代器 -func (it *Iterator) Reset() { - it.index = -1 -} - -// Clear 清空 MemTable -func (m *MemTable) Clear() { - m.mu.Lock() - defer m.mu.Unlock() - - m.data = make(map[int64][]byte) - m.keys = make([]int64, 0) - m.size = 0 -} diff --git a/memtable/memtable_test.go b/memtable/memtable_test.go deleted file mode 100644 index 0767830..0000000 --- a/memtable/memtable_test.go +++ /dev/null @@ -1,121 +0,0 @@ -package memtable - -import ( - "testing" -) - -func TestMemTable(t *testing.T) { - mt := New() - - // 1. 插入数据 - for i := int64(1); i <= 100; i++ { - mt.Put(i, []byte("value_"+string(rune(i)))) - } - - if mt.Count() != 100 { - t.Errorf("Expected 100 entries, got %d", mt.Count()) - } - - t.Logf("Inserted 100 entries, size: %d bytes", mt.Size()) - - // 2. 查询数据 - for i := int64(1); i <= 100; i++ { - value, exists := mt.Get(i) - if !exists { - t.Errorf("Key %d not found", i) - } - if value == nil { - t.Errorf("Key %d: value is nil", i) - } - } - - // 3. 查询不存在的 key - _, exists := mt.Get(101) - if exists { - t.Error("Key 101 should not exist") - } - - t.Log("All tests passed!") -} - -func TestMemTableIterator(t *testing.T) { - mt := New() - - // 插入数据 (乱序) - keys := []int64{5, 2, 8, 1, 9, 3, 7, 4, 6, 10} - for _, key := range keys { - mt.Put(key, []byte("value")) - } - - // 迭代器应该按顺序返回 - iter := mt.NewIterator() - var result []int64 - for iter.Next() { - result = append(result, iter.Key()) - } - - // 验证顺序 - for i := 0; i < len(result)-1; i++ { - if result[i] >= result[i+1] { - t.Errorf("Keys not in order: %v", result) - break - } - } - - if len(result) != 10 { - t.Errorf("Expected 10 keys, got %d", len(result)) - } - - t.Logf("Iterator returned keys in order: %v", result) -} - -func TestMemTableClear(t *testing.T) { - mt := New() - - // 插入数据 - for i := int64(1); i <= 10; i++ { - mt.Put(i, []byte("value")) - } - - if mt.Count() != 10 { - t.Errorf("Expected 10 entries, got %d", mt.Count()) - } - - // 清空 - mt.Clear() - - if mt.Count() != 0 { - t.Errorf("Expected 0 entries after clear, got %d", mt.Count()) - } - - if mt.Size() != 0 { - t.Errorf("Expected size 0 after clear, got %d", mt.Size()) - } - - t.Log("Clear test passed!") -} - -func BenchmarkMemTablePut(b *testing.B) { - mt := New() - value := make([]byte, 100) - - b.ResetTimer() - for i := 0; i < b.N; i++ { - mt.Put(int64(i), value) - } -} - -func BenchmarkMemTableGet(b *testing.B) { - mt := New() - value := make([]byte, 100) - - // 预先插入数据 - for i := int64(0); i < 10000; i++ { - mt.Put(i, value) - } - - b.ResetTimer() - for i := 0; i < b.N; i++ { - mt.Get(int64(i % 10000)) - } -} diff --git a/memtable/manager_test.go b/memtable_test.go similarity index 57% rename from memtable/manager_test.go rename to memtable_test.go index 9adfda4..e51660f 100644 --- a/memtable/manager_test.go +++ b/memtable_test.go @@ -1,11 +1,125 @@ -package memtable +package srdb import ( "testing" ) -func TestManagerBasic(t *testing.T) { - mgr := NewManager(1024) // 1KB +func TestMemTable(t *testing.T) { + mt := NewMemTable() + + // 1. 插入数据 + for i := int64(1); i <= 100; i++ { + mt.Put(i, []byte("value_"+string(rune(i)))) + } + + if mt.Count() != 100 { + t.Errorf("Expected 100 entries, got %d", mt.Count()) + } + + t.Logf("Inserted 100 entries, size: %d bytes", mt.Size()) + + // 2. 查询数据 + for i := int64(1); i <= 100; i++ { + value, exists := mt.Get(i) + if !exists { + t.Errorf("Key %d not found", i) + } + if value == nil { + t.Errorf("Key %d: value is nil", i) + } + } + + // 3. 查询不存在的 key + _, exists := mt.Get(101) + if exists { + t.Error("Key 101 should not exist") + } + + t.Log("All tests passed!") +} + +func TestMemTableIterator(t *testing.T) { + mt := NewMemTable() + + // 插入数据 (乱序) + keys := []int64{5, 2, 8, 1, 9, 3, 7, 4, 6, 10} + for _, key := range keys { + mt.Put(key, []byte("value")) + } + + // 迭代器应该按顺序返回 + iter := mt.NewIterator() + var result []int64 + for iter.Next() { + result = append(result, iter.Key()) + } + + // 验证顺序 + for i := 0; i < len(result)-1; i++ { + if result[i] >= result[i+1] { + t.Errorf("Keys not in order: %v", result) + break + } + } + + if len(result) != 10 { + t.Errorf("Expected 10 keys, got %d", len(result)) + } + + t.Logf("Iterator returned keys in order: %v", result) +} + +func TestMemTableClear(t *testing.T) { + mt := NewMemTable() + + // 插入数据 + for i := int64(1); i <= 10; i++ { + mt.Put(i, []byte("value")) + } + + if mt.Count() != 10 { + t.Errorf("Expected 10 entries, got %d", mt.Count()) + } + + // 清空 + mt.Clear() + + if mt.Count() != 0 { + t.Errorf("Expected 0 entries after clear, got %d", mt.Count()) + } + + if mt.Size() != 0 { + t.Errorf("Expected size 0 after clear, got %d", mt.Size()) + } + + t.Log("Clear test passed!") +} + +func BenchmarkMemTablePut(b *testing.B) { + mt := NewMemTable() + value := make([]byte, 100) + + for i := 0; b.Loop(); i++ { + mt.Put(int64(i), value) + } +} + +func BenchmarkMemTableGet(b *testing.B) { + mt := NewMemTable() + value := make([]byte, 100) + + // 预先插入数据 + for i := range int64(10000) { + mt.Put(i, value) + } + + for i := 0; b.Loop(); i++ { + mt.Get(int64(i % 10000)) + } +} + +func TestMemTableManagerBasic(t *testing.T) { + mgr := NewMemTableManager(1024) // 1KB // 测试写入 mgr.Put(1, []byte("value1")) @@ -26,8 +140,8 @@ func TestManagerBasic(t *testing.T) { t.Log("Manager basic test passed!") } -func TestManagerSwitch(t *testing.T) { - mgr := NewManager(50) // 50 bytes +func TestMemTableManagerSwitch(t *testing.T) { + mgr := NewMemTableManager(50) // 50 bytes mgr.SetActiveWAL(1) // 写入数据 @@ -69,8 +183,8 @@ func TestManagerSwitch(t *testing.T) { t.Log("Manager switch test passed!") } -func TestManagerMultipleImmutables(t *testing.T) { - mgr := NewManager(50) +func TestMemTableManagerMultipleImmutables(t *testing.T) { + mgr := NewMemTableManager(50) mgr.SetActiveWAL(1) // 第一批数据 @@ -100,8 +214,8 @@ func TestManagerMultipleImmutables(t *testing.T) { t.Log("Manager multiple immutables test passed!") } -func TestManagerRemoveImmutable(t *testing.T) { - mgr := NewManager(50) +func TestMemTableManagerRemoveImmutable(t *testing.T) { + mgr := NewMemTableManager(50) mgr.SetActiveWAL(1) // 创建 Immutable @@ -124,8 +238,8 @@ func TestManagerRemoveImmutable(t *testing.T) { t.Log("Manager remove immutable test passed!") } -func TestManagerStats(t *testing.T) { - mgr := NewManager(100) +func TestMemTableManagerStats(t *testing.T) { + mgr := NewMemTableManager(100) mgr.SetActiveWAL(1) // Active 数据 @@ -161,15 +275,15 @@ func TestManagerStats(t *testing.T) { t.Log("Manager stats test passed!") } -func TestManagerConcurrent(t *testing.T) { - mgr := NewManager(1024) +func TestMemTableManagerConcurrent(t *testing.T) { + mgr := NewMemTableManager(1024) mgr.SetActiveWAL(1) // 并发写入 done := make(chan bool) - for i := 0; i < 10; i++ { + for i := range 10 { go func(id int) { - for j := 0; j < 100; j++ { + for j := range 100 { key := int64(id*100 + j) mgr.Put(key, []byte("value")) } @@ -178,7 +292,7 @@ func TestManagerConcurrent(t *testing.T) { } // 等待完成 - for i := 0; i < 10; i++ { + for range 10 { <-done } diff --git a/query.go b/query.go index b424a6b..943624b 100644 --- a/query.go +++ b/query.go @@ -3,9 +3,9 @@ package srdb import ( "encoding/json" "fmt" + "maps" + "slices" "strings" - - "code.tczkiot.com/srdb/sst" ) type Fieldset interface { @@ -489,29 +489,101 @@ func (qb *QueryBuilder) Rows() (*Rows, error) { visited: make(map[int64]bool), } - // 初始化 Active MemTable 迭代器 + // 收集所有数据源的 keys 并全局排序 + // 立即读取数据避免 compaction 期间文件被删除 + keyToRow := make(map[int64]*SSTableRow) // 存储已读取的行数据 + var allKeys []int64 + + // 1. 从 Active MemTable 读取数据 activeMemTable := qb.engine.memtableManager.GetActive() if activeMemTable != nil { activeKeys := activeMemTable.Keys() - if len(activeKeys) > 0 { - rows.memIterator = newMemtableIterator(activeKeys) + for _, key := range activeKeys { + if data, ok := activeMemTable.Get(key); ok { + var row SSTableRow + if err := json.Unmarshal(data, &row); err == nil { + keyToRow[key] = &row + allKeys = append(allKeys, key) + } + } } } - // 准备 Immutable MemTables（延迟初始化） - rows.immutableIndex = 0 + // 2. 从所有 Immutable MemTables 读取数据 + immutables := qb.engine.memtableManager.GetImmutables() + for _, imm := range immutables { + immKeys := imm.MemTable.Keys() + for _, key := range immKeys { + // 如果 key 已存在（来自更新的数据源），跳过 + if _, exists := keyToRow[key]; exists { + continue + } - // 初始化 SST 文件 readers + if data, ok := imm.MemTable.Get(key); ok { + var row SSTableRow + if err := json.Unmarshal(data, &row); err == nil { + keyToRow[key] = &row + allKeys = append(allKeys, key) + } + } + } + } + + // 3. 收集所有 SST 文件的 keys sstReaders := qb.engine.sstManager.GetReaders() + for _, reader := range sstReaders { - // 获取文件中实际存在的 key 列表（已排序） - // 这比 minKey→maxKey 逐个尝试高效 100-1000 倍（对于稀疏 key） + // 获取文件中实际存在的 key 列表（已在 GetAllKeys 中排序） keys := reader.GetAllKeys() - rows.sstReaders = append(rows.sstReaders, &sstReader{ - reader: reader, - keys: keys, - index: 0, - }) + + // 记录所有 keys（实际数据稍后统一从 engine 读取） + for _, key := range keys { + // 如果 key 已存在（来自更新的数据源），跳过 + if _, exists := keyToRow[key]; !exists { + allKeys = append(allKeys, key) + keyToRow[key] = nil // 占位，表示需要读取 + } + } + } + + // 4. 对所有 keys 排序 + if len(allKeys) > 0 { + // 去重（使用 map 已经去重了，但 allKeys 可能有重复） + keySet := make(map[int64]bool) + uniqueKeys := make([]int64, 0, len(allKeys)) + for _, key := range allKeys { + if !keySet[key] { + keySet[key] = true + uniqueKeys = append(uniqueKeys, key) + } + } + + // 排序 + slices.Sort(uniqueKeys) + + // 统一从 engine 读取所有数据（避免 compaction 导致的文件删除） + rows.cachedRows = make([]*SSTableRow, 0, len(uniqueKeys)) + for _, seq := range uniqueKeys { + // 如果已经从 MemTable 读取，直接使用 + row := keyToRow[seq] + if row == nil { + // 从 engine 读取（会搜索 MemTable + 所有 SST，包括 compaction 后的新文件） + var err error + row, err = qb.engine.Get(seq) + if err != nil { + // 数据不存在（理论上不应该发生，因为 key 来自索引） + continue + } + } + + if qb.Match(row.Data) { + rows.cachedRows = append(rows.cachedRows, row) + } + } + + // 使用缓存模式 + rows.cached = true + rows.cachedIndex = -1 } return rows, nil @@ -553,7 +625,7 @@ func (qb *QueryBuilder) Scan(value any) error { type Row struct { schema *Schema fields []string // 要选择的字段，nil 表示选择所有字段 - inner *sst.Row + inner *SSTableRow } // Data 获取行数据（根据 Select 过滤字段） @@ -563,13 +635,11 @@ func (r *Row) Data() map[string]any { } // 如果没有指定字段，返回所有数据（包括 _seq 和 _time） - if r.fields == nil || len(r.fields) == 0 { + if len(r.fields) == 0 { result := make(map[string]any) result["_seq"] = r.inner.Seq result["_time"] = r.inner.Time - for k, v := range r.inner.Data { - result[k] = v - } + maps.Copy(result, r.inner.Data) return result } @@ -636,7 +706,7 @@ type Rows struct { // 缓存模式（用于 Collect/Data 等方法） cached bool - cachedRows []*sst.Row + cachedRows []*SSTableRow cachedIndex int // 缓存模式下的迭代位置 } @@ -663,9 +733,8 @@ func (m *memtableIterator) next() (int64, bool) { // sstReader 包装 SST Reader 的迭代状态 type sstReader struct { - reader any // 实际的 SST reader - keys []int64 // 文件中实际存在的 key 列表（已排序） - index int // 当前迭代位置 + keys []int64 // 文件中实际存在的 key 列表（已排序） + index int // 当前迭代位置 } // Next 移动到下一行，返回是否还有数据 @@ -769,7 +838,11 @@ func (r *Rows) nextFromCache() bool { if r.cachedIndex >= len(r.cachedRows) { return false } - r.currentRow = &Row{schema: r.schema, fields: r.fields, inner: r.cachedRows[r.cachedIndex]} + r.currentRow = &Row{ + schema: r.schema, + fields: r.fields, + inner: r.cachedRows[r.cachedIndex], + } return true } @@ -860,7 +933,11 @@ func (r *Rows) Last() (*Row, error) { if len(r.cachedRows) == 0 { return nil, fmt.Errorf("no rows") } - return &Row{schema: r.schema, fields: r.fields, inner: r.cachedRows[len(r.cachedRows)-1]}, nil + return &Row{ + schema: r.schema, + fields: r.fields, + inner: r.cachedRows[len(r.cachedRows)-1], + }, nil } // Count 返回总行数（别名） diff --git a/schema_test.go b/schema_test.go index a9fec69..c281d2e 100644 --- a/schema_test.go +++ b/schema_test.go @@ -155,7 +155,7 @@ func TestPredefinedSchemas(t *testing.T) { // TestChecksumDeterminism 测试 checksum 的确定性 func TestChecksumDeterminism(t *testing.T) { // 创建相同的 Schema 多次 - for i := 0; i < 10; i++ { + for i := range 10 { s1 := NewSchema("users", []Field{ {Name: "name", Type: FieldTypeString, Indexed: true, Comment: "用户名"}, {Name: "age", Type: FieldTypeInt64, Indexed: false, Comment: "年龄"}, diff --git a/sst/encoding.go b/sst/encoding.go deleted file mode 100644 index 537172a..0000000 --- a/sst/encoding.go +++ /dev/null @@ -1,98 +0,0 @@ -package sst - -import ( - "bytes" - "encoding/binary" - "encoding/json" - "fmt" -) - -// 二进制编码格式: -// [Magic: 4 bytes][Seq: 8 bytes][Time: 8 bytes][DataLen: 4 bytes][Data: variable] - -const ( - RowMagic = 0x524F5733 // "ROW3" -) - -// encodeRowBinary 使用二进制格式编码行数据 -func encodeRowBinary(row *Row) ([]byte, error) { - buf := new(bytes.Buffer) - - // 写入 Magic Number (用于验证) - if err := binary.Write(buf, binary.LittleEndian, uint32(RowMagic)); err != nil { - return nil, err - } - - // 写入 Seq - if err := binary.Write(buf, binary.LittleEndian, row.Seq); err != nil { - return nil, err - } - - // 写入 Time - if err := binary.Write(buf, binary.LittleEndian, row.Time); err != nil { - return nil, err - } - - // 序列化用户数据 (仍使用 JSON，但只序列化用户数据部分) - dataBytes, err := json.Marshal(row.Data) - if err != nil { - return nil, err - } - - // 写入数据长度 - if err := binary.Write(buf, binary.LittleEndian, uint32(len(dataBytes))); err != nil { - return nil, err - } - - // 写入数据 - if _, err := buf.Write(dataBytes); err != nil { - return nil, err - } - - return buf.Bytes(), nil -} - -// decodeRowBinary 解码二进制格式的行数据 -func decodeRowBinary(data []byte) (*Row, error) { - buf := bytes.NewReader(data) - - // 读取并验证 Magic Number - var magic uint32 - if err := binary.Read(buf, binary.LittleEndian, &magic); err != nil { - return nil, err - } - if magic != RowMagic { - return nil, fmt.Errorf("invalid row magic: %x", magic) - } - - row := &Row{} - - // 读取 Seq - if err := binary.Read(buf, binary.LittleEndian, &row.Seq); err != nil { - return nil, err - } - - // 读取 Time - if err := binary.Read(buf, binary.LittleEndian, &row.Time); err != nil { - return nil, err - } - - // 读取数据长度 - var dataLen uint32 - if err := binary.Read(buf, binary.LittleEndian, &dataLen); err != nil { - return nil, err - } - - // 读取数据 - dataBytes := make([]byte, dataLen) - if _, err := buf.Read(dataBytes); err != nil { - return nil, err - } - - // 反序列化用户数据 - if err := json.Unmarshal(dataBytes, &row.Data); err != nil { - return nil, err - } - - return row, nil -} diff --git a/sst/encoding_test.go b/sst/encoding_test.go deleted file mode 100644 index 6e73a35..0000000 --- a/sst/encoding_test.go +++ /dev/null @@ -1,117 +0,0 @@ -package sst - -import ( - "encoding/json" - "testing" -) - -func TestBinaryEncoding(t *testing.T) { - // 创建测试数据 - row := &Row{ - Seq: 12345, - Time: 1234567890, - Data: map[string]interface{}{ - "name": "test_user", - "age": 25, - "email": "test@example.com", - }, - } - - // 编码 - encoded, err := encodeRowBinary(row) - if err != nil { - t.Fatal(err) - } - - t.Logf("Encoded size: %d bytes", len(encoded)) - - // 解码 - decoded, err := decodeRowBinary(encoded) - if err != nil { - t.Fatal(err) - } - - // 验证 - if decoded.Seq != row.Seq { - t.Errorf("Seq mismatch: expected %d, got %d", row.Seq, decoded.Seq) - } - if decoded.Time != row.Time { - t.Errorf("Time mismatch: expected %d, got %d", row.Time, decoded.Time) - } - if decoded.Data["name"] != row.Data["name"] { - t.Errorf("Name mismatch") - } - - t.Log("Binary encoding test passed!") -} - -func TestEncodingComparison(t *testing.T) { - row := &Row{ - Seq: 12345, - Time: 1234567890, - Data: map[string]interface{}{ - "name": "test_user", - "age": 25, - "email": "test@example.com", - }, - } - - // 二进制编码 - binaryEncoded, _ := encodeRowBinary(row) - - // JSON 编码 (旧方式) - jsonData := map[string]interface{}{ - "_seq": row.Seq, - "_time": row.Time, - "data": row.Data, - } - jsonEncoded, _ := json.Marshal(jsonData) - - t.Logf("Binary size: %d bytes", len(binaryEncoded)) - t.Logf("JSON size: %d bytes", len(jsonEncoded)) - t.Logf("Space saved: %.1f%%", float64(len(jsonEncoded)-len(binaryEncoded))/float64(len(jsonEncoded))*100) - - if len(binaryEncoded) >= len(jsonEncoded) { - t.Error("Binary encoding should be smaller than JSON") - } -} - -func BenchmarkBinaryEncoding(b *testing.B) { - row := &Row{ - Seq: 12345, - Time: 1234567890, - Data: map[string]interface{}{ - "name": "test_user", - "age": 25, - "email": "test@example.com", - }, - } - - b.ResetTimer() - for i := 0; i < b.N; i++ { - encodeRowBinary(row) - } -} - -func BenchmarkJSONEncoding(b *testing.B) { - row := &Row{ - Seq: 12345, - Time: 1234567890, - Data: map[string]interface{}{ - "name": "test_user", - "age": 25, - "email": "test@example.com", - }, - } - - data := map[string]interface{}{ - "_seq": row.Seq, - "_time": row.Time, - "data": row.Data, - } - - b.ResetTimer() - for i := 0; i < b.N; i++ { - json.Marshal(data) - } -} diff --git a/sst/format.go b/sst/format.go deleted file mode 100644 index b7ba491..0000000 --- a/sst/format.go +++ /dev/null @@ -1,142 +0,0 @@ -package sst - -import ( - "encoding/binary" -) - -const ( - // 文件格式 - MagicNumber = 0x53535433 // "SST3" - Version = 1 - HeaderSize = 256 // 文件头大小 - BlockSize = 64 * 1024 // 数据块大小 (64 KB) - - // 压缩类型 - CompressionNone = 0 - CompressionSnappy = 1 -) - -// Header SST 文件头 (256 bytes) -type Header struct { - // 基础信息 (32 bytes) - Magic uint32 // Magic Number: 0x53535433 - Version uint32 // 版本号 - Compression uint8 // 压缩类型 - Reserved1 [3]byte - Flags uint32 // 标志位 - Reserved2 [16]byte - - // 索引信息 (32 bytes) - IndexOffset int64 // B+Tree 索引起始位置 - IndexSize int64 // B+Tree 索引大小 - RootOffset int64 // B+Tree 根节点位置 - Reserved3 [8]byte - - // 数据信息 (32 bytes) - DataOffset int64 // 数据块起始位置 - DataSize int64 // 数据块总大小 - RowCount int64 // 行数 - Reserved4 [8]byte - - // 统计信息 (32 bytes) - MinKey int64 // 最小 key (_seq) - MaxKey int64 // 最大 key (_seq) - MinTime int64 // 最小时间戳 - MaxTime int64 // 最大时间戳 - - // CRC 校验 (8 bytes) - CRC32 uint32 // Header CRC32 - Reserved5 [4]byte - - // 预留空间 (120 bytes) - Reserved6 [120]byte -} - -// Marshal 序列化 Header -func (h *Header) Marshal() []byte { - buf := make([]byte, HeaderSize) - - // 基础信息 - binary.LittleEndian.PutUint32(buf[0:4], h.Magic) - binary.LittleEndian.PutUint32(buf[4:8], h.Version) - buf[8] = h.Compression - copy(buf[9:12], h.Reserved1[:]) - binary.LittleEndian.PutUint32(buf[12:16], h.Flags) - copy(buf[16:32], h.Reserved2[:]) - - // 索引信息 - binary.LittleEndian.PutUint64(buf[32:40], uint64(h.IndexOffset)) - binary.LittleEndian.PutUint64(buf[40:48], uint64(h.IndexSize)) - binary.LittleEndian.PutUint64(buf[48:56], uint64(h.RootOffset)) - copy(buf[56:64], h.Reserved3[:]) - - // 数据信息 - binary.LittleEndian.PutUint64(buf[64:72], uint64(h.DataOffset)) - binary.LittleEndian.PutUint64(buf[72:80], uint64(h.DataSize)) - binary.LittleEndian.PutUint64(buf[80:88], uint64(h.RowCount)) - copy(buf[88:96], h.Reserved4[:]) - - // 统计信息 - binary.LittleEndian.PutUint64(buf[96:104], uint64(h.MinKey)) - binary.LittleEndian.PutUint64(buf[104:112], uint64(h.MaxKey)) - binary.LittleEndian.PutUint64(buf[112:120], uint64(h.MinTime)) - binary.LittleEndian.PutUint64(buf[120:128], uint64(h.MaxTime)) - - // CRC 校验 - binary.LittleEndian.PutUint32(buf[128:132], h.CRC32) - copy(buf[132:136], h.Reserved5[:]) - - // 预留空间 - copy(buf[136:256], h.Reserved6[:]) - - return buf -} - -// Unmarshal 反序列化 Header -func UnmarshalHeader(data []byte) *Header { - if len(data) < HeaderSize { - return nil - } - - h := &Header{} - - // 基础信息 - h.Magic = binary.LittleEndian.Uint32(data[0:4]) - h.Version = binary.LittleEndian.Uint32(data[4:8]) - h.Compression = data[8] - copy(h.Reserved1[:], data[9:12]) - h.Flags = binary.LittleEndian.Uint32(data[12:16]) - copy(h.Reserved2[:], data[16:32]) - - // 索引信息 - h.IndexOffset = int64(binary.LittleEndian.Uint64(data[32:40])) - h.IndexSize = int64(binary.LittleEndian.Uint64(data[40:48])) - h.RootOffset = int64(binary.LittleEndian.Uint64(data[48:56])) - copy(h.Reserved3[:], data[56:64]) - - // 数据信息 - h.DataOffset = int64(binary.LittleEndian.Uint64(data[64:72])) - h.DataSize = int64(binary.LittleEndian.Uint64(data[72:80])) - h.RowCount = int64(binary.LittleEndian.Uint64(data[80:88])) - copy(h.Reserved4[:], data[88:96]) - - // 统计信息 - h.MinKey = int64(binary.LittleEndian.Uint64(data[96:104])) - h.MaxKey = int64(binary.LittleEndian.Uint64(data[104:112])) - h.MinTime = int64(binary.LittleEndian.Uint64(data[112:120])) - h.MaxTime = int64(binary.LittleEndian.Uint64(data[120:128])) - - // CRC 校验 - h.CRC32 = binary.LittleEndian.Uint32(data[128:132]) - copy(h.Reserved5[:], data[132:136]) - - // 预留空间 - copy(h.Reserved6[:], data[136:256]) - - return h -} - -// Validate 验证 Header -func (h *Header) Validate() bool { - return h.Magic == MagicNumber && h.Version == Version -} diff --git a/sst/manager.go b/sst/manager.go deleted file mode 100644 index 0de7ca3..0000000 --- a/sst/manager.go +++ /dev/null @@ -1,284 +0,0 @@ -package sst - -import ( - "fmt" - "os" - "path/filepath" - "sort" - "strings" - "sync" -) - -// Manager SST 文件管理器 -type Manager struct { - dir string - readers []*Reader - mu sync.RWMutex -} - -// NewManager 创建 SST 管理器 -func NewManager(dir string) (*Manager, error) { - // 确保目录存在 - err := os.MkdirAll(dir, 0755) - if err != nil { - return nil, err - } - - mgr := &Manager{ - dir: dir, - readers: make([]*Reader, 0), - } - - // 恢复现有的 SST 文件 - err = mgr.recover() - if err != nil { - return nil, err - } - - return mgr, nil -} - -// recover 恢复现有的 SST 文件 -func (m *Manager) recover() error { - // 查找所有 SST 文件 - files, err := filepath.Glob(filepath.Join(m.dir, "*.sst")) - if err != nil { - return err - } - - for _, file := range files { - // 跳过索引文件 - filename := filepath.Base(file) - if strings.HasPrefix(filename, "idx_") { - continue - } - - // 打开 SST Reader - reader, err := NewReader(file) - if err != nil { - return err - } - - m.readers = append(m.readers, reader) - } - - return nil -} - -// CreateSST 创建新的 SST 文件 -// fileNumber: 文件编号（由 VersionSet 分配） -func (m *Manager) CreateSST(fileNumber int64, rows []*Row) (*Reader, error) { - return m.CreateSSTWithLevel(fileNumber, rows, 0) // 默认创建到 L0 -} - -// CreateSSTWithLevel 创建新的 SST 文件到指定层级 -// fileNumber: 文件编号（由 VersionSet 分配） -func (m *Manager) CreateSSTWithLevel(fileNumber int64, rows []*Row, level int) (*Reader, error) { - m.mu.Lock() - defer m.mu.Unlock() - - sstPath := filepath.Join(m.dir, fmt.Sprintf("%06d.sst", fileNumber)) - - // 创建文件 - file, err := os.Create(sstPath) - if err != nil { - return nil, err - } - - writer := NewWriter(file) - - // 写入所有行 - for _, row := range rows { - err = writer.Add(row) - if err != nil { - file.Close() - os.Remove(sstPath) - return nil, err - } - } - - // 完成写入 - err = writer.Finish() - if err != nil { - file.Close() - os.Remove(sstPath) - return nil, err - } - - file.Close() - - // 打开 SST Reader - reader, err := NewReader(sstPath) - if err != nil { - return nil, err - } - - // 添加到 readers 列表 - m.readers = append(m.readers, reader) - - return reader, nil -} - -// Get 从所有 SST 文件中查找数据 -func (m *Manager) Get(seq int64) (*Row, error) { - m.mu.RLock() - defer m.mu.RUnlock() - - // 从后往前查找（新的文件优先） - for i := len(m.readers) - 1; i >= 0; i-- { - reader := m.readers[i] - row, err := reader.Get(seq) - if err == nil { - return row, nil - } - } - - return nil, fmt.Errorf("key not found: %d", seq) -} - -// GetReaders 获取所有 Readers（用于扫描） -func (m *Manager) GetReaders() []*Reader { - m.mu.RLock() - defer m.mu.RUnlock() - - // 返回副本 - readers := make([]*Reader, len(m.readers)) - copy(readers, m.readers) - return readers -} - -// GetMaxSeq 获取所有 SST 中的最大 seq -func (m *Manager) GetMaxSeq() int64 { - m.mu.RLock() - defer m.mu.RUnlock() - - maxSeq := int64(0) - for _, reader := range m.readers { - header := reader.GetHeader() - if header.MaxKey > maxSeq { - maxSeq = header.MaxKey - } - } - - return maxSeq -} - -// Count 获取 SST 文件数量 -func (m *Manager) Count() int { - m.mu.RLock() - defer m.mu.RUnlock() - - return len(m.readers) -} - -// ListFiles 列出所有 SST 文件 -func (m *Manager) ListFiles() []string { - m.mu.RLock() - defer m.mu.RUnlock() - - files := make([]string, 0, len(m.readers)) - for _, reader := range m.readers { - files = append(files, reader.path) - } - - return files -} - -// CompactionConfig Compaction 配置 -// 已废弃：请使用 compaction 包中的 Manager -type CompactionConfig struct { - Threshold int // 触发阈值（SST 文件数量） - BatchSize int // 每次合并的文件数量 -} - -// DefaultCompactionConfig 默认配置 -// 已废弃：请使用 compaction 包中的 Manager -var DefaultCompactionConfig = CompactionConfig{ - Threshold: 10, - BatchSize: 10, -} - -// ShouldCompact 检查是否需要 Compaction -// 已废弃：请使用 compaction 包中的 Manager -func (m *Manager) ShouldCompact(config CompactionConfig) bool { - m.mu.RLock() - defer m.mu.RUnlock() - - return len(m.readers) > config.Threshold -} - -// Compact 执行 Compaction -// 已废弃：请使用 compaction 包中的 Manager -// 注意：此方法已不再维护，不应在新代码中使用 -func (m *Manager) Compact(config CompactionConfig) error { - // 此方法已废弃，不再实现 - return fmt.Errorf("Compact is deprecated, please use compaction.Manager") -} - -// sortRows 按 seq 排序 -func sortRows(rows []*Row) { - sort.Slice(rows, func(i, j int) bool { - return rows[i].Seq < rows[j].Seq - }) -} - -// Delete 删除指定的 SST 文件（预留接口） -func (m *Manager) Delete(fileNumber int64) error { - m.mu.Lock() - defer m.mu.Unlock() - - sstPath := filepath.Join(m.dir, fmt.Sprintf("%06d.sst", fileNumber)) - return os.Remove(sstPath) -} - -// Close 关闭所有 SST Readers -func (m *Manager) Close() error { - m.mu.Lock() - defer m.mu.Unlock() - - for _, reader := range m.readers { - reader.Close() - } - - m.readers = nil - return nil -} - -// Stats 统计信息 -type Stats struct { - FileCount int - TotalSize int64 - MinSeq int64 - MaxSeq int64 -} - -// GetStats 获取统计信息 -func (m *Manager) GetStats() *Stats { - m.mu.RLock() - defer m.mu.RUnlock() - - stats := &Stats{ - FileCount: len(m.readers), - MinSeq: -1, - MaxSeq: -1, - } - - for _, reader := range m.readers { - header := reader.GetHeader() - - if stats.MinSeq == -1 || header.MinKey < stats.MinSeq { - stats.MinSeq = header.MinKey - } - - if stats.MaxSeq == -1 || header.MaxKey > stats.MaxSeq { - stats.MaxSeq = header.MaxKey - } - - // 获取文件大小 - if stat, err := os.Stat(reader.path); err == nil { - stats.TotalSize += stat.Size() - } - } - - return stats -} diff --git a/sst/reader.go b/sst/reader.go deleted file mode 100644 index 3124b61..0000000 --- a/sst/reader.go +++ /dev/null @@ -1,152 +0,0 @@ -package sst - -import ( - "encoding/json" - "fmt" - "os" - - "code.tczkiot.com/srdb/btree" - "github.com/edsrzf/mmap-go" - "github.com/golang/snappy" -) - -// Reader SST 文件读取器 -type Reader struct { - path string - file *os.File - mmap mmap.MMap - header *Header - btReader *btree.Reader -} - -// NewReader 创建 SST 读取器 -func NewReader(path string) (*Reader, error) { - // 1. 打开文件 - file, err := os.Open(path) - if err != nil { - return nil, err - } - - // 2. mmap 映射 - mmapData, err := mmap.Map(file, mmap.RDONLY, 0) - if err != nil { - file.Close() - return nil, err - } - - // 3. 读取 Header - if len(mmapData) < HeaderSize { - mmapData.Unmap() - file.Close() - return nil, fmt.Errorf("file too small") - } - - header := UnmarshalHeader(mmapData[:HeaderSize]) - if header == nil || !header.Validate() { - mmapData.Unmap() - file.Close() - return nil, fmt.Errorf("invalid header") - } - - // 4. 创建 B+Tree Reader - btReader := btree.NewReader(mmapData, header.RootOffset) - - return &Reader{ - path: path, - file: file, - mmap: mmapData, - header: header, - btReader: btReader, - }, nil -} - -// Get 查询一行数据 -func (r *Reader) Get(key int64) (*Row, error) { - // 1. 检查范围 - if key < r.header.MinKey || key > r.header.MaxKey { - return nil, fmt.Errorf("key out of range") - } - - // 2. 在 B+Tree 中查找 - dataOffset, dataSize, found := r.btReader.Get(key) - if !found { - return nil, fmt.Errorf("key not found") - } - - // 3. 读取数据 - if dataOffset+int64(dataSize) > int64(len(r.mmap)) { - return nil, fmt.Errorf("invalid data offset") - } - - compressed := r.mmap[dataOffset : dataOffset+int64(dataSize)] - - // 4. 解压缩 - var data []byte - var err error - if r.header.Compression == CompressionSnappy { - data, err = snappy.Decode(nil, compressed) - if err != nil { - return nil, err - } - } else { - data = compressed - } - - // 5. 反序列化 - row, err := decodeRow(data) - if err != nil { - return nil, err - } - - return row, nil -} - -// GetHeader 获取文件头信息 -func (r *Reader) GetHeader() *Header { - return r.header -} - -// GetPath 获取文件路径 -func (r *Reader) GetPath() string { - return r.path -} - -// GetAllKeys 获取文件中所有的 key（按顺序） -func (r *Reader) GetAllKeys() []int64 { - return r.btReader.GetAllKeys() -} - -// Close 关闭读取器 -func (r *Reader) Close() error { - if r.mmap != nil { - r.mmap.Unmap() - } - if r.file != nil { - return r.file.Close() - } - return nil -} - -// decodeRow 解码行数据 -func decodeRow(data []byte) (*Row, error) { - // 尝试使用二进制格式解码 - row, err := decodeRowBinary(data) - if err == nil { - return row, nil - } - - // 降级到 JSON (兼容旧数据) - var decoded map[string]interface{} - err = json.Unmarshal(data, &decoded) - if err != nil { - return nil, err - } - - row = &Row{ - Seq: int64(decoded["_seq"].(float64)), - Time: int64(decoded["_time"].(float64)), - Data: decoded["data"].(map[string]interface{}), - } - - return row, nil -} diff --git a/sst/sst_test.go b/sst/sst_test.go deleted file mode 100644 index 5195d3e..0000000 --- a/sst/sst_test.go +++ /dev/null @@ -1,183 +0,0 @@ -package sst - -import ( - "os" - "testing" -) - -func TestSST(t *testing.T) { - // 1. 创建测试文件 - file, err := os.Create("test.sst") - if err != nil { - t.Fatal(err) - } - defer os.Remove("test.sst") - - // 2. 写入数据 - writer := NewWriter(file) - - // 添加 1000 行数据 - for i := int64(1); i <= 1000; i++ { - row := &Row{ - Seq: i, - Time: 1000000 + i, - Data: map[string]interface{}{ - "name": "user_" + string(rune(i)), - "age": 20 + i%50, - }, - } - err := writer.Add(row) - if err != nil { - t.Fatal(err) - } - } - - // 完成写入 - err = writer.Finish() - if err != nil { - t.Fatal(err) - } - - file.Close() - - t.Logf("Written 1000 rows") - - // 3. 读取数据 - reader, err := NewReader("test.sst") - if err != nil { - t.Fatal(err) - } - defer reader.Close() - - // 验证 Header - header := reader.GetHeader() - if header.RowCount != 1000 { - t.Errorf("Expected 1000 rows, got %d", header.RowCount) - } - if header.MinKey != 1 { - t.Errorf("Expected MinKey=1, got %d", header.MinKey) - } - if header.MaxKey != 1000 { - t.Errorf("Expected MaxKey=1000, got %d", header.MaxKey) - } - - t.Logf("Header: RowCount=%d, MinKey=%d, MaxKey=%d", - header.RowCount, header.MinKey, header.MaxKey) - - // 4. 查询测试 - for i := int64(1); i <= 1000; i++ { - row, err := reader.Get(i) - if err != nil { - t.Errorf("Failed to get key %d: %v", i, err) - continue - } - if row.Seq != i { - t.Errorf("Key %d: expected Seq=%d, got %d", i, i, row.Seq) - } - if row.Time != 1000000+i { - t.Errorf("Key %d: expected Time=%d, got %d", i, 1000000+i, row.Time) - } - } - - // 测试不存在的 key - _, err = reader.Get(1001) - if err == nil { - t.Error("Key 1001 should not exist") - } - - _, err = reader.Get(0) - if err == nil { - t.Error("Key 0 should not exist") - } - - t.Log("All tests passed!") -} - -func TestHeaderSerialization(t *testing.T) { - // 创建 Header - header := &Header{ - Magic: MagicNumber, - Version: Version, - Compression: CompressionSnappy, - IndexOffset: 256, - IndexSize: 1024, - RootOffset: 512, - DataOffset: 2048, - DataSize: 10240, - RowCount: 100, - MinKey: 1, - MaxKey: 100, - MinTime: 1000000, - MaxTime: 1000100, - } - - // 序列化 - data := header.Marshal() - if len(data) != HeaderSize { - t.Errorf("Expected size %d, got %d", HeaderSize, len(data)) - } - - // 反序列化 - header2 := UnmarshalHeader(data) - if header2 == nil { - t.Fatal("Unmarshal failed") - } - - // 验证 - if header2.Magic != header.Magic { - t.Error("Magic mismatch") - } - if header2.Version != header.Version { - t.Error("Version mismatch") - } - if header2.Compression != header.Compression { - t.Error("Compression mismatch") - } - if header2.RowCount != header.RowCount { - t.Error("RowCount mismatch") - } - if header2.MinKey != header.MinKey { - t.Error("MinKey mismatch") - } - if header2.MaxKey != header.MaxKey { - t.Error("MaxKey mismatch") - } - - // 验证 - if !header2.Validate() { - t.Error("Header validation failed") - } - - t.Log("Header serialization test passed!") -} - -func BenchmarkSSTGet(b *testing.B) { - // 创建测试文件 - file, _ := os.Create("bench.sst") - defer os.Remove("bench.sst") - - writer := NewWriter(file) - for i := int64(1); i <= 10000; i++ { - row := &Row{ - Seq: i, - Time: 1000000 + i, - Data: map[string]interface{}{ - "value": i, - }, - } - writer.Add(row) - } - writer.Finish() - file.Close() - - // 打开读取器 - reader, _ := NewReader("bench.sst") - defer reader.Close() - - // 性能测试 - b.ResetTimer() - for i := 0; i < b.N; i++ { - key := int64(i%10000 + 1) - reader.Get(key) - } -} diff --git a/sst/writer.go b/sst/writer.go deleted file mode 100644 index 7508058..0000000 --- a/sst/writer.go +++ /dev/null @@ -1,155 +0,0 @@ -package sst - -import ( - "encoding/json" - "os" - - "code.tczkiot.com/srdb/btree" - "github.com/golang/snappy" -) - -// Writer SST 文件写入器 -type Writer struct { - file *os.File - builder *btree.Builder - dataOffset int64 - dataStart int64 // 数据起始位置 - rowCount int64 - minKey int64 - maxKey int64 - minTime int64 - maxTime int64 - compression uint8 -} - -// NewWriter 创建 SST 写入器 -func NewWriter(file *os.File) *Writer { - return &Writer{ - file: file, - builder: btree.NewBuilder(file, HeaderSize), - dataOffset: 0, // 先写数据，后面会更新 - compression: CompressionSnappy, - minKey: -1, - maxKey: -1, - minTime: -1, - maxTime: -1, - } -} - -// Row 表示一行数据 -type Row struct { - Seq int64 // _seq - Time int64 // _time - Data map[string]any // 用户数据 -} - -// Add 添加一行数据 -func (w *Writer) Add(row *Row) error { - // 更新统计信息 - if w.minKey == -1 || row.Seq < w.minKey { - w.minKey = row.Seq - } - if w.maxKey == -1 || row.Seq > w.maxKey { - w.maxKey = row.Seq - } - if w.minTime == -1 || row.Time < w.minTime { - w.minTime = row.Time - } - if w.maxTime == -1 || row.Time > w.maxTime { - w.maxTime = row.Time - } - w.rowCount++ - - // 序列化数据 (简单的 JSON 序列化，后续可以优化) - data := encodeRow(row) - - // 压缩数据 - var compressed []byte - if w.compression == CompressionSnappy { - compressed = snappy.Encode(nil, data) - } else { - compressed = data - } - - // 写入数据块 - // 第一次写入时，确定数据起始位置 - if w.dataStart == 0 { - // 预留足够空间给 B+Tree 索引 - // 假设索引最多占用 10% 的空间，最少 1 MB - estimatedIndexSize := int64(10 * 1024 * 1024) // 10 MB - w.dataStart = HeaderSize + estimatedIndexSize - w.dataOffset = w.dataStart - } - - offset := w.dataOffset - _, err := w.file.WriteAt(compressed, offset) - if err != nil { - return err - } - - // 添加到 B+Tree - err = w.builder.Add(row.Seq, offset, int32(len(compressed))) - if err != nil { - return err - } - - // 更新数据偏移 - w.dataOffset += int64(len(compressed)) - - return nil -} - -// Finish 完成写入 -func (w *Writer) Finish() error { - // 1. 构建 B+Tree 索引 - rootOffset, err := w.builder.Build() - if err != nil { - return err - } - - // 2. 计算索引大小 - indexSize := w.dataStart - HeaderSize - - // 3. 创建 Header - header := &Header{ - Magic: MagicNumber, - Version: Version, - Compression: w.compression, - IndexOffset: HeaderSize, - IndexSize: indexSize, - RootOffset: rootOffset, - DataOffset: w.dataStart, - DataSize: w.dataOffset - w.dataStart, - RowCount: w.rowCount, - MinKey: w.minKey, - MaxKey: w.maxKey, - MinTime: w.minTime, - MaxTime: w.maxTime, - } - - // 4. 写入 Header - headerData := header.Marshal() - _, err = w.file.WriteAt(headerData, 0) - if err != nil { - return err - } - - // 5. Sync 到磁盘 - return w.file.Sync() -} - -// encodeRow 编码行数据 (使用二进制格式) -func encodeRow(row *Row) []byte { - // 使用二进制格式编码 - encoded, err := encodeRowBinary(row) - if err != nil { - // 降级到 JSON (不应该发生) - data := map[string]interface{}{ - "_seq": row.Seq, - "_time": row.Time, - "data": row.Data, - } - encoded, _ = json.Marshal(data) - } - return encoded -} diff --git a/sstable.go b/sstable.go new file mode 100644 index 0000000..0d3931a --- /dev/null +++ b/sstable.go @@ -0,0 +1,1134 @@ +package srdb + +import ( + "bytes" + "encoding/binary" + "encoding/json" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + "sync" + + "github.com/edsrzf/mmap-go" + "github.com/golang/snappy" +) + +const ( + // 文件格式 + SSTableMagicNumber = 0x53535433 // "SST3" + SSTableVersion = 1 + SSTableHeaderSize = 256 // 文件头大小 + SSTableBlockSize = 64 * 1024 // 数据块大小 (64 KB) + + // 压缩类型 + SSTableCompressionNone = 0 + SSTableCompressionSnappy = 1 + + // 二进制编码格式: + // [Magic: 4 bytes][Seq: 8 bytes][Time: 8 bytes][DataLen: 4 bytes][Data: variable] + SSTableRowMagic = 0x524F5733 // "ROW3" +) + +// SSTableHeader SST 文件头 (256 bytes) +type SSTableHeader struct { + // 基础信息 (32 bytes) + Magic uint32 // Magic Number: 0x53535433 + Version uint32 // 版本号 + Compression uint8 // 压缩类型 + Reserved1 [3]byte + Flags uint32 // 标志位 + Reserved2 [16]byte + + // 索引信息 (32 bytes) + IndexOffset int64 // B+Tree 索引起始位置 + IndexSize int64 // B+Tree 索引大小 + RootOffset int64 // B+Tree 根节点位置 + Reserved3 [8]byte + + // 数据信息 (32 bytes) + DataOffset int64 // 数据块起始位置 + DataSize int64 // 数据块总大小 + RowCount int64 // 行数 + Reserved4 [8]byte + + // 统计信息 (32 bytes) + MinKey int64 // 最小 key (_seq) + MaxKey int64 // 最大 key (_seq) + MinTime int64 // 最小时间戳 + MaxTime int64 // 最大时间戳 + + // CRC 校验 (8 bytes) + CRC32 uint32 // Header CRC32 + Reserved5 [4]byte + + // 预留空间 (120 bytes) + Reserved6 [120]byte +} + +// Marshal 序列化 Header +func (h *SSTableHeader) Marshal() []byte { + buf := make([]byte, SSTableHeaderSize) + + // 基础信息 + binary.LittleEndian.PutUint32(buf[0:4], h.Magic) + binary.LittleEndian.PutUint32(buf[4:8], h.Version) + buf[8] = h.Compression + copy(buf[9:12], h.Reserved1[:]) + binary.LittleEndian.PutUint32(buf[12:16], h.Flags) + copy(buf[16:32], h.Reserved2[:]) + + // 索引信息 + binary.LittleEndian.PutUint64(buf[32:40], uint64(h.IndexOffset)) + binary.LittleEndian.PutUint64(buf[40:48], uint64(h.IndexSize)) + binary.LittleEndian.PutUint64(buf[48:56], uint64(h.RootOffset)) + copy(buf[56:64], h.Reserved3[:]) + + // 数据信息 + binary.LittleEndian.PutUint64(buf[64:72], uint64(h.DataOffset)) + binary.LittleEndian.PutUint64(buf[72:80], uint64(h.DataSize)) + binary.LittleEndian.PutUint64(buf[80:88], uint64(h.RowCount)) + copy(buf[88:96], h.Reserved4[:]) + + // 统计信息 + binary.LittleEndian.PutUint64(buf[96:104], uint64(h.MinKey)) + binary.LittleEndian.PutUint64(buf[104:112], uint64(h.MaxKey)) + binary.LittleEndian.PutUint64(buf[112:120], uint64(h.MinTime)) + binary.LittleEndian.PutUint64(buf[120:128], uint64(h.MaxTime)) + + // CRC 校验 + binary.LittleEndian.PutUint32(buf[128:132], h.CRC32) + copy(buf[132:136], h.Reserved5[:]) + + // 预留空间 + copy(buf[136:256], h.Reserved6[:]) + + return buf +} + +// Unmarshal 反序列化 Header +func UnmarshalSSTableHeader(data []byte) *SSTableHeader { + if len(data) < SSTableHeaderSize { + return nil + } + + h := &SSTableHeader{} + + // 基础信息 + h.Magic = binary.LittleEndian.Uint32(data[0:4]) + h.Version = binary.LittleEndian.Uint32(data[4:8]) + h.Compression = data[8] + copy(h.Reserved1[:], data[9:12]) + h.Flags = binary.LittleEndian.Uint32(data[12:16]) + copy(h.Reserved2[:], data[16:32]) + + // 索引信息 + h.IndexOffset = int64(binary.LittleEndian.Uint64(data[32:40])) + h.IndexSize = int64(binary.LittleEndian.Uint64(data[40:48])) + h.RootOffset = int64(binary.LittleEndian.Uint64(data[48:56])) + copy(h.Reserved3[:], data[56:64]) + + // 数据信息 + h.DataOffset = int64(binary.LittleEndian.Uint64(data[64:72])) + h.DataSize = int64(binary.LittleEndian.Uint64(data[72:80])) + h.RowCount = int64(binary.LittleEndian.Uint64(data[80:88])) + copy(h.Reserved4[:], data[88:96]) + + // 统计信息 + h.MinKey = int64(binary.LittleEndian.Uint64(data[96:104])) + h.MaxKey = int64(binary.LittleEndian.Uint64(data[104:112])) + h.MinTime = int64(binary.LittleEndian.Uint64(data[112:120])) + h.MaxTime = int64(binary.LittleEndian.Uint64(data[120:128])) + + // CRC 校验 + h.CRC32 = binary.LittleEndian.Uint32(data[128:132]) + copy(h.Reserved5[:], data[132:136]) + + // 预留空间 + copy(h.Reserved6[:], data[136:256]) + + return h +} + +// Validate 验证 Header +func (h *SSTableHeader) Validate() bool { + return h.Magic == SSTableMagicNumber && h.Version == SSTableVersion +} + +// encodeSSTableRowBinary 使用二进制格式编码行数据（按字段压缩） +func encodeSSTableRowBinary(row *SSTableRow, schema *Schema) ([]byte, error) { + buf := new(bytes.Buffer) + + // 写入 Magic Number (用于验证) + if err := binary.Write(buf, binary.LittleEndian, uint32(SSTableRowMagic)); err != nil { + return nil, err + } + + // 写入 Seq + if err := binary.Write(buf, binary.LittleEndian, row.Seq); err != nil { + return nil, err + } + + // 写入 Time + if err := binary.Write(buf, binary.LittleEndian, row.Time); err != nil { + return nil, err + } + + // 如果没有 Schema，回退到 JSON 编码（只编码 Data 部分，Seq/Time 已经写入） + if schema == nil { + dataJSON, err := json.Marshal(row.Data) + if err != nil { + return nil, err + } + buf.Write(dataJSON) + return buf.Bytes(), nil + } + + // 按字段分别编码和压缩 + fieldCount := uint16(len(schema.Fields)) + if err := binary.Write(buf, binary.LittleEndian, fieldCount); err != nil { + return nil, err + } + + // 1. 先编码所有字段到各自的 buffer + compressedFields := make([][]byte, len(schema.Fields)) + + for i, field := range schema.Fields { + fieldBuf := new(bytes.Buffer) + value, exists := row.Data[field.Name] + + if !exists { + // 字段不存在，写入零值 + if err := writeFieldZeroValue(fieldBuf, field.Type); err != nil { + return nil, fmt.Errorf("write zero value for field %s: %w", field.Name, err) + } + } else { + if err := writeFieldBinaryValue(fieldBuf, field.Type, value); err != nil { + return nil, fmt.Errorf("write field %s: %w", field.Name, err) + } + } + + // 压缩每个字段 + compressedFields[i] = snappy.Encode(nil, fieldBuf.Bytes()) + } + + // 2. 写入字段偏移表（相对于数据区起始位置） + currentOffset := 0 + + for _, compressed := range compressedFields { + // 写入字段偏移（相对于数据区） + if err := binary.Write(buf, binary.LittleEndian, uint32(currentOffset)); err != nil { + return nil, err + } + // 写入压缩后大小 + if err := binary.Write(buf, binary.LittleEndian, uint32(len(compressed))); err != nil { + return nil, err + } + currentOffset += len(compressed) + } + + // 3. 写入压缩后的字段数据 + for _, compressed := range compressedFields { + if _, err := buf.Write(compressed); err != nil { + return nil, err + } + } + + return buf.Bytes(), nil +} + +// writeFieldBinaryValue 写入字段值（二进制格式） +func writeFieldBinaryValue(buf *bytes.Buffer, typ FieldType, value any) error { + switch typ { + case FieldTypeInt64: + var v int64 + switch val := value.(type) { + case int: + v = int64(val) + case int64: + v = val + case int32: + v = int64(val) + case int16: + v = int64(val) + case int8: + v = int64(val) + case float64: + v = int64(val) + default: + return fmt.Errorf("cannot convert %T to int64", value) + } + return binary.Write(buf, binary.LittleEndian, v) + + case FieldTypeFloat: + var v float64 + switch val := value.(type) { + case float64: + v = val + case float32: + v = float64(val) + default: + return fmt.Errorf("cannot convert %T to float64", value) + } + return binary.Write(buf, binary.LittleEndian, v) + + case FieldTypeBool: + var b byte + if value.(bool) { + b = 1 + } + return buf.WriteByte(b) + + case FieldTypeString: + s := value.(string) + // 写入长度 + if err := binary.Write(buf, binary.LittleEndian, uint32(len(s))); err != nil { + return err + } + // 写入内容 + _, err := buf.WriteString(s) + return err + + default: + return fmt.Errorf("unsupported field type: %d", typ) + } +} + +// writeFieldZeroValue 写入字段零值 +func writeFieldZeroValue(buf *bytes.Buffer, typ FieldType) error { + switch typ { + case FieldTypeInt64: + return binary.Write(buf, binary.LittleEndian, int64(0)) + case FieldTypeFloat: + return binary.Write(buf, binary.LittleEndian, float64(0)) + case FieldTypeBool: + return buf.WriteByte(0) + case FieldTypeString: + return binary.Write(buf, binary.LittleEndian, uint32(0)) + default: + return fmt.Errorf("unsupported field type: %d", typ) + } +} + +// decodeSSTableRowBinary 解码二进制格式的行数据（完整解码） +func decodeSSTableRowBinary(data []byte, schema *Schema) (*SSTableRow, error) { + return decodeSSTableRowBinaryPartial(data, schema, nil) +} + +// decodeSSTableRowBinaryPartial 按需解码（只读取和解压指定字段） +func decodeSSTableRowBinaryPartial(data []byte, schema *Schema, fields []string) (*SSTableRow, error) { + buf := bytes.NewReader(data) + + // 读取并验证 Magic Number + var magic uint32 + if err := binary.Read(buf, binary.LittleEndian, &magic); err != nil { + return nil, err + } + if magic != SSTableRowMagic { + return nil, fmt.Errorf("invalid row magic: %x", magic) + } + + row := &SSTableRow{Data: make(map[string]any)} + + // 读取 Seq + if err := binary.Read(buf, binary.LittleEndian, &row.Seq); err != nil { + return nil, err + } + + // 读取 Time + if err := binary.Read(buf, binary.LittleEndian, &row.Time); err != nil { + return nil, err + } + + // 如果没有 Schema，回退到 JSON 解码（读取剩余的 JSON 数据） + if schema == nil { + remainingData := data[16:] // Skip Seq (8 bytes) + Time (8 bytes) + row.Data = make(map[string]any) + if err := json.Unmarshal(remainingData, &row.Data); err != nil { + return nil, err + } + return row, nil + } + + // 读取字段数量 + var fieldCount uint16 + if err := binary.Read(buf, binary.LittleEndian, &fieldCount); err != nil { + return nil, err + } + + // 读取字段偏移表 + type fieldInfo struct { + offset uint32 + size uint32 + } + fieldInfos := make([]fieldInfo, fieldCount) + for i := range fieldInfos { + if err := binary.Read(buf, binary.LittleEndian, &fieldInfos[i].offset); err != nil { + return nil, err + } + if err := binary.Read(buf, binary.LittleEndian, &fieldInfos[i].size); err != nil { + return nil, err + } + } + + // 构建需要读取的字段集合 + needFields := make(map[string]bool) + if fields == nil { + // nil 表示读取所有字段 + for _, f := range schema.Fields { + needFields[f.Name] = true + } + } else { + for _, f := range fields { + needFields[f] = true + } + } + + // 数据区起始位置（当前 buf 位置） + dataStart := buf.Size() - int64(buf.Len()) + + // 按需读取和解压字段 + for i, field := range schema.Fields { + if i >= int(fieldCount) { + break + } + + need := needFields[field.Name] + if !need { + // 跳过不需要的字段（不读取，不解压） + continue + } + + // 读取压缩的字段数据 + info := fieldInfos[i] + compressedPos := dataStart + int64(info.offset) + + // Seek 到字段位置 + if _, err := buf.Seek(compressedPos, 0); err != nil { + return nil, fmt.Errorf("seek to field %s: %w", field.Name, err) + } + + compressedData := make([]byte, info.size) + if _, err := buf.Read(compressedData); err != nil { + return nil, fmt.Errorf("read field %s: %w", field.Name, err) + } + + // 解压字段数据 + decompressed, err := snappy.Decode(nil, compressedData) + if err != nil { + return nil, fmt.Errorf("decompress field %s: %w", field.Name, err) + } + + // 解析字段值 + fieldBuf := bytes.NewReader(decompressed) + value, err := readFieldBinaryValue(fieldBuf, field.Type, true) + if err != nil { + return nil, fmt.Errorf("parse field %s: %w", field.Name, err) + } + + if value != nil { + row.Data[field.Name] = value + } + } + + return row, nil +} + +// readFieldBinaryValue 读取字段值（二进制格式） +func readFieldBinaryValue(buf *bytes.Reader, typ FieldType, keep bool) (any, error) { + switch typ { + case FieldTypeInt64: + var v int64 + if err := binary.Read(buf, binary.LittleEndian, &v); err != nil { + return nil, err + } + if keep { + return v, nil + } + return nil, nil + + case FieldTypeFloat: + var v float64 + if err := binary.Read(buf, binary.LittleEndian, &v); err != nil { + return nil, err + } + if keep { + return v, nil + } + return nil, nil + + case FieldTypeBool: + b, err := buf.ReadByte() + if err != nil { + return nil, err + } + if keep { + return b == 1, nil + } + return nil, nil + + case FieldTypeString: + var length uint32 + if err := binary.Read(buf, binary.LittleEndian, &length); err != nil { + return nil, err + } + str := make([]byte, length) + if _, err := buf.Read(str); err != nil { + return nil, err + } + if keep { + return string(str), nil + } + return nil, nil + + default: + return nil, fmt.Errorf("unsupported field type: %d", typ) + } +} + +// SSTableWriter SST 文件写入器 +type SSTableWriter struct { + file *os.File + builder *BTreeBuilder + dataOffset int64 + dataStart int64 // 数据起始位置 + rowCount int64 + minKey int64 + maxKey int64 + minTime int64 + maxTime int64 + compression uint8 + schema *Schema // Schema 用于优化编码 +} + +// NewSSTableWriter 创建 SST 写入器 +func NewSSTableWriter(file *os.File, schema *Schema) *SSTableWriter { + return &SSTableWriter{ + file: file, + builder: NewBTreeBuilder(file, SSTableHeaderSize), + dataOffset: 0, // 先写数据，后面会更新 + compression: SSTableCompressionSnappy, + minKey: -1, + maxKey: -1, + minTime: -1, + maxTime: -1, + schema: schema, + } +} + +// SSTableRow 表示一行数据 +type SSTableRow struct { + Seq int64 // _seq + Time int64 // _time + Data map[string]any // 用户数据 +} + +// Add 添加一行数据 +func (w *SSTableWriter) Add(row *SSTableRow) error { + // 更新统计信息 + if w.minKey == -1 || row.Seq < w.minKey { + w.minKey = row.Seq + } + if w.maxKey == -1 || row.Seq > w.maxKey { + w.maxKey = row.Seq + } + if w.minTime == -1 || row.Time < w.minTime { + w.minTime = row.Time + } + if w.maxTime == -1 || row.Time > w.maxTime { + w.maxTime = row.Time + } + w.rowCount++ + + // 序列化数据（使用 Schema 优化的二进制格式） + data := encodeSSTableRow(row, w.schema) + + // 压缩数据 + var compressed []byte + if w.compression == SSTableCompressionSnappy { + compressed = snappy.Encode(nil, data) + } else { + compressed = data + } + + // 写入数据块 + // 第一次写入时，确定数据起始位置 + if w.dataStart == 0 { + // 预留足够空间给 B+Tree 索引 + // 假设索引最多占用 10% 的空间，最少 1 MB + estimatedIndexSize := int64(10 * 1024 * 1024) // 10 MB + w.dataStart = SSTableHeaderSize + estimatedIndexSize + w.dataOffset = w.dataStart + } + + offset := w.dataOffset + _, err := w.file.WriteAt(compressed, offset) + if err != nil { + return err + } + + // 添加到 B+Tree + err = w.builder.Add(row.Seq, offset, int32(len(compressed))) + if err != nil { + return err + } + + // 更新数据偏移 + w.dataOffset += int64(len(compressed)) + + return nil +} + +// Finish 完成写入 +func (w *SSTableWriter) Finish() error { + // 1. 构建 B+Tree 索引 + rootOffset, err := w.builder.Build() + if err != nil { + return err + } + + // 2. 计算索引大小 + indexSize := w.dataStart - SSTableHeaderSize + + // 3. 创建 Header + header := &SSTableHeader{ + Magic: SSTableMagicNumber, + Version: SSTableVersion, + Compression: w.compression, + IndexOffset: SSTableHeaderSize, + IndexSize: indexSize, + RootOffset: rootOffset, + DataOffset: w.dataStart, + DataSize: w.dataOffset - w.dataStart, + RowCount: w.rowCount, + MinKey: w.minKey, + MaxKey: w.maxKey, + MinTime: w.minTime, + MaxTime: w.maxTime, + } + + // 4. 写入 Header + headerData := header.Marshal() + _, err = w.file.WriteAt(headerData, 0) + if err != nil { + return err + } + + // 5. Sync 到磁盘 + return w.file.Sync() +} + +// encodeSSTableRow 编码行数据 (使用二进制格式) +func encodeSSTableRow(row *SSTableRow, schema *Schema) []byte { + // 使用二进制格式编码 + encoded, err := encodeSSTableRowBinary(row, schema) + if err != nil { + // 降级到 JSON (不应该发生) + data := map[string]any{ + "_seq": row.Seq, + "_time": row.Time, + "data": row.Data, + } + encoded, _ = json.Marshal(data) + } + return encoded +} + +// SSTableReader SST 文件读取器 +type SSTableReader struct { + path string + file *os.File + mmap mmap.MMap + header *SSTableHeader + btReader *BTreeReader + schema *Schema // Schema 用于优化解码 +} + +// NewSSTableReader 创建 SST 读取器 +func NewSSTableReader(path string) (*SSTableReader, error) { + // 1. 打开文件 + file, err := os.Open(path) + if err != nil { + return nil, err + } + + // 2. mmap 映射 + mmapData, err := mmap.Map(file, mmap.RDONLY, 0) + if err != nil { + file.Close() + return nil, err + } + + // 3. 读取 Header + if len(mmapData) < SSTableHeaderSize { + mmapData.Unmap() + file.Close() + return nil, fmt.Errorf("file too small") + } + + header := UnmarshalSSTableHeader(mmapData[:SSTableHeaderSize]) + if header == nil || !header.Validate() { + mmapData.Unmap() + file.Close() + return nil, fmt.Errorf("invalid header") + } + + // 4. 创建 B+Tree Reader + btReader := NewBTreeReader(mmapData, header.RootOffset) + + return &SSTableReader{ + path: path, + file: file, + mmap: mmapData, + header: header, + btReader: btReader, + }, nil +} + +// Get 查询一行数据 +func (r *SSTableReader) Get(key int64) (*SSTableRow, error) { + // 1. 检查范围 + if key < r.header.MinKey || key > r.header.MaxKey { + return nil, fmt.Errorf("key out of range") + } + + // 2. 在 B+Tree 中查找 + dataOffset, dataSize, found := r.btReader.Get(key) + if !found { + return nil, fmt.Errorf("key not found") + } + + // 3. 读取数据 + if dataOffset+int64(dataSize) > int64(len(r.mmap)) { + return nil, fmt.Errorf("invalid data offset") + } + + compressed := r.mmap[dataOffset : dataOffset+int64(dataSize)] + + // 4. 解压缩 + var data []byte + var err error + if r.header.Compression == SSTableCompressionSnappy { + data, err = snappy.Decode(nil, compressed) + if err != nil { + return nil, err + } + } else { + data = compressed + } + + // 5. 反序列化 + row, err := decodeSSTableRow(data, r.schema) + if err != nil { + return nil, err + } + + return row, nil +} + +// GetPartial 按需查询一行数据（只读取指定字段） +func (r *SSTableReader) GetPartial(key int64, fields []string) (*SSTableRow, error) { + // 1. 检查范围 + if key < r.header.MinKey || key > r.header.MaxKey { + return nil, fmt.Errorf("key out of range") + } + + // 2. 在 B+Tree 中查找 + dataOffset, dataSize, found := r.btReader.Get(key) + if !found { + return nil, fmt.Errorf("key not found") + } + + // 3. 读取数据 + if dataOffset+int64(dataSize) > int64(len(r.mmap)) { + return nil, fmt.Errorf("invalid data offset") + } + + compressed := r.mmap[dataOffset : dataOffset+int64(dataSize)] + + // 4. 解压缩 + var data []byte + var err error + if r.header.Compression == SSTableCompressionSnappy { + data, err = snappy.Decode(nil, compressed) + if err != nil { + return nil, err + } + } else { + data = compressed + } + + // 5. 按需反序列化（只解析需要的字段） + row, err := decodeSSTableRowBinaryPartial(data, r.schema, fields) + if err != nil { + return nil, err + } + + return row, nil +} + +// SetSchema 设置 Schema（用于优化编解码） +func (r *SSTableReader) SetSchema(schema *Schema) { + r.schema = schema +} + +// GetHeader 获取文件头信息 +func (r *SSTableReader) GetHeader() *SSTableHeader { + return r.header +} + +// GetPath 获取文件路径 +func (r *SSTableReader) GetPath() string { + return r.path +} + +// GetAllKeys 获取文件中所有的 key（按顺序） +func (r *SSTableReader) GetAllKeys() []int64 { + return r.btReader.GetAllKeys() +} + +// Close 关闭读取器 +func (r *SSTableReader) Close() error { + if r.mmap != nil { + r.mmap.Unmap() + } + if r.file != nil { + return r.file.Close() + } + return nil +} + +// decodeSSTableRow 解码行数据 +func decodeSSTableRow(data []byte, schema *Schema) (*SSTableRow, error) { + // 尝试使用二进制格式解码 + row, err := decodeSSTableRowBinary(data, schema) + if err == nil { + return row, nil + } + + // 降级到 JSON (兼容旧数据) + var decoded map[string]any + err = json.Unmarshal(data, &decoded) + if err != nil { + return nil, err + } + + row = &SSTableRow{ + Seq: int64(decoded["_seq"].(float64)), + Time: int64(decoded["_time"].(float64)), + Data: decoded["data"].(map[string]any), + } + + return row, nil +} + +// SSTableManager SST 文件管理器 +type SSTableManager struct { + dir string + readers []*SSTableReader + mu sync.RWMutex + schema *Schema // Schema 用于优化编解码 +} + +// NewSSTableManager 创建 SST 管理器 +func NewSSTableManager(dir string) (*SSTableManager, error) { + // 确保目录存在 + err := os.MkdirAll(dir, 0755) + if err != nil { + return nil, err + } + + mgr := &SSTableManager{ + dir: dir, + readers: make([]*SSTableReader, 0), + } + + // 恢复现有的 SST 文件 + err = mgr.recover() + if err != nil { + return nil, err + } + + return mgr, nil +} + +// recover 恢复现有的 SST 文件 +func (m *SSTableManager) recover() error { + // 查找所有 SST 文件 + files, err := filepath.Glob(filepath.Join(m.dir, "*.sst")) + if err != nil { + return err + } + + for _, file := range files { + // 跳过索引文件 + filename := filepath.Base(file) + if strings.HasPrefix(filename, "idx_") { + continue + } + + // 打开 SST Reader + reader, err := NewSSTableReader(file) + if err != nil { + return err + } + + // 设置 Schema + if m.schema != nil { + reader.SetSchema(m.schema) + } + + m.readers = append(m.readers, reader) + } + + return nil +} + +// CreateSST 创建新的 SST 文件 +// fileNumber: 文件编号（由 VersionSet 分配） +func (m *SSTableManager) CreateSST(fileNumber int64, rows []*SSTableRow) (*SSTableReader, error) { + return m.CreateSSTWithLevel(fileNumber, rows, 0) // 默认创建到 L0 +} + +// CreateSSTWithLevel 创建新的 SST 文件到指定层级 +// fileNumber: 文件编号（由 VersionSet 分配） +func (m *SSTableManager) CreateSSTWithLevel(fileNumber int64, rows []*SSTableRow, level int) (*SSTableReader, error) { + m.mu.Lock() + defer m.mu.Unlock() + + sstPath := filepath.Join(m.dir, fmt.Sprintf("%06d.sst", fileNumber)) + + // 创建文件 + file, err := os.Create(sstPath) + if err != nil { + return nil, err + } + + writer := NewSSTableWriter(file, m.schema) + + // 写入所有行 + for _, row := range rows { + err = writer.Add(row) + if err != nil { + file.Close() + os.Remove(sstPath) + return nil, err + } + } + + // 完成写入 + err = writer.Finish() + if err != nil { + file.Close() + os.Remove(sstPath) + return nil, err + } + + file.Close() + + // 打开 SST Reader + reader, err := NewSSTableReader(sstPath) + if err != nil { + return nil, err + } + + // 设置 Schema + if m.schema != nil { + reader.SetSchema(m.schema) + } + + // 添加到 readers 列表 + m.readers = append(m.readers, reader) + + return reader, nil +} + +// SetSchema 设置 Schema（用于优化编解码） +func (m *SSTableManager) SetSchema(schema *Schema) { + m.mu.Lock() + defer m.mu.Unlock() + + m.schema = schema + + // 为所有已存在的 readers 设置 schema + for _, reader := range m.readers { + reader.SetSchema(schema) + } +} + +// Get 从所有 SST 文件中查找数据 +func (m *SSTableManager) Get(seq int64) (*SSTableRow, error) { + m.mu.RLock() + defer m.mu.RUnlock() + + // 从后往前查找（新的文件优先） + for i := len(m.readers) - 1; i >= 0; i-- { + reader := m.readers[i] + row, err := reader.Get(seq) + if err == nil { + return row, nil + } + } + + return nil, fmt.Errorf("key not found: %d", seq) +} + +// GetPartial 从所有 SST 文件中按需查找数据（只读取指定字段） +func (m *SSTableManager) GetPartial(seq int64, fields []string) (*SSTableRow, error) { + m.mu.RLock() + defer m.mu.RUnlock() + + // 从后往前查找（新的文件优先） + for i := len(m.readers) - 1; i >= 0; i-- { + reader := m.readers[i] + row, err := reader.GetPartial(seq, fields) + if err == nil { + return row, nil + } + } + + return nil, fmt.Errorf("key not found: %d", seq) +} + +// RemoveReader 移除指定文件编号的 reader（用于 compaction） +func (m *SSTableManager) RemoveReader(fileNumber int64) error { + m.mu.Lock() + defer m.mu.Unlock() + + // 查找并移除对应的 reader + for i, reader := range m.readers { + // 从文件名中提取文件编号 + filename := filepath.Base(reader.path) + var readerFileNum int64 + if _, err := fmt.Sscanf(filename, "%d.sst", &readerFileNum); err == nil { + if readerFileNum == fileNumber { + // 关闭 reader + reader.Close() + + // 从列表中移除 + m.readers = append(m.readers[:i], m.readers[i+1:]...) + return nil + } + } + } + + return fmt.Errorf("reader for file %d not found", fileNumber) +} + +// AddReader 添加 reader 到管理器（用于 compaction 创建的新文件） +func (m *SSTableManager) AddReader(reader *SSTableReader) { + m.mu.Lock() + defer m.mu.Unlock() + + m.readers = append(m.readers, reader) +} + +// GetReaders 获取所有 Readers（用于扫描） +func (m *SSTableManager) GetReaders() []*SSTableReader { + m.mu.RLock() + defer m.mu.RUnlock() + + // 返回副本 + readers := make([]*SSTableReader, len(m.readers)) + copy(readers, m.readers) + + // 按 MinKey 排序，确保查询时按 seq 顺序遍历 + // 这对于 compaction 后的文件顺序至关重要： + // compaction 生成的新文件包含旧 seq，但被添加到 readers 末尾 + // 排序后保证查询结果有序 + sort.Slice(readers, func(i, j int) bool { + return readers[i].header.MinKey < readers[j].header.MinKey + }) + + return readers +} + +// GetMaxSeq 获取所有 SST 中的最大 seq +func (m *SSTableManager) GetMaxSeq() int64 { + m.mu.RLock() + defer m.mu.RUnlock() + + maxSeq := int64(0) + for _, reader := range m.readers { + header := reader.GetHeader() + if header.MaxKey > maxSeq { + maxSeq = header.MaxKey + } + } + + return maxSeq +} + +// Count 获取 SST 文件数量 +func (m *SSTableManager) Count() int { + m.mu.RLock() + defer m.mu.RUnlock() + + return len(m.readers) +} + +// ListFiles 列出所有 SST 文件 +func (m *SSTableManager) ListFiles() []string { + m.mu.RLock() + defer m.mu.RUnlock() + + files := make([]string, 0, len(m.readers)) + for _, reader := range m.readers { + files = append(files, reader.path) + } + + return files +} + +// Close 关闭所有 SST Readers +func (m *SSTableManager) Close() error { + m.mu.Lock() + defer m.mu.Unlock() + + for _, reader := range m.readers { + reader.Close() + } + + m.readers = nil + return nil +} + +// SSTableStats 统计信息 +type SSTableStats struct { + FileCount int + TotalSize int64 + MinSeq int64 + MaxSeq int64 +} + +// GetStats 获取统计信息 +func (m *SSTableManager) GetStats() *SSTableStats { + m.mu.RLock() + defer m.mu.RUnlock() + + stats := &SSTableStats{ + FileCount: len(m.readers), + MinSeq: -1, + MaxSeq: -1, + } + + for _, reader := range m.readers { + header := reader.GetHeader() + + if stats.MinSeq == -1 || header.MinKey < stats.MinSeq { + stats.MinSeq = header.MinKey + } + + if stats.MaxSeq == -1 || header.MaxKey > stats.MaxSeq { + stats.MaxSeq = header.MaxKey + } + + // 获取文件大小 + if stat, err := os.Stat(reader.path); err == nil { + stats.TotalSize += stat.Size() + } + } + + return stats +} diff --git a/sstable_test.go b/sstable_test.go new file mode 100644 index 0000000..f6ba7bc --- /dev/null +++ b/sstable_test.go @@ -0,0 +1,466 @@ +package srdb + +import ( + "encoding/json" + "fmt" + "os" + "testing" +) + +func TestSSTable(t *testing.T) { + // 1. 创建测试文件 + file, err := os.Create("test.sst") + if err != nil { + t.Fatal(err) + } + defer os.Remove("test.sst") + + // 2. 写入数据 + writer := NewSSTableWriter(file, nil) + + // 添加 1000 行数据 + for i := int64(1); i <= 1000; i++ { + row := &SSTableRow{ + Seq: i, + Time: 1000000 + i, + Data: map[string]any{ + "name": "user_" + string(rune(i)), + "age": 20 + i%50, + }, + } + err := writer.Add(row) + if err != nil { + t.Fatal(err) + } + } + + // 完成写入 + err = writer.Finish() + if err != nil { + t.Fatal(err) + } + + file.Close() + + t.Logf("Written 1000 rows") + + // 3. 读取数据 + reader, err := NewSSTableReader("test.sst") + if err != nil { + t.Fatal(err) + } + defer reader.Close() + + // 验证 Header + header := reader.GetHeader() + if header.RowCount != 1000 { + t.Errorf("Expected 1000 rows, got %d", header.RowCount) + } + if header.MinKey != 1 { + t.Errorf("Expected MinKey=1, got %d", header.MinKey) + } + if header.MaxKey != 1000 { + t.Errorf("Expected MaxKey=1000, got %d", header.MaxKey) + } + + t.Logf("Header: RowCount=%d, MinKey=%d, MaxKey=%d", + header.RowCount, header.MinKey, header.MaxKey) + + // 4. 查询测试 + for i := int64(1); i <= 1000; i++ { + row, err := reader.Get(i) + if err != nil { + t.Errorf("Failed to get key %d: %v", i, err) + continue + } + if row.Seq != i { + t.Errorf("Key %d: expected Seq=%d, got %d", i, i, row.Seq) + } + if row.Time != 1000000+i { + t.Errorf("Key %d: expected Time=%d, got %d", i, 1000000+i, row.Time) + } + } + + // 测试不存在的 key + _, err = reader.Get(1001) + if err == nil { + t.Error("Key 1001 should not exist") + } + + _, err = reader.Get(0) + if err == nil { + t.Error("Key 0 should not exist") + } + + t.Log("All tests passed!") +} + +func TestSSTableHeaderSerialization(t *testing.T) { + // 创建 Header + header := &SSTableHeader{ + Magic: SSTableMagicNumber, + Version: SSTableVersion, + Compression: SSTableCompressionSnappy, + IndexOffset: 256, + IndexSize: 1024, + RootOffset: 512, + DataOffset: 2048, + DataSize: 10240, + RowCount: 100, + MinKey: 1, + MaxKey: 100, + MinTime: 1000000, + MaxTime: 1000100, + } + + // 序列化 + data := header.Marshal() + if len(data) != SSTableHeaderSize { + t.Errorf("Expected size %d, got %d", SSTableHeaderSize, len(data)) + } + + // 反序列化 + header2 := UnmarshalSSTableHeader(data) + if header2 == nil { + t.Fatal("Unmarshal failed") + } + + // 验证 + if header2.Magic != header.Magic { + t.Error("Magic mismatch") + } + if header2.Version != header.Version { + t.Error("Version mismatch") + } + if header2.Compression != header.Compression { + t.Error("Compression mismatch") + } + if header2.RowCount != header.RowCount { + t.Error("RowCount mismatch") + } + if header2.MinKey != header.MinKey { + t.Error("MinKey mismatch") + } + if header2.MaxKey != header.MaxKey { + t.Error("MaxKey mismatch") + } + + // 验证 + if !header2.Validate() { + t.Error("Header validation failed") + } + + t.Log("Header serialization test passed!") +} + +func BenchmarkSSTableGet(b *testing.B) { + // 创建测试文件 + file, _ := os.Create("bench.sst") + defer os.Remove("bench.sst") + + writer := NewSSTableWriter(file, nil) + for i := int64(1); i <= 10000; i++ { + row := &SSTableRow{ + Seq: i, + Time: 1000000 + i, + Data: map[string]any{ + "value": i, + }, + } + writer.Add(row) + } + writer.Finish() + file.Close() + + // 打开读取器 + reader, _ := NewSSTableReader("bench.sst") + defer reader.Close() + + // 性能测试 + + for i := 0; b.Loop(); i++ { + key := int64(i%10000 + 1) + reader.Get(key) + } +} + +func TestSSTableBinaryEncoding(t *testing.T) { + // 创建 Schema + schema := &Schema{ + Name: "users", + Fields: []Field{ + {Name: "name", Type: FieldTypeString}, + {Name: "age", Type: FieldTypeInt64}, + {Name: "email", Type: FieldTypeString}, + }, + } + + // 创建测试数据 + row := &SSTableRow{ + Seq: 12345, + Time: 1234567890, + Data: map[string]any{ + "name": "test_user", + "age": int64(25), + "email": "test@example.com", + }, + } + + // 编码 + encoded, err := encodeSSTableRowBinary(row, schema) + if err != nil { + t.Fatal(err) + } + + t.Logf("Encoded size: %d bytes", len(encoded)) + + // 解码 + decoded, err := decodeSSTableRowBinary(encoded, schema) + if err != nil { + t.Fatal(err) + } + + // 验证 + if decoded.Seq != row.Seq { + t.Errorf("Seq mismatch: expected %d, got %d", row.Seq, decoded.Seq) + } + if decoded.Time != row.Time { + t.Errorf("Time mismatch: expected %d, got %d", row.Time, decoded.Time) + } + if decoded.Data["name"] != row.Data["name"] { + t.Errorf("Name mismatch") + } + + t.Log("Binary encoding test passed!") +} + +func TestSSTableEncodingComparison(t *testing.T) { + // 创建 Schema + schema := &Schema{ + Name: "users", + Fields: []Field{ + {Name: "name", Type: FieldTypeString}, + {Name: "age", Type: FieldTypeInt64}, + {Name: "email", Type: FieldTypeString}, + }, + } + + row := &SSTableRow{ + Seq: 12345, + Time: 1234567890, + Data: map[string]any{ + "name": "test_user", + "age": int64(25), + "email": "test@example.com", + }, + } + + // 二进制编码 + binaryEncoded, _ := encodeSSTableRowBinary(row, schema) + + // JSON 编码 (旧方式) + jsonData := map[string]any{ + "_seq": row.Seq, + "_time": row.Time, + "data": row.Data, + } + jsonEncoded, _ := json.Marshal(jsonData) + + t.Logf("Binary size: %d bytes", len(binaryEncoded)) + t.Logf("JSON size: %d bytes", len(jsonEncoded)) + t.Logf("Space saved: %.1f%%", float64(len(jsonEncoded)-len(binaryEncoded))/float64(len(jsonEncoded))*100) + + if len(binaryEncoded) >= len(jsonEncoded) { + t.Error("Binary encoding should be smaller than JSON") + } +} + +func BenchmarkSSTableBinaryEncoding(b *testing.B) { + // 创建 Schema + schema := &Schema{ + Name: "users", + Fields: []Field{ + {Name: "name", Type: FieldTypeString}, + {Name: "age", Type: FieldTypeInt64}, + {Name: "email", Type: FieldTypeString}, + }, + } + + row := &SSTableRow{ + Seq: 12345, + Time: 1234567890, + Data: map[string]any{ + "name": "test_user", + "age": int64(25), + "email": "test@example.com", + }, + } + + for b.Loop() { + encodeSSTableRowBinary(row, schema) + } +} + +func BenchmarkSSTableJSONEncoding(b *testing.B) { + row := &SSTableRow{ + Seq: 12345, + Time: 1234567890, + Data: map[string]any{ + "name": "test_user", + "age": 25, + "email": "test@example.com", + }, + } + + data := map[string]any{ + "_seq": row.Seq, + "_time": row.Time, + "data": row.Data, + } + + for b.Loop() { + json.Marshal(data) + } +} + +func TestSSTablePerFieldCompression(t *testing.T) { + // 创建 Schema + schema := &Schema{ + Name: "users", + Fields: []Field{ + {Name: "name", Type: FieldTypeString, Indexed: false}, + {Name: "age", Type: FieldTypeInt64, Indexed: false}, + {Name: "email", Type: FieldTypeString, Indexed: false}, + {Name: "score", Type: FieldTypeFloat, Indexed: false}, + }, + } + + // 创建测试数据 + row := &SSTableRow{ + Seq: 12345, + Time: 1234567890, + Data: map[string]any{ + "name": "test_user", + "age": int64(25), + "email": "test@example.com", + "score": 95.5, + }, + } + + // 使用 Schema 编码（按字段压缩） + encoded, err := encodeSSTableRowBinary(row, schema) + if err != nil { + t.Fatal(err) + } + + t.Logf("Per-field compressed size: %d bytes", len(encoded)) + + // 完整解码 + decoded, err := decodeSSTableRowBinary(encoded, schema) + if err != nil { + t.Fatal(err) + } + + // 验证完整数据 + if decoded.Seq != row.Seq { + t.Errorf("Seq mismatch: expected %d, got %d", row.Seq, decoded.Seq) + } + if decoded.Time != row.Time { + t.Errorf("Time mismatch: expected %d, got %d", row.Time, decoded.Time) + } + if decoded.Data["name"] != row.Data["name"] { + t.Errorf("Name mismatch") + } + if decoded.Data["age"] != row.Data["age"] { + t.Errorf("Age mismatch") + } + + // 部分解码 - 只读取 name 和 age 字段 + partialDecoded, err := decodeSSTableRowBinaryPartial(encoded, schema, []string{"name", "age"}) + if err != nil { + t.Fatal(err) + } + + // 验证只包含请求的字段 + if len(partialDecoded.Data) != 2 { + t.Errorf("Expected 2 fields, got %d", len(partialDecoded.Data)) + } + if _, ok := partialDecoded.Data["name"]; !ok { + t.Error("Missing field: name") + } + if _, ok := partialDecoded.Data["age"]; !ok { + t.Error("Missing field: age") + } + if _, ok := partialDecoded.Data["email"]; ok { + t.Error("Should not have field: email") + } + if _, ok := partialDecoded.Data["score"]; ok { + t.Error("Should not have field: score") + } + + // 验证字段值正确 + if partialDecoded.Data["name"] != "test_user" { + t.Errorf("Name mismatch: got %v", partialDecoded.Data["name"]) + } + if partialDecoded.Data["age"] != int64(25) { + t.Errorf("Age mismatch: got %v", partialDecoded.Data["age"]) + } + + t.Log("Per-field compression test passed!") +} + +func TestSSTablePartialReadingPerformance(t *testing.T) { + // 创建包含多个字段的 Schema + schema := &Schema{ + Name: "events", + Fields: []Field{ + {Name: "field1", Type: FieldTypeString, Indexed: false}, + {Name: "field2", Type: FieldTypeString, Indexed: false}, + {Name: "field3", Type: FieldTypeString, Indexed: false}, + {Name: "field4", Type: FieldTypeString, Indexed: false}, + {Name: "field5", Type: FieldTypeString, Indexed: false}, + {Name: "field6", Type: FieldTypeString, Indexed: false}, + {Name: "field7", Type: FieldTypeString, Indexed: false}, + {Name: "field8", Type: FieldTypeString, Indexed: false}, + {Name: "field9", Type: FieldTypeString, Indexed: false}, + {Name: "field10", Type: FieldTypeString, Indexed: false}, + }, + } + + // 创建包含大量数据��行 + largeData := make(map[string]any) + for i := 1; i <= 10; i++ { + // 每个字段包含较大的字符串数据 + largeData[fmt.Sprintf("field%d", i)] = fmt.Sprintf("This is a large data field %d with lots of content to compress", i) + + " Lorem ipsum dolor sit amet, consectetur adipiscing elit." + } + + row := &SSTableRow{ + Seq: 12345, + Time: 1234567890, + Data: largeData, + } + + // 编码 + encoded, err := encodeSSTableRowBinary(row, schema) + if err != nil { + t.Fatal(err) + } + + t.Logf("Total encoded size with 10 fields: %d bytes", len(encoded)) + + // 完整解码 + fullDecoded, _ := decodeSSTableRowBinary(encoded, schema) + t.Logf("Full decode: %d fields", len(fullDecoded.Data)) + + // 部分解码 - 只读取 1 个字段 + partialDecoded, _ := decodeSSTableRowBinaryPartial(encoded, schema, []string{"field1"}) + t.Logf("Partial decode (1 field): %d fields", len(partialDecoded.Data)) + + // 验证部分读取确实只返回请求的字段 + if len(partialDecoded.Data) != 1 { + t.Errorf("Expected 1 field, got %d", len(partialDecoded.Data)) + } + + t.Log("Partial reading performance test passed!") +} diff --git a/table.go b/table.go index 8169f88..be0f421 100644 --- a/table.go +++ b/table.go @@ -4,8 +4,6 @@ import ( "os" "path/filepath" "time" - - "code.tczkiot.com/srdb/sst" ) // Table 表 @@ -95,7 +93,7 @@ func (t *Table) Insert(data map[string]any) error { } // Get 查询数据 -func (t *Table) Get(seq int64) (*sst.Row, error) { +func (t *Table) Get(seq int64) (*SSTableRow, error) { return t.engine.Get(seq) } @@ -120,7 +118,7 @@ func (t *Table) ListIndexes() []string { } // Stats 获取统计信息 -func (t *Table) Stats() *Stats { +func (t *Table) Stats() *TableStats { return t.engine.Stats() } diff --git a/version.go b/version.go new file mode 100644 index 0000000..3e1297f --- /dev/null +++ b/version.go @@ -0,0 +1,614 @@ +package srdb + +import ( + "encoding/binary" + "encoding/json" + "fmt" + "hash/crc32" + "io" + "os" + "path/filepath" + "strings" + "sync" + "sync/atomic" +) + +// FileMetadata SST 文件元数据 +type FileMetadata struct { + FileNumber int64 // 文件编号 + Level int // 所在层级 (0-6) + FileSize int64 // 文件大小 + MinKey int64 // 最小 key + MaxKey int64 // 最大 key + RowCount int64 // 行数 +} + +const ( + NumLevels = 7 // L0-L6 +) + +// Version 数据库的一个版本快照 +type Version struct { + // 分层存储 SST 文件 (L0-L6) + Levels [NumLevels][]*FileMetadata + + // 下一个文件编号 + NextFileNumber int64 + + // 最后序列号 + LastSequence int64 + + // 版本号 + VersionNumber int64 + + mu sync.RWMutex +} + +// NewVersion 创建新版本 +func NewVersion() *Version { + v := &Version{ + NextFileNumber: 1, + LastSequence: 0, + VersionNumber: 0, + } + // 初始化每一层 + for i := range NumLevels { + v.Levels[i] = make([]*FileMetadata, 0) + } + return v +} + +// Clone 克隆版本 +func (v *Version) Clone() *Version { + v.mu.RLock() + defer v.mu.RUnlock() + + newVersion := &Version{ + NextFileNumber: v.NextFileNumber, + LastSequence: v.LastSequence, + VersionNumber: v.VersionNumber + 1, + } + + // 克隆每一层 + for level := range NumLevels { + newVersion.Levels[level] = make([]*FileMetadata, len(v.Levels[level])) + copy(newVersion.Levels[level], v.Levels[level]) + } + + return newVersion +} + +// Apply 应用版本变更 +func (v *Version) Apply(edit *VersionEdit) { + v.mu.Lock() + defer v.mu.Unlock() + + // 删除文件（按层级删除） + if len(edit.DeletedFiles) > 0 { + deleteSet := make(map[int64]bool) + for _, fileNum := range edit.DeletedFiles { + deleteSet[fileNum] = true + } + + // 遍历每一层，删除文件 + for level := range NumLevels { + newFiles := make([]*FileMetadata, 0) + for _, file := range v.Levels[level] { + if !deleteSet[file.FileNumber] { + newFiles = append(newFiles, file) + } + } + v.Levels[level] = newFiles + } + } + + // 添加文件（按层级添加） + if len(edit.AddedFiles) > 0 { + for _, file := range edit.AddedFiles { + if file.Level >= 0 && file.Level < NumLevels { + v.Levels[file.Level] = append(v.Levels[file.Level], file) + } + } + } + + // 更新下一个文件编号 + if edit.NextFileNumber != nil { + v.NextFileNumber = *edit.NextFileNumber + } + + // 更新最后序列号 + if edit.LastSequence != nil { + v.LastSequence = *edit.LastSequence + } +} + +// GetLevel 获取指定层级的文件 +func (v *Version) GetLevel(level int) []*FileMetadata { + v.mu.RLock() + defer v.mu.RUnlock() + + if level < 0 || level >= NumLevels { + return nil + } + + files := make([]*FileMetadata, len(v.Levels[level])) + copy(files, v.Levels[level]) + return files +} + +// GetSSTFiles 获取所有 SST 文件（副本，兼容旧接口） +func (v *Version) GetSSTFiles() []*FileMetadata { + v.mu.RLock() + defer v.mu.RUnlock() + + // 收集所有层级的文件 + allFiles := make([]*FileMetadata, 0) + for level := range NumLevels { + allFiles = append(allFiles, v.Levels[level]...) + } + return allFiles +} + +// GetNextFileNumber 获取下一个文件编号 +func (v *Version) GetNextFileNumber() int64 { + v.mu.RLock() + defer v.mu.RUnlock() + return v.NextFileNumber +} + +// GetLastSequence 获取最后序列号 +func (v *Version) GetLastSequence() int64 { + v.mu.RLock() + defer v.mu.RUnlock() + return v.LastSequence +} + +// GetFileCount 获取文件数量 +func (v *Version) GetFileCount() int { + v.mu.RLock() + defer v.mu.RUnlock() + + total := 0 + for level := range NumLevels { + total += len(v.Levels[level]) + } + return total +} + +// GetLevelFileCount 获取指定层级的文件数量 +func (v *Version) GetLevelFileCount(level int) int { + v.mu.RLock() + defer v.mu.RUnlock() + + if level < 0 || level >= NumLevels { + return 0 + } + return len(v.Levels[level]) +} + +// ManifestReader MANIFEST 读取器 +type ManifestReader struct { + file io.Reader +} + +// NewManifestReader 创建 MANIFEST 读取器 +func NewManifestReader(file io.Reader) *ManifestReader { + return &ManifestReader{ + file: file, + } +} + +// ReadEdit 读取版本变更 +func (r *ManifestReader) ReadEdit() (*VersionEdit, error) { + // 读取 CRC32 和 Length + header := make([]byte, 8) + _, err := io.ReadFull(r.file, header) + if err != nil { + return nil, err + } + + // 读取长度 + length := binary.LittleEndian.Uint32(header[4:8]) + + // 读取数据 + data := make([]byte, 8+length) + copy(data[0:8], header) + _, err = io.ReadFull(r.file, data[8:]) + if err != nil { + return nil, err + } + + // 解码 + edit := NewVersionEdit() + err = edit.Decode(data) + if err != nil { + return nil, err + } + + return edit, nil +} + +// ManifestWriter MANIFEST 写入器 +type ManifestWriter struct { + file io.Writer + mu sync.Mutex +} + +// NewManifestWriter 创建 MANIFEST 写入器 +func NewManifestWriter(file io.Writer) *ManifestWriter { + return &ManifestWriter{ + file: file, + } +} + +// WriteEdit 写入版本变更 +func (w *ManifestWriter) WriteEdit(edit *VersionEdit) error { + w.mu.Lock() + defer w.mu.Unlock() + + // 编码 + data, err := edit.Encode() + if err != nil { + return err + } + + // 写入 + _, err = w.file.Write(data) + return err +} + +// EditType 变更类型 +type EditType byte + +const ( + EditTypeAddFile EditType = 1 // 添加文件 + EditTypeDeleteFile EditType = 2 // 删除文件 + EditTypeSetNextFile EditType = 3 // 设置下一个文件编号 + EditTypeSetLastSeq EditType = 4 // 设置最后序列号 +) + +// VersionEdit 版本变更记录 +type VersionEdit struct { + // 添加的文件 + AddedFiles []*FileMetadata + + // 删除的文件（文件编号列表） + DeletedFiles []int64 + + // 下一个文件编号 + NextFileNumber *int64 + + // 最后序列号 + LastSequence *int64 +} + +// NewVersionEdit 创建版本变更 +func NewVersionEdit() *VersionEdit { + return &VersionEdit{ + AddedFiles: make([]*FileMetadata, 0), + DeletedFiles: make([]int64, 0), + } +} + +// AddFile 添加文件 +func (e *VersionEdit) AddFile(file *FileMetadata) { + e.AddedFiles = append(e.AddedFiles, file) +} + +// DeleteFile 删除文件 +func (e *VersionEdit) DeleteFile(fileNumber int64) { + e.DeletedFiles = append(e.DeletedFiles, fileNumber) +} + +// SetNextFileNumber 设置下一个文件编号 +func (e *VersionEdit) SetNextFileNumber(num int64) { + e.NextFileNumber = &num +} + +// SetLastSequence 设置最后序列号 +func (e *VersionEdit) SetLastSequence(seq int64) { + e.LastSequence = &seq +} + +// Encode 编码为字节 +func (e *VersionEdit) Encode() ([]byte, error) { + // 使用 JSON 编码（简单实现） + data, err := json.Marshal(e) + if err != nil { + return nil, err + } + + // 格式: CRC32(4) + Length(4) + Data + totalLen := 8 + len(data) + buf := make([]byte, totalLen) + + // 计算 CRC32 + crc := crc32.ChecksumIEEE(data) + binary.LittleEndian.PutUint32(buf[0:4], crc) + + // 写入长度 + binary.LittleEndian.PutUint32(buf[4:8], uint32(len(data))) + + // 写入数据 + copy(buf[8:], data) + + return buf, nil +} + +// Decode 从字节解码 +func (e *VersionEdit) Decode(data []byte) error { + if len(data) < 8 { + return io.ErrUnexpectedEOF + } + + // 读取 CRC32 + crc := binary.LittleEndian.Uint32(data[0:4]) + + // 读取长度 + length := binary.LittleEndian.Uint32(data[4:8]) + + if len(data) < int(8+length) { + return io.ErrUnexpectedEOF + } + + // 读取数据 + editData := data[8 : 8+length] + + // 验证 CRC32 + if crc32.ChecksumIEEE(editData) != crc { + return io.ErrUnexpectedEOF + } + + // JSON 解码 + return json.Unmarshal(editData, e) +} + +// VersionSet 版本集合管理器 +type VersionSet struct { + // 当前版本 + current *Version + + // MANIFEST 文件 + manifestFile *os.File + manifestWriter *ManifestWriter + manifestNumber int64 + + // 下一个文件编号 + nextFileNumber atomic.Int64 + + // 最后序列号 + lastSequence atomic.Int64 + + // 目录 + dir string + + // 锁 + mu sync.RWMutex +} + +// NewVersionSet 创建版本集合 +func NewVersionSet(dir string) (*VersionSet, error) { + vs := &VersionSet{ + dir: dir, + } + + // 确保目录存在 + err := os.MkdirAll(dir, 0755) + if err != nil { + return nil, err + } + + // 读取 CURRENT 文件 + currentFile := filepath.Join(dir, "CURRENT") + data, err := os.ReadFile(currentFile) + + if err != nil { + // CURRENT 不存在，创建新的 MANIFEST + return vs, vs.createNewManifest() + } + + // 读取 MANIFEST 文件 + manifestName := strings.TrimSpace(string(data)) + manifestPath := filepath.Join(dir, manifestName) + + // 恢复版本信息 + version, err := vs.recoverFromManifest(manifestPath) + if err != nil { + return nil, err + } + + vs.current = version + vs.nextFileNumber.Store(version.NextFileNumber) + vs.lastSequence.Store(version.LastSequence) + + // 解析 MANIFEST 编号 + fmt.Sscanf(manifestName, "MANIFEST-%d", &vs.manifestNumber) + + // 打开 MANIFEST 用于追加 + file, err := os.OpenFile(manifestPath, os.O_APPEND|os.O_WRONLY, 0644) + if err != nil { + return nil, err + } + vs.manifestFile = file + vs.manifestWriter = NewManifestWriter(file) + + return vs, nil +} + +// createNewManifest 创建新的 MANIFEST +func (vs *VersionSet) createNewManifest() error { + // 生成新的 MANIFEST 文件名 + vs.manifestNumber = vs.nextFileNumber.Add(1) + manifestName := fmt.Sprintf("MANIFEST-%06d", vs.manifestNumber) + manifestPath := filepath.Join(vs.dir, manifestName) + + // 创建 MANIFEST 文件 + file, err := os.Create(manifestPath) + if err != nil { + return err + } + + vs.manifestFile = file + vs.manifestWriter = NewManifestWriter(file) + + // 创建初始版本 + vs.current = NewVersion() + + // 写入初始版本 + edit := NewVersionEdit() + nextFile := vs.manifestNumber + edit.SetNextFileNumber(nextFile) + lastSeq := int64(0) + edit.SetLastSequence(lastSeq) + + err = vs.manifestWriter.WriteEdit(edit) + if err != nil { + return err + } + + // 同步到磁盘 + err = vs.manifestFile.Sync() + if err != nil { + return err + } + + // 更新 CURRENT 文件 + return vs.updateCurrent(manifestName) +} + +// recoverFromManifest 从 MANIFEST 恢复版本 +func (vs *VersionSet) recoverFromManifest(manifestPath string) (*Version, error) { + // 打开 MANIFEST 文件 + file, err := os.Open(manifestPath) + if err != nil { + return nil, err + } + defer file.Close() + + reader := NewManifestReader(file) + + // 创建初始版本 + version := NewVersion() + + // 读取所有 VersionEdit + for { + edit, err := reader.ReadEdit() + if err == io.EOF { + break + } + if err != nil { + return nil, err + } + + // 应用变更 + version.Apply(edit) + } + + return version, nil +} + +// updateCurrent 更新 CURRENT 文件 +func (vs *VersionSet) updateCurrent(manifestName string) error { + currentPath := filepath.Join(vs.dir, "CURRENT") + tmpPath := currentPath + ".tmp" + + // 1. 写入临时文件 + err := os.WriteFile(tmpPath, []byte(manifestName+"\n"), 0644) + if err != nil { + return err + } + + // 2. 原子性重命名 + err = os.Rename(tmpPath, currentPath) + if err != nil { + os.Remove(tmpPath) + return err + } + + return nil +} + +// LogAndApply 记录并应用版本变更 +func (vs *VersionSet) LogAndApply(edit *VersionEdit) error { + vs.mu.Lock() + defer vs.mu.Unlock() + + // 1. 创建新版本 + newVersion := vs.current.Clone() + + // 2. 应用变更 + newVersion.Apply(edit) + + // 3. 写入 MANIFEST + err := vs.manifestWriter.WriteEdit(edit) + if err != nil { + return err + } + + // 4. 同步到磁盘 + err = vs.manifestFile.Sync() + if err != nil { + return err + } + + // 5. 更新当前版本 + vs.current = newVersion + + // 6. 更新原子变量 + if edit.NextFileNumber != nil { + // 使用 CAS 循环确保只在新值更大时更新，避免并发回退 + for { + old := vs.nextFileNumber.Load() + if *edit.NextFileNumber <= old { + break // 新值不大于当前值，不更新 + } + if vs.nextFileNumber.CompareAndSwap(old, *edit.NextFileNumber) { + break // 更新成功 + } + // CAS 失败，重试 + } + } + if edit.LastSequence != nil { + vs.lastSequence.Store(*edit.LastSequence) + } + + return nil +} + +// GetCurrent 获取当前版本 +func (vs *VersionSet) GetCurrent() *Version { + vs.mu.RLock() + defer vs.mu.RUnlock() + return vs.current +} + +// GetNextFileNumber 获取下一个文件编号 +func (vs *VersionSet) GetNextFileNumber() int64 { + return vs.nextFileNumber.Load() +} + +// AllocateFileNumber 分配文件编号 +func (vs *VersionSet) AllocateFileNumber() int64 { + return vs.nextFileNumber.Add(1) +} + +// GetLastSequence 获取最后序列号 +func (vs *VersionSet) GetLastSequence() int64 { + return vs.lastSequence.Load() +} + +// SetLastSequence 设置最后序列号 +func (vs *VersionSet) SetLastSequence(seq int64) { + vs.lastSequence.Store(seq) +} + +// Close 关闭 VersionSet +func (vs *VersionSet) Close() error { + vs.mu.Lock() + defer vs.mu.Unlock() + + if vs.manifestFile != nil { + return vs.manifestFile.Close() + } + return nil +} diff --git a/manifest/version_set_test.go b/version_test.go similarity index 99% rename from manifest/version_set_test.go rename to version_test.go index 0b6f905..b05ec92 100644 --- a/manifest/version_set_test.go +++ b/version_test.go @@ -1,4 +1,4 @@ -package manifest +package srdb import ( "os" diff --git a/wal.go b/wal.go new file mode 100644 index 0000000..9540c43 --- /dev/null +++ b/wal.go @@ -0,0 +1,408 @@ +package srdb + +import ( + "encoding/binary" + "fmt" + "hash/crc32" + "io" + "os" + "path/filepath" + "sort" + "strconv" + "strings" + "sync" +) + +const ( + // Entry 类型 + WALEntryTypePut = 1 + WALEntryTypeDelete = 2 // 预留，暂不支持 + + // Entry Header 大小 + WALEntryHeaderSize = 17 // CRC32(4) + Length(4) + Type(1) + Seq(8) +) + +// WALEntry WAL 条目 +type WALEntry struct { + Type byte // 操作类型 + Seq int64 // _seq + Data []byte // 数据 + CRC32 uint32 // 校验和 +} + +// WAL Write-Ahead Log +type WAL struct { + file *os.File + offset int64 + mu sync.Mutex +} + +// OpenWAL 打开 WAL 文件 +func OpenWAL(path string) (*WAL, error) { + file, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR|os.O_APPEND, 0644) + if err != nil { + return nil, err + } + + // 获取当前文件大小 + stat, err := file.Stat() + if err != nil { + file.Close() + return nil, err + } + + return &WAL{ + file: file, + offset: stat.Size(), + }, nil +} + +// Append 追加一条记录 +func (w *WAL) Append(entry *WALEntry) error { + w.mu.Lock() + defer w.mu.Unlock() + + // 序列化 Entry + data := w.marshalEntry(entry) + + // 写入文件 + _, err := w.file.Write(data) + if err != nil { + return err + } + + w.offset += int64(len(data)) + + return nil +} + +// Sync 同步到磁盘 +func (w *WAL) Sync() error { + w.mu.Lock() + defer w.mu.Unlock() + + return w.file.Sync() +} + +// Close 关闭 WAL +func (w *WAL) Close() error { + w.mu.Lock() + defer w.mu.Unlock() + + return w.file.Close() +} + +// Truncate 清空 WAL +func (w *WAL) Truncate() error { + w.mu.Lock() + defer w.mu.Unlock() + + err := w.file.Truncate(0) + if err != nil { + return err + } + + _, err = w.file.Seek(0, 0) + if err != nil { + return err + } + + w.offset = 0 + return nil +} + +// marshalEntry 序列化 Entry +func (w *WAL) marshalEntry(entry *WALEntry) []byte { + dataLen := len(entry.Data) + totalLen := WALEntryHeaderSize + dataLen + + buf := make([]byte, totalLen) + + // 计算 CRC32 (不包括 CRC32 字段本身) + crcData := buf[4:totalLen] + binary.LittleEndian.PutUint32(crcData[0:4], uint32(dataLen)) + crcData[4] = entry.Type + binary.LittleEndian.PutUint64(crcData[5:13], uint64(entry.Seq)) + copy(crcData[13:], entry.Data) + + crc := crc32.ChecksumIEEE(crcData) + + // 写入 CRC32 + binary.LittleEndian.PutUint32(buf[0:4], crc) + + return buf +} + +// WALReader WAL 读取器 +type WALReader struct { + file *os.File +} + +// NewWALReader 创建 WAL 读取器 +func NewWALReader(path string) (*WALReader, error) { + file, err := os.Open(path) + if err != nil { + return nil, err + } + + return &WALReader{ + file: file, + }, nil +} + +// Read 读取所有 Entry +func (r *WALReader) Read() ([]*WALEntry, error) { + var entries []*WALEntry + + for { + entry, err := r.readEntry() + if err == io.EOF { + break + } + if err != nil { + return nil, err + } + entries = append(entries, entry) + } + + return entries, nil +} + +// Close 关闭读取器 +func (r *WALReader) Close() error { + return r.file.Close() +} + +// readEntry 读取一条 Entry +func (r *WALReader) readEntry() (*WALEntry, error) { + // 读取 Header + header := make([]byte, WALEntryHeaderSize) + _, err := io.ReadFull(r.file, header) + if err != nil { + return nil, err + } + + // 解析 Header + crc := binary.LittleEndian.Uint32(header[0:4]) + dataLen := binary.LittleEndian.Uint32(header[4:8]) + entryType := header[8] + seq := int64(binary.LittleEndian.Uint64(header[9:17])) + + // 读取 Data + data := make([]byte, dataLen) + _, err = io.ReadFull(r.file, data) + if err != nil { + return nil, err + } + + // 验证 CRC32 + crcData := make([]byte, WALEntryHeaderSize-4+int(dataLen)) + copy(crcData[0:WALEntryHeaderSize-4], header[4:]) + copy(crcData[WALEntryHeaderSize-4:], data) + + if crc32.ChecksumIEEE(crcData) != crc { + return nil, io.ErrUnexpectedEOF // CRC 校验失败 + } + + return &WALEntry{ + Type: entryType, + Seq: seq, + Data: data, + CRC32: crc, + }, nil +} + +// WALManager WAL 管理器，管理多个 WAL 文件 +type WALManager struct { + dir string + currentWAL *WAL + currentNumber int64 + mu sync.Mutex +} + +// NewWALManager 创建 WAL 管理器 +func NewWALManager(dir string) (*WALManager, error) { + // 确保目录存在 + err := os.MkdirAll(dir, 0755) + if err != nil { + return nil, err + } + + // 读取当前 WAL 编号 + number, err := readWALCurrentNumber(dir) + if err != nil { + // 如果读取失败，从 1 开始 + number = 1 + } + + // 打开当前 WAL + walPath := filepath.Join(dir, fmt.Sprintf("%06d.wal", number)) + wal, err := OpenWAL(walPath) + if err != nil { + return nil, err + } + + // 保存当前编号 + err = saveWALCurrentNumber(dir, number) + if err != nil { + wal.Close() + return nil, err + } + + return &WALManager{ + dir: dir, + currentWAL: wal, + currentNumber: number, + }, nil +} + +// Append 追加记录到当前 WAL +func (m *WALManager) Append(entry *WALEntry) error { + m.mu.Lock() + defer m.mu.Unlock() + + return m.currentWAL.Append(entry) +} + +// Sync 同步当前 WAL 到磁盘 +func (m *WALManager) Sync() error { + m.mu.Lock() + defer m.mu.Unlock() + + return m.currentWAL.Sync() +} + +// Rotate 切换到新的 WAL 文件 +func (m *WALManager) Rotate() (int64, error) { + m.mu.Lock() + defer m.mu.Unlock() + + // 记录旧的 WAL 编号 + oldNumber := m.currentNumber + + // 关闭当前 WAL + err := m.currentWAL.Close() + if err != nil { + return 0, err + } + + // 创建新 WAL + m.currentNumber++ + walPath := filepath.Join(m.dir, fmt.Sprintf("%06d.wal", m.currentNumber)) + wal, err := OpenWAL(walPath) + if err != nil { + return 0, err + } + + m.currentWAL = wal + + // 更新 CURRENT 文件 + err = saveWALCurrentNumber(m.dir, m.currentNumber) + if err != nil { + return 0, err + } + + return oldNumber, nil +} + +// Delete 删除指定的 WAL 文件 +func (m *WALManager) Delete(number int64) error { + m.mu.Lock() + defer m.mu.Unlock() + + walPath := filepath.Join(m.dir, fmt.Sprintf("%06d.wal", number)) + return os.Remove(walPath) +} + +// GetCurrentNumber 获取当前 WAL 编号 +func (m *WALManager) GetCurrentNumber() int64 { + m.mu.Lock() + defer m.mu.Unlock() + + return m.currentNumber +} + +// RecoverAll 恢复所有 WAL 文件 +func (m *WALManager) RecoverAll() ([]*WALEntry, error) { + // 查找所有 WAL 文件 + pattern := filepath.Join(m.dir, "*.wal") + files, err := filepath.Glob(pattern) + if err != nil { + return nil, err + } + + if len(files) == 0 { + return nil, nil + } + + // 按文件名排序（确保按时间顺序） + sort.Strings(files) + + var allEntries []*WALEntry + + // 依次读取每个 WAL + for _, file := range files { + reader, err := NewWALReader(file) + if err != nil { + continue + } + + entries, err := reader.Read() + reader.Close() + + if err != nil { + continue + } + + allEntries = append(allEntries, entries...) + } + + return allEntries, nil +} + +// ListWALFiles 列出所有 WAL 文件 +func (m *WALManager) ListWALFiles() ([]string, error) { + pattern := filepath.Join(m.dir, "*.wal") + files, err := filepath.Glob(pattern) + if err != nil { + return nil, err + } + + sort.Strings(files) + return files, nil +} + +// Close 关闭 WAL 管理器 +func (m *WALManager) Close() error { + m.mu.Lock() + defer m.mu.Unlock() + + if m.currentWAL != nil { + return m.currentWAL.Close() + } + + return nil +} + +// readWALCurrentNumber 读取当前 WAL 编号 +func readWALCurrentNumber(dir string) (int64, error) { + currentPath := filepath.Join(dir, "CURRENT") + data, err := os.ReadFile(currentPath) + if err != nil { + return 0, err + } + + number, err := strconv.ParseInt(strings.TrimSpace(string(data)), 10, 64) + if err != nil { + return 0, err + } + + return number, nil +} + +// saveWALCurrentNumber 保存当前 WAL 编号 +func saveWALCurrentNumber(dir string, number int64) error { + currentPath := filepath.Join(dir, "CURRENT") + data := fmt.Appendf(nil, "%d\n", number) + return os.WriteFile(currentPath, data, 0644) +} diff --git a/wal/manager.go b/wal/manager.go deleted file mode 100644 index 4509064..0000000 --- a/wal/manager.go +++ /dev/null @@ -1,206 +0,0 @@ -package wal - -import ( - "fmt" - "os" - "path/filepath" - "sort" - "strconv" - "strings" - "sync" -) - -// Manager WAL 管理器，管理多个 WAL 文件 -type Manager struct { - dir string - currentWAL *WAL - currentNumber int64 - mu sync.Mutex -} - -// NewManager 创建 WAL 管理器 -func NewManager(dir string) (*Manager, error) { - // 确保目录存在 - err := os.MkdirAll(dir, 0755) - if err != nil { - return nil, err - } - - // 读取当前 WAL 编号 - number, err := readCurrentNumber(dir) - if err != nil { - // 如果读取失败，从 1 开始 - number = 1 - } - - // 打开当前 WAL - walPath := filepath.Join(dir, fmt.Sprintf("%06d.wal", number)) - wal, err := Open(walPath) - if err != nil { - return nil, err - } - - // 保存当前编号 - err = saveCurrentNumber(dir, number) - if err != nil { - wal.Close() - return nil, err - } - - return &Manager{ - dir: dir, - currentWAL: wal, - currentNumber: number, - }, nil -} - -// Append 追加记录到当前 WAL -func (m *Manager) Append(entry *Entry) error { - m.mu.Lock() - defer m.mu.Unlock() - - return m.currentWAL.Append(entry) -} - -// Sync 同步当前 WAL 到磁盘 -func (m *Manager) Sync() error { - m.mu.Lock() - defer m.mu.Unlock() - - return m.currentWAL.Sync() -} - -// Rotate 切换到新的 WAL 文件 -func (m *Manager) Rotate() (int64, error) { - m.mu.Lock() - defer m.mu.Unlock() - - // 记录旧的 WAL 编号 - oldNumber := m.currentNumber - - // 关闭当前 WAL - err := m.currentWAL.Close() - if err != nil { - return 0, err - } - - // 创建新 WAL - m.currentNumber++ - walPath := filepath.Join(m.dir, fmt.Sprintf("%06d.wal", m.currentNumber)) - wal, err := Open(walPath) - if err != nil { - return 0, err - } - - m.currentWAL = wal - - // 更新 CURRENT 文件 - err = saveCurrentNumber(m.dir, m.currentNumber) - if err != nil { - return 0, err - } - - return oldNumber, nil -} - -// Delete 删除指定的 WAL 文件 -func (m *Manager) Delete(number int64) error { - m.mu.Lock() - defer m.mu.Unlock() - - walPath := filepath.Join(m.dir, fmt.Sprintf("%06d.wal", number)) - return os.Remove(walPath) -} - -// GetCurrentNumber 获取当前 WAL 编号 -func (m *Manager) GetCurrentNumber() int64 { - m.mu.Lock() - defer m.mu.Unlock() - - return m.currentNumber -} - -// RecoverAll 恢复所有 WAL 文件 -func (m *Manager) RecoverAll() ([]*Entry, error) { - // 查找所有 WAL 文件 - pattern := filepath.Join(m.dir, "*.wal") - files, err := filepath.Glob(pattern) - if err != nil { - return nil, err - } - - if len(files) == 0 { - return nil, nil - } - - // 按文件名排序（确保按时间顺序） - sort.Strings(files) - - var allEntries []*Entry - - // 依次读取每个 WAL - for _, file := range files { - reader, err := NewReader(file) - if err != nil { - continue - } - - entries, err := reader.Read() - reader.Close() - - if err != nil { - continue - } - - allEntries = append(allEntries, entries...) - } - - return allEntries, nil -} - -// ListWALFiles 列出所有 WAL 文件 -func (m *Manager) ListWALFiles() ([]string, error) { - pattern := filepath.Join(m.dir, "*.wal") - files, err := filepath.Glob(pattern) - if err != nil { - return nil, err - } - - sort.Strings(files) - return files, nil -} - -// Close 关闭 WAL 管理器 -func (m *Manager) Close() error { - m.mu.Lock() - defer m.mu.Unlock() - - if m.currentWAL != nil { - return m.currentWAL.Close() - } - - return nil -} - -// readCurrentNumber 读取当前 WAL 编号 -func readCurrentNumber(dir string) (int64, error) { - currentPath := filepath.Join(dir, "CURRENT") - data, err := os.ReadFile(currentPath) - if err != nil { - return 0, err - } - - number, err := strconv.ParseInt(strings.TrimSpace(string(data)), 10, 64) - if err != nil { - return 0, err - } - - return number, nil -} - -// saveCurrentNumber 保存当前 WAL 编号 -func saveCurrentNumber(dir string, number int64) error { - currentPath := filepath.Join(dir, "CURRENT") - data := []byte(fmt.Sprintf("%d\n", number)) - return os.WriteFile(currentPath, data, 0644) -} diff --git a/wal/wal.go b/wal/wal.go deleted file mode 100644 index 5c10f13..0000000 --- a/wal/wal.go +++ /dev/null @@ -1,208 +0,0 @@ -package wal - -import ( - "encoding/binary" - "hash/crc32" - "io" - "os" - "sync" -) - -const ( - // Entry 类型 - EntryTypePut = 1 - EntryTypeDelete = 2 // 预留，暂不支持 - - // Entry Header 大小 - EntryHeaderSize = 17 // CRC32(4) + Length(4) + Type(1) + Seq(8) -) - -// Entry WAL 条目 -type Entry struct { - Type byte // 操作类型 - Seq int64 // _seq - Data []byte // 数据 - CRC32 uint32 // 校验和 -} - -// WAL Write-Ahead Log -type WAL struct { - file *os.File - offset int64 - mu sync.Mutex -} - -// Open 打开 WAL 文件 -func Open(path string) (*WAL, error) { - file, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR|os.O_APPEND, 0644) - if err != nil { - return nil, err - } - - // 获取当前文件大小 - stat, err := file.Stat() - if err != nil { - file.Close() - return nil, err - } - - return &WAL{ - file: file, - offset: stat.Size(), - }, nil -} - -// Append 追加一条记录 -func (w *WAL) Append(entry *Entry) error { - w.mu.Lock() - defer w.mu.Unlock() - - // 序列化 Entry - data := w.marshalEntry(entry) - - // 写入文件 - _, err := w.file.Write(data) - if err != nil { - return err - } - - w.offset += int64(len(data)) - - return nil -} - -// Sync 同步到磁盘 -func (w *WAL) Sync() error { - w.mu.Lock() - defer w.mu.Unlock() - - return w.file.Sync() -} - -// Close 关闭 WAL -func (w *WAL) Close() error { - w.mu.Lock() - defer w.mu.Unlock() - - return w.file.Close() -} - -// Truncate 清空 WAL -func (w *WAL) Truncate() error { - w.mu.Lock() - defer w.mu.Unlock() - - err := w.file.Truncate(0) - if err != nil { - return err - } - - _, err = w.file.Seek(0, 0) - if err != nil { - return err - } - - w.offset = 0 - return nil -} - -// marshalEntry 序列化 Entry -func (w *WAL) marshalEntry(entry *Entry) []byte { - dataLen := len(entry.Data) - totalLen := EntryHeaderSize + dataLen - - buf := make([]byte, totalLen) - - // 计算 CRC32 (不包括 CRC32 字段本身) - crcData := buf[4:totalLen] - binary.LittleEndian.PutUint32(crcData[0:4], uint32(dataLen)) - crcData[4] = entry.Type - binary.LittleEndian.PutUint64(crcData[5:13], uint64(entry.Seq)) - copy(crcData[13:], entry.Data) - - crc := crc32.ChecksumIEEE(crcData) - - // 写入 CRC32 - binary.LittleEndian.PutUint32(buf[0:4], crc) - - return buf -} - -// Reader WAL 读取器 -type Reader struct { - file *os.File -} - -// NewReader 创建 WAL 读取器 -func NewReader(path string) (*Reader, error) { - file, err := os.Open(path) - if err != nil { - return nil, err - } - - return &Reader{ - file: file, - }, nil -} - -// Read 读取所有 Entry -func (r *Reader) Read() ([]*Entry, error) { - var entries []*Entry - - for { - entry, err := r.readEntry() - if err == io.EOF { - break - } - if err != nil { - return nil, err - } - entries = append(entries, entry) - } - - return entries, nil -} - -// Close 关闭读取器 -func (r *Reader) Close() error { - return r.file.Close() -} - -// readEntry 读取一条 Entry -func (r *Reader) readEntry() (*Entry, error) { - // 读取 Header - header := make([]byte, EntryHeaderSize) - _, err := io.ReadFull(r.file, header) - if err != nil { - return nil, err - } - - // 解析 Header - crc := binary.LittleEndian.Uint32(header[0:4]) - dataLen := binary.LittleEndian.Uint32(header[4:8]) - entryType := header[8] - seq := int64(binary.LittleEndian.Uint64(header[9:17])) - - // 读取 Data - data := make([]byte, dataLen) - _, err = io.ReadFull(r.file, data) - if err != nil { - return nil, err - } - - // 验证 CRC32 - crcData := make([]byte, EntryHeaderSize-4+int(dataLen)) - copy(crcData[0:EntryHeaderSize-4], header[4:]) - copy(crcData[EntryHeaderSize-4:], data) - - if crc32.ChecksumIEEE(crcData) != crc { - return nil, io.ErrUnexpectedEOF // CRC 校验失败 - } - - return &Entry{ - Type: entryType, - Seq: seq, - Data: data, - CRC32: crc, - }, nil -} diff --git a/wal/wal_test.go b/wal_test.go similarity index 77% rename from wal/wal_test.go rename to wal_test.go index dc94e2f..35da8dc 100644 --- a/wal/wal_test.go +++ b/wal_test.go @@ -1,4 +1,4 @@ -package wal +package srdb import ( "os" @@ -7,7 +7,7 @@ import ( func TestWAL(t *testing.T) { // 1. 创建 WAL - wal, err := Open("test.wal") + wal, err := OpenWAL("test.wal") if err != nil { t.Fatal(err) } @@ -15,8 +15,8 @@ func TestWAL(t *testing.T) { // 2. 写入数据 for i := int64(1); i <= 100; i++ { - entry := &Entry{ - Type: EntryTypePut, + entry := &WALEntry{ + Type: WALEntryTypePut, Seq: i, Data: []byte("value_" + string(rune(i))), } @@ -37,7 +37,7 @@ func TestWAL(t *testing.T) { t.Log("Written 100 entries") // 4. 读取数据 - reader, err := NewReader("test.wal") + reader, err := NewWALReader("test.wal") if err != nil { t.Fatal(err) } @@ -58,8 +58,8 @@ func TestWAL(t *testing.T) { if entry.Seq != expectedSeq { t.Errorf("Entry %d: expected Seq=%d, got %d", i, expectedSeq, entry.Seq) } - if entry.Type != EntryTypePut { - t.Errorf("Entry %d: expected Type=%d, got %d", i, EntryTypePut, entry.Type) + if entry.Type != WALEntryTypePut { + t.Errorf("Entry %d: expected Type=%d, got %d", i, WALEntryTypePut, entry.Type) } } @@ -68,7 +68,7 @@ func TestWAL(t *testing.T) { func TestWALTruncate(t *testing.T) { // 创建 WAL - wal, err := Open("test_truncate.wal") + wal, err := OpenWAL("test_truncate.wal") if err != nil { t.Fatal(err) } @@ -76,8 +76,8 @@ func TestWALTruncate(t *testing.T) { // 写入数据 for i := int64(1); i <= 10; i++ { - entry := &Entry{ - Type: EntryTypePut, + entry := &WALEntry{ + Type: WALEntryTypePut, Seq: i, Data: []byte("value"), } @@ -93,7 +93,7 @@ func TestWALTruncate(t *testing.T) { wal.Close() // 验证文件为空 - reader, err := NewReader("test_truncate.wal") + reader, err := NewWALReader("test_truncate.wal") if err != nil { t.Fatal(err) } @@ -112,18 +112,17 @@ func TestWALTruncate(t *testing.T) { } func BenchmarkWALAppend(b *testing.B) { - wal, _ := Open("bench.wal") + wal, _ := OpenWAL("bench.wal") defer os.Remove("bench.wal") defer wal.Close() - entry := &Entry{ - Type: EntryTypePut, + entry := &WALEntry{ + Type: WALEntryTypePut, Seq: 1, Data: make([]byte, 100), } - b.ResetTimer() - for i := 0; i < b.N; i++ { + for i := 0; b.Loop(); i++ { entry.Seq = int64(i) wal.Append(entry) } diff --git a/webui/htmx.go b/webui/htmx.go deleted file mode 100644 index a4065cf..0000000 --- a/webui/htmx.go +++ /dev/null @@ -1,552 +0,0 @@ -package webui - -import ( - "bytes" - "fmt" - "html" - "strings" -) - -// HTML 渲染辅助函数 - -// renderTablesHTML 渲染表列表 HTML -func renderTablesHTML(tables []TableListItem) string { - var buf bytes.Buffer - - for _, table := range tables { - buf.WriteString(`
`) - buf.WriteString(`
`) - - // 左侧：展开图标和表名 - buf.WriteString(`
`) - buf.WriteString(`▶`) - buf.WriteString(``) - buf.WriteString(html.EscapeString(table.Name)) - buf.WriteString(`
`) - - // 右侧：字段数量 - buf.WriteString(``) - buf.WriteString(formatCount(int64(len(table.Fields)))) - buf.WriteString(` fields`) - buf.WriteString(`
`) - - // Schema 字段列表（默认隐藏） - if len(table.Fields) > 0 { - buf.WriteString(`
`) - for _, field := range table.Fields { - buf.WriteString(`
`) - buf.WriteString(``) - buf.WriteString(html.EscapeString(field.Name)) - buf.WriteString(``) - buf.WriteString(``) - buf.WriteString(html.EscapeString(field.Type)) - buf.WriteString(``) - if field.Indexed { - buf.WriteString(`●indexed`) - } - buf.WriteString(`
`) - } - buf.WriteString(`
`) - } - - buf.WriteString(`
`) - } - - return buf.String() -} - -// renderDataViewHTML 渲染数据视图 HTML -func renderDataViewHTML(tableName string, schema SchemaInfo, tableData TableDataResponse) string { - var buf bytes.Buffer - - // 标题 - buf.WriteString(`

`) - buf.WriteString(html.EscapeString(tableName)) - buf.WriteString(`

`) - - // 视图切换标签 - buf.WriteString(`
`) - buf.WriteString(``) - buf.WriteString(``) - buf.WriteString(`
`) - - // Schema 部分 - if len(schema.Fields) > 0 { - buf.WriteString(`
`) - buf.WriteString(`

Schema (点击字段卡片选择要显示的列)

`) - buf.WriteString(`
`) - for _, field := range schema.Fields { - buf.WriteString(`
`) - buf.WriteString(`
`) - buf.WriteString(``) - buf.WriteString(html.EscapeString(field.Name)) - buf.WriteString(``) - buf.WriteString(``) - buf.WriteString(html.EscapeString(field.Type)) - buf.WriteString(``) - if field.Indexed { - buf.WriteString(`●indexed`) - } - buf.WriteString(`
`) - buf.WriteString(`
`) - if field.Comment != "" { - buf.WriteString(html.EscapeString(field.Comment)) - } - buf.WriteString(`
`) - buf.WriteString(`
`) - } - buf.WriteString(`
`) - buf.WriteString(`
`) - } - - // 数据表格 - buf.WriteString(`

Data (`) - buf.WriteString(formatCount(tableData.TotalRows)) - buf.WriteString(` rows)

`) - - if len(tableData.Data) == 0 { - buf.WriteString(`

No data available

`) - return buf.String() - } - - // 获取列并排序：_seq 第1列，_time 倒数第2列 - columns := []string{} - otherColumns := []string{} - hasSeq := false - hasTime := false - - if len(tableData.Data) > 0 { - for key := range tableData.Data[0] { - if !strings.HasSuffix(key, "_truncated") { - if key == "_seq" { - hasSeq = true - } else if key == "_time" { - hasTime = true - } else { - otherColumns = append(otherColumns, key) - } - } - } - } - - // 按顺序组装：_seq, 其他列, _time - if hasSeq { - columns = append(columns, "_seq") - } - columns = append(columns, otherColumns...) - if hasTime { - columns = append(columns, "_time") - } - - // 表格 - buf.WriteString(`
`) - buf.WriteString(``) - buf.WriteString(``) - for _, col := range columns { - buf.WriteString(``) - } - buf.WriteString(``) - buf.WriteString(``) - - buf.WriteString(``) - for _, row := range tableData.Data { - buf.WriteString(``) - for _, col := range columns { - value := row[col] - buf.WriteString(``) - } - - // Actions 列 - buf.WriteString(``) - - buf.WriteString(``) - } - buf.WriteString(``) - buf.WriteString(`
`) - buf.WriteString(html.EscapeString(col)) - buf.WriteString(`Actions
`) - buf.WriteString(html.EscapeString(fmt.Sprintf("%v", value))) - - // 检查是否被截断 - if truncated, ok := row[col+"_truncated"]; ok && truncated == true { - buf.WriteString(`✂️`) - } - - buf.WriteString(``) - buf.WriteString(``) - buf.WriteString(`
`) - buf.WriteString(`
`) - - // 分页 - buf.WriteString(renderPagination(tableData)) - - return buf.String() -} - -// renderManifestViewHTML 渲染 Manifest 视图 HTML -func renderManifestViewHTML(tableName string, manifest ManifestResponse) string { - var buf bytes.Buffer - - // 标题 - buf.WriteString(`

`) - buf.WriteString(html.EscapeString(tableName)) - buf.WriteString(`

`) - - // 视图切换标签 - buf.WriteString(`
`) - buf.WriteString(``) - buf.WriteString(``) - buf.WriteString(`
`) - - // 标题和控制按钮 - buf.WriteString(`
`) - buf.WriteString(`

LSM-Tree Structure

`) - buf.WriteString(`
`) - buf.WriteString(``) - buf.WriteString(``) - buf.WriteString(`
`) - buf.WriteString(`
`) - - // 统计卡片 - totalLevels := len(manifest.Levels) - totalFiles := 0 - totalSize := int64(0) - for _, level := range manifest.Levels { - totalFiles += level.FileCount - totalSize += level.TotalSize - } - - buf.WriteString(`
`) - - // Active Levels - buf.WriteString(`
`) - buf.WriteString(`
Active Levels
`) - buf.WriteString(`
`) - buf.WriteString(fmt.Sprintf("%d", totalLevels)) - buf.WriteString(`
`) - - // Total Files - buf.WriteString(`
`) - buf.WriteString(`
Total Files
`) - buf.WriteString(`
`) - buf.WriteString(fmt.Sprintf("%d", totalFiles)) - buf.WriteString(`
`) - - // Total Size - buf.WriteString(`
`) - buf.WriteString(`
Total Size
`) - buf.WriteString(`
`) - buf.WriteString(formatBytes(totalSize)) - buf.WriteString(`
`) - - // Next File Number - buf.WriteString(`
`) - buf.WriteString(`
Next File Number
`) - buf.WriteString(`
`) - buf.WriteString(fmt.Sprintf("%d", manifest.NextFileNumber)) - buf.WriteString(`
`) - - // Last Sequence - buf.WriteString(`
`) - buf.WriteString(`
Last Sequence
`) - buf.WriteString(`
`) - buf.WriteString(fmt.Sprintf("%d", manifest.LastSequence)) - buf.WriteString(`
`) - - // Total Compactions - buf.WriteString(`
`) - buf.WriteString(`
Total Compactions
`) - buf.WriteString(`
`) - totalCompactions := 0 - if manifest.CompactionStats != nil { - if tc, ok := manifest.CompactionStats["total_compactions"]; ok { - if tcInt, ok := tc.(float64); ok { - totalCompactions = int(tcInt) - } - } - } - buf.WriteString(fmt.Sprintf("%d", totalCompactions)) - buf.WriteString(`
`) - - buf.WriteString(`
`) - - // 渲染所有层级（L0-L6） - for i := 0; i <= 6; i++ { - var level *LevelInfo - for j := range manifest.Levels { - if manifest.Levels[j].Level == i { - level = &manifest.Levels[j] - break - } - } - - if level == nil { - // 创建空层级 - level = &LevelInfo{ - Level: i, - FileCount: 0, - TotalSize: 0, - Score: 0, - Files: []FileInfo{}, - } - } - - buf.WriteString(renderLevelCard(*level)) - } - - return buf.String() -} - -// renderLevelCard 渲染层级卡片 -func renderLevelCard(level LevelInfo) string { - var buf bytes.Buffer - - scoreClass := "normal" - if level.Score >= 1.0 { - scoreClass = "critical" - } else if level.Score >= 0.8 { - scoreClass = "warning" - } - - buf.WriteString(`
`) - buf.WriteString(`
`) - - // 左侧：展开图标和标题 - buf.WriteString(`
`) - buf.WriteString(`▶`) - buf.WriteString(`
Level `) - buf.WriteString(fmt.Sprintf("%d", level.Level)) - buf.WriteString(`
`) - - // 右侧：统计信息 - buf.WriteString(`
`) - buf.WriteString(``) - buf.WriteString(fmt.Sprintf("%d", level.FileCount)) - buf.WriteString(` files`) - buf.WriteString(``) - buf.WriteString(formatBytes(level.TotalSize)) - buf.WriteString(``) - buf.WriteString(`Score: `) - buf.WriteString(fmt.Sprintf("%.2f", level.Score)) - buf.WriteString(``) - buf.WriteString(`
`) - - buf.WriteString(`
`) - - // 文件列表（默认隐藏） - buf.WriteString(`
`) - if len(level.Files) == 0 { - buf.WriteString(`
No files in this level
`) - } else { - for _, file := range level.Files { - buf.WriteString(`
`) - buf.WriteString(`
`) - buf.WriteString(`File #`) - buf.WriteString(fmt.Sprintf("%d", file.FileNumber)) - buf.WriteString(``) - buf.WriteString(``) - buf.WriteString(formatBytes(file.FileSize)) - buf.WriteString(``) - buf.WriteString(`
`) - - buf.WriteString(`
`) - buf.WriteString(`Key Range:`) - buf.WriteString(``) - buf.WriteString(fmt.Sprintf("%d - %d", file.MinKey, file.MaxKey)) - buf.WriteString(`
`) - - buf.WriteString(`
`) - buf.WriteString(`Rows:`) - buf.WriteString(``) - buf.WriteString(formatCount(file.RowCount)) - buf.WriteString(`
`) - - buf.WriteString(`
`) - } - } - buf.WriteString(`
`) - - buf.WriteString(`
`) - return buf.String() -} - -// renderPagination 渲染分页 HTML -func renderPagination(data TableDataResponse) string { - var buf bytes.Buffer - - buf.WriteString(``) - return buf.String() -} - -// formatBytes 格式化字节数 -func formatBytes(bytes int64) string { - if bytes == 0 { - return "0 B" - } - const k = 1024 - sizes := []string{"B", "KB", "MB", "GB", "TB"} - i := 0 - size := float64(bytes) - for size >= k && i < len(sizes)-1 { - size /= k - i++ - } - return fmt.Sprintf("%.2f %s", size, sizes[i]) -} - -// formatCount 格式化数量（K/M） -func formatCount(count int64) string { - if count >= 1000000 { - return fmt.Sprintf("%.1fM", float64(count)/1000000) - } - if count >= 1000 { - return fmt.Sprintf("%.1fK", float64(count)/1000) - } - return fmt.Sprintf("%d", count) -} - -// escapeJSString 转义 JavaScript 字符串 -func escapeJSString(s string) string { - s = strings.ReplaceAll(s, `\`, `\\`) - s = strings.ReplaceAll(s, `'`, `\'`) - s = strings.ReplaceAll(s, `"`, `\"`) - s = strings.ReplaceAll(s, "\n", `\n`) - s = strings.ReplaceAll(s, "\r", `\r`) - s = strings.ReplaceAll(s, "\t", `\t`) - return s -} - -// 数据结构定义 -type TableListItem struct { - Name string `json:"name"` - CreatedAt int64 `json:"created_at"` - Fields []FieldInfo `json:"fields"` -} - -type FieldInfo struct { - Name string `json:"name"` - Type string `json:"type"` - Indexed bool `json:"indexed"` - Comment string `json:"comment"` -} - -type SchemaInfo struct { - Name string `json:"name"` - Fields []FieldInfo `json:"fields"` -} - -type TableDataResponse struct { - Data []map[string]any `json:"data"` - Page int64 `json:"page"` - PageSize int64 `json:"pageSize"` - TotalRows int64 `json:"totalRows"` - TotalPages int64 `json:"totalPages"` -} - -type ManifestResponse struct { - Levels []LevelInfo `json:"levels"` - NextFileNumber int64 `json:"next_file_number"` - LastSequence int64 `json:"last_sequence"` - CompactionStats map[string]any `json:"compaction_stats"` -} - -type LevelInfo struct { - Level int `json:"level"` - FileCount int `json:"file_count"` - TotalSize int64 `json:"total_size"` - Score float64 `json:"score"` - Files []FileInfo `json:"files"` -} - -type FileInfo struct { - FileNumber int64 `json:"file_number"` - Level int `json:"level"` - FileSize int64 `json:"file_size"` - MinKey int64 `json:"min_key"` - MaxKey int64 `json:"max_key"` - RowCount int64 `json:"row_count"` -} diff --git a/webui/static/css/styles.css b/webui/static/css/styles.css index 10de36c..580fbd1 100644 --- a/webui/static/css/styles.css +++ b/webui/static/css/styles.css @@ -1,4 +1,4 @@ -/* SRDB WebUI - Modern Design */ +/* SRDB WebUI - Modern Design with Lit */ :root { /* 主色调 - 优雅的紫蓝色 */ @@ -53,6 +53,11 @@ padding: 0; } +:root { + /* 主题过渡动画 */ + transition: background-color 0.3s ease, color 0.3s ease; +} + body { font-family: "Inter", @@ -67,837 +72,9 @@ body { font-size: 14px; -webkit-font-smoothing: antialiased; -moz-osx-font-smoothing: grayscale; -} - -/* 布局 */ -.container { - display: flex; + margin: 0; + padding: 0; height: 100vh; overflow: hidden; -} - -/* 侧边栏 */ -.sidebar { - width: 280px; - background: var(--bg-surface); - border-right: 1px solid var(--border-color); - overflow-y: auto; - overflow-x: hidden; - padding: 16px 12px; - display: flex; - flex-direction: column; - gap: 8px; -} - -.sidebar::-webkit-scrollbar { - width: 6px; -} - -.sidebar::-webkit-scrollbar-track { - background: transparent; -} - -.sidebar::-webkit-scrollbar-thumb { - background: rgba(255, 255, 255, 0.1); - border-radius: 3px; -} - -.sidebar::-webkit-scrollbar-thumb:hover { - background: rgba(255, 255, 255, 0.15); -} - -.sidebar h1 { - font-size: 18px; - font-weight: 700; - letter-spacing: -0.02em; - background: linear-gradient(135deg, var(--primary-light), var(--primary)); - -webkit-background-clip: text; - -webkit-text-fill-color: transparent; - margin-bottom: 4px; -} - -/* 主内容区 */ -.main { - flex: 1; - padding: 20px; - overflow-y: auto; - overflow-x: hidden; - background: var(--bg-main); -} - -.main h2 { - font-size: 24px; - font-weight: 700; - margin-bottom: 16px; - background: linear-gradient( - 135deg, - var(--text-primary) 0%, - var(--primary-light) 100% - ); - -webkit-background-clip: text; - -webkit-text-fill-color: transparent; - letter-spacing: -0.02em; -} - -.main h3 { - font-size: 16px; - font-weight: 600; - margin-bottom: 12px; - margin-top: 20px; - color: var(--text-primary); - display: flex; - align-items: center; - gap: 8px; -} - -.main h3::before { - content: ""; - width: 3px; - height: 18px; - background: linear-gradient(135deg, var(--primary), var(--primary-light)); - border-radius: 2px; -} - -.main::-webkit-scrollbar { - width: 8px; -} - -.main::-webkit-scrollbar-track { - background: transparent; -} - -.main::-webkit-scrollbar-thumb { - background: rgba(255, 255, 255, 0.1); - border-radius: 4px; -} - -.main::-webkit-scrollbar-thumb:hover { - background: rgba(255, 255, 255, 0.15); -} - -/* 表列表卡片 */ -.table-item { - margin-bottom: 6px; - border-radius: var(--radius-md); - overflow: hidden; - transition: var(--transition); -} - -.table-header { - padding: 10px 12px; - background: var(--bg-elevated); - border: 1px solid var(--border-color); - border-radius: var(--radius-md); - cursor: pointer; - display: flex; - justify-content: space-between; - align-items: center; - transition: var(--transition); -} - -.table-header:hover { - background: var(--bg-hover); - border-color: var(--border-hover); - /*transform: translateX(2px);*/ -} - -.table-header.selected, -.table-item.selected .table-header { - background: linear-gradient(135deg, var(--primary), var(--primary-dark)); - border-color: var(--primary); - box-shadow: 0 0 0 3px var(--primary-bg); -} - -.table-header-left { - display: flex; - align-items: center; - gap: 10px; - flex: 1; - min-width: 0; -} - -.expand-icon { - font-size: 10px; - color: var(--text-secondary); - transition: var(--transition); - user-select: none; - flex-shrink: 0; -} - -.expand-icon.expanded { - transform: rotate(90deg); -} - -.table-name { - font-weight: 600; - font-size: 14px; - overflow: hidden; - text-overflow: ellipsis; - white-space: nowrap; - flex: 1; -} - -.table-count { - font-size: 11px; - font-weight: 500; - padding: 3px 8px; - background: rgba(255, 255, 255, 0.1); - border-radius: 12px; - color: var(--text-secondary); - flex-shrink: 0; -} - -.table-item.selected .table-count { - background: rgba(255, 255, 255, 0.2); - color: white; -} - -/* Schema 字段列表 */ -.schema-fields { - display: none; - margin-top: 8px; - padding: 10px 12px; - background: rgba(255, 255, 255, 0.03); - border-left: 2px solid var(--primary); - border-radius: var(--radius-md); - gap: 6px; - flex-direction: column; -} - -.field-item { - display: flex; - align-items: center; - justify-content: space-between; - gap: 8px; - padding: 4px 0; -} - -.field-name { - font-size: 13px; - font-weight: 500; - color: var(--text-primary); - min-width: 90px; -} - -.field-type { - font-size: 12px; - font-family: "SF Mono", Monaco, monospace; - color: var(--primary-light); - background: rgba(99, 102, 241, 0.15); - padding: 2px 8px; - border-radius: 4px; -} - -.field-indexed { - font-size: 10px; - font-weight: 600; - color: var(--success); - text-transform: uppercase; - letter-spacing: 0.05em; -} - -.field-comment { - font-size: 12px; - color: #999; - margin-top: 4px; -} - -/* 视图切换标签 */ -.view-tabs { - display: flex; - gap: 8px; - margin-bottom: 20px; - padding: 4px; - background: var(--bg-surface); - border-radius: var(--radius-lg); - border: 1px solid var(--border-color); - box-shadow: var(--shadow-sm); -} - -.view-tab { - padding: 10px 20px; - background: transparent; - border: none; - border-radius: var(--radius-md); - cursor: pointer; - font-size: 14px; - font-weight: 600; - color: var(--text-secondary); - transition: var(--transition); - position: relative; - letter-spacing: -0.01em; -} - -.view-tab:hover { - color: var(--text-primary); - background: rgba(255, 255, 255, 0.05); -} - -.view-tab.active { - color: white; - background: linear-gradient( - 135deg, - var(--primary) 0%, - var(--primary-dark) 100% - ); - box-shadow: 0 2px 8px rgba(99, 102, 241, 0.3); -} - -/* Schema 展示 */ -/*.schema-section { - background: linear-gradient( - 135deg, - rgba(99, 102, 241, 0.05) 0%, - rgba(99, 102, 241, 0.02) 100% - ); - border: 1px solid var(--border-color); - border-radius: var(--radius-lg); - padding: 18px; - margin-bottom: 20px; - box-shadow: var(--shadow-sm); -}*/ - -.schema-section h3 { - font-size: 15px; - font-weight: 600; - margin-bottom: 14px; - margin-top: 0; - color: var(--text-primary); -} - -.schema-grid { - display: grid; - grid-template-columns: repeat(auto-fill, minmax(280px, 1fr)); - gap: 10px; -} - -.schema-field-card { - background: var(--bg-elevated); - border: 1px solid var(--border-color); - border-radius: var(--radius-md); - padding: 12px; - transition: var(--transition); - cursor: pointer; - position: relative; - opacity: 0.5; -} - -.schema-field-card.selected { - opacity: 1; - border-color: var(--primary); - background: linear-gradient( - 135deg, - rgba(99, 102, 241, 0.1) 0%, - rgba(99, 102, 241, 0.05) 100% - ); -} - -/*.schema-field-card::after { - content: "✓"; - position: absolute; - bottom: 8px; - right: 8px; - font-size: 14px; - font-weight: bold; - color: var(--primary); - opacity: 0; - transition: var(--transition); -} - -.schema-field-card.selected::after { - opacity: 1; -}*/ - -.schema-field-card:hover { - border-color: var(--primary-light); - transform: translateY(-2px); - box-shadow: var(--shadow-md); -} - -/* 数据表格 */ -.table-wrapper { - overflow-x: auto; - margin-bottom: 16px; - border-radius: var(--radius-lg); - border: 1px solid var(--border-color); - background: var(--bg-surface); -} - -.data-table { - width: 100%; - border-collapse: collapse; -} - -.data-table th { - background: var(--bg-elevated); - padding: 10px 12px; - text-align: left; - font-size: 11px; - font-weight: 600; - color: var(--text-secondary); - text-transform: uppercase; - letter-spacing: 0.05em; - border-bottom: 1px solid var(--border-color); - position: sticky; - top: 0; - z-index: 10; -} - -.data-table td { - padding: 10px 12px; - border-bottom: 1px solid var(--border-color); - font-size: 13px; - color: var(--text-primary); - max-width: 400px; - overflow: hidden; - text-overflow: ellipsis; - white-space: nowrap; -} - -.data-table tbody tr { - transition: var(--transition); -} - -.data-table tbody tr:hover { - background: rgba(255, 255, 255, 0.03); -} - -.data-table tbody tr:last-child td { - border-bottom: none; -} - -/* 分页 */ -.pagination { - margin-top: 16px; - display: flex; - justify-content: center; - align-items: center; - gap: 8px; -} - -.pagination button, -.pagination select, -.pagination input { - padding: 8px 12px; - background: var(--bg-surface); - border: 1px solid var(--border-color); - border-radius: var(--radius-md); - color: var(--text-primary); - font-size: 13px; - font-weight: 500; - cursor: pointer; - transition: var(--transition); -} - -.pagination button:hover:not(:disabled) { - background: var(--primary); - border-color: var(--primary); - box-shadow: var(--shadow-md); -} - -.pagination button:disabled { - opacity: 0.4; - cursor: not-allowed; -} - -.pagination input[type="number"] { - width: 80px; - text-align: center; -} - -.pagination select { - cursor: pointer; -} - -/* Manifest / LSM-Tree */ -.level-card { - background: var(--bg-surface); - border: 1px solid var(--border-color); - border-radius: var(--radius-lg); - padding: 14px; - margin-bottom: 12px; - transition: var(--transition); -} - -.level-card:hover { - border-color: var(--border-hover); - box-shadow: var(--shadow-md); -} - -.level-header { - display: flex; - justify-content: space-between; - align-items: center; - cursor: pointer; -} - -.level-title { - font-size: 16px; - font-weight: 600; - background: linear-gradient( - 135deg, - var(--text-primary), - var(--text-secondary) - ); - -webkit-background-clip: text; - -webkit-text-fill-color: transparent; -} - -.level-stats { - display: flex; - gap: 16px; - font-size: 12px; - color: var(--text-secondary); -} - -.score-badge { - padding: 4px 12px; - border-radius: 12px; - font-size: 12px; - font-weight: 600; -} - -.score-badge.normal { - background: rgba(16, 185, 129, 0.15); - color: var(--success); -} - -.score-badge.warning { - background: rgba(245, 158, 11, 0.15); - color: var(--warning); -} - -.score-badge.critical { - background: rgba(239, 68, 68, 0.15); - color: var(--danger); -} - -.file-list { - display: none; - margin-top: 12px; - grid-template-columns: repeat(auto-fill, minmax(350px, 1fr)); - gap: 10px; - padding-top: 8px; - /*border-top: 1px solid var(--border-color);*/ -} - -.file-card { - background: var(--bg-elevated); - border: 1px solid var(--border-color); - border-radius: var(--radius-md); - padding: 12px; - font-size: 12px; -} - -.file-header { - display: flex; - align-items: center; - justify-content: space-between; - font-weight: 600; - margin-bottom: 12px; - color: var(--text-primary); -} - -.file-detail { - display: flex; - justify-content: space-between; - /*padding: 4px 0;*/ - color: var(--text-secondary); - font-size: 12px; -} - -/* Modal */ -.modal { - display: none; - position: fixed; - top: 0; - left: 0; - width: 100%; - height: 100%; - background: rgba(0, 0, 0, 0.8); - backdrop-filter: blur(8px); - z-index: 1000; - justify-content: center; - align-items: center; - animation: fadeIn 0.2s ease-out; -} - -@keyframes fadeIn { - from { - opacity: 0; - } - to { - opacity: 1; - } -} - -.modal-content { - background: var(--bg-surface); - border: 1px solid var(--border-color); - border-radius: var(--radius-xl); - max-width: 90%; - max-height: 85%; - display: flex; - flex-direction: column; - box-shadow: var(--shadow-xl); - animation: slideUp 0.3s ease-out; -} - -@keyframes slideUp { - from { - transform: translateY(20px); - opacity: 0; - } - to { - transform: translateY(0); - opacity: 1; - } -} - -.modal-header { - padding: 16px 20px; - border-bottom: 1px solid var(--border-color); - display: flex; - justify-content: space-between; - align-items: center; - gap: 24px; -} - -.modal-header h3 { - font-size: 16px; - font-weight: 600; - color: var(--text-primary); -} - -.modal-close { - background: transparent; - color: var(--text-secondary); - border: 1px solid var(--border-color); - width: 32px; - height: 32px; - padding: 0; - display: flex; - align-items: center; - justify-content: center; - border-radius: 50%; - cursor: pointer; - transition: var(--transition); - flex-shrink: 0; -} - -.modal-close svg { - width: 18px; - height: 18px; - transition: inherit; -} - -.modal-close:hover { - background: rgba(239, 68, 68, 0.1); - border-color: var(--danger); - color: var(--danger); - transform: rotate(90deg); -} - -.modal-body { - padding: 16px; - overflow: auto; - font-family: "SF Mono", Monaco, monospace; - font-size: 12px; - line-height: 1.6; -} - -.modal-body pre { - white-space: pre-wrap; - word-break: break-word; - margin: 0; - color: var(--text-primary); -} - -/* 按钮 */ -button { - cursor: pointer; - transition: var(--transition); - font-family: inherit; -} - -.row-detail-btn { - background: var(--primary); - color: white; - border: none; - padding: 6px 12px; - border-radius: var(--radius-sm); - font-size: 12px; - font-weight: 600; - transition: var(--transition); -} - -.row-detail-btn:hover { - background: var(--primary-dark); - box-shadow: var(--shadow-md); - transform: translateY(-1px); -} - -/* 空状态和加载 */ -.loading, -.empty, -.error { - text-align: center; - padding: 60px 30px; -} - -.empty h2 { - font-size: 20px; - font-weight: 600; - margin-bottom: 10px; - color: var(--text-primary); -} - -.empty p { - font-size: 13px; - color: var(--text-secondary); -} - -.error { - color: var(--danger); -} - -/* Manifest stats */ -.manifest-stats { - display: grid; - grid-template-columns: repeat(auto-fit, minmax(200px, 1fr)); - gap: 12px; - margin-bottom: 16px; -} - -.stat-card { - background: var(--bg-surface); - border: 1px solid var(--border-color); - border-radius: var(--radius-lg); - padding: 14px; - transition: var(--transition); -} - -.stat-card:hover { - border-color: var(--border-hover); - transform: translateY(-2px); - box-shadow: var(--shadow-md); -} - -.stat-label { - font-size: 12px; - font-weight: 500; - color: var(--text-secondary); - text-transform: uppercase; - letter-spacing: 0.05em; - margin-bottom: 8px; -} - -.stat-value { - font-size: 28px; - font-weight: 700; - background: linear-gradient(135deg, var(--primary-light), var(--primary)); - -webkit-background-clip: text; - -webkit-text-fill-color: transparent; -} - -/* 响应式设计 */ -@media (max-width: 768px) { - .container { - flex-direction: column; - } - - .sidebar { - width: 100%; - border-right: none; - border-bottom: 1px solid var(--border-color); - max-height: 40vh; - } - - .main { - padding: 16px; - } - - .schema-grid { - grid-template-columns: 1fr; - } - - .manifest-stats { - grid-template-columns: repeat(2, 1fr); - } -} - -/* 列选择器 */ -.column-selector { - margin-bottom: 16px; - padding: 14px; - background: var(--bg-surface); - border: 1px solid var(--border-color); - border-radius: var(--radius-lg); -} - -.columns-container { - display: flex; - flex-wrap: wrap; - gap: 8px; -} - -.column-checkbox { - display: flex; - align-items: center; - padding: 8px 14px; - background: var(--bg-elevated); - border: 1px solid var(--border-color); - border-radius: var(--radius-md); - cursor: pointer; - font-size: 13px; - transition: var(--transition); -} - -.column-checkbox:hover { - border-color: var(--border-hover); -} - -.column-checkbox.selected { - background: var(--primary-bg); - border-color: var(--primary); -} - -.column-checkbox input { - margin-right: 8px; -} - -/* 工具按钮 */ -.control-buttons { - display: flex; - gap: 8px; -} - -.control-buttons button { - padding: 6px 14px; - background: var(--bg-surface); - border: 1px solid var(--border-color); - border-radius: var(--radius-md); - color: var(--text-primary); - font-size: 12px; - font-weight: 500; - transition: var(--transition); -} - -.control-buttons button:hover { - background: var(--bg-hover); - border-color: var(--border-hover); -} - -/* 单元格大小指示器 */ -.cell-size { - font-size: 11px; - color: var(--text-tertiary); - font-style: italic; - margin-left: 6px; -} - -/* 截断指示器 */ -.truncated-icon { - color: var(--warning); - margin-left: 6px; + transition: background-color 0.3s ease, color 0.3s ease; } diff --git a/webui/static/index.html b/webui/static/index.html index 7186d35..aadeb8a 100644 --- a/webui/static/index.html +++ b/webui/static/index.html @@ -1,5 +1,5 @@ - + @@ -11,59 +11,15 @@ rel="stylesheet" /> - -
- - - - -
-
-

Select a table to view data

-

Choose a table from the sidebar to get started

-
-
-
+ + - + - + + diff --git a/webui/static/js/app.js b/webui/static/js/app.js index 06e521d..80931a2 100644 --- a/webui/static/js/app.js +++ b/webui/static/js/app.js @@ -1,199 +1,110 @@ -// SRDB WebUI - htmx 版本 +import './components/app.js'; +import './components/table-list.js'; +import './components/table-view.js'; +import './components/modal-dialog.js'; +import './components/theme-toggle.js'; +import './components/badge.js'; +import './components/field-icon.js'; +import './components/data-view.js'; +import './components/manifest-view.js'; +import './components/page-header.js'; -// 全局状态 -window.srdbState = { - selectedTable: null, - currentPage: 1, - pageSize: 20, - selectedColumns: [], - expandedTables: new Set(), - expandedLevels: new Set([0, 1]), -}; +class App { + constructor() { + // 等待 srdb-app 组件渲染完成 + this.appContainer = document.querySelector('srdb-app'); + this.modal = document.querySelector('srdb-modal-dialog'); + + // 等待组件初始化 + if (this.appContainer) { + // 使用 updateComplete 等待组件渲染完成 + this.appContainer.updateComplete.then(() => { + this.tableList = this.appContainer.shadowRoot.querySelector('srdb-table-list'); + this.tableView = this.appContainer.shadowRoot.querySelector('srdb-table-view'); + this.pageHeader = this.appContainer.shadowRoot.querySelector('srdb-page-header'); + this.setupEventListeners(); + }); + } else { + // 如果组件还未定义，等待它被定义 + customElements.whenDefined('srdb-app').then(() => { + this.appContainer = document.querySelector('srdb-app'); + this.appContainer.updateComplete.then(() => { + this.tableList = this.appContainer.shadowRoot.querySelector('srdb-table-list'); + this.tableView = this.appContainer.shadowRoot.querySelector('srdb-table-view'); + this.pageHeader = this.appContainer.shadowRoot.querySelector('srdb-page-header'); + this.setupEventListeners(); + }); + }); + } + } -// 选择表格 -function selectTable(tableName) { - window.srdbState.selectedTable = tableName; - window.srdbState.currentPage = 1; - - // 高亮选中的表 - document.querySelectorAll(".table-item").forEach((el) => { - el.classList.toggle("selected", el.dataset.table === tableName); - }); - - // 加载表数据 - loadTableData(tableName); -} - -// 加载表数据 -function loadTableData(tableName) { - const mainContent = document.getElementById("main-content"); - mainContent.innerHTML = '
Loading...
'; - - fetch( - `/api/tables-view/${tableName}?page=${window.srdbState.currentPage}&pageSize=${window.srdbState.pageSize}`, - ) - .then((res) => res.text()) - .then((html) => { - mainContent.innerHTML = html; - }) - .catch((err) => { - console.error("Failed to load table data:", err); - mainContent.innerHTML = - '
Failed to load table data
'; + setupEventListeners() { + // Listen for table selection + document.addEventListener('table-selected', (e) => { + const tableName = e.detail.tableName; + this.pageHeader.tableName = tableName; + this.pageHeader.view = 'data'; + this.tableView.tableName = tableName; + this.tableView.view = 'data'; + this.tableView.page = 1; }); -} -// 切换视图 (Data / Manifest) -function switchView(tableName, mode) { - const mainContent = document.getElementById("main-content"); - mainContent.innerHTML = '
Loading...
'; - - const endpoint = - mode === "manifest" - ? `/api/tables-view/${tableName}/manifest` - : `/api/tables-view/${tableName}?page=${window.srdbState.currentPage}&pageSize=${window.srdbState.pageSize}`; - - fetch(endpoint) - .then((res) => res.text()) - .then((html) => { - mainContent.innerHTML = html; + // Listen for view change from page-header + document.addEventListener('view-changed', (e) => { + this.tableView.view = e.detail.view; }); -} -// 分页 -function changePage(delta) { - window.srdbState.currentPage += delta; - if (window.srdbState.selectedTable) { - loadTableData(window.srdbState.selectedTable); - } -} - -function jumpToPage(page) { - window.srdbState.currentPage = parseInt(page); - if (window.srdbState.selectedTable) { - loadTableData(window.srdbState.selectedTable); - } -} - -function changePageSize(newSize) { - window.srdbState.pageSize = parseInt(newSize); - window.srdbState.currentPage = 1; - if (window.srdbState.selectedTable) { - loadTableData(window.srdbState.selectedTable); - } -} - -// Modal 相关 -function showModal(title, content) { - document.getElementById("modal-title").textContent = title; - document.getElementById("modal-body-content").textContent = content; - document.getElementById("modal").style.display = "flex"; -} - -function closeModal() { - document.getElementById("modal").style.display = "none"; -} - -function showCellContent(content) { - showModal("Cell Content", content); -} - -function showRowDetail(tableName, seq) { - fetch(`/api/tables/${tableName}/data/${seq}`) - .then((res) => res.json()) - .then((data) => { - const formatted = JSON.stringify(data, null, 2); - showModal(`Row Detail (Seq: ${seq})`, formatted); - }) - .catch((err) => { - console.error("Failed to load row detail:", err); - alert("Failed to load row detail"); + // Listen for refresh request from page-header + document.addEventListener('refresh-view', (e) => { + this.tableView.loadData(); }); -} -// 折叠展开 -function toggleExpand(tableName) { - const item = document.querySelector(`[data-table="${tableName}"]`); - const fieldsDiv = item.querySelector(".schema-fields"); - const icon = item.querySelector(".expand-icon"); + // Listen for row detail request + document.addEventListener('show-row-detail', async (e) => { + const { tableName, seq } = e.detail; + await this.showRowDetail(tableName, seq); + }); - if (window.srdbState.expandedTables.has(tableName)) { - window.srdbState.expandedTables.delete(tableName); - fieldsDiv.style.display = "none"; - icon.classList.remove("expanded"); - } else { - window.srdbState.expandedTables.add(tableName); - fieldsDiv.style.display = "block"; - icon.classList.add("expanded"); + // Listen for cell content request + document.addEventListener('show-cell-content', (e) => { + this.showCellContent(e.detail.content); + }); + + // Close modal on backdrop click + this.modal.addEventListener('click', () => { + this.modal.open = false; + }); + } + + async showRowDetail(tableName, seq) { + try { + const response = await fetch(`/api/tables/${tableName}/data/${seq}`); + if (!response.ok) throw new Error('Failed to load row detail'); + + const data = await response.json(); + const content = JSON.stringify(data, null, 2); + + this.modal.title = `Row Detail - Seq: ${seq}`; + this.modal.content = content; + this.modal.open = true; + } catch (error) { + console.error('Error loading row detail:', error); + this.modal.title = 'Error'; + this.modal.content = `Failed to load row detail: ${error.message}`; + this.modal.open = true; + } + } + + showCellContent(content) { + this.modal.title = 'Cell Content'; + this.modal.content = String(content); + this.modal.open = true; } } -function toggleLevel(level) { - const levelCard = document.querySelector(`[data-level="${level}"]`); - const fileList = levelCard.querySelector(".file-list"); - const icon = levelCard.querySelector(".expand-icon"); - - if (window.srdbState.expandedLevels.has(level)) { - window.srdbState.expandedLevels.delete(level); - fileList.style.display = "none"; - icon.classList.remove("expanded"); - } else { - window.srdbState.expandedLevels.add(level); - fileList.style.display = "grid"; - icon.classList.add("expanded"); - } -} - -// 格式化工具 -function formatBytes(bytes) { - if (bytes === 0) return "0 B"; - const k = 1024; - const sizes = ["B", "KB", "MB", "GB"]; - const i = Math.floor(Math.log(bytes) / Math.log(k)); - return (bytes / Math.pow(k, i)).toFixed(2) + " " + sizes[i]; -} - -function formatCount(count) { - if (count >= 1000000) return (count / 1000000).toFixed(1) + "M"; - if (count >= 1000) return (count / 1000).toFixed(1) + "K"; - return count.toString(); -} - -// 点击 modal 外部关闭 -document.addEventListener("click", (e) => { - const modal = document.getElementById("modal"); - if (e.target === modal) { - closeModal(); - } -}); - -// ESC 键关闭 modal -document.addEventListener("keydown", (e) => { - if (e.key === "Escape") { - closeModal(); - } -}); - -// 切换列显示 -function toggleColumn(columnName) { - // 切换 schema-field-card 的选中状态 - const card = document.querySelector( - `.schema-field-card[data-column="${columnName}"]`, - ); - if (!card) return; - - card.classList.toggle("selected"); - const isSelected = card.classList.contains("selected"); - - // 切换表格列的显示/隐藏 - const headers = document.querySelectorAll(`th[data-column="${columnName}"]`); - const cells = document.querySelectorAll(`td[data-column="${columnName}"]`); - - headers.forEach((header) => { - header.style.display = isSelected ? "" : "none"; - }); - - cells.forEach((cell) => { - cell.style.display = isSelected ? "" : "none"; - }); +// Initialize app when DOM is ready +if (document.readyState === 'loading') { + document.addEventListener('DOMContentLoaded', () => new App()); +} else { + new App(); } diff --git a/webui/static/js/components/app.js b/webui/static/js/components/app.js new file mode 100644 index 0000000..d0981de --- /dev/null +++ b/webui/static/js/components/app.js @@ -0,0 +1,145 @@ +import { LitElement, html, css } from 'https://cdn.jsdelivr.net/gh/lit/dist@3/core/lit-core.min.js'; +import { sharedStyles, cssVariables } from '../styles/shared-styles.js'; + +export class AppContainer extends LitElement { + static properties = { + mobileMenuOpen: { type: Boolean } + }; + + constructor() { + super(); + this.mobileMenuOpen = false; + } + + toggleMobileMenu() { + this.mobileMenuOpen = !this.mobileMenuOpen; + } + + static styles = [ + sharedStyles, + cssVariables, + css` + :host { + display: block; + height: 100vh; + } + + .container { + display: flex; + height: 100%; + overflow: hidden; + } + + .sidebar { + width: 280px; + background: var(--bg-surface); + border-right: 1px solid var(--border-color); + overflow-y: auto; + overflow-x: hidden; + padding: 16px 12px; + display: flex; + flex-direction: column; + gap: 8px; + } + + .sidebar::-webkit-scrollbar { + width: 6px; + } + + .sidebar-header { + display: flex; + align-items: center; + justify-content: space-between; + margin-bottom: 4px; + } + + .sidebar h1 { + font-size: 18px; + font-weight: 700; + letter-spacing: -0.02em; + background: linear-gradient(135deg, var(--primary-light), var(--primary)); + -webkit-background-clip: text; + -webkit-text-fill-color: transparent; + } + + .main { + flex: 1; + overflow-y: auto; + overflow-x: hidden; + background: var(--bg-main); + display: flex; + flex-direction: column; + } + + + /* 移动端遮罩 */ + .mobile-overlay { + display: none; + position: fixed; + top: 0; + left: 0; + width: 100%; + height: 100%; + background: rgba(0, 0, 0, 0.5); + z-index: 999; + } + + @media (max-width: 768px) { + .mobile-overlay.show { + display: block; + } + + .container { + flex-direction: column; + } + + .sidebar { + position: fixed; + top: 0; + left: 0; + width: 280px; + height: 100vh; + border-right: 1px solid var(--border-color); + border-bottom: none; + transform: translateX(-100%); + transition: transform 0.3s ease; + z-index: 1000; + } + + .sidebar.open { + transform: translateX(0); + } + + .main { + padding-top: 0; + } + } + ` + ]; + + render() { + return html` + +
+ +
+ +
+
+

SRDB Tables

+ +
+ +
+ + +
+ + +
+
+ `; + } +} + +customElements.define('srdb-app', AppContainer); diff --git a/webui/static/js/components/badge.js b/webui/static/js/components/badge.js new file mode 100644 index 0000000..82e1a16 --- /dev/null +++ b/webui/static/js/components/badge.js @@ -0,0 +1,106 @@ +import { LitElement, html, css } from 'https://cdn.jsdelivr.net/gh/lit/dist@3/core/lit-core.min.js'; +import { sharedStyles, cssVariables } from '../styles/shared-styles.js'; + +export class Badge extends LitElement { + static properties = { + variant: { type: String }, // 'primary', 'success', 'warning', 'danger', 'info' + icon: { type: String }, + size: { type: String } // 'sm', 'md' + }; + + static styles = [ + sharedStyles, + cssVariables, + css` + :host { + display: inline-flex; + } + + .badge { + position: relative; + display: inline-flex; + align-items: center; + gap: 4px; + padding: 2px 8px; + font-size: 11px; + font-weight: 600; + border-radius: var(--radius-sm); + white-space: nowrap; + } + + .badge.size-md { + padding: 4px 10px; + font-size: 12px; + } + + /* Primary variant */ + .badge.variant-primary { + --badge-border-color: rgba(99, 102, 241, 0.2); + background: rgba(99, 102, 241, 0.15); + color: var(--primary); + } + + /* Success variant */ + .badge.variant-success { + --badge-border-color: rgba(16, 185, 129, 0.2); + background: rgba(16, 185, 129, 0.15); + color: var(--success); + } + + /* Warning variant */ + .badge.variant-warning { + --badge-border-color: rgba(245, 158, 11, 0.2); + background: rgba(245, 158, 11, 0.15); + color: var(--warning); + } + + /* Danger variant */ + .badge.variant-danger { + --badge-border-color: rgba(239, 68, 68, 0.2); + background: rgba(239, 68, 68, 0.15); + color: var(--danger); + } + + /* Info variant */ + .badge.variant-info { + --badge-border-color: rgba(59, 130, 246, 0.2); + background: rgba(59, 130, 246, 0.15); + color: var(--info); + } + + /* Secondary variant */ + .badge.variant-secondary { + --badge-border-color: rgba(160, 160, 176, 0.2); + background: rgba(160, 160, 176, 0.15); + color: var(--text-secondary); + } + + .icon { + font-size: 12px; + line-height: 1; + } + + .badge.size-md .icon { + font-size: 14px; + } + ` + ]; + + constructor() { + super(); + this.variant = 'primary'; + this.icon = ''; + this.size = 'sm'; + } + + render() { + return html` + + ${this.icon ? html`${this.icon}` : ''} + + + `; + } +} + +customElements.define('srdb-badge', Badge); diff --git a/webui/static/js/components/data-view.js b/webui/static/js/components/data-view.js new file mode 100644 index 0000000..0f6d9c5 --- /dev/null +++ b/webui/static/js/components/data-view.js @@ -0,0 +1,351 @@ +import { LitElement, html, css } from 'https://cdn.jsdelivr.net/gh/lit/dist@3/core/lit-core.min.js'; +import { sharedStyles, cssVariables } from '../styles/shared-styles.js'; + +export class DataView extends LitElement { + static properties = { + tableName: { type: String }, + schema: { type: Object }, + tableData: { type: Object }, + selectedColumns: { type: Array }, + loading: { type: Boolean } + }; + + static styles = [ + sharedStyles, + cssVariables, + css` + :host { + display: block; + } + + h3 { + font-size: 16px; + font-weight: 600; + margin: 20px 0 12px 0; + color: var(--text-primary); + } + + .schema-section { + margin-bottom: 24px; + } + + .schema-grid { + display: grid; + grid-template-columns: repeat(auto-fill, minmax(200px, 1fr)); + gap: 12px; + } + + .schema-field-card { + padding: 12px; + background: var(--bg-elevated); + border: 1px solid var(--border-color); + border-radius: var(--radius-md); + cursor: pointer; + transition: var(--transition); + } + + .schema-field-card:hover { + background: var(--bg-hover); + } + + .schema-field-card.selected { + border-color: var(--primary); + background: var(--primary-bg); + } + + .field-item { + display: flex; + flex-direction: column; + gap: 4px; + } + + .field-item-row { + display: flex; + align-items: center; + gap: 8px; + } + + .field-index-icon { + font-size: 14px; + flex-shrink: 0; + } + + .field-name { + font-weight: 500; + color: var(--text-primary); + font-size: 13px; + flex: 1; + } + + .field-type { + font-family: 'Courier New', monospace; + } + + .field-comment { + color: var(--text-tertiary); + font-size: 11px; + margin-top: 4px; + font-style: italic; + min-height: 16px; + } + + .field-comment:empty::before { + content: "No comment"; + opacity: 0.5; + } + + .table-wrapper { + overflow-x: auto; + background: var(--bg-elevated); + border-radius: var(--radius-md); + border: 1px solid var(--border-color); + } + + .data-table { + width: 100%; + border-collapse: collapse; + font-size: 13px; + } + + .data-table th { + background: var(--bg-surface); + color: var(--text-secondary); + font-weight: 600; + text-align: left; + padding: 12px; + border-bottom: 1px solid var(--border-color); + position: sticky; + top: 0; + z-index: 1; + } + + .data-table td { + padding: 10px 12px; + border-bottom: 1px solid var(--border-color); + color: var(--text-primary); + max-width: 300px; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; + } + + .data-table td:hover { + white-space: normal; + word-break: break-all; + } + + .data-table tr:hover { + background: var(--bg-hover); + } + + .truncated-icon { + margin-left: 4px; + font-size: 10px; + } + + .row-detail-btn { + padding: 4px 12px; + background: var(--primary); + color: white; + border: none; + border-radius: var(--radius-sm); + cursor: pointer; + font-size: 12px; + transition: var(--transition); + } + + .row-detail-btn:hover { + background: var(--primary-dark); + } + ` + ]; + + constructor() { + super(); + this.tableName = ''; + this.schema = null; + this.tableData = null; + this.selectedColumns = []; + this.loading = false; + } + + updated(changedProperties) { + // 当 tableName 或 schema 改变时，尝试加载保存的列选择 + if ((changedProperties.has('tableName') || changedProperties.has('schema')) && this.tableName && this.schema) { + const saved = this.loadSelectedColumns(); + if (saved && saved.length > 0) { + // 验证保存的列是否仍然存在于当前 schema 中 + const validColumns = saved.filter(col => + this.schema.fields.some(field => field.name === col) + ); + if (validColumns.length > 0) { + this.selectedColumns = validColumns; + } + } + } + } + + toggleColumn(columnName) { + const index = this.selectedColumns.indexOf(columnName); + if (index > -1) { + this.selectedColumns = this.selectedColumns.filter(c => c !== columnName); + } else { + this.selectedColumns = [...this.selectedColumns, columnName]; + } + + // 持久化到 localStorage + this.saveSelectedColumns(); + + this.dispatchEvent(new CustomEvent('columns-changed', { + detail: { columns: this.selectedColumns }, + bubbles: true, + composed: true + })); + } + + saveSelectedColumns() { + if (!this.tableName) return; + const key = `srdb_columns_${this.tableName}`; + localStorage.setItem(key, JSON.stringify(this.selectedColumns)); + } + + loadSelectedColumns() { + if (!this.tableName) return null; + const key = `srdb_columns_${this.tableName}`; + const saved = localStorage.getItem(key); + return saved ? JSON.parse(saved) : null; + } + + showRowDetail(seq) { + this.dispatchEvent(new CustomEvent('show-row-detail', { + detail: { tableName: this.tableName, seq }, + bubbles: true, + composed: true + })); + } + + formatCount(count) { + if (count >= 1000000) return (count / 1000000).toFixed(1) + 'M'; + if (count >= 1000) return (count / 1000).toFixed(1) + 'K'; + return count.toString(); + } + + formatTime(nanoTime) { + if (!nanoTime) return ''; + // 将纳秒转换为毫秒 + const date = new Date(nanoTime / 1000000); + // 格式化为 YYYY-MM-DD HH:mm:ss + const year = date.getFullYear(); + const month = String(date.getMonth() + 1).padStart(2, '0'); + const day = String(date.getDate()).padStart(2, '0'); + const hours = String(date.getHours()).padStart(2, '0'); + const minutes = String(date.getMinutes()).padStart(2, '0'); + const seconds = String(date.getSeconds()).padStart(2, '0'); + return `${year}-${month}-${day} ${hours}:${minutes}:${seconds}`; + } + + getColumns() { + let columns = []; + + if (this.selectedColumns.length > 0) { + columns = [...this.selectedColumns]; + } else { + columns = this.schema?.fields?.map(f => f.name) || []; + } + + // 确保系统字段的顺序：_seq 在开头，_time 在倒数第二 + const filtered = columns.filter(c => c !== '_seq' && c !== '_time'); + + // _seq 放开头 + const result = ['_seq', ...filtered]; + + // _time 放倒数第二（Actions 列之前） + result.push('_time'); + + return result; + } + + render() { + if (this.loading || !this.schema || !this.tableData) { + return html`
Loading data...
`; + } + + const columns = this.getColumns(); + + return html` + ${this.renderSchemaSection()} + +

Data (${this.formatCount(this.tableData.totalRows)} rows)

+ + ${this.tableData.data.length === 0 ? html` +

No data available

+ ` : html` +
+ + + + ${columns.map(col => html``)} + + + + + ${this.tableData.data.map(row => html` + + ${columns.map(col => html` + + `)} + + + `)} + +
${col}Actions
+ ${col === '_time' ? this.formatTime(row[col]) : row[col]} + ${row[col + '_truncated'] ? html`✂️` : ''} + + +
+
+ `} + `; + } + + renderSchemaSection() { + if (!this.schema || !this.schema.fields) return ''; + + return html` +
+

Schema (点击字段卡片选择要显示的列)

+
+ ${this.schema.fields.map(field => html` +
this.toggleColumn(field.name)} + > +
+
+ + ${field.name} + + ${field.type} + +
+
${field.comment || ''}
+
+
+ `)} +
+
+ `; + } +} + +customElements.define('srdb-data-view', DataView); diff --git a/webui/static/js/components/field-icon.js b/webui/static/js/components/field-icon.js new file mode 100644 index 0000000..4e5c986 --- /dev/null +++ b/webui/static/js/components/field-icon.js @@ -0,0 +1,59 @@ +import { LitElement, html, css, svg } from 'https://cdn.jsdelivr.net/gh/lit/dist@3/core/lit-core.min.js'; + +export class FieldIcon extends LitElement { + static properties = { + indexed: { type: Boolean } + }; + + static styles = css` + :host { + display: inline-flex; + align-items: center; + justify-content: center; + width: 20px; + height: 20px; + } + + svg { + width: 16px; + height: 16px; + } + + .indexed { + fill: var(--success); + color: var(--success); + opacity: 1; + } + + .not-indexed { + fill: var(--text-secondary); + color: var(--text-secondary); + opacity: 0.6; + } + `; + + constructor() { + super(); + this.indexed = false; + } + + render() { + if (this.indexed) { + // 闪电图标 - 已索引（快速） + return html` + + + + `; + } else { + // 圆点图标 - 未索引 + return html` + + + + `; + } + } +} + +customElements.define('srdb-field-icon', FieldIcon); diff --git a/webui/static/js/components/manifest-view.js b/webui/static/js/components/manifest-view.js new file mode 100644 index 0000000..eec7a41 --- /dev/null +++ b/webui/static/js/components/manifest-view.js @@ -0,0 +1,301 @@ +import { LitElement, html, css } from 'https://cdn.jsdelivr.net/gh/lit/dist@3/core/lit-core.min.js'; +import { sharedStyles, cssVariables } from '../styles/shared-styles.js'; + +export class ManifestView extends LitElement { + static properties = { + manifestData: { type: Object }, + loading: { type: Boolean }, + expandedLevels: { type: Set } + }; + + static styles = [ + sharedStyles, + cssVariables, + css` + :host { + display: block; + } + + h3 { + font-size: 16px; + font-weight: 600; + margin: 20px 0 12px 0; + color: var(--text-primary); + } + + .manifest-stats { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(200px, 1fr)); + gap: 16px; + margin-bottom: 24px; + } + + .stat-card { + padding: 16px; + background: var(--bg-elevated); + border: 1px solid var(--border-color); + border-radius: var(--radius-md); + } + + .stat-label { + font-size: 12px; + color: var(--text-tertiary); + margin-bottom: 8px; + } + + .stat-value { + font-size: 20px; + font-weight: 600; + color: var(--text-primary); + } + + .level-card { + margin-bottom: 12px; + background: var(--bg-elevated); + border: 1px solid var(--border-color); + border-radius: var(--radius-md); + overflow: hidden; + } + + .level-header { + display: flex; + align-items: center; + justify-content: space-between; + padding: 16px; + cursor: pointer; + transition: var(--transition); + } + + .level-header:hover { + background: var(--bg-hover); + } + + .level-header-left { + display: flex; + align-items: center; + gap: 12px; + flex: 1; + } + + .expand-icon { + font-size: 12px; + color: var(--text-secondary); + transition: transform 0.2s ease; + user-select: none; + } + + .expand-icon.expanded { + transform: rotate(90deg); + } + + .level-title { + font-size: 16px; + font-weight: 600; + color: var(--text-primary); + } + + .level-stats { + display: flex; + gap: 16px; + font-size: 13px; + color: var(--text-secondary); + } + + .level-files { + padding: 16px; + background: var(--bg-surface); + border-top: 1px solid var(--border-color); + display: none; + } + + .level-files.expanded { + display: block; + } + + .file-list { + display: grid; + grid-template-columns: repeat(auto-fill, minmax(280px, 1fr)); + gap: 12px; + } + + .file-item { + padding: 12px; + background: var(--bg-elevated); + border: 1px solid var(--border-color); + border-radius: var(--radius-sm); + transition: var(--transition); + } + + .file-item:hover { + border-color: var(--primary); + box-shadow: 0 2px 8px rgba(0, 0, 0, 0.1); + } + + .file-name { + font-family: 'Courier New', monospace; + font-size: 13px; + color: var(--text-primary); + font-weight: 500; + margin-bottom: 8px; + } + + .file-detail { + display: flex; + flex-direction: column; + gap: 4px; + font-size: 12px; + color: var(--text-secondary); + } + + .file-detail-row { + display: flex; + justify-content: space-between; + } + + @media (max-width: 768px) { + .file-list { + grid-template-columns: 1fr; + } + } + + .empty { + background: var(--bg-elevated); + border-radius: var(--radius-md); + border: 1px dashed var(--border-color); + margin-top: 24px; + } + + .empty p { + margin: 0; + } + ` + ]; + + constructor() { + super(); + this.manifestData = null; + this.loading = false; + this.expandedLevels = new Set(); + } + + toggleLevel(levelNum) { + if (this.expandedLevels.has(levelNum)) { + this.expandedLevels.delete(levelNum); + } else { + this.expandedLevels.add(levelNum); + } + this.requestUpdate(); + } + + formatSize(bytes) { + if (bytes >= 1024 * 1024 * 1024) return (bytes / (1024 * 1024 * 1024)).toFixed(2) + ' GB'; + if (bytes >= 1024 * 1024) return (bytes / (1024 * 1024)).toFixed(2) + ' MB'; + if (bytes >= 1024) return (bytes / 1024).toFixed(2) + ' KB'; + return bytes + ' B'; + } + + getScoreVariant(score) { + if (score >= 0.8) return 'danger'; // 高分 = 需要紧急 compaction + if (score >= 0.5) return 'warning'; // 中分 = 需要关注 + return 'success'; // 低分 = 健康状态 + } + + render() { + if (this.loading || !this.manifestData) { + return html`
Loading manifest...
`; + } + + const totalFiles = this.manifestData.levels.reduce((sum, l) => sum + l.file_count, 0); + const totalSize = this.manifestData.levels.reduce((sum, l) => sum + l.total_size, 0); + const totalCompactions = this.manifestData.compaction_stats?.total_compactions || 0; + + return html` +

LSM-Tree Structure

+ +
+
+
Active Levels
+
${this.manifestData.levels.filter(l => l.file_count > 0).length}
+
+
+
Total Files
+
${totalFiles}
+
+
+
Total Size
+
${this.formatSize(totalSize)}
+
+
+
Compactions
+
${totalCompactions}
+
+
+ + ${this.manifestData.levels && this.manifestData.levels.length > 0 + ? this.manifestData.levels.map(level => this.renderLevelCard(level)) + : html` +
+

No SSTable files in this table yet.

+

Insert some data to see the LSM-Tree structure.

+
+ ` + } + `; + } + + renderLevelCard(level) { + if (level.file_count === 0) return ''; + + const isExpanded = this.expandedLevels.has(level.level); + + return html` +
+
this.toggleLevel(level.level)}> +
+ ▶ +
+
Level ${level.level}
+
+ ${level.file_count} files + ${this.formatSize(level.total_size)} + ${level.score !== undefined ? html` + + Score: ${(level.score * 100).toFixed(0)}% + + ` : ''} +
+
+
+
+ + ${level.files && level.files.length > 0 ? html` +
+
+ ${level.files.map(file => html` +
+
${file.file_number}.sst
+
+
+ Size: + ${this.formatSize(file.file_size)} +
+
+ Rows: + ${file.row_count || 0} +
+
+ Seq Range: + ${file.min_key} - ${file.max_key} +
+
+
+ `)} +
+
+ ` : ''} +
+ `; + } +} + +customElements.define('srdb-manifest-view', ManifestView); diff --git a/webui/static/js/components/modal-dialog.js b/webui/static/js/components/modal-dialog.js new file mode 100644 index 0000000..26660d2 --- /dev/null +++ b/webui/static/js/components/modal-dialog.js @@ -0,0 +1,179 @@ +import { LitElement, html, css } from 'https://cdn.jsdelivr.net/gh/lit/dist@3/core/lit-core.min.js'; +import { sharedStyles, cssVariables } from '../styles/shared-styles.js'; + +export class ModalDialog extends LitElement { + static properties = { + open: { type: Boolean }, + title: { type: String }, + content: { type: String } + }; + + static styles = [ + sharedStyles, + cssVariables, + css` + :host { + display: none; + position: fixed; + top: 0; + left: 0; + width: 100%; + height: 100%; + background: rgba(0, 0, 0, 0.7); + z-index: 1000; + align-items: center; + justify-content: center; + } + + :host([open]) { + display: flex; + } + + .modal-content { + background: var(--bg-surface); + border-radius: var(--radius-lg); + box-shadow: var(--shadow-xl); + min-width: 324px; + max-width: 90vw; + max-height: 80vh; + width: fit-content; + display: flex; + flex-direction: column; + border: 1px solid var(--border-color); + } + + @media (max-width: 768px) { + .modal-content { + min-width: 300px; + max-width: 95vw; + } + } + + .modal-header { + display: flex; + align-items: center; + justify-content: space-between; + padding: 20px 24px; + border-bottom: 1px solid var(--border-color); + } + + .modal-header h3 { + margin: 0; + font-size: 18px; + font-weight: 600; + color: var(--text-primary); + } + + .modal-close { + background: transparent; + border: none; + color: var(--text-secondary); + cursor: pointer; + padding: 4px; + display: flex; + align-items: center; + justify-content: center; + border-radius: var(--radius-sm); + transition: var(--transition); + } + + .modal-close:hover { + background: var(--bg-hover); + color: var(--text-primary); + } + + .modal-body { + padding: 24px; + overflow-y: auto; + flex: 1; + } + + pre { + /*background: var(--bg-elevated);*/ + padding: 16px; + border-radius: var(--radius-md); + overflow-x: auto; + color: var(--text-primary); + font-family: 'Courier New', monospace; + font-size: 13px; + line-height: 1.5; + margin: 0; + white-space: pre-wrap; + word-wrap: break-word; + } + ` + ]; + + constructor() { + super(); + this.open = false; + this.title = 'Content'; + this.content = ''; + this._handleKeyDown = this._handleKeyDown.bind(this); + } + + connectedCallback() { + super.connectedCallback(); + document.addEventListener('keydown', this._handleKeyDown); + } + + disconnectedCallback() { + super.disconnectedCallback(); + document.removeEventListener('keydown', this._handleKeyDown); + } + + updated(changedProperties) { + if (changedProperties.has('open')) { + if (this.open) { + this.setAttribute('open', ''); + } else { + this.removeAttribute('open'); + } + } + } + + _handleKeyDown(e) { + if (this.open && e.key === 'Escape') { + this.close(); + } + } + + close() { + this.open = false; + this.dispatchEvent(new CustomEvent('modal-close', { + bubbles: true, + composed: true + })); + } + + render() { + return html` +
e.stopPropagation()}> +
+

${this.title}

+ +
+
+ 
${this.content}
+
+
+ `; + } +} + +customElements.define('srdb-modal-dialog', ModalDialog); diff --git a/webui/static/js/components/page-header.js b/webui/static/js/components/page-header.js new file mode 100644 index 0000000..67da256 --- /dev/null +++ b/webui/static/js/components/page-header.js @@ -0,0 +1,267 @@ +import { LitElement, html, css } from 'https://cdn.jsdelivr.net/gh/lit/dist@3/core/lit-core.min.js'; +import { sharedStyles, cssVariables } from '../styles/shared-styles.js'; + +export class PageHeader extends LitElement { + static properties = { + tableName: { type: String }, + view: { type: String } + }; + + static styles = [ + sharedStyles, + cssVariables, + css` + :host { + display: block; + background: var(--bg-surface); + border-bottom: 1px solid var(--border-color); + position: sticky; + top: 0; + z-index: 10; + } + + .header-content { + padding: 16px 24px; + } + + .header-top { + display: flex; + align-items: center; + gap: 16px; + margin-bottom: 12px; + } + + .mobile-menu-btn { + display: none; + width: 40px; + height: 40px; + background: var(--primary); + border: none; + border-radius: var(--radius-md); + color: white; + cursor: pointer; + flex-shrink: 0; + transition: var(--transition); + } + + .mobile-menu-btn:hover { + background: var(--primary-dark); + } + + .mobile-menu-btn svg { + width: 20px; + height: 20px; + } + + h2 { + font-size: 24px; + font-weight: 600; + margin: 0; + color: var(--text-primary); + } + + .empty-state { + text-align: center; + padding: 20px; + color: var(--text-secondary); + } + + .view-tabs { + display: flex; + align-items: center; + gap: 8px; + border-bottom: 1px solid var(--border-color); + margin: 0 -12px; + padding: 0 24px; + } + + .refresh-btn { + margin-left: auto; + padding: 8px 12px; + background: transparent; + border: 1px solid var(--border-color); + border-radius: var(--radius-sm); + color: var(--text-secondary); + cursor: pointer; + transition: var(--transition); + display: flex; + align-items: center; + gap: 6px; + font-size: 13px; + } + + .refresh-btn:hover { + background: var(--bg-hover); + border-color: var(--border-hover); + color: var(--text-primary); + } + + .refresh-btn svg { + width: 16px; + height: 16px; + } + + .view-tab { + position: relative; + padding: 16px 20px; + background: transparent; + border: none; + color: var(--text-secondary); + cursor: pointer; + font-size: 14px; + font-weight: 500; + transition: var(--transition); + } + + .view-tab:hover { + color: var(--text-primary); + /*background: var(--bg-hover);*/ + } + + .view-tab.active { + color: var(--primary); + } + + .view-tab::before, + .view-tab::after { + position: absolute; + display: block; + content: ""; + z-index: 0; + opacity: 0; + inset-inline: 8px; + } + + .view-tab::before { + inset-block: 8px; + border-radius: var(--radius-md); + background: var(--bg-elevated); + } + + .view-tab::after { + border-radius: var(--radius-md); + background: var(--primary); + bottom: -2px; + height: 4px; + } + + .view-tab.active::before, + .view-tab.active::after, + .view-tab:hover::before { + opacity: 1; + } + + .view-tab span { + position: relative; + z-index: 1; + } + + @media (max-width: 768px) { + .mobile-menu-btn { + display: flex; + align-items: center; + justify-content: center; + } + + .header-content { + padding: 12px 16px; + } + + h2 { + font-size: 18px; + flex: 1; + } + + .view-tabs { + margin: 0 -16px; + padding: 0 16px; + } + + .view-tab { + padding: 8px 16px; + font-size: 13px; + } + } + ` + ]; + + constructor() { + super(); + this.tableName = ''; + this.view = 'data'; + } + + switchView(newView) { + this.view = newView; + this.dispatchEvent(new CustomEvent('view-changed', { + detail: { view: newView }, + bubbles: true, + composed: true + })); + } + + toggleMobileMenu() { + this.dispatchEvent(new CustomEvent('toggle-mobile-menu', { + bubbles: true, + composed: true + })); + } + + refreshView() { + this.dispatchEvent(new CustomEvent('refresh-view', { + detail: { view: this.view }, + bubbles: true, + composed: true + })); + } + + render() { + if (!this.tableName) { + return html` +
+
+

Select a table from the sidebar

+
+
+ `; + } + + return html` +
+
+ +

${this.tableName}

+
+
+ +
+ + + +
+ `; + } +} + +customElements.define('srdb-page-header', PageHeader); diff --git a/webui/static/js/components/table-list.js b/webui/static/js/components/table-list.js new file mode 100644 index 0000000..cab6710 --- /dev/null +++ b/webui/static/js/components/table-list.js @@ -0,0 +1,284 @@ +import { LitElement, html, css } from 'https://cdn.jsdelivr.net/gh/lit/dist@3/core/lit-core.min.js'; +import { sharedStyles, cssVariables } from '../styles/shared-styles.js'; + +export class TableList extends LitElement { + static properties = { + tables: { type: Array }, + selectedTable: { type: String }, + expandedTables: { type: Set } + }; + + static styles = [ + sharedStyles, + cssVariables, + css` + :host { + display: block; + width: 100%; + } + + .table-item { + margin-bottom: 8px; + background: var(--bg-elevated); + border: 1px solid var(--border-color); + border-radius: var(--radius-md); + overflow: hidden; + transition: var(--transition); + } + + .table-item:hover { + border-color: var(--border-hover); + } + + .table-item.selected { + border-color: var(--primary); + box-shadow: 0 0 0 1px var(--primary); + } + + .table-item.selected .table-header { + background: var(--primary-bg); + } + + .table-header { + display: flex; + align-items: center; + justify-content: space-between; + padding: 10px 12px; + cursor: pointer; + transition: var(--transition); + } + + .table-header:hover { + background: var(--bg-hover); + } + + .table-item.has-expanded .table-header { + border-bottom-color: var(--border-color); + } + + .table-header-left { + display: flex; + align-items: center; + gap: 8px; + flex: 1; + min-width: 0; + } + + .expand-icon { + display: flex; + align-items: center; + justify-content: center; + width: 24px; + height: 24px; + font-size: 12px; + transition: var(--transition); + flex-shrink: 0; + border-radius: var(--radius-sm); + cursor: pointer; + color: var(--text-secondary); + } + + .expand-icon:hover { + background: var(--bg-hover); + color: var(--text-primary); + } + + .expand-icon.expanded { + transform: rotate(90deg); + color: var(--primary); + } + + .table-name { + font-weight: 500; + color: var(--text-primary); + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; + } + .table-count { + font-size: 12px; + color: var(--text-tertiary); + white-space: nowrap; + flex-shrink: 0; + } + + /* Schema 字段列表 */ + .schema-fields { + display: none; + flex-direction: column; + border-top: 1px solid transparent; + transition: var(--transition); + position: relative; + } + + .schema-fields.expanded { + display: block; + border-top-color: var(--border-color); + } + + /* 共享的垂直线 */ + .schema-fields.expanded::before { + z-index: 2; + content: ""; + position: absolute; + left: 24px; + top: 0; + bottom: 24px; + width: 1px; + background: var(--border-color); + } + + .field-item { + z-index: 1; + display: flex; + align-items: center; + gap: 8px; + padding: 8px 12px 8px 24px; + font-size: 12px; + transition: var(--transition); + position: relative; + } + + /* 每个字段的水平线 */ + .field-item::before { + content: ""; + width: 8px; + height: 1px; + background: var(--border-color); + } + + .field-item:hover { + background: var(--bg-hover); + } + + .field-item:last-child { + padding-bottom: 12px; + } + + .field-index-icon { + display: flex; + align-items: center; + justify-content: center; + width: 20px; + height: 20px; + font-size: 14px; + flex-shrink: 0; + } + + .field-index-icon.indexed { + color: var(--success); + } + + .field-index-icon.not-indexed { + color: var(--text-tertiary); + opacity: 0.5; + } + + .field-name { + font-weight: 500; + color: var(--text-secondary); + flex: 1; + } + .field-type { + font-family: 'Courier New', monospace; + } + + .loading { + padding: 20px; + } + + .error { + text-align: center; + padding: 20px; + color: var(--danger); + } + ` + ]; + + constructor() { + super(); + this.tables = []; + this.selectedTable = ''; + this.expandedTables = new Set(); + } + + connectedCallback() { + super.connectedCallback(); + this.loadTables(); + } + + async loadTables() { + try { + const response = await fetch('/api/tables'); + if (!response.ok) throw new Error('Failed to load tables'); + this.tables = await response.json(); + } catch (error) { + console.error('Error loading tables:', error); + } + } + + toggleExpand(tableName, event) { + event.stopPropagation(); + if (this.expandedTables.has(tableName)) { + this.expandedTables.delete(tableName); + } else { + this.expandedTables.add(tableName); + } + this.requestUpdate(); + } + + selectTable(tableName) { + this.selectedTable = tableName; + this.dispatchEvent(new CustomEvent('table-selected', { + detail: { tableName }, + bubbles: true, + composed: true + })); + } + + render() { + if (this.tables.length === 0) { + return html`
Loading tables...
`; + } + + return html` + ${this.tables.map(table => html` +
+
this.selectTable(table.name)} + > +
+ this.toggleExpand(table.name, e)} + > + ▶ + + ${table.name} +
+ ${table.fields.length} fields +
+ +
+ ${table.fields.map(field => html` +
+ + ${field.name} + + ${field.type} + +
+ `)} +
+
+ `)} + `; + } +} + +customElements.define('srdb-table-list', TableList); diff --git a/webui/static/js/components/table-view.js b/webui/static/js/components/table-view.js new file mode 100644 index 0000000..c064c46 --- /dev/null +++ b/webui/static/js/components/table-view.js @@ -0,0 +1,364 @@ +import { LitElement, html, css } from 'https://cdn.jsdelivr.net/gh/lit/dist@3/core/lit-core.min.js'; +import { sharedStyles, cssVariables } from '../styles/shared-styles.js'; + +export class TableView extends LitElement { + static properties = { + tableName: { type: String }, + view: { type: String }, // 'data' or 'manifest' + schema: { type: Object }, + tableData: { type: Object }, + manifestData: { type: Object }, + selectedColumns: { type: Array }, + page: { type: Number }, + pageSize: { type: Number }, + loading: { type: Boolean } + }; + + static styles = [ + sharedStyles, + cssVariables, + css` + :host { + display: flex; + flex-direction: column; + width: 100%; + flex: 1; + position: relative; + overflow: hidden; + } + + .content-wrapper { + flex: 1; + overflow-y: auto; + padding: 24px; + padding-bottom: 80px; + } + + .pagination { + position: fixed; + bottom: 0; + left: 280px; + right: 0; + display: flex; + align-items: center; + justify-content: center; + gap: 12px; + padding: 16px 24px; + background: var(--bg-elevated); + border-top: 1px solid var(--border-color); + z-index: 10; + } + + @media (max-width: 768px) { + .pagination { + left: 0; + } + } + + .pagination button { + padding: 8px 16px; + background: var(--bg-surface); + color: var(--text-primary); + border: 1px solid var(--border-color); + border-radius: var(--radius-sm); + cursor: pointer; + transition: var(--transition); + } + + .pagination button:hover:not(:disabled) { + background: var(--bg-hover); + border-color: var(--border-hover); + } + + .pagination button:disabled { + opacity: 0.5; + cursor: not-allowed; + } + + .pagination select, + .pagination input { + padding: 8px 12px; + background: var(--bg-surface); + color: var(--text-primary); + border: 1px solid var(--border-color); + border-radius: var(--radius-sm); + } + + .pagination input { + width: 80px; + } + + .pagination span { + color: var(--text-primary); + font-size: 14px; + } + ` + ]; + + constructor() { + super(); + this.tableName = ''; + this.view = 'data'; + this.schema = null; + this.tableData = null; + this.manifestData = null; + this.selectedColumns = []; + this.page = 1; + this.pageSize = 20; + this.loading = false; + } + + updated(changedProperties) { + if (changedProperties.has('tableName') && this.tableName) { + // 切换表时重置选中的列 + this.selectedColumns = []; + this.page = 1; + this.loadData(); + } + if (changedProperties.has('view') && this.tableName) { + this.loadData(); + } + } + + async loadData() { + if (!this.tableName) return; + + this.loading = true; + + try { + // Load schema + const schemaResponse = await fetch(`/api/tables/${this.tableName}/schema`); + if (!schemaResponse.ok) throw new Error('Failed to load schema'); + this.schema = await schemaResponse.json(); + + // Initialize selected columns (all by default) + if (this.schema.fields) { + this.selectedColumns = this.schema.fields.map(f => f.name); + } + + if (this.view === 'data') { + await this.loadTableData(); + } else if (this.view === 'manifest') { + await this.loadManifestData(); + } + } catch (error) { + console.error('Error loading data:', error); + } finally { + this.loading = false; + } + } + + async loadTableData() { + const selectParam = this.selectedColumns.join(','); + const url = `/api/tables/${this.tableName}/data?page=${this.page}&pageSize=${this.pageSize}&select=${selectParam}`; + + const response = await fetch(url); + if (!response.ok) throw new Error('Failed to load table data'); + this.tableData = await response.json(); + } + + async loadManifestData() { + const response = await fetch(`/api/tables/${this.tableName}/manifest`); + if (!response.ok) throw new Error('Failed to load manifest data'); + this.manifestData = await response.json(); + } + + switchView(newView) { + this.view = newView; + } + + toggleColumn(columnName) { + const index = this.selectedColumns.indexOf(columnName); + if (index > -1) { + this.selectedColumns = this.selectedColumns.filter(c => c !== columnName); + } else { + this.selectedColumns = [...this.selectedColumns, columnName]; + } + this.loadTableData(); + } + + changePage(delta) { + this.page = Math.max(1, this.page + delta); + this.loadTableData(); + } + + changePageSize(newSize) { + this.pageSize = parseInt(newSize); + this.page = 1; + this.loadTableData(); + } + + jumpToPage(pageNum) { + const num = parseInt(pageNum); + if (num > 0 && this.tableData && num <= this.tableData.totalPages) { + this.page = num; + this.loadTableData(); + } + } + + showRowDetail(seq) { + this.dispatchEvent(new CustomEvent('show-row-detail', { + detail: { tableName: this.tableName, seq }, + bubbles: true, + composed: true + })); + } + + toggleLevel(level) { + const levelCard = this.shadowRoot.querySelector(`[data-level="${level}"]`); + if (levelCard) { + const fileList = levelCard.querySelector('.file-list'); + const icon = levelCard.querySelector('.expand-icon'); + if (fileList.classList.contains('expanded')) { + fileList.classList.remove('expanded'); + icon.style.transform = 'rotate(0deg)'; + } else { + fileList.classList.add('expanded'); + icon.style.transform = 'rotate(90deg)'; + } + } + } + + formatBytes(bytes) { + if (bytes === 0) return '0 B'; + const k = 1024; + const sizes = ['B', 'KB', 'MB', 'GB', 'TB']; + const i = Math.floor(Math.log(bytes) / Math.log(k)); + return (bytes / Math.pow(k, i)).toFixed(2) + ' ' + sizes[i]; + } + + formatCount(count) { + if (count >= 1000000) return (count / 1000000).toFixed(1) + 'M'; + if (count >= 1000) return (count / 1000).toFixed(1) + 'K'; + return count.toString(); + } + + render() { + if (!this.tableName) { + return html` +
+

Select a table to view data

+

Choose a table from the sidebar to get started

+
+ `; + } + + if (this.loading) { + return html`
Loading...
`; + } + + return html` +
+ ${this.view === 'data' ? html` + { + this.selectedColumns = e.detail.columns; + this.loadTableData(); + }} + @show-row-detail=${(e) => this.showRowDetail(e.detail.seq)} + > + ` : html` + + `} +
+ + ${this.view === 'data' && this.tableData ? this.renderPagination() : ''} + `; + } + + renderPagination() { + return html` + + `; + } + + formatCount(count) { + if (count >= 1000000) return (count / 1000000).toFixed(1) + 'M'; + if (count >= 1000) return (count / 1000).toFixed(1) + 'K'; + return count.toString(); + } + + changePage(delta) { + this.page = Math.max(1, this.page + delta); + this.loadTableData(); + } + + changePageSize(newSize) { + this.pageSize = parseInt(newSize); + this.page = 1; + this.loadTableData(); + } + + jumpToPage(pageNum) { + const num = parseInt(pageNum); + if (num > 0 && this.tableData && num <= this.tableData.totalPages) { + this.page = num; + this.loadTableData(); + } + } + + showRowDetail(seq) { + this.dispatchEvent(new CustomEvent('show-row-detail', { + detail: { tableName: this.tableName, seq }, + bubbles: true, + composed: true + })); + } + + showCellContent(content) { + this.dispatchEvent(new CustomEvent('show-cell-content', { + detail: { content }, + bubbles: true, + composed: true + })); + } +} + +customElements.define('srdb-table-view', TableView); diff --git a/webui/static/js/components/theme-toggle.js b/webui/static/js/components/theme-toggle.js new file mode 100644 index 0000000..8551a7d --- /dev/null +++ b/webui/static/js/components/theme-toggle.js @@ -0,0 +1,123 @@ +import { LitElement, html, css } from 'https://cdn.jsdelivr.net/gh/lit/dist@3/core/lit-core.min.js'; +import { sharedStyles } from '../styles/shared-styles.js'; + +export class ThemeToggle extends LitElement { + static properties = { + theme: { type: String } + }; + + static styles = [ + sharedStyles, + css` + :host { + display: inline-block; + } + + .theme-toggle { + display: flex; + align-items: center; + gap: 8px; + padding: 8px 12px; + background: var(--bg-elevated); + border: 1px solid var(--border-color); + border-radius: var(--radius-md); + cursor: pointer; + transition: var(--transition); + font-size: 14px; + color: var(--text-primary); + } + + .theme-toggle:hover { + background: var(--bg-hover); + border-color: var(--border-hover); + } + + .icon { + font-size: 18px; + display: flex; + align-items: center; + } + + .label { + font-weight: 500; + } + ` + ]; + + constructor() { + super(); + // 从 localStorage 读取主题，默认为 dark + this.theme = localStorage.getItem('srdb-theme') || 'dark'; + this.applyTheme(); + } + + toggleTheme() { + this.theme = this.theme === 'dark' ? 'light' : 'dark'; + localStorage.setItem('srdb-theme', this.theme); + this.applyTheme(); + + // 触发主题变化事件 + this.dispatchEvent(new CustomEvent('theme-changed', { + detail: { theme: this.theme }, + bubbles: true, + composed: true + })); + } + + applyTheme() { + const root = document.documentElement; + + if (this.theme === 'light') { + // 浅色主题 + root.style.setProperty('--srdb-bg-main', '#ffffff'); + root.style.setProperty('--srdb-bg-surface', '#f5f5f5'); + root.style.setProperty('--srdb-bg-elevated', '#e5e5e5'); + root.style.setProperty('--srdb-bg-hover', '#d4d4d4'); + + root.style.setProperty('--srdb-text-primary', '#1a1a1a'); + root.style.setProperty('--srdb-text-secondary', '#666666'); + root.style.setProperty('--srdb-text-tertiary', '#999999'); + + root.style.setProperty('--srdb-border-color', 'rgba(0, 0, 0, 0.1)'); + root.style.setProperty('--srdb-border-hover', 'rgba(0, 0, 0, 0.2)'); + + root.style.setProperty('--srdb-shadow-sm', '0 1px 2px 0 rgba(0, 0, 0, 0.05)'); + root.style.setProperty('--srdb-shadow-md', '0 4px 6px -1px rgba(0, 0, 0, 0.1), 0 2px 4px -1px rgba(0, 0, 0, 0.06)'); + root.style.setProperty('--srdb-shadow-lg', '0 10px 15px -3px rgba(0, 0, 0, 0.1), 0 4px 6px -2px rgba(0, 0, 0, 0.05)'); + root.style.setProperty('--srdb-shadow-xl', '0 20px 25px -5px rgba(0, 0, 0, 0.1), 0 10px 10px -5px rgba(0, 0, 0, 0.04)'); + } else { + // 深色主题（默认值） + root.style.setProperty('--srdb-bg-main', '#0f0f1a'); + root.style.setProperty('--srdb-bg-surface', '#1a1a2e'); + root.style.setProperty('--srdb-bg-elevated', '#222236'); + root.style.setProperty('--srdb-bg-hover', '#2a2a3e'); + + root.style.setProperty('--srdb-text-primary', '#ffffff'); + root.style.setProperty('--srdb-text-secondary', '#a0a0b0'); + root.style.setProperty('--srdb-text-tertiary', '#6b6b7b'); + + root.style.setProperty('--srdb-border-color', 'rgba(255, 255, 255, 0.1)'); + root.style.setProperty('--srdb-border-hover', 'rgba(255, 255, 255, 0.2)'); + + root.style.setProperty('--srdb-shadow-sm', '0 1px 2px 0 rgba(0, 0, 0, 0.3)'); + root.style.setProperty('--srdb-shadow-md', '0 4px 6px -1px rgba(0, 0, 0, 0.4), 0 2px 4px -1px rgba(0, 0, 0, 0.3)'); + root.style.setProperty('--srdb-shadow-lg', '0 10px 15px -3px rgba(0, 0, 0, 0.5), 0 4px 6px -2px rgba(0, 0, 0, 0.3)'); + root.style.setProperty('--srdb-shadow-xl', '0 20px 25px -5px rgba(0, 0, 0, 0.5), 0 10px 10px -5px rgba(0, 0, 0, 0.3)'); + } + } + + render() { + return html` + + `; + } +} + +customElements.define('srdb-theme-toggle', ThemeToggle); diff --git a/webui/static/js/styles/shared-styles.js b/webui/static/js/styles/shared-styles.js new file mode 100644 index 0000000..dff0178 --- /dev/null +++ b/webui/static/js/styles/shared-styles.js @@ -0,0 +1,89 @@ +import { css } from 'https://cdn.jsdelivr.net/gh/lit/dist@3/core/lit-core.min.js'; + +// 共享的基础样式 +export const sharedStyles = css` + * { + box-sizing: border-box; + margin: 0; + padding: 0; + } + + /* 自定义滚动条样式 */ + *::-webkit-scrollbar { + width: 8px; + height: 8px; + } + + *::-webkit-scrollbar-track { + background: transparent; + } + + *::-webkit-scrollbar-thumb { + background: rgba(255, 255, 255, 0.1); + border-radius: 4px; + } + + *::-webkit-scrollbar-thumb:hover { + background: rgba(255, 255, 255, 0.15); + } + + /* 通用状态样式 */ + .empty { + text-align: center; + padding: 60px 20px; + color: var(--text-secondary); + } + + .loading { + text-align: center; + padding: 60px 20px; + color: var(--text-secondary); + } +`; + +// CSS 变量（可以在组件中使用，优先使用外部定义的变量） +export const cssVariables = css` + :host { + /* 主色调 - 优雅的紫蓝色 */ + --primary: var(--srdb-primary, #6366f1); + --primary-dark: var(--srdb-primary-dark, #4f46e5); + --primary-light: var(--srdb-primary-light, #818cf8); + --primary-bg: var(--srdb-primary-bg, rgba(99, 102, 241, 0.1)); + + /* 背景色 */ + --bg-main: var(--srdb-bg-main, #0f0f1a); + --bg-surface: var(--srdb-bg-surface, #1a1a2e); + --bg-elevated: var(--srdb-bg-elevated, #222236); + --bg-hover: var(--srdb-bg-hover, #2a2a3e); + + /* 文字颜色 */ + --text-primary: var(--srdb-text-primary, #ffffff); + --text-secondary: var(--srdb-text-secondary, #a0a0b0); + --text-tertiary: var(--srdb-text-tertiary, #6b6b7b); + + /* 边框和分隔线 */ + --border-color: var(--srdb-border-color, rgba(255, 255, 255, 0.1)); + --border-hover: var(--srdb-border-hover, rgba(255, 255, 255, 0.2)); + + /* 状态颜色 */ + --success: var(--srdb-success, #10b981); + --warning: var(--srdb-warning, #f59e0b); + --danger: var(--srdb-danger, #ef4444); + --info: var(--srdb-info, #3b82f6); + + /* 阴影 */ + --shadow-sm: var(--srdb-shadow-sm, 0 1px 2px 0 rgba(0, 0, 0, 0.3)); + --shadow-md: var(--srdb-shadow-md, 0 4px 6px -1px rgba(0, 0, 0, 0.4), 0 2px 4px -1px rgba(0, 0, 0, 0.3)); + --shadow-lg: var(--srdb-shadow-lg, 0 10px 15px -3px rgba(0, 0, 0, 0.5), 0 4px 6px -2px rgba(0, 0, 0, 0.3)); + --shadow-xl: var(--srdb-shadow-xl, 0 20px 25px -5px rgba(0, 0, 0, 0.5), 0 10px 10px -5px rgba(0, 0, 0, 0.3)); + + /* 圆角 */ + --radius-sm: var(--srdb-radius-sm, 6px); + --radius-md: var(--srdb-radius-md, 8px); + --radius-lg: var(--srdb-radius-lg, 12px); + --radius-xl: var(--srdb-radius-xl, 16px); + + /* 过渡 */ + --transition: var(--srdb-transition, all 0.2s cubic-bezier(0.4, 0, 0.2, 1)); + } +`; diff --git a/webui/webui.go b/webui/webui.go index 3f7e924..d16011a 100644 --- a/webui/webui.go +++ b/webui/webui.go @@ -5,12 +5,13 @@ import ( "encoding/json" "fmt" "io/fs" + "maps" "net/http" + "sort" "strconv" "strings" "code.tczkiot.com/srdb" - "code.tczkiot.com/srdb/sst" ) //go:embed static @@ -33,17 +34,10 @@ func NewWebUI(db *srdb.Database) *WebUI { func (ui *WebUI) setupHandler() http.Handler { mux := http.NewServeMux() - // API endpoints - JSON + // API endpoints - 纯 JSON API mux.HandleFunc("/api/tables", ui.handleListTables) mux.HandleFunc("/api/tables/", ui.handleTableAPI) - // API endpoints - HTML (for htmx) - mux.HandleFunc("/api/tables-html", ui.handleTablesHTML) - mux.HandleFunc("/api/tables-view/", ui.handleTableViewHTML) - - // Debug endpoint - list embedded files - mux.HandleFunc("/debug/files", ui.handleDebugFiles) - // 静态文件服务 staticFiles, _ := fs.Sub(staticFS, "static") mux.Handle("/static/", http.StripPrefix("/static/", http.FileServer(http.FS(staticFiles)))) @@ -105,6 +99,11 @@ func (ui *WebUI) handleListTables(w http.ResponseWriter, r *http.Request) { }) } + // 按表名排序 + sort.Slice(tables, func(i, j int) bool { + return tables[i].Name < tables[j].Name + }) + w.Header().Set("Content-Type", "application/json") json.NewEncoder(w).Encode(tables) } @@ -221,11 +220,8 @@ func (ui *WebUI) handleTableManifest(w http.ResponseWriter, r *http.Request, tab picker := compactionMgr.GetPicker() levels := make([]LevelInfo, 0) - for level := 0; level < 7; level++ { + for level := range 7 { files := version.GetLevel(level) - if len(files) == 0 { - continue - } totalSize := int64(0) fileInfos := make([]FileInfo, 0, len(files)) @@ -241,7 +237,10 @@ func (ui *WebUI) handleTableManifest(w http.ResponseWriter, r *http.Request, tab }) } - score := picker.GetLevelScore(version, level) + score := 0.0 + if len(files) > 0 { + score = picker.GetLevelScore(version, level) + } levels = append(levels, LevelInfo{ Level: level, @@ -294,12 +293,10 @@ func (ui *WebUI) handleTableDataBySeq(w http.ResponseWriter, r *http.Request, ta } // 构造响应（不进行剪裁，返回完整数据） - rowData := make(map[string]interface{}) + rowData := make(map[string]any) rowData["_seq"] = row.Seq rowData["_time"] = row.Time - for k, v := range row.Data { - rowData[k] = v - } + maps.Copy(rowData, row.Data) w.Header().Set("Content-Type", "application/json") json.NewEncoder(w).Encode(rowData) @@ -342,26 +339,52 @@ func (ui *WebUI) handleTableData(w http.ResponseWriter, r *http.Request, tableNa var selectedFields []string if selectParam != "" { selectedFields = strings.Split(selectParam, ",") + // 清理字段名（去除空格） + for i := range selectedFields { + selectedFields[i] = strings.TrimSpace(selectedFields[i]) + } } // 获取 schema 用于字段类型判断 tableSchema := table.GetSchema() - // 使用 Query API 获取所有数据（高效） - queryRows, err := table.Query().Rows() + // 使用 Query API 获取数据，如果指定了字段则只查询指定字段（按字段压缩优化） + queryBuilder := table.Query() + if len(selectedFields) > 0 { + // 确保 _seq 和 _time 总是被查询（用于构造响应） + fieldsWithMeta := make([]string, 0, len(selectedFields)+2) + hasSeq := false + hasTime := false + for _, field := range selectedFields { + switch field { + case "_seq": + hasSeq = true + case "_time": + hasTime = true + } + } + if !hasSeq { + fieldsWithMeta = append(fieldsWithMeta, "_seq") + } + if !hasTime { + fieldsWithMeta = append(fieldsWithMeta, "_time") + } + fieldsWithMeta = append(fieldsWithMeta, selectedFields...) + + queryBuilder = queryBuilder.Select(fieldsWithMeta...) + } + queryRows, err := queryBuilder.Rows() if err != nil { http.Error(w, fmt.Sprintf("Failed to query table: %v", err), http.StatusInternalServerError) return } defer queryRows.Close() - // 收集所有 rows 到内存中用于分页（对于大数据集，后续可以优化为流式处理） - allRows := make([]*sst.Row, 0) + // 收集所有 rows 到内存中用于分页 + allRows := make([]*srdb.SSTableRow, 0) for queryRows.Next() { row := queryRows.Row() - // Row 是 query.Row 类型，需要获取其内部的 sst.Row - // 直接构造 sst.Row - sstRow := &sst.Row{ + sstRow := &srdb.SSTableRow{ Seq: row.Data()["_seq"].(int64), Time: row.Data()["_time"].(int64), Data: make(map[string]any), @@ -378,73 +401,43 @@ func (ui *WebUI) handleTableData(w http.ResponseWriter, r *http.Request, tableNa // 计算分页 totalRows := int64(len(allRows)) offset := (page - 1) * pageSize - end := offset + pageSize - if end > int(totalRows) { - end = int(totalRows) - } + end := min(offset+pageSize, int(totalRows)) // 获取当前页数据 - rows := make([]*sst.Row, 0, pageSize) + rows := make([]*srdb.SSTableRow, 0, pageSize) if offset < int(totalRows) { rows = allRows[offset:end] } // 构造响应，对 string 字段进行剪裁 - const maxStringLength = 100 // 最大字符串长度（按字符计数，非字节） + const maxStringLength = 100 // 最大字符串长度 data := make([]map[string]any, 0, len(rows)) for _, row := range rows { rowData := make(map[string]any) + rowData["_seq"] = row.Seq + rowData["_time"] = row.Time - // 如果指定了字段，只返回选定的字段 - if len(selectedFields) > 0 { - for _, field := range selectedFields { - field = strings.TrimSpace(field) - if field == "_seq" { - rowData["_seq"] = row.Seq - } else if field == "_time" { - rowData["_time"] = row.Time - } else if v, ok := row.Data[field]; ok { - // 检查字段类型 - fieldDef, err := tableSchema.GetField(field) - if err == nil && fieldDef.Type == srdb.FieldTypeString { - // 对字符串字段进行剪裁 - if str, ok := v.(string); ok { - runes := []rune(str) - if len(runes) > maxStringLength { - rowData[field] = string(runes[:maxStringLength]) + "..." - rowData[field+"_truncated"] = true - continue - } - } - } - rowData[field] = v - } - } - } else { - // 返回所有字段 - rowData["_seq"] = row.Seq - rowData["_time"] = row.Time - for k, v := range row.Data { - // 检查字段类型 - field, err := tableSchema.GetField(k) - if err == nil && field.Type == srdb.FieldTypeString { - // 对字符串字段进行剪裁（按 rune 截取，避免 CJK 等多字节字符乱码） - if str, ok := v.(string); ok { - runes := []rune(str) - if len(runes) > maxStringLength { - rowData[k] = string(runes[:maxStringLength]) + "..." - rowData[k+"_truncated"] = true - continue - } + // 遍历所有字段 + for k, v := range row.Data { + // 检查字段类型 + field, err := tableSchema.GetField(k) + if err == nil && field.Type == srdb.FieldTypeString { + // 对字符串字段进行剪裁 + if str, ok := v.(string); ok { + runes := []rune(str) + if len(runes) > maxStringLength { + rowData[k] = string(runes[:maxStringLength]) + "..." + rowData[k+"_truncated"] = true + continue } } - rowData[k] = v } + rowData[k] = v } data = append(data, rowData) } - response := map[string]interface{}{ + response := map[string]any{ "data": data, "page": page, "pageSize": pageSize, @@ -456,25 +449,6 @@ func (ui *WebUI) handleTableData(w http.ResponseWriter, r *http.Request, tableNa json.NewEncoder(w).Encode(response) } -// handleDebugFiles 列出所有嵌入的文件（调试用） -func (ui *WebUI) handleDebugFiles(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Content-Type", "text/plain; charset=utf-8") - fmt.Fprintln(w, "Embedded files in staticFS:") - fs.WalkDir(staticFS, ".", func(path string, d fs.DirEntry, err error) error { - if err != nil { - fmt.Fprintf(w, "ERROR walking %s: %v\n", path, err) - return err - } - if d.IsDir() { - fmt.Fprintf(w, "[DIR] %s/\n", path) - } else { - info, _ := d.Info() - fmt.Fprintf(w, "[FILE] %s (%d bytes)\n", path, info.Size()) - } - return nil - }) -} - // handleIndex 处理首页请求 func (ui *WebUI) handleIndex(w http.ResponseWriter, r *http.Request) { if r.URL.Path != "/" { @@ -493,238 +467,3 @@ func (ui *WebUI) handleIndex(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "text/html; charset=utf-8") w.Write(content) } - -// handleTablesHTML 处理获取表列表 HTML 请求（for htmx） -func (ui *WebUI) handleTablesHTML(w http.ResponseWriter, r *http.Request) { - if r.Method != http.MethodGet { - http.Error(w, "Method not allowed", http.StatusMethodNotAllowed) - return - } - - allTables := ui.db.GetAllTablesInfo() - tables := make([]TableListItem, 0, len(allTables)) - for name, table := range allTables { - schema := table.GetSchema() - fields := make([]FieldInfo, 0, len(schema.Fields)) - for _, field := range schema.Fields { - fields = append(fields, FieldInfo{ - Name: field.Name, - Type: field.Type.String(), - Indexed: field.Indexed, - Comment: field.Comment, - }) - } - - tables = append(tables, TableListItem{ - Name: name, - CreatedAt: table.GetCreatedAt(), - Fields: fields, - }) - } - - html := renderTablesHTML(tables) - w.Header().Set("Content-Type", "text/html; charset=utf-8") - w.Write([]byte(html)) -} - -// handleTableViewHTML 处理获取表视图 HTML 请求（for htmx） -func (ui *WebUI) handleTableViewHTML(w http.ResponseWriter, r *http.Request) { - if r.Method != http.MethodGet { - http.Error(w, "Method not allowed", http.StatusMethodNotAllowed) - return - } - - // 解析路径: /api/tables-view/{name} 或 /api/tables-view/{name}/manifest - path := strings.TrimPrefix(r.URL.Path, "/api/tables-view/") - parts := strings.Split(path, "/") - - if len(parts) < 1 || parts[0] == "" { - http.Error(w, "Invalid path", http.StatusBadRequest) - return - } - - tableName := parts[0] - isManifest := len(parts) >= 2 && parts[1] == "manifest" - - table, err := ui.db.GetTable(tableName) - if err != nil { - http.Error(w, err.Error(), http.StatusNotFound) - return - } - - if isManifest { - // 返回 Manifest 视图 HTML - ui.renderManifestHTML(w, r, tableName, table) - } else { - // 返回 Data 视图 HTML - ui.renderDataHTML(w, r, tableName, table) - } -} - -// renderDataHTML 渲染数据视图 HTML -func (ui *WebUI) renderDataHTML(w http.ResponseWriter, r *http.Request, tableName string, table *srdb.Table) { - // 解析分页参数 - pageStr := r.URL.Query().Get("page") - pageSizeStr := r.URL.Query().Get("pageSize") - - page := 1 - pageSize := 20 - - if pageStr != "" { - if p, err := strconv.Atoi(pageStr); err == nil && p > 0 { - page = p - } - } - - if pageSizeStr != "" { - if ps, err := strconv.Atoi(pageSizeStr); err == nil && ps > 0 && ps <= 1000 { - pageSize = ps - } - } - - // 获取 schema - tableSchema := table.GetSchema() - schemaInfo := SchemaInfo{ - Name: tableSchema.Name, - Fields: make([]FieldInfo, 0, len(tableSchema.Fields)), - } - for _, field := range tableSchema.Fields { - schemaInfo.Fields = append(schemaInfo.Fields, FieldInfo{ - Name: field.Name, - Type: field.Type.String(), - Indexed: field.Indexed, - Comment: field.Comment, - }) - } - - // 使用 Query API 获取所有数据 - queryRows, err := table.Query().Rows() - if err != nil { - http.Error(w, fmt.Sprintf("Failed to query table: %v", err), http.StatusInternalServerError) - return - } - defer queryRows.Close() - - // 收集所有 rows - allRows := make([]*sst.Row, 0) - for queryRows.Next() { - row := queryRows.Row() - sstRow := &sst.Row{ - Seq: row.Data()["_seq"].(int64), - Time: row.Data()["_time"].(int64), - Data: make(map[string]any), - } - for k, v := range row.Data() { - if k != "_seq" && k != "_time" { - sstRow.Data[k] = v - } - } - allRows = append(allRows, sstRow) - } - - // 计算分页 - totalRows := int64(len(allRows)) - offset := (page - 1) * pageSize - end := offset + pageSize - if end > int(totalRows) { - end = int(totalRows) - } - - // 获取当前页数据 - rows := make([]*sst.Row, 0, pageSize) - if offset < int(totalRows) { - rows = allRows[offset:end] - } - - // 构造 TableDataResponse - const maxStringLength = 100 - data := make([]map[string]any, 0, len(rows)) - for _, row := range rows { - rowData := make(map[string]any) - rowData["_seq"] = row.Seq - rowData["_time"] = row.Time - for k, v := range row.Data { - field, err := tableSchema.GetField(k) - if err == nil && field.Type == srdb.FieldTypeString { - if str, ok := v.(string); ok { - runes := []rune(str) - if len(runes) > maxStringLength { - rowData[k] = string(runes[:maxStringLength]) + "..." - rowData[k+"_truncated"] = true - continue - } - } - } - rowData[k] = v - } - data = append(data, rowData) - } - - tableData := TableDataResponse{ - Data: data, - Page: int64(page), - PageSize: int64(pageSize), - TotalRows: totalRows, - TotalPages: (totalRows + int64(pageSize) - 1) / int64(pageSize), - } - - html := renderDataViewHTML(tableName, schemaInfo, tableData) - w.Header().Set("Content-Type", "text/html; charset=utf-8") - w.Write([]byte(html)) -} - -// renderManifestHTML 渲染 Manifest 视图 HTML -func (ui *WebUI) renderManifestHTML(w http.ResponseWriter, r *http.Request, tableName string, table *srdb.Table) { - engine := table.GetEngine() - versionSet := engine.GetVersionSet() - version := versionSet.GetCurrent() - - // 获取 Compaction Manager 和 Picker - compactionMgr := engine.GetCompactionManager() - picker := compactionMgr.GetPicker() - - levels := make([]LevelInfo, 0) - for level := 0; level < 7; level++ { - files := version.GetLevel(level) - - totalSize := int64(0) - fileInfos := make([]FileInfo, 0, len(files)) - for _, f := range files { - totalSize += f.FileSize - fileInfos = append(fileInfos, FileInfo{ - FileNumber: f.FileNumber, - Level: f.Level, - FileSize: f.FileSize, - MinKey: f.MinKey, - MaxKey: f.MaxKey, - RowCount: f.RowCount, - }) - } - - score := 0.0 - if len(files) > 0 { - score = picker.GetLevelScore(version, level) - } - - levels = append(levels, LevelInfo{ - Level: level, - FileCount: len(files), - TotalSize: totalSize, - Score: score, - Files: fileInfos, - }) - } - - stats := compactionMgr.GetStats() - - manifest := ManifestResponse{ - Levels: levels, - NextFileNumber: versionSet.GetNextFileNumber(), - LastSequence: versionSet.GetLastSequence(), - CompactionStats: stats, - } - - html := renderManifestViewHTML(tableName, manifest) - w.Header().Set("Content-Type", "text/html; charset=utf-8") - w.Write([]byte(html)) -}