Initial commit: SRDB - High-performance LSM-Tree database

- Core engine with MemTable, SST, WAL
- B+Tree indexing for SST files  
- Leveled compaction strategy
- Multi-table database management
- Schema validation and secondary indexes
- Query builder with complex conditions
- Web UI with HTMX for data visualization
- Command-line tools for diagnostics
This commit is contained in:
2025-10-08 06:38:12 +08:00
commit ae87c38776
61 changed files with 15475 additions and 0 deletions

View File

@@ -0,0 +1,40 @@
package commands
import (
"fmt"
"log"
"code.tczkiot.com/srdb"
)
// CheckData 检查数据库中的数据
func CheckData(dbPath string) {
// 打开数据库
db, err := srdb.Open(dbPath)
if err != nil {
log.Fatal(err)
}
defer db.Close()
// 列出所有表
tables := db.ListTables()
fmt.Printf("Found %d tables: %v\n", len(tables), tables)
// 检查每个表的记录数
for _, tableName := range tables {
table, err := db.GetTable(tableName)
if err != nil {
fmt.Printf("Error getting table %s: %v\n", tableName, err)
continue
}
result, err := table.Query().Rows()
if err != nil {
fmt.Printf("Error querying table %s: %v\n", tableName, err)
continue
}
count := result.Count()
fmt.Printf("Table '%s': %d rows\n", tableName, count)
}
}

View File

@@ -0,0 +1,69 @@
package commands
import (
"fmt"
"log"
"code.tczkiot.com/srdb"
)
// CheckSeq 检查特定序列号的数据
func CheckSeq(dbPath string) {
db, err := srdb.Open(dbPath)
if err != nil {
log.Fatal(err)
}
defer db.Close()
table, err := db.GetTable("logs")
if err != nil {
log.Fatal(err)
}
// Check seq 1
row1, err := table.Get(1)
if err != nil {
fmt.Printf("Error getting seq=1: %v\n", err)
} else if row1 == nil {
fmt.Println("Seq=1: NOT FOUND")
} else {
fmt.Printf("Seq=1: FOUND (time=%d)\n", row1.Time)
}
// Check seq 100
row100, err := table.Get(100)
if err != nil {
fmt.Printf("Error getting seq=100: %v\n", err)
} else if row100 == nil {
fmt.Println("Seq=100: NOT FOUND")
} else {
fmt.Printf("Seq=100: FOUND (time=%d)\n", row100.Time)
}
// Check seq 729
row729, err := table.Get(729)
if err != nil {
fmt.Printf("Error getting seq=729: %v\n", err)
} else if row729 == nil {
fmt.Println("Seq=729: NOT FOUND")
} else {
fmt.Printf("Seq=729: FOUND (time=%d)\n", row729.Time)
}
// Query all records
result, err := table.Query().Rows()
if err != nil {
log.Fatal(err)
}
count := result.Count()
fmt.Printf("\nTotal rows from Query: %d\n", count)
if count > 0 {
first, _ := result.First()
if first != nil {
data := first.Data()
fmt.Printf("First row _seq: %v\n", data["_seq"])
}
}
}

View File

@@ -0,0 +1,58 @@
package commands
import (
"fmt"
"log"
"code.tczkiot.com/srdb"
)
// DumpManifest 导出 manifest 信息
func DumpManifest(dbPath string) {
db, err := srdb.Open(dbPath)
if err != nil {
log.Fatal(err)
}
defer db.Close()
table, err := db.GetTable("logs")
if err != nil {
log.Fatal(err)
}
engine := table.GetEngine()
versionSet := engine.GetVersionSet()
version := versionSet.GetCurrent()
// Check for duplicates in each level
for level := 0; level < 7; level++ {
files := version.GetLevel(level)
if len(files) == 0 {
continue
}
// Track file numbers
fileMap := make(map[int64][]struct {
minKey int64
maxKey int64
})
for _, f := range files {
fileMap[f.FileNumber] = append(fileMap[f.FileNumber], struct {
minKey int64
maxKey int64
}{f.MinKey, f.MaxKey})
}
// Report duplicates
fmt.Printf("Level %d: %d files\n", level, len(files))
for fileNum, entries := range fileMap {
if len(entries) > 1 {
fmt.Printf(" [DUPLICATE] File #%d appears %d times:\n", fileNum, len(entries))
for i, e := range entries {
fmt.Printf(" Entry %d: min=%d max=%d\n", i+1, e.minKey, e.maxKey)
}
}
}
}
}

View File

@@ -0,0 +1,72 @@
package commands
import (
"fmt"
"log"
"os"
"path/filepath"
"sort"
"strconv"
"strings"
"code.tczkiot.com/srdb/sst"
)
// InspectAllSST 检查所有 SST 文件
func InspectAllSST(sstDir string) {
// List all SST files
files, err := os.ReadDir(sstDir)
if err != nil {
log.Fatal(err)
}
var sstFiles []string
for _, file := range files {
if strings.HasSuffix(file.Name(), ".sst") {
sstFiles = append(sstFiles, file.Name())
}
}
sort.Strings(sstFiles)
fmt.Printf("Found %d SST files\n\n", len(sstFiles))
// Inspect each file
for _, filename := range sstFiles {
sstPath := filepath.Join(sstDir, filename)
reader, err := sst.NewReader(sstPath)
if err != nil {
fmt.Printf("%s: ERROR - %v\n", filename, err)
continue
}
header := reader.GetHeader()
allKeys := reader.GetAllKeys()
// Extract file number
numStr := strings.TrimPrefix(filename, "000")
numStr = strings.TrimPrefix(numStr, "00")
numStr = strings.TrimPrefix(numStr, "0")
numStr = strings.TrimSuffix(numStr, ".sst")
fileNum, _ := strconv.Atoi(numStr)
fmt.Printf("File #%d (%s):\n", fileNum, filename)
fmt.Printf(" Header: MinKey=%d MaxKey=%d RowCount=%d\n", header.MinKey, header.MaxKey, header.RowCount)
fmt.Printf(" Actual: %d keys", len(allKeys))
if len(allKeys) > 0 {
fmt.Printf(" [%d ... %d]", allKeys[0], allKeys[len(allKeys)-1])
}
fmt.Printf("\n")
// Check if header matches actual keys
if len(allKeys) > 0 {
if header.MinKey != allKeys[0] || header.MaxKey != allKeys[len(allKeys)-1] {
fmt.Printf(" *** MISMATCH: Header says %d-%d but file has %d-%d ***\n",
header.MinKey, header.MaxKey, allKeys[0], allKeys[len(allKeys)-1])
}
}
reader.Close()
}
}

View File

@@ -0,0 +1,75 @@
package commands
import (
"fmt"
"log"
"os"
"code.tczkiot.com/srdb/sst"
)
// InspectSST 检查特定 SST 文件
func InspectSST(sstPath string) {
// Check if file exists
info, err := os.Stat(sstPath)
if err != nil {
log.Fatal(err)
}
fmt.Printf("File: %s\n", sstPath)
fmt.Printf("Size: %d bytes\n", info.Size())
// Open reader
reader, err := sst.NewReader(sstPath)
if err != nil {
log.Fatal(err)
}
defer reader.Close()
// Get header
header := reader.GetHeader()
fmt.Printf("\nHeader:\n")
fmt.Printf(" RowCount: %d\n", header.RowCount)
fmt.Printf(" MinKey: %d\n", header.MinKey)
fmt.Printf(" MaxKey: %d\n", header.MaxKey)
fmt.Printf(" DataSize: %d bytes\n", header.DataSize)
// Get all keys using GetAllKeys()
allKeys := reader.GetAllKeys()
fmt.Printf("\nActual keys in file: %d keys\n", len(allKeys))
if len(allKeys) > 0 {
fmt.Printf(" First key: %d\n", allKeys[0])
fmt.Printf(" Last key: %d\n", allKeys[len(allKeys)-1])
if len(allKeys) <= 30 {
fmt.Printf(" All keys: %v\n", allKeys)
} else {
fmt.Printf(" First 15: %v\n", allKeys[:15])
fmt.Printf(" Last 15: %v\n", allKeys[len(allKeys)-15:])
}
}
// Try to get a specific key
fmt.Printf("\nTrying to get key 332:\n")
row, err := reader.Get(332)
if err != nil {
fmt.Printf(" Error: %v\n", err)
} else if row == nil {
fmt.Printf(" NULL\n")
} else {
fmt.Printf(" FOUND: seq=%d, time=%d\n", row.Seq, row.Time)
}
// Try to get key based on actual first key
if len(allKeys) > 0 {
firstKey := allKeys[0]
fmt.Printf("\nTrying to get actual first key %d:\n", firstKey)
row, err := reader.Get(firstKey)
if err != nil {
fmt.Printf(" Error: %v\n", err)
} else if row == nil {
fmt.Printf(" NULL\n")
} else {
fmt.Printf(" FOUND: seq=%d, time=%d\n", row.Seq, row.Time)
}
}
}

View File

@@ -0,0 +1,59 @@
package commands
import (
"fmt"
"log"
"code.tczkiot.com/srdb"
)
// TestFix 测试修复
func TestFix(dbPath string) {
db, err := srdb.Open(dbPath)
if err != nil {
log.Fatal(err)
}
defer db.Close()
table, err := db.GetTable("logs")
if err != nil {
log.Fatal(err)
}
// Get total count
result, err := table.Query().Rows()
if err != nil {
log.Fatal(err)
}
totalCount := result.Count()
fmt.Printf("Total rows in Query(): %d\n", totalCount)
// Test Get() for first 10, middle 10, and last 10
testRanges := []struct {
name string
start int64
end int64
}{
{"First 10", 1, 10},
{"Middle 10", 50, 59},
{"Last 10", int64(totalCount) - 9, int64(totalCount)},
}
for _, tr := range testRanges {
fmt.Printf("\n%s (keys %d-%d):\n", tr.name, tr.start, tr.end)
foundCount := 0
for seq := tr.start; seq <= tr.end; seq++ {
row, err := table.Get(seq)
if err != nil {
fmt.Printf(" Seq %d: ERROR - %v\n", seq, err)
} else if row == nil {
fmt.Printf(" Seq %d: NULL\n", seq)
} else {
foundCount++
}
}
fmt.Printf(" Found: %d/%d\n", foundCount, tr.end-tr.start+1)
}
fmt.Printf("\n✅ If all keys found, the bug is FIXED!\n")
}

View File

@@ -0,0 +1,66 @@
package commands
import (
"fmt"
"log"
"code.tczkiot.com/srdb"
)
// TestKeys 测试键
func TestKeys(dbPath string) {
db, err := srdb.Open(dbPath)
if err != nil {
log.Fatal(err)
}
defer db.Close()
table, err := db.GetTable("logs")
if err != nil {
log.Fatal(err)
}
// Test keys from different ranges
testKeys := []int64{
1, 100, 331, 332, 350, 400, 447, 500, 600, 700, 800, 850, 861, 862, 900, 1000, 1500, 1665, 1666, 1723,
}
fmt.Println("Testing key existence:")
foundCount := 0
for _, key := range testKeys {
row, err := table.Get(key)
if err != nil {
fmt.Printf("Key %4d: NOT FOUND (%v)\n", key, err)
} else if row == nil {
fmt.Printf("Key %4d: NULL\n", key)
} else {
fmt.Printf("Key %4d: FOUND (time=%d)\n", key, row.Time)
foundCount++
}
}
fmt.Printf("\nFound %d out of %d test keys\n", foundCount, len(testKeys))
// Query all
result, err := table.Query().Rows()
if err != nil {
log.Fatal(err)
}
count := result.Count()
fmt.Printf("Total rows from Query: %d\n", count)
if count > 0 {
first, _ := result.First()
if first != nil {
data := first.Data()
fmt.Printf("First row _seq: %v\n", data["_seq"])
}
last, _ := result.Last()
if last != nil {
data := last.Data()
fmt.Printf("Last row _seq: %v\n", data["_seq"])
}
}
}

View File

@@ -0,0 +1,192 @@
package commands
import (
"crypto/rand"
"fmt"
"log"
"math/big"
"net/http"
"slices"
"time"
"code.tczkiot.com/srdb"
"code.tczkiot.com/srdb/webui"
)
// StartWebUI 启动 WebUI 服务器
func StartWebUI(dbPath string, addr string) {
// 打开数据库
db, err := srdb.Open(dbPath)
if err != nil {
log.Fatal(err)
}
defer db.Close()
// 创建示例 Schema
userSchema := srdb.NewSchema("users", []srdb.Field{
{Name: "name", Type: srdb.FieldTypeString, Indexed: false, Comment: "User name"},
{Name: "email", Type: srdb.FieldTypeString, Indexed: false, Comment: "Email address"},
{Name: "age", Type: srdb.FieldTypeInt64, Indexed: false, Comment: "Age"},
{Name: "city", Type: srdb.FieldTypeString, Indexed: false, Comment: "City"},
})
productSchema := srdb.NewSchema("products", []srdb.Field{
{Name: "product_name", Type: srdb.FieldTypeString, Indexed: false, Comment: "Product name"},
{Name: "price", Type: srdb.FieldTypeFloat, Indexed: false, Comment: "Price"},
{Name: "quantity", Type: srdb.FieldTypeInt64, Indexed: false, Comment: "Quantity"},
{Name: "category", Type: srdb.FieldTypeString, Indexed: false, Comment: "Category"},
})
// 创建表(如果不存在)
tables := db.ListTables()
hasUsers := false
hasProducts := false
for _, t := range tables {
if t == "users" {
hasUsers = true
}
if t == "products" {
hasProducts = true
}
}
if !hasUsers {
table, err := db.CreateTable("users", userSchema)
if err != nil {
log.Printf("Create users table failed: %v", err)
} else {
// 插入一些示例数据
users := []map[string]interface{}{
{"name": "Alice", "email": "alice@example.com", "age": 30, "city": "Beijing"},
{"name": "Bob", "email": "bob@example.com", "age": 25, "city": "Shanghai"},
{"name": "Charlie", "email": "charlie@example.com", "age": 35, "city": "Guangzhou"},
{"name": "David", "email": "david@example.com", "age": 28, "city": "Shenzhen"},
{"name": "Eve", "email": "eve@example.com", "age": 32, "city": "Hangzhou"},
}
for _, user := range users {
table.Insert(user)
}
log.Printf("Created users table with %d records", len(users))
}
}
if !hasProducts {
table, err := db.CreateTable("products", productSchema)
if err != nil {
log.Printf("Create products table failed: %v", err)
} else {
// 插入一些示例数据
products := []map[string]interface{}{
{"product_name": "Laptop", "price": 999.99, "quantity": 10, "category": "Electronics"},
{"product_name": "Mouse", "price": 29.99, "quantity": 50, "category": "Electronics"},
{"product_name": "Keyboard", "price": 79.99, "quantity": 30, "category": "Electronics"},
{"product_name": "Monitor", "price": 299.99, "quantity": 15, "category": "Electronics"},
{"product_name": "Desk", "price": 199.99, "quantity": 5, "category": "Furniture"},
{"product_name": "Chair", "price": 149.99, "quantity": 8, "category": "Furniture"},
}
for _, product := range products {
table.Insert(product)
}
log.Printf("Created products table with %d records", len(products))
}
}
// 启动后台数据插入协程
go autoInsertData(db)
// 启动 Web UI
handler := webui.NewWebUI(db)
fmt.Printf("SRDB Web UI is running at http://%s\n", addr)
fmt.Println("Press Ctrl+C to stop")
fmt.Println("Background data insertion is running...")
if err := http.ListenAndServe(addr, handler); err != nil {
log.Fatal(err)
}
}
// generateRandomData 生成指定大小的随机数据 (2KB ~ 512KB)
func generateRandomData() string {
minSize := 2 * 1024 // 2KB
maxSize := (1 * 1024 * 1024) / 2 // 512KB
sizeBig, _ := rand.Int(rand.Reader, big.NewInt(int64(maxSize-minSize)))
size := int(sizeBig.Int64()) + minSize
data := make([]byte, size)
rand.Read(data)
return fmt.Sprintf("%x", data)
}
// autoInsertData 在后台自动插入数据
func autoInsertData(db *srdb.Database) {
ticker := time.NewTicker(time.Second)
defer ticker.Stop()
counter := 1
for range ticker.C {
tables := db.ListTables()
var logsTable *srdb.Table
hasLogs := slices.Contains(tables, "logs")
if !hasLogs {
logsSchema := srdb.NewSchema("logs", []srdb.Field{
{Name: "timestamp", Type: srdb.FieldTypeString, Indexed: false, Comment: "Timestamp"},
{Name: "data", Type: srdb.FieldTypeString, Indexed: false, Comment: "Random data"},
{Name: "size_bytes", Type: srdb.FieldTypeInt64, Indexed: false, Comment: "Data size in bytes"},
})
var err error
logsTable, err = db.CreateTable("logs", logsSchema)
if err != nil {
log.Printf("Failed to create logs table: %v", err)
continue
}
log.Println("Created logs table for background data insertion")
} else {
var err error
logsTable, err = db.GetTable("logs")
if err != nil || logsTable == nil {
log.Printf("Failed to get logs table: %v", err)
continue
}
}
data := generateRandomData()
sizeBytes := len(data)
record := map[string]any{
"timestamp": time.Now().Format(time.RFC3339),
"data": data,
"size_bytes": int64(sizeBytes),
}
err := logsTable.Insert(record)
if err != nil {
log.Printf("Failed to insert data: %v", err)
} else {
sizeStr := formatBytes(sizeBytes)
log.Printf("Inserted record #%d, size: %s", counter, sizeStr)
counter++
}
}
}
// formatBytes 格式化字节大小显示
func formatBytes(bytes int) string {
const unit = 1024
if bytes < unit {
return fmt.Sprintf("%d B", bytes)
}
div, exp := int64(unit), 0
for n := bytes / unit; n >= unit; n /= unit {
div *= unit
exp++
}
units := []string{"KB", "MB", "GB", "TB"}
return fmt.Sprintf("%.2f %s", float64(bytes)/float64(div), units[exp])
}