Initial commit: SRDB - High-performance LSM-Tree database
- Core engine with MemTable, SST, WAL - B+Tree indexing for SST files - Leveled compaction strategy - Multi-table database management - Schema validation and secondary indexes - Query builder with complex conditions - Web UI with HTMX for data visualization - Command-line tools for diagnostics
This commit is contained in:
206
wal/manager.go
Normal file
206
wal/manager.go
Normal file
@@ -0,0 +1,206 @@
|
||||
package wal
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
)
|
||||
|
||||
// Manager WAL 管理器，管理多个 WAL 文件
|
||||
type Manager struct {
|
||||
dir string
|
||||
currentWAL *WAL
|
||||
currentNumber int64
|
||||
mu sync.Mutex
|
||||
}
|
||||
|
||||
// NewManager 创建 WAL 管理器
|
||||
func NewManager(dir string) (*Manager, error) {
|
||||
// 确保目录存在
|
||||
err := os.MkdirAll(dir, 0755)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// 读取当前 WAL 编号
|
||||
number, err := readCurrentNumber(dir)
|
||||
if err != nil {
|
||||
// 如果读取失败，从 1 开始
|
||||
number = 1
|
||||
}
|
||||
|
||||
// 打开当前 WAL
|
||||
walPath := filepath.Join(dir, fmt.Sprintf("%06d.wal", number))
|
||||
wal, err := Open(walPath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// 保存当前编号
|
||||
err = saveCurrentNumber(dir, number)
|
||||
if err != nil {
|
||||
wal.Close()
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &Manager{
|
||||
dir: dir,
|
||||
currentWAL: wal,
|
||||
currentNumber: number,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Append 追加记录到当前 WAL
|
||||
func (m *Manager) Append(entry *Entry) error {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
|
||||
return m.currentWAL.Append(entry)
|
||||
}
|
||||
|
||||
// Sync 同步当前 WAL 到磁盘
|
||||
func (m *Manager) Sync() error {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
|
||||
return m.currentWAL.Sync()
|
||||
}
|
||||
|
||||
// Rotate 切换到新的 WAL 文件
|
||||
func (m *Manager) Rotate() (int64, error) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
|
||||
// 记录旧的 WAL 编号
|
||||
oldNumber := m.currentNumber
|
||||
|
||||
// 关闭当前 WAL
|
||||
err := m.currentWAL.Close()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
// 创建新 WAL
|
||||
m.currentNumber++
|
||||
walPath := filepath.Join(m.dir, fmt.Sprintf("%06d.wal", m.currentNumber))
|
||||
wal, err := Open(walPath)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
m.currentWAL = wal
|
||||
|
||||
// 更新 CURRENT 文件
|
||||
err = saveCurrentNumber(m.dir, m.currentNumber)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
return oldNumber, nil
|
||||
}
|
||||
|
||||
// Delete 删除指定的 WAL 文件
|
||||
func (m *Manager) Delete(number int64) error {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
|
||||
walPath := filepath.Join(m.dir, fmt.Sprintf("%06d.wal", number))
|
||||
return os.Remove(walPath)
|
||||
}
|
||||
|
||||
// GetCurrentNumber 获取当前 WAL 编号
|
||||
func (m *Manager) GetCurrentNumber() int64 {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
|
||||
return m.currentNumber
|
||||
}
|
||||
|
||||
// RecoverAll 恢复所有 WAL 文件
|
||||
func (m *Manager) RecoverAll() ([]*Entry, error) {
|
||||
// 查找所有 WAL 文件
|
||||
pattern := filepath.Join(m.dir, "*.wal")
|
||||
files, err := filepath.Glob(pattern)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if len(files) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// 按文件名排序（确保按时间顺序）
|
||||
sort.Strings(files)
|
||||
|
||||
var allEntries []*Entry
|
||||
|
||||
// 依次读取每个 WAL
|
||||
for _, file := range files {
|
||||
reader, err := NewReader(file)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
entries, err := reader.Read()
|
||||
reader.Close()
|
||||
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
allEntries = append(allEntries, entries...)
|
||||
}
|
||||
|
||||
return allEntries, nil
|
||||
}
|
||||
|
||||
// ListWALFiles 列出所有 WAL 文件
|
||||
func (m *Manager) ListWALFiles() ([]string, error) {
|
||||
pattern := filepath.Join(m.dir, "*.wal")
|
||||
files, err := filepath.Glob(pattern)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
sort.Strings(files)
|
||||
return files, nil
|
||||
}
|
||||
|
||||
// Close 关闭 WAL 管理器
|
||||
func (m *Manager) Close() error {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
|
||||
if m.currentWAL != nil {
|
||||
return m.currentWAL.Close()
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// readCurrentNumber 读取当前 WAL 编号
|
||||
func readCurrentNumber(dir string) (int64, error) {
|
||||
currentPath := filepath.Join(dir, "CURRENT")
|
||||
data, err := os.ReadFile(currentPath)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
number, err := strconv.ParseInt(strings.TrimSpace(string(data)), 10, 64)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
return number, nil
|
||||
}
|
||||
|
||||
// saveCurrentNumber 保存当前 WAL 编号
|
||||
func saveCurrentNumber(dir string, number int64) error {
|
||||
currentPath := filepath.Join(dir, "CURRENT")
|
||||
data := []byte(fmt.Sprintf("%d\n", number))
|
||||
return os.WriteFile(currentPath, data, 0644)
|
||||
}
|
||||
208
wal/wal.go
Normal file
208
wal/wal.go
Normal file
@@ -0,0 +1,208 @@
|
||||
package wal
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"hash/crc32"
|
||||
"io"
|
||||
"os"
|
||||
"sync"
|
||||
)
|
||||
|
||||
const (
|
||||
// Entry 类型
|
||||
EntryTypePut = 1
|
||||
EntryTypeDelete = 2 // 预留，暂不支持
|
||||
|
||||
// Entry Header 大小
|
||||
EntryHeaderSize = 17 // CRC32(4) + Length(4) + Type(1) + Seq(8)
|
||||
)
|
||||
|
||||
// Entry WAL 条目
|
||||
type Entry struct {
|
||||
Type byte // 操作类型
|
||||
Seq int64 // _seq
|
||||
Data []byte // 数据
|
||||
CRC32 uint32 // 校验和
|
||||
}
|
||||
|
||||
// WAL Write-Ahead Log
|
||||
type WAL struct {
|
||||
file *os.File
|
||||
offset int64
|
||||
mu sync.Mutex
|
||||
}
|
||||
|
||||
// Open 打开 WAL 文件
|
||||
func Open(path string) (*WAL, error) {
|
||||
file, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR|os.O_APPEND, 0644)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// 获取当前文件大小
|
||||
stat, err := file.Stat()
|
||||
if err != nil {
|
||||
file.Close()
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &WAL{
|
||||
file: file,
|
||||
offset: stat.Size(),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Append 追加一条记录
|
||||
func (w *WAL) Append(entry *Entry) error {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
|
||||
// 序列化 Entry
|
||||
data := w.marshalEntry(entry)
|
||||
|
||||
// 写入文件
|
||||
_, err := w.file.Write(data)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
w.offset += int64(len(data))
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Sync 同步到磁盘
|
||||
func (w *WAL) Sync() error {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
|
||||
return w.file.Sync()
|
||||
}
|
||||
|
||||
// Close 关闭 WAL
|
||||
func (w *WAL) Close() error {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
|
||||
return w.file.Close()
|
||||
}
|
||||
|
||||
// Truncate 清空 WAL
|
||||
func (w *WAL) Truncate() error {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
|
||||
err := w.file.Truncate(0)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
_, err = w.file.Seek(0, 0)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
w.offset = 0
|
||||
return nil
|
||||
}
|
||||
|
||||
// marshalEntry 序列化 Entry
|
||||
func (w *WAL) marshalEntry(entry *Entry) []byte {
|
||||
dataLen := len(entry.Data)
|
||||
totalLen := EntryHeaderSize + dataLen
|
||||
|
||||
buf := make([]byte, totalLen)
|
||||
|
||||
// 计算 CRC32 (不包括 CRC32 字段本身)
|
||||
crcData := buf[4:totalLen]
|
||||
binary.LittleEndian.PutUint32(crcData[0:4], uint32(dataLen))
|
||||
crcData[4] = entry.Type
|
||||
binary.LittleEndian.PutUint64(crcData[5:13], uint64(entry.Seq))
|
||||
copy(crcData[13:], entry.Data)
|
||||
|
||||
crc := crc32.ChecksumIEEE(crcData)
|
||||
|
||||
// 写入 CRC32
|
||||
binary.LittleEndian.PutUint32(buf[0:4], crc)
|
||||
|
||||
return buf
|
||||
}
|
||||
|
||||
// Reader WAL 读取器
|
||||
type Reader struct {
|
||||
file *os.File
|
||||
}
|
||||
|
||||
// NewReader 创建 WAL 读取器
|
||||
func NewReader(path string) (*Reader, error) {
|
||||
file, err := os.Open(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &Reader{
|
||||
file: file,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Read 读取所有 Entry
|
||||
func (r *Reader) Read() ([]*Entry, error) {
|
||||
var entries []*Entry
|
||||
|
||||
for {
|
||||
entry, err := r.readEntry()
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
entries = append(entries, entry)
|
||||
}
|
||||
|
||||
return entries, nil
|
||||
}
|
||||
|
||||
// Close 关闭读取器
|
||||
func (r *Reader) Close() error {
|
||||
return r.file.Close()
|
||||
}
|
||||
|
||||
// readEntry 读取一条 Entry
|
||||
func (r *Reader) readEntry() (*Entry, error) {
|
||||
// 读取 Header
|
||||
header := make([]byte, EntryHeaderSize)
|
||||
_, err := io.ReadFull(r.file, header)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// 解析 Header
|
||||
crc := binary.LittleEndian.Uint32(header[0:4])
|
||||
dataLen := binary.LittleEndian.Uint32(header[4:8])
|
||||
entryType := header[8]
|
||||
seq := int64(binary.LittleEndian.Uint64(header[9:17]))
|
||||
|
||||
// 读取 Data
|
||||
data := make([]byte, dataLen)
|
||||
_, err = io.ReadFull(r.file, data)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// 验证 CRC32
|
||||
crcData := make([]byte, EntryHeaderSize-4+int(dataLen))
|
||||
copy(crcData[0:EntryHeaderSize-4], header[4:])
|
||||
copy(crcData[EntryHeaderSize-4:], data)
|
||||
|
||||
if crc32.ChecksumIEEE(crcData) != crc {
|
||||
return nil, io.ErrUnexpectedEOF // CRC 校验失败
|
||||
}
|
||||
|
||||
return &Entry{
|
||||
Type: entryType,
|
||||
Seq: seq,
|
||||
Data: data,
|
||||
CRC32: crc,
|
||||
}, nil
|
||||
}
|
||||
130
wal/wal_test.go
Normal file
130
wal/wal_test.go
Normal file
@@ -0,0 +1,130 @@
|
||||
package wal
|
||||
|
||||
import (
|
||||
"os"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestWAL(t *testing.T) {
|
||||
// 1. 创建 WAL
|
||||
wal, err := Open("test.wal")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer os.Remove("test.wal")
|
||||
|
||||
// 2. 写入数据
|
||||
for i := int64(1); i <= 100; i++ {
|
||||
entry := &Entry{
|
||||
Type: EntryTypePut,
|
||||
Seq: i,
|
||||
Data: []byte("value_" + string(rune(i))),
|
||||
}
|
||||
err := wal.Append(entry)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// 3. Sync
|
||||
err = wal.Sync()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
wal.Close()
|
||||
|
||||
t.Log("Written 100 entries")
|
||||
|
||||
// 4. 读取数据
|
||||
reader, err := NewReader("test.wal")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer reader.Close()
|
||||
|
||||
entries, err := reader.Read()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if len(entries) != 100 {
|
||||
t.Errorf("Expected 100 entries, got %d", len(entries))
|
||||
}
|
||||
|
||||
// 验证数据
|
||||
for i, entry := range entries {
|
||||
expectedSeq := int64(i + 1)
|
||||
if entry.Seq != expectedSeq {
|
||||
t.Errorf("Entry %d: expected Seq=%d, got %d", i, expectedSeq, entry.Seq)
|
||||
}
|
||||
if entry.Type != EntryTypePut {
|
||||
t.Errorf("Entry %d: expected Type=%d, got %d", i, EntryTypePut, entry.Type)
|
||||
}
|
||||
}
|
||||
|
||||
t.Log("All tests passed!")
|
||||
}
|
||||
|
||||
func TestWALTruncate(t *testing.T) {
|
||||
// 创建 WAL
|
||||
wal, err := Open("test_truncate.wal")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer os.Remove("test_truncate.wal")
|
||||
|
||||
// 写入数据
|
||||
for i := int64(1); i <= 10; i++ {
|
||||
entry := &Entry{
|
||||
Type: EntryTypePut,
|
||||
Seq: i,
|
||||
Data: []byte("value"),
|
||||
}
|
||||
wal.Append(entry)
|
||||
}
|
||||
|
||||
// Truncate
|
||||
err = wal.Truncate()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
wal.Close()
|
||||
|
||||
// 验证文件为空
|
||||
reader, err := NewReader("test_truncate.wal")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer reader.Close()
|
||||
|
||||
entries, err := reader.Read()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if len(entries) != 0 {
|
||||
t.Errorf("Expected 0 entries after truncate, got %d", len(entries))
|
||||
}
|
||||
|
||||
t.Log("Truncate test passed!")
|
||||
}
|
||||
|
||||
func BenchmarkWALAppend(b *testing.B) {
|
||||
wal, _ := Open("bench.wal")
|
||||
defer os.Remove("bench.wal")
|
||||
defer wal.Close()
|
||||
|
||||
entry := &Entry{
|
||||
Type: EntryTypePut,
|
||||
Seq: 1,
|
||||
Data: make([]byte, 100),
|
||||
}
|
||||
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
entry.Seq = int64(i)
|
||||
wal.Append(entry)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user