bourdon/seqlog
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

重构：重命名核心组件并增强查询功能

Browse Source 
主要更改：

1. 核心重命名
   - Seqlog -> LogHub (更准确地反映其作为日志中枢的角色)
   - NewSeqlog() -> NewLogHub()
   - LogCursor -> ProcessCursor (更准确地反映其用于处理场景)
   - seqlog_manager.go -> loghub.go (文件名与结构体名对应)

2. TopicProcessor.Reset 增强
   - 如果正在运行且没有待处理的日志，会自动停止后重置
   - 如果有待处理的日志，返回详细错误（显示已处理/总记录数）
   - 简化了 LogHub.ResetTopic，移除显式 Stop 调用

3. 新增查询方法
   - TopicProcessor.QueryFromFirst(count) - 从第一条记录向索引递增方向查询
   - TopicProcessor.QueryFromLast(count) - 从最后一条记录向索引递减方向查询
   - LogHub.QueryFromFirst(topic, count)
   - LogHub.QueryFromLast(topic, count)

4. 测试覆盖
   - 添加 query_test.go - QueryFromProcessing 测试
   - 添加 TestQueryFromFirstAndLast - TopicProcessor 查询测试
   - 添加 TestLogHubQueryFromFirstAndLast - LogHub 查询测试
   - 添加 TestTopicResetWithPendingRecords - Reset 增强功能测试

5. 示例代码
   - 添加 example/get_record/ - 演示 QueryFromProcessing 用法
   - 更新所有示例以使用 LogHub 和新 API

所有测试通过 ✅

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
bourdon
2025-10-04 13:26:21 +08:00
parent dfdc27c67f
commit 90cc9e21c9
14 changed files with 795 additions and 275 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
example/README.md
View File

@@ -192,10 +192,10 @@ for i := 0; i < 20; i++ {
**1. "no such file or directory" 错误**
确保在创建 Seqlog 之前先创建目录：
确保在创建 LogHub 之前先创建目录：
```go
os.MkdirAll("log_dir", 0755)
seq := seqlog.NewSeqlog("log_dir", logger, nil)
seq := seqlog.NewLogHub("log_dir", logger, nil)
```
**2. 查询时出现 EOF 错误**

2
example/concurrent/main.go
View File

@@ -40,7 +40,7 @@ func main() {
}
// 创建 Seqlog 实例（默认处理器）
seq := seqlog.NewSeqlog(testDir, logger, nil)
seq := seqlog.NewLogHub(testDir, logger, nil)
topics := []string{"app", "access", "error"}
for _, topic := range topics {

74
example/get_record/main.go Normal file
View File

@@ -0,0 +1,74 @@
package main
import (
"fmt"
"log"
"time"
"code.tczkiot.com/seqlog"
)
func main() {
// 创建 Seqlog 实例
mgr := seqlog.NewLogHub("./logs", nil, nil)
// 注册 topic handler
processedCount := 0
err := mgr.RegisterHandler("app", func(record *seqlog.Record) error {
processedCount++
fmt.Printf("处理记录 #%d: %s\n", processedCount, string(record.Data))
time.Sleep(100 * time.Millisecond) // 模拟处理耗时
return nil
})
if err != nil {
log.Fatal(err)
}
// 写入一些记录
fmt.Println("=== 写入记录 ===")
for i := 0; i < 10; i++ {
data := fmt.Sprintf("日志消息 #%d", i)
offset, err := mgr.Write("app", []byte(data))
if err != nil {
log.Fatal(err)
}
fmt.Printf("写入: offset=%d, data=%s\n", offset, data)
}
// 启动处理
fmt.Println("\n=== 启动日志处理 ===")
err = mgr.Start()
if err != nil {
log.Fatal(err)
}
// 等待一段时间让处理器处理一些记录
time.Sleep(500 * time.Millisecond)
// 查询当前处理窗口的记录
fmt.Println("\n=== 查询当前处理窗口记录 ===")
records, err := mgr.QueryFromProcessing("app", 5)
if err != nil {
log.Fatal(err)
}
fmt.Printf("从处理窗口开始位置查询到 %d 条记录:\n", len(records))
for _, rec := range records {
fmt.Printf(" [索引 %d] %s - 状态: %s\n", rec.Index, string(rec.Record.Data), rec.Status)
}
// 查询更多记录
fmt.Println("\n=== 查询后续记录 ===")
moreRecords, err := mgr.QueryFromProcessing("app", 10)
if err != nil {
log.Fatal(err)
}
fmt.Printf("查询到 %d 条记录:\n", len(moreRecords))
for _, rec := range moreRecords {
fmt.Printf(" [索引 %d] %s - 状态: %s\n", rec.Index, string(rec.Record.Data), rec.Status)
}
// 清理
mgr.Stop()
fmt.Println("\n=== 示例完成 ===")
}

20
example/index/main.go
View File

@@ -70,25 +70,25 @@ func main() {
fmt.Printf("从第 %d 条记录开始查询\n", startIndex)
// 向索引递减方向查询（查询更早的记录）
// QueryNewest(4, 3) 查询索引 2, 3, 4，返回按索引递增排序
backward, err := query.QueryNewest(startIndex-1, 3)
// QueryOldest(5, 3) 查询索引 2, 3, 4（不包含 5），返回按索引递增排序
backward, err := query.QueryOldest(startIndex, 3)
if err != nil {
log.Fatal(err)
}
fmt.Printf("向索引递减方向查询 3 条记录（索引 2-4）:\n")
for i, rec := range backward {
fmt.Printf(" [%d] 数据=%s\n", i, string(rec.Data))
fmt.Printf("向索引递减方向查询 3 条记录:\n")
for _, rec := range backward {
fmt.Printf(" [索引 %d] 数据=%s\n", rec.Index, string(rec.Record.Data))
}
// 向索引递增方向查询（查询更新的记录）
// QueryOldest(5, 3) 查询索引 5, 6, 7，返回按索引递增排序
forward, err := query.QueryOldest(startIndex, 3)
// QueryNewest(5, 3) 查询索引 6, 7, 8（不包含 5），返回按索引递增排序
forward, err := query.QueryNewest(startIndex, 3)
if err != nil {
log.Fatal(err)
}
fmt.Printf("向索引递增方向查询 3 条记录（索引 5-7）:\n")
for i, rec := range forward {
fmt.Printf(" [%d] 数据=%s\n", i, string(rec.Data))
fmt.Printf("向索引递增方向查询 3 条记录:\n")
for _, rec := range forward {
fmt.Printf(" [索引 %d] 数据=%s\n", rec.Index, string(rec.Record.Data))
}
fmt.Println()

44
example/topic_processor/main.go
View File

@@ -40,58 +40,48 @@ func main() {
count := tp.GetRecordCount()
fmt.Printf(" 总共 %d 条记录\n\n", count)
// ===== 3. 获取索引 =====
fmt.Println("3. 使用索引：")
index := tp.Index()
fmt.Printf(" 索引记录数: %d\n", index.Count())
fmt.Printf(" 最后偏移: %d\n\n", index.LastOffset())
// ===== 3. 获取记录数 =====
fmt.Println("3. 查看记录统计：")
totalCount := tp.GetRecordCount()
fmt.Printf(" 记录总数: %d\n\n", totalCount)
// ===== 4. 使用查询器查询 =====
fmt.Println("4. 查询记录：")
// 查询最老的 3 条记录（从索引 0 开始）
oldest, err := tp.QueryOldest(0, 3)
oldest, err := tp.QueryOldest(3, 3)
if err != nil {
log.Fatal(err)
}
fmt.Println(" 查询最老的 3 条:")
for i, rws := range oldest {
fmt.Printf(" [%d] 状态=%s, 数据=%s\n", i, rws.Status, string(rws.Record.Data))
fmt.Printf(" [%d] 索引=%d, 状态=%s, 数据=%s\n", i, rws.Index, rws.Status, string(rws.Record.Data))
}
// 查询最新的 2 条记录（从最后一条开始）
totalCount := tp.GetRecordCount()
newest, err := tp.QueryNewest(totalCount-1, 2)
newest, err := tp.QueryNewest(totalCount-3, 2)
if err != nil {
log.Fatal(err)
}
fmt.Println(" 查询最新的 2 条:")
for i, rws := range newest {
fmt.Printf(" [%d] 状态=%s, 数据=%s\n", i, rws.Status, string(rws.Record.Data))
fmt.Printf(" [%d] 索引=%d, 状态=%s, 数据=%s\n", i, rws.Index, rws.Status, string(rws.Record.Data))
}
fmt.Println()
// ===== 5. 使用游标读取 =====
fmt.Println("5. 使用游标读取：")
cursor, err := tp.Cursor()
// ===== 5. 从处理窗口查询 =====
fmt.Println("5. 从处理窗口查询：")
// 从处理窗口开始位置查询 3 条记录
processing, err := tp.QueryFromProcessing(3)
if err != nil {
log.Fatal(err)
}
defer cursor.Close()
// 读取 3 条记录
records, err := cursor.NextRange(3)
if err != nil {
log.Fatal(err)
fmt.Printf(" 从处理窗口查询到 %d 条记录:\n", len(processing))
for i, rec := range processing {
fmt.Printf(" [%d] 索引=%d, 状态=%s, 数据=%s\n", i, rec.Index, rec.Status, string(rec.Record.Data))
}
fmt.Printf(" 读取了 %d 条记录:\n", len(records))
for i, rec := range records {
fmt.Printf(" [%d] %s\n", i, string(rec.Data))
}
// 提交游标位置
cursor.Commit()
fmt.Printf(" 游标位置: start=%d, end=%d\n\n", cursor.StartIndex(), cursor.EndIndex())
fmt.Println()
// ===== 6. 继续写入 =====
fmt.Println("6. 继续写入：")

128
example/webapp/main.go
View File

@@ -14,7 +14,7 @@ import (
)
var (
seq *seqlog.Seqlog
seq *seqlog.LogHub
logger *slog.Logger
)
@@ -25,7 +25,7 @@ func main() {
}))
// 创建 Seqlog
seq = seqlog.NewSeqlog("logs", logger, func(topic string, rec *seqlog.Record) error {
seq = seqlog.NewLogHub("logs", logger, func(topic string, rec *seqlog.Record) error {
// 简单的日志处理：只打印摘要信息
dataPreview := string(rec.Data)
if len(dataPreview) > 100 {
@@ -155,6 +155,12 @@ func formatBytes(bytes int64) string {
return fmt.Sprintf("%.2f MB", float64(bytes)/1024/1024)
}
type Record struct {
Index int `json:"index"`
Status string `json:"status"`
Data string `json:"data"`
}
// 首页
func handleIndex(w http.ResponseWriter, r *http.Request) {
html := `<!DOCTYPE html>
@@ -366,7 +372,7 @@ func handleIndex(w http.ResponseWriter, r *http.Request) {
// 选择 topic
function selectTopic(topic) {
currentTopic = topic;
direction = 'forward'; // 切换 topic 时重置方向
direction = ''; // 切换 topic 时重置方向
startIndex = null;
endIndex = null;
@@ -405,14 +411,14 @@ func handleIndex(w http.ResponseWriter, r *http.Request) {
const forward = document.getElementById('forwardCount').value;
// 构建查询 URL
let url = '/api/query?topic=' + currentTopic +
'&backward=' + backward + '&forward=' + forward +
'&direction=' + direction;
let url = '/api/query?topic=' + currentTopic;
if (direction === 'backward' && startIndex != null) {
url += '&index=' + startIndex;
url += '&direction=backward&index=' + startIndex + '&count=' + backward;
} else if (direction === 'forward' && endIndex != null) {
url += '&index=' + endIndex;
url += '&direction=forward&index=' + endIndex + '&count=' + forward;
} else {
url += '&count=10';
}
const response = await fetch(url);
@@ -449,11 +455,16 @@ func handleIndex(w http.ResponseWriter, r *http.Request) {
}).join('');
if (data.records.length > 0) {
startIndex = data.records[0].index;
endIndex = data.records[data.records.length - 1].index;
} else {
startIndex = null;
endIndex = null;
if (startIndex === null) {
startIndex = data.records[0].index;
} else {
startIndex = Math.min(startIndex, data.records[0].index);
}
if (endIndex === null) {
endIndex = data.records[data.records.length - 1].index;
} else {
endIndex = Math.max(endIndex, data.records[data.records.length - 1].index);
}
}
container.innerHTML = html;
@@ -526,14 +537,10 @@ func handleQuery(w http.ResponseWriter, r *http.Request) {
// 获取查询参数
indexParam := r.URL.Query().Get("index")
direction := r.URL.Query().Get("direction")
backward, _ := strconv.Atoi(r.URL.Query().Get("backward"))
forward, _ := strconv.Atoi(r.URL.Query().Get("forward"))
count, _ := strconv.Atoi(r.URL.Query().Get("count"))
if backward == 0 {
backward = 10
}
if forward == 0 {
forward = 10
if count <= 0 {
count = 10
}
// 获取 processor
@@ -546,79 +553,68 @@ func handleQuery(w http.ResponseWriter, r *http.Request) {
// 获取记录总数
totalCount := processor.GetRecordCount()
// 获取当前处理索引和读取索引（用于状态判断）
startIdx := seq.GetProcessingIndex(topic)
endIdx := seq.GetReadIndex(topic)
// 执行查询
var results []*seqlog.RecordWithStatus
var startRecordIndex int
if direction == "backward" {
// 向前翻页：查询更早的记录
var index int
if indexParam == "" {
index = startIdx
} else {
index, _ = strconv.Atoi(indexParam)
if direction == "" {
results, err = processor.QueryFromProcessing(count)
if err != nil {
http.Error(w, err.Error(), http.StatusNotFound)
return
}
if index > 0 {
var queryErr error
results, queryErr = processor.QueryNewest(index-1, backward)
if queryErr != nil {
http.Error(w, queryErr.Error(), http.StatusInternalServerError)
if len(results) == 0 {
results, err = processor.QueryFromLast(count)
if err != nil {
http.Error(w, err.Error(), http.StatusNotFound)
return
}
if len(results) > 0 {
// QueryNewest 返回的结果按索引递增排列
startRecordIndex = index - len(results)
}
}
} else {
// 向后翻页：查询更新的记录
var index int
var refIndex int
if indexParam == "" {
index = startIdx
http.Error(w, "参数错误", http.StatusNotFound)
return
} else {
index, _ = strconv.Atoi(indexParam)
refIndex, err = strconv.Atoi(indexParam)
if err != nil {
http.Error(w, err.Error(), http.StatusNotFound)
return
}
}
if index < totalCount {
if direction == "backward" {
var queryErr error
results, queryErr = processor.QueryOldest(index, forward)
results, queryErr = processor.QueryNewest(refIndex, count)
if queryErr != nil {
http.Error(w, queryErr.Error(), http.StatusInternalServerError)
return
}
if len(results) > 0 {
// QueryOldest 返回的结果按索引递增排列
startRecordIndex = index
} else if direction == "forward" {
var queryErr error
results, queryErr = processor.QueryOldest(refIndex, count)
if queryErr != nil {
http.Error(w, queryErr.Error(), http.StatusInternalServerError)
return
}
} else {
http.Error(w, "参数错误", http.StatusNotFound)
return
}
}
type Record struct {
Index int `json:"index"`
Status string `json:"status"`
Data string `json:"data"`
}
records := make([]Record, len(results))
for i, r := range results {
for i, result := range results {
records[i] = Record{
Index: startRecordIndex + i,
Status: r.Status.String(),
Data: string(r.Record.Data),
Index: result.Index,
Status: result.Status.String(),
Data: string(result.Record.Data),
}
}
json.NewEncoder(w).Encode(map[string]interface{}{
"records": records,
"total": len(records),
"totalCount": totalCount,
"processingIndex": startIdx,
"readIndex": endIdx,
"records": records,
"total": len(records),
"totalCount": totalCount,
})
}