重构:优化记录格式并修复核心功能
- 修改记录存储格式为 [4B len][8B offset][4B CRC][16B UUID][data] - 修复 TopicProcessor 中 WaitGroup 使用错误导致 handler 不执行的问题 - 修复写入保护逻辑,避免 dirtyOffset=-1 时误判为写入中 - 添加统计信息定期持久化功能 - 改进 UTF-8 字符截断处理,防止 CJK 字符乱码 - 优化 Web UI:显示人类可读的文件大小,支持点击外部关闭弹窗 - 重构示例代码,添加 webui 和 webui_integration 示例 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -1,74 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log"
|
||||
"time"
|
||||
|
||||
"code.tczkiot.com/seqlog"
|
||||
)
|
||||
|
||||
func main() {
|
||||
// 创建 Seqlog 实例
|
||||
mgr := seqlog.NewLogHub("./logs", nil, nil)
|
||||
|
||||
// 注册 topic handler
|
||||
processedCount := 0
|
||||
err := mgr.RegisterHandler("app", func(record *seqlog.Record) error {
|
||||
processedCount++
|
||||
fmt.Printf("处理记录 #%d: %s\n", processedCount, string(record.Data))
|
||||
time.Sleep(100 * time.Millisecond) // 模拟处理耗时
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
// 写入一些记录
|
||||
fmt.Println("=== 写入记录 ===")
|
||||
for i := 0; i < 10; i++ {
|
||||
data := fmt.Sprintf("日志消息 #%d", i)
|
||||
offset, err := mgr.Write("app", []byte(data))
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
fmt.Printf("写入: offset=%d, data=%s\n", offset, data)
|
||||
}
|
||||
|
||||
// 启动处理
|
||||
fmt.Println("\n=== 启动日志处理 ===")
|
||||
err = mgr.Start()
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
// 等待一段时间让处理器处理一些记录
|
||||
time.Sleep(500 * time.Millisecond)
|
||||
|
||||
// 查询当前处理窗口的记录
|
||||
fmt.Println("\n=== 查询当前处理窗口记录 ===")
|
||||
records, err := mgr.QueryFromProcessing("app", 5)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
fmt.Printf("从处理窗口开始位置查询到 %d 条记录:\n", len(records))
|
||||
for _, rec := range records {
|
||||
fmt.Printf(" [索引 %d] %s - 状态: %s\n", rec.Index, string(rec.Record.Data), rec.Status)
|
||||
}
|
||||
|
||||
// 查询更多记录
|
||||
fmt.Println("\n=== 查询后续记录 ===")
|
||||
moreRecords, err := mgr.QueryFromProcessing("app", 10)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
fmt.Printf("查询到 %d 条记录:\n", len(moreRecords))
|
||||
for _, rec := range moreRecords {
|
||||
fmt.Printf(" [索引 %d] %s - 状态: %s\n", rec.Index, string(rec.Record.Data), rec.Status)
|
||||
}
|
||||
|
||||
// 清理
|
||||
mgr.Stop()
|
||||
fmt.Println("\n=== 示例完成 ===")
|
||||
}
|
||||
@@ -37,8 +37,8 @@ func main() {
|
||||
lastOffset = offset
|
||||
fmt.Printf("写入: offset=%d, data=%s\n", offset, data)
|
||||
}
|
||||
defer writer.Close()
|
||||
|
||||
writer.Close()
|
||||
fmt.Printf("索引文件已创建: %s.idx\n\n", logPath)
|
||||
|
||||
// ===== 示例 2:使用索引进行快速查询 =====
|
||||
@@ -52,7 +52,7 @@ func main() {
|
||||
defer index2.Close()
|
||||
|
||||
// 创建查询器(使用外部索引)
|
||||
query, err := seqlog.NewRecordQuery(logPath, index2)
|
||||
query, err := seqlog.NewRecordQuery(logPath, index2, writer)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
@@ -1,65 +0,0 @@
|
||||
# Seqlog Web 演示
|
||||
|
||||
一个简单的 Web 应用,展示 Seqlog 的实际使用场景。
|
||||
|
||||
## 功能
|
||||
|
||||
### 后端模拟业务
|
||||
- 每 2 秒自动生成业务日志
|
||||
- 随机生成不同 topic(app、api、database、cache)
|
||||
- 随机生成不同操作(查询、插入、更新、删除、备份、恢复、同步等)
|
||||
- **随机日志大小**(2KB ~ 10MB):
|
||||
- 80% 小日志(2KB - 100KB)
|
||||
- 15% 中日志(100KB - 1MB)
|
||||
- 5% 大日志(1MB - 10MB)
|
||||
|
||||
### Web 查询界面
|
||||
- 查看所有 topics
|
||||
- 查看每个 topic 的统计信息(显示实际字节数)
|
||||
- 查询日志(支持向前/向后翻页)
|
||||
- 实时自动刷新
|
||||
- 日志状态标注(已处理/处理中/待处理)
|
||||
|
||||
## 快速启动
|
||||
|
||||
```bash
|
||||
cd example/webapp
|
||||
go run main.go
|
||||
```
|
||||
|
||||
访问: http://localhost:8080
|
||||
|
||||
## 使用说明
|
||||
|
||||
1. **选择 Topic**: 点击左侧的 topic 列表
|
||||
2. **查看统计**: 左侧会显示该 topic 的统计信息(包括总字节数)
|
||||
3. **查看日志**: 右侧显示日志内容,带状态标注
|
||||
4. **刷新**: 点击"刷新日志"按钮或等待自动刷新
|
||||
5. **翻页**: 使用"向前翻页"和"向后翻页"按钮
|
||||
6. **自定义范围**: 修改显示范围的数字,控制查询条数
|
||||
|
||||
## 界面说明
|
||||
|
||||
- **绿色边框**: 已处理的日志
|
||||
- **黄色边框**: 正在处理的日志
|
||||
- **灰色边框**: 待处理的日志
|
||||
|
||||
## 性能测试
|
||||
|
||||
由于日志大小范围很大(2KB ~ 10MB),可以观察到:
|
||||
- 小日志处理速度很快
|
||||
- 大日志会占用更多存储空间
|
||||
- 统计信息会显示真实的字节数增长
|
||||
|
||||
## API 接口
|
||||
|
||||
- `GET /api/topics` - 获取所有 topics
|
||||
- `GET /api/stats?topic=<name>` - 获取统计信息
|
||||
- `GET /api/query?topic=<name>&backward=10&forward=10` - 查询日志
|
||||
- `POST /api/write` - 手动写入日志
|
||||
|
||||
## 技术栈
|
||||
|
||||
- 后端: Go + Seqlog
|
||||
- 前端: 原生 HTML/CSS/JavaScript
|
||||
- 无需额外依赖
|
||||
@@ -1,648 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"math/rand"
|
||||
"net/http"
|
||||
"os"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"code.tczkiot.com/seqlog"
|
||||
)
|
||||
|
||||
var (
|
||||
seq *seqlog.LogHub
|
||||
logger *slog.Logger
|
||||
)
|
||||
|
||||
func main() {
|
||||
// 初始化
|
||||
logger = slog.New(slog.NewTextHandler(os.Stdout, &slog.HandlerOptions{
|
||||
Level: slog.LevelInfo,
|
||||
}))
|
||||
|
||||
// 创建 Seqlog
|
||||
seq = seqlog.NewLogHub("logs", logger, func(topic string, rec *seqlog.Record) error {
|
||||
// 简单的日志处理:只打印摘要信息
|
||||
dataPreview := string(rec.Data)
|
||||
if len(dataPreview) > 100 {
|
||||
dataPreview = dataPreview[:100] + "..."
|
||||
}
|
||||
logger.Info("处理日志", "topic", topic, "size", len(rec.Data), "preview", dataPreview)
|
||||
return nil
|
||||
})
|
||||
|
||||
if err := seq.Start(); err != nil {
|
||||
logger.Error("启动失败", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
defer seq.Stop()
|
||||
|
||||
logger.Info("Seqlog 已启动")
|
||||
|
||||
// 启动后台业务模拟
|
||||
go simulateBusiness()
|
||||
|
||||
// 启动 Web 服务器
|
||||
http.HandleFunc("/", handleIndex)
|
||||
http.HandleFunc("/api/topics", handleTopics)
|
||||
http.HandleFunc("/api/stats", handleStats)
|
||||
http.HandleFunc("/api/query", handleQuery)
|
||||
http.HandleFunc("/api/write", handleWrite)
|
||||
|
||||
addr := ":8080"
|
||||
logger.Info("Web 服务器启动", "地址", "http://localhost"+addr)
|
||||
if err := http.ListenAndServe(addr, nil); err != nil {
|
||||
logger.Error("服务器错误", "error", err)
|
||||
}
|
||||
}
|
||||
|
||||
// 模拟业务写日志
|
||||
func simulateBusiness() {
|
||||
topics := []string{"app", "api", "database", "cache"}
|
||||
actions := []string{"查询", "插入", "更新", "删除", "连接", "断开", "备份", "恢复", "同步"}
|
||||
status := []string{"成功", "失败", "超时", "重试"}
|
||||
|
||||
ticker := time.NewTicker(2 * time.Second)
|
||||
defer ticker.Stop()
|
||||
|
||||
for range ticker.C {
|
||||
// 随机选择 topic 和内容
|
||||
topic := topics[rand.Intn(len(topics))]
|
||||
action := actions[rand.Intn(len(actions))]
|
||||
st := status[rand.Intn(len(status))]
|
||||
|
||||
// 随机生成日志大小:2KB 到 10MB
|
||||
// 80% 概率生成小日志(2KB-100KB)
|
||||
// 15% 概率生成中日志(100KB-1MB)
|
||||
// 5% 概率生成大日志(1MB-10MB)
|
||||
var logSize int
|
||||
prob := rand.Intn(100)
|
||||
if prob < 80 {
|
||||
// 2KB - 100KB
|
||||
logSize = 2*1024 + rand.Intn(98*1024)
|
||||
} else if prob < 95 {
|
||||
// 100KB - 1MB
|
||||
logSize = 100*1024 + rand.Intn(924*1024)
|
||||
} else {
|
||||
// 1MB - 10MB
|
||||
logSize = 1024*1024 + rand.Intn(9*1024*1024)
|
||||
}
|
||||
|
||||
// 生成日志内容
|
||||
header := fmt.Sprintf("[%s] %s %s - 用时: %dms | 数据大小: %s | ",
|
||||
time.Now().Format("15:04:05"),
|
||||
action,
|
||||
st,
|
||||
rand.Intn(1000),
|
||||
formatBytes(int64(logSize)))
|
||||
|
||||
// 填充随机数据到指定大小
|
||||
data := make([]byte, logSize)
|
||||
copy(data, []byte(header))
|
||||
|
||||
// 填充可读的模拟数据
|
||||
fillOffset := len(header)
|
||||
patterns := []string{
|
||||
"user_id=%d, session=%x, ip=%d.%d.%d.%d, ",
|
||||
"query_time=%dms, rows=%d, cached=%v, ",
|
||||
"error_code=%d, retry_count=%d, ",
|
||||
"request_id=%x, trace_id=%x, ",
|
||||
}
|
||||
|
||||
for fillOffset < logSize-100 {
|
||||
pattern := patterns[rand.Intn(len(patterns))]
|
||||
var chunk string
|
||||
switch pattern {
|
||||
case patterns[0]:
|
||||
chunk = fmt.Sprintf(pattern, rand.Intn(10000), rand.Intn(0xFFFFFF),
|
||||
rand.Intn(256), rand.Intn(256), rand.Intn(256), rand.Intn(256))
|
||||
case patterns[1]:
|
||||
chunk = fmt.Sprintf(pattern, rand.Intn(1000), rand.Intn(10000), rand.Intn(2) == 1)
|
||||
case patterns[2]:
|
||||
chunk = fmt.Sprintf(pattern, rand.Intn(500), rand.Intn(5))
|
||||
case patterns[3]:
|
||||
chunk = fmt.Sprintf(pattern, rand.Intn(0xFFFFFFFF), rand.Intn(0xFFFFFFFF))
|
||||
}
|
||||
|
||||
remaining := logSize - fillOffset
|
||||
if len(chunk) > remaining {
|
||||
chunk = chunk[:remaining]
|
||||
}
|
||||
copy(data[fillOffset:], []byte(chunk))
|
||||
fillOffset += len(chunk)
|
||||
}
|
||||
|
||||
// 写入日志
|
||||
if _, err := seq.Write(topic, data); err != nil {
|
||||
logger.Error("写入日志失败", "error", err, "size", logSize)
|
||||
} else {
|
||||
logger.Info("写入日志", "topic", topic, "size", formatBytes(int64(logSize)))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func formatBytes(bytes int64) string {
|
||||
if bytes < 1024 {
|
||||
return fmt.Sprintf("%d B", bytes)
|
||||
}
|
||||
if bytes < 1024*1024 {
|
||||
return fmt.Sprintf("%.1f KB", float64(bytes)/1024)
|
||||
}
|
||||
return fmt.Sprintf("%.2f MB", float64(bytes)/1024/1024)
|
||||
}
|
||||
|
||||
type Record struct {
|
||||
Index int `json:"index"`
|
||||
Status string `json:"status"`
|
||||
Data string `json:"data"`
|
||||
}
|
||||
|
||||
// 首页
|
||||
func handleIndex(w http.ResponseWriter, r *http.Request) {
|
||||
html := `<!DOCTYPE html>
|
||||
<html>
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<title>Seqlog 日志查询</title>
|
||||
<style>
|
||||
body {
|
||||
font-family: 'Segoe UI', Tahoma, Geneva, Verdana, sans-serif;
|
||||
max-width: 1400px;
|
||||
margin: 0 auto;
|
||||
padding: 20px;
|
||||
background: #f5f5f5;
|
||||
}
|
||||
.header {
|
||||
background: white;
|
||||
padding: 20px;
|
||||
border-radius: 8px;
|
||||
margin-bottom: 20px;
|
||||
box-shadow: 0 2px 4px rgba(0,0,0,0.1);
|
||||
}
|
||||
h1 {
|
||||
margin: 0;
|
||||
color: #333;
|
||||
}
|
||||
.subtitle {
|
||||
color: #666;
|
||||
margin-top: 5px;
|
||||
}
|
||||
.container {
|
||||
display: grid;
|
||||
grid-template-columns: 250px 1fr;
|
||||
gap: 20px;
|
||||
}
|
||||
.sidebar {
|
||||
background: white;
|
||||
padding: 20px;
|
||||
border-radius: 8px;
|
||||
box-shadow: 0 2px 4px rgba(0,0,0,0.1);
|
||||
}
|
||||
.main {
|
||||
background: white;
|
||||
padding: 20px;
|
||||
border-radius: 8px;
|
||||
box-shadow: 0 2px 4px rgba(0,0,0,0.1);
|
||||
}
|
||||
.topic-list {
|
||||
list-style: none;
|
||||
padding: 0;
|
||||
margin: 0;
|
||||
}
|
||||
.topic-item {
|
||||
padding: 10px;
|
||||
margin-bottom: 5px;
|
||||
cursor: pointer;
|
||||
border-radius: 4px;
|
||||
transition: background 0.2s;
|
||||
}
|
||||
.topic-item:hover {
|
||||
background: #f0f0f0;
|
||||
}
|
||||
.topic-item.active {
|
||||
background: #007bff;
|
||||
color: white;
|
||||
}
|
||||
.stats {
|
||||
margin-top: 20px;
|
||||
padding: 15px;
|
||||
background: #f8f9fa;
|
||||
border-radius: 4px;
|
||||
}
|
||||
.stat-item {
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
margin-bottom: 8px;
|
||||
font-size: 14px;
|
||||
}
|
||||
.controls {
|
||||
margin-bottom: 20px;
|
||||
}
|
||||
.btn {
|
||||
padding: 8px 16px;
|
||||
margin-right: 10px;
|
||||
border: none;
|
||||
border-radius: 4px;
|
||||
cursor: pointer;
|
||||
font-size: 14px;
|
||||
}
|
||||
.btn-primary {
|
||||
background: #007bff;
|
||||
color: white;
|
||||
}
|
||||
.btn-secondary {
|
||||
background: #6c757d;
|
||||
color: white;
|
||||
}
|
||||
.log-container {
|
||||
height: 500px;
|
||||
overflow-y: auto;
|
||||
border: 1px solid #ddd;
|
||||
border-radius: 4px;
|
||||
padding: 10px;
|
||||
background: #f8f9fa;
|
||||
font-family: 'Courier New', monospace;
|
||||
font-size: 13px;
|
||||
}
|
||||
.log-entry {
|
||||
padding: 6px 10px;
|
||||
margin-bottom: 4px;
|
||||
background: white;
|
||||
border-left: 3px solid #007bff;
|
||||
border-radius: 2px;
|
||||
word-wrap: break-word;
|
||||
overflow-wrap: break-word;
|
||||
display: -webkit-box;
|
||||
-webkit-line-clamp: 3;
|
||||
-webkit-box-orient: vertical;
|
||||
overflow: hidden;
|
||||
line-height: 1.5;
|
||||
}
|
||||
.log-entry.processed {
|
||||
border-left-color: #28a745;
|
||||
opacity: 0.8;
|
||||
}
|
||||
.log-entry.processing {
|
||||
border-left-color: #ffc107;
|
||||
background: #fff9e6;
|
||||
}
|
||||
.log-entry.pending {
|
||||
border-left-color: #6c757d;
|
||||
opacity: 0.6;
|
||||
}
|
||||
.status-badge {
|
||||
display: inline-block;
|
||||
padding: 2px 8px;
|
||||
border-radius: 3px;
|
||||
font-size: 11px;
|
||||
margin-right: 8px;
|
||||
}
|
||||
.status-processed {
|
||||
background: #d4edda;
|
||||
color: #155724;
|
||||
}
|
||||
.status-processing {
|
||||
background: #fff3cd;
|
||||
color: #856404;
|
||||
}
|
||||
.status-pending {
|
||||
background: #e2e3e5;
|
||||
color: #383d41;
|
||||
}
|
||||
.loading {
|
||||
text-align: center;
|
||||
padding: 20px;
|
||||
color: #666;
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div class="header">
|
||||
<h1>Seqlog 日志查询系统</h1>
|
||||
<div class="subtitle">实时查看和管理应用日志</div>
|
||||
</div>
|
||||
|
||||
<div class="container">
|
||||
<div class="sidebar">
|
||||
<h3>Topics</h3>
|
||||
<ul class="topic-list" id="topicList"></ul>
|
||||
|
||||
<div class="stats" id="stats">
|
||||
<h4>统计信息</h4>
|
||||
<div id="statsContent">选择一个 topic 查看统计</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="main">
|
||||
<div class="controls">
|
||||
<button class="btn btn-primary" onclick="loadLogs()">刷新日志</button>
|
||||
<button class="btn btn-secondary" onclick="queryBackward()">向前翻页</button>
|
||||
<button class="btn btn-secondary" onclick="queryForward()">向后翻页</button>
|
||||
<span style="margin-left: 20px;">显示范围: 前 <input type="number" id="backwardCount" value="10" style="width: 60px;"> 条, 后 <input type="number" id="forwardCount" value="10" style="width: 60px;"> 条</span>
|
||||
</div>
|
||||
|
||||
<div class="log-container" id="logContainer">
|
||||
<div class="loading">选择一个 topic 开始查看日志</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<script>
|
||||
let currentTopic = null;
|
||||
let currentCenterIndex = null; // 追踪当前中心索引位置
|
||||
let direction = 'forward'; // 默认方向为向前翻页
|
||||
let startIndex = null;
|
||||
let endIndex = null;
|
||||
|
||||
// 加载 topics
|
||||
async function loadTopics() {
|
||||
const response = await fetch('/api/topics');
|
||||
const topics = await response.json();
|
||||
|
||||
const list = document.getElementById('topicList');
|
||||
list.innerHTML = topics.sort().map(topic =>
|
||||
'<li class="topic-item" onclick="selectTopic(\'' + topic + '\')">' + topic + '</li>'
|
||||
).join('');
|
||||
}
|
||||
|
||||
// 选择 topic
|
||||
function selectTopic(topic) {
|
||||
currentTopic = topic;
|
||||
direction = ''; // 切换 topic 时重置方向
|
||||
startIndex = null;
|
||||
endIndex = null;
|
||||
|
||||
// 更新选中状态
|
||||
document.querySelectorAll('.topic-item').forEach(item => {
|
||||
item.classList.remove('active');
|
||||
if (item.textContent === topic) {
|
||||
item.classList.add('active');
|
||||
}
|
||||
});
|
||||
|
||||
// 清空容器并重新加载
|
||||
document.getElementById('logContainer').innerHTML = '';
|
||||
loadStats(topic);
|
||||
loadLogs();
|
||||
}
|
||||
|
||||
// 加载统计
|
||||
async function loadStats(topic) {
|
||||
const response = await fetch('/api/stats?topic=' + topic);
|
||||
const stats = await response.json();
|
||||
|
||||
const content = document.getElementById('statsContent');
|
||||
content.innerHTML =
|
||||
'<div class="stat-item"><span>写入:</span><span>' + stats.write_count + ' 条</span></div>' +
|
||||
'<div class="stat-item"><span>处理:</span><span>' + stats.processed_count + ' 条</span></div>' +
|
||||
'<div class="stat-item"><span>错误:</span><span>' + stats.error_count + ' 次</span></div>' +
|
||||
'<div class="stat-item"><span>大小:</span><span>' + formatBytes(stats.write_bytes) + '</span></div>';
|
||||
}
|
||||
|
||||
// 加载日志
|
||||
async function loadLogs() {
|
||||
if (!currentTopic) return;
|
||||
|
||||
const backward = document.getElementById('backwardCount').value;
|
||||
const forward = document.getElementById('forwardCount').value;
|
||||
|
||||
// 构建查询 URL
|
||||
let url = '/api/query?topic=' + currentTopic;
|
||||
|
||||
if (direction === 'backward' && startIndex != null) {
|
||||
url += '&direction=backward&index=' + startIndex + '&count=' + backward;
|
||||
} else if (direction === 'forward' && endIndex != null) {
|
||||
url += '&direction=forward&index=' + endIndex + '&count=' + forward;
|
||||
} else {
|
||||
url += '&count=10';
|
||||
}
|
||||
|
||||
const response = await fetch(url);
|
||||
const data = await response.json();
|
||||
|
||||
const container = document.getElementById('logContainer');
|
||||
|
||||
if (data.records.length === 0) {
|
||||
container.innerHTML = '<div class="loading">暂无日志</div>';
|
||||
return;
|
||||
}
|
||||
|
||||
// 清空并重新渲染所有记录
|
||||
const html = data.records.map(r => {
|
||||
// 解析状态
|
||||
let statusClass = 'pending';
|
||||
let statusText = '待处理';
|
||||
let badgeClass = 'status-pending';
|
||||
|
||||
if (r.status === 'StatusProcessed' || r.status === 'processed') {
|
||||
statusClass = 'processed';
|
||||
statusText = '已处理';
|
||||
badgeClass = 'status-processed';
|
||||
} else if (r.status === 'StatusProcessing' || r.status === 'processing') {
|
||||
statusClass = 'processing';
|
||||
statusText = '处理中';
|
||||
badgeClass = 'status-processing';
|
||||
}
|
||||
|
||||
return '<div class="log-entry ' + statusClass + '" data-index="' + r.index + '">' +
|
||||
'<span class="status-badge ' + badgeClass + '">[#' + r.index + '] ' + statusText + '</span>' +
|
||||
r.data +
|
||||
'</div>';
|
||||
}).join('');
|
||||
|
||||
if (data.records.length > 0) {
|
||||
if (startIndex === null) {
|
||||
startIndex = data.records[0].index;
|
||||
} else {
|
||||
startIndex = Math.min(startIndex, data.records[0].index);
|
||||
}
|
||||
if (endIndex === null) {
|
||||
endIndex = data.records[data.records.length - 1].index;
|
||||
} else {
|
||||
endIndex = Math.max(endIndex, data.records[data.records.length - 1].index);
|
||||
}
|
||||
}
|
||||
|
||||
container.innerHTML = html;
|
||||
}
|
||||
|
||||
function queryBackward() {
|
||||
direction = 'backward';
|
||||
loadLogs();
|
||||
}
|
||||
|
||||
function queryForward() {
|
||||
direction = 'forward';
|
||||
loadLogs();
|
||||
}
|
||||
|
||||
function formatBytes(bytes) {
|
||||
if (bytes < 1024) return bytes + ' B';
|
||||
if (bytes < 1024 * 1024) return (bytes / 1024).toFixed(1) + ' KB';
|
||||
return (bytes / 1024 / 1024).toFixed(1) + ' MB';
|
||||
}
|
||||
|
||||
// 初始化
|
||||
loadTopics();
|
||||
|
||||
// 只自动刷新统计信息,不刷新日志
|
||||
setInterval(() => {
|
||||
if (currentTopic) {
|
||||
loadStats(currentTopic);
|
||||
}
|
||||
}, 3000); // 每 3 秒刷新统计
|
||||
</script>
|
||||
</body>
|
||||
</html>`
|
||||
|
||||
w.Header().Set("Content-Type", "text/html; charset=utf-8")
|
||||
fmt.Fprint(w, html)
|
||||
}
|
||||
|
||||
// API: 获取所有 topics
|
||||
func handleTopics(w http.ResponseWriter, r *http.Request) {
|
||||
topics := seq.GetTopics()
|
||||
json.NewEncoder(w).Encode(topics)
|
||||
}
|
||||
|
||||
// API: 获取统计信息
|
||||
func handleStats(w http.ResponseWriter, r *http.Request) {
|
||||
topic := r.URL.Query().Get("topic")
|
||||
if topic == "" {
|
||||
http.Error(w, "缺少 topic 参数", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
stats, err := seq.GetTopicStats(topic)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
|
||||
json.NewEncoder(w).Encode(stats)
|
||||
}
|
||||
|
||||
// API: 查询日志
|
||||
func handleQuery(w http.ResponseWriter, r *http.Request) {
|
||||
topic := r.URL.Query().Get("topic")
|
||||
if topic == "" {
|
||||
http.Error(w, "缺少 topic 参数", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
// 获取查询参数
|
||||
indexParam := r.URL.Query().Get("index")
|
||||
direction := r.URL.Query().Get("direction")
|
||||
count, _ := strconv.Atoi(r.URL.Query().Get("count"))
|
||||
|
||||
if count <= 0 {
|
||||
count = 10
|
||||
}
|
||||
|
||||
// 获取 processor
|
||||
processor, err := seq.GetProcessor(topic)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
|
||||
// 获取记录总数
|
||||
totalCount := processor.GetRecordCount()
|
||||
|
||||
// 执行查询
|
||||
var results []*seqlog.RecordWithStatus
|
||||
|
||||
if direction == "" {
|
||||
results, err = processor.QueryFromProcessing(count)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
if len(results) == 0 {
|
||||
results, err = processor.QueryFromLast(count)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
}
|
||||
} else {
|
||||
var refIndex int
|
||||
if indexParam == "" {
|
||||
http.Error(w, "参数错误", http.StatusNotFound)
|
||||
return
|
||||
} else {
|
||||
refIndex, err = strconv.Atoi(indexParam)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
if direction == "backward" {
|
||||
var queryErr error
|
||||
results, queryErr = processor.QueryNewest(refIndex, count)
|
||||
if queryErr != nil {
|
||||
http.Error(w, queryErr.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
} else if direction == "forward" {
|
||||
var queryErr error
|
||||
results, queryErr = processor.QueryOldest(refIndex, count)
|
||||
if queryErr != nil {
|
||||
http.Error(w, queryErr.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
} else {
|
||||
http.Error(w, "参数错误", http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
records := make([]Record, len(results))
|
||||
for i, result := range results {
|
||||
records[i] = Record{
|
||||
Index: result.Index,
|
||||
Status: result.Status.String(),
|
||||
Data: string(result.Record.Data),
|
||||
}
|
||||
}
|
||||
|
||||
json.NewEncoder(w).Encode(map[string]interface{}{
|
||||
"records": records,
|
||||
"total": len(records),
|
||||
"totalCount": totalCount,
|
||||
})
|
||||
}
|
||||
|
||||
// API: 手动写入日志
|
||||
func handleWrite(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != "POST" {
|
||||
http.Error(w, "只支持 POST", http.StatusMethodNotAllowed)
|
||||
return
|
||||
}
|
||||
|
||||
var req struct {
|
||||
Topic string `json:"topic"`
|
||||
Data string `json:"data"`
|
||||
}
|
||||
|
||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
offset, err := seq.Write(req.Topic, []byte(req.Data))
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
json.NewEncoder(w).Encode(map[string]interface{}{
|
||||
"success": true,
|
||||
"offset": offset,
|
||||
})
|
||||
}
|
||||
140
example/webui/README.md
Normal file
140
example/webui/README.md
Normal file
@@ -0,0 +1,140 @@
|
||||
# Seqlog Web UI 示例
|
||||
|
||||
这个示例展示了如何使用 seqlog 的内置 Web UI 功能。
|
||||
|
||||
## 特性
|
||||
|
||||
- **零编译前端**:使用 Vue 3 和 Tailwind CSS CDN,无需前端构建步骤
|
||||
- **实时日志查看**:查看多个 topic 的日志记录
|
||||
- **统计信息**:实时显示每个 topic 的统计数据
|
||||
- **日志查询**:支持查询最早、最新的日志记录
|
||||
- **状态追踪**:显示日志的处理状态(待处理、处理中、已处理)
|
||||
- **灵活集成**:可以集成到现有的 HTTP 服务器或其他框架
|
||||
|
||||
## 运行示例
|
||||
|
||||
```bash
|
||||
cd example/webui
|
||||
go run main.go
|
||||
```
|
||||
|
||||
然后在浏览器中访问 http://localhost:8080
|
||||
|
||||
## API 端点
|
||||
|
||||
Web UI 提供以下 API 端点:
|
||||
|
||||
- `GET /api/topics` - 获取所有 topic 列表
|
||||
- `GET /api/logs?topic=xxx&count=N` - 查询最新的 N 条日志(从最后一条向前查询)
|
||||
- `GET /api/logs/first?topic=xxx&count=N` - 查询从第一条开始的 N 条日志(从索引 0 向后查询)
|
||||
- `GET /api/logs/last?topic=xxx&count=N` - 查询最后的 N 条日志(从最后一条向前查询)
|
||||
- `GET /api/stats?topic=xxx` - 获取指定 topic 的统计信息
|
||||
|
||||
## 使用方式
|
||||
|
||||
### 方式 1:独立 Web UI 服务器
|
||||
|
||||
最简单的方式,直接启动一个独立的 Web UI 服务器:
|
||||
|
||||
```go
|
||||
import "code.tczkiot.com/seqlog"
|
||||
|
||||
// 创建 LogHub
|
||||
hub := seqlog.NewLogHub("./logs", logger, handler)
|
||||
|
||||
// 启动服务
|
||||
hub.Start()
|
||||
|
||||
// 启动 Web UI(会阻塞)
|
||||
hub.ServeUI(":8080")
|
||||
```
|
||||
|
||||
如果需要在后台运行 Web UI:
|
||||
|
||||
```go
|
||||
go func() {
|
||||
if err := hub.ServeUI(":8080"); err != nil {
|
||||
logger.Error("Web UI 错误", "error", err)
|
||||
}
|
||||
}()
|
||||
```
|
||||
|
||||
### 方式 2:集成到现有 HTTP 服务器
|
||||
|
||||
如果你已经有一个 HTTP 服务器,可以将 Web UI 集成到现有的 ServeMux:
|
||||
|
||||
```go
|
||||
import (
|
||||
"net/http"
|
||||
"code.tczkiot.com/seqlog"
|
||||
)
|
||||
|
||||
// 创建 LogHub
|
||||
hub := seqlog.NewLogHub("./logs", logger, handler)
|
||||
hub.Start()
|
||||
|
||||
// 创建自己的 ServeMux
|
||||
mux := http.NewServeMux()
|
||||
|
||||
// 注册业务端点
|
||||
mux.HandleFunc("/api/users", handleUsers)
|
||||
mux.HandleFunc("/health", handleHealth)
|
||||
|
||||
// 注册 seqlog Web UI 到根路径
|
||||
hub.RegisterWebUIRoutes(mux)
|
||||
|
||||
// 启动服务器
|
||||
http.ListenAndServe(":8080", mux)
|
||||
```
|
||||
|
||||
### 方式 3:集成到子路径
|
||||
|
||||
如果你想将 Web UI 放在子路径下(比如 `/logs/`):
|
||||
|
||||
```go
|
||||
// 创建主 ServeMux
|
||||
mux := http.NewServeMux()
|
||||
|
||||
// 注册业务端点
|
||||
mux.HandleFunc("/health", handleHealth)
|
||||
|
||||
// 创建 Web UI 的 ServeMux
|
||||
logsMux := http.NewServeMux()
|
||||
hub.RegisterWebUIRoutes(logsMux)
|
||||
|
||||
// 挂载到 /logs/ 路径
|
||||
mux.Handle("/logs/", http.StripPrefix("/logs", logsMux))
|
||||
|
||||
// 启动服务器
|
||||
http.ListenAndServe(":8080", mux)
|
||||
```
|
||||
|
||||
访问 http://localhost:8080/logs/ 查看 Web UI。
|
||||
|
||||
完整的集成示例请参考 [example/webui_integration](../webui_integration/main.go)。
|
||||
|
||||
### 方式 4:集成到其他 Web 框架
|
||||
|
||||
对于 gin、echo 等框架,可以通过适配器集成:
|
||||
|
||||
```go
|
||||
// Gin 框架示例
|
||||
import (
|
||||
"github.com/gin-gonic/gin"
|
||||
"net/http"
|
||||
)
|
||||
|
||||
r := gin.Default()
|
||||
|
||||
// 业务路由
|
||||
r.GET("/api/users", handleUsers)
|
||||
|
||||
// 创建 seqlog Web UI 的 ServeMux
|
||||
logsMux := http.NewServeMux()
|
||||
hub.RegisterWebUIRoutes(logsMux)
|
||||
|
||||
// 使用 gin.WrapH 包装 http.Handler
|
||||
r.Any("/logs/*path", gin.WrapH(http.StripPrefix("/logs", logsMux)))
|
||||
|
||||
r.Run(":8080")
|
||||
```
|
||||
114
example/webui/main.go
Normal file
114
example/webui/main.go
Normal file
@@ -0,0 +1,114 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"math/rand"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
|
||||
"code.tczkiot.com/seqlog"
|
||||
)
|
||||
|
||||
func main() {
|
||||
// 创建日志目录
|
||||
baseDir := "./logs"
|
||||
if err := os.MkdirAll(baseDir, 0755); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
// 创建 LogHub
|
||||
logger := slog.New(slog.NewTextHandler(os.Stdout, &slog.HandlerOptions{
|
||||
Level: slog.LevelInfo,
|
||||
}))
|
||||
|
||||
// 自定义处理器:打印处理的记录
|
||||
handler := func(topic string, record *seqlog.Record) error {
|
||||
previewSize := min(int(record.Len), 100)
|
||||
validPreviewSize := previewSize
|
||||
if previewSize > 0 && previewSize < int(record.Len) {
|
||||
// 只有在截断的情况下才需要检查
|
||||
// 从后往前最多检查 3 个字节,找到最后一个完整的 UTF-8 字符边界
|
||||
for i := 0; i < 3 && validPreviewSize > 0; i++ {
|
||||
if utf8.Valid(record.Data[:validPreviewSize]) {
|
||||
break
|
||||
}
|
||||
validPreviewSize--
|
||||
}
|
||||
}
|
||||
|
||||
fmt.Printf("[%s] 处理记录: %s\n", topic, string(record.Data[:validPreviewSize]))
|
||||
return nil
|
||||
}
|
||||
|
||||
hub := seqlog.NewLogHub(baseDir, logger, handler)
|
||||
|
||||
// 启动 LogHub(会自动发现和启动所有 topic)
|
||||
if err := hub.Start(); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
defer hub.Stop()
|
||||
|
||||
// topic 列表(会在第一次写入时自动创建)
|
||||
topics := []string{"app", "system", "access"}
|
||||
|
||||
// 在后台启动 Web UI 服务器
|
||||
go func() {
|
||||
fmt.Println("启动 Web UI 服务器: http://localhost:8080")
|
||||
if err := hub.ServeUI(":8080"); err != nil {
|
||||
fmt.Printf("Web UI 服务器错误: %v\n", err)
|
||||
}
|
||||
}()
|
||||
|
||||
// 生成随机大小的数据(2KB 到 10MB)
|
||||
generateRandomData := func(minSize, maxSize int) []byte {
|
||||
size := minSize + rand.Intn(maxSize-minSize)
|
||||
// 使用重复字符填充,模拟实际日志内容
|
||||
return []byte(strings.Repeat("X", size))
|
||||
}
|
||||
|
||||
// 启动多个并发写入器(提高并发数)
|
||||
var wg sync.WaitGroup
|
||||
concurrentWriters := 10 // 10 个并发写入器
|
||||
|
||||
for i := range concurrentWriters {
|
||||
wg.Add(1)
|
||||
go func(writerID int) {
|
||||
defer wg.Done()
|
||||
count := 0
|
||||
ticker := time.NewTicker(100 * time.Millisecond)
|
||||
defer ticker.Stop()
|
||||
|
||||
for range ticker.C {
|
||||
count++
|
||||
// 随机选择 topic
|
||||
topic := topics[rand.Intn(len(topics))]
|
||||
|
||||
// 生成随机大小的数据(2KB 到 2MB)
|
||||
minSize := 2 * 1024 // 2KB
|
||||
maxSize := 2 * 1024 * 1024 // 2MB
|
||||
randomData := generateRandomData(minSize, maxSize)
|
||||
|
||||
// 组合消息头和随机数据
|
||||
message := fmt.Sprintf("[Writer-%d] 日志 #%d - %s - 大小: %d bytes\n",
|
||||
writerID, count, time.Now().Format(time.RFC3339), len(randomData))
|
||||
fullData := append([]byte(message), randomData...)
|
||||
|
||||
if _, err := hub.Write(topic, fullData); err != nil {
|
||||
fmt.Printf("写入失败: %v\n", err)
|
||||
}
|
||||
}
|
||||
}(i)
|
||||
}
|
||||
|
||||
// 等待用户中断
|
||||
fmt.Println("\n==== Seqlog Web UI 示例 ====")
|
||||
fmt.Println("访问 http://localhost:8080 查看 Web UI")
|
||||
fmt.Println("按 Ctrl+C 退出")
|
||||
|
||||
// 阻塞主线程
|
||||
select {}
|
||||
}
|
||||
91
example/webui_integration/main.go
Normal file
91
example/webui_integration/main.go
Normal file
@@ -0,0 +1,91 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"code.tczkiot.com/seqlog"
|
||||
)
|
||||
|
||||
func main() {
|
||||
// 创建日志目录
|
||||
baseDir := "./logs"
|
||||
if err := os.MkdirAll(baseDir, 0755); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
// 创建 LogHub
|
||||
logger := slog.New(slog.NewTextHandler(os.Stdout, &slog.HandlerOptions{
|
||||
Level: slog.LevelInfo,
|
||||
}))
|
||||
|
||||
handler := func(topic string, record *seqlog.Record) error {
|
||||
fmt.Printf("[%s] 处理记录: %s\n", topic, string(record.Data))
|
||||
return nil
|
||||
}
|
||||
|
||||
hub := seqlog.NewLogHub(baseDir, logger, handler)
|
||||
|
||||
// 启动 LogHub
|
||||
if err := hub.Start(); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
defer hub.Stop()
|
||||
|
||||
// 创建自己的 ServeMux
|
||||
mux := http.NewServeMux()
|
||||
|
||||
// 注册自己的业务端点
|
||||
mux.HandleFunc("/health", func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
fmt.Fprintf(w, `{"status":"ok","service":"my-app"}`)
|
||||
})
|
||||
|
||||
mux.HandleFunc("/metrics", func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "text/plain")
|
||||
fmt.Fprintf(w, "# HELP my_app_requests_total Total requests\n")
|
||||
fmt.Fprintf(w, "my_app_requests_total 12345\n")
|
||||
})
|
||||
|
||||
// 在 /logs 路径下集成 seqlog Web UI
|
||||
// 方法 1:使用子路径(需要创建一个包装 ServeMux)
|
||||
logsMux := http.NewServeMux()
|
||||
if err := hub.RegisterWebUIRoutes(logsMux); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
mux.Handle("/logs/", http.StripPrefix("/logs", logsMux))
|
||||
|
||||
// 启动模拟写日志
|
||||
go func() {
|
||||
topics := []string{"app", "system", "access"}
|
||||
count := 0
|
||||
ticker := time.NewTicker(500 * time.Millisecond)
|
||||
defer ticker.Stop()
|
||||
|
||||
for range ticker.C {
|
||||
count++
|
||||
topic := topics[count%len(topics)]
|
||||
message := fmt.Sprintf("日志消息 %d - %s", count, time.Now().Format(time.RFC3339))
|
||||
|
||||
if _, err := hub.Write(topic, []byte(message)); err != nil {
|
||||
fmt.Printf("写入失败: %v\n", err)
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
// 启动服务器
|
||||
fmt.Println("\n==== Seqlog 集成示例 ====")
|
||||
fmt.Println("业务端点:")
|
||||
fmt.Println(" - http://localhost:8080/health")
|
||||
fmt.Println(" - http://localhost:8080/metrics")
|
||||
fmt.Println("Web UI:")
|
||||
fmt.Println(" - http://localhost:8080/logs/")
|
||||
fmt.Println("按 Ctrl+C 退出")
|
||||
|
||||
if err := http.ListenAndServe(":8080", mux); err != nil {
|
||||
fmt.Printf("服务器错误: %v\n", err)
|
||||
}
|
||||
}
|
||||
BIN
example/webui_integration/webui_integration
Executable file
BIN
example/webui_integration/webui_integration
Executable file
Binary file not shown.
Reference in New Issue
Block a user