bourdon
90cc9e21c9
主要更改： 1. 核心重命名 - Seqlog -> LogHub (更准确地反映其作为日志中枢的角色) - NewSeqlog() -> NewLogHub() - LogCursor -> ProcessCursor (更准确地反映其用于处理场景) - seqlog_manager.go -> loghub.go (文件名与结构体名对应) 2. TopicProcessor.Reset 增强 - 如果正在运行且没有待处理的日志，会自动停止后重置 - 如果有待处理的日志，返回详细错误（显示已处理/总记录数） - 简化了 LogHub.ResetTopic，移除显式 Stop 调用 3. 新增查询方法 - TopicProcessor.QueryFromFirst(count) - 从第一条记录向索引递增方向查询 - TopicProcessor.QueryFromLast(count) - 从最后一条记录向索引递减方向查询 - LogHub.QueryFromFirst(topic, count) - LogHub.QueryFromLast(topic, count) 4. 测试覆盖 - 添加 query_test.go - QueryFromProcessing 测试 - 添加 TestQueryFromFirstAndLast - TopicProcessor 查询测试 - 添加 TestLogHubQueryFromFirstAndLast - LogHub 查询测试 - 添加 TestTopicResetWithPendingRecords - Reset 增强功能测试 5. 示例代码 - 添加 example/get_record/ - 演示 QueryFromProcessing 用法 - 更新所有示例以使用 LogHub 和新 API 所有测试通过 ✅ 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com>
649 lines
18 KiB
Go
649 lines
18 KiB
Go
package main
|
||
|
||
import (
|
||
"encoding/json"
|
||
"fmt"
|
||
"log/slog"
|
||
"math/rand"
|
||
"net/http"
|
||
"os"
|
||
"strconv"
|
||
"time"
|
||
|
||
"code.tczkiot.com/seqlog"
|
||
)
|
||
|
||
var (
|
||
seq *seqlog.LogHub
|
||
logger *slog.Logger
|
||
)
|
||
|
||
func main() {
|
||
// 初始化
|
||
logger = slog.New(slog.NewTextHandler(os.Stdout, &slog.HandlerOptions{
|
||
Level: slog.LevelInfo,
|
||
}))
|
||
|
||
// 创建 Seqlog
|
||
seq = seqlog.NewLogHub("logs", logger, func(topic string, rec *seqlog.Record) error {
|
||
// 简单的日志处理：只打印摘要信息
|
||
dataPreview := string(rec.Data)
|
||
if len(dataPreview) > 100 {
|
||
dataPreview = dataPreview[:100] + "..."
|
||
}
|
||
logger.Info("处理日志", "topic", topic, "size", len(rec.Data), "preview", dataPreview)
|
||
return nil
|
||
})
|
||
|
||
if err := seq.Start(); err != nil {
|
||
logger.Error("启动失败", "error", err)
|
||
os.Exit(1)
|
||
}
|
||
defer seq.Stop()
|
||
|
||
logger.Info("Seqlog 已启动")
|
||
|
||
// 启动后台业务模拟
|
||
go simulateBusiness()
|
||
|
||
// 启动 Web 服务器
|
||
http.HandleFunc("/", handleIndex)
|
||
http.HandleFunc("/api/topics", handleTopics)
|
||
http.HandleFunc("/api/stats", handleStats)
|
||
http.HandleFunc("/api/query", handleQuery)
|
||
http.HandleFunc("/api/write", handleWrite)
|
||
|
||
addr := ":8080"
|
||
logger.Info("Web 服务器启动", "地址", "http://localhost"+addr)
|
||
if err := http.ListenAndServe(addr, nil); err != nil {
|
||
logger.Error("服务器错误", "error", err)
|
||
}
|
||
}
|
||
|
||
// 模拟业务写日志
|
||
func simulateBusiness() {
|
||
topics := []string{"app", "api", "database", "cache"}
|
||
actions := []string{"查询", "插入", "更新", "删除", "连接", "断开", "备份", "恢复", "同步"}
|
||
status := []string{"成功", "失败", "超时", "重试"}
|
||
|
||
ticker := time.NewTicker(2 * time.Second)
|
||
defer ticker.Stop()
|
||
|
||
for range ticker.C {
|
||
// 随机选择 topic 和内容
|
||
topic := topics[rand.Intn(len(topics))]
|
||
action := actions[rand.Intn(len(actions))]
|
||
st := status[rand.Intn(len(status))]
|
||
|
||
// 随机生成日志大小：2KB 到 10MB
|
||
// 80% 概率生成小日志（2KB-100KB）
|
||
// 15% 概率生成中日志（100KB-1MB）
|
||
// 5% 概率生成大日志（1MB-10MB）
|
||
var logSize int
|
||
prob := rand.Intn(100)
|
||
if prob < 80 {
|
||
// 2KB - 100KB
|
||
logSize = 2*1024 + rand.Intn(98*1024)
|
||
} else if prob < 95 {
|
||
// 100KB - 1MB
|
||
logSize = 100*1024 + rand.Intn(924*1024)
|
||
} else {
|
||
// 1MB - 10MB
|
||
logSize = 1024*1024 + rand.Intn(9*1024*1024)
|
||
}
|
||
|
||
// 生成日志内容
|
||
header := fmt.Sprintf("[%s] %s %s - 用时: %dms | 数据大小: %s | ",
|
||
time.Now().Format("15:04:05"),
|
||
action,
|
||
st,
|
||
rand.Intn(1000),
|
||
formatBytes(int64(logSize)))
|
||
|
||
// 填充随机数据到指定大小
|
||
data := make([]byte, logSize)
|
||
copy(data, []byte(header))
|
||
|
||
// 填充可读的模拟数据
|
||
fillOffset := len(header)
|
||
patterns := []string{
|
||
"user_id=%d, session=%x, ip=%d.%d.%d.%d, ",
|
||
"query_time=%dms, rows=%d, cached=%v, ",
|
||
"error_code=%d, retry_count=%d, ",
|
||
"request_id=%x, trace_id=%x, ",
|
||
}
|
||
|
||
for fillOffset < logSize-100 {
|
||
pattern := patterns[rand.Intn(len(patterns))]
|
||
var chunk string
|
||
switch pattern {
|
||
case patterns[0]:
|
||
chunk = fmt.Sprintf(pattern, rand.Intn(10000), rand.Intn(0xFFFFFF),
|
||
rand.Intn(256), rand.Intn(256), rand.Intn(256), rand.Intn(256))
|
||
case patterns[1]:
|
||
chunk = fmt.Sprintf(pattern, rand.Intn(1000), rand.Intn(10000), rand.Intn(2) == 1)
|
||
case patterns[2]:
|
||
chunk = fmt.Sprintf(pattern, rand.Intn(500), rand.Intn(5))
|
||
case patterns[3]:
|
||
chunk = fmt.Sprintf(pattern, rand.Intn(0xFFFFFFFF), rand.Intn(0xFFFFFFFF))
|
||
}
|
||
|
||
remaining := logSize - fillOffset
|
||
if len(chunk) > remaining {
|
||
chunk = chunk[:remaining]
|
||
}
|
||
copy(data[fillOffset:], []byte(chunk))
|
||
fillOffset += len(chunk)
|
||
}
|
||
|
||
// 写入日志
|
||
if _, err := seq.Write(topic, data); err != nil {
|
||
logger.Error("写入日志失败", "error", err, "size", logSize)
|
||
} else {
|
||
logger.Info("写入日志", "topic", topic, "size", formatBytes(int64(logSize)))
|
||
}
|
||
}
|
||
}
|
||
|
||
func formatBytes(bytes int64) string {
|
||
if bytes < 1024 {
|
||
return fmt.Sprintf("%d B", bytes)
|
||
}
|
||
if bytes < 1024*1024 {
|
||
return fmt.Sprintf("%.1f KB", float64(bytes)/1024)
|
||
}
|
||
return fmt.Sprintf("%.2f MB", float64(bytes)/1024/1024)
|
||
}
|
||
|
||
type Record struct {
|
||
Index int `json:"index"`
|
||
Status string `json:"status"`
|
||
Data string `json:"data"`
|
||
}
|
||
|
||
// 首页
|
||
func handleIndex(w http.ResponseWriter, r *http.Request) {
|
||
html := `<!DOCTYPE html>
|
||
<html>
|
||
<head>
|
||
<meta charset="UTF-8">
|
||
<title>Seqlog 日志查询</title>
|
||
<style>
|
||
body {
|
||
font-family: 'Segoe UI', Tahoma, Geneva, Verdana, sans-serif;
|
||
max-width: 1400px;
|
||
margin: 0 auto;
|
||
padding: 20px;
|
||
background: #f5f5f5;
|
||
}
|
||
.header {
|
||
background: white;
|
||
padding: 20px;
|
||
border-radius: 8px;
|
||
margin-bottom: 20px;
|
||
box-shadow: 0 2px 4px rgba(0,0,0,0.1);
|
||
}
|
||
h1 {
|
||
margin: 0;
|
||
color: #333;
|
||
}
|
||
.subtitle {
|
||
color: #666;
|
||
margin-top: 5px;
|
||
}
|
||
.container {
|
||
display: grid;
|
||
grid-template-columns: 250px 1fr;
|
||
gap: 20px;
|
||
}
|
||
.sidebar {
|
||
background: white;
|
||
padding: 20px;
|
||
border-radius: 8px;
|
||
box-shadow: 0 2px 4px rgba(0,0,0,0.1);
|
||
}
|
||
.main {
|
||
background: white;
|
||
padding: 20px;
|
||
border-radius: 8px;
|
||
box-shadow: 0 2px 4px rgba(0,0,0,0.1);
|
||
}
|
||
.topic-list {
|
||
list-style: none;
|
||
padding: 0;
|
||
margin: 0;
|
||
}
|
||
.topic-item {
|
||
padding: 10px;
|
||
margin-bottom: 5px;
|
||
cursor: pointer;
|
||
border-radius: 4px;
|
||
transition: background 0.2s;
|
||
}
|
||
.topic-item:hover {
|
||
background: #f0f0f0;
|
||
}
|
||
.topic-item.active {
|
||
background: #007bff;
|
||
color: white;
|
||
}
|
||
.stats {
|
||
margin-top: 20px;
|
||
padding: 15px;
|
||
background: #f8f9fa;
|
||
border-radius: 4px;
|
||
}
|
||
.stat-item {
|
||
display: flex;
|
||
justify-content: space-between;
|
||
margin-bottom: 8px;
|
||
font-size: 14px;
|
||
}
|
||
.controls {
|
||
margin-bottom: 20px;
|
||
}
|
||
.btn {
|
||
padding: 8px 16px;
|
||
margin-right: 10px;
|
||
border: none;
|
||
border-radius: 4px;
|
||
cursor: pointer;
|
||
font-size: 14px;
|
||
}
|
||
.btn-primary {
|
||
background: #007bff;
|
||
color: white;
|
||
}
|
||
.btn-secondary {
|
||
background: #6c757d;
|
||
color: white;
|
||
}
|
||
.log-container {
|
||
height: 500px;
|
||
overflow-y: auto;
|
||
border: 1px solid #ddd;
|
||
border-radius: 4px;
|
||
padding: 10px;
|
||
background: #f8f9fa;
|
||
font-family: 'Courier New', monospace;
|
||
font-size: 13px;
|
||
}
|
||
.log-entry {
|
||
padding: 6px 10px;
|
||
margin-bottom: 4px;
|
||
background: white;
|
||
border-left: 3px solid #007bff;
|
||
border-radius: 2px;
|
||
word-wrap: break-word;
|
||
overflow-wrap: break-word;
|
||
display: -webkit-box;
|
||
-webkit-line-clamp: 3;
|
||
-webkit-box-orient: vertical;
|
||
overflow: hidden;
|
||
line-height: 1.5;
|
||
}
|
||
.log-entry.processed {
|
||
border-left-color: #28a745;
|
||
opacity: 0.8;
|
||
}
|
||
.log-entry.processing {
|
||
border-left-color: #ffc107;
|
||
background: #fff9e6;
|
||
}
|
||
.log-entry.pending {
|
||
border-left-color: #6c757d;
|
||
opacity: 0.6;
|
||
}
|
||
.status-badge {
|
||
display: inline-block;
|
||
padding: 2px 8px;
|
||
border-radius: 3px;
|
||
font-size: 11px;
|
||
margin-right: 8px;
|
||
}
|
||
.status-processed {
|
||
background: #d4edda;
|
||
color: #155724;
|
||
}
|
||
.status-processing {
|
||
background: #fff3cd;
|
||
color: #856404;
|
||
}
|
||
.status-pending {
|
||
background: #e2e3e5;
|
||
color: #383d41;
|
||
}
|
||
.loading {
|
||
text-align: center;
|
||
padding: 20px;
|
||
color: #666;
|
||
}
|
||
</style>
|
||
</head>
|
||
<body>
|
||
<div class="header">
|
||
<h1>Seqlog 日志查询系统</h1>
|
||
<div class="subtitle">实时查看和管理应用日志</div>
|
||
</div>
|
||
|
||
<div class="container">
|
||
<div class="sidebar">
|
||
<h3>Topics</h3>
|
||
<ul class="topic-list" id="topicList"></ul>
|
||
|
||
<div class="stats" id="stats">
|
||
<h4>统计信息</h4>
|
||
<div id="statsContent">选择一个 topic 查看统计</div>
|
||
</div>
|
||
</div>
|
||
|
||
<div class="main">
|
||
<div class="controls">
|
||
<button class="btn btn-primary" onclick="loadLogs()">刷新日志</button>
|
||
<button class="btn btn-secondary" onclick="queryBackward()">向前翻页</button>
|
||
<button class="btn btn-secondary" onclick="queryForward()">向后翻页</button>
|
||
<span style="margin-left: 20px;">显示范围: 前 <input type="number" id="backwardCount" value="10" style="width: 60px;"> 条, 后 <input type="number" id="forwardCount" value="10" style="width: 60px;"> 条</span>
|
||
</div>
|
||
|
||
<div class="log-container" id="logContainer">
|
||
<div class="loading">选择一个 topic 开始查看日志</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
|
||
<script>
|
||
let currentTopic = null;
|
||
let currentCenterIndex = null; // 追踪当前中心索引位置
|
||
let direction = 'forward'; // 默认方向为向前翻页
|
||
let startIndex = null;
|
||
let endIndex = null;
|
||
|
||
// 加载 topics
|
||
async function loadTopics() {
|
||
const response = await fetch('/api/topics');
|
||
const topics = await response.json();
|
||
|
||
const list = document.getElementById('topicList');
|
||
list.innerHTML = topics.sort().map(topic =>
|
||
'<li class="topic-item" onclick="selectTopic(\'' + topic + '\')">' + topic + '</li>'
|
||
).join('');
|
||
}
|
||
|
||
// 选择 topic
|
||
function selectTopic(topic) {
|
||
currentTopic = topic;
|
||
direction = ''; // 切换 topic 时重置方向
|
||
startIndex = null;
|
||
endIndex = null;
|
||
|
||
// 更新选中状态
|
||
document.querySelectorAll('.topic-item').forEach(item => {
|
||
item.classList.remove('active');
|
||
if (item.textContent === topic) {
|
||
item.classList.add('active');
|
||
}
|
||
});
|
||
|
||
// 清空容器并重新加载
|
||
document.getElementById('logContainer').innerHTML = '';
|
||
loadStats(topic);
|
||
loadLogs();
|
||
}
|
||
|
||
// 加载统计
|
||
async function loadStats(topic) {
|
||
const response = await fetch('/api/stats?topic=' + topic);
|
||
const stats = await response.json();
|
||
|
||
const content = document.getElementById('statsContent');
|
||
content.innerHTML =
|
||
'<div class="stat-item"><span>写入:</span><span>' + stats.write_count + ' 条</span></div>' +
|
||
'<div class="stat-item"><span>处理:</span><span>' + stats.processed_count + ' 条</span></div>' +
|
||
'<div class="stat-item"><span>错误:</span><span>' + stats.error_count + ' 次</span></div>' +
|
||
'<div class="stat-item"><span>大小:</span><span>' + formatBytes(stats.write_bytes) + '</span></div>';
|
||
}
|
||
|
||
// 加载日志
|
||
async function loadLogs() {
|
||
if (!currentTopic) return;
|
||
|
||
const backward = document.getElementById('backwardCount').value;
|
||
const forward = document.getElementById('forwardCount').value;
|
||
|
||
// 构建查询 URL
|
||
let url = '/api/query?topic=' + currentTopic;
|
||
|
||
if (direction === 'backward' && startIndex != null) {
|
||
url += '&direction=backward&index=' + startIndex + '&count=' + backward;
|
||
} else if (direction === 'forward' && endIndex != null) {
|
||
url += '&direction=forward&index=' + endIndex + '&count=' + forward;
|
||
} else {
|
||
url += '&count=10';
|
||
}
|
||
|
||
const response = await fetch(url);
|
||
const data = await response.json();
|
||
|
||
const container = document.getElementById('logContainer');
|
||
|
||
if (data.records.length === 0) {
|
||
container.innerHTML = '<div class="loading">暂无日志</div>';
|
||
return;
|
||
}
|
||
|
||
// 清空并重新渲染所有记录
|
||
const html = data.records.map(r => {
|
||
// 解析状态
|
||
let statusClass = 'pending';
|
||
let statusText = '待处理';
|
||
let badgeClass = 'status-pending';
|
||
|
||
if (r.status === 'StatusProcessed' || r.status === 'processed') {
|
||
statusClass = 'processed';
|
||
statusText = '已处理';
|
||
badgeClass = 'status-processed';
|
||
} else if (r.status === 'StatusProcessing' || r.status === 'processing') {
|
||
statusClass = 'processing';
|
||
statusText = '处理中';
|
||
badgeClass = 'status-processing';
|
||
}
|
||
|
||
return '<div class="log-entry ' + statusClass + '" data-index="' + r.index + '">' +
|
||
'<span class="status-badge ' + badgeClass + '">[#' + r.index + '] ' + statusText + '</span>' +
|
||
r.data +
|
||
'</div>';
|
||
}).join('');
|
||
|
||
if (data.records.length > 0) {
|
||
if (startIndex === null) {
|
||
startIndex = data.records[0].index;
|
||
} else {
|
||
startIndex = Math.min(startIndex, data.records[0].index);
|
||
}
|
||
if (endIndex === null) {
|
||
endIndex = data.records[data.records.length - 1].index;
|
||
} else {
|
||
endIndex = Math.max(endIndex, data.records[data.records.length - 1].index);
|
||
}
|
||
}
|
||
|
||
container.innerHTML = html;
|
||
}
|
||
|
||
function queryBackward() {
|
||
direction = 'backward';
|
||
loadLogs();
|
||
}
|
||
|
||
function queryForward() {
|
||
direction = 'forward';
|
||
loadLogs();
|
||
}
|
||
|
||
function formatBytes(bytes) {
|
||
if (bytes < 1024) return bytes + ' B';
|
||
if (bytes < 1024 * 1024) return (bytes / 1024).toFixed(1) + ' KB';
|
||
return (bytes / 1024 / 1024).toFixed(1) + ' MB';
|
||
}
|
||
|
||
// 初始化
|
||
loadTopics();
|
||
|
||
// 只自动刷新统计信息，不刷新日志
|
||
setInterval(() => {
|
||
if (currentTopic) {
|
||
loadStats(currentTopic);
|
||
}
|
||
}, 3000); // 每 3 秒刷新统计
|
||
</script>
|
||
</body>
|
||
</html>`
|
||
|
||
w.Header().Set("Content-Type", "text/html; charset=utf-8")
|
||
fmt.Fprint(w, html)
|
||
}
|
||
|
||
// API: 获取所有 topics
|
||
func handleTopics(w http.ResponseWriter, r *http.Request) {
|
||
topics := seq.GetTopics()
|
||
json.NewEncoder(w).Encode(topics)
|
||
}
|
||
|
||
// API: 获取统计信息
|
||
func handleStats(w http.ResponseWriter, r *http.Request) {
|
||
topic := r.URL.Query().Get("topic")
|
||
if topic == "" {
|
||
http.Error(w, "缺少 topic 参数", http.StatusBadRequest)
|
||
return
|
||
}
|
||
|
||
stats, err := seq.GetTopicStats(topic)
|
||
if err != nil {
|
||
http.Error(w, err.Error(), http.StatusNotFound)
|
||
return
|
||
}
|
||
|
||
json.NewEncoder(w).Encode(stats)
|
||
}
|
||
|
||
// API: 查询日志
|
||
func handleQuery(w http.ResponseWriter, r *http.Request) {
|
||
topic := r.URL.Query().Get("topic")
|
||
if topic == "" {
|
||
http.Error(w, "缺少 topic 参数", http.StatusBadRequest)
|
||
return
|
||
}
|
||
|
||
// 获取查询参数
|
||
indexParam := r.URL.Query().Get("index")
|
||
direction := r.URL.Query().Get("direction")
|
||
count, _ := strconv.Atoi(r.URL.Query().Get("count"))
|
||
|
||
if count <= 0 {
|
||
count = 10
|
||
}
|
||
|
||
// 获取 processor
|
||
processor, err := seq.GetProcessor(topic)
|
||
if err != nil {
|
||
http.Error(w, err.Error(), http.StatusNotFound)
|
||
return
|
||
}
|
||
|
||
// 获取记录总数
|
||
totalCount := processor.GetRecordCount()
|
||
|
||
// 执行查询
|
||
var results []*seqlog.RecordWithStatus
|
||
|
||
if direction == "" {
|
||
results, err = processor.QueryFromProcessing(count)
|
||
if err != nil {
|
||
http.Error(w, err.Error(), http.StatusNotFound)
|
||
return
|
||
}
|
||
if len(results) == 0 {
|
||
results, err = processor.QueryFromLast(count)
|
||
if err != nil {
|
||
http.Error(w, err.Error(), http.StatusNotFound)
|
||
return
|
||
}
|
||
}
|
||
} else {
|
||
var refIndex int
|
||
if indexParam == "" {
|
||
http.Error(w, "参数错误", http.StatusNotFound)
|
||
return
|
||
} else {
|
||
refIndex, err = strconv.Atoi(indexParam)
|
||
if err != nil {
|
||
http.Error(w, err.Error(), http.StatusNotFound)
|
||
return
|
||
}
|
||
}
|
||
|
||
if direction == "backward" {
|
||
var queryErr error
|
||
results, queryErr = processor.QueryNewest(refIndex, count)
|
||
if queryErr != nil {
|
||
http.Error(w, queryErr.Error(), http.StatusInternalServerError)
|
||
return
|
||
}
|
||
} else if direction == "forward" {
|
||
var queryErr error
|
||
results, queryErr = processor.QueryOldest(refIndex, count)
|
||
if queryErr != nil {
|
||
http.Error(w, queryErr.Error(), http.StatusInternalServerError)
|
||
return
|
||
}
|
||
} else {
|
||
http.Error(w, "参数错误", http.StatusNotFound)
|
||
return
|
||
}
|
||
}
|
||
|
||
records := make([]Record, len(results))
|
||
for i, result := range results {
|
||
records[i] = Record{
|
||
Index: result.Index,
|
||
Status: result.Status.String(),
|
||
Data: string(result.Record.Data),
|
||
}
|
||
}
|
||
|
||
json.NewEncoder(w).Encode(map[string]interface{}{
|
||
"records": records,
|
||
"total": len(records),
|
||
"totalCount": totalCount,
|
||
})
|
||
}
|
||
|
||
// API: 手动写入日志
|
||
func handleWrite(w http.ResponseWriter, r *http.Request) {
|
||
if r.Method != "POST" {
|
||
http.Error(w, "只支持 POST", http.StatusMethodNotAllowed)
|
||
return
|
||
}
|
||
|
||
var req struct {
|
||
Topic string `json:"topic"`
|
||
Data string `json:"data"`
|
||
}
|
||
|
||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||
return
|
||
}
|
||
|
||
offset, err := seq.Write(req.Topic, []byte(req.Data))
|
||
if err != nil {
|
||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||
return
|
||
}
|
||
|
||
json.NewEncoder(w).Encode(map[string]interface{}{
|
||
"success": true,
|
||
"offset": offset,
|
||
})
|
||
}
|