Initial commit: Pipeline Database

This commit is contained in:
Pipeline Database
2025-09-30 15:05:56 +08:00
commit e81dcba515
44 changed files with 12972 additions and 0 deletions

View File

@@ -0,0 +1,237 @@
name: 性能基准测试
on:
push:
tags:
- 'v*'
workflow_dispatch:
inputs:
benchtime:
description: '基准测试运行时间'
required: false
default: '10s'
count:
description: '基准测试运行次数'
required: false
default: '5'
jobs:
benchmark:
name: 完整性能基准测试
runs-on: ubuntu-latest
steps:
- name: 检出代码
uses: actions/checkout@v4
- name: 设置 Go 环境
uses: actions/setup-go@v4
with:
go-version: 1.24.x
- name: 系统信息
run: |
echo "=== 系统信息 ==="
uname -a
cat /proc/cpuinfo | grep "model name" | head -1
cat /proc/meminfo | grep MemTotal
df -h /
- name: Go 环境信息
run: |
echo "=== Go 环境信息 ==="
go version
go env
- name: 预热系统
run: |
echo "=== 系统预热 ==="
go test -bench=BenchmarkPipelineDBOpen -benchtime=100ms -run=^$ ./...
- name: 数据库核心功能基准测试
run: |
echo "=== 数据库核心功能基准测试 ==="
go test -bench=BenchmarkPipelineDB -benchmem -count=${{ github.event.inputs.count || '5' }} -benchtime=${{ github.event.inputs.benchtime || '2s' }} -run=^$ ./... | tee core_bench.txt
- name: 并发性能基准测试
run: |
echo "=== 并发性能基准测试 ==="
go test -bench=BenchmarkConcurrent -benchmem -count=${{ github.event.inputs.count || '5' }} -benchtime=${{ github.event.inputs.benchtime || '2s' }} -run=^$ ./... | tee concurrent_bench.txt
- name: 存储引擎基准测试
run: |
echo "=== 存储引擎基准测试 ==="
go test -bench=BenchmarkStorage -benchmem -count=${{ github.event.inputs.count || '5' }} -benchtime=${{ github.event.inputs.benchtime || '2s' }} -run=^$ ./... | tee storage_bench.txt
- name: 缓存系统基准测试
run: |
echo "=== 缓存系统基准测试 ==="
go test -bench=BenchmarkCache -benchmem -count=${{ github.event.inputs.count || '5' }} -benchtime=${{ github.event.inputs.benchtime || '2s' }} -run=^$ ./... | tee cache_bench.txt
- name: 索引管理基准测试
run: |
echo "=== 索引管理基准测试 ==="
go test -bench=BenchmarkIndex -benchmem -count=${{ github.event.inputs.count || '5' }} -benchtime=${{ github.event.inputs.benchtime || '2s' }} -run=^$ ./... | tee index_bench.txt
- name: 组管理基准测试
run: |
echo "=== 组管理基准测试 ==="
go test -bench=BenchmarkGroup -benchmem -count=${{ github.event.inputs.count || '5' }} -benchtime=${{ github.event.inputs.benchtime || '2s' }} -run=^$ ./... | tee group_bench.txt
- name: Handler 处理器基准测试
run: |
echo "=== Handler 处理器基准测试 ==="
go test -bench=BenchmarkHandler -benchmem -count=${{ github.event.inputs.count || '5' }} -benchtime=${{ github.event.inputs.benchtime || '2s' }} -run=^$ ./... | tee handler_bench.txt
- name: 高并发压力测试
run: |
echo "=== 高并发压力测试 ==="
go test -bench=BenchmarkHighConcurrency -benchmem -count=3 -benchtime=5s -run=^$ ./... | tee high_concurrency_bench.txt
- name: 内存分析
run: |
echo "=== 内存分析 ==="
go test -bench=BenchmarkConcurrentAcceptData -benchmem -memprofile=mem.prof -run=^$ ./...
go tool pprof -text mem.prof | head -20
- name: CPU 分析
run: |
echo "=== CPU 分析 ==="
go test -bench=BenchmarkConcurrentAcceptData -cpuprofile=cpu.prof -run=^$ ./...
go tool pprof -text cpu.prof | head -20
- name: 生成性能报告
run: |
echo "=== 性能基准测试报告 ===" > benchmark_report.md
echo "**测试时间:** $(date)" >> benchmark_report.md
echo "**Go 版本:** $(go version)" >> benchmark_report.md
echo "**系统信息:** $(uname -a)" >> benchmark_report.md
echo "" >> benchmark_report.md
echo "## 📊 核心功能性能" >> benchmark_report.md
echo "\`\`\`" >> benchmark_report.md
cat core_bench.txt | grep "Benchmark" >> benchmark_report.md
echo "\`\`\`" >> benchmark_report.md
echo "" >> benchmark_report.md
echo "## 🚀 并发性能" >> benchmark_report.md
echo "\`\`\`" >> benchmark_report.md
cat concurrent_bench.txt | grep "Benchmark" >> benchmark_report.md
echo "\`\`\`" >> benchmark_report.md
echo "" >> benchmark_report.md
echo "## 💾 存储引擎性能" >> benchmark_report.md
echo "\`\`\`" >> benchmark_report.md
cat storage_bench.txt | grep "Benchmark" >> benchmark_report.md
echo "\`\`\`" >> benchmark_report.md
echo "" >> benchmark_report.md
echo "## ⚡ 缓存系统性能" >> benchmark_report.md
echo "\`\`\`" >> benchmark_report.md
cat cache_bench.txt | grep "Benchmark" >> benchmark_report.md
echo "\`\`\`" >> benchmark_report.md
echo "" >> benchmark_report.md
echo "## 🔍 索引管理性能" >> benchmark_report.md
echo "\`\`\`" >> benchmark_report.md
cat index_bench.txt | grep "Benchmark" >> benchmark_report.md
echo "\`\`\`" >> benchmark_report.md
echo "" >> benchmark_report.md
echo "## 📁 组管理性能" >> benchmark_report.md
echo "\`\`\`" >> benchmark_report.md
cat group_bench.txt | grep "Benchmark" >> benchmark_report.md
echo "\`\`\`" >> benchmark_report.md
echo "" >> benchmark_report.md
echo "## 🔧 Handler 处理器性能" >> benchmark_report.md
echo "\`\`\`" >> benchmark_report.md
cat handler_bench.txt | grep "Benchmark" >> benchmark_report.md
echo "\`\`\`" >> benchmark_report.md
echo "" >> benchmark_report.md
echo "## 💥 高并发压力测试" >> benchmark_report.md
echo "\`\`\`" >> benchmark_report.md
cat high_concurrency_bench.txt | grep "Benchmark" >> benchmark_report.md
echo "\`\`\`" >> benchmark_report.md
cat benchmark_report.md
- name: 性能趋势分析
run: |
echo "=== 性能趋势分析 ==="
# 提取关键性能指标
ACCEPT_DATA_QPS=$(cat core_bench.txt | grep "BenchmarkAcceptData" | awk '{print $3}' | tail -1)
QUERY_QPS=$(cat concurrent_bench.txt | grep "BenchmarkConcurrentQuery" | awk '{print $3}' | tail -1)
CACHE_OPS=$(cat cache_bench.txt | grep "BenchmarkCacheOperations" | awk '{print $3}' | tail -1)
echo "关键性能指标:"
echo "- AcceptData QPS: $ACCEPT_DATA_QPS"
echo "- 并发查询 QPS: $QUERY_QPS"
echo "- 缓存操作 QPS: $CACHE_OPS"
# 性能阈值检查
if [ ! -z "$ACCEPT_DATA_QPS" ] && [ "$ACCEPT_DATA_QPS" -lt 1000 ]; then
echo "⚠️ AcceptData 性能低于预期阈值 (1000 ops/sec)"
fi
if [ ! -z "$CACHE_OPS" ] && [ "$CACHE_OPS" -lt 5000000 ]; then
echo "⚠️ 缓存操作性能低于预期阈值 (5M ops/sec)"
fi
- name: 上传基准测试结果
uses: actions/upload-artifact@v3
with:
name: benchmark-results-${{ github.run_number }}
path: |
*_bench.txt
benchmark_report.md
*.prof
- name: 上传性能分析文件
uses: actions/upload-artifact@v3
with:
name: performance-profiles-${{ github.run_number }}
path: |
mem.prof
cpu.prof
stress_test:
name: 压力测试
runs-on: ubuntu-latest
needs: benchmark
steps:
- name: 检出代码
uses: actions/checkout@v4
- name: 设置 Go 环境
uses: actions/setup-go@v4
with:
go-version: 1.24.x
- name: 长时间压力测试
timeout-minutes: 30
run: |
echo "=== 长时间压力测试 (20分钟) ==="
timeout 1200s go test -bench=BenchmarkHighConcurrencyWrite -benchtime=20m -run=^$ ./... || true
- name: 内存泄漏检测
run: |
echo "=== 内存泄漏检测 ==="
go test -bench=BenchmarkConcurrentAcceptData -benchtime=30s -memprofile=leak_test.prof -run=^$ ./...
go tool pprof -alloc_space -text leak_test.prof | head -20
- name: 竞态条件检测
run: |
echo "=== 竞态条件检测 ==="
go test -race -bench=BenchmarkConcurrent -benchtime=10s -run=^$ ./...
- name: 上传压力测试结果
uses: actions/upload-artifact@v3
if: always()
with:
name: stress-test-results-${{ github.run_number }}
path: |
leak_test.prof

272
.gitea/workflows/ci.yml Normal file
View File

@@ -0,0 +1,272 @@
name: CI Pipeline
on:
push:
branches: [ main, develop ]
pull_request:
branches: [ main, develop ]
jobs:
test:
name: 测试和基准测试
runs-on: ubuntu-latest
strategy:
matrix:
go-version: [1.24.x, 1.25.x]
steps:
- name: 检出代码
uses: actions/checkout@v4
- name: 设置 Go 环境
uses: actions/setup-go@v4
with:
go-version: ${{ matrix.go-version }}
- name: 验证 Go 模块
run: |
go mod tidy
git diff --exit-code go.mod go.sum
- name: 下载依赖
run: go mod download
- name: 代码格式检查
run: |
if [ "$(gofmt -s -l . | wc -l)" -gt 0 ]; then
echo "以下文件需要格式化:"
gofmt -s -l .
exit 1
fi
- name: 静态代码检查
run: |
go vet ./...
- name: 运行单元测试
run: |
go test -v -race -coverprofile=coverage.out ./...
- name: 生成测试覆盖率报告
run: |
go tool cover -html=coverage.out -o coverage.html
go tool cover -func=coverage.out
- name: 运行基准测试
run: |
echo "=== 基础性能基准测试 ==="
go test -bench=BenchmarkPipelineDB -benchmem -benchtime=1s -run=^$ ./...
echo "=== 并发性能基准测试 ==="
go test -bench=BenchmarkConcurrent -benchmem -benchtime=500ms -run=^$ ./...
echo "=== 存储引擎基准测试 ==="
go test -bench=BenchmarkStorage -benchmem -benchtime=500ms -run=^$ ./...
echo "=== 缓存系统基准测试 ==="
go test -bench=BenchmarkCache -benchmem -benchtime=500ms -run=^$ ./...
echo "=== Handler 基准测试 ==="
go test -bench=BenchmarkHandler -benchmem -benchtime=500ms -run=^$ ./...
- name: 运行示例程序
run: |
echo "=== 测试基础使用示例 ==="
cd examples/basic-usage && timeout 10s go run main.go || true
echo "=== 测试组管理示例 ==="
cd ../group-management && timeout 10s go run main.go || true
echo "=== 测试外部处理器示例 ==="
cd ../external-handler && timeout 10s go run main.go || true
- name: 上传测试覆盖率
uses: actions/upload-artifact@v3
with:
name: coverage-report-go${{ matrix.go-version }}
path: |
coverage.out
coverage.html
- name: 检查测试覆盖率
run: |
COVERAGE=$(go tool cover -func=coverage.out | grep total | awk '{print $3}' | sed 's/%//')
echo "测试覆盖率: ${COVERAGE}%"
if (( $(echo "$COVERAGE < 80" | bc -l) )); then
echo "⚠️ 测试覆盖率低于 80%"
exit 1
fi
echo "✅ 测试覆盖率达标"
build:
name: 构建验证
runs-on: ubuntu-latest
needs: test
strategy:
matrix:
goos: [linux, darwin, windows]
goarch: [amd64, arm64]
exclude:
- goos: windows
goarch: arm64
steps:
- name: 检出代码
uses: actions/checkout@v4
- name: 设置 Go 环境
uses: actions/setup-go@v4
with:
go-version: 1.24.x
- name: 构建二进制文件
env:
GOOS: ${{ matrix.goos }}
GOARCH: ${{ matrix.goarch }}
run: |
echo "构建 ${{ matrix.goos }}/${{ matrix.goarch }}"
go build -v -ldflags="-s -w" ./...
- name: 构建示例程序
env:
GOOS: ${{ matrix.goos }}
GOARCH: ${{ matrix.goarch }}
run: |
cd examples/basic-usage && go build -v -ldflags="-s -w" .
cd ../group-management && go build -v -ldflags="-s -w" .
cd ../external-handler && go build -v -ldflags="-s -w" .
cd ../data-analytics && go build -v -ldflags="-s -w" .
cd ../concurrent-processing && go build -v -ldflags="-s -w" .
cd ../high-concurrency && go build -v -ldflags="-s -w" .
performance:
name: 性能回归测试
runs-on: ubuntu-latest
needs: test
if: github.event_name == 'pull_request'
steps:
- name: 检出代码
uses: actions/checkout@v4
- name: 设置 Go 环境
uses: actions/setup-go@v4
with:
go-version: 1.24.x
- name: 运行性能基准测试
run: |
echo "=== 性能基准测试 - 当前分支 ==="
go test -bench=. -benchmem -count=3 -benchtime=2s -run=^$ ./... > current_bench.txt
- name: 检出主分支
run: |
git fetch origin main
git checkout origin/main
- name: 运行性能基准测试 - 主分支
run: |
echo "=== 性能基准测试 - 主分支 ==="
go test -bench=. -benchmem -count=3 -benchtime=2s -run=^$ ./... > main_bench.txt || true
- name: 性能对比分析
run: |
echo "=== 性能对比报告 ==="
echo "当前分支性能:"
cat current_bench.txt | grep "Benchmark" | head -10
echo ""
echo "主分支性能:"
cat main_bench.txt | grep "Benchmark" | head -10 || echo "主分支基准测试数据不可用"
- name: 上传性能报告
uses: actions/upload-artifact@v3
with:
name: performance-report
path: |
current_bench.txt
main_bench.txt
security:
name: 安全扫描
runs-on: ubuntu-latest
steps:
- name: 检出代码
uses: actions/checkout@v4
- name: 设置 Go 环境
uses: actions/setup-go@v4
with:
go-version: 1.24.x
- name: 安装 gosec
run: go install github.com/securego/gosec/v2/cmd/gosec@latest
- name: 运行安全扫描
run: |
gosec -fmt json -out gosec-report.json ./...
gosec ./...
- name: 上传安全报告
uses: actions/upload-artifact@v3
if: always()
with:
name: security-report
path: gosec-report.json
quality:
name: 代码质量检查
runs-on: ubuntu-latest
steps:
- name: 检出代码
uses: actions/checkout@v4
- name: 设置 Go 环境
uses: actions/setup-go@v4
with:
go-version: 1.24.x
- name: 安装代码质量工具
run: |
go install honnef.co/go/tools/cmd/staticcheck@latest
go install github.com/golangci/golangci-lint/cmd/golangci-lint@latest
- name: 运行 staticcheck
run: staticcheck ./...
- name: 运行 golangci-lint
run: golangci-lint run ./...
- name: 检查循环复杂度
run: |
go install github.com/fzipp/gocyclo/cmd/gocyclo@latest
gocyclo -over 15 .
- name: 检查代码重复
run: |
go install github.com/mibk/dupl@latest
dupl -threshold 100 .
notification:
name: 通知
runs-on: ubuntu-latest
needs: [test, build, security, quality]
if: always()
steps:
- name: 构建状态通知
run: |
if [[ "${{ needs.test.result }}" == "success" && "${{ needs.build.result }}" == "success" && "${{ needs.security.result }}" == "success" && "${{ needs.quality.result }}" == "success" ]]; then
echo "✅ 所有检查通过!"
echo "STATUS=success" >> $GITHUB_ENV
else
echo "❌ 部分检查失败"
echo "测试结果: ${{ needs.test.result }}"
echo "构建结果: ${{ needs.build.result }}"
echo "安全扫描: ${{ needs.security.result }}"
echo "质量检查: ${{ needs.quality.result }}"
echo "STATUS=failure" >> $GITHUB_ENV
fi

View File

@@ -0,0 +1,250 @@
name: 发布流程
on:
push:
tags:
- 'v*'
jobs:
test:
name: 发布前测试
runs-on: ubuntu-latest
steps:
- name: 检出代码
uses: actions/checkout@v4
- name: 设置 Go 环境
uses: actions/setup-go@v4
with:
go-version: 1.24.x
- name: 运行完整测试套件
run: |
go test -v -race -coverprofile=coverage.out ./...
- name: 运行基准测试验证
run: |
go test -bench=. -benchtime=1s -run=^$ ./...
- name: 检查测试覆盖率
run: |
COVERAGE=$(go tool cover -func=coverage.out | grep total | awk '{print $3}' | sed 's/%//')
echo "测试覆盖率: ${COVERAGE}%"
if (( $(echo "$COVERAGE < 80" | bc -l) )); then
echo "❌ 测试覆盖率低于 80%,无法发布"
exit 1
fi
build:
name: 构建发布包
runs-on: ubuntu-latest
needs: test
strategy:
matrix:
include:
- goos: linux
goarch: amd64
- goos: linux
goarch: arm64
- goos: darwin
goarch: amd64
- goos: darwin
goarch: arm64
- goos: windows
goarch: amd64
steps:
- name: 检出代码
uses: actions/checkout@v4
- name: 设置 Go 环境
uses: actions/setup-go@v4
with:
go-version: 1.24.x
- name: 获取版本信息
id: version
run: |
VERSION=${GITHUB_REF#refs/tags/}
echo "VERSION=$VERSION" >> $GITHUB_OUTPUT
echo "发布版本: $VERSION"
- name: 构建示例程序
env:
GOOS: ${{ matrix.goos }}
GOARCH: ${{ matrix.goarch }}
CGO_ENABLED: 0
run: |
VERSION=${{ steps.version.outputs.VERSION }}
LDFLAGS="-s -w -X main.version=$VERSION"
mkdir -p dist/${{ matrix.goos }}_${{ matrix.goarch }}
# 构建所有示例程序
cd examples/basic-usage
go build -ldflags="$LDFLAGS" -o ../../dist/${{ matrix.goos }}_${{ matrix.goarch }}/basic-usage${{ matrix.goos == 'windows' && '.exe' || '' }} .
cd ../group-management
go build -ldflags="$LDFLAGS" -o ../../dist/${{ matrix.goos }}_${{ matrix.goarch }}/group-management${{ matrix.goos == 'windows' && '.exe' || '' }} .
cd ../external-handler
go build -ldflags="$LDFLAGS" -o ../../dist/${{ matrix.goos }}_${{ matrix.goarch }}/external-handler${{ matrix.goos == 'windows' && '.exe' || '' }} .
cd ../data-analytics
go build -ldflags="$LDFLAGS" -o ../../dist/${{ matrix.goos }}_${{ matrix.goarch }}/data-analytics${{ matrix.goos == 'windows' && '.exe' || '' }} .
cd ../concurrent-processing
go build -ldflags="$LDFLAGS" -o ../../dist/${{ matrix.goos }}_${{ matrix.goarch }}/concurrent-processing${{ matrix.goos == 'windows' && '.exe' || '' }} .
cd ../high-concurrency
go build -ldflags="$LDFLAGS" -o ../../dist/${{ matrix.goos }}_${{ matrix.goarch }}/high-concurrency${{ matrix.goos == 'windows' && '.exe' || '' }} .
- name: 创建发布包
run: |
cd dist/${{ matrix.goos }}_${{ matrix.goarch }}
# 复制文档文件
cp ../../README.md .
cp ../../LICENSE .
cp -r ../../examples .
# 创建压缩包
if [ "${{ matrix.goos }}" = "windows" ]; then
zip -r ../pipelinedb-${{ steps.version.outputs.VERSION }}-${{ matrix.goos }}-${{ matrix.goarch }}.zip .
else
tar -czf ../pipelinedb-${{ steps.version.outputs.VERSION }}-${{ matrix.goos }}-${{ matrix.goarch }}.tar.gz .
fi
- name: 上传构建产物
uses: actions/upload-artifact@v3
with:
name: pipelinedb-${{ steps.version.outputs.VERSION }}-${{ matrix.goos }}-${{ matrix.goarch }}
path: dist/pipelinedb-${{ steps.version.outputs.VERSION }}-${{ matrix.goos }}-${{ matrix.goarch }}.*
release:
name: 创建发布
runs-on: ubuntu-latest
needs: build
steps:
- name: 检出代码
uses: actions/checkout@v4
- name: 获取版本信息
id: version
run: |
VERSION=${GITHUB_REF#refs/tags/}
echo "VERSION=$VERSION" >> $GITHUB_OUTPUT
- name: 下载所有构建产物
uses: actions/download-artifact@v3
with:
path: release-assets
- name: 生成发布说明
run: |
VERSION=${{ steps.version.outputs.VERSION }}
cat > release_notes.md << EOF
# Pipeline Database $VERSION
## 🚀 新特性
- 基于页面的高性能存储引擎
- 支持分组数据管理和统计
- Hot → Warm → Cold 三级数据状态流转
- 自定义 Handler 接口支持
- 高并发线程安全操作
- 内置页面缓存系统
## 📊 性能指标
- 并发写入:~1,900 QPS
- 缓存操作:~7,700,000 QPS
- 索引操作:~6,200,000 QPS
- 支持 TB 级数据存储
## 📦 下载
选择适合您系统的版本:
- **Linux (x64)**: pipelinedb-$VERSION-linux-amd64.tar.gz
- **Linux (ARM64)**: pipelinedb-$VERSION-linux-arm64.tar.gz
- **macOS (Intel)**: pipelinedb-$VERSION-darwin-amd64.tar.gz
- **macOS (Apple Silicon)**: pipelinedb-$VERSION-darwin-arm64.tar.gz
- **Windows (x64)**: pipelinedb-$VERSION-windows-amd64.zip
## 🛠️ 快速开始
\`\`\`bash
# 安装
go get code.tczkiot.com/wlw/pipelinedb@$VERSION
# 运行示例
./basic-usage
\`\`\`
## 📚 文档
- [README](https://code.tczkiot.com/wlw/pipelinedb/src/branch/main/README.md)
- [示例代码](https://code.tczkiot.com/wlw/pipelinedb/src/branch/main/examples)
- [API 文档](https://pkg.go.dev/code.tczkiot.com/wlw/pipelinedb)
## 🔄 更新日志
查看完整的更新日志和技术细节,请参考 [CHANGELOG.md](https://code.tczkiot.com/wlw/pipelinedb/src/branch/main/CHANGELOG.md)
EOF
- name: 创建 GitHub Release
uses: softprops/action-gh-release@v1
with:
name: Pipeline Database ${{ steps.version.outputs.VERSION }}
body_path: release_notes.md
files: |
release-assets/**/*.tar.gz
release-assets/**/*.zip
draft: false
prerelease: false
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
docker:
name: 构建 Docker 镜像
runs-on: ubuntu-latest
needs: test
steps:
- name: 检出代码
uses: actions/checkout@v4
- name: 获取版本信息
id: version
run: |
VERSION=${GITHUB_REF#refs/tags/}
echo "VERSION=$VERSION" >> $GITHUB_OUTPUT
- name: 设置 Docker Buildx
uses: docker/setup-buildx-action@v2
- name: 登录 Docker Hub
uses: docker/login-action@v2
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
- name: 构建并推送 Docker 镜像
uses: docker/build-push-action@v4
with:
context: .
platforms: linux/amd64,linux/arm64
push: true
tags: |
bourdon/pipelinedb:${{ steps.version.outputs.VERSION }}
bourdon/pipelinedb:latest
labels: |
org.opencontainers.image.title=Pipeline Database
org.opencontainers.image.description=集成数据库存储和业务管道处理的一体化解决方案
org.opencontainers.image.version=${{ steps.version.outputs.VERSION }}
org.opencontainers.image.source=https://code.tczkiot.com/wlw/pipelinedb