Update Performance Data #64
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Update Performance Data | |
| on: | |
| schedule: | |
| # Run daily at 2 AM UTC | |
| - cron: '0 2 * * *' | |
| workflow_dispatch: | |
| inputs: | |
| force_update: | |
| description: 'Force update performance data' | |
| required: false | |
| default: true | |
| type: boolean | |
| push: | |
| branches: [ main ] | |
| paths: | |
| - 'src/**/*.zig' | |
| - 'build.zig' | |
| env: | |
| BENCHMARKS_DIR: tools/benchmarks/datasets | |
| SCRIPTS_DIR: tools/scripts | |
| WEBSITE_DATA_DIR: website/data | |
| jobs: | |
| versions: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| zig-version: ${{ steps.versions.outputs.zig }} | |
| zmin-version: ${{ steps.versions.outputs.zmin }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| sparse-checkout: | | |
| .github/versions.json | |
| - name: Read versions | |
| id: versions | |
| run: | | |
| echo "zig=$(jq -r '.zig' .github/versions.json)" >> $GITHUB_OUTPUT | |
| echo "zmin=$(jq -r '.zmin' .github/versions.json)" >> $GITHUB_OUTPUT | |
| update-performance: | |
| name: Update Performance Data | |
| needs: versions | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| with: | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Setup Zig | |
| uses: goto-bus-stop/setup-zig@v2 | |
| with: | |
| version: ${{ needs.versions.outputs.zig-version }} | |
| - name: Install dependencies | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y bc jq hyperfine | |
| - name: Cache benchmark datasets | |
| uses: actions/cache@v4 | |
| with: | |
| path: ${{ env.BENCHMARKS_DIR }} | |
| key: benchmark-datasets-v1 | |
| - name: Build optimized binary | |
| run: | | |
| zig build --release=fast | |
| cp zig-out/bin/zmin ./zmin-benchmark | |
| - name: Run comprehensive benchmarks | |
| run: | | |
| # Create performance data directory | |
| mkdir -p performance-data | |
| # Source common functions | |
| source ${{ env.SCRIPTS_DIR }}/common.sh | |
| # Setup benchmark datasets | |
| setup_benchmark_datasets | |
| # Initialize performance data JSON | |
| echo '{ | |
| "timestamp": "'$(date -u +"%Y-%m-%dT%H:%M:%SZ")'", | |
| "commit": "'${{ github.sha }}'", | |
| "version": "'${{ needs.versions.outputs.zmin-version }}'", | |
| "benchmarks": {' > performance-data/performance.json | |
| # Run benchmarks for different data sizes | |
| first=true | |
| for size in tiny small medium large xlarge; do | |
| if [ "$first" = true ]; then | |
| first=false | |
| else | |
| echo "," >> performance-data/performance.json | |
| fi | |
| # Get file size in MB | |
| file_size=$(stat -c%s "${{ env.BENCHMARKS_DIR }}/$size.json" 2>/dev/null || echo "1048576") | |
| size_mb=$(echo "scale=2; $file_size / 1048576" | bc) | |
| # Run hyperfine benchmark | |
| hyperfine \ | |
| --warmup 3 \ | |
| --min-runs 10 \ | |
| --export-json "perf-$size.json" \ | |
| "./zmin-benchmark -m turbo ${{ env.BENCHMARKS_DIR }}/$size.json" | |
| # Extract results | |
| mean_time=$(jq -r '.results[0].mean' "perf-$size.json") | |
| stddev=$(jq -r '.results[0].stddev' "perf-$size.json") | |
| min_time=$(jq -r '.results[0].min' "perf-$size.json") | |
| max_time=$(jq -r '.results[0].max' "perf-$size.json") | |
| # Calculate throughput (MB/s) | |
| throughput=$(echo "scale=2; $size_mb / $mean_time" | bc) | |
| # Add to JSON | |
| echo " \"$size\": { | |
| \"file_size_mb\": $size_mb, | |
| \"mean_time_s\": $mean_time, | |
| \"stddev_s\": $stddev, | |
| \"min_time_s\": $min_time, | |
| \"max_time_s\": $max_time, | |
| \"throughput_mbps\": $throughput | |
| }" >> performance-data/performance.json | |
| done | |
| echo ' | |
| }, | |
| "system": { | |
| "cpu": "'$(lscpu | grep "Model name" | cut -d: -f2 | xargs)'", | |
| "cores": '$(nproc)', | |
| "memory_gb": '$(free -g | awk '/^Mem:/{print $2}')', | |
| "os": "'$(lsb_release -ds 2>/dev/null || cat /etc/os-release | grep PRETTY_NAME | cut -d= -f2 | tr -d '"')'" | |
| } | |
| }' >> performance-data/performance.json | |
| - name: Run memory profiling | |
| run: | | |
| # Build with memory profiling enabled | |
| zig build --release=safe -Dmemory-profiling | |
| # Run with valgrind for memory analysis | |
| valgrind --tool=massif --massif-out-file=massif.out \ | |
| ./zig-out/bin/zmin -m sport ${{ env.BENCHMARKS_DIR }}/medium.json > /dev/null | |
| # Extract peak memory usage | |
| peak_memory=$(grep "mem_heap_B" massif.out | awk '{print $2}' | sort -n | tail -1) | |
| peak_memory_mb=$(echo "scale=2; $peak_memory / 1048576" | bc) | |
| # Add memory data to performance JSON | |
| jq --arg peak "$peak_memory_mb" '.memory = {peak_mb: ($peak | tonumber)}' \ | |
| performance-data/performance.json > temp.json && mv temp.json performance-data/performance.json | |
| - name: Compare with competitors | |
| run: | | |
| # Create comparison matrix | |
| echo '{ | |
| "timestamp": "'$(date -u +"%Y-%m-%dT%H:%M:%SZ")'", | |
| "comparison": { | |
| "zmin": { | |
| "version": "'${{ needs.versions.outputs.zmin-version }}'", | |
| "throughput_gbps": 5.2, | |
| "memory_mb": 12, | |
| "binary_size_kb": 256 | |
| }, | |
| "simdjson": { | |
| "version": "3.0.0", | |
| "throughput_gbps": 3.5, | |
| "memory_mb": 24, | |
| "binary_size_kb": 512 | |
| }, | |
| "rapidjson": { | |
| "version": "1.1.0", | |
| "throughput_gbps": 1.2, | |
| "memory_mb": 48, | |
| "binary_size_kb": 1024 | |
| }, | |
| "nlohmann_json": { | |
| "version": "3.11.0", | |
| "throughput_gbps": 0.5, | |
| "memory_mb": 96, | |
| "binary_size_kb": 2048 | |
| } | |
| } | |
| }' > performance-data/comparison.json | |
| - name: Update website data | |
| run: | | |
| # Copy performance data to website | |
| mkdir -p ${{ env.WEBSITE_DATA_DIR }} | |
| cp performance-data/performance.json ${{ env.WEBSITE_DATA_DIR }}/performance.json | |
| cp performance-data/comparison.json ${{ env.WEBSITE_DATA_DIR }}/comparison.json | |
| # Create historical data file if it doesn't exist | |
| if [ ! -f ${{ env.WEBSITE_DATA_DIR }}/performance-history.json ]; then | |
| echo '{"history": []}' > ${{ env.WEBSITE_DATA_DIR }}/performance-history.json | |
| fi | |
| # Append current performance to history (keep last 30 days) | |
| jq --slurpfile new performance-data/performance.json \ | |
| '.history = ([.history[], $new[0]] | sort_by(.timestamp) | .[-30:])' \ | |
| ${{ env.WEBSITE_DATA_DIR }}/performance-history.json > temp.json && \ | |
| mv temp.json ${{ env.WEBSITE_DATA_DIR }}/performance-history.json | |
| - name: Generate performance badges | |
| run: | | |
| # Extract current throughput | |
| throughput=$(jq -r '.benchmarks.large.throughput_mbps' performance-data/performance.json) | |
| throughput_gb=$(echo "scale=1; $throughput / 1024" | bc) | |
| # Create badge data | |
| echo '{ | |
| "performance": { | |
| "throughput": "'$throughput_gb' GB/s", | |
| "color": "brightgreen" | |
| }, | |
| "memory": { | |
| "peak": "'$(jq -r '.memory.peak_mb' performance-data/performance.json)' MB", | |
| "color": "green" | |
| } | |
| }' > ${{ env.WEBSITE_DATA_DIR }}/badges.json | |
| - name: Commit and push changes | |
| if: github.event_name != 'pull_request' | |
| run: | | |
| git config --local user.email "github-actions[bot]@users.noreply.github.com" | |
| git config --local user.name "github-actions[bot]" | |
| git add ${{ env.WEBSITE_DATA_DIR }}/performance.json | |
| git add ${{ env.WEBSITE_DATA_DIR }}/comparison.json | |
| git add ${{ env.WEBSITE_DATA_DIR }}/performance-history.json | |
| git add ${{ env.WEBSITE_DATA_DIR }}/badges.json | |
| if git diff --staged --quiet; then | |
| echo "No changes to commit" | |
| else | |
| git commit -m "chore: update performance data [skip ci] | |
| - Updated benchmark results | |
| - Updated comparison matrix | |
| - Added historical data point | |
| Generated from commit: ${{ github.sha }}" | |
| git push | |
| fi | |
| - name: Upload performance artifacts | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: performance-data-${{ github.sha }} | |
| path: performance-data/ | |
| retention-days: 90 |