[libc++] Add a job that runs the benchmarks as part of the CI

ldionne · ldionne · commit c7306f453076 · 2025-09-11T12:21:29.000-04:00
This will make it easier to see the performance impact of individual
patches. This is a first step towards better performance tracking, but
many things still need to be figured out:
- We should get a dedicated set of machines to run these tests on
- We could probably reuse regularly-generated baseline results published
  to LNT instead of re-generating the baselines in this job every time.
diff --git a/.github/workflows/libcxx-build-and-test.yaml b/.github/workflows/libcxx-build-and-test.yaml
@@ -284,3 +284,25 @@ jobs:
       - name: Build and test
         run: |
           bash libcxx/utils/ci/run-buildbot ${{ matrix.config }}
+
+  benchmarks:
+    runs-on: llvm-premerge-libcxx-next-runners # TODO: This should run on a dedicated set of machines
+    # needs: [ stage3 ] # TODO: Run this at the end of the pipeline once we're done testing the job
+    steps:
+      - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
+      - name: benchmarks
+        run: libcxx/utils/ci/run-buildbot benchmark-comparison
+        env:
+          CC: clang-22
+          CXX: clang++-22
+      - uses: actions/upload-artifact@26f96dfa697d77e81fd5907df203aa23a56210a8 # v4.3.0
+        if: always()
+        with:
+          name: benchmarks-results
+          path: |
+            **/test-results.xml
+            **/*.abilist
+            **/CMakeConfigureLog.yaml
+            **/CMakeError.log
+            **/CMakeOutput.log
+            **/crash_diagnostics/*
diff --git a/libcxx/utils/ci/run-buildbot b/libcxx/utils/ci/run-buildbot
@@ -542,6 +542,24 @@ generic-optimized-speed)
     generate-cmake -C "${MONOREPO_ROOT}/libcxx/cmake/caches/Generic-optimized-speed.cmake"
     check-runtimes
 ;;
+benchmark-comparison)
+    clean
+    LIT_ARGS="-sv -j1 --param optimization=speed libcxx/test/benchmarks/hash.bench.cpp libcxx/test/benchmarks/join_view.bench.cpp"
+
+    baseline_commit=$(git merge-base main HEAD)
+    step "Building and running the baseline at ${baseline_commit}"
+    rm -rf "${BUILD_DIR}-baseline"
+    ./libcxx/utils/test-at-commit --commit ${baseline_commit} -B "${BUILD_DIR}-baseline" -- ${LIT_ARGS}
+
+    # Build and run the candidate
+    step "Building and running the candidate"
+    generate-cmake -C "${MONOREPO_ROOT}/libcxx/cmake/caches/Generic-optimized-speed.cmake"
+    ./libcxx/utils/libcxx-lit "${BUILD_DIR}" ${LIT_ARGS}
+
+    step "Comparison of the baseline and the candidate"
+    ./libcxx/utils/compare-benchmarks <(./libcxx/utils/consolidate-benchmarks "${BUILD_DIR}-baseline") \
+                                      <(./libcxx/utils/consolidate-benchmarks "${BUILD_DIR}")
+;;
 apple-configuration)
     clean
 
