diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d1c9c730..6df75834 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -89,16 +89,34 @@ jobs: - name: Check divan_compat MSRV run: cargo msrv --path crates/divan_compat verify -- cargo check --all-features --config codspeed=true - tests: + tests-without-cargo-codspeed: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 with: submodules: true - uses: moonrepo/setup-rust@v1 + with: + bins: cargo-nextest + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - run: cargo nextest run --workspace --exclude cargo-codspeed + + test-cargo-codspeed: + runs-on: ubuntu-latest + strategy: + matrix: + partition: [1, 2, 3, 4, 5] + steps: + - uses: actions/checkout@v4 + with: + submodules: true + - uses: moonrepo/setup-rust@v1 + with: + bins: cargo-nextest env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - run: cargo test --all + - run: cargo nextest run -p cargo-codspeed --partition hash:${{ matrix.partition }}/5 compat-integration-test-instrumentation: runs-on: ubuntu-latest @@ -198,8 +216,9 @@ jobs: needs: - lint - test-codspeed + - tests-without-cargo-codspeed + - test-cargo-codspeed - msrv-check - - tests - compat-integration-test-instrumentation - compat-integration-test-walltime - musl-build-check diff --git a/Cargo.lock b/Cargo.lock index 8bf501dd..3d01ed78 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -436,16 +436,16 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "camino" -version = "1.1.9" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b96ec4966b5813e2c0507c1f86115c8c5abaadc3980879c3424042a02fd1ad3" +checksum = "276a59bf2b2c967788139340c9f0c5b12d7fd6630315c15c217e559de85d2609" dependencies = [ - "serde", + "serde_core", ] [[package]] name = "cargo-codspeed" -version = "4.0.5" +version = "4.1.0" dependencies = [ "anstyle", "anyhow", @@ -459,23 +459,23 @@ dependencies = [ "serde", "serde_json", "termcolor", - "uuid", + "toml", ] [[package]] name = "cargo-platform" -version = "0.1.8" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24b1f0365a6c6bb4020cd05806fd0d33c44d38046b8bd7f0e40814b9763cabfc" +checksum = "122ec45a44b270afd1402f351b782c676b173e3c3fb28d86ff7ebfb4d86a4ee4" dependencies = [ "serde", ] [[package]] name = "cargo_metadata" -version = "0.19.2" +version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd5eb614ed4c27c5d706420e4320fbe3216ab31fa1c33cd8246ac36dae4479ba" +checksum = "981a6f317983eec002839b90fae7411a85621410ae591a9cab2ecf5cb5744873" dependencies = [ "camino", "cargo-platform", @@ -583,11 +583,12 @@ checksum = "b94f61472cee1439c0b966b47e3aca9ae07e45d070759512cd390ea2bebc6675" [[package]] name = "codspeed" -version = "4.0.5" +version = "4.1.0" dependencies = [ "anyhow", "cc", "colored", + "getrandom", "glob", "libc", "nix", @@ -595,12 +596,11 @@ dependencies = [ "serde_json", "statrs", "tempfile", - "uuid", ] [[package]] name = "codspeed-bencher-compat" -version = "4.0.5" +version = "4.1.0" dependencies = [ "bencher", "codspeed", @@ -608,7 +608,7 @@ dependencies = [ [[package]] name = "codspeed-criterion-compat" -version = "4.0.5" +version = "4.1.0" dependencies = [ "async-std", "clap", @@ -623,7 +623,7 @@ dependencies = [ [[package]] name = "codspeed-criterion-compat-walltime" -version = "4.0.5" +version = "4.1.0" dependencies = [ "anes", "approx", @@ -657,7 +657,7 @@ dependencies = [ [[package]] name = "codspeed-divan-compat" -version = "4.0.5" +version = "4.1.0" dependencies = [ "clap", "codspeed", @@ -678,7 +678,7 @@ dependencies = [ [[package]] name = "codspeed-divan-compat-macros" -version = "4.0.5" +version = "4.1.0" dependencies = [ "divan-macros", "itertools 0.14.0", @@ -690,7 +690,7 @@ dependencies = [ [[package]] name = "codspeed-divan-compat-walltime" -version = "4.0.5" +version = "4.1.0" dependencies = [ "cfg-if", "clap", @@ -1618,27 +1618,38 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.23" +version = "1.0.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61697e0a1c7e512e84a621326239844a24d8207b4669b41bc18b32ea5cbf988b" +checksum = "d767eb0aabc880b29956c35734170f26ed551a859dbd361d140cdbeca61ab1e2" dependencies = [ "serde", + "serde_core", ] [[package]] name = "serde" -version = "1.0.217" +version = "1.0.228" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02fc4265df13d6fa1d00ecff087228cc0a2b5f3c0e87e258d8b94a156e984c70" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.217" +version = "1.0.228" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a9bf7cf98d04a2b28aead066b7496853d4779c9cc183c440dbac457641e19a0" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", @@ -1647,14 +1658,24 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.138" +version = "1.0.145" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d434192e7da787e94a6ea7e9670b26a036d0ca41e0b7efb2676dd32bae872949" +checksum = "402a6f66d8c709116cf22f558eab210f5a50187f702eb4d7e5ef38d9a7f1c79c" dependencies = [ "itoa", "memchr", "ryu", "serde", + "serde_core", +] + +[[package]] +name = "serde_spanned" +version = "0.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf41e0cfaf7226dca15e8197172c295a782857fcb97fad1808a166870dee75a3" +dependencies = [ + "serde", ] [[package]] @@ -1830,11 +1851,26 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "toml" +version = "0.8.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd87a5cdd6ffab733b2f74bc4fd7ee5fff6634124999ac278c35fc78c6120148" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit", +] + [[package]] name = "toml_datetime" version = "0.6.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0dd7358ecb8fc2f8d014bf86f6f638ce72ba252a2c3a2572f2a795f1d23efb41" +dependencies = [ + "serde", +] [[package]] name = "toml_edit" @@ -1843,6 +1879,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "02a8b472d1a3d7c18e2d61a489aee3453fd9031c33e4f55bd533f4a7adca1bee" dependencies = [ "indexmap", + "serde", + "serde_spanned", "toml_datetime", "winnow", ] @@ -1875,15 +1913,6 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" -[[package]] -name = "uuid" -version = "1.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3758f5e68192bb96cc8f9b7e2c2cfdabb435499a28499a42f8f984092adad4b" -dependencies = [ - "getrandom", -] - [[package]] name = "value-bag" version = "1.9.0" diff --git a/crates/bencher_compat/Cargo.toml b/crates/bencher_compat/Cargo.toml index d7b06da6..9d2c9963 100644 --- a/crates/bencher_compat/Cargo.toml +++ b/crates/bencher_compat/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "codspeed-bencher-compat" -version = "4.0.5" +version = "4.1.0" rust-version = "1.74" # MSRV edition = "2021" description = "Bencher compatibility layer for CodSpeed" @@ -19,7 +19,7 @@ keywords = ["codspeed", "benchmark", "bencher"] [dependencies] bencher = "0.1.5" -codspeed = { path = "../codspeed", version = "=4.0.5" } +codspeed = { path = "../codspeed", version = "=4.1.0" } [[bench]] name = "bencher_example" diff --git a/crates/cargo-codspeed/Cargo.toml b/crates/cargo-codspeed/Cargo.toml index e9dbd01e..2a233a21 100644 --- a/crates/cargo-codspeed/Cargo.toml +++ b/crates/cargo-codspeed/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "cargo-codspeed" -version = "4.0.5" +version = "4.1.0" rust-version = "1.74" # MSRV edition = "2021" description = "Cargo extension to build & run your codspeed benchmarks" @@ -19,7 +19,7 @@ categories = [ keywords = ["codspeed", "benchmark", "cargo"] [dependencies] -cargo_metadata = "0.19.2" +cargo_metadata = "0.23.0" clap = { version = "=4.5.17", features = ["derive", "env"] } termcolor = "1.4" anyhow = { workspace = true } @@ -27,13 +27,13 @@ itertools = { workspace = true } anstyle = "1.0.8" serde = { workspace = true } serde_json = { workspace = true } -codspeed = { path = "../codspeed", version = "=4.0.5" } +toml = "0.8" +codspeed = { path = "../codspeed", version = "=4.1.0" } [dev-dependencies] assert_cmd = "2.0.15" fs_extra = "1.3.0" predicates = "3.1.2" -uuid = { version = "1.10.0", features = ["v4"] } [features] vendored-openssl = [] diff --git a/crates/cargo-codspeed/src/build.rs b/crates/cargo-codspeed/src/build.rs index 8d44deee..70e88a66 100644 --- a/crates/cargo-codspeed/src/build.rs +++ b/crates/cargo-codspeed/src/build.rs @@ -4,7 +4,9 @@ use crate::{ measurement_mode::MeasurementMode, prelude::*, }; +use anyhow::Context; use cargo_metadata::{camino::Utf8PathBuf, Message, Metadata, TargetKind}; +use std::collections::HashMap; use std::process::{exit, Command, Stdio}; struct BuildOptions<'a> { @@ -31,6 +33,50 @@ pub struct BuildConfig { pub passthrough_flags: Vec, } +fn get_bench_harness_value( + manifest_path: &Utf8PathBuf, + bench_name: &str, + cache: &mut HashMap, +) -> Result { + let manifest_table = if let Some(table) = cache.get(manifest_path) { + table + } else { + // Read and parse the Cargo.toml file + let manifest_content = std::fs::read_to_string(manifest_path) + .with_context(|| format!("Failed to read manifest at {manifest_path}"))?; + let table: toml::Table = toml::from_str(&manifest_content) + .with_context(|| format!("Failed to parse TOML in {manifest_path}"))?; + cache.insert(manifest_path.clone(), table); + cache.get(manifest_path).unwrap() + }; + + // Look for [[bench]] sections + let Some(benches) = manifest_table.get("bench").and_then(|v| v.as_array()) else { + // If no [[bench]] sections, it's not an error, benches present in /benches/.rs + // are still collected with harness = true + return Ok(true); + }; + + // Find the bench entry with matching name + let matching_bench = benches + .iter() + .filter_map(|bench| bench.as_table()) + .find(|bench_table| { + bench_table + .get("name") + .and_then(|v| v.as_str()) + .is_some_and(|name| name == bench_name) + }); + + // Check if harness is enabled (defaults to true) + let harness_enabled = matching_bench + .and_then(|bench_table| bench_table.get("harness")) + .and_then(|v| v.as_bool()) + .unwrap_or(true); + + Ok(harness_enabled) +} + impl BuildOptions<'_> { /// Builds the benchmarks by invoking cargo /// Returns a list of built benchmarks, with path to associated executables @@ -60,6 +106,8 @@ impl BuildOptions<'_> { ); let mut built_benches = Vec::new(); + let mut bench_targets_with_default_harness = Vec::new(); + let mut manifest_cache = HashMap::new(); let package_names = self .package_filters @@ -95,8 +143,17 @@ impl BuildOptions<'_> { let add_bench_to_codspeed_dir = package_names.iter().contains(&package.name); if add_bench_to_codspeed_dir { + if get_bench_harness_value( + &package.manifest_path, + &bench_target_name, + &mut manifest_cache, + )? { + bench_targets_with_default_harness + .push((package.name.to_string(), bench_target_name.clone())); + } + built_benches.push(BuiltBench { - package: package.name.clone(), + package: package.name.to_string(), bench: bench_target_name, executable_path: artifact .executable @@ -114,6 +171,25 @@ impl BuildOptions<'_> { exit(status.code().expect("Could not get exit code")); } + if !bench_targets_with_default_harness.is_empty() { + let targets_list = bench_targets_with_default_harness + .into_iter() + .map(|(package, bench)| format!(" - `{bench}` in package `{package}`")) + .join("\n"); + + bail!("\ +CodSpeed will not work with the following benchmark targets: +{targets_list} + +CodSpeed requires benchmark targets to disable the default test harness because benchmark frameworks handle harnessing themselves. + +Either disable the default harness by adding `harness = false` to the corresponding \ +`[[bench]]` section in the Cargo.toml, or specify which targets to build by using \ +`cargo codspeed build -p package_name --bench first_target --bench second_target`. + +See `cargo codspeed build --help` for more information."); + } + for built_bench in &built_benches { eprintln!( "Built benchmark `{}` in package `{}`", diff --git a/crates/cargo-codspeed/src/run.rs b/crates/cargo-codspeed/src/run.rs index 1dc10393..7fc6268a 100644 --- a/crates/cargo-codspeed/src/run.rs +++ b/crates/cargo-codspeed/src/run.rs @@ -58,7 +58,7 @@ impl PackageFilters { let mut benches = vec![]; for package in packages { let package_name = &package.name; - let package_target_dir = codspeed_target_dir.join(package_name); + let package_target_dir = codspeed_target_dir.join(package_name.to_string()); let working_directory = package.manifest_path.parent().ok_or_else(|| { Error::msg(format!("Failed to get root dir for package {package_name}")) })?; @@ -79,7 +79,7 @@ impl PackageFilters { } if !bench_path.is_dir() { benches.push(BenchToRun { - package_name: package_name.clone(), + package_name: package_name.to_string(), working_directory: working_directory.into(), bench_path, bench_target_name, @@ -134,7 +134,7 @@ pub fn run_benches( command .status() - .map_err(|e| anyhow!("failed to execute the benchmark process: {}", e)) + .map_err(|e| anyhow!("failed to execute the benchmark process: {e}")) .and_then(|status| { if status.success() { Ok(()) diff --git a/crates/cargo-codspeed/tests/default_harness_error.in/.gitignore b/crates/cargo-codspeed/tests/default_harness_error.in/.gitignore new file mode 100644 index 00000000..2c96eb1b --- /dev/null +++ b/crates/cargo-codspeed/tests/default_harness_error.in/.gitignore @@ -0,0 +1,2 @@ +target/ +Cargo.lock diff --git a/crates/cargo-codspeed/tests/default_harness_error.in/Cargo.toml b/crates/cargo-codspeed/tests/default_harness_error.in/Cargo.toml new file mode 100644 index 00000000..bc149a25 --- /dev/null +++ b/crates/cargo-codspeed/tests/default_harness_error.in/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "default-harness-error-test" +version = "0.1.0" +edition = "2021" +publish = false + +[dependencies] +bencher = "0.1.5" +codspeed = { path = "../../../codspeed" } +codspeed-bencher-compat = { path = "../../../bencher_compat" } + +[[bench]] +name = "bencher_example" +# Missing harness = false to trigger error + +[[bench]] +name = "bencher_correct" +harness = false diff --git a/crates/cargo-codspeed/tests/default_harness_error.in/benches/bencher_correct.rs b/crates/cargo-codspeed/tests/default_harness_error.in/benches/bencher_correct.rs new file mode 100644 index 00000000..8fee2c6a --- /dev/null +++ b/crates/cargo-codspeed/tests/default_harness_error.in/benches/bencher_correct.rs @@ -0,0 +1,9 @@ +use codspeed::codspeed::black_box; +use codspeed_bencher_compat::{benchmark_group, benchmark_main, Bencher}; + +pub fn b(bench: &mut Bencher) { + bench.iter(|| (0..50).fold(0, |x, y| black_box(x + y))) +} + +benchmark_group!(benches, b); +benchmark_main!(benches); diff --git a/crates/cargo-codspeed/tests/default_harness_error.in/benches/bencher_example.rs b/crates/cargo-codspeed/tests/default_harness_error.in/benches/bencher_example.rs new file mode 100644 index 00000000..a5430914 --- /dev/null +++ b/crates/cargo-codspeed/tests/default_harness_error.in/benches/bencher_example.rs @@ -0,0 +1,9 @@ +use codspeed::codspeed::black_box; +use codspeed_bencher_compat::{benchmark_group, benchmark_main, Bencher}; + +pub fn a(bench: &mut Bencher) { + bench.iter(|| (0..100).fold(0, |x, y| black_box(x + y))) +} + +benchmark_group!(benches, a); +benchmark_main!(benches); diff --git a/crates/cargo-codspeed/tests/default_harness_error.in/benches/bencher_no_section.rs b/crates/cargo-codspeed/tests/default_harness_error.in/benches/bencher_no_section.rs new file mode 100644 index 00000000..2c8f7ab3 --- /dev/null +++ b/crates/cargo-codspeed/tests/default_harness_error.in/benches/bencher_no_section.rs @@ -0,0 +1,9 @@ +use codspeed::codspeed::black_box; +use codspeed_bencher_compat::{benchmark_group, benchmark_main, Bencher}; + +pub fn c(bench: &mut Bencher) { + bench.iter(|| (0..75).fold(0, |x, y| black_box(x + y))) +} + +benchmark_group!(benches, c); +benchmark_main!(benches); diff --git a/crates/cargo-codspeed/tests/default_harness_error.rs b/crates/cargo-codspeed/tests/default_harness_error.rs new file mode 100644 index 00000000..78ec4cf0 --- /dev/null +++ b/crates/cargo-codspeed/tests/default_harness_error.rs @@ -0,0 +1,33 @@ +use predicates::prelude::*; +use predicates::str::contains; + +mod helpers; +use helpers::*; + +const DIR: &str = "tests/default_harness_error.in"; + +#[test] +fn test_default_harness_error() { + let dir = setup(DIR, Project::DefaultHarnessError); + cargo_codspeed(&dir) + .arg("build") + .assert() + .failure() + .stderr(contains( + "Error: CodSpeed will not work with the following benchmark targets:", + )) + // Should report bencher_example (explicit bench with missing harness = false) + .stderr(contains( + "`bencher_example` in package `default-harness-error-test`", + )) + // Should report bencher_no_section (no [[bench]] section means default harness) + .stderr(contains( + "`bencher_no_section` in package `default-harness-error-test`", + )) + .stderr(contains( + "CodSpeed requires benchmark targets to disable the default test harness", + )) + // Ensure the correct benchmark with harness = false is NOT reported + .stderr(contains("bencher_correct").not()); + teardown(dir); +} diff --git a/crates/cargo-codspeed/tests/helpers.rs b/crates/cargo-codspeed/tests/helpers.rs index f9259d14..74e85cae 100644 --- a/crates/cargo-codspeed/tests/helpers.rs +++ b/crates/cargo-codspeed/tests/helpers.rs @@ -7,7 +7,6 @@ use assert_cmd::Command; use fs_extra::dir::copy; use fs_extra::dir::create; use fs_extra::dir::remove; -use uuid::Uuid; fn replace_in_file(path: &str, from: &str, to: &str) { let mut contents = std::fs::read_to_string(path).unwrap(); @@ -22,11 +21,13 @@ pub enum Project { Workspace, PackageInDeps, CratesWorkingDirectory, + DefaultHarnessError, } pub fn setup(dir: &str, project: Project) -> String { //Create a new unique named temp directory - let tmp_dir = temp_dir().join(format!("cargo-codspeed-test-{}", Uuid::new_v4())); + let unique_id = codspeed::utils::generate_unique_id(); + let tmp_dir = temp_dir().join(format!("cargo-codspeed-test-{unique_id}")); create(&tmp_dir, false).unwrap(); let mut copy_opts = fs_extra::dir::CopyOptions::new(); copy_opts.content_only = true; @@ -38,7 +39,7 @@ pub fn setup(dir: &str, project: Project) -> String { let package_root = PathBuf::from_str(env!("CARGO_MANIFEST_DIR")).unwrap(); let workspace_root = package_root.parent().unwrap().parent().unwrap(); match project { - Project::Simple | Project::Features => { + Project::Simple | Project::Features | Project::DefaultHarnessError => { replace_in_file( tmp_dir.join("Cargo.toml").to_str().unwrap(), "../../..", diff --git a/crates/cargo-codspeed/tests/workspace.in/b/Cargo.toml b/crates/cargo-codspeed/tests/workspace.in/b/Cargo.toml index 5fb96d32..d9297c79 100644 --- a/crates/cargo-codspeed/tests/workspace.in/b/Cargo.toml +++ b/crates/cargo-codspeed/tests/workspace.in/b/Cargo.toml @@ -12,3 +12,7 @@ codspeed-bencher-compat = { path = "../../../../bencher_compat" } [[bench]] name = "bencher_example" harness = false + +[[bench]] +name = "another_bencher_example" +harness = false diff --git a/crates/codspeed/Cargo.toml b/crates/codspeed/Cargo.toml index dd3eeda8..ce991dcf 100644 --- a/crates/codspeed/Cargo.toml +++ b/crates/codspeed/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "codspeed" -version = "4.0.5" +version = "4.1.0" rust-version = "1.74" # MSRV edition = "2021" description = "Core instrumentation library for CodSpeed" @@ -23,10 +23,10 @@ colored = "2.0.0" glob = "0.3.2" libc = "^0.2" nix = { version = "0.30.1", features = ["time"] } +getrandom = "0.2" serde = { workspace = true } serde_json = { workspace = true } statrs = { version = "0.18.0", default-features = false } -uuid = { version = "1.12.1", features = ["v4"] } [[bench]] name = "native" diff --git a/crates/codspeed/src/utils.rs b/crates/codspeed/src/utils.rs index ffbfb6d3..472c1f84 100644 --- a/crates/codspeed/src/utils.rs +++ b/crates/codspeed/src/utils.rs @@ -46,6 +46,29 @@ pub fn is_perf_enabled() -> bool { std::env::var("CODSPEED_PERF_ENABLED").is_ok() } +/// Generate a statistically unique ID in a format resembling UUID v4. +pub fn generate_unique_id() -> String { + // Generate random bytes for UUID v4 + let mut bytes = [0u8; 16]; + getrandom::getrandom(&mut bytes).expect("Failed to generate random bytes"); + + // Extract values from bytes + let r1 = u32::from_be_bytes([bytes[0], bytes[1], bytes[2], bytes[3]]); + let r2 = u16::from_be_bytes([bytes[4], bytes[5]]); + let r3 = u16::from_be_bytes([bytes[6], bytes[7]]); + let r4 = u16::from_be_bytes([bytes[8], bytes[9]]); + let r5 = u32::from_be_bytes([bytes[10], bytes[11], bytes[12], bytes[13]]); + let r6 = u16::from_be_bytes([bytes[14], bytes[15]]); + + // Set version (4) and variant bits according to UUID v4 spec + let r3_v4 = (r3 & 0x0fff) | 0x4000; // Version 4 + let r4_variant = (r4 & 0x3fff) | 0x8000; // Variant 10 + + // Format as standard UUID: xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx + // where y is one of 8, 9, A, or B + format!("{r1:08x}-{r2:04x}-{r3_v4:04x}-{r4_variant:04x}-{r5:08x}{r6:04x}") +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/codspeed/src/walltime_results.rs b/crates/codspeed/src/walltime_results.rs index b7f0c491..dc67de84 100644 --- a/crates/codspeed/src/walltime_results.rs +++ b/crates/codspeed/src/walltime_results.rs @@ -7,6 +7,8 @@ use std::{ use serde::{Deserialize, Serialize}; use statrs::statistics::{Data, Distribution, Max, Min, OrderStatistics}; +use crate::utils::generate_unique_id; + const IQR_OUTLIER_FACTOR: f64 = 1.5; const STDEV_OUTLIER_FACTOR: f64 = 3.0; @@ -191,7 +193,7 @@ impl WalltimeBenchmark { fn dump_to_results(&self, workspace_root: &Path, scope: &str) { let output_dir = result_dir_from_workspace_root(workspace_root).join(scope); std::fs::create_dir_all(&output_dir).unwrap(); - let bench_id = uuid::Uuid::new_v4().to_string(); + let bench_id = generate_unique_id(); let output_path = output_dir.join(format!("{bench_id}.json")); let mut writer = std::fs::File::create(&output_path).expect("Failed to create the file"); serde_json::to_writer_pretty(&mut writer, self).expect("Failed to write the data"); diff --git a/crates/criterion_compat/Cargo.toml b/crates/criterion_compat/Cargo.toml index 54b9f96c..dcc1a13f 100644 --- a/crates/criterion_compat/Cargo.toml +++ b/crates/criterion_compat/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "codspeed-criterion-compat" -version = "4.0.5" +version = "4.1.0" rust-version = "1.74" # MSRV edition = "2021" description = "Criterion.rs compatibility layer for CodSpeed" @@ -11,14 +11,14 @@ repository = "https://github.com/CodSpeedHQ/codspeed-rust" homepage = "https://codspeed.io" license = "MIT OR Apache-2.0" categories = [ - "development-tools", - "development-tools::profiling", - "development-tools::testing", + "development-tools", + "development-tools::profiling", + "development-tools::testing", ] keywords = ["codspeed", "benchmark", "criterion"] [dependencies] -criterion = { package = "codspeed-criterion-compat-walltime", path = "./criterion_fork", version = "=4.0.5", default-features = false } -codspeed = { path = "../codspeed", version = "=4.0.5" } +criterion = { package = "codspeed-criterion-compat-walltime", path = "./criterion_fork", version = "=4.1.0", default-features = false } +codspeed = { path = "../codspeed", version = "=4.1.0" } colored = "2.1.0" clap = { version = "4", default-features = false, features = ["std"] } regex = { version = "1.5", default-features = false, features = ["std"] } @@ -26,7 +26,7 @@ regex = { version = "1.5", default-features = false, features = ["std"] } futures = { version = "0.3", default-features = false, optional = true } smol = { version = "2.0", default-features = false, optional = true } tokio = { version = "1.39", default-features = false, features = [ - "rt", + "rt", ], optional = true } async-std = { version = "1.12", optional = true } diff --git a/crates/criterion_compat/criterion_fork/Cargo.toml b/crates/criterion_compat/criterion_fork/Cargo.toml index 22b98af4..0666850d 100644 --- a/crates/criterion_compat/criterion_fork/Cargo.toml +++ b/crates/criterion_compat/criterion_fork/Cargo.toml @@ -4,7 +4,7 @@ authors = [ "Brook Heisler ", ] name = "codspeed-criterion-compat-walltime" -version = "4.0.5" +version = "4.1.0" edition = "2018" description = "Statistics-driven micro-benchmarking library" @@ -17,7 +17,7 @@ license = "Apache-2.0 OR MIT" exclude = ["book/*"] [dependencies] -codspeed = { path = "../../codspeed", version = "=4.0.5" } +codspeed = { path = "../../codspeed", version = "=4.1.0" } anes = "0.1.4" once_cell = "1.14" criterion-plot = { version = "0.5.0" } diff --git a/crates/divan_compat/Cargo.toml b/crates/divan_compat/Cargo.toml index b45effe7..ea9c73a9 100644 --- a/crates/divan_compat/Cargo.toml +++ b/crates/divan_compat/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "codspeed-divan-compat" -version = "4.0.5" +version = "4.1.0" rust-version = "1.80" edition = "2021" description = "Divan compatibility layer for CodSpeed" @@ -11,16 +11,16 @@ repository = "https://github.com/CodSpeedHQ/codspeed-rust" homepage = "https://codspeed.io" license = "MIT OR Apache-2.0" categories = [ - "development-tools", - "development-tools::profiling", - "development-tools::testing", + "development-tools", + "development-tools::profiling", + "development-tools::testing", ] keywords = ["codspeed", "benchmark", "divan"] [dependencies] -codspeed = { path = "../codspeed", version = "=4.0.5" } -divan = { package = "codspeed-divan-compat-walltime", path = "./divan_fork", version = "=4.0.5" } -codspeed-divan-compat-macros = { version = "=4.0.5", path = './macros' } +codspeed = { path = "../codspeed", version = "=4.1.0" } +divan = { package = "codspeed-divan-compat-walltime", path = "./divan_fork", version = "=4.1.0" } +codspeed-divan-compat-macros = { version = "=4.1.0", path = './macros' } regex = "1.11.3" clap = { version = "4", default-features = false, features = ["std", "env"] } diff --git a/crates/divan_compat/divan_fork/Cargo.toml b/crates/divan_compat/divan_fork/Cargo.toml index c1e2f286..af228c7d 100644 --- a/crates/divan_compat/divan_fork/Cargo.toml +++ b/crates/divan_compat/divan_fork/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "codspeed-divan-compat-walltime" -version = "4.0.5" +version = "4.1.0" rust-version = "1.80.0" edition = "2021" authors = ["Nikolai Vazquez"] @@ -23,7 +23,7 @@ regex = { package = "regex-lite", version = "0.1", default-features = false, fea "std", "string", ] } -codspeed = { path = "../../codspeed", version = "=4.0.5" } +codspeed = { path = "../../codspeed", version = "=4.1.0" } [target.'cfg(unix)'.dependencies] libc = "0.2.148" diff --git a/crates/divan_compat/examples/Cargo.toml b/crates/divan_compat/examples/Cargo.toml index 188a5c0c..1e9ea793 100644 --- a/crates/divan_compat/examples/Cargo.toml +++ b/crates/divan_compat/examples/Cargo.toml @@ -40,3 +40,7 @@ harness = false [[bench]] name = "the_algorithms" harness = false + +[[bench]] +name = "counters" +harness = false diff --git a/crates/divan_compat/examples/benches/counters.rs b/crates/divan_compat/examples/benches/counters.rs new file mode 100644 index 00000000..6cb4bd11 --- /dev/null +++ b/crates/divan_compat/examples/benches/counters.rs @@ -0,0 +1,160 @@ +//! Example benchmarks demonstrating divan counter usage +//! +//! This file shows how to use different types of counters with divan: +//! - BytesCount: for measuring throughput in bytes +//! - ItemsCount: for counting processed items +//! - CharsCount: for counting processed characters +//! - CyclesCount: for counting processing cycles + +use divan::{counter::*, AllocProfiler, Bencher}; + +#[global_allocator] +static ALLOC: AllocProfiler = AllocProfiler::system(); + +fn main() { + divan::main(); +} + +/// Example data for benchmarks +const SAMPLE_DATA: &[i32] = &[1, 2, 3, 4, 5, 6, 7, 8, 9, 10]; +const SAMPLE_TEXT: &str = "Hello, world! This is a sample string for benchmarking."; + +mod bytes_counter_examples { + use super::*; + + #[divan::bench] + fn vec_copy_with_bytes_counter(bencher: Bencher) { + let data = SAMPLE_DATA; + let bytes = BytesCount::of_slice(data); + + bencher + .counter(bytes) + .bench(|| -> Vec { divan::black_box(data).to_vec() }); + } + + #[divan::bench] + fn string_copy_with_bytes_counter(bencher: Bencher) { + let text = SAMPLE_TEXT; + let bytes = BytesCount::of_str(text); + + bencher + .counter(bytes) + .bench(|| -> String { divan::black_box(text).to_owned() }); + } + + #[divan::bench] + fn slice_into_vec_with_bytes(bencher: Bencher) { + let ints = &[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]; + let bytes = BytesCount::of_slice(ints); + + bencher + .counter(bytes) + .bench(|| -> Vec { divan::black_box(ints).into() }); + } +} + +mod items_counter_examples { + use super::*; + + #[divan::bench] + fn process_items_with_counter(bencher: Bencher) { + let data = SAMPLE_DATA; + let items = ItemsCount::new(data.len()); + + bencher + .counter(items) + .bench(|| -> Vec { divan::black_box(data).iter().map(|x| x * 2).collect() }); + } + + #[divan::bench] + fn filter_items_with_counter(bencher: Bencher) { + let data = (1..=100).collect::>(); + let items = ItemsCount::new(data.len()); + + bencher.counter(items).bench(|| -> Vec { + divan::black_box(&data) + .iter() + .filter(|&&x| x % 2 == 0) + .copied() + .collect() + }); + } +} + +mod chars_counter_examples { + use super::*; + + #[divan::bench] + fn count_chars_in_string(bencher: Bencher) { + let text = SAMPLE_TEXT; + let chars = CharsCount::of_str(text); + + bencher + .counter(chars) + .bench(|| -> usize { divan::black_box(text).chars().count() }); + } + + #[divan::bench] + fn uppercase_chars_with_counter(bencher: Bencher) { + let text = "hello world with unicode: café naïve résumé"; + let chars = CharsCount::of_str(text); + + bencher + .counter(chars) + .bench(|| -> String { divan::black_box(text).to_uppercase() }); + } +} + +mod cycles_counter_examples { + use super::*; + + #[divan::bench] + fn simulated_processing_cycles(bencher: Bencher) { + // Simulate processing 1000 "cycles" of work + let cycles = CyclesCount::new(1000u32); + + bencher.counter(cycles).bench(|| { + // Simulate some work that processes 1000 cycles + let mut sum = 0u64; + for i in 0..1000 { + sum = sum.wrapping_add(divan::black_box(i)); + } + sum + }); + } + + #[divan::bench] + fn hash_computation_cycles(bencher: Bencher) { + let data = SAMPLE_DATA; + // Treat each hash operation as processing N cycles where N = data length + let cycles = CyclesCount::new(data.len()); + + bencher.counter(cycles).bench(|| -> u64 { + use std::collections::hash_map::DefaultHasher; + use std::hash::{Hash, Hasher}; + + let mut hasher = DefaultHasher::new(); + divan::black_box(data).hash(&mut hasher); + hasher.finish() + }); + } +} + +mod multiple_counters_examples { + use super::*; + + #[divan::bench(counters = [BytesCount::of_slice(SAMPLE_DATA), ItemsCount::new(SAMPLE_DATA.len())])] + fn process_with_multiple_counters() -> Vec { + SAMPLE_DATA.iter().map(|x| x * x).collect() + } + + #[divan::bench] + fn string_processing_multi_counter(bencher: Bencher) { + let text = "Processing this text with multiple counters"; + + bencher + .counter(BytesCount::of_str(text)) + .counter(CharsCount::of_str(text)) + .bench(|| -> Vec { divan::black_box(text).chars().collect() }); + } +} diff --git a/crates/divan_compat/examples/benches/time_scale.rs b/crates/divan_compat/examples/benches/time_scale.rs index c4c8baa7..05f314e1 100644 --- a/crates/divan_compat/examples/benches/time_scale.rs +++ b/crates/divan_compat/examples/benches/time_scale.rs @@ -2,24 +2,29 @@ fn main() { divan::main(); } +fn busy_sleep(duration: std::time::Duration) { + let start = std::time::Instant::now(); + while start.elapsed() < duration {} +} + #[divan::bench] fn sleep_1ns() { - std::thread::sleep(std::time::Duration::from_nanos(1)); + busy_sleep(std::time::Duration::from_nanos(1)); } #[divan::bench] fn sleep_100ns() { - std::thread::sleep(std::time::Duration::from_nanos(100)); + busy_sleep(std::time::Duration::from_nanos(100)); } #[divan::bench] fn sleep_1us() { - std::thread::sleep(std::time::Duration::from_micros(1)); + busy_sleep(std::time::Duration::from_micros(1)); } #[divan::bench] fn sleep_100us() { - std::thread::sleep(std::time::Duration::from_micros(100)); + busy_sleep(std::time::Duration::from_micros(100)); } #[divan::bench] diff --git a/crates/divan_compat/macros/Cargo.toml b/crates/divan_compat/macros/Cargo.toml index 8231cd3e..5267b853 100644 --- a/crates/divan_compat/macros/Cargo.toml +++ b/crates/divan_compat/macros/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "codspeed-divan-compat-macros" -version = "4.0.5" +version = "4.1.0" rust-version = "1.80" edition = "2021" description = "Macros for the Divan compatibility layer for CodSpeed" diff --git a/crates/divan_compat/macros/src/args.rs b/crates/divan_compat/macros/src/args.rs index da32978c..4a854358 100644 --- a/crates/divan_compat/macros/src/args.rs +++ b/crates/divan_compat/macros/src/args.rs @@ -45,6 +45,13 @@ impl AttrOptions { // These arguments are ignored for codspeed runs meta.value()?.parse::()?; // Discard the value } + "counters" => { + // Counters are not yet supported, but we parse them to avoid errors + eprintln!( + "Warning: Counter feature is not yet supported by codspeed-divan-compat" + ); + meta.value()?.parse::()?; // Discard the value + } _ => { let path = meta.path.clone(); let parsed_meta = if meta.input.is_empty() { diff --git a/crates/divan_compat/src/compat/bench/mod.rs b/crates/divan_compat/src/compat/bench/mod.rs index 5ffd3fd3..5ec824b9 100644 --- a/crates/divan_compat/src/compat/bench/mod.rs +++ b/crates/divan_compat/src/compat/bench/mod.rs @@ -51,6 +51,27 @@ impl<'a, 'b> Bencher<'a, 'b> { _marker: self._marker, } } + + /// Add a counter to this benchmark (placeholder implementation). + /// + /// Note: Counters are not yet supported by codspeed-divan-compat. + /// This method is provided for API compatibility but does not affect benchmarking. + pub fn counter(self, _counter: C) -> Self { + eprintln!("Warning: Counter feature is not yet supported by codspeed-divan-compat"); + self + } + + /// Add a counter based on input to this benchmark (placeholder implementation). + /// + /// Note: Counters are not yet supported by codspeed-divan-compat. + /// This method is provided for API compatibility but does not affect benchmarking. + pub fn input_counter(self, _counter_fn: F) -> Self + where + F: Fn() -> C, + { + eprintln!("Warning: Counter feature is not yet supported by codspeed-divan-compat"); + self + } } impl<'a, 'b> Bencher<'a, 'b> { @@ -73,6 +94,26 @@ impl<'a, 'b, I, GenI> Bencher<'a, 'b, BencherConfig> where GenI: FnMut() -> I, { + /// Add a counter to this benchmark (placeholder implementation). + /// + /// Note: Counters are not yet supported by codspeed-divan-compat. + /// This method is provided for API compatibility but does not affect benchmarking. + pub fn counter(self, _counter: C) -> Self { + eprintln!("Warning: Counter feature is not yet supported by codspeed-divan-compat"); + self + } + + /// Add a counter based on input to this benchmark (placeholder implementation). + /// + /// Note: Counters are not yet supported by codspeed-divan-compat. + /// This method is provided for API compatibility but does not affect benchmarking. + pub fn input_counter(self, _counter_fn: F) -> Self + where + F: Fn(&I) -> C, + { + eprintln!("Warning: Counter feature is not yet supported by codspeed-divan-compat"); + self + } pub fn bench_values(self, benched: B) where B: Fn(I) -> O + Sync, diff --git a/crates/divan_compat/src/compat/bench/options.rs b/crates/divan_compat/src/compat/bench/options.rs index c55e0fd6..3b596c8e 100644 --- a/crates/divan_compat/src/compat/bench/options.rs +++ b/crates/divan_compat/src/compat/bench/options.rs @@ -10,4 +10,10 @@ pub struct BenchOptions { /// This may be set within the attribute or with a separate /// [`#[ignore]`](https://doc.rust-lang.org/reference/attributes/testing.html#the-ignore-attribute). pub ignore: Option, + + /// Counters to be used with this benchmark. + /// + /// Note: Counters are not yet supported by codspeed-divan-compat. + /// This field is provided for API compatibility but does not affect benchmarking. + pub counters: Option>, } diff --git a/crates/divan_compat/src/compat/mod.rs b/crates/divan_compat/src/compat/mod.rs index 99b695b6..5fa6d558 100644 --- a/crates/divan_compat/src/compat/mod.rs +++ b/crates/divan_compat/src/compat/mod.rs @@ -20,6 +20,141 @@ mod uri; mod util; pub use bench::*; + +// Counter types (placeholder implementations) +pub mod counter { + /// Process N bytes. + #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] + pub struct BytesCount { + count: u64, + } + + /// Process N [`char`s](char). + #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] + pub struct CharsCount { + count: u64, + } + + /// Process N cycles, displayed as Hertz. + #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] + pub struct CyclesCount { + count: u64, + } + + /// Process N items. + #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] + pub struct ItemsCount { + count: u64, + } + + impl BytesCount { + /// Count N bytes. + #[inline] + pub fn new(count: N) -> Self + where + N: TryInto, + N::Error: std::fmt::Debug, + { + eprintln!("Warning: Counter feature is not yet supported by codspeed-divan-compat"); + Self { + count: count.try_into().unwrap(), + } + } + + /// Counts the size of a type with [`size_of`]. + #[inline] + pub fn of() -> Self { + eprintln!("Warning: Counter feature is not yet supported by codspeed-divan-compat"); + Self { + count: std::mem::size_of::() as u64, + } + } + + /// Counts the size of multiple instances of a type with [`size_of`]. + #[inline] + pub fn of_many(n: usize) -> Self { + eprintln!("Warning: Counter feature is not yet supported by codspeed-divan-compat"); + Self { + count: (std::mem::size_of::() * n) as u64, + } + } + + /// Counts the size of a value with [`size_of_val`]. + #[inline] + pub fn of_val(val: &T) -> Self { + eprintln!("Warning: Counter feature is not yet supported by codspeed-divan-compat"); + Self { + count: std::mem::size_of_val(val) as u64, + } + } + + /// Counts the bytes of a [`&str`]. + #[inline] + pub fn of_str>(s: &S) -> Self { + eprintln!("Warning: Counter feature is not yet supported by codspeed-divan-compat"); + Self::of_val(s.as_ref()) + } + + /// Counts the bytes of a [slice](prim@slice). + #[inline] + pub fn of_slice>(s: &S) -> Self { + eprintln!("Warning: Counter feature is not yet supported by codspeed-divan-compat"); + Self::of_val(s.as_ref()) + } + } + + impl CharsCount { + /// Count N [`char`s](char). + #[inline] + pub fn new(count: N) -> Self + where + N: TryInto, + N::Error: std::fmt::Debug, + { + eprintln!("Warning: Counter feature is not yet supported by codspeed-divan-compat"); + Self { + count: count.try_into().unwrap(), + } + } + + /// Counts the [`char`s](prim@char) of a [`&str`](prim@str). + #[inline] + pub fn of_str>(s: &S) -> Self { + eprintln!("Warning: Counter feature is not yet supported by codspeed-divan-compat"); + Self::new(s.as_ref().chars().count() as u64) + } + } + + impl CyclesCount { + /// Count N cycles. + #[inline] + pub fn new(count: N) -> Self + where + N: TryInto, + N::Error: std::fmt::Debug, + { + eprintln!("Warning: Counter feature is not yet supported by codspeed-divan-compat"); + Self { + count: count.try_into().unwrap(), + } + } + } + + impl ItemsCount { + /// Count N items. + #[inline] + pub fn new(count: N) -> Self + where + N: TryInto, + N::Error: std::fmt::Debug, + { + eprintln!("Warning: Counter feature is not yet supported by codspeed-divan-compat"); + Self { + count: count.try_into().unwrap(), + } + } + } +} use codspeed::codspeed::CodSpeed; use config::Filter; use entry::AnyBenchEntry;