From 9815ee4093735a1d594b45eab843daa17a2e8fad Mon Sep 17 00:00:00 2001 From: Siew Kam Onn Date: Thu, 26 Feb 2026 15:39:25 +0800 Subject: [PATCH 01/12] Add timing summary feature for SQL logic tests --- datafusion/sqllogictest/bin/sqllogictests.rs | 127 +++++++++++++++++-- 1 file changed, 118 insertions(+), 9 deletions(-) diff --git a/datafusion/sqllogictest/bin/sqllogictests.rs b/datafusion/sqllogictest/bin/sqllogictests.rs index 463b7b03a760c..7c643e61347ee 100644 --- a/datafusion/sqllogictest/bin/sqllogictests.rs +++ b/datafusion/sqllogictest/bin/sqllogictests.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -use clap::{ColorChoice, Parser}; +use clap::{ColorChoice, Parser, ValueEnum}; use datafusion::common::instant::Instant; use datafusion::common::utils::get_available_parallelism; use datafusion::common::{DataFusionError, Result, exec_datafusion_err, exec_err}; @@ -49,6 +49,7 @@ use std::path::{Path, PathBuf}; use std::str::FromStr; use std::sync::Arc; use std::sync::atomic::{AtomicUsize, Ordering}; +use std::time::Duration; #[cfg(feature = "postgres")] mod postgres_container; @@ -59,6 +60,20 @@ const PG_COMPAT_FILE_PREFIX: &str = "pg_compat_"; const SQLITE_PREFIX: &str = "sqlite"; const ERRS_PER_FILE_LIMIT: usize = 10; +#[derive(Clone, Copy, Debug, Eq, PartialEq, ValueEnum)] +enum TimingSummaryMode { + Auto, + Off, + Top, + Full, +} + +#[derive(Debug)] +struct FileTiming { + relative_path: PathBuf, + elapsed: Duration, +} + pub fn main() -> Result<()> { tokio::runtime::Builder::new_multi_thread() .enable_all() @@ -160,7 +175,7 @@ async fn run_tests() -> Result<()> { let is_ci = !stderr().is_terminal(); let completed_count = Arc::new(AtomicUsize::new(0)); - let errors: Vec<_> = futures::stream::iter(test_files) + let file_results: Vec<_> = futures::stream::iter(test_files) .map(|test_file| { let validator = if options.include_sqlite && test_file.relative_path.starts_with(SQLITE_PREFIX) @@ -255,7 +270,14 @@ async fn run_tests() -> Result<()> { Ok(()) }) .join() - .map(move |result| (result, relative_path, currently_running_sql_tracker)) + .map(move |result| { + ( + result, + relative_path, + currently_running_sql_tracker, + file_start.elapsed(), + ) + }) }) // run up to num_cpus streams in parallel .buffer_unordered(options.test_threads) @@ -274,10 +296,30 @@ async fn run_tests() -> Result<()> { } } }) - .flat_map(|(result, test_file_path, current_sql)| { + .collect() + .await; + + let mut file_timings: Vec = file_results + .iter() + .map(|(_, path, _, elapsed)| FileTiming { + relative_path: path.clone(), + elapsed: *elapsed, + }) + .collect(); + + file_timings.sort_by(|a, b| { + b.elapsed + .cmp(&a.elapsed) + .then_with(|| a.relative_path.cmp(&b.relative_path)) + }); + + print_timing_summary(&options, &m, is_ci, &file_timings)?; + + let errors: Vec<_> = file_results + .into_iter() + .filter_map(|(result, test_file_path, current_sql, _)| { // Filter out any Ok() leaving only the DataFusionErrors - futures::stream::iter(match result { - // Tokio panic error + match result { Err(e) => { let error = DataFusionError::External(Box::new(e)); let current_sql = current_sql.get_currently_running_sqls(); @@ -307,10 +349,9 @@ async fn run_tests() -> Result<()> { } } Ok(thread_result) => thread_result.err(), - }) + } }) - .collect() - .await; + .collect(); m.println(format!( "Completed {} test files in {}", @@ -332,6 +373,44 @@ async fn run_tests() -> Result<()> { } } +fn print_timing_summary( + options: &Options, + progress: &MultiProgress, + is_ci: bool, + file_timings: &[FileTiming], +) -> Result<()> { + let mode = options.timing_summary_mode(is_ci); + if mode == TimingSummaryMode::Off || file_timings.is_empty() { + return Ok(()); + } + + let top_n = options.timing_top_n.max(1); + let count = match mode { + TimingSummaryMode::Off => 0, + TimingSummaryMode::Auto | TimingSummaryMode::Top => top_n, + TimingSummaryMode::Full => file_timings.len(), + }; + + progress.println("Per-file elapsed summary (deterministic):")?; + for (idx, timing) in file_timings.iter().take(count).enumerate() { + progress.println(format!( + "{:>3}. {:>8.3}s {}", + idx + 1, + timing.elapsed.as_secs_f64(), + timing.relative_path.display() + ))?; + } + + if mode != TimingSummaryMode::Full && file_timings.len() > count { + progress.println(format!( + "... {} more files omitted (use --timing-summary full to show all)", + file_timings.len() - count + ))?; + } + + Ok(()) +} + async fn run_test_file_substrait_round_trip( test_file: TestFile, validator: Validator, @@ -825,6 +904,23 @@ struct Options { )] test_threads: usize, + #[clap( + long, + env = "SLT_TIMING_SUMMARY", + value_enum, + default_value_t = TimingSummaryMode::Auto, + help = "Per-file timing summary mode: auto|off|top|full" + )] + timing_summary: TimingSummaryMode, + + #[clap( + long, + env = "SLT_TIMING_TOP_N", + default_value_t = 10, + help = "Number of files to show when timing summary mode is auto/top" + )] + timing_top_n: usize, + #[clap( long, value_name = "MODE", @@ -835,6 +931,19 @@ struct Options { } impl Options { + fn timing_summary_mode(&self, is_ci: bool) -> TimingSummaryMode { + match self.timing_summary { + TimingSummaryMode::Auto => { + if is_ci { + TimingSummaryMode::Top + } else { + TimingSummaryMode::Off + } + } + mode => mode, + } + } + /// Because this test can be run as a cargo test, commands like /// /// ```shell From 76c797916c88d9d645b9c3cdc3fe80afc3ade9c7 Mon Sep 17 00:00:00 2001 From: Siew Kam Onn Date: Thu, 26 Feb 2026 15:45:21 +0800 Subject: [PATCH 02/12] Add per-file timing summary feature to README for sqllogictests --- datafusion/sqllogictest/README.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/datafusion/sqllogictest/README.md b/datafusion/sqllogictest/README.md index 8768deee3d87e..60d5758b88f73 100644 --- a/datafusion/sqllogictest/README.md +++ b/datafusion/sqllogictest/README.md @@ -70,6 +70,26 @@ cargo test --test sqllogictests -- ddl --complete RUST_LOG=debug cargo test --test sqllogictests -- ddl ``` +### Per-file timing summary + +The sqllogictest runner can emit deterministic per-file elapsed timings to help +identify slow test files. + +```shell +# Show top 10 slowest files (good for CI) +cargo test --test sqllogictests -- --timing-summary top --timing-top-n 10 +``` + +```shell +# Show full per-file timing table +cargo test --test sqllogictests -- --timing-summary full +``` + +```shell +# Same controls via environment variables +SLT_TIMING_SUMMARY=top SLT_TIMING_TOP_N=15 cargo test --test sqllogictests +``` + ## Cookbook: Adding Tests 1. Add queries From 5b81cfefda72aad3de4cd8bb4ba4825bf10036d0 Mon Sep 17 00:00:00 2001 From: Siew Kam Onn Date: Thu, 26 Feb 2026 15:57:32 +0800 Subject: [PATCH 03/12] Refactor color choice parsing to use associated type for clarity --- datafusion/sqllogictest/bin/sqllogictests.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datafusion/sqllogictest/bin/sqllogictests.rs b/datafusion/sqllogictest/bin/sqllogictests.rs index 7c643e61347ee..dfc961c24f9f8 100644 --- a/datafusion/sqllogictest/bin/sqllogictests.rs +++ b/datafusion/sqllogictest/bin/sqllogictests.rs @@ -995,7 +995,7 @@ impl Options { ColorChoice::Never => false, ColorChoice::Auto => { // CARGO_TERM_COLOR takes precedence over auto-detection - let cargo_term_color = ColorChoice::from_str( + let cargo_term_color = ::from_str( &std::env::var("CARGO_TERM_COLOR") .unwrap_or_else(|_| "auto".to_string()), ) From 6f1879910972e18a5a43f03abf05747450c1cca3 Mon Sep 17 00:00:00 2001 From: Siew Kam Onn Date: Thu, 26 Feb 2026 16:11:21 +0800 Subject: [PATCH 04/12] Add optional debug logging for slow test files in sqllogictests --- datafusion/sqllogictest/README.md | 5 +++ datafusion/sqllogictest/bin/sqllogictests.rs | 32 +++++++++++++++----- 2 files changed, 29 insertions(+), 8 deletions(-) diff --git a/datafusion/sqllogictest/README.md b/datafusion/sqllogictest/README.md index 60d5758b88f73..4052578d3eea8 100644 --- a/datafusion/sqllogictest/README.md +++ b/datafusion/sqllogictest/README.md @@ -90,6 +90,11 @@ cargo test --test sqllogictests -- --timing-summary full SLT_TIMING_SUMMARY=top SLT_TIMING_TOP_N=15 cargo test --test sqllogictests ``` +```shell +# Optional debug logging for per-task slow files (>30s), disabled by default +SLT_TIMING_DEBUG_SLOW_FILES=1 cargo test --test sqllogictests +``` + ## Cookbook: Adding Tests 1. Add queries diff --git a/datafusion/sqllogictest/bin/sqllogictests.rs b/datafusion/sqllogictest/bin/sqllogictests.rs index dfc961c24f9f8..1b542c7486883 100644 --- a/datafusion/sqllogictest/bin/sqllogictests.rs +++ b/datafusion/sqllogictest/bin/sqllogictests.rs @@ -59,6 +59,7 @@ const DATAFUSION_TESTING_TEST_DIRECTORY: &str = "../../datafusion-testing/data/" const PG_COMPAT_FILE_PREFIX: &str = "pg_compat_"; const SQLITE_PREFIX: &str = "sqlite"; const ERRS_PER_FILE_LIMIT: usize = 10; +const TIMING_DEBUG_SLOW_FILES_ENV: &str = "SLT_TIMING_DEBUG_SLOW_FILES"; #[derive(Clone, Copy, Debug, Eq, PartialEq, ValueEnum)] enum TimingSummaryMode { @@ -115,6 +116,7 @@ async fn run_tests() -> Result<()> { env_logger::init(); let options: Options = Parser::parse(); + let timing_debug_slow_files = is_env_truthy(TIMING_DEBUG_SLOW_FILES_ENV); if options.list { // nextest parses stdout, so print messages to stderr eprintln!("NOTICE: --list option unsupported, quitting"); @@ -188,6 +190,7 @@ async fn run_tests() -> Result<()> { let m_clone = m.clone(); let m_style_clone = m_style.clone(); let filters = options.filters.clone(); + let timing_debug_slow_files = timing_debug_slow_files; let relative_path = test_file.relative_path.clone(); let relative_path_for_timing = test_file.relative_path.clone(); @@ -258,14 +261,16 @@ async fn run_tests() -> Result<()> { .await? } }; - // Log slow files (>30s) for CI debugging - let elapsed = file_start.elapsed(); - if elapsed.as_secs() > 30 { - eprintln!( - "Slow file: {} took {:.1}s", - relative_path_for_timing.display(), - elapsed.as_secs_f64() - ); + + if timing_debug_slow_files { + let elapsed = file_start.elapsed(); + if elapsed.as_secs() > 30 { + eprintln!( + "Slow file: {} took {:.1}s", + relative_path_for_timing.display(), + elapsed.as_secs_f64() + ); + } } Ok(()) }) @@ -411,6 +416,17 @@ fn print_timing_summary( Ok(()) } +fn is_env_truthy(name: &str) -> bool { + std::env::var_os(name) + .and_then(|value| value.into_string().ok()) + .is_some_and(|value| { + matches!( + value.trim().to_ascii_lowercase().as_str(), + "1" | "true" | "yes" | "on" + ) + }) +} + async fn run_test_file_substrait_round_trip( test_file: TestFile, validator: Validator, From b302f7c1479072c8fc3594129a0ba3280f395300 Mon Sep 17 00:00:00 2001 From: Siew Kam Onn Date: Thu, 26 Feb 2026 16:25:17 +0800 Subject: [PATCH 05/12] Update elapsed time handling in sqllogictests.rs Capture elapsed time once in spawned per-file task and reuse after join. Remove redundant post-join measurement while maintaining existing error behavior. Implement safe fallback to Duration::ZERO for join-level panics or errors where elapsed time is not available. --- datafusion/sqllogictest/bin/sqllogictests.rs | 26 ++++++++++++-------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/datafusion/sqllogictest/bin/sqllogictests.rs b/datafusion/sqllogictest/bin/sqllogictests.rs index 1b542c7486883..1867f5ebfc4ab 100644 --- a/datafusion/sqllogictest/bin/sqllogictests.rs +++ b/datafusion/sqllogictest/bin/sqllogictests.rs @@ -200,7 +200,7 @@ async fn run_tests() -> Result<()> { currently_running_sql_tracker.clone(); let file_start = Instant::now(); SpawnedTask::spawn(async move { - match ( + let result = match ( options.postgres_runner, options.complete, options.substrait_round_trip, @@ -215,7 +215,7 @@ async fn run_tests() -> Result<()> { currently_running_sql_tracker_clone, colored_output, ) - .await? + .await } (false, false, _) => { run_test_file( @@ -227,7 +227,7 @@ async fn run_tests() -> Result<()> { currently_running_sql_tracker_clone, colored_output, ) - .await? + .await } (false, true, _) => { run_complete_file( @@ -237,7 +237,7 @@ async fn run_tests() -> Result<()> { m_style_clone, currently_running_sql_tracker_clone, ) - .await? + .await } (true, false, _) => { run_test_file_with_postgres( @@ -248,7 +248,7 @@ async fn run_tests() -> Result<()> { filters.as_ref(), currently_running_sql_tracker_clone, ) - .await? + .await } (true, true, _) => { run_complete_file_with_postgres( @@ -258,12 +258,12 @@ async fn run_tests() -> Result<()> { m_style_clone, currently_running_sql_tracker_clone, ) - .await? + .await } }; + let elapsed = file_start.elapsed(); if timing_debug_slow_files { - let elapsed = file_start.elapsed(); if elapsed.as_secs() > 30 { eprintln!( "Slow file: {} took {:.1}s", @@ -272,15 +272,21 @@ async fn run_tests() -> Result<()> { ); } } - Ok(()) + + (result, elapsed) }) .join() .map(move |result| { + let elapsed = match &result { + Ok((_, elapsed)) => *elapsed, + Err(_) => Duration::ZERO, + }; + ( - result, + result.map(|(thread_result, _)| thread_result), relative_path, currently_running_sql_tracker, - file_start.elapsed(), + elapsed, ) }) }) From 080e53a58f93e9c46e2e6aa12708287b5b482383 Mon Sep 17 00:00:00 2001 From: Siew Kam Onn Date: Thu, 26 Feb 2026 16:27:11 +0800 Subject: [PATCH 06/12] Enforce parse-time validation for --timing-top-n Ensure --timing-top-n accepts only values >= 1 by using clap's value parser with a defined range. Update help text to reflect this new requirement and clarify in README.md to avoid silent runtime coercion. --- datafusion/sqllogictest/README.md | 2 ++ datafusion/sqllogictest/bin/sqllogictests.rs | 5 +++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/datafusion/sqllogictest/README.md b/datafusion/sqllogictest/README.md index 4052578d3eea8..05e24853ec972 100644 --- a/datafusion/sqllogictest/README.md +++ b/datafusion/sqllogictest/README.md @@ -75,6 +75,8 @@ RUST_LOG=debug cargo test --test sqllogictests -- ddl The sqllogictest runner can emit deterministic per-file elapsed timings to help identify slow test files. +`--timing-top-n` / `SLT_TIMING_TOP_N` must be a positive integer (`>= 1`). + ```shell # Show top 10 slowest files (good for CI) cargo test --test sqllogictests -- --timing-summary top --timing-top-n 10 diff --git a/datafusion/sqllogictest/bin/sqllogictests.rs b/datafusion/sqllogictest/bin/sqllogictests.rs index 1867f5ebfc4ab..12ae3a815e4f5 100644 --- a/datafusion/sqllogictest/bin/sqllogictests.rs +++ b/datafusion/sqllogictest/bin/sqllogictests.rs @@ -395,7 +395,7 @@ fn print_timing_summary( return Ok(()); } - let top_n = options.timing_top_n.max(1); + let top_n = options.timing_top_n; let count = match mode { TimingSummaryMode::Off => 0, TimingSummaryMode::Auto | TimingSummaryMode::Top => top_n, @@ -939,7 +939,8 @@ struct Options { long, env = "SLT_TIMING_TOP_N", default_value_t = 10, - help = "Number of files to show when timing summary mode is auto/top" + value_parser = clap::value_parser!(usize).range(1..), + help = "Number of files to show when timing summary mode is auto/top (must be >= 1)" )] timing_top_n: usize, From 7f1ee856f584b6d5fffa44931ddedbabe53101f9 Mon Sep 17 00:00:00 2001 From: Siew Kam Onn Date: Thu, 26 Feb 2026 16:28:09 +0800 Subject: [PATCH 07/12] Update README.md for timing summary behavior Clarify default behavior for timing summaries in TTY and non-TTY/CI runs. Maintain conciseness within the existing timing-summary section. --- datafusion/sqllogictest/README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/datafusion/sqllogictest/README.md b/datafusion/sqllogictest/README.md index 05e24853ec972..7d84ad23d5905 100644 --- a/datafusion/sqllogictest/README.md +++ b/datafusion/sqllogictest/README.md @@ -75,6 +75,9 @@ RUST_LOG=debug cargo test --test sqllogictests -- ddl The sqllogictest runner can emit deterministic per-file elapsed timings to help identify slow test files. +By default (`--timing-summary auto`), timing summary output is disabled in local +TTY runs and shows a top-slowest summary in non-TTY/CI runs. + `--timing-top-n` / `SLT_TIMING_TOP_N` must be a positive integer (`>= 1`). ```shell From 57c472eec2db0cec7e92421d4fc156b1a70362d1 Mon Sep 17 00:00:00 2001 From: Siew Kam Onn Date: Thu, 26 Feb 2026 16:47:51 +0800 Subject: [PATCH 08/12] Fix parser for usize in sqllogictests.rs Replace Clap parser call in sqllogictests.rs:949 with a custom parser function. Add validation to ensure usize values are >= 1 in lines 433-443, providing a clear error message for any input of 0. --- datafusion/sqllogictest/bin/sqllogictests.rs | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/datafusion/sqllogictest/bin/sqllogictests.rs b/datafusion/sqllogictest/bin/sqllogictests.rs index 12ae3a815e4f5..a4b5fd28d9e47 100644 --- a/datafusion/sqllogictest/bin/sqllogictests.rs +++ b/datafusion/sqllogictest/bin/sqllogictests.rs @@ -433,6 +433,16 @@ fn is_env_truthy(name: &str) -> bool { }) } +fn parse_timing_top_n(arg: &str) -> std::result::Result { + let parsed = arg + .parse::() + .map_err(|error| format!("invalid value '{arg}': {error}"))?; + if parsed == 0 { + return Err("must be >= 1".to_string()); + } + Ok(parsed) +} + async fn run_test_file_substrait_round_trip( test_file: TestFile, validator: Validator, @@ -939,7 +949,7 @@ struct Options { long, env = "SLT_TIMING_TOP_N", default_value_t = 10, - value_parser = clap::value_parser!(usize).range(1..), + value_parser = parse_timing_top_n, help = "Number of files to show when timing summary mode is auto/top (must be >= 1)" )] timing_top_n: usize, From 88714e98b7b070b1e0eaece030ff8e06417e109e Mon Sep 17 00:00:00 2001 From: Siew Kam Onn Date: Thu, 26 Feb 2026 17:00:11 +0800 Subject: [PATCH 09/12] clippy fix --- datafusion/sqllogictest/bin/sqllogictests.rs | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/datafusion/sqllogictest/bin/sqllogictests.rs b/datafusion/sqllogictest/bin/sqllogictests.rs index a4b5fd28d9e47..4e5ff81b5ef32 100644 --- a/datafusion/sqllogictest/bin/sqllogictests.rs +++ b/datafusion/sqllogictest/bin/sqllogictests.rs @@ -190,7 +190,6 @@ async fn run_tests() -> Result<()> { let m_clone = m.clone(); let m_style_clone = m_style.clone(); let filters = options.filters.clone(); - let timing_debug_slow_files = timing_debug_slow_files; let relative_path = test_file.relative_path.clone(); let relative_path_for_timing = test_file.relative_path.clone(); @@ -263,14 +262,12 @@ async fn run_tests() -> Result<()> { }; let elapsed = file_start.elapsed(); - if timing_debug_slow_files { - if elapsed.as_secs() > 30 { - eprintln!( - "Slow file: {} took {:.1}s", - relative_path_for_timing.display(), - elapsed.as_secs_f64() - ); - } + if timing_debug_slow_files && elapsed.as_secs() > 30 { + eprintln!( + "Slow file: {} took {:.1}s", + relative_path_for_timing.display(), + elapsed.as_secs_f64() + ); } (result, elapsed) From 789e64e31cc1d3a0870f891fa97f82dc45c6521c Mon Sep 17 00:00:00 2001 From: Siew Kam Onn Date: Thu, 26 Feb 2026 18:55:28 +0800 Subject: [PATCH 10/12] Refactor timing summary logic to simplify count determination based on mode --- datafusion/sqllogictest/bin/sqllogictests.rs | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/datafusion/sqllogictest/bin/sqllogictests.rs b/datafusion/sqllogictest/bin/sqllogictests.rs index 4e5ff81b5ef32..e067f2488d81f 100644 --- a/datafusion/sqllogictest/bin/sqllogictests.rs +++ b/datafusion/sqllogictest/bin/sqllogictests.rs @@ -393,10 +393,14 @@ fn print_timing_summary( } let top_n = options.timing_top_n; - let count = match mode { - TimingSummaryMode::Off => 0, - TimingSummaryMode::Auto | TimingSummaryMode::Top => top_n, - TimingSummaryMode::Full => file_timings.len(), + debug_assert!(matches!( + mode, + TimingSummaryMode::Top | TimingSummaryMode::Full + )); + let count = if mode == TimingSummaryMode::Full { + file_timings.len() + } else { + top_n }; progress.println("Per-file elapsed summary (deterministic):")?; From 2eb94d408c7c839b30409af2219c7256bff6270a Mon Sep 17 00:00:00 2001 From: Siew Kam Onn Date: Fri, 27 Feb 2026 17:01:41 +0800 Subject: [PATCH 11/12] Simplify timing UX and remove complexity Streamline timing summary to a single switch, enabling full deterministic per-file timings sorted slowest-first. Eliminate all mode and top-N options in sqllogictests.rs, including the removal of TimingSummaryMode and related auto branching for summary output. Update README.md to recommend Unix post-processing with `| head -n 10`. --- datafusion/sqllogictest/README.md | 18 +++-- datafusion/sqllogictest/bin/sqllogictests.rs | 75 ++------------------ 2 files changed, 15 insertions(+), 78 deletions(-) diff --git a/datafusion/sqllogictest/README.md b/datafusion/sqllogictest/README.md index 7d84ad23d5905..45d2d77058692 100644 --- a/datafusion/sqllogictest/README.md +++ b/datafusion/sqllogictest/README.md @@ -75,24 +75,22 @@ RUST_LOG=debug cargo test --test sqllogictests -- ddl The sqllogictest runner can emit deterministic per-file elapsed timings to help identify slow test files. -By default (`--timing-summary auto`), timing summary output is disabled in local -TTY runs and shows a top-slowest summary in non-TTY/CI runs. - -`--timing-top-n` / `SLT_TIMING_TOP_N` must be a positive integer (`>= 1`). +Timing summary output is disabled by default and enabled with +`--timing-summary` (or `SLT_TIMING_SUMMARY=1`). ```shell -# Show top 10 slowest files (good for CI) -cargo test --test sqllogictests -- --timing-summary top --timing-top-n 10 +# Show deterministic per-file elapsed timings (sorted slowest first) +cargo test --test sqllogictests -- --timing-summary ``` ```shell -# Show full per-file timing table -cargo test --test sqllogictests -- --timing-summary full +# Keep only the top 10 lines using standard shell tooling +cargo test --test sqllogictests -- --timing-summary | head -n 10 ``` ```shell -# Same controls via environment variables -SLT_TIMING_SUMMARY=top SLT_TIMING_TOP_N=15 cargo test --test sqllogictests +# Enable via environment variable +SLT_TIMING_SUMMARY=1 cargo test --test sqllogictests ``` ```shell diff --git a/datafusion/sqllogictest/bin/sqllogictests.rs b/datafusion/sqllogictest/bin/sqllogictests.rs index e067f2488d81f..a2bdd30ceef77 100644 --- a/datafusion/sqllogictest/bin/sqllogictests.rs +++ b/datafusion/sqllogictest/bin/sqllogictests.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -use clap::{ColorChoice, Parser, ValueEnum}; +use clap::{ColorChoice, Parser}; use datafusion::common::instant::Instant; use datafusion::common::utils::get_available_parallelism; use datafusion::common::{DataFusionError, Result, exec_datafusion_err, exec_err}; @@ -61,14 +61,6 @@ const SQLITE_PREFIX: &str = "sqlite"; const ERRS_PER_FILE_LIMIT: usize = 10; const TIMING_DEBUG_SLOW_FILES_ENV: &str = "SLT_TIMING_DEBUG_SLOW_FILES"; -#[derive(Clone, Copy, Debug, Eq, PartialEq, ValueEnum)] -enum TimingSummaryMode { - Auto, - Off, - Top, - Full, -} - #[derive(Debug)] struct FileTiming { relative_path: PathBuf, @@ -321,7 +313,7 @@ async fn run_tests() -> Result<()> { .then_with(|| a.relative_path.cmp(&b.relative_path)) }); - print_timing_summary(&options, &m, is_ci, &file_timings)?; + print_timing_summary(&options, &m, &file_timings)?; let errors: Vec<_> = file_results .into_iter() @@ -384,27 +376,14 @@ async fn run_tests() -> Result<()> { fn print_timing_summary( options: &Options, progress: &MultiProgress, - is_ci: bool, file_timings: &[FileTiming], ) -> Result<()> { - let mode = options.timing_summary_mode(is_ci); - if mode == TimingSummaryMode::Off || file_timings.is_empty() { + if !options.timing_summary || file_timings.is_empty() { return Ok(()); } - let top_n = options.timing_top_n; - debug_assert!(matches!( - mode, - TimingSummaryMode::Top | TimingSummaryMode::Full - )); - let count = if mode == TimingSummaryMode::Full { - file_timings.len() - } else { - top_n - }; - progress.println("Per-file elapsed summary (deterministic):")?; - for (idx, timing) in file_timings.iter().take(count).enumerate() { + for (idx, timing) in file_timings.iter().enumerate() { progress.println(format!( "{:>3}. {:>8.3}s {}", idx + 1, @@ -413,13 +392,6 @@ fn print_timing_summary( ))?; } - if mode != TimingSummaryMode::Full && file_timings.len() > count { - progress.println(format!( - "... {} more files omitted (use --timing-summary full to show all)", - file_timings.len() - count - ))?; - } - Ok(()) } @@ -434,16 +406,6 @@ fn is_env_truthy(name: &str) -> bool { }) } -fn parse_timing_top_n(arg: &str) -> std::result::Result { - let parsed = arg - .parse::() - .map_err(|error| format!("invalid value '{arg}': {error}"))?; - if parsed == 0 { - return Err("must be >= 1".to_string()); - } - Ok(parsed) -} - async fn run_test_file_substrait_round_trip( test_file: TestFile, validator: Validator, @@ -940,20 +902,10 @@ struct Options { #[clap( long, env = "SLT_TIMING_SUMMARY", - value_enum, - default_value_t = TimingSummaryMode::Auto, - help = "Per-file timing summary mode: auto|off|top|full" - )] - timing_summary: TimingSummaryMode, - - #[clap( - long, - env = "SLT_TIMING_TOP_N", - default_value_t = 10, - value_parser = parse_timing_top_n, - help = "Number of files to show when timing summary mode is auto/top (must be >= 1)" + default_value_t = false, + help = "Print deterministic per-file timing summary" )] - timing_top_n: usize, + timing_summary: bool, #[clap( long, @@ -965,19 +917,6 @@ struct Options { } impl Options { - fn timing_summary_mode(&self, is_ci: bool) -> TimingSummaryMode { - match self.timing_summary { - TimingSummaryMode::Auto => { - if is_ci { - TimingSummaryMode::Top - } else { - TimingSummaryMode::Off - } - } - mode => mode, - } - } - /// Because this test can be run as a cargo test, commands like /// /// ```shell From b133317d22cc406cf74bee41e0636d078156e6c7 Mon Sep 17 00:00:00 2001 From: Siew Kam Onn Date: Fri, 27 Feb 2026 17:28:30 +0800 Subject: [PATCH 12/12] Revert "Simplify timing UX and remove complexity" This reverts commit 2eb94d408c7c839b30409af2219c7256bff6270a. --- datafusion/sqllogictest/README.md | 18 ++--- datafusion/sqllogictest/bin/sqllogictests.rs | 75 ++++++++++++++++++-- 2 files changed, 78 insertions(+), 15 deletions(-) diff --git a/datafusion/sqllogictest/README.md b/datafusion/sqllogictest/README.md index 45d2d77058692..7d84ad23d5905 100644 --- a/datafusion/sqllogictest/README.md +++ b/datafusion/sqllogictest/README.md @@ -75,22 +75,24 @@ RUST_LOG=debug cargo test --test sqllogictests -- ddl The sqllogictest runner can emit deterministic per-file elapsed timings to help identify slow test files. -Timing summary output is disabled by default and enabled with -`--timing-summary` (or `SLT_TIMING_SUMMARY=1`). +By default (`--timing-summary auto`), timing summary output is disabled in local +TTY runs and shows a top-slowest summary in non-TTY/CI runs. + +`--timing-top-n` / `SLT_TIMING_TOP_N` must be a positive integer (`>= 1`). ```shell -# Show deterministic per-file elapsed timings (sorted slowest first) -cargo test --test sqllogictests -- --timing-summary +# Show top 10 slowest files (good for CI) +cargo test --test sqllogictests -- --timing-summary top --timing-top-n 10 ``` ```shell -# Keep only the top 10 lines using standard shell tooling -cargo test --test sqllogictests -- --timing-summary | head -n 10 +# Show full per-file timing table +cargo test --test sqllogictests -- --timing-summary full ``` ```shell -# Enable via environment variable -SLT_TIMING_SUMMARY=1 cargo test --test sqllogictests +# Same controls via environment variables +SLT_TIMING_SUMMARY=top SLT_TIMING_TOP_N=15 cargo test --test sqllogictests ``` ```shell diff --git a/datafusion/sqllogictest/bin/sqllogictests.rs b/datafusion/sqllogictest/bin/sqllogictests.rs index a2bdd30ceef77..e067f2488d81f 100644 --- a/datafusion/sqllogictest/bin/sqllogictests.rs +++ b/datafusion/sqllogictest/bin/sqllogictests.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -use clap::{ColorChoice, Parser}; +use clap::{ColorChoice, Parser, ValueEnum}; use datafusion::common::instant::Instant; use datafusion::common::utils::get_available_parallelism; use datafusion::common::{DataFusionError, Result, exec_datafusion_err, exec_err}; @@ -61,6 +61,14 @@ const SQLITE_PREFIX: &str = "sqlite"; const ERRS_PER_FILE_LIMIT: usize = 10; const TIMING_DEBUG_SLOW_FILES_ENV: &str = "SLT_TIMING_DEBUG_SLOW_FILES"; +#[derive(Clone, Copy, Debug, Eq, PartialEq, ValueEnum)] +enum TimingSummaryMode { + Auto, + Off, + Top, + Full, +} + #[derive(Debug)] struct FileTiming { relative_path: PathBuf, @@ -313,7 +321,7 @@ async fn run_tests() -> Result<()> { .then_with(|| a.relative_path.cmp(&b.relative_path)) }); - print_timing_summary(&options, &m, &file_timings)?; + print_timing_summary(&options, &m, is_ci, &file_timings)?; let errors: Vec<_> = file_results .into_iter() @@ -376,14 +384,27 @@ async fn run_tests() -> Result<()> { fn print_timing_summary( options: &Options, progress: &MultiProgress, + is_ci: bool, file_timings: &[FileTiming], ) -> Result<()> { - if !options.timing_summary || file_timings.is_empty() { + let mode = options.timing_summary_mode(is_ci); + if mode == TimingSummaryMode::Off || file_timings.is_empty() { return Ok(()); } + let top_n = options.timing_top_n; + debug_assert!(matches!( + mode, + TimingSummaryMode::Top | TimingSummaryMode::Full + )); + let count = if mode == TimingSummaryMode::Full { + file_timings.len() + } else { + top_n + }; + progress.println("Per-file elapsed summary (deterministic):")?; - for (idx, timing) in file_timings.iter().enumerate() { + for (idx, timing) in file_timings.iter().take(count).enumerate() { progress.println(format!( "{:>3}. {:>8.3}s {}", idx + 1, @@ -392,6 +413,13 @@ fn print_timing_summary( ))?; } + if mode != TimingSummaryMode::Full && file_timings.len() > count { + progress.println(format!( + "... {} more files omitted (use --timing-summary full to show all)", + file_timings.len() - count + ))?; + } + Ok(()) } @@ -406,6 +434,16 @@ fn is_env_truthy(name: &str) -> bool { }) } +fn parse_timing_top_n(arg: &str) -> std::result::Result { + let parsed = arg + .parse::() + .map_err(|error| format!("invalid value '{arg}': {error}"))?; + if parsed == 0 { + return Err("must be >= 1".to_string()); + } + Ok(parsed) +} + async fn run_test_file_substrait_round_trip( test_file: TestFile, validator: Validator, @@ -902,10 +940,20 @@ struct Options { #[clap( long, env = "SLT_TIMING_SUMMARY", - default_value_t = false, - help = "Print deterministic per-file timing summary" + value_enum, + default_value_t = TimingSummaryMode::Auto, + help = "Per-file timing summary mode: auto|off|top|full" + )] + timing_summary: TimingSummaryMode, + + #[clap( + long, + env = "SLT_TIMING_TOP_N", + default_value_t = 10, + value_parser = parse_timing_top_n, + help = "Number of files to show when timing summary mode is auto/top (must be >= 1)" )] - timing_summary: bool, + timing_top_n: usize, #[clap( long, @@ -917,6 +965,19 @@ struct Options { } impl Options { + fn timing_summary_mode(&self, is_ci: bool) -> TimingSummaryMode { + match self.timing_summary { + TimingSummaryMode::Auto => { + if is_ci { + TimingSummaryMode::Top + } else { + TimingSummaryMode::Off + } + } + mode => mode, + } + } + /// Because this test can be run as a cargo test, commands like /// /// ```shell