From 55993f0ffa6bb896e1cc6ea56e68da8da5ecbc80 Mon Sep 17 00:00:00 2001 From: Peter Darton Date: Wed, 3 Dec 2025 16:01:42 +0000 Subject: [PATCH 1/4] Ensure we only output a test-failure when the test failed --- libexec/bats-core/bats-format-junit | 54 ++++++++++++++++++----------- 1 file changed, 33 insertions(+), 21 deletions(-) diff --git a/libexec/bats-core/bats-format-junit b/libexec/bats-core/bats-format-junit index 52b6ad8db4..d1214bee84 100755 --- a/libexec/bats-core/bats-format-junit +++ b/libexec/bats-core/bats-format-junit @@ -20,7 +20,7 @@ init_file() { test_exec_time=0 name="" _buffer="" - _buffer_log="" + _system_err_log="" _system_out_log="" test_result_state="" # mark that no test has run in this file so far } @@ -53,12 +53,12 @@ suite_header() { file_header() { timestamp=$(date -u +"%Y-%m-%dT%H:%M:%S") - printf "\n" \ + printf " \n" \ "$(xml_escape "${class}")" "${file_count}" "${file_failures}" "${file_skipped}" "$(milliseconds_to_seconds "${file_exec_time}")" "${timestamp}" "$(host)" } file_footer() { - printf "\n" + printf " \n" } suite_footer() { @@ -66,19 +66,32 @@ suite_footer() { } print_test_case() { - if [[ "$test_result_state" == ok && -z "$_system_out_log" && -z "$_buffer_log" ]]; then + if [[ "$test_result_state" == ok && -z "$_system_out_log" && -z "$_system_err_log" ]]; then # pass and no output can be shortened printf " \n" "$(xml_escape "${class}")" "$(xml_escape "${name}")" "$(milliseconds_to_seconds "${test_exec_time}")" else printf " \n" "$(xml_escape "${class}")" "$(xml_escape "${name}")" "$(milliseconds_to_seconds "${test_exec_time}")" if [[ -n "$_system_out_log" ]]; then - printf " %s\n" "$(xml_escape "${_system_out_log}")" + printf " %s\n" "$(xml_escape "${_system_out_log}")" fi - if [[ -n "$_buffer_log" || "$test_result_state" == not_ok ]]; then - printf " %s\n" "$(xml_escape "${_buffer_log}")" + if [[ "$test_result_state" == not_ok ]]; then # Failed tests need a element. + # If we have system err output, we use that as the failure text. + if [[ -n "$_system_err_log" ]]; then + printf " %s\n" "$(xml_escape "${_system_err_log}")" + else + printf " \n" + fi + else + if [[ -n "$_system_err_log" ]]; then + printf " %s\n" "$(xml_escape "${_system_err_log}")" + fi fi if [[ "$test_result_state" == skipped ]]; then - printf " %s\n" "$(xml_escape "$test_skip_message")" + if [[ -n "$test_skip_message" ]]; then + printf " %s\n" "$(xml_escape "$test_skip_message")" + else + printf " \n" + fi fi printf " \n" fi @@ -127,12 +140,12 @@ flush() { _buffer="" } -log() { - if [[ -n "$_buffer_log" ]]; then - _buffer_log="${_buffer_log} +log_system_err() { + if [[ -n "$_system_err_log" ]]; then + _system_err_log="${_system_err_log} $1" else - _buffer_log="$1" + _system_err_log="$1" fi } @@ -140,7 +153,7 @@ flush_log() { if [[ -n "$test_result_state" ]]; then buffer print_test_case fi - _buffer_log="" + _system_err_log="" _system_out_log="" test_result_state="" # Clean out result from last test. Retried tests will have multiple begin calls. } @@ -157,7 +170,7 @@ $1" finish_file() { if [[ "${class}" != JUNIT_FORMATTER_NO_FILE_ENCOUNTERED ]]; then file_header - printf "%s\n" "${_buffer}" + printf "%s" "${_buffer}" file_footer class='' name='' @@ -226,7 +239,7 @@ bats_tap_stream_comment() { # ;; *) # everything else is considered error output - log "$1" + log_system_err "$1" ;; esac } @@ -239,15 +252,16 @@ bats_tap_stream_suite() { # } bats_tap_stream_unknown() { # - : + local line="$1" scope="$2" + log_system_out "$line" } main() { - local _buffer_log="" \ + local \ name="" \ class="JUNIT_FORMATTER_NO_FILE_ENCOUNTERED" \ _buffer="" \ - _buffer_log="" \ + _system_err_log="" \ _suite_buffer="" \ _system_out_log="" \ test_result_state="" \ @@ -282,6 +296,4 @@ main() { finish_suite # ensure we run this with the local variables still in scope } -# [\x00-\x08\x0B\x0C\x0E-\x1F] can't use $'\x00' as this terminates the string early -BATS_JUNIT_FORMATTER_DELETE_CHARS_DEFAULT='\000'$'\x01\x02\x03\x04\x05\x06\x07\x08\x0B\x0C\x0E\x0F\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1A\x1B\x1C\x1D\x1E\x1F' -main "$@" | tr -d "${BATS_JUNIT_FORMATTER_DELETE_CHARS-$BATS_JUNIT_FORMATTER_DELETE_CHARS_DEFAULT}" +main "$@" From 6787d2fc097cd196841b577b72e20efe89b71fe0 Mon Sep 17 00:00:00 2001 From: Peter Darton Date: Wed, 10 Dec 2025 19:16:52 +0000 Subject: [PATCH 2/4] Add unit-test suite that demonstrates the issue --- .../referenced_connectors_tests.bats | 27 +++++++++++++++++++ .../issue1180/setup_suite.bash | 14 ++++++++++ 2 files changed, 41 insertions(+) create mode 100755 test/fixtures/junit-formatter/issue1180/referenced_connectors_tests.bats create mode 100644 test/fixtures/junit-formatter/issue1180/setup_suite.bash diff --git a/test/fixtures/junit-formatter/issue1180/referenced_connectors_tests.bats b/test/fixtures/junit-formatter/issue1180/referenced_connectors_tests.bats new file mode 100755 index 0000000000..dcd1e71c46 --- /dev/null +++ b/test/fixtures/junit-formatter/issue1180/referenced_connectors_tests.bats @@ -0,0 +1,27 @@ +#!/usr/bin/env bats +# shellcheck shell=bash + +function setup_file() { + skip +} + +function teardown_file() { + echo "Teardown_file stdout" + echo "# Hash Teardown_file stdout" + echo "Teardown_file stderr" >&2 + echo "# Hash Teardown_file stderr" >&2 + echo "Teardown_file fd3" >&3 + echo "# Hash Teardown_file fd3" >&3 +} + +@test "skipped-and-junit-agrees" { + echo "unexpected stdout as this should be skipped" + echo "unexpected stderr as this should be skipped" >&2 + echo "unexpected fd3 as this should be skipped" >&3 +} + +@test "skipped-but-junit-reports-failure" { + echo "unexpected stdout as this should be skipped" + echo "unexpected stderr as this should be skipped" >&2 + echo "unexpected fd3 as this should be skipped" >&3 +} diff --git a/test/fixtures/junit-formatter/issue1180/setup_suite.bash b/test/fixtures/junit-formatter/issue1180/setup_suite.bash new file mode 100644 index 0000000000..12e2e7ee7d --- /dev/null +++ b/test/fixtures/junit-formatter/issue1180/setup_suite.bash @@ -0,0 +1,14 @@ +# shellcheck shell=bash + +function setup_suite() { + true +} + +teardown_suite() { + echo "Teardown_suite stdout" + echo "# Hash teardown_suite stdout" + echo "Teardown_suite stderr" >&2 + echo "# Hash teardown_suite stderr" >&2 + echo "Teardown_suite fd3" >&3 + echo "# Hash teardown_suite fd3" >&3 +} From 03f7f7d60ea5be5c93b7194824ae65113142a8fe Mon Sep 17 00:00:00 2001 From: Peter Darton Date: Thu, 11 Dec 2025 16:47:29 +0000 Subject: [PATCH 3/4] Reduce unnecessary changes --- libexec/bats-core/bats-format-junit | 42 +++++++++++++---------------- 1 file changed, 18 insertions(+), 24 deletions(-) diff --git a/libexec/bats-core/bats-format-junit b/libexec/bats-core/bats-format-junit index d1214bee84..3a922d0c9a 100755 --- a/libexec/bats-core/bats-format-junit +++ b/libexec/bats-core/bats-format-junit @@ -20,7 +20,7 @@ init_file() { test_exec_time=0 name="" _buffer="" - _system_err_log="" + _buffer_log="" _system_out_log="" test_result_state="" # mark that no test has run in this file so far } @@ -53,12 +53,12 @@ suite_header() { file_header() { timestamp=$(date -u +"%Y-%m-%dT%H:%M:%S") - printf " \n" \ + printf "\n" \ "$(xml_escape "${class}")" "${file_count}" "${file_failures}" "${file_skipped}" "$(milliseconds_to_seconds "${file_exec_time}")" "${timestamp}" "$(host)" } file_footer() { - printf " \n" + printf "\n" } suite_footer() { @@ -66,32 +66,24 @@ suite_footer() { } print_test_case() { - if [[ "$test_result_state" == ok && -z "$_system_out_log" && -z "$_system_err_log" ]]; then + if [[ "$test_result_state" == ok && -z "$_system_out_log" && -z "$_buffer_log" ]]; then # pass and no output can be shortened printf " \n" "$(xml_escape "${class}")" "$(xml_escape "${name}")" "$(milliseconds_to_seconds "${test_exec_time}")" else printf " \n" "$(xml_escape "${class}")" "$(xml_escape "${name}")" "$(milliseconds_to_seconds "${test_exec_time}")" if [[ -n "$_system_out_log" ]]; then - printf " %s\n" "$(xml_escape "${_system_out_log}")" + printf " %s\n" "$(xml_escape "${_system_out_log}")" fi if [[ "$test_result_state" == not_ok ]]; then # Failed tests need a element. # If we have system err output, we use that as the failure text. - if [[ -n "$_system_err_log" ]]; then - printf " %s\n" "$(xml_escape "${_system_err_log}")" - else - printf " \n" - fi + printf " %s\n" "$(xml_escape "${_buffer_log}")" else - if [[ -n "$_system_err_log" ]]; then - printf " %s\n" "$(xml_escape "${_system_err_log}")" + if [[ -n "$_buffer_log" ]]; then + printf " %s\n" "$(xml_escape "${_buffer_log}")" fi fi if [[ "$test_result_state" == skipped ]]; then - if [[ -n "$test_skip_message" ]]; then - printf " %s\n" "$(xml_escape "$test_skip_message")" - else - printf " \n" - fi + printf " %s\n" "$(xml_escape "$test_skip_message")" fi printf " \n" fi @@ -141,11 +133,11 @@ flush() { } log_system_err() { - if [[ -n "$_system_err_log" ]]; then - _system_err_log="${_system_err_log} + if [[ -n "$_buffer_log" ]]; then + _buffer_log="${_buffer_log} $1" else - _system_err_log="$1" + _buffer_log="$1" fi } @@ -153,7 +145,7 @@ flush_log() { if [[ -n "$test_result_state" ]]; then buffer print_test_case fi - _system_err_log="" + _buffer_log="" _system_out_log="" test_result_state="" # Clean out result from last test. Retried tests will have multiple begin calls. } @@ -257,11 +249,11 @@ bats_tap_stream_unknown() { # } main() { - local \ + local _buffer_log="" \ name="" \ class="JUNIT_FORMATTER_NO_FILE_ENCOUNTERED" \ _buffer="" \ - _system_err_log="" \ + _buffer_log="" \ _suite_buffer="" \ _system_out_log="" \ test_result_state="" \ @@ -296,4 +288,6 @@ main() { finish_suite # ensure we run this with the local variables still in scope } -main "$@" +# [\x00-\x08\x0B\x0C\x0E-\x1F] can't use $'\x00' as this terminates the string early +BATS_JUNIT_FORMATTER_DELETE_CHARS_DEFAULT='\000'$'\x01\x02\x03\x04\x05\x06\x07\x08\x0B\x0C\x0E\x0F\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1A\x1B\x1C\x1D\x1E\x1F' +main "$@" | tr -d "${BATS_JUNIT_FORMATTER_DELETE_CHARS-$BATS_JUNIT_FORMATTER_DELETE_CHARS_DEFAULT}" From 31a45933716a4ee62db5010308fad304e9ea5e7b Mon Sep 17 00:00:00 2001 From: Peter Darton Date: Thu, 11 Dec 2025 16:49:23 +0000 Subject: [PATCH 4/4] Add unit-test to demonstrate the issue --- .../issue1180/referenced_connectors_tests.bats | 12 ++++++------ .../junit-formatter/issue1180/setup_suite.bash | 6 +++--- test/junit-formatter.bats | 9 +++++++++ 3 files changed, 18 insertions(+), 9 deletions(-) diff --git a/test/fixtures/junit-formatter/issue1180/referenced_connectors_tests.bats b/test/fixtures/junit-formatter/issue1180/referenced_connectors_tests.bats index dcd1e71c46..54394fda9c 100755 --- a/test/fixtures/junit-formatter/issue1180/referenced_connectors_tests.bats +++ b/test/fixtures/junit-formatter/issue1180/referenced_connectors_tests.bats @@ -6,12 +6,12 @@ function setup_file() { } function teardown_file() { - echo "Teardown_file stdout" - echo "# Hash Teardown_file stdout" - echo "Teardown_file stderr" >&2 - echo "# Hash Teardown_file stderr" >&2 - echo "Teardown_file fd3" >&3 - echo "# Hash Teardown_file fd3" >&3 + echo "normal teardown_file stdout" + echo "# Hash teardown_file stdout" + echo "normal teardown_file stderr" >&2 + echo "# Hash teardown_file stderr" >&2 + echo "normal teardown_file fd3" >&3 + echo "# Hash teardown_file fd3" >&3 } @test "skipped-and-junit-agrees" { diff --git a/test/fixtures/junit-formatter/issue1180/setup_suite.bash b/test/fixtures/junit-formatter/issue1180/setup_suite.bash index 12e2e7ee7d..facdb5017f 100644 --- a/test/fixtures/junit-formatter/issue1180/setup_suite.bash +++ b/test/fixtures/junit-formatter/issue1180/setup_suite.bash @@ -5,10 +5,10 @@ function setup_suite() { } teardown_suite() { - echo "Teardown_suite stdout" + echo "normal teardown_suite stdout" echo "# Hash teardown_suite stdout" - echo "Teardown_suite stderr" >&2 + echo "normal teardown_suite stderr" >&2 echo "# Hash teardown_suite stderr" >&2 - echo "Teardown_suite fd3" >&3 + echo "normal teardown_suite fd3" >&3 echo "# Hash teardown_suite fd3" >&3 } diff --git a/test/junit-formatter.bats b/test/junit-formatter.bats index 3ed7f80fe9..0f831e16ce 100644 --- a/test/junit-formatter.bats +++ b/test/junit-formatter.bats @@ -149,6 +149,15 @@ TESTSUITES_REGEX="" [[ "${lines[7]}" == '' ]] } +@test "junit does not mark tests with FD 3 output in teardown_suite as failed (issue #1180)" { + bats_require_minimum_version 1.5.0 + local stderr='' # silence shellcheck + name=non-empty reentrant_run -0 --separate-stderr bats --formatter junit "$FIXTURE_ROOT/issue1180" + [ "${stderr}" == "" ] || { echo "stderr should be empty but was: ${stderr}" >&3; return 1; } + [[ "${output}" != *'