From d0fd77248b63e7a797bb5f6d2a5e16d32d7a128b Mon Sep 17 00:00:00 2001 From: Chemaclass Date: Tue, 16 Jun 2026 08:33:22 +0200 Subject: [PATCH] feat(runner): add per-test timeout to abort hanging tests Add an opt-in `--test-timeout ` flag and `BASHUNIT_TEST_TIMEOUT` env var. When set, a test that runs longer than the threshold is killed and reported as a failure, then the run continues instead of hanging forever (e.g. a mock left without an implementation that blocks on input). The test body runs as a backgrounded job in its own process group so a watchdog can SIGTERM/SIGKILL the whole tree; a hanging test typically blocks in a child process that signalling the subshell alone cannot reach. Disabled by default (0), needs no external `timeout` command and works on Bash 3.2+ (including the default macOS Bash). Also reset the accumulated per-test exit code before parsing each result, so a non-zero/timed-out test no longer poisons the next test in the file. Closes #721 --- CHANGELOG.md | 6 + docs/command-line.md | 29 +++ docs/configuration.md | 24 ++ src/console_header.sh | 1 + src/env.sh | 21 ++ src/main.sh | 4 + src/runner.sh | 207 ++++++++++++++---- tests/acceptance/bashunit_timeout_test.sh | 36 +++ .../fixtures/test_bashunit_timeout.sh | 10 + tests/unit/env_test.sh | 37 ++++ 10 files changed, 327 insertions(+), 48 deletions(-) create mode 100644 tests/acceptance/bashunit_timeout_test.sh create mode 100644 tests/acceptance/fixtures/test_bashunit_timeout.sh diff --git a/CHANGELOG.md b/CHANGELOG.md index ef546e08..60989397 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,12 @@ ## Unreleased +### Added +- `--test-timeout ` flag and `BASHUNIT_TEST_TIMEOUT` env var: abort an individual test that runs longer than the given number of seconds, report it as a failure and keep running the rest. Disabled by default (`0`); needs no external `timeout` command and works on Bash 3.2+ (#721) + +### Fixed +- A test that exited non-zero no longer poisons the exit code of subsequent tests in the same file (the per-test exit code was accumulated instead of reset) + ### Changed - Documentation and project URLs now point to the new primary domain `bashunit.com` (old `bashunit.typeddevs.com` continues to work as a redirect) - The docs site now deploys to GitHub Pages on the `bashunit.com` custom domain (`deploy-gh-pages.yml`); the installer is copied into the site root from the repo's canonical `install.sh` diff --git a/docs/command-line.md b/docs/command-line.md index 75153d67..ada1daf1 100644 --- a/docs/command-line.md +++ b/docs/command-line.md @@ -75,6 +75,7 @@ bashunit test tests/ --parallel --simple | `-s, --simple` | Simple output (dots) | | `--detailed` | Detailed output (default) | | `-S, --stop-on-failure` | Stop on first failure | +| `--test-timeout ` | Fail a test if it runs longer than N seconds | | `--show-skipped` | Show skipped tests summary at end | | `--show-incomplete` | Show incomplete tests summary at end | | `-vvv, --verbose` | Show execution details | @@ -431,6 +432,34 @@ BASHUNIT_PROFILE_COUNT=3 bashunit test tests/ --profile ``` ::: +### Test Timeout + +> `bashunit test --test-timeout ` + +Abort an individual test if it runs longer than the given number of seconds and +report it as a failure, then continue with the remaining tests. This protects a +run from hanging forever on a blocked test — for example a mock that was never +given an implementation and waits on input that never arrives. + +The timeout is **disabled by default** (`0`). It applies per test (set up and +tear down included) and is expressed in whole seconds. It needs no external +`timeout` command and works on Bash 3.2+ (including the default macOS Bash). + +::: code-group +```bash [Example] +bashunit test tests/ --test-timeout 5 +``` +```[Output] +✗ Error: Test hangs forever + Test timed out after 5s + +Tests: 1 passed, 1 failed, 2 total +``` +::: + +It can also be set via the `BASHUNIT_TEST_TIMEOUT` environment variable (see +[configuration](/configuration#test-timeout)). + ### No Progress > `bashunit test --no-progress` diff --git a/docs/configuration.md b/docs/configuration.md index dc1fae0e..267e72d0 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -127,6 +127,30 @@ By default, when an assertion fails within a test, subsequent assertions in the The `--stop-on-failure` flag is separate – it stops the entire test runner after a failing **test**, while assertion-level stopping happens within each test. ::: +## Test timeout + +> `BASHUNIT_TEST_TIMEOUT=` + +Abort an individual test if it runs longer than the given number of seconds, +report it as a failure and keep running the remaining tests. `0` (disabled) by +default. Useful to stop a run from hanging forever on a blocked test, such as a +mock left without an implementation. + +The value is expressed in whole seconds and applies per test (set up and tear +down included). It needs no external `timeout` command and works on Bash 3.2+, +including the default macOS Bash. + +Similar as using `--test-timeout` option on the [command line](/command-line#test-timeout). + +::: code-group +```bash [Enable a 5s timeout] +BASHUNIT_TEST_TIMEOUT=5 +``` +```bash [Disabled (default)] +BASHUNIT_TEST_TIMEOUT=0 +``` +::: + ## Stop on assertion failure > `BASHUNIT_STOP_ON_ASSERTION_FAILURE=true|false` diff --git a/src/console_header.sh b/src/console_header.sh index 83f27516..48f4fb2a 100644 --- a/src/console_header.sh +++ b/src/console_header.sh @@ -118,6 +118,7 @@ Options: --output Output format: tap (TAP version 13) -R, --run-all Run all assertions (don't stop on first failure) -S, --stop-on-failure Stop on first failure + --test-timeout Fail a test if it runs longer than N seconds (0 = off) -vvv, --verbose Show execution details --debug [file] Enable shell debug mode --no-output Suppress all output diff --git a/src/env.sh b/src/env.sh index 43e4c446..784ccd2a 100644 --- a/src/env.sh +++ b/src/env.sh @@ -115,6 +115,8 @@ _BASHUNIT_DEFAULT_OUTPUT_FORMAT="" _BASHUNIT_DEFAULT_FAIL_ON_RISKY="false" _BASHUNIT_DEFAULT_PROFILE="false" _BASHUNIT_DEFAULT_PROFILE_COUNT="10" +# Per-test timeout in seconds (0 = disabled) +_BASHUNIT_DEFAULT_TEST_TIMEOUT="0" : "${BASHUNIT_PARALLEL_RUN:=${PARALLEL_RUN:=$_BASHUNIT_DEFAULT_PARALLEL_RUN}}" : "${BASHUNIT_PARALLEL_JOBS:=0}" @@ -140,6 +142,7 @@ _BASHUNIT_DEFAULT_PROFILE_COUNT="10" : "${BASHUNIT_FAIL_ON_RISKY:=${FAIL_ON_RISKY:=$_BASHUNIT_DEFAULT_FAIL_ON_RISKY}}" : "${BASHUNIT_PROFILE:=${PROFILE:=$_BASHUNIT_DEFAULT_PROFILE}}" : "${BASHUNIT_PROFILE_COUNT:=${PROFILE_COUNT:=$_BASHUNIT_DEFAULT_PROFILE_COUNT}}" +: "${BASHUNIT_TEST_TIMEOUT:=${TEST_TIMEOUT:=$_BASHUNIT_DEFAULT_TEST_TIMEOUT}}" # Support NO_COLOR standard (https://no-color.org) if [ -n "${NO_COLOR:-}" ]; then BASHUNIT_NO_COLOR="true" @@ -151,6 +154,24 @@ function bashunit::env::is_parallel_run_enabled() { [ "$BASHUNIT_PARALLEL_RUN" = "true" ] } +## +# Whether a per-test timeout is configured (a positive integer number of seconds). +# Returns: 0 when enabled, 1 otherwise. +## +function bashunit::env::is_test_timeout_enabled() { + case "${BASHUNIT_TEST_TIMEOUT:-0}" in + '' | *[!0-9]*) return 1 ;; + esac + [ "${BASHUNIT_TEST_TIMEOUT:-0}" -gt 0 ] +} + +## +# Prints the configured per-test timeout in seconds (0 when disabled). +## +function bashunit::env::test_timeout_secs() { + printf '%s' "${BASHUNIT_TEST_TIMEOUT:-0}" +} + function bashunit::env::is_show_header_enabled() { [ "$BASHUNIT_SHOW_HEADER" = "true" ] } diff --git a/src/main.sh b/src/main.sh index 415829fe..42d264cc 100644 --- a/src/main.sh +++ b/src/main.sh @@ -74,6 +74,10 @@ function bashunit::main::cmd_test() { --no-parallel) export BASHUNIT_PARALLEL_RUN=false ;; + --test-timeout) + export BASHUNIT_TEST_TIMEOUT="$2" + shift + ;; -w | --watch) export BASHUNIT_WATCH_MODE=true ;; diff --git a/src/runner.sh b/src/runner.sh index fd71e0bf..b47bf547 100755 --- a/src/runner.sh +++ b/src/runner.sh @@ -843,6 +843,147 @@ function bashunit::runner::render_running_file_header() { fi } +# Result slots for the timeout-aware execution path (see run_with_timeout). +_BASHUNIT_RUNNER_EXEC_OUT="" +_BASHUNIT_RUNNER_TIMED_OUT="false" + +## +# Runs a single test inside the capture subshell: sets up the EXIT trap that +# encodes assertion counts/exit code, runs set_up, applies the shell mode and +# finally invokes the test function. Meant to be called from a subshell (either +# the `$(...)` capture or a backgrounded job), so its `set`/`trap`/`exit` calls +# stay isolated. Emits the test stdout (with stderr merged) followed by the +# encoded context from cleanup_on_exit. +# Arguments: $1 test file, $2 function name, $@ test args +## +function bashunit::runner::execute_test_body() { + local test_file=$1 + shift + local fn_name=$1 + shift + + # Save subshell stdout to FD 5 so the EXIT trap can restore it. + # When set -e kills the subshell during a redirected block in + # execute_test_hook, the redirect leaks into the EXIT trap, + # causing export_subshell_context output to be lost. + exec 5>&1 + # shellcheck disable=SC2064 + trap "exit_code=\$?; bashunit::runner::cleanup_on_exit \"$test_file\" \"\$exit_code\"" EXIT + bashunit::state::initialize_assertions_count + + if bashunit::env::is_login_shell_enabled; then + bashunit::runner::source_login_shell_profiles + fi + + # Enable coverage tracking early to include set_up/tear_down hooks + if [ "${_BASHUNIT_COVERAGE_ON:-0}" = 1 ]; then + bashunit::coverage::enable_trap + fi + + # Run set_up and capture exit code without || to preserve errexit behavior + # shellcheck disable=SC2030 + _BASHUNIT_SETUP_COMPLETED=false + local setup_exit_code=0 + bashunit::runner::run_set_up "$test_file" + setup_exit_code=$? + _BASHUNIT_SETUP_COMPLETED=true + if [ $setup_exit_code -ne 0 ]; then + exit $setup_exit_code + fi + + # Apply shell mode setting for test execution + if bashunit::env::is_strict_mode_enabled; then + set -eu + # Bash 3.0 ships a broken pipefail; only enable it where it is reliable. + if bashunit::runner::_supports_reliable_pipefail; then + set -o pipefail + else + set +o pipefail + fi + else + set +euo pipefail + fi + + # 2>&1: Redirects the std-error (FD 2) to the std-output (FD 1). + # points to the original std-output. + "$fn_name" "$@" 2>&1 +} + +## +# Prints an encoded subshell result for a test that timed out: empty assertion +# counters and exit code 124 (the conventional "timed out" code, already mapped +# by classify_kill_signal). The empty TEST_HOOK_MESSAGE/TITLE/OUTPUT fields would +# base64-encode to an empty string anyway, so the line is emitted directly rather +# than mutating the shared _BASHUNIT_* globals (it mirrors the layout produced by +# bashunit::state::export_subshell_context). Bash 3.0+ compatible. +## +function bashunit::runner::build_timeout_result() { + printf '%s' "##ASSERTIONS_FAILED=0##ASSERTIONS_PASSED=0##ASSERTIONS_SKIPPED=0\ +##ASSERTIONS_INCOMPLETE=0##ASSERTIONS_SNAPSHOT=0##TEST_EXIT_CODE=124\ +##TEST_HOOK_FAILURE=##TEST_HOOK_MESSAGE=##TEST_TITLE=##TEST_OUTPUT=##" +} + +## +# Runs the test body with a watchdog that kills it after BASHUNIT_TEST_TIMEOUT +# seconds. The body runs as a backgrounded job in its own process group (set -m) +# so the watchdog can SIGTERM/SIGKILL the whole tree — a hanging test usually +# blocks in a child process, which signalling the subshell alone cannot reach. +# Writes the captured result to _BASHUNIT_RUNNER_EXEC_OUT and "true"/"false" to +# _BASHUNIT_RUNNER_TIMED_OUT. Bash 3.0+ compatible (validated on Bash 3.2). +# Arguments: $1 test file, $2 function name, $@ test args +## +function bashunit::runner::run_with_timeout() { + local test_file=$1 + shift + local fn_name=$1 + shift + local secs + secs=$(bashunit::env::test_timeout_secs) + + # NOTE: these must NOT use bashunit::temp_file — that prefixes the current + # test id, and cleanup_on_exit (run inside the test subshell) would unlink + # them via cleanup_testcase_temp_files before we read them back here. + local tmp_dir="${BASHUNIT_TEMP_DIR:-${TMPDIR:-/tmp}}" + local out_file marker_file + out_file="$("$MKTEMP" "$tmp_dir/bashunit_timeout_out.XXXXXXX")" + marker_file="$("$MKTEMP" "$tmp_dir/bashunit_timeout_marker.XXXXXXX")" + rm -f "$marker_file" + + # Both jobs run in their own process group (set -m) so each can be killed as a + # whole tree. The body MUST run in an explicit ( ) subshell: a backgrounded { } + # group does not run its EXIT trap on normal completion, which would drop the + # encoded assertion context. The watchdog's fds are detached from the caller so + # a lingering `sleep` can never hold a captured stdout pipe open. + set -m + (bashunit::runner::execute_test_body "$test_file" "$fn_name" "$@") >"$out_file" 2>&1 & + local test_pid=$! + ( + sleep "$secs" + : >"$marker_file" + kill -TERM -"$test_pid" 2>/dev/null + sleep 0.3 + kill -KILL -"$test_pid" 2>/dev/null + ) /dev/null 2>&1 & + local watchdog_pid=$! + set +m + + wait "$test_pid" 2>/dev/null + # Tear down the watchdog group (its `sleep` child included) so it cannot fire + # late or block the caller. + kill -TERM -"$watchdog_pid" 2>/dev/null + wait "$watchdog_pid" 2>/dev/null + + if [ -f "$marker_file" ]; then + _BASHUNIT_RUNNER_TIMED_OUT="true" + _BASHUNIT_RUNNER_EXEC_OUT="$(bashunit::runner::build_timeout_result)" + else + _BASHUNIT_RUNNER_TIMED_OUT="false" + _BASHUNIT_RUNNER_EXEC_OUT="$(cat "$out_file" 2>/dev/null)" + fi + + rm -f "$out_file" "$marker_file" +} + function bashunit::runner::run_test() { local start_time start_time=$(bashunit::clock::now) @@ -868,54 +1009,15 @@ function bashunit::runner::run_test() { # This means that FD 3 now points to wherever the std-output was pointing. exec 3>&1 - local test_execution_result=$( - # Save subshell stdout to FD 5 so the EXIT trap can restore it. - # When set -e kills the subshell during a redirected block in - # execute_test_hook, the redirect leaks into the EXIT trap, - # causing export_subshell_context output to be lost. - exec 5>&1 - # shellcheck disable=SC2064 - trap "exit_code=\$?; bashunit::runner::cleanup_on_exit \"$test_file\" \"\$exit_code\"" EXIT - bashunit::state::initialize_assertions_count - - if bashunit::env::is_login_shell_enabled; then - bashunit::runner::source_login_shell_profiles - fi - - # Enable coverage tracking early to include set_up/tear_down hooks - if [ "${_BASHUNIT_COVERAGE_ON:-0}" = 1 ]; then - bashunit::coverage::enable_trap - fi - - # Run set_up and capture exit code without || to preserve errexit behavior - # shellcheck disable=SC2030 - _BASHUNIT_SETUP_COMPLETED=false - local setup_exit_code=0 - bashunit::runner::run_set_up "$test_file" - setup_exit_code=$? - _BASHUNIT_SETUP_COMPLETED=true - if [ $setup_exit_code -ne 0 ]; then - exit $setup_exit_code - fi - - # Apply shell mode setting for test execution - if bashunit::env::is_strict_mode_enabled; then - set -eu - # Bash 3.0 ships a broken pipefail; only enable it where it is reliable. - if bashunit::runner::_supports_reliable_pipefail; then - set -o pipefail - else - set +o pipefail - fi - else - set +euo pipefail - fi - - # 2>&1: Redirects the std-error (FD 2) to the std-output (FD 1). - # points to the original std-output. - "$fn_name" "$@" 2>&1 - - ) + local test_execution_result + local timed_out="false" + if bashunit::env::is_test_timeout_enabled; then + bashunit::runner::run_with_timeout "$test_file" "$fn_name" "$@" + test_execution_result="$_BASHUNIT_RUNNER_EXEC_OUT" + timed_out="$_BASHUNIT_RUNNER_TIMED_OUT" + else + test_execution_result=$(bashunit::runner::execute_test_body "$test_file" "$fn_name" "$@") + fi # Closes FD 3, which was used temporarily to hold the original stdout. exec 3>&- @@ -950,6 +1052,10 @@ function bashunit::runner::run_test() { local runtime_error runtime_error=$(bashunit::runner::detect_runtime_error "$runtime_output") + # parse_result accumulates _BASHUNIT_TEST_EXIT_CODE; reset it so each test's + # exit code is read in isolation (a non-zero/timed-out test must not poison + # the next one). + _BASHUNIT_TEST_EXIT_CODE=0 bashunit::runner::parse_result "$fn_name" "$test_execution_result" "$@" local test_exit_code="$_BASHUNIT_TEST_EXIT_CODE" @@ -1003,6 +1109,11 @@ function bashunit::runner::run_test() { fi fi + # A test that exceeded BASHUNIT_TEST_TIMEOUT gets a clear, specific message. + if [ "$timed_out" = "true" ]; then + error_message="Test timed out after $(bashunit::env::test_timeout_secs)s" + fi + bashunit::console_results::print_error_test "$failure_function" "$error_message" "$runtime_output" bashunit::reports::add_test_failed "$test_file" "$failure_label" "$duration" "$total_assertions" "$error_message" bashunit::runner::write_failure_result_output "$test_file" "$failure_function" "$error_message" "$runtime_output" diff --git a/tests/acceptance/bashunit_timeout_test.sh b/tests/acceptance/bashunit_timeout_test.sh new file mode 100644 index 00000000..e58c964f --- /dev/null +++ b/tests/acceptance/bashunit_timeout_test.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +set -euo pipefail + +function set_up_before_script() { + TEST_ENV_FILE="tests/acceptance/fixtures/.env.default" + FIXTURE="tests/acceptance/fixtures/test_bashunit_timeout.sh" +} + +function test_bashunit_terminates_a_hanging_test_with_timeout() { + local output + output="$(./bashunit --no-parallel --env "$TEST_ENV_FILE" --test-timeout 1 "$FIXTURE")" + + assert_contains "Test timed out after 1s" "$output" +} + +function test_bashunit_keeps_running_tests_after_a_timed_out_one() { + local output + output="$(./bashunit --no-parallel --env "$TEST_ENV_FILE" --test-timeout 1 "$FIXTURE")" + + # The fast test still ran and passed and the run reached its summary instead + # of hanging forever on the blocked test. + assert_contains "1 passed" "$output" + assert_contains "1 failed" "$output" +} + +function test_bashunit_returns_error_when_a_test_times_out() { + assert_general_error \ + "$(./bashunit --no-parallel --env "$TEST_ENV_FILE" --test-timeout 1 "$FIXTURE")" +} + +function test_bashunit_does_not_time_out_a_fast_test() { + local fast_only=./tests/acceptance/fixtures/test_bashunit_when_a_test_passes.sh + + assert_successful_code \ + "$(./bashunit --no-parallel --env "$TEST_ENV_FILE" --test-timeout 5 "$fast_only")" +} diff --git a/tests/acceptance/fixtures/test_bashunit_timeout.sh b/tests/acceptance/fixtures/test_bashunit_timeout.sh new file mode 100644 index 00000000..38174e54 --- /dev/null +++ b/tests/acceptance/fixtures/test_bashunit_timeout.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +function test_passes_quickly() { + assert_same "ok" "ok" +} + +function test_hangs_until_killed() { + sleep 30 + assert_same "never" "reached" +} diff --git a/tests/unit/env_test.sh b/tests/unit/env_test.sh index e68c1539..19689277 100644 --- a/tests/unit/env_test.sh +++ b/tests/unit/env_test.sh @@ -95,6 +95,43 @@ function test_is_dev_mode_disabled_when_dev_log_empty() { assert_successful_code 0 } +# @data_provider provide_test_timeout_enabled +function test_is_test_timeout_enabled(){ + local value="$1" + local expected="$2" + + local original="${BASHUNIT_TEST_TIMEOUT:-0}" + export BASHUNIT_TEST_TIMEOUT="$value" + + if bashunit::env::is_test_timeout_enabled; then + local actual="enabled" + else + local actual="disabled" + fi + + export BASHUNIT_TEST_TIMEOUT="$original" + assert_equals "$expected" "$actual" +} + +function provide_test_timeout_enabled() { + bashunit::data_set "0" "disabled" + bashunit::data_set "" "disabled" + bashunit::data_set "abc" "disabled" + bashunit::data_set "1" "enabled" + bashunit::data_set "5" "enabled" +} + +function test_test_timeout_secs_returns_the_configured_value() { + local original="${BASHUNIT_TEST_TIMEOUT:-0}" + export BASHUNIT_TEST_TIMEOUT="7" + + local result + result=$(bashunit::env::test_timeout_secs) + + export BASHUNIT_TEST_TIMEOUT="$original" + assert_equals "7" "$result" +} + function test_is_tap_output_enabled_when_format_is_tap() { local original="$BASHUNIT_OUTPUT_FORMAT" export BASHUNIT_OUTPUT_FORMAT="tap"