diff --git a/.editorconfig b/.editorconfig index f8e831d647..457107ead5 100644 --- a/.editorconfig +++ b/.editorconfig @@ -30,3 +30,6 @@ tab_width = 4 # trailing spaces indicates word wrap trim_trailing_spaces = false trim_trailing_whitespace = false + +[test/fixtures/bats/*_no_shellcheck.bats] +ignore = true \ No newline at end of file diff --git a/.github/workflows/check_pr_label.sh b/.github/workflows/check_pr_label.sh index 197335bd15..2ff5723c71 100755 --- a/.github/workflows/check_pr_label.sh +++ b/.github/workflows/check_pr_label.sh @@ -1,7 +1,7 @@ #!/usr/bin/bash get_pr_json() { - curl -s -H "Accept: application/vnd.github.v3+json" "https://api.github.com/repos/bats-core/bats-core/pulls/$1" + curl -s -H "Accept: application/vnd.github.v3+json" "https://api.github.com/repos/bats-core/bats-core/pulls/$1" } PR_NUMBER="$1" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 5fbb478bf7..b38f8201d9 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -23,7 +23,7 @@ jobs: - uses: actions/setup-node@v2 with: registry-url: "https://npm.pkg.github.com" - - name: scope package name as required by GHPR + - name: scope package name as required by GitHub Packages run: npm init -y --scope ${{ github.repository_owner }} - run: npm publish --ignore-scripts env: diff --git a/.github/workflows/release_dockerhub.yml b/.github/workflows/release_dockerhub.yml index 584c9b9bb4..e5faec1c89 100644 --- a/.github/workflows/release_dockerhub.yml +++ b/.github/workflows/release_dockerhub.yml @@ -13,26 +13,35 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 + - id: version run: | EXPECTED_VERSION=${{ github.event.inputs.version }} TAG_VERSION=${GITHUB_REF#refs/tags/v} # refs/tags/v1.2.3 -> 1.2.3 echo ::set-output name=version::${EXPECTED_VERSION:-$TAG_VERSION} - - - name: Set up QEMU + - name: Set up QEMU uses: docker/setup-qemu-action@v1 - - - name: Login to DockerHub + + + - name: Login to DockerHub uses: docker/login-action@v1 with: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - - - name: Set up Docker Buildx + + - name: Set up Docker Buildx id: buildx uses: docker/setup-buildx-action@v1 + - uses: docker/build-push-action@v2 with: platforms: linux/amd64,linux/arm64,linux/ppc64le,linux/s390x,linux/386,linux/arm/v7,linux/arm/v6 tags: ${{ secrets.DOCKER_USERNAME }}/bats:${{ steps.version.outputs.version }},${{ secrets.DOCKER_USERNAME }}/bats:latest push: true + + - uses: docker/build-push-action@v2 + with: + platforms: linux/amd64,linux/arm64,linux/ppc64le,linux/s390x,linux/386,linux/arm/v7,linux/arm/v6 + tags: ${{ secrets.DOCKER_USERNAME }}/bats:${{ steps.version.outputs.version }}-no-faccessat2,${{ secrets.DOCKER_USERNAME }}/bats:latest-no-faccessat2 + push: true + build-args: bashver=5.1.4 diff --git a/.github/workflows/set_nounset.bash b/.github/workflows/set_nounset.bash new file mode 100644 index 0000000000..8925c2d815 --- /dev/null +++ b/.github/workflows/set_nounset.bash @@ -0,0 +1 @@ +set -u diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index bd8fecfadb..332ec3cee1 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -14,6 +14,16 @@ jobs: grep "#${{github.event.pull_request.number}}" docs/CHANGELOG.md fi if: ${{github.event.pull_request}} + + shfmt: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v2 + - run: | + curl https://github.com/mvdan/sh/releases/download/v3.5.1/shfmt_v3.5.1_linux_amd64 -o shfmt + chmod a+x shfmt + - run: ./shfmt --diff . + shellcheck: runs-on: ubuntu-20.04 steps: @@ -27,7 +37,7 @@ jobs: linux: strategy: matrix: - os: ['ubuntu-20.04', 'ubuntu-18.04'] + os: ['ubuntu-20.04', 'ubuntu-22.04'] env_vars: - '' # allow for some parallelity without GNU parallel, since it is not installed by default @@ -43,10 +53,21 @@ jobs: bash --version bash -c "time ${{ matrix.env_vars }} bin/bats --print-output-on-failure --formatter tap test" + unset_variables: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Check unset variables + shell: 'script -q -e -c "bash {0}"' # work around tty issues + env: + TERM: linux # fix tput for tty issue work around + BASH_ENV: ${GITHUB_WORKSPACE}/.github/workflows/set_nounset.bash + run: bin/bats test --print-output-on-failure + npm_on_linux: strategy: matrix: - os: ['ubuntu-20.04', 'ubuntu-18.04'] + os: ['ubuntu-20.04', 'ubuntu-22.04'] runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v2 @@ -83,7 +104,7 @@ jobs: macos: strategy: matrix: - os: ['macos-10.15'] + os: ['macos-11', 'macos-12'] env_vars: - '' # allow for some parallelity without GNU parallel, since it is not installed by default @@ -104,7 +125,7 @@ jobs: npm_on_macos: strategy: matrix: - os: ['macos-10.15'] + os: ['macos-11', 'macos-12'] runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v2 @@ -117,14 +138,14 @@ jobs: TERM: linux # fix tput for tty issue work around run: | npm pack ./ - # somehow there is already an intalled bats version around + # somehow there is already an installed bats version around npm install --force -g ./bats-*.tgz bats --print-output-on-failure test bash-version: strategy: matrix: - version: ['3.2', '4.0', '4.1', '4.2', '4.3', '4.4', '4', '5.0', '5.1', '5', 'latest'] + version: ['3.2', '4.0', '4.1', '4.2', '4.3', '4.4', '4', '5.0', '5.1', '5', 'rc'] env_vars: - '' # also test running (recursively!) in parallel @@ -142,16 +163,20 @@ jobs: alpine: runs-on: ubuntu-latest + container: alpine:latest steps: - uses: actions/checkout@v2 + - name: Install dependencies + run: apk add bash ncurses util-linux - name: Run test on bash version ${{ matrix.version }} shell: 'script -q -e -c "bash {0}"' # work around tty issues - run: | - set -e - time docker run -it -v $PWD:/opt/bats alpine sh -c "apk add bash ncurses; /opt/bats/bin/bats --print-output-on-failure --tap /opt/bats/test" + env: + TERM: linux # fix tput for tty issue work around + run: + time ./bin/bats --print-output-on-failure test/ freebsd: - runs-on: macos-10.15 + runs-on: macos-12 strategy: matrix: packages: @@ -159,7 +184,7 @@ jobs: - "" steps: - uses: actions/checkout@v2 - - uses: vmactions/freebsd-vm@v0.1.5 + - uses: vmactions/freebsd-vm@v0 with: prepare: pkg install -y bash parallel ${{ matrix.packages }} run: | @@ -172,3 +197,23 @@ jobs: # list symlinks that are broken and force non-zero exit if there are any - run: "! find . -xtype l | grep ." + rpm: + runs-on: ubuntu-latest + container: almalinux:8 + steps: + - uses: actions/checkout@v2 + - run: dnf install -y rpm-build rpmdevtools + - name: Build and install RPM and dependencies + run: | + rpmdev-setuptree + version=$(rpmspec -q --qf '%{version}' contrib/rpm/bats.spec) + tar --transform "s,^,bats-core-${version}/," -cf /github/home/rpmbuild/SOURCES/v${version}.tar.gz ./ + rpmbuild -v -bb ./contrib/rpm/bats.spec + ls -al /github/home/rpmbuild/RPMS/noarch/ + dnf install -y /github/home/rpmbuild/RPMS/noarch/bats-*.rpm + dnf -y install procps-ng # avoid timeout failure + - name: Run tests + shell: 'script -q -e -c "bash {0}"' # work around tty issues + env: + TERM: linux # fix tput for tty issue work around + run: bats --print-output-on-failure --filter-tags !dep:install_sh test/ diff --git a/.gitignore b/.gitignore index f94915443e..432021d3cf 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ /bats-*.tgz # we don't have any deps; un-ignore if that changes /package-lock.json +test/.bats/run-logs/ \ No newline at end of file diff --git a/bin/bats b/bin/bats index 1f7ce2e970..892470c1e8 100755 --- a/bin/bats +++ b/bin/bats @@ -26,8 +26,8 @@ fallback_to_readlinkf_posix() { while [ "$max_symlinks" -ge 0 ] && max_symlinks=$((max_symlinks - 1)); do if [ ! "$target" = "${target%/*}" ]; then case $target in - /*) cd -P "${target%/*}/" 2>/dev/null || break ;; - *) cd -P "./${target%/*}" 2>/dev/null || break ;; + /*) cd -P "${target%/*}/" 2>/dev/null || break ;; + *) cd -P "./${target%/*}" 2>/dev/null || break ;; esac target=${target##*/} fi diff --git a/contrib/release.sh b/contrib/release.sh index fec7463773..2e4805e1f0 100755 --- a/contrib/release.sh +++ b/contrib/release.sh @@ -65,14 +65,14 @@ git commit -m "feat: release Bats v${NEW_BATS_VERSION}" # changelog start EOF -local DELIM -DELIM=$(echo -en "\001"); -sed -E -n "\\${DELIM}^## \[${NEW_BATS_VERSION}\]${DELIM},\\${DELIM}^## ${DELIM}p" docs/CHANGELOG.md \ - | head -n -1 \ - | sed -E \ - -e 's,^## \[([0-9\.]+)] - (.*),Bats \1\n\nReleased: \2,' \ - -e 's,^### (.*),\1:,g' \ - | tee "${BATS_RELEASE_NOTES}" + local DELIM + DELIM=$(echo -en "\001") + sed -E -n "\\${DELIM}^## \[${NEW_BATS_VERSION}\]${DELIM},\\${DELIM}^## ${DELIM}p" docs/CHANGELOG.md | + head -n -1 | + sed -E \ + -e 's,^## \[([0-9\.]+)] - (.*),Bats \1\n\nReleased: \2,' \ + -e 's,^### (.*),\1:,g' | + tee "${BATS_RELEASE_NOTES}" cat < - 1.1.0-1 +* Wed Sep 07 2022 Marcel Hecko - 1.2.0-1 +- Fix and test RPM build on Rocky Linux release 8.6 + +* Sun Jul 08 2018 mbland - 1.1.0-1 - Increase version to match upstream release * Mon Jun 18 2018 pixdrift - 1.0.2-1 diff --git a/contrib/semver b/contrib/semver index 176ade0fc8..ab75586cc6 100755 --- a/contrib/semver +++ b/contrib/semver @@ -101,23 +101,35 @@ function validate-version { } function is-nat { - [[ "$1" =~ ^($NAT)$ ]] + [[ "$1" =~ ^($NAT)$ ]] } function is-null { - [ -z "$1" ] + [ -z "$1" ] } function order-nat { - [ "$1" -lt "$2" ] && { echo -1 ; return ; } - [ "$1" -gt "$2" ] && { echo 1 ; return ; } - echo 0 + [ "$1" -lt "$2" ] && { + echo -1 + return + } + [ "$1" -gt "$2" ] && { + echo 1 + return + } + echo 0 } function order-string { - [[ $1 < $2 ]] && { echo -1 ; return ; } - [[ $1 > $2 ]] && { echo 1 ; return ; } - echo 0 + [[ $1 < $2 ]] && { + echo -1 + return + } + [[ $1 > $2 ]] && { + echo 1 + return + } + echo 0 } # given two (named) arrays containing NAT and/or ALPHANUM fields, compare them @@ -126,33 +138,56 @@ function order-string { # is considered greater-than the shorter if the shorter is a prefix of the longer. # function compare-fields { - local l="$1[@]" - local r="$2[@]" - local leftfield=( "${!l}" ) - local rightfield=( "${!r}" ) - local left - local right - - local i=$(( -1 )) - local order=$(( 0 )) - - while true - do - [ $order -ne 0 ] && { echo $order ; return ; } - - : $(( i++ )) - left="${leftfield[$i]}" - right="${rightfield[$i]}" - - is-null "$left" && is-null "$right" && { echo 0 ; return ; } - is-null "$left" && { echo -1 ; return ; } - is-null "$right" && { echo 1 ; return ; } - - is-nat "$left" && is-nat "$right" && { order=$(order-nat "$left" "$right") ; continue ; } - is-nat "$left" && { echo -1 ; return ; } - is-nat "$right" && { echo 1 ; return ; } - { order=$(order-string "$left" "$right") ; continue ; } - done + local l="$1[@]" + local r="$2[@]" + local leftfield=("${!l}") + local rightfield=("${!r}") + local left + local right + + local i=$((-1)) + local order=$((0)) + + while true; do + [ $order -ne 0 ] && { + echo $order + return + } + + : $((i++)) + left="${leftfield[$i]}" + right="${rightfield[$i]}" + + is-null "$left" && is-null "$right" && { + echo 0 + return + } + is-null "$left" && { + echo -1 + return + } + is-null "$right" && { + echo 1 + return + } + + is-nat "$left" && is-nat "$right" && { + order=$(order-nat "$left" "$right") + continue + } + is-nat "$left" && { + echo -1 + return + } + is-nat "$right" && { + echo 1 + return + } + { + order=$(order-string "$left" "$right") + continue + } + done } # shellcheck disable=SC2206 # checked by "validate"; ok to expand prerel id's into array @@ -163,25 +198,37 @@ function compare-version { # compare major, minor, patch - local left=( "${V[0]}" "${V[1]}" "${V[2]}" ) - local right=( "${V_[0]}" "${V_[1]}" "${V_[2]}" ) + local left=("${V[0]}" "${V[1]}" "${V[2]}") + local right=("${V_[0]}" "${V_[1]}" "${V_[2]}") order=$(compare-fields left right) - [ "$order" -ne 0 ] && { echo "$order" ; return ; } + [ "$order" -ne 0 ] && { + echo "$order" + return + } # compare pre-release ids when M.m.p are equal local prerel="${V[3]:1}" local prerel_="${V_[3]:1}" - local left=( ${prerel//./ } ) - local right=( ${prerel_//./ } ) + local left=(${prerel//./ }) + local right=(${prerel_//./ }) # if left and right have no pre-release part, then left equals right # if only one of left/right has pre-release part, that one is less than simple M.m.p - [ -z "$prerel" ] && [ -z "$prerel_" ] && { echo 0 ; return ; } - [ -z "$prerel" ] && { echo 1 ; return ; } - [ -z "$prerel_" ] && { echo -1 ; return ; } + [ -z "$prerel" ] && [ -z "$prerel_" ] && { + echo 0 + return + } + [ -z "$prerel" ] && { + echo 1 + return + } + [ -z "$prerel_" ] && { + echo -1 + return + } # otherwise, compare the pre-release id's @@ -189,18 +236,27 @@ function compare-version { } function command-bump { - local new; local version; local sub_version; local command; + local new + local version + local sub_version + local command case $# in - 2) case $1 in - major|minor|patch|release) command=$1; version=$2;; - *) usage-help;; - esac ;; - 3) case $1 in - prerel|build) command=$1; sub_version=$2 version=$3 ;; - *) usage-help;; - esac ;; - *) usage-help;; + 2) case $1 in + major | minor | patch | release) + command=$1 + version=$2 + ;; + *) usage-help ;; + esac ;; + 3) case $1 in + prerel | build) + command=$1 + sub_version=$2 version=$3 + ;; + *) usage-help ;; + esac ;; + *) usage-help ;; esac validate-version "$version" parts @@ -212,13 +268,13 @@ function command-bump { local build="${parts[4]}" case "$command" in - major) new="$((major + 1)).0.0";; - minor) new="${major}.$((minor + 1)).0";; - patch) new="${major}.${minor}.$((patch + 1))";; - release) new="${major}.${minor}.${patch}";; - prerel) new=$(validate-version "${major}.${minor}.${patch}-${sub_version}");; - build) new=$(validate-version "${major}.${minor}.${patch}${prere}+${sub_version}");; - *) usage-help ;; + major) new="$((major + 1)).0.0" ;; + minor) new="${major}.$((minor + 1)).0" ;; + patch) new="${major}.${minor}.$((patch + 1))" ;; + release) new="${major}.${minor}.${patch}" ;; + prerel) new=$(validate-version "${major}.${minor}.${patch}-${sub_version}") ;; + build) new=$(validate-version "${major}.${minor}.${patch}${prere}+${sub_version}") ;; + *) usage-help ;; esac echo "$new" @@ -226,56 +282,77 @@ function command-bump { } function command-compare { - local v; local v_; + local v + local v_ case $# in - 2) v=$(validate-version "$1"); v_=$(validate-version "$2") ;; - *) usage-help ;; + 2) + v=$(validate-version "$1") + v_=$(validate-version "$2") + ;; + *) usage-help ;; esac - set +u # need unset array element to evaluate to null + set +u # need unset array element to evaluate to null compare-version "$v" "$v_" exit 0 } - # shellcheck disable=SC2034 function command-get { - local part version + local part version - if [[ "$#" -ne "2" ]] || [[ -z "$1" ]] || [[ -z "$2" ]]; then - usage-help - exit 0 - fi + if [[ "$#" -ne "2" ]] || [[ -z "$1" ]] || [[ -z "$2" ]]; then + usage-help + exit 0 + fi - part="$1" - version="$2" + part="$1" + version="$2" - validate-version "$version" parts - local major="${parts[0]}" - local minor="${parts[1]}" - local patch="${parts[2]}" - local prerel="${parts[3]:1}" - local build="${parts[4]:1}" - local release="${major}.${minor}.${patch}" + validate-version "$version" parts + local major="${parts[0]}" + local minor="${parts[1]}" + local patch="${parts[2]}" + local prerel="${parts[3]:1}" + local build="${parts[4]:1}" + local release="${major}.${minor}.${patch}" - case "$part" in - major|minor|patch|release|prerel|build) echo "${!part}" ;; - *) usage-help ;; - esac + case "$part" in + major | minor | patch | release | prerel | build) echo "${!part}" ;; + *) usage-help ;; + esac - exit 0 + exit 0 } case $# in - 0) echo "Unknown command: $*"; usage-help;; +0) + echo "Unknown command: $*" + usage-help + ;; esac case $1 in - --help|-h) echo -e "$USAGE"; exit 0;; - --version|-v) usage-version ;; - bump) shift; command-bump "$@";; - get) shift; command-get "$@";; - compare) shift; command-compare "$@";; - *) echo "Unknown arguments: $*"; usage-help;; +--help | -h) + echo -e "$USAGE" + exit 0 + ;; +--version | -v) usage-version ;; +bump) + shift + command-bump "$@" + ;; +get) + shift + command-get "$@" + ;; +compare) + shift + command-compare "$@" + ;; +*) + echo "Unknown arguments: $*" + usage-help + ;; esac diff --git a/docker/install_tini.sh b/docker/install_tini.sh index cd4f419cf2..8d98da14c6 100755 --- a/docker/install_tini.sh +++ b/docker/install_tini.sh @@ -3,18 +3,18 @@ set -e case ${1#linux/} in - 386) - TINI_PLATFORM=i386 - ;; - arm/v7) - TINI_PLATFORM=armhf - ;; - arm/v6) - TINI_PLATFORM=armel - ;; - *) - TINI_PLATFORM=${1#linux/} - ;; +386) + TINI_PLATFORM=i386 + ;; +arm/v7) + TINI_PLATFORM=armhf + ;; +arm/v6) + TINI_PLATFORM=armel + ;; +*) + TINI_PLATFORM=${1#linux/} + ;; esac echo "Installing tini for $TINI_PLATFORM" @@ -25,6 +25,6 @@ wget "https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini-sta chmod +x /tini apk add gnupg -gpg --import < /tmp/docker/tini.pubkey.gpg +gpg --import ` (without space) (#657) + +## [1.8.0] - 2022-09-15 + +### Added + +* using external formatters via `--formatter ` (also works for + `--report-formatter`) (#602) +* running only tests that failed in the last run via `--filter-status failed` (#483) +* variable `BATS_TEST_RETRIES` that specifies how often a test should be + reattempted before it is considered failed (#618) +* Docker tags `latest-no-faccessat2` and `-no-faccessat2` for + avoiding `bash: bats: No such file or directory` on `docker<20.10` (or + `runc`/`# bats file_tags=` and + `--filter-tags ` for tagging tests for execution filters (#642) +* warning BW03: inform about `setup_suite` in wrong file (`.bats` instead of `setup_suite.bash`) (#652) + +#### Documentation + +* update gotcha about negated statements: Recommend using `run !` on Bats + versions >=1.5.0 (#593) +* add documentation for `bats_require_minimum_version` (#595) +* improve documentation about `setup_suite` (#652) + +### Fixed + +* added missing shebang (#597) +* remaining instances of `run -` being incorrectly documented as `run =` (#599) +* allow `--gather-test-outputs-in ` to work with existing, empty + directories (#603) + * also add `--clean-and-gather-test-outputs-in ` for improved UX +* double slashes in paths derived from TMPDIR on MacOS (#607) +* fix `load` in `teardown` marking failed tests as not run (#612) +* fix unset variable errors (with set -u) and add regression test (#621) +* `teardown_file` errors don't swallow `setup_file` errors anymore, the behavior + is more like `teardown`'s now (only `return`/last command can trigger `teardown` + errors) (#623) +* upgraded from deprecated CI envs for MacOS (10 -> 11,12) and Ubuntu + (18.04 -> 22.04) (#630) +* add `/usr/lib/bats` as default value for `BATS_LIB_PATH` (#628) +* fix unset variable in `bats-formatter-junit` when `setup_file` fails (#632) +* unify error behavior of `teardown`/`teardown_file`/`teardown_suite` functions: + only fail via return code, not via ERREXIT (#633) +* fix unbound variable errors with `set -u` on `setup_suite` failures (#643) +* fix `load` not being available in `setup_suite` (#644) +* fix RPM spec, add regression test (#648) +* fix handling of `IFS` by `run` (#650) +* only print `setup_suite`'s stderr on errors (#649) + +#### Documentation + +* fix typos, spelling and links (#596, #604, #619, #627) +* fix redirection order of an example in the tutorial (#617) + ## [1.7.0] - 2022-05-14 ### Added @@ -43,6 +113,22 @@ The format is based on [Keep a Changelog][kac] and this project adheres to * remove 2018 in title, update copyright dates in README.md (#567) * fix broken links (#568) +* corrected invalid documentation of `run -N` (had `=N` instead) (#579) + * **CRITICAL**: using the incorrect form can lead to silent errors. See + [issue #578](https://github.com/bats-core/bats-core/issues/578) for more + details and how to find out if your tests are affected. + +## [1.6.1] - 2022-05-14 + +### Fixed + +* prevent `teardown`, `teardown_file`, and `teardown_suite` from overriding bats' + exit code by setting `$status` (e.g. via calling `run`) (#581, #575) + * **CRITICAL**: this can return exit code 0 despite failed tests, thus preventing + your CI from reporting test failures! The regression happened in version 1.6.0. + +#### Documentation + * corrected invalid documentation of `run -N` (had `=N` instead) (#579) * **CRITICAL**: using the incorrect form can lead to silent errors. See [issue #578](https://github.com/bats-core/bats-core/issues/578) for more @@ -166,7 +252,7 @@ quotes around code blocks in error output (#506) * custom test-file extension via `BATS_FILE_EXTENSION` when searching for test files in a directory (#376) * TAP13 formatter, including millisecond timing (#337) -* automatic release to NPM via Github Actions (#406) +* automatic release to NPM via GitHub Actions (#406) #### Documentation @@ -182,7 +268,7 @@ quotes around code blocks in error output (#506) `--report-formatter junit` to obtain the `.xml` report file! * removed `--parallel-preserve-environment` flag, as this is the default behavior (#324) -* moved CI from Travis/Appveyor to Github Actions (#405) +* moved CI from Travis/AppVeyor to GitHub Actions (#405) * preprocessed files are no longer removed if `--no-tempdir-cleanup` is specified (#395) @@ -368,7 +454,16 @@ Changes: * Initial public release. -[Unreleased]: https://github.com/bats-core/bats-core/compare/v1.1.0...HEAD +[Unreleased]: https://github.com/bats-core/bats-core/compare/v1.7.0...HEAD +[1.7.0]: https://github.com/bats-core/bats-core/compare/v1.6.1...v1.7.0 +[1.6.1]: https://github.com/bats-core/bats-core/compare/v1.6.0...v1.6.1 +[1.6.0]: https://github.com/bats-core/bats-core/compare/v1.5.0...v1.6.0 +[1.5.0]: https://github.com/bats-core/bats-core/compare/v1.4.1...v1.5.0 +[1.4.1]: https://github.com/bats-core/bats-core/compare/v1.4.0...v1.4.1 +[1.4.0]: https://github.com/bats-core/bats-core/compare/v1.3.0...v1.4.0 +[1.3.0]: https://github.com/bats-core/bats-core/compare/v1.2.1...v1.3.0 +[1.2.1]: https://github.com/bats-core/bats-core/compare/v1.2.0...v1.2.1 +[1.2.0]: https://github.com/bats-core/bats-core/compare/v1.1.0...v1.2.0 [1.1.0]: https://github.com/bats-core/bats-core/compare/v1.0.2...v1.1.0 [1.0.2]: https://github.com/bats-core/bats-core/compare/v1.0.1...v1.0.2 [1.0.1]: https://github.com/bats-core/bats-core/compare/v1.0.0...v1.0.1 diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md index b8501627f3..263e41197f 100644 --- a/docs/CONTRIBUTING.md +++ b/docs/CONTRIBUTING.md @@ -34,7 +34,7 @@ CONTRIBUTING.md file][atom]. * [Contributing Guidelines](#contributing-guidelines) * [Welcome!](#welcome) * [Table of contents](#table-of-contents) - * [Quick links 🔗](#quick-links-) + * [Quick links](#quick-links) * [Contributor License Agreement](#contributor-license-agreement) * [Code of conduct](#code-of-conduct) * [Asking questions and reporting issues](#asking-questions-and-reporting-issues) @@ -55,7 +55,7 @@ CONTRIBUTING.md file][atom]. * [Open Source License](#open-source-license) * [Credits](#credits) -## Quick links 🔗 +## Quick links - [Gitter channel →][gitterurl]: Feel free to come chat with us on Gitter - [README →][README] @@ -90,7 +90,7 @@ See also: ["Does my project need an additional contributor agreement? Probably ## Code of conduct -Harrassment or rudeness of any kind will not be tolerated, period. For +Harassment or rudeness of any kind will not be tolerated, period. For specifics, see the [CODE_OF_CONDUCT][] file. ## Asking questions and reporting issues @@ -144,7 +144,7 @@ Also consider using: 1. DO add information if you're facing a similar issue to someone else, but within a different context (e.g. different steps needed to reproduce the issue than previous stated, different version of Bash or BATS, different OS, etc.) -You can read on how to do that here: [Information to include][#information-to-include] +You can read on how to do that here: [Information to include](#information-to-include) 1. DO remember that you can use the *Subscribe* button on the right side of the page to receive notifications of further conversations or a resolution. @@ -295,9 +295,9 @@ The following are intended to prevent too-compact code: the difference avoiding subshells makes.) Bash is quite powerful; see if you can do what you need in pure Bash first. - If you need to capture the output from a function, store the output using - `printf -v` instead if possible. `-v` specfies the name of the variable into + `printf -v` instead if possible. `-v` specifies the name of the variable into which to write the result; the caller can supply this name as a parameter. -- If you must use command substituion, use `$()` instead of backticks, as it's +- If you must use command substitution, use `$()` instead of backticks, as it's more robust, more searchable, and can be nested. [win-slow]: https://rufflewind.com/2014-08-23/windows-bash-slow diff --git a/docs/source/faq.rst b/docs/source/faq.rst index ebe1edf75c..0cecfa7aaa 100644 --- a/docs/source/faq.rst +++ b/docs/source/faq.rst @@ -145,7 +145,8 @@ If you need the library inside `setup_file` or `teardown_file` you need to load How to set a test timeout in bats? ---------------------------------- -Unfortunately, this is not possible yet. Please contribute to issue `#396 `_ for further progress. +Set the variable `$BATS_TEST_TIMEOUT` before `setup()` starts. This means you can set it either on the command line, +in free code in the test file or in `setup_file()`. How can I lint/shell-format my bats tests? ------------------------------------------ diff --git a/docs/source/gotchas.rst b/docs/source/gotchas.rst index 17dfc05dee..485b675030 100644 --- a/docs/source/gotchas.rst +++ b/docs/source/gotchas.rst @@ -12,10 +12,11 @@ Please adhere to this idiom while using bats, or you will constantly work agains My negated statement (e.g. ! true) does not fail the test, even when it should. ------------------------------------------------------------------------------- -Bash deliberately excludes negated return values from causing a pipeline to exit (see bash's `-e` option). You'll need to use the form `! x || false` or (recommended) use `run` and check for `[ $status != 0 ]`. +Bash deliberately excludes negated return values from causing a pipeline to exit (see bash's `-e` option). +Use `run !` on Bats 1.5.0 and above. For older bats versions, use one of `! x || false` or `run` with `[ $status != 0 ]`. If the negated command is the final statement in a test, that final statement's (negated) exit status will propagate through to the test's return code as usual. -Negated statements of the form `! x || false` will explicitly fail the test when the pipeline returns true, regardless of where they occur in the test. +Negated statements of one of the correct forms mentioned above will explicitly fail the test when the pipeline returns true, regardless of where they occur in the test. I cannot register a test multiple times via for loop. ----------------------------------------------------- diff --git a/docs/source/tutorial.rst b/docs/source/tutorial.rst index 8cd8eb28c7..72a5409d86 100644 --- a/docs/source/tutorial.rst +++ b/docs/source/tutorial.rst @@ -43,7 +43,7 @@ Your first test Now we want to add our first test. In the tutorial repository, we want to build up our project in a TDD fashion. -Thus, we start with an empty project and our first test is to just run our (non existing) shell script. +Thus, we start with an empty project and our first test is to just run our (nonexistent) shell script. We start by creating a new test file `test/test.bats` @@ -231,7 +231,7 @@ Then, `run` sucks up the stdout and stderr of the command it ran and stores it i This means `run` never fails the test and won't generate any context/output in the log of a failed test on its own. Marking the test as failed and printing context information is up to the consumers of `$status` and `$output`. -`assert_output` is such a consumer, it compares `$output` to the the parameter it got and tells us quite succinctly that it did not match in this case. +`assert_output` is such a consumer, it compares `$output` to the parameter it got and tells us quite succinctly that it did not match in this case. For our current test we don't care about any other output or the error message, so we want it gone. `grep` is always at our fingertips, so we tape together this ramshackle construct @@ -257,7 +257,7 @@ This is a common mistake that can happen when our mind parses the file different `run` is just a function, so the pipe won't actually be forwarded into the function. Bash reads this as `(run project.sh) | grep Welcome`, instead of our intended `run (project.sh | grep Welcome)`. -Unfortunately, the latter is no valid bash syntax, so we have to work around it, e.g. by using a function: +Unfortunately, the latter is not valid bash syntax, so we have to work around it, e.g. by using a function: .. code-block:: bash @@ -567,7 +567,7 @@ As an example, we want to add an echo server capability to our project. First, w setup_file() { load 'test_helper/common-setup' _common_setup - PORT=$(project.sh start-echo-server >/dev/null 2>&1) + PORT=$(project.sh start-echo-server 2>&1 >/dev/null) export PORT } diff --git a/docs/source/usage.md b/docs/source/usage.md index e4b1d87b7d..bdbe407d50 100644 --- a/docs/source/usage.md +++ b/docs/source/usage.md @@ -54,14 +54,33 @@ ok 1 addition using bc ok 2 addition using dc ``` -Test reports will be output in the executing directory, but may be placed elsewhere -by specifying the `--output` flag. +If you have your own formatter, you can use an absolute path to the executable +to use it: + +```bash +$ bats --formatter /absolute/path/to/my-formatter addition.bats +addition using bc WORKED +addition using dc FAILED +``` + +You can also generate test report files via `--report-formatter` which accepts +the same options as `--formatter`. By default, the file is stored in the current +workdir. However, it may be placed elsewhere by specifying the `--output` flag. ```text -$ bats --formatter junit addition.bats --output /tmp +$ bats --report-formatter junit addition.bats --output /tmp 1..2 ok 1 addition using bc ok 2 addition using dc + +$ cat /tmp/report.xml + + + + + + + ``` ## Parallel Execution @@ -76,7 +95,7 @@ a compatible replacement installed) using the `--jobs` parameter. This can result in your tests completing faster (depending on your tests and the testing hardware). -Ordering of parallised tests is not guaranteed, so this mode may break suites +Ordering of parallelised tests is not guaranteed, so this mode may break suites with dependencies between tests (or tests that write to shared locations). When enabling `--jobs` for the first time be sure to re-run bats multiple times to identify any inter-test dependencies or non-deterministic test behaviour. @@ -88,7 +107,7 @@ sequentially. If you have files where tests within the file would interfere with each other, you can use `--no-parallelize-within-files` to disable parallelization within all files. -If you want more finegrained control, you can `export BATS_NO_PARALLELIZE_WITHIN_FILE=true` in `setup_file()` +If you want more fine-grained control, you can `export BATS_NO_PARALLELIZE_WITHIN_FILE=true` in `setup_file()` or outside any function to disable parallelization only within the containing file. [tap-format]: https://testanything.org diff --git a/docs/source/warnings/BW03.rst b/docs/source/warnings/BW03.rst new file mode 100644 index 0000000000..2a0adc0dce --- /dev/null +++ b/docs/source/warnings/BW03.rst @@ -0,0 +1,15 @@ +BW03: `setup_suite` is visible to test file '', but was not executed. It belongs into 'setup_suite.bash' to be picked up automatically. +============================================================================================================================================= + +In contrast to the other setup functions, `setup_suite` must not be defined in `*.bats` files but in `setup_suite.bash`. +When a file is executed and sees `setup_suite` defined but not run before the tests, this warning will be printed. + +How to fix +---------- + +The fix depends on your actual intention. There are basically two cases: + +1. You want a setup before all tests and accidentally put `setup_suite` into a test file instead of `setup_suite.bash`. + Simply move `setup_suite` (and `teardown_suite`!) into `setup_suite.bash`. +2. You did not mean to run a setup before any test but need to defined a function named `setup_suite` in your test file. + In this case, you can silence this warning by assigning `BATS_SETUP_SUITE_COMPLETED='suppress BW03'`. \ No newline at end of file diff --git a/docs/source/warnings/index.rst b/docs/source/warnings/index.rst index 5a315000e3..957fc4305f 100644 --- a/docs/source/warnings/index.rst +++ b/docs/source/warnings/index.rst @@ -24,4 +24,5 @@ Currently, Bats emits the following warnings: .. toctree:: BW01 - BW02 \ No newline at end of file + BW02 + BW03 \ No newline at end of file diff --git a/docs/source/writing-tests.md b/docs/source/writing-tests.md index 89d06bdf0d..fdb685d13f 100644 --- a/docs/source/writing-tests.md +++ b/docs/source/writing-tests.md @@ -12,6 +12,85 @@ For sample test files, see [examples](https://github.com/bats-core/bats-core/tre [bats-eval]: https://github.com/bats-core/bats-core/wiki/Bats-Evaluation-Process +## Tagging tests + +Startig with version 1.8.0, Bats comes with a tagging system, that allows users +to categorize their tests and filter according to those categories. + +Each test has a list of tags attached to it. Without specification, this list is empty. +Tags can be defined in two ways. The first being `# bats test_tags=`: + +```bash +# bats test_tags=tag:1, tag:2, tag:3 +@test "second test" { + # ... +} + +@test "second test" { + # ... +} +``` + +These tags (`tag:1`, `tag:2`, `tag:3`) will be attached to the test `first test`. +The second test will have no tags attached. Values defined in the `# bats test_tags=` +directive will be assigned to the next `@test` that is being encountered in the +file and forgotten after that. Only the value of the last `# bats test_tags=` directive +before a given test will be used. + +Sometimes, we want to give all tests in a file a set of the same tags. This can +be achieved via `# bats file_tags=`. They will be added to all tests in the file +after that directive. An additional `# bats file_tags=` directive will override +the previously defined values: + +```bash +@test "Zeroth test" { + # will have no tags +} + +# bats file_tags=a:b +# bats test_tags=c:d + +@test "First test" { + # will be tagged a:b, c:d +} + +# bats file_tags= + +@test "Second test" { + # will have no tags +} +``` + +Tags are case sensitive and must only consist of alphanumeric characters and `_`, + `-`, or `:`. They must not contain whitespaces! +The colon is intended as a separator for (recursive) namespacing. + +Tag lists must be separated by commas and are allowed to contain whitespace. +They must not contain empty tags like `test_tags=,b` (first tag is empty), +`test_tags=a,,c`, `test_tags=a, ,c` (second tag is only whitespace/empty), +`test_tags=a,b,` (third tag is empty). + +Every tag starting with a `bats:` (case insensitive!) is reserved for Bats' +internal use. + +### Filtering execution + +Tags can be used for more finegrained filtering of which tests to run via `--filter-tags`. +This accepts a comma separated list of tags. Only tests that match all of these +tags will be executed. For example, `bats --filter-tags a,b,c` will pick up tests +with tags `a,b,c`, but not tests that miss one or more of those tags. + +Additionally, you can specify negative tags via `bats --filter-tags a,!b,c`, +which now won't match tests with tags `a,b,c`, due to the `b`, but will select `a,c`. +To put it more formally, `--filter-tags` is a boolean conjunction. + +To allow for more complex queries, you can specify multiple `--filter-tags`. +A test will be executed, if it matches at least one of them. +This means multiple `--filter-tags` form a boolean disjunction. + +A query of `--filter-tags a,!b --filter-tags b,c` can be translated to: +Execute only tests that (have tag a, but not tag b) or (have tag b and c). + ## Comment syntax External tools (like `shellcheck`, `shfmt`, and various IDE's) may not support @@ -183,7 +262,7 @@ Apart from the changed lookup rules, `bats_load_library` behaves like `load`. __Note:__ As seen above `load.bash` is the entry point for libraries and meant to load more files from its directory or other libraries. -__Note:__ Obviously, the actual `BATS_LIB_PATH` is highly dependant on the environment. +__Note:__ Obviously, the actual `BATS_LIB_PATH` is highly dependent on the environment. To maintain a uniform location across systems, (distribution) package maintainers are encouraged to use `/usr/lib/bats/` as the install path for libraries where possible. However, if the package manager has another preferred location, like `npm` or `brew`, @@ -241,11 +320,18 @@ functions (`setup`, the test itself, `teardown`, `teardown_file`). Similarly, there is `setup_suite` (and `teardown_suite`) which run once before (and after) all tests of the test run. +__Note:__ As `setup_suite` and `teardown_suite` are intended for all files in a suite, +they must be defined in a separate `setup_suite.bash` file. Automatic discovery works +by searching for `setup_suite.bash` in the folder of the first `*.bats` file of the suite. +If this automatism does not work for your usecase, you can work around by specifying +`--setup-suite-file` on the `bats` command. If you have a `setup_suite.bash`, it must define +`setup_suite`! However, defining `teardown_suite` is optional. +
Example of setup/{,_file,_suite} (and teardown{,_file,_suite}) call order For example the following call order would result from two files (file 1 with -tests 1 and 2, and file 2 with test3) with a corresponding `setup_suite.bash` file beeing tested: +tests 1 and 2, and file 2 with test3) with a corresponding `setup_suite.bash` file being tested: ```text setup_suite # from setup_suite.bash @@ -268,6 +354,64 @@ teardown_suite # from setup_suite.bash
+Note that the `teardown*` functions can fail a test, if their return code is nonzero. +This means, using `return 1` or having the last command in teardown fail, will +fail the teardown. Unlike `@test`, failing commands within `teardown` won't +trigger failure as ERREXIT is disabled. + + +
+ Example of different teardown failure modes + +```bash +teardown() { + false # this will fail the test, as it determines the return code +} + +teardown() { + false # this won't fail the test ... + echo some more code # ... and this will be executed too! +} + +teardown() { + return 1 # this will fail the test, but the rest won't be executed + echo some more code +} + +teardown() { + if true; then + false # this will also fail the test, as it is the last command in this function + else + true + fi +} +``` + +
+ + +## `bats_require_minimum_version ` + +Added in [v1.7.0](https://github.com/bats-core/bats-core/releases/tag/v1.7.0) + +Code for newer versions of Bats can be incompatible with older versions. +In the best case this will lead to an error message and a failed test suite. +In the worst case, the tests will pass erroneously, potentially masking a failure. + +Use `bats_require_minimum_version ` to avoid this. +It communicates in a concise manner, that you intend the following code to be run +under the given Bats version or higher. + +Additionally, this function will communicate the current Bats version floor to +subsequent code, allowing e.g. Bats' internal warning to give more informed warnings. + +__Note__: By default, calling `bats_require_minimum_version` with versions before +Bats 1.7.0 will fail regardless of the required version as the function is not +available. However, you can use the +[bats-backports plugin](https://github.com/bats-core/bats-backports) to make +your code usable with older versions, e.g. during migration while your CI system +is not yet upgraded. + ## Code outside of test cases You can include code in your test file outside of `@test` functions. For @@ -354,6 +498,13 @@ There are several global variables you can use to introspect on Bats tests: - `BATS_TEST_NAME_PREFIX` will be prepended to the description of each test on stdout and in reports. - `$BATS_TEST_DESCRIPTION` is the description of the current test case. +- `BATS_TEST_RETRIES` is the maximum number of additional attempts that will be + made on a failed test before it is finally considered failed. + The default of 0 means the test must pass on the first attempt. +- `BATS_TEST_TIMEOUT` is the number of seconds after which a test (including setup) + will be aborted and marked as failed. Updates to this value in `setup()` or `@test` + cannot change the running timeout countdown, so the latest useful update location + is `setup_file()`. - `$BATS_TEST_NUMBER` is the (1-based) index of the current test case in the test file. - `$BATS_SUITE_TEST_NUMBER` is the (1-based) index of the current test case in the test suite (over all files). - `$BATS_TMPDIR` is the base temporary directory used by bats to create its diff --git a/lib/bats-core/common.bash b/lib/bats-core/common.bash index 8da47794cd..756d31ad0b 100644 --- a/lib/bats-core/common.bash +++ b/lib/bats-core/common.bash @@ -1,12 +1,12 @@ #!/usr/bin/env bash bats_prefix_lines_for_tap_output() { - while IFS= read -r line; do - printf '# %s\n' "$line" || break # avoid feedback loop when errors are redirected into BATS_OUT (see #353) - done - if [[ -n "$line" ]]; then - printf '# %s\n' "$line" - fi + while IFS= read -r line; do + printf '# %s\n' "$line" || break # avoid feedback loop when errors are redirected into BATS_OUT (see #353) + done + if [[ -n "$line" ]]; then + printf '# %s\n' "$line" + fi } function bats_replace_filename() { @@ -20,7 +20,7 @@ function bats_replace_filename() { } bats_quote_code() { # - printf -v "$1" -- "%s%s%s" "$BATS_BEGIN_CODE_QUOTE" "$2" "$BATS_END_CODE_QUOTE" + printf -v "$1" -- "%s%s%s" "$BATS_BEGIN_CODE_QUOTE" "$2" "$BATS_END_CODE_QUOTE" } bats_check_valid_version() { @@ -36,13 +36,13 @@ bats_version_lt() { # bats_check_valid_version "$2" local -a version1_parts version2_parts - IFS=. read -ra version1_parts <<< "$1" - IFS=. read -ra version2_parts <<< "$2" + IFS=. read -ra version1_parts <<<"$1" + IFS=. read -ra version2_parts <<<"$2" for i in {0..2}; do - if (( version1_parts[i] < version2_parts[i] )); then + if ((version1_parts[i] < version2_parts[i])); then return 0 - elif (( version1_parts[i] > version2_parts[i] )); then + elif ((version1_parts[i] > version2_parts[i])); then return 1 fi done @@ -63,3 +63,151 @@ bats_require_minimum_version() { # BATS_GUARANTEED_MINIMUM_VERSION="$required_minimum_version" fi } + +bats_binary_search() { # + if [[ $# -ne 2 ]]; then + printf "ERROR: bats_binary_search requires exactly 2 arguments: \n" >&2 + return 2 + fi + + local -r search_value=$1 array_name=$2 + + # we'd like to test if array is set but we cannot distinguish unset from empty arrays, so we need to skip that + + local start=0 mid end mid_value + # start is inclusive, end is exclusive ... + eval "end=\${#${array_name}[@]}" + + # so start == end means empty search space + while ((start < end)); do + mid=$(((start + end) / 2)) + eval "mid_value=\${${array_name}[$mid]}" + if [[ "$mid_value" == "$search_value" ]]; then + return 0 + elif [[ "$mid_value" < "$search_value" ]]; then + # This branch excludes equality -> +1 to skip the mid element. + # This +1 also avoids endless recursion on odd sized search ranges. + start=$((mid + 1)) + else + end=$mid + fi + done + + # did not find it -> its not there + return 1 +} + +# store the values in ascending order in result array +# Intended for short lists! +bats_sort() { # + local -r result_name=$1 + shift + + if (($# == 0)); then + eval "$result_name=()" + return 0 + fi + + local -a sorted_array=() + local -i j i=0 + for ((j = 1; j <= $#; ++j)); do + for ((i = ${#sorted_array[@]}; i >= 0; --i)); do + if [[ $i -eq 0 || ${sorted_array[$((i - 1))]} < ${!j} ]]; then + sorted_array[$i]=${!j} + break + else + sorted_array[$i]=${sorted_array[$((i - 1))]} + fi + done + done + + eval "$result_name=(\"\${sorted_array[@]}\")" +} + +# check if all search values (must be sorted!) are in the (sorted!) array +# Intended for short lists/arrays! +bats_all_in() { # + local -r haystack_array=$1 + shift + + local -i haystack_length # just to appease shellcheck + eval "local -r haystack_length=\${#${haystack_array}[@]}" + + local -i haystack_index=0 # initialize only here to continue from last search position + local search_value haystack_value # just to appease shellcheck + for ((i = 1; i <= $#; ++i)); do + eval "local search_value=${!i}" + for (( ; haystack_index < haystack_length; ++haystack_index)); do + eval "local haystack_value=\${${haystack_array}[$haystack_index]}" + if [[ $haystack_value > "$search_value" ]]; then + # we passed the location this value would have been at -> not found + return 1 + elif [[ $haystack_value == "$search_value" ]]; then + continue 2 # search value found -> try the next one + fi + done + return 1 # we ran of the end of the haystack without finding the value! + done + + # did not return from loop above -> all search values were found + return 0 +} + +# check if any search value (must be sorted!) is in the (sorted!) array +# intended for short lists/arrays +bats_any_in() { # + local -r haystack_array=$1 + shift + + local -i haystack_length # just to appease shellcheck + eval "local -r haystack_length=\${#${haystack_array}[@]}" + + local -i haystack_index=0 # initialize only here to continue from last search position + local search_value haystack_value # just to appease shellcheck + for ((i = 1; i <= $#; ++i)); do + eval "local search_value=${!i}" + for (( ; haystack_index < haystack_length; ++haystack_index)); do + eval "local haystack_value=\${${haystack_array}[$haystack_index]}" + if [[ $haystack_value > "$search_value" ]]; then + continue 2 # search value not in array! -> try next + elif [[ $haystack_value == "$search_value" ]]; then + return 0 # search value found + fi + done + done + + # did not return from loop above -> no search value was found + return 1 +} + +bats_trim() { # + local -r bats_trim_ltrimmed=${2#"${2%%[![:space:]]*}"} # cut off leading whitespace + # shellcheck disable=SC2034 # used in eval! + local -r bats_trim_trimmed=${bats_trim_ltrimmed%"${bats_trim_ltrimmed##*[![:space:]]}"} # cut off trailing whitespace + eval "$1=\$bats_trim_trimmed" +} + +# a helper function to work around unbound variable errors with ${arr[@]} on Bash 3 +bats_append_arrays_as_args() { # -- + local -a trailing_args=() + while (($# > 0)) && [[ $1 != -- ]]; do + local array=$1 + shift + + if eval "(( \${#${array}[@]} > 0 ))"; then + eval "trailing_args+=(\"\${${array}[@]}\")" + fi + done + shift # remove -- separator + + if (($# == 0)); then + printf "Error: append_arrays_as_args is missing a command or -- separator\n" >&2 + return 1 + fi + + if ((${#trailing_args[@]} > 0)); then + "$@" "${trailing_args[@]}" + else + "$@" + fi +} diff --git a/lib/bats-core/formatter.bash b/lib/bats-core/formatter.bash index 1a3b982391..b774e16730 100644 --- a/lib/bats-core/formatter.bash +++ b/lib/bats-core/formatter.bash @@ -1,116 +1,143 @@ #!/usr/bin/env bash # reads (extended) bats tap streams from stdin and calls callback functions for each line +# +# Segmenting functions +# ==================== # bats_tap_stream_plan -> when the test plan is encountered +# bats_tap_stream_suite -> when a new file is begun WARNING: extended only # bats_tap_stream_begin -> when a new test is begun WARNING: extended only -# bats_tap_stream_ok [--duration -> when a test was successful -# bats_tap_stream_not_ok [--duration ] -> when a test has failed +# +# Test result functions +# ===================== +# If timing was enabled, BATS_FORMATTER_TEST_DURATION will be set to their duration in milliseconds +# bats_tap_stream_ok -> when a test was successful +# bats_tap_stream_not_ok -> when a test has failed. If the failure was due to a timeout, +# BATS_FORMATTER_TEST_TIMEOUT is set to the timeout duration in seconds # bats_tap_stream_skipped -> when a test was skipped -# bats_tap_stream_comment -> when a comment line was encountered, +# +# Context functions +# ================= +# bats_tap_stream_comment -> when a comment line was encountered, # scope tells the last encountered of plan, begin, ok, not_ok, skipped, suite -# bats_tap_stream_suite -> when a new file is begun WARNING: extended only # bats_tap_stream_unknown -> when a line is encountered that does not match the previous entries, # scope @see bats_tap_stream_comment # forwards all input as is, when there is no TAP test plan header function bats_parse_internal_extended_tap() { - local header_pattern='[0-9]+\.\.[0-9]+' - IFS= read -r header + local header_pattern='[0-9]+\.\.[0-9]+' + IFS= read -r header - if [[ "$header" =~ $header_pattern ]]; then - bats_tap_stream_plan "${header:3}" - else - # If the first line isn't a TAP plan, print it and pass the rest through - printf '%s\n' "$header" - exec cat - fi + if [[ "$header" =~ $header_pattern ]]; then + bats_tap_stream_plan "${header:3}" + else + # If the first line isn't a TAP plan, print it and pass the rest through + printf '%s\n' "$header" + exec cat + fi - ok_line_regexpr="ok ([0-9]+) (.*)" - skip_line_regexpr="ok ([0-9]+) (.*) # skip( (.*))?$" - not_ok_line_regexpr="not ok ([0-9]+) (.*)" + ok_line_regexpr="ok ([0-9]+) (.*)" + skip_line_regexpr="ok ([0-9]+) (.*) # skip( (.*))?$" + timeout_line_regexpr="not ok ([0-9]+) (.*) # timeout after ([0-9]+)s$" + not_ok_line_regexpr="not ok ([0-9]+) (.*)" - timing_expr="in ([0-9]+)ms$" - local test_name begin_index ok_index not_ok_index index scope - begin_index=0 - index=0 - scope=plan - while IFS= read -r line; do - case "$line" in - 'begin '*) # this might only be called in extended tap output - ((++begin_index)) - scope=begin - test_name="${line#* "$begin_index" }" - bats_tap_stream_begin "$begin_index" "$test_name" - ;; - 'ok '*) - ((++index)) - if [[ "$line" =~ $ok_line_regexpr ]]; then - ok_index="${BASH_REMATCH[1]}" - test_name="${BASH_REMATCH[2]}" - if [[ "$line" =~ $skip_line_regexpr ]]; then - scope=skipped - test_name="${BASH_REMATCH[2]}" # cut off name before "# skip" - local skip_reason="${BASH_REMATCH[4]}" - bats_tap_stream_skipped "$ok_index" "$test_name" "$skip_reason" - else - scope=ok - if [[ "$line" =~ $timing_expr ]]; then - bats_tap_stream_ok --duration "${BASH_REMATCH[1]}" "$ok_index" "$test_name" - else - bats_tap_stream_ok "$ok_index" "$test_name" - fi - fi - else - printf "ERROR: could not match ok line: %s" "$line" >&2 - exit 1 - fi - ;; - 'not ok '*) - ((++index)) - scope=not_ok - if [[ "$line" =~ $not_ok_line_regexpr ]]; then - not_ok_index="${BASH_REMATCH[1]}" - test_name="${BASH_REMATCH[2]}" - if [[ "$line" =~ $timing_expr ]]; then - bats_tap_stream_not_ok --duration "${BASH_REMATCH[1]}" "$not_ok_index" "$test_name" - else - bats_tap_stream_not_ok "$not_ok_index" "$test_name" - fi - else - printf "ERROR: could not match not ok line: %s" "$line" >&2 - exit 1 - fi - ;; - '# '*) - bats_tap_stream_comment "${line:2}" "$scope" - ;; - '#') - bats_tap_stream_comment "" "$scope" - ;; - 'suite '*) - scope=suite - # pass on the - bats_tap_stream_suite "${line:6}" - ;; - *) - bats_tap_stream_unknown "$line" "$scope" - ;; - esac - done + timing_expr="in ([0-9]+)ms$" + local test_name begin_index ok_index not_ok_index index scope + begin_index=0 + index=0 + scope=plan + while IFS= read -r line; do + unset BATS_FORMATTER_TEST_DURATION BATS_FORMATTER_TEST_TIMEOUT + case "$line" in + 'begin '*) # this might only be called in extended tap output + ((++begin_index)) + scope=begin + test_name="${line#* "$begin_index" }" + bats_tap_stream_begin "$begin_index" "$test_name" + ;; + 'ok '*) + ((++index)) + if [[ "$line" =~ $ok_line_regexpr ]]; then + ok_index="${BASH_REMATCH[1]}" + test_name="${BASH_REMATCH[2]}" + if [[ "$line" =~ $skip_line_regexpr ]]; then + scope=skipped + test_name="${BASH_REMATCH[2]}" # cut off name before "# skip" + local skip_reason="${BASH_REMATCH[4]}" + if [[ "$test_name" =~ $timing_expr ]]; then + local BATS_FORMATTER_TEST_DURATION="${BASH_REMATCH[1]}" + test_name="${test_name% in "${BATS_FORMATTER_TEST_DURATION}"ms}" + bats_tap_stream_skipped "$ok_index" "$test_name" "$skip_reason" + else + bats_tap_stream_skipped "$ok_index" "$test_name" "$skip_reason" + fi + else + scope=ok + if [[ "$line" =~ $timing_expr ]]; then + local BATS_FORMATTER_TEST_DURATION="${BASH_REMATCH[1]}" + bats_tap_stream_ok "$ok_index" "${test_name% in "${BASH_REMATCH[1]}"ms}" + else + bats_tap_stream_ok "$ok_index" "$test_name" + fi + fi + else + printf "ERROR: could not match ok line: %s" "$line" >&2 + exit 1 + fi + ;; + 'not ok '*) + ((++index)) + scope=not_ok + if [[ "$line" =~ $not_ok_line_regexpr ]]; then + not_ok_index="${BASH_REMATCH[1]}" + test_name="${BASH_REMATCH[2]}" + if [[ "$line" =~ $timeout_line_regexpr ]]; then + not_ok_index="${BASH_REMATCH[1]}" + test_name="${BASH_REMATCH[2]}" + # shellcheck disable=SC2034 # used in bats_tap_stream_ok + local BATS_FORMATTER_TEST_TIMEOUT="${BASH_REMATCH[3]}" + fi + if [[ "$test_name" =~ $timing_expr ]]; then + # shellcheck disable=SC2034 # used in bats_tap_stream_ok + local BATS_FORMATTER_TEST_DURATION="${BASH_REMATCH[1]}" + test_name="${test_name% in "${BASH_REMATCH[1]}"ms}" + fi + bats_tap_stream_not_ok "$not_ok_index" "$test_name" + else + printf "ERROR: could not match not ok line: %s" "$line" >&2 + exit 1 + fi + ;; + '# '*) + bats_tap_stream_comment "${line:2}" "$scope" + ;; + '#') + bats_tap_stream_comment "" "$scope" + ;; + 'suite '*) + scope=suite + # pass on the + bats_tap_stream_suite "${line:6}" + ;; + *) + bats_tap_stream_unknown "$line" "$scope" + ;; + esac + done } normalize_base_path() { # - # the relative path root to use for reporting filenames - # this is mainly intended for suite mode, where this will be the suite root folder - local base_path="$2" - # use the containing directory when --base-path is a file - if [[ ! -d "$base_path" ]]; then - base_path="$(dirname "$base_path")" - fi - # get the absolute path - base_path="$(cd "$base_path" && pwd)" - # ensure the path ends with / to strip that later on - if [[ "${base_path}" != *"/" ]]; then - base_path="$base_path/" - fi - printf -v "$1" "%s" "$base_path" + # the relative path root to use for reporting filenames + # this is mainly intended for suite mode, where this will be the suite root folder + local base_path="$2" + # use the containing directory when --base-path is a file + if [[ ! -d "$base_path" ]]; then + base_path="$(dirname "$base_path")" + fi + # get the absolute path + base_path="$(cd "$base_path" && pwd)" + # ensure the path ends with / to strip that later on + if [[ "${base_path}" != *"/" ]]; then + base_path="$base_path/" + fi + printf -v "$1" "%s" "$base_path" } diff --git a/lib/bats-core/preprocessing.bash b/lib/bats-core/preprocessing.bash index 6804770a7d..5d9a7652c6 100644 --- a/lib/bats-core/preprocessing.bash +++ b/lib/bats-core/preprocessing.bash @@ -6,17 +6,17 @@ BATS_PARENT_TMPNAME="$BATS_RUN_TMPDIR/bats.$PPID" BATS_OUT="${BATS_TMPNAME}.out" # used in bats-exec-file bats_preprocess_source() { - # export to make it visible to bats_evaluate_preprocessed_source - # since the latter runs in bats-exec-test's bash while this runs in bats-exec-file's - export BATS_TEST_SOURCE="${BATS_TMPNAME}.src" - bats-preprocess "$BATS_TEST_FILENAME" >"$BATS_TEST_SOURCE" + # export to make it visible to bats_evaluate_preprocessed_source + # since the latter runs in bats-exec-test's bash while this runs in bats-exec-file's + export BATS_TEST_SOURCE="${BATS_TMPNAME}.src" + CHECK_BATS_COMMENT_COMMANDS=1 bats-preprocess "$BATS_TEST_FILENAME" >"$BATS_TEST_SOURCE" } bats_evaluate_preprocessed_source() { - if [[ -z "${BATS_TEST_SOURCE:-}" ]]; then - BATS_TEST_SOURCE="${BATS_PARENT_TMPNAME}.src" - fi - # Dynamically loaded user files provided outside of Bats. - # shellcheck disable=SC1090 - source "$BATS_TEST_SOURCE" + if [[ -z "${BATS_TEST_SOURCE:-}" ]]; then + BATS_TEST_SOURCE="${BATS_PARENT_TMPNAME}.src" + fi + # Dynamically loaded user files provided outside of Bats. + # shellcheck disable=SC1090 + source "$BATS_TEST_SOURCE" } diff --git a/lib/bats-core/semaphore.bash b/lib/bats-core/semaphore.bash index acf50f187c..e299293c76 100644 --- a/lib/bats-core/semaphore.bash +++ b/lib/bats-core/semaphore.bash @@ -2,31 +2,31 @@ # setup the semaphore environment for the loading file bats_semaphore_setup() { - export -f bats_semaphore_get_free_slot_count - export -f bats_semaphore_acquire_while_locked - export BATS_SEMAPHORE_DIR="$BATS_RUN_TMPDIR/semaphores" + export -f bats_semaphore_get_free_slot_count + export -f bats_semaphore_acquire_while_locked + export BATS_SEMAPHORE_DIR="$BATS_RUN_TMPDIR/semaphores" - if command -v flock >/dev/null; then + if command -v flock >/dev/null; then bats_run_under_lock() { - flock "$BATS_SEMAPHORE_DIR" "$@" + flock "$BATS_SEMAPHORE_DIR" "$@" } - elif command -v shlock >/dev/null; then - bats_run_under_lock() { - local lockfile="$BATS_SEMAPHORE_DIR/shlock.lock" - while ! shlock -p $$ -f "$lockfile"; do - sleep 1 - done - # we got the lock now, execute the command - "$@" - local status=$? - # free the lock - rm -f "$lockfile" - return $status - } - else - printf "ERROR: flock/shlock is required for parallelization within files!\n" >&2 - exit 1 - fi + elif command -v shlock >/dev/null; then + bats_run_under_lock() { + local lockfile="$BATS_SEMAPHORE_DIR/shlock.lock" + while ! shlock -p $$ -f "$lockfile"; do + sleep 1 + done + # we got the lock now, execute the command + "$@" + local status=$? + # free the lock + rm -f "$lockfile" + return $status + } + else + printf "ERROR: flock/shlock is required for parallelization within files!\n" >&2 + exit 1 + fi } # $1 - output directory for stdout/stderr @@ -36,72 +36,72 @@ bats_semaphore_setup() { # when there is a free slot, run the command in background # gather the output of the command in files in the given directory bats_semaphore_run() { - local output_dir=$1 - shift - local semaphore_slot - semaphore_slot=$(bats_semaphore_acquire_slot) - bats_semaphore_release_wrapper "$output_dir" "$semaphore_slot" "$@" & - printf "%d\n" "$!" + local output_dir=$1 + shift + local semaphore_slot + semaphore_slot=$(bats_semaphore_acquire_slot) + bats_semaphore_release_wrapper "$output_dir" "$semaphore_slot" "$@" & + printf "%d\n" "$!" } # $1 - output directory for stdout/stderr # $@ - command to run # this wraps the actual function call to install some traps on exiting bats_semaphore_release_wrapper() { - local output_dir="$1" - local semaphore_name="$2" - shift 2 # all other parameters will be use for the command to execute + local output_dir="$1" + local semaphore_name="$2" + shift 2 # all other parameters will be use for the command to execute - # shellcheck disable=SC2064 # we want to expand the semaphore_name right now! - trap "status=$?; bats_semaphore_release_slot '$semaphore_name'; exit $status" EXIT + # shellcheck disable=SC2064 # we want to expand the semaphore_name right now! + trap "status=$?; bats_semaphore_release_slot '$semaphore_name'; exit $status" EXIT - mkdir -p "$output_dir" - "$@" 2>"$output_dir/stderr" >"$output_dir/stdout" - local status=$? + mkdir -p "$output_dir" + "$@" 2>"$output_dir/stderr" >"$output_dir/stdout" + local status=$? - # bash bug: the exit trap is not called for the background process - bats_semaphore_release_slot "$semaphore_name" - trap - EXIT # avoid calling release twice - return $status + # bash bug: the exit trap is not called for the background process + bats_semaphore_release_slot "$semaphore_name" + trap - EXIT # avoid calling release twice + return $status } bats_semaphore_acquire_while_locked() { - if [[ $(bats_semaphore_get_free_slot_count) -gt 0 ]]; then - local slot=0 - while [[ -e "$BATS_SEMAPHORE_DIR/slot-$slot" ]]; do - (( ++slot )) - done - if [[ $slot -lt $BATS_SEMAPHORE_NUMBER_OF_SLOTS ]]; then - touch "$BATS_SEMAPHORE_DIR/slot-$slot" && printf "%d\n" "$slot" && return 0 - fi + if [[ $(bats_semaphore_get_free_slot_count) -gt 0 ]]; then + local slot=0 + while [[ -e "$BATS_SEMAPHORE_DIR/slot-$slot" ]]; do + ((++slot)) + done + if [[ $slot -lt $BATS_SEMAPHORE_NUMBER_OF_SLOTS ]]; then + touch "$BATS_SEMAPHORE_DIR/slot-$slot" && printf "%d\n" "$slot" && return 0 fi - return 1 + fi + return 1 } # block until a semaphore slot becomes free # prints the number of the slot that it received bats_semaphore_acquire_slot() { - mkdir -p "$BATS_SEMAPHORE_DIR" - # wait for a slot to become free - # TODO: avoid busy waiting by using signals -> this opens op prioritizing possibilities as well - while true; do - # don't lock for reading, we are fine with spuriously getting no free slot - if [[ $(bats_semaphore_get_free_slot_count) -gt 0 ]]; then - bats_run_under_lock bash -c bats_semaphore_acquire_while_locked && break - fi - sleep 1 - done + mkdir -p "$BATS_SEMAPHORE_DIR" + # wait for a slot to become free + # TODO: avoid busy waiting by using signals -> this opens op prioritizing possibilities as well + while true; do + # don't lock for reading, we are fine with spuriously getting no free slot + if [[ $(bats_semaphore_get_free_slot_count) -gt 0 ]]; then + bats_run_under_lock bash -c bats_semaphore_acquire_while_locked && break + fi + sleep 1 + done } bats_semaphore_release_slot() { - # we don't need to lock this, since only our process owns this file - # and freeing a semaphore cannot lead to conflicts with others - rm "$BATS_SEMAPHORE_DIR/slot-$1" # this will fail if we had not aqcuired a semaphore! + # we don't need to lock this, since only our process owns this file + # and freeing a semaphore cannot lead to conflicts with others + rm "$BATS_SEMAPHORE_DIR/slot-$1" # this will fail if we had not acquired a semaphore! } bats_semaphore_get_free_slot_count() { - # find might error out without returning something useful when a file is deleted, - # while the directory is traversed -> only continue when there was no error - until used_slots=$(find "$BATS_SEMAPHORE_DIR" -name 'slot-*' 2>/dev/null | wc -l); do :; done - echo $(( BATS_SEMAPHORE_NUMBER_OF_SLOTS - used_slots )) + # find might error out without returning something useful when a file is deleted, + # while the directory is traversed -> only continue when there was no error + until used_slots=$(find "$BATS_SEMAPHORE_DIR" -name 'slot-*' 2>/dev/null | wc -l); do :; done + echo $((BATS_SEMAPHORE_NUMBER_OF_SLOTS - used_slots)) } diff --git a/lib/bats-core/test_functions.bash b/lib/bats-core/test_functions.bash index 5865af8168..72a337f720 100644 --- a/lib/bats-core/test_functions.bash +++ b/lib/bats-core/test_functions.bash @@ -20,17 +20,17 @@ find_in_bats_lib_path() { # local library_name="${2:?}" local -a bats_lib_paths - IFS=: read -ra bats_lib_paths <<< "$BATS_LIB_PATH" + IFS=: read -ra bats_lib_paths <<<"$BATS_LIB_PATH" for path in "${bats_lib_paths[@]}"; do if [[ -f "$path/$library_name" ]]; then - printf -v "$return_var" "%s" "$path/$library_name" + printf -v "$return_var" "%s" "$path/$library_name" # A library load path was found, return - return + return 0 elif [[ -f "$path/$library_name/load.bash" ]]; then - printf -v "$return_var" "%s" "$path/$library_name/load.bash" + printf -v "$return_var" "%s" "$path/$library_name/load.bash" # A library load path was found, return - return + return 0 fi done @@ -60,12 +60,12 @@ bats_internal_load() { # library_load_path is a library loader if [[ -f "$library_load_path" ]]; then - # shellcheck disable=SC1090 - if ! source "$library_load_path"; then - printf "Error while sourcing library loader at '%s'\n" "$library_load_path" >&2 - return 1 - fi - return + # shellcheck disable=SC1090 + if ! source "$library_load_path"; then + printf "Error while sourcing library loader at '%s'\n" "$library_load_path" >&2 + return 1 + fi + return 0 fi printf "Passed library load path is neither a library loader nor library directory: %s\n" "$library_load_path" >&2 @@ -99,17 +99,17 @@ bats_load_safe() { if [[ -f "$slug.bash" ]]; then bats_internal_load "$slug.bash" - return + return $? elif [[ -f "$slug" ]]; then bats_internal_load "$slug" - return + return $? fi # loading from PATH (retained for backwards compatibility) if [[ ! -f "$1" ]] && type -P "$1" >/dev/null; then # shellcheck disable=SC1090 source "$1" - return + return $? fi # No library load path can be found @@ -117,22 +117,12 @@ bats_load_safe() { return 1 } -bats_require_lib_path() { - if [[ -z "${BATS_LIB_PATH:-}" ]]; then - printf "%s: requires BATS_LIB_PATH to be set!\n" "${FUNCNAME[1]}" >&2 - exit 1 - fi -} - bats_load_library_safe() { # local slug="${1:?}" library_path - bats_require_lib_path - # Check for library load paths in BATS_TEST_DIRNAME and BATS_LIB_PATH if [[ ${slug:0:1} != / ]]; then - find_in_bats_lib_path library_path "$slug" - if [[ -z "$library_path" ]]; then + if ! find_in_bats_lib_path library_path "$slug"; then printf "Could not find library '%s' relative to test file or in BATS_LIB_PATH\n" "$slug" >&2 return 1 fi @@ -151,7 +141,6 @@ bats_load_library_safe() { # # immediately exit on error, use bats_load_library_safe to catch and handle errors bats_load_library() { # - bats_require_lib_path if ! bats_load_library_safe "$@"; then exit 1 fi @@ -159,9 +148,9 @@ bats_load_library() { # # load acts like bats_load_safe but exits the shell instead of returning 1. load() { - if ! bats_load_safe "$@"; then - exit 1 - fi + if ! bats_load_safe "$@"; then + exit 1 + fi } bats_redirect_stderr_into_file() { @@ -203,32 +192,32 @@ run() { # [!|-N] [--keep-empty-lines] [--separate-stderr] [--] &2 - return 1 - elif [[ $expected_rc -gt 255 ]]; then - printf "Usage error: run: '=NNN': NNN must be <= 255 (got: %d)\n" "$expected_rc" >&2 - return 1 - fi + -[0-9]*) + expected_rc=${1#-} + if [[ $expected_rc =~ [^0-9] ]]; then + printf "Usage error: run: '-NNN' requires numeric NNN (got: %s)\n" "$expected_rc" >&2 + return 1 + elif [[ $expected_rc -gt 255 ]]; then + printf "Usage error: run: '-NNN': NNN must be <= 255 (got: %d)\n" "$expected_rc" >&2 + return 1 + fi ;; - --keep-empty-lines) - keep_empty_lines=1 + --keep-empty-lines) + keep_empty_lines=1 ;; - --separate-stderr) - output_case="separate" + --separate-stderr) + output_case="separate" ;; - --) - shift # eat the -- before breaking away - break + --) + shift # eat the -- before breaking away + break ;; - *) - printf "Usage error: unknown flag '%s'" "$1" >&2 - return 1 + *) + printf "Usage error: unknown flag '%s'" "$1" >&2 + return 1 ;; esac shift @@ -241,49 +230,51 @@ run() { # [!|-N] [--keep-empty-lines] [--separate-stderr] [--] - local output_var="$1" path="$2" - if [[ "$output_var" != NORMALIZED_INPUT ]]; then - local NORMALIZED_INPUT - fi - if [[ $path == ?:* ]]; then - NORMALIZED_INPUT="$(cd "$path" || exit 1; pwd)" - else - NORMALIZED_INPUT="$path" - fi - printf -v "$output_var" "%s" "$NORMALIZED_INPUT" + local output_var="$1" path="$2" + if [[ "$output_var" != NORMALIZED_INPUT ]]; then + local NORMALIZED_INPUT + fi + if [[ $path == ?:* ]]; then + NORMALIZED_INPUT="$( + cd "$path" || exit 1 + pwd + )" + else + NORMALIZED_INPUT="$path" + fi + printf -v "$output_var" "%s" "$NORMALIZED_INPUT" } bats_emit_trace() { - if [[ $BATS_TRACE_LEVEL -gt 0 ]]; then - local line=${BASH_LINENO[1]} - # shellcheck disable=SC2016 - if [[ $BASH_COMMAND != '"$BATS_TEST_NAME" >> "$BATS_OUT" 2>&1 4>&1' && $BASH_COMMAND != "bats_test_begin "* ]] && # don't emit these internal calls - [[ $BASH_COMMAND != "$BATS_LAST_BASH_COMMAND" || $line != "$BATS_LAST_BASH_LINENO" ]] && - # avoid printing a function twice (at call site and at definiton site) - [[ $BASH_COMMAND != "$BATS_LAST_BASH_COMMAND" || ${BASH_LINENO[2]} != "$BATS_LAST_BASH_LINENO" || ${BASH_SOURCE[3]} != "$BATS_LAST_BASH_SOURCE" ]]; then - local file="${BASH_SOURCE[2]}" # index 2: skip over bats_emit_trace and bats_debug_trap - if [[ $file == "${BATS_TEST_SOURCE}" ]]; then - file="$BATS_TEST_FILENAME" - fi - local padding='$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$' - if (( BATS_LAST_STACK_DEPTH != ${#BASH_LINENO[@]} )); then - printf '%s [%s:%d]\n' "${padding::${#BASH_LINENO[@]}-4}" "${file##*/}" "$line" >&4 - fi - printf '%s %s\n' "${padding::${#BASH_LINENO[@]}-4}" "$BASH_COMMAND" >&4 - BATS_LAST_BASH_COMMAND="$BASH_COMMAND" - BATS_LAST_BASH_LINENO="$line" - BATS_LAST_BASH_SOURCE="${BASH_SOURCE[2]}" - BATS_LAST_STACK_DEPTH="${#BASH_LINENO[@]}" - fi - fi + if [[ $BATS_TRACE_LEVEL -gt 0 ]]; then + local line=${BASH_LINENO[1]} + # shellcheck disable=SC2016 + if [[ $BASH_COMMAND != '"$BATS_TEST_NAME" >> "$BATS_OUT" 2>&1 4>&1' && $BASH_COMMAND != "bats_test_begin "* ]] && # don't emit these internal calls + [[ $BASH_COMMAND != "$BATS_LAST_BASH_COMMAND" || $line != "$BATS_LAST_BASH_LINENO" ]] && + # avoid printing a function twice (at call site and at definition site) + [[ $BASH_COMMAND != "$BATS_LAST_BASH_COMMAND" || ${BASH_LINENO[2]} != "$BATS_LAST_BASH_LINENO" || ${BASH_SOURCE[3]} != "$BATS_LAST_BASH_SOURCE" ]]; then + local file="${BASH_SOURCE[2]}" # index 2: skip over bats_emit_trace and bats_debug_trap + if [[ $file == "${BATS_TEST_SOURCE}" ]]; then + file="$BATS_TEST_FILENAME" + fi + local padding='$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$' + if ((BATS_LAST_STACK_DEPTH != ${#BASH_LINENO[@]})); then + printf '%s [%s:%d]\n' "${padding::${#BASH_LINENO[@]}-4}" "${file##*/}" "$line" >&4 + fi + printf '%s %s\n' "${padding::${#BASH_LINENO[@]}-4}" "$BASH_COMMAND" >&4 + BATS_LAST_BASH_COMMAND="$BASH_COMMAND" + BATS_LAST_BASH_LINENO="$line" + BATS_LAST_BASH_SOURCE="${BASH_SOURCE[2]}" + BATS_LAST_STACK_DEPTH="${#BASH_LINENO[@]}" + fi + fi } # bats_debug_trap tracks the last line of code executed within a test. This is @@ -236,31 +245,33 @@ bats_emit_trace() { # [4] The reported source location is that of the call to the function calling # the command. bats_debug_trap() { - # on windows we sometimes get a mix of paths (when install via nmp install -g) - # which have C:/... or /c/... comparing them is going to be problematic. - # We need to normalize them to a common format! - local NORMALIZED_INPUT - bats_normalize_windows_dir_path NORMALIZED_INPUT "${1%/*}" - local file_excluded='' path - for path in "${BATS_DEBUG_EXCLUDE_PATHS[@]}"; do - if [[ "$NORMALIZED_INPUT" == "$path"* ]]; then - file_excluded=1 - break - fi - done - - # don't update the trace within library functions or we get backtraces from inside traps - # also don't record new stack traces while handling interruptions, to avoid overriding the interrupted command - if [[ -z "$file_excluded" && "${BATS_INTERRUPTED-NOTSET}" == NOTSET ]]; then - BATS_DEBUG_LASTLAST_STACK_TRACE=( - ${BATS_DEBUG_LAST_STACK_TRACE[@]+"${BATS_DEBUG_LAST_STACK_TRACE[@]}"} - ) - - BATS_DEBUG_LAST_LINENO=(${BASH_LINENO[@]+"${BASH_LINENO[@]}"}) - BATS_DEBUG_LAST_SOURCE=(${BASH_SOURCE[@]+"${BASH_SOURCE[@]}"}) - bats_capture_stack_trace - bats_emit_trace - fi + # on windows we sometimes get a mix of paths (when install via nmp install -g) + # which have C:/... or /c/... comparing them is going to be problematic. + # We need to normalize them to a common format! + local NORMALIZED_INPUT + bats_normalize_windows_dir_path NORMALIZED_INPUT "${1%/*}" + local file_excluded='' path + for path in "${BATS_DEBUG_EXCLUDE_PATHS[@]}"; do + if [[ "$NORMALIZED_INPUT" == "$path"* ]]; then + file_excluded=1 + break + fi + done + + # don't update the trace within library functions or we get backtraces from inside traps + # also don't record new stack traces while handling interruptions, to avoid overriding the interrupted command + if [[ -z "$file_excluded" && + "${BATS_INTERRUPTED-NOTSET}" == NOTSET && + "${BATS_TIMED_OUT-NOTSET}" == NOTSET ]]; then + BATS_DEBUG_LASTLAST_STACK_TRACE=( + ${BATS_DEBUG_LAST_STACK_TRACE[@]+"${BATS_DEBUG_LAST_STACK_TRACE[@]}"} + ) + + BATS_DEBUG_LAST_LINENO=(${BASH_LINENO[@]+"${BASH_LINENO[@]}"}) + BATS_DEBUG_LAST_SOURCE=(${BASH_SOURCE[@]+"${BASH_SOURCE[@]}"}) + bats_capture_stack_trace + bats_emit_trace + fi } # For some versions of Bash, the `ERR` trap may not always fire for every @@ -274,62 +285,81 @@ bats_debug_trap() { # See `bats_exit_trap` for an additional EXIT error handling case when `$?` # isn't set properly during `teardown()` errors. bats_check_status_from_trap() { - local status="$?" - if [[ -z "$BATS_TEST_COMPLETED" ]]; then - BATS_ERROR_STATUS="${BATS_ERROR_STATUS:-$status}" - if [[ "$BATS_ERROR_STATUS" -eq 0 ]]; then - BATS_ERROR_STATUS=1 - fi - trap - DEBUG - fi + local status="$?" + if [[ -z "${BATS_TEST_COMPLETED:-}" ]]; then + BATS_ERROR_STATUS="${BATS_ERROR_STATUS:-$status}" + if [[ "$BATS_ERROR_STATUS" -eq 0 ]]; then + BATS_ERROR_STATUS=1 + fi + trap - DEBUG + fi } bats_add_debug_exclude_path() { # - if [[ -z "$1" ]]; then # don't exclude everything - printf "bats_add_debug_exclude_path: Exclude path must not be empty!\n" >&2 - return 1 - fi - if [[ "$OSTYPE" == cygwin || "$OSTYPE" == msys ]]; then - local normalized_dir - bats_normalize_windows_dir_path normalized_dir "$1" - BATS_DEBUG_EXCLUDE_PATHS+=("$normalized_dir") - else - BATS_DEBUG_EXCLUDE_PATHS+=("$1") - fi + if [[ -z "$1" ]]; then # don't exclude everything + printf "bats_add_debug_exclude_path: Exclude path must not be empty!\n" >&2 + return 1 + fi + if [[ "$OSTYPE" == cygwin || "$OSTYPE" == msys ]]; then + local normalized_dir + bats_normalize_windows_dir_path normalized_dir "$1" + BATS_DEBUG_EXCLUDE_PATHS+=("$normalized_dir") + else + BATS_DEBUG_EXCLUDE_PATHS+=("$1") + fi } bats_setup_tracing() { - BATS_DEBUG_EXCLUDE_PATHS=() - # exclude some paths by default - bats_add_debug_exclude_path "$BATS_ROOT/lib/" - bats_add_debug_exclude_path "$BATS_ROOT/libexec/" - - - exec 4<&1 # used for tracing - if [[ "${BATS_TRACE_LEVEL:-0}" -gt 0 ]]; then - # avoid undefined variable errors - BATS_LAST_BASH_COMMAND= - BATS_LAST_BASH_LINENO= - BATS_LAST_BASH_SOURCE= - BATS_LAST_STACK_DEPTH= - # try to exclude helper libraries if found, this is only relevant for tracing - while read -r path; do - bats_add_debug_exclude_path "$path" - done < <(find "$PWD" -type d -name bats-assert -o -name bats-support) - fi - - local exclude_paths path - # exclude user defined libraries - IFS=':' read -r exclude_paths <<< "${BATS_DEBUG_EXCLUDE_PATHS:-}" - for path in "${exclude_paths[@]}"; do - if [[ -n "$path" ]]; then - bats_add_debug_exclude_path "$path" - fi - done - - # turn on traps after setting excludedes to avoid tracing the exclude setup - trap 'bats_debug_trap "$BASH_SOURCE"' DEBUG - trap 'bats_error_trap' ERR + # Variables for capturing accurate stack traces. See bats_debug_trap for + # details. + # + # BATS_DEBUG_LAST_LINENO, BATS_DEBUG_LAST_SOURCE, and + # BATS_DEBUG_LAST_STACK_TRACE hold data from the most recent call to + # bats_debug_trap. + # + # BATS_DEBUG_LASTLAST_STACK_TRACE holds data from two bats_debug_trap calls + # ago. + # + # BATS_DEBUG_LAST_STACK_TRACE_IS_VALID indicates that + # BATS_DEBUG_LAST_STACK_TRACE contains the stack trace of the test's error. If + # unset, BATS_DEBUG_LAST_STACK_TRACE is unreliable and + # BATS_DEBUG_LASTLAST_STACK_TRACE should be used instead. + BATS_DEBUG_LASTLAST_STACK_TRACE=() + BATS_DEBUG_LAST_LINENO=() + BATS_DEBUG_LAST_SOURCE=() + BATS_DEBUG_LAST_STACK_TRACE=() + BATS_DEBUG_LAST_STACK_TRACE_IS_VALID= + BATS_ERROR_SUFFIX= + BATS_DEBUG_EXCLUDE_PATHS=() + # exclude some paths by default + bats_add_debug_exclude_path "$BATS_ROOT/lib/" + bats_add_debug_exclude_path "$BATS_ROOT/libexec/" + + exec 4<&1 # used for tracing + if [[ "${BATS_TRACE_LEVEL:-0}" -gt 0 ]]; then + # avoid undefined variable errors + BATS_LAST_BASH_COMMAND= + BATS_LAST_BASH_LINENO= + BATS_LAST_BASH_SOURCE= + BATS_LAST_STACK_DEPTH= + # try to exclude helper libraries if found, this is only relevant for tracing + while read -r path; do + bats_add_debug_exclude_path "$path" + done < <(find "$PWD" -type d -name bats-assert -o -name bats-support) + fi + + local exclude_paths path + # exclude user defined libraries + IFS=':' read -r exclude_paths <<<"${BATS_DEBUG_EXCLUDE_PATHS:-}" + for path in "${exclude_paths[@]}"; do + if [[ -n "$path" ]]; then + bats_add_debug_exclude_path "$path" + fi + done + + # turn on traps after setting excludes to avoid tracing the exclude setup + trap 'bats_debug_trap "$BASH_SOURCE"' DEBUG + trap 'bats_error_trap' ERR } bats_error_trap() { @@ -337,9 +367,9 @@ bats_error_trap() { # If necessary, undo the most recent stack trace captured by bats_debug_trap. # See bats_debug_trap for details. - if [[ "${BASH_LINENO[*]}" = "${BATS_DEBUG_LAST_LINENO[*]:-}" - && "${BASH_SOURCE[*]}" = "${BATS_DEBUG_LAST_SOURCE[*]:-}" - && -z "$BATS_DEBUG_LAST_STACK_TRACE_IS_VALID" ]]; then + if [[ "${BASH_LINENO[*]}" = "${BATS_DEBUG_LAST_LINENO[*]:-}" && + "${BASH_SOURCE[*]}" = "${BATS_DEBUG_LAST_SOURCE[*]:-}" && + -z "$BATS_DEBUG_LAST_STACK_TRACE_IS_VALID" ]]; then BATS_DEBUG_LAST_STACK_TRACE=( ${BATS_DEBUG_LASTLAST_STACK_TRACE[@]+"${BATS_DEBUG_LASTLAST_STACK_TRACE[@]}"} ) @@ -352,8 +382,7 @@ bats_interrupt_trap() { BATS_INTERRUPTED=true BATS_ERROR_STATUS=130 # debug trap fires before interrupt trap but gets wrong linenumber (line 1) - # -> use last last stack trace - exit $BATS_ERROR_STATUS + # -> use last stack trace instead of BATS_DEBUG_LAST_STACK_TRACE_IS_VALID=true } # this is used inside run() @@ -362,5 +391,5 @@ bats_interrupt_trap_in_run() { BATS_INTERRUPTED=true BATS_ERROR_STATUS=130 BATS_DEBUG_LAST_STACK_TRACE_IS_VALID=true - exit $BATS_ERROR_STATUS + exit 130 } diff --git a/lib/bats-core/validator.bash b/lib/bats-core/validator.bash index f41109a07f..59fc2c1e69 100644 --- a/lib/bats-core/validator.bash +++ b/lib/bats-core/validator.bash @@ -14,24 +14,24 @@ bats_test_count_validator() { # ... count the actual number of [not ] oks... local actual_number_of_tests=0 while IFS= read -r line; do - # forward line - printf "%s\n" "$line" - case "$line" in - 'ok '*) - (( ++actual_number_of_tests )) + # forward line + printf "%s\n" "$line" + case "$line" in + 'ok '*) + ((++actual_number_of_tests)) ;; - 'not ok'*) - (( ++actual_number_of_tests )) + 'not ok'*) + ((++actual_number_of_tests)) ;; - esac + esac done # ... and error if they are not the same if [[ "${actual_number_of_tests}" != "${expected_number_of_tests}" ]]; then - printf '# bats warning: Executed %s instead of expected %s tests\n' "$actual_number_of_tests" "$expected_number_of_tests" - return 1 + printf '# bats warning: Executed %s instead of expected %s tests\n' "$actual_number_of_tests" "$expected_number_of_tests" + return 1 fi else # forward output unchanged cat fi -} \ No newline at end of file +} diff --git a/lib/bats-core/warnings.bash b/lib/bats-core/warnings.bash index 9d25105455..fbb5186a46 100644 --- a/lib/bats-core/warnings.bash +++ b/lib/bats-core/warnings.bash @@ -1,24 +1,25 @@ +#!/usr/bin/env bash + # shellcheck source=lib/bats-core/tracing.bash source "$BATS_ROOT/lib/bats-core/tracing.bash" -BATS_WARNING_SHORT_DESCS=( - # to start with 1 - 'PADDING' - # see issue #578 for context - "\`run\`'s command \`%s\` exited with code 127, indicating 'Command not found'. Use run's return code checks, e.g. \`run -127\`, to fix this message." - "%s requires at least BATS_VERSION=%s. Use \`bats_require_minimum_version %s\` to fix this message." -) - # generate a warning report for the parent call's call site -bats_generate_warning() { # [...] - local warning_number="$1" padding="00" +bats_generate_warning() { # [--no-stacktrace] [...] + local warning_number="${1-}" padding="00" shift - if [[ $warning_number =~ [0-9]+ ]] && ((warning_number < ${#BATS_WARNING_SHORT_DESCS[@]} )); then + local no_stacktrace= + if [[ ${1-} == --no-stacktrace ]]; then + no_stacktrace=1 + shift + fi + if [[ $warning_number =~ [0-9]+ ]] && ((warning_number < ${#BATS_WARNING_SHORT_DESCS[@]})); then { - printf "BW%s: ${BATS_WARNING_SHORT_DESCS[$warning_number]}\n" "${padding:${#warning_number}}${warning_number}" "$@" + printf "BW%s: ${BATS_WARNING_SHORT_DESCS[$warning_number]}\n" "${padding:${#warning_number}}${warning_number}" "$@" + if [[ -z "$no_stacktrace" ]]; then bats_capture_stack_trace BATS_STACK_TRACE_PREFIX=' ' bats_print_stack_trace "${BATS_DEBUG_LAST_STACK_TRACE[@]}" - } >> "$BATS_WARNING_FILE" 2>&3 + fi + } >>"$BATS_WARNING_FILE" 2>&3 else printf "Invalid Bats warning number '%s'. It must be an integer between 1 and %d." "$warning_number" "$((${#BATS_WARNING_SHORT_DESCS[@]} - 1))" >&2 exit 1 @@ -30,4 +31,14 @@ bats_warn_minimum_guaranteed_version() { # if bats_version_lt "$BATS_GUARANTEED_MINIMUM_VERSION" "$2"; then bats_generate_warning 2 "$1" "$2" "$2" fi -} \ No newline at end of file +} + +# put after functions to avoid line changes in tests when new ones get added +BATS_WARNING_SHORT_DESCS=( + # to start with 1 + 'PADDING' + # see issue #578 for context + "\`run\`'s command \`%s\` exited with code 127, indicating 'Command not found'. Use run's return code checks, e.g. \`run -127\`, to fix this message." + "%s requires at least BATS_VERSION=%s. Use \`bats_require_minimum_version %s\` to fix this message." + "\`setup_suite\` is visible to test file '%s', but was not executed. It belongs into 'setup_suite.bash' to be picked up automatically." +) diff --git a/libexec/bats-core/bats b/libexec/bats-core/bats index 0400fc888f..05b70e6118 100755 --- a/libexec/bats-core/bats +++ b/libexec/bats-core/bats @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -e -export BATS_VERSION='1.7.0' +export BATS_VERSION='1.8.2' VALID_FORMATTERS="pretty, junit, tap, tap13" version() { @@ -9,8 +9,15 @@ version() { } abort() { + local print_usage=1 + if [[ ${1:-} == --no-print-usage ]]; then + print_usage= + shift + fi printf 'Error: %s\n' "$1" >&2 - usage >&2 + if [[ -n $print_usage ]]; then + usage >&2 + fi exit 1 } @@ -34,11 +41,15 @@ HELP_TEXT_HEADER or 'custom' which requires setting $BATS_BEGIN_CODE_QUOTE and $BATS_END_CODE_QUOTE. Can also be set via $BATS_CODE_QUOTE_STYLE -f, --filter Only run tests that match the regular expression + --filter-status Only run tests with the given status in the last completed (no CTRL+C/SIGINT) run. + Valid values are: + failed - runs tests that failed or were not present in the last run + missed - runs tests that were not present in the last run -F, --formatter Switch between formatters: pretty (default), - tap (default w/o term), tap13, junit + tap (default w/o term), tap13, junit, / --gather-test-outputs-in - Gather the output of failing *and* passing tests - as files in directory + Gather the output of failing *and* passing tests + as files in directory (if existing, must be empty) -h, --help Display this help message -j, --jobs Number of parallel jobs (requires GNU parallel) --no-tempdir-cleanup Preserve test output temporary directory @@ -49,7 +60,7 @@ HELP_TEXT_HEADER Serialize test execution within files instead of running them in parallel (requires --jobs >1) --report-formatter Switch between reporters (same options as --formatter) - -o, --output Directory to write report files + -o, --output Directory to write report files (must exist) -p, --pretty Shorthand for "--formatter pretty" --print-output-on-failure Automatically print the value of `$output` on failed tests -r, --recursive Include tests in subdirectories @@ -80,20 +91,26 @@ expand_path() { printf -v "$result" '%s/%s' "$dirname" "${path##*/}" } -BATS_LIBEXEC="$(cd "$(dirname "$(bats_readlinkf "${BASH_SOURCE[0]}")")"; pwd)" +BATS_LIBEXEC="$( + cd "$(dirname "$(bats_readlinkf "${BASH_SOURCE[0]}")")" + pwd +)" export BATS_LIBEXEC export BATS_CWD="$PWD" export BATS_TEST_FILTER= export PATH="$BATS_LIBEXEC:$PATH" export BATS_ROOT_PID=$$ export BATS_TMPDIR="${TMPDIR:-/tmp}" +BATS_TMPDIR=${BATS_TMPDIR%/} # chop off trailing / to avoid duplication export BATS_RUN_TMPDIR= export BATS_GUARANTEED_MINIMUM_VERSION=0.0.0 +export BATS_LIB_PATH=${BATS_LIB_PATH-/usr/lib/bats} +BATS_REPORT_OUTPUT_DIR=${BATS_REPORT_OUTPUT_DIR-.} -if [[ ! -d "${BATS_TMPDIR}" ]];then +if [[ ! -d "${BATS_TMPDIR}" ]]; then printf "Error: BATS_TMPDIR (%s) does not exist or is not a directory" "${BATS_TMPDIR}" >&2 exit 1 -elif [[ ! -w "${BATS_TMPDIR}" ]];then +elif [[ ! -w "${BATS_TMPDIR}" ]]; then printf "Error: BATS_TMPDIR (%s) is not writable" "${BATS_TMPDIR}" >&2 exit 1 fi @@ -120,14 +137,13 @@ set -- "${arguments[@]}" arguments=() unset flags recursive formatter_flags -flags=('--dummy-flag') # add a dummy flag to prevent unset variable errors on empty array expansion in old bash versions +flags=('--dummy-flag') # add a dummy flag to prevent unset variable errors on empty array expansion in old bash versions formatter_flags=('--dummy-flag') # add a dummy flag to prevent unset variable errors on empty array expansion in old bash versions -formatter='tap' +formatter=${BATS_FORMATTER:-'tap'} report_formatter='' recursive= setup_suite_file='' export BATS_TEMPDIR_CLEANUP=1 -output= if [[ -z "${CI:-}" && -t 0 && -t 1 ]] && command -v tput >/dev/null; then formatter='pretty' fi @@ -153,7 +169,7 @@ while [[ "$#" -ne 0 ]]; do -F | --formatter) shift # allow cat formatter to see extended output but don't advertise to users - if [[ $1 =~ ^(pretty|junit|tap|tap13|cat)$ ]]; then + if [[ $1 =~ ^(pretty|junit|tap|tap13|cat|/.*)$ ]]; then formatter="$1" else printf "Unknown formatter '%s', valid options are %s\n" "$1" "${VALID_FORMATTERS}" @@ -162,7 +178,7 @@ while [[ "$#" -ne 0 ]]; do ;; --report-formatter) shift - if [[ $1 =~ ^(pretty|junit|tap|tap13)$ ]]; then + if [[ $1 =~ ^(cat|pretty|junit|tap|tap13)$ ]]; then report_formatter="$1" else printf "Unknown report formatter '%s', valid options are %s\n" "$1" "${VALID_FORMATTERS}" @@ -171,7 +187,7 @@ while [[ "$#" -ne 0 ]]; do ;; -o | --output) shift - output="$1" + BATS_REPORT_OUTPUT_DIR="$1" ;; -p | --pretty) formatter='pretty' @@ -191,8 +207,8 @@ while [[ "$#" -ne 0 ]]; do formatter_flags+=('-T') ;; # this flag is now a no-op, as it is the parallel default - --parallel-preserve-environment) - ;; + --parallel-preserve-environment) ;; + --no-parallelize-across-files) flags+=("--no-parallelize-across-files") ;; @@ -221,8 +237,12 @@ while [[ "$#" -ne 0 ]]; do --gather-test-outputs-in) shift output_dir="$1" - if ! mkdir "$output_dir"; then - abort "Could not create $output_dir for --gather-test-outputs-in" + if [ -d "$output_dir" ]; then + if ! find "$output_dir" -mindepth 1 -exec false {} + 2>/dev/null; then + abort --no-print-usage "Directory '$output_dir' must be empty for --gather-test-outputs-in" + fi + elif ! mkdir "$output_dir" 2>/dev/null; then + abort --no-print-usage "Could not create '$output_dir' for --gather-test-outputs-in" fi flags+=(--gather-test-outputs-in "$output_dir") ;; @@ -233,7 +253,15 @@ while [[ "$#" -ne 0 ]]; do --code-quote-style) shift BATS_CODE_QUOTE_STYLE="$1" - ;; + ;; + --filter-status) + shift + flags+=('--filter-status' "$1") + ;; + --filter-tags) + shift + flags+=('--filter-tags' "$1") + ;; -*) abort "Bad command line option '$1'" ;; @@ -244,16 +272,16 @@ while [[ "$#" -ne 0 ]]; do shift done -if [[ -n "${BATS_RUN_TMPDIR:-}" ]];then +if [[ -n "${BATS_RUN_TMPDIR:-}" ]]; then if [[ -d "$BATS_RUN_TMPDIR" ]]; then printf "Error: BATS_RUN_TMPDIR (%s) already exists\n" "$BATS_RUN_TMPDIR" >&2 printf "Reusing old run directories can lead to unexpected results ... aborting!\n" >&2 exit 1 - elif ! mkdir -p "$BATS_RUN_TMPDIR" ;then + elif ! mkdir -p "$BATS_RUN_TMPDIR"; then printf "Error: Failed to create BATS_RUN_TMPDIR (%s)\n" "$BATS_RUN_TMPDIR" >&2 exit 1 fi -elif ! BATS_RUN_TMPDIR=$(mktemp -d "${BATS_TMPDIR}/bats-run-XXXXXX");then +elif ! BATS_RUN_TMPDIR=$(mktemp -d "${BATS_TMPDIR}/bats-run-XXXXXX"); then printf "Error: Failed to create BATS_RUN_TMPDIR (%s) with mktemp\n" "${BATS_TMPDIR}/bats-run-XXXXXX" >&2 exit 1 fi @@ -262,13 +290,14 @@ export BATS_WARNING_FILE="${BATS_RUN_TMPDIR}/warnings.log" bats_exit_trap() { if [[ -s "$BATS_WARNING_FILE" ]]; then + local pre_cat='' post_cat='' if [[ $formatter == pretty ]]; then - PRE_CAT=$'\x1B[31m' - POST_CAT=$'\x1B[0m' + pre_cat=$'\x1B[31m' + post_cat=$'\x1B[0m' fi - printf "\nThe following warnings were encountered during tests:\n%s" "$PRE_CAT" + printf "\nThe following warnings were encountered during tests:\n%s" "$pre_cat" cat "$BATS_WARNING_FILE" - printf "%s" "$POST_CAT" + printf "%s" "$post_cat" fi >&2 if [[ -n "$BATS_TEMPDIR_CLEANUP" ]]; then @@ -306,26 +335,53 @@ if [[ "$formatter" == tap && -z "$report_formatter" ]]; then formatter="cat" fi +bats_check_formatter() { # + local -r formatter="$1" + if [[ ! -f "$formatter" ]]; then + printf "ERROR: Formatter '%s' is not readable!\n" "$formatter" + exit 1 + elif [[ ! -x "$formatter" ]]; then + printf "ERROR: Formatter '%s' is not executable!\n" "$formatter" + exit 1 + fi +} + +if [[ $formatter == /* ]]; then # absolute paths are direct references to formatters + bats_check_formatter "$formatter" + interpolated_formatter="$formatter" +else + interpolated_formatter="bats-format-${formatter}" +fi + if [[ "${#arguments[@]}" -eq 0 ]]; then abort 'Must specify at least one ' fi if [[ -n "$report_formatter" ]]; then - # default to the current directory for output - if [[ -z "$output" ]]; then - output=. + if [[ ! -w "${BATS_REPORT_OUTPUT_DIR}" ]]; then + abort "Output path ${BATS_REPORT_OUTPUT_DIR} is not writeable" fi - case "$report_formatter" in - tap|tap13) - BATS_REPORT_FILE_NAME="report.tap" + # only set BATS_REPORT_FILENAME if none was given + if [[ -z "${BATS_REPORT_FILENAME:-}" ]]; then + case "$report_formatter" in + tap | tap13) + BATS_REPORT_FILENAME="report.tap" ;; junit) - BATS_REPORT_FILE_NAME="report.xml" + BATS_REPORT_FILENAME="report.xml" ;; - pretty) - BATS_REPORT_FILE_NAME="report.log" + *) + BATS_REPORT_FILENAME="report.log" ;; - esac + esac + fi +fi + +if [[ $report_formatter == /* ]]; then # absolute paths are direct references to formatters + bats_check_formatter "$report_formatter" + interpolated_report_formatter="${report_formatter}" +else + interpolated_report_formatter="bats-format-${report_formatter}" fi if [[ "${BATS_CODE_QUOTE_STYLE-BATS_CODE_QUOTE_STYLE_UNSET}" == BATS_CODE_QUOTE_STYLE_UNSET ]]; then @@ -333,31 +389,24 @@ if [[ "${BATS_CODE_QUOTE_STYLE-BATS_CODE_QUOTE_STYLE_UNSET}" == BATS_CODE_QUOTE_ fi case "${BATS_CODE_QUOTE_STYLE}" in - ??) - BATS_BEGIN_CODE_QUOTE="${BATS_CODE_QUOTE_STYLE::1}" - BATS_END_CODE_QUOTE="${BATS_CODE_QUOTE_STYLE:1:1}" - export BATS_BEGIN_CODE_QUOTE BATS_END_CODE_QUOTE +??) + BATS_BEGIN_CODE_QUOTE="${BATS_CODE_QUOTE_STYLE::1}" + BATS_END_CODE_QUOTE="${BATS_CODE_QUOTE_STYLE:1:1}" + export BATS_BEGIN_CODE_QUOTE BATS_END_CODE_QUOTE ;; - custom) - if [[ ${BATS_BEGIN_CODE_QUOTE-BATS_BEGIN_CODE_QUOTE_UNSET} == BATS_BEGIN_CODE_QUOTE_UNSET - || ${BATS_END_CODE_QUOTE-BATS_BEGIN_CODE_QUOTE_UNSET} == BATS_BEGIN_CODE_QUOTE_UNSET ]]; then - printf "ERROR: BATS_CODE_QUOTE_STYLE=custom requires BATS_BEGIN_CODE_QUOTE and BATS_END_CODE_QUOTE to be set\n" >&2 - exit 1 - fi - ;; - *) - printf "ERROR: Unknown BATS_CODE_QUOTE_STYLE: %s\n" "$BATS_CODE_QUOTE_STYLE" >&2 +custom) + if [[ ${BATS_BEGIN_CODE_QUOTE-BATS_BEGIN_CODE_QUOTE_UNSET} == BATS_BEGIN_CODE_QUOTE_UNSET || + ${BATS_END_CODE_QUOTE-BATS_BEGIN_CODE_QUOTE_UNSET} == BATS_BEGIN_CODE_QUOTE_UNSET ]]; then + printf "ERROR: BATS_CODE_QUOTE_STYLE=custom requires BATS_BEGIN_CODE_QUOTE and BATS_END_CODE_QUOTE to be set\n" >&2 exit 1 + fi + ;; +*) + printf "ERROR: Unknown BATS_CODE_QUOTE_STYLE: %s\n" "$BATS_CODE_QUOTE_STYLE" >&2 + exit 1 ;; esac -if [[ -n "$output" ]]; then - if [[ ! -w "${output}" ]]; then - abort "Output path ${output} is not writeable" - fi - export BATS_REPORT_OUTPUT_PATH="$output" -fi - if [[ -n "$setup_suite_file" && ! -f "$setup_suite_file" ]]; then abort "--setup-suite-file $setup_suite_file does not exist!" fi @@ -407,7 +456,12 @@ trap 'BATS_INTERRUPTED=true' INT # let the lower levels handle the interruption set -o pipefail execfail if [[ -n "$report_formatter" ]]; then - exec bats-exec-suite "${flags[@]}" "${filenames[@]}" | tee >("bats-format-${report_formatter}" "${report_formatter_flags[@]}" >"${BATS_REPORT_OUTPUT_PATH}/${BATS_REPORT_FILE_NAME}") | bats_test_count_validator | "bats-format-${formatter}" "${formatter_flags[@]}" + exec bats-exec-suite "${flags[@]}" "${filenames[@]}" | + tee >("$interpolated_report_formatter" "${report_formatter_flags[@]}" >"${BATS_REPORT_OUTPUT_DIR}/${BATS_REPORT_FILENAME}") | + bats_test_count_validator | + "$interpolated_formatter" "${formatter_flags[@]}" else - exec bats-exec-suite "${flags[@]}" "${filenames[@]}" | bats_test_count_validator | "bats-format-${formatter}" "${formatter_flags[@]}" + exec bats-exec-suite "${flags[@]}" "${filenames[@]}" | + bats_test_count_validator | + "$interpolated_formatter" "${formatter_flags[@]}" fi diff --git a/libexec/bats-core/bats-exec-file b/libexec/bats-core/bats-exec-file index 63452c7d3f..2669f313a4 100755 --- a/libexec/bats-core/bats-exec-file +++ b/libexec/bats-core/bats-exec-file @@ -1,21 +1,15 @@ #!/usr/bin/env bash set -eET -export flags=('--dummy-flag') +flags=('--dummy-flag') num_jobs=${BATS_NUMBER_OF_PARALLEL_JOBS:-1} -filter='' extended_syntax='' BATS_TRACE_LEVEL="${BATS_TRACE_LEVEL:-0}" +declare -r BATS_RETRY_RETURN_CODE=126 +export BATS_TEST_RETRIES=0 # no retries by default while [[ "$#" -ne 0 ]]; do case "$1" in - -c) ;; - - -f) - shift - filter="$1" - flags+=('-f' "$filter") - ;; -j) shift num_jobs="$1" @@ -31,8 +25,8 @@ while [[ "$#" -ne 0 ]]; do # use singular to allow for users to override in file BATS_NO_PARALLELIZE_WITHIN_FILE=1 ;; - --dummy-flag) - ;; + --dummy-flag) ;; + --trace) flags+=('--trace') ;; @@ -82,7 +76,7 @@ bats_run_setup_file() { # these are defined only to avoid errors when referencing undefined variables down the line # shellcheck disable=2034 - BATS_TEST_NAME= # used in tracing.bash + BATS_TEST_NAME= # used in tracing.bash # shellcheck disable=2034 BATS_TEST_COMPLETED= # used in tracing.bash @@ -98,7 +92,7 @@ bats_run_setup_file() { local status=0 # get the setup_file/teardown_file functions for this file (if it has them) # shellcheck disable=SC1090 - source "$BATS_TEST_SOURCE" >>"$BATS_OUT" 2>&1 + source "$BATS_TEST_SOURCE" >>"$BATS_OUT" 2>&1 BATS_SOURCE_FILE_COMPLETED=1 @@ -108,43 +102,64 @@ bats_run_setup_file() { } bats_run_teardown_file() { + local bats_teardown_file_status=0 # avoid running the therdown trap due to errors in teardown_file trap 'bats_file_exit_trap' EXIT - # rely on bats_error_trap to catch failures - teardown_file >>"$BATS_OUT" 2>&1 - BATS_TEARDOWN_FILE_COMPLETED=1 + # rely on bats_error_trap to catch failures + teardown_file >>"$BATS_OUT" 2>&1 || bats_teardown_file_status=$? + + if ((bats_teardown_file_status == 0)); then + BATS_TEARDOWN_FILE_COMPLETED=1 + elif [[ -n "${BATS_SETUP_FILE_COMPLETED:-}" ]]; then + BATS_DEBUG_LAST_STACK_TRACE_IS_VALID=1 + BATS_ERROR_STATUS=$bats_teardown_file_status + return $BATS_ERROR_STATUS + fi } bats_file_teardown_trap() { bats_run_teardown_file - bats_file_exit_trap + bats_file_exit_trap in-teardown_trap } # shellcheck source=lib/bats-core/common.bash source "$BATS_ROOT/lib/bats-core/common.bash" bats_file_exit_trap() { + local -r last_return_code=$? + if [[ ${1:-} != in-teardown_trap ]]; then + BATS_ERROR_STATUS=$last_return_code + fi trap - ERR EXIT + local failure_reason + local -i failure_test_index=$((BATS_FILE_FIRST_TEST_NUMBER_IN_SUITE + 1)) if [[ -z "$BATS_SETUP_FILE_COMPLETED" || -z "$BATS_TEARDOWN_FILE_COMPLETED" ]]; then if [[ -z "$BATS_SETUP_FILE_COMPLETED" ]]; then - FAILURE_REASON='setup_file' + failure_reason='setup_file' elif [[ -z "$BATS_TEARDOWN_FILE_COMPLETED" ]]; then - FAILURE_REASON='teardown_file' + failure_reason='teardown_file' + failure_test_index=$((BATS_FILE_FIRST_TEST_NUMBER_IN_SUITE + ${#tests_to_run[@]} + 1)) elif [[ -z "$BATS_SOURCE_FILE_COMPLETED" ]]; then - FAILURE_REASON='source' + failure_reason='source' else - FAILURE_REASON='unknown internal' + failure_reason='unknown internal' fi - printf "not ok %d %s\n" "$((test_number_in_suite + 1))" "$FAILURE_REASON failed" >&3 + printf "not ok %d %s\n" "$failure_test_index" "$failure_reason failed" >&3 local stack_trace bats_get_failure_stack_trace stack_trace bats_print_stack_trace "${stack_trace[@]}" >&3 bats_print_failed_command "${stack_trace[@]}" >&3 - bats_prefix_lines_for_tap_output < "$BATS_OUT" | bats_replace_filename >&3 + bats_prefix_lines_for_tap_output <"$BATS_OUT" | bats_replace_filename >&3 rm -rf "$BATS_OUT" bats_exec_file_status=1 fi + + # setup_file not executed but defined in this test file? -> might be defined in the wrong file + if [[ -z "${BATS_SETUP_SUITE_COMPLETED-}" ]] && declare -F setup_suite >/dev/null; then + bats_generate_warning 3 --no-stacktrace "$BATS_TEST_FILENAME" + fi + exit $bats_exec_file_status } @@ -168,7 +183,7 @@ bats_forward_output_of_parallel_test() { bats_is_next_parallel_test_finished() { local PID # get the pid of the next potentially finished test - PID=$(cat "$output_folder/$(( test_number_in_suite_of_last_finished_test + 1 ))/pid") + PID=$(cat "$output_folder/$((test_number_in_suite_of_last_finished_test + 1))/pid") # try to send a signal to this process # if it fails, the process exited, # if it succeeds, the process is still running @@ -183,11 +198,11 @@ bats_is_next_parallel_test_finished() { bats_forward_output_for_parallel_tests() { local status=0 # was the next test already started? - while [[ $(( test_number_in_suite_of_last_finished_test + 1 )) -le $test_number_in_suite ]]; do + while ((test_number_in_suite_of_last_finished_test + 1 <= test_number_in_suite)); do # if we are okay with waiting or if the test has already been finished - if [[ "$1" == "blocking" ]] || bats_is_next_parallel_test_finished ; then - (( ++test_number_in_suite_of_last_finished_test )) - bats_forward_output_of_parallel_test "$test_number_in_suite_of_last_finished_test" || status=1 + if [[ "$1" == "blocking" ]] || bats_is_next_parallel_test_finished; then + ((++test_number_in_suite_of_last_finished_test)) + bats_forward_output_of_parallel_test "$test_number_in_suite_of_last_finished_test" || status=$? else # non-blocking and the process has not finished -> abort the printing break @@ -196,6 +211,26 @@ bats_forward_output_for_parallel_tests() { return $status } +bats_run_test_with_retries() { # + local status=0 + local should_try_again=1 try_number + for ((try_number = 1; should_try_again; ++try_number)); do + if "$BATS_LIBEXEC/bats-exec-test" "$@" "$try_number"; then + should_try_again=0 + else + status=$? + if ((status == BATS_RETRY_RETURN_CODE)); then + should_try_again=1 + status=0 # this is not the last try -> reset status + else + should_try_again=0 + bats_exec_file_status=$status + fi + fi + done + return $status +} + bats_run_tests_in_parallel() { local output_folder="$BATS_RUN_TMPDIR/parallel_output" local status=0 @@ -204,7 +239,8 @@ bats_run_tests_in_parallel() { source "$BATS_ROOT/lib/bats-core/semaphore.bash" bats_semaphore_setup # the test_number_in_file is not yet incremented -> one before the next test to run - local test_number_in_suite_of_last_finished_test="$test_number_in_suite" # stores which test was printed last + local test_number_in_suite_of_last_finished_test="$BATS_FILE_FIRST_TEST_NUMBER_IN_SUITE" # stores which test was printed last + local test_number_in_file=0 test_number_in_suite=$BATS_FILE_FIRST_TEST_NUMBER_IN_SUITE for test_name in "${tests_to_run[@]}"; do # Only handle non-empty lines if [[ $test_name ]]; then @@ -212,8 +248,8 @@ bats_run_tests_in_parallel() { ((++test_number_in_file)) mkdir -p "$output_folder/$test_number_in_suite" bats_semaphore_run "$output_folder/$test_number_in_suite" \ - "$BATS_LIBEXEC/bats-exec-test" "${flags[@]}" "$filename" "$test_name" "$test_number_in_suite" "$test_number_in_file" \ - > "$output_folder/$test_number_in_suite/pid" + bats_run_test_with_retries "${flags[@]}" "$filename" "$test_name" "$test_number_in_suite" "$test_number_in_file" \ + >"$output_folder/$test_number_in_suite/pid" fi # print results early to get interactive feedback bats_forward_output_for_parallel_tests non-blocking || status=1 # ignore if we did not finish yet @@ -226,35 +262,41 @@ bats_read_tests_list_file() { local line_number=0 tests_to_run=() # the global test number must be visible to traps -> not local - first_test_number_in_suite='' + local test_number_in_suite='' while read -r test_line; do # check if the line begins with filename # filename might contain some hard to parse characters, # use simple string operations to work around that issue if [[ "$filename" == "${test_line::${#filename}}" ]]; then # get the rest of the line without the separator \t - test_name=${test_line:$((1 + ${#filename} ))} + test_name=${test_line:$((1 + ${#filename}))} tests_to_run+=("$test_name") # save the first test's number for later iteration # this assumes that tests for a file are stored consecutive in the file! - if [[ -z "$first_test_number_in_suite" ]]; then - first_test_number_in_suite=$line_number + if [[ -z "$test_number_in_suite" ]]; then + test_number_in_suite=$line_number fi fi ((++line_number)) done <"$TESTS_FILE" - - test_number_in_suite="$first_test_number_in_suite" - test_number_in_file=0 + BATS_FILE_FIRST_TEST_NUMBER_IN_SUITE="$test_number_in_suite" + declare -ri BATS_FILE_FIRST_TEST_NUMBER_IN_SUITE # mark readonly (cannot merge assignment, because value would be lost) } bats_run_tests() { bats_exec_file_status=0 + if [[ "$num_jobs" -lt 1 ]]; then + printf 'Invalid number of jobs: %s\n' "$num_jobs" >&2 + exit 1 + fi + if [[ "$num_jobs" != 1 && "${BATS_NO_PARALLELIZE_WITHIN_FILE-False}" == False ]]; then export BATS_SEMAPHORE_NUMBER_OF_SLOTS="$num_jobs" bats_run_tests_in_parallel "$BATS_RUN_TMPDIR/parallel_output" || bats_exec_file_status=1 else + local test_number_in_suite=$BATS_FILE_FIRST_TEST_NUMBER_IN_SUITE \ + test_number_in_file=0 for test_name in "${tests_to_run[@]}"; do if [[ "${BATS_INTERRUPTED-NOTSET}" != NOTSET ]]; then bats_exec_file_status=130 # bash's code for SIGINT exits @@ -264,12 +306,8 @@ bats_run_tests() { if [[ $test_name ]]; then ((++test_number_in_suite)) ((++test_number_in_file)) - # deal with empty flags to avoid spurious "unbound variable" errors on Bash 4.3 and lower - if [[ "${#flags[@]}" -gt 0 ]]; then - "$BATS_LIBEXEC/bats-exec-test" "${flags[@]}" "$filename" "$test_name" "$test_number_in_suite" "$test_number_in_file" || bats_exec_file_status=1 - else - "$BATS_LIBEXEC/bats-exec-test" "$filename" "$test_name" "$test_number_in_suite" "$test_number_in_file" || bats_exec_file_status=1 - fi + bats_run_test_with_retries "${flags[@]}" "$filename" "$test_name" \ + "$test_number_in_suite" "$test_number_in_file" || bats_exec_file_status=$? fi done fi @@ -281,7 +319,7 @@ bats_create_file_tempdirs() { printf 'Failed to create %s\n' "$bats_files_tmpdir" >&2 exit 1 fi - BATS_FILE_TMPDIR="$bats_files_tmpdir/$first_test_number_in_suite" + BATS_FILE_TMPDIR="$bats_files_tmpdir/${BATS_FILE_FIRST_TEST_NUMBER_IN_SUITE?}" if ! mkdir "$BATS_FILE_TMPDIR"; then printf 'Failed to create BATS_FILE_TMPDIR=%s\n' "$BATS_FILE_TMPDIR" >&2 exit 1 @@ -296,6 +334,7 @@ if [[ -n "$extended_syntax" ]]; then printf "suite %s\n" "$filename" fi +BATS_FILE_FIRST_TEST_NUMBER_IN_SUITE=0 # predeclare as Bash 3.2 does not support declare -g bats_read_tests_list_file # don't run potentially expensive setup/teardown_file diff --git a/libexec/bats-core/bats-exec-suite b/libexec/bats-core/bats-exec-suite index e5c887077d..cc8d429aa9 100755 --- a/libexec/bats-core/bats-exec-suite +++ b/libexec/bats-core/bats-exec-suite @@ -6,15 +6,21 @@ filter='' num_jobs=${BATS_NUMBER_OF_PARALLEL_JOBS:-1} bats_no_parallelize_across_files=${BATS_NO_PARALLELIZE_ACROSS_FILES-} bats_no_parallelize_within_files= -flags=('--dummy-flag') # add a dummy flag to prevent unset varialeb errors on empty array expansion in old bash versions +filter_status='' +filter_tags_list=() +flags=('--dummy-flag') # add a dummy flag to prevent unset variable errors on empty array expansion in old bash versions setup_suite_file='' BATS_TRACE_LEVEL="${BATS_TRACE_LEVEL:-0}" +BATS_SHOW_OUTPUT_OF_SUCCEEDING_TESTS= abort() { printf 'Error: %s\n' "$1" >&2 exit 1 } +# shellcheck source=lib/bats-core/common.bash disable=SC2153 +source "$BATS_ROOT/lib/bats-core/common.bash" + while [[ "$#" -ne 0 ]]; do case "$1" in -c) @@ -23,7 +29,6 @@ while [[ "$#" -ne 0 ]]; do -f) shift filter="$1" - flags+=('-f' "$filter") ;; -j) shift @@ -43,17 +48,35 @@ while [[ "$#" -ne 0 ]]; do bats_no_parallelize_within_files=1 flags+=("--no-parallelize-within-files") ;; - --dummy-flag) + --filter-status) + shift + filter_status="$1" ;; + --filter-tags) + shift + IFS=, read -ra tags <<<"$1" || true + if ((${#tags[@]} > 0)); then + for ((i = 0; i < ${#tags[@]}; ++i)); do + bats_trim "tags[$i]" "${tags[$i]}" + done + bats_sort sorted_tags "${tags[@]}" + IFS=, filter_tags_list+=("${sorted_tags[*]}") + else + filter_tags_list+=("") + fi + ;; + --dummy-flag) ;; + --trace) flags+=('--trace') - (( ++BATS_TRACE_LEVEL )) # avoid returning 0 + ((++BATS_TRACE_LEVEL)) # avoid returning 0 ;; --print-output-on-failure) flags+=(--print-output-on-failure) ;; --show-output-of-passing-tests) flags+=(--show-output-of-passing-tests) + BATS_SHOW_OUTPUT_OF_SUCCEEDING_TESTS=1 ;; --verbose-run) flags+=(--verbose-run) @@ -86,37 +109,178 @@ fi # create a file that contains all (filtered) tests to run from all files TESTS_LIST_FILE="${BATS_RUN_TMPDIR}/test_list_file.txt" -all_tests=() -for filename in "$@"; do - if [[ ! -f "$filename" ]]; then - abort "Test file \"${filename}\" does not exist" - fi - - test_names=() - test_dupes=() - while read -r line; do - if [[ ! "$line" =~ ^bats_test_function\ ]]; then - continue +bats_gather_tests() { + local line test_line tags + all_tests=() + for filename in "$@"; do + if [[ ! -f "$filename" ]]; then + abort "Test file \"${filename}\" does not exist" fi - line="${line%$'\r'}" - line="${line#* }" - test_line=$(printf "%s\t%s" "$filename" "$line") - all_tests+=("$test_line") - printf "%s\n" "$test_line" >>"$TESTS_LIST_FILE" - # avoid unbound variable errors on empty array expansion with old bash versions - if [[ ${#test_names[@]} -gt 0 && " ${test_names[*]} " == *" $line "* ]]; then - test_dupes+=("$line") - continue + + test_names=() + test_dupes=() + while read -r line; do + if [[ ! "$line" =~ ^bats_test_function\ ]]; then + continue + fi + line="${line%$'\r'}" + line="${line#* }" + TAG_REGEX="--tags '(.*)' (.*)" + if [[ "$line" =~ $TAG_REGEX ]]; then + IFS=, read -ra tags <<<"${BASH_REMATCH[1]}" || true + line="${BASH_REMATCH[2]}" + else + tags=() + fi + if [[ ${#filter_tags_list[@]} -gt 0 ]]; then + local match= + for filter_tags in "${filter_tags_list[@]}"; do + # empty search tags only match empty test tags! + if [[ -z "$filter_tags" ]]; then + if [[ ${#tags[@]} -eq 0 ]]; then + match=1 + break + fi + continue + fi + local -a positive_filter_tags=() negative_filter_tags=() + IFS=, read -ra filter_tags <<<"$filter_tags" || true + for filter_tag in "${filter_tags[@]}"; do + if [[ $filter_tag == !* ]]; then + bats_trim filter_tag "${filter_tag#!}" + negative_filter_tags+=("${filter_tag}") + else + positive_filter_tags+=("${filter_tag}") + fi + done + if bats_append_arrays_as_args positive_filter_tags -- bats_all_in tags && + ! bats_append_arrays_as_args negative_filter_tags -- bats_any_in tags; then + match=1 + fi + done + if [[ -z "$match" ]]; then + continue + fi + fi + test_line=$(printf "%s\t%s" "$filename" "$line") + all_tests+=("$test_line") + printf "%s\n" "$test_line" >>"$TESTS_LIST_FILE" + # avoid unbound variable errors on empty array expansion with old bash versions + if [[ ${#test_names[@]} -gt 0 && " ${test_names[*]} " == *" $line "* ]]; then + test_dupes+=("$line") + continue + fi + test_names+=("$line") + done < <(BATS_TEST_FILTER="$filter" bats-preprocess "$filename") + + if [[ "${#test_dupes[@]}" -ne 0 ]]; then + abort "Duplicate test name(s) in file \"${filename}\": ${test_dupes[*]}" fi - test_names+=("$line") - done < <(BATS_TEST_FILTER="$filter" bats-preprocess "$filename") + done - if [[ "${#test_dupes[@]}" -ne 0 ]]; then - abort "Duplicate test name(s) in file \"${filename}\": ${test_dupes[*]}" + test_count="${#all_tests[@]}" +} + +TEST_ROOT=${1-} +TEST_ROOT=${TEST_ROOT%/*} +BATS_RUN_LOGS_DIRECTORY="$TEST_ROOT/.bats/run-logs" +if [[ ! -d "$BATS_RUN_LOGS_DIRECTORY" ]]; then + if [[ -n "$filter_status" ]]; then + printf "Error: --filter-status needs '%s/' to save failed tests. Please create this folder, add it to .gitignore and try again.\n" "$BATS_RUN_LOGS_DIRECTORY" + exit 1 + else + BATS_RUN_LOGS_DIRECTORY= fi -done + # discard via sink instead of having a conditional later + export BATS_RUNLOG_FILE='/dev/null' +else + # use UTC (-u) to avoid problems with TZ changes + BATS_RUNLOG_DATE=$(date -u '+%Y-%m-%d %H:%M:%S UTC') + export BATS_RUNLOG_FILE="$BATS_RUN_LOGS_DIRECTORY/${BATS_RUNLOG_DATE}.log" +fi + +bats_gather_tests "$@" + +if [[ -n "$filter_status" ]]; then + case "$filter_status" in + failed) + bats_filter_test_by_status() { # + ! bats_binary_search "$1" "passed_tests" + } + ;; + passed) + bats_filter_test_by_status() { + ! bats_binary_search "$1" "failed_tests" + } + ;; + missed) + bats_filter_test_by_status() { + ! bats_binary_search "$1" "failed_tests" && ! bats_binary_search "$1" "passed_tests" + } + ;; + *) + printf "Error: Unknown value '%s' for --filter-status. Valid values are 'failed' and 'missed'.\n" "$filter_status" >&2 + exit 1 + ;; + esac + + if IFS='' read -d $'\n' -r BATS_PREVIOUS_RUNLOG_FILE < <(ls -1r "$BATS_RUN_LOGS_DIRECTORY"); then + BATS_PREVIOUS_RUNLOG_FILE="$BATS_RUN_LOGS_DIRECTORY/$BATS_PREVIOUS_RUNLOG_FILE" + if [[ $BATS_PREVIOUS_RUNLOG_FILE == "$BATS_RUNLOG_FILE" ]]; then + count=$(find "$BATS_RUN_LOGS_DIRECTORY" -name "$BATS_RUNLOG_DATE*" | wc -l) + BATS_RUNLOG_FILE="$BATS_RUN_LOGS_DIRECTORY/${BATS_RUNLOG_DATE}-$count.log" + fi + failed_tests=() + passed_tests=() + # store tests that were already filtered out in the last run for the same filter reason + last_filtered_tests=() + i=0 + while read -rd $'\n' line; do + ((++i)) + case "$line" in + "passed "*) + passed_tests+=("${line#passed }") + ;; + "failed "*) + failed_tests+=("${line#failed }") + ;; + "status-filtered $filter_status"*) # pick up tests that were filtered in the last round for the same status + last_filtered_tests+=("${line#status-filtered "$filter_status" }") + ;; + "status-filtered "*) # ignore other status-filtered lines + ;; + "#"*) # allow for comments + ;; + *) + printf "Error: %s:%d: Invalid format: %s\n" "$BATS_PREVIOUS_RUNLOG_FILE" "$i" "$line" >&2 + exit 1 + ;; + esac + done < <(sort "$BATS_PREVIOUS_RUNLOG_FILE") -test_count="${#all_tests[@]}" + filtered_tests=() + for line in "${all_tests[@]}"; do + if bats_filter_test_by_status "$line" && ! bats_binary_search "$line" last_filtered_tests; then + printf "%s\n" "$line" + filtered_tests+=("$line") + else + printf "status-filtered %s %s\n" "$filter_status" "$line" >>"$BATS_RUNLOG_FILE" + fi + done >"$TESTS_LIST_FILE" + + # save filtered tests to exclude them again in next round + for test_line in "${last_filtered_tests[@]}"; do + printf "status-filtered %s %s\n" "$filter_status" "$test_line" + done >>"$BATS_RUNLOG_FILE" + + test_count="${#filtered_tests[@]}" + if [[ ${#failed_tests[@]} -eq 0 && ${#filtered_tests[@]} -eq 0 ]]; then + printf "There where no failed tests in the last recorded run.\n" >&2 + fi + else + printf "No recording of previous runs found. Running all tests!\n" >&2 + fi +fi if [[ -n "$count_only_flag" ]]; then printf '%d\n' "${test_count}" @@ -154,7 +318,7 @@ fi # (see https://github.com/bats-core/bats-core/issues/329) # If a file was specified multiple times, we already got it repeatedly in our TESTS_LIST_FILE. # Thus, it suffices to bats-exec-file it once to run all repeated tests on it. -IFS=$'\n' read -d '' -r -a BATS_UNIQUE_TEST_FILENAMES < <(printf "%s\n" "$@"| nl | sort -k 2 | uniq -f 1 | sort -n | cut -f 2-) || true +IFS=$'\n' read -d '' -r -a BATS_UNIQUE_TEST_FILENAMES < <(printf "%s\n" "$@" | nl | sort -k 2 | uniq -f 1 | sort -n | cut -f 2-) || true # shellcheck source=lib/bats-core/tracing.bash source "$BATS_ROOT/lib/bats-core/tracing.bash" @@ -162,34 +326,53 @@ bats_setup_tracing trap bats_suite_exit_trap EXIT +exec 3<&1 + bats_suite_exit_trap() { + local print_bats_out="${BATS_SHOW_OUTPUT_OF_SUCCEEDING_TESTS}" if [[ -z "${BATS_SETUP_SUITE_COMPLETED}" || -z "${BATS_TEARDOWN_SUITE_COMPLETED}" ]]; then if [[ -z "${BATS_SETUP_SUITE_COMPLETED}" ]]; then printf "not ok 1 setup_suite\n" elif [[ -z "${BATS_TEARDOWN_SUITE_COMPLETED}" ]]; then - printf "not ok %d teardown_suite\n" $((test_count+1)) + printf "not ok %d teardown_suite\n" $((test_count + 1)) fi local stack_trace bats_get_failure_stack_trace stack_trace bats_print_stack_trace "${stack_trace[@]}" bats_print_failed_command "${stack_trace[@]}" + print_bats_out=1 bats_exec_suite_status=1 fi + + if [[ -n "$print_bats_out" ]]; then + bats_prefix_lines_for_tap_output <"$BATS_OUT" + fi + if [[ ${BATS_INTERRUPTED-NOTSET} != NOTSET ]]; then printf "\n# Received SIGINT, aborting ...\n\n" fi + + if [[ -d "$BATS_RUN_LOGS_DIRECTORY" && -n "${BATS_INTERRUPTED:-}" ]]; then + # aborting a test run with CTRL+C does not save the runlog file + rm "$BATS_RUNLOG_FILE" + fi exit "$bats_exec_suite_status" -} +} >&3 bats_run_teardown_suite() { + local bats_teardown_suite_status=0 # avoid being called twice, in case this is not called through bats_teardown_suite_trap # but from the end of file trap bats_suite_exit_trap EXIT - set -eET BATS_TEARDOWN_SUITE_COMPLETED= - teardown_suite - BATS_TEARDOWN_SUITE_COMPLETED=1 - set +ET + teardown_suite >>"$BATS_OUT" 2>&1 || bats_teardown_suite_status=$? + if ((bats_teardown_suite_status == 0)); then + BATS_TEARDOWN_SUITE_COMPLETED=1 + elif [[ -n "${BATS_SETUP_SUITE_COMPLETED:-}" ]]; then + BATS_DEBUG_LAST_STACK_TRACE_IS_VALID=1 + BATS_ERROR_STATUS=$bats_teardown_suite_status + return $BATS_ERROR_STATUS + fi } bats_teardown_suite_trap() { @@ -197,38 +380,44 @@ bats_teardown_suite_trap() { bats_suite_exit_trap } -setup_suite() { - : -} - teardown_suite() { - : + : } trap bats_teardown_suite_trap EXIT +BATS_OUT="$BATS_RUN_TMPDIR/suite.out" if [[ -n "$setup_suite_file" ]]; then setup_suite() { printf "%s does not define \`setup_suite()\`\n" "$setup_suite_file" >&2 - exit 1 + return 1 } + # shellcheck disable=SC2034 # will be used in the sourced file below + BATS_TEST_FILENAME="$setup_suite_file" + # shellcheck source=lib/bats-core/test_functions.bash + source "$BATS_ROOT/lib/bats-core/test_functions.bash" + # shellcheck disable=SC1090 source "$setup_suite_file" -fi -set -eET -BATS_SETUP_SUITE_COMPLETED= -setup_suite -BATS_SETUP_SUITE_COMPLETED=1 -set +ET + set -eET + export BATS_SETUP_SUITE_COMPLETED= + setup_suite >>"$BATS_OUT" 2>&1 + BATS_SETUP_SUITE_COMPLETED=1 + set +ET +else + # prevent exit trap from printing an error because of incomplete setup_suite, + # when there was none to execute + BATS_SETUP_SUITE_COMPLETED=1 +fi if [[ "$num_jobs" -gt 1 ]] && [[ -z "$bats_no_parallelize_across_files" ]]; then # run files in parallel to get the maximum pool of parallel tasks # shellcheck disable=SC2086,SC2068 # we need to handle the quoting of ${flags[@]} ourselves, # because parallel can only quote it as one - parallel --keep-order --jobs "$num_jobs" bats-exec-file "$(printf "%q " "${flags[@]}")" "{}" "$TESTS_LIST_FILE" ::: "${BATS_UNIQUE_TEST_FILENAMES[@]}" 2>&1 || bats_exec_suite_status=1 + parallel --keep-order --jobs "$num_jobs" bats-exec-file "$(printf "%q " "${flags[@]}")" "{}" "$TESTS_LIST_FILE" ::: "${BATS_UNIQUE_TEST_FILENAMES[@]}" 2>&1 || bats_exec_suite_status=1 else for filename in "${BATS_UNIQUE_TEST_FILENAMES[@]}"; do if [[ "${BATS_INTERRUPTED-NOTSET}" != NOTSET ]]; then diff --git a/libexec/bats-core/bats-exec-test b/libexec/bats-core/bats-exec-test index 241a5cfe8a..0130b0c5c5 100755 --- a/libexec/bats-core/bats-exec-test +++ b/libexec/bats-core/bats-exec-test @@ -2,8 +2,6 @@ set -eET # Variables used in other scripts. -BATS_COUNT_ONLY='' -BATS_TEST_FILTER='' BATS_ENABLE_TIMING='' BATS_EXTENDED_SYNTAX='' BATS_TRACE_LEVEL="${BATS_TRACE_LEVEL:-0}" @@ -15,15 +13,6 @@ BATS_TEST_NAME_PREFIX="${BATS_TEST_NAME_PREFIX:-}" while [[ "$#" -ne 0 ]]; do case "$1" in - -c) - # shellcheck disable=SC2034 - BATS_COUNT_ONLY=1 - ;; - -f) - shift - # shellcheck disable=SC2034 - BATS_TEST_FILTER="$1" - ;; -T) BATS_ENABLE_TIMING='-T' ;; @@ -31,10 +20,10 @@ while [[ "$#" -ne 0 ]]; do # shellcheck disable=SC2034 BATS_EXTENDED_SYNTAX='-x' ;; - --dummy-flag) - ;; + --dummy-flag) ;; + --trace) - (( ++BATS_TRACE_LEVEL )) # avoid returning 0 + ((++BATS_TRACE_LEVEL)) # avoid returning 0 ;; --print-output-on-failure) BATS_PRINT_OUTPUT_ON_FAILURE=1 @@ -60,6 +49,7 @@ export BATS_TEST_FILENAME="$1" export BATS_TEST_NAME="$2" export BATS_SUITE_TEST_NUMBER="$3" export BATS_TEST_NUMBER="$4" +BATS_TEST_TRY_NUMBER="$5" if [[ -z "$BATS_TEST_FILENAME" ]]; then printf 'usage: bats-exec-test \n' >&2 @@ -70,7 +60,7 @@ elif [[ ! -f "$BATS_TEST_FILENAME" ]]; then fi bats_create_test_tmpdirs() { - local tests_tmpdir="${BATS_RUN_TMPDIR}/test/" + local tests_tmpdir="${BATS_RUN_TMPDIR}/test" if ! mkdir -p "$tests_tmpdir"; then printf 'Failed to create: %s\n' "$tests_tmpdir" >&2 exit 1 @@ -78,11 +68,11 @@ bats_create_test_tmpdirs() { BATS_TEST_TMPDIR="$tests_tmpdir/$BATS_SUITE_TEST_NUMBER" if ! mkdir "$BATS_TEST_TMPDIR"; then - printf 'Failed to create BATS_TEST_TMPDIR: %s\n' "$BATS_TEST_TMPDIR" >&2 + printf 'Failed to create BATS_TEST_TMPDIR%d: %s\n' "$BATS_TEST_TRY_NUMBER" "$BATS_TEST_TMPDIR" >&2 exit 1 fi - printf "%s\n" "$BATS_TEST_NAME" > "$BATS_TEST_TMPDIR.name" + printf "%s\n" "$BATS_TEST_NAME" >"$BATS_TEST_TMPDIR.name" export BATS_TEST_TMPDIR } @@ -96,6 +86,12 @@ source "$BATS_ROOT/lib/bats-core/tracing.bash" bats_teardown_trap() { bats_check_status_from_trap local bats_teardown_trap_status=0 + + # `bats_teardown_trap` is always called with one parameter (BATS_TEARDOWN_STARTED) + # The second parameter is optional and corresponds for to the killer pid + # that will be forwarded to the exit trap to kill it if necesary + local killer_pid=${2:-} + # mark the start of this function to distinguish where skip is called # parameter 1 will signify the reason why this function was called # this is used to identify when this is called as exit trap function @@ -109,7 +105,7 @@ bats_teardown_trap() { BATS_ERROR_STATUS="$bats_teardown_trap_status" fi - bats_exit_trap + bats_exit_trap "$killer_pid" } # shellcheck source=lib/bats-core/common.bash @@ -117,23 +113,31 @@ source "$BATS_ROOT/lib/bats-core/common.bash" bats_exit_trap() { local status - local skipped='' + local exit_metadata='' + local killer_pid=${1:-} trap - ERR EXIT + if [[ -n "${BATS_TEST_TIMEOUT:-}" ]]; then + # Kill the watchdog in the case of of kernel finished before the timeout + bats_abort_timeout_countdown "$killer_pid" || status=1 + fi if [[ -n "$BATS_TEST_SKIPPED" ]]; then - skipped=' # skip' + exit_metadata=' # skip' if [[ "$BATS_TEST_SKIPPED" != '1' ]]; then - skipped+=" $BATS_TEST_SKIPPED" + exit_metadata+=" $BATS_TEST_SKIPPED" fi + elif [[ "${BATS_TIMED_OUT-NOTSET}" != NOTSET ]]; then + exit_metadata=" # timeout after ${BATS_TEST_TIMEOUT}s" fi BATS_TEST_TIME='' - if [[ -z "${skipped}" && -n "$BATS_ENABLE_TIMING" ]]; then - BATS_TEST_TIME=" in "$(( $(get_mills_since_epoch) - BATS_TEST_START_TIME ))"ms" + if [[ -n "$BATS_ENABLE_TIMING" ]]; then + BATS_TEST_TIME=" in "$(($(get_mills_since_epoch) - BATS_TEST_START_TIME))"ms" fi local print_bats_out="${BATS_SHOW_OUTPUT_OF_SUCCEEDING_TESTS}" + local should_retry='' if [[ -z "$BATS_TEST_COMPLETED" || -z "$BATS_TEARDOWN_COMPLETED" || "${BATS_INTERRUPTED-NOTSET}" != NOTSET ]]; then if [[ "$BATS_ERROR_STATUS" -eq 0 ]]; then # For some versions of bash, `$?` may not be set properly for some error @@ -147,43 +151,141 @@ bats_exit_trap() { # output, since there's no way to reach the `bats_exit_trap` call. BATS_ERROR_STATUS=1 fi - printf 'not ok %d %s\n' "$BATS_SUITE_TEST_NUMBER" "${BATS_TEST_NAME_PREFIX:-}${BATS_TEST_DESCRIPTION}${BATS_TEST_TIME}" >&3 - local stack_trace - bats_get_failure_stack_trace stack_trace - bats_print_stack_trace "${stack_trace[@]}" >&3 - bats_print_failed_command "${stack_trace[@]}" >&3 - - if [[ $BATS_PRINT_OUTPUT_ON_FAILURE && -n "${output:-}" ]]; then - printf "Last output:\n%s\n" "$output" >> "$BATS_OUT" - fi - - print_bats_out=1 - status=1 + if bats_should_retry_test; then + should_retry=1 + status=126 # signify retry + rm -r "$BATS_TEST_TMPDIR" # clean up for retry + else + printf 'not ok %d %s%s\n' "$BATS_SUITE_TEST_NUMBER" "${BATS_TEST_NAME_PREFIX:-}${BATS_TEST_DESCRIPTION}${BATS_TEST_TIME}" "$exit_metadata" >&3 + local stack_trace + bats_get_failure_stack_trace stack_trace + bats_print_stack_trace "${stack_trace[@]}" >&3 + bats_print_failed_command "${stack_trace[@]}" >&3 + + if [[ $BATS_PRINT_OUTPUT_ON_FAILURE ]]; then + if [[ -n "${output:-}" ]]; then + printf "Last output:\n%s\n" "$output" + fi + if [[ -n "${stderr:-}" ]]; then + printf "Last stderr: \n%s\n" "$stderr" + fi + fi >>"$BATS_OUT" + + print_bats_out=1 + status=1 + local state=failed + fi else printf 'ok %d %s%s\n' "$BATS_SUITE_TEST_NUMBER" "${BATS_TEST_NAME_PREFIX:-}${BATS_TEST_DESCRIPTION}${BATS_TEST_TIME}" \ - "$skipped" >&3 + "$exit_metadata" >&3 status=0 + local state=passed fi - if [[ $print_bats_out ]]; then - bats_prefix_lines_for_tap_output < "$BATS_OUT" | bats_replace_filename >&3 - fi + if [[ -z "$should_retry" ]]; then + printf "%s %s\t%s\n" "$state" "$BATS_TEST_FILENAME" "$BATS_TEST_NAME" >>"$BATS_RUNLOG_FILE" + if [[ $print_bats_out ]]; then + bats_prefix_lines_for_tap_output <"$BATS_OUT" | bats_replace_filename >&3 + fi + fi if [[ $BATS_GATHER_TEST_OUTPUTS_IN ]]; then - cp "$BATS_OUT" "$BATS_GATHER_TEST_OUTPUTS_IN/$BATS_SUITE_TEST_NUMBER-$BATS_TEST_DESCRIPTION.log" + local try_suffix= + if [[ -n "$should_retry" ]]; then + try_suffix="-try$BATS_TEST_TRY_NUMBER" + fi + cp "$BATS_OUT" "$BATS_GATHER_TEST_OUTPUTS_IN/$BATS_SUITE_TEST_NUMBER$try_suffix-$BATS_TEST_DESCRIPTION.log" fi - rm -f "$BATS_OUT" exit "$status" } +# Marks the test as failed due to timeout. +# The actual termination of subprocesses is done via pkill in the background +# process in bats_start_timeout_countdown +bats_timeout_trap() { + BATS_TIMED_OUT=1 + BATS_DEBUG_LAST_STACK_TRACE_IS_VALID= + exit 1 +} + +bats_get_child_processes_of() { # + local -ri parent_pid=${1?} + { + read -ra header + local pid_col ppid_col + for ((i = 0; i < ${#header[@]}; ++i)); do + if [[ ${header[$i]} == "PID" ]]; then + pid_col=$i + fi + if [[ ${header[$i]} == "PPID" ]]; then + ppid_col=$i + fi + done + while read -ra row; do + if ((${row[$ppid_col]} == parent_pid)); then + printf "%d\n" "${row[$pid_col]}" + fi + done + } < <(ps -ef "$parent_pid") +} + +bats_kill_childprocesses_of() { # + local -ir parent_pid="${1?}" + if command -v pkill; then + pkill -P "$parent_pid" + else + # kill in reverse order (latest first) + while read -r pid; do + kill "$pid" + done < <(bats_get_child_processes_of "$parent_pid" | sort -r) + fi >/dev/null +} + +# sets a timeout for this process +# +# using SIGABRT for interprocess communication. +# Ruled out: +# USR1/2 - not available on Windows +# SIGALRM - interferes with sleep: +# "sleep(3) may be implemented using SIGALRM; mixing calls to alarm() +# and sleep(3) is a bad idea." ~ https://linux.die.net/man/2/alarm +bats_start_timeout_countdown() { # + local -ri timeout=$1 + local -ri target_pid=$$ + # shellcheck disable=SC2064 + trap "bats_timeout_trap $target_pid" ABRT + if ! (command -v ps || command -v pkill) >/dev/null; then + printf "Error: Cannot execute timeout because neither pkill nor ps are available on this system!\n" >&2 + exit 1 + fi + # Start another process to kill the children of this process + ( + sleep "$timeout" & + # sleep won't recieve signals, so we use wait below + # and kill sleep explicitly when signalled to do so + # shellcheck disable=SC2064 + trap "kill $!; exit 0" ABRT + wait + if kill -ABRT "$target_pid"; then + # get rid of signal blocking child processes (like sleep) + bats_kill_childprocesses_of "$target_pid" + fi &>/dev/null + ) & +} + +bats_abort_timeout_countdown() { + # kill the countdown process, don't care if its still there + kill -ABRT "$1" &>/dev/null || true +} + get_mills_since_epoch() { local ms_since_epoch ms_since_epoch=$(date +%s%N) if [[ "$ms_since_epoch" == *N || "${#ms_since_epoch}" -lt 19 ]]; then - ms_since_epoch=$(( $(date +%s) * 1000 )) + ms_since_epoch=$(($(date +%s) * 1000)) else - ms_since_epoch=$(( ms_since_epoch / 1000000 )) + ms_since_epoch=$((ms_since_epoch / 1000000)) fi printf "%d\n" "$ms_since_epoch" @@ -197,41 +299,27 @@ bats_perform_test() { exit 1 fi - # Variables for capturing accurate stack traces. See bats_debug_trap for - # details. - # - # BATS_DEBUG_LAST_LINENO, BATS_DEBUG_LAST_SOURCE, and - # BATS_DEBUG_LAST_STACK_TRACE hold data from the most recent call to - # bats_debug_trap. - # - # BATS_DEBUG_LASTLAST_STACK_TRACE holds data from two bats_debug_trap calls - # ago. - # - # BATS_DEBUG_LAST_STACK_TRACE_IS_VALID indicates that - # BATS_DEBUG_LAST_STACK_TRACE contains the stack trace of the test's error. If - # unset, BATS_DEBUG_LAST_STACK_TRACE is unreliable and - # BATS_DEBUG_LASTLAST_STACK_TRACE should be used instead. - BATS_DEBUG_LASTLAST_STACK_TRACE=() - BATS_DEBUG_LAST_LINENO=() - BATS_DEBUG_LAST_SOURCE=() - BATS_DEBUG_LAST_STACK_TRACE=() - BATS_DEBUG_LAST_STACK_TRACE_IS_VALID= - + local BATS_killer_pid='' + if [[ -n "${BATS_TEST_TIMEOUT:-}" ]]; then + bats_start_timeout_countdown "$BATS_TEST_TIMEOUT" + BATS_killer_pid=$! + fi BATS_TEST_COMPLETED= BATS_TEST_SKIPPED= BATS_TEARDOWN_COMPLETED= BATS_ERROR_STATUS= - BATS_ERROR_SUFFIX= bats_setup_tracing - # mark this call as trap call - trap 'bats_teardown_trap as-exit-trap' EXIT + # use parameter to mark this call as trap call + # shellcheck disable=SC2064 + trap "bats_teardown_trap as-exit-trap $BATS_killer_pid" EXIT BATS_TEST_START_TIME=$(get_mills_since_epoch) "$BATS_TEST_NAME" >>"$BATS_OUT" 2>&1 4>&1 BATS_TEST_COMPLETED=1 - trap 'bats_exit_trap' EXIT - bats_teardown_trap "" # pass empty parameter to signify call outside trap + # shellcheck disable=SC2064 + trap "bats_exit_trap $BATS_killer_pid" EXIT + bats_teardown_trap "" "$BATS_killer_pid" # pass empty parameter to signify call outside trap } trap bats_interrupt_trap INT diff --git a/libexec/bats-core/bats-format-junit b/libexec/bats-core/bats-format-junit index ab1eb9d851..5436157e05 100755 --- a/libexec/bats-core/bats-format-junit +++ b/libexec/bats-core/bats-format-junit @@ -6,12 +6,11 @@ source "$BATS_ROOT/lib/bats-core/formatter.bash" BASE_PATH=. - while [[ "$#" -ne 0 ]]; do case "$1" in - --base-path) - shift - normalize_base_path BASE_PATH "$1" + --base-path) + shift + normalize_base_path BASE_PATH "$1" ;; esac shift @@ -19,7 +18,7 @@ done init_suite() { suite_test_exec_time=0 - # since we have to print the suite header before its contents but we don't know the contents before the header, + # since we have to print the suite header before its contents but we don't know the contents before the header, # we have to buffer the contents _suite_buffer="" test_result_state="" # declare for the first flush, when no test has been encountered @@ -32,6 +31,7 @@ init_file() { file_skipped=0 file_exec_time=0 test_exec_time=0 + name="" _buffer="" _buffer_log="" _system_out_log="" @@ -67,7 +67,7 @@ suite_header() { file_header() { timestamp=$(date -u +"%Y-%m-%dT%H:%M:%S") printf "\n" \ - "$(xml_escape "${class}")" "${file_count}" "${file_failures}" "${file_skipped}" "$(milliseconds_to_seconds "${file_exec_time}")" "${timestamp}" "$(host)" + "$(xml_escape "${class}")" "${file_count}" "${file_failures}" "${file_skipped}" "$(milliseconds_to_seconds "${file_exec_time}")" "${timestamp}" "$(host)" } file_footer() { @@ -98,19 +98,22 @@ print_test_case() { } xml_escape() { - output=${1//&/&} - output=${output///>} - output=${output//'"'/"} - output=${output//\'/'} + output=${1//&/\&} + output=${output///\>} + output=${output//'"'/\"} + output=${output//\'/\'} local CONTROL_CHAR=$'\033' - output="${output//$CONTROL_CHAR/}" + output=${output//$CONTROL_CHAR/\} printf "%s" "$output" } suite_buffer() { local output - output="$("$@"; printf "x")" # use x marker to avoid losing trailing newlines + output="$( + "$@" + printf "x" + )" # use x marker to avoid losing trailing newlines _suite_buffer="${_suite_buffer}${output%x}" } @@ -121,7 +124,10 @@ suite_flush() { buffer() { local output - output="$("$@"; printf "x")" # use x marker to avoid losing trailing newlines + output="$( + "$@" + printf "x" + )" # use x marker to avoid losing trailing newlines _buffer="${_buffer}${output%x}" } @@ -186,12 +192,8 @@ bats_tap_stream_begin() { # name="$2" } -bats_tap_stream_ok() { # [--duration - if [[ "$1" == "--duration" ]]; then - test_exec_time="${BASH_REMATCH[1]}" - else - test_exec_time=0 - fi +bats_tap_stream_ok() { # + test_exec_time=${BATS_FORMATTER_TEST_DURATION:-0} ((file_count += 1)) test_result_state='ok' file_exec_time="$((file_exec_time + test_exec_time))" @@ -199,6 +201,7 @@ bats_tap_stream_ok() { # [--duration } bats_tap_stream_skipped() { # + test_exec_time=${BATS_FORMATTER_TEST_DURATION:-0} ((file_count += 1)) ((file_skipped += 1)) test_result_state='skipped' @@ -206,14 +209,10 @@ bats_tap_stream_skipped() { # test_skip_message="$3" } -bats_tap_stream_not_ok() { # [--duration ] +bats_tap_stream_not_ok() { # + test_exec_time=${BATS_FORMATTER_TEST_DURATION:-0} ((file_count += 1)) ((file_failures += 1)) - if [[ "$1" == "--duration" ]]; then - test_exec_time="${BASH_REMATCH[1]}" - else - test_exec_time=0 - fi test_result_state=not_ok file_exec_time="$((file_exec_time + test_exec_time))" suite_test_exec_time=$((suite_test_exec_time + test_exec_time)) @@ -222,17 +221,17 @@ bats_tap_stream_not_ok() { # [--duration ] local comment="$1" scope="$2" case "$scope" in - begin) - # everything that happens between begin and [not] ok is FD3 output from the test - log_system_out "$comment" + begin) + # everything that happens between begin and [not] ok is FD3 output from the test + log_system_out "$comment" ;; - ok) - # non failed tests can produce FD3 output - log_system_out "$comment" + ok) + # non failed tests can produce FD3 output + log_system_out "$comment" ;; - *) - # everything else is considered error output - log "$1" + *) + # everything else is considered error output + log "$1" ;; esac } @@ -241,7 +240,7 @@ bats_tap_stream_suite() { # flush_log suite_buffer finish_file init_file - class="${1/$BASE_PATH}" + class="${1/$BASE_PATH/}" } bats_tap_stream_unknown() { # diff --git a/libexec/bats-core/bats-format-pretty b/libexec/bats-core/bats-format-pretty index f4d492e85f..2b03eed8ea 100755 --- a/libexec/bats-core/bats-format-pretty +++ b/libexec/bats-core/bats-format-pretty @@ -69,51 +69,70 @@ finish_test() { move_up $line_backoff_count go_to_column 0 buffer "$@" - if [[ -n "$BATS_ENABLE_TIMING" ]]; then + if [[ -n "${TIMEOUT-}" ]]; then set_color 2 - buffer ' [%s]' "$TIMING" + if [[ -n "$BATS_ENABLE_TIMING" ]]; then + buffer ' [%s (timeout: %s)]' "$TIMING" "$TIMEOUT" + else + buffer ' [timeout: %s]' "$TIMEOUT" + fi + else + if [[ -n "$BATS_ENABLE_TIMING" ]]; then + set_color 2 + buffer ' [%s]' "$TIMING" + fi fi advance - move_down $(( line_backoff_count - 1 )) + move_down $((line_backoff_count - 1)) } pass() { - TIMING="${1:-}" + local TIMING="${1:-}" finish_test ' ✓ %s' "$name" test_result=pass } skip() { - local reason="$1" + local reason="$1" TIMING="${2:-}" if [[ -n "$reason" ]]; then reason=": $reason" fi - BATS_ENABLE_TIMING='' finish_test ' - %s (skipped%s)' "$name" "$reason" + finish_test ' - %s (skipped%s)' "$name" "$reason" test_result=skip } fail() { + local TIMING="${1:-}" set_color 1 bold - TIMING="${1:-}" finish_test ' ✗ %s' "$name" test_result=fail } +timeout() { + local TIMING="${1:-}" + set_color 3 bold + TIMEOUT="${2:-}" finish_test ' ✗ %s' "$name" + test_result=timeout +} + log() { case ${test_result} in - pass) + pass) clear_color ;; - fail) + fail) set_color 1 ;; + timeout) + set_color 3 + ;; esac buffer ' %s\n' "$1" clear_color } summary() { - if [ "$failures" -eq 0 ] ; then + if [ "$failures" -eq 0 ]; then set_color 2 bold else set_color 1 bold @@ -133,7 +152,11 @@ summary() { buffer ', %d skipped' "$skipped" fi - not_run=$((count - passed - failures - skipped)) + if ((timed_out > 0)); then + buffer ', %d timed out' "$timed_out" + fi + + not_run=$((count - passed - failures - skipped - timed_out)) if [[ "$not_run" -gt 0 ]]; then buffer ', %d not run' "$not_run" fi @@ -237,6 +260,7 @@ bats_tap_stream_plan() { passed=0 failures=0 skipped=0 + timed_out=0 name= update_count_column_width } @@ -249,36 +273,32 @@ bats_tap_stream_begin() { } bats_tap_stream_ok() { - local duration= - if [[ "$1" == "--duration" ]]; then - duration="$2" - shift 2 - fi index="$1" name="$2" ((++passed)) - - pass "$duration" + + pass "${BATS_FORMATTER_TEST_DURATION:-}" } bats_tap_stream_skipped() { index="$1" name="$2" ((++skipped)) - skip "$3" + skip "$3" "${BATS_FORMATTER_TEST_DURATION:-}" } bats_tap_stream_not_ok() { - local duration= - if [[ "$1" == "--duration" ]]; then - duration="$2" - shift 2 - fi index="$1" name="$2" - ((++failures)) - - fail "$duration" + + if [[ ${BATS_FORMATTER_TEST_TIMEOUT-x} != x ]]; then + timeout "${BATS_FORMATTER_TEST_DURATION:-}" "${BATS_FORMATTER_TEST_TIMEOUT}s" + ((++timed_out)) + else + fail "${BATS_FORMATTER_TEST_DURATION:-}" + ((++failures)) + fi + } bats_tap_stream_comment() { # @@ -287,11 +307,11 @@ bats_tap_stream_comment() { # if [[ $line_backoff_count -eq 0 && $scope == begin ]]; then # if this is the first line after begin, go down one line buffer "\n" - (( ++line_backoff_count )) # prefix-increment to avoid "error" due to returning 0 + ((++line_backoff_count)) # prefix-increment to avoid "error" due to returning 0 fi - (( ++line_backoff_count )) - (( line_backoff_count += ${#1} / screen_width)) # account for linebreaks due to length + ((++line_backoff_count)) + ((line_backoff_count += ${#1} / screen_width)) # account for linebreaks due to length log "$1" } @@ -300,7 +320,7 @@ bats_tap_stream_suite() { line_backoff_count=0 index= # indicate filename for failures - local file_name="${1/$BASE_PATH}" + local file_name="${1#"$BASE_PATH"}" name="File $file_name" set_color 4 bold buffer "%s\n" "$file_name" @@ -309,18 +329,18 @@ bats_tap_stream_suite() { line_backoff_count=0 bats_tap_stream_unknown() { # - local scope=$2 - # count the lines we printed after the begin text, (or after suite, in case of syntax errors) - if [[ $line_backoff_count -eq 0 && ( $scope == begin || $scope == suite )]]; then - # if this is the first line after begin, go down one line - buffer "\n" - (( ++line_backoff_count )) # prefix-increment to avoid "error" due to returning 0 - fi + local scope=$2 + # count the lines we printed after the begin text, (or after suite, in case of syntax errors) + if [[ $line_backoff_count -eq 0 && ($scope == begin || $scope == suite) ]]; then + # if this is the first line after begin, go down one line + buffer "\n" + ((++line_backoff_count)) # prefix-increment to avoid "error" due to returning 0 + fi - (( ++line_backoff_count )) - (( line_backoff_count += ${#1} / screen_width)) # account for linebreaks due to length - buffer "%s\n" "$1" - flush + ((++line_backoff_count)) + ((line_backoff_count += ${#1} / screen_width)) # account for linebreaks due to length + buffer "%s\n" "$1" + flush } bats_parse_internal_extended_tap diff --git a/libexec/bats-core/bats-format-tap b/libexec/bats-core/bats-format-tap index fd8bdb7893..4aed2c140c 100755 --- a/libexec/bats-core/bats-format-tap +++ b/libexec/bats-core/bats-format-tap @@ -6,47 +6,50 @@ trap '' INT source "$BATS_ROOT/lib/bats-core/formatter.bash" bats_tap_stream_plan() { - printf "1..%d\n" "$1" + printf "1..%d\n" "$1" } bats_tap_stream_begin() { # - : + : } -bats_tap_stream_ok() { # [--duration - if [[ "$1" == "--duration" ]]; then - printf "ok %d %s # in %d ms\n" "$3" "$4" "$2" - else - printf "ok %d %s\n" "$1" "$2" - fi +bats_tap_stream_ok() { # [ + printf "ok %d %s" "$1" "$2" + if [[ "${BATS_FORMATTER_TEST_DURATION-x}" != x ]]; then + printf " # in %d ms" "$BATS_FORMATTER_TEST_DURATION" + fi + printf "\n" } -bats_tap_stream_not_ok() { # [--duration ] - if [[ "$1" == "--duration" ]]; then - printf "not ok %d %s # in %d ms\n" "$3" "$4" "$2" - else - printf "not ok %d %s\n" "$1" "$2" - fi +bats_tap_stream_not_ok() { # + printf "not ok %d %s" "$1" "$2" + if [[ "${BATS_FORMATTER_TEST_DURATION-x}" != x ]]; then + printf " # in %d ms" "$BATS_FORMATTER_TEST_DURATION" + fi + if [[ "${BATS_FORMATTER_TEST_TIMEOUT-x}" != x ]]; then + printf " # timeout after %d s" "${BATS_FORMATTER_TEST_TIMEOUT}" + fi + printf "\n" } bats_tap_stream_skipped() { # - if [[ -n "$3" ]]; then - printf "ok %d %s # skip %s\n" "$1" "$2" "$3" - else - printf "ok %d %s # skip\n" "$1" "$2" - fi + if [[ $# -eq 3 ]]; then + printf "ok %d %s # skip %s\n" "$1" "$2" "$3" + else + printf "ok %d %s # skip\n" "$1" "$2" + fi } bats_tap_stream_comment() { # - printf "# %s\n" "$1" + printf "# %s\n" "$1" } bats_tap_stream_suite() { # - : + : } -bats_tap_stream_unknown() { # - printf "%s\n" "$1" +bats_tap_stream_unknown() { # + printf "%s\n" "$1" } -bats_parse_internal_extended_tap \ No newline at end of file +bats_parse_internal_extended_tap diff --git a/libexec/bats-core/bats-format-tap13 b/libexec/bats-core/bats-format-tap13 index 1f95a7a8d1..98a3ffafbf 100755 --- a/libexec/bats-core/bats-format-tap13 +++ b/libexec/bats-core/bats-format-tap13 @@ -1,27 +1,6 @@ #!/usr/bin/env bash set -e -while [[ "$#" -ne 0 ]]; do - case "$1" in - -T) - BATS_ENABLE_TIMING="-T" - ;; - esac - shift -done - -header_pattern='[0-9]+\.\.[0-9]+' -IFS= read -r header - -if [[ "$header" =~ $header_pattern ]]; then - printf "TAP version 13\n" - printf "%s\n" "$header" -else - # If the first line isn't a TAP plan, print it and pass the rest through - printf '%s\n' "$header" - exec cat -fi - yaml_block_open='' add_yaml_entry() { if [[ -z "$yaml_block_open" ]]; then @@ -42,50 +21,69 @@ trap '' INT number_of_printed_log_lines_for_this_test_so_far=0 -while IFS= read -r line; do - case "$line" in - 'begin '*) ;; - 'ok '*) - close_previous_yaml_block - number_of_printed_log_lines_for_this_test_so_far=0 - if [[ -n "${BATS_ENABLE_TIMING-}" ]]; then - timing_expr="(ok [0-9]+ .+) in ([0-9]+)ms$" - if [[ "$line" =~ $timing_expr ]]; then - printf "%s\n" "${BASH_REMATCH[1]}" - add_yaml_entry "duration_ms" "${BASH_REMATCH[2]}" - else - echo "Could not match output line to timing regex: $line" >&2 - exit 1 - fi - else - printf "%s\n" "${line}" - fi - ;; - 'not ok '*) - close_previous_yaml_block - number_of_printed_log_lines_for_this_test_so_far=0 - timing_expr="(not ok [0-9]+ .+) in ([0-9])+ms$" - if [[ -n "${BATS_ENABLE_TIMING-}" ]]; then - if [[ "$line" =~ $timing_expr ]]; then - printf "%s\n" "${BASH_REMATCH[1]}" - add_yaml_entry "duration_ms" "${BASH_REMATCH[2]}" - else - echo "Could not match failure line to timing regex: $line" >&2 - exit 1 - fi - else - printf "%s\n" "${line}" - fi - ;; - '# '*) - if [[ $number_of_printed_log_lines_for_this_test_so_far -eq 0 ]]; then - add_yaml_entry "message" "|" # use a multiline string for this entry - fi - ((++number_of_printed_log_lines_for_this_test_so_far)) - printf " %s\n" "${line:2}" - ;; - 'suite '*) ;; - esac -done +# shellcheck source=lib/bats-core/formatter.bash +source "$BATS_ROOT/lib/bats-core/formatter.bash" + +bats_tap_stream_plan() { + printf "TAP version 13\n" + printf "1..%d\n" "$1" +} + +bats_tap_stream_begin() { # + : +} + +bats_tap_stream_ok() { # + close_previous_yaml_block + number_of_printed_log_lines_for_this_test_so_far=0 + printf "ok %d %s\n" "$1" "$2" + if [[ "${BATS_FORMATTER_TEST_DURATION-x}" != x ]]; then + add_yaml_entry "duration_ms" "${BATS_FORMATTER_TEST_DURATION}" + fi +} + +pass_on_optional_data() { + if [[ "${BATS_FORMATTER_TEST_DURATION-x}" != x ]]; then + add_yaml_entry "duration_ms" "${BATS_FORMATTER_TEST_DURATION}" + fi + if [[ "${BATS_FORMATTER_TEST_TIMEOUT-x}" != x ]]; then + add_yaml_entry "timeout_sec" "${BATS_FORMATTER_TEST_TIMEOUT}" + fi +} + +bats_tap_stream_not_ok() { # + close_previous_yaml_block + number_of_printed_log_lines_for_this_test_so_far=0 + + printf "not ok %d %s\n" "$1" "$2" + pass_on_optional_data +} + +bats_tap_stream_skipped() { # + close_previous_yaml_block + number_of_printed_log_lines_for_this_test_so_far=0 + + printf "not ok %d %s # SKIP %s\n" "$1" "$2" "$3" + pass_on_optional_data +} + +bats_tap_stream_comment() { # + if [[ $number_of_printed_log_lines_for_this_test_so_far -eq 0 ]]; then + add_yaml_entry "message" "|" # use a multiline string for this entry + fi + ((++number_of_printed_log_lines_for_this_test_so_far)) + printf " %s\n" "$1" +} + +bats_tap_stream_suite() { # + : +} + +bats_tap_stream_unknown() { # + : +} + +bats_parse_internal_extended_tap + # close the final block if there was one close_previous_yaml_block diff --git a/libexec/bats-core/bats-preprocess b/libexec/bats-core/bats-preprocess index 0391298f4b..5ef831043d 100755 --- a/libexec/bats-core/bats-preprocess +++ b/libexec/bats-core/bats-preprocess @@ -33,23 +33,80 @@ bats_encode_test_name() { BATS_TEST_PATTERN="^[[:blank:]]*@test[[:blank:]]+(.*[^[:blank:]])[[:blank:]]+\{(.*)\$" BATS_TEST_PATTERN_COMMENT="[[:blank:]]*([^[:blank:]()]+)[[:blank:]]*\(?\)?[[:blank:]]+\{[[:blank:]]+#[[:blank:]]*@test[[:blank:]]*\$" +BATS_COMMENT_COMMAND_PATTERN="^[[:blank:]]*#[[:blank:]]*bats[[:blank:]]+(.*)$" +BATS_VALID_TAG_PATTERN="[-_:[:alnum:]]+" +BATS_VALID_TAGS_PATTERN="^ *($BATS_VALID_TAG_PATTERN)?( *, *$BATS_VALID_TAG_PATTERN)* *$" + +# shellcheck source=lib/bats-core/common.bash +source "$BATS_ROOT/lib/bats-core/common.bash" + +extract_tags() { # + local -r tag_type=$1 tags_string=$2 + local -a tags=() + + if [[ $tags_string =~ $BATS_VALID_TAGS_PATTERN ]]; then + IFS=, read -ra tags <<<"$tags_string" + local -ri length=${#tags[@]} + for ((i = 0; i < length; ++i)); do + local element="tags[$i]" + bats_trim "$element" "${!element}" 2>/dev/null # printf on bash 3 will complain but work anyways + if [[ -z "${!element}" && -n "${CHECK_BATS_COMMENT_COMMANDS:-}" ]]; then + printf "%s:%d: Error: Invalid %s: '%s'. " "$test_file" "$line_number" "$tag_type" "$tags_string" + printf "Tags must not be empty. Please remove redundant commas!\n" + exit_code=1 + fi + done + elif [[ -n "${CHECK_BATS_COMMENT_COMMANDS:-}" ]]; then + printf "%s:%d: Error: Invalid %s: '%s'. " "$test_file" "$line_number" "$tag_type" "$tags_string" + printf "Valid tags must match %s and be separated with comma (and optional spaces)\n" "$BATS_VALID_TAG_PATTERN" + exit_code=1 + fi >&2 + if ((${#tags[@]} > 0)); then + eval "$tag_type=(\"\${tags[@]}\")" + else + eval "$tag_type=()" + fi +} test_file="$1" tests=() +test_tags=() +# shellcheck disable=SC2034 # used in `bats_sort tags`/`extract_tags`` +file_tags=() +line_number=0 +exit_code=0 { while IFS= read -r line; do + ((++line_number)) line="${line//$'\r'/}" if [[ "$line" =~ $BATS_TEST_PATTERN ]] || [[ "$line" =~ $BATS_TEST_PATTERN_COMMENT ]]; then name="${BASH_REMATCH[1]#[\'\"]}" name="${name%[\'\"]}" - body="${BASH_REMATCH[2]}" + body="${BASH_REMATCH[2]:-}" bats_encode_test_name "$name" 'encoded_name' printf '%s() { bats_test_begin "%s"; %s\n' "${encoded_name:?}" "$name" "$body" || : + bats_append_arrays_as_args \ + test_tags file_tags \ + -- bats_sort tags if [[ -z "$BATS_TEST_FILTER" || "$name" =~ $BATS_TEST_FILTER ]]; then - tests+=("$encoded_name") + IFS=, + tests+=("--tags '${tags[*]-}' $encoded_name") fi + # shellcheck disable=SC2034 # used in `bats_sort tags`/`extract_tags` + test_tags=() # reset test tags for next test else + if [[ "$line" =~ $BATS_COMMENT_COMMAND_PATTERN ]]; then + command=${BASH_REMATCH[1]} + case $command in + 'test_tags='*) + extract_tags test_tags "${command#test_tags=}" + ;; + 'file_tags='*) + extract_tags file_tags "${command#file_tags=}" + ;; + esac + fi printf '%s\n' "$line" fi done @@ -58,3 +115,5 @@ tests=() for test_name in "${tests[@]}"; do printf 'bats_test_function %s\n' "$test_name" done + +exit $exit_code diff --git a/man/bats.1.ronn b/man/bats.1.ronn index 854286e997..f97d4f6d31 100644 --- a/man/bats.1.ronn +++ b/man/bats.1.ronn @@ -57,7 +57,13 @@ OPTIONS * `-f`, `--filter `: Filter test cases by names matching the regular expression * `-F`, `--formatter `: - Switch between formatters: pretty (default), tap (default w/o term), tap13, junit + Switch between formatters: pretty (default), tap (default w/o term), tap13, junit, + `/` + * `--filter-status ` + Only run tests with the given status in the last completed (no CTRL+C/SIGINT) run. + Valid values are: + failed - runs tests that failed or were not present in the last run + missed - runs tests that were not present in the last run * `--gather-test-outputs-in `: Gather the output of failing *and* passing tests as files in directory * `-h`, `--help`: diff --git a/man/bats.7 b/man/bats.7 index b2d19ff43e..2554b354a3 100644 --- a/man/bats.7 +++ b/man/bats.7 @@ -1,7 +1,7 @@ .\" generated with Ronn/v0.7.3 .\" http://github.com/rtomayko/ronn/tree/0.7.3 . -.TH "BATS" "7" "November 2021" "bats-core" "Bash Automated Testing System" +.TH "BATS" "7" "July 2022" "bats-core" "Bash Automated Testing System" . .SH "NAME" \fBbats\fR \- Bats test file format @@ -33,7 +33,7 @@ A Bats test file is a Bash script with special syntax for defining test cases\. Each Bats test file is evaluated n+1 times, where \fIn\fR is the number of test cases in the file\. The first run counts the number of test cases, then iterates over the test cases and executes each one in its own process\. . .SH "THE RUN HELPER" -Usage: run [OPTIONS] [\-\-] Options: ! check for non zero exit code \-\fIN\fR check that exit code is \fIN\fR \-\-separate\-stderr split stderr and stdout \-\-keep\-empty\-lines retain emtpy lines in \fB${lines[@]}\fR/\fB${stderr_lines[@]}\fR +Usage: run [OPTIONS] [\-\-] Options: ! check for non zero exit code \-\fIN\fR check that exit code is \fIN\fR \-\-separate\-stderr split stderr and stdout \-\-keep\-empty\-lines retain empty lines in \fB${lines[@]}\fR/\fB${stderr_lines[@]}\fR . .P Many Bats tests need to run a command and then make assertions about its exit status and output\. Bats includes a \fBrun\fR helper that invokes its arguments as a command, saves the exit status and output into special global variables, and (optionally) checks exit status against a given expected value\. If successful, \fBrun\fR returns with a \fB0\fR status code so you can continue to make assertions in your test case\. @@ -115,6 +115,66 @@ load test_helper .P will source the script \fBtest/test_helper\.bash\fR in your test file\. This can be useful for sharing functions to set up your environment or load fixtures\. . +.SH "THE BATS_LOAD_LIBRARY COMMAND" +Some libraries are installed on the system, e\.g\. by \fBnpm\fR or \fBbrew\fR\. These should not be \fBload\fRed, as their path depends on the installation method\. Instead, one should use \fBbats_load_library\fR together with setting \fBBATS_LIB_PATH\fR, a \fBPATH\fR\-like colon\-delimited variable\. +. +.P +\fBbats_load_library\fR has two modes of resolving requests: +. +.IP "1." 4 +by relative path from the \fBBATS_LIB_PATH\fR to a file in the library +. +.IP "2." 4 +by library name, expecting libraries to have a \fBload\.bash\fR entrypoint +. +.IP "" 0 +. +.P +For example if your \fBBATS_LIB_PATH\fR is set to \fB~/\.bats/libs:/usr/lib/bats\fR, then \fBbats_load_library test_helper\fR would look for existing files with the following paths: +. +.IP "\(bu" 4 +\fB~/\.bats/libs/test_helper\fR +. +.IP "\(bu" 4 +\fB~/\.bats/libs/test_helper/load\.bash\fR +. +.IP "\(bu" 4 +\fB/usr/lib/bats/test_helper\fR +. +.IP "\(bu" 4 +\fB/usr/lib/bats/test_helper/load\.bash\fR +. +.IP "" 0 +. +.P +The first existing file in this list will be sourced\. +. +.P +If you want to load only part of a library or the entry point is not named \fBload\.bash\fR, you have to include it in the argument: \fBbats_load_library library_name/file_to_load\fR will try +. +.IP "\(bu" 4 +\fB~/\.bats/libs/library_name/file_to_load\fR +. +.IP "\(bu" 4 +\fB~/\.bats/libs/library_name/file_to_load/load\.bash\fR +. +.IP "\(bu" 4 +\fB/usr/lib/bats/library_name/file_to_load\fR +. +.IP "\(bu" 4 +\fB/usr/lib/bats/library_name/file_to_load/load\.bash\fR +. +.IP "" 0 +. +.P +Apart from the changed lookup rules, \fBbats_load_library\fR behaves like \fBload\fR\. +. +.P +\fBNote\fR: As seen above \fBload\.bash\fR is the entry point for libraries and meant to load more files from its directory or other libraries\. +. +.P +\fBNote\fR: Obviously, the actual \fBBATS_LIB_PATH\fR is highly dependent on the environment\. To maintain a uniform location across systems, (distribution) package maintainers are encouraged to use \fB/usr/lib/bats/\fR as the install path for libraries where possible\. However, if the package manager has another preferred location, like \fBnpm\fR or \fBbrew\fR, you should use this instead\. +. .SH "THE SKIP COMMAND" Tests can be skipped by using the \fBskip\fR command at the point in a test you wish to skip\. . @@ -166,6 +226,18 @@ Or you can skip conditionally: . .IP "" 0 . +.SH "THE BATS_REQUIRE_MINIMUM_VERSION COMMAND" +Code for newer versions of Bats can be incompatible with older versions\. In the best case this will lead to an error message and a failed test suite\. In the worst case, the tests will pass erroneously, potentially masking a failure\. +. +.P +Use \fBbats_require_minimum_version \fR to avoid this\. It communicates in a concise manner, that you intend the following code to be run under the given Bats version or higher\. +. +.P +Additionally, this function will communicate the current Bats version floor to subsequent code, allowing e\.g\. Bats\' internal warning to give more informed warnings\. +. +.P +\fBNote\fR: By default, calling \fBbats_require_minimum_version\fR with versions before Bats 1\.7\.0 will fail regardless of the required version as the function is not available\. However, you can use the bats\-backports plugin (https://github\.com/bats\-core/bats\-backports) to make your code usable with older versions, e\.g\. during migration while your CI system is not yet upgraded\. +. .SH "SETUP AND TEARDOWN FUNCTIONS" You can define special \fBsetup\fR and \fBteardown\fR functions which run before and after each test case, respectively\. Use these to load fixtures, set up your environment, and clean up when you\'re done\. . @@ -188,9 +260,15 @@ There are several global variables you can use to introspect on Bats tests: \fB$BATS_TEST_NAME\fR is the name of the function containing the current test case\. . .IP "\(bu" 4 +\fBBATS_TEST_NAME_PREFIX\fR will be prepended to the description of each test on stdout and in reports\. +. +.IP "\(bu" 4 \fB$BATS_TEST_DESCRIPTION\fR is the description of the current test case\. . .IP "\(bu" 4 +\fBBATS_TEST_RETRIES\fR is the maximum number of additional attempts that will be made on a failed test before it is finally considered failed\. The default of 0 means the test must pass on the first attempt\. +. +.IP "\(bu" 4 \fB$BATS_TEST_NUMBER\fR is the (1\-based) index of the current test case in the test file\. . .IP "\(bu" 4 diff --git a/man/bats.7.ronn b/man/bats.7.ronn index 396a208eeb..1ac951c0cb 100644 --- a/man/bats.7.ronn +++ b/man/bats.7.ronn @@ -38,7 +38,7 @@ Options: --separate-stderr split stderr and stdout --keep-empty-lines - retain emtpy lines in `${lines[@]}`/`${stderr_lines[@]}` + retain empty lines in `${lines[@]}`/`${stderr_lines[@]}` Many Bats tests need to run a command and then make assertions about its exit status and output. Bats includes a `run` helper that invokes @@ -105,6 +105,49 @@ will source the script `test/test_helper.bash` in your test file. This can be useful for sharing functions to set up your environment or load fixtures. +THE BATS_LOAD_LIBRARY COMMAND +----------------------------- + +Some libraries are installed on the system, e.g. by `npm` or `brew`. +These should not be `load`ed, as their path depends on the installation method. +Instead, one should use `bats_load_library` together with setting +`BATS_LIB_PATH`, a `PATH`-like colon-delimited variable. + +`bats_load_library` has two modes of resolving requests: + +1. by relative path from the `BATS_LIB_PATH` to a file in the library +2. by library name, expecting libraries to have a `load.bash` entrypoint + +For example if your `BATS_LIB_PATH` is set to +`~/.bats/libs:/usr/lib/bats`, then `bats_load_library test_helper` +would look for existing files with the following paths: + +- `~/.bats/libs/test_helper` +- `~/.bats/libs/test_helper/load.bash` +- `/usr/lib/bats/test_helper` +- `/usr/lib/bats/test_helper/load.bash` + +The first existing file in this list will be sourced. + +If you want to load only part of a library or the entry point is not named `load.bash`, +you have to include it in the argument: +`bats_load_library library_name/file_to_load` will try + +- `~/.bats/libs/library_name/file_to_load` +- `~/.bats/libs/library_name/file_to_load/load.bash` +- `/usr/lib/bats/library_name/file_to_load` +- `/usr/lib/bats/library_name/file_to_load/load.bash` + +Apart from the changed lookup rules, `bats_load_library` behaves like `load`. + +**Note**: As seen above `load.bash` is the entry point for libraries and +meant to load more files from its directory or other libraries. + +**Note**: Obviously, the actual `BATS_LIB_PATH` is highly dependent on the environment. +To maintain a uniform location across systems, (distribution) package maintainers +are encouraged to use `/usr/lib/bats/` as the install path for libraries where possible. +However, if the package manager has another preferred location, like `npm` or `brew`, +you should use this instead. THE SKIP COMMAND ---------------- @@ -135,6 +178,27 @@ Or you can skip conditionally: } +THE BATS_REQUIRE_MINIMUM_VERSION COMMAND +---------------------------------------- + +Code for newer versions of Bats can be incompatible with older versions. +In the best case this will lead to an error message and a failed test suite. +In the worst case, the tests will pass erroneously, potentially masking a failure. + +Use `bats_require_minimum_version ` to avoid this. +It communicates in a concise manner, that you intend the following code to be run +under the given Bats version or higher. + +Additionally, this function will communicate the current Bats version floor to +subsequent code, allowing e.g. Bats' internal warning to give more informed warnings. + +**Note**: By default, calling `bats_require_minimum_version` with versions before +Bats 1.7.0 will fail regardless of the required version as the function is not +available. However, you can use the +bats-backports plugin (https://github.com/bats-core/bats-backports) to make +your code usable with older versions, e.g. during migration while your CI system +is not yet upgraded. + SETUP AND TEARDOWN FUNCTIONS ---------------------------- @@ -171,6 +235,12 @@ test case. on stdout and in reports. * `$BATS_TEST_DESCRIPTION` is the description of the current test case. +* `BATS_TEST_RETRIES` is the maximum number of additional attempts that will be + made on a failed test before it is finally considered failed. + The default of 0 means the test must pass on the first attempt. +* `BATS_TEST_TIMEOUT` is the number of seconds after which a test (including setup) + will be aborted and marked as failed. Updates to this value in `setup()` or `@test` + cannot change the running timeout countdown, so the latest useful update location is `setup_file()`. * `$BATS_TEST_NUMBER` is the (1-based) index of the current test case in the test file. * `$BATS_SUITE_TEST_NUMBER` is the (1-based) index of the current test diff --git a/package.json b/package.json index 770abaab35..142e6461a3 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "bats", - "version": "1.7.0", + "version": "1.8.2", "description": "Bash Automated Testing System", "homepage": "https://github.com/bats-core/bats-core#readme", "license": "MIT", diff --git a/shellcheck.sh b/shellcheck.sh index b7900ac6ec..3087264452 100755 --- a/shellcheck.sh +++ b/shellcheck.sh @@ -3,17 +3,18 @@ set -e targets=() -while IFS= read -r -d $'\0'; do - targets+=("$REPLY") +while IFS= read -r -d $'\0'; do + targets+=("$REPLY") done < <( - find . -type f \( -name \*.bash -o -name \*.sh \) -print0; \ - find . -name '*.bats' -not -name '*_no_shellcheck*' -print0; \ - find libexec -type f -print0; - find bin -type f -print0) + find . -type f \( -name \*.bash -o -name \*.sh \) -print0 + find . -type f -name '*.bats' -not -name '*_no_shellcheck*' -print0 + find libexec -type f -print0 + find bin -type f -print0 +) if [[ $1 == --list ]]; then - printf "%s\n" "${targets[@]}" - exit 0 + printf "%s\n" "${targets[@]}" + exit 0 fi LC_ALL=C.UTF-8 shellcheck "${targets[@]}" diff --git a/test.bats b/test.bats deleted file mode 100644 index 35a7f2a997..0000000000 --- a/test.bats +++ /dev/null @@ -1,7 +0,0 @@ -teardown() { - false -} - -@test test { - false -} diff --git a/test/.bats/run-logs/.gitkeep b/test/.bats/run-logs/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/bats.bats b/test/bats.bats index 41f70b6020..8c6dab8116 100755 --- a/test/bats.bats +++ b/test/bats.bats @@ -3,96 +3,79 @@ setup() { load test_helper fixtures bats + REENTRANT_RUN_PRESERVE+=(BATS_LIBEXEC) } @test "no arguments prints message and usage instructions" { - run bats + reentrant_run bats [ $status -eq 1 ] [ "${lines[0]}" == 'Error: Must specify at least one ' ] [ "${lines[1]%% *}" == 'Usage:' ] } @test "invalid option prints message and usage instructions" { - run bats --invalid-option + reentrant_run bats --invalid-option [ $status -eq 1 ] [ "${lines[0]}" == "Error: Bad command line option '--invalid-option'" ] [ "${lines[1]%% *}" == 'Usage:' ] } @test "-v and --version print version number" { - run bats -v + reentrant_run bats -v [ $status -eq 0 ] [ "$(expr "$output" : "Bats [0-9][0-9.]*")" -ne 0 ] } @test "-h and --help print help" { - run bats -h + reentrant_run bats -h [ $status -eq 0 ] [ "${#lines[@]}" -gt 3 ] } @test "invalid filename prints an error" { - run bats nonexistent + reentrant_run bats nonexistent [ $status -eq 1 ] [ "$(expr "$output" : ".*does not exist")" -ne 0 ] } @test "empty test file runs zero tests" { - run bats "$FIXTURE_ROOT/empty.bats" + reentrant_run bats "$FIXTURE_ROOT/empty.bats" [ $status -eq 0 ] [ "$output" = "1..0" ] } @test "one passing test" { - run bats "$FIXTURE_ROOT/passing.bats" + reentrant_run bats "$FIXTURE_ROOT/passing.bats" [ $status -eq 0 ] [ "${lines[0]}" = "1..1" ] [ "${lines[1]}" = "ok 1 a passing test" ] } @test "summary passing tests" { - run filter_control_sequences bats -p "$FIXTURE_ROOT/passing.bats" + reentrant_run filter_control_sequences bats -p "$FIXTURE_ROOT/passing.bats" echo "$output" [ $status -eq 0 ] [ "${lines[2]}" = "1 test, 0 failures" ] } @test "summary passing and skipping tests" { - run filter_control_sequences bats -p "$FIXTURE_ROOT/passing_and_skipping.bats" + reentrant_run filter_control_sequences bats -p "$FIXTURE_ROOT/passing_and_skipping.bats" [ $status -eq 0 ] [ "${lines[4]}" = "3 tests, 0 failures, 2 skipped" ] } -@test "tap passing and skipping tests" { - run filter_control_sequences bats --formatter tap "$FIXTURE_ROOT/passing_and_skipping.bats" - [ $status -eq 0 ] - [ "${lines[0]}" = "1..3" ] - [ "${lines[1]}" = "ok 1 a passing test" ] - [ "${lines[2]}" = "ok 2 a skipped test with no reason # skip" ] - [ "${lines[3]}" = "ok 3 a skipped test with a reason # skip for a really good reason" ] -} - @test "summary passing and failing tests" { - run filter_control_sequences bats -p "$FIXTURE_ROOT/failing_and_passing.bats" + reentrant_run filter_control_sequences bats -p "$FIXTURE_ROOT/failing_and_passing.bats" [ $status -eq 0 ] [ "${lines[5]}" = "2 tests, 1 failure" ] } @test "summary passing, failing and skipping tests" { - run filter_control_sequences bats -p "$FIXTURE_ROOT/passing_failing_and_skipping.bats" + reentrant_run filter_control_sequences bats -p "$FIXTURE_ROOT/passing_failing_and_skipping.bats" [ $status -eq 0 ] [ "${lines[6]}" = "3 tests, 1 failure, 1 skipped" ] } -@test "tap passing, failing and skipping tests" { - run filter_control_sequences bats --formatter tap "$FIXTURE_ROOT/passing_failing_and_skipping.bats" - [ $status -eq 0 ] - [ "${lines[0]}" = "1..3" ] - [ "${lines[1]}" = "ok 1 a passing test" ] - [ "${lines[2]}" = "ok 2 a skipping test # skip" ] - [ "${lines[3]}" = "not ok 3 a failing test" ] -} - @test "BATS_CWD is correctly set to PWD as validated by bats_trim_filename" { local trimmed bats_trim_filename "$PWD/foo/bar" 'trimmed' @@ -101,7 +84,7 @@ setup() { } @test "one failing test" { - run bats "$FIXTURE_ROOT/failing.bats" + reentrant_run bats "$FIXTURE_ROOT/failing.bats" [ $status -eq 1 ] [ "${lines[0]}" = '1..1' ] [ "${lines[1]}" = 'not ok 1 a failing test' ] @@ -110,7 +93,7 @@ setup() { } @test "one failing and one passing test" { - run bats "$FIXTURE_ROOT/failing_and_passing.bats" + reentrant_run bats "$FIXTURE_ROOT/failing_and_passing.bats" [ $status -eq 1 ] [ "${lines[0]}" = '1..2' ] [ "${lines[1]}" = 'not ok 1 a failing test' ] @@ -120,13 +103,13 @@ setup() { } @test "failing test with significant status" { - STATUS=2 run bats "$FIXTURE_ROOT/failing.bats" + STATUS=2 reentrant_run bats "$FIXTURE_ROOT/failing.bats" [ $status -eq 1 ] [ "${lines[3]}" = "# \`eval \"( exit \${STATUS:-1} )\"' failed with status 2" ] } @test "failing helper function logs the test case's line number" { - run bats "$FIXTURE_ROOT/failing_helper.bats" + reentrant_run bats "$FIXTURE_ROOT/failing_helper.bats" [ $status -eq 1 ] [ "${lines[1]}" = 'not ok 1 failing helper function' ] [ "${lines[2]}" = "# (from function \`failing_helper' in file $RELATIVE_FIXTURE_ROOT/test_helper.bash, line 6," ] @@ -135,7 +118,7 @@ setup() { } @test "failing bash condition logs correct line number" { - run bats "$FIXTURE_ROOT/failing_with_bash_cond.bats" + reentrant_run bats "$FIXTURE_ROOT/failing_with_bash_cond.bats" [ "$status" -eq 1 ] [ "${#lines[@]}" -eq 4 ] [ "${lines[1]}" = 'not ok 1 a failing test' ] @@ -144,16 +127,16 @@ setup() { } @test "failing bash expression logs correct line number" { - run bats "$FIXTURE_ROOT/failing_with_bash_expression.bats" + reentrant_run bats "$FIXTURE_ROOT/failing_with_bash_expression.bats" [ "$status" -eq 1 ] [ "${#lines[@]}" -eq 4 ] [ "${lines[1]}" = 'not ok 1 a failing test' ] [ "${lines[2]}" = "# (in test file $RELATIVE_FIXTURE_ROOT/failing_with_bash_expression.bats, line 3)" ] - [ "${lines[3]}" = "# \`(( 1 == 2 ))' failed" ] + [ "${lines[3]}" = "# \`((1 == 2))' failed" ] } @test "failing negated command logs correct line number" { - run bats "$FIXTURE_ROOT/failing_with_negated_command.bats" + reentrant_run bats "$FIXTURE_ROOT/failing_with_negated_command.bats" [ "$status" -eq 1 ] [ "${#lines[@]}" -eq 4 ] [ "${lines[1]}" = 'not ok 1 a failing test' ] @@ -162,32 +145,40 @@ setup() { } @test "test environments are isolated" { - run bats "$FIXTURE_ROOT/environment.bats" + reentrant_run bats "$FIXTURE_ROOT/environment.bats" [ $status -eq 0 ] } @test "setup is run once before each test" { unset BATS_NUMBER_OF_PARALLEL_JOBS BATS_NO_PARALLELIZE_ACROSS_FILES + # shellcheck disable=SC2031,SC2030 export BATS_TEST_SUITE_TMPDIR="${BATS_TEST_TMPDIR}" - run bats "$FIXTURE_ROOT/setup.bats" + # shellcheck disable=SC2030 + REENTRANT_RUN_PRESERVE+=(BATS_TEST_SUITE_TMPDIR) + + reentrant_run bats "$FIXTURE_ROOT/setup.bats" [ $status -eq 0 ] - run cat "$BATS_TEST_SUITE_TMPDIR/setup.log" + reentrant_run cat "$BATS_TEST_SUITE_TMPDIR/setup.log" [ ${#lines[@]} -eq 3 ] } @test "teardown is run once after each test, even if it fails" { unset BATS_NUMBER_OF_PARALLEL_JOBS BATS_NO_PARALLELIZE_ACROSS_FILES + # shellcheck disable=SC2031,SC2030 export BATS_TEST_SUITE_TMPDIR="${BATS_TEST_TMPDIR}" - run bats "$FIXTURE_ROOT/teardown.bats" + # shellcheck disable=SC2030,SC2031 + REENTRANT_RUN_PRESERVE+=(BATS_TEST_SUITE_TMPDIR) + + reentrant_run bats "$FIXTURE_ROOT/teardown.bats" [ $status -eq 1 ] - run cat "$BATS_TEST_SUITE_TMPDIR/teardown.log" + reentrant_run cat "$BATS_TEST_SUITE_TMPDIR/teardown.log" [ ${#lines[@]} -eq 3 ] } @test "setup failure" { - run bats "$FIXTURE_ROOT/failing_setup.bats" + reentrant_run bats "$FIXTURE_ROOT/failing_setup.bats" [ $status -eq 1 ] [ "${lines[1]}" = 'not ok 1 truth' ] [ "${lines[2]}" = "# (from function \`setup' in test file $RELATIVE_FIXTURE_ROOT/failing_setup.bats, line 2)" ] @@ -195,7 +186,7 @@ setup() { } @test "passing test with teardown failure" { - PASS=1 run bats "$FIXTURE_ROOT/failing_teardown.bats" + PASS=1 reentrant_run bats "$FIXTURE_ROOT/failing_teardown.bats" [ $status -eq 1 ] echo "$output" [ "${lines[1]}" = 'not ok 1 truth' ] @@ -204,81 +195,71 @@ setup() { } @test "failing test with teardown failure" { - PASS=0 run bats "$FIXTURE_ROOT/failing_teardown.bats" + PASS=0 reentrant_run bats "$FIXTURE_ROOT/failing_teardown.bats" [ $status -eq 1 ] - [ "${lines[1]}" = 'not ok 1 truth' ] - [ "${lines[2]}" = "# (in test file $RELATIVE_FIXTURE_ROOT/failing_teardown.bats, line 6)" ] + [ "${lines[1]}" = 'not ok 1 truth' ] + [ "${lines[2]}" = "# (in test file $RELATIVE_FIXTURE_ROOT/failing_teardown.bats, line 6)" ] [ "${lines[3]}" = $'# `[ "$PASS" = 1 ]\' failed' ] } @test "teardown failure with significant status" { - PASS=1 STATUS=2 run bats "$FIXTURE_ROOT/failing_teardown.bats" + PASS=1 STATUS=2 reentrant_run bats "$FIXTURE_ROOT/failing_teardown.bats" [ $status -eq 1 ] [ "${lines[3]}" = "# \`eval \"( exit \${STATUS:-1} )\"' failed with status 2" ] } @test "failing test file outside of BATS_CWD" { cd "${BATS_TEST_TMPDIR}" - run bats "$FIXTURE_ROOT/failing.bats" + reentrant_run bats "$FIXTURE_ROOT/failing.bats" [ $status -eq 1 ] [ "${lines[2]}" = "# (in test file $FIXTURE_ROOT/failing.bats, line 4)" ] } @test "output is discarded for passing tests and printed for failing tests" { - run bats "$FIXTURE_ROOT/output.bats" + reentrant_run bats "$FIXTURE_ROOT/output.bats" [ $status -eq 1 ] - [ "${lines[6]}" = '# failure stdout 1' ] - [ "${lines[7]}" = '# failure stdout 2' ] + [ "${lines[6]}" = '# failure stdout 1' ] + [ "${lines[7]}" = '# failure stdout 2' ] [ "${lines[11]}" = '# failure stderr' ] } @test "-c prints the number of tests" { - run bats -c "$FIXTURE_ROOT/empty.bats" + reentrant_run bats -c "$FIXTURE_ROOT/empty.bats" [ $status -eq 0 ] [ "$output" = 0 ] - run bats -c "$FIXTURE_ROOT/output.bats" + reentrant_run bats -c "$FIXTURE_ROOT/output.bats" [ $status -eq 0 ] [ "$output" = 4 ] } @test "dash-e is not mangled on beginning of line" { - run bats "$FIXTURE_ROOT/intact.bats" + reentrant_run bats "$FIXTURE_ROOT/intact.bats" [ $status -eq 0 ] [ "${lines[1]}" = "ok 1 dash-e on beginning of line" ] } @test "dos line endings are stripped before testing" { - run bats "$FIXTURE_ROOT/dos_line_no_shellcheck.bats" + reentrant_run bats "$FIXTURE_ROOT/dos_line_no_shellcheck.bats" [ $status -eq 0 ] } @test "test file without trailing newline" { - run bats "$FIXTURE_ROOT/without_trailing_newline.bats" + reentrant_run bats "$FIXTURE_ROOT/without_trailing_newline.bats" [ $status -eq 0 ] [ "${lines[1]}" = "ok 1 truth" ] } @test "skipped tests" { - run bats "$FIXTURE_ROOT/skipped.bats" + reentrant_run bats "$FIXTURE_ROOT/skipped.bats" [ $status -eq 0 ] [ "${lines[1]}" = "ok 1 a skipped test # skip" ] [ "${lines[2]}" = "ok 2 a skipped test with a reason # skip a reason" ] } -@test "skipped test with parens (pretty formatter)" { - run bats --pretty "$FIXTURE_ROOT/skipped_with_parens.bats" - [ $status -eq 0 ] - - # Some systems (Alpine, for example) seem to emit an extra whitespace into - # entries in the 'lines' array when a carriage return is present from the - # pretty formatter. This is why a '+' is used after the 'skipped' note. - [[ "${lines[*]}" =~ "- a skipped test with parentheses in the reason (skipped: "+"a reason (with parentheses))" ]] -} - @test "extended syntax" { emulate_bats_env - run bats-exec-suite -x "$FIXTURE_ROOT/failing_and_passing.bats" + reentrant_run bats-exec-suite -x "$FIXTURE_ROOT/failing_and_passing.bats" echo "$output" [ $status -eq 1 ] [ "${lines[1]}" = "suite $FIXTURE_ROOT/failing_and_passing.bats" ] @@ -289,7 +270,7 @@ setup() { } @test "timing syntax" { - run bats -T "$FIXTURE_ROOT/failing_and_passing.bats" + reentrant_run bats -T "$FIXTURE_ROOT/failing_and_passing.bats" echo "$output" [ $status -eq 1 ] regex='not ok 1 a failing test in [0-9]+ms' @@ -300,7 +281,7 @@ setup() { @test "extended timing syntax" { emulate_bats_env - run bats-exec-suite -x -T "$FIXTURE_ROOT/failing_and_passing.bats" + reentrant_run bats-exec-suite -x -T "$FIXTURE_ROOT/failing_and_passing.bats" echo "$output" [ $status -eq 1 ] regex="not ok 1 a failing test in [0-9]+ms" @@ -313,57 +294,38 @@ setup() { @test "time is greater than 0ms for long test" { emulate_bats_env - run bats-exec-suite -x -T "$FIXTURE_ROOT/run_long_command.bats" + reentrant_run bats-exec-suite -x -T "$FIXTURE_ROOT/run_long_command.bats" echo "$output" [ $status -eq 0 ] regex="ok 1 run long command in [1-9][0-9]*ms" [[ "${lines[3]}" =~ $regex ]] } -@test "pretty and tap formats" { - run bats --formatter tap "$FIXTURE_ROOT/passing.bats" - tap_output="$output" - [ $status -eq 0 ] - - run bats --pretty "$FIXTURE_ROOT/passing.bats" - pretty_output="$output" - [ $status -eq 0 ] - - [ "$tap_output" != "$pretty_output" ] -} - -@test "pretty formatter bails on invalid tap" { - run bats-format-pretty < <(printf "This isn't TAP!\nGood day to you\n") - [ $status -eq 0 ] - [ "${lines[0]}" = "This isn't TAP!" ] - [ "${lines[1]}" = "Good day to you" ] -} - @test "single-line tests" { - run bats "$FIXTURE_ROOT/single_line_no_shellcheck.bats" + reentrant_run bats --no-tempdir-cleanup "$FIXTURE_ROOT/single_line_no_shellcheck.bats" [ $status -eq 1 ] - [ "${lines[1]}" = 'ok 1 empty' ] - [ "${lines[2]}" = 'ok 2 passing' ] - [ "${lines[3]}" = 'ok 3 input redirection' ] - [ "${lines[4]}" = 'not ok 4 failing' ] - [ "${lines[5]}" = "# (in test file $RELATIVE_FIXTURE_ROOT/single_line_no_shellcheck.bats, line 9)" ] + [ "${lines[1]}" = 'ok 1 empty' ] + [ "${lines[2]}" = 'ok 2 passing' ] + [ "${lines[3]}" = 'ok 3 input redirection' ] + [ "${lines[4]}" = 'not ok 4 failing' ] + [ "${lines[5]}" = "# (in test file $RELATIVE_FIXTURE_ROOT/single_line_no_shellcheck.bats, line 9)" ] [ "${lines[6]}" = $'# `@test "failing" { false; }\' failed' ] } @test "testing IFS not modified by run" { - run bats "$FIXTURE_ROOT/loop_keep_IFS.bats" + reentrant_run bats "$FIXTURE_ROOT/loop_keep_IFS.bats" [ $status -eq 0 ] [ "${lines[1]}" = "ok 1 loop_func" ] } @test "expand variables in test name" { - SUITE='test/suite' run bats "$FIXTURE_ROOT/expand_var_in_test_name.bats" + SUITE='test/suite' reentrant_run bats "$FIXTURE_ROOT/expand_var_in_test_name.bats" [ $status -eq 0 ] [ "${lines[1]}" = "ok 1 test/suite: test with variable in name" ] } @test "handle quoted and unquoted test names" { - run bats "$FIXTURE_ROOT/quoted_and_unquoted_test_names_no_shellcheck.bats" + reentrant_run bats "$FIXTURE_ROOT/quoted_and_unquoted_test_names_no_shellcheck.bats" [ $status -eq 0 ] [ "${lines[1]}" = "ok 1 single-quoted name" ] [ "${lines[2]}" = "ok 2 double-quoted name" ] @@ -373,8 +335,8 @@ setup() { @test 'ensure compatibility with unofficial Bash strict mode' { local expected='ok 1 unofficial Bash strict mode conditions met' - if [[ -n "$BATS_NUMBER_OF_PARALLEL_JOBS" ]]; then - if [[ -z "$BATS_NO_PARALLELIZE_ACROSS_FILES" ]]; then + if [[ -n "${BATS_NUMBER_OF_PARALLEL_JOBS:-}" ]]; then + if [[ -z "${BATS_NO_PARALLELIZE_ACROSS_FILES:-}" ]]; then type -p parallel &>/dev/null || skip "Don't check file parallelized without GNU parallel" fi (type -p flock &>/dev/null || type -p shlock &>/dev/null) || skip "Don't check parallelized without flock/shlock " @@ -382,15 +344,15 @@ setup() { # PATH required for windows # HOME required to avoid error from GNU Parallel - # Run Bats under SHELLOPTS=nounset (recursive `set -u`) to catch + # Run Bats under SHELLOPTS=nounset (recursive `set -u`) to catch # as many unset variable accesses as possible. run env - \ - "PATH=$PATH" \ - "HOME=$HOME" \ - "BATS_NO_PARALLELIZE_ACROSS_FILES=$BATS_NO_PARALLELIZE_ACROSS_FILES" \ - "BATS_NUMBER_OF_PARALLEL_JOBS=$BATS_NUMBER_OF_PARALLEL_JOBS" \ - SHELLOPTS=nounset \ - "${BATS_ROOT}/bin/bats" "$FIXTURE_ROOT/unofficial_bash_strict_mode.bats" + "PATH=$PATH" \ + "HOME=$HOME" \ + "BATS_NO_PARALLELIZE_ACROSS_FILES=${BATS_NO_PARALLELIZE_ACROSS_FILES:-}" \ + "BATS_NUMBER_OF_PARALLEL_JOBS=${BATS_NUMBER_OF_PARALLEL_JOBS:-}" \ + SHELLOPTS=nounset \ + "${BATS_ROOT}/bin/bats" "$FIXTURE_ROOT/unofficial_bash_strict_mode.bats" if [[ "$status" -ne 0 || "${lines[1]}" != "$expected" ]]; then cat <&2 printf 'LINE: %s\n' "${lines[@]}" >&2 [ "$status" -eq 1 ] @@ -535,7 +497,7 @@ END_OF_ERR_MSG } @test "run tests which consume stdin (see #197)" { - run bats "$FIXTURE_ROOT/read_from_stdin.bats" + reentrant_run bats "$FIXTURE_ROOT/read_from_stdin.bats" [ "$status" -eq 0 ] [[ "${lines[0]}" == "1..3" ]] [[ "${lines[1]}" == "ok 1 test 1" ]] @@ -544,7 +506,7 @@ END_OF_ERR_MSG } @test "report correct line on unset variables" { - LANG=C run bats "$FIXTURE_ROOT/unbound_variable.bats" + LANG=C reentrant_run bats "$FIXTURE_ROOT/unbound_variable.bats" [ "$status" -eq 1 ] [ "${#lines[@]}" -eq 9 ] [ "${lines[1]}" = 'not ok 1 access unbound variable' ] @@ -560,7 +522,7 @@ END_OF_ERR_MSG } @test "report correct line on external function calls" { - run bats "$FIXTURE_ROOT/external_function_calls.bats" + reentrant_run bats "$FIXTURE_ROOT/external_function_calls.bats" [ "$status" -eq 1 ] expectedNumberOfTests=12 @@ -576,18 +538,19 @@ END_OF_ERR_MSG [[ "${lines[$outputOffset]}" =~ stackdepth=([0-9]+) ]] stackdepth="${BASH_REMATCH[1]}" case "${stackdepth}" in - 1) - [ "${lines[$((outputOffset + 1))]}" = "# (in test file $RELATIVE_FIXTURE_ROOT/external_function_calls.bats, line $currentErrorLine)" ] - outputOffset=$((outputOffset + 3)) - ;; - 2) - [[ "${lines[$((outputOffset + 1))]}" =~ ^'# (from function `'.*\'' in file '.*'/test_helper.bash, line '[0-9]+,$ ]] - [ "${lines[$((outputOffset + 2))]}" = "# in test file $RELATIVE_FIXTURE_ROOT/external_function_calls.bats, line $currentErrorLine)" ] - outputOffset=$((outputOffset + 4)) - ;; - *) - printf 'error: stackdepth=%s not implemented\n' "${stackdepth}" >&2 - return 1 + 1) + [ "${lines[$((outputOffset + 1))]}" = "# (in test file $RELATIVE_FIXTURE_ROOT/external_function_calls.bats, line $currentErrorLine)" ] + outputOffset=$((outputOffset + 3)) + ;; + 2) + [[ "${lines[$((outputOffset + 1))]}" =~ ^'# (from function `'.*\'' in file '.*'/test_helper.bash, line '[0-9]+,$ ]] + [ "${lines[$((outputOffset + 2))]}" = "# in test file $RELATIVE_FIXTURE_ROOT/external_function_calls.bats, line $currentErrorLine)" ] + outputOffset=$((outputOffset + 4)) + ;; + *) + printf 'error: stackdepth=%s not implemented\n' "${stackdepth}" >&2 + return 1 + ;; esac currentErrorLine=$((currentErrorLine + linesPerTest)) done @@ -597,25 +560,25 @@ END_OF_ERR_MSG # shellcheck source=lib/bats-core/validator.bash source "$BATS_ROOT/lib/bats-core/validator.bash" export -f bats_test_count_validator - run bash -c "echo $'1..1\n' | bats_test_count_validator" + reentrant_run bash -c "echo $'1..1\n' | bats_test_count_validator" [[ $status -ne 0 ]] - run bash -c "echo $'1..1\nok 1\nok 2' | bats_test_count_validator" + reentrant_run bash -c "echo $'1..1\nok 1\nok 2' | bats_test_count_validator" [[ $status -ne 0 ]] - run bash -c "echo $'1..1\nok 1' | bats_test_count_validator" + reentrant_run bash -c "echo $'1..1\nok 1' | bats_test_count_validator" [[ $status -eq 0 ]] } @test "running the same file twice runs its tests twice without errors" { - run bats "$FIXTURE_ROOT/passing.bats" "$FIXTURE_ROOT/passing.bats" + reentrant_run bats "$FIXTURE_ROOT/passing.bats" "$FIXTURE_ROOT/passing.bats" echo "$output" [[ $status -eq 0 ]] [[ "${lines[0]}" == "1..2" ]] # got 2x1 tests } @test "Don't use unbound variables inside bats (issue #340)" { - run bats "$FIXTURE_ROOT/set_-eu_in_setup_and_teardown.bats" + reentrant_run bats "$FIXTURE_ROOT/set_-eu_in_setup_and_teardown.bats" echo "$output" [[ "${lines[0]}" == "1..4" ]] [[ "${lines[1]}" == "ok 1 skipped test # skip" ]] @@ -637,7 +600,7 @@ END_OF_ERR_MSG @test "each file is evaluated n+1 times" { # shellcheck disable=SC2031,SC2030 export TEMPFILE="$BATS_TEST_TMPDIR/$BATS_TEST_NAME.log" - run bats "$FIXTURE_ROOT/evaluation_count/" + reentrant_run bats "$FIXTURE_ROOT/evaluation_count/" cat "$TEMPFILE" @@ -649,7 +612,7 @@ END_OF_ERR_MSG } @test "Don't hang on CTRL-C (issue #353)" { - if [[ "$BATS_NUMBER_OF_PARALLEL_JOBS" -gt 1 ]]; then + if [[ "${BATS_NUMBER_OF_PARALLEL_JOBS:-1}" -gt 1 ]]; then skip "Aborts don't work in parallel mode" fi load 'concurrent-coordination' @@ -670,14 +633,17 @@ END_OF_ERR_MSG # when the process is gone, we cannot deliver a signal anymore, getting non-zero from kill run kill -0 -- -$SUBPROCESS_PID - [[ $status -ne 0 ]] \ - || (kill -9 -- -$SUBPROCESS_PID; false) - # ^ kill the process for good when SIGINT failed, - # to avoid waiting endlessly for stuck children to finish + [[ $status -ne 0 ]] || + ( + kill -9 -- -$SUBPROCESS_PID + false + ) + # ^ kill the process for good when SIGINT failed, + # to avoid waiting endlessly for stuck children to finish } @test "test comment style" { - run bats "$FIXTURE_ROOT/comment_style.bats" + reentrant_run bats "$FIXTURE_ROOT/comment_style.bats" [ $status -eq 0 ] [ "${lines[0]}" = '1..6' ] [ "${lines[1]}" = 'ok 1 should_be_found' ] @@ -689,7 +655,7 @@ END_OF_ERR_MSG } @test "test works even if PATH is reset" { - run bats "$FIXTURE_ROOT/update_path_env.bats" + reentrant_run bats "$FIXTURE_ROOT/update_path_env.bats" [ "$status" -eq 1 ] [ "${lines[4]}" = "# /usr/local/bin:/usr/bin:/bin" ] } @@ -697,7 +663,7 @@ END_OF_ERR_MSG @test "Test nounset does not trip up bats' internals (see #385)" { # don't export nounset within this file or we might trip up the testsuite itself, # getting bad diagnostics - run bash -c "set -o nounset; export SHELLOPTS; bats --tap '$FIXTURE_ROOT/passing.bats'" + reentrant_run bash -c "set -o nounset; export SHELLOPTS; bats --tap '$FIXTURE_ROOT/passing.bats'" echo "$output" [ "${lines[0]}" = "1..1" ] [ "${lines[1]}" = "ok 1 a passing test" ] @@ -721,38 +687,10 @@ END_OF_ERR_MSG [ "$(find "$TEST_TMPDIR" -name '*.src' | wc -l)" -eq 1 ] } -@test "All formatters (except cat) implement the callback interface" { - cd "$BATS_ROOT/libexec/bats-core/" - for formatter in bats-format-*; do - # the cat formatter is not expected to implement this interface - if [[ "$formatter" == *"bats-format-cat" ]]; then - continue - fi - tested_at_least_one_formatter=1 - echo "Formatter: ${formatter}" - # the replay should be possible without errors - bash -u "$formatter" >/dev/null </dev/null || skip "--jobs requires GNU parallel" (type -p flock &>/dev/null || type -p shlock &>/dev/null) || skip "--jobs requires flock/shlock" - run bats -j 2 "$FIXTURE_ROOT/issue-433" + reentrant_run bats -j 2 "$FIXTURE_ROOT/issue-433" [ "$status" -eq 0 ] [[ "$output" != *"No such file or directory"* ]] || exit 1 # ensure failures are detected with old bash } @test "Failure in free code (see #399)" { - run bats --tap "$FIXTURE_ROOT/failure_in_free_code.bats" + reentrant_run bats --tap "$FIXTURE_ROOT/failure_in_free_code.bats" echo "$output" [ "$status" -ne 0 ] [ "${lines[0]}" == 1..1 ] [ "${lines[1]}" == 'not ok 1 setup_file failed' ] - [ "${lines[2]}" == "# (from function \`helper' in file $RELATIVE_FIXTURE_ROOT/failure_in_free_code.bats, line 4," ] - [ "${lines[3]}" == "# in test file $RELATIVE_FIXTURE_ROOT/failure_in_free_code.bats, line 7)" ] + [ "${lines[2]}" == "# (from function \`helper' in file $RELATIVE_FIXTURE_ROOT/failure_in_free_code.bats, line 2," ] + [ "${lines[3]}" == "# in test file $RELATIVE_FIXTURE_ROOT/failure_in_free_code.bats, line 5)" ] [ "${lines[4]}" == "# \`helper' failed" ] } @test "CTRL-C aborts and fails the current test" { - if [[ "$BATS_NUMBER_OF_PARALLEL_JOBS" -gt 1 ]]; then + if [[ "${BATS_NUMBER_OF_PARALLEL_JOBS:-1}" -gt 1 ]]; then skip "Aborts don't work in parallel mode" fi - + # shellcheck disable=SC2031,SC2030 export TEMPFILE="$BATS_TEST_TMPDIR/$BATS_TEST_NAME.log" # guarantee that background processes get their own process group -> pid=pgid set -m - + load 'concurrent-coordination' # shellcheck disable=SC2031,SC2030 export SINGLE_USE_LATCH_DIR="${BATS_SUITE_TMPDIR}" # we cannot use run for a background task, so we have to store the output for later - bats "$FIXTURE_ROOT/hang_in_test.bats" --tap >"$TEMPFILE" 2>&1 & # don't block execution, or we cannot send signals + bats "$FIXTURE_ROOT/hang_in_test.bats" --tap >"$TEMPFILE" 2>&1 & # don't block execution, or we cannot send signals SUBPROCESS_PID=$! - single-use-latch::wait hang_in_test 1 10 || (cat "$TEMPFILE"; false) # still forward output on timeout + single-use-latch::wait hang_in_test 1 10 || ( + cat "$TEMPFILE" + false + ) # still forward output on timeout # emulate CTRL-C by sending SIGINT to the whole process group - kill -SIGINT -- -$SUBPROCESS_PID || (cat "$TEMPFILE"; false) + kill -SIGINT -- -$SUBPROCESS_PID || ( + cat "$TEMPFILE" + false + ) # the test suite must be marked as failed! wait $SUBPROCESS_PID && return 1 @@ -837,12 +781,17 @@ EOF [ "${lines[1]}" == "not ok 1 test" ] # due to scheduling the exact line will vary but we should exit with 130 + [[ "${lines[2]}" == "# (in test file "*")" ]] || false [[ "${lines[3]}" == *"failed with status 130" ]] || false [ "${lines[4]}" == "# Received SIGINT, aborting ..." ] + [ ${#lines[@]} -eq 5 ] } @test "CTRL-C aborts and fails the current run" { - if [[ "$BATS_NUMBER_OF_PARALLEL_JOBS" -gt 1 ]]; then + # shellcheck disable=SC2034 + BATS_TEST_RETRIES=2 + + if [[ "${BATS_NUMBER_OF_PARALLEL_JOBS:-1}" -gt 1 ]]; then skip "Aborts don't work in parallel mode" fi @@ -851,25 +800,28 @@ EOF # guarantee that background processes get their own process group -> pid=pgid set -m - + load 'concurrent-coordination' # shellcheck disable=SC2031,SC2030 export SINGLE_USE_LATCH_DIR="${BATS_SUITE_TMPDIR}" # we cannot use run for a background task, so we have to store the output for later - bats "$FIXTURE_ROOT/hang_in_run.bats" --tap >"$TEMPFILE" 2>&1 & # don't block execution, or we cannot send signals + bats "$FIXTURE_ROOT/hang_in_run.bats" --tap >"$TEMPFILE" 2>&1 & # don't block execution, or we cannot send signals SUBPROCESS_PID=$! - + single-use-latch::wait hang_in_run 1 10 # emulate CTRL-C by sending SIGINT to the whole process group - kill -SIGINT -- -$SUBPROCESS_PID || (cat "$TEMPFILE"; false) + kill -SIGINT -- -$SUBPROCESS_PID || ( + cat "$TEMPFILE" + false + ) # the test suite must be marked as failed! wait $SUBPROCESS_PID && return 1 run cat "$TEMPFILE" - + [ "${lines[1]}" == "not ok 1 test" ] # due to scheduling the exact line will vary but we should exit with 130 [[ "${lines[3]}" == *"failed with status 130" ]] || false @@ -877,7 +829,9 @@ EOF } @test "CTRL-C aborts and fails after run" { - if [[ "$BATS_NUMBER_OF_PARALLEL_JOBS" -gt 1 ]]; then + # shellcheck disable=SC2034 + BATS_TEST_RETRIES=2 + if [[ "${BATS_NUMBER_OF_PARALLEL_JOBS:-1}" -gt 1 ]]; then skip "Aborts don't work in parallel mode" fi @@ -886,25 +840,28 @@ EOF # guarantee that background processes get their own process group -> pid=pgid set -m - + load 'concurrent-coordination' # shellcheck disable=SC2031,SC2030 export SINGLE_USE_LATCH_DIR="${BATS_SUITE_TMPDIR}" # we cannot use run for a background task, so we have to store the output for later - bats "$FIXTURE_ROOT/hang_after_run.bats" --tap >"$TEMPFILE" 2>&1 & # don't block execution, or we cannot send signals + bats "$FIXTURE_ROOT/hang_after_run.bats" --tap >"$TEMPFILE" 2>&1 & # don't block execution, or we cannot send signals SUBPROCESS_PID=$! - + single-use-latch::wait hang_after_run 1 10 # emulate CTRL-C by sending SIGINT to the whole process group - kill -SIGINT -- -$SUBPROCESS_PID || (cat "$TEMPFILE"; false) + kill -SIGINT -- -$SUBPROCESS_PID || ( + cat "$TEMPFILE" + false + ) # the test suite must be marked as failed! wait $SUBPROCESS_PID && return 1 run cat "$TEMPFILE" - + [ "${lines[1]}" == "not ok 1 test" ] # due to scheduling the exact line will vary but we should exit with 130 [[ "${lines[3]}" == *"failed with status 130" ]] || false @@ -912,7 +869,7 @@ EOF } @test "CTRL-C aborts and fails the current teardown" { - if [[ "$BATS_NUMBER_OF_PARALLEL_JOBS" -gt 1 ]]; then + if [[ "${BATS_NUMBER_OF_PARALLEL_JOBS:-1}" -gt 1 ]]; then skip "Aborts don't work in parallel mode" fi @@ -921,19 +878,22 @@ EOF # guarantee that background processes get their own process group -> pid=pgid set -m - + load 'concurrent-coordination' # shellcheck disable=SC2031,SC2030 export SINGLE_USE_LATCH_DIR="${BATS_SUITE_TMPDIR}" # we cannot use run for a background task, so we have to store the output for later - bats "$FIXTURE_ROOT/hang_in_teardown.bats" --tap >"$TEMPFILE" 2>&1 & # don't block execution, or we cannot send signals + bats "$FIXTURE_ROOT/hang_in_teardown.bats" --tap >"$TEMPFILE" 2>&1 & # don't block execution, or we cannot send signals SUBPROCESS_PID=$! - + single-use-latch::wait hang_in_teardown 1 10 # emulate CTRL-C by sending SIGINT to the whole process group - kill -SIGINT -- -$SUBPROCESS_PID || (cat "$TEMPFILE"; false) + kill -SIGINT -- -$SUBPROCESS_PID || ( + cat "$TEMPFILE" + false + ) # the test suite must be marked as failed! wait $SUBPROCESS_PID && return 1 @@ -948,7 +908,7 @@ EOF } @test "CTRL-C aborts and fails the current setup_file" { - if [[ "$BATS_NUMBER_OF_PARALLEL_JOBS" -gt 1 ]]; then + if [[ "${BATS_NUMBER_OF_PARALLEL_JOBS:-1}" -gt 1 ]]; then skip "Aborts don't work in parallel mode" fi @@ -957,19 +917,22 @@ EOF # guarantee that background processes get their own process group -> pid=pgid set -m - + load 'concurrent-coordination' # shellcheck disable=SC2031,SC2030 export SINGLE_USE_LATCH_DIR="${BATS_SUITE_TMPDIR}" # we cannot use run for a background task, so we have to store the output for later - bats "$FIXTURE_ROOT/hang_in_setup_file.bats" --tap >"$TEMPFILE" 2>&1 & # don't block execution, or we cannot send signals + bats "$FIXTURE_ROOT/hang_in_setup_file.bats" --tap >"$TEMPFILE" 2>&1 & # don't block execution, or we cannot send signals SUBPROCESS_PID=$! - + single-use-latch::wait hang_in_setup_file 1 10 # emulate CTRL-C by sending SIGINT to the whole process group - kill -SIGINT -- -$SUBPROCESS_PID || (cat "$TEMPFILE"; false) + kill -SIGINT -- -$SUBPROCESS_PID || ( + cat "$TEMPFILE" + false + ) # the test suite must be marked as failed! wait $SUBPROCESS_PID && return 1 @@ -984,7 +947,7 @@ EOF } @test "CTRL-C aborts and fails the current teardown_file" { - if [[ "$BATS_NUMBER_OF_PARALLEL_JOBS" -gt 1 ]]; then + if [[ "${BATS_NUMBER_OF_PARALLEL_JOBS:-1}" -gt 1 ]]; then skip "Aborts don't work in parallel mode" fi # shellcheck disable=SC2031 @@ -992,19 +955,22 @@ EOF # guarantee that background processes get their own process group -> pid=pgid set -m - + load 'concurrent-coordination' # shellcheck disable=SC2031 export SINGLE_USE_LATCH_DIR="${BATS_SUITE_TMPDIR}" # we cannot use run for a background task, so we have to store the output for later - bats "$FIXTURE_ROOT/hang_in_teardown_file.bats" --tap >"$TEMPFILE" 2>&1 & # don't block execution, or we cannot send signals + bats "$FIXTURE_ROOT/hang_in_teardown_file.bats" --tap >"$TEMPFILE" 2>&1 & # don't block execution, or we cannot send signals SUBPROCESS_PID=$! - + single-use-latch::wait hang_in_teardown_file 1 10 # emulate CTRL-C by sending SIGINT to the whole process group - kill -SIGINT -- -$SUBPROCESS_PID || (cat "$TEMPFILE"; false) + kill -SIGINT -- -$SUBPROCESS_PID || ( + cat "$TEMPFILE" + false + ) # the test suite must be marked as failed! wait $SUBPROCESS_PID && return 1 @@ -1021,17 +987,16 @@ EOF [ "${lines[6]}" == "# bats warning: Executed 2 instead of expected 1 tests" ] } - @test "single star in output is not treated as a glob" { - star(){ echo '*'; } - + star() { echo '*'; } + run star [ "${lines[0]}" = '*' ] } @test "multiple stars in output are not treated as a glob" { - stars(){ echo '**'; } - + stars() { echo '**'; } + run stars [ "${lines[0]}" = '**' ] } @@ -1044,13 +1009,13 @@ EOF run "./shellcheck.sh" --list echo "$output" - grep bin/bats <<< "$output" - grep contrib/ <<< "$output" - grep docker/ <<< "$output" - grep lib/bats-core/ <<< "$output" - grep libexec/bats-core/ <<< "$output" - grep test/fixtures <<< "$output" - grep install.sh <<< "$output" + grep bin/bats <<<"$output" + grep contrib/ <<<"$output" + grep docker/ <<<"$output" + grep lib/bats-core/ <<<"$output" + grep libexec/bats-core/ <<<"$output" + grep test/fixtures <<<"$output" + grep install.sh <<<"$output" } @test "BATS_RUN_COMMAND: test content of variable" { @@ -1061,26 +1026,8 @@ EOF [[ "$BATS_RUN_COMMAND" == "bats BATS_RUN_COMMAND: test content of variable" ]] } -@test "pretty formatter summary is colorized red on failure" { - bats_require_minimum_version 1.5.0 - run -1 bats --pretty "$FIXTURE_ROOT/failing.bats" - - [ "${lines[4]}" == $'\033[0m\033[31;1m' ] # TODO: avoid checking for the leading reset too - [ "${lines[5]}" == '1 test, 1 failure' ] - [ "${lines[6]}" == $'\033[0m' ] -} - -@test "pretty formatter summary is colorized green on success" { - bats_require_minimum_version 1.5.0 - run -0 bats --pretty "$FIXTURE_ROOT/passing.bats" - - [ "${lines[2]}" == $'\033[0m\033[32;1m' ] # TODO: avoid checking for the leading reset too - [ "${lines[3]}" == '1 test, 0 failures' ] - [ "${lines[4]}" == $'\033[0m' ] -} - @test "--print-output-on-failure works as expected" { - run bats --print-output-on-failure --show-output-of-passing-tests "$FIXTURE_ROOT/print_output_on_failure.bats" + reentrant_run bats --print-output-on-failure --show-output-of-passing-tests "$FIXTURE_ROOT/print_output_on_failure.bats" [ "${lines[0]}" == '1..3' ] [ "${lines[1]}" == 'ok 1 no failure prints no output' ] # ^ no output despite --show-output-of-passing-tests, because there is no failure @@ -1095,9 +1042,27 @@ EOF [ ${#lines[@]} -eq 10 ] } +@test "--print-output-on-failure also shows stderr (for run --separate-stderr)" { + reentrant_run bats --print-output-on-failure --show-output-of-passing-tests "$FIXTURE_ROOT/print_output_on_failure_with_stderr.bats" + [ "${lines[0]}" == '1..3' ] + [ "${lines[1]}" == 'ok 1 no failure prints no output' ] + # ^ no output despite --show-output-of-passing-tests, because there is no failure + [ "${lines[2]}" == 'not ok 2 failure prints output' ] + [ "${lines[3]}" == "# (in test file $RELATIVE_FIXTURE_ROOT/print_output_on_failure_with_stderr.bats, line 7)" ] + [ "${lines[4]}" == "# \`run -1 --separate-stderr bash -c 'echo \"fail hard\"; echo with stderr >&2'' failed, expected exit code 1, got 0" ] + [ "${lines[5]}" == '# Last output:' ] + [ "${lines[6]}" == '# fail hard' ] + [ "${lines[7]}" == '# Last stderr:' ] + [ "${lines[8]}" == '# with stderr' ] + [ "${lines[9]}" == 'not ok 3 empty output on failure' ] + [ "${lines[10]}" == "# (in test file $RELATIVE_FIXTURE_ROOT/print_output_on_failure_with_stderr.bats, line 11)" ] + [ "${lines[11]}" == "# \`false' failed" ] + [ ${#lines[@]} -eq 12 ] +} + @test "--show-output-of-passing-tests works as expected" { bats_require_minimum_version 1.5.0 - run -0 bats --show-output-of-passing-tests "$FIXTURE_ROOT/show-output-of-passing-tests.bats" + reentrant_run -0 bats --show-output-of-passing-tests "$FIXTURE_ROOT/show-output-of-passing-tests.bats" [ "${lines[0]}" == '1..1' ] [ "${lines[1]}" == 'ok 1 test' ] [ "${lines[2]}" == '# output' ] @@ -1106,7 +1071,7 @@ EOF @test "--verbose-run prints output" { bats_require_minimum_version 1.5.0 - run -1 bats --verbose-run "$FIXTURE_ROOT/verbose-run.bats" + reentrant_run -1 bats --verbose-run "$FIXTURE_ROOT/verbose-run.bats" [ "${lines[0]}" == '1..1' ] [ "${lines[1]}" == 'not ok 1 test' ] [ "${lines[2]}" == "# (in test file $RELATIVE_FIXTURE_ROOT/verbose-run.bats, line 3)" ] @@ -1117,7 +1082,7 @@ EOF @test "BATS_VERBOSE_RUN=1 also prints output" { bats_require_minimum_version 1.5.0 - run -1 env BATS_VERBOSE_RUN=1 bats "$FIXTURE_ROOT/verbose-run.bats" + reentrant_run -1 env BATS_VERBOSE_RUN=1 bats "$FIXTURE_ROOT/verbose-run.bats" [ "${lines[0]}" == '1..1' ] [ "${lines[1]}" == 'not ok 1 test' ] [ "${lines[2]}" == "# (in test file $RELATIVE_FIXTURE_ROOT/verbose-run.bats, line 3)" ] @@ -1128,14 +1093,14 @@ EOF @test "--gather-test-outputs-in gathers outputs of all tests (even succeeding!)" { local OUTPUT_DIR="$BATS_TEST_TMPDIR/logs" - run bats --verbose-run --gather-test-outputs-in "$OUTPUT_DIR" "$FIXTURE_ROOT/print_output_on_failure.bats" + reentrant_run bats --verbose-run --gather-test-outputs-in "$OUTPUT_DIR" "$FIXTURE_ROOT/print_output_on_failure.bats" [ -d "$OUTPUT_DIR" ] # will be generated! # even outputs of successful tests are generated OUTPUT=$(<"$OUTPUT_DIR/1-no failure prints no output.log") # own line to trigger failure if file does not exist - [ "$OUTPUT" == "success" ] - + [ "$OUTPUT" == "success" ] + OUTPUT=$(<"$OUTPUT_DIR/2-failure prints output.log") [ "$OUTPUT" == "fail hard" ] @@ -1146,6 +1111,22 @@ EOF [ "$(find "$OUTPUT_DIR" -type f | wc -l)" -eq 3 ] } +@test "--gather-test-outputs-in allows directory to exist (only if empty)" { + local OUTPUT_DIR="$BATS_TEST_TMPDIR/logs" + bats_require_minimum_version 1.5.0 + + # anything existing, even if empty, 'hidden', etc. should cause failure + mkdir "$OUTPUT_DIR" && touch "$OUTPUT_DIR/.oops" + reentrant_run -1 bats --verbose-run --gather-test-outputs-in "$OUTPUT_DIR" "$FIXTURE_ROOT/passing.bats" + [ "${lines[0]}" == "Error: Directory '$OUTPUT_DIR' must be empty for --gather-test-outputs-in" ] + + # empty directory is just fine + rm "$OUTPUT_DIR/.oops" && rmdir "$OUTPUT_DIR" # avoiding rm -fr to avoid goofs + mkdir "$OUTPUT_DIR" + reentrant_run -0 bats --verbose-run --gather-test-outputs-in "$OUTPUT_DIR" "$FIXTURE_ROOT/passing.bats" + [ "$(find "$OUTPUT_DIR" -type f | wc -l)" -eq 1 ] +} + @test "Tell about missing flock and shlock" { if ! command -v parallel; then skip "this test requires GNU parallel to be installed" @@ -1158,7 +1139,7 @@ EOF fi bats_require_minimum_version 1.5.0 - run ! bats --jobs 2 "$FIXTURE_ROOT/parallel.bats" + reentrant_run ! bats --jobs 2 "$FIXTURE_ROOT/parallel.bats" [ "${lines[0]}" == "ERROR: flock/shlock is required for parallelization within files!" ] [ "${#lines[@]}" -eq 1 ] } @@ -1169,11 +1150,13 @@ EOF @test "BATS_CODE_QUOTE_STYLE works with any two characters (even unicode)" { bats_require_minimum_version 1.5.0 - BATS_CODE_QUOTE_STYLE='``' run -1 bats --tap "${FIXTURE_ROOT}/failing.bats" + + # shellcheck disable=SC2030,SC2031 + REENTRANT_RUN_PRESERVE+=(BATS_CODE_QUOTE_STYLE) + BATS_CODE_QUOTE_STYLE='``' reentrant_run -1 bats --tap "${FIXTURE_ROOT}/failing.bats" # shellcheck disable=SC2016 [ "${lines[3]}" == '# `eval "( exit ${STATUS:-1} )"` failed' ] - export BATS_CODE_QUOTE_STYLE='😁😂' if [[ ${#BATS_CODE_QUOTE_STYLE} -ne 2 ]]; then # for example, this happens on windows! @@ -1181,7 +1164,7 @@ EOF fi bats_require_minimum_version 1.5.0 - run -1 bats --tap "${FIXTURE_ROOT}/failing.bats" + reentrant_run -1 bats --tap "${FIXTURE_ROOT}/failing.bats" # shellcheck disable=SC2016 [ "${lines[3]}" == '# 😁eval "( exit ${STATUS:-1} )"😂 failed' ] } @@ -1192,31 +1175,38 @@ EOF bats_require_minimum_version 1.5.0 - BATS_CODE_QUOTE_STYLE=custom run -1 bats --tap "${FIXTURE_ROOT}/passing.bats" + # shellcheck disable=SC2030,SC2031 + REENTRANT_RUN_PRESERVE+=(BATS_CODE_QUOTE_STYLE) + BATS_CODE_QUOTE_STYLE=custom reentrant_run -1 bats --tap "${FIXTURE_ROOT}/passing.bats" [ "${lines[0]}" == 'ERROR: BATS_CODE_QUOTE_STYLE=custom requires BATS_BEGIN_CODE_QUOTE and BATS_END_CODE_QUOTE to be set' ] + REENTRANT_RUN_PRESERVE+=(BATS_BEGIN_CODE_QUOTE BATS_END_CODE_QUOTE) # shellcheck disable=SC2016 BATS_CODE_QUOTE_STYLE=custom \ - BATS_BEGIN_CODE_QUOTE='$(' \ - BATS_END_CODE_QUOTE=')' \ - run -1 bats --tap "${FIXTURE_ROOT}/failing.bats" + BATS_BEGIN_CODE_QUOTE='$(' \ + BATS_END_CODE_QUOTE=')' \ + reentrant_run -1 bats --tap "${FIXTURE_ROOT}/failing.bats" # shellcheck disable=SC2016 [ "${lines[3]}" == '# $(eval "( exit ${STATUS:-1} )") failed' ] } @test "Warn about invalid BATS_CODE_QUOTE_STYLE" { bats_require_minimum_version 1.5.0 - BATS_CODE_QUOTE_STYLE='' run -1 bats --tap "${FIXTURE_ROOT}/passing.bats" + + # shellcheck disable=SC2030,SC2031 + REENTRANT_RUN_PRESERVE+=(BATS_CODE_QUOTE_STYLE) + + BATS_CODE_QUOTE_STYLE='' reentrant_run -1 bats --tap "${FIXTURE_ROOT}/passing.bats" [ "${lines[0]}" == 'ERROR: Unknown BATS_CODE_QUOTE_STYLE: ' ] - BATS_CODE_QUOTE_STYLE='1' run -1 bats --tap "${FIXTURE_ROOT}/passing.bats" + BATS_CODE_QUOTE_STYLE='1' reentrant_run -1 bats --tap "${FIXTURE_ROOT}/passing.bats" [ "${lines[0]}" == 'ERROR: Unknown BATS_CODE_QUOTE_STYLE: 1' ] - BATS_CODE_QUOTE_STYLE='three' run -1 bats --tap "${FIXTURE_ROOT}/passing.bats" + BATS_CODE_QUOTE_STYLE='three' reentrant_run -1 bats --tap "${FIXTURE_ROOT}/passing.bats" [ "${lines[0]}" == 'ERROR: Unknown BATS_CODE_QUOTE_STYLE: three' ] } -@test "Debug trap must only override variables that are prefixed with bats_ (issue #519)" { +@test "Debug trap must only override variables that are prefixed with BATS_ (issue #519)" { # use declare -p to gather variables in pristine bash and bats @test environment # then compare which ones are introduced in @test compared to bash @@ -1241,9 +1231,9 @@ EOF normalize_variable_list() { # `declare -p`: declare -X VAR_NAME="VALUE" while IFS=' =' read -r _declare _ variable _; do - if [[ "$_declare" == declare ]]; then # skip multiline variables' values - printf "%s\n" "$variable" - fi + if [[ "$_declare" == declare ]]; then # skip multiline variables' values + printf "%s\n" "$variable" + fi done | sort } fi @@ -1254,55 +1244,250 @@ EOF local BATS_DECLARED_VARIABLES_FILE="${BATS_TEST_TMPDIR}/variables.log" bats_require_minimum_version 1.5.0 # now capture bats @test environment - run -0 env -i PATH="$PATH" BATS_DECLARED_VARIABLES_FILE="$BATS_DECLARED_VARIABLES_FILE" bash "${BATS_ROOT}/bin/bats" "${FIXTURE_ROOT}/issue-519.bats" - # use function to allow failing via !, run is a bit unwiedly with the pipe and subshells + reentrant_run -0 env -i PATH="$PATH" BATS_DECLARED_VARIABLES_FILE="$BATS_DECLARED_VARIABLES_FILE" bash "${BATS_ROOT}/bin/bats" "${FIXTURE_ROOT}/issue-519.bats" + # use function to allow failing via !, run is a bit unwieldy with the pipe and subshells check_no_new_variables() { # -23 -> only look at additions on the bats list ! comm -23 <(normalize_variable_list <"$BATS_DECLARED_VARIABLES_FILE") \ - <(normalize_variable_list <<< "$BASH_DECLARED_VARIABLES" ) \ - | grep -v '^BATS_' # variables that are prefixed with BATS_ don't count + <(normalize_variable_list <<<"$BASH_DECLARED_VARIABLES") | + grep -v '^BATS_' # variables that are prefixed with BATS_ don't count } check_no_new_variables } @test "Don't wait for disowned background jobs to finish because of open FDs (#205)" { - SECONDS=0 - export LOG_FILE="$BATS_TEST_TMPDIR/fds.log" - bats_require_minimum_version 1.5.0 - run -0 bats --show-output-of-passing-tests --tap "${FIXTURE_ROOT}/issue-205.bats" - echo "Whole suite took: $SECONDS seconds" - FDS_LOG=$(<"$LOG_FILE") - echo "$FDS_LOG" - [ $SECONDS -lt 10 ] - [[ $FDS_LOG == *'otherfunc fds after: (0 1 2)'* ]] || false - [[ $FDS_LOG == *'setup_file fds after: (0 1 2)'* ]] || false + SECONDS=0 + export LOG_FILE="$BATS_TEST_TMPDIR/fds.log" + bats_require_minimum_version 1.5.0 + reentrant_run -0 bats --show-output-of-passing-tests --tap "${FIXTURE_ROOT}/issue-205.bats" + echo "Whole suite took: $SECONDS seconds" + FDS_LOG=$(<"$LOG_FILE") + echo "$FDS_LOG" + [ $SECONDS -lt 10 ] + [[ $FDS_LOG == *'otherfunc fds after: (0 1 2)'* ]] || false + [[ $FDS_LOG == *'setup_file fds after: (0 1 2)'* ]] || false } @test "Allow for prefixing tests' names with BATS_TEST_NAME_PREFIX" { - BATS_TEST_NAME_PREFIX='PREFIX: ' run bats "${FIXTURE_ROOT}/passing.bats" + # shellcheck disable=SC2030,SC2031 + REENTRANT_RUN_PRESERVE+=(BATS_TEST_NAME_PREFIX) + BATS_TEST_NAME_PREFIX='PREFIX: ' reentrant_run bats "${FIXTURE_ROOT}/passing.bats" [ "${lines[1]}" == "ok 1 PREFIX: a passing test" ] } @test "Setting status in teardown* does not override exit code (see issue #575)" { bats_require_minimum_version 1.5.0 - TEARDOWN_RETURN_CODE=0 TEST_RETURN_CODE=0 STATUS=0 run -0 bats "$FIXTURE_ROOT/teardown_override_status.bats" - TEARDOWN_RETURN_CODE=1 TEST_RETURN_CODE=0 STATUS=0 run -1 bats "$FIXTURE_ROOT/teardown_override_status.bats" - TEARDOWN_RETURN_CODE=0 TEST_RETURN_CODE=1 STATUS=0 run -1 bats "$FIXTURE_ROOT/teardown_override_status.bats" - TEARDOWN_RETURN_CODE=1 TEST_RETURN_CODE=1 STATUS=0 run -1 bats "$FIXTURE_ROOT/teardown_override_status.bats" - TEARDOWN_RETURN_CODE=0 TEST_RETURN_CODE=0 STATUS=1 run -0 bats "$FIXTURE_ROOT/teardown_override_status.bats" - TEARDOWN_RETURN_CODE=1 TEST_RETURN_CODE=0 STATUS=1 run -1 bats "$FIXTURE_ROOT/teardown_override_status.bats" - - TEARDOWN_RETURN_CODE=0 TEST_RETURN_CODE=0 STATUS=0 run -0 bats "$FIXTURE_ROOT/teardown_file_override_status.bats" - TEARDOWN_RETURN_CODE=1 TEST_RETURN_CODE=0 STATUS=0 run -1 bats "$FIXTURE_ROOT/teardown_file_override_status.bats" - TEARDOWN_RETURN_CODE=0 TEST_RETURN_CODE=1 STATUS=0 run -1 bats "$FIXTURE_ROOT/teardown_file_override_status.bats" - TEARDOWN_RETURN_CODE=1 TEST_RETURN_CODE=1 STATUS=0 run -1 bats "$FIXTURE_ROOT/teardown_file_override_status.bats" - TEARDOWN_RETURN_CODE=0 TEST_RETURN_CODE=0 STATUS=1 run -0 bats "$FIXTURE_ROOT/teardown_file_override_status.bats" - TEARDOWN_RETURN_CODE=1 TEST_RETURN_CODE=0 STATUS=1 run -1 bats "$FIXTURE_ROOT/teardown_file_override_status.bats" - - TEARDOWN_RETURN_CODE=0 TEST_RETURN_CODE=0 STATUS=0 run -0 bats "$FIXTURE_ROOT/teardown_suite_override_status/" - TEARDOWN_RETURN_CODE=1 TEST_RETURN_CODE=0 STATUS=0 run -1 bats "$FIXTURE_ROOT/teardown_suite_override_status/" - TEARDOWN_RETURN_CODE=0 TEST_RETURN_CODE=1 STATUS=0 run -1 bats "$FIXTURE_ROOT/teardown_suite_override_status/" - TEARDOWN_RETURN_CODE=1 TEST_RETURN_CODE=1 STATUS=0 run -1 bats "$FIXTURE_ROOT/teardown_suite_override_status/" - TEARDOWN_RETURN_CODE=0 TEST_RETURN_CODE=0 STATUS=1 run -0 bats "$FIXTURE_ROOT/teardown_suite_override_status/" - TEARDOWN_RETURN_CODE=1 TEST_RETURN_CODE=0 STATUS=1 run -1 bats "$FIXTURE_ROOT/teardown_suite_override_status/" -} \ No newline at end of file + TEARDOWN_RETURN_CODE=0 TEST_RETURN_CODE=0 STATUS=0 reentrant_run -0 bats "$FIXTURE_ROOT/teardown_override_status.bats" + TEARDOWN_RETURN_CODE=1 TEST_RETURN_CODE=0 STATUS=0 reentrant_run -1 bats "$FIXTURE_ROOT/teardown_override_status.bats" + TEARDOWN_RETURN_CODE=0 TEST_RETURN_CODE=1 STATUS=0 reentrant_run -1 bats "$FIXTURE_ROOT/teardown_override_status.bats" + TEARDOWN_RETURN_CODE=1 TEST_RETURN_CODE=1 STATUS=0 reentrant_run -1 bats "$FIXTURE_ROOT/teardown_override_status.bats" + TEARDOWN_RETURN_CODE=0 TEST_RETURN_CODE=0 STATUS=1 reentrant_run -0 bats "$FIXTURE_ROOT/teardown_override_status.bats" + TEARDOWN_RETURN_CODE=1 TEST_RETURN_CODE=0 STATUS=1 reentrant_run -1 bats "$FIXTURE_ROOT/teardown_override_status.bats" + + TEARDOWN_RETURN_CODE=0 TEST_RETURN_CODE=0 STATUS=0 reentrant_run -0 bats "$FIXTURE_ROOT/teardown_file_override_status.bats" + TEARDOWN_RETURN_CODE=1 TEST_RETURN_CODE=0 STATUS=0 reentrant_run -1 bats "$FIXTURE_ROOT/teardown_file_override_status.bats" + TEARDOWN_RETURN_CODE=0 TEST_RETURN_CODE=1 STATUS=0 reentrant_run -1 bats "$FIXTURE_ROOT/teardown_file_override_status.bats" + TEARDOWN_RETURN_CODE=1 TEST_RETURN_CODE=1 STATUS=0 reentrant_run -1 bats "$FIXTURE_ROOT/teardown_file_override_status.bats" + TEARDOWN_RETURN_CODE=0 TEST_RETURN_CODE=0 STATUS=1 reentrant_run -0 bats "$FIXTURE_ROOT/teardown_file_override_status.bats" + TEARDOWN_RETURN_CODE=1 TEST_RETURN_CODE=0 STATUS=1 reentrant_run -1 bats "$FIXTURE_ROOT/teardown_file_override_status.bats" + + TEARDOWN_RETURN_CODE=0 TEST_RETURN_CODE=0 STATUS=0 reentrant_run -0 bats "$FIXTURE_ROOT/teardown_suite_override_status/" + TEARDOWN_RETURN_CODE=1 TEST_RETURN_CODE=0 STATUS=0 reentrant_run -1 bats "$FIXTURE_ROOT/teardown_suite_override_status/" + TEARDOWN_RETURN_CODE=0 TEST_RETURN_CODE=1 STATUS=0 reentrant_run -1 bats "$FIXTURE_ROOT/teardown_suite_override_status/" + TEARDOWN_RETURN_CODE=1 TEST_RETURN_CODE=1 STATUS=0 reentrant_run -1 bats "$FIXTURE_ROOT/teardown_suite_override_status/" + TEARDOWN_RETURN_CODE=0 TEST_RETURN_CODE=0 STATUS=1 reentrant_run -0 bats "$FIXTURE_ROOT/teardown_suite_override_status/" + TEARDOWN_RETURN_CODE=1 TEST_RETURN_CODE=0 STATUS=1 reentrant_run -1 bats "$FIXTURE_ROOT/teardown_suite_override_status/" +} + +@test "BATS_* variables don't contain double slashes" { + TMPDIR=/tmp/ bats "$FIXTURE_ROOT/BATS_variables_dont_contain_double_slashes.bats" +} + +@test "Without .bats/run-logs --filter-status failed returns an error" { + bats_require_minimum_version 1.5.0 + reentrant_run -1 bats --filter-status failed "$FIXTURE_ROOT/passing_and_failing.bats" + [[ "${lines[0]}" == "Error: --filter-status needs '"*".bats/run-logs/' to save failed tests. Please create this folder, add it to .gitignore and try again." ]] || false +} + +@test "Without previous recording --filter-status failed runs all tests and then runs only failed and missed tests" { + cd "$BATS_TEST_TMPDIR" # don't pollute the source folder + cp "$FIXTURE_ROOT/many_passing_and_one_failing.bats" . + mkdir -p .bats/run-logs + bats_require_minimum_version 1.5.0 + reentrant_run -1 bats --filter-status failed "many_passing_and_one_failing.bats" + # without previous recording, all tests should be run + [ "${lines[0]}" == 'No recording of previous runs found. Running all tests!' ] + [ "${lines[1]}" == '1..4' ] + [ "$(grep -c 'not ok' <<<"$output")" -eq 1 ] + + reentrant_run -1 bats --tap --filter-status failed "many_passing_and_one_failing.bats" + # now we should only run the failing test + [ "${lines[0]}" == 1..1 ] + [ "${lines[1]}" == "not ok 1 a failing test" ] + + # add a new test that was missed before + echo $'@test missed { :; }' >>"many_passing_and_one_failing.bats" + + find .bats/run-logs/ -type f -print -exec cat {} \; + + reentrant_run -1 bats --tap --filter-status failed "many_passing_and_one_failing.bats" + # now we should only run the failing test + [ "${lines[0]}" == 1..2 ] + [ "${lines[1]}" == "not ok 1 a failing test" ] + [ "${lines[4]}" == "ok 2 missed" ] +} + +@test "Without previous recording --filter-status passed runs all tests and then runs only passed and missed tests" { + cd "$BATS_TEST_TMPDIR" # don't pollute the source folder + cp "$FIXTURE_ROOT/many_passing_and_one_failing.bats" . + mkdir -p .bats/run-logs + bats_require_minimum_version 1.5.0 + reentrant_run -1 bats --filter-status passed "many_passing_and_one_failing.bats" + # without previous recording, all tests should be run + [ "${lines[0]}" == 'No recording of previous runs found. Running all tests!' ] + [ "${lines[1]}" == '1..4' ] + [ "$(grep -c 'not ok' <<<"$output")" -eq 1 ] + + reentrant_run -0 bats --tap --filter-status passed "many_passing_and_one_failing.bats" + # now we should only run the passed tests + [ "${lines[0]}" == 1..3 ] + [ "$(grep -c 'not ok' <<<"$output")" -eq 0 ] + + # add a new test that was missed before + echo $'@test missed { :; }' >>"many_passing_and_one_failing.bats" + + reentrant_run -0 bats --tap --filter-status passed "many_passing_and_one_failing.bats" + # now we should only run the passed and missed tests + [ "${lines[0]}" == 1..4 ] + [ "$(grep -c 'not ok' <<<"$output")" -eq 0 ] + [ "${lines[4]}" == "ok 4 missed" ] +} + +@test "Without previous recording --filter-status missed runs all tests and then runs only missed tests" { + cd "$BATS_TEST_TMPDIR" # don't pollute the source folder + cp "$FIXTURE_ROOT/many_passing_and_one_failing.bats" . + mkdir -p .bats/run-logs + bats_require_minimum_version 1.5.0 + reentrant_run -1 bats --filter-status missed "many_passing_and_one_failing.bats" + # without previous recording, all tests should be run + [ "${lines[0]}" == 'No recording of previous runs found. Running all tests!' ] + [ "${lines[1]}" == '1..4' ] + [ "$(grep -c -E '^ok' <<<"$output")" -eq 3 ] + + # add a new test that was missed before + echo $'@test missed { :; }' >>"many_passing_and_one_failing.bats" + + reentrant_run -0 bats --tap --filter-status missed "many_passing_and_one_failing.bats" + # now we should only run the missed test + [ "${lines[0]}" == 1..1 ] + [ "${lines[1]}" == "ok 1 missed" ] +} + +@test "--filter-status failed gives warning on empty failed test list" { + cd "$BATS_TEST_TMPDIR" # don't pollute the source folder + cp "$FIXTURE_ROOT/passing.bats" . + mkdir -p .bats/run-logs + bats_require_minimum_version 1.5.0 + # have no failing tests + reentrant_run -0 bats --filter-status failed "passing.bats" + # try to run the empty list of failing tests + reentrant_run -0 bats --filter-status failed "passing.bats" + [ "${lines[0]}" == "There where no failed tests in the last recorded run." ] + [ "${lines[1]}" == "1..0" ] + [ "${#lines[@]}" -eq 2 ] +} + +enforce_own_process_group() { + set -m + "$@" +} + +@test "--filter-status failed does not update list when run is aborted" { + if [[ "${BATS_NUMBER_OF_PARALLEL_JOBS:-1}" -gt 1 ]]; then + skip "Aborts don't work in parallel mode" + fi + + cd "$BATS_TEST_TMPDIR" # don't pollute the source folder + cp "$FIXTURE_ROOT/sigint_in_failing_test.bats" . + mkdir -p .bats/run-logs + + bats_require_minimum_version 1.5.0 + # don't hang yet, so we get a useful rerun file + reentrant_run -1 env DONT_ABORT=1 bats "sigint_in_failing_test.bats" + + # check that we have exactly one log + reentrant_run find .bats/run-logs -name '*.log' + [[ "${lines[0]}" == *.log ]] || false + [ ${#lines[@]} -eq 1 ] + + local first_run_logs="$output" + + sleep 1 # ensure we would get different timestamps for each run + + # now rerun but abort midrun + reentrant_run -1 enforce_own_process_group bats --rerun-failed "sigint_in_failing_test.bats" + + # should not have produced a new log + reentrant_run find .bats/run-logs -name '*.log' + [ "$first_run_logs" == "$output" ] +} + +@test "BATS_TEST_RETRIES allows for retrying tests" { + # shellcheck disable=SC2030 + export LOG="$BATS_TEST_TMPDIR/call.log" + bats_require_minimum_version 1.5.0 + reentrant_run ! bats "$FIXTURE_ROOT/retry.bats" + [ "${lines[0]}" == '1..3' ] + [ "${lines[1]}" == 'not ok 1 Fail all' ] + [ "${lines[4]}" == 'ok 2 Fail once' ] + [ "${lines[5]}" == 'not ok 3 Override retries' ] + + run cat "$LOG" + [ "${lines[0]}" == ' setup_file ' ] # should only be executed once + [ "${lines[22]}" == ' teardown_file ' ] # should only be executed once + [ "${#lines[@]}" -eq 23 ] + + # 3x Fail All (give up after 3 tries/2 retries) + run grep test_Fail_all <"$LOG" + [ "${lines[0]}" == 'test_Fail_all setup 1' ] # should be executed for each try + [ "${lines[1]}" == 'test_Fail_all test_Fail_all 1' ] + [ "${lines[2]}" == 'test_Fail_all teardown 1' ] # should be executed for each try + [ "${lines[3]}" == 'test_Fail_all setup 2' ] + [ "${lines[4]}" == 'test_Fail_all test_Fail_all 2' ] + [ "${lines[5]}" == 'test_Fail_all teardown 2' ] + [ "${lines[6]}" == 'test_Fail_all setup 3' ] + [ "${lines[7]}" == 'test_Fail_all test_Fail_all 3' ] + [ "${lines[8]}" == 'test_Fail_all teardown 3' ] + [ "${#lines[@]}" -eq 9 ] + + # 2x Fail once (pass second try/first retry) + run grep test_Fail_once <"$LOG" + [ "${lines[0]}" == 'test_Fail_once setup 1' ] + [ "${lines[1]}" == 'test_Fail_once test_Fail_once 1' ] + [ "${lines[2]}" == 'test_Fail_once teardown 1' ] + [ "${lines[3]}" == 'test_Fail_once setup 2' ] + [ "${lines[4]}" == 'test_Fail_once test_Fail_once 2' ] + [ "${lines[5]}" == 'test_Fail_once teardown 2' ] + [ "${#lines[@]}" -eq 6 ] + + # 2x Override retries (give up after second try/first retry) + run grep test_Override_retries <"$LOG" + [ "${lines[0]}" == 'test_Override_retries setup 1' ] + [ "${lines[1]}" == 'test_Override_retries test_Override_retries 1' ] + [ "${lines[2]}" == 'test_Override_retries teardown 1' ] + [ "${lines[3]}" == 'test_Override_retries setup 2' ] + [ "${lines[4]}" == 'test_Override_retries test_Override_retries 2' ] + [ "${lines[5]}" == 'test_Override_retries teardown 2' ] + [ "${#lines[@]}" -eq 6 ] + +} + +@test "Exit code is zero after successful retry (see #660)" { + # shellcheck disable=SC2031 + export LOG="$BATS_TEST_TMPDIR/call.log" + bats_require_minimum_version 1.5.0 + reentrant_run -0 bats "$FIXTURE_ROOT/retry_success.bats" + [ "${lines[0]}" == '1..1' ] + [ "${lines[1]}" == 'ok 1 Fail once' ] + [ ${#lines[@]} == 2 ] +} diff --git a/test/common.bats b/test/common.bats index 7fa0f6834e..8d01a10740 100644 --- a/test/common.bats +++ b/test/common.bats @@ -1,41 +1,204 @@ - @test bats_version_lt { - bats_require_minimum_version 1.5.0 - run ! bats_version_lt 1.0.0 1.0 - [ "$output" = "ERROR: version '1.0' must be of format ..!" ] + bats_require_minimum_version 1.5.0 + run ! bats_version_lt 1.0.0 1.0 + [ "$output" = "ERROR: version '1.0' must be of format ..!" ] - run ! bats_version_lt 1.0 1.0.0 - [ "$output" = "ERROR: version '1.0' must be of format ..!" ] + run ! bats_version_lt 1.0 1.0.0 + [ "$output" = "ERROR: version '1.0' must be of format ..!" ] - - run -0 bats_version_lt 1.0.0 2.0.0 - run -0 bats_version_lt 1.2.0 2.0.0 - run -0 bats_version_lt 1.2.3 2.0.0 - run -0 bats_version_lt 1.0.0 1.1.0 - run -0 bats_version_lt 1.0.2 1.1.0 - run -0 bats_version_lt 1.0.0 1.0.1 + run -0 bats_version_lt 1.0.0 2.0.0 + run -0 bats_version_lt 1.2.0 2.0.0 + run -0 bats_version_lt 1.2.3 2.0.0 + run -0 bats_version_lt 1.0.0 1.1.0 + run -0 bats_version_lt 1.0.2 1.1.0 + run -0 bats_version_lt 1.0.0 1.0.1 - run -1 bats_version_lt 2.0.0 1.0.0 - run -1 bats_version_lt 2.0.0 1.2.0 - run -1 bats_version_lt 2.0.0 1.2.3 - run -1 bats_version_lt 1.1.0 1.0.0 - run -1 bats_version_lt 1.1.0 1.0.2 - run -1 bats_version_lt 1.0.1 1.0.0 + run -1 bats_version_lt 2.0.0 1.0.0 + run -1 bats_version_lt 2.0.0 1.2.0 + run -1 bats_version_lt 2.0.0 1.2.3 + run -1 bats_version_lt 1.1.0 1.0.0 + run -1 bats_version_lt 1.1.0 1.0.2 + run -1 bats_version_lt 1.0.1 1.0.0 - run -2 bats_version_lt 1.0.0 1.0.0 + run -2 bats_version_lt 1.0.0 1.0.0 } @test bats_require_minimum_version { - [ "$BATS_GUARANTEED_MINIMUM_VERSION" = 0.0.0 ] # check default + [ "$BATS_GUARANTEED_MINIMUM_VERSION" = 0.0.0 ] # check default + + bats_require_minimum_version 0.1.2 # (a version that should be safe not to fail) + [ "${BATS_GUARANTEED_MINIMUM_VERSION}" = 0.1.2 ] + + # a higher version should upgrade + bats_require_minimum_version 0.2.3 + [ "${BATS_GUARANTEED_MINIMUM_VERSION}" = 0.2.3 ] + + # a lower version should not change + bats_require_minimum_version 0.1.2 + [ "${BATS_GUARANTEED_MINIMUM_VERSION}" = 0.2.3 ] +} + +@test bats_binary_search { + bats_require_minimum_version 1.5.0 + + run -2 bats_binary_search "search-value" + [ "$output" = "ERROR: bats_binary_search requires exactly 2 arguments: " ] + + # unset array = empty array: a bit unfortunate but we can't tell the difference (on older Bash?) + unset no_array + run -1 bats_binary_search "search-value" "no_array" + + # shellcheck disable=SC2034 + empty_array=() + run -1 bats_binary_search "search-value" "empty_array" + + # shellcheck disable=SC2034 + odd_length_array=(1 2 3) + run -1 bats_binary_search a odd_length_array + run -0 bats_binary_search 1 odd_length_array + run -0 bats_binary_search 2 odd_length_array + run -0 bats_binary_search 3 odd_length_array + + # shellcheck disable=SC2034 + even_length_array=(1 2 3 4) + run -1 bats_binary_search a even_length_array + run -0 bats_binary_search 1 even_length_array + run -0 bats_binary_search 2 even_length_array + run -0 bats_binary_search 3 even_length_array + run -0 bats_binary_search 4 even_length_array + +} + +@test bats_sort { + local -a empty one_element two_sorted two_elements_reversed three_elements_scrambled + + bats_sort empty + echo "empty(${#empty[@]}): ${empty[*]}" + [ ${#empty[@]} -eq 0 ] + + bats_sort one_element 1 + echo "one_element(${#one_element[@]}): ${one_element[*]}" + [ ${#one_element[@]} -eq 1 ] + [ "${one_element[0]}" = 1 ] + + bats_sort two_sorted 1 2 + echo "two_sorted(${#two_sorted[@]}): ${two_sorted[*]}" + [ ${#two_sorted[@]} -eq 2 ] + [ "${two_sorted[0]}" = 1 ] + [ "${two_sorted[1]}" = 2 ] + + bats_sort two_elements_reversed 2 1 + echo "two_elements_reversed(${#two_elements_reversed[@]}): ${two_elements_reversed[*]}" + [ ${#two_elements_reversed[@]} -eq 2 ] + [ "${two_elements_reversed[0]}" = 1 ] + [ "${two_elements_reversed[1]}" = 2 ] + + bats_sort three_elements_scrambled 2 1 3 + echo "three_elements_scrambled(${#three_elements_scrambled[@]}): ${three_elements_scrambled[*]}" + [ ${#three_elements_scrambled[@]} -eq 3 ] + [ "${three_elements_scrambled[0]}" = 1 ] + [ "${three_elements_scrambled[1]}" = 2 ] + [ "${three_elements_scrambled[2]}" = 3 ] +} - bats_require_minimum_version 0.1.2 # (a version that should be safe not to fail) - [ "${BATS_GUARANTEED_MINIMUM_VERSION}" = 0.1.2 ] +@test bats_all_in { + bats_require_minimum_version 1.5.0 - # a higher version should upgrade - bats_require_minimum_version 0.2.3 - [ "${BATS_GUARANTEED_MINIMUM_VERSION}" = 0.2.3 ] + local -ra empty=() one=(1) onetwo=(1 2) + # find nothing in any array + run -0 bats_all_in empty + run -0 bats_all_in one + run -0 bats_all_in onetwo + # find single search value in single element array + run -0 bats_all_in one 1 + # find single search values in multi element array + run -0 bats_all_in onetwo 1 + # find multiple search values in multi element array + run -0 bats_all_in onetwo 1 2 - # a lower version shoudl not change - bats_require_minimum_version 0.1.2 - [ "${BATS_GUARANTEED_MINIMUM_VERSION}" = 0.2.3 ] -} \ No newline at end of file + # don't find in empty array + run -1 bats_all_in empty 1 + # don't find in non empty + run -1 bats_all_in one 2 + # don't find smaller values + run -1 bats_all_in onetwo 0 1 2 + # don't find greater values + run -1 bats_all_in onetwo 1 2 3 +} + +@test bats_any_in { + bats_require_minimum_version 1.5.0 + + # shellcheck disable=SC2030,SC2034 + local -ra empty=() one=(1) onetwo=(1 2) + # empty search set is always false + run -1 bats_any_in empty + run -1 bats_any_in one + run -1 bats_any_in onetwo + + # find single search value in single element array + run -0 bats_any_in one 1 + # find single search values in multi element array + run -0 bats_any_in onetwo 2 + # find multiple search values in multi element array + run -0 bats_any_in onetwo 1 2 + + # don't find in empty array + run -1 bats_any_in empty 1 + # don't find in non empty + run -1 bats_any_in one 2 + # don't find smaller values + run -1 bats_any_in onetwo 0 + # don't find greater values + run -1 bats_any_in onetwo 3 +} + +@test bats_trim { + local empty already_trimmed trimmed whitespaces_within + bats_trim empty "" + # shellcheck disable=SC2031 + [ "${empty-NOTSET}" = "" ] + + bats_trim already_trimmed "abc" + [ "$already_trimmed" = abc ] + + bats_trim trimmed " abc " + [ "$trimmed" = abc ] + + bats_trim whitespaces_within " a b " + [ "$whitespaces_within" = "a b" ] +} + +@test bats_append_arrays_as_args { + bats_require_minimum_version 1.5.0 + count_and_print_args() { + echo "$# $*" + } + + run -1 bats_append_arrays_as_args + [ "${lines[0]}" == "Error: append_arrays_as_args is missing a command or -- separator" ] + + run -1 bats_append_arrays_as_args -- + [ "${lines[0]}" == "Error: append_arrays_as_args is missing a command or -- separator" ] + + # shellcheck disable=SC2034 + empty=() + run -0 bats_append_arrays_as_args empty -- count_and_print_args + [ "${lines[0]}" == '0 ' ] + + run -0 bats_append_arrays_as_args -- count_and_print_args + [ "${lines[0]}" == '0 ' ] + + # shellcheck disable=SC2034 + arr=(a) + run -0 bats_append_arrays_as_args arr -- count_and_print_args + [ "${lines[0]}" == '1 a' ] + + # shellcheck disable=SC2034 + arr2=(b) + run -0 bats_append_arrays_as_args arr arr2 -- count_and_print_args + [ "${lines[0]}" == '2 a b' ] + + run -0 bats_append_arrays_as_args arr empty arr2 -- count_and_print_args + [ "${lines[0]}" == '2 a b' ] +} diff --git a/test/concurrent-coordination.bash b/test/concurrent-coordination.bash index c1ee8d0258..387686be4b 100644 --- a/test/concurrent-coordination.bash +++ b/test/concurrent-coordination.bash @@ -2,64 +2,64 @@ # once this happened, all latecomers will go through immediately! # WARNING: a barrier group consists of all processes with the same barrier name *and* size! single-use-barrier() { # [ []] - local barrier_name="$1" - local barrier_size="$2" - local timeout_in_seconds="${3:-0}" - local sleep_cycle_time="${4:-1}" - # use name and size to distinguish between invocations - # this will block inconsistent sizes on the same name! - local BARRIER_SUFFIX=${barrier_name//\//_}-$barrier_size - local BARRIER_FILE="$BATS_SUITE_TMPDIR/barrier-$BARRIER_SUFFIX" - # mark our entry for all others - # concurrent writes may interleave but should not lose their newlines - echo "in-$$" >> "$BARRIER_FILE" - local start="$SECONDS" - # wait for others to enter - while [[ $(wc -l <"$BARRIER_FILE" ) -lt $barrier_size ]]; do - if [[ $timeout_in_seconds -ne 0 && $(( SECONDS - start )) -gt $timeout_in_seconds ]]; then - mv "$BARRIER_FILE" "$BARRIER_FILE-timeout" - printf "ERROR: single-use-barrier %s timed out\n" "$BARRIER_SUFFIX" >&2 - return 1 - fi - sleep "$sleep_cycle_time" - done - # mark our exit - echo "out-$$" >> "$BARRIER_FILE" + local barrier_name="$1" + local barrier_size="$2" + local timeout_in_seconds="${3:-0}" + local sleep_cycle_time="${4:-1}" + # use name and size to distinguish between invocations + # this will block inconsistent sizes on the same name! + local BARRIER_SUFFIX=${barrier_name//\//_}-$barrier_size + local BARRIER_FILE="$BATS_SUITE_TMPDIR/barrier-$BARRIER_SUFFIX" + # mark our entry for all others + # concurrent writes may interleave but should not lose their newlines + echo "in-$$" >>"$BARRIER_FILE" + local start="$SECONDS" + # wait for others to enter + while [[ $(wc -l <"$BARRIER_FILE") -lt $barrier_size ]]; do + if [[ $timeout_in_seconds -ne 0 && $((SECONDS - start)) -gt $timeout_in_seconds ]]; then + mv "$BARRIER_FILE" "$BARRIER_FILE-timeout" + printf "ERROR: single-use-barrier %s timed out\n" "$BARRIER_SUFFIX" >&2 + return 1 + fi + sleep "$sleep_cycle_time" + done + # mark our exit + echo "out-$$" >>"$BARRIER_FILE" } # block until at least signalling threads have passed the latch # SINGLE_USE_LATCH_DIR must be exported! single-use-latch::wait() { # [ []] - local latch_name="$1" - local latch_size="$2" - local timeout_in_seconds="${3:-0}" - local sleep_cycle_time="${4:-1}" - - local LATCH_FILE - LATCH_FILE="$(single-use-latch::_filename "$latch_name")" - local start="$SECONDS" - while [[ (! -e "$LATCH_FILE") || $(wc -l <"$LATCH_FILE" ) -lt $latch_size ]]; do - if [[ $timeout_in_seconds -ne 0 && $(( SECONDS - start )) -gt $timeout_in_seconds ]]; then - printf "ERROR: single-use-latch %s timed out\n" "$latch_name" >&2 - mv "$LATCH_FILE" "$LATCH_FILE-timeout" - return 1 - fi - sleep "$sleep_cycle_time" - done + local latch_name="$1" + local latch_size="$2" + local timeout_in_seconds="${3:-0}" + local sleep_cycle_time="${4:-1}" + + local LATCH_FILE + LATCH_FILE="$(single-use-latch::_filename "$latch_name")" + local start="$SECONDS" + while [[ (! -e "$LATCH_FILE") || $(wc -l <"$LATCH_FILE") -lt $latch_size ]]; do + if [[ $timeout_in_seconds -ne 0 && $((SECONDS - start)) -gt $timeout_in_seconds ]]; then + printf "ERROR: single-use-latch %s timed out\n" "$latch_name" >&2 + mv "$LATCH_FILE" "$LATCH_FILE-timeout" + return 1 + fi + sleep "$sleep_cycle_time" + done } # signal the waiting process that the latch was passed # this does not block # SINGLE_USE_LATCH_DIR must be exported! single-use-latch::signal() { # - local latch_name="$1" - local LATCH_FILE - LATCH_FILE="$(single-use-latch::_filename "$latch_name")" - # mark our passing - # concurrent process might interleave but will still post their newline - echo "passed-$$" >> "$LATCH_FILE" + local latch_name="$1" + local LATCH_FILE + LATCH_FILE="$(single-use-latch::_filename "$latch_name")" + # mark our passing + # concurrent process might interleave but will still post their newline + echo "passed-$$" >>"$LATCH_FILE" } -single-use-latch::_filename() { # - printf "%s\n" "${SINGLE_USE_LATCH_DIR?}/latch-${1//\//_}" +single-use-latch::_filename() { # + printf "%s\n" "${SINGLE_USE_LATCH_DIR?}/latch-${1//\//_}" } diff --git a/test/file_setup_teardown.bats b/test/file_setup_teardown.bats index e5a30eaf67..6c00eb41e9 100644 --- a/test/file_setup_teardown.bats +++ b/test/file_setup_teardown.bats @@ -8,43 +8,43 @@ setup_file() { } @test "setup_file is run once per file" { - # shellcheck disable=SC2031,SC2030 - export LOG="$BATS_TEST_TMPDIR/setup_file_once.log" - bats "$FIXTURE_ROOT/setup_file.bats" + # shellcheck disable=SC2031,SC2030 + export LOG="$BATS_TEST_TMPDIR/setup_file_once.log" + bats "$FIXTURE_ROOT/setup_file.bats" } @test "teardown_file is run once per file" { # shellcheck disable=SC2031,SC2030 export LOG="$BATS_TEST_TMPDIR/teardown_file_once.log" - run bats "$FIXTURE_ROOT/teardown_file.bats" + reentrant_run bats "$FIXTURE_ROOT/teardown_file.bats" [[ $status -eq 0 ]] # output the log for faster debugging cat "$LOG" # expect to find an entry for the tested file grep 'teardown_file.bats' "$LOG" # it should be the only entry - run wc -l < "$LOG" + run wc -l <"$LOG" [[ $output -eq 1 ]] } @test "setup_file is called correctly in multi file suite" { - # shellcheck disable=SC2031,SC2030 - export LOG="$BATS_TEST_TMPDIR/setup_file_multi_file_suite.log" - run bats "$FIXTURE_ROOT/setup_file.bats" "$FIXTURE_ROOT/no_setup_file.bats" "$FIXTURE_ROOT/setup_file2.bats" - [[ $status -eq 0 ]] - run wc -l < "$LOG" - # each setup_file[2].bats is in the log exactly once! - [[ $output -eq 2 ]] - grep setup_file.bats "$LOG" - grep setup_file2.bats "$LOG" + # shellcheck disable=SC2031,SC2030 + export LOG="$BATS_TEST_TMPDIR/setup_file_multi_file_suite.log" + reentrant_run bats "$FIXTURE_ROOT/setup_file.bats" "$FIXTURE_ROOT/no_setup_file.bats" "$FIXTURE_ROOT/setup_file2.bats" + [[ $status -eq 0 ]] + run wc -l <"$LOG" + # each setup_file[2].bats is in the log exactly once! + [[ $output -eq 2 ]] + grep setup_file.bats "$LOG" + grep setup_file2.bats "$LOG" } @test "teardown_file is called correctly in multi file suite" { # shellcheck disable=SC2031,SC2030 export LOG="$BATS_TEST_TMPDIR/teardown_file_multi_file_suite.log" - run bats "$FIXTURE_ROOT/teardown_file.bats" "$FIXTURE_ROOT/no_teardown_file.bats" "$FIXTURE_ROOT/teardown_file2.bats" + reentrant_run bats "$FIXTURE_ROOT/teardown_file.bats" "$FIXTURE_ROOT/no_teardown_file.bats" "$FIXTURE_ROOT/teardown_file2.bats" [[ $status -eq 0 ]] - run wc -l < "$LOG" + run wc -l <"$LOG" # each teardown_file[2].bats is in the log exactly once! [[ $output -eq 2 ]] grep teardown_file.bats "$LOG" @@ -54,30 +54,30 @@ setup_file() { @test "setup_file failure aborts tests for this file" { # this might need to mark them as skipped as the test count is already determined at this point - run bats "$FIXTURE_ROOT/setup_file_failed.bats" + reentrant_run bats "$FIXTURE_ROOT/setup_file_failed.bats" echo "$output" [[ "${lines[0]}" == "1..2" ]] [[ "${lines[1]}" == "not ok 1 setup_file failed" ]] [[ "${lines[2]}" == "# (from function \`setup_file' in test file $RELATIVE_FIXTURE_ROOT/setup_file_failed.bats, line 2)" ]] [[ "${lines[3]}" == "# \`false' failed" ]] [[ "${lines[4]}" == "# bats warning: Executed 1 instead of expected 2 tests" ]] # this warning is expected - # to appease the count validator, we would have to reduce the expected number of tests (retroactively?) or + # to appease the count validator, we would have to reduce the expected number of tests (retroactively?) or # output even those tests that should be skipped due to a failed setup_file. # Since we are already in a failure mode, the additional error does not hurt and is less verbose than # printing all the failed/skipped tests due to the setup failure. } @test "teardown_file failure fails at least one test from the file" { - run bats "$FIXTURE_ROOT/teardown_file_failed.bats" + reentrant_run bats "$FIXTURE_ROOT/teardown_file_failed.bats" [[ $status -ne 0 ]] echo "$output" [[ "${lines[0]}" == "1..1" ]] [[ "${lines[1]}" == "ok 1 test" ]] [[ "${lines[2]}" == "not ok 2 teardown_file failed" ]] - [[ "${lines[3]}" == "# (from function \`teardown_file' in test file $RELATIVE_FIXTURE_ROOT/teardown_file_failed.bats, line 3)" ]] + [[ "${lines[3]}" == "# (from function \`teardown_file' in test file $RELATIVE_FIXTURE_ROOT/teardown_file_failed.bats, line 2)" ]] [[ "${lines[4]}" == "# \`false' failed" ]] [[ "${lines[5]}" == "# bats warning: Executed 2 instead of expected 1 tests" ]] # for now this warning is expected - # for a failed teardown_file not to change the number of tests being reported, we would have to alter at least one provious test result report + # for a failed teardown_file not to change the number of tests being reported, we would have to alter at least one previous test result report # this would require arbitrary amounts of buffering so we simply add our own line with a fake test number # tripping the count validator won't change the overall result, as we already are in a failure mode } @@ -85,7 +85,7 @@ setup_file() { @test "teardown_file runs even if any test in the file failed" { # shellcheck disable=SC2031,SC2030 export LOG="$BATS_TEST_TMPDIR/teardown_file_failed.log" - run bats "$FIXTURE_ROOT/teardown_file_after_failing_test.bats" + reentrant_run bats "$FIXTURE_ROOT/teardown_file_after_failing_test.bats" [[ $status -ne 0 ]] grep teardown_file_after_failing_test.bats "$LOG" echo "$output" @@ -96,7 +96,7 @@ not ok 1 failing test } @test "teardown_file should run even after user abort via CTRL-C" { - if [[ "$BATS_NUMBER_OF_PARALLEL_JOBS" -gt 1 ]]; then + if [[ "${BATS_NUMBER_OF_PARALLEL_JOBS:-1}" -gt 1 ]]; then skip "Aborts don't work in parallel mode" fi # shellcheck disable=SC2031,SC2030 @@ -105,7 +105,7 @@ not ok 1 failing test set -m SECONDS=0 # run testsubprocess in background to not avoid blocking this test - bats "$FIXTURE_ROOT/teardown_file_after_long_test.bats"& + bats "$FIXTURE_ROOT/teardown_file_after_long_test.bats" & SUBPROCESS_PID=$! # wait until we enter the test sleep 2 @@ -123,16 +123,16 @@ not ok 1 failing test @test "setup_file runs even if all tests in the file are skipped" { # shellcheck disable=SC2031,SC2030 - export LOG="$BATS_TEST_TMPDIR/setup_file_skipped.log" - run bats "$FIXTURE_ROOT/setup_file_even_if_all_tests_are_skipped.bats" + export LOG="$BATS_TEST_TMPDIR/setup_file_skipped.log" + reentrant_run bats "$FIXTURE_ROOT/setup_file_even_if_all_tests_are_skipped.bats" [[ -f "$LOG" ]] grep setup_file_even_if_all_tests_are_skipped.bats "$LOG" } @test "teardown_file runs even if all tests in the file are skipped" { # shellcheck disable=SC2031,SC2030 - export LOG="$BATS_TEST_TMPDIR/teardown_file_skipped.log" - run bats "$FIXTURE_ROOT/teardown_file_even_if_all_tests_are_skipped.bats" + export LOG="$BATS_TEST_TMPDIR/teardown_file_skipped.log" + reentrant_run bats "$FIXTURE_ROOT/teardown_file_even_if_all_tests_are_skipped.bats" [[ $status -eq 0 ]] [[ -f "$LOG" ]] grep teardown_file_even_if_all_tests_are_skipped.bats "$LOG" @@ -141,14 +141,14 @@ not ok 1 failing test @test "setup_file must not leak context between tests in the same suite" { # example: BATS_ROOT was unset in one test but used in others, therefore, the suite failed # Simulate leaking env var from first to second test by: export SETUP_FILE_VAR="LEAK!" - run bats "$FIXTURE_ROOT/setup_file_does_not_leak_env.bats" "$FIXTURE_ROOT/setup_file_does_not_leak_env2.bats" + reentrant_run bats "$FIXTURE_ROOT/setup_file_does_not_leak_env.bats" "$FIXTURE_ROOT/setup_file_does_not_leak_env2.bats" echo "$output" [[ $status -eq 0 ]] } @test "teardown_file must not leak context between tests in the same suite" { # example: BATS_ROOT was unset in one test but used in others, therefore, the suite failed - run bats "$FIXTURE_ROOT/teardown_file_does_not_leak.bats" "$FIXTURE_ROOT/teardown_file_does_not_leak2.bats" + reentrant_run bats "$FIXTURE_ROOT/teardown_file_does_not_leak.bats" "$FIXTURE_ROOT/teardown_file_does_not_leak2.bats" echo "$output" [[ $status -eq 0 ]] [[ $output == "1..2 @@ -157,7 +157,7 @@ ok 2 must not see variable from first run" ]] } @test "halfway setup_file errors are caught and reported" { - run bats "$FIXTURE_ROOT/setup_file_halfway_error.bats" + reentrant_run bats "$FIXTURE_ROOT/setup_file_halfway_error.bats" [ $status -ne 0 ] echo "$output" [ "${lines[0]}" == "1..1" ] @@ -166,16 +166,12 @@ ok 2 must not see variable from first run" ]] [ "${lines[3]}" == "# \`false' failed" ] } -@test "halfway teardown_file errors are caught and reported" { - run bats "$FIXTURE_ROOT/teardown_file_halfway_error.bats" - echo "$output" - [[ $status -ne 0 ]] - [[ "${lines[0]}" == "1..1" ]] - [[ "${lines[1]}" == "ok 1 empty" ]] - [[ "${lines[2]}" == "not ok 2 teardown_file failed" ]] - [[ "${lines[3]}" == "# (from function \`teardown_file' in test file $RELATIVE_FIXTURE_ROOT/teardown_file_halfway_error.bats, line 3)" ]] - [[ "${lines[4]}" == "# \`false' failed" ]] - [[ "${lines[5]}" == "# bats warning: Executed 2 instead of expected 1 tests" ]] # for now this warning is expected +@test "halfway teardown_file errors are ignored" { + reentrant_run -0 bats "$FIXTURE_ROOT/teardown_file_halfway_error.bats" + [ "${lines[0]}" == "1..1" ] + [ "${lines[1]}" == "ok 1 empty" ] + [ "${#lines[@]}" -eq 2 ] + } @test "variables exported in setup_file are visible in tests" { @@ -186,8 +182,17 @@ ok 2 must not see variable from first run" ]] # shellcheck disable=SC2031 export LOG="$BATS_TEST_TMPDIR/setup_file.log" # only select the test from no_setup_file - run bats -f test "$FIXTURE_ROOT/setup_file.bats" "$FIXTURE_ROOT/no_setup_file.bats" + reentrant_run bats -f test "$FIXTURE_ROOT/setup_file.bats" "$FIXTURE_ROOT/no_setup_file.bats" - [ ! -f "$LOG" ] # setup_file must not have been executed! + [ ! -f "$LOG" ] # setup_file must not have been executed! [ "${lines[0]}" == '1..1' ] # but at least one test should have been run } + +@test "Failure in setup_file and teardown_file still prints error message" { + reentrant_run ! bats "$FIXTURE_ROOT/error_in_setup_and_teardown_file.bats" + [ "${lines[0]}" == '1..1' ] + [ "${lines[1]}" == 'not ok 1 setup_file failed' ] + [ "${lines[2]}" == "# (from function \`setup_file' in test file $RELATIVE_FIXTURE_ROOT/error_in_setup_and_teardown_file.bats, line 2)" ] + [ "${lines[3]}" == "# \`false' failed" ] + [ "${#lines[@]}" -eq 4 ] +} diff --git a/test/fixtures/bats/BATS_variables_dont_contain_double_slashes.bats b/test/fixtures/bats/BATS_variables_dont_contain_double_slashes.bats new file mode 100644 index 0000000000..2994bbe84f --- /dev/null +++ b/test/fixtures/bats/BATS_variables_dont_contain_double_slashes.bats @@ -0,0 +1,9 @@ +@test "BATS_* variables don't contain double slashes" { + for var_name in ${!BATS_@}; do + local var_value="${!var_name}" + if [[ "$var_value" == *'//'* ]]; then + + echo "$var_name contains // ${#var_value}: ${var_value}" && false + fi + done +} diff --git a/test/fixtures/bats/comment_style.bats b/test/fixtures/bats/comment_style.bats index 5363bd9601..5696948e67 100644 --- a/test/fixtures/bats/comment_style.bats +++ b/test/fixtures/bats/comment_style.bats @@ -2,7 +2,7 @@ function should_be_found { # @test true } -function should_be_found_with_trailing_whitespace { # @test +function should_be_found_with_trailing_whitespace { # @test true } @@ -10,7 +10,7 @@ should_be_found_with_parens() { #@test true } -should_be_found_with_parens_and_whitespace () { #@test +should_be_found_with_parens_and_whitespace() { #@test true } @@ -18,15 +18,15 @@ function should_be_found_with_function_and_parens() { #@test true } -function should_be_found_with_function_parens_and_whitespace () { #@test +function should_be_found_with_function_parens_and_whitespace() { #@test true } -should_not_be_found() { - false - #@test -} +should_not_be_found() { + false + #@test +} -should_not_be_found() { - false -} #@test +should_not_be_found() { + false +} #@test diff --git a/test/fixtures/bats/dos_line_no_shellcheck.bats b/test/fixtures/bats/dos_line_no_shellcheck.bats index b5f65c67b0..d04c17f17b 100644 --- a/test/fixtures/bats/dos_line_no_shellcheck.bats +++ b/test/fixtures/bats/dos_line_no_shellcheck.bats @@ -1,3 +1,3 @@ -@test "foo" { - echo "foo" -} +@test "foo" { + echo "foo" +} diff --git a/test/fixtures/bats/empty.bats b/test/fixtures/bats/empty.bats index e69de29bb2..8b13789179 100644 --- a/test/fixtures/bats/empty.bats +++ b/test/fixtures/bats/empty.bats @@ -0,0 +1 @@ + diff --git a/test/fixtures/bats/environment.bats b/test/fixtures/bats/environment.bats index f643e1bd16..0d045da81b 100644 --- a/test/fixtures/bats/environment.bats +++ b/test/fixtures/bats/environment.bats @@ -6,5 +6,5 @@ @test "variables do not persist across tests" { # shellcheck disable=SC2031 - [ -z "$variable" ] + [ -z "${variable:-}" ] } diff --git a/test/fixtures/bats/evaluation_count/file1.bats b/test/fixtures/bats/evaluation_count/file1.bats index eb7d85b8ed..c2ea77f37e 100644 --- a/test/fixtures/bats/evaluation_count/file1.bats +++ b/test/fixtures/bats/evaluation_count/file1.bats @@ -1,5 +1,5 @@ -echo "file1" >> "$TEMPFILE" +echo "file1" >>"$TEMPFILE" @test "test1" { - : -} \ No newline at end of file + : +} diff --git a/test/fixtures/bats/evaluation_count/file2.bats b/test/fixtures/bats/evaluation_count/file2.bats index fee360f76c..49ff64af68 100644 --- a/test/fixtures/bats/evaluation_count/file2.bats +++ b/test/fixtures/bats/evaluation_count/file2.bats @@ -1,9 +1,9 @@ -echo "file2" >> "$TEMPFILE" +echo "file2" >>"$TEMPFILE" @test "test 1" { - : + : } @test "test 2" { - : + : } diff --git a/test/fixtures/bats/external_function_calls.bats b/test/fixtures/bats/external_function_calls.bats index 5b009ebf2e..6a9a5fadf0 100644 --- a/test/fixtures/bats/external_function_calls.bats +++ b/test/fixtures/bats/external_function_calls.bats @@ -5,61 +5,61 @@ load test_helper # All tests fail on the same line so checking can be automated @test "Call true function && false stackdepth=1" { - help_me - help_me && false + help_me + help_me && false } @test "Call true function && return 1 stackdepth=1" { - help_me - help_me && return 1 + help_me + help_me && return 1 } @test "Call true function and invert stackdepth=2" { - help_me - ! help_me + help_me + ! help_me } @test "Call false function || false stackdepth=1" { - ! failing_helper - failing_helper || false + ! failing_helper + failing_helper || false } @test "Call false function && return 1 stackdepth=1" { - ! failing_helper - failing_helper || return 1 + ! failing_helper + failing_helper || return 1 } @test "Call false function stackdepth=2" { - ! failing_helper - failing_helper + ! failing_helper + failing_helper } @test "Call return_0 function && false stackdepth=1" { - return_0 - return_0 && false + return_0 + return_0 && false } @test "Call return_0 function && return 1 stackdepth=1" { - return_0 - return_0 && return 1 + return_0 + return_0 && return 1 } @test "Call return_0 function and invert stackdepth=2" { - return_0 - ! return_0 + return_0 + ! return_0 } @test "Call return_1 function || false stackdepth=1" { - ! return_1 - return_1 || false + ! return_1 + return_1 || false } @test "Call return_1 function && return 1 stackdepth=1" { - ! return_1 - return_1 || return 1 + ! return_1 + return_1 || return 1 } @test "Call return_1 function stackdepth=2" { - ! return_1 - return_1 + ! return_1 + return_1 } diff --git a/test/fixtures/bats/external_functions.bash b/test/fixtures/bats/external_functions.bash index 12057c2b75..4ed5d5980b 100644 --- a/test/fixtures/bats/external_functions.bash +++ b/test/fixtures/bats/external_functions.bash @@ -1,15 +1,15 @@ setup() { - true + true } teardown() { - true + true } setup_file() { - true + true } teardown_file() { - true -} \ No newline at end of file + true +} diff --git a/test/fixtures/bats/external_functions.bats b/test/fixtures/bats/external_functions.bats index 5709a84b0d..946fca248f 100644 --- a/test/fixtures/bats/external_functions.bats +++ b/test/fixtures/bats/external_functions.bats @@ -1,5 +1,5 @@ load external_functions @test test { - true -} \ No newline at end of file + true +} diff --git a/test/fixtures/bats/failing_with_bash_expression.bats b/test/fixtures/bats/failing_with_bash_expression.bats index 71e8ee363d..4574da54c8 100644 --- a/test/fixtures/bats/failing_with_bash_expression.bats +++ b/test/fixtures/bats/failing_with_bash_expression.bats @@ -1,4 +1,4 @@ @test "a failing test" { true - (( 1 == 2 )) + ((1 == 2)) } diff --git a/test/fixtures/bats/failure_in_free_code.bats b/test/fixtures/bats/failure_in_free_code.bats index 42afad1553..63ef844a38 100644 --- a/test/fixtures/bats/failure_in_free_code.bats +++ b/test/fixtures/bats/failure_in_free_code.bats @@ -1,5 +1,3 @@ - - helper() { false } @@ -8,4 +6,4 @@ helper @test "everything is ok" { true -} \ No newline at end of file +} diff --git a/test/fixtures/bats/hang_after_run.bats b/test/fixtures/bats/hang_after_run.bats index 702e5fd387..3cfae39f12 100644 --- a/test/fixtures/bats/hang_after_run.bats +++ b/test/fixtures/bats/hang_after_run.bats @@ -1,9 +1,9 @@ setup() { - load '../../concurrent-coordination' + load '../../concurrent-coordination' } @test "test" { - single-use-latch::signal hang_after_run - run true - sleep 10 -} \ No newline at end of file + single-use-latch::signal hang_after_run + run true + sleep 10 +} diff --git a/test/fixtures/bats/hang_in_run.bats b/test/fixtures/bats/hang_in_run.bats index ded0789b87..b08c2a8ae4 100644 --- a/test/fixtures/bats/hang_in_run.bats +++ b/test/fixtures/bats/hang_in_run.bats @@ -1,8 +1,8 @@ setup() { - load '../../concurrent-coordination' + load '../../concurrent-coordination' } @test "test" { - single-use-latch::signal hang_in_run - run sleep 10 -} \ No newline at end of file + single-use-latch::signal hang_in_run + run sleep 10 +} diff --git a/test/fixtures/bats/hang_in_setup_file.bats b/test/fixtures/bats/hang_in_setup_file.bats index d372815329..b21ef7b890 100644 --- a/test/fixtures/bats/hang_in_setup_file.bats +++ b/test/fixtures/bats/hang_in_setup_file.bats @@ -1,9 +1,9 @@ setup_file() { - load '../../concurrent-coordination' - single-use-latch::signal hang_in_setup_file - sleep 10 + load '../../concurrent-coordination' + single-use-latch::signal hang_in_setup_file + sleep 10 } @test "empty" { - : -} \ No newline at end of file + : +} diff --git a/test/fixtures/bats/hang_in_teardown.bats b/test/fixtures/bats/hang_in_teardown.bats index 314838b02c..229920da6f 100644 --- a/test/fixtures/bats/hang_in_teardown.bats +++ b/test/fixtures/bats/hang_in_teardown.bats @@ -1,9 +1,9 @@ teardown() { - load '../../concurrent-coordination' - single-use-latch::signal hang_in_teardown - sleep 10 + load '../../concurrent-coordination' + single-use-latch::signal hang_in_teardown + sleep 10 } @test "empty" { - : -} \ No newline at end of file + : +} diff --git a/test/fixtures/bats/hang_in_teardown_file.bats b/test/fixtures/bats/hang_in_teardown_file.bats index 640d8ce8ae..40143ed42d 100644 --- a/test/fixtures/bats/hang_in_teardown_file.bats +++ b/test/fixtures/bats/hang_in_teardown_file.bats @@ -1,9 +1,9 @@ teardown_file() { - load '../../concurrent-coordination' - single-use-latch::signal hang_in_teardown_file - sleep 10 + load '../../concurrent-coordination' + single-use-latch::signal hang_in_teardown_file + sleep 10 } @test "empty" { - : -} \ No newline at end of file + : +} diff --git a/test/fixtures/bats/hang_in_test.bats b/test/fixtures/bats/hang_in_test.bats index 52479b1727..13b5e32a8c 100644 --- a/test/fixtures/bats/hang_in_test.bats +++ b/test/fixtures/bats/hang_in_test.bats @@ -1,8 +1,9 @@ setup() { - load '../../concurrent-coordination' + load '../../concurrent-coordination' } @test "test" { - single-use-latch::signal hang_in_test - sleep 10 -} \ No newline at end of file + single-use-latch::signal hang_in_test + sleep 10 + echo "after sleep" # this should not be printed +} diff --git a/test/fixtures/bats/issue-205.bats b/test/fixtures/bats/issue-205.bats index 4c8a7ec44b..ddc826a34b 100644 --- a/test/fixtures/bats/issue-205.bats +++ b/test/fixtures/bats/issue-205.bats @@ -1,103 +1,103 @@ #!/usr/bin/env bats function bgfunc { - get_open_fds - echo "${FUNCNAME[1]} fds before: (${open_fds[*]})" >>"${LOG_FILE}" - close_non_std_fds - get_open_fds - echo "${FUNCNAME[1]} fds after: (${open_fds[*]})" >>"${LOG_FILE}" - sleep 10 - echo "bgfunc done" - return 0 + get_open_fds + echo "${FUNCNAME[1]} fds before: (${open_fds[*]})" >>"${LOG_FILE}" + close_non_std_fds + get_open_fds + echo "${FUNCNAME[1]} fds after: (${open_fds[*]})" >>"${LOG_FILE}" + sleep 10 + echo "bgfunc done" + return 0 } # store the list of open FDs in array open_fds function get_open_fds() { - open_fds=() # reset output array in case it was already set - if [[ ${BASH_VERSINFO[0]} == 3 ]]; then - local BASHPID - BASHPID=$(bash -c 'echo $PPID') - fi - local tmpfile - tmpfile=$(mktemp "$BATS_SUITE_TMPDIR/fds-XXXXXX") - # Avoid opening a new fd to read fds: Don't use <(), glob expansion. - # Instead, redirect stdout to file which does not create an extra FD. - if [[ -d /proc/$BASHPID/fd ]]; then # Linux - ls -1 "/proc/$BASHPID/fd" > "$tmpfile" - IFS=$'\n' read -d '' -ra open_fds <"$tmpfile" || true - elif command -v lsof >/dev/null ; then # MacOS - local -a fds - lsof -F f -p "$BASHPID" >"$tmpfile" - IFS=$'\n' read -d '' -ra fds < "$tmpfile" || true - for fd in "${fds[@]}"; do - case $fd in - f[0-9]*) # filter non fd entries (mainly pid?) - open_fds+=("${fd#f}") # cut off f prefix - ;; - esac - done - elif command -v procstat >/dev/null ; then # BSDs - local -a columns header - procstat fds "$BASHPID" > "$tmpfile" - { - read -r -a header - local fd_column_index=-1 - for ((i=0; i<${#header[@]}; ++i)); do - if [[ ${header[$i]} == *FD* ]]; then - fd_column_index=$i - break - fi - done - if [[ $fd_column_index -eq -1 ]]; then - printf "Could not find FD column in procstat" >&2 - exit 1 - fi - while read -r -a columns; do - local fd=${columns[$fd_column_index]} - if [[ $fd == [0-9]* ]]; then # only take up numeric entries - open_fds+=("$fd") - fi - done - } < "$tmpfile" - else - # TODO: MSYS (Windows) - printf "Neither FD discovery mechanism available\n" >&2 + open_fds=() # reset output array in case it was already set + if [[ ${BASH_VERSINFO[0]} == 3 ]]; then + local BASHPID + BASHPID=$(bash -c 'echo $PPID') + fi + local tmpfile + tmpfile=$(mktemp "$BATS_SUITE_TMPDIR/fds-XXXXXX") + # Avoid opening a new fd to read fds: Don't use <(), glob expansion. + # Instead, redirect stdout to file which does not create an extra FD. + if [[ -d /proc/$BASHPID/fd ]]; then # Linux + ls -1 "/proc/$BASHPID/fd" >"$tmpfile" + IFS=$'\n' read -d '' -ra open_fds <"$tmpfile" || true + elif command -v lsof >/dev/null; then # MacOS + local -a fds + lsof -F f -p "$BASHPID" >"$tmpfile" + IFS=$'\n' read -d '' -ra fds <"$tmpfile" || true + for fd in "${fds[@]}"; do + case $fd in + f[0-9]*) # filter non fd entries (mainly pid?) + open_fds+=("${fd#f}") # cut off f prefix + ;; + esac + done + elif command -v procstat >/dev/null; then # BSDs + local -a columns header + procstat fds "$BASHPID" >"$tmpfile" + { + read -r -a header + local fd_column_index=-1 + for ((i = 0; i < ${#header[@]}; ++i)); do + if [[ ${header[$i]} == *FD* ]]; then + fd_column_index=$i + break + fi + done + if [[ $fd_column_index -eq -1 ]]; then + printf "Could not find FD column in procstat" >&2 exit 1 - fi + fi + while read -r -a columns; do + local fd=${columns[$fd_column_index]} + if [[ $fd == [0-9]* ]]; then # only take up numeric entries + open_fds+=("$fd") + fi + done + } <"$tmpfile" + else + # TODO: MSYS (Windows) + printf "Neither FD discovery mechanism available\n" >&2 + exit 1 + fi } function close_non_std_fds() { - local open_fds non_std_fds=() - get_open_fds - for fd in "${open_fds[@]}"; do - if [[ $fd -gt 2 ]]; then - non_std_fds+=("$fd") - fi - done - close_fds "${non_std_fds[@]}" + local open_fds non_std_fds=() + get_open_fds + for fd in "${open_fds[@]}"; do + if [[ $fd -gt 2 ]]; then + non_std_fds+=("$fd") + fi + done + close_fds "${non_std_fds[@]}" } function close_fds() { # - for fd in "$@"; do - eval "exec $fd>&-" - done + for fd in "$@"; do + eval "exec $fd>&-" + done } function otherfunc { - bgfunc & - PID=$! - disown - return 0 + bgfunc & + PID=$! + disown + return 0 } -setup_file (){ # see issue #530 - bgfunc & +setup_file() { # see issue #530 + bgfunc & } @test "min bg" { - echo "sec: $SECONDS" - otherfunc - sleep 1 # leave some space for the background job to print/fail early - kill -s 0 -- $PID # fail it the process already finished due to error! - echo "sec: $SECONDS" + echo "sec: $SECONDS" + otherfunc + sleep 1 # leave some space for the background job to print/fail early + kill -s 0 -- $PID # fail it the process already finished due to error! + echo "sec: $SECONDS" } diff --git a/test/fixtures/bats/issue-433/repro1.bats b/test/fixtures/bats/issue-433/repro1.bats index fc488e746c..d37d4b607e 100644 --- a/test/fixtures/bats/issue-433/repro1.bats +++ b/test/fixtures/bats/issue-433/repro1.bats @@ -1,8 +1,7 @@ @test "1" { - sleep 1 + sleep 1 } @test "2" { - sleep 1 + sleep 1 } - diff --git a/test/fixtures/bats/issue-433/repro2.bats b/test/fixtures/bats/issue-433/repro2.bats index 2863d959b9..92d37971fe 100644 --- a/test/fixtures/bats/issue-433/repro2.bats +++ b/test/fixtures/bats/issue-433/repro2.bats @@ -1,8 +1,7 @@ @test "11" { - sleep 1 + sleep 1 } @test "12" { - sleep 1 + sleep 1 } - diff --git a/test/fixtures/bats/issue-519.bats b/test/fixtures/bats/issue-519.bats index a28f38c4e1..307bbb5857 100644 --- a/test/fixtures/bats/issue-519.bats +++ b/test/fixtures/bats/issue-519.bats @@ -1,3 +1,3 @@ @test "no unprefixed variables" { - declare -p >"${BATS_DECLARED_VARIABLES_FILE?}" -} \ No newline at end of file + declare -p >"${BATS_DECLARED_VARIABLES_FILE?}" +} diff --git a/test/fixtures/bats/loop_keep_IFS.bats b/test/fixtures/bats/loop_keep_IFS.bats index 4af343dc30..a08c39c05f 100644 --- a/test/fixtures/bats/loop_keep_IFS.bats +++ b/test/fixtures/bats/loop_keep_IFS.bats @@ -3,7 +3,7 @@ loop_func() { local search="none one two tree" local d - for d in $search ; do + for d in $search; do echo "$d" done } diff --git a/test/fixtures/bats/many_passing_and_one_failing.bats b/test/fixtures/bats/many_passing_and_one_failing.bats new file mode 100644 index 0000000000..da487a1569 --- /dev/null +++ b/test/fixtures/bats/many_passing_and_one_failing.bats @@ -0,0 +1,15 @@ +@test "a passing test" { + true +} + +@test "another passing test" { + true +} + +@test "a failing test" { + false +} + +@test "yet another passing test" { + true +} diff --git a/test/fixtures/bats/no-final-newline.bats b/test/fixtures/bats/no-final-newline.bats index 09014f0eee..a7061a1dac 100644 --- a/test/fixtures/bats/no-final-newline.bats +++ b/test/fixtures/bats/no-final-newline.bats @@ -6,4 +6,4 @@ @test "test function returns nonzero" { printf 'foo\nbar' return 1 -} \ No newline at end of file +} diff --git a/test/fixtures/bats/print_output_on_failure.bats b/test/fixtures/bats/print_output_on_failure.bats index 793b972369..e7f9168fad 100644 --- a/test/fixtures/bats/print_output_on_failure.bats +++ b/test/fixtures/bats/print_output_on_failure.bats @@ -1,11 +1,11 @@ @test "no failure prints no output" { - run echo success + run echo success } bats_require_minimum_version 1.5.0 # don't be fooled by order, this will run before the test above! @test "failure prints output" { - run -1 echo "fail hard" + run -1 echo "fail hard" } @test "empty output on failure" { - false -} \ No newline at end of file + false +} diff --git a/test/fixtures/bats/print_output_on_failure_with_stderr.bats b/test/fixtures/bats/print_output_on_failure_with_stderr.bats new file mode 100644 index 0000000000..4ad5e412dc --- /dev/null +++ b/test/fixtures/bats/print_output_on_failure_with_stderr.bats @@ -0,0 +1,12 @@ +@test "no failure prints no output" { + run echo success +} + +@test "failure prints output" { + bats_require_minimum_version 1.5.0 + run -1 --separate-stderr bash -c 'echo "fail hard"; echo with stderr >&2' +} + +@test "empty output on failure" { + false +} diff --git a/test/fixtures/bats/read_from_stdin.bats b/test/fixtures/bats/read_from_stdin.bats index 0ff76e7035..7bb41b63d1 100644 --- a/test/fixtures/bats/read_from_stdin.bats +++ b/test/fixtures/bats/read_from_stdin.bats @@ -1,21 +1,21 @@ #!/usr/bin/env bats @test "test 1" { - # Don't print anything - run bash -c "$BATS_TEST_DIRNAME/cmd_using_stdin.bash" - [ "$status" -eq 1 ] - [ "$output" = "Not found" ] + # Don't print anything + run bash -c "$BATS_TEST_DIRNAME/cmd_using_stdin.bash" + [ "$status" -eq 1 ] + [ "$output" = "Not found" ] } @test "test 2 with TAB in name" { - run bash -c "echo EXIT | $BATS_TEST_DIRNAME/cmd_using_stdin.bash" - [ "$status" -eq 0 ] - echo "$output" - [ "$output" = "Found" ] + run bash -c "echo EXIT | $BATS_TEST_DIRNAME/cmd_using_stdin.bash" + [ "$status" -eq 0 ] + echo "$output" + [ "$output" = "Found" ] } @test "test 3" { - run bash -c "echo EXIT | $BATS_TEST_DIRNAME/cmd_using_stdin.bash" - [ "$status" -eq 0 ] - [ "$output" = "Found" ] + run bash -c "echo EXIT | $BATS_TEST_DIRNAME/cmd_using_stdin.bash" + [ "$status" -eq 0 ] + [ "$output" = "Found" ] } diff --git a/test/fixtures/bats/retry.bats b/test/fixtures/bats/retry.bats new file mode 100644 index 0000000000..085672841d --- /dev/null +++ b/test/fixtures/bats/retry.bats @@ -0,0 +1,38 @@ +BATS_TEST_RETRIES=2 # means three tries per test + +log_caller() { + printf "%s %s %s\n" "${BATS_TEST_NAME:-}" "${FUNCNAME[1]}" "${BATS_TEST_TRY_NUMBER:-}" >>"${LOG?}" +} + +setup_file() { + log_caller +} + +teardown_file() { + log_caller +} + +setup() { + log_caller +} + +teardown() { + log_caller +} + +@test "Fail all" { + log_caller + false +} + +@test "Fail once" { + log_caller + ((BATS_TEST_TRY_NUMBER > 1)) || false +} + +@test "Override retries" { + log_caller + # shellcheck disable=SC2034 + BATS_TEST_RETRIES=1 + ((BATS_TEST_TRY_NUMBER > 2)) || false +} diff --git a/test/fixtures/bats/retry_success.bats b/test/fixtures/bats/retry_success.bats new file mode 100644 index 0000000000..babfec831b --- /dev/null +++ b/test/fixtures/bats/retry_success.bats @@ -0,0 +1,6 @@ +# shellcheck disable=SC2034 +BATS_TEST_RETRIES=2 # means three tries per test + +@test "Fail once" { + ((BATS_TEST_TRY_NUMBER > 1)) || false +} \ No newline at end of file diff --git a/test/fixtures/bats/run_long_command.bats b/test/fixtures/bats/run_long_command.bats index c725f43b66..f49a8dee26 100644 --- a/test/fixtures/bats/run_long_command.bats +++ b/test/fixtures/bats/run_long_command.bats @@ -1,3 +1,3 @@ @test "run long command" { - run sleep 3 -} \ No newline at end of file + run sleep 3 +} diff --git a/test/fixtures/bats/set_-eu_in_setup_and_teardown.bats b/test/fixtures/bats/set_-eu_in_setup_and_teardown.bats index e9879f0ab5..49a25c9a5f 100644 --- a/test/fixtures/bats/set_-eu_in_setup_and_teardown.bats +++ b/test/fixtures/bats/set_-eu_in_setup_and_teardown.bats @@ -1,23 +1,23 @@ setup() { - set -eu + set -eu } teardown() { - set -eu + set -eu } @test "skipped test" { - skip + skip } @test "skipped test with reason" { - skip "reason" + skip "reason" } @test "passing test" { - run true + run true } @test "failing test" { - false -} \ No newline at end of file + false +} diff --git a/test/fixtures/bats/setup.bats b/test/fixtures/bats/setup.bats index 0fc9eea6ec..ee88b162fa 100644 --- a/test/fixtures/bats/setup.bats +++ b/test/fixtures/bats/setup.bats @@ -1,7 +1,7 @@ LOG="$BATS_TEST_SUITE_TMPDIR/setup.log" setup() { - echo "$BATS_TEST_NAME" >> "$LOG" + echo "$BATS_TEST_NAME" >>"$LOG" } @test "one" { diff --git a/test/fixtures/bats/show-output-of-passing-tests.bats b/test/fixtures/bats/show-output-of-passing-tests.bats index 421da017ca..acd8b2fe44 100644 --- a/test/fixtures/bats/show-output-of-passing-tests.bats +++ b/test/fixtures/bats/show-output-of-passing-tests.bats @@ -1,3 +1,3 @@ @test "test" { - echo output -} \ No newline at end of file + echo output +} diff --git a/test/fixtures/bats/sigint_in_failing_test.bats b/test/fixtures/bats/sigint_in_failing_test.bats new file mode 100644 index 0000000000..0755cc96ab --- /dev/null +++ b/test/fixtures/bats/sigint_in_failing_test.bats @@ -0,0 +1,11 @@ +@test "failing" { + if [[ -z "${DONT_ABORT:-}" ]]; then + # emulate CTRL-C by sending SIGINT to the whole process group + kill -SIGINT -- -"$BATS_ROOT_PID" + fi + false +} + +@test "passing" { + : +} diff --git a/test/fixtures/bats/single_line_no_shellcheck.bats b/test/fixtures/bats/single_line_no_shellcheck.bats index fc342d9666..060d77224f 100644 --- a/test/fixtures/bats/single_line_no_shellcheck.bats +++ b/test/fixtures/bats/single_line_no_shellcheck.bats @@ -2,7 +2,7 @@ @test "passing" { true; } -@test "input redirection" { diff - <( echo hello ); } <> "$LOG" + echo "$BATS_TEST_NAME" >>"$LOG" } @test "one" { diff --git a/test/fixtures/bats/teardown_file_override_status.bats b/test/fixtures/bats/teardown_file_override_status.bats index 61b6de6b0e..9717d94c79 100644 --- a/test/fixtures/bats/teardown_file_override_status.bats +++ b/test/fixtures/bats/teardown_file_override_status.bats @@ -1,9 +1,9 @@ teardown_file() { - # shellcheck disable=SC2034 - status=${STATUS?} - return "${TEARDOWN_RETURN_CODE?}" + # shellcheck disable=SC2034 + status=${STATUS?} + return "${TEARDOWN_RETURN_CODE?}" } @test "return expected code" { - return "${TEST_RETURN_CODE?}" -} \ No newline at end of file + return "${TEST_RETURN_CODE?}" +} diff --git a/test/fixtures/bats/teardown_override_status.bats b/test/fixtures/bats/teardown_override_status.bats index b45996cc05..69fb4200a7 100644 --- a/test/fixtures/bats/teardown_override_status.bats +++ b/test/fixtures/bats/teardown_override_status.bats @@ -1,8 +1,8 @@ teardown() { - status=${STATUS?} - return "${TEARDOWN_RETURN_CODE?}" + status=${STATUS?} + return "${TEARDOWN_RETURN_CODE?}" } @test "return expected code" { - return "${TEST_RETURN_CODE?}" -} \ No newline at end of file + return "${TEST_RETURN_CODE?}" +} diff --git a/test/fixtures/bats/teardown_suite_override_status/setup_suite.bash b/test/fixtures/bats/teardown_suite_override_status/setup_suite.bash index 0eb3b0cd9a..79a1c76231 100644 --- a/test/fixtures/bats/teardown_suite_override_status/setup_suite.bash +++ b/test/fixtures/bats/teardown_suite_override_status/setup_suite.bash @@ -1,9 +1,9 @@ setup_suite() { - : + : } teardown_suite() { - # shellcheck disable=SC2034 - status=${STATUS?} - return "${TEARDOWN_RETURN_CODE?}" -} \ No newline at end of file + # shellcheck disable=SC2034 + status=${STATUS?} + return "${TEARDOWN_RETURN_CODE?}" +} diff --git a/test/fixtures/bats/teardown_suite_override_status/test.bats b/test/fixtures/bats/teardown_suite_override_status/test.bats index 7c1c779209..4cde17a383 100644 --- a/test/fixtures/bats/teardown_suite_override_status/test.bats +++ b/test/fixtures/bats/teardown_suite_override_status/test.bats @@ -1,3 +1,3 @@ @test test { - return "${TEST_RETURN_CODE?}" -} \ No newline at end of file + return "${TEST_RETURN_CODE?}" +} diff --git a/test/fixtures/bats/unbound_variable.bats b/test/fixtures/bats/unbound_variable.bats index 4b8c2d956c..a8219fa14c 100644 --- a/test/fixtures/bats/unbound_variable.bats +++ b/test/fixtures/bats/unbound_variable.bats @@ -1,16 +1,16 @@ - set -u +set -u # This file is used to test line number offsets. Any changes to lines will affect tests @test "access unbound variable" { - unset unset_variable - # Add a line for checking line number - # shellcheck disable=SC2154 - foo=$unset_variable + unset unset_variable + # Add a line for checking line number + # shellcheck disable=SC2154 + foo=$unset_variable } @test "access second unbound variable" { - unset second_unset_variable - # shellcheck disable=SC2034,SC2154 - foo=$second_unset_variable + unset second_unset_variable + # shellcheck disable=SC2034,SC2154 + foo=$second_unset_variable } diff --git a/test/fixtures/bats/verbose-run.bats b/test/fixtures/bats/verbose-run.bats index de2942b507..1fb2d00fcb 100644 --- a/test/fixtures/bats/verbose-run.bats +++ b/test/fixtures/bats/verbose-run.bats @@ -1,4 +1,4 @@ @test "test" { - bats_require_minimum_version 1.5.0 - run ! echo test -} \ No newline at end of file + bats_require_minimum_version 1.5.0 + run ! echo test +} diff --git a/test/fixtures/bats/without_trailing_newline.bats b/test/fixtures/bats/without_trailing_newline.bats index e3ace8b578..fbc1f3833a 100644 --- a/test/fixtures/bats/without_trailing_newline.bats +++ b/test/fixtures/bats/without_trailing_newline.bats @@ -1,3 +1,3 @@ @test "truth" { true -} \ No newline at end of file +} diff --git a/test/fixtures/file_setup_teardown/error_in_setup_and_teardown_file.bats b/test/fixtures/file_setup_teardown/error_in_setup_and_teardown_file.bats new file mode 100644 index 0000000000..c0a2582093 --- /dev/null +++ b/test/fixtures/file_setup_teardown/error_in_setup_and_teardown_file.bats @@ -0,0 +1,11 @@ +setup_file() { + false +} + +teardown_file() { + false +} + +@test dummy { + : +} diff --git a/test/fixtures/file_setup_teardown/no_setup_file.bats b/test/fixtures/file_setup_teardown/no_setup_file.bats index f66092a863..5e6619ea0e 100644 --- a/test/fixtures/file_setup_teardown/no_setup_file.bats +++ b/test/fixtures/file_setup_teardown/no_setup_file.bats @@ -1,3 +1,3 @@ @test "test" { - true -} \ No newline at end of file + true +} diff --git a/test/fixtures/file_setup_teardown/no_teardown_file.bats b/test/fixtures/file_setup_teardown/no_teardown_file.bats index 55379e4a1a..e9e48c36f7 100644 --- a/test/fixtures/file_setup_teardown/no_teardown_file.bats +++ b/test/fixtures/file_setup_teardown/no_teardown_file.bats @@ -1,3 +1,3 @@ @test "first" { - true -} \ No newline at end of file + true +} diff --git a/test/fixtures/file_setup_teardown/setup_file.bats b/test/fixtures/file_setup_teardown/setup_file.bats index 3a84ca897a..116f7e0146 100644 --- a/test/fixtures/file_setup_teardown/setup_file.bats +++ b/test/fixtures/file_setup_teardown/setup_file.bats @@ -1,11 +1,11 @@ setup_file() { - echo "$BATS_TEST_FILENAME" >> "$LOG" + echo "$BATS_TEST_FILENAME" >>"$LOG" } @test "Test 1" { - true + true } @test "Test 2" { - true -} \ No newline at end of file + true +} diff --git a/test/fixtures/file_setup_teardown/setup_file2.bats b/test/fixtures/file_setup_teardown/setup_file2.bats index bd9fd59244..0ea1de67bf 100644 --- a/test/fixtures/file_setup_teardown/setup_file2.bats +++ b/test/fixtures/file_setup_teardown/setup_file2.bats @@ -1,7 +1,7 @@ setup_file() { - echo "$BATS_TEST_FILENAME" >> "$LOG" + echo "$BATS_TEST_FILENAME" >>"$LOG" } @test "test" { - true -} \ No newline at end of file + true +} diff --git a/test/fixtures/file_setup_teardown/setup_file_does_not_leak_env.bats b/test/fixtures/file_setup_teardown/setup_file_does_not_leak_env.bats index 0db3d9cfa0..44ae813601 100644 --- a/test/fixtures/file_setup_teardown/setup_file_does_not_leak_env.bats +++ b/test/fixtures/file_setup_teardown/setup_file_does_not_leak_env.bats @@ -1,7 +1,7 @@ setup_file() { - export SETUP_FILE_VAR="$BATS_TEST_FILENAME" + export SETUP_FILE_VAR="$BATS_TEST_FILENAME" } @test "test" { - true -} \ No newline at end of file + true +} diff --git a/test/fixtures/file_setup_teardown/setup_file_does_not_leak_env2.bats b/test/fixtures/file_setup_teardown/setup_file_does_not_leak_env2.bats index cb71b3adbd..fc10d6e204 100644 --- a/test/fixtures/file_setup_teardown/setup_file_does_not_leak_env2.bats +++ b/test/fixtures/file_setup_teardown/setup_file_does_not_leak_env2.bats @@ -1,3 +1,3 @@ @test "test" { - [[ "${SETUP_FILE_VAR-"NOT_SET"}" == "NOT_SET" ]] -} \ No newline at end of file + [[ "${SETUP_FILE_VAR-"NOT_SET"}" == "NOT_SET" ]] +} diff --git a/test/fixtures/file_setup_teardown/setup_file_even_if_all_tests_are_skipped.bats b/test/fixtures/file_setup_teardown/setup_file_even_if_all_tests_are_skipped.bats index ae339957a5..791b14830b 100644 --- a/test/fixtures/file_setup_teardown/setup_file_even_if_all_tests_are_skipped.bats +++ b/test/fixtures/file_setup_teardown/setup_file_even_if_all_tests_are_skipped.bats @@ -1,7 +1,7 @@ setup_file() { - echo "$BATS_TEST_FILENAME" >> "$LOG" + echo "$BATS_TEST_FILENAME" >>"$LOG" } @test "test" { - skip "We only want to see if setup file runs" -} \ No newline at end of file + skip "We only want to see if setup file runs" +} diff --git a/test/fixtures/file_setup_teardown/setup_file_failed.bats b/test/fixtures/file_setup_teardown/setup_file_failed.bats index 2a8e3287b1..4811c513e9 100644 --- a/test/fixtures/file_setup_teardown/setup_file_failed.bats +++ b/test/fixtures/file_setup_teardown/setup_file_failed.bats @@ -1,11 +1,11 @@ setup_file() { - false + false } @test "test 1" { - true + true } @test "test 2" { - true -} \ No newline at end of file + true +} diff --git a/test/fixtures/file_setup_teardown/setup_file_halfway_error.bats b/test/fixtures/file_setup_teardown/setup_file_halfway_error.bats index 929bf29929..3a156a54fc 100644 --- a/test/fixtures/file_setup_teardown/setup_file_halfway_error.bats +++ b/test/fixtures/file_setup_teardown/setup_file_halfway_error.bats @@ -1,9 +1,9 @@ setup_file() { - true - false - true + true + false + true } @test "test" { - true -} \ No newline at end of file + true +} diff --git a/test/fixtures/file_setup_teardown/teardown_file.bats b/test/fixtures/file_setup_teardown/teardown_file.bats index 50886fb71e..7ae78d1121 100644 --- a/test/fixtures/file_setup_teardown/teardown_file.bats +++ b/test/fixtures/file_setup_teardown/teardown_file.bats @@ -1,11 +1,11 @@ teardown_file() { - echo "$BATS_TEST_FILENAME" >> "$LOG" + echo "$BATS_TEST_FILENAME" >>"$LOG" } @test "first" { - true + true } @test "second" { - true -} \ No newline at end of file + true +} diff --git a/test/fixtures/file_setup_teardown/teardown_file2.bats b/test/fixtures/file_setup_teardown/teardown_file2.bats index 50886fb71e..7ae78d1121 100644 --- a/test/fixtures/file_setup_teardown/teardown_file2.bats +++ b/test/fixtures/file_setup_teardown/teardown_file2.bats @@ -1,11 +1,11 @@ teardown_file() { - echo "$BATS_TEST_FILENAME" >> "$LOG" + echo "$BATS_TEST_FILENAME" >>"$LOG" } @test "first" { - true + true } @test "second" { - true -} \ No newline at end of file + true +} diff --git a/test/fixtures/file_setup_teardown/teardown_file_after_failing_test.bats b/test/fixtures/file_setup_teardown/teardown_file_after_failing_test.bats index 05d8b14de9..3e8fb54ac6 100644 --- a/test/fixtures/file_setup_teardown/teardown_file_after_failing_test.bats +++ b/test/fixtures/file_setup_teardown/teardown_file_after_failing_test.bats @@ -1,7 +1,7 @@ teardown_file() { - echo "$BATS_TEST_FILENAME" >> "$LOG" + echo "$BATS_TEST_FILENAME" >>"$LOG" } @test "failing test" { - false -} \ No newline at end of file + false +} diff --git a/test/fixtures/file_setup_teardown/teardown_file_after_long_test.bats b/test/fixtures/file_setup_teardown/teardown_file_after_long_test.bats index c3acdf1ae3..c266c462e3 100644 --- a/test/fixtures/file_setup_teardown/teardown_file_after_long_test.bats +++ b/test/fixtures/file_setup_teardown/teardown_file_after_long_test.bats @@ -1,8 +1,8 @@ teardown_file() { - echo "$BATS_TEST_FILENAME" >> "$LOG" + echo "$BATS_TEST_FILENAME" >>"$LOG" } @test "long running test" { - sleep 10 - echo "test finished successfully" >> "$LOG" -} \ No newline at end of file + sleep 10 + echo "test finished successfully" >>"$LOG" +} diff --git a/test/fixtures/file_setup_teardown/teardown_file_does_not_leak.bats b/test/fixtures/file_setup_teardown/teardown_file_does_not_leak.bats index cf620cdffc..9288045213 100644 --- a/test/fixtures/file_setup_teardown/teardown_file_does_not_leak.bats +++ b/test/fixtures/file_setup_teardown/teardown_file_does_not_leak.bats @@ -1,7 +1,7 @@ teardown_file() { - export POTENTIALLY_LEAKING_VARIABLE="$BATS_TEST_FILENAME" + export POTENTIALLY_LEAKING_VARIABLE="$BATS_TEST_FILENAME" } @test "test" { - true -} \ No newline at end of file + true +} diff --git a/test/fixtures/file_setup_teardown/teardown_file_does_not_leak2.bats b/test/fixtures/file_setup_teardown/teardown_file_does_not_leak2.bats index 798bc8cc1f..2af74fd3f3 100644 --- a/test/fixtures/file_setup_teardown/teardown_file_does_not_leak2.bats +++ b/test/fixtures/file_setup_teardown/teardown_file_does_not_leak2.bats @@ -1,3 +1,3 @@ @test "must not see variable from first run" { - [[ -z "$POTENTIALLY_LEAKING_VARIABLE" ]] -} \ No newline at end of file + [[ -z "${POTENTIALLY_LEAKING_VARIABLE:-}" ]] +} diff --git a/test/fixtures/file_setup_teardown/teardown_file_even_if_all_tests_are_skipped.bats b/test/fixtures/file_setup_teardown/teardown_file_even_if_all_tests_are_skipped.bats index ba63ece26d..6263ffa673 100644 --- a/test/fixtures/file_setup_teardown/teardown_file_even_if_all_tests_are_skipped.bats +++ b/test/fixtures/file_setup_teardown/teardown_file_even_if_all_tests_are_skipped.bats @@ -1,7 +1,7 @@ teardown_file() { - echo "$BATS_TEST_FILENAME" >> "$LOG" + echo "$BATS_TEST_FILENAME" >>"$LOG" } @test "skipped test" { - skip 'All tests in this file are skipped! Teardown_file runs anyways' -} \ No newline at end of file + skip 'All tests in this file are skipped! Teardown_file runs anyways' +} diff --git a/test/fixtures/file_setup_teardown/teardown_file_failed.bats b/test/fixtures/file_setup_teardown/teardown_file_failed.bats index 627b1201f6..14b22f74e5 100644 --- a/test/fixtures/file_setup_teardown/teardown_file_failed.bats +++ b/test/fixtures/file_setup_teardown/teardown_file_failed.bats @@ -1,8 +1,7 @@ - teardown_file() { - false + false } @test "test" { - true -} \ No newline at end of file + true +} diff --git a/test/fixtures/file_setup_teardown/teardown_file_halfway_error.bats b/test/fixtures/file_setup_teardown/teardown_file_halfway_error.bats index 1337805826..326b16a99e 100644 --- a/test/fixtures/file_setup_teardown/teardown_file_halfway_error.bats +++ b/test/fixtures/file_setup_teardown/teardown_file_halfway_error.bats @@ -1,9 +1,9 @@ teardown_file() { - true - false - true + true + false + true } @test "empty" { - true -} \ No newline at end of file + true +} diff --git a/test/fixtures/formatter/dummy-formatter b/test/fixtures/formatter/dummy-formatter new file mode 100755 index 0000000000..255acc0544 --- /dev/null +++ b/test/fixtures/formatter/dummy-formatter @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +cat >/dev/null # ignore input + +echo "Dummy Formatter!" diff --git a/test/fixtures/formatter/failing.bats b/test/fixtures/formatter/failing.bats new file mode 100644 index 0000000000..13be78cfb3 --- /dev/null +++ b/test/fixtures/formatter/failing.bats @@ -0,0 +1,5 @@ +@test "a failing test" { + true + true + eval "( exit ${STATUS:-1} )" +} diff --git a/test/fixtures/formatter/passing.bats b/test/fixtures/formatter/passing.bats new file mode 100644 index 0000000000..e8182ce094 --- /dev/null +++ b/test/fixtures/formatter/passing.bats @@ -0,0 +1,3 @@ +@test "a passing test" { + true +} diff --git a/test/fixtures/formatter/passing_and_skipping.bats b/test/fixtures/formatter/passing_and_skipping.bats new file mode 100644 index 0000000000..83fc76bfca --- /dev/null +++ b/test/fixtures/formatter/passing_and_skipping.bats @@ -0,0 +1,11 @@ +@test "a passing test" { + true +} + +@test "a skipped test with no reason" { + skip +} + +@test "a skipped test with a reason" { + skip "for a really good reason" +} diff --git a/test/fixtures/formatter/passing_failing_and_skipping.bats b/test/fixtures/formatter/passing_failing_and_skipping.bats new file mode 100644 index 0000000000..3c9f17fa93 --- /dev/null +++ b/test/fixtures/formatter/passing_failing_and_skipping.bats @@ -0,0 +1,11 @@ +@test "a passing test" { + true +} + +@test "a skipping test" { + skip +} + +@test "a failing test" { + false +} diff --git a/test/fixtures/formatter/skipped_with_parens.bats b/test/fixtures/formatter/skipped_with_parens.bats new file mode 100644 index 0000000000..3f8f628357 --- /dev/null +++ b/test/fixtures/formatter/skipped_with_parens.bats @@ -0,0 +1,3 @@ +@test "a skipped test with parentheses in the reason" { + skip "a reason (with parentheses)" +} diff --git a/test/fixtures/junit-formatter/duplicate/first/file1.bats b/test/fixtures/junit-formatter/duplicate/first/file1.bats index e1df58729d..3084334434 100644 --- a/test/fixtures/junit-formatter/duplicate/first/file1.bats +++ b/test/fixtures/junit-formatter/duplicate/first/file1.bats @@ -1,3 +1,3 @@ @test "test in duplicate file" { - true -} \ No newline at end of file + true +} diff --git a/test/fixtures/junit-formatter/duplicate/second/file1.bats b/test/fixtures/junit-formatter/duplicate/second/file1.bats index e1df58729d..3084334434 100644 --- a/test/fixtures/junit-formatter/duplicate/second/file1.bats +++ b/test/fixtures/junit-formatter/duplicate/second/file1.bats @@ -1,3 +1,3 @@ @test "test in duplicate file" { - true -} \ No newline at end of file + true +} diff --git a/test/fixtures/junit-formatter/issue_360.bats b/test/fixtures/junit-formatter/issue_360.bats index f4bd850bdb..3cf103636d 100644 --- a/test/fixtures/junit-formatter/issue_360.bats +++ b/test/fixtures/junit-formatter/issue_360.bats @@ -1,22 +1,22 @@ #!/usr/bin/env bats setup() { - echo "# setup stdout" - echo "# setup FD3" >&3 + echo "# setup stdout" + echo "# setup FD3" >&3 } teardown() { - echo "# teardown stdout" - echo "# teardown FD3" >&3 + echo "# teardown stdout" + echo "# teardown FD3" >&3 } @test "say hello to Biblo" { - echo "# hello stdout" - echo "# hello Bilbo" >&3 + echo "# hello stdout" + echo "# hello Bilbo" >&3 } @test "fail to say hello to Biblo" { - echo "# hello stdout" - echo "# hello Bilbo" >&3 - false -} \ No newline at end of file + echo "# hello stdout" + echo "# hello Bilbo" >&3 + false +} diff --git a/test/fixtures/junit-formatter/issue_531.bats b/test/fixtures/junit-formatter/issue_531.bats index b4add812ae..d7135e027b 100644 --- a/test/fixtures/junit-formatter/issue_531.bats +++ b/test/fixtures/junit-formatter/issue_531.bats @@ -1,16 +1,16 @@ #!/usr/bin/env bats setup_file() { - echo "# setup_file stdout" - echo "# setup_file fd3" >&3 + echo "# setup_file stdout" + echo "# setup_file fd3" >&3 } teardown_file() { - echo "# teardown_file stdout" - echo "# teardown_file fd3" >&3 + echo "# teardown_file stdout" + echo "# teardown_file fd3" >&3 } @test "My test" { - echo "# test stdout" - echo "# test fd3" >&3 -} \ No newline at end of file + echo "# test stdout" + echo "# test fd3" >&3 +} diff --git a/test/fixtures/junit-formatter/skipped.bats b/test/fixtures/junit-formatter/skipped.bats index aeedda181a..3a37a6776f 100644 --- a/test/fixtures/junit-formatter/skipped.bats +++ b/test/fixtures/junit-formatter/skipped.bats @@ -6,4 +6,4 @@ @test "a skipped test with a reason" { skip "a reason" -} \ No newline at end of file +} diff --git a/test/fixtures/junit-formatter/suite/file1.bats b/test/fixtures/junit-formatter/suite/file1.bats index e2be967be4..f39c513874 100644 --- a/test/fixtures/junit-formatter/suite/file1.bats +++ b/test/fixtures/junit-formatter/suite/file1.bats @@ -1,3 +1,3 @@ @test "one test" { - true -} \ No newline at end of file + true +} diff --git a/test/fixtures/junit-formatter/suite/file2.bats b/test/fixtures/junit-formatter/suite/file2.bats index 7f6bd39558..624efe4ee8 100644 --- a/test/fixtures/junit-formatter/suite/file2.bats +++ b/test/fixtures/junit-formatter/suite/file2.bats @@ -1,3 +1,3 @@ @test "another test" { - true -} \ No newline at end of file + true +} diff --git a/test/fixtures/junit-formatter/xml-escape.bats b/test/fixtures/junit-formatter/xml-escape.bats index 71a8101d0f..da0b1e00ad 100644 --- a/test/fixtures/junit-formatter/xml-escape.bats +++ b/test/fixtures/junit-formatter/xml-escape.bats @@ -8,4 +8,4 @@ @test "Skipped test with escape characters: \"'<>& (0x1b)" { skip "\"'<>&" -} \ No newline at end of file +} diff --git a/test/fixtures/load/bats_load_library.bats b/test/fixtures/load/bats_load_library.bats index 5f9b566915..69bd1f723e 100644 --- a/test/fixtures/load/bats_load_library.bats +++ b/test/fixtures/load/bats_load_library.bats @@ -1,4 +1,4 @@ -[ -n "$HELPER_NAME" ] || HELPER_NAME="test_helper" +[ -n "${HELPER_NAME:-}" ] || HELPER_NAME="test_helper" bats_load_library "$HELPER_NAME" @test "calling a loaded helper" { diff --git a/test/fixtures/load/failing_bats_load_library.bats b/test/fixtures/load/failing_bats_load_library.bats index 3bff0baf24..0adb3be623 100644 --- a/test/fixtures/load/failing_bats_load_library.bats +++ b/test/fixtures/load/failing_bats_load_library.bats @@ -1,5 +1,5 @@ bats_load_library "return1" @test "true" { - true + true } diff --git a/test/fixtures/load/failing_load.bats b/test/fixtures/load/failing_load.bats index f17935ca81..f437dacdf6 100644 --- a/test/fixtures/load/failing_load.bats +++ b/test/fixtures/load/failing_load.bats @@ -1,5 +1,5 @@ load "return1" @test "true" { - true + true } diff --git a/test/fixtures/load/find_library_helper.bats b/test/fixtures/load/find_library_helper.bats index d42b7458ac..a95cd04536 100644 --- a/test/fixtures/load/find_library_helper.bats +++ b/test/fixtures/load/find_library_helper.bats @@ -1,5 +1,5 @@ @test "find a library" { - run find_in_bats_lib_path "$LIBRARY_NAME" - [ $status -eq 0 ] - [ "${lines[0]}" = "$LIBRARY_PATH" ] + run find_in_bats_lib_path "$LIBRARY_NAME" + [ $status -eq 0 ] + [ "${lines[0]}" = "$LIBRARY_PATH" ] } diff --git a/test/fixtures/load/find_library_helper_err.bats b/test/fixtures/load/find_library_helper_err.bats index 0ff3345743..e38491dd6f 100644 --- a/test/fixtures/load/find_library_helper_err.bats +++ b/test/fixtures/load/find_library_helper_err.bats @@ -1,4 +1,4 @@ @test "does not find a library" { - run find_in_bats_lib_path "$LIBRARY_NAME" - [ $status -eq 1 ] + run find_in_bats_lib_path "$LIBRARY_NAME" + [ $status -eq 1 ] } diff --git a/test/fixtures/load/load.bats b/test/fixtures/load/load.bats index 975b6b8c1b..ba656c68bc 100644 --- a/test/fixtures/load/load.bats +++ b/test/fixtures/load/load.bats @@ -1,4 +1,4 @@ -[ -n "$HELPER_NAME" ] || HELPER_NAME="test_helper" +[ -n "${HELPER_NAME:-}" ] || HELPER_NAME="test_helper" load "$HELPER_NAME" @test "calling a loaded helper" { diff --git a/test/fixtures/load/load_in_teardown_after_failure.bats b/test/fixtures/load/load_in_teardown_after_failure.bats new file mode 100644 index 0000000000..a24d6dc7d3 --- /dev/null +++ b/test/fixtures/load/load_in_teardown_after_failure.bats @@ -0,0 +1,7 @@ +teardown() { + load 'test_helper' +} + +@test failed { + false +} diff --git a/test/fixtures/parallel/must_not_parallelize_across_files/file1.bats b/test/fixtures/parallel/must_not_parallelize_across_files/file1.bats index 144390ed39..86c96a0d5a 100644 --- a/test/fixtures/parallel/must_not_parallelize_across_files/file1.bats +++ b/test/fixtures/parallel/must_not_parallelize_across_files/file1.bats @@ -1,14 +1,14 @@ setup() { - echo "start $BATS_TEST_NAME" >> "$FILE_MARKER" + echo "start $BATS_TEST_NAME" >>"$FILE_MARKER" } teardown() { - echo "end $BATS_TEST_NAME" >> "$FILE_MARKER" + echo "end $BATS_TEST_NAME" >>"$FILE_MARKER" } @test "test 1" { - # stretch the time this test runs to prevent accidental serialization by the scheduler - # if both tests could run in parallel, this will increase the likelihood of detecting it - # by delaying this test's teardown past the other's - sleep 3 + # stretch the time this test runs to prevent accidental serialization by the scheduler + # if both tests could run in parallel, this will increase the likelihood of detecting it + # by delaying this test's teardown past the other's + sleep 3 } diff --git a/test/fixtures/parallel/must_not_parallelize_across_files/file2.bats b/test/fixtures/parallel/must_not_parallelize_across_files/file2.bats index 6ec54bc570..df29fd918d 100644 --- a/test/fixtures/parallel/must_not_parallelize_across_files/file2.bats +++ b/test/fixtures/parallel/must_not_parallelize_across_files/file2.bats @@ -1,19 +1,19 @@ setup() { - echo "start $BATS_TEST_NAME" >> "$FILE_MARKER" + echo "start $BATS_TEST_NAME" >>"$FILE_MARKER" } teardown() { - echo "end $BATS_TEST_NAME" >> "$FILE_MARKER" + echo "end $BATS_TEST_NAME" >>"$FILE_MARKER" } @test "test 2" { - run cat "$FILE_MARKER" - echo "$output" + run cat "$FILE_MARKER" + echo "$output" - # assuming serialized, ordered execution we will always see the first test start and end before this runs - [[ "${lines[0]}" == "start"* ]] - OTHER_TEST_NAME="${lines[0]:6}" - [[ "$OTHER_TEST_NAME" != "$BATS_TEST_NAME" ]] - [[ "${lines[1]}" == "end $OTHER_TEST_NAME" ]] - [[ "${lines[2]}" == "start $BATS_TEST_NAME" ]] -} \ No newline at end of file + # assuming serialized, ordered execution we will always see the first test start and end before this runs + [[ "${lines[0]}" == "start"* ]] + OTHER_TEST_NAME="${lines[0]:6}" + [[ "$OTHER_TEST_NAME" != "$BATS_TEST_NAME" ]] + [[ "${lines[1]}" == "end $OTHER_TEST_NAME" ]] + [[ "${lines[2]}" == "start $BATS_TEST_NAME" ]] +} diff --git a/test/fixtures/parallel/must_not_parallelize_within_file.bats b/test/fixtures/parallel/must_not_parallelize_within_file.bats index fbf97d8d23..b8380df16d 100644 --- a/test/fixtures/parallel/must_not_parallelize_within_file.bats +++ b/test/fixtures/parallel/must_not_parallelize_within_file.bats @@ -1,52 +1,52 @@ setup_file() { - export FILE_MARKER - FILE_MARKER=$(mktemp "${BATS_RUN_TMPDIR}/file_marker.XXXXXX") - if [[ -n "${DISABLE_IN_SETUP_FILE_FUNCTION}" ]]; then - export BATS_NO_PARALLELIZE_WITHIN_FILE=true - echo "setup_file() sets BATS_NO_PARALLELIZE_WITHIN_FILE=true" >&2 - fi + export FILE_MARKER + FILE_MARKER=$(mktemp "${BATS_RUN_TMPDIR}/file_marker.XXXXXX") + if [[ -n "${DISABLE_IN_SETUP_FILE_FUNCTION:-}" ]]; then + export BATS_NO_PARALLELIZE_WITHIN_FILE=true + echo "setup_file() sets BATS_NO_PARALLELIZE_WITHIN_FILE=true" >&2 + fi } -if [[ -n "${DISABLE_OUTSIDE_ALL_FUNCTIONS}" ]]; then - export BATS_NO_PARALLELIZE_WITHIN_FILE=true - echo "File sets BATS_NO_PARALLELIZE_WITHIN_FILE=true" >&2 +if [[ -n "${DISABLE_OUTSIDE_ALL_FUNCTIONS:-}" ]]; then + export BATS_NO_PARALLELIZE_WITHIN_FILE=true + echo "File sets BATS_NO_PARALLELIZE_WITHIN_FILE=true" >&2 fi teardown_file() { - rm "$FILE_MARKER" + rm "$FILE_MARKER" } setup() { - if [[ -n "${DISABLE_IN_SETUP_FUNCTION}" ]]; then - export BATS_NO_PARALLELIZE_WITHIN_FILE=true - echo "setup() sets BATS_NO_PARALLELIZE_WITHIN_FILE=true" >&3 - fi - echo "start $BATS_TEST_NAME" >> "$FILE_MARKER" + if [[ -n "${DISABLE_IN_SETUP_FUNCTION:-}" ]]; then + export BATS_NO_PARALLELIZE_WITHIN_FILE=true + echo "setup() sets BATS_NO_PARALLELIZE_WITHIN_FILE=true" >&3 + fi + echo "start $BATS_TEST_NAME" >>"$FILE_MARKER" } teardown() { - echo "end $BATS_TEST_NAME" >> "$FILE_MARKER" + echo "end $BATS_TEST_NAME" >>"$FILE_MARKER" } @test "test 1" { - if [[ -n "${DISABLE_IN_TEST_FUNCTION}" ]]; then - export BATS_NO_PARALLELIZE_WITHIN_FILE=true - echo "Test function sets BATS_NO_PARALLELIZE_WITHIN_FILE=true" >&3 - fi - # stretch the time this test runs to prevent accidental serialization by the scheduler - # if both tests could run in parallel, this will increase the likelyhood of detecting it - # by delaying this test's teardown past the other's - sleep 3 + if [[ -n "${DISABLE_IN_TEST_FUNCTION:-}" ]]; then + export BATS_NO_PARALLELIZE_WITHIN_FILE=true + echo "Test function sets BATS_NO_PARALLELIZE_WITHIN_FILE=true" >&3 + fi + # stretch the time this test runs to prevent accidental serialization by the scheduler + # if both tests could run in parallel, this will increase the likelihood of detecting it + # by delaying this test's teardown past the other's + sleep 3 } @test "test 2" { - run cat "$FILE_MARKER" - echo "$output" + run cat "$FILE_MARKER" + echo "$output" - # assuming serialized, ordered execution we will always see the first test start and end before this runs - [[ "${lines[0]}" == "start"* ]] - OTHER_TEST_NAME="${lines[0]:6}" - [[ "$OTHER_TEST_NAME" != "$BATS_TEST_NAME" ]] - [[ "${lines[1]}" == "end $OTHER_TEST_NAME" ]] - [[ "${lines[2]}" == "start $BATS_TEST_NAME" ]] + # assuming serialized, ordered execution we will always see the first test start and end before this runs + [[ "${lines[0]}" == "start"* ]] + OTHER_TEST_NAME="${lines[0]:6}" + [[ "$OTHER_TEST_NAME" != "$BATS_TEST_NAME" ]] + [[ "${lines[1]}" == "end $OTHER_TEST_NAME" ]] + [[ "${lines[2]}" == "start $BATS_TEST_NAME" ]] } diff --git a/test/fixtures/parallel/parallel-preserve-environment.bats b/test/fixtures/parallel/parallel-preserve-environment.bats index c40322b2db..729e1c66cc 100644 --- a/test/fixtures/parallel/parallel-preserve-environment.bats +++ b/test/fixtures/parallel/parallel-preserve-environment.bats @@ -1,8 +1,8 @@ setup_file() { - export OTHER_ENV_VARIABLE='my-value' + export OTHER_ENV_VARIABLE='my-value' } @test "check env variables are set" { - [[ "$TEST_ENV_VARIABLE" == "test-value" ]] - [[ "$OTHER_ENV_VARIABLE" == "my-value" ]] -} \ No newline at end of file + [[ "$TEST_ENV_VARIABLE" == "test-value" ]] + [[ "$OTHER_ENV_VARIABLE" == "my-value" ]] +} diff --git a/test/fixtures/parallel/parallel.bats b/test/fixtures/parallel/parallel.bats index 5a3eba4386..046a165f09 100644 --- a/test/fixtures/parallel/parallel.bats +++ b/test/fixtures/parallel/parallel.bats @@ -1,10 +1,10 @@ setup() { load '../../concurrent-coordination' - echo "start $BATS_TEST_NAME $BATS_TEST_FILENAME" >> "$FILE_MARKER" + echo "start $BATS_TEST_NAME $BATS_TEST_FILENAME" >>"$FILE_MARKER" } teardown() { - echo "stop $BATS_TEST_NAME $BATS_TEST_FILENAME" >> "$FILE_MARKER" + echo "stop $BATS_TEST_NAME $BATS_TEST_FILENAME" >>"$FILE_MARKER" } @test "slow test 1" { @@ -17,4 +17,4 @@ teardown() { @test "slow test 3" { single-use-barrier "parallel" "$PARALLELITY" -} \ No newline at end of file +} diff --git a/test/fixtures/parallel/parallel_factor.bats b/test/fixtures/parallel/parallel_factor.bats index f687fbdf5f..67b83f81e7 100644 --- a/test/fixtures/parallel/parallel_factor.bats +++ b/test/fixtures/parallel/parallel_factor.bats @@ -1,10 +1,10 @@ setup() { load '../../concurrent-coordination' - echo setup "$BATS_TEST_NUMBER" >> "${MARKER_FILE?}" + echo setup "$BATS_TEST_NUMBER" >>"${MARKER_FILE?}" } teardown() { - echo teardown "$BATS_TEST_NUMBER" >> "${MARKER_FILE?}" + echo teardown "$BATS_TEST_NUMBER" >>"${MARKER_FILE?}" } @test "slow test 1" { diff --git a/test/fixtures/parallel/setup_file/setup_file.bats b/test/fixtures/parallel/setup_file/setup_file.bats index 2f687f93f1..40c8a5ba64 100644 --- a/test/fixtures/parallel/setup_file/setup_file.bats +++ b/test/fixtures/parallel/setup_file/setup_file.bats @@ -1,13 +1,13 @@ setup_file() { - load '../../../concurrent-coordination' - echo "start $BATS_TEST_FILENAME" >> "${FILE_MARKER?}" - single-use-barrier setup-file "${PARALLELITY?}" 10 + load '../../../concurrent-coordination' + echo "start $BATS_TEST_FILENAME" >>"${FILE_MARKER?}" + single-use-barrier setup-file "${PARALLELITY?}" 10 } teardown_file() { - echo "stop $BATS_TEST_FILENAME" >> "$FILE_MARKER" + echo "stop $BATS_TEST_FILENAME" >>"$FILE_MARKER" } @test "nothing" { - true -} \ No newline at end of file + true +} diff --git a/test/fixtures/parallel/suite/parallel1.bats b/test/fixtures/parallel/suite/parallel1.bats index 6bc967dd41..caf557c428 100644 --- a/test/fixtures/parallel/suite/parallel1.bats +++ b/test/fixtures/parallel/suite/parallel1.bats @@ -1,10 +1,10 @@ setup() { load '../../../concurrent-coordination' - echo "start $BATS_TEST_NAME $BATS_TEST_FILENAME" >> "$FILE_MARKER" + echo "start $BATS_TEST_NAME $BATS_TEST_FILENAME" >>"$FILE_MARKER" } teardown() { - echo "stop $BATS_TEST_NAME $BATS_TEST_FILENAME" >> "$FILE_MARKER" + echo "stop $BATS_TEST_NAME $BATS_TEST_FILENAME" >>"$FILE_MARKER" } @test "slow test 1" { @@ -17,4 +17,4 @@ teardown() { @test "slow test 3" { single-use-barrier "parallel" "$PARALLELITY" -} \ No newline at end of file +} diff --git a/test/fixtures/run/failing.bats b/test/fixtures/run/failing.bats index 6b154177f8..fd61992c39 100644 --- a/test/fixtures/run/failing.bats +++ b/test/fixtures/run/failing.bats @@ -20,4 +20,4 @@ bats_require_minimum_version 1.5.0 run -0 echo hi run -127 /no/such/cmd run -1 /etc -} \ No newline at end of file +} diff --git a/test/fixtures/run/invalid.bats b/test/fixtures/run/invalid.bats index 97d9733303..e810d12da1 100644 --- a/test/fixtures/run/invalid.bats +++ b/test/fixtures/run/invalid.bats @@ -4,4 +4,4 @@ @test "run -256 echo hi" { run -256 echo hi -} \ No newline at end of file +} diff --git a/test/fixtures/suite/multiple/b.bats b/test/fixtures/suite/multiple/b.bats index bb965a4f86..327ec8dfe5 100644 --- a/test/fixtures/suite/multiple/b.bats +++ b/test/fixtures/suite/multiple/b.bats @@ -3,5 +3,5 @@ } @test "quasi-truth" { - [ -z "$FLUNK" ] + [ -z "${FLUNK:-}" ] } diff --git a/test/fixtures/suite/override_BATS_FILE_EXTENSION/test.bats b/test/fixtures/suite/override_BATS_FILE_EXTENSION/test.bats index 1328af14e0..135f3a185b 100644 --- a/test/fixtures/suite/override_BATS_FILE_EXTENSION/test.bats +++ b/test/fixtures/suite/override_BATS_FILE_EXTENSION/test.bats @@ -1,3 +1,3 @@ @test "test.bats" { - true + true } diff --git a/test/fixtures/suite/skip/skip-in-setup-and-teardown.bats b/test/fixtures/suite/skip/skip-in-setup-and-teardown.bats index 6992a96419..8c58532277 100644 --- a/test/fixtures/suite/skip/skip-in-setup-and-teardown.bats +++ b/test/fixtures/suite/skip/skip-in-setup-and-teardown.bats @@ -1,7 +1,7 @@ #!/usr/bin/env bats -setup () { - skip "This is not working (https://github.com/kata-containers/runtime/issues/175)" +setup() { + skip "This is not working (https://github.com/kata-containers/runtime/issues/175)" } @test "skip in setup and teardown" { @@ -13,5 +13,5 @@ setup () { } teardown() { - skip "This is not working (https://github.com/clearcontainers/runtime/issues/1042)" + skip "This is not working (https://github.com/clearcontainers/runtime/issues/1042)" } diff --git a/test/fixtures/suite/skip/skip-in-setup.bats b/test/fixtures/suite/skip/skip-in-setup.bats index 8ca1c8e2a7..05b72b09bf 100644 --- a/test/fixtures/suite/skip/skip-in-setup.bats +++ b/test/fixtures/suite/skip/skip-in-setup.bats @@ -10,4 +10,4 @@ setup() { @test "skip in setup and test" { skip -} \ No newline at end of file +} diff --git a/test/fixtures/suite/skip/skip-in-teardown.bats b/test/fixtures/suite/skip/skip-in-teardown.bats index 209832b5ee..89896caa65 100644 --- a/test/fixtures/suite/skip/skip-in-teardown.bats +++ b/test/fixtures/suite/skip/skip-in-teardown.bats @@ -5,5 +5,5 @@ } teardown() { - skip "This is not working (https://github.com/clearcontainers/runtime/issues/1042)" + skip "This is not working (https://github.com/clearcontainers/runtime/issues/1042)" } diff --git a/test/fixtures/suite/skip/skip-in-test-and-teardown.bats b/test/fixtures/suite/skip/skip-in-test-and-teardown.bats index aa48ea8559..6c34d9db6d 100644 --- a/test/fixtures/suite/skip/skip-in-test-and-teardown.bats +++ b/test/fixtures/suite/skip/skip-in-test-and-teardown.bats @@ -1,9 +1,9 @@ #!/usr/bin/env bats teardown() { - skip "This is not working (https://github.com/clearcontainers/runtime/issues/1042)" + skip "This is not working (https://github.com/clearcontainers/runtime/issues/1042)" } @test "skip in test and teardown" { - skip "This is not working (https://github.com/clearcontainers/runtime/issues/1042)" + skip "This is not working (https://github.com/clearcontainers/runtime/issues/1042)" } diff --git a/test/fixtures/suite/skip/skip-in-test.bats b/test/fixtures/suite/skip/skip-in-test.bats index 0e340c1f56..f1f2dcdb24 100644 --- a/test/fixtures/suite/skip/skip-in-test.bats +++ b/test/fixtures/suite/skip/skip-in-test.bats @@ -1,5 +1,5 @@ #!/usr/bin/env bats @test "skip in test" { - skip "This is not working (https://github.com/clearcontainers/runtime/issues/1042)" + skip "This is not working (https://github.com/clearcontainers/runtime/issues/1042)" } diff --git a/test/fixtures/suite/test_number/file1.bats b/test/fixtures/suite/test_number/file1.bats index 1b692ec16c..8ab3a27d42 100644 --- a/test/fixtures/suite/test_number/file1.bats +++ b/test/fixtures/suite/test_number/file1.bats @@ -1,18 +1,18 @@ #!/usr/bin/env bats @test "first test in file 1" { - echo "BATS_TEST_NUMBER=$BATS_TEST_NUMBER" - [[ "$BATS_TEST_NUMBER" == 1 ]] - echo "BATS_SUITE_TEST_NUMBER=$BATS_SUITE_TEST_NUMBER" - [[ "$BATS_SUITE_TEST_NUMBER" == 1 ]] + echo "BATS_TEST_NUMBER=$BATS_TEST_NUMBER" + [[ "$BATS_TEST_NUMBER" == 1 ]] + echo "BATS_SUITE_TEST_NUMBER=$BATS_SUITE_TEST_NUMBER" + [[ "$BATS_SUITE_TEST_NUMBER" == 1 ]] } @test "second test in file 1" { - [[ "$BATS_TEST_NUMBER" == 2 ]] - [[ "$BATS_SUITE_TEST_NUMBER" == 2 ]] + [[ "$BATS_TEST_NUMBER" == 2 ]] + [[ "$BATS_SUITE_TEST_NUMBER" == 2 ]] } @test "BATS_TEST_NAMES is per file" { - echo "${#BATS_TEST_NAMES[@]}" - [[ "${#BATS_TEST_NAMES[@]}" == 3 ]] + echo "${#BATS_TEST_NAMES[@]}" + [[ "${#BATS_TEST_NAMES[@]}" == 3 ]] } diff --git a/test/fixtures/suite/test_number/file2.bats b/test/fixtures/suite/test_number/file2.bats index 636689fc28..c62d10b8f9 100644 --- a/test/fixtures/suite/test_number/file2.bats +++ b/test/fixtures/suite/test_number/file2.bats @@ -1,23 +1,23 @@ #!/usr/bin/env bats @test "first test in file 2" { - echo "BATS_TEST_NUMBER=$BATS_TEST_NUMBER" - [[ "$BATS_TEST_NUMBER" == 1 ]] - echo "BATS_SUITE_TEST_NUMBER=$BATS_SUITE_TEST_NUMBER" - [[ "$BATS_SUITE_TEST_NUMBER" == 4 ]] + echo "BATS_TEST_NUMBER=$BATS_TEST_NUMBER" + [[ "$BATS_TEST_NUMBER" == 1 ]] + echo "BATS_SUITE_TEST_NUMBER=$BATS_SUITE_TEST_NUMBER" + [[ "$BATS_SUITE_TEST_NUMBER" == 4 ]] } @test "second test in file 2" { - [[ "$BATS_TEST_NUMBER" == 2 ]] - [[ "$BATS_SUITE_TEST_NUMBER" == 5 ]] + [[ "$BATS_TEST_NUMBER" == 2 ]] + [[ "$BATS_SUITE_TEST_NUMBER" == 5 ]] } @test "second test in file 3" { - [[ "$BATS_TEST_NUMBER" == 3 ]] - [[ "$BATS_SUITE_TEST_NUMBER" == 6 ]] + [[ "$BATS_TEST_NUMBER" == 3 ]] + [[ "$BATS_SUITE_TEST_NUMBER" == 6 ]] } @test "BATS_TEST_NAMES is per file" { - echo "${#BATS_TEST_NAMES[@]}" - [[ "${#BATS_TEST_NAMES[@]}" == 4 ]] + echo "${#BATS_TEST_NAMES[@]}" + [[ "${#BATS_TEST_NAMES[@]}" == 4 ]] } diff --git a/test/fixtures/suite_setup_teardown/call_load/setup_suite.bash b/test/fixtures/suite_setup_teardown/call_load/setup_suite.bash new file mode 100644 index 0000000000..1bc38a83ea --- /dev/null +++ b/test/fixtures/suite_setup_teardown/call_load/setup_suite.bash @@ -0,0 +1,3 @@ +setup_suite() { + load test_helper +} diff --git a/test/fixtures/suite_setup_teardown/call_load/test.bats b/test/fixtures/suite_setup_teardown/call_load/test.bats new file mode 100644 index 0000000000..23670ca12a --- /dev/null +++ b/test/fixtures/suite_setup_teardown/call_load/test.bats @@ -0,0 +1,3 @@ +@test passing { + : +} diff --git a/test/fixtures/suite_setup_teardown/call_load/test_helper.bash b/test/fixtures/suite_setup_teardown/call_load/test_helper.bash new file mode 100644 index 0000000000..8b13789179 --- /dev/null +++ b/test/fixtures/suite_setup_teardown/call_load/test_helper.bash @@ -0,0 +1 @@ + diff --git a/test/fixtures/suite_setup_teardown/default_name/setup_suite.bash b/test/fixtures/suite_setup_teardown/default_name/setup_suite.bash index e074c802c3..df42a37c5d 100644 --- a/test/fixtures/suite_setup_teardown/default_name/setup_suite.bash +++ b/test/fixtures/suite_setup_teardown/default_name/setup_suite.bash @@ -1,7 +1,7 @@ setup_suite() { - echo setup_suite >> "$LOGFILE" + echo setup_suite >>"$LOGFILE" } teardown_suite() { - echo teardown_suite >> "$LOGFILE" -} \ No newline at end of file + echo teardown_suite >>"$LOGFILE" +} diff --git a/test/fixtures/suite_setup_teardown/default_name/test.bats b/test/fixtures/suite_setup_teardown/default_name/test.bats index d2a7628f6c..23670ca12a 100644 --- a/test/fixtures/suite_setup_teardown/default_name/test.bats +++ b/test/fixtures/suite_setup_teardown/default_name/test.bats @@ -1,3 +1,3 @@ @test passing { - : -} \ No newline at end of file + : +} diff --git a/test/fixtures/suite_setup_teardown/error_in_free_code/setup_suite.bash b/test/fixtures/suite_setup_teardown/error_in_free_code/setup_suite.bash index 578e6cc0e1..7ecf291255 100644 --- a/test/fixtures/suite_setup_teardown/error_in_free_code/setup_suite.bash +++ b/test/fixtures/suite_setup_teardown/error_in_free_code/setup_suite.bash @@ -1,5 +1,5 @@ call-to-undefined-command setup_suite() { - : -} \ No newline at end of file + : +} diff --git a/test/fixtures/suite_setup_teardown/error_in_free_code/test.bats b/test/fixtures/suite_setup_teardown/error_in_free_code/test.bats index 6c9a238ea3..581252daa9 100644 --- a/test/fixtures/suite_setup_teardown/error_in_free_code/test.bats +++ b/test/fixtures/suite_setup_teardown/error_in_free_code/test.bats @@ -1 +1 @@ -@test test { :; } \ No newline at end of file +@test test { :; } diff --git a/test/fixtures/suite_setup_teardown/error_in_setup_suite/setup_suite.bash b/test/fixtures/suite_setup_teardown/error_in_setup_suite/setup_suite.bash index 3c08bb23f5..844add6640 100644 --- a/test/fixtures/suite_setup_teardown/error_in_setup_suite/setup_suite.bash +++ b/test/fixtures/suite_setup_teardown/error_in_setup_suite/setup_suite.bash @@ -1,3 +1,3 @@ setup_suite() { - call-to-undefined-command -} \ No newline at end of file + call-to-undefined-command +} diff --git a/test/fixtures/suite_setup_teardown/error_in_setup_suite/test.bats b/test/fixtures/suite_setup_teardown/error_in_setup_suite/test.bats index 6c9a238ea3..581252daa9 100644 --- a/test/fixtures/suite_setup_teardown/error_in_setup_suite/test.bats +++ b/test/fixtures/suite_setup_teardown/error_in_setup_suite/test.bats @@ -1 +1 @@ -@test test { :; } \ No newline at end of file +@test test { :; } diff --git a/test/fixtures/suite_setup_teardown/error_in_teardown_suite/setup_suite.bash b/test/fixtures/suite_setup_teardown/error_in_teardown_suite/setup_suite.bash index 393927f1cf..b3b4302c66 100644 --- a/test/fixtures/suite_setup_teardown/error_in_teardown_suite/setup_suite.bash +++ b/test/fixtures/suite_setup_teardown/error_in_teardown_suite/setup_suite.bash @@ -1,7 +1,7 @@ setup_suite() { - : + : } teardown_suite() { - call-to-undefined-command -} \ No newline at end of file + call-to-undefined-command +} diff --git a/test/fixtures/suite_setup_teardown/error_in_teardown_suite/test.bats b/test/fixtures/suite_setup_teardown/error_in_teardown_suite/test.bats index 6c9a238ea3..581252daa9 100644 --- a/test/fixtures/suite_setup_teardown/error_in_teardown_suite/test.bats +++ b/test/fixtures/suite_setup_teardown/error_in_teardown_suite/test.bats @@ -1 +1 @@ -@test test { :; } \ No newline at end of file +@test test { :; } diff --git a/test/fixtures/suite_setup_teardown/exported_vars/setup_suite.bash b/test/fixtures/suite_setup_teardown/exported_vars/setup_suite.bash index 5bc7e2248a..e4334fdd82 100644 --- a/test/fixtures/suite_setup_teardown/exported_vars/setup_suite.bash +++ b/test/fixtures/suite_setup_teardown/exported_vars/setup_suite.bash @@ -1,3 +1,3 @@ setup_suite() { - export EXPORTED_VAR="${EXPECTED_VALUE?}" -} \ No newline at end of file + export EXPORTED_VAR="${EXPECTED_VALUE?}" +} diff --git a/test/fixtures/suite_setup_teardown/exported_vars/test.bats b/test/fixtures/suite_setup_teardown/exported_vars/test.bats index 17acc27498..42ea96d90f 100644 --- a/test/fixtures/suite_setup_teardown/exported_vars/test.bats +++ b/test/fixtures/suite_setup_teardown/exported_vars/test.bats @@ -1,11 +1,11 @@ setup_file() { - [ "$EXPORTED_VAR" = "$EXPECTED_VALUE" ] + [ "$EXPORTED_VAR" = "$EXPECTED_VALUE" ] } setup() { - [ "$EXPORTED_VAR" = "$EXPECTED_VALUE" ] + [ "$EXPORTED_VAR" = "$EXPECTED_VALUE" ] } @test test { - [ "$EXPORTED_VAR" = "$EXPECTED_VALUE" ] -} \ No newline at end of file + [ "$EXPORTED_VAR" = "$EXPECTED_VALUE" ] +} diff --git a/test/fixtures/suite_setup_teardown/failure_in_setup_suite/setup_suite.bash b/test/fixtures/suite_setup_teardown/failure_in_setup_suite/setup_suite.bash index 0ec42913c2..42c24cfde4 100644 --- a/test/fixtures/suite_setup_teardown/failure_in_setup_suite/setup_suite.bash +++ b/test/fixtures/suite_setup_teardown/failure_in_setup_suite/setup_suite.bash @@ -1,9 +1,9 @@ setup_suite() { - echo "setup_suite before" >&2 - false - echo "setup_suite after" >&2 + echo "setup_suite before" >&2 + false + echo "setup_suite after" >&2 } teardown_suite() { - echo "teardown_suite" >&2 -} \ No newline at end of file + echo "teardown_suite" >&2 +} diff --git a/test/fixtures/suite_setup_teardown/failure_in_setup_suite/test.bats b/test/fixtures/suite_setup_teardown/failure_in_setup_suite/test.bats index 6c9a238ea3..581252daa9 100644 --- a/test/fixtures/suite_setup_teardown/failure_in_setup_suite/test.bats +++ b/test/fixtures/suite_setup_teardown/failure_in_setup_suite/test.bats @@ -1 +1 @@ -@test test { :; } \ No newline at end of file +@test test { :; } diff --git a/test/fixtures/suite_setup_teardown/failure_in_teardown_suite/setup_suite.bash b/test/fixtures/suite_setup_teardown/failure_in_teardown_suite/setup_suite.bash index 3c8d1a708c..d8c3397c54 100644 --- a/test/fixtures/suite_setup_teardown/failure_in_teardown_suite/setup_suite.bash +++ b/test/fixtures/suite_setup_teardown/failure_in_teardown_suite/setup_suite.bash @@ -1,9 +1,9 @@ setup_suite() { - : + : } teardown_suite() { - echo "teardown_suite before" >&2 - false - echo "teardown_suite after" >&2 -} \ No newline at end of file + echo "teardown_suite before" >&2 + false + echo "teardown_suite after" >&2 +} diff --git a/test/fixtures/suite_setup_teardown/failure_in_teardown_suite/test.bats b/test/fixtures/suite_setup_teardown/failure_in_teardown_suite/test.bats index 6c9a238ea3..581252daa9 100644 --- a/test/fixtures/suite_setup_teardown/failure_in_teardown_suite/test.bats +++ b/test/fixtures/suite_setup_teardown/failure_in_teardown_suite/test.bats @@ -1 +1 @@ -@test test { :; } \ No newline at end of file +@test test { :; } diff --git a/test/fixtures/suite_setup_teardown/no_failure_no_output/setup_suite.bash b/test/fixtures/suite_setup_teardown/no_failure_no_output/setup_suite.bash new file mode 100644 index 0000000000..7c31b7e91f --- /dev/null +++ b/test/fixtures/suite_setup_teardown/no_failure_no_output/setup_suite.bash @@ -0,0 +1,7 @@ +setup_suite() { + echo setup_suite +} + +teardown_suite() { + echo teardown_suite +} diff --git a/test/fixtures/suite_setup_teardown/no_failure_no_output/test.bats b/test/fixtures/suite_setup_teardown/no_failure_no_output/test.bats new file mode 100644 index 0000000000..23670ca12a --- /dev/null +++ b/test/fixtures/suite_setup_teardown/no_failure_no_output/test.bats @@ -0,0 +1,3 @@ +@test passing { + : +} diff --git a/test/fixtures/suite_setup_teardown/no_setup_suite_function/setup_suite.bash b/test/fixtures/suite_setup_teardown/no_setup_suite_function/setup_suite.bash index 2894c97549..f7e517c5ee 100644 --- a/test/fixtures/suite_setup_teardown/no_setup_suite_function/setup_suite.bash +++ b/test/fixtures/suite_setup_teardown/no_setup_suite_function/setup_suite.bash @@ -1,3 +1,3 @@ teardown_suite() { - : -} \ No newline at end of file + : +} diff --git a/test/fixtures/suite_setup_teardown/no_setup_suite_function/test.bats b/test/fixtures/suite_setup_teardown/no_setup_suite_function/test.bats index ff320b68bd..8c40677f50 100644 --- a/test/fixtures/suite_setup_teardown/no_setup_suite_function/test.bats +++ b/test/fixtures/suite_setup_teardown/no_setup_suite_function/test.bats @@ -1,3 +1,3 @@ @test test { - : -} \ No newline at end of file + : +} diff --git a/test/fixtures/suite_setup_teardown/non_default_name/setup_suite_non_default.bash b/test/fixtures/suite_setup_teardown/non_default_name/setup_suite_non_default.bash index dfff9e2ef7..f6baf0fa3b 100644 --- a/test/fixtures/suite_setup_teardown/non_default_name/setup_suite_non_default.bash +++ b/test/fixtures/suite_setup_teardown/non_default_name/setup_suite_non_default.bash @@ -1,7 +1,7 @@ setup_suite() { - echo setup_suite non_default >> "$LOGFILE" + echo setup_suite non_default >>"$LOGFILE" } teardown_suite() { - echo teardown_suite non_default >> "$LOGFILE" -} \ No newline at end of file + echo teardown_suite non_default >>"$LOGFILE" +} diff --git a/test/fixtures/suite_setup_teardown/non_default_name/test.bats b/test/fixtures/suite_setup_teardown/non_default_name/test.bats index d2a7628f6c..23670ca12a 100644 --- a/test/fixtures/suite_setup_teardown/non_default_name/test.bats +++ b/test/fixtures/suite_setup_teardown/non_default_name/test.bats @@ -1,3 +1,3 @@ @test passing { - : -} \ No newline at end of file + : +} diff --git a/test/fixtures/suite_setup_teardown/output_with_failure/setup_suite.bash b/test/fixtures/suite_setup_teardown/output_with_failure/setup_suite.bash new file mode 100644 index 0000000000..df9ee78210 --- /dev/null +++ b/test/fixtures/suite_setup_teardown/output_with_failure/setup_suite.bash @@ -0,0 +1,8 @@ +setup_suite() { + echo setup_suite + false +} + +teardown_suite() { + echo teardown_suite +} diff --git a/test/fixtures/suite_setup_teardown/output_with_failure/test.bats b/test/fixtures/suite_setup_teardown/output_with_failure/test.bats new file mode 100644 index 0000000000..23670ca12a --- /dev/null +++ b/test/fixtures/suite_setup_teardown/output_with_failure/test.bats @@ -0,0 +1,3 @@ +@test passing { + : +} diff --git a/test/fixtures/suite_setup_teardown/pick_up_toplevel/folder1/test.bats b/test/fixtures/suite_setup_teardown/pick_up_toplevel/folder1/test.bats index d2a7628f6c..23670ca12a 100644 --- a/test/fixtures/suite_setup_teardown/pick_up_toplevel/folder1/test.bats +++ b/test/fixtures/suite_setup_teardown/pick_up_toplevel/folder1/test.bats @@ -1,3 +1,3 @@ @test passing { - : -} \ No newline at end of file + : +} diff --git a/test/fixtures/suite_setup_teardown/pick_up_toplevel/folder2/test.bats b/test/fixtures/suite_setup_teardown/pick_up_toplevel/folder2/test.bats index d2a7628f6c..23670ca12a 100644 --- a/test/fixtures/suite_setup_teardown/pick_up_toplevel/folder2/test.bats +++ b/test/fixtures/suite_setup_teardown/pick_up_toplevel/folder2/test.bats @@ -1,3 +1,3 @@ @test passing { - : -} \ No newline at end of file + : +} diff --git a/test/fixtures/suite_setup_teardown/pick_up_toplevel/setup_suite.bash b/test/fixtures/suite_setup_teardown/pick_up_toplevel/setup_suite.bash index e5f142c91c..72f3527335 100644 --- a/test/fixtures/suite_setup_teardown/pick_up_toplevel/setup_suite.bash +++ b/test/fixtures/suite_setup_teardown/pick_up_toplevel/setup_suite.bash @@ -1,7 +1,7 @@ setup_suite() { - echo "${BASH_SOURCE[0]}" setup_suite >> "$LOGFILE" + echo "${BASH_SOURCE[0]}" setup_suite >>"$LOGFILE" } teardown_suite() { - echo "${BASH_SOURCE[0]}" teardown_suite >> "$LOGFILE" -} \ No newline at end of file + echo "${BASH_SOURCE[0]}" teardown_suite >>"$LOGFILE" +} diff --git a/test/fixtures/suite_setup_teardown/pick_up_toplevel/test.bats b/test/fixtures/suite_setup_teardown/pick_up_toplevel/test.bats index d2a7628f6c..23670ca12a 100644 --- a/test/fixtures/suite_setup_teardown/pick_up_toplevel/test.bats +++ b/test/fixtures/suite_setup_teardown/pick_up_toplevel/test.bats @@ -1,3 +1,3 @@ @test passing { - : -} \ No newline at end of file + : +} diff --git a/test/fixtures/suite_setup_teardown/return_nonzero_in_teardown_suite/setup_suite.bash b/test/fixtures/suite_setup_teardown/return_nonzero_in_teardown_suite/setup_suite.bash new file mode 100644 index 0000000000..8a1a18aec5 --- /dev/null +++ b/test/fixtures/suite_setup_teardown/return_nonzero_in_teardown_suite/setup_suite.bash @@ -0,0 +1,9 @@ +setup_suite() { + : +} + +teardown_suite() { + echo "teardown_suite before" >&2 + return 1 + echo "teardown_suite after" >&2 +} diff --git a/test/fixtures/suite_setup_teardown/return_nonzero_in_teardown_suite/test.bats b/test/fixtures/suite_setup_teardown/return_nonzero_in_teardown_suite/test.bats new file mode 100644 index 0000000000..581252daa9 --- /dev/null +++ b/test/fixtures/suite_setup_teardown/return_nonzero_in_teardown_suite/test.bats @@ -0,0 +1 @@ +@test test { :; } diff --git a/test/fixtures/suite_setup_teardown/stderr_in_setup_teardown_suite/setup_suite.bash b/test/fixtures/suite_setup_teardown/stderr_in_setup_teardown_suite/setup_suite.bash new file mode 100644 index 0000000000..2a3ec9f4e3 --- /dev/null +++ b/test/fixtures/suite_setup_teardown/stderr_in_setup_teardown_suite/setup_suite.bash @@ -0,0 +1,10 @@ +setup_suite() { + echo setup_suite stdout + echo setup_suite stderr >&2 +} + +teardown_suite() { + echo teardown_suite stdout + echo teardown_suite stderr >&2 + false +} diff --git a/test/fixtures/suite_setup_teardown/stderr_in_setup_teardown_suite/test.bats b/test/fixtures/suite_setup_teardown/stderr_in_setup_teardown_suite/test.bats new file mode 100644 index 0000000000..8c40677f50 --- /dev/null +++ b/test/fixtures/suite_setup_teardown/stderr_in_setup_teardown_suite/test.bats @@ -0,0 +1,3 @@ +@test test { + : +} diff --git a/test/fixtures/suite_setup_teardown/syntax_error/test.bats b/test/fixtures/suite_setup_teardown/syntax_error/test.bats index 6c9a238ea3..581252daa9 100644 --- a/test/fixtures/suite_setup_teardown/syntax_error/test.bats +++ b/test/fixtures/suite_setup_teardown/syntax_error/test.bats @@ -1 +1 @@ -@test test { :; } \ No newline at end of file +@test test { :; } diff --git a/test/fixtures/tagging/invalid_tags.bats b/test/fixtures/tagging/invalid_tags.bats new file mode 100644 index 0000000000..fe8d228e7e --- /dev/null +++ b/test/fixtures/tagging/invalid_tags.bats @@ -0,0 +1,8 @@ +# bats file_tags=,bc +# bats file_tags=a+b + +# bats test_tags=,bc +# bats test_tags=a+b +@test test { + : +} diff --git a/test/fixtures/tagging/tagged.bats b/test/fixtures/tagging/tagged.bats new file mode 100644 index 0000000000..9e6bfb2872 --- /dev/null +++ b/test/fixtures/tagging/tagged.bats @@ -0,0 +1,25 @@ +@test "No tags" { + : +} + +# bats file_tags=file:tag:1 + +@test "Only file tags" { + : +} + +# bats test_tags=test:tag:1 +@test "File and test tags" { + : +} + +# bats test_tags=test:tag:2 +@test "File and other test tags" { + : +} + +# bats file_tags= +# bats test_tags=test:tag:3 +@test "Only test tags" { + : +} diff --git a/test/fixtures/tagging/trimming.bats b/test/fixtures/tagging/trimming.bats new file mode 100644 index 0000000000..bd94b1cbd1 --- /dev/null +++ b/test/fixtures/tagging/trimming.bats @@ -0,0 +1,5 @@ +# bats test_tags= test:trim , test:me +# bats file_tags= file:trim , file:me +@test test { + : +} diff --git a/test/fixtures/timeout/sleep2.bats b/test/fixtures/timeout/sleep2.bats new file mode 100644 index 0000000000..5ad1db0983 --- /dev/null +++ b/test/fixtures/timeout/sleep2.bats @@ -0,0 +1,3 @@ +@test "my sleep ${SLEEP}" { + sleep "${SLEEP?}" +} diff --git a/test/fixtures/warnings/BW01.bats b/test/fixtures/warnings/BW01.bats index b914aca39e..3776626b8c 100644 --- a/test/fixtures/warnings/BW01.bats +++ b/test/fixtures/warnings/BW01.bats @@ -1,5 +1,5 @@ @test 'Trigger BW01' { - # shellcheck disable=SC2283,SC1068 - run =0 actually-intended-command with some args # see issue #578 - # no $status check because that should be done above (but isn't!) -} \ No newline at end of file + # shellcheck disable=SC2283,SC1068 + run =0 actually-intended-command with some args # see issue #578 + # no $status check because that should be done above (but isn't!) +} diff --git a/test/fixtures/warnings/BW01_check_exit_code_is_127.bats b/test/fixtures/warnings/BW01_check_exit_code_is_127.bats index 9ce261846a..3ff635862e 100644 --- a/test/fixtures/warnings/BW01_check_exit_code_is_127.bats +++ b/test/fixtures/warnings/BW01_check_exit_code_is_127.bats @@ -1,4 +1,4 @@ @test "Don't trigger BW01 with checked exit code 127" { - bats_require_minimum_version 1.5.0 - run -127 =0 actually-intended-command with some args -} \ No newline at end of file + bats_require_minimum_version 1.5.0 + run -127 =0 actually-intended-command with some args +} diff --git a/test/fixtures/warnings/BW01_no_exit_code_check_no_exit_code_127.bats b/test/fixtures/warnings/BW01_no_exit_code_check_no_exit_code_127.bats index 2ea49539d9..e9b721478a 100644 --- a/test/fixtures/warnings/BW01_no_exit_code_check_no_exit_code_127.bats +++ b/test/fixtures/warnings/BW01_no_exit_code_check_no_exit_code_127.bats @@ -1,3 +1,3 @@ @test "Don't trigger BW01 with exit code !=127 and no check" { - run exit 1 -} \ No newline at end of file + run exit 1 +} diff --git a/test/fixtures/warnings/BW02.bats b/test/fixtures/warnings/BW02.bats index 8b8f29e278..8e14c61822 100644 --- a/test/fixtures/warnings/BW02.bats +++ b/test/fixtures/warnings/BW02.bats @@ -1,3 +1,3 @@ @test "Trigger BW02" { - run --keep-empty-lines true -} \ No newline at end of file + run --keep-empty-lines true +} diff --git a/test/fixtures/warnings/BW03/define_setup_suite_in_wrong_file.bats b/test/fixtures/warnings/BW03/define_setup_suite_in_wrong_file.bats new file mode 100644 index 0000000000..fa8042f293 --- /dev/null +++ b/test/fixtures/warnings/BW03/define_setup_suite_in_wrong_file.bats @@ -0,0 +1,7 @@ +setup_suite() { + : +} + +@test test { + : +} diff --git a/test/fixtures/warnings/BW03/non_default_setup_suite.bash b/test/fixtures/warnings/BW03/non_default_setup_suite.bash new file mode 100644 index 0000000000..cacaef2340 --- /dev/null +++ b/test/fixtures/warnings/BW03/non_default_setup_suite.bash @@ -0,0 +1,3 @@ +setup_suite() { + : +} diff --git a/test/fixtures/warnings/BW03/suppress_warning.bats b/test/fixtures/warnings/BW03/suppress_warning.bats new file mode 100644 index 0000000000..70a8fd3c82 --- /dev/null +++ b/test/fixtures/warnings/BW03/suppress_warning.bats @@ -0,0 +1,10 @@ +setup_suite() { + : +} + +# shellcheck disable=SC2034 +BATS_SETUP_SUITE_COMPLETED='suppress BW03' + +@test test { + : +} diff --git a/test/formatter.bats b/test/formatter.bats new file mode 100644 index 0000000000..a87f8bfe40 --- /dev/null +++ b/test/formatter.bats @@ -0,0 +1,101 @@ +#!/usr/bin/env bats + +setup() { + load test_helper + fixtures formatter +} + +@test "tap passing and skipping tests" { + reentrant_run filter_control_sequences bats --formatter tap "$FIXTURE_ROOT/passing_and_skipping.bats" + [ $status -eq 0 ] + [ "${lines[0]}" = "1..3" ] + [ "${lines[1]}" = "ok 1 a passing test" ] + [ "${lines[2]}" = "ok 2 a skipped test with no reason # skip" ] + [ "${lines[3]}" = "ok 3 a skipped test with a reason # skip for a really good reason" ] +} + +@test "tap passing, failing and skipping tests" { + reentrant_run filter_control_sequences bats --formatter tap "$FIXTURE_ROOT/passing_failing_and_skipping.bats" + [ $status -eq 0 ] + [ "${lines[0]}" = "1..3" ] + [ "${lines[1]}" = "ok 1 a passing test" ] + [ "${lines[2]}" = "ok 2 a skipping test # skip" ] + [ "${lines[3]}" = "not ok 3 a failing test" ] +} + +@test "skipped test with parens (pretty formatter)" { + reentrant_run bats --pretty "$FIXTURE_ROOT/skipped_with_parens.bats" + [ $status -eq 0 ] + + # Some systems (Alpine, for example) seem to emit an extra whitespace into + # entries in the 'lines' array when a carriage return is present from the + # pretty formatter. This is why a '+' is used after the 'skipped' note. + [[ "${lines[*]}" =~ "- a skipped test with parentheses in the reason (skipped: "+"a reason (with parentheses))" ]] +} + +@test "pretty and tap formats" { + reentrant_run bats --formatter tap "$FIXTURE_ROOT/passing.bats" + tap_output="$output" + [ $status -eq 0 ] + + reentrant_run bats --pretty "$FIXTURE_ROOT/passing.bats" + pretty_output="$output" + [ $status -eq 0 ] + + [ "$tap_output" != "$pretty_output" ] +} + +@test "pretty formatter bails on invalid tap" { + reentrant_run bats-format-pretty < <(printf "This isn't TAP!\nGood day to you\n") + [ $status -eq 0 ] + [ "${lines[0]}" = "This isn't TAP!" ] + [ "${lines[1]}" = "Good day to you" ] +} + +@test "All formatters (except cat) implement the callback interface" { + cd "$BATS_ROOT/libexec/bats-core/" + for formatter in bats-format-*; do + # the cat formatter is not expected to implement this interface + if [[ "$formatter" == *"bats-format-cat" ]]; then + continue + fi + tested_at_least_one_formatter=1 + echo "Formatter: ${formatter}" + # the replay should be possible without errors + bash -u "$formatter" >/dev/null <"$bats_symlink" chmod 700 "$bats_symlink" - run "$bats_symlink" -v + reentrant_run "$bats_symlink" -v [ "$status" -eq 0 ] [ "${output%% *}" == 'Bats' ] } diff --git a/test/junit-formatter.bats b/test/junit-formatter.bats index a6883efdac..62e2477ba1 100644 --- a/test/junit-formatter.bats +++ b/test/junit-formatter.bats @@ -8,11 +8,11 @@ TIMESTAMP_REGEX='[0-9]+-[0-1][0-9]-[0-3][0-9]T[0-2][0-9]:[0-5][0-9]:[0-5][0-9]' TESTSUITES_REGEX="" @test "junit formatter with skipped test does not fail" { - run bats --formatter junit "$FIXTURE_ROOT/skipped.bats" + reentrant_run bats --formatter junit "$FIXTURE_ROOT/skipped.bats" echo "$output" [[ $status -eq 0 ]] [[ "${lines[0]}" == '' ]] - + [[ "${lines[1]}" =~ $TESTSUITES_REGEX ]] TESTSUITE_REGEX="" @@ -29,28 +29,28 @@ TESTSUITES_REGEX="" [[ "${lines[6]}" =~ $TESTCASE_REGEX ]] [[ "${lines[7]}" == *"a reason"* ]] [[ "${lines[8]}" == *""* ]] - + [[ "${lines[9]}" == *""* ]] [[ "${lines[10]}" == *""* ]] } @test "junit formatter: escapes xml special chars" { case $OSTYPE in - linux*|darwin) - # their CI can handle special chars on filename - TEST_FILE_NAME="xml-escape-\"<>'&.bats" - ESCAPED_TEST_FILE_NAME="xml-escape-"<>'&.bats" - TEST_FILE_PATH="$BATS_TEST_TMPDIR/$TEST_FILE_NAME" - cp "$FIXTURE_ROOT/xml-escape.bats" "$TEST_FILE_PATH" + linux* | darwin) + # their CI can handle special chars on filename + TEST_FILE_NAME="xml-escape-\"<>'&.bats" + ESCAPED_TEST_FILE_NAME="xml-escape-"<>'&.bats" + TEST_FILE_PATH="$BATS_TEST_TMPDIR/$TEST_FILE_NAME" + cp "$FIXTURE_ROOT/xml-escape.bats" "$TEST_FILE_PATH" ;; - *) - # use the filename without special chars - TEST_FILE_NAME="xml-escape.bats" - ESCAPED_TEST_FILE_NAME="$TEST_FILE_NAME" - TEST_FILE_PATH="$FIXTURE_ROOT/$TEST_FILE_NAME" + *) + # use the filename without special chars + TEST_FILE_NAME="xml-escape.bats" + ESCAPED_TEST_FILE_NAME="$TEST_FILE_NAME" + TEST_FILE_PATH="$FIXTURE_ROOT/$TEST_FILE_NAME" ;; esac - run bats --formatter junit "$TEST_FILE_PATH" + reentrant_run bats --formatter junit "$TEST_FILE_PATH" echo "$output" [[ "${lines[2]}" == "" ]] @@ -63,7 +63,7 @@ TESTSUITES_REGEX="" } @test "junit formatter: test suites" { - run bats --formatter junit "$FIXTURE_ROOT/suite/" + reentrant_run bats --formatter junit "$FIXTURE_ROOT/suite/" echo "$output" [[ "${lines[0]}" == '' ]] @@ -79,7 +79,7 @@ TESTSUITES_REGEX="" @test "junit formatter: test suites relative path" { cd "$FIXTURE_ROOT" - run bats --formatter junit "suite/" + reentrant_run bats --formatter junit "suite/" echo "$output" [[ "${lines[0]}" == '' ]] @@ -94,7 +94,7 @@ TESTSUITES_REGEX="" } @test "junit formatter: files with the same name are distinguishable" { - run bats --formatter junit -r "$FIXTURE_ROOT/duplicate/" + reentrant_run bats --formatter junit -r "$FIXTURE_ROOT/duplicate/" echo "$output" [[ "${lines[2]}" == *"" @test "junit formatter as report formatter creates report.xml" { cd "$BATS_TEST_TMPDIR" # don't litter sources with output files - run bats --report-formatter junit "$FIXTURE_ROOT/suite/" + reentrant_run bats --report-formatter junit "$FIXTURE_ROOT/suite/" echo "$output" [[ -e "report.xml" ]] run cat "report.xml" @@ -113,7 +113,7 @@ TESTSUITES_REGEX="" } @test "junit does not mark tests with FD 3 output as failed (issue #360)" { - run bats --formatter junit "$FIXTURE_ROOT/issue_360.bats" + reentrant_run bats --formatter junit "$FIXTURE_ROOT/issue_360.bats" echo "$output" @@ -141,7 +141,7 @@ TESTSUITES_REGEX="" @test "junit does not mark tests with FD 3 output in teardown_file as failed (issue #531)" { bats_require_minimum_version 1.5.0 - run -0 bats --formatter junit "$FIXTURE_ROOT/issue_531.bats" + reentrant_run -0 bats --formatter junit "$FIXTURE_ROOT/issue_531.bats" [[ "${lines[2]}" == '' ]] [[ "${lines[3]}" == ' ' ]] @@ -151,3 +151,10 @@ TESTSUITES_REGEX="" [[ "${lines[6]}" == ' ' ]] [[ "${lines[7]}" == '' ]] } + +@test "don't choke on setup_file errors" { + bats_require_minimum_version 1.5.0 + local stderr='' # silence shellcheck + reentrant_run -1 --separate-stderr bats --formatter junit "$FIXTURE_ROOT/../file_setup_teardown/setup_file_failed.bats" + [ "${stderr}" == "" ] +} diff --git a/test/load.bats b/test/load.bats index 5a4791b49d..75934b46f4 100644 --- a/test/load.bats +++ b/test/load.bats @@ -5,23 +5,24 @@ bats_require_minimum_version 1.5.0 setup() { load test_helper fixtures load + REENTRANT_RUN_PRESERVE+=(BATS_LIB_PATH) } @test "find_in_bats_lib_path recognizes files relative to test file" { - test_dir="$BATS_TEST_TMPDIR/find_in_bats_lib_path/bats_test_dirname_priorty" + test_dir="$BATS_TEST_TMPDIR/find_in_bats_lib_path/bats_test_dirname_priority" mkdir -p "$test_dir" cp "$FIXTURE_ROOT/test_helper.bash" "$test_dir/" cp "$FIXTURE_ROOT/find_library_helper.bats" "$test_dir" - BATS_LIB_PATH="" LIBRARY_NAME="test_helper" LIBRARY_PATH="$test_dir/test_helper.bash" run bats "$test_dir/find_library_helper.bats" + BATS_LIB_PATH="" LIBRARY_NAME="test_helper" LIBRARY_PATH="$test_dir/test_helper.bash" reentrant_run bats "$test_dir/find_library_helper.bats" } @test "find_in_bats_lib_path recognizes files in BATS_LIB_PATH" { - test_dir="$BATS_TEST_TMPDIR/find_in_bats_lib_path/bats_test_dirname_priorty" + test_dir="$BATS_TEST_TMPDIR/find_in_bats_lib_path/bats_test_dirname_priority" mkdir -p "$test_dir" cp "$FIXTURE_ROOT/test_helper.bash" "$test_dir/" - BATS_LIB_PATH="$test_dir" LIBRARY_NAME="test_helper" LIBRARY_PATH="$test_dir/test_helper.bash" run bats "$FIXTURE_ROOT/find_library_helper.bats" + BATS_LIB_PATH="$test_dir" LIBRARY_NAME="test_helper" LIBRARY_PATH="$test_dir/test_helper.bash" reentrant_run bats "$FIXTURE_ROOT/find_library_helper.bats" } @test "find_in_bats_lib_path returns 1 if no load path is found" { @@ -29,7 +30,7 @@ setup() { mkdir -p "$test_dir" cp "$FIXTURE_ROOT/test_helper.bash" "$test_dir/" - BATS_LIB_PATH="$test_dir" LIBRARY_NAME="test_helper" run bats "$FIXTURE_ROOT/find_library_helper_err.bats" + BATS_LIB_PATH="$test_dir" LIBRARY_NAME="test_helper" reentrant_run bats "$FIXTURE_ROOT/find_library_helper_err.bats" } @test "find_in_bats_lib_path follows the priority of BATS_LIB_PATH" { @@ -43,36 +44,36 @@ setup() { mkdir -p "$second_dir" cp "$FIXTURE_ROOT/exit1.bash" "$second_dir/target.bash" - BATS_LIB_PATH="$first_dir:$second_dir" LIBRARY_NAME="target" LIBRARY_PATH="$first_dir/target.bash" run bats "$FIXTURE_ROOT/find_library_helper.bats" + BATS_LIB_PATH="$first_dir:$second_dir" LIBRARY_NAME="target" LIBRARY_PATH="$first_dir/target.bash" reentrant_run bats "$FIXTURE_ROOT/find_library_helper.bats" } @test "load sources scripts relative to the current test file" { - run bats "$FIXTURE_ROOT/load.bats" + reentrant_run bats "$FIXTURE_ROOT/load.bats" [ $status -eq 0 ] } @test "load sources relative scripts with filename extension" { - HELPER_NAME="test_helper.bash" run bats "$FIXTURE_ROOT/load.bats" + HELPER_NAME="test_helper.bash" reentrant_run bats "$FIXTURE_ROOT/load.bats" [ $status -eq 0 ] } @test "load aborts if the specified script does not exist" { - HELPER_NAME="nonexistent" run bats "$FIXTURE_ROOT/load.bats" + HELPER_NAME="nonexistent" reentrant_run bats "$FIXTURE_ROOT/load.bats" [ $status -eq 1 ] } @test "load sources scripts by absolute path" { - HELPER_NAME="${FIXTURE_ROOT}/test_helper.bash" run bats "$FIXTURE_ROOT/load.bats" + HELPER_NAME="${FIXTURE_ROOT}/test_helper.bash" reentrant_run bats "$FIXTURE_ROOT/load.bats" [ $status -eq 0 ] } @test "load aborts if the script, specified by an absolute path, does not exist" { - HELPER_NAME="${FIXTURE_ROOT}/nonexistent" run bats "$FIXTURE_ROOT/load.bats" + HELPER_NAME="${FIXTURE_ROOT}/nonexistent" reentrant_run bats "$FIXTURE_ROOT/load.bats" [ $status -eq 1 ] } @test "load relative script with ambiguous name" { - HELPER_NAME="ambiguous" run bats "$FIXTURE_ROOT/load.bats" + HELPER_NAME="ambiguous" reentrant_run bats "$FIXTURE_ROOT/load.bats" [ $status -eq 0 ] } @@ -81,9 +82,9 @@ setup() { mkdir -p "$path_dir/on_path" cp "${FIXTURE_ROOT}/test_helper.bash" "${path_dir}/on_path/load.bash" # shellcheck disable=SC2030,SC2031 - export BATS_LIB_PATH="${path_dir}" HELPER_NAME="on_path" - run ! bats "$FIXTURE_ROOT/load.bats" - run -0 bats "$FIXTURE_ROOT/bats_load_library.bats" + export BATS_LIB_PATH="${path_dir}" HELPER_NAME="on_path" + reentrant_run ! bats "$FIXTURE_ROOT/load.bats" + reentrant_run -0 bats "$FIXTURE_ROOT/bats_load_library.bats" } @test "load supports plain symbols" { @@ -91,7 +92,7 @@ setup() { { echo "plain_variable='value of plain variable'" echo "plain_array=(test me hard)" - } > "${helper}" + } >"${helper}" load "${helper}" # shellcheck disable=SC2154 @@ -110,13 +111,13 @@ setup() { echo "declare -i an_integer=0x7e4" echo "declare -a an_array=(test me hard)" echo "declare -x exported_variable='value of exported variable'" - } > "${helper}" + } >"${helper}" load "${helper}" [ "${declared_variable:-}" != 'value of declared variable' ] [ "${a_constant:-}" != 'constant value' ] - (( "${an_integer:-2019}" != 2020 )) + (("${an_integer:-2019}" != 2020)) [ "${an_array[2]:-}" != 'hard' ] [ "${exported_variable:-}" != 'value of exported variable' ] @@ -128,14 +129,8 @@ setup() { mkdir -p "$path_dir" cp "${FIXTURE_ROOT}/test_helper.bash" "${path_dir}/on_path" # shellcheck disable=SC2030,SC2031 - export PATH="${path_dir}:$PATH" HELPER_NAME="on_path" - run -0 bats "$FIXTURE_ROOT/load.bats" -} - -@test "bats_load_library requires BATS_LIB_PATH to be set" { - unset BATS_LIB_PATH - run ! bats "$FIXTURE_ROOT/bats_load_library.bats" - [ "${lines[4]}" == '# bats_load_library: requires BATS_LIB_PATH to be set!' ] + export PATH="${path_dir}:$PATH" HELPER_NAME="on_path" + reentrant_run -0 bats "$FIXTURE_ROOT/load.bats" } @test "bats_load_library supports libraries with loaders on the BATS_LIB_PATH" { @@ -145,35 +140,35 @@ setup() { cp "${FIXTURE_ROOT}/exit1.bash" "${path_dir}/exit1.bash" # shellcheck disable=SC2030,SC2031 export BATS_LIB_PATH="${BATS_TEST_TMPDIR}/libraries" HELPER_NAME="$BATS_TEST_NAME" - run -0 bats "$FIXTURE_ROOT/bats_load_library.bats" - run ! bats "$FIXTURE_ROOT/bats_load.bats" # load does not use BATS_LIB_PATH! + reentrant_run -0 bats "$FIXTURE_ROOT/bats_load_library.bats" + reentrant_run ! bats "$FIXTURE_ROOT/bats_load.bats" # load does not use BATS_LIB_PATH! } @test "bats_load_library supports libraries with loaders on the BATS_LIB_PATH with multiple libraries" { path_dir="$BATS_TEST_TMPDIR/libraries/" for lib in liba libb libc; do - mkdir -p "$path_dir/$lib" - cp "${FIXTURE_ROOT}/exit1.bash" "$path_dir/$lib/load.bash" + mkdir -p "$path_dir/$lib" + cp "${FIXTURE_ROOT}/exit1.bash" "$path_dir/$lib/load.bash" done mkdir -p "$path_dir/$BATS_TEST_NAME" cp "${FIXTURE_ROOT}/test_helper.bash" "$path_dir/$BATS_TEST_NAME/load.bash" # shellcheck disable=SC2030,SC2031 export BATS_LIB_PATH="$path_dir" HELPER_NAME="$BATS_TEST_NAME" - run -0 bats "$FIXTURE_ROOT/bats_load_library.bats" - run ! bats "$FIXTURE_ROOT/load.bats" # load does not use BATS_LIB_PATH! + reentrant_run -0 bats "$FIXTURE_ROOT/bats_load_library.bats" + reentrant_run ! bats "$FIXTURE_ROOT/load.bats" # load does not use BATS_LIB_PATH! } @test "bats_load_library can handle whitespaces in BATS_LIB_PATH" { path_dir="$BATS_TEST_TMPDIR/libraries with spaces/" for lib in liba libb libc; do - mkdir -p "$path_dir/$lib" - cp "${FIXTURE_ROOT}/exit1.bash" "$path_dir/$lib/load.bash" + mkdir -p "$path_dir/$lib" + cp "${FIXTURE_ROOT}/exit1.bash" "$path_dir/$lib/load.bash" done mkdir -p "$path_dir/$BATS_TEST_NAME" cp "${FIXTURE_ROOT}/test_helper.bash" "$path_dir/$BATS_TEST_NAME/load.bash" # shellcheck disable=SC2030,SC2031 export BATS_LIB_PATH="$path_dir" HELPER_NAME="$BATS_TEST_NAME" - run -0 bats "$FIXTURE_ROOT/bats_load_library.bats" + reentrant_run -0 bats "$FIXTURE_ROOT/bats_load_library.bats" } @test "bats_load_library errors when a library errors while sourcing" { @@ -183,5 +178,13 @@ setup() { # shellcheck disable=SC2030,SC2031 export BATS_LIB_PATH="$path_dir" - run -1 bats "$FIXTURE_ROOT/failing_bats_load_library.bats" -} \ No newline at end of file + reentrant_run -1 bats "$FIXTURE_ROOT/failing_bats_load_library.bats" +} + +@test "load in teardown after failure does not prevent test from being counted (see #609)" { + reentrant_run -1 bats "$FIXTURE_ROOT/load_in_teardown_after_failure.bats" + [ "${lines[0]}" = 1..1 ] + [ "${lines[1]}" = "not ok 1 failed" ] + [ "${lines[3]}" = "# \`false' failed" ] + [ ${#lines[@]} -eq 4 ] +} diff --git a/test/parallel.bats b/test/parallel.bats index 705f36c713..6bc94d2fc9 100644 --- a/test/parallel.bats +++ b/test/parallel.bats @@ -5,6 +5,9 @@ bats_require_minimum_version 1.5.0 load test_helper fixtures parallel +# shellcheck disable=SC2034 +BATS_TEST_TIMEOUT=10 # only intended for the "short form ..."" test + setup() { type -p parallel &>/dev/null || skip "--jobs requires GNU parallel" (type -p flock &>/dev/null || type -p shlock &>/dev/null) || skip "--jobs requires flock/shlock" @@ -18,15 +21,15 @@ check_parallel_tests() { # started_tests=0 read_lines=0 while IFS= read -r line; do - (( ++read_lines )) + ((++read_lines)) case "$line" in - "start "*) - if (( ++started_tests > max_parallel_tests )); then - max_parallel_tests="$started_tests" - fi + "start "*) + if ((++started_tests > max_parallel_tests)); then + max_parallel_tests="$started_tests" + fi ;; - "stop "*) - (( started_tests-- )) + "stop "*) + ((started_tests--)) ;; esac done <"$FILE_MARKER" @@ -45,8 +48,8 @@ check_parallel_tests() { # FILE_MARKER=$(mktemp "${BATS_RUN_TMPDIR}/file_marker.XXXXXX") # shellcheck disable=SC2030 export PARALLELITY=3 - run bats --jobs $PARALLELITY "$FIXTURE_ROOT/parallel.bats" - + reentrant_run bats --jobs $PARALLELITY "$FIXTURE_ROOT/parallel.bats" + [ "$status" -eq 0 ] # Make sure the lines are in-order. [[ "${lines[0]}" == "1..3" ]] @@ -59,7 +62,7 @@ check_parallel_tests() { # @test "parallel can preserve environment variables" { export TEST_ENV_VARIABLE='test-value' - run bats --jobs 2 "$FIXTURE_ROOT/parallel-preserve-environment.bats" + reentrant_run bats --jobs 2 "$FIXTURE_ROOT/parallel-preserve-environment.bats" echo "$output" [[ "$status" -eq 0 ]] } @@ -74,8 +77,8 @@ check_parallel_tests() { # # file parallelization is needed for maximum parallelity! # If we got over the skip (if no GNU parallel) in setup() we can reenable it safely! - unset BATS_NO_PARALLELIZE_ACROSS_FILES - run bash -c "bats --jobs $PARALLELITY \"${FIXTURE_ROOT}/suite/\" 2> >(grep -v '^parallel: Warning: ')" + unset BATS_NO_PARALLELIZE_ACROSS_FILES + reentrant_run bash -c "bats --jobs $PARALLELITY \"${FIXTURE_ROOT}/suite/\" 2> >(grep -v '^parallel: Warning: ')" echo "$output" [ "$status" -eq 0 ] @@ -102,11 +105,14 @@ check_parallel_tests() { # # file parallelization is needed for this test! # If we got over the skip (if no GNU parallel) in setup() we can reenable it safely! - unset BATS_NO_PARALLELIZE_ACROSS_FILES + unset BATS_NO_PARALLELIZE_ACROSS_FILES # run 4 files with parallelity of 2 -> serialize 2 - run bats --jobs $PARALLELITY "$FIXTURE_ROOT/setup_file" + reentrant_run bats --jobs $PARALLELITY "$FIXTURE_ROOT/setup_file" - [[ $status -eq 0 ]] || (echo "$output"; false) + [[ $status -eq 0 ]] || ( + echo "$output" + false + ) cat "$FILE_MARKER" @@ -117,7 +123,7 @@ check_parallel_tests() { # } @test "running the same file twice runs its tests twice without errors" { - run bats --jobs 2 "$FIXTURE_ROOT/../bats/passing.bats" "$FIXTURE_ROOT/../bats/passing.bats" + reentrant_run bats --jobs 2 "$FIXTURE_ROOT/../bats/passing.bats" "$FIXTURE_ROOT/../bats/passing.bats" echo "$output" [[ $status -eq 0 ]] [[ "${lines[0]}" == "1..2" ]] # got 2x1 tests @@ -132,20 +138,20 @@ check_parallel_tests() { # bats --jobs $PARALLELITY "$FIXTURE_ROOT/parallel_factor.bats" local current_parallel_count=0 maximum_parallel_count=0 total_count=0 while read -r line; do - case "$line" in - setup*) - ((++current_parallel_count)) - ((++total_count)) + case "$line" in + setup*) + ((++current_parallel_count)) + ((++total_count)) + ;; + teardown*) + ((current_parallel_count--)) ;; - teardown*) - ((current_parallel_count--)) - ;; esac - if (( current_parallel_count > maximum_parallel_count )); then + if ((current_parallel_count > maximum_parallel_count)); then maximum_parallel_count=$current_parallel_count fi - done < "$MARKER_FILE" - + done <"$MARKER_FILE" + cat "$MARKER_FILE" # for debugging purposes [[ "$maximum_parallel_count" -eq $PARALLELITY ]] [[ "$current_parallel_count" -eq 0 ]] @@ -153,7 +159,7 @@ check_parallel_tests() { # } @test "parallel mode correctly forwards failure return code" { - run bats --jobs 2 "$FIXTURE_ROOT/../bats/failing.bats" + reentrant_run bats --jobs 2 "$FIXTURE_ROOT/../bats/failing.bats" [[ "$status" -eq 1 ]] } @@ -162,7 +168,7 @@ check_parallel_tests() { # # (setup should have skipped already, if there was no GNU parallel) unset BATS_NO_PARALLELIZE_ACROSS_FILES FILE_MARKER=$(mktemp "${BATS_RUN_TMPDIR}/file_marker.XXXXXX") \ - run ! bats --jobs 2 "$FIXTURE_ROOT/must_not_parallelize_across_files/" + reentrant_run ! bats --jobs 2 "$FIXTURE_ROOT/must_not_parallelize_across_files/" } @test "--no-parallelize-across-files prevents parallelization across files" { @@ -171,11 +177,11 @@ check_parallel_tests() { # } @test "--no-parallelize-across-files does not prevent parallelization within files" { - run ! bats --jobs 2 --no-parallelize-across-files "$FIXTURE_ROOT/must_not_parallelize_within_file.bats" + reentrant_run ! bats --jobs 2 --no-parallelize-across-files "$FIXTURE_ROOT/must_not_parallelize_within_file.bats" } @test "--no-parallelize-within-files test file detects parallel execution" { - run ! bats --jobs 2 "$FIXTURE_ROOT/must_not_parallelize_within_file.bats" + reentrant_run ! bats --jobs 2 "$FIXTURE_ROOT/must_not_parallelize_within_file.bats" } @test "--no-parallelize-within-files prevents parallelization within files" { @@ -187,7 +193,7 @@ check_parallel_tests() { # # (setup should have skipped already, if there was no GNU parallel) unset BATS_NO_PARALLELIZE_ACROSS_FILES FILEMARKER=$(mktemp "${BATS_RUN_TMPDIR}/file_marker.XXXXXX") \ - run ! bats --jobs 2 --no-parallelize-within-files "$FIXTURE_ROOT/must_not_parallelize_across_files/" + reentrant_run ! bats --jobs 2 --no-parallelize-within-files "$FIXTURE_ROOT/must_not_parallelize_across_files/" } @test "BATS_NO_PARALLELIZE_WITHIN_FILE works from inside setup_file()" { @@ -199,9 +205,16 @@ check_parallel_tests() { # } @test "BATS_NO_PARALLELIZE_WITHIN_FILE does not work from inside setup()" { - DISABLE_IN_SETUP_FUNCTION=1 run ! bats --jobs 2 "$FIXTURE_ROOT/must_not_parallelize_within_file.bats" + DISABLE_IN_SETUP_FUNCTION=1 reentrant_run ! bats --jobs 2 "$FIXTURE_ROOT/must_not_parallelize_within_file.bats" } @test "BATS_NO_PARALLELIZE_WITHIN_FILE does not work from inside test function" { - DISABLE_IN_TEST_FUNCTION=1 run ! bats --jobs 2 "$FIXTURE_ROOT/must_not_parallelize_within_file.bats" + DISABLE_IN_TEST_FUNCTION=1 reentrant_run ! bats --jobs 2 "$FIXTURE_ROOT/must_not_parallelize_within_file.bats" +} + +@test "Short form typo does not run endlessly" { + unset BATS_NO_PARALLELIZE_ACROSS_FILES + run bats -j2 "$FIXTURE_ROOT/../bats/passing.bats" + (( SECONDS < 5 )) + [ "${lines[1]}" = 'Invalid number of jobs: -2' ] } diff --git a/test/pretty-formatter.bats b/test/pretty-formatter.bats index 488354392c..a345e86d86 100644 --- a/test/pretty-formatter.bats +++ b/test/pretty-formatter.bats @@ -1,14 +1,78 @@ +#!/usr/bin/env bats + +setup() { + load test_helper + fixtures bats +} @test "Timing printout shows milliseconds" { - format_example_stream() { - bats-format-pretty -T < /usr/bin/bar (relative executable) + # - /usr/bin/foo => /usr/bin/bar (relative executable) ln -s bar usr/bin/foo -# - /usr/bin/bar => /opt/bats/bin0/bar (absolute executable) + # - /usr/bin/bar => /opt/bats/bin0/bar (absolute executable) ln -s "$BATS_TEST_TMPDIR/opt/bats/bin0/bar" usr/bin/bar -# - /opt/bats/bin0 => /opt/bats/bin1 (relative directory) + # - /opt/bats/bin0 => /opt/bats/bin1 (relative directory) ln -s bin1 opt/bats/bin0 -# - /opt/bats/bin1 => /opt/bats/bin2 (absolute directory) + # - /opt/bats/bin1 => /opt/bats/bin2 (absolute directory) ln -s "$BATS_TEST_TMPDIR/opt/bats/bin2" opt/bats/bin1 -# - /opt/bats/bin2/bar => /opt/bats-core/bin/bar (absolute executable) + # - /opt/bats/bin2/bar => /opt/bats-core/bin/bar (absolute executable) ln -s "$BATS_TEST_TMPDIR/opt/bats-core/bin/bar" opt/bats/bin2/bar -# - /opt/bats-core/bin/bar => /opt/bats-core/bin/baz (relative executable) + # - /opt/bats-core/bin/bar => /opt/bats-core/bin/baz (relative executable) ln -s baz opt/bats-core/bin/bar -# - /opt/bats-core/bin/baz => /opt/bats-core/bin/bats (relative executable) + # - /opt/bats-core/bin/baz => /opt/bats-core/bin/bats (relative executable) ln -s bats opt/bats-core/bin/baz # shellcheck disable=SC2103,SC2164 cd - >/dev/null - run "$BATS_TEST_TMPDIR/bin/foo" -v + reentrant_run "$BATS_TEST_TMPDIR/bin/foo" -v echo "$output" [ "$status" -eq 0 ] [ "${output%% *}" == 'Bats' ] @@ -76,7 +78,7 @@ setup() { @test "set BATS_ROOT when calling from same dir" { cd "$BATS_TEST_TMPDIR" - run ./bin/bats -v + reentrant_run ./bin/bats -v [ "$status" -eq 0 ] [ "${output%% *}" == 'Bats' ] } @@ -85,7 +87,7 @@ setup() { cd /tmp # shellcheck disable=SC2031,SC2030 PATH="$PATH:$BATS_TEST_TMPDIR/bin" - run bats -v + reentrant_run bats -v [ "$status" -eq 0 ] [ "${output%% *}" == 'Bats' ] } @@ -94,7 +96,7 @@ setup() { cd /tmp # shellcheck disable=SC2031,SC2030 PATH="$PATH:$BATS_TEST_TMPDIR/bin" - run bash bats -v + reentrant_run bash bats -v [ "$status" -eq 0 ] [ "${output%% *}" == 'Bats' ] } diff --git a/test/run.bats b/test/run.bats index 2fc0e53449..10a0595249 100644 --- a/test/run.bats +++ b/test/run.bats @@ -3,91 +3,91 @@ fixtures run bats_require_minimum_version 1.5.0 @test "run --keep-empty-lines preserves leading empty lines" { - run --keep-empty-lines -- echo -n $'\na' - printf "'%s'\n" "${lines[@]}" - [ "${lines[0]}" == '' ] - [ "${lines[1]}" == a ] - [ ${#lines[@]} -eq 2 ] + run --keep-empty-lines -- echo -n $'\na' + printf "'%s'\n" "${lines[@]}" + [ "${lines[0]}" == '' ] + [ "${lines[1]}" == a ] + [ ${#lines[@]} -eq 2 ] } @test "run --keep-empty-lines preserves inner empty lines" { - run --keep-empty-lines -- echo -n $'a\n\nb' - printf "'%s'\n" "${lines[@]}" - [ "${lines[0]}" == a ] - [ "${lines[1]}" == '' ] - [ "${lines[2]}" == b ] - [ ${#lines[@]} -eq 3 ] + run --keep-empty-lines -- echo -n $'a\n\nb' + printf "'%s'\n" "${lines[@]}" + [ "${lines[0]}" == a ] + [ "${lines[1]}" == '' ] + [ "${lines[2]}" == b ] + [ ${#lines[@]} -eq 3 ] } @test "run --keep-empty-lines preserves trailing empty lines" { - run --keep-empty-lines -- echo -n $'a\n' - printf "'%s'\n" "${lines[@]}" - [ "${lines[0]}" == a ] - [ "${lines[1]}" == '' ] - [ ${#lines[@]} -eq 2 ] + run --keep-empty-lines -- echo -n $'a\n' + printf "'%s'\n" "${lines[@]}" + [ "${lines[0]}" == a ] + [ "${lines[1]}" == '' ] + [ ${#lines[@]} -eq 2 ] } @test "run --keep-empty-lines preserves multiple trailing empty lines" { - run --keep-empty-lines -- echo -n $'a\n\n' - printf "'%s'\n" "${lines[@]}" - [ "${lines[0]}" == a ] - [ "${lines[1]}" == '' ] - [ "${lines[2]}" == '' ] - [ ${#lines[@]} -eq 3 ] + run --keep-empty-lines -- echo -n $'a\n\n' + printf "'%s'\n" "${lines[@]}" + [ "${lines[0]}" == a ] + [ "${lines[1]}" == '' ] + [ "${lines[2]}" == '' ] + [ ${#lines[@]} -eq 3 ] } @test "run --keep-empty-lines preserves non-empty trailing line" { - run --keep-empty-lines -- echo -n $'a\nb' - printf "'%s'\n" "${lines[@]}" - [ "${lines[0]}" == a ] - [ "${lines[1]}" == b ] - [ ${#lines[@]} -eq 2 ] + run --keep-empty-lines -- echo -n $'a\nb' + printf "'%s'\n" "${lines[@]}" + [ "${lines[0]}" == a ] + [ "${lines[1]}" == b ] + [ ${#lines[@]} -eq 2 ] } @test "--keep-empty-lines has zero lines for empty output (see #573)" { - run --keep-empty-lines true - printf "'%s'\n" "${lines[@]}" - [ ${#lines[@]} -eq 0 ] + run --keep-empty-lines true + printf "'%s'\n" "${lines[@]}" + [ ${#lines[@]} -eq 0 ] } print-stderr-stdout() { - printf stdout - printf stderr >&2 + printf stdout + printf stderr >&2 } @test "run --separate-stderr splits output" { - local stderr stderr_lines # silence shellcheck - run --separate-stderr -- print-stderr-stdout - echo "output='$output' stderr='$stderr'" - [ "$output" = stdout ] - [ ${#lines[@]} -eq 1 ] + local stderr stderr_lines # silence shellcheck + run --separate-stderr -- print-stderr-stdout + echo "output='$output' stderr='$stderr'" + [ "$output" = stdout ] + [ ${#lines[@]} -eq 1 ] - [ "$stderr" = stderr ] - [ ${#stderr_lines[@]} -eq 1 ] + [ "$stderr" = stderr ] + [ ${#stderr_lines[@]} -eq 1 ] } @test "run does not change set flags" { - old_flags="$-" - run true - echo -- "$-" == "$old_flags" - [ "$-" == "$old_flags" ] + old_flags="$-" + run true + echo -- "$-" == "$old_flags" + [ "$-" == "$old_flags" ] } # Positive assertions: each of these should succeed @test "basic return-code checking" { - run true - run -0 true - run '!' false - run -1 false - run -3 exit 3 - run -5 exit 5 + run true + run -0 true + run '!' false + run -1 false + run -3 exit 3 + run -5 exit 5 run -111 exit 111 run -255 exit 255 run -127 /no/such/command } @test "run exit code check output " { - run ! bats --tap "${FIXTURE_ROOT}/failing.bats" + reentrant_run ! bats --tap "${FIXTURE_ROOT}/failing.bats" echo "$output" [ "${lines[0]}" == 1..5 ] [ "${lines[1]}" == "not ok 1 run -0 false" ] @@ -108,15 +108,33 @@ print-stderr-stdout() { } @test "run invalid exit code check error message" { - run ! bats --tap "${FIXTURE_ROOT}/invalid.bats" + reentrant_run ! bats --tap "${FIXTURE_ROOT}/invalid.bats" echo "$output" [ "${lines[0]}" == 1..2 ] [ "${lines[1]}" == "not ok 1 run '-4evah' echo hi" ] [ "${lines[2]}" == "# (in test file ${RELATIVE_FIXTURE_ROOT}/invalid.bats, line 2)" ] [ "${lines[3]}" == "# \`run '-4evah' echo hi' failed" ] - [ "${lines[4]}" == "# Usage error: run: '=NNN' requires numeric NNN (got: 4evah)" ] + [ "${lines[4]}" == "# Usage error: run: '-NNN' requires numeric NNN (got: 4evah)" ] [ "${lines[5]}" == "not ok 2 run -256 echo hi" ] [ "${lines[6]}" == "# (in test file ${RELATIVE_FIXTURE_ROOT}/invalid.bats, line 6)" ] [ "${lines[7]}" == "# \`run -256 echo hi' failed" ] - [ "${lines[8]}" == "# Usage error: run: '=NNN': NNN must be <= 255 (got: 256)" ] + [ "${lines[8]}" == "# Usage error: run: '-NNN': NNN must be <= 255 (got: 256)" ] +} + +@test "run is not affected by IFS" { + IFS=_ + run true + [ "$status" -eq 0 ] + IFS=0 + run true + [ "$status" -eq 0 ] +} + +@test "run does not change IFS" { + local IFS=_ + run true + [ "$IFS" = _ ] + unset IFS + run true + [ "${IFS-(unset)}" = '(unset)' ] } diff --git a/test/suite.bats b/test/suite.bats index 01539d7875..31f9fa502b 100755 --- a/test/suite.bats +++ b/test/suite.bats @@ -1,33 +1,34 @@ -#!/usr/bin/env bats - -load test_helper -fixtures suite +setup() { + load test_helper + fixtures suite + REENTRANT_RUN_PRESERVE+=(BATS_LIBEXEC) +} @test "running a suite with no test files" { - run bats "$FIXTURE_ROOT/empty" + reentrant_run bats "$FIXTURE_ROOT/empty" [ $status -eq 0 ] [ "$output" = "1..0" ] } @test "running a suite with one test file" { - run bats "$FIXTURE_ROOT/single" + reentrant_run bats "$FIXTURE_ROOT/single" [ $status -eq 0 ] [ "${lines[0]}" = "1..1" ] [ "${lines[1]}" = "ok 1 a passing test" ] } @test "counting tests in a suite" { - run bats -c "$FIXTURE_ROOT/single" + reentrant_run bats -c "$FIXTURE_ROOT/single" [ $status -eq 0 ] [ "$output" -eq 1 ] - run bats -c "$FIXTURE_ROOT/multiple" + reentrant_run bats -c "$FIXTURE_ROOT/multiple" [ $status -eq 0 ] [ "$output" -eq 3 ] } @test "aggregated output of multiple tests in a suite" { - run bats "$FIXTURE_ROOT/multiple" + reentrant_run bats "$FIXTURE_ROOT/multiple" [ $status -eq 0 ] [ "${lines[0]}" = "1..3" ] echo "$output" | grep "^ok . truth" @@ -36,14 +37,14 @@ fixtures suite } @test "a failing test in a suite results in an error exit code" { - FLUNK=1 run bats "$FIXTURE_ROOT/multiple" + FLUNK=1 reentrant_run bats "$FIXTURE_ROOT/multiple" [ $status -eq 1 ] [ "${lines[0]}" = "1..3" ] echo "$output" | grep "^not ok . quasi-truth" } @test "running an ad-hoc suite by specifying multiple test files" { - run bats "$FIXTURE_ROOT/multiple/a.bats" "$FIXTURE_ROOT/multiple/b.bats" + reentrant_run bats "$FIXTURE_ROOT/multiple/a.bats" "$FIXTURE_ROOT/multiple/b.bats" [ $status -eq 0 ] [ "${lines[0]}" = "1..3" ] echo "$output" | grep "^ok . truth" @@ -53,7 +54,7 @@ fixtures suite @test "extended syntax in suite" { emulate_bats_env - FLUNK=1 run bats-exec-suite -x "$FIXTURE_ROOT/multiple/"*.bats + FLUNK=1 reentrant_run bats-exec-suite -x "$FIXTURE_ROOT/multiple/"*.bats echo "output: $output" [ $status -eq 1 ] [ "${lines[0]}" = "1..3" ] @@ -69,21 +70,21 @@ fixtures suite @test "timing syntax in suite" { emulate_bats_env - FLUNK=1 run bats-exec-suite -T "$FIXTURE_ROOT/multiple/"*.bats + FLUNK=1 reentrant_run bats-exec-suite -T "$FIXTURE_ROOT/multiple/"*.bats echo "$output" [ $status -eq 1 ] [ "${lines[0]}" = "1..3" ] regex="ok 1 truth in [0-9]+ms" [[ "${lines[1]}" =~ $regex ]] regex="ok 2 more truth in [0-9]+ms" - [[ "${lines[2]}" =~ $regex ]] + [[ "${lines[2]}" =~ $regex ]] regex="not ok 3 quasi-truth in [0-9]+ms" - [[ "${lines[3]}" =~ $regex ]] + [[ "${lines[3]}" =~ $regex ]] } @test "extended timing syntax in suite" { emulate_bats_env - FLUNK=1 run bats-exec-suite -x -T "$FIXTURE_ROOT/multiple/"*.bats + FLUNK=1 reentrant_run bats-exec-suite -x -T "$FIXTURE_ROOT/multiple/"*.bats echo "$output" [ $status -eq 1 ] [ "${lines[0]}" = "1..3" ] @@ -101,7 +102,7 @@ fixtures suite } @test "recursive support (short option)" { - run bats -r "${FIXTURE_ROOT}/recursive" + reentrant_run bats -r "${FIXTURE_ROOT}/recursive" [ $status -eq 0 ] [ "${lines[0]}" = "1..2" ] [ "${lines[1]}" = "ok 1 another passing test" ] @@ -109,7 +110,7 @@ fixtures suite } @test "recursive support (long option)" { - run bats --recursive "${FIXTURE_ROOT}/recursive" + reentrant_run bats --recursive "${FIXTURE_ROOT}/recursive" [ $status -eq 0 ] [ "${lines[0]}" = "1..2" ] [ "${lines[1]}" = "ok 1 another passing test" ] @@ -121,7 +122,7 @@ fixtures suite skip "symbolic links aren't functional on OSTYPE=$OSTYPE" fi - run bats -r "${FIXTURE_ROOT}/recursive_with_symlinks" + reentrant_run bats -r "${FIXTURE_ROOT}/recursive_with_symlinks" [ $status -eq 0 ] [ "${lines[0]}" = "1..2" ] [ "${lines[1]}" = "ok 1 another passing test" ] @@ -129,7 +130,7 @@ fixtures suite } @test "run entire suite when --filter isn't set" { - run bats "${FIXTURE_ROOT}/filter" + reentrant_run bats "${FIXTURE_ROOT}/filter" [ "$status" -eq 0 ] [ "${lines[0]}" = '1..9' ] [ "${lines[1]}" = 'ok 1 foo in a' ] @@ -144,7 +145,7 @@ fixtures suite } @test "use --filter to run subset of test cases from across the suite" { - run bats -f 'ba[rz]' "${FIXTURE_ROOT}/filter" + reentrant_run bats -f 'ba[rz]' "${FIXTURE_ROOT}/filter" [ "$status" -eq 0 ] [ "${lines[0]}" = '1..4' ] [ "${lines[1]}" = 'ok 1 --bar in a' ] @@ -154,13 +155,13 @@ fixtures suite local prev_output="$output" - run bats --filter 'ba[rz]' "${FIXTURE_ROOT}/filter" + reentrant_run bats --filter 'ba[rz]' "${FIXTURE_ROOT}/filter" [ "$status" -eq 0 ] [ "$output" = "$prev_output" ] } @test "--filter can handle regular expressions that contain [_- ]" { - run bats -f '--ba[rz][ _]in' "${FIXTURE_ROOT}/filter" + reentrant_run bats -f '--ba[rz][ _]in' "${FIXTURE_ROOT}/filter" [ "$status" -eq 0 ] [ "${lines[0]}" = '1..2' ] [ "${lines[1]}" = 'ok 1 --bar in a' ] @@ -168,7 +169,7 @@ fixtures suite } @test "--filter can handle regular expressions that start with ^" { - run bats -f '^ba[rz]' "${FIXTURE_ROOT}/filter" + reentrant_run bats -f '^ba[rz]' "${FIXTURE_ROOT}/filter" [ "$status" -eq 0 ] [ "${lines[0]}" = '1..2' ] [ "${lines[1]}" = 'ok 1 baz in a' ] @@ -180,13 +181,15 @@ fixtures suite } @test "BATS_TEST_NUMBER starts at 1 in each individual test file" { - run bats "${FIXTURE_ROOT}/test_number" + reentrant_run bats "${FIXTURE_ROOT}/test_number" echo "$output" [ "$status" -eq 0 ] } @test "Override BATS_FILE_EXTENSION with suite" { - BATS_FILE_EXTENSION="test" run bats "${FIXTURE_ROOT}/override_BATS_FILE_EXTENSION" + # shellcheck disable=SC2030 + REENTRANT_RUN_PRESERVE+=(BATS_FILE_EXTENSION) + BATS_FILE_EXTENSION="test" reentrant_run bats "${FIXTURE_ROOT}/override_BATS_FILE_EXTENSION" echo "$output" [ "$status" -eq 0 ] [ "${#lines[@]}" -eq 2 ] @@ -195,7 +198,9 @@ fixtures suite } @test "Override BATS_FILE_EXTENSION with suite recursive" { - BATS_FILE_EXTENSION="other_extension" run bats -r "${FIXTURE_ROOT}/override_BATS_FILE_EXTENSION" + # shellcheck disable=SC2030,SC2031 + REENTRANT_RUN_PRESERVE+=(BATS_FILE_EXTENSION) + BATS_FILE_EXTENSION="other_extension" reentrant_run bats -r "${FIXTURE_ROOT}/override_BATS_FILE_EXTENSION" echo "$output" [ "$status" -eq 0 ] [ "${#lines[@]}" -eq 2 ] diff --git a/test/suite_setup_teardown.bats b/test/suite_setup_teardown.bats index 3b60323ea0..235549fb36 100644 --- a/test/suite_setup_teardown.bats +++ b/test/suite_setup_teardown.bats @@ -3,98 +3,143 @@ fixtures suite_setup_teardown bats_require_minimum_version 1.5.0 setup() { - export LOGFILE="$BATS_TEST_TMPDIR/log" + export LOGFILE="$BATS_TEST_TMPDIR/log" } @test "setup_suite.bash is picked up in toplevel folder of suite" { - run -0 bats -r "$FIXTURE_ROOT/pick_up_toplevel" - run cat "$LOGFILE" + reentrant_run -0 bats -r "$FIXTURE_ROOT/pick_up_toplevel" + run cat "$LOGFILE" - [ "${lines[0]}" = "$FIXTURE_ROOT/pick_up_toplevel/setup_suite.bash setup_suite" ] - [ "${lines[1]}" = "$FIXTURE_ROOT/pick_up_toplevel/setup_suite.bash teardown_suite" ] + [ "${lines[0]}" = "$FIXTURE_ROOT/pick_up_toplevel/setup_suite.bash setup_suite" ] + [ "${lines[1]}" = "$FIXTURE_ROOT/pick_up_toplevel/setup_suite.bash teardown_suite" ] } @test "setup_suite.bash is picked up in folder of first test file" { - run -0 bats "$FIXTURE_ROOT/pick_up_toplevel/folder1/test.bats" "$FIXTURE_ROOT/pick_up_toplevel/folder2/test.bats" - run cat "$LOGFILE" + reentrant_run -0 bats "$FIXTURE_ROOT/pick_up_toplevel/folder1/test.bats" "$FIXTURE_ROOT/pick_up_toplevel/folder2/test.bats" + run cat "$LOGFILE" - [ "${lines[0]}" = "$FIXTURE_ROOT/pick_up_toplevel/folder1/setup_suite.bash setup_suite" ] - [ "${lines[1]}" = "$FIXTURE_ROOT/pick_up_toplevel/folder1/setup_suite.bash teardown_suite" ] + [ "${lines[0]}" = "$FIXTURE_ROOT/pick_up_toplevel/folder1/setup_suite.bash setup_suite" ] + [ "${lines[1]}" = "$FIXTURE_ROOT/pick_up_toplevel/folder1/setup_suite.bash teardown_suite" ] } @test "setup_suite is not picked up from wrongly named file" { - run -0 bats "$FIXTURE_ROOT/non_default_name/" - run cat "$LOGFILE" - [[ "${output}" != *"setup_suite"* ]] || false - [[ "${output}" != *"teardown_suite"* ]] || false + reentrant_run -0 bats "$FIXTURE_ROOT/non_default_name/" + run cat "$LOGFILE" + [[ "${output}" != *"setup_suite"* ]] || false + [[ "${output}" != *"teardown_suite"* ]] || false } @test "setup_suite is picked up from --setup-suite-file" { - run -0 bats "$FIXTURE_ROOT/non_default_name/" \ - --setup-suite-file "$FIXTURE_ROOT/non_default_name/setup_suite_non_default.bash" - run cat "$LOGFILE" - [ "${lines[0]}" == "setup_suite non_default" ] - [ "${lines[1]}" == "teardown_suite non_default" ] + reentrant_run -0 bats "$FIXTURE_ROOT/non_default_name/" \ + --setup-suite-file "$FIXTURE_ROOT/non_default_name/setup_suite_non_default.bash" + run cat "$LOGFILE" + [ "${lines[0]}" == "setup_suite non_default" ] + [ "${lines[1]}" == "teardown_suite non_default" ] } @test "--setup-suite-file takes precedence over convention" { - run -0 bats "$FIXTURE_ROOT/default_name/" \ - --setup-suite-file "$FIXTURE_ROOT/non_default_name/setup_suite_non_default.bash" - run cat "$LOGFILE" - [ "${lines[0]}" == "setup_suite non_default" ] - [ "${lines[1]}" == "teardown_suite non_default" ] + reentrant_run -0 bats "$FIXTURE_ROOT/default_name/" \ + --setup-suite-file "$FIXTURE_ROOT/non_default_name/setup_suite_non_default.bash" + run cat "$LOGFILE" + [ "${lines[0]}" == "setup_suite non_default" ] + [ "${lines[1]}" == "teardown_suite non_default" ] } @test "passing a nonexisting file to --setup-suite-file prints an error message" { - run -1 bats "$FIXTURE_ROOT/default_name/" \ - --setup-suite-file "/non-existing/setup_suite.bash" - [ "${lines[0]}" == "Error: --setup-suite-file /non-existing/setup_suite.bash does not exist!" ] + reentrant_run -1 bats "$FIXTURE_ROOT/default_name/" \ + --setup-suite-file "/non-existing/setup_suite.bash" + [ "${lines[0]}" == "Error: --setup-suite-file /non-existing/setup_suite.bash does not exist!" ] } @test "setup_suite.bash without setup_suite() is an error" { - run ! bats "$FIXTURE_ROOT/no_setup_suite_function/" - [ "${lines[1]}" == "$FIXTURE_ROOT/no_setup_suite_function/setup_suite.bash does not define \`setup_suite()\`" ] + reentrant_run ! bats "$FIXTURE_ROOT/no_setup_suite_function/" + [ "${lines[0]}" == "1..1" ] + [ "${lines[1]}" == "not ok 1 setup_suite" ] + [ "${lines[2]}" == "# $FIXTURE_ROOT/no_setup_suite_function/setup_suite.bash does not define \`setup_suite()\`" ] + [ "${#lines[@]}" -eq 3 ] } @test "exported variables from setup_suite are visible in setup_file, setup and @test" { - unset EXPORTED_VAR - EXPECTED_VALUE=exported_var run -0 bats "$FIXTURE_ROOT/exported_vars/" + unset EXPORTED_VAR + EXPECTED_VALUE=exported_var reentrant_run -0 bats "$FIXTURE_ROOT/exported_vars/" } @test "syntax errors in setup_suite.bash are reported and lead to non zero exit code" { - LANG=C run ! bats --setup-suite-file "$FIXTURE_ROOT/syntax_error/setup_suite_no_shellcheck" "$FIXTURE_ROOT/syntax_error/" - [ "${lines[1]}" == "$FIXTURE_ROOT/syntax_error/setup_suite_no_shellcheck: line 2: syntax error: unexpected end of file" ] + LANG=C reentrant_run ! bats --setup-suite-file "$FIXTURE_ROOT/syntax_error/setup_suite_no_shellcheck" "$FIXTURE_ROOT/syntax_error/" + [ "${lines[1]}" == "$FIXTURE_ROOT/syntax_error/setup_suite_no_shellcheck: line 2: syntax error: unexpected end of file" ] } @test "errors in setup_suite.bash's free code reported correctly" { - LANG=C run ! bats "$FIXTURE_ROOT/error_in_free_code/" - [ "${lines[1]}" == "$FIXTURE_ROOT/error_in_free_code/setup_suite.bash: line 1: call-to-undefined-command: command not found" ] + LANG=C reentrant_run ! bats "$FIXTURE_ROOT/error_in_free_code/" + [ "${lines[1]}" == "$FIXTURE_ROOT/error_in_free_code/setup_suite.bash: line 1: call-to-undefined-command: command not found" ] } @test "errors in setup_suite reported correctly" { - LANG=C run ! bats "$FIXTURE_ROOT/error_in_setup_suite/" - [ "${lines[1]}" == "$FIXTURE_ROOT/error_in_setup_suite/setup_suite.bash: line 2: call-to-undefined-command: command not found" ] + LANG=C reentrant_run ! bats "$FIXTURE_ROOT/error_in_setup_suite/" + [ "${lines[4]}" == "# $FIXTURE_ROOT/error_in_setup_suite/setup_suite.bash: line 2: call-to-undefined-command: command not found" ] } @test "errors in teardown_suite reported correctly" { - LANG=C run ! bats "$FIXTURE_ROOT/error_in_teardown_suite/" - [ "${lines[2]}" == "$FIXTURE_ROOT/error_in_teardown_suite/setup_suite.bash: line 6: call-to-undefined-command: command not found" ] + LANG=C reentrant_run ! bats "$FIXTURE_ROOT/error_in_teardown_suite/" + [ "${lines[5]}" == "# $FIXTURE_ROOT/error_in_teardown_suite/setup_suite.bash: line 6: call-to-undefined-command: command not found" ] } @test "failure in setup_suite skips further setup and suite but runs teardown_suite" { - run ! bats "$FIXTURE_ROOT/failure_in_setup_suite/" - [ "${lines[1]}" == "setup_suite before" ] # <- only setup_suite code before failure is run - [ "${lines[2]}" == "teardown_suite" ] # <- teardown is run - # get a nice error message - [ "${lines[3]}" == "not ok 1 setup_suite" ] - [ "${lines[4]}" == "# (from function \`setup_suite' in test file $RELATIVE_FIXTURE_ROOT/failure_in_setup_suite/setup_suite.bash, line 3)" ] - [ "${lines[5]}" == "# \`false' failed" ] -} - -@test "failure in teardown_suite is reported and fails test suite, remaining code is skipped" { - run ! bats "$FIXTURE_ROOT/failure_in_teardown_suite/" - [ "${lines[2]}" == "teardown_suite before" ] - [ "${lines[3]}" == "not ok 2 teardown_suite" ] - [ "${lines[4]}" == "# (from function \`teardown_suite' in test file $RELATIVE_FIXTURE_ROOT/failure_in_teardown_suite/setup_suite.bash, line 7)" ] - [ "${lines[5]}" == "# \`false' failed" ] -} \ No newline at end of file + reentrant_run ! bats "$FIXTURE_ROOT/failure_in_setup_suite/" + [ "${lines[0]}" == "1..1" ] + # get a nice error message + [ "${lines[1]}" == "not ok 1 setup_suite" ] + [ "${lines[2]}" == "# (from function \`setup_suite' in test file $RELATIVE_FIXTURE_ROOT/failure_in_setup_suite/setup_suite.bash, line 3)" ] + [ "${lines[3]}" == "# \`false' failed" ] + [ "${lines[4]}" == "# setup_suite before" ] # <- only setup_suite code before failure is run + [ "${lines[5]}" == "# teardown_suite" ] # <- teardown is run + [ ${#lines[@]} -eq 6 ] +} + +@test "midway failure in teardown_suite does not fail test suite, remaining code is executed" { + reentrant_run -0 bats --show-output-of-passing-tests "$FIXTURE_ROOT/failure_in_teardown_suite/" + [ "${lines[2]}" == "# teardown_suite before" ] + [ "${lines[3]}" == "# teardown_suite after" ] + [ "${#lines[@]}" -eq 4 ] +} + +@test "nonzero return in teardown_suite does fails test suite" { + reentrant_run -1 bats "$FIXTURE_ROOT/return_nonzero_in_teardown_suite/" + [ "${lines[0]}" == "1..1" ] + [ "${lines[1]}" == "ok 1 test" ] + [ "${lines[2]}" == "not ok 2 teardown_suite" ] + [ "${lines[3]}" == "# (from function \`teardown_suite' in test file $RELATIVE_FIXTURE_ROOT/return_nonzero_in_teardown_suite/setup_suite.bash, line 7)" ] + [ "${lines[4]}" == "# \`return 1' failed" ] + [ "${lines[5]}" == "# teardown_suite before" ] + [ "${lines[6]}" == "# bats warning: Executed 2 instead of expected 1 tests" ] + [ "${#lines[@]}" -eq 7 ] +} + +@test "stderr from setup/teardown_suite does not overtake stdout" { + reentrant_run -1 --separate-stderr bats "$FIXTURE_ROOT/stderr_in_setup_teardown_suite/" + [[ "$output" == *$'setup_suite stdout\n'*'setup_suite stderr'* ]] || false + [[ "$output" == *$'teardown_suite stdout\n'*'teardown_suite stderr'* ]] || false +} + +@test "load is available in setup_suite" { + reentrant_run -0 bats "$FIXTURE_ROOT/call_load/" + [ "${lines[0]}" = "1..1" ] + [ "${lines[1]}" = "ok 1 passing" ] + [ "${#lines[@]}" -eq 2 ] +} + +@test "output frorm setup_suite is only visible on failure" { + reentrant_run -0 bats "$FIXTURE_ROOT/default_name/" + [ "${lines[0]}" = "1..1" ] + [ "${lines[1]}" = "ok 1 passing" ] + [ "${#lines[@]}" -eq 2 ] + + reentrant_run -1 bats "$FIXTURE_ROOT/output_with_failure/" + [ "${lines[0]}" = "1..1" ] + [ "${lines[1]}" = "not ok 1 setup_suite" ] + [ "${lines[2]}" = "# (from function \`setup_suite' in test file $RELATIVE_FIXTURE_ROOT/output_with_failure/setup_suite.bash, line 3)" ] + [ "${lines[3]}" = "# \`false' failed" ] + [ "${lines[4]}" = "# setup_suite" ] + [ "${lines[5]}" = "# teardown_suite" ] + [ "${#lines[@]}" -eq 6 ] +} diff --git a/test/tagging.bats b/test/tagging.bats new file mode 100644 index 0000000000..b8f027e1a3 --- /dev/null +++ b/test/tagging.bats @@ -0,0 +1,75 @@ +#!/usr/bin/env bats + +bats_require_minimum_version 1.5.0 + +setup() { + load test_helper + fixtures tagging +} + +@test "No tag filter runs all tests" { + run -0 bats "$FIXTURE_ROOT/tagged.bats" + [ "${lines[0]}" == "1..5" ] + [ "${lines[1]}" == "ok 1 No tags" ] + [ "${lines[2]}" == "ok 2 Only file tags" ] + [ "${lines[3]}" == "ok 3 File and test tags" ] + [ "${lines[4]}" == "ok 4 File and other test tags" ] + [ "${lines[5]}" == "ok 5 Only test tags" ] + [ ${#lines[@]} -eq 6 ] +} + +@test "Empty tag filter runs tests without tag" { + run -0 bats --filter-tags '' "$FIXTURE_ROOT/tagged.bats" + [ "${lines[0]}" == "1..1" ] + [ "${lines[1]}" == "ok 1 No tags" ] + [ ${#lines[@]} -eq 2 ] +} + +@test "--filter-tags (also) selects tests that contain additional tags" { + run -0 bats --filter-tags 'file:tag:1' "$FIXTURE_ROOT/tagged.bats" + [ "${lines[0]}" == "1..3" ] + [ "${lines[1]}" == "ok 1 Only file tags" ] + [ "${lines[2]}" == "ok 2 File and test tags" ] + [ "${lines[3]}" == "ok 3 File and other test tags" ] + [ ${#lines[@]} -eq 4 ] +} + +@test "--filter-tags only selects tests that match all tags (logic and)" { + run -0 bats --filter-tags 'test:tag:1,file:tag:1' "$FIXTURE_ROOT/tagged.bats" + [ "${lines[0]}" == "1..1" ] + [ "${lines[1]}" == "ok 1 File and test tags" ] + [ ${#lines[@]} -eq 2 ] +} + +@test "multiple --filter-tags work as logical or" { + run -0 bats --filter-tags 'test:tag:1,file:tag:1' --filter-tags 'test:tag:2,file:tag:1' "$FIXTURE_ROOT/tagged.bats" + [ "${lines[0]}" == "1..2" ] + [ "${lines[1]}" == "ok 1 File and test tags" ] + [ "${lines[2]}" == "ok 2 File and other test tags" ] + [ ${#lines[@]} -eq 3 ] +} + +@test "--filter-tags order of tags does not matter" { + # note the reversed order in comparison to test above + run -0 bats --filter-tags 'file:tag:1,test:tag:1' "$FIXTURE_ROOT/tagged.bats" + [ "${lines[0]}" == "1..1" ] + [ "${lines[1]}" == "ok 1 File and test tags" ] + [ ${#lines[@]} -eq 2 ] +} + +@test "exit with error on invalid tags in .bats file" { + run -1 bats "$FIXTURE_ROOT/invalid_tags.bats" + [ "${lines[0]}" = "1..1" ] + [ "${lines[1]}" = "$FIXTURE_ROOT/invalid_tags.bats:1: Error: Invalid file_tags: ',bc'. Tags must not be empty. Please remove redundant commas!" ] + [ "${lines[2]}" = "$FIXTURE_ROOT/invalid_tags.bats:2: Error: Invalid file_tags: 'a+b'. Valid tags must match [-_:[:alnum:]]+ and be separated with comma (and optional spaces)" ] + [ "${lines[3]}" = "$FIXTURE_ROOT/invalid_tags.bats:4: Error: Invalid test_tags: ',bc'. Tags must not be empty. Please remove redundant commas!" ] + [ "${lines[4]}" = "$FIXTURE_ROOT/invalid_tags.bats:5: Error: Invalid test_tags: 'a+b'. Valid tags must match [-_:[:alnum:]]+ and be separated with comma (and optional spaces)" ] +} + +@test "--filter-tags allows for negation via !" { + run -0 bats --filter-tags '!file:tag:1' "$FIXTURE_ROOT/tagged.bats" + [ "${lines[0]}" = '1..2' ] + [ "${lines[1]}" = 'ok 1 No tags' ] + [ "${lines[2]}" = 'ok 2 Only test tags' ] + [ "${#lines[@]}" -eq 3 ] +} diff --git a/test/tap13-formatter.bats b/test/tap13-formatter.bats index 74e71dba24..075c8a8fc9 100644 --- a/test/tap13-formatter.bats +++ b/test/tap13-formatter.bats @@ -2,52 +2,52 @@ load 'test_helper' fixtures bats # reuse bats fixtures @test "passing test" { - run bats --formatter tap13 "${FIXTURE_ROOT}/passing.bats" - - [ "${lines[0]}" == 'TAP version 13' ] - [ "${lines[1]}" == '1..1' ] - [ "${lines[2]}" == 'ok 1 a passing test' ] - [ "${#lines[@]}" -eq 3 ] + reentrant_run bats --formatter tap13 "${FIXTURE_ROOT}/passing.bats" + + [ "${lines[0]}" == 'TAP version 13' ] + [ "${lines[1]}" == '1..1' ] + [ "${lines[2]}" == 'ok 1 a passing test' ] + [ "${#lines[@]}" -eq 3 ] } @test "failing test" { - run bats --formatter tap13 "${FIXTURE_ROOT}/failing.bats" - - [ "${lines[0]}" == 'TAP version 13' ] - [ "${lines[1]}" == '1..1' ] - [ "${lines[2]}" == 'not ok 1 a failing test' ] - [ "${lines[3]}" == ' ---' ] - [ "${lines[4]}" == ' message: |' ] - [ "${lines[5]}" == " (in test file ${RELATIVE_FIXTURE_ROOT}/failing.bats, line 4)" ] - [ "${lines[6]}" == " \`eval \"( exit \${STATUS:-1} )\"' failed" ] - [ "${lines[7]}" == ' ...' ] - [ "${#lines[@]}" -eq 8 ] + reentrant_run bats --formatter tap13 "${FIXTURE_ROOT}/failing.bats" + + [ "${lines[0]}" == 'TAP version 13' ] + [ "${lines[1]}" == '1..1' ] + [ "${lines[2]}" == 'not ok 1 a failing test' ] + [ "${lines[3]}" == ' ---' ] + [ "${lines[4]}" == ' message: |' ] + [ "${lines[5]}" == " (in test file ${RELATIVE_FIXTURE_ROOT}/failing.bats, line 4)" ] + [ "${lines[6]}" == " \`eval \"( exit \${STATUS:-1} )\"' failed" ] + [ "${lines[7]}" == ' ...' ] + [ "${#lines[@]}" -eq 8 ] } @test "passing test with timing" { - run bats --formatter tap13 --timing "${FIXTURE_ROOT}/passing.bats" - - [ "${lines[0]}" == 'TAP version 13' ] - [ "${lines[1]}" == '1..1' ] - [ "${lines[2]}" == 'ok 1 a passing test' ] - [ "${lines[3]}" == ' ---' ] - [ "${lines[4]::15}" == ' duration_ms: ' ] - [ "${lines[5]}" == ' ...' ] - - [ "${#lines[@]}" -eq 6 ] + reentrant_run bats --formatter tap13 --timing "${FIXTURE_ROOT}/passing.bats" + + [ "${lines[0]}" == 'TAP version 13' ] + [ "${lines[1]}" == '1..1' ] + [ "${lines[2]}" == 'ok 1 a passing test' ] + [ "${lines[3]}" == ' ---' ] + [ "${lines[4]::15}" == ' duration_ms: ' ] + [ "${lines[5]}" == ' ...' ] + + [ "${#lines[@]}" -eq 6 ] } @test "failing test with timing" { - run bats --formatter tap13 --timing "${FIXTURE_ROOT}/failing.bats" - - [ "${lines[0]}" == 'TAP version 13' ] - [ "${lines[1]}" == '1..1' ] - [ "${lines[2]}" == 'not ok 1 a failing test' ] - [ "${lines[3]}" == ' ---' ] - [ "${lines[4]::15}" == ' duration_ms: ' ] - [ "${lines[5]}" == ' message: |' ] - [ "${lines[6]}" == " (in test file ${RELATIVE_FIXTURE_ROOT}/failing.bats, line 4)" ] - [ "${lines[7]}" == " \`eval \"( exit \${STATUS:-1} )\"' failed" ] - [ "${lines[8]}" == ' ...' ] - [ "${#lines[@]}" -eq 9 ] -} \ No newline at end of file + reentrant_run bats --formatter tap13 --timing "${FIXTURE_ROOT}/failing.bats" + + [ "${lines[0]}" == 'TAP version 13' ] + [ "${lines[1]}" == '1..1' ] + [ "${lines[2]}" == 'not ok 1 a failing test' ] + [ "${lines[3]}" == ' ---' ] + [ "${lines[4]::15}" == ' duration_ms: ' ] + [ "${lines[5]}" == ' message: |' ] + [ "${lines[6]}" == " (in test file ${RELATIVE_FIXTURE_ROOT}/failing.bats, line 4)" ] + [ "${lines[7]}" == " \`eval \"( exit \${STATUS:-1} )\"' failed" ] + [ "${lines[8]}" == ' ...' ] + [ "${#lines[@]}" -eq 9 ] +} diff --git a/test/test_helper.bash b/test/test_helper.bash index dcb16d9ebe..4b46ad3a69 100644 --- a/test/test_helper.bash +++ b/test/test_helper.bash @@ -4,6 +4,7 @@ emulate_bats_env() { export BATS_ROOT_PID=$$ export BATS_RUN_TMPDIR BATS_RUN_TMPDIR=$(mktemp -d "${BATS_RUN_TMPDIR}/emulated-tmpdir-${BATS_ROOT_PID}-XXXXXX") + REENTRANT_RUN_PRESERVE+=(BATS_CWD BATS_TEST_FILTER BATS_ROOT_PID BATS_RUN_TMPDIR) } fixtures() { @@ -13,7 +14,9 @@ fixtures() { } filter_control_sequences() { - "$@" | sed $'s,\x1b\\[[0-9;]*[a-zA-Z],,g' + local status=0 + "$@" | sed $'s,\x1b\\[[0-9;]*[a-zA-Z],,g' || status=$? + return "$status" } if ! command -v tput >/dev/null; then @@ -27,3 +30,40 @@ emit_debug_output() { # shellcheck disable=SC2154 printf '%s\n' 'output:' "$output" >&2 } + +execute_with_unset_bats_vars() { # + for var_to_delete in "${!BATS_@}"; do + for var_to_exclude in "${REENTRANT_RUN_PRESERVE[@]-}"; do + # is this var excluded -> skip unset + if [[ $var_to_delete == "$var_to_exclude" ]]; then + continue 2 + fi + done + unset "$var_to_delete" + done + "$@" +} + +REENTRANT_RUN_PRESERVE+=(BATS_ROOT) + +# call run with all BATS_* variables purged from the environment +reentrant_run() { # + # take up all args to run except the command, + # to avoid having to deal with empty arrays in Bash 3 + local -a pre_command_args=() + + # remove all flags to run to leave the command in $@ + while [[ $1 == -* || $1 == ! ]]; do + pre_command_args+=("$1") + if [[ "$1" == -- ]]; then + shift + break + fi + shift + done + + # put that here to ensure Bash 3 won't have to deal with empty pre_command_args + pre_command_args+=(execute_with_unset_bats_vars) + + run "${pre_command_args[@]}" "$@" +} diff --git a/test/timeout.bats b/test/timeout.bats new file mode 100644 index 0000000000..79d10e42a2 --- /dev/null +++ b/test/timeout.bats @@ -0,0 +1,29 @@ +load test_helper +fixtures timeout + +bats_require_minimum_version 1.5.0 + +@test "test faster than timeout" { + SECONDS=0 + TIMEOUT=10 + reentrant_run -0 env BATS_TEST_TIMEOUT=$TIMEOUT SLEEP=1 bats -T "$FIXTURE_ROOT" + [ "${lines[0]}" == '1..1' ] + [[ "${lines[1]}" == 'ok 1 my sleep 1 in '*'ms' ]] || false + [ "${#lines[@]}" -eq 2 ] + # the timeout background process should not hold up execution + ((SECONDS < TIMEOUT)) || false +} + +@test "test longer than timeout" { + SECONDS=0 + reentrant_run ! env BATS_TEST_TIMEOUT=1 SLEEP=10 bats -T "$FIXTURE_ROOT" + [ "${lines[0]}" == '1..1' ] + [[ "${lines[1]}" == 'not ok 1 my sleep 10 in '*'ms # timeout after 1s' ]] || false + [ "${lines[2]}" == "# (in test file $RELATIVE_FIXTURE_ROOT/sleep2.bats, line 2)" ] + [ "${lines[3]}" == "# \`sleep \"\${SLEEP?}\"' failed due to timeout" ] + ((SECONDS < 10)) || false +} + +@test "sleep in run" { + run sleep 10 +} diff --git a/test/trace.bats b/test/trace.bats index 3454c730f2..b32a5e5933 100644 --- a/test/trace.bats +++ b/test/trace.bats @@ -1,12 +1,12 @@ #!/usr/bin/env bats bats_require_minimum_version 1.5.0 setup() { - load test_helper - fixtures trace + load test_helper + fixtures trace } @test "no --trace doesn't show anything on failure" { - run -1 bats "$FIXTURE_ROOT/failing_complex.bats" + reentrant_run -1 bats "$FIXTURE_ROOT/failing_complex.bats" [ "${lines[0]}" = "1..1" ] [ "${lines[1]}" = "not ok 1 a complex failing test" ] [ "${lines[2]}" = "# (in test file $RELATIVE_FIXTURE_ROOT/failing_complex.bats, line 4)" ] @@ -16,69 +16,69 @@ setup() { } @test "--trace recurses into functions but not into run" { - run -1 bats --trace "$FIXTURE_ROOT/failing_recursive.bats" - - [ "${lines[0]}" = '1..1' ] - [ "${lines[1]}" = 'not ok 1 a recursive failing test' ] - [ "${lines[2]}" = "# (in test file $RELATIVE_FIXTURE_ROOT/failing_recursive.bats, line 12)" ] - [ "${lines[3]}" = "# \`false' failed" ] - [ "${lines[4]}" = '# $ [failing_recursive.bats:9]' ] - [ "${lines[5]}" = '# $ echo Outer' ] - [ "${lines[6]}" = '# Outer' ] - [ "${lines[7]}" = '# $ fun 2' ] - [ "${lines[8]}" = '# $$ [failing_recursive.bats:2]' ] - # shellcheck disable=SC2016 - [ "${lines[9]}" = '# $$ echo "$1"' ] - [ "${lines[10]}" = '# 2' ] - # shellcheck disable=SC2016 - [ "${lines[11]}" = '# $$ [[ $1 -gt 0 ]]' ] - # shellcheck disable=SC2016 - [ "${lines[12]}" = '# $$ fun $(($1 - 1))' ] - [ "${lines[13]}" = '# $$$ [failing_recursive.bats:2]' ] - # shellcheck disable=SC2016 - [ "${lines[14]}" = '# $$$ echo "$1"' ] - [ "${lines[15]}" = '# 1' ] - # shellcheck disable=SC2016 - [ "${lines[16]}" = '# $$$ [[ $1 -gt 0 ]]' ] - # shellcheck disable=SC2016 - [ "${lines[17]}" = '# $$$ fun $(($1 - 1))' ] - [ "${lines[18]}" = '# $$$$ [failing_recursive.bats:2]' ] - # shellcheck disable=SC2016 - [ "${lines[19]}" = '# $$$$ echo "$1"' ] - [ "${lines[20]}" = '# 0' ] - # shellcheck disable=SC2016 - [ "${lines[21]}" = '# $$$$ [[ $1 -gt 0 ]]' ] - [ "${lines[22]}" = '# $ [failing_recursive.bats:11]' ] - [ "${lines[23]}" = '# $ run fun 2' ] - [ "${lines[24]}" = '# $ false' ] + reentrant_run -1 bats --trace "$FIXTURE_ROOT/failing_recursive.bats" - # the trace on return from a function differs between bash versions - check_bash_5() { - [ ${#lines[@]} -eq 25 ] - } + [ "${lines[0]}" = '1..1' ] + [ "${lines[1]}" = 'not ok 1 a recursive failing test' ] + [ "${lines[2]}" = "# (in test file $RELATIVE_FIXTURE_ROOT/failing_recursive.bats, line 12)" ] + [ "${lines[3]}" = "# \`false' failed" ] + [ "${lines[4]}" = '# $ [failing_recursive.bats:9]' ] + [ "${lines[5]}" = '# $ echo Outer' ] + [ "${lines[6]}" = '# Outer' ] + [ "${lines[7]}" = '# $ fun 2' ] + [ "${lines[8]}" = '# $$ [failing_recursive.bats:2]' ] + # shellcheck disable=SC2016 + [ "${lines[9]}" = '# $$ echo "$1"' ] + [ "${lines[10]}" = '# 2' ] + # shellcheck disable=SC2016 + [ "${lines[11]}" = '# $$ [[ $1 -gt 0 ]]' ] + # shellcheck disable=SC2016 + [ "${lines[12]}" = '# $$ fun $(($1 - 1))' ] + [ "${lines[13]}" = '# $$$ [failing_recursive.bats:2]' ] + # shellcheck disable=SC2016 + [ "${lines[14]}" = '# $$$ echo "$1"' ] + [ "${lines[15]}" = '# 1' ] + # shellcheck disable=SC2016 + [ "${lines[16]}" = '# $$$ [[ $1 -gt 0 ]]' ] + # shellcheck disable=SC2016 + [ "${lines[17]}" = '# $$$ fun $(($1 - 1))' ] + [ "${lines[18]}" = '# $$$$ [failing_recursive.bats:2]' ] + # shellcheck disable=SC2016 + [ "${lines[19]}" = '# $$$$ echo "$1"' ] + [ "${lines[20]}" = '# 0' ] + # shellcheck disable=SC2016 + [ "${lines[21]}" = '# $$$$ [[ $1 -gt 0 ]]' ] + [ "${lines[22]}" = '# $ [failing_recursive.bats:11]' ] + [ "${lines[23]}" = '# $ run fun 2' ] + [ "${lines[24]}" = '# $ false' ] - # "alias" same behavior to have single point of truth - check_bash_4_4() { check_bash_5; } - check_bash_4_3() { check_bash_5; } - check_bash_4_2() { check_bash_4_0; } - check_bash_4_1() { check_bash_4_0; } + # the trace on return from a function differs between bash versions + check_bash_5() { + [ ${#lines[@]} -eq 25 ] + } - check_bash_4_0() { - # bash bug: the lineno from the debug_trap spills over -> ignore it - [ "${lines[25]}" = '# $ false' ] - [ ${#lines[@]} -eq 26 ] - } + # "alias" same behavior to have single point of truth + check_bash_4_4() { check_bash_5; } + check_bash_4_3() { check_bash_5; } + check_bash_4_2() { check_bash_4_0; } + check_bash_4_1() { check_bash_4_0; } - check_bash_3_2() { - # lineno from function definition - [ "${lines[25]}" = '# $ false' ] - [ ${#lines[@]} -eq 26 ] - } + check_bash_4_0() { + # bash bug: the lineno from the debug_trap spills over -> ignore it + [ "${lines[25]}" = '# $ false' ] + [ ${#lines[@]} -eq 26 ] + } - IFS=. read -r -a bash_version <<< "${BASH_VERSION}" - check_func="check_bash_${bash_version[0]}" - if [[ $(type -t "$check_func") != function ]]; then - check_func="check_bash_${bash_version[0]}_${bash_version[1]}" - fi - $check_func -} \ No newline at end of file + check_bash_3_2() { + # lineno from function definition + [ "${lines[25]}" = '# $ false' ] + [ ${#lines[@]} -eq 26 ] + } + + IFS=. read -r -a bash_version <<<"${BASH_VERSION}" + check_func="check_bash_${bash_version[0]}" + if [[ $(type -t "$check_func") != function ]]; then + check_func="check_bash_${bash_version[0]}_${bash_version[1]}" + fi + $check_func +} diff --git a/test/warnings.bats b/test/warnings.bats index 6623c2a0fc..ef94e7618d 100644 --- a/test/warnings.bats +++ b/test/warnings.bats @@ -3,55 +3,79 @@ bats_require_minimum_version 1.5.0 setup() { - load test_helper - fixtures warnings - if [[ $BATS_ROOT == "$BATS_CWD" ]]; then - RELATIVE_BATS_ROOT='' - else - RELATIVE_BATS_ROOT=${BATS_ROOT#"$BATS_CWD"/} - fi - if [[ -n "$RELATIVE_BATS_ROOT" && "$RELATIVE_BATS_ROOT" != */ ]]; then - RELATIVE_BATS_ROOT+=/ - fi - echo "RELATIVE_BATS_ROOT=$RELATIVE_BATS_ROOT" "BATS_ROOT=$BATS_ROOT" "BATS_CWD=$BATS_CWD" + load test_helper + fixtures warnings + if [[ $BATS_ROOT == "$BATS_CWD" ]]; then + RELATIVE_BATS_ROOT='' + else + RELATIVE_BATS_ROOT=${BATS_ROOT#"$BATS_CWD"/} + fi + if [[ -n "$RELATIVE_BATS_ROOT" && "$RELATIVE_BATS_ROOT" != */ ]]; then + RELATIVE_BATS_ROOT+=/ + fi + echo "RELATIVE_BATS_ROOT=$RELATIVE_BATS_ROOT" "BATS_ROOT=$BATS_ROOT" "BATS_CWD=$BATS_CWD" } @test "invalid warning is an error" { - run -1 bats_generate_warning invalid-number - [[ "$output" == "Invalid Bats warning number 'invalid-number'. It must be an integer between 1 and "* ]] + REENTRANT_RUN_PRESERVE+=(BATS_WARNING_SHORT_DESCS) + reentrant_run -1 bats_generate_warning invalid-number + [[ "$output" == "Invalid Bats warning number 'invalid-number'. It must be an integer between 1 and "* ]] } @test "BW01 is printed when \`run\`ing a (non-existant) command with exit code 127 without exit code check" { - run -0 bats "$FIXTURE_ROOT/BW01.bats" - [ "${lines[0]}" == "1..1" ] - [ "${lines[1]}" == "ok 1 Trigger BW01" ] - [ "${lines[2]}" == "The following warnings were encountered during tests:" ] - [ "${lines[3]}" == "BW01: \`run\`'s command \`=0 actually-intended-command with some args\` exited with code 127, indicating 'Command not found'. Use run's return code checks, e.g. \`run -127\`, to fix this message." ] - [[ "${lines[4]}" == " (from function \`run' in file ${RELATIVE_BATS_ROOT}lib/bats-core/test_functions.bash, line"* ]] - [ "${lines[5]}" == " in test file $RELATIVE_FIXTURE_ROOT/BW01.bats, line 3)" ] + reentrant_run -0 bats "$FIXTURE_ROOT/BW01.bats" + [ "${lines[0]}" == "1..1" ] + [ "${lines[1]}" == "ok 1 Trigger BW01" ] + [ "${lines[2]}" == "The following warnings were encountered during tests:" ] + [ "${lines[3]}" == "BW01: \`run\`'s command \`=0 actually-intended-command with some args\` exited with code 127, indicating 'Command not found'. Use run's return code checks, e.g. \`run -127\`, to fix this message." ] + [[ "${lines[4]}" == " (from function \`run' in file ${RELATIVE_BATS_ROOT}lib/bats-core/test_functions.bash, line"* ]] + [ "${lines[5]}" == " in test file $RELATIVE_FIXTURE_ROOT/BW01.bats, line 3)" ] } @test "BW01 is not printed when \`run\`ing a (non-existant) command with exit code 127 with exit code check" { - run -0 bats "$FIXTURE_ROOT/BW01_check_exit_code_is_127.bats" - [ "${lines[0]}" == "1..1" ] - [ "${lines[1]}" == "ok 1 Don't trigger BW01 with checked exit code 127" ] - [ "${#lines[@]}" -eq 2 ] + reentrant_run -0 bats "$FIXTURE_ROOT/BW01_check_exit_code_is_127.bats" + [ "${lines[0]}" == "1..1" ] + [ "${lines[1]}" == "ok 1 Don't trigger BW01 with checked exit code 127" ] + [ "${#lines[@]}" -eq 2 ] } @test "BW01 is not printed when \`run\`ing a command with exit code !=127 without exit code check" { - run -0 bats "$FIXTURE_ROOT/BW01_no_exit_code_check_no_exit_code_127.bats" - [ "${lines[0]}" == "1..1" ] - [ "${lines[1]}" == "ok 1 Don't trigger BW01 with exit code !=127 and no check" ] - [ "${#lines[@]}" -eq 2 ] + reentrant_run -0 bats "$FIXTURE_ROOT/BW01_no_exit_code_check_no_exit_code_127.bats" + [ "${lines[0]}" == "1..1" ] + [ "${lines[1]}" == "ok 1 Don't trigger BW01 with exit code !=127 and no check" ] + [ "${#lines[@]}" -eq 2 ] } @test "BW02 is printed when run uses parameters without guaranteed version >= 1.5.0" { - run -0 bats "$FIXTURE_ROOT/BW02.bats" - [ "${lines[0]}" == "1..1" ] - [ "${lines[1]}" == "ok 1 Trigger BW02" ] - [ "${lines[2]}" == "The following warnings were encountered during tests:" ] - [ "${lines[3]}" == "BW02: Using flags on \`run\` requires at least BATS_VERSION=1.5.0. Use \`bats_require_minimum_version 1.5.0\` to fix this message." ] - [[ "${lines[4]}" == " (from function \`bats_warn_minimum_guaranteed_version' in file ${RELATIVE_BATS_ROOT}lib/bats-core/warnings.bash, line 31,"* ]] - [[ "${lines[5]}" == " from function \`run' in file ${RELATIVE_BATS_ROOT}lib/bats-core/test_functions.bash, line"* ]] - [ "${lines[6]}" == " in test file $RELATIVE_FIXTURE_ROOT/BW02.bats, line 2)" ] -} \ No newline at end of file + reentrant_run -0 bats "$FIXTURE_ROOT/BW02.bats" + [ "${lines[0]}" == "1..1" ] + [ "${lines[1]}" == "ok 1 Trigger BW02" ] + [ "${lines[2]}" == "The following warnings were encountered during tests:" ] + [ "${lines[3]}" == "BW02: Using flags on \`run\` requires at least BATS_VERSION=1.5.0. Use \`bats_require_minimum_version 1.5.0\` to fix this message." ] + [[ "${lines[4]}" == " (from function \`bats_warn_minimum_guaranteed_version' in file ${RELATIVE_BATS_ROOT}lib/bats-core/warnings.bash, line "* ]] + [[ "${lines[5]}" == " from function \`run' in file ${RELATIVE_BATS_ROOT}lib/bats-core/test_functions.bash, line"* ]] + [ "${lines[6]}" == " in test file $RELATIVE_FIXTURE_ROOT/BW02.bats, line 2)" ] +} + +@test "BW03 is printed when a test file defines setup_suite and setup_suite is not defined" { + reentrant_run -0 bats "$FIXTURE_ROOT/BW03/define_setup_suite_in_wrong_file.bats" + [ "${lines[0]}" == "1..1" ] + [ "${lines[1]}" == "ok 1 test" ] + [ "${lines[2]}" == "The following warnings were encountered during tests:" ] + [ "${lines[3]}" == "BW03: \`setup_suite\` is visible to test file '${FIXTURE_ROOT}/BW03/define_setup_suite_in_wrong_file.bats', but was not executed. It belongs into 'setup_suite.bash' to be picked up automatically." ] + [ "${#lines[@]}" -eq 4 ] +} + +@test "BW03 is not printed when a test file defines setup_suite but setup_suite was completed" { + reentrant_run -0 bats "$FIXTURE_ROOT/BW03/define_setup_suite_in_wrong_file.bats" --setup-suite-file "$FIXTURE_ROOT/BW03/non_default_setup_suite.bash" + [ "${lines[0]}" == "1..1" ] + [ "${lines[1]}" == "ok 1 test" ] + [ "${#lines[@]}" -eq 2 ] +} + +@test "BW03 can be suppressed by setting BATS_SETUP_SUITE_COMPLETED" { + reentrant_run -0 bats "$FIXTURE_ROOT/BW03/suppress_warning.bats" + [ "${lines[0]}" == "1..1" ] + [ "${lines[1]}" == "ok 1 test" ] + [ "${#lines[@]}" -eq 2 ] +}