From 9337a5d7d2b4c1bc3fbe222eb2cdf6a97f22d5df Mon Sep 17 00:00:00 2001 From: Samuel Dolt Date: Fri, 18 Aug 2023 10:31:44 +0200 Subject: [PATCH] docs(bats): add info on how to use bats --- documentation/.vscode/extensions.json | 7 + documentation/.vscode/settings.json | 12 + documentation/index.rst | 1 + documentation/testing/bats.rst | 354 ++++++++++++++++++++++++++ documentation/testing/index.rst | 15 ++ 5 files changed, 389 insertions(+) create mode 100644 documentation/.vscode/extensions.json create mode 100644 documentation/.vscode/settings.json create mode 100644 documentation/testing/bats.rst create mode 100644 documentation/testing/index.rst diff --git a/documentation/.vscode/extensions.json b/documentation/.vscode/extensions.json new file mode 100644 index 0000000..4c8cf79 --- /dev/null +++ b/documentation/.vscode/extensions.json @@ -0,0 +1,7 @@ +{ + "recommendations": [ + "ms-vscode.makefile-tools", + "lextudio.restructuredtext", + "trond-snekvik.simple-rst" + ] +} diff --git a/documentation/.vscode/settings.json b/documentation/.vscode/settings.json new file mode 100644 index 0000000..7ce9e0e --- /dev/null +++ b/documentation/.vscode/settings.json @@ -0,0 +1,12 @@ +{ + "files.watcherExclude": { + "**/_build/**": true, + }, + "python.formatting.provider": "black", + "editor.rulers": [ + 80, + 100, + 120 + ], + "esbonio.sphinx.confDir": "" +} diff --git a/documentation/index.rst b/documentation/index.rst index fa6029f..3bea34e 100644 --- a/documentation/index.rst +++ b/documentation/index.rst @@ -40,6 +40,7 @@ same structures. Installation Manual Reference Manual + Testing Manual Boot Concepts Best Practices diff --git a/documentation/testing/bats.rst b/documentation/testing/bats.rst new file mode 100644 index 0000000..66a6946 --- /dev/null +++ b/documentation/testing/bats.rst @@ -0,0 +1,354 @@ +.. index:: BATS + +************************************ +BATS - Bash Automated Testing System +************************************ + +The CoreOS distribution supports writing tests using shell syntax by providing the `bats` command. + +If you want to use `bats`, you will need the following CoreOS packages: + +- bats +- bats-file +- bats-assert + +Overview of BATS +================ + +A BATS test can be as simple as a single .bats file. For example: + +.. code-block:: bash + + #!/usr/bin/env bats + + bats_load_library bats-support + bats_load_library bats-assert + + @test "can output to stdout" { + run echo hello + assert_output 'hello' + } + +You can run it using the command `bats .bats` + +This will give you the following output: + +.. code-block:: bash + + sam@SAVE:~/Projects/tests$ bats .bats + .bats + ✓ can output to stdout + + 1 test, 0 failures + +The run command +================ + +In shell tests, you often need to run commands and capture their output, exit +status, and error messages. The run command provided by `bats` allows you to +execute commands within your test cases and collect this information for later +assertion and validation. + +The run command will make the following variables available: + +- `${status}`: exit code of the command run by `run` +- `${output}`: combined content of `stdout` and `stderr` +- `${lines[@]}`: array of lines of the output +- `${BATS_RUN_COMMAND}`: command run by the `run` command + +.. code-block:: bash + + @test "invoking foo with a nonexistent file prints an error" { + run foo nonexistent_filename + [ "$status" -eq 1 ] + [ "$output" = "foo: no such file 'nonexistent_filename'" ] + [ "$BATS_RUN_COMMAND" = "foo nonexistent_filename" ] + + } + +The `run` command accepts some parameters: + +- `-N`: Expect N as exit status and fail otherwise +- `-!`: Expect non-zero exit status and fail if the command succeeds. +- `--keep-empty-lines`: don't remove empty lines from `${lines}` +- `--separate-stderr`: Use separate variables for stderr `${stderr}` and `${stderr_lines[@]}` + +.. code-block:: bash + + @test "invoking foo without arguments prints usage" { + run -1 foo + [ "${lines[0]}" = "usage: foo " ] + } + +The bats-assert helper +====================== + +The `bats-assert` helper provides some functions to create more readable tests. +These assertions use the variables created by the `run` command and can be used +as follows: + +.. code-block:: bash + + @test 'assert_output()' { + run echo 'have' + assert_output 'want' + } + +The following functions are provided: + +- `assert` and `refute`: Assert that a given expression evaluates to true or false. +- `assert_equal`: Assert that two parameters are equal. +- `assert_not_equal`: Assert that two parameters are not equal. +- `assert_success` and `assert_failure`: Assert that the exit status is 0 or 1. +- `assert_output` and `refute_output`: Assert that the output does (or does not) contain the given content. +- `assert_line` and `refute_line`: Assert that a specific line of the output does (or does not) contain the given content. +- `assert_regex` and `refute_regex`: Assert that a parameter matches (or does not match) the given pattern. + +The bats-file helper +==================== + +The `bats-file` helper provides functions to help work with files in tests: + +**Test File Types:** + +- `assert_exists` and `assert_not_exists`: Check if a file or directory exists. +- `assert_file_exists` and `assert_file_not_exists`: Check if a file exists. +- `assert_dir_exists` and `assert_dir_not_exists`: Check if a directory exists. +- `assert_link_exists` and `assert_link_not_exists`: Check if a link exists. +- `assert_block_exists` and `assert_block_not_exists`: Check if a block special file exists. +- `assert_character_exists` and `assert_character_not_exists`: Check if a character special file exists. +- `assert_socket_exists` and `assert_socket_not_exists`: Check if a socket exists. +- `assert_fifo_exists` and `assert_fifo_not_exists`: Check if a fifo special file exists. + +**Test File Attributes:** + +- `assert_file_executable` and `assert_file_not_executable` +- `assert_file_owner` and `assert_file_not_owner` +- `assert_file_permission` and `assert_not_file_permission` +- `assert_file_size_equals` +- `assert_size_zero` and `assert_size_not_zero` +- `assert_file_group_id_set` and `assert_file_not_group_id_set` +- `assert_file_user_id_set` and `assert_file_not_user_id_set` +- `assert_sticky_bit` and `assert_no_sticky_bit` + +**Test File Content:** + +- `assert_file_empty` and `assert_file_not_empty` +- `assert_file_contains` and `assert_file_not_contains` +- `assert_symlink_to` and `assert_not_symlink_to` + +**Working with a temporary directory:** + +- `temp_make` and `temp_del` + +Pre- and Post-test case hooks +============================== + +In some cases, it's useful to have a function that runs before or after each test +case in a bats file. + +A function named `setup` will run before each test case, and a function +named `teardown` will run after each test case. + +This example creates a directory in the setup function but lacks a teardown +that removes the directory. The second time the setup function is run, the +setup will fail as the directory already exists: + +.. code-block:: bash + + #!/usr/bin/env bats + + bats_load_library bats-support + bats_load_library bats-assert + bats_load_library bats-file + + setup() { + mkdir tmp + echo 'a' >> ./tmp/test + } + + @test "test contains a single a I" { + assert_file_contains ./tmp/test '^a$' + } + + @test "test contains a single a II" { + assert_file_contains ./tmp/test '^a$' + } + +.. code-block:: bash + + sam@SAVE:~/Projects/tests$ bats test.bats + test.bats + ✓ test contains a single a I + ✗ test contains a single a II + (from function `setup' in test file test.bats, line 8) + `mkdir tmp' failed + mkdir: cannot create directory ‘tmp’: File exists + + 2 tests, 1 failure + +This can be easily fixed by adding a teardown function: + +.. code-block:: bash + + #!/usr/bin/env bats + + bats_load_library bats-support + bats_load_library bats-assert + bats_load_library bats-file + + setup() { + mkdir tmp + echo 'a' >> ./tmp/test + } + + teardown() { + rm -rf ./tmp + } + + + + @test "test contains a single a I" { + assert_file_contains ./tmp/test '^a$' + } + + @test "test contains a single a II" { + assert_file_contains ./tmp/test '^a$' + } + +.. code-block:: bash + + sam@SAVE:~/Projects/tests$ bats test.bats + test.bats + ✓ test contains a single a I + ✓ test contains a single a II + + 2 tests, 0 failures + +Pre- and Post-test file hooks +============================= + +To run some code before executing a test file or after executing it, the +functions `setup_file` and `teardown_file` can be used. + +The last example could be refactored to only create the tmp directory once: + +.. code-block:: bash + + #!/usr/bin/env bats + + bats_load_library bats-support + bats_load_library bats-assert + bats_load_library bats-file + + setup_file() { + export DIR="./tmp" + export FILE="${DIR}/test" + mkdir "${DIR}" + } + + teardown_file() { + rm -rf "${DIR}" + } + + setup() { + echo 'a' >> "${FILE}" + } + + teardown() { + rm "${FILE}" + } + + @test "test contains a single a I" { + assert_file_contains "${FILE}" '^a$' + } + + @test "test contains a single a II" { + assert_file_contains "${FILE}" '^a$' + } + +Multiple files +============== + +With `bats`, a file is a test suite. If you have multiple `bats` files in a +directory and you provide the directory in the `bats` command line, `bats` +will execute all the test suites. + +Example: `bats .` + +.. code-block:: bash + + sam@SAVE:~/Projects/tests$ bats . + ./first.bats + ✓ can run our script + ✗ second test + (in test file ./first.bats, line 27) + `false' failed + ./second.bats + ✓ multi file + ./test.bats + ✓ test contains a single a I + ✓ test contains a single a II + + 5 tests, 1 failure + +Pre- and Post-suite hooks +========================= + +If you want to execute the same function before each test suite or after +each test suite, create a file named `setup_suite.bash`. In this file, +create a function named `setup_suite()` and another named `teardown_suite()`. + +Exporting the test results +========================== + +Test results can be exported using the JUnit XML format. This can then be +used in other tools and merged with other JUnit XML formats to generate a final +test report. + +Example: + +.. code-block:: bash + + sam@SAVE:~/Projects/tests$ bats . -F junit + +This will produce the following XML content on stdout: + +.. code-block:: xml + + + + + + + (in test file ./first.bats, line 27) + `false' failed + + + + + + + + + + + + + + +Going further +============= + +`bats` scripts can be checked with shellcheck for common mistakes. + +The `bats-assert` add-on provides many helper functions to perform +assertions with a more readable syntax than the shell's built-in syntax. + +See https://github.com/bats-core/bats-assert + +The `bats-file` add-on provides helper functions to check for files. See +https://github.com/bats-core/bats-file/ + +You can find a list of projects using `bats` on this page: +https://github.com/bats-core/bats-core/wiki/Projects-Using-Bats \ No newline at end of file diff --git a/documentation/testing/index.rst b/documentation/testing/index.rst new file mode 100644 index 0000000..bfd8db5 --- /dev/null +++ b/documentation/testing/index.rst @@ -0,0 +1,15 @@ + +============================== +Belden CoreOS Testing Manual +============================== + +This manual is a work on progress on how to test and how to write test for +CoreOS or CoreOS based distribution. + +| + +.. toctree:: + :caption: Table of Contents + :numbered: + + bats