From 4f406b0d1dce0431708d79a0403eb13584f99c26 Mon Sep 17 00:00:00 2001 From: Andrew Kaster Date: Tue, 4 Jun 2024 16:27:48 -0600 Subject: [PATCH] Documentation: Update RunningTests to refer to CMakePresets --- CMakePresets.json | 29 ++++++++- Documentation/RunningTests.md | 107 +++++++++------------------------- 2 files changed, 56 insertions(+), 80 deletions(-) diff --git a/CMakePresets.json b/CMakePresets.json index a1ad820b89a..2470877a19e 100644 --- a/CMakePresets.json +++ b/CMakePresets.json @@ -38,6 +38,26 @@ } } ], + "buildPresets": [ + { + "name": "default", + "configurePreset": "default", + "displayName": "Build", + "description": "Build the project", + "targets": [ + "all" + ] + }, + { + "name": "Sanitizer", + "configurePreset": "Sanitizer", + "displayName": "Build with Sanitizers", + "description": "Build the project with Sanitizers", + "targets": [ + "all" + ] + } + ], "testPresets": [ { "name": "default", @@ -48,12 +68,19 @@ "execution": { "noTestsAction": "error", "stopOnFailure": true + }, + "environment": { + "LADYBIRD_SOURCE_DIR": "${sourceDir}" } }, { "name": "Sanitizer", "inherits": "default", - "configurePreset": "Sanitizer" + "configurePreset": "Sanitizer", + "environment": { + "ASAN_OPTIONS": "strict_string_checks=1:check_initialization_order=1:strict_init_order=1:detect_stack_use_after_return=1:allocator_may_return_null=1", + "UBSAN_OPTIONS": "print_stacktrace=1:print_summary=1:halt_on_error=1" + } } ] } diff --git a/Documentation/RunningTests.md b/Documentation/RunningTests.md index 7affe22f861..987c2dafec9 100644 --- a/Documentation/RunningTests.md +++ b/Documentation/RunningTests.md @@ -1,31 +1,27 @@ -# Running SerenityOS Tests +# Running Tests -There are two classes of tests built during a SerenityOS build: host tests and target tests. Host tests run on the build -machine, and use [Lagom](../Meta/Lagom/ReadMe.md) to build SerenityOS userspace libraries for the host platform. Target -tests run on the SerenityOS machine, either emulated or bare metal. +To reproduce a CI failure, see the section on [Running with Sanitizers](#running-with-sanitizers). -## Running Host Tests - -There are two ways to build host tests: from a full build, or from a Lagom-only build. The only difference is the CMake -command used to initialize the build directory. - -For a full build, pass `-DBUILD_LAGOM=ON` to the CMake command. +The simplest way to run tests locally is to use the `default` preset from ``CMakePresets.json``: ```sh -cmake -GNinja -S Meta/CMake/Superbuild -B Build/superbuild-x86_64 -DBUILD_LAGOM=ON +cmake --preset default +cmake --build --preset default +ctest --preset default ``` -For a Lagom-only build, pass the Lagom directory to CMake. The `BUILD_LAGOM` CMake option is still required. +If you want to avoid building and running LibWeb tests, you can use a Lagom-only build. +For a Lagom-only build, pass the Lagom source directory to CMake. The `BUILD_LAGOM` CMake option is required. ```sh cmake -GNinja -S Meta/Lagom -B Build/lagom -DBUILD_LAGOM=ON ``` -In both cases, the tests can be run via ninja after doing a build. Note that `test-js` requires the `LADYBIRD_SOURCE_DIR` environment variable to be set -to the root of the serenity source tree when running on a non-SerenityOS host. +The tests can be run via ninja after doing a build. Note that `test-js` requires the `LADYBIRD_SOURCE_DIR` environment variable to be set +to the root of the ladybird source tree. ```sh -# /path/to/serenity repository +# /path/to/ladybird repository export LADYBIRD_SOURCE_DIR=${PWD} cd Build/lagom ninja @@ -41,7 +37,7 @@ CTEST_OUTPUT_ON_FAILURE=1 ninja test ctest --output-on-failure ``` -### Running with Sanitizers +# Running with Sanitizers CI runs host tests with Address Sanitizer and Undefined Sanitizer instrumentation enabled. These tools catch many classes of common C++ errors, including memory leaks, out of bounds access to stack and heap allocations, and @@ -49,73 +45,26 @@ signed integer overflow. For more info on the sanitizers, check out the Address or the Undefined Sanitizer [documentation](https://clang.llvm.org/docs/UndefinedBehaviorSanitizer.html) from clang. Note that a sanitizer build will take significantly longer than a non-sanitizer build, and will mess with caches in tools such as `ccache`. -The sanitizers can be enabled with the `-DENABLE_FOO_SANITIZER` set of flags. For the Serenity target, only the Undefined Sanitizers is supported. +The sanitizers can be enabled with the `-DENABLE_FOO_SANITIZER` set of flags. + +The simplest way to enable sanitizers is to use the `Sanitizer` preset. ```sh +cmake --preset Sanitizer +cmake --build --preset Sanitizer +ctest --preset Sanitizer +``` + +Or from a Lagom build: + +To ensure that the test behaves the same way as CI, make sure to set the ASAN_OPTIONS and UBSAN_OPTIONS appropriately. +The Sanitizer test preset already sets these environment variables. + +```sh +export ASAN_OPTIONS='strict_string_checks=1:check_initialization_order=1:strict_init_order=1:detect_stack_use_after_return=1:allocator_may_return_null=1' +export UBSAN_OPTIONS='print_stacktrace=1:print_summary=1:halt_on_error=1' cmake -GNinja -S Meta/Lagom -B Build/lagom -DBUILD_LAGOM=ON -DENABLE_ADDRESS_SANITIZER=ON -DENABLE_UNDEFINED_SANITIZER=ON cd Build/lagom ninja CTEST_OUTPUT_ON_FAILURE=1 LADYBIRD_SOURCE_DIR=${PWD}/../.. ninja test ``` - -To ensure that Undefined Sanitizer errors fail the test, the `halt_on_error` flag should be set to 1 in the environment variable `UBSAN_OPTIONS`. - -```sh -UBSAN_OPTIONS=halt_on_error=1 CTEST_OUTPUT_ON_FAILURE=1 LADYBIRD_SOURCE_DIR=${PWD}/.. ninja test -``` - -## Running Target Tests - -Tests built for the SerenityOS target get installed either into `/usr/Tests` or `/bin`. `/usr/Tests` is preferred, but -some system tests are installed into `/bin` for historical reasons. - -The easiest way to run all of the known tests in the system is to use the `run-tests-and-shutdown.sh` script that gets -installed into `/home/anon/Tests`. When running in CI, the environment variable `$DO_SHUTDOWN_AFTER_TESTS` is set, which -will run `shutdown -n` after running all the tests. - -For completeness, a basic on-target test run will need the SerenityOS image built and run via QEMU. - -```sh -cmake -GNinja -S Meta/CMake/Superbuild -B Build/superbuild-x86_64 -cmake --build Build/superbuild-x86_64 -cd Build/x86_64 -ninja install && ninja qemu-image && ninja run -``` - -In the initial terminal, one can easily run the test runner script: - -``` -courage ~ $ ./Tests/run-tests-and-shutdown.sh -=== Running Tests on SerenityOS === -... -``` - -CI runs the tests in self-test mode, using the 'ci' run options and the TestRunner entry in /etc/SystemServer.ini to run -tests automatically on startup. - -The system server entry looks as below: - -```ini -[TestRunner@ttyS0] -Executable=/home/anon/Tests/run-tests-and-shutdown.sh -StdIO=/dev/ttyS0 -Environment=DO_SHUTDOWN_AFTER_TESTS=1 TERM=xterm PATH=/usr/local/bin:/usr/bin:/bin -User=anon -WorkingDirectory=/home/anon -SystemModes=self-test -``` - -`/dev/ttyS0` is used as stdio because that serial port is connected when qemu is run with `-display none` and -`-serial stdio`, and output to it will show up in the stdout of the qemu window. Separately, the CI run script redirects -the serial debug output to `./debug.log` so that both stdout of the tests and the dbgln from the kernel/tests can be -captured. - -To run with CI's TestRunner system server entry, SerenityOS needs booted in self-test mode. Running the following shell -lines will boot SerenityOS in self-test mode, run tests, and exit. Note that CI also sets `panic=shutdown` to terminate qemu; -the default value `halt` keeps qemu around, which allows you to inspect the state. - -```sh -export SERENITY_RUN=ci -export SERENITY_KERNEL_CMDLINE="graphics_subsystem_mode=off system_mode=self-test" -ninja run -```