From edaa5061c46ff2d11f773b499c7b5e1117d6f794 Mon Sep 17 00:00:00 2001 From: Sam Atkins Date: Fri, 10 Nov 2023 16:50:39 +0000 Subject: [PATCH] headless-browser: Add flag to dump screenshots of failing ref-tests When the `--dump-failed-ref-tests` flag is provided, screenshots of the actual and reference pages will be placed in `Build/lagom/ladbybird/test-dumps`. This makes it a lot easier to spot what's wrong with a failing test. :^) --- Ladybird/CMakeLists.txt | 2 +- Userland/Utilities/headless-browser.cpp | 32 ++++++++++++++++++++----- 2 files changed, 27 insertions(+), 7 deletions(-) diff --git a/Ladybird/CMakeLists.txt b/Ladybird/CMakeLists.txt index d2d52aa811a..6275e718792 100644 --- a/Ladybird/CMakeLists.txt +++ b/Ladybird/CMakeLists.txt @@ -260,7 +260,7 @@ include(CTest) if (BUILD_TESTING) add_test( NAME LibWeb - COMMAND ${CMAKE_CURRENT_BINARY_DIR}/../bin/headless-browser --resources "${SERENITY_SOURCE_DIR}/Base/res" --run-tests ${SERENITY_SOURCE_DIR}/Tests/LibWeb + COMMAND ${CMAKE_CURRENT_BINARY_DIR}/../bin/headless-browser --resources "${SERENITY_SOURCE_DIR}/Base/res" --run-tests ${SERENITY_SOURCE_DIR}/Tests/LibWeb --dump-failed-ref-tests ) add_test( NAME WPT diff --git a/Userland/Utilities/headless-browser.cpp b/Userland/Utilities/headless-browser.cpp index 820879d1b7e..d041c055c1e 100644 --- a/Userland/Utilities/headless-browser.cpp +++ b/Userland/Utilities/headless-browser.cpp @@ -234,7 +234,7 @@ static ErrorOr run_dump_test(HeadlessWebContentView& view, StringVie return TestResult::Fail; } -static ErrorOr run_ref_test(HeadlessWebContentView& view, StringView input_path, int timeout_in_milliseconds = 15000) +static ErrorOr run_ref_test(HeadlessWebContentView& view, StringView input_path, bool dump_failed_ref_tests, int timeout_in_milliseconds = 15000) { Core::EventLoop loop; bool did_timeout = false; @@ -269,17 +269,35 @@ static ErrorOr run_ref_test(HeadlessWebContentView& view, StringView if (actual_screenshot->visually_equals(*expectation_screenshot)) return TestResult::Pass; + if (dump_failed_ref_tests) { + warnln("\033[33;1mRef test {} failed; dumping screenshots\033[0m", input_path); + auto title = LexicalPath::title(input_path); + auto dump_screenshot = [&](Gfx::Bitmap& bitmap, StringView path) -> ErrorOr { + auto screenshot_file = TRY(Core::File::open(path, Core::File::OpenMode::Write)); + auto encoded_data = TRY(Gfx::PNGWriter::encode(bitmap)); + TRY(screenshot_file->write_until_depleted(encoded_data)); + warnln("\033[33;1mDumped {}\033[0m", TRY(FileSystem::real_path(path))); + return {}; + }; + + auto mkdir_result = Core::System::mkdir("test-dumps"sv, 0755); + if (mkdir_result.is_error() && mkdir_result.error().code() != EEXIST) + return mkdir_result.release_error(); + TRY(dump_screenshot(*actual_screenshot, TRY(String::formatted("test-dumps/{}.png", title)))); + TRY(dump_screenshot(*expectation_screenshot, TRY(String::formatted("test-dumps/{}-ref.png", title)))); + } + return TestResult::Fail; } -static ErrorOr run_test(HeadlessWebContentView& view, StringView input_path, StringView expectation_path, TestMode mode) +static ErrorOr run_test(HeadlessWebContentView& view, StringView input_path, StringView expectation_path, TestMode mode, bool dump_failed_ref_tests) { switch (mode) { case TestMode::Text: case TestMode::Layout: return run_dump_test(view, input_path, expectation_path, mode); case TestMode::Ref: - return run_ref_test(view, input_path); + return run_ref_test(view, input_path, dump_failed_ref_tests); default: VERIFY_NOT_REACHED(); } @@ -325,7 +343,7 @@ static ErrorOr collect_ref_tests(Vector& tests, StringView path) return {}; } -static ErrorOr run_tests(HeadlessWebContentView& view, StringView test_root_path) +static ErrorOr run_tests(HeadlessWebContentView& view, StringView test_root_path, bool dump_failed_ref_tests) { view.clear_content_filters(); @@ -356,7 +374,7 @@ static ErrorOr run_tests(HeadlessWebContentView& view, StringView test_root else outln(""); - test.result = TRY(run_test(view, test.input_path, test.expectation_path, test.mode)); + test.result = TRY(run_test(view, test.input_path, test.expectation_path, test.mode, dump_failed_ref_tests)); switch (*test.result) { case TestResult::Pass: ++pass_count; @@ -395,6 +413,7 @@ ErrorOr serenity_main(Main::Arguments arguments) StringView raw_url; auto resources_folder = "/res"sv; StringView web_driver_ipc_path; + bool dump_failed_ref_tests = false; bool dump_layout_tree = false; bool dump_text = false; bool is_layout_test_mode = false; @@ -406,6 +425,7 @@ ErrorOr serenity_main(Main::Arguments arguments) args_parser.add_option(dump_layout_tree, "Dump layout tree and exit", "dump-layout-tree", 'd'); args_parser.add_option(dump_text, "Dump text and exit", "dump-text", 'T'); args_parser.add_option(test_root_path, "Run tests in path", "run-tests", 'R', "test-root-path"); + args_parser.add_option(dump_failed_ref_tests, "Dump screenshots of failing ref tests", "dump-failed-ref-tests", 'D'); args_parser.add_option(resources_folder, "Path of the base resources folder (defaults to /res)", "resources", 'r', "resources-root-path"); args_parser.add_option(web_driver_ipc_path, "Path to the WebDriver IPC socket", "webdriver-ipc-path", 0, "path"); args_parser.add_option(is_layout_test_mode, "Enable layout test mode", "layout-test-mode", 0); @@ -433,7 +453,7 @@ ErrorOr serenity_main(Main::Arguments arguments) RefPtr timer; if (!test_root_path.is_empty()) { - return run_tests(*view, test_root_path); + return run_tests(*view, test_root_path, dump_failed_ref_tests); } if (dump_layout_tree) {