diff --git a/Tests/LibJS/test-js.cpp b/Tests/LibJS/test-js.cpp index c9936dd910d..ce9301576bf 100644 --- a/Tests/LibJS/test-js.cpp +++ b/Tests/LibJS/test-js.cpp @@ -132,12 +132,13 @@ TESTJS_RUN_FILE_FUNCTION(String const& test_file, JS::Interpreter& interpreter, } auto test_result = test_passed ? Test::Result::Pass : Test::Result::Fail; - + auto test_path = LexicalPath::relative_path(test_file, Test::JS::g_test_root); + auto duration_ms = Test::get_time_in_ms() - start_time; return Test::JS::JSFileResult { - LexicalPath::relative_path(test_file, Test::JS::g_test_root), + test_path, {}, - Test::get_time_in_ms() - start_time, + duration_ms, test_result, - { Test::Suite { "Parse file", test_result, { { expectation_string, test_result, message } } } } + { Test::Suite { test_path, "Parse file", test_result, { { expectation_string, test_result, message, static_cast(duration_ms) * 1000u } } } } }; } diff --git a/Userland/Libraries/LibJS/Tests/test-common.js b/Userland/Libraries/LibJS/Tests/test-common.js index a43477902e3..bbdc80fc9bd 100644 --- a/Userland/Libraries/LibJS/Tests/test-common.js +++ b/Userland/Libraries/LibJS/Tests/test-common.js @@ -536,6 +536,7 @@ class ExpectationError extends Error { __TestResults__[suiteMessage][defaultSuiteMessage] = { result: "fail", details: String(e), + duration: 0, }; } suiteMessage = defaultSuiteMessage; @@ -549,19 +550,25 @@ class ExpectationError extends Error { suite[message] = { result: "fail", details: "Another test with the same message did already run", + duration: 0, }; return; } + const now = () => Temporal.Now.instant().epochNanoseconds; + const start = now(); + const time_us = () => Number(BigInt.asIntN(53, (now() - start) / 1000n)); try { callback(); suite[message] = { result: "pass", + duration: time_us(), }; } catch (e) { suite[message] = { result: "fail", details: String(e), + duration: time_us(), }; } }; @@ -577,12 +584,14 @@ class ExpectationError extends Error { suite[message] = { result: "fail", details: "Another test with the same message did already run", + duration: 0, }; return; } suite[message] = { result: "skip", + duration: 0, }; }; })(); diff --git a/Userland/Libraries/LibTest/JavaScriptTestRunner.h b/Userland/Libraries/LibTest/JavaScriptTestRunner.h index b9d34b5d142..c10bc05049d 100644 --- a/Userland/Libraries/LibTest/JavaScriptTestRunner.h +++ b/Userland/Libraries/LibTest/JavaScriptTestRunner.h @@ -168,8 +168,8 @@ extern IntermediateRunFileResult (*g_run_file)(const String&, JS::Interpreter&, class TestRunner : public ::Test::TestRunner { public: - TestRunner(String test_root, String common_path, bool print_times, bool print_progress, bool print_json) - : ::Test::TestRunner(move(test_root), print_times, print_progress, print_json) + TestRunner(String test_root, String common_path, bool print_times, bool print_progress, bool print_json, bool detailed_json) + : ::Test::TestRunner(move(test_root), print_times, print_progress, print_json, detailed_json) , m_common_path(move(common_path)) { g_test_root = m_test_root; @@ -266,6 +266,9 @@ inline void TestRunner::do_run_single_test(const String& test_path, size_t, size auto file_result = run_file_test(test_path); if (!m_print_json) print_file_result(file_result); + + if (needs_detailed_suites()) + ensure_suites().extend(file_result.suites); } inline Vector TestRunner::get_test_paths() const @@ -402,12 +405,12 @@ inline JSFileResult TestRunner::run_file_test(const String& test_path) } test_json.value().as_object().for_each_member([&](const String& suite_name, const JsonValue& suite_value) { - Test::Suite suite { suite_name }; + Test::Suite suite { test_path, suite_name }; VERIFY(suite_value.is_object()); suite_value.as_object().for_each_member([&](const String& test_name, const JsonValue& test_value) { - Test::Case test { test_name, Test::Result::Fail, "" }; + Test::Case test { test_name, Test::Result::Fail, "", 0 }; VERIFY(test_value.is_object()); VERIFY(test_value.as_object().has("result")); @@ -433,6 +436,8 @@ inline JSFileResult TestRunner::run_file_test(const String& test_path) m_counts.tests_skipped++; } + test.duration_us = test_value.as_object().get("duration").to_u64(0); + suite.tests.append(test); }); diff --git a/Userland/Libraries/LibTest/JavaScriptTestRunnerMain.cpp b/Userland/Libraries/LibTest/JavaScriptTestRunnerMain.cpp index ebff8e9e5a8..7dc4027aaa6 100644 --- a/Userland/Libraries/LibTest/JavaScriptTestRunnerMain.cpp +++ b/Userland/Libraries/LibTest/JavaScriptTestRunnerMain.cpp @@ -87,6 +87,7 @@ int main(int argc, char** argv) false; #endif bool print_json = false; + bool per_file = false; const char* specified_test_root = nullptr; String common_path; String test_glob; @@ -109,6 +110,7 @@ int main(int argc, char** argv) }, }); args_parser.add_option(print_json, "Show results as JSON", "json", 'j'); + args_parser.add_option(per_file, "Show detailed per-file results as JSON (implies -j)", "per-file", 0); args_parser.add_option(g_collect_on_every_allocation, "Collect garbage after every allocation", "collect-often", 'g'); args_parser.add_option(g_run_bytecode, "Use the bytecode interpreter", "run-bytecode", 'b'); args_parser.add_option(JS::Bytecode::g_dump_bytecode, "Dump the bytecode", "dump-bytecode", 'd'); @@ -119,6 +121,9 @@ int main(int argc, char** argv) args_parser.add_positional_argument(common_path, "Path to tests-common.js", "common-path", Core::ArgsParser::Required::No); args_parser.parse(argc, argv); + if (per_file) + print_json = true; + test_glob = String::formatted("*{}*", test_glob); if (getenv("DISABLE_DBG_OUTPUT")) { @@ -182,7 +187,7 @@ int main(int argc, char** argv) g_vm->enable_default_host_import_module_dynamically_hook(); } - Test::JS::TestRunner test_runner(test_root, common_path, print_times, print_progress, print_json); + Test::JS::TestRunner test_runner(test_root, common_path, print_times, print_progress, print_json, per_file); test_runner.run(test_glob); g_vm = nullptr; diff --git a/Userland/Libraries/LibTest/Results.h b/Userland/Libraries/LibTest/Results.h index 69f8622dd45..c21a9920e5b 100644 --- a/Userland/Libraries/LibTest/Results.h +++ b/Userland/Libraries/LibTest/Results.h @@ -24,9 +24,11 @@ struct Case { String name; Result result; String details; + u64 duration_us; }; struct Suite { + String path; String name; // A failed test takes precedence over a skipped test, which both have // precedence over a passed test diff --git a/Userland/Libraries/LibTest/TestRunner.h b/Userland/Libraries/LibTest/TestRunner.h index a7d461c54db..4e8bad2a400 100644 --- a/Userland/Libraries/LibTest/TestRunner.h +++ b/Userland/Libraries/LibTest/TestRunner.h @@ -29,11 +29,12 @@ public: return s_the; } - TestRunner(String test_root, bool print_times, bool print_progress, bool print_json) + TestRunner(String test_root, bool print_times, bool print_progress, bool print_json, bool detailed_json = false) : m_test_root(move(test_root)) , m_print_times(print_times) , m_print_progress(print_progress) , m_print_json(print_json) + , m_detailed_json(detailed_json) { VERIFY(!s_the); s_the = this; @@ -47,6 +48,16 @@ public: bool is_printing_progress() const { return m_print_progress; } + bool needs_detailed_suites() const { return m_detailed_json; } + Vector const& suites() const { return *m_suites; } + + Vector& ensure_suites() + { + if (!m_suites.has_value()) + m_suites = Vector {}; + return *m_suites; + } + protected: static TestRunner* s_the; @@ -61,9 +72,11 @@ protected: bool m_print_times; bool m_print_progress; bool m_print_json; + bool m_detailed_json; double m_total_elapsed_time_in_ms { 0 }; Test::Counts m_counts; + Optional> m_suites; }; inline void cleanup() @@ -223,26 +236,63 @@ inline void TestRunner::print_test_results() const inline void TestRunner::print_test_results_as_json() const { - JsonObject suites; - suites.set("failed", m_counts.suites_failed); - suites.set("passed", m_counts.suites_passed); - suites.set("total", m_counts.suites_failed + m_counts.suites_passed); - - JsonObject tests; - tests.set("failed", m_counts.tests_failed); - tests.set("passed", m_counts.tests_passed); - tests.set("skipped", m_counts.tests_skipped); - tests.set("total", m_counts.tests_failed + m_counts.tests_passed + m_counts.tests_skipped); - - JsonObject results; - results.set("suites", suites); - results.set("tests", tests); - JsonObject root; - root.set("results", results); - root.set("files_total", m_counts.files_total); - root.set("duration", m_total_elapsed_time_in_ms / 1000.0); + if (needs_detailed_suites()) { + auto& suites = this->suites(); + u64 duration_us = 0; + JsonObject tests; + for (auto& suite : suites) { + for (auto& case_ : suite.tests) { + duration_us += case_.duration_us; + StringView result_name; + switch (case_.result) { + case Result::Pass: + result_name = "PASSED"; + break; + case Result::Fail: + result_name = "FAILED"; + break; + case Result::Skip: + result_name = "SKIPPED"; + break; + case Result::Crashed: + result_name = "PROCESS_ERROR"; + break; + } + + auto name = suite.name; + if (name == "__$$TOP_LEVEL$$__"sv) + name = String::empty(); + + auto path = LexicalPath::relative_path(suite.path, m_test_root); + + tests.set(String::formatted("{}/{}::{}", path, name, case_.name), result_name); + } + } + + root.set("duration", static_cast(duration_us) / 1000000.); + root.set("results", move(tests)); + } else { + JsonObject suites; + suites.set("failed", m_counts.suites_failed); + suites.set("passed", m_counts.suites_passed); + suites.set("total", m_counts.suites_failed + m_counts.suites_passed); + + JsonObject tests; + tests.set("failed", m_counts.tests_failed); + tests.set("passed", m_counts.tests_passed); + tests.set("skipped", m_counts.tests_skipped); + tests.set("total", m_counts.tests_failed + m_counts.tests_passed + m_counts.tests_skipped); + + JsonObject results; + results.set("suites", suites); + results.set("tests", tests); + + root.set("results", results); + root.set("files_total", m_counts.files_total); + root.set("duration", m_total_elapsed_time_in_ms / 1000.0); + } outln("{}", root.to_string()); }