LibTest: Provide detailed per-file JSON output with --per-file

This makes test-js style runners dump out output in the same format as
libjs-test262's per-file output.

Tests/LibJS/test-js.cpp

@@ -132,12 +132,13 @@ TESTJS_RUN_FILE_FUNCTION(String const& test_file, JS::Interpreter& interpreter,
     }
 
     auto test_result = test_passed ? Test::Result::Pass : Test::Result::Fail;
-
+    auto test_path = LexicalPath::relative_path(test_file, Test::JS::g_test_root);
+    auto duration_ms = Test::get_time_in_ms() - start_time;
     return Test::JS::JSFileResult {
-        LexicalPath::relative_path(test_file, Test::JS::g_test_root),
+        test_path,
         {},
-        Test::get_time_in_ms() - start_time,
+        duration_ms,
         test_result,
-        { Test::Suite { "Parse file", test_result, { { expectation_string, test_result, message } } } }
+        { Test::Suite { test_path, "Parse file", test_result, { { expectation_string, test_result, message, static_cast<u64>(duration_ms) * 1000u } } } }
     };
 }

Userland/Libraries/LibJS/Tests/test-common.js

@@ -536,6 +536,7 @@ class ExpectationError extends Error {
             __TestResults__[suiteMessage][defaultSuiteMessage] = {
                 result: "fail",
                 details: String(e),
+                duration: 0,
             };
         }
         suiteMessage = defaultSuiteMessage;
@@ -549,19 +550,25 @@ class ExpectationError extends Error {
             suite[message] = {
                 result: "fail",
                 details: "Another test with the same message did already run",
+                duration: 0,
             };
             return;
         }
 
+        const now = () => Temporal.Now.instant().epochNanoseconds;
+        const start = now();
+        const time_us = () => Number(BigInt.asIntN(53, (now() - start) / 1000n));
         try {
             callback();
             suite[message] = {
                 result: "pass",
+                duration: time_us(),
             };
         } catch (e) {
             suite[message] = {
                 result: "fail",
                 details: String(e),
+                duration: time_us(),
             };
         }
     };
@@ -577,12 +584,14 @@ class ExpectationError extends Error {
             suite[message] = {
                 result: "fail",
                 details: "Another test with the same message did already run",
+                duration: 0,
             };
             return;
         }
 
         suite[message] = {
             result: "skip",
+            duration: 0,
         };
     };
 })();

Userland/Libraries/LibTest/JavaScriptTestRunner.h

@@ -168,8 +168,8 @@ extern IntermediateRunFileResult (*g_run_file)(const String&, JS::Interpreter&,
 
 class TestRunner : public ::Test::TestRunner {
 public:
-    TestRunner(String test_root, String common_path, bool print_times, bool print_progress, bool print_json)
+    TestRunner(String test_root, String common_path, bool print_times, bool print_progress, bool print_json, bool detailed_json)
-        : ::Test::TestRunner(move(test_root), print_times, print_progress, print_json)
+        : ::Test::TestRunner(move(test_root), print_times, print_progress, print_json, detailed_json)
         , m_common_path(move(common_path))
     {
         g_test_root = m_test_root;
@@ -266,6 +266,9 @@ inline void TestRunner::do_run_single_test(const String& test_path, size_t, size
     auto file_result = run_file_test(test_path);
     if (!m_print_json)
         print_file_result(file_result);
+
+    if (needs_detailed_suites())
+        ensure_suites().extend(file_result.suites);
 }
 
 inline Vector<String> TestRunner::get_test_paths() const
@@ -402,12 +405,12 @@ inline JSFileResult TestRunner::run_file_test(const String& test_path)
     }
 
     test_json.value().as_object().for_each_member([&](const String& suite_name, const JsonValue& suite_value) {
-        Test::Suite suite { suite_name };
+        Test::Suite suite { test_path, suite_name };
 
         VERIFY(suite_value.is_object());
 
         suite_value.as_object().for_each_member([&](const String& test_name, const JsonValue& test_value) {
-            Test::Case test { test_name, Test::Result::Fail, "" };
+            Test::Case test { test_name, Test::Result::Fail, "", 0 };
 
             VERIFY(test_value.is_object());
             VERIFY(test_value.as_object().has("result"));
@@ -433,6 +436,8 @@ inline JSFileResult TestRunner::run_file_test(const String& test_path)
                 m_counts.tests_skipped++;
             }
 
+            test.duration_us = test_value.as_object().get("duration").to_u64(0);
+
             suite.tests.append(test);
         });
 

Userland/Libraries/LibTest/JavaScriptTestRunnerMain.cpp

@@ -87,6 +87,7 @@ int main(int argc, char** argv)
         false;
 #endif
     bool print_json = false;
+    bool per_file = false;
     const char* specified_test_root = nullptr;
     String common_path;
     String test_glob;
@@ -109,6 +110,7 @@ int main(int argc, char** argv)
         },
     });
     args_parser.add_option(print_json, "Show results as JSON", "json", 'j');
+    args_parser.add_option(per_file, "Show detailed per-file results as JSON (implies -j)", "per-file", 0);
     args_parser.add_option(g_collect_on_every_allocation, "Collect garbage after every allocation", "collect-often", 'g');
     args_parser.add_option(g_run_bytecode, "Use the bytecode interpreter", "run-bytecode", 'b');
     args_parser.add_option(JS::Bytecode::g_dump_bytecode, "Dump the bytecode", "dump-bytecode", 'd');
@@ -119,6 +121,9 @@ int main(int argc, char** argv)
     args_parser.add_positional_argument(common_path, "Path to tests-common.js", "common-path", Core::ArgsParser::Required::No);
     args_parser.parse(argc, argv);
 
+    if (per_file)
+        print_json = true;
+
     test_glob = String::formatted("*{}*", test_glob);
 
     if (getenv("DISABLE_DBG_OUTPUT")) {
@@ -182,7 +187,7 @@ int main(int argc, char** argv)
         g_vm->enable_default_host_import_module_dynamically_hook();
     }
 
-    Test::JS::TestRunner test_runner(test_root, common_path, print_times, print_progress, print_json);
+    Test::JS::TestRunner test_runner(test_root, common_path, print_times, print_progress, print_json, per_file);
     test_runner.run(test_glob);
 
     g_vm = nullptr;

Userland/Libraries/LibTest/Results.h

@@ -24,9 +24,11 @@ struct Case {
     String name;
     Result result;
     String details;
+    u64 duration_us;
 };
 
 struct Suite {
+    String path;
     String name;
     // A failed test takes precedence over a skipped test, which both have
     // precedence over a passed test

Userland/Libraries/LibTest/TestRunner.h

@@ -29,11 +29,12 @@ public:
         return s_the;
     }
 
-    TestRunner(String test_root, bool print_times, bool print_progress, bool print_json)
+    TestRunner(String test_root, bool print_times, bool print_progress, bool print_json, bool detailed_json = false)
         : m_test_root(move(test_root))
         , m_print_times(print_times)
         , m_print_progress(print_progress)
         , m_print_json(print_json)
+        , m_detailed_json(detailed_json)
     {
         VERIFY(!s_the);
         s_the = this;
@@ -47,6 +48,16 @@ public:
 
     bool is_printing_progress() const { return m_print_progress; }
 
+    bool needs_detailed_suites() const { return m_detailed_json; }
+    Vector<Test::Suite> const& suites() const { return *m_suites; }
+
+    Vector<Test::Suite>& ensure_suites()
+    {
+        if (!m_suites.has_value())
+            m_suites = Vector<Suite> {};
+        return *m_suites;
+    }
+
 protected:
     static TestRunner* s_the;
 
@@ -61,9 +72,11 @@ protected:
     bool m_print_times;
     bool m_print_progress;
     bool m_print_json;
+    bool m_detailed_json;
 
     double m_total_elapsed_time_in_ms { 0 };
     Test::Counts m_counts;
+    Optional<Vector<Test::Suite>> m_suites;
 };
 
 inline void cleanup()
@@ -223,26 +236,63 @@ inline void TestRunner::print_test_results() const
 
 inline void TestRunner::print_test_results_as_json() const
 {
-    JsonObject suites;
-    suites.set("failed", m_counts.suites_failed);
-    suites.set("passed", m_counts.suites_passed);
-    suites.set("total", m_counts.suites_failed + m_counts.suites_passed);
-
-    JsonObject tests;
-    tests.set("failed", m_counts.tests_failed);
-    tests.set("passed", m_counts.tests_passed);
-    tests.set("skipped", m_counts.tests_skipped);
-    tests.set("total", m_counts.tests_failed + m_counts.tests_passed + m_counts.tests_skipped);
-
-    JsonObject results;
-    results.set("suites", suites);
-    results.set("tests", tests);
-
     JsonObject root;
-    root.set("results", results);
+    if (needs_detailed_suites()) {
-    root.set("files_total", m_counts.files_total);
+        auto& suites = this->suites();
-    root.set("duration", m_total_elapsed_time_in_ms / 1000.0);
+        u64 duration_us = 0;
+        JsonObject tests;
 
+        for (auto& suite : suites) {
+            for (auto& case_ : suite.tests) {
+                duration_us += case_.duration_us;
+                StringView result_name;
+                switch (case_.result) {
+                case Result::Pass:
+                    result_name = "PASSED";
+                    break;
+                case Result::Fail:
+                    result_name = "FAILED";
+                    break;
+                case Result::Skip:
+                    result_name = "SKIPPED";
+                    break;
+                case Result::Crashed:
+                    result_name = "PROCESS_ERROR";
+                    break;
+                }
+
+                auto name = suite.name;
+                if (name == "__$$TOP_LEVEL$$__"sv)
+                    name = String::empty();
+
+                auto path = LexicalPath::relative_path(suite.path, m_test_root);
+
+                tests.set(String::formatted("{}/{}::{}", path, name, case_.name), result_name);
+            }
+        }
+
+        root.set("duration", static_cast<double>(duration_us) / 1000000.);
+        root.set("results", move(tests));
+    } else {
+        JsonObject suites;
+        suites.set("failed", m_counts.suites_failed);
+        suites.set("passed", m_counts.suites_passed);
+        suites.set("total", m_counts.suites_failed + m_counts.suites_passed);
+
+        JsonObject tests;
+        tests.set("failed", m_counts.tests_failed);
+        tests.set("passed", m_counts.tests_passed);
+        tests.set("skipped", m_counts.tests_skipped);
+        tests.set("total", m_counts.tests_failed + m_counts.tests_passed + m_counts.tests_skipped);
+
+        JsonObject results;
+        results.set("suites", suites);
+        results.set("tests", tests);
+
+        root.set("results", results);
+        root.set("files_total", m_counts.files_total);
+        root.set("duration", m_total_elapsed_time_in_ms / 1000.0);
+    }
     outln("{}", root.to_string());
 }