beenull/ladybird
1
0
Fork 0
mirror of https://github.com/LadybirdBrowser/ladybird.git synced 2024-11-22 07:30:19 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 6

LibTest: Provide detailed per-file JSON output with --per-file

Browse source 
This makes test-js style runners dump out output in the same format as
libjs-test262's per-file output.
This commit is contained in:
Ali Mohammad Pur 2022-03-08 07:09:41 +03:30 committed by Linus Groh
parent 3063aedb0c
commit 8b50009e9b
Notes: sideshowbarker 2024-07-17 17:18:45 +09:00 
Author: https://github.com/alimpfard
Commit: https://github.com/SerenityOS/serenity/commit/8b50009e9b
Pull-request: https://github.com/SerenityOS/serenity/pull/12998
Reviewed-by: https://github.com/IdanHo
Reviewed-by: https://github.com/davidot ✅
6 changed files with 100 additions and 28 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
Tests/LibJS/test-js.cpp
View file

@ -132,12 +132,13 @@ TESTJS_RUN_FILE_FUNCTION(String const& test_file, JS::Interpreter& interpreter,
}
auto test_result = test_passed ? Test::Result::Pass : Test::Result::Fail;
auto test_path = LexicalPath::relative_path(test_file, Test::JS::g_test_root);
auto duration_ms = Test::get_time_in_ms() - start_time;
return Test::JS::JSFileResult {
LexicalPath::relative_path(test_file, Test::JS::g_test_root),
test_path,
{},
Test::get_time_in_ms() - start_time,
duration_ms,
test_result,
{ Test::Suite { "Parse file", test_result, { { expectation_string, test_result, message } } } }
{ Test::Suite { test_path, "Parse file", test_result, { { expectation_string, test_result, message, static_cast<u64>(duration_ms) * 1000u } } } }
};
}

9
Userland/Libraries/LibJS/Tests/test-common.js
View file

@ -536,6 +536,7 @@ class ExpectationError extends Error {
__TestResults__[suiteMessage][defaultSuiteMessage] = {
result: "fail",
details: String(e),
duration: 0,
};
}
suiteMessage = defaultSuiteMessage;
@ -549,19 +550,25 @@ class ExpectationError extends Error {
suite[message] = {
result: "fail",
details: "Another test with the same message did already run",
duration: 0,
};
return;
}
const now = () => Temporal.Now.instant().epochNanoseconds;
const start = now();
const time_us = () => Number(BigInt.asIntN(53, (now() - start) / 1000n));
try {
callback();
suite[message] = {
result: "pass",
duration: time_us(),
};
} catch (e) {
suite[message] = {
result: "fail",
details: String(e),
duration: time_us(),
};
}
};
@ -577,12 +584,14 @@ class ExpectationError extends Error {
suite[message] = {
result: "fail",
details: "Another test with the same message did already run",
duration: 0,
};
return;
}
suite[message] = {
result: "skip",
duration: 0,
};
};
})();

13
Userland/Libraries/LibTest/JavaScriptTestRunner.h
View file

@ -168,8 +168,8 @@ extern IntermediateRunFileResult (*g_run_file)(const String&, JS::Interpreter&,
class TestRunner : public ::Test::TestRunner {
public:
TestRunner(String test_root, String common_path, bool print_times, bool print_progress, bool print_json)
: ::Test::TestRunner(move(test_root), print_times, print_progress, print_json)
TestRunner(String test_root, String common_path, bool print_times, bool print_progress, bool print_json, bool detailed_json)
: ::Test::TestRunner(move(test_root), print_times, print_progress, print_json, detailed_json)
, m_common_path(move(common_path))
{
g_test_root = m_test_root;
@ -266,6 +266,9 @@ inline void TestRunner::do_run_single_test(const String& test_path, size_t, size
auto file_result = run_file_test(test_path);
if (!m_print_json)
print_file_result(file_result);
if (needs_detailed_suites())
ensure_suites().extend(file_result.suites);
}
inline Vector<String> TestRunner::get_test_paths() const
@ -402,12 +405,12 @@ inline JSFileResult TestRunner::run_file_test(const String& test_path)
}
test_json.value().as_object().for_each_member([&](const String& suite_name, const JsonValue& suite_value) {
Test::Suite suite { suite_name };
Test::Suite suite { test_path, suite_name };
VERIFY(suite_value.is_object());
suite_value.as_object().for_each_member([&](const String& test_name, const JsonValue& test_value) {
Test::Case test { test_name, Test::Result::Fail, "" };
Test::Case test { test_name, Test::Result::Fail, "", 0 };
VERIFY(test_value.is_object());
VERIFY(test_value.as_object().has("result"));
@ -433,6 +436,8 @@ inline JSFileResult TestRunner::run_file_test(const String& test_path)
m_counts.tests_skipped++;
}
test.duration_us = test_value.as_object().get("duration").to_u64(0);
suite.tests.append(test);
});

7
Userland/Libraries/LibTest/JavaScriptTestRunnerMain.cpp
View file

@ -87,6 +87,7 @@ int main(int argc, char** argv)
false;
#endif
bool print_json = false;
bool per_file = false;
const char* specified_test_root = nullptr;
String common_path;
String test_glob;
@ -109,6 +110,7 @@ int main(int argc, char** argv)
},
});
args_parser.add_option(print_json, "Show results as JSON", "json", 'j');
args_parser.add_option(per_file, "Show detailed per-file results as JSON (implies -j)", "per-file", 0);
args_parser.add_option(g_collect_on_every_allocation, "Collect garbage after every allocation", "collect-often", 'g');
args_parser.add_option(g_run_bytecode, "Use the bytecode interpreter", "run-bytecode", 'b');
args_parser.add_option(JS::Bytecode::g_dump_bytecode, "Dump the bytecode", "dump-bytecode", 'd');
@ -119,6 +121,9 @@ int main(int argc, char** argv)
args_parser.add_positional_argument(common_path, "Path to tests-common.js", "common-path", Core::ArgsParser::Required::No);
args_parser.parse(argc, argv);
if (per_file)
print_json = true;
test_glob = String::formatted("*{}*", test_glob);
if (getenv("DISABLE_DBG_OUTPUT")) {
@ -182,7 +187,7 @@ int main(int argc, char** argv)
g_vm->enable_default_host_import_module_dynamically_hook();
}
Test::JS::TestRunner test_runner(test_root, common_path, print_times, print_progress, print_json);
Test::JS::TestRunner test_runner(test_root, common_path, print_times, print_progress, print_json, per_file);
test_runner.run(test_glob);
g_vm = nullptr;

2
Userland/Libraries/LibTest/Results.h
View file

@ -24,9 +24,11 @@ struct Case {
String name;
Result result;
String details;
u64 duration_us;
};
struct Suite {
String path;
String name;
// A failed test takes precedence over a skipped test, which both have
// precedence over a passed test

88
Userland/Libraries/LibTest/TestRunner.h
View file

@ -29,11 +29,12 @@ public:
return s_the;
}
TestRunner(String test_root, bool print_times, bool print_progress, bool print_json)
TestRunner(String test_root, bool print_times, bool print_progress, bool print_json, bool detailed_json = false)
: m_test_root(move(test_root))
, m_print_times(print_times)
, m_print_progress(print_progress)
, m_print_json(print_json)
, m_detailed_json(detailed_json)
{
VERIFY(!s_the);
s_the = this;
@ -47,6 +48,16 @@ public:
bool is_printing_progress() const { return m_print_progress; }
bool needs_detailed_suites() const { return m_detailed_json; }
Vector<Test::Suite> const& suites() const { return *m_suites; }
Vector<Test::Suite>& ensure_suites()
{
if (!m_suites.has_value())
m_suites = Vector<Suite> {};
return *m_suites;
}
protected:
static TestRunner* s_the;
@ -61,9 +72,11 @@ protected:
bool m_print_times;
bool m_print_progress;
bool m_print_json;
bool m_detailed_json;
double m_total_elapsed_time_in_ms { 0 };
Test::Counts m_counts;
Optional<Vector<Test::Suite>> m_suites;
};
inline void cleanup()
@ -223,26 +236,63 @@ inline void TestRunner::print_test_results() const
inline void TestRunner::print_test_results_as_json() const
{
JsonObject suites;
suites.set("failed", m_counts.suites_failed);
suites.set("passed", m_counts.suites_passed);
suites.set("total", m_counts.suites_failed + m_counts.suites_passed);
JsonObject tests;
tests.set("failed", m_counts.tests_failed);
tests.set("passed", m_counts.tests_passed);
tests.set("skipped", m_counts.tests_skipped);
tests.set("total", m_counts.tests_failed + m_counts.tests_passed + m_counts.tests_skipped);
JsonObject results;
results.set("suites", suites);
results.set("tests", tests);
JsonObject root;
root.set("results", results);
root.set("files_total", m_counts.files_total);
root.set("duration", m_total_elapsed_time_in_ms / 1000.0);
if (needs_detailed_suites()) {
auto& suites = this->suites();
u64 duration_us = 0;
JsonObject tests;
for (auto& suite : suites) {
for (auto& case_ : suite.tests) {
duration_us += case_.duration_us;
StringView result_name;
switch (case_.result) {
case Result::Pass:
result_name = "PASSED";
break;
case Result::Fail:
result_name = "FAILED";
break;
case Result::Skip:
result_name = "SKIPPED";
break;
case Result::Crashed:
result_name = "PROCESS_ERROR";
break;
}
auto name = suite.name;
if (name == "__$$TOP_LEVEL$$__"sv)
name = String::empty();
auto path = LexicalPath::relative_path(suite.path, m_test_root);
tests.set(String::formatted("{}/{}::{}", path, name, case_.name), result_name);
}
}
root.set("duration", static_cast<double>(duration_us) / 1000000.);
root.set("results", move(tests));
} else {
JsonObject suites;
suites.set("failed", m_counts.suites_failed);
suites.set("passed", m_counts.suites_passed);
suites.set("total", m_counts.suites_failed + m_counts.suites_passed);
JsonObject tests;
tests.set("failed", m_counts.tests_failed);
tests.set("passed", m_counts.tests_passed);
tests.set("skipped", m_counts.tests_skipped);
tests.set("total", m_counts.tests_failed + m_counts.tests_passed + m_counts.tests_skipped);
JsonObject results;
results.set("suites", suites);
results.set("tests", tests);
root.set("results", results);
root.set("files_total", m_counts.files_total);
root.set("duration", m_total_elapsed_time_in_ms / 1000.0);
}
outln("{}", root.to_string());
}