test262-runner.cpp 27 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791
  1. /*
  2. * Copyright (c) 2021, Linus Groh <linusg@serenityos.org>
  3. * Copyright (c) 2021-2022, David Tuin <davidot@serenityos.org>
  4. *
  5. * SPDX-License-Identifier: BSD-2-Clause
  6. */
  7. #include <AK/Format.h>
  8. #include <AK/JsonObject.h>
  9. #include <AK/Result.h>
  10. #include <AK/ScopeGuard.h>
  11. #include <AK/String.h>
  12. #include <AK/Vector.h>
  13. #include <LibCore/ArgsParser.h>
  14. #include <LibCore/File.h>
  15. #include <LibJS/Bytecode/BasicBlock.h>
  16. #include <LibJS/Bytecode/Generator.h>
  17. #include <LibJS/Bytecode/Interpreter.h>
  18. #include <LibJS/Bytecode/PassManager.h>
  19. #include <LibJS/Contrib/Test262/GlobalObject.h>
  20. #include <LibJS/Interpreter.h>
  21. #include <LibJS/Runtime/VM.h>
  22. #include <LibJS/Script.h>
  23. #include <fcntl.h>
  24. #include <signal.h>
  25. #include <unistd.h>
  26. #ifndef AK_OS_MACOS
  27. // Only used to disable core dumps
  28. # include <sys/prctl.h>
  29. #endif
  30. static String s_current_test = "";
  31. static bool s_use_bytecode = false;
  32. static bool s_parse_only = false;
  33. static String s_harness_file_directory;
  34. static bool s_automatic_harness_detection_mode = false;
  35. enum class NegativePhase {
  36. ParseOrEarly,
  37. Resolution,
  38. Runtime,
  39. Harness
  40. };
  41. struct TestError {
  42. NegativePhase phase { NegativePhase::ParseOrEarly };
  43. String type;
  44. String details;
  45. String harness_file;
  46. };
  47. using ScriptOrModuleProgram = Variant<JS::NonnullGCPtr<JS::Script>, JS::NonnullGCPtr<JS::SourceTextModule>>;
  48. template<typename ScriptType>
  49. static Result<ScriptOrModuleProgram, TestError> parse_program(JS::Realm& realm, StringView source, StringView filepath)
  50. {
  51. auto script_or_error = ScriptType::parse(source, realm, filepath);
  52. if (script_or_error.is_error()) {
  53. return TestError {
  54. NegativePhase::ParseOrEarly,
  55. "SyntaxError",
  56. script_or_error.error()[0].to_string(),
  57. ""
  58. };
  59. }
  60. return ScriptOrModuleProgram { script_or_error.release_value() };
  61. }
  62. static Result<ScriptOrModuleProgram, TestError> parse_program(JS::Realm& realm, StringView source, StringView filepath, JS::Program::Type program_type)
  63. {
  64. if (program_type == JS::Program::Type::Script)
  65. return parse_program<JS::Script>(realm, source, filepath);
  66. return parse_program<JS::SourceTextModule>(realm, source, filepath);
  67. }
  68. template<typename InterpreterT>
  69. static Result<void, TestError> run_program(InterpreterT& interpreter, ScriptOrModuleProgram& program)
  70. {
  71. auto result = JS::ThrowCompletionOr<JS::Value> { JS::js_undefined() };
  72. if constexpr (IsSame<InterpreterT, JS::Interpreter>) {
  73. result = program.visit(
  74. [&](auto& visitor) {
  75. return interpreter.run(*visitor);
  76. });
  77. } else {
  78. auto program_node = program.visit(
  79. [](auto& visitor) -> NonnullRefPtr<JS::Program> {
  80. return visitor->parse_node();
  81. });
  82. auto unit_result = JS::Bytecode::Generator::generate(program_node);
  83. if (unit_result.is_error()) {
  84. result = JS::throw_completion(JS::InternalError::create(interpreter.realm(), String::formatted("TODO({})", unit_result.error().to_string())));
  85. } else {
  86. auto unit = unit_result.release_value();
  87. auto& passes = JS::Bytecode::Interpreter::optimization_pipeline();
  88. passes.perform(*unit);
  89. result = interpreter.run(*unit);
  90. }
  91. }
  92. if (result.is_error()) {
  93. auto error_value = *result.throw_completion().value();
  94. TestError error;
  95. error.phase = NegativePhase::Runtime;
  96. if (error_value.is_object()) {
  97. auto& object = error_value.as_object();
  98. auto name = object.get_without_side_effects("name");
  99. if (!name.is_empty() && !name.is_accessor()) {
  100. error.type = name.to_string_without_side_effects();
  101. } else {
  102. auto constructor = object.get_without_side_effects("constructor");
  103. if (constructor.is_object()) {
  104. name = constructor.as_object().get_without_side_effects("name");
  105. if (!name.is_undefined())
  106. error.type = name.to_string_without_side_effects();
  107. }
  108. }
  109. auto message = object.get_without_side_effects("message");
  110. if (!message.is_empty() && !message.is_accessor())
  111. error.details = message.to_string_without_side_effects();
  112. }
  113. if (error.type.is_empty())
  114. error.type = error_value.to_string_without_side_effects();
  115. return error;
  116. }
  117. return {};
  118. }
  119. static HashMap<String, String> s_cached_harness_files;
  120. static Result<StringView, TestError> read_harness_file(StringView harness_file)
  121. {
  122. auto cache = s_cached_harness_files.find(harness_file);
  123. if (cache == s_cached_harness_files.end()) {
  124. auto file = Core::File::construct(String::formatted("{}{}", s_harness_file_directory, harness_file));
  125. if (!file->open(Core::OpenMode::ReadOnly)) {
  126. return TestError {
  127. NegativePhase::Harness,
  128. "filesystem",
  129. String::formatted("Could not open file: {}", harness_file),
  130. harness_file
  131. };
  132. }
  133. auto contents = file->read_all();
  134. StringView contents_view = contents;
  135. s_cached_harness_files.set(harness_file, contents_view.to_string());
  136. cache = s_cached_harness_files.find(harness_file);
  137. VERIFY(cache != s_cached_harness_files.end());
  138. }
  139. return cache->value.view();
  140. }
  141. static Result<JS::NonnullGCPtr<JS::Script>, TestError> parse_harness_files(JS::Realm& realm, StringView harness_file)
  142. {
  143. auto source_or_error = read_harness_file(harness_file);
  144. if (source_or_error.is_error())
  145. return source_or_error.release_error();
  146. auto program_or_error = parse_program<JS::Script>(realm, source_or_error.value(), harness_file);
  147. if (program_or_error.is_error()) {
  148. return TestError {
  149. NegativePhase::Harness,
  150. program_or_error.error().type,
  151. program_or_error.error().details,
  152. harness_file
  153. };
  154. }
  155. return program_or_error.release_value().get<JS::NonnullGCPtr<JS::Script>>();
  156. }
  157. enum class StrictMode {
  158. Both,
  159. NoStrict,
  160. OnlyStrict
  161. };
  162. static constexpr auto sta_harness_file = "sta.js"sv;
  163. static constexpr auto assert_harness_file = "assert.js"sv;
  164. static constexpr auto async_include = "doneprintHandle.js"sv;
  165. struct TestMetadata {
  166. Vector<StringView> harness_files { sta_harness_file, assert_harness_file };
  167. StrictMode strict_mode { StrictMode::Both };
  168. JS::Program::Type program_type { JS::Program::Type::Script };
  169. bool is_async { false };
  170. bool is_negative { false };
  171. NegativePhase phase { NegativePhase::ParseOrEarly };
  172. StringView type;
  173. };
  174. static Result<void, TestError> run_test(StringView source, StringView filepath, TestMetadata const& metadata)
  175. {
  176. if (s_parse_only || (metadata.is_negative && metadata.phase == NegativePhase::ParseOrEarly && metadata.program_type != JS::Program::Type::Module)) {
  177. // Creating the vm and interpreter is heavy so we just parse directly here.
  178. // We can also skip if we know the test is supposed to fail during parse
  179. // time. Unfortunately the phases of modules are not as clear and thus we
  180. // only do this for scripts. See also the comment at the end of verify_test.
  181. auto parser = JS::Parser(JS::Lexer(source, filepath), metadata.program_type);
  182. auto program_or_error = parser.parse_program();
  183. if (parser.has_errors()) {
  184. return TestError {
  185. NegativePhase::ParseOrEarly,
  186. "SyntaxError",
  187. parser.errors()[0].to_string(),
  188. ""
  189. };
  190. }
  191. return {};
  192. }
  193. auto vm = JS::VM::create();
  194. vm->enable_default_host_import_module_dynamically_hook();
  195. auto ast_interpreter = JS::Interpreter::create<JS::Test262::GlobalObject>(*vm);
  196. auto& realm = ast_interpreter->realm();
  197. auto program_or_error = parse_program(realm, source, filepath, metadata.program_type);
  198. if (program_or_error.is_error())
  199. return program_or_error.release_error();
  200. OwnPtr<JS::Bytecode::Interpreter> bytecode_interpreter = nullptr;
  201. if (s_use_bytecode)
  202. bytecode_interpreter = make<JS::Bytecode::Interpreter>(realm);
  203. auto run_with_interpreter = [&](ScriptOrModuleProgram& program) {
  204. if (s_use_bytecode)
  205. return run_program(*bytecode_interpreter, program);
  206. return run_program(*ast_interpreter, program);
  207. };
  208. for (auto& harness_file : metadata.harness_files) {
  209. auto harness_program_or_error = parse_harness_files(realm, harness_file);
  210. if (harness_program_or_error.is_error())
  211. return harness_program_or_error.release_error();
  212. ScriptOrModuleProgram harness_program { harness_program_or_error.release_value() };
  213. auto result = run_with_interpreter(harness_program);
  214. if (result.is_error()) {
  215. return TestError {
  216. NegativePhase::Harness,
  217. result.error().type,
  218. result.error().details,
  219. harness_file
  220. };
  221. }
  222. }
  223. return run_with_interpreter(program_or_error.value());
  224. }
  225. static Result<TestMetadata, String> extract_metadata(StringView source)
  226. {
  227. auto lines = source.lines();
  228. TestMetadata metadata;
  229. bool parsing_negative = false;
  230. String failed_message;
  231. auto parse_list = [&](StringView line) {
  232. auto start = line.find('[');
  233. if (!start.has_value())
  234. return Vector<StringView> {};
  235. Vector<StringView> items;
  236. auto end = line.find_last(']');
  237. if (!end.has_value() || end.value() <= start.value()) {
  238. failed_message = String::formatted("Can't parse list in '{}'", line);
  239. return items;
  240. }
  241. auto list = line.substring_view(start.value() + 1, end.value() - start.value() - 1);
  242. for (auto const& item : list.split_view(","sv))
  243. items.append(item.trim_whitespace(TrimMode::Both));
  244. return items;
  245. };
  246. auto second_word = [&](StringView line) {
  247. auto separator = line.find(' ');
  248. if (!separator.has_value() || separator.value() >= (line.length() - 1u)) {
  249. failed_message = String::formatted("Can't parse value after space in '{}'", line);
  250. return ""sv;
  251. }
  252. return line.substring_view(separator.value() + 1);
  253. };
  254. Vector<StringView> include_list;
  255. bool parsing_includes_list = false;
  256. bool has_phase = false;
  257. for (auto raw_line : lines) {
  258. if (!failed_message.is_empty())
  259. break;
  260. if (raw_line.starts_with("---*/"sv)) {
  261. if (parsing_includes_list) {
  262. for (auto& file : include_list)
  263. metadata.harness_files.append(file);
  264. }
  265. return metadata;
  266. }
  267. auto line = raw_line.trim_whitespace();
  268. if (parsing_includes_list) {
  269. if (line.starts_with('-')) {
  270. include_list.append(second_word(line));
  271. continue;
  272. } else {
  273. if (include_list.is_empty()) {
  274. failed_message = "Supposed to parse a list but found no entries";
  275. break;
  276. }
  277. for (auto& file : include_list)
  278. metadata.harness_files.append(file);
  279. include_list.clear();
  280. parsing_includes_list = false;
  281. }
  282. }
  283. if (parsing_negative) {
  284. if (line.starts_with("phase:"sv)) {
  285. auto phase = second_word(line);
  286. has_phase = true;
  287. if (phase == "early"sv || phase == "parse"sv) {
  288. metadata.phase = NegativePhase::ParseOrEarly;
  289. } else if (phase == "resolution"sv) {
  290. metadata.phase = NegativePhase::Resolution;
  291. } else if (phase == "runtime"sv) {
  292. metadata.phase = NegativePhase::Runtime;
  293. } else {
  294. has_phase = false;
  295. failed_message = String::formatted("Unknown negative phase: {}", phase);
  296. break;
  297. }
  298. } else if (line.starts_with("type:"sv)) {
  299. metadata.type = second_word(line);
  300. } else {
  301. if (!has_phase) {
  302. failed_message = "Failed to find phase in negative attributes";
  303. break;
  304. }
  305. if (metadata.type.is_null()) {
  306. failed_message = "Failed to find type in negative attributes";
  307. break;
  308. }
  309. parsing_negative = false;
  310. }
  311. }
  312. if (line.starts_with("flags:"sv)) {
  313. auto flags = parse_list(line);
  314. if (flags.is_empty()) {
  315. failed_message = String::formatted("Failed to find flags in '{}'", line);
  316. break;
  317. }
  318. for (auto flag : flags) {
  319. if (flag == "raw"sv) {
  320. metadata.strict_mode = StrictMode::NoStrict;
  321. metadata.harness_files.clear();
  322. } else if (flag == "noStrict"sv) {
  323. metadata.strict_mode = StrictMode::NoStrict;
  324. } else if (flag == "onlyStrict"sv) {
  325. metadata.strict_mode = StrictMode::OnlyStrict;
  326. } else if (flag == "module"sv) {
  327. VERIFY(metadata.strict_mode == StrictMode::Both);
  328. metadata.program_type = JS::Program::Type::Module;
  329. metadata.strict_mode = StrictMode::NoStrict;
  330. } else if (flag == "async"sv) {
  331. metadata.harness_files.append(async_include);
  332. metadata.is_async = true;
  333. }
  334. }
  335. } else if (line.starts_with("includes:"sv)) {
  336. auto files = parse_list(line);
  337. if (files.is_empty()) {
  338. parsing_includes_list = true;
  339. } else {
  340. for (auto& file : files)
  341. metadata.harness_files.append(file);
  342. }
  343. } else if (line.starts_with("negative:"sv)) {
  344. metadata.is_negative = true;
  345. parsing_negative = true;
  346. }
  347. }
  348. if (failed_message.is_empty())
  349. failed_message = String::formatted("Never reached end of comment '---*/'");
  350. return failed_message;
  351. }
  352. static bool verify_test(Result<void, TestError>& result, TestMetadata const& metadata, JsonObject& output)
  353. {
  354. if (result.is_error()) {
  355. if (result.error().phase == NegativePhase::Harness) {
  356. output.set("harness_error", true);
  357. output.set("harness_file", result.error().harness_file);
  358. output.set("result", "harness_error");
  359. } else if (result.error().phase == NegativePhase::Runtime) {
  360. auto& error_type = result.error().type;
  361. auto& error_details = result.error().details;
  362. if ((error_type == "InternalError"sv && error_details.starts_with("TODO("sv))
  363. || (error_type == "Test262Error"sv && error_details.ends_with(" but got a InternalError"sv))) {
  364. output.set("todo_error", true);
  365. output.set("result", "todo_error");
  366. }
  367. }
  368. }
  369. if (metadata.is_async && output.has("output"sv)) {
  370. auto& output_messages = output.get("output"sv);
  371. VERIFY(output_messages.is_string());
  372. if (output_messages.as_string().contains("AsyncTestFailure:InternalError: TODO("sv)) {
  373. output.set("todo_error", true);
  374. output.set("result", "todo_error");
  375. }
  376. }
  377. auto phase_to_string = [](NegativePhase phase) {
  378. switch (phase) {
  379. case NegativePhase::ParseOrEarly:
  380. return "parse";
  381. case NegativePhase::Resolution:
  382. return "resolution";
  383. case NegativePhase::Runtime:
  384. return "runtime";
  385. case NegativePhase::Harness:
  386. return "harness";
  387. }
  388. VERIFY_NOT_REACHED();
  389. };
  390. auto error_to_json = [&phase_to_string](TestError const& error) {
  391. JsonObject error_object;
  392. error_object.set("phase", phase_to_string(error.phase));
  393. error_object.set("type", error.type);
  394. error_object.set("details", error.details);
  395. return error_object;
  396. };
  397. JsonValue expected_error;
  398. JsonValue got_error;
  399. ScopeGuard set_error = [&] {
  400. JsonObject error_object;
  401. error_object.set("expected", expected_error);
  402. error_object.set("got", got_error);
  403. output.set("error", error_object);
  404. };
  405. if (!metadata.is_negative) {
  406. if (!result.is_error())
  407. return true;
  408. got_error = JsonValue { error_to_json(result.error()) };
  409. return false;
  410. }
  411. JsonObject expected_error_object;
  412. expected_error_object.set("phase", phase_to_string(metadata.phase));
  413. expected_error_object.set("type", metadata.type.to_string());
  414. expected_error = expected_error_object;
  415. if (!result.is_error()) {
  416. if (s_parse_only && metadata.phase != NegativePhase::ParseOrEarly) {
  417. // Expected non-parse error but did not get it but we never got to that phase.
  418. return true;
  419. }
  420. return false;
  421. }
  422. auto const& error = result.error();
  423. got_error = JsonValue { error_to_json(error) };
  424. if (metadata.program_type == JS::Program::Type::Module && metadata.type == "SyntaxError"sv) {
  425. // NOTE: Since the "phase" of negative results is both not defined and hard to
  426. // track throughout the entire Module life span we will just accept any
  427. // SyntaxError as the correct one.
  428. // See for example:
  429. // - test/language/module-code/instn-star-err-not-found.js
  430. // - test/language/module-code/instn-resolve-err-syntax-1.js
  431. // - test/language/import/json-invalid.js
  432. // The first fails in runtime because there is no 'x' to export
  433. // However this is during the linking phase of the upper module.
  434. // Whereas the second fails with a SyntaxError because the linked module
  435. // has one.
  436. // The third test is the same as the second, upper module is fine but
  437. // import a module with SyntaxError, however here the phase is runtime.
  438. // In conclusion all the test which would cause the initial module to not
  439. // be evaluated !should! have '$DONOTEVALUATE();' at the top causing a
  440. // Reference error, meaning we just ignore the phase in the SyntaxError case.
  441. return error.type == metadata.type;
  442. }
  443. return error.phase == metadata.phase && error.type == metadata.type;
  444. }
  445. static bool extract_harness_directory(String const& test_file_path)
  446. {
  447. auto test_directory_index = test_file_path.find("test/"sv);
  448. if (!test_directory_index.has_value()) {
  449. warnln("Attempted to find harness directory from test file '{}', but did not find 'test/'", test_file_path);
  450. return false;
  451. }
  452. s_harness_file_directory = String::formatted("{}harness/", test_file_path.substring_view(0, test_directory_index.value()));
  453. return true;
  454. }
  455. static FILE* saved_stdout_fd;
  456. static bool g_in_assert = false;
  457. [[noreturn]] static void handle_failed_assert(char const* assert_failed_message)
  458. {
  459. if (!g_in_assert) {
  460. // Just in case we trigger an assert while creating the JSON output just
  461. // immediately stop if we are already in a failed assert.
  462. g_in_assert = true;
  463. JsonObject assert_fail_result;
  464. assert_fail_result.set("test", s_current_test);
  465. assert_fail_result.set("assert_fail", true);
  466. assert_fail_result.set("result", "assert_fail");
  467. assert_fail_result.set("output", assert_failed_message);
  468. outln(saved_stdout_fd, "RESULT {}{}", assert_fail_result.to_string(), '\0');
  469. // (Attempt to) Ensure that messages are written before quitting.
  470. fflush(saved_stdout_fd);
  471. fflush(stderr);
  472. }
  473. exit(12);
  474. }
  475. #ifdef __serenity__
  476. void __assertion_failed(char const* assertion)
  477. {
  478. handle_failed_assert(assertion);
  479. }
  480. #else
  481. extern "C" __attribute__((__noreturn__)) void __assert_fail(char const* assertion, char const* file, unsigned int line, char const* function)
  482. {
  483. auto full_message = String::formatted("{}:{}: {}: Assertion `{}' failed.", file, line, function, assertion);
  484. handle_failed_assert(full_message.characters());
  485. }
  486. #endif
  487. int main(int argc, char** argv)
  488. {
  489. int timeout = 10;
  490. bool enable_debug_printing = false;
  491. bool disable_core_dumping = false;
  492. Core::ArgsParser args_parser;
  493. args_parser.set_general_help("LibJS test262 runner for streaming tests");
  494. args_parser.add_option(s_harness_file_directory, "Directory containing the harness files", "harness-location", 'l', "harness-files");
  495. args_parser.add_option(s_use_bytecode, "Use the bytecode interpreter", "use-bytecode", 'b');
  496. args_parser.add_option(s_parse_only, "Only parse the files", "parse-only", 'p');
  497. args_parser.add_option(timeout, "Seconds before test should timeout", "timeout", 't', "seconds");
  498. args_parser.add_option(enable_debug_printing, "Enable debug printing", "debug", 'd');
  499. args_parser.add_option(disable_core_dumping, "Disable core dumping", "disable-core-dump", 0);
  500. args_parser.parse(argc, argv);
  501. #ifndef AK_OS_MACOS
  502. if (disable_core_dumping && prctl(PR_SET_DUMPABLE, 0, 0) < 0) {
  503. perror("prctl(PR_SET_DUMPABLE)");
  504. return 2;
  505. }
  506. #endif
  507. if (s_harness_file_directory.is_empty()) {
  508. s_automatic_harness_detection_mode = true;
  509. } else if (!s_harness_file_directory.ends_with('/')) {
  510. s_harness_file_directory = String::formatted("{}/", s_harness_file_directory);
  511. }
  512. if (timeout <= 0) {
  513. warnln("timeout must be at least 1");
  514. return 2;
  515. }
  516. AK::set_debug_enabled(enable_debug_printing);
  517. // The piping stuff is based on https://stackoverflow.com/a/956269.
  518. constexpr auto BUFFER_SIZE = 1 * KiB;
  519. char buffer[BUFFER_SIZE] = {};
  520. auto saved_stdout = dup(STDOUT_FILENO);
  521. if (saved_stdout < 0) {
  522. perror("dup");
  523. return 1;
  524. }
  525. saved_stdout_fd = fdopen(saved_stdout, "w");
  526. if (!saved_stdout_fd) {
  527. perror("fdopen");
  528. return 1;
  529. }
  530. int stdout_pipe[2];
  531. if (pipe(stdout_pipe) < 0) {
  532. perror("pipe");
  533. return 1;
  534. }
  535. auto flags = fcntl(stdout_pipe[0], F_GETFL);
  536. flags |= O_NONBLOCK;
  537. fcntl(stdout_pipe[0], F_SETFL, flags);
  538. auto flags2 = fcntl(stdout_pipe[1], F_GETFL);
  539. flags2 |= O_NONBLOCK;
  540. fcntl(stdout_pipe[1], F_SETFL, flags2);
  541. if (dup2(stdout_pipe[1], STDOUT_FILENO) < 0) {
  542. perror("dup2");
  543. return 1;
  544. }
  545. if (close(stdout_pipe[1]) < 0) {
  546. perror("close");
  547. return 1;
  548. }
  549. auto collect_output = [&] {
  550. fflush(stdout);
  551. auto nread = read(stdout_pipe[0], buffer, BUFFER_SIZE);
  552. String value;
  553. if (nread > 0) {
  554. value = String { buffer, static_cast<size_t>(nread) };
  555. while (nread > 0) {
  556. nread = read(stdout_pipe[0], buffer, BUFFER_SIZE);
  557. }
  558. }
  559. return value;
  560. };
  561. #define ARM_TIMER() \
  562. alarm(timeout)
  563. #define DISARM_TIMER() \
  564. alarm(0)
  565. auto standard_input = Core::File::standard_input();
  566. size_t count = 0;
  567. while (!standard_input->eof()) {
  568. auto path = standard_input->read_line();
  569. if (path.is_empty()) {
  570. continue;
  571. }
  572. s_current_test = path;
  573. if (s_automatic_harness_detection_mode) {
  574. if (!extract_harness_directory(path))
  575. return 4;
  576. s_automatic_harness_detection_mode = false;
  577. VERIFY(!s_harness_file_directory.is_empty());
  578. }
  579. auto file = Core::File::construct(path);
  580. if (!file->open(Core::OpenMode::ReadOnly)) {
  581. warnln("Could not open file: {}", path);
  582. return 3;
  583. }
  584. count++;
  585. String source_with_strict;
  586. static String use_strict = "'use strict';\n";
  587. static size_t strict_length = use_strict.length();
  588. {
  589. auto contents = file->read_all();
  590. StringBuilder builder { contents.size() + strict_length };
  591. builder.append(use_strict);
  592. builder.append(contents);
  593. source_with_strict = builder.to_string();
  594. }
  595. StringView with_strict = source_with_strict.view();
  596. StringView original_contents = source_with_strict.substring_view(strict_length);
  597. JsonObject result_object;
  598. result_object.set("test", path);
  599. ScopeGuard output_guard = [&] {
  600. outln(saved_stdout_fd, "RESULT {}{}", result_object.to_string(), '\0');
  601. fflush(saved_stdout_fd);
  602. };
  603. auto metadata_or_error = extract_metadata(original_contents);
  604. if (metadata_or_error.is_error()) {
  605. result_object.set("result", "metadata_error");
  606. result_object.set("metadata_error", true);
  607. result_object.set("metadata_output", metadata_or_error.error());
  608. continue;
  609. }
  610. auto& metadata = metadata_or_error.value();
  611. bool passed = true;
  612. if (metadata.strict_mode != StrictMode::OnlyStrict) {
  613. result_object.set("strict_mode", false);
  614. ARM_TIMER();
  615. auto result = run_test(original_contents, path, metadata);
  616. DISARM_TIMER();
  617. String first_output = collect_output();
  618. if (!first_output.is_null())
  619. result_object.set("output", first_output);
  620. passed = verify_test(result, metadata, result_object);
  621. if (metadata.is_async && !s_parse_only) {
  622. if (!first_output.contains("Test262:AsyncTestComplete"sv) || first_output.contains("Test262:AsyncTestFailure"sv)) {
  623. result_object.set("async_fail", true);
  624. if (first_output.is_null())
  625. result_object.set("output", JsonValue { AK::JsonValue::Type::Null });
  626. passed = false;
  627. }
  628. }
  629. }
  630. if (passed && metadata.strict_mode != StrictMode::NoStrict) {
  631. result_object.set("strict_mode", true);
  632. ARM_TIMER();
  633. auto result = run_test(with_strict, path, metadata);
  634. DISARM_TIMER();
  635. String first_output = collect_output();
  636. if (!first_output.is_null())
  637. result_object.set("strict_output", first_output);
  638. passed = verify_test(result, metadata, result_object);
  639. if (metadata.is_async && !s_parse_only) {
  640. if (!first_output.contains("Test262:AsyncTestComplete"sv) || first_output.contains("Test262:AsyncTestFailure"sv)) {
  641. result_object.set("async_fail", true);
  642. if (first_output.is_null())
  643. result_object.set("output", JsonValue { AK::JsonValue::Type::Null });
  644. passed = false;
  645. }
  646. }
  647. }
  648. if (passed)
  649. result_object.remove("strict_mode"sv);
  650. if (!result_object.has("result"sv))
  651. result_object.set("result"sv, passed ? "passed"sv : "failed"sv);
  652. }
  653. s_current_test = "";
  654. outln(saved_stdout_fd, "DONE {}", count);
  655. // After this point we have already written our output so pretend everything is fine if we get an error.
  656. if (dup2(saved_stdout, STDOUT_FILENO) < 0) {
  657. perror("dup2");
  658. return 0;
  659. }
  660. if (fclose(saved_stdout_fd) < 0) {
  661. perror("fclose");
  662. return 0;
  663. }
  664. if (close(stdout_pipe[0]) < 0) {
  665. perror("close");
  666. return 0;
  667. }
  668. return 0;
  669. }