test-runner.cpp 5.8 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173
  1. /*
  2. * Copyright (c) 2023, Dan Klishch <danilklishch@gmail.com>
  3. *
  4. * SPDX-License-Identifier: BSD-2-Clause
  5. */
  6. #include <AK/Variant.h>
  7. #include <LibCore/Directory.h>
  8. #include <LibCore/Process.h>
  9. #include <LibCore/System.h>
  10. #include <LibDiff/Format.h>
  11. #include <LibDiff/Generator.h>
  12. #include <LibFileSystem/FileSystem.h>
  13. #include <LibFileSystem/TempFile.h>
  14. #include <LibTest/TestCase.h>
  15. struct TestDescription {
  16. struct Flag {
  17. StringView name;
  18. bool dump_ast = false;
  19. bool dump_cfg = false;
  20. };
  21. Vector<StringView> sources;
  22. Vector<Flag> flags;
  23. };
  24. constexpr StringView stderr_capture_filename = "stderr"sv;
  25. constexpr StringView compiler_binary_name = "JSSpecCompiler"sv;
  26. constexpr StringView relative_path_to_test = "Tests"sv;
  27. constexpr TestDescription::Flag always_dump_all = {
  28. .name = "all"sv,
  29. .dump_ast = true,
  30. .dump_cfg = true
  31. };
  32. const Array regression_tests = {
  33. TestDescription {
  34. .sources = { "simple.cpp"sv },
  35. .flags = { always_dump_all },
  36. },
  37. };
  38. static const LexicalPath path_to_compiler_binary = [] {
  39. auto path_to_self = LexicalPath(MUST(Core::System::current_executable_path())).parent();
  40. return LexicalPath::join(path_to_self.string(), compiler_binary_name);
  41. }();
  42. static const LexicalPath path_to_tests_directory { relative_path_to_test };
  43. Vector<DeprecatedString> build_command_line_arguments(LexicalPath const& test_source, TestDescription const& description)
  44. {
  45. Vector<DeprecatedString> result;
  46. StringBuilder dump_ast_option;
  47. StringBuilder dump_cfg_option;
  48. for (auto const& flag : description.flags) {
  49. if (flag.dump_ast) {
  50. if (!dump_ast_option.is_empty())
  51. dump_ast_option.append(","sv);
  52. dump_ast_option.append(flag.name);
  53. }
  54. if (flag.dump_cfg) {
  55. if (!dump_cfg_option.is_empty())
  56. dump_cfg_option.append(","sv);
  57. dump_cfg_option.append(flag.name);
  58. }
  59. }
  60. if (!dump_ast_option.is_empty())
  61. result.append(DeprecatedString::formatted("--dump-ast={}", dump_ast_option.string_view()));
  62. if (!dump_cfg_option.is_empty())
  63. result.append(DeprecatedString::formatted("--dump-cfg={}", dump_cfg_option.string_view()));
  64. if (test_source.has_extension(".cpp"sv))
  65. result.append("-xc++"sv);
  66. result.append(test_source.string());
  67. return result;
  68. }
  69. ErrorOr<ByteBuffer> read(LexicalPath const& path)
  70. {
  71. auto file = TRY(Core::File::open(path.string(), Core::File::OpenMode::Read));
  72. return MUST(file->read_until_eof());
  73. }
  74. void check_expectations(LexicalPath const& path_to_expectation, LexicalPath const& path_to_captured_output, bool should_update_expectations)
  75. {
  76. struct PathPair {
  77. LexicalPath expectation;
  78. LexicalPath result;
  79. };
  80. Vector<PathPair> file_pairs_to_check;
  81. file_pairs_to_check.append({
  82. .expectation = path_to_expectation,
  83. .result = path_to_captured_output,
  84. });
  85. auto out = MUST(Core::File::standard_error());
  86. for (auto const& [expectation_path, result_path] : file_pairs_to_check) {
  87. auto result_content = read(result_path);
  88. if (should_update_expectations && !result_content.is_error()) {
  89. using namespace FileSystem;
  90. MUST(copy_file_or_directory(expectation_path.string(), result_path.string(),
  91. RecursionMode::Disallowed, LinkMode::Disallowed, AddDuplicateFileMarker::No));
  92. }
  93. auto expectation = read(expectation_path);
  94. bool read_successfully = !(expectation.is_error() || result_content.is_error());
  95. EXPECT(read_successfully);
  96. if (read_successfully) {
  97. bool are_equal = expectation.value() == result_content.value();
  98. EXPECT(are_equal);
  99. if (!are_equal) {
  100. dbgln("Files {} and {} do not match!", expectation_path.string(), result_path.string());
  101. auto maybe_diff = Diff::from_text(expectation.value(), result_content.value());
  102. if (!maybe_diff.is_error()) {
  103. for (auto const& hunk : maybe_diff.value())
  104. MUST(Diff::write_unified(hunk, *out, Diff::ColorOutput::Yes));
  105. }
  106. }
  107. }
  108. }
  109. }
  110. TEST_CASE(test_regression)
  111. {
  112. auto* update_expectations_env = getenv("JSSC_UPDATE_EXPECTATIONS");
  113. bool should_update_expectations = false;
  114. if (update_expectations_env != nullptr && strcmp(update_expectations_env, "1") == 0)
  115. should_update_expectations = true;
  116. auto temp_directory = MUST(FileSystem::TempFile::create_temp_directory());
  117. auto path_to_captured_stderr = LexicalPath::join(temp_directory->path(), stderr_capture_filename);
  118. for (auto const& test_description : regression_tests) {
  119. for (auto const& source : test_description.sources) {
  120. dbgln("Running {}...", source);
  121. auto path_to_test = LexicalPath::join(path_to_tests_directory.string(), source);
  122. auto path_to_expectation = LexicalPath::join(path_to_tests_directory.string(), DeprecatedString::formatted("{}.expectation", source));
  123. auto process = MUST(Core::Process::spawn({
  124. .path = path_to_compiler_binary.string(),
  125. .arguments = build_command_line_arguments(path_to_test, test_description),
  126. .file_actions = {
  127. Core::FileAction::OpenFile {
  128. .path = path_to_captured_stderr.string(),
  129. .mode = Core::File::OpenMode::Write,
  130. .fd = STDERR_FILENO,
  131. },
  132. },
  133. }));
  134. bool exited_with_code_0 = MUST(process.wait_for_termination());
  135. EXPECT(exited_with_code_0);
  136. if (exited_with_code_0)
  137. check_expectations(path_to_expectation, path_to_captured_stderr, should_update_expectations);
  138. }
  139. }
  140. }