JSSpecCompiler: Add regression test runner

Author: https://github.com/DanShaders
Commit: https://github.com/SerenityOS/serenity/commit/107a3b44fa
Pull-request: https://github.com/SerenityOS/serenity/pull/21612
Reviewed-by: https://github.com/ADKaster

Meta/Lagom/CMakeLists.txt

@@ -647,6 +647,7 @@ if (BUILD_LAGOM)
         # LibTest tests from Tests/
         set(TEST_DIRECTORIES
             AK
+            JSSpecCompiler
             LibCrypto
             LibCompress
             LibGL

Tests/JSSpecCompiler/CMakeLists.txt

@@ -0,0 +1,6 @@
+lagom_test(
+    test-runner.cpp
+    NAME TestJSSpecCompiler
+    LIBS LibDiff
+    WORKING_DIRECTORY "${SERENITY_PROJECT_ROOT}/Meta/Lagom/Tools/CodeGenerators/JSSpecCompiler/"
+)

Tests/JSSpecCompiler/test-runner.cpp

@@ -0,0 +1,153 @@
+/*
+ * Copyright (c) 2023, Dan Klishch <danilklishch@gmail.com>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#include <AK/Variant.h>
+#include <LibCore/Directory.h>
+#include <LibCore/Process.h>
+#include <LibCore/System.h>
+#include <LibDiff/Format.h>
+#include <LibDiff/Generator.h>
+#include <LibFileSystem/FileSystem.h>
+#include <LibFileSystem/TempFile.h>
+#include <LibTest/TestCase.h>
+
+struct TestDescription {
+    struct Flag {
+        StringView name;
+        bool dump_ast = false;
+    };
+
+    Vector<StringView> sources;
+    Vector<Flag> flags;
+};
+
+constexpr StringView stderr_capture_filename = "stderr"sv;
+
+constexpr StringView compiler_binary_name = "JSSpecCompiler"sv;
+constexpr StringView relative_path_to_test = "Tests"sv;
+
+Array<TestDescription, 0> const regression_tests = {};
+
+static const LexicalPath path_to_compiler_binary = [] {
+    auto path_to_self = LexicalPath(MUST(Core::System::current_executable_path())).parent();
+    return LexicalPath::join(path_to_self.string(), compiler_binary_name);
+}();
+static const LexicalPath path_to_tests_directory { relative_path_to_test };
+
+Vector<DeprecatedString> build_command_line_arguments(LexicalPath const& test_source, TestDescription const& description)
+{
+    Vector<DeprecatedString> result;
+
+    StringBuilder dump_ast_option;
+
+    for (auto const& flag : description.flags) {
+        if (flag.dump_ast) {
+            if (!dump_ast_option.is_empty())
+                dump_ast_option.append(","sv);
+            dump_ast_option.append(flag.name);
+        }
+    }
+    if (!dump_ast_option.is_empty())
+        result.append(DeprecatedString::formatted("--dump-ast={}", dump_ast_option.string_view()));
+
+    if (test_source.has_extension(".cpp"sv))
+        result.append("-xc++"sv);
+
+    result.append(test_source.string());
+
+    return result;
+}
+
+ErrorOr<ByteBuffer> read(LexicalPath const& path)
+{
+    auto file = TRY(Core::File::open(path.string(), Core::File::OpenMode::Read));
+    return MUST(file->read_until_eof());
+}
+
+void check_expectations(LexicalPath const& path_to_expectation, LexicalPath const& path_to_captured_output, bool should_update_expectations)
+{
+    struct PathPair {
+        LexicalPath expectation;
+        LexicalPath result;
+    };
+    Vector<PathPair> file_pairs_to_check;
+
+    file_pairs_to_check.append({
+        .expectation = path_to_expectation,
+        .result = path_to_captured_output,
+    });
+
+    auto out = MUST(Core::File::standard_error());
+
+    for (auto const& [expectation_path, result_path] : file_pairs_to_check) {
+        auto result_content = read(result_path);
+
+        if (should_update_expectations && !result_content.is_error()) {
+            using namespace FileSystem;
+            MUST(copy_file_or_directory(expectation_path.string(), result_path.string(),
+                RecursionMode::Disallowed, LinkMode::Disallowed, AddDuplicateFileMarker::No));
+        }
+
+        auto expectation = read(expectation_path);
+
+        bool read_successfully = !(expectation.is_error() || result_content.is_error());
+        EXPECT(read_successfully);
+
+        if (read_successfully) {
+            bool are_equal = expectation.value() == result_content.value();
+            EXPECT(are_equal);
+
+            if (!are_equal) {
+                dbgln("Files {} and {} do not match!", expectation_path.string(), result_path.string());
+                auto maybe_diff = Diff::from_text(expectation.value(), result_content.value());
+                if (!maybe_diff.is_error()) {
+                    for (auto const& hunk : maybe_diff.value())
+                        MUST(Diff::write_unified(hunk, *out, Diff::ColorOutput::Yes));
+                }
+            }
+        }
+    }
+}
+
+TEST_CASE(test_regression)
+{
+    auto* update_expectations_env = getenv("JSSC_UPDATE_EXPECTATIONS");
+
+    bool should_update_expectations = false;
+    if (update_expectations_env != nullptr && strcmp(update_expectations_env, "1") == 0)
+        should_update_expectations = true;
+
+    auto temp_directory = MUST(FileSystem::TempFile::create_temp_directory());
+
+    auto path_to_captured_stderr = LexicalPath::join(temp_directory->path(), stderr_capture_filename);
+
+    for (auto const& test_description : regression_tests) {
+        for (auto const& source : test_description.sources) {
+            dbgln("Running {}...", source);
+
+            auto path_to_test = LexicalPath::join(path_to_tests_directory.string(), source);
+            auto path_to_expectation = LexicalPath::join(path_to_tests_directory.string(), DeprecatedString::formatted("{}.expectation", source));
+
+            auto process = MUST(Core::Process::spawn({
+                .path = path_to_compiler_binary.string(),
+                .arguments = build_command_line_arguments(path_to_test, test_description),
+                .file_actions = {
+                    Core::FileAction::OpenFile {
+                        .path = path_to_captured_stderr.string(),
+                        .mode = Core::File::OpenMode::Write,
+                        .fd = STDERR_FILENO,
+                    },
+                },
+            }));
+
+            bool exited_with_code_0 = MUST(process.wait_for_termination());
+            EXPECT(exited_with_code_0);
+
+            if (exited_with_code_0)
+                check_expectations(path_to_expectation, path_to_captured_stderr, should_update_expectations);
+        }
+    }
+}