Meta: Run the Wasm spec tests in CI

Since LibWasm is still not capable of passing all of the spec tests,
ignore failing tests, only fail the build if some segfault/abort/etc
occurs.

.github/workflows/cmake.yml

@@ -198,15 +198,15 @@ jobs:
         sudo add-apt-repository "deb http://apt.llvm.org/focal/ llvm-toolchain-focal-12 main"
         sudo apt-get purge -y clang-10 clang-11
         sudo apt-get update
-        sudo apt-get install clang-12 ninja-build
+        sudo apt-get install clang-12 ninja-build binaryen
         sudo update-alternatives --install /usr/bin/clang clang /usr/bin/clang-12 100
         sudo update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-12 100
       if: ${{ runner.os == 'Linux' }}
     - name: Install macOS dependencies
-      run: brew install ninja
+      run: brew install ninja binaryen
       if: ${{ runner.os == 'macOS' }}
     - name: Check versions
-      run: set +e; clang --version; clang++ --version; ninja --version
+      run: set +e; clang --version; clang++ --version; ninja --version; wasm-as --version
 
     # === PREPARE FOR BUILDING ===
 
@@ -227,7 +227,7 @@ jobs:
       run: |
         mkdir -p Build
         cd Build
-        cmake -GNinja -DBUILD_LAGOM=ON -DENABLE_UNDEFINED_SANITIZER=ON -DENABLE_ADDRESS_SANITIZER=ON -DENABLE_PCI_IDS_DOWNLOAD=OFF -DCMAKE_C_COMPILER=gcc-10 -DCMAKE_CXX_COMPILER=g++-10 ..
+        cmake -GNinja -DBUILD_LAGOM=ON -DINCLUDE_WASM_SPEC_TESTS=ON -DWASM_SPEC_TEST_SKIP_FORMATTING=ON -DENABLE_UNDEFINED_SANITIZER=ON -DENABLE_ADDRESS_SANITIZER=ON -DENABLE_PCI_IDS_DOWNLOAD=OFF -DCMAKE_C_COMPILER=gcc-10 -DCMAKE_CXX_COMPILER=g++-10 ..
       if: ${{ matrix.with-fuzzers == 'NO_FUZZ' }}
 
     # === ACTUALLY BUILD AND TEST ===

CMakeLists.txt

@@ -29,7 +29,8 @@ option(ENABLE_COMPILETIME_FORMAT_CHECK "Enable compiletime format string checks"
 option(ENABLE_PCI_IDS_DOWNLOAD "Enable download of the pci.ids database at build time" ON)
 option(BUILD_LAGOM "Build parts of the system targeting the host OS for fuzzing/testing" OFF)
 option(ENABLE_KERNEL_LTO "Build the kernel with link-time optimization" OFF)
-option(INCLUDE_WASM_SPEC_TESTS "Download and include the WebAssembly spec testsuite" OFF)
+
+include(Meta/CMake/wasm_spec_tests.cmake)
 
 add_custom_target(run
     COMMAND ${CMAKE_SOURCE_DIR}/Meta/run.sh
@@ -253,31 +254,3 @@ if(EXISTS ${PCI_IDS_GZ_PATH} AND NOT EXISTS ${PCI_IDS_INSTALL_PATH})
     file(MAKE_DIRECTORY ${CMAKE_INSTALL_DATAROOTDIR})
     file(RENAME ${PCI_IDS_PATH} ${PCI_IDS_INSTALL_PATH})
 endif()
-
-if(INCLUDE_WASM_SPEC_TESTS)
-    set(WASM_SPEC_TEST_GZ_URL https://github.com/WebAssembly/testsuite/archive/refs/heads/master.tar.gz)
-    set(WASM_SPEC_TEST_GZ_PATH ${CMAKE_BINARY_DIR}/wasm-spec-testsuite.tar.gz)
-    set(WASM_SPEC_TEST_TAR_PATH ${CMAKE_BINARY_DIR}/wasm-spec-testsuite.tar)
-    set(WASM_SPEC_TEST_PATH ${CMAKE_SOURCE_DIR}/Userland/Libraries/LibWasm/Tests/Fixtures/SpecTests)
-
-    if(NOT EXISTS ${WASM_SPEC_TEST_GZ_PATH})
-        message(STATUS "Downloading the WebAssembly testsuite from ${WASM_SPEC_TEST_GZ_URL}...")
-        file(DOWNLOAD ${WASM_SPEC_TEST_GZ_URL} ${WASM_SPEC_TEST_GZ_PATH} INACTIVITY_TIMEOUT 10)
-    endif()
-
-    if(EXISTS ${WASM_SPEC_TEST_GZ_PATH} AND NOT EXISTS ${WASM_SPEC_TEST_PATH})
-        message(STATUS "Extracting the WebAssembly testsuite from ${WASM_SPEC_TEST_GZ_PATH}...")
-        file(MAKE_DIRECTORY ${WASM_SPEC_TEST_PATH})
-        execute_process(COMMAND gzip -k -d ${WASM_SPEC_TEST_GZ_PATH})
-        execute_process(COMMAND tar -xf ${WASM_SPEC_TEST_TAR_PATH})
-        execute_process(COMMAND rm ${WASM_SPEC_TEST_TAR_PATH})
-        file(GLOB WASM_TESTS "${CMAKE_BINARY_DIR}/testsuite-master/*.wast")
-        foreach(PATH ${WASM_TESTS})
-            get_filename_component(NAME ${PATH} NAME_WLE)
-            message(STATUS "Generating test cases for WebAssembly test ${NAME}...")
-            execute_process(
-                COMMAND bash ${CMAKE_SOURCE_DIR}/Meta/generate-libwasm-spec-test.sh "${PATH}" "${CMAKE_SOURCE_DIR}/Userland/Libraries/LibWasm/Tests/Spec" "${NAME}" "${WASM_SPEC_TEST_PATH}")
-        endforeach()
-        file(REMOVE testsuite-master)
-    endif()
-endif()

Meta/CMake/wasm_spec_tests.cmake

@@ -0,0 +1,38 @@
+option(INCLUDE_WASM_SPEC_TESTS "Download and include the WebAssembly spec testsuite" OFF)
+
+if(INCLUDE_WASM_SPEC_TESTS)
+    set(SOURCE_DIR ${CMAKE_SOURCE_DIR})
+    if (CMAKE_SOURCE_DIR MATCHES ".*/Lagom")
+        set(SOURCE_DIR ${SOURCE_DIR}/../..)
+    endif()
+    set(WASM_SPEC_TEST_GZ_URL https://github.com/WebAssembly/testsuite/archive/refs/heads/master.tar.gz)
+    set(WASM_SPEC_TEST_GZ_PATH ${CMAKE_BINARY_DIR}/wasm-spec-testsuite.tar.gz)
+    set(WASM_SPEC_TEST_TAR_PATH ${CMAKE_BINARY_DIR}/wasm-spec-testsuite.tar)
+    set(WASM_SPEC_TEST_PATH ${SOURCE_DIR}/Userland/Libraries/LibWasm/Tests/Fixtures/SpecTests)
+
+    if(NOT EXISTS ${WASM_SPEC_TEST_GZ_PATH})
+        message(STATUS "Downloading the WebAssembly testsuite from ${WASM_SPEC_TEST_GZ_URL}...")
+        file(DOWNLOAD ${WASM_SPEC_TEST_GZ_URL} ${WASM_SPEC_TEST_GZ_PATH} INACTIVITY_TIMEOUT 10)
+    endif()
+
+    set(SKIP_PRETTIER false)
+    if (WASM_SPEC_TEST_SKIP_FORMATTING)
+        set(SKIP_PRETTIER true)
+    endif()
+
+    if(EXISTS ${WASM_SPEC_TEST_GZ_PATH} AND NOT EXISTS ${WASM_SPEC_TEST_PATH})
+        message(STATUS "Extracting the WebAssembly testsuite from ${WASM_SPEC_TEST_GZ_PATH}...")
+        file(MAKE_DIRECTORY ${WASM_SPEC_TEST_PATH})
+        execute_process(COMMAND gzip -k -d ${WASM_SPEC_TEST_GZ_PATH})
+        execute_process(COMMAND tar -xf ${WASM_SPEC_TEST_TAR_PATH})
+        execute_process(COMMAND rm ${WASM_SPEC_TEST_TAR_PATH})
+        file(GLOB WASM_TESTS "${CMAKE_BINARY_DIR}/testsuite-master/*.wast")
+        foreach(PATH ${WASM_TESTS})
+            get_filename_component(NAME ${PATH} NAME_WLE)
+            message(STATUS "Generating test cases for WebAssembly test ${NAME}...")
+            execute_process(
+                COMMAND env SKIP_PRETTIER=${SKIP_PRETTIER} bash ${SOURCE_DIR}/Meta/generate-libwasm-spec-test.sh "${PATH}" "${SOURCE_DIR}/Userland/Libraries/LibWasm/Tests/Spec" "${NAME}" "${WASM_SPEC_TEST_PATH}")
+        endforeach()
+        file(REMOVE testsuite-master)
+    endif()
+endif()

Meta/Lagom/CMakeLists.txt

@@ -1,6 +1,8 @@
 cmake_minimum_required (VERSION 3.0)
 project (Lagom)
 
+include(../CMake/wasm_spec_tests.cmake)
+
 if (NOT ENABLE_OSS_FUZZ)
     set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-unknown-warning-option -Wno-literal-suffix -O2 -Wall -Wextra -Werror -std=c++2a -fPIC -g -Wno-deprecated-copy")
 else()
@@ -168,7 +170,8 @@ if (BUILD_LAGOM)
             WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
         )
         set_tests_properties(WasmParser PROPERTIES
-            ENVIRONMENT SERENITY_SOURCE_DIR=${CMAKE_CURRENT_SOURCE_DIR}/../..)
+            ENVIRONMENT SERENITY_SOURCE_DIR=${CMAKE_CURRENT_SOURCE_DIR}/../..
+            SKIP_RETURN_CODE 1)
 
         add_executable(disasm_lagom ../../Userland/Utilities/disasm.cpp)
         set_target_properties(disasm_lagom PROPERTIES OUTPUT_NAME disasm)

Meta/generate-libwasm-spec-test.sh

@@ -13,4 +13,9 @@ MODULE_OUTPUT_PATH="$4"
 mkdir -p "$OUTPUT_PATH"
 mkdir -p "$MODULE_OUTPUT_PATH"
 
-python3 "$(dirname "$0")/generate-libwasm-spec-test.py" "$INPUT_FILE" "$NAME" "$MODULE_OUTPUT_PATH" | prettier --stdin-filepath "test-$NAME.js" > "$OUTPUT_PATH/$NAME.js"
+python3 "$(dirname "$0")/generate-libwasm-spec-test.py" "$INPUT_FILE" "$NAME" "$MODULE_OUTPUT_PATH" |\
+    if $SKIP_PRETTIER; then
+        cat
+    else
+        prettier --stdin-filepath "test-$NAME.js"
+    fi > "$OUTPUT_PATH/$NAME.js"