diff --git a/.github/workflows/libjs-test262.yml b/.github/workflows/libjs-test262.yml new file mode 100644 index 00000000000..b5b290c7dab --- /dev/null +++ b/.github/workflows/libjs-test262.yml @@ -0,0 +1,159 @@ +name: Run test262 and test-wasm + +on: [push] + +env: + SERENITY_SOURCE_DIR: ${{ github.workspace }} + +jobs: + run_and_update_results: + runs-on: self-hosted + if: always() && github.repository == 'SerenityOS/serenity' && github.ref == 'refs/heads/master' + + concurrency: libjs-test262 + + steps: + - name: Cleanup + run: | + echo "Cleaning up previous run" + rm -rf "${{ github.workspace }}/*" + + - name: Checkout SerenityOS/serenity + uses: actions/checkout@v3 + + - name: Checkout SerenityOS/libjs-test262 + uses: actions/checkout@v3 + with: + repository: SerenityOS/libjs-test262 + path: libjs-test262 + + - name: Checkout SerenityOS/libjs-test262-data + uses: actions/checkout@v3 + with: + repository: SerenityOS/libjs-test262-data + path: libjs-test262-data + lfs: true + + - name: Checkout SerenityOS/libjs-test262-data LFS objects + working-directory: libjs-test262-data + run: git lfs checkout + + - name: Checkout tc39/test262 + uses: actions/checkout@v3 + with: + repository: tc39/test262 + path: test262 + + - name: Checkout tc39/test262-parser-tests + uses: actions/checkout@v3 + with: + repository: tc39/test262-parser-tests + path: test262-parser-tests + + - name: Install dependencies + run: | + sudo apt-get update + sudo apt-get install -y ninja-build unzip gcc-12 g++-12 jq wget + test -e /opt/wabt-1.0.27 || ( + cd /tmp + wget https://github.com/WebAssembly/wabt/releases/download/1.0.27/wabt-1.0.27-ubuntu.tar.gz + sudo tar xf wabt-1.0.27-ubuntu.tar.gz -C /opt + rm wabt-1.0.27-ubuntu.tar.gz + ) + + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: '3.x' + + - name: Install Python dependencies + # The setup-python action set default python to python3.x. Note that we are not using system python here. + run: | + python -m pip install --upgrade pip + pip install -r libjs-test262/requirements.txt + + - name: Check versions + run: set +e; g++ --version; g++-12 --version; python --version; python3 --version; ninja --version + + - name: TimeZoneData cache + uses: actions/cache@v3 + with: + path: ${{ github.workspace }}/libjs-test262/Build/TZDB + key: TimeZoneData-${{ hashFiles('Meta/CMake/time_zone_data.cmake') }} + + - name: UnicodeData cache + uses: actions/cache@v3 + with: + path: ${{ github.workspace }}/libjs-test262/Build/UCD + key: UnicodeData-${{ hashFiles('Meta/CMake/unicode_data.cmake') }} + + - name: UnicodeLocale cache + uses: actions/cache@v3 + with: + path: ${{ github.workspace }}/libjs-test262/Build/CLDR + key: UnicodeLocale-${{ hashFiles('Meta/CMake/locale_data.cmake') }} + + - name: Get previous results + run: | + mkdir -p old-libjs-test262-data + cp -R libjs-test262-data/test262 libjs-test262-data/wasm old-libjs-test262-data + + - name: Build test262-runner, test-js and test-wasm + working-directory: libjs-test262 + run: | + env PATH="/opt/wabt-1.0.27/bin:$PATH" \ + cmake -GNinja -B Build -DCMAKE_C_COMPILER=gcc-12 -DCMAKE_CXX_COMPILER=g++-12 -DWASM_SPEC_TEST_SKIP_FORMATTING=ON -DINCLUDE_WASM_SPEC_TESTS=ON -DSERENITY_SOURCE_DIR=${{ env.SERENITY_SOURCE_DIR }} + ninja -C Build test262-runner test-js test-wasm + + - name: Run test262 and test262-parser-tests + working-directory: libjs-test262 + run: | + python3 run_all_and_update_results.py \ + --serenity .. \ + --test262 ../test262 \ + --test262-parser-tests ../test262-parser-tests \ + --results-json ../libjs-test262-data/test262/results.json \ + --per-file-output ../libjs-test262-data/test262/per-file-master.json + + - name: Run test-wasm + working-directory: libjs-test262 + run: | + Build/_deps/lagom-build/bin/test-wasm --per-file Build/_deps/lagom-build/Userland/Libraries/LibWasm/Tests > ../libjs-test262-data/wasm/per-file-master.json || true + jq -nc -f /dev/stdin <<-EOF --slurpfile previous ../libjs-test262-data/wasm/results.json --slurpfile details ../libjs-test262-data/wasm/per-file-master.json > wasm-new-results.json + \$details[0] as \$details | \$previous[0] + [{ + "commit_timestamp": $(git -C .. log -1 --format=%ct), + "run_timestamp": $(date +%s), + "versions": { + "serenity": "$(git -C .. rev-parse HEAD)" + }, + "tests": { + "spectest": { + "duration": (\$details.duration), + "results": { + "total": (\$details.results | keys | length), + "passed": ([\$details.results | values[] | select(. == "PASSED")] | length), + "failed": ([\$details.results | values[] | select(. == "FAILED")] | length), + "skipped": ([\$details.results | values[] | select(. == "SKIPPED")] | length), + "process_error": ([\$details.results | values[] | select(. == "PROCESS_ERROR")] | length) + } + } + } + }] + EOF + mv wasm-new-results.json ../libjs-test262-data/wasm/results.json + + - name: Compare test262 results + run: ./libjs-test262/per_file_result_diff.py -o old-libjs-test262-data/test262/per-file-master.json -n libjs-test262-data/test262/per-file-master.json + + - name: Compare Wasm results + run: ./libjs-test262/per_file_result_diff.py -o old-libjs-test262-data/wasm/per-file-master.json -n libjs-test262-data/wasm/per-file-master.json + + - name: Deploy to GitHub + uses: JamesIves/github-pages-deploy-action@v4.4.3 + with: + git-config-name: BuggieBot + git-config-email: buggiebot@serenityos.org + branch: master + repository-name: SerenityOS/libjs-test262-data + token: ${{ secrets.BUGGIEBOT_TOKEN }} + folder: libjs-test262-data