diff --git a/Documentation/RunningTests.md b/Documentation/RunningTests.md index 262733947d1..f28ac8e851d 100644 --- a/Documentation/RunningTests.md +++ b/Documentation/RunningTests.md @@ -67,3 +67,24 @@ cd Build/lagom ninja CTEST_OUTPUT_ON_FAILURE=1 LADYBIRD_SOURCE_DIR=${PWD}/../.. ninja test ``` + +# Running the Web Platform Tests + +The Web Platform Tests can be run with the `WPT.sh` script. This script can also be used to compare the results of two +test runs. + +Example usage: + +```sh +# Run the WPT tests then run them again, comparing the results from the two runs +./Meta/WPT.sh run --log expectations.log css +git checkout my-css-change +./Meta/WPT.sh compare --log results.log expectations.log css +``` + +```sh +# Pull the latest changes from the upstream WPT repository +./Meta/WPT.sh update +# Run all of the Web Platform Tests, outputting the results to results.log +./Meta/WPT.sh run --log results.log +``` diff --git a/Meta/WPT.sh b/Meta/WPT.sh new file mode 100755 index 00000000000..1719cf4f3c7 --- /dev/null +++ b/Meta/WPT.sh @@ -0,0 +1,155 @@ +#!/usr/bin/env bash + +set -e + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +LADYBIRD_SOURCE_DIR="$(realpath "${DIR}"/..)" +WPT_SOURCE_DIR=${WPT_SOURCE_DIR:-"${LADYBIRD_SOURCE_DIR}/Tests/LibWeb/WPT/wpt"} +WPT_REPOSITORY_URL=${WPT_REPOSITORY_URL:-"https://github.com/web-platform-tests/wpt.git"} + +LADYBIRD_BINARY=${LADYBIRD_BINARY:-"${LADYBIRD_SOURCE_DIR}/Build/ladybird/bin/Ladybird"} +WEBDRIVER_BINARY=${WEBDRIVER_BINARY:-"${LADYBIRD_SOURCE_DIR}/Build/ladybird/bin/WebDriver"} + +WPT_PROCESSES=${WPT_PROCESSES:-$(nproc)} +WPT_CERTIFICATES=( + "tools/certs/cacert.pem" + "${LADYBIRD_SOURCE_DIR}/Build/ladybird/Lagom/cacert.pem" +) +WPT_ARGS=( "--binary=${LADYBIRD_BINARY}" + "--webdriver-binary=${WEBDRIVER_BINARY}" + "--install-webdriver" + "--processes=${WPT_PROCESSES}" + "--webdriver-arg=--force-cpu-painting" + "--no-pause-after-test" + "-f" + "${EXTRA_WPT_ARGS[@]}" +) + +ARG0=$0 +print_help() { + NAME=$(basename "$ARG0") + cat <&2 print_help + exit 1 +} + +CMD=$1 +[ -n "$CMD" ] || usage +shift +if [ "$CMD" = "--help" ] || [ "$CMD" = "help" ]; then + print_help + exit 0 +fi + +ARG=$1 +if [ "$ARG" = "--log" ]; then + shift + LOG_NAME="$(pwd -P)/$1" + [ -n "$LOG_NAME" ] || usage; + shift + WPT_ARGS+=( "--log-raw=${LOG_NAME}" ) +fi +TEST_LIST=( "$@" ) + +# shellcheck source=/dev/null +. "${DIR}/shell_include.sh" + +exit_if_running_as_root "Do not run WPT.sh as root" + +ensure_wpt_repository() { + mkdir -p "${WPT_SOURCE_DIR}" + pushd "${WPT_SOURCE_DIR}" > /dev/null + if [ ! -d .git ]; then + git clone --depth 1 "${WPT_REPOSITORY_URL}" "${WPT_SOURCE_DIR}" + fi + popd > /dev/null +} + +build_ladybird_and_webdriver() { + "${DIR}"/ladybird.sh build WebDriver +} + +update_wpt() { + ensure_wpt_repository + pushd "${WPT_SOURCE_DIR}" > /dev/null + git pull + popd > /dev/null +} + +execute_wpt() { + pushd "${WPT_SOURCE_DIR}" > /dev/null + for certificate_path in "${WPT_CERTIFICATES[@]}"; do + if [ ! -f "${certificate_path}" ]; then + echo "Certificate not found: \"${certificate_path}\"" + exit 1 + fi + WPT_ARGS+=( "--webdriver-arg=--certificate=${certificate_path}" ) + done + QT_QPA_PLATFORM="minimal" ./wpt run "${WPT_ARGS[@]}" ladybird "${TEST_LIST[@]}" + popd > /dev/null +} + +run_wpt() { + ensure_wpt_repository + build_ladybird_and_webdriver + execute_wpt +} + +compare_wpt() { + ensure_wpt_repository + METADATA_DIR=$(mktemp -d) + pushd "${WPT_SOURCE_DIR}" > /dev/null + ./wpt update-expectations --product ladybird --full --metadata="${METADATA_DIR}" "${INPUT_LOG_NAME}" + popd > /dev/null + WPT_ARGS+=( "--metadata=${METADATA_DIR}" ) + build_ladybird_and_webdriver + execute_wpt + rm -rf "${METADATA_DIR}" +} + +if [[ "$CMD" =~ ^(update|run|compare)$ ]]; then + case "$CMD" in + update) + update_wpt + ;; + run) + run_wpt + ;; + compare) + INPUT_LOG_NAME="$(pwd -P)/$1" + if [ ! -f "$INPUT_LOG_NAME" ]; then + echo "Log file not found: \"${INPUT_LOG_NAME}\"" + usage; + fi + shift + compare_wpt + ;; + esac +else + >&2 echo "Unknown command: $CMD" + usage +fi