beenull/ladybird
1
0
Fork 0
mirror of https://github.com/LadybirdBrowser/ladybird.git synced 2024-11-21 23:20:20 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 6

Meta: Add a script to run the Web Platform tests locally

Browse source 
This change adds a script that can run the Web Platform Tests and
compare different test runs.
This commit is contained in:
Tim Ledbetter 2024-08-12 15:33:28 +01:00 committed by Andreas Kling
parent e0c8a14fab
commit c9caa4262e
Notes: github-actions[bot] 2024-08-13 12:11:45 +00:00 
Author: https://github.com/tcl3
Commit: https://github.com/LadybirdBrowser/ladybird/commit/c9caa4262e6
Pull-request: https://github.com/LadybirdBrowser/ladybird/pull/1051
2 changed files with 176 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

21
Documentation/RunningTests.md
View file

@ -67,3 +67,24 @@ cd Build/lagom
ninja
CTEST_OUTPUT_ON_FAILURE=1 LADYBIRD_SOURCE_DIR=${PWD}/../.. ninja test
```
# Running the Web Platform Tests
The Web Platform Tests can be run with the `WPT.sh` script. This script can also be used to compare the results of two
test runs.
Example usage:
```sh
# Run the WPT tests then run them again, comparing the results from the two runs
./Meta/WPT.sh run --log expectations.log css
git checkout my-css-change
./Meta/WPT.sh compare --log results.log expectations.log css
```
```sh
# Pull the latest changes from the upstream WPT repository
./Meta/WPT.sh update
# Run all of the Web Platform Tests, outputting the results to results.log
./Meta/WPT.sh run --log results.log
```

155
Meta/WPT.sh Executable file
View file

@ -0,0 +1,155 @@
#!/usr/bin/env bash
set -e
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
LADYBIRD_SOURCE_DIR="$(realpath "${DIR}"/..)"
WPT_SOURCE_DIR=${WPT_SOURCE_DIR:-"${LADYBIRD_SOURCE_DIR}/Tests/LibWeb/WPT/wpt"}
WPT_REPOSITORY_URL=${WPT_REPOSITORY_URL:-"https://github.com/web-platform-tests/wpt.git"}
LADYBIRD_BINARY=${LADYBIRD_BINARY:-"${LADYBIRD_SOURCE_DIR}/Build/ladybird/bin/Ladybird"}
WEBDRIVER_BINARY=${WEBDRIVER_BINARY:-"${LADYBIRD_SOURCE_DIR}/Build/ladybird/bin/WebDriver"}
WPT_PROCESSES=${WPT_PROCESSES:-$(nproc)}
WPT_CERTIFICATES=(
"tools/certs/cacert.pem"
"${LADYBIRD_SOURCE_DIR}/Build/ladybird/Lagom/cacert.pem"
)
WPT_ARGS=( "--binary=${LADYBIRD_BINARY}"
"--webdriver-binary=${WEBDRIVER_BINARY}"
"--install-webdriver"
"--processes=${WPT_PROCESSES}"
"--webdriver-arg=--force-cpu-painting"
"--no-pause-after-test"
"-f"
"${EXTRA_WPT_ARGS[@]}"
)
ARG0=$0
print_help() {
NAME=$(basename "$ARG0")
cat <<EOF
Usage: $NAME COMMAND [OPTIONS..] [TESTS...]
Supported COMMANDs:
update: Update the Web Platform Tests repository.
run: $NAME run [OPTIONS...] [TESTS...]
Run the Web Platform Tests.
compare: $NAME compare [OPTIONS...] LOG_FILE [TESTS...]
Run the Web Platform Tests comparing the results to the expectations in LOG_FILE.
Examples:
$NAME update
Updates the Web Platform Tests repository.
$NAME run
Run all of the Web Platform Tests.
$NAME run --log expectations.log css dom
Run the Web Platform Tests in the 'css' and 'dom' directories and save the output to expectations.log.
$NAME compare expectations.log
Run all of the Web Platform Tests comparing the results to the expectations in before.log.
$NAME compare --log results.log expectations.log css/CSS2
Run the Web Platform Tests in the 'css/CSS2' directory, comparing the results to the expectations in expectations.log; output the results to results.log.
EOF
}
usage() {
>&2 print_help
exit 1
}
CMD=$1
[ -n "$CMD" ] || usage
shift
if [ "$CMD" = "--help" ] || [ "$CMD" = "help" ]; then
print_help
exit 0
fi
ARG=$1
if [ "$ARG" = "--log" ]; then
shift
LOG_NAME="$(pwd -P)/$1"
[ -n "$LOG_NAME" ] || usage;
shift
WPT_ARGS+=( "--log-raw=${LOG_NAME}" )
fi
TEST_LIST=( "$@" )
# shellcheck source=/dev/null
. "${DIR}/shell_include.sh"
exit_if_running_as_root "Do not run WPT.sh as root"
ensure_wpt_repository() {
mkdir -p "${WPT_SOURCE_DIR}"
pushd "${WPT_SOURCE_DIR}" > /dev/null
if [ ! -d .git ]; then
git clone --depth 1 "${WPT_REPOSITORY_URL}" "${WPT_SOURCE_DIR}"
fi
popd > /dev/null
}
build_ladybird_and_webdriver() {
"${DIR}"/ladybird.sh build WebDriver
}
update_wpt() {
ensure_wpt_repository
pushd "${WPT_SOURCE_DIR}" > /dev/null
git pull
popd > /dev/null
}
execute_wpt() {
pushd "${WPT_SOURCE_DIR}" > /dev/null
for certificate_path in "${WPT_CERTIFICATES[@]}"; do
if [ ! -f "${certificate_path}" ]; then
echo "Certificate not found: \"${certificate_path}\""
exit 1
fi
WPT_ARGS+=( "--webdriver-arg=--certificate=${certificate_path}" )
done
QT_QPA_PLATFORM="minimal" ./wpt run "${WPT_ARGS[@]}" ladybird "${TEST_LIST[@]}"
popd > /dev/null
}
run_wpt() {
ensure_wpt_repository
build_ladybird_and_webdriver
execute_wpt
}
compare_wpt() {
ensure_wpt_repository
METADATA_DIR=$(mktemp -d)
pushd "${WPT_SOURCE_DIR}" > /dev/null
./wpt update-expectations --product ladybird --full --metadata="${METADATA_DIR}" "${INPUT_LOG_NAME}"
popd > /dev/null
WPT_ARGS+=( "--metadata=${METADATA_DIR}" )
build_ladybird_and_webdriver
execute_wpt
rm -rf "${METADATA_DIR}"
}
if [[ "$CMD" =~ ^(update|run|compare)$ ]]; then
case "$CMD" in
update)
update_wpt
;;
run)
run_wpt
;;
compare)
INPUT_LOG_NAME="$(pwd -P)/$1"
if [ ! -f "$INPUT_LOG_NAME" ]; then
echo "Log file not found: \"${INPUT_LOG_NAME}\""
usage;
fi
shift
compare_wpt
;;
esac
else
>&2 echo "Unknown command: $CMD"
usage
fi