beenull/ladybird
1
0
Fork 0
mirror of https://github.com/LadybirdBrowser/ladybird.git synced 2024-11-25 00:50:22 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 7

LibWeb: Add a script to run Web Platform Tests

Browse source 
Introduce very initial and basic support for running Web Platform Tests
for Ladybird. This change includes simple bash script that currently
works only on Debian and could run tests with patched runner.

For now script to run WPT is not integrated in CI.

There is also a bunch of metadata required to run WPT. To avoid
introducing thousands of files in the initial commit for now it is
limited to run only css/CSS2/floats tests subdirectory.
This commit is contained in:
Aliaksandr Kalenik 2023-06-01 19:05:23 +03:00 committed by Andreas Kling
parent fb262de7cb
commit a414ddcbf3
Notes: sideshowbarker 2024-07-17 10:31:19 +09:00 
Author: https://github.com/kalenikaliaksandr
Commit: https://github.com/SerenityOS/serenity/commit/a414ddcbf3
Pull-request: https://github.com/SerenityOS/serenity/pull/19207
Reviewed-by: https://github.com/awesomekling
7 changed files with 754523 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
.gitignore vendored
View file

@ -32,3 +32,6 @@ sync-local.sh
Userland/Libraries/LibWasm/Tests/Fixtures/SpecTests
Userland/Libraries/LibWasm/Tests/Spec
Tests/LibWeb/WPT/wpt
Tests/LibWeb/WPT/metadata

754021
Tests/LibWeb/WPT/MANIFEST.json Normal file
View file

File diff suppressed because it is too large Load diff

76
Tests/LibWeb/WPT/concat-extract-metadata.py Executable file
View file

@ -0,0 +1,76 @@
#!/usr/bin/env python3
"""
Usage:
- To concatenate all WPT expectation files in the metadata directory into metadata.tx:
concat-extract-metadata.py --concat metadata > metadata.txt
- To extract expectation files from metadata.txt into the metadata directory:
concat-extract-metadata.py --extract metadata.txt metadata
"""
import argparse
import os
def concat_metadata_files(metadata_directory):
concatenated_metadata = ""
for root, _, files in os.walk(metadata_directory):
for filename in files:
if filename.endswith(".ini"):
filepath = os.path.join(root, filename)
relative_path = os.path.relpath(filepath, metadata_directory)
with open(filepath, "r") as file:
file_content = file.read()
concatenated_metadata += f"--- {relative_path} ---\n"
concatenated_metadata += file_content + "\n"
return concatenated_metadata
def extract_metadata_files(concatenated_metadata_file, output_metadata_directory):
with open(concatenated_metadata_file, "r") as file:
concatenated_metadata = file.read()
lines = concatenated_metadata.splitlines()
current_filename = None
current_content = []
def flush_content_to_file(filename, content, output_directory):
filepath = os.path.join(output_directory, filename)
os.makedirs(os.path.dirname(filepath), exist_ok=True)
with open(filepath, "w") as file:
file.write("\n".join(content))
return filepath
for line in lines:
if line.startswith("---"):
if current_filename:
flush_content_to_file(current_filename, current_content, output_metadata_directory)
current_filename = line[4:-4].strip()
current_content = []
else:
current_content.append(line)
if current_filename:
flush_content_to_file(current_filename, current_content, output_metadata_directory)
parser = argparse.ArgumentParser(description="Concatenate and extract WPT metadata files")
parser.add_argument("--concat", help="Concatenate expectation files from specified directory")
parser.add_argument("--extract", nargs=2, help="Extract expectation files from specified input concatenated file")
args = parser.parse_args()
if args.concat:
metadata_directory_path = args.concat
print(concat_metadata_files(metadata_directory_path))
if args.extract:
input_concatenated_metadata_filename, output_metadata_directory = args.extract
extract_metadata_files(input_concatenated_metadata_filename, output_metadata_directory)

7
Tests/LibWeb/WPT/include.ini Normal file
View file

@ -0,0 +1,7 @@
skip: true
[css]
skip: true
[CSS2]
skip: true
[floats]
skip: false

140
Tests/LibWeb/WPT/ladybird_runner.patch Normal file
View file

@ -0,0 +1,140 @@
diff --git a/tools/wpt/browser.py b/tools/wpt/browser.py
index 5e8bfdab1..779b3b427 100644
--- a/tools/wpt/browser.py
+++ b/tools/wpt/browser.py
@@ -1918,6 +1918,27 @@ class WebKit(Browser):
def version(self, binary=None, webdriver_binary=None):
return None
+class Ladybird(Browser):
+ product = "ladybird"
+ requirements = None
+
+ def download(self, dest=None, channel=None, rename=None):
+ raise NotImplementedError
+
+ def install(self, dest=None, channel=None):
+ raise NotImplementedError
+
+ def find_binary(self, venv_path=None, channel=None):
+ return None
+
+ def find_webdriver(self, venv_path=None, channel=None):
+ return None
+
+ def install_webdriver(self, dest=None, channel=None, browser_binary=None):
+ raise NotImplementedError
+
+ def version(self, binary=None, webdriver_binary=None):
+ return None
class WebKitTestRunner(Browser):
"""Interface for WebKitTestRunner.
diff --git a/tools/wpt/run.py b/tools/wpt/run.py
index 58e0004f9..8eb5399d3 100644
--- a/tools/wpt/run.py
+++ b/tools/wpt/run.py
@@ -110,7 +110,7 @@ otherwise install OpenSSL and ensure that it's on your $PATH.""")
def check_environ(product):
if product not in ("android_weblayer", "android_webview", "chrome",
"chrome_android", "chrome_ios", "content_shell",
- "firefox", "firefox_android", "servo", "wktr"):
+ "firefox", "firefox_android", "servo", "wktr", "ladybird"):
config_builder = serve.build_config(os.path.join(wpt_root, "config.json"))
# Override the ports to avoid looking for free ports
config_builder.ssl = {"type": "none"}
@@ -687,6 +687,15 @@ class WebKit(BrowserSetup):
def setup_kwargs(self, kwargs):
pass
+class Ladybird(BrowserSetup):
+ name = "ladybird"
+ browser_cls = browser.Ladybird
+
+ def install(self, channel=None):
+ raise NotImplementedError
+
+ def setup_kwargs(self, kwargs):
+ pass
class WebKitTestRunner(BrowserSetup):
name = "wktr"
@@ -777,6 +786,7 @@ product_setup = {
"wktr": WebKitTestRunner,
"webkitgtk_minibrowser": WebKitGTKMiniBrowser,
"epiphany": Epiphany,
+ "ladybird": Ladybird
}
diff --git a/tools/wptrunner/wptrunner/browsers/__init__.py b/tools/wptrunner/wptrunner/browsers/__init__.py
index 9724bb957..4d1045769 100644
--- a/tools/wptrunner/wptrunner/browsers/__init__.py
+++ b/tools/wptrunner/wptrunner/browsers/__init__.py
@@ -43,4 +43,5 @@ product_list = ["android_weblayer",
"webkit",
"webkitgtk_minibrowser",
"wktr",
- "epiphany"]
+ "epiphany",
+ "ladybird"]
diff --git a/tools/wptrunner/wptrunner/browsers/ladybird.py b/tools/wptrunner/wptrunner/browsers/ladybird.py
new file mode 100644
index 000000000..cfcf285b4
--- /dev/null
+++ b/tools/wptrunner/wptrunner/browsers/ladybird.py
@@ -0,0 +1,54 @@
+from .base import WebDriverBrowser, require_arg
+from .base import get_timeout_multiplier
+from ..executors import executor_kwargs as base_executor_kwargs
+from ..executors.executorwebdriver import (WebDriverTestharnessExecutor, # noqa: F401
+ WebDriverRefTestExecutor, # noqa: F401
+ WebDriverCrashtestExecutor) # noqa: F401
+from ..executors.base import WdspecExecutor
+
+__wptrunner__ = {
+ "product": "ladybird",
+ "check_args": "check_args",
+ "browser": "LadybirdBrowser",
+ "browser_kwargs": "browser_kwargs",
+ "executor_kwargs": "executor_kwargs",
+ "env_options": "env_options",
+ "env_extras": "env_extras",
+ "timeout_multiplier": "get_timeout_multiplier",
+ "executor": {
+ # "testharness": "WebDriverTestharnessExecutor",
+ "reftest": "WebDriverRefTestExecutor",
+ # "wdspec": "WdspecExecutor",
+ # "crashtest": "WebDriverCrashtestExecutor"
+ }
+}
+
+def check_args(**kwargs):
+ pass
+
+def browser_kwargs(logger, test_type, run_info_data, config, **kwargs):
+ return {}
+
+def executor_kwargs(logger, test_type, test_environment, run_info_data,
+ **kwargs):
+ executor_kwargs = base_executor_kwargs(test_type, test_environment, run_info_data, **kwargs)
+ executor_kwargs["capabilities"] = {}
+ return executor_kwargs
+
+def env_options():
+ return {}
+
+def env_extras(**kwargs):
+ return []
+
+class LadybirdBrowser(WebDriverBrowser):
+ def __init__(self, logger, webdriver_args=None,
+ host="localhost", port=None, base_path="/", env=None, **kwargs):
+ webdriver_bin = "WebDriver"
+
+ super().__init__(logger, "binary???", webdriver_bin, webdriver_args=webdriver_args,
+ host=host, port=port, base_path=base_path, env=env, **kwargs)
+ self.host = "localhost"
+
+ def make_command(self):
+ return [self.webdriver_binary, "--port", str(self.port)] + self.webdriver_args

241
Tests/LibWeb/WPT/metadata.txt Normal file
View file

@ -0,0 +1,241 @@
--- css/CSS2/floats/floats-wrap-bfc-with-margin-003.tentative.html.ini ---
[floats-wrap-bfc-with-margin-003.tentative.html]
expected: FAIL
--- css/CSS2/floats/float-nowrap-2.html.ini ---
[float-nowrap-2.html]
expected: FAIL
--- css/CSS2/floats/float-nowrap-3.html.ini ---
[float-nowrap-3.html]
expected: FAIL
--- css/CSS2/floats/float-root.html.ini ---
[float-root.html]
expected: FAIL
--- css/CSS2/floats/floats-wrap-bfc-outside-001.xht.ini ---
[floats-wrap-bfc-outside-001.xht]
expected: FAIL
--- css/CSS2/floats/floats-rule3-outside-right-001.xht.ini ---
[floats-rule3-outside-right-001.xht]
expected: FAIL
--- css/CSS2/floats/overhanging-float-paint-order.html.ini ---
[overhanging-float-paint-order.html]
expected: FAIL
--- css/CSS2/floats/float-nowrap-8.html.ini ---
[float-nowrap-8.html]
expected: FAIL
--- css/CSS2/floats/float-nowrap-9.html.ini ---
[float-nowrap-9.html]
expected: FAIL
--- css/CSS2/floats/floats-line-wrap-shifted-001.html.ini ---
[floats-line-wrap-shifted-001.html]
expected: FAIL
--- css/CSS2/floats/floats-placement-002.html.ini ---
[floats-placement-002.html]
expected: FAIL
--- css/CSS2/floats/floats-placement-003.html.ini ---
[floats-placement-003.html]
expected: FAIL
--- css/CSS2/floats/new-fc-beside-adjoining-float.html.ini ---
[new-fc-beside-adjoining-float.html]
expected: PASS
--- css/CSS2/floats/floats-placement-008.html.ini ---
[floats-placement-008.html]
expected: FAIL
--- css/CSS2/floats/floats-wrap-top-below-inline-001l.xht.ini ---
[floats-wrap-top-below-inline-001l.xht]
expected: FAIL
--- css/CSS2/floats/new-fc-beside-adjoining-float-2.html.ini ---
[new-fc-beside-adjoining-float-2.html]
expected: PASS
--- css/CSS2/floats/floats-wrap-top-below-inline-001r.xht.ini ---
[floats-wrap-top-below-inline-001r.xht]
expected: FAIL
--- css/CSS2/floats/floats-placement-005.html.ini ---
[floats-placement-005.html]
expected: FAIL
--- css/CSS2/floats/floats-placement-004.html.ini ---
[floats-placement-004.html]
expected: FAIL
--- css/CSS2/floats/remove-block-between-inline-and-float.html.ini ---
[remove-block-between-inline-and-float.html]
expected: FAIL
--- css/CSS2/floats/zero-width-floats.html.ini ---
[zero-width-floats.html]
expected: FAIL
--- css/CSS2/floats/negative-margin-float-positioning.html.ini ---
[negative-margin-float-positioning.html]
expected: FAIL
--- css/CSS2/floats/intrinsic-size-float-and-line.html.ini ---
[intrinsic-size-float-and-line.html]
expected: PASS
--- css/CSS2/floats/floats-rule3-outside-right-002.xht.ini ---
[floats-rule3-outside-right-002.xht]
expected: FAIL
--- css/CSS2/floats/float-nowrap-5.html.ini ---
[float-nowrap-5.html]
expected: FAIL
--- css/CSS2/floats/float-nowrap-4.html.ini ---
[float-nowrap-4.html]
expected: FAIL
--- css/CSS2/floats/floats-wrap-bfc-with-margin-005.html.ini ---
[floats-wrap-bfc-with-margin-005.html]
expected: FAIL
--- css/CSS2/floats/floats-wrap-bfc-with-margin-004.html.ini ---
[floats-wrap-bfc-with-margin-004.html]
expected: FAIL
--- css/CSS2/floats/overflow-scroll-float-paint-order.html.ini ---
[overflow-scroll-float-paint-order.html]
expected: FAIL
--- css/CSS2/floats/floats-rule3-outside-left-001.xht.ini ---
[floats-rule3-outside-left-001.xht]
expected: FAIL
--- css/CSS2/floats/floats-rule7-outside-right-001.xht.ini ---
[floats-rule7-outside-right-001.xht]
expected: FAIL
--- css/CSS2/floats/floats-rule7-outside-left-001.xht.ini ---
[floats-rule7-outside-left-001.xht]
expected: FAIL
--- css/CSS2/floats/zero-width-floats-positioning.tentative.html.ini ---
[zero-width-floats-positioning.tentative.html]
expected: FAIL
--- css/CSS2/floats/floats-placement-001.html.ini ---
[floats-placement-001.html]
expected: FAIL
--- css/CSS2/floats/floats-wrap-bfc-with-margin-001a.tentative.html.ini ---
[floats-wrap-bfc-with-margin-001a.tentative.html]
expected: FAIL
--- css/CSS2/floats/floated-table-wider-than-specified.html.ini ---
[floated-table-wider-than-specified.html]
expected: FAIL
--- css/CSS2/floats/new-fc-separates-from-float-2.html.ini ---
[new-fc-separates-from-float-2.html]
expected: FAIL
--- css/CSS2/floats/floats-in-table-caption-001.html.ini ---
[floats-in-table-caption-001.html]
expected: FAIL
--- css/CSS2/floats/new-fc-separates-from-float-3.html.ini ---
[new-fc-separates-from-float-3.html]
expected: FAIL
--- css/CSS2/floats/floats-wrap-top-below-bfc-001l.xht.ini ---
[floats-wrap-top-below-bfc-001l.xht]
expected: FAIL
--- css/CSS2/floats/float-under-flatten-under-preserve-3d.html.ini ---
[float-under-flatten-under-preserve-3d.html]
expected: FAIL
--- css/CSS2/floats/floats-zero-height-wrap-002.xht.ini ---
[floats-zero-height-wrap-002.xht]
expected: FAIL
--- css/CSS2/floats/new-fc-beside-float-with-margin.html.ini ---
[new-fc-beside-float-with-margin.html]
expected: FAIL
--- css/CSS2/floats/floats-wrap-bfc-with-margin-001.html.ini ---
[floats-wrap-bfc-with-margin-001.html]
expected: FAIL
--- css/CSS2/floats/new-fc-separates-from-float.html.ini ---
[new-fc-separates-from-float.html]
expected: FAIL
--- css/CSS2/floats/float-paint-relayout.html.ini ---
[float-paint-relayout.html]
expected: FAIL
--- css/CSS2/floats/float-no-content-beside-001.html.ini ---
[float-no-content-beside-001.html]
expected: FAIL
--- css/CSS2/floats/new-fc-beside-float-with-margin-rtl.html.ini ---
[new-fc-beside-float-with-margin-rtl.html]
expected: FAIL
--- css/CSS2/floats/floats-wrap-top-below-bfc-001r.xht.ini ---
[floats-wrap-top-below-bfc-001r.xht]
expected: FAIL
--- css/CSS2/floats/floats-rule3-outside-left-002.xht.ini ---
[floats-rule3-outside-left-002.xht]
expected: FAIL
--- css/CSS2/floats/float-nowrap-6.html.ini ---
[float-nowrap-6.html]
expected: FAIL
--- css/CSS2/floats/float-nowrap-7.html.ini ---
[float-nowrap-7.html]
expected: FAIL
--- css/CSS2/floats/floats-wrap-bfc-with-margin-002.tentative.html.ini ---
[floats-wrap-bfc-with-margin-002.tentative.html]
expected: FAIL
--- css/CSS2/floats/float-nowrap-3-ref.html.ini ---
[float-nowrap-3-ref.html]
expected: FAIL
--- css/CSS2/floats/adjoining-floats-dynamic.html.ini ---
[adjoining-floats-dynamic.html]
expected: FAIL
--- css/CSS2/floats/floats-placement-006.html.ini ---
[floats-placement-006.html]
expected: FAIL
--- css/CSS2/floats/floats-placement-007.html.ini ---
[floats-placement-007.html]
expected: FAIL
--- css/CSS2/floats/zero-available-space-float-positioning.html.ini ---
[zero-available-space-float-positioning.html]
expected: FAIL
--- css/CSS2/floats/floats-zero-height-wrap-001.xht.ini ---
[floats-zero-height-wrap-001.xht]
expected: FAIL
--- css/CSS2/floats/float-table-align-left-quirk.html.ini ---
[float-table-align-left-quirk.html]
expected: FAIL

35
Tests/LibWeb/WPT/run.sh Executable file
View file

@ -0,0 +1,35 @@
#!/usr/bin/env bash
set -eo pipefail
if [ -z "$SERENITY_SOURCE_DIR" ]
then
echo "SERENITY_SOURCE_DIR is not set. Exiting."
fi
# NOTE: WPT runner assumes Ladybird, WebContent and WebDriver are available in $PATH.
export PATH="${SERENITY_SOURCE_DIR}/Build/lagom/bin:${SERENITY_SOURCE_DIR}/Meta/Lagom/Build/bin:${PATH}"
# Install dependencies.
sudo apt-get install -y git python3 python3-pip python3-venv libssl-dev
# Ensure a `python` binary exists
sudo apt-get install -y python-is-python3
# Clone patched web-platform-tests repository
git clone --depth 10000 https://github.com/web-platform-tests/wpt.git
# Switch to the commit that was used to generate tests expectations. Requires periodic updates.
(cd wpt; git checkout 4c27189ed2db4ddad8e727d4ea9ae8329c3e1672)
# Apply WPT patch with Ladybird runner
(cd wpt; git apply ../ladybird_runner.patch)
# Update hosts file
./wpt/wpt make-hosts-file | sudo tee -a /etc/hosts
# Extract metadata.txt into directory with expectation files expected by WPT runner
./concat-extract-metadata.py --extract metadata.txt metadata
# Run tests.
./wpt/wpt run ladybird --include-manifest include.ini --metadata ./metadata --manifest ./MANIFEST.json