Source code
Revision control
Copy as Markdown
Other Tools
# mypy: allow-untyped-defs
import argparse
import errno
import logging
import os
import shutil
import socket
import subprocess
import sys
import time
from unittest import mock
from urllib.request import urlopen
from urllib.error import URLError
import pytest
from tools.wpt import browser, utils, wpt
here = os.path.abspath(os.path.dirname(__file__))
def is_port_8000_in_use():
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
s.bind(("127.0.0.1", 8000))
except OSError as e:
if e.errno == errno.EADDRINUSE:
return True
else:
raise e
finally:
s.close()
return False
def get_persistent_manifest_path():
directory = ("~/meta" if os.environ.get('TRAVIS') == "true"
else wpt.localpaths.repo_root)
return os.path.join(directory, "MANIFEST.json")
@pytest.fixture(scope="module", autouse=True)
def init_manifest():
with pytest.raises(SystemExit) as excinfo:
wpt.main(argv=["manifest", "--no-download",
"--path", get_persistent_manifest_path()])
assert excinfo.value.code == 0
@pytest.fixture
def manifest_dir(tmp_path):
shutil.copyfile(get_persistent_manifest_path(),
os.path.join(tmp_path, "MANIFEST.json"))
return str(tmp_path)
@pytest.fixture(scope="module")
def firefox_binary_path(tmp_path_factory):
try:
logger = logging.getLogger("firefox_binary_path")
path = str(tmp_path_factory.mktemp("firefox_binary"))
firefox = browser.Firefox(logger)
bin_path = firefox.install(dest=path)
assert os.path.exists(bin_path) and os.path.isfile(bin_path)
yield bin_path
finally:
utils.rmtree(path)
@pytest.fixture
def temp_test():
os.makedirs("../../.tools-tests")
test_count = {"value": 0}
def make_test(body):
test_count["value"] += 1
test_name = ".tools-tests/%s.html" % test_count["value"]
test_path = "../../%s" % test_name
with open(test_path, "w") as handle:
handle.write("""
<!DOCTYPE html>
<script src="/resources/testharness.js"></script>
<script src="/resources/testharnessreport.js"></script>
<script>%s</script>
""" % body)
return test_name
yield make_test
utils.rmtree("../../.tools-tests")
def test_missing():
with pytest.raises(SystemExit):
wpt.main(argv=["#missing-command"])
def test_help():
# TODO: It seems like there's a bug in argparse that makes this argument order required
# should try to work around that
with pytest.raises(SystemExit) as excinfo:
wpt.main(argv=["--help"])
assert excinfo.value.code == 0
def test_load_commands():
commands = wpt.load_commands()
# The `wpt run` command has conditional requirements.
assert "conditional_requirements" in commands["run"]
def test_dispatcher_uncaught_exception_exits_70(monkeypatch, capsys):
def boom(*args, **kwargs):
raise RuntimeError("boom")
def fake_import_command(prog, command, props):
return boom, argparse.ArgumentParser()
monkeypatch.setattr(wpt, "import_command", fake_import_command)
with pytest.raises(SystemExit) as excinfo:
wpt.main(argv=["files-changed"])
assert excinfo.value.code == getattr(os, "EX_SOFTWARE", 70)
err = capsys.readouterr().err
assert "Traceback (most recent call last):" in err
assert "RuntimeError: boom" in err
def test_dispatcher_uncaught_exception_debug_invokes_pdb(monkeypatch):
def boom(*args, **kwargs):
raise RuntimeError("boom")
def fake_import_command(prog, command, props):
return boom, argparse.ArgumentParser()
monkeypatch.setattr(wpt, "import_command", fake_import_command)
with mock.patch("pdb.post_mortem", autospec=True, spec_set=True) as mock_pm:
with pytest.raises(SystemExit) as excinfo:
wpt.main(argv=["--debug", "files-changed"])
# Falls through to the trailing sys.exit(0) after pdb.post_mortem returns.
assert excinfo.value.code == 0
mock_pm.assert_called_once_with()
@pytest.mark.slow
@pytest.mark.skipif(sys.platform == "win32",
def test_list_tests(manifest_dir, firefox_binary_path):
"""The `--list-tests` option should not produce an error under normal
conditions."""
with pytest.raises(SystemExit) as excinfo:
wpt.main(argv=["run", "--metadata", manifest_dir, "--list-tests",
"--yes",
# WebTransport server is not needed (web-platform-tests/wpt#41675).
"--no-enable-webtransport-h3",
"--binary", firefox_binary_path,
"firefox", "/dom/nodes/Element-tagName.html"])
assert excinfo.value.code == 0
@pytest.mark.slow
@pytest.mark.remote_network
def test_list_tests_missing_manifest(manifest_dir, firefox_binary_path):
"""The `--list-tests` option should not produce an error in the absence of
a test manifest file."""
os.remove(os.path.join(manifest_dir, "MANIFEST.json"))
with pytest.raises(SystemExit) as excinfo:
wpt.main(argv=["run",
# This test triggers the creation of a new manifest
# file which is not necessary to ensure successful
# process completion. Specifying the current directory
# as the tests source via the --tests` option
# drastically reduces the time to execute the test.
"--tests", here,
"--metadata", manifest_dir,
"--log-mach", "-",
"--list-tests",
"--yes",
# WebTransport server is not needed (web-platform-tests/wpt#41675).
"--no-enable-webtransport-h3",
"--binary", firefox_binary_path,
"firefox", "/dom/nodes/Element-tagName.html"])
assert excinfo.value.code == 0
@pytest.mark.slow
@pytest.mark.remote_network
def test_list_tests_invalid_manifest(manifest_dir, firefox_binary_path):
"""The `--list-tests` option should not produce an error in the presence of
a malformed test manifest file."""
manifest_filename = os.path.join(manifest_dir, "MANIFEST.json")
assert os.path.isfile(manifest_filename)
with open(manifest_filename, "a+") as handle:
handle.write("extra text which invalidates the file")
with pytest.raises(SystemExit) as excinfo:
wpt.main(argv=["run",
# This test triggers the creation of a new manifest
# file which is not necessary to ensure successful
# process completion. Specifying the current directory
# as the tests source via the --tests` option
# drastically reduces the time to execute the test.
"--tests", here,
"--metadata", manifest_dir,
"--log-mach", "-",
"--list-tests",
"--yes",
# WebTransport server is not needed (web-platform-tests/wpt#41675).
"--no-enable-webtransport-h3",
"--binary", firefox_binary_path,
"firefox", "/dom/nodes/Element-tagName.html"])
assert excinfo.value.code == 0
@pytest.mark.slow
@pytest.mark.remote_network
@pytest.mark.skipif(sys.platform == "win32",
def test_run_zero_tests(firefox_binary_path):
"""A test execution describing zero tests should be reported as an error
even in the presence of the `--no-fail-on-unexpected` option."""
if is_port_8000_in_use():
pytest.skip("port 8000 already in use")
with pytest.raises(SystemExit) as excinfo:
wpt.main(argv=["run", "--yes", "--no-pause",
# WebTransport server is not needed (web-platform-tests/wpt#41675).
"--no-enable-webtransport-h3",
"--binary", firefox_binary_path,
"firefox", "/non-existent-dir/non-existent-file.html"])
assert excinfo.value.code != 0
with pytest.raises(SystemExit) as excinfo:
wpt.main(argv=["run", "--yes", "--no-pause", "--no-fail-on-unexpected",
# WebTransport server is not needed (web-platform-tests/wpt#41675).
"--no-enable-webtransport-h3",
"--binary", firefox_binary_path,
"firefox", "/non-existent-dir/non-existent-file.html"])
assert excinfo.value.code != 0
@pytest.mark.slow
@pytest.mark.remote_network
@pytest.mark.skipif(sys.platform == "win32",
def test_run_failing_test(firefox_binary_path):
"""Failing tests should be reported with a non-zero exit status unless the
`--no-fail-on-unexpected` option has been specified."""
if is_port_8000_in_use():
pytest.skip("port 8000 already in use")
failing_test = "/infrastructure/expected-fail/failing-test.html"
assert os.path.isfile("../../%s" % failing_test)
with pytest.raises(SystemExit) as excinfo:
wpt.main(argv=["run", "--yes", "--no-pause",
# WebTransport server is not needed (web-platform-tests/wpt#41675).
"--no-enable-webtransport-h3",
"--binary", firefox_binary_path,
"firefox", failing_test])
assert excinfo.value.code != 0
with pytest.raises(SystemExit) as excinfo:
wpt.main(argv=["run", "--yes", "--no-pause", "--no-fail-on-unexpected",
"--channel", "dev",
# WebTransport server is not needed (web-platform-tests/wpt#41675).
"--no-enable-webtransport-h3",
"--binary", firefox_binary_path,
"firefox", failing_test])
assert excinfo.value.code == 0
@pytest.mark.slow
@pytest.mark.remote_network
@pytest.mark.skipif(sys.platform == "win32",
def test_run_verify_unstable(temp_test, firefox_binary_path):
"""Unstable tests should be reported with a non-zero exit status. Stable
tests should be reported with a zero exit status."""
if is_port_8000_in_use():
pytest.skip("port 8000 already in use")
unstable_test = temp_test("""
test(function() {
if (localStorage.getItem('wpt-unstable-test-flag')) {
throw new Error();
}
localStorage.setItem('wpt-unstable-test-flag', 'x');
}, 'my test');
""")
with pytest.raises(SystemExit) as excinfo:
wpt.main(argv=["run", "--yes", "--verify",
# WebTransport server is not needed (web-platform-tests/wpt#41675).
"--no-enable-webtransport-h3",
"--binary", firefox_binary_path,
"firefox", unstable_test])
assert excinfo.value.code != 0
stable_test = temp_test("test(function() {}, 'my test');")
with pytest.raises(SystemExit) as excinfo:
wpt.main(argv=["run", "--yes", "--verify",
# WebTransport server is not needed (web-platform-tests/wpt#41675).
"--no-enable-webtransport-h3",
"--binary", firefox_binary_path,
"firefox", stable_test])
assert excinfo.value.code == 0
def test_files_changed(capsys):
commit = "9047ac1d9f51b1e9faa4f9fad9c47d109609ab09"
with pytest.raises(SystemExit) as excinfo:
wpt.main(argv=["files-changed", f"{commit}~..{commit}"])
assert excinfo.value.code == 0
out, err = capsys.readouterr()
expected = """html/browsers/offline/appcache/workers/appcache-worker.html
html/browsers/offline/appcache/workers/resources/appcache-dedicated-worker-not-in-cache.js
html/browsers/offline/appcache/workers/resources/appcache-shared-worker-not-in-cache.js
html/browsers/offline/appcache/workers/resources/appcache-worker-data.py
html/browsers/offline/appcache/workers/resources/appcache-worker-import.py
html/browsers/offline/appcache/workers/resources/appcache-worker.manifest
html/browsers/offline/appcache/workers/resources/appcache-worker.py
""".replace("/", os.path.sep)
assert out == expected
assert err == ""
def test_files_changed_null(capsys):
commit = "9047ac1d9f51b1e9faa4f9fad9c47d109609ab09"
with pytest.raises(SystemExit) as excinfo:
wpt.main(argv=["files-changed", "--null", f"{commit}~..{commit}"])
assert excinfo.value.code == 0
out, err = capsys.readouterr()
expected = "\0".join(["html/browsers/offline/appcache/workers/appcache-worker.html",
"html/browsers/offline/appcache/workers/resources/appcache-dedicated-worker-not-in-cache.js",
"html/browsers/offline/appcache/workers/resources/appcache-shared-worker-not-in-cache.js",
"html/browsers/offline/appcache/workers/resources/appcache-worker-data.py",
"html/browsers/offline/appcache/workers/resources/appcache-worker-import.py",
"html/browsers/offline/appcache/workers/resources/appcache-worker.manifest",
"html/browsers/offline/appcache/workers/resources/appcache-worker.py",
""]).replace("/", os.path.sep)
assert out == expected
assert err == ""
def test_files_changed_ignore():
from tools.wpt.testfiles import exclude_ignored
files = ["resources/testharness.js", "resources/webidl2/index.js", "test/test.js"]
changed, ignored = exclude_ignored(files, ignore_rules=["resources/testharness*"])
assert changed == [os.path.join(wpt.wpt_root, item) for item in
["resources/webidl2/index.js", "test/test.js"]]
assert ignored == [os.path.join(wpt.wpt_root, item) for item in
["resources/testharness.js"]]
def test_files_changed_ignore_rules():
from tools.wpt.testfiles import compile_ignore_rule
assert compile_ignore_rule("foo*bar*/baz").pattern == r"^foo\*bar[^/]*/baz$"
assert compile_ignore_rule("foo**bar**/baz").pattern == r"^foo\*\*bar.*/baz$"
assert compile_ignore_rule("foobar/baz/*").pattern == "^foobar/baz/[^/]*$"
assert compile_ignore_rule("foobar/baz/**").pattern == "^foobar/baz/.*$"
@pytest.mark.slow # this updates the manifest
@pytest.mark.xfail(sys.platform == "win32",
reason="Tests currently don't work on Windows for path reasons")
@pytest.mark.skipif(sys.platform == "win32",
def test_tests_affected(capsys, manifest_dir):
# This doesn't really work properly for random commits because we test the files in
# the current working directory for references to the changed files, not the ones at
# that specific commit. But we can at least test it returns something sensible.
# The test will fail if the file we assert is renamed, so we choose a stable one.
commit = "3a055e818218f548db240c316654f3cc1aeeb733"
with pytest.raises(SystemExit) as excinfo:
wpt.main(argv=["tests-affected", "--metadata", manifest_dir, f"{commit}~..{commit}"])
assert excinfo.value.code == 0
out, err = capsys.readouterr()
assert "infrastructure/reftest-wait.html" in out
@pytest.mark.slow # this updates the manifest
@pytest.mark.xfail(sys.platform == "win32",
reason="Tests currently don't work on Windows for path reasons")
@pytest.mark.skipif(sys.platform == "win32",
def test_tests_affected_idlharness(capsys, manifest_dir):
commit = "47cea8c38b88c0ddd3854e4edec0c5b6f2697e62"
with pytest.raises(SystemExit) as excinfo:
wpt.main(argv=["tests-affected", "--metadata", manifest_dir, f"{commit}~..{commit}"])
assert excinfo.value.code == 0
out, err = capsys.readouterr()
assert ("mst-content-hint/idlharness.window.js\n" +
"webrtc-encoded-transform/idlharness.https.window.js\n" +
"webrtc-identity/idlharness.https.window.js\n" +
"webrtc-stats/idlharness.window.js\n" +
"webrtc-stats/supported-stats.https.html\n" +
"webrtc/idlharness.https.window.js\n") == out
@pytest.mark.slow # this updates the manifest
@pytest.mark.xfail(sys.platform == "win32",
reason="Tests currently don't work on Windows for path reasons")
@pytest.mark.skipif(sys.platform == "win32",
def test_tests_affected_null(capsys, manifest_dir):
# This doesn't really work properly for random commits because we test the files in
# the current working directory for references to the changed files, not the ones at
# that specific commit. But we can at least test it returns something sensible.
# The test will fail if the file we assert is renamed, so we choose a stable one.
commit = "2614e3316f1d3d1a744ed3af088d19516552a5de"
with pytest.raises(SystemExit) as excinfo:
wpt.main(argv=["tests-affected", "--null", "--metadata", manifest_dir, f"{commit}~..{commit}"])
assert excinfo.value.code == 0
out, err = capsys.readouterr()
tests = out.split("\0")
assert "dom/idlharness.any.js" in tests
assert "xhr/idlharness.any.js" in tests
@pytest.mark.slow
@pytest.mark.skipif(sys.platform == "win32",
reason="no os.setsid/killpg to easily cleanup the process tree")
def test_serve():
if is_port_8000_in_use():
pytest.skip("port 8000 already in use")
p = subprocess.Popen([os.path.join(wpt.localpaths.repo_root, "wpt"), "serve"],
preexec_fn=os.setsid)
start = time.time()
try:
while True:
if p.poll() is not None:
assert False, "server not running"
if time.time() - start > 60:
assert False, "server did not start responding within 60s"
try:
print(resp)
except URLError:
print("URLError")
time.sleep(1)
else:
assert resp.code == 200
break
finally:
os.killpg(p.pid, 15)
p.wait(10)
# The following commands are slow running and used implicitly in other CI
# jobs, so we skip them here:
# wpt manifest
# wpt lint