Source code
Revision control
Copy as Markdown
Other Tools
import json
import random
from unittest import mock
import pytest
import requests
import mozperftest.test.webpagetest as webpagetest
from mozperftest.test.webpagetest import (
ACCEPTED_STATISTICS,
WPT_API_EXPIRED_MESSAGE,
WPTBrowserSelectionError,
WPTDataProcessingError,
WPTExpiredAPIKeyError,
WPTInvalidConnectionSelection,
WPTInvalidStatisticsError,
WPTInvalidURLError,
WPTLocationSelectionError,
WPTTimeOutError,
)
from mozperftest.tests.support import EXAMPLE_WPT_TEST, get_running_env
WPT_METRICS = [
"firstContentfulPaint",
"timeToContentfulPaint",
"visualComplete90",
"firstPaint",
"visualComplete99",
"visualComplete",
"SpeedIndex",
"bytesIn",
"bytesOut",
"TTFB",
"fullyLoadedCPUms",
"fullyLoadedCPUpct",
"domElements",
"domContentLoadedEventStart",
"domContentLoadedEventEnd",
"loadEventStart",
"loadEventEnd",
]
class WPTTests:
def __init__(self, log):
self.log = log
def runTests(self, args):
return True
def running_env(**kw):
return get_running_env(flavor="webpagetest", **kw)
def init_placeholder_wpt_data(fvonly=False, invalid_results=False):
views = {"firstView": {}}
if not fvonly:
views["repeatView"] = {}
placeholder_data = {
"data": {
"summary": "websitelink.com",
"location": "ec2-us-east-1:Firefox",
"testRuns": 3,
"successfulFVRuns": 3,
"successfulRVRuns": 3,
"fvonly": fvonly,
"average": views,
"standardDeviation": views,
"median": views,
"runs": {"1": {"firstView": {"browserVersion": 101.1}}},
"url": "testurl.ca",
},
"webPagetestVersion": 21.0,
}
exclude_metrics = 0 if not invalid_results else 2
for metric in WPT_METRICS[exclude_metrics:]:
for view in views:
for stat in ACCEPTED_STATISTICS:
placeholder_data["data"][stat][view][metric] = random.randint(0, 10000)
placeholder_data["data"][stat][view][metric] = random.randint(0, 10000)
placeholder_data["data"][stat][view][metric] = random.randint(0, 10000)
return placeholder_data
def init_mocked_request(
status_code, WPT_test_status_code=200, WPT_test_status_text="Ok", **kwargs
):
mock_data = {
"data": {
"ec2-us-east-1": {"PendingTests": {"Queued": 3}, "Label": "California"},
"jsonUrl": "mock_test.com",
"summary": "Just a pageload test",
"url": "testurl.ca",
"remaining": 2000,
},
"statusCode": WPT_test_status_code,
"statusText": WPT_test_status_text,
}
for key, value in kwargs.items():
mock_data["data"][key] = value
mock_request = requests.Response()
mock_request.status_code = status_code
mock_request._content = json.dumps(mock_data).encode("utf-8")
return mock_request
@mock.patch(
"mozperftest.test.webpagetest.WebPageTest.get_WPT_results",
return_value=init_placeholder_wpt_data(),
)
@mock.patch("mozperftest.utils.get_tc_secret", return_value={"wpt_key": "fake_key"})
@mock.patch(
"mozperftest.test.webpagetest.WebPageTest.location_queue", return_value=None
)
@mock.patch("requests.get", return_value=init_mocked_request(200))
@mock.patch("mozperftest.test.webpagetest.WPT_KEY_FILE", "tests/data/WPT_fakekey.txt")
def test_webpagetest_no_issues_mocked_results(*mocked):
mach_cmd, metadata, env = running_env(tests=[str(EXAMPLE_WPT_TEST)])
test = webpagetest.WebPageTest(env, mach_cmd)
metadata.script["options"]["test_parameters"]["wait_between_requests"] = 1
metadata.script["options"]["test_parameters"]["first_view_only"] = 0
metadata.script["options"]["test_parameters"]["test_list"] = ["google.ca"]
test.run(metadata)
@mock.patch("mozperftest.utils.get_tc_secret", return_value={"wpt_key": "fake_key"})
@mock.patch(
"mozperftest.test.webpagetest.WebPageTest.location_queue", return_value=None
)
@mock.patch("requests.get", return_value=init_mocked_request(200))
@mock.patch("mozperftest.test.webpagetest.WPT_KEY_FILE", "tests/data/WPT_fakekey.txt")
def test_webpagetest_test_invalid_browser(*mocked):
mach_cmd, metadata, env = running_env(tests=[str(EXAMPLE_WPT_TEST)])
metadata.script["options"]["test_parameters"]["browser"] = "Invalid Browser"
test = webpagetest.WebPageTest(env, mach_cmd)
with pytest.raises(WPTBrowserSelectionError):
test.run(metadata)
@mock.patch("mozperftest.utils.get_tc_secret", return_value={"wpt_key": "fake_key"})
@mock.patch(
"mozperftest.test.webpagetest.WebPageTest.location_queue", return_value=None
)
@mock.patch("requests.get", return_value=init_mocked_request(200))
@mock.patch("mozperftest.test.webpagetest.WPT_KEY_FILE", "tests/data/WPT_fakekey.txt")
def test_webpagetest_test_invalid_connection(*mocked):
mach_cmd, metadata, env = running_env(tests=[str(EXAMPLE_WPT_TEST)])
test = webpagetest.WebPageTest(env, mach_cmd)
metadata.script["options"]["test_parameters"]["connection"] = "Invalid Connection"
with pytest.raises(WPTInvalidConnectionSelection):
test.run(metadata)
@mock.patch("mozperftest.utils.get_tc_secret", return_value={"wpt_key": "fake_key"})
@mock.patch(
"mozperftest.test.webpagetest.WebPageTest.location_queue", return_value=None
)
@mock.patch("requests.get", return_value=init_mocked_request(200))
@mock.patch("mozperftest.test.webpagetest.WPT_KEY_FILE", "tests/data/WPT_fakekey.txt")
def test_webpagetest_test_invalid_url(*mocked):
mach_cmd, metadata, env = running_env(tests=[str(EXAMPLE_WPT_TEST)])
test = webpagetest.WebPageTest(env, mach_cmd)
metadata.script["options"]["test_list"] = ["InvalidUrl"]
with pytest.raises(WPTInvalidURLError):
test.run(metadata)
@mock.patch("mozperftest.utils.get_tc_secret", return_value={"wpt_key": "fake_key"})
@mock.patch(
"mozperftest.test.webpagetest.WebPageTest.location_queue", return_value=None
)
@mock.patch("requests.get", return_value=init_mocked_request(200))
@mock.patch("mozperftest.test.webpagetest.WPT_KEY_FILE", "tests/data/WPT_fakekey.txt")
def test_webpagetest_test_invalid_statistic(*mocked):
mach_cmd, metadata, env = running_env(tests=[str(EXAMPLE_WPT_TEST)])
test = webpagetest.WebPageTest(env, mach_cmd)
metadata.script["options"]["test_parameters"]["statistics"] = ["Invalid Statistic"]
with pytest.raises(WPTInvalidStatisticsError):
test.run(metadata)
assert True
@mock.patch("requests.get", return_value=init_mocked_request(200))
@mock.patch("mozperftest.utils.get_tc_secret", return_value={"wpt_key": "fake_key"})
@mock.patch(
"mozperftest.test.webpagetest.WebPageTest.request_with_timeout",
return_value={"data": {}},
)
@mock.patch("mozperftest.test.webpagetest.WPT_KEY_FILE", "tests/data/WPT_fakekey.txt")
def test_webpagetest_test_invalid_location(*mocked):
mach_cmd, metadata, env = running_env(tests=[str(EXAMPLE_WPT_TEST)])
test = webpagetest.WebPageTest(env, mach_cmd)
metadata.script["options"]["test_parameters"]["location"] = "Invalid Location"
with pytest.raises(WPTLocationSelectionError):
test.run(metadata)
@mock.patch("requests.get", return_value=init_mocked_request(200, 101))
@mock.patch("mozperftest.utils.get_tc_secret", return_value={"wpt_key": "fake_key"})
@mock.patch(
"mozperftest.test.webpagetest.WebPageTest.location_queue", return_value=None
)
@mock.patch("mozperftest.test.webpagetest.WPT_KEY_FILE", "tests/data/WPT_fakekey.txt")
def test_webpagetest_test_timeout(*mocked):
mach_cmd, metadata, env = running_env(tests=[str(EXAMPLE_WPT_TEST)])
test = webpagetest.WebPageTest(env, mach_cmd)
metadata.script["options"]["test_parameters"]["timeout_limit"] = 2
metadata.script["options"]["test_parameters"]["wait_between_requests"] = 1
with pytest.raises(WPTTimeOutError):
test.run(metadata)
assert True
@mock.patch(
"requests.get",
return_value=init_mocked_request(
200, testRuns=3, successfulFVRuns=3, fvonly=True, location="BadLocation"
),
)
@mock.patch("mozperftest.utils.get_tc_secret", return_value={"wpt_key": "fake_key"})
@mock.patch(
"mozperftest.test.webpagetest.WebPageTest.location_queue", return_value=None
)
@mock.patch("mozperftest.test.webpagetest.WPT_KEY_FILE", "tests/data/WPT_fakekey.txt")
def test_webpagetest_test_wrong_browserlocation(*mocked):
mach_cmd, metadata, env = running_env(tests=[str(EXAMPLE_WPT_TEST)])
metadata.script["options"]["test_list"] = ["google.ca"]
metadata.script["options"]["test_parameters"]["wait_between_requests"] = 1
test = webpagetest.WebPageTest(env, mach_cmd)
with pytest.raises(WPTBrowserSelectionError):
test.run(metadata)
@mock.patch(
"mozperftest.test.webpagetest.WebPageTest.get_WPT_results",
return_value=init_placeholder_wpt_data(invalid_results=True),
)
@mock.patch("mozperftest.utils.get_tc_secret", return_value={"wpt_key": "fake_key"})
@mock.patch(
"mozperftest.test.webpagetest.WebPageTest.location_queue", return_value=None
)
@mock.patch("requests.get", return_value=init_mocked_request(200))
@mock.patch("mozperftest.test.webpagetest.WPT_KEY_FILE", "tests/data/WPT_fakekey.txt")
def test_webpagetest_test_metric_not_found(*mocked):
mach_cmd, metadata, env = running_env(tests=[str(EXAMPLE_WPT_TEST)])
metadata.script["options"]["test_list"] = ["google.ca"]
metadata.script["options"]["test_parameters"]["wait_between_requests"] = 1
test = webpagetest.WebPageTest(env, mach_cmd)
with pytest.raises(WPTDataProcessingError):
test.run(metadata)
@mock.patch("mozperftest.utils.get_tc_secret", return_value={"wpt_key": "fake_key"})
@mock.patch(
"mozperftest.test.webpagetest.WebPageTest.location_queue", return_value=None
)
@mock.patch(
"requests.get",
return_value=init_mocked_request(
200, WPT_test_status_code=400, WPT_test_status_text=WPT_API_EXPIRED_MESSAGE
),
)
@mock.patch("mozperftest.test.webpagetest.WPT_KEY_FILE", "tests/data/WPT_fakekey.txt")
def test_webpagetest_test_expired_api_key(*mocked):
mach_cmd, metadata, env = running_env(tests=[str(EXAMPLE_WPT_TEST)])
metadata.script["options"]["test_list"] = ["google.ca"]
metadata.script["options"]["test_parameters"]["wait_between_requests"] = 1
test = webpagetest.WebPageTest(env, mach_cmd)
with pytest.raises(WPTExpiredAPIKeyError):
test.run(metadata)