Source code
Revision control
Copy as Markdown
Other Tools
Test Info:
- Manifest: testing/test/python.toml
# -*- coding: utf-8 -*-"
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
import io
import json
import os
from pathlib import Path
import pytest
from mozunit import main
from skipfails import Kind, Skipfails
DATA_PATH = Path(__file__).with_name("data")
def test_get_revision():
"""Test get_revision"""
sf = Skipfails()
with pytest.raises(ValueError) as e_info:
sf.get_revision("")
assert str(e_info.value) == "try_url scheme not https"
with pytest.raises(ValueError) as e_info:
assert str(e_info.value) == "try_url server not treeherder.mozilla.org"
with pytest.raises(ValueError) as e_info:
assert str(e_info.value) == "try_url query missing"
with pytest.raises(ValueError) as e_info:
assert str(e_info.value) == "try_url query missing revision"
revision, repo = sf.get_revision(
)
assert revision == "5b1738d0af571777199ff3c694b1590ff574946b"
assert repo == "try"
def test_get_tasks():
"""Test get_tasks import of mozci"""
from mozci.push import Push
revision = "5b1738d0af571777199ff3c694b1590ff574946b"
repo = "try"
push = Push(revision, repo)
assert push is not None
def get_failures(
tasks_name, exp_f_name, task_details=None, error_summary=None, implicit_vars=False
):
"""Runs Skipfails.get_failures on tasks to compare with failures"""
sf = Skipfails(implicit_vars=implicit_vars)
assert sf.implicit_vars == implicit_vars
if task_details is not None: # preload task details cache, if needed
if isinstance(task_details, str): # read file
task_details = sf.read_json(DATA_PATH.joinpath(task_details))
sf.tasks = task_details
if error_summary is not None: # preload task details cache, if needed
if isinstance(error_summary, str): # read file
error_summary = sf.read_json(DATA_PATH.joinpath(error_summary))
sf.error_summary = error_summary
tasks = sf.read_tasks(DATA_PATH.joinpath(tasks_name))
exp_f = sf.read_failures(DATA_PATH.joinpath(exp_f_name))
expected_failures = json.dumps(exp_f, indent=2, sort_keys=True).strip()
failures = sf.get_failures(tasks)
actual_failures = json.dumps(failures, indent=2, sort_keys=True).strip()
assert actual_failures == expected_failures
def test_get_failures_1():
"""Test get_failures 1"""
task_details = {"dwOJ8M9ERSmk6oI2KXg6hg": {}}
get_failures("wayland-tasks-1.json", "wayland-failures-1.json", task_details)
def test_get_failures_2():
"""Test get_failures 2"""
task_details = {
"Y7r1q2xWSu-2bRAofEfeBw": {},
"Z7r1q2xWSu-2bRAofEfeBw": {},
"X7r1q2xWSu-2bRAofEfeBw": {},
}
get_failures("wayland-tasks-2.json", "wayland-failures-2.json", task_details)
def test_get_failures_3():
"""Test get_failures 3"""
task_details = {
"b7_ahjGtQ_-ZMNBG_hUZUw": {},
"WVczuxkuSRKZg_jMiGyQsA": {},
"UOZUIVAaTZKmRwArq5WkDw": {},
}
get_failures("wayland-tasks-3.json", "wayland-failures-3.json", task_details)
def test_get_failures_4():
"""Test get_failures 4"""
task_details = {
"bxMVPbPMTru_bfAivc1sPA": {},
"EDql3NKPR3W6OEU3mLeKbg": {},
"FDql3NKPR3W6OEU3mLeKbg": {},
}
get_failures("wayland-tasks-4.json", "wayland-failures-4.json", task_details)
def test_get_failures_5():
"""Test get_failures 5"""
task_details = {
"Bgc6We1sSjakIo3V9crldw": {
"expires": "2024-01-09T16:05:56.825Z",
"extra": {
"suite": "mochitest-browser",
"test-setting": {
"build": {"type": "opt"},
"platform": {
"arch": "64",
"os": {"name": "linux", "version": "22.04"},
"display": "wayland",
},
"runtime": {},
},
},
}
}
get_failures("wayland-tasks-5.json", "wayland-failures-5.json", task_details)
def test_get_failures_6():
"""Test get_failures 6"""
task_details = {
"AKYqxtoWStigj_5yHVqAeg": {
"expires": "2024-03-19T03:29:11.050Z",
"extra": {
"suite": "web-platform-tests",
"test-setting": {
"build": {
"type": "opt",
"shippable": True,
},
"platform": {
"arch": "32",
"os": {"name": "linux", "version": "1804"},
},
"runtime": {},
},
},
}
}
get_failures("wpt-tasks-1.json", "wpt-failures-1.json", task_details)
def test_get_failures_7():
"""Test get_failures 7"""
get_failures(
"reftest-tasks-1.json",
"reftest-failures-1.json",
"reftest-extra-1.json",
"reftest-summary-1.json",
True,
)
def test_get_bug_by_id():
"""Test get_bug_by_id"""
sf = Skipfails()
id = 1682371
# preload bug cache
bugs_filename = f"bug-{id}.json"
sf.bugs = sf.read_bugs(DATA_PATH.joinpath(bugs_filename))
# function under test
bug = sf.get_bug_by_id(id)
assert bug is not None
assert bug.id == id
assert bug.product == "Testing"
assert bug.component == "General"
assert (
bug.summary
== "create tool to quickly parse and identify all failures from a try push and ideally annotate manifests"
)
def test_get_bugs_by_summary():
"""Test get_bugs_by_summary"""
sf = Skipfails()
id = 1682371
# preload bug cache
bugs_filename = f"bug-{id}.json"
sf.bugs = sf.read_bugs(DATA_PATH.joinpath(bugs_filename))
# function under test
summary = "create tool to quickly parse and identify all failures from a try push and ideally annotate manifests"
bugs = sf.get_bugs_by_summary(summary)
assert len(bugs) == 1
assert bugs[0].id == id
assert bugs[0].product == "Testing"
assert bugs[0].component == "General"
assert bugs[0].summary == summary
def test_get_variants():
"""Test get_variants"""
sf = Skipfails()
variants = sf.get_variants()
assert "1proc" in variants
assert variants["1proc"] == "e10s"
assert "webrender-sw" in variants
assert variants["webrender-sw"] == "swgl"
assert "aab" in variants
assert variants["aab"] == "aab"
def test_task_to_skip_if():
"""Test task_to_skip_if"""
sf = Skipfails()
# preload task cache
task_id = "UP-t3xrGSDWvUNjFGIt_aQ"
task_details = {
"expires": "2024-01-09T16:05:56.825Z",
"extra": {
"suite": "mochitest-plain",
"test-setting": {
"build": {"type": "debug"},
"platform": {
"arch": "32",
"os": {"build": "2009", "name": "windows", "version": "11"},
},
"runtime": {},
},
},
}
sf.tasks[task_id] = task_details
# function under test
skip_if = sf.task_to_skip_if(task_id, Kind.TOML)
assert skip_if == "win11_2009 && processor == 'x86' && debug"
task_id = "I3iXyGDATDSDyzGh4YfNJw"
task_details = {
"expires": "2024-01-09T16:05:56.825Z",
"extra": {
"suite": "web-platform-tests-crashtest",
"test-setting": {
"build": {"type": "debug"},
"platform": {
"arch": "64",
"os": {"name": "macosx", "version": "1015"},
},
"runtime": {"webrender-sw": True},
},
},
}
sf.tasks[task_id] = task_details
# function under test
skip_if = sf.task_to_skip_if(task_id, Kind.TOML)
assert (
skip_if
== "os == 'mac' && os_version == '10.15' && processor == 'x86_64' && debug && swgl"
)
task_id = "bAkMaQIVQp6oeEIW6fzBDw"
task_details = {
"expires": "2024-01-09T16:05:56.825Z",
"extra": {
"suite": "mochitest-media",
"test-setting": {
"build": {"type": "debug"},
"platform": {
"arch": "aarch64",
"os": {"name": "macosx", "version": "1100"},
},
"runtime": {"webrender-sw": True},
},
},
}
sf.tasks[task_id] = task_details
# function under test
skip_if = sf.task_to_skip_if(task_id, Kind.TOML)
assert (
skip_if
== "os == 'mac' && os_version == '11.00' && processor == 'aarch64' && debug && swgl"
)
def test_task_to_skip_if_wpt():
"""Test task_to_skip_if_wpt"""
# preload task cache
task_id = "AKYqxtoWStigj_5yHVqAeg"
task_details = {
"expires": "2024-03-19T03:29:11.050Z",
"extra": {
"suite": "web-platform-tests",
"test-setting": {
"build": {
"type": "opt",
"shippable": True,
},
"platform": {
"arch": "32",
"os": {"name": "linux", "version": "1804"},
},
"runtime": {},
},
},
}
sf = Skipfails()
sf.tasks[task_id] = task_details
# function under test
skip_if = sf.task_to_skip_if(task_id, Kind.WPT)
assert (
skip_if
== 'os == "linux" and os_version == "18.04" and processor == "x86" and not debug'
)
def test_task_to_skip_if_reftest():
"""Test task_to_skip_if_reftest"""
# preload task cache
task_id = "AKYqxtoWStigj_5yHVqAeg"
task_details = {
"expires": "2024-03-19T03:29:11.050Z",
"extra": {
"suite": "reftest",
"test-setting": {
"build": {
"type": "opt",
"shippable": True,
},
"platform": {
"arch": "32",
"os": {"name": "linux", "version": "1804"},
},
"runtime": {},
},
},
}
sf = Skipfails(implicit_vars=True)
sf.tasks[task_id] = task_details
# function under test
skip_if = sf.task_to_skip_if(task_id, Kind.LIST)
assert skip_if == "gtkWidget&&optimized&&!is64Bit"
def test_task_to_skip_if_reftest2():
"""Test task_to_skip_if_reftest2"""
# preload task cache
task_id = "ajp7DRgGQbyfnIAklKA7Tw"
task_details = {
"expires": "2024-03-19T03:29:11.050Z",
"extra": {
"suite": "reftest",
"test-setting": {
"build": {"tsan": True, "type": "opt"},
"runtime": {"webrender-sw": True},
"platform": {"os": {"name": "linux", "version": "1804"}, "arch": "64"},
},
},
}
sf = Skipfails(implicit_vars=True)
sf.tasks[task_id] = task_details
# function under test
skip_if = sf.task_to_skip_if(task_id, Kind.LIST)
assert skip_if == "gtkWidget&&ThreadSanitizer&&swgl"
def test_task_to_skip_if_reftest3():
"""Test task_to_skip_if_reftest3"""
# preload task cache
task_id = "UP-t3xrGSDWvUNjFGIt_aQ"
task_details = {
"expires": "2024-01-09T16:05:56.825Z",
"extra": {
"suite": "mochitest-plain",
"test-setting": {
"build": {"type": "debug"},
"platform": {
"arch": "32",
"os": {"build": "2009", "name": "windows", "version": "11"},
},
"runtime": {},
},
},
}
sf = Skipfails(implicit_vars=False)
sf.tasks[task_id] = task_details
# function under test
skip_if = sf.task_to_skip_if(task_id, Kind.LIST)
assert skip_if == "winWidget&&isDebugBuild&&fission&&!is64Bit&&!swgl"
def test_task_to_skip_if_reftest4():
"""Test task_to_skip_if_reftest4"""
# preload task cache
task_id = "ajp7DRgGQbyfnIAklKA7Tw"
task_details = {
"expires": "2024-03-19T03:29:11.050Z",
"extra": {
"suite": "reftest",
"test-setting": {
"build": {"tsan": True, "type": "opt"},
"runtime": {},
"platform": {"os": {"name": "linux", "version": "1804"}, "arch": "64"},
},
},
}
sf = Skipfails(implicit_vars=False)
sf.tasks[task_id] = task_details
# function under test
skip_if = sf.task_to_skip_if(task_id, Kind.LIST)
assert skip_if == "gtkWidget&&ThreadSanitizer&&fission&&!swgl"
def test_wpt_add_skip_if():
"""Test wpt_add_skip_if"""
sf = Skipfails()
manifest_before1 = ""
anyjs = {}
filename = "myfile.html"
anyjs[filename] = False
skip_if = 'os == "linux" and processor == "x86" and not debug'
disabled = " disabled:\n"
condition = " if " + skip_if + ": " + bug_reference + "\n"
manifest_str, additional_comment_ = sf.wpt_add_skip_if(
manifest_before1, anyjs, skip_if, bug_reference
)
manifest_expected1 = "[myfile.html]\n" + disabled + condition + "\n"
assert manifest_str == manifest_expected1
manifest_before2 = """[myfile.html]
expected:
if fission: [OK, FAIL]
[< [Test 5\\] 1 out of 2 assertions were failed.]
expected: FAIL
"""
manifest_expected2 = (
"""[myfile.html]
expected:
if fission: [OK, FAIL]
"""
+ disabled
+ condition
+ """
[< [Test 5\\] 1 out of 2 assertions were failed.]
expected: FAIL
"""
)
anyjs[filename] = False
manifest_str, additional_comment_ = sf.wpt_add_skip_if(
manifest_before2, anyjs, skip_if, bug_reference
)
assert manifest_str == manifest_expected2
anyjs[filename] = False
manifest_str, additional_comment_ = sf.wpt_add_skip_if(
manifest_expected2, anyjs, skip_if, bug_reference
)
assert manifest_str == manifest_expected2
manifest_before4 = """;https: //bugzilla.mozilla.org/show_bug.cgi?id=1838684
expected: [FAIL, PASS]
[custom-highlight-painting-overlapping-highlights-002.html]
expected:
[PASS, FAIL]
"""
anyjs[filename] = False
manifest_str, additional_comment_ = sf.wpt_add_skip_if(
manifest_before4, anyjs, skip_if, bug_reference
)
manifest_expected4 = manifest_before4 + "\n" + manifest_expected1
assert manifest_str == manifest_expected4
manifest_before5 = """[myfile.html]
disabled:
"""
anyjs[filename] = False
manifest_str, additional_comment_ = sf.wpt_add_skip_if(
manifest_before5, anyjs, skip_if, bug_reference
)
manifest_expected5 = manifest_before5 + condition + "\n"
assert manifest_str == manifest_expected5
manifest_before6 = """[myfile.html]
[Window Size]
expected:
if product == "firefox_android": FAIL
"""
anyjs[filename] = False
manifest_str, additional_comment_ = sf.wpt_add_skip_if(
manifest_before6, anyjs, skip_if, bug_reference
)
manifest_expected6 = (
"""[myfile.html]
disabled:
"""
+ condition
+ """
[Window Size]
expected:
if product == "firefox_android": FAIL
"""
)
assert manifest_str == manifest_expected6
manifest_before7 = """[myfile.html]
fuzzy:
maxDifference=0-1;totalPixels=0-80
disabled:
"""
anyjs[filename] = False
manifest_str, additional_comment_ = sf.wpt_add_skip_if(
manifest_before7, anyjs, skip_if, bug_reference
)
manifest_expected7 = manifest_before7 + condition + "\n"
assert manifest_str == manifest_expected7
manifest_before8 = """[myfile.html]
disabled:
if os == "linux" and os_version == "22.04" and processor == "x86_64" and debug and display == "wayland": Bug TBD
expected:
if swgl and (os == "linux") and debug and not fission: [PASS, FAIL]
"""
anyjs[filename] = False
manifest_str, additional_comment_ = sf.wpt_add_skip_if(
manifest_before8, anyjs, skip_if, bug_reference
)
manifest_expected8 = (
"""[myfile.html]
disabled:
if os == "linux" and os_version == "22.04" and processor == "x86_64" and debug and display == "wayland": Bug TBD
"""
+ condition
+ """ expected:
if swgl and (os == "linux") and debug and not fission: [PASS, FAIL]
"""
)
assert manifest_str == manifest_expected8
manifest_before9 = """[myfile.html]
disabled:
if os == "linux" and os_version == "22.04" and processor == "x86_64" and debug and display == "wayland": Bug TBD
"""
anyjs[filename] = False
manifest_str, additional_comment_ = sf.wpt_add_skip_if(
manifest_before9, anyjs, skip_if, bug_reference
)
manifest_expected9 = (
"""[myfile.html]
disabled:
if os == "linux" and os_version == "22.04" and processor == "x86_64" and debug and display == "wayland": Bug TBD
"""
+ condition
+ "\n"
)
assert manifest_str == manifest_expected9
manifest_before10 = """[myfile.html]
disabled:
if os == "linux" and os_version == "22.04" and processor == "x86_64" and not debug and display == "wayland": Bug TBD
[3P fetch: Cross site window setting HTTP cookies]
"""
anyjs[filename] = False
manifest_str, additional_comment_ = sf.wpt_add_skip_if(
manifest_before10, anyjs, skip_if, bug_reference
)
manifest_expected10 = (
"""[myfile.html]
disabled:
if os == "linux" and os_version == "22.04" and processor == "x86_64" and not debug and display == "wayland": Bug TBD
"""
+ condition
+ """\n [3P fetch: Cross site window setting HTTP cookies]
"""
)
assert manifest_str == manifest_expected10
def test_get_filename_in_manifest():
"""Test get_filename_in_manifest"""
sf = Skipfails()
assert (
sf.get_filename_in_manifest(
"browser/components/sessionstore/test/browser.toml",
"browser/components/sessionstore/test/browser_closed_tabs_windows.js",
)
== "browser_closed_tabs_windows.js"
)
assert (
sf.get_filename_in_manifest(
"browser/base/content/test/webrtc/gracePeriod/browser.toml",
"browser/base/content/test/webrtc/browser_devices_get_user_media_grace.js",
)
== "../browser_devices_get_user_media_grace.js"
)
assert (
sf.get_filename_in_manifest(
"dom/animation/test/mochitest.toml",
"dom/animation/test/document-timeline/test_document-timeline.html",
)
== "document-timeline/test_document-timeline.html"
)
def test_label_to_platform_testname():
"""Test label_to_platform_testname"""
sf = Skipfails()
label = "test-linux2204-64-wayland/opt-mochitest-browser-chrome-swr-13"
platform, testname = sf.label_to_platform_testname(label)
assert platform == "test-linux2204-64-wayland/opt"
assert testname == "mochitest-browser-chrome"
def test_reftest_add_fuzzy_if():
"""Test reftest_add_fuzzy_if"""
sf = Skipfails(implicit_vars=True)
manifest_path = DATA_PATH.joinpath("reftest-1-before.list")
manifest_str = io.open(manifest_path, "r", encoding="utf-8").read()
after_path = DATA_PATH.joinpath("reftest-1-after.list")
after_str = io.open(after_path, "r", encoding="utf-8").read()
manifest_str, additional_comment = sf.reftest_add_fuzzy_if(
manifest_str,
"fuzzy(0-11,0-7155) == blur-inside-clipPath.svg blur-inside-clipPath-ref.svg",
"gtkWidget",
[10, 12],
[7000, 8000],
2,
False,
"Bug TBD",
)
assert additional_comment == ""
assert manifest_str == after_str
def test_reftest_add_fuzzy_if2():
"""Test reftest_add_fuzzy_if2"""
sf = Skipfails(implicit_vars=True)
manifest_path = DATA_PATH.joinpath("reftest-2-before.list")
manifest_str = io.open(manifest_path, "r", encoding="utf-8").read()
after_path = DATA_PATH.joinpath("reftest-2-after.list")
after_str = io.open(after_path, "r", encoding="utf-8").read()
manifest_str, additional_comment = sf.reftest_add_fuzzy_if(
manifest_str,
"fuzzy(0-11,0-7155) HTTP(..) == blur-inside-clipPath.svg blur-inside-clipPath-ref.svg",
"gtkWidget",
[10, 12],
[7000, 8000],
2,
False,
"Bug TBD",
)
assert additional_comment == ""
assert manifest_str == after_str
def test_reftest_add_fuzzy_if3():
"""Test reftest_add_fuzzy_if3"""
sf = Skipfails(implicit_vars=True)
manifest_path = DATA_PATH.joinpath("reftest-2-after.list")
manifest_str = io.open(manifest_path, "r", encoding="utf-8").read()
after_path = DATA_PATH.joinpath("reftest-3-after.list")
after_str = io.open(after_path, "r", encoding="utf-8").read()
manifest_str, additional_comment = sf.reftest_add_fuzzy_if(
manifest_str,
"fuzzy(0-11,0-7155) fuzzy-if(gtkWidget,10-12,7000-8400) HTTP(..) == blur-inside-clipPath.svg blur-inside-clipPath-ref.svg",
"gtkWidget",
[9, 13],
[7500, 7900],
2,
False,
"Bug TBD",
)
assert additional_comment == ""
assert manifest_str == after_str
def test_reftest_add_fuzzy_if4():
"""Test reftest_add_fuzzy_if4"""
# Now with the relaxed merge policy the additional condition has been merged
# instead of added....
# fuzzy-if(winWidget,0-177,0-1) fuzzy-if(winWidget&&!is64Bit&&isDebugBuild,0-152,0-1) == 23605-5.html 23605-5-ref.html # Bug TBD
sf = Skipfails(implicit_vars=True)
manifest_path = DATA_PATH.joinpath("reftest-4-before.list")
manifest_str = io.open(manifest_path, "r", encoding="utf-8").read()
after_path = DATA_PATH.joinpath("reftest-4-after.list")
after_str = io.open(after_path, "r", encoding="utf-8").read()
manifest_str, additional_comment = sf.reftest_add_fuzzy_if(
manifest_str,
"fuzzy-if(winWidget,0-177,0-1) == 23605-5.html 23605-5-ref.html",
"winWidget&&!is64Bit&&isDebugBuild",
[139, 145],
[1, 1],
2,
True,
"Bug TBD",
)
assert additional_comment == ""
assert manifest_str == after_str
# assert (
# additional_comment
# == "NOTE: more than one fuzzy-if for the OS = winWidget ==> may require manual review"
# )
def test_reftest_add_fuzzy_if5():
"""Test reftest_add_fuzzy_if5"""
sf = Skipfails(implicit_vars=True)
manifest_path = DATA_PATH.joinpath("reftest-2-after.list")
manifest_str = io.open(manifest_path, "r", encoding="utf-8").read()
after_path = DATA_PATH.joinpath("reftest-5-after.list")
after_str = io.open(after_path, "r", encoding="utf-8").read()
manifest_str, additional_comment = sf.reftest_add_fuzzy_if(
manifest_str,
"fuzzy(0-11,0-7155) fuzzy-if(gtkWidget,10-12,7000-8400) HTTP(..) == blur-inside-clipPath.svg blur-inside-clipPath-ref.svg",
"gtkWidget&&!is64Bit",
[5, 6],
[3000, 4000],
2,
False,
"Bug TBD",
)
assert additional_comment == ""
assert manifest_str == after_str
def test_reftest_add_fuzzy_if6():
"""Test reftest_add_fuzzy_if6"""
sf = Skipfails(implicit_vars=True)
manifest_path = DATA_PATH.joinpath("reftest-6-before.list")
manifest_str = io.open(manifest_path, "r", encoding="utf-8").read()
after_path = DATA_PATH.joinpath("reftest-6-after.list")
after_str = io.open(after_path, "r", encoding="utf-8").read()
manifest_str, additional_comment = sf.reftest_add_fuzzy_if(
manifest_str,
"fuzzy(16-51,5234-5622) fuzzy-if(swgl,32-38,1600-91746) fuzzy-if(useDrawSnapshot,16-16,11600-11600) fuzzy-if(cocoaWidget,16-73,5212-5622) == ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.tv.yuv420p.av1.webm ../reftest_img.html?src=color_quads/720p.png",
"Android&&isDebugBuild&&!fission&&swgl",
[1, 32],
[1, 1760],
2,
True,
"Bug TBD",
)
assert additional_comment == ""
assert manifest_str == after_str
def test_reftest_add_fuzzy_if7():
"""Test reftest_add_fuzzy_if7"""
sf = Skipfails(implicit_vars=False)
manifest_path = DATA_PATH.joinpath("reftest-7-before.list")
manifest_str = io.open(manifest_path, "r", encoding="utf-8").read()
after_path = DATA_PATH.joinpath("reftest-7-after.list")
after_str = io.open(after_path, "r", encoding="utf-8").read()
manifest_str, additional_comment = sf.reftest_add_fuzzy_if(
manifest_str,
"skip-if(winWidget&&isCoverageBuild) fuzzy(0-16,75-1941) fuzzy-if(Android,28-255,273680-359920) fuzzy-if(appleSilicon,30-48,1835-187409) fuzzy-if(cocoaWidget,30-32,187326-187407) == ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.tv.yuv420p.h264.mp4 ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.tv.yuv420p.av1.webm",
"cocoaWidget",
[],
[],
2,
False,
"Bug TBD",
)
assert additional_comment == ""
assert manifest_str == after_str
def test_reftest_add_fuzzy_if8():
"""Test reftest_add_fuzzy_if8"""
sf = Skipfails(implicit_vars=True)
manifest_path = DATA_PATH.joinpath("reftest-6-after.list")
manifest_str = io.open(manifest_path, "r", encoding="utf-8").read()
after_path = DATA_PATH.joinpath("reftest-8-after.list")
after_str = io.open(after_path, "r", encoding="utf-8").read()
manifest_str, additional_comment = sf.reftest_add_fuzzy_if(
manifest_str,
"fuzzy(16-51,5234-5622) fuzzy-if(swgl,32-38,1600-91746) fuzzy-if(useDrawSnapshot,16-16,11600-11600) fuzzy-if(Android&&isDebugBuild&&!fission&&swgl,0-33,0-1848) fuzzy-if(cocoaWidget,16-73,5212-5622) == ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.tv.yuv420p.av1.webm ../reftest_img.html?src=color_quads/720p.png",
"cocoaWidget",
[0, 0],
[0, 0],
2,
True,
"Bug TBD",
)
assert additional_comment == ""
assert manifest_str == after_str
def test_reftest_add_fuzzy_if9():
"""Test reftest_add_fuzzy_if9"""
sf = Skipfails(implicit_vars=True)
manifest_path = DATA_PATH.joinpath("reftest-9-before.list")
manifest_str = io.open(manifest_path, "r", encoding="utf-8").read()
after_path = DATA_PATH.joinpath("reftest-9-after.list")
after_str = io.open(after_path, "r", encoding="utf-8").read()
manifest_str, additional_comment = sf.reftest_add_fuzzy_if(
manifest_str,
"fuzzy(0-201,0-1486) fuzzy-if(Android&&(fission||!fission),201-267,42-1560) fuzzy-if(Android&&!fission&&(swgl||!swgl),201-267,40-1560) fuzzy-if(Android&&isDebugBuild&&!fission&&swgl,198-207,1439-1510) fuzzy-if(cocoaWidget&&isDebugBuild&&swgl,198-267,40-1510) fuzzy-if(gtkWidget&&swgl,198-267,40-1510) fuzzy-if(gtkWidget&&!fission&&swgl,0-267,0-1510) fuzzy-if(gtkWidget&&!fission&&(swgl||!swgl),201-267,40-1560) == element-paint-transform-02.html element-paint-transform-02-ref.html",
"gtkWidget&&isDebugBuild&&!fission&&swgl",
[198],
[1439],
2,
True,
"Bug TBD",
)
assert manifest_str == after_str
def test_reftest_add_fuzzy_if10():
"""Test reftest_add_fuzzy_if10"""
sf = Skipfails(implicit_vars=False)
manifest_path = DATA_PATH.joinpath("reftest-10-before.list")
manifest_str = io.open(manifest_path, "r", encoding="utf-8").read()
after_path = DATA_PATH.joinpath("reftest-10-after.list")
after_str = io.open(after_path, "r", encoding="utf-8").read()
manifest_str, additional_comment = sf.reftest_add_fuzzy_if(
manifest_str,
"skip-if(winWidget&&isCoverageBuild) fuzzy(0-16,75-1941) fuzzy-if(Android,28-255,273680-359920) fuzzy-if(appleSilicon,30-48,1835-187409) fuzzy-if(cocoaWidget,0-0,0-0) == ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.tv.yuv420p.h264.mp4 ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.tv.yuv420p.av1.webm",
"cocoaWidget",
[],
[],
2,
True,
"Bug TBD",
)
assert additional_comment == "fuzzy-if removed as calculated range is 0-0,0-0"
assert manifest_str == after_str
def test_reftest_add_fuzzy_if11():
"""Test reftest_add_fuzzy_if11"""
sf = Skipfails(implicit_vars=False)
manifest_path = DATA_PATH.joinpath("reftest-10-before.list")
manifest_str = io.open(manifest_path, "r", encoding="utf-8").read()
manifest_str, additional_comment = sf.reftest_add_fuzzy_if(
manifest_str,
"skip-if(winWidget&&isCoverageBuild) fuzzy(0-16,75-1941) fuzzy-if(Android,28-255,273680-359920) fuzzy-if(cocoaWidget,30-32,187326-187407) fuzzy-if(appleSilicon,30-48,1835-187409) == ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.tv.yuv420p.h264.mp4 ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.tv.yuv420p.av1.webm",
"winWidget",
[],
[],
2,
True,
"Bug TBD",
)
assert additional_comment == "fuzzy-if not added as calculated range is 0-0,0-0"
assert manifest_str == ""
def test_reftest_get_lineno():
"""Test reftest_get_lineno"""
sf = Skipfails()
mods = [
"pref(gfx.font_rendering.colr_v1.enabled,true)",
"fuzzy(0-8,0-10100)",
"==",
"colrv1-01.html#C",
"colrv1-01-ref.html#C",
]
allmods = " ".join(mods)
lineno = sf.reftest_find_lineno(
"layout/reftests/font-face/reftest.list", mods, allmods
)
assert lineno == 171
def test_reftest_get_lineno2():
"""Test reftest_get_lineno2"""
sf = Skipfails()
mods = [
"pref(image.downscale-during-decode.enabled,true)",
"fuzzy(0-53,0-6391)",
"fuzzy-if(appleSilicon,0-20,0-11605)",
"fuzzy-if(gtkWidget,18-19,5502-5568)",
"skip-if(Android)",
"==",
"downscale-moz-icon-1.html",
"downscale-moz-icon-1-ref.html",
]
allmods = " ".join(mods)
lineno = sf.reftest_find_lineno(
"image/test/reftest/downscaling/reftest.list", mods, allmods
)
assert lineno == 177
def test_reftest_get_lineno3():
"""Test reftest_get_lineno3"""
sf = Skipfails()
mods = [
"pref(webgl.force-enabled,true)",
"skip-if(Android)",
"fuzzy(0-235,0-3104)",
"==",
"1177726-text-stroke-bounds.html",
"1177726-text-stroke-bounds-ref.html",
]
allmods = " ".join(mods)
lineno = sf.reftest_find_lineno(
"dom/canvas/test/reftest/reftest.list", mods, allmods
)
assert lineno == 233
def test_reftest_skip_failure_win_32(capsys):
"""Test reftest_skip_failure_win_32"""
sf = Skipfails(verbose=True, bugzilla="disable", implicit_vars=True)
manifest = "layout/reftests/svg/reftest.list"
kind = Kind.LIST
path = "fuzzy(0-1,0-5) fuzzy-if(winWidget,0-96,0-21713) skip-if(winWidget&&isCoverageBuild) fuzzy-if(Android&&device,0-4,0-946) == radialGradient-basic-03.svg radialGradient-basic-03-ref.html"
anyjs = None
differences = [68]
pixels = [21668]
lineno = 419
status = "FAIL"
label = "test-windows11-32-2009-qr/debug-reftest-1"
classification = "disable_recommended"
task_id = "BpoP8I2CRZekXUKoSIZjUQ"
revision = "3e54b0b81de7d6a3e6a2c3408892ffd6430bc137"
repo = "try"
meta_bug_id = None
task_details = { # pre-cache task details
"expires": "2024-01-09T16:05:56.825Z",
"extra": {
"suite": "reftest",
"test-setting": {
"build": {"type": "debug"},
"platform": {
"arch": "32",
"os": {"build": "2009", "name": "windows", "version": "11"},
},
"runtime": {},
},
},
}
sf.tasks[task_id] = task_details
sf.skip_failure(
manifest,
kind,
path,
anyjs,
differences,
pixels,
lineno,
status,
label,
classification,
task_id,
try_url,
revision,
repo,
meta_bug_id,
)
capsys.readouterr()
# assert (
# captured.err.find(
# "Skipping failures for Windows 32-bit are temporarily disabled"
# )
# > 0
# )
def test_reftest_skip_failure_reorder(capsys):
"""Test reftest_skip_failure_reorder"""
manifest_before_path = DATA_PATH.joinpath("reftest-reorder-before.list")
manifest_before = io.open(manifest_before_path, "r", encoding="utf-8").read()
manifest_path = DATA_PATH.joinpath("reftest-reorder.list")
manifest_fp = io.open(manifest_path, "w", encoding="utf-8")
manifest_fp.write(manifest_before)
manifest_fp.close()
manifest_after_path = DATA_PATH.joinpath("reftest-reorder-after.list")
manifest_after = io.open(manifest_after_path, "r", encoding="utf-8").read()
sf = Skipfails(verbose=True, bugzilla="disable", implicit_vars=True)
manifest = "testing/test/data/reftest-reorder.list"
kind = Kind.LIST
path = "fuzzy-if(cocoaWidget,0-80,0-76800) fuzzy-if(appleSilicon,0-80,0-76800) skip-if(Android) fuzzy-if(winWidget,0-63,0-76799) fuzzy-if(gtkWidget,0-70,0-2032) HTTP(..) == short.mp4.firstframe.html short.mp4.firstframe-ref.html"
anyjs = None
differences = [59, 61]
pixels = [2032, 2133]
lineno = 2
status = "FAIL"
label = "test-linux1804-64-asan-qr/opt-reftest-nofis-7"
classification = "disable_failure"
task_id = "a9Pz5yFJTxOL_uM-BEvGoQ"
revision = "3e54b0b81de7d6a3e6a2c3408892ffd6430bc137"
repo = "try"
meta_bug_id = None
task_details = { # pre-cache task details
"expires": "2024-01-09T16:05:56.825Z",
"extra": {
"suite": "reftest",
"test-setting": {
"build": {"asan": True, "type": "opt"},
"platform": {
"arch": "64",
"os": {"name": "linux", "version": "1804"},
},
"runtime": {"no-fission": True},
},
},
}
sf.tasks[task_id] = task_details
sf.skip_failure(
manifest,
kind,
path,
anyjs,
differences,
pixels,
lineno,
status,
label,
classification,
task_id,
try_url,
revision,
repo,
meta_bug_id,
)
assert os.path.exists(manifest_path)
manifest_str = io.open(manifest_path, "r", encoding="utf-8").read()
assert manifest_str == manifest_after
os.remove(manifest_path)
# captured = capsys.readouterr()
# assert (
# captured.err.find(
# "NOTE: more than one fuzzy-if for the OS = gtkWidget ==> may require manual review"
# )
# > 0
# )
if __name__ == "__main__":
main()