firefox-main/testing/test/test_modes.py

Source code

Revision control

Copy as Markdown

Other Tools

Test Info:

# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
import os
from pathlib import Path
import mozunit
import pytest
from manifestparser.toml import Mode
from skipfails import FAILURE_RATIO, Skipfails, read_json, write_json
DATA_PATH = Path(__file__).with_name("data")
REVISION: str = "7cf7a9720f4ead03213f1799f3dcc00a413c7a02"
RETRIEVING = "Retrieving"
META_BUG_ID: str = "1991977"
TEXT_FILE = 0
JSON_FILE = 1
TASKS_FILE = 1
def clear_cache(sf: Skipfails):
sf.clear_cache = META_BUG_ID
sf.check_cache()
sf.clear_cache = REVISION
sf.check_cache()
def copy_to_cache(skipfails: Skipfails, filename: str, kind: int = JSON_FILE):
from_path = DATA_PATH.joinpath(filename)
to_path = skipfails.cached_path(REVISION, filename)
if kind == TEXT_FILE:
with open(from_path) as from_fp, open(to_path, "w") as to_fp:
to_fp.write(from_fp.read())
elif kind == JSON_FILE:
data = read_json(from_path)
write_json(to_path, data)
else: # TASKS_FILE
data = skipfails.read_tasks(from_path)
skipfails.write_tasks(to_path, data)
def cache_vs_expected(
skipfails: Skipfails,
to_filename: str,
from_filename: str = "",
cache_dir: str = REVISION,
):
from_path = DATA_PATH.joinpath(from_filename if from_filename else to_filename)
to_path = skipfails.cached_path(cache_dir, to_filename)
with open(from_path) as from_fp, open(to_path) as to_fp:
from_data = from_fp.read()
to_data = to_fp.read()
return (to_data, from_data)
@pytest.fixture(scope="session")
def skipfails():
sf = Skipfails(
None, # command_context
TRY_URL,
True, # verbose
"disable", # bugzilla
False, # dry_run
False, # turbo
False, # implicit_vars
None, # new_version
None, # task_id
None, # user_agent
None, # clear_cache
)
clear_cache(sf)
yield sf
def test_carryover_mode(skipfails: Skipfails, capsys):
"Test --carryover"
copy_to_cache(skipfails, "tasks.json", TASKS_FILE)
copy_to_cache(skipfails, "job_ids.json")
copy_to_cache(skipfails, "browser.toml", TEXT_FILE)
copy_to_cache(skipfails, "suggest-531522970.json")
copy_to_cache(skipfails, "suggest-531522979.json")
copy_to_cache(skipfails, "suggest-531523119.json")
copy_to_cache(skipfails, "context-O304PG2lSOuef7JzFoSaow-12581.txt", TEXT_FILE)
copy_to_cache(skipfails, "context-BarnNoFwSCGQtnNGH-o08w-4028.txt", TEXT_FILE)
mode: int = Mode.CARRYOVER
skipfails.run(
META_BUG_ID,
None, # save_tasks
None, # use_tasks
None, # save_failures
None, # use_failures
-1, # max_failures
FAILURE_RATIO, # failure_ratio: float = FAILURE_RATIO,
mode,
)
out, err = capsys.readouterr()
# save STDERR for debugging (don't clear cache at the end)
err_path = skipfails.cached_path(REVISION, "err-carryover.log")
with open(err_path, "w") as fp:
fp.write(err)
mode_string = "Carryover mode: only platform match conditions considered, no bugs created or updated"
assert mode_string in err
assert RETRIEVING not in err
failures, failures_expected = cache_vs_expected(skipfails, "failures.json")
assert failures == failures_expected
manifest, manifest_expected = cache_vs_expected(
skipfails, "browser.toml", "browser-carryover.toml"
)
assert manifest == manifest_expected
actions, actions_expected = cache_vs_expected(
skipfails, "actions.json", "actions-carryover.json", META_BUG_ID
)
assert actions == actions_expected
def test_known_intermittents_mode(skipfails: Skipfails, capsys):
"Test --known-intermittents"
mode: int = Mode.KNOWN_INTERMITTENT
skipfails.run(
META_BUG_ID,
None, # save_tasks
None, # use_tasks
None, # save_failures
None, # use_failures
-1, # max_failures
FAILURE_RATIO, # failure_ratio: float = FAILURE_RATIO,
mode,
)
out, err = capsys.readouterr()
# save STDERR for debugging (don't clear cache at the end)
err_path = skipfails.cached_path(REVISION, "err-known.log")
with open(err_path, "w") as fp:
fp.write(err)
mode_string = "Known Intermittents mode: only failures with known intermittents considered, no bugs created or updated"
assert mode_string in err
assert RETRIEVING not in err
manifest, manifest_expected = cache_vs_expected(
skipfails, "browser.toml", "browser-known.toml"
)
assert manifest == manifest_expected
actions, actions_expected = cache_vs_expected(
skipfails, "actions.json", "actions-known.json", META_BUG_ID
)
assert actions == actions_expected
def test_new_failures_mode(skipfails: Skipfails, capsys):
"Test --new-failures"
mode: int = Mode.NEW_FAILURE
skipfails.run(
META_BUG_ID,
None, # save_tasks
None, # use_tasks
None, # save_failures
None, # use_failures
-1, # max_failures
FAILURE_RATIO, # failure_ratio: float = FAILURE_RATIO,
mode,
)
out, err = capsys.readouterr()
# save STDERR for debugging (don't clear cache at the end)
err_path = skipfails.cached_path(REVISION, "err-new.log")
with open(err_path, "w") as fp:
fp.write(err)
mode_string = "New failures mode: Will only edit manifest skip-if conditions for new failures (i.e. not carryover nor known intermittents)"
assert mode_string in err
assert RETRIEVING not in err
manifest, manifest_expected = cache_vs_expected(
skipfails, "browser.toml", "browser-new.toml"
)
assert manifest == manifest_expected
actions, actions_expected = cache_vs_expected(
skipfails, "actions.json", "actions-new.json", META_BUG_ID
)
assert actions == actions_expected
def test_replace_tbd_mode(skipfails: Skipfails, capsys):
"Test --replace-tbd"
mode: int = Mode.REPLACE_TBD
skipfails.run(
META_BUG_ID,
None, # save_tasks
None, # use_tasks
None, # save_failures
None, # use_failures
-1, # max_failures
FAILURE_RATIO, # failure_ratio: float = FAILURE_RATIO,
mode,
)
out, err = capsys.readouterr()
# save STDERR for debugging (don't clear cache at the end)
err_path = skipfails.cached_path(REVISION, "err-replace.log")
with open(err_path, "w") as fp:
fp.write(err)
mode_string = "Replace TBD mode: Will only edit manifest skip-if conditions for new failures by filing new bugs and replacing TBD with actual bug number."
assert mode_string in err
assert RETRIEVING not in err
carryover = "Bugzilla has been disabled: comment not added to Bug 1111111"
assert carryover in err
assert intermittent in err
assert new in err
def test_cleanup(skipfails: Skipfails):
clear_cache(skipfails)
tasks_cached = skipfails.cached_path(REVISION, "tasks.json")
assert not os.path.exists(tasks_cached)
if __name__ == "__main__":
mozunit.main()