Revision control
Copy as Markdown
#!/usr/bin/env python3
"""
Run application-services tests
This script provides an interface to run our test/linting/formating scripts
The first argument specifies the operating mode. This is either a particular
type of test (rust-tests, swift-lint, etc.), or it's "changes" which is for devs
to test their changes.
Changes Mode:
- Runs tests/clippy against the changes in your branch as determined by:
- `git merge-base` to find the last commit from the main branch
- `git diff` to find changed files from that commit
- Prioritizes executing the tests in a reasonable amount of time.
- Runs linting and formatting (clippy, rustfmt, etc).
- Lets rustfmt fix any issues it finds. It's recommended to
add or commit code before running the tests so that you can inspect any
changes with git diff.
Other Modes:
- rust-tests
- rust-min-version-tests
- rust-clippy
- rust-fmt
- ktlint
- swiftlint
- swiftformat
- nss-bindings
- gradle
- ios-tests
- python-tests
"""
from enum import Enum
from pathlib import Path
import argparse
import json
import os
import platform
import shlex
import subprocess
import sys
import traceback
PROJECT_ROOT = Path(__file__).parent.parent
AUTOMATION_DIR = PROJECT_ROOT / 'automation'
COMPONENTS_DIR = PROJECT_ROOT / 'components'
GRADLE = PROJECT_ROOT / 'gradlew'
# Ensure this is a proper path, so we can execute it without searching $PATH.
GRADLE = GRADLE.resolve()
IGNORE_PATHS = set([
# let's not run tests just for dependency changes
'megazords/full/DEPENDENCIES.md',
'megazords/full/android/dependency-licenses.xml',
'megazords/ios-rust/DEPENDENCIES.md',
])
def blue_text(text):
if not sys.stdout.isatty():
return text
return '\033[96m{}\033[0m'.format(text)
def yellow_text(text):
if not sys.stdout.isatty():
return text
return '\033[93m{}\033[0m'.format(text)
def get_output(cmdline, **kwargs):
output = subprocess.check_output(cmdline, **kwargs).decode('utf8')
return output
def run_command(cmdline, **kwargs):
print(yellow_text(' '.join(shlex.quote(str(part)) for part in cmdline)))
subprocess.check_call(cmdline, **kwargs)
def path_is_relative_to(path, other):
"""
Implementation of Path.is_relative_to() which was only added in python 3.9
"""
return str(path.resolve()).startswith(str(other.resolve()))
def parse_args():
parser = argparse.ArgumentParser(
description=__doc__,
formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument('mode', help='Testing mode', metavar='MODE')
parser.add_argument('--base-branch', dest='base_branch', default='main',
help='git base branch')
return parser.parse_args()
def on_darwin():
return platform.system() == 'Darwin'
def get_default_target():
return get_output([PROJECT_ROOT / 'taskcluster' / 'scripts' / 'detect-target.sh', '--use-rustc'])
def should_run_rust_tests(package, min_version):
# There are no tests in examples/ packages, so don't waste time on them.
if "examples" in package.manifest_path.parts:
return False
# Skip the error support trybuild tests when running the min version tests, since we keep the
# trybuild output files pinned to the latest rust version.
if min_version and package.name == "error-support-tests":
return False
return True
class BranchChanges:
"""
Tracks which files have been changed in this branch
"""
def __init__(self, base_branch):
# Calculate the merge base, this is the last commit from the base
# branch that's present in this branch.
self.merge_base = get_output([
'git', 'merge-base', 'HEAD', base_branch,
]).strip()
# Use the merge base to calculate which files have changed from the
# base branch.
raw_paths = get_output([
'git', 'diff', '--name-only', self.merge_base,
]).split('\n')
raw_paths = [p for p in raw_paths if p not in IGNORE_PATHS]
self.paths = [PROJECT_ROOT.joinpath(p) for p in raw_paths]
@staticmethod
def has_unstanged_changes():
output = get_output(['git', 'status', '--porcelain=v1'])
return any(line and line[1] == 'M' for line in output.split('\n'))
class RustPackage:
"""
A rust package that we want to test and run clippy on
"""
def __init__(self, cargo_metadata):
self.cargo_metadata = cargo_metadata
self.name = cargo_metadata['name']
# Use manifest path to select the package in cargo. This works better
# when using the --no-default-features flag
self.manifest_path = Path(cargo_metadata['manifest_path'])
self.directory = self.manifest_path.parent
def has_default_features(self):
return bool(self.cargo_metadata.get('features').get('default'))
def has_features(self):
return bool(self.cargo_metadata.get('features'))
def has_changes(self, branch_changes):
return any(path_is_relative_to(p, self.directory)
for p in branch_changes.paths)
class RustFeatures(Enum):
"""
A set of features to for testing/clippy
Rust tests and clippy can be affected by which features are enabled.
Therefore we want to use different combinations of features when running
tests/clippy.
"""
DEFAULT = 'default features'
ALL = 'all features'
NONE = 'no features'
def label(self):
return self.value
def cmdline_args(self):
if self == RustFeatures.DEFAULT:
return []
elif self == RustFeatures.ALL:
return ['--all-features']
elif self == RustFeatures.NONE:
return ['--no-default-features']
def calc_rust_items(branch_changes=None, default_features_only=False):
"""
Calculate which items we want to test and run clippy on
Args:
branch_changes: only yield items for rust packages that have changes
default_features_only: only test with the default features
Returns: list of (RustPackage, RustFeatures) items
"""
json_data = json.loads(get_output([
'cargo', 'metadata', '--no-deps', '--format-version', '1',
]))
packages = [RustPackage(p) for p in json_data['packages']]
if branch_changes:
packages = [p for p in packages if p.has_changes(branch_changes)]
for p in packages:
yield p, RustFeatures.DEFAULT
if default_features_only:
return
for p in packages:
if p.has_features():
yield p, RustFeatures.ALL
for p in packages:
if p.has_default_features():
yield p, RustFeatures.NONE
def calc_non_workspace_rust_items(branch_changes=None, default_features_only=False):
"""
Calculate which items are not in our default workspace, but we might want to
do certain things with.
Returns the same as calc_rust_items
"""
for path in ["testing/sync-test/Cargo.toml"]:
json_data = json.loads(get_output([
'cargo', 'metadata', '--no-deps', '--format-version', '1', '--manifest-path', path
]))
packages = [RustPackage(p) for p in json_data['packages']]
if branch_changes:
packages = [p for p in packages if p.has_changes(branch_changes)]
for p in packages:
yield p, RustFeatures.DEFAULT
if default_features_only:
return
for p in packages:
if p.has_features():
yield p, RustFeatures.ALL
for p in packages:
if p.has_default_features():
yield p, RustFeatures.NONE
# Define a couple functions to avoid this clippy issue:
#
# The safest way to avoid the issue is running cargo clean. For changes mode
# we use the faster method of touching the changed files so only they get
# rebuilt.
def cargo_clean():
"""
Force cargo to rebuild rust files
"""
run_command(['cargo', 'clean'])
def touch_changed_paths(branch_changes):
"""
Quick version of force_rebuild() for change mode
This version just touches any changed paths, which causes them to be
rebuilt, but leaves the rest of the files alone.
"""
for path in branch_changes.paths:
if path.exists():
path.touch()
def print_rust_environment():
print('platform: {}'.format(platform.uname()))
print('rustc version: {}'.format(
get_output(['rustc', '--version']).strip()))
print('cargo version: {}'.format(
get_output(['cargo', '--version']).strip()))
print('rustfmt version: {}'.format(
get_output(['rustfmt', '--version']).strip()))
print('GCC version: {}'.format(
get_output(['gcc', '--version']).split('\n')[0]))
print()
def calc_rust_env(package, features):
if features == RustFeatures.ALL:
# nss-sys's --features handling is broken. Workaround it by using a
# custom --cfg. This shouldn't be this way!
return {
**os.environ,
'RUSTFLAGS' : "--cfg __appsvc_ci_hack"
}
else:
return None
def run_rust_test(package, features):
run_command([
'cargo', 'test',
'--manifest-path', package.manifest_path,
] + features.cmdline_args(), env=calc_rust_env(package, features))
def run_nss_bindings_test():
run_command([
'cargo', 'run', '-p', 'systest',
])
def run_clippy(package, features):
run_command([
'cargo', 'clippy', '--all-targets',
'--manifest-path', package.manifest_path,
] + features.cmdline_args() + [
'--', '-D', 'warnings'
], env=calc_rust_env(package, features))
def run_ktlint():
run_command([GRADLE, 'ktlint', 'detekt'])
def run_swiftlint():
if on_darwin():
run_command(['swiftlint', '--strict'])
elif not docker_installed():
print("WARNING: On non-Darwin hosts, docker is required to run swiftlint")
print("WARNING: skipping swiftlint on non-Darwin host")
else:
cwd = os.getcwd()
run_command(
[
"docker",
"run",
"-it",
"--rm",
"-v",
f"{cwd}:{cwd}",
"-w",
cwd,
"ghcr.io/realm/swiftlint:latest",
"swiftlint",
"--strict"
]
)
def run_gradle_tests():
run_command([GRADLE, 'test'])
def run_ios_tests():
if on_darwin():
run_command([AUTOMATION_DIR / 'run_ios_tests.sh'])
else:
print("WARNING: skipping iOS tests on non-Darwin host")
def run_python_tests():
target=get_default_target()
run_command([PROJECT_ROOT / 'taskcluster/scripts/server-megazord-build.py', 'cirrus', target])
run_command([PROJECT_ROOT / 'taskcluster/scripts/server-megazord-build.py', 'nimbus-experimenter', target])
def cargo_fmt(package=None, fix_issues=False):
cmdline = ['cargo', 'fmt']
if package:
cmdline.extend(['--manifest-path', package.manifest_path])
else:
cmdline.append('--all')
if not fix_issues:
cmdline.extend(['--', '--check'])
run_command(cmdline)
def swift_format():
swift_format_args = [
'megazords',
'components/*/ios',
'--exclude',
'**/Generated',
'--exclude',
'components/nimbus/ios/Nimbus/Utils',
'--lint',
'--swiftversion',
'5',
]
if on_darwin():
run_command(["swiftformat", *swift_format_args])
elif not docker_installed():
print("WARNING: On non-Darwin hosts, docker is required to run swiftformat")
print("WARNING: skipping swiftformat on non-Darwin host")
else:
cwd = os.getcwd()
run_command(
[
"docker",
"run",
"-it",
"--rm",
"-v",
f"{cwd}:{cwd}",
"-w",
cwd,
"ghcr.io/nicklockwood/swiftformat:latest",
*swift_format_args,
]
)
def check_for_fmt_changes(branch_changes):
print()
if branch_changes.has_unstanged_changes():
print("cargo fmt made changes. Make sure to check and commit them.")
else:
print("All checks passed!")
class Step:
"""
Represents a single step of the testing process
"""
def __init__(self, name, func, *args, **kwargs):
self.name = name
self.func = func
self.args = args
self.kwargs = kwargs
def run(self):
print()
print(blue_text('Running {}'.format(self.name)))
try:
self.func(*self.args, **self.kwargs)
except subprocess.CalledProcessError:
exit_with_error(1, 'Error while running {}'.format(self.name))
except:
exit_with_error(
2, 'Unexpected exception while running {}'.format(self.name),
print_exception=True)
def calc_steps(args):
"""
Calculate the steps needed to run the tests
Yields a list of (name, func) items.
"""
if args.mode == 'changes':
# changes mode is complicated enough that it's split off into its own
# function
for step in calc_steps_change_mode(args):
yield step
elif args.mode == 'rust-tests':
print_rust_environment()
yield Step('cargo clean', cargo_clean)
for package, features in calc_rust_items():
if should_run_rust_tests(package, False):
yield Step(
'tests for {} ({})'.format(package.name, features.label()),
run_rust_test, package, features)
elif args.mode == 'rust-min-version-tests':
print_rust_environment()
yield Step('cargo clean', cargo_clean)
for package, features in calc_rust_items():
if should_run_rust_tests(package, True):
yield Step(
'tests for {} ({})'.format(package.name, features.label()),
run_rust_test, package, features)
elif args.mode == 'rust-clippy':
print_rust_environment()
yield Step('cargo clean', cargo_clean)
for package, features in calc_rust_items():
yield Step(
'clippy for {} ({})'.format(package.name, features.label()),
run_clippy, package, features)
# non-workspace items aren't tested, but we do run clippy on them to
# make sure they don't go stale.
for package, features in calc_non_workspace_rust_items():
yield Step(
'clippy for {} ({})'.format(package.name, features.label()),
run_clippy, package, features)
elif args.mode == 'rust-fmt':
print_rust_environment()
yield Step('cargo fmt', cargo_fmt)
elif args.mode == 'ktlint':
yield Step('ktlint', run_ktlint)
elif args.mode == 'swiftlint':
yield Step('swiftlint', run_swiftlint)
elif args.mode == 'swiftformat':
yield Step('swiftformat', swift_format)
elif args.mode == 'nss-bindings':
print_rust_environment()
yield Step('NSS bindings test', run_nss_bindings_test)
elif args.mode == 'gradle':
yield Step('gradle tests', run_gradle_tests)
elif args.mode == 'ios-tests':
yield Step('ios tests', run_ios_tests)
elif args.mode == 'python-tests':
yield Step('python tests', run_python_tests)
else:
print('Invalid mode: {}'.format(args.mode))
sys.exit(1)
def calc_steps_change_mode(args):
"""
Calculate the steps needed for change mode
"""
print_rust_environment()
branch_changes = BranchChanges(args.base_branch)
rust_items = list(calc_rust_items(branch_changes,
default_features_only=True))
rust_packages = list(set(package for package, _ in rust_items))
if not rust_items:
print('no changes found.')
return
if branch_changes.has_unstanged_changes():
subprocess.run(['git', 'status'])
print()
print('WARNING: unstaged changes in your branch:')
print('Consider git add or git commit to stage them since this '
'script will run cargo fmt')
print("Continue (Y/N)?")
if input().lower() != 'y':
sys.exit(0)
yield Step('touch changed paths', touch_changed_paths, branch_changes)
for package, features in rust_items:
yield Step('tests for {} ({})'.format(package.name, features.label()),
run_rust_test, package, features)
for package, features in rust_items:
yield Step('clippy for {} ({})'.format(package.name, features.label()),
run_clippy, package, features)
for package in rust_packages:
yield Step('rustfmt for {}'.format(package.name), cargo_fmt, package,
fix_issues=True)
yield Step('Check for changes', check_for_fmt_changes, branch_changes)
def main():
args = parse_args()
os.chdir(PROJECT_ROOT)
for step in calc_steps(args):
step.run()
def exit_with_error(code, text, print_exception=False):
print()
print('-' * 78)
print()
print(text)
if print_exception:
traceback.print_exc()
sys.exit(code)
def docker_installed():
result = subprocess.run(
["docker"],
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
)
return result.returncode == 0
if __name__ == '__main__':
main()