Revision control
Copy as Markdown
#!/usr/bin/python3
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
import argparse
import subprocess
import pathlib
import os
import shutil
import tempfile
# Repository root dir
SRC_ROOT = pathlib.Path(
subprocess.check_output(['git', 'rev-parse', '--show-toplevel']).decode('utf8').strip()
).resolve()
PATH_NOT_SPECIFIED = pathlib.Path('/not specified').resolve()
PWD = pathlib.Path().resolve()
DEBUG = False
TARGET_DETECTOR_SCRIPT = SRC_ROOT / 'taskcluster' / 'scripts'/ 'detect-target.sh'
def main():
args = parse_args()
megazord = args.megazord
target = args.target
out_dir = PWD / args.out_dir
_dir = SRC_ROOT / 'megazords' / megazord
if not _dir.is_dir():
raise NotADirectoryError(f'Megazord {megazord} does not exist to build')
if DEBUG:
temp_dir = None
dist_dir = PWD / 'dist'
os.makedirs(dist_dir, exist_ok=True)
else:
temp_dir = tempfile.TemporaryDirectory()
dist_dir = pathlib.Path(temp_dir.name)
try:
filename = _build_shared_library(megazord, target, dist_dir)
if _target_matches_host(target):
_run_python_tests(megazord, dist_dir)
_prepare_artifact(megazord, target, filename, dist_dir)
if str(out_dir) != str(PATH_NOT_SPECIFIED):
os.makedirs(out_dir, exist_ok=True)
_create_artifact(megazord, target, dist_dir, out_dir)
finally:
if not DEBUG:
temp_dir.cleanup()
def _build_shared_library(megazord, target, dist_dir):
env = os.environ.copy()
binary=megazord.replace('-', '_')
if '-linux' in target:
filename = f'lib{binary}.so'
elif '-darwin' in target:
filename = f'lib{binary}.dylib'
elif '-win' in target:
filename = f'{binary}.dll'
else:
raise NotImplementedError(f'Only targets for linux, darwin or windows available')
if "-musl" in target:
env["RUSTFLAGS"] = (
env.get("RUSTFLAGS", "") + " -C target-feature=-crt-static"
)
if _host_os() == 'unknown-linux':
env["RUSTFLAGS"] = (
env.get("RUSTFLAGS", "") + " -C link-arg=-lgcc"
)
elif _host_os() == 'apple-darwin':
if "x86_64" in target:
env["CARGO_TARGET_X86_64_UNKNOWN_LINUX_MUSL_LINKER"] = "x86_64-linux-musl-gcc"
env["TARGET_CC"] = "x86_64-linux-musl-gcc"
elif "aarch64" in target:
env["CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_LINKER"] = "aarch64-linux-musl-gcc"
if target == "x86_64-pc-windows-gnu":
env["RUSTFLAGS"] = env.get("RUSTFLAGS", "") + " -C panic=abort"
elif target == 'aarch64-unknown-linux-gnu':
env["CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER"] = 'aarch64-linux-gnu-gcc'
subprocess.check_call([
'cargo', 'build', '--manifest-path', f'{SRC_ROOT}/megazords/{megazord}/Cargo.toml', '--release', '--target', target,
], env=env, cwd=SRC_ROOT)
# Move the .so file to the dist_directory
shutil.move(SRC_ROOT / 'target' / target / 'release' / filename, dist_dir / filename)
# This is only temporary, until cirrus uses pre-built binaries.
_patch_uniffi_tomls()
# Generate the Python FFI. We do this with `--library` so we don't have to specify the UDL or the uniffi.toml file.
subprocess.check_call([
'cargo', 'uniffi-bindgen', 'generate', '--library', dist_dir / filename, '--language', 'python', '--out-dir', dist_dir,
], env=env, cwd=SRC_ROOT)
return filename
def _patch_uniffi_tomls():
_replace_text(SRC_ROOT / 'components' / 'support' / 'nimbus-fml' / 'uniffi.toml', '\ncdylib_name', '\n# cdylib_name')
_replace_text(SRC_ROOT / 'components' / 'nimbus' / 'uniffi.toml', '\ncdylib_name', '\n# cdylib_name')
def _replace_text(filename, search, replace):
with open(filename, 'r') as file:
data = file.read()
data = data.replace(search, replace)
with open(filename, 'w') as file:
file.write(data)
def _run_python_tests(megazord, dist_dir):
env = os.environ.copy()
existing = env.get("PYTHONPATH", None)
dist_path = [str(dist_dir)] + _python_sources(megazord)
if existing is None:
env["PYTHONPATH"] = ":".join(dist_path)
else:
env["PYTHONPATH"] = ":".join(existing.split(":") + dist_path)
test_dirs = _python_tests(megazord)
for d in test_dirs:
subprocess.check_call([
'pytest', '-s', d,
], env=env, cwd=SRC_ROOT)
def _target_matches_host(target):
return _host_os() in target and _host_machine() in target
def _host_machine():
import platform
m = platform.machine().lower()
if m in ('i386', 'amd64', 'x86_64'):
return 'x86_64'
elif m in ('arm64', 'aarch64'):
return 'aarch64'
else:
return m
def _host_os():
import platform
s = platform.system().lower()
if 'windows' in s:
return 'windows'
elif 'linux' in s:
return 'unknown-linux'
elif 'darwin' in s:
return 'apple-darwin'
else:
return s
def _python_sources(megazord):
return _dirs(f'{SRC_ROOT}/megazords/{megazord}', ['python/lib', 'python/src'])
def _python_tests(megazord):
return _dirs(f'{SRC_ROOT}/megazords/{megazord}', ['tests/python-tests', 'python/test'])
def _dirs(prefix, list):
return [f'{prefix}/{f}' for f in list if os.path.isdir(f'{prefix}/{f}')]
def _prepare_artifact(megazord, target, filename, dist_dir):
for f in _python_sources(megazord):
shutil.copytree(f, dist_dir)
# Move the binary into a target specific directory.
# This is so shared libraries for the same OS, but different architectures
# don't overwrite one another.
target_dir = dist_dir / target
os.makedirs(target_dir, exist_ok=True)
shutil.move(dist_dir / filename, target_dir / filename)
scripts_dir = dist_dir / 'scripts'
os.makedirs(scripts_dir, exist_ok=True)
shutil.copy(TARGET_DETECTOR_SCRIPT, scripts_dir)
def _create_artifact(megazord, target, dist_dir, out_dir):
archive = out_dir / f'{megazord}-{target}.zip'
subprocess.check_call([
'zip', archive,
'-r', '.',
'-x', '*/__pycache__/*', '__pycache__/*',
], cwd=dist_dir)
print(f"Archive complete: {archive}")
def parse_args():
parser = argparse.ArgumentParser(prog='server-megazord-build.py')
parser.add_argument('megazord')
parser.add_argument('target')
parser.add_argument('out_dir', nargs='?', type=pathlib.Path, default=PATH_NOT_SPECIFIED)
return parser.parse_args()
if __name__ == '__main__':
main()