Revision control
Copy as Markdown
Other Tools
#!/usr/bin/env python3
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# This script provides one-line bootstrap support to configure systems to build
# the tree. It does so by cloning the repo before calling directly into `mach
# bootstrap`.
# mozboot bootstrap.py was mangled and maimed in the creation of this script.
# Note that this script can't assume anything in particular about the host
# Python environment (except that it's run with a sufficiently recent version of
# Python 3), so we are restricted to stdlib modules.
import sys
major, minor = sys.version_info[:2]
if (major < 3) or (major == 3 and minor < 8):
print("Bootstrap currently only runs on Python 3.8+." "Please try re-running with python3.8+.")
sys.exit(1)
import ctypes
import os
import shutil
import subprocess
import tempfile
from optparse import OptionParser
from pathlib import Path
CLONE_MERCURIAL_PULL_FAIL = """
Failed to pull from hg.mozilla.org.
This is most likely because of unstable network connection.
or download a mercurial bundle and use it:
WINDOWS = sys.platform.startswith("win32") or sys.platform.startswith("msys")
VCS_HUMAN_READABLE = {
"hg": "Mercurial",
"git": "Git",
}
def which(name):
"""Python implementation of which.
It returns the path of an executable or None if it couldn't be found.
"""
search_dirs = os.environ["PATH"].split(os.pathsep)
potential_names = [name]
if WINDOWS:
potential_names.insert(0, name + ".exe")
for path in search_dirs:
for executable_name in potential_names:
test = Path(path) / executable_name
if test.is_file() and os.access(test, os.X_OK):
return test
return None
def validate_clone_dest(dest: Path):
dest = dest.resolve()
if not dest.exists():
return dest
if not dest.is_dir():
print(f"ERROR! Destination {dest} exists but is not a directory.")
return None
if not any(dest.iterdir()):
return dest
else:
print(f"ERROR! Destination directory {dest} exists but is nonempty.")
print(
f"To re-bootstrap the existing checkout, go into '{dest}' and run './mach bootstrap'."
)
return None
def input_clone_dest(vcs, no_interactive):
repo_name = "mozilla-unified"
print(f"Cloning into {repo_name} using {VCS_HUMAN_READABLE[vcs]}...")
while True:
dest = None
if not no_interactive:
dest = input(
f"Destination directory for clone (leave empty to use "
f"default destination of {repo_name}): "
).strip()
if not dest:
dest = repo_name
dest = validate_clone_dest(Path(dest).expanduser())
if dest:
return dest
if no_interactive:
return None
def hg_clone(hg: Path, repo, dest: Path, watchman, head_repo, head_rev):
print(f"Cloning {repo} to {dest}...")
# We create an empty repo then modify the config before adding data.
# This is necessary to ensure storage settings are optimally
# configured.
args = [
str(hg),
# The unified repo is generaldelta, so ensure the client is as
# well.
"--config",
"format.generaldelta=true",
"init",
str(dest),
]
res = subprocess.call(args)
if res:
print("unable to create destination repo; please try cloning manually")
return None
# Strictly speaking, this could overwrite a config based on a template
# the user has installed. Let's pretend this problem doesn't exist
# unless someone complains about it.
with open(dest / ".hg" / "hgrc", "a") as fh:
fh.write("[paths]\n")
fh.write("\n")
# The server uses aggressivemergedeltas which can blow up delta chain
# length. This can cause performance to tank due to delta chains being
# too long. Limit the delta chain length to something reasonable
# to bound revlog read time.
fh.write("[format]\n")
fh.write("# This is necessary to keep performance in check\n")
fh.write("maxchainlen = 10000\n")
# Pulling a specific revision into an empty repository induces a lot of
# load on the Mercurial server, so we always pull from mozilla-unified (which,
# when done from an empty repository, is equivalent to a clone), and then pull
# the specific revision we want (if we want a specific one, otherwise we just
# use the "central" bookmark), at which point it will be an incremental pull,
# that the server can process more easily.
# This is the same thing that robustcheckout does on automation.
res = subprocess.call(
)
if not res and head_repo:
res = subprocess.call([str(hg), "pull", head_repo, "-r", head_rev], cwd=str(dest))
print("")
if res:
print(CLONE_MERCURIAL_PULL_FAIL % dest)
return None
update_rev = {"mozilla-unified": "central", "comm-unified": "comm"}[repo]
head_rev = head_rev or update_rev
print("updating to {}".format(head_rev))
res = subprocess.call([str(hg), "update", "-r", head_rev], cwd=str(dest))
if res:
print(f"error updating; you will need to `cd {dest} && hg update -r {head_rev}` manually")
return dest
def git_clone(git: Path, repo, dest: Path, watchman: Path, head_repo, head_rev):
print(f"Cloning {repo} to {dest}...")
tempdir = None
cinnabar = None
env = dict(os.environ)
try:
cinnabar = which("git-cinnabar")
if not cinnabar:
from urllib.request import urlopen
# If git-cinnabar isn't installed already, that's fine; we can
# download a temporary copy. `mach bootstrap` will install a copy
# in the state dir; we don't want to copy all that logic to this
# tiny bootstrapping script.
tempdir = Path(tempfile.mkdtemp())
with open(tempdir / "download.py", "wb") as fh:
shutil.copyfileobj(urlopen(f"{cinnabar_url}/raw/master/download.py"), fh)
subprocess.check_call(
[sys.executable, str(tempdir / "download.py")],
cwd=str(tempdir),
)
env["PATH"] = str(tempdir) + os.pathsep + env["PATH"]
print(
"WARNING! git-cinnabar is required for Firefox development "
"with git. After the clone is complete, the bootstrapper "
"will ask if you would like to configure git; answer yes, "
"and be sure to add git-cinnabar to your PATH according to "
"the bootstrapper output."
)
# We're guaranteed to have `git-cinnabar` installed now.
# Configure git per the git-cinnabar requirements.
subprocess.check_call(
[
str(git),
"clone",
"--no-checkout",
str(dest),
],
env=env,
)
subprocess.check_call([str(git), "config", "fetch.prune", "true"], cwd=str(dest), env=env)
subprocess.check_call([str(git), "config", "pull.ff", "only"], cwd=str(dest), env=env)
if head_repo:
subprocess.check_call(
[str(git), "cinnabar", "fetch", f"hg::{head_repo}", head_rev],
cwd=str(dest),
env=env,
)
bookmark = {"mozilla-unified": "bookmarks/central", "comm-unified": "bookmarks/comm"}[repo]
subprocess.check_call(
[str(git), "checkout", "FETCH_HEAD" if head_rev else bookmark],
cwd=str(dest),
env=env,
)
watchman_sample = dest / ".git/hooks/fsmonitor-watchman.sample"
# Older versions of git didn't include fsmonitor-watchman.sample.
if watchman and watchman_sample.exists():
print("Configuring watchman")
watchman_config = dest / ".git/hooks/query-watchman"
if not watchman_config.exists():
print(f"Copying {watchman_sample} to {watchman_config}")
copy_args = [
"cp",
".git/hooks/fsmonitor-watchman.sample",
".git/hooks/query-watchman",
]
subprocess.check_call(copy_args, cwd=str(dest))
config_args = [
str(git),
"config",
"core.fsmonitor",
".git/hooks/query-watchman",
]
subprocess.check_call(config_args, cwd=str(dest), env=env)
return dest
finally:
if tempdir:
shutil.rmtree(str(tempdir))
def add_microsoft_defender_antivirus_exclusions(dest, no_system_changes):
if no_system_changes:
return
if not WINDOWS:
return
powershell_exe = which("powershell")
if not powershell_exe:
return
def print_attempt_exclusion(path):
print(f"Attempting to add exclusion path to Microsoft Defender Antivirus for: {path}")
powershell_exe = str(powershell_exe)
paths = []
# mozilla-unified / clone dest
repo_dir = Path.cwd() / dest
paths.append(repo_dir)
print_attempt_exclusion(repo_dir)
# MOZILLABUILD
mozillabuild_dir = os.getenv("MOZILLABUILD")
if mozillabuild_dir:
paths.append(mozillabuild_dir)
print_attempt_exclusion(mozillabuild_dir)
# .mozbuild
mozbuild_dir = Path.home() / ".mozbuild"
paths.append(mozbuild_dir)
print_attempt_exclusion(mozbuild_dir)
args = ";".join(f"Add-MpPreference -ExclusionPath '{path}'" for path in paths)
command = f'-Command "{args}"'
# This will attempt to run as administrator by triggering a UAC prompt
# for admin credentials. If "No" is selected, no exclusions are added.
ctypes.windll.shell32.ShellExecuteW(None, "runas", powershell_exe, command, None, 0)
def clone(options):
vcs = options.vcs
no_interactive = options.no_interactive
no_system_changes = options.no_system_changes
if vcs == "hg":
hg = which("hg")
if not hg:
print("Mercurial is not installed. Mercurial is required to clone Thunderbird.")
try:
# We're going to recommend people install the Mercurial package with
# pip3. That will work if `pip3` installs binaries to a location
# that's in the PATH, but it might not be. To help out, if we CAN
# import "mercurial" (in which case it's already been installed),
# offer that as a solution.
import mercurial # noqa: F401
print(
"Hint: have you made sure that Mercurial is installed to a "
"location in your PATH?"
)
except ImportError:
print("Try installing hg with `pip3 install Mercurial`.")
return None
binary = hg
else:
binary = which(vcs)
if not binary:
print("Git is not installed.")
print("Try installing git using your system package manager.")
return None
dest = input_clone_dest(vcs, no_interactive)
if not dest:
return None
add_microsoft_defender_antivirus_exclusions(dest, no_system_changes)
if vcs == "hg":
clone_func = hg_clone
else:
clone_func = git_clone
watchman = which("watchman")
print(f"Cloning Thunderbird {VCS_HUMAN_READABLE[vcs]} repository to {dest}")
gecko_head_repo = os.environ.get("GECKO_HEAD_REPOSITORY")
gecko_head_rev = os.environ.get("GECKO_HEAD_REV")
comm_head_repo = os.environ.get("COMM_HEAD_REPOSITORY")
comm_head_rev = os.environ.get("COMM_HEAD_REV")
mc = clone_func(binary, "mozilla-unified", dest, watchman, gecko_head_repo, gecko_head_rev)
# Funny logic... the return value if successful needs to be the path
# to mozilla-central. Only return "cc" if cloning comm-central
# fails.
if mc == dest:
cc_dest = Path(dest) / "comm"
cc = clone_func(binary, "comm-unified", cc_dest, watchman, comm_head_repo, comm_head_rev)
if cc:
return mc
else:
return cc
return mc
def bootstrap(srcdir: Path, artifact_mode, no_interactive, no_system_changes):
args = [sys.executable, "mach"]
if no_interactive:
# --no-interactive is a global argument, not a command argument,
# so it needs to be specified before "bootstrap" is appended.
args += ["--no-interactive"]
args += ["bootstrap"]
if artifact_mode:
args += ["--application-choice", "Firefox for Desktop Artifact Mode"]
else:
args += ["--application-choice", "Firefox for Desktop"]
if no_system_changes:
args += ["--no-system-changes"]
print("Running `%s`" % " ".join(args))
return subprocess.call(args, cwd=str(srcdir))
def mozconfig(srcdir):
"""Build Thunderbird, not Firefox!"""
mozconfig = os.path.join(srcdir, "mozconfig")
with open(mozconfig, "a") as mfp:
mfp.write("ac_add_options --enable-project=comm/mail")
return True
def main(args):
parser = OptionParser()
parser.add_option(
"--artifact-mode",
dest="artifact_mode",
help="Build Thunderbird in Artifact mode. "
"/artifact_builds.html for details.",
)
parser.add_option(
"--vcs",
dest="vcs",
default="hg",
choices=["git", "hg"],
help="VCS (hg or git) to use for downloading the source code, "
"instead of using the default interactive prompt.",
)
parser.add_option(
"--no-interactive",
dest="no_interactive",
action="store_true",
help="Answer yes to any (Y/n) interactive prompts.",
)
parser.add_option(
"--no-system-changes",
dest="no_system_changes",
action="store_true",
help="Only executes actions that leave the system " "configuration alone.",
)
options, leftover = parser.parse_args(args)
try:
srcdir = clone(options)
if not srcdir:
return 1
print("Clone complete.")
print(
"If you need to run the tooling bootstrapping again, "
"then consider running './mach bootstrap' instead."
)
if not options.no_interactive:
remove_bootstrap_file = input(
"Unless you are going to have more local copies of Firefox source code, "
"this 'bootstrap.py' file is no longer needed and can be deleted. "
"Clean up the bootstrap.py file? (Y/n)"
)
if not remove_bootstrap_file:
remove_bootstrap_file = "y"
if options.no_interactive or remove_bootstrap_file == "y":
try:
Path(sys.argv[0]).unlink()
except FileNotFoundError:
print("File could not be found !")
bootstrap(
srcdir,
options.artifact_mode,
options.no_interactive,
options.no_system_changes,
)
return mozconfig(srcdir)
except Exception:
print("Could not bootstrap Thunderbird! Consider filing a bug.")
raise
if __name__ == "__main__":
sys.exit(main(sys.argv))