Source code
Revision control
Copy as Markdown
Other Tools
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
import argparse
import atexit
import os
import re
from filter_git_changes import filter_git_changes
from run_operations import get_last_line, run_hg, run_shell, update_resume_state
# This script vendors moz-libwebrtc, handles add/deletes/renames and
# commits the newly vendored code with the provided commit message.
error_help = None
script_name = os.path.basename(__file__)
def early_exit_handler():
print("*** ERROR *** {} did not complete successfully".format(script_name))
if error_help is not None:
print(error_help)
print(
"Please resolve the error and then continue running {}".format(script_name)
)
def log_output_lines(lines, log_dir, filename):
if len(lines) == 0:
return
with open(os.path.join(log_dir, filename), "w") as ofile:
for line in lines:
ofile.write(line)
ofile.write("\n")
def vendor_current_stack(github_branch, github_path, script_dir):
print("-------")
print("------- Vendor {} from {}".format(github_branch, github_path))
print("-------")
cmd = [
"./mach",
"python",
"{}/vendor-libwebrtc.py".format(script_dir),
"--from-local",
github_path,
"--commit",
github_branch,
"libwebrtc",
]
cmd = " ".join(cmd)
run_shell(cmd)
def restore_mozbuild_files(target_dir, log_dir):
print("-------")
print("------- Restore moz.build files from repo")
print("-------")
cmd = 'hg revert --include "{}/**moz.build" {}'.format(target_dir, target_dir)
stdout_lines = run_shell(cmd) # run_shell to allow file wildcard
log_output_lines(stdout_lines, log_dir, "log-regen-mozbuild-files.txt")
def remove_deleted_upstream_files(
github_path, github_sha, target_dir, log_dir, handle_noop_commit
):
if handle_noop_commit:
return
deleted_paths = filter_git_changes(github_path, github_sha, "D")
deleted_paths = [re.sub("^.\t", "", x) for x in deleted_paths]
deleted_paths = [os.path.join(target_dir, x) for x in deleted_paths]
if len(deleted_paths) != 0:
print("-------")
print("------- Remove deleted upstream files")
print("-------")
cmd = "hg rm {}".format(" ".join(deleted_paths))
stdout_lines = run_hg(cmd)
log_output_lines(stdout_lines, log_dir, "log-deleted-upstream-files.txt")
def add_new_upstream_files(
github_path, github_sha, target_dir, log_dir, handle_noop_commit
):
if handle_noop_commit:
return
added_paths = filter_git_changes(github_path, github_sha, "A")
added_paths = [re.sub("^.\t", "", x) for x in added_paths]
added_paths = [os.path.join(target_dir, x) for x in added_paths]
if len(added_paths) != 0:
print("-------")
print("------- Add new upstream files")
print("-------")
cmd = "hg add {}".format(" ".join(added_paths))
stdout_lines = run_hg(cmd)
log_output_lines(stdout_lines, log_dir, "log-new-upstream-files.txt")
def handle_renamed_upstream_files(
github_path, github_sha, target_dir, log_dir, handle_noop_commit
):
if handle_noop_commit:
return
renamed_paths = filter_git_changes(github_path, github_sha, "R")
renamed_paths = [re.sub("^.[0-9]+\t", "", x) for x in renamed_paths]
renamed_paths = [x.split("\t") for x in renamed_paths]
renamed_paths = [
" ".join([os.path.join(target_dir, file) for file in rename_pair])
for rename_pair in renamed_paths
]
if len(renamed_paths) != 0:
print("-------")
print("------- Handle renamed upstream files")
print("-------")
for x in renamed_paths:
cmd = "hg rename --after {}".format(x)
stdout_lines = run_hg(cmd)
log_output_lines(stdout_lines, log_dir, "log-renamed-upstream-files.txt")
def commit_all_changes(github_sha, commit_msg_filename, target_dir):
print("-------")
print("------- Commit vendored changes from {}".format(github_sha))
print("-------")
cmd = "hg commit -l {} {}".format(commit_msg_filename, target_dir)
run_hg(cmd)
def vendor_and_commit(
script_dir,
github_path,
github_branch,
github_sha,
target_dir,
state_dir,
log_dir,
commit_msg_filename,
):
# register the exit handler after the arg parser completes so '--help' doesn't exit with
# an error.
atexit.register(early_exit_handler)
print("script_dir: {}".format(script_dir))
print("github_path: {}".format(github_path))
print("github_branch: {}".format(github_branch))
print("github_sha: {}".format(github_sha))
print("target_dir: {}".format(target_dir))
print("state_dir: {}".format(state_dir))
print("log_dir: {}".format(log_dir))
print("commit_msg_filename: {}".format(commit_msg_filename))
resume_state_filename = os.path.join(state_dir, "vendor_and_commit.resume")
noop_commit_path = os.path.join(state_dir, "{}.no-op-cherry-pick-msg").format(
github_sha
)
handle_noop_commit = os.path.exists(noop_commit_path)
print("noop_commit_path: {}".format(noop_commit_path))
print("handle_noop_commit: {}".format(handle_noop_commit))
if handle_noop_commit:
print("***")
print("*** Detected special commit msg, setting handle_noop_commit.")
print("*** This commit is flagged as having been handled by a")
print("*** previous commit, meaning the changed file count between")
print("*** upstream and our vendored commit will not match. The")
print("*** commit message is annotated with info on the previous")
print("*** commit.")
print("***")
global error_help
resume_state = ""
if os.path.exists(resume_state_filename):
resume_state = get_last_line(resume_state_filename).strip()
print("resume_state: '{}'".format(resume_state))
if len(resume_state) == 0:
update_resume_state("resume2", resume_state_filename)
error_help = (
"Running script '{}/vendor-libwebrtc.py' failed.\n"
"Please manually confirm that all changes from git ({})\n"
"are reflected in the output of 'hg diff'"
).format(script_dir, github_path)
vendor_current_stack(github_branch, github_path, script_dir)
error_help = None
if len(resume_state) == 0 or resume_state == "resume2":
resume_state = ""
update_resume_state("resume3", resume_state_filename)
error_help = (
"An error occurred while restoring moz.build files after vendoring.\n"
"Verify no moz.build files are modified, missing, or changed."
)
restore_mozbuild_files(target_dir, log_dir)
error_help = None
if len(resume_state) == 0 or resume_state == "resume3":
resume_state = ""
update_resume_state("resume4", resume_state_filename)
error_help = (
"An error occurred while removing deleted upstream files.\n"
"Verify files deleted in the newest upstream commit are also\n"
"shown as deleted in the output of 'hg status'"
)
remove_deleted_upstream_files(
github_path, github_sha, target_dir, log_dir, handle_noop_commit
)
error_help = None
if len(resume_state) == 0 or resume_state == "resume4":
resume_state = ""
update_resume_state("resume5", resume_state_filename)
error_help = (
"An error occurred while adding new upstream files.\n"
"Verify files added in the newest upstream commit are also\n"
"shown as added in the output of 'hg status'"
)
add_new_upstream_files(
github_path, github_sha, target_dir, log_dir, handle_noop_commit
)
error_help = None
if len(resume_state) == 0 or resume_state == "resume5":
resume_state = ""
update_resume_state("resume6", resume_state_filename)
error_help = (
"An error occurred while adding handling renamed upstream files.\n"
"Verify files renamed in the newest upstream commit are also\n"
"shown as renamed/moved in the output of 'hg status'"
)
handle_renamed_upstream_files(
github_path, github_sha, target_dir, log_dir, handle_noop_commit
)
error_help = None
if len(resume_state) == 0 or resume_state == "resume6":
resume_state = ""
update_resume_state("", resume_state_filename)
error_help = (
"An error occurred while committing the vendored changes to Mercurial.\n"
)
commit_all_changes(github_sha, commit_msg_filename, target_dir)
error_help = None
# unregister the exit handler so the normal exit doesn't falsely
# report as an error.
atexit.unregister(early_exit_handler)
if __name__ == "__main__":
default_target_dir = "third_party/libwebrtc"
default_state_dir = ".moz-fast-forward"
default_log_dir = ".moz-fast-forward/logs"
parser = argparse.ArgumentParser(
description="Vendor from local copy of moz-libwebrtc and commit"
)
parser.add_argument(
"--repo-path",
required=True,
help="path to libwebrtc repo",
)
parser.add_argument(
"--script-path",
required=True,
help="path to script directory",
)
parser.add_argument(
"--commit-sha",
required=True,
help="sha of commit to examine",
)
parser.add_argument(
"--branch",
default="mozpatches",
help="moz-libwebrtc branch (defaults to mozpatches)",
)
parser.add_argument(
"--target-path",
default=default_target_dir,
help="target path for vendoring (defaults to {})".format(default_target_dir),
)
parser.add_argument(
"--state-path",
default=default_state_dir,
help="path to state directory (defaults to {})".format(default_state_dir),
)
parser.add_argument(
"--log-path",
default=default_log_dir,
help="path to log directory (defaults to {})".format(default_log_dir),
)
parser.add_argument(
"--commit-msg-path",
required=True,
help="path to file containing commit message",
)
args = parser.parse_args()
vendor_and_commit(
args.script_path,
args.repo_path,
args.branch,
args.commit_sha,
args.target_path, # os.path.abspath(args.target_path),
args.state_path,
args.log_path,
args.commit_msg_path,
)