Source code
Revision control
Copy as Markdown
Other Tools
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# Utility package for working with moz.yaml files.
#
# Requires `pyyaml` and `voluptuous`
# (both are in-tree under third_party/python)
import errno
import os
import re
import voluptuous
import yaml
from voluptuous import (
All,
Boolean,
FqdnUrl,
In,
Invalid,
Length,
Match,
Msg,
Required,
Schema,
Unique,
)
from yaml.error import MarkedYAMLError
# TODO ensure this matches the approved list of licenses
VALID_LICENSES = [
"Apache-2.0",
"BSD-2-Clause",
"BSD-3-Clause",
"BSD-3-Clause-Clear",
"BSL-1.0",
"CC0-1.0",
"ISC",
"ICU",
"LGPL-2.1",
"LGPL-3.0",
"MIT",
"MPL-1.1",
"MPL-2.0",
"Public Domain",
"Unlicense",
"WTFPL",
"Zlib",
# Unique Licenses
]
VALID_SOURCE_HOSTS = [
"gitlab",
"googlesource",
"github",
"angle",
"codeberg",
"git",
"yaml-dir",
]
"""
---
# Third-Party Library Template
# All fields are mandatory unless otherwise noted
# Version of this schema
schema: 1
bugzilla:
# Bugzilla product and component for this directory and subdirectories
product: product name
component: component name
# Document the source of externally hosted code
origin:
# Short name of the package/library
name: name of the package
description: short (one line) description
# Full URL for the package's homepage/etc
# Usually different from repository url
url: package's homepage url
# Human-readable identifier for this version/release
# Generally "version NNN", "tag SSS", "bookmark SSS"
release: identifier
# Revision to pull in
# Must be a long or short commit SHA (long preferred)
revision: sha
# The package's license, where possible using the mnemonic from
# Multiple licenses can be specified (as a YAML list)
# A "LICENSE" file must exist containing the full license text
license: MPL-2.0
# If the package's license is specified in a particular file,
# this is the name of the file.
# optional
license-file: COPYING
# If there are any mozilla-specific notes you want to put
# about a library, they can be put here.
notes: Notes about the library
# Configuration for the automated vendoring system.
# optional
vendoring:
# Repository URL to vendor from
# Any repository host can be specified here, however initially we'll only
# support automated vendoring from selected sources.
url: source url (generally repository clone url)
# Type of hosting for the upstream repository
# Valid values are 'gitlab', 'github', googlesource
source-hosting: gitlab
# Type of Vendoring
# This is either 'regular', 'individual-files', or 'rust'
# If omitted, will default to 'regular'
flavor: rust
# Type of git reference (commit, tag) to track updates from.
# You cannot use tag tracking with the individual-files flavor
# If omitted, will default to tracking commits.
tracking: commit
# When using tag tracking (only on Github currently) use a release artifact
# for the source code instead of the automatically built git-archive exports.
# The source repository must build these artifacts with consistent filenames
# for every tag. This is useful when the Github repository uses submodules
# since they are not included in the git-archives.
# Substitution is performed on the filename, {tag} is replaced with the tag name.
# optional
release-artifact: "rnp-{tag}.tar.gz"
# Base directory of the location where the source files will live in-tree.
# If omitted, will default to the location the moz.yaml file is in.
vendor-directory: third_party/directory
# Allows skipping certain steps of the vendoring process.
# Most useful if e.g. vendoring upstream is complicated and should be done by a script
# The valid steps that can be skipped are listed below
skip-vendoring-steps:
- fetch
- keep
- include
- exclude
- move-contents
- hg-add
- spurious-check
- update-moz-yaml
- update-moz-build
# List of patch files to apply after vendoring. Applied in the order
# specified, and alphabetically if globbing is used. Patches must apply
# cleanly before changes are pushed.
# Patch files should be relative to the vendor-directory rather than the gecko
# root directory.
# All patch files are implicitly added to the keep file list.
# optional
patches:
- file
- path/to/file
- path/*.patch
- path/** # Captures all files and subdirectories below path
- path/* # Captures all files but _not_ subdirectories below path. Equivalent to `path/`
# List of files that are not removed from the destination directory while vendoring
# in a new version of the library. Intended for mozilla files not present in upstream.
# Implicitly contains "moz.yaml", "moz.build", and any files referenced in
# "patches"
# optional
keep:
- file
- path/to/file
- another/path
- *.mozilla
# Files/paths that will not be vendored from the upstream repository
# Implicitly contains ".git", and ".gitignore"
# optional
exclude:
- file
- path/to/file
- another/path
- docs
- src/*.test
# Files/paths that will always be vendored from source repository, even if
# they would otherwise be excluded by "exclude".
# optional
include:
- file
- path/to/file
- another/path
- docs/LICENSE.*
# Files that are modified as part of the update process.
# To avoid creating updates that don't update anything, ./mach vendor will detect
# if any in-tree files have changed. If there are files that are always changed
# during an update process (e.g. version numbers or source revisions), list them
# here to avoid having them counted as substative changes.
# This field does NOT support directories or globbing
# optional
generated:
- '{yaml_dir}/vcs_version.h'
# If neither "exclude" or "include" are set, all files will be vendored
# Files/paths in "include" will always be vendored, even if excluded
# eg. excluding "docs/" then including "docs/LICENSE" will vendor just the
# LICENSE file from the docs directory
# All three file/path parameters ("keep", "exclude", and "include") support
# filenames, directory names, and globs/wildcards.
# Actions to take after updating. Applied in order.
# The action subfield is required. It must be one of:
# - copy-file
# - move-file
# - move-dir
# - replace-in-file
# - replace-in-file-regex
# - delete-path
# - run-script
# Unless otherwise noted, all subfields of action are required.
#
# If the action is copy-file, move-file, or move-dir:
# from is the source file
# to is the destination
#
# If the action is replace-in-file or replace-in-file-regex:
# pattern is what in the file to search for. It is an exact strng match.
# with is the string to replace it with. Accepts the special keyword
# '{revision}' for the commit we are updating to.
# File is the file to replace it in.
#
# If the action is delete-path
# path is the file or directory to recursively delete
#
# If the action is run-script:
# script is the script to run
# cwd is the directory the script should run with as its cwd
# args is a list of arguments to pass to the script
#
# If the action is run-command:
# command is the command to run
# Unlike run-script, `command` is _not_ processed to be relative
# to the vendor directory, and is passed directly to python's
# execution code without any path substitution or manipulation
# cwd is the directory the command should run with as its cwd
# args is a list of arguments to pass to the command
#
#
# Unless specified otherwise, all files/directories are relative to the
# vendor-directory. If the vendor-directory is different from the
# directory of the yaml file, the keyword '{yaml_dir}' may be used
# to make the path relative to that directory.
# 'run-script' supports the addictional keyword {cwd} which, if used,
# must only be used at the beginning of the path.
#
# optional
update-actions:
- action: copy-file
from: include/vcs_version.h.in
to: '{yaml_dir}/vcs_version.h'
- action: replace-in-file
pattern: '@VCS_TAG@'
with: '{revision}'
file: '{yaml_dir}/vcs_version.h'
- action: delete-path
path: '{yaml_dir}/config'
- action: run-script
script: '{cwd}/generate_sources.sh'
cwd: '{yaml_dir}'
# Configuration for automatic updating system.
# optional
updatebot:
# TODO: allow multiple users to be specified
# Phabricator username for a maintainer of the library, used for assigning
# reviewers. For a review group, preface with #, such as "#build""
maintainer-phab: tjr
# Bugzilla email address for a maintainer of the library, used for needinfos
maintainer-bz: tom@mozilla.com
# Optional: A preset for ./mach try to use. If present, fuzzy-query and fuzzy-paths will
# be ignored. If it, fuzzy-query, and fuzzy-path are omitted, ./mach try auto will be used
try-preset: media
# Optional: A query string for ./mach try fuzzy. If try-preset, it and fuzzy-paths are omitted
# then ./mach try auto will be used
fuzzy-query: media
# Optional: An array of test paths for ./mach try fuzzy. If try-preset, it and fuzzy-query are
# omitted then ./mach try auto will be used
fuzzy-paths: ['media']
# The tasks that Updatebot can run. Only one of each task is currently permitted
# optional
tasks:
- type: commit-alert
branch: upstream-branch-name
cc: ["bugzilla@email.address", "another@example.com"]
needinfo: ["bugzilla@email.address", "another@example.com"]
enabled: True
filter: security
frequency: every
platform: windows
blocking: 1234
- type: vendoring
branch: master
enabled: False
# frequency can be 'every', 'release', 'N weeks', 'N commits'
# or 'N weeks, M commits' requiring satisfying both constraints.
frequency: 2 weeks
"""
RE_SECTION = re.compile(r"^(\S[^:]*):").search
RE_FIELD = re.compile(r"^\s\s([^:]+):\s+(\S+)$").search
class MozYamlVerifyError(Exception):
def __init__(self, filename, error):
self.filename = filename
self.error = error
def __str__(self):
return "%s: %s" % (self.filename, self.error)
def load_moz_yaml(filename, verify=True, require_license_file=True):
"""Loads and verifies the specified manifest."""
# Load and parse YAML.
try:
with open(filename, "r") as f:
manifest = yaml.load(f, Loader=yaml.BaseLoader)
except IOError as e:
if e.errno == errno.ENOENT:
raise MozYamlVerifyError(filename, "Failed to find manifest: %s" % filename)
raise
except MarkedYAMLError as e:
raise MozYamlVerifyError(filename, e)
if not verify:
return manifest
# Verify schema.
if "schema" not in manifest:
raise MozYamlVerifyError(filename, 'Missing manifest "schema"')
if manifest["schema"] == "1":
schema = _schema_1()
schema_additional = _schema_1_additional
schema_transform = _schema_1_transform
else:
raise MozYamlVerifyError(filename, "Unsupported manifest schema")
try:
schema(manifest)
schema_additional(filename, manifest, require_license_file=require_license_file)
manifest = schema_transform(manifest)
except (voluptuous.Error, ValueError) as e:
raise MozYamlVerifyError(filename, e)
return manifest
def _schema_1():
"""Returns Voluptuous Schema object."""
return Schema(
{
Required("schema"): "1",
Required("bugzilla"): {
Required("product"): All(str, Length(min=1)),
Required("component"): All(str, Length(min=1)),
},
"origin": {
Required("name"): All(str, Length(min=1)),
Required("description"): All(str, Length(min=1)),
"notes": All(str, Length(min=1)),
Required("url"): FqdnUrl(),
Required("license"): Msg(License(), msg="Unsupported License"),
"license-file": All(str, Length(min=1)),
Required("release"): All(str, Length(min=1)),
# The following regex defines a valid git reference
# The first group [^ ~^:?*[\]] matches 0 or more times anything
# that isn't a Space, ~, ^, :, ?, *, or ]
# The second group [^ ~^:?*[\]\.]+ matches 1 or more times
# anything that isn't a Space, ~, ^, :, ?, *, [, ], or .
"revision": Match(r"^[^ ~^:?*[\]]*[^ ~^:?*[\]\.]+$"),
},
"updatebot": {
Required("maintainer-phab"): All(str, Length(min=1)),
Required("maintainer-bz"): All(str, Length(min=1)),
"try-preset": All(str, Length(min=1)),
"fuzzy-query": All(str, Length(min=1)),
"fuzzy-paths": All([str], Length(min=1)),
"tasks": All(
UpdatebotTasks(),
[
{
Required("type"): In(
["vendoring", "commit-alert"],
msg="Invalid type specified in tasks",
),
"branch": All(str, Length(min=1)),
"enabled": Boolean(),
"cc": Unique([str]),
"needinfo": Unique([str]),
"filter": In(
["none", "security", "source-extensions"],
msg="Invalid filter value specified in tasks",
),
"source-extensions": Unique([str]),
"blocking": Match(r"^[0-9]+$"),
"frequency": Match(
r"^(every|release|[1-9][0-9]* weeks?|[1-9][0-9]* commits?|"
+ r"[1-9][0-9]* weeks?, ?[1-9][0-9]* commits?)$"
),
"platform": Match(r"^(windows|linux)$"),
}
],
),
},
"vendoring": {
Required("url"): FqdnUrl(),
Required("source-hosting"): All(
str,
Length(min=1),
In(VALID_SOURCE_HOSTS, msg="Unsupported Source Hosting"),
),
"source-host-path": str,
"tracking": Match(r"^(commit|tag)$"),
"release-artifact": All(str, Length(min=1)),
"flavor": Match(r"^(regular|rust|individual-files)$"),
"skip-vendoring-steps": Unique([str]),
"vendor-directory": All(str, Length(min=1)),
"patches": Unique([str]),
"keep": Unique([str]),
"exclude": Unique([str]),
"include": Unique([str]),
"generated": Unique([str]),
"individual-files": [
{
Required("upstream"): All(str, Length(min=1)),
Required("destination"): All(str, Length(min=1)),
}
],
"individual-files-default-upstream": All(str, Length(min=1)),
"individual-files-default-destination": All(str, Length(min=1)),
"individual-files-list": Unique([str]),
"update-actions": All(
UpdateActions(),
[
{
Required("action"): In(
[
"copy-file",
"move-file",
"move-dir",
"replace-in-file",
"replace-in-file-regex",
"run-script",
"run-command",
"delete-path",
],
msg="Invalid action specified in update-actions",
),
"from": All(str, Length(min=1)),
"to": All(str, Length(min=1)),
"pattern": All(str, Length(min=1)),
"with": All(str, Length(min=1)),
"file": All(str, Length(min=1)),
"script": All(str, Length(min=1)),
"command": All(str, Length(min=1)),
"args": All([All(str, Length(min=1))]),
"cwd": All(str, Length(min=1)),
"path": All(str, Length(min=1)),
}
],
),
},
}
)
def _schema_1_additional(filename, manifest, require_license_file=True):
"""Additional schema/validity checks"""
vendor_directory = os.path.dirname(filename)
if "vendoring" in manifest and "vendor-directory" in manifest["vendoring"]:
vendor_directory = manifest["vendoring"]["vendor-directory"]
# LICENSE file must exist, except for Rust crates which are exempted
# because the license is required to be specified in the Cargo.toml file
if require_license_file and "origin" in manifest:
files = [f.lower() for f in os.listdir(vendor_directory)]
if (
not (
"license-file" in manifest["origin"]
and manifest["origin"]["license-file"].lower() in files
)
and not (
"license" in files
or "license.txt" in files
or "license.rst" in files
or "license.html" in files
or "license.md" in files
)
and not (
"vendoring" in manifest
and manifest["vendoring"].get("flavor", "regular") == "rust"
)
):
license = manifest["origin"]["license"]
if isinstance(license, list):
license = "/".join(license)
raise ValueError("Failed to find %s LICENSE file" % license)
# Cannot vendor without an origin.
if "vendoring" in manifest and "origin" not in manifest:
raise ValueError('"vendoring" requires an "origin"')
# Cannot vendor without a computer-readable revision.
if "vendoring" in manifest and "revision" not in manifest["origin"]:
raise ValueError(
'If "vendoring" is present, "revision" must be present in "origin"'
)
# The Rust and Individual Flavor type precludes a lot of options
# individual-files could, in theory, use several of these, but until we have a use case let's
# disallow them so we're not worrying about whether they work. When we need them we can make
# sure they do.
if (
"vendoring" in manifest
and manifest["vendoring"].get("flavor", "regular") != "regular"
):
for i in [
"skip-vendoring-steps",
"keep",
"exclude",
"include",
"generated",
]:
if i in manifest["vendoring"]:
raise ValueError("A non-regular flavor of update cannot use '%s'" % i)
if manifest["vendoring"].get("flavor", "regular") == "rust":
for i in [
"update-actions",
]:
if i in manifest["vendoring"]:
raise ValueError("A rust flavor of update cannot use '%s'" % i)
# Ensure that only individual-files flavor uses those options
if (
"vendoring" in manifest
and manifest["vendoring"].get("flavor", "regular") != "individual-files"
):
if (
"individual-files" in manifest["vendoring"]
or "individual-files-list" in manifest["vendoring"]
):
raise ValueError(
"Only individual-files flavor of update can use 'individual-files'"
)
# Ensure that release-artifact is only used with tag tracking
if "vendoring" in manifest and "release-artifact" in manifest["vendoring"]:
if (
manifest["vendoring"].get("source-hosting") != "github"
or manifest["vendoring"].get("tracking", "commit") != "tag"
):
raise ValueError(
"You can only use release-artifact with tag tracking from Github."
)
# Ensure that the individual-files flavor has all the correct options
if (
"vendoring" in manifest
and manifest["vendoring"].get("flavor", "regular") == "individual-files"
):
# Because the only way we can determine the latest tag is by doing a local clone,
# we don't want to do that for individual-files flavors because those flavors are
# usually on gigantic repos we don't want to clone for such a simple thing.
if manifest["vendoring"].get("tracking", "commit") == "tag":
raise ValueError(
"You cannot use tag tracking with the individual-files flavor. (Sorry.)"
)
# We need either individual-files or individual-files-list
if (
"individual-files" not in manifest["vendoring"]
and "individual-files-list" not in manifest["vendoring"]
):
raise ValueError(
"The individual-files flavor must include either "
+ "'individual-files' or 'individual-files-list'"
)
# For whichever we have, make sure we don't have the other and we don't have
# options we shouldn't or lack ones we should.
if "individual-files" in manifest["vendoring"]:
if "individual-files-list" in manifest["vendoring"]:
raise ValueError(
"individual-files-list is mutually exclusive with individual-files"
)
if "individual-files-default-upstream" in manifest["vendoring"]:
raise ValueError(
"individual-files-default-upstream can only be used with individual-files-list"
)
if "individual-files-default-destination" in manifest["vendoring"]:
raise ValueError(
"individual-files-default-destination can only be used "
+ "with individual-files-list"
)
if "individual-files-list" in manifest["vendoring"]:
if "individual-files" in manifest["vendoring"]:
raise ValueError(
"individual-files is mutually exclusive with individual-files-list"
)
if "individual-files-default-upstream" not in manifest["vendoring"]:
raise ValueError(
"individual-files-default-upstream must be used with individual-files-list"
)
if "individual-files-default-destination" not in manifest["vendoring"]:
raise ValueError(
"individual-files-default-destination must be used with individual-files-list"
)
if "updatebot" in manifest:
# If there are Updatebot tasks, then certain fields must be present and
# defaults need to be set.
if "tasks" in manifest["updatebot"]:
if "vendoring" not in manifest or "url" not in manifest["vendoring"]:
raise ValueError(
"If Updatebot tasks are specified, a vendoring url must be included."
)
if "try-preset" in manifest["updatebot"]:
for f in ["fuzzy-query", "fuzzy-paths"]:
if f in manifest["updatebot"]:
raise ValueError(
"If 'try-preset' is specified, then %s cannot be" % f
)
# Check for a simple YAML file
with open(filename, "r") as f:
has_schema = False
for line in f.readlines():
m = RE_SECTION(line)
if m:
if m.group(1) == "schema":
has_schema = True
break
if not has_schema:
raise ValueError("Not simple YAML")
# Do type conversion for the few things that need it.
# Everythig is parsed as a string to (a) not cause problems with revisions that
# are only numerals and (b) not strip leading zeros from the numbers if we just
# converted them to string
def _schema_1_transform(manifest):
if "updatebot" in manifest:
if "tasks" in manifest["updatebot"]:
for i in range(len(manifest["updatebot"]["tasks"])):
if "enabled" in manifest["updatebot"]["tasks"][i]:
val = manifest["updatebot"]["tasks"][i]["enabled"]
manifest["updatebot"]["tasks"][i]["enabled"] = (
val.lower() == "true" or val.lower() == "yes"
)
return manifest
class UpdateActions(object):
"""Voluptuous validator which verifies the update actions(s) are valid."""
def __call__(self, values):
for v in values:
if "action" not in v:
raise Invalid("All file-update entries must specify a valid action")
if v["action"] in ["copy-file", "move-file", "move-dir"]:
if "from" not in v or "to" not in v or len(v.keys()) != 3:
raise Invalid(
"%s action must (only) specify 'from' and 'to' keys"
% v["action"]
)
elif v["action"] in ["replace-in-file", "replace-in-file-regex"]:
if (
"pattern" not in v
or "with" not in v
or "file" not in v
or len(v.keys()) != 4
):
raise Invalid(
"replace-in-file action must (only) specify "
+ "'pattern', 'with', and 'file' keys"
)
elif v["action"] == "delete-path":
if "path" not in v or len(v.keys()) != 2:
raise Invalid(
"delete-path action must (only) specify the 'path' key"
)
elif v["action"] == "run-script":
if "script" not in v or "cwd" not in v:
raise Invalid(
"run-script action must specify 'script' and 'cwd' keys"
)
if set(v.keys()) - set(["args", "cwd", "script", "action"]) != set():
raise Invalid(
"run-script action may only specify 'script', 'cwd', and 'args' keys"
)
elif v["action"] == "run-command":
if "command" not in v or "cwd" not in v:
raise Invalid(
"run-command action must specify 'command' and 'cwd' keys"
)
if set(v.keys()) - set(["args", "cwd", "command", "action"]) != set():
raise Invalid(
"run-command action may only specify 'command', 'cwd', and 'args' keys"
)
else:
# This check occurs before the validator above, so the above is
# redundant but we leave it to be verbose.
raise Invalid("Supplied action " + v["action"] + " is invalid.")
return values
def __repr__(self):
return "UpdateActions"
class UpdatebotTasks(object):
"""Voluptuous validator which verifies the updatebot task(s) are valid."""
def __call__(self, values):
seenTaskTypes = set()
for v in values:
if "type" not in v:
raise Invalid("All updatebot tasks must specify a valid type")
if v["type"] in seenTaskTypes:
raise Invalid("Only one type of each task is currently supported")
seenTaskTypes.add(v["type"])
if v["type"] == "vendoring":
for i in ["filter", "branch", "source-extensions"]:
if i in v:
raise Invalid(
"'%s' is only valid for commit-alert task types" % i
)
elif v["type"] == "commit-alert":
pass
else:
# This check occurs before the validator above, so the above is
# redundant but we leave it to be verbose.
raise Invalid("Supplied type " + v["type"] + " is invalid.")
return values
def __repr__(self):
return "UpdatebotTasks"
class License(object):
"""Voluptuous validator which verifies the license(s) are valid as per our
allow list."""
def __call__(self, values):
if isinstance(values, str):
values = [values]
elif not isinstance(values, list):
raise Invalid("Must be string or list")
for v in values:
if v not in VALID_LICENSES:
raise Invalid("Bad License")
return values
def __repr__(self):
return "License"