Source code

Revision control

Copy as Markdown

Other Tools

# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at
from __future__ import absolute_import, print_function, unicode_literals
import os
import shutil
import subprocess
import sys
from pathlib import Path
import mozfile
from mozbuild.base import MozbuildObject
from mozfile import TemporaryDirectory
from mozpack.files import FileFinder
# dlmanager's package on PyPI only has metadata, but is missing the code.
# gyp's package on PyPI doesn't have any downloadable files.
# We keep some wheels vendored in "_venv" for use in Mozharness
# We manage vendoring "vsdownload" with a moz.yaml file (there is no module
# on PyPI).
# The file isn't a vendored module, so don't delete it.
# The ansicon package contains DLLs and we don't want to arbitrarily vendor
# them since they could be unsafe. This module should rarely be used in practice
# (it's a fallback for old versions of windows). We've intentionally vendored a
# modified 'dummy' version of it so that the dependency checks still succeed, but
# if it ever is attempted to be used, it will fail gracefully.
class VendorPython(MozbuildObject):
def __init__(self, *args, **kwargs):
MozbuildObject.__init__(self, *args, virtualenv_name="vendor", **kwargs)
def vendor(self, keep_extra_files=False):
from mach.python_lockfile import PoetryHandle
vendor_dir = Path(self.topsrcdir) / "third_party" / "python"
requirements_in = vendor_dir / ""
poetry_lockfile = vendor_dir / "poetry.lock"
with TemporaryDirectory() as work_dir:
work_dir = Path(work_dir)
poetry = PoetryHandle(work_dir)
lockfiles = poetry.generate_lockfiles(do_update=False)
# Vendoring packages is only viable if it's possible to have a single
# set of packages that work regardless of which environment they're used in.
# So, we scrub environment markers, so that we essentially ask pip to
# download "all dependencies for all environments". Pip will then either
# fetch them as requested, or intelligently raise an error if that's not
# possible (e.g.: if different versions of Python would result in different
# packages/package versions).
pip_lockfile_without_markers = work_dir / ""
shutil.copy(str(lockfiles.pip_lockfile), str(pip_lockfile_without_markers))
with TemporaryDirectory() as tmp:
# use requirements.txt to download archived source distributions of all
# packages
self._extract(tmp, vendor_dir, keep_extra_files)
requirements_out = vendor_dir / "requirements.txt"
# since requirements.out and poetry.lockfile are both outputs from
# third party code, they may contain carriage returns on Windows. We
# should strip the carriage returns to maintain consistency in our output
# regardless of which platform is doing the vendoring. We can do this and
# the copying at the same time to minimize reads and writes.
_copy_file_strip_carriage_return(lockfiles.pip_lockfile, requirements_out)
_copy_file_strip_carriage_return(lockfiles.poetry_lockfile, poetry_lockfile)
def _extract(self, src, dest, keep_extra_files=False):
"""extract source distribution into vendor directory"""
ignore = ()
if not keep_extra_files:
ignore = ("*/doc", "*/docs", "*/test", "*/tests", "**/.git")
finder = FileFinder(src)
for archive, _ in finder.find("*"):
_, ext = os.path.splitext(archive)
archive_path = os.path.join(finder.base, archive)
if ext == ".whl":
# Archive is named like "$package-name-1.0-py2.py3-none-any.whl", and should
# have four dashes that aren't part of the package name.
package_name, version, spec, abi, platform_and_suffix = archive.rsplit(
"-", 4
if package_name in EXCLUDED_PACKAGES:
f"'{package_name}' is on the exclusion list and will not be vendored."
target_package_dir = os.path.join(dest, package_name)
# Extract all the contents of the wheel into the package subdirectory.
# We're expecting at least a code directory and a ".dist-info" directory,
# though there may be a ".data" directory as well.
mozfile.extract(archive_path, target_package_dir, ignore=ignore)
# Archive is named like "$package-name-1.0.tar.gz", and the rightmost
# dash should separate the package name from the rest of the archive
# specifier.
package_name, archive_postfix = archive.rsplit("-", 1)
package_dir = os.path.join(dest, package_name)
if package_name in EXCLUDED_PACKAGES:
f"'{package_name}' is on the exclusion list and will not be vendored."
# The archive should only contain one top-level directory, which has
# the source files. We extract this directory directly to
# the vendor directory.
extracted_files = mozfile.extract(archive_path, dest, ignore=ignore)
assert len(extracted_files) == 1
extracted_package_dir = extracted_files[0]
# The extracted package dir includes the version in the name,
# which we don't we don't want.
mozfile.move(extracted_package_dir, package_dir)
def apply_patches(self):
def _patch_poetry_pypi_repository(self):
# There is a bug in Poetry 1.2.0a2 caused by a breaking change
# This bug was fixed in Poetry 1.2.0b3, but 1.2.0b1 dropped
# support for Python 3.6, which is a requirement for us.
# As a temporary workaround, we can patch the fix into our
# virtualenv's copy of Poetry. This patch should be removed
# once we switch to using a newer version of Poetry.
venv = self.virtualenv_manager._virtualenv
# Get the last element since on Windows the
# first element is the virtualenv root
site_packages = Path(venv.site_packages_dirs()[-1])
expected_poetry_dist_info = site_packages / "poetry-1.2.0a2.dist-info"
# If the specific release of poetry isn't in the site-packages directory
# we should not attempt to patch.
if not expected_poetry_dist_info.exists():
f'The version of Poetry that needs patching ("{}") '
f'could not be found in the "{self.virtualenv_manager._site_name}" virtualenv. '
f"Can this patch be removed?"
file_to_patch = site_packages / "poetry" / "repositories" / ""
with"r") as file:
contents =
contents = contents.replace(
'version_info = json_data["releases"][version]',
'version_info = json_data["urls"]',
print(f'Patching "{file_to_patch}"')
with"w") as file:
def _sort_requirements_in(requirements_in: Path):
requirements = {}
with"r", newline="\n") as f:
comments = []
for line in f.readlines():
line = line.strip()
if not line or line.startswith("#"):
name, version = line.split("==")
requirements[name] = version, comments
comments = []
with"w", newline="\n") as f:
for name, (version, comments) in sorted(requirements.items()):
if comments:
f.write("{}=={}\n".format(name, version))
def remove_environment_markers_from_requirements_txt(requirements_txt: Path):
with"r", newline="\n") as f:
lines = f.readlines()
markerless_lines = []
continuation_token = " \\"
for line in lines:
line = line.rstrip()
if not line.startswith(" ") and not line.startswith("#") and ";" in line:
has_continuation_token = line.endswith(continuation_token)
# The first line of each requirement looks something like:
# package-name==X.Y; python_version>=3.7
# We can scrub the environment marker by splitting on the semicolon
line = line.split(";")[0]
if has_continuation_token:
line += continuation_token
with"w", newline="\n") as f:
def _purge_vendor_dir(vendor_dir):
for child in Path(vendor_dir).iterdir():
def _denormalize_symlinks(target):
# If any files inside the vendored package were symlinks, turn them into normal files
# because forbids symlinks in the repository.
link_finder = FileFinder(target)
for _, f in link_finder.find("**"):
if os.path.islink(f.path):
link_target = os.path.realpath(f.path)
shutil.copyfile(link_target, f.path)
def _copy_file_strip_carriage_return(file_src: Path, file_dst):
shutil.copyfileobj("r"),"w", newline="\n"))