Source code

Revision control

Other Tools

# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at
from __future__ import absolute_import, print_function, unicode_literals
import os
import shutil
import subprocess
import sys
from pathlib import Path
import mozfile
from mozbuild.base import MozbuildObject
from mozfile import TemporaryDirectory
from mozpack.files import FileFinder
class VendorPython(MozbuildObject):
def vendor(self, keep_extra_files=False):
from mach.python_lockfile import PoetryHandle
vendor_dir = Path(self.topsrcdir) / "third_party" / "python"
requirements_in = vendor_dir / ""
poetry_lockfile = vendor_dir / "poetry.lock"
with TemporaryDirectory() as work_dir:
work_dir = Path(work_dir)
poetry = PoetryHandle(work_dir)
lockfiles = poetry.generate_lockfiles(do_update=False)
# Vendoring packages is only viable if it's possible to have a single
# set of packages that work regardless of which environment they're used in.
# So, we scrub environment markers, so that we essentially ask pip to
# download "all dependencies for all environments". Pip will then either
# fetch them as requested, or intelligently raise an error if that's not
# possible (e.g.: if different versions of Python would result in different
# packages/package versions).
pip_lockfile_without_markers = work_dir / ""
shutil.copy(str(lockfiles.pip_lockfile), str(pip_lockfile_without_markers))
with TemporaryDirectory() as tmp:
# use requirements.txt to download archived source distributions of all
# packages
self._extract(tmp, vendor_dir, keep_extra_files)
shutil.copy(lockfiles.pip_lockfile, str(vendor_dir / "requirements.txt"))
shutil.copy(lockfiles.poetry_lockfile, str(poetry_lockfile))
def _extract(self, src, dest, keep_extra_files=False):
"""extract source distribution into vendor directory"""
ignore = ()
if not keep_extra_files:
ignore = ("*/doc", "*/docs", "*/test", "*/tests", "**/.git")
finder = FileFinder(src)
for archive, _ in finder.find("*"):
_, ext = os.path.splitext(archive)
archive_path = os.path.join(finder.base, archive)
if ext == ".whl":
# Archive is named like "$package-name-1.0-py2.py3-none-any.whl", and should
# have four dashes that aren't part of the package name.
package_name, version, spec, abi, platform_and_suffix = archive.rsplit(
"-", 4
target_package_dir = os.path.join(dest, package_name)
# Extract all the contents of the wheel into the package subdirectory.
# We're expecting at least a code directory and a ".dist-info" directory,
# though there may be a ".data" directory as well.
mozfile.extract(archive_path, target_package_dir, ignore=ignore)
# Archive is named like "$package-name-1.0.tar.gz", and the rightmost
# dash should separate the package name from the rest of the archive
# specifier.
package_name, archive_postfix = archive.rsplit("-", 1)
package_dir = os.path.join(dest, package_name)
# The archive should only contain one top-level directory, which has
# the source files. We extract this directory directly to
# the vendor directory.
extracted_files = mozfile.extract(archive_path, dest, ignore=ignore)
assert len(extracted_files) == 1
extracted_package_dir = extracted_files[0]
# The extracted package dir includes the version in the name,
# which we don't we don't want.
mozfile.move(extracted_package_dir, package_dir)
def _sort_requirements_in(requirements_in: Path):
requirements = {}
with open(requirements_in) as f:
comments = []
for line in f.readlines():
line = line.strip()
if not line or line.startswith("#"):
name, version = line.split("==")
requirements[name] = version, comments
comments = []
with open(requirements_in, "w") as f:
for name, (version, comments) in sorted(requirements.items()):
if comments:
f.write("{}=={}\n".format(name, version))
def remove_environment_markers_from_requirements_txt(requirements_txt: Path):
with open(requirements_txt) as f:
lines = f.readlines()
markerless_lines = []
for line in lines:
if not line.startswith(" ") and not line.startswith("#"):
# The first line of each requirement looks something like:
# package-name==X.Y; python_version>=3.7
# We can scrub the environment marker by splitting on the
# semicolon
with open(requirements_txt, "w") as f:
def _purge_vendor_dir(vendor_dir):
excluded_packages = [
# dlmanager's package on PyPI only has metadata, but is missing the code.
# gyp's package on PyPI doesn't have any downloadable files.
# We manage installing "virtualenv" package manually, and we have a custom
# "" entry module.
# We manage vendoring "vsdownload" with a moz.yaml file (there is no module
# on PyPI).
# The file isn't a vendored module, so don't delete it.
for child in Path(vendor_dir).iterdir():
if not in excluded_packages:
def _denormalize_symlinks(target):
# If any files inside the vendored package were symlinks, turn them into normal files
# because forbids symlinks in the repository.
link_finder = FileFinder(target)
for _, f in link_finder.find("**"):
if os.path.islink(f.path):
link_target = os.path.realpath(f.path)
shutil.copyfile(link_target, f.path)