Source code
Revision control
Copy as Markdown
Other Tools
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
import json
import os
import sys
from collections import OrderedDict
from collections.abc import Iterable
from pathlib import Path
from types import ModuleType
import mozpack.path as mozpath
import six
from mozbuild.shellutil import quote as shell_quote
from mozbuild.util import (
FileAvoidWrite,
ReadOnlyDict,
memoized_property,
system_encoding,
)
class ConfigStatusFailure(Exception):
"""Error loading config.status"""
class BuildConfig(object):
"""Represents the output of configure."""
_CODE_CACHE = {}
def __init__(self):
self.topsrcdir = None
self.topobjdir = None
self.defines = {}
self.substs = {}
self.files = []
self.mozconfig = None
@classmethod
def from_config_status(cls, path):
"""Create an instance from a config.status file."""
code_cache = cls._CODE_CACHE
mtime = os.path.getmtime(path)
# cache the compiled code as it can be reused
# we cache it the first time, or if the file changed
if path not in code_cache or code_cache[path][0] != mtime:
# Add config.status manually to sys.modules so it gets picked up by
# iter_modules_in_path() for automatic dependencies.
mod = ModuleType("config.status")
mod.__file__ = path
sys.modules["config.status"] = mod
with open(path, "rt") as fh:
source = fh.read()
code_cache[path] = (
mtime,
compile(source, path, "exec", dont_inherit=1),
)
g = {"__builtins__": __builtins__, "__file__": path}
l = {}
try:
exec(code_cache[path][1], g, l)
except Exception:
raise ConfigStatusFailure()
config = BuildConfig()
for name in l["__all__"]:
setattr(config, name, l[name])
return config
class ConfigEnvironment(object):
"""Perform actions associated with a configured but bare objdir.
The purpose of this class is to preprocess files from the source directory
and output results in the object directory.
There are two types of files: config files and config headers,
each treated through a different member function.
Creating a ConfigEnvironment requires a few arguments:
- topsrcdir and topobjdir are, respectively, the top source and
the top object directory.
- defines is a dict filled from AC_DEFINE and AC_DEFINE_UNQUOTED in autoconf.
- substs is a dict filled from AC_SUBST in autoconf.
ConfigEnvironment automatically defines one additional substs variable
from all the defines:
- ACDEFINES contains the defines in the form -DNAME=VALUE, for use on
preprocessor command lines. The order in which defines were given
when creating the ConfigEnvironment is preserved.
and two other additional subst variables from all the other substs:
- ALLSUBSTS contains the substs in the form NAME = VALUE, in sorted
order, for use in autoconf.mk. It includes ACDEFINES.
Only substs with a VALUE are included, such that the resulting file
doesn't change when new empty substs are added.
This results in less invalidation of build dependencies in the case
of autoconf.mk..
- ALLEMPTYSUBSTS contains the substs with an empty value, in the form NAME =.
ConfigEnvironment expects a "top_srcdir" subst to be set with the top
source directory, in msys format on windows. It is used to derive a
"srcdir" subst when treating config files. It can either be an absolute
path or a path relative to the topobjdir.
"""
def __init__(
self,
topsrcdir,
topobjdir,
defines=None,
substs=None,
source=None,
mozconfig=None,
):
if not source:
source = mozpath.join(topobjdir, "config.status")
self.source = source
self.defines = ReadOnlyDict(defines or {})
self.substs = dict(substs or {})
self.topsrcdir = mozpath.abspath(topsrcdir)
self.topobjdir = mozpath.abspath(topobjdir)
self.mozconfig = mozpath.abspath(mozconfig) if mozconfig else None
self.lib_prefix = self.substs.get("LIB_PREFIX", "")
if "LIB_SUFFIX" in self.substs:
self.lib_suffix = ".%s" % self.substs["LIB_SUFFIX"]
self.dll_prefix = self.substs.get("DLL_PREFIX", "")
self.dll_suffix = self.substs.get("DLL_SUFFIX", "")
self.host_dll_prefix = self.substs.get("HOST_DLL_PREFIX", "")
self.host_dll_suffix = self.substs.get("HOST_DLL_SUFFIX", "")
if self.substs.get("IMPORT_LIB_SUFFIX"):
self.import_prefix = self.lib_prefix
self.import_suffix = ".%s" % self.substs["IMPORT_LIB_SUFFIX"]
else:
self.import_prefix = self.dll_prefix
self.import_suffix = self.dll_suffix
if self.substs.get("HOST_IMPORT_LIB_SUFFIX"):
self.host_import_prefix = self.substs.get("HOST_LIB_PREFIX", "")
self.host_import_suffix = ".%s" % self.substs["HOST_IMPORT_LIB_SUFFIX"]
else:
self.host_import_prefix = self.host_dll_prefix
self.host_import_suffix = self.host_dll_suffix
self.bin_suffix = self.substs.get("BIN_SUFFIX", "")
global_defines = [name for name in self.defines]
self.substs["ACDEFINES"] = " ".join(
[
"-D%s=%s" % (name, shell_quote(self.defines[name]).replace("$", "$$"))
for name in sorted(global_defines)
]
)
def serialize(name, obj):
if isinstance(obj, six.string_types):
return obj
if isinstance(obj, Iterable):
return " ".join(obj)
raise Exception("Unhandled type %s for %s", type(obj), str(name))
self.substs["ALLSUBSTS"] = "\n".join(
sorted(
[
"%s = %s" % (name, serialize(name, self.substs[name]))
for name in self.substs
if self.substs[name]
]
)
)
self.substs["ALLEMPTYSUBSTS"] = "\n".join(
sorted(["%s =" % name for name in self.substs if not self.substs[name]])
)
self.substs = ReadOnlyDict(self.substs)
@property
def is_artifact_build(self):
return self.substs.get("MOZ_ARTIFACT_BUILDS", False)
@memoized_property
def acdefines(self):
acdefines = dict((name, self.defines[name]) for name in self.defines)
return ReadOnlyDict(acdefines)
@staticmethod
def from_config_status(path):
config = BuildConfig.from_config_status(path)
return ConfigEnvironment(
config.topsrcdir, config.topobjdir, config.defines, config.substs, path
)
class PartialConfigDict(object):
"""Facilitates mapping the config.statusd defines & substs with dict-like access.
This allows a buildconfig client to use buildconfig.defines['FOO'] (and
similar for substs), where the value of FOO is delay-loaded until it is
needed.
"""
def __init__(self, config_statusd, typ, environ_override=False):
self._dict = {}
self._datadir = mozpath.join(config_statusd, typ)
self._config_track = mozpath.join(self._datadir, "config.track")
self._files = set()
self._environ_override = environ_override
def _load_config_track(self):
existing_files = set()
try:
with open(self._config_track) as fh:
existing_files.update(fh.read().splitlines())
except IOError:
pass
return existing_files
def _write_file(self, key, value):
filename = mozpath.join(self._datadir, key)
with FileAvoidWrite(filename) as fh:
to_write = json.dumps(value, indent=4)
fh.write(to_write.encode(system_encoding))
return filename
def _fill_group(self, values):
# Clear out any cached values. This is mostly for tests that will check
# the environment, write out a new set of variables, and then check the
# environment again. Normally only configure ends up calling this
# function, and other consumers create their own
# PartialConfigEnvironments in new python processes.
self._dict = {}
existing_files = self._load_config_track()
existing_files = {Path(f) for f in existing_files}
new_files = set()
for k, v in six.iteritems(values):
new_files.add(Path(self._write_file(k, v)))
for filename in existing_files - new_files:
# We can't actually os.remove() here, since make would not see that the
# file has been removed and that the target needs to be updated. Instead
# we just overwrite the file with a value of None, which is equivalent
# to a non-existing file.
with FileAvoidWrite(filename) as fh:
json.dump(None, fh)
with FileAvoidWrite(self._config_track) as fh:
for f in sorted(new_files):
fh.write("%s\n" % f)
def __getitem__(self, key):
if self._environ_override:
if (key not in ("CPP", "CXXCPP", "SHELL")) and (key in os.environ):
return os.environ[key]
if key not in self._dict:
data = None
try:
filename = mozpath.join(self._datadir, key)
self._files.add(filename)
with open(filename) as f:
data = json.load(f)
except IOError:
pass
self._dict[key] = data
if self._dict[key] is None:
raise KeyError("'%s'" % key)
return self._dict[key]
def __setitem__(self, key, value):
self._dict[key] = value
def get(self, key, default=None):
return self[key] if key in self else default
def __contains__(self, key):
try:
return self[key] is not None
except KeyError:
return False
def iteritems(self):
existing_files = self._load_config_track()
for f in existing_files:
# The track file contains filenames, and the basename is the
# variable name.
var = mozpath.basename(f)
yield var, self[var]
class PartialConfigEnvironment(object):
"""Allows access to individual config.status items via config.statusd/* files.
This class is similar to the full ConfigEnvironment, which uses
config.status, except this allows access and tracks dependencies to
individual configure values. It is intended to be used during the build
process to handle things like GENERATED_FILES, CONFIGURE_DEFINE_FILES, and
anything else that may need to access specific substs or defines.
Creating a PartialConfigEnvironment requires only the topobjdir, which is
needed to distinguish between the top-level environment and the js/src
environment.
The PartialConfigEnvironment automatically defines one additional subst variable
from all the defines:
- ACDEFINES contains the defines in the form -DNAME=VALUE, for use on
preprocessor command lines. The order in which defines were given
when creating the ConfigEnvironment is preserved.
and one additional define from all the defines as a dictionary:
- ALLDEFINES contains all of the global defines as a dictionary. This is
intended to be used instead of the defines structure from config.status so
that scripts can depend directly on its value.
"""
def __init__(self, topobjdir):
config_statusd = mozpath.join(topobjdir, "config.statusd")
self.substs = PartialConfigDict(config_statusd, "substs", environ_override=True)
self.defines = PartialConfigDict(config_statusd, "defines")
self.topobjdir = topobjdir
def write_vars(self, config):
substs = config["substs"].copy()
defines = config["defines"].copy()
global_defines = [name for name in config["defines"]]
acdefines = " ".join(
[
"-D%s=%s"
% (name, shell_quote(config["defines"][name]).replace("$", "$$"))
for name in sorted(global_defines)
]
)
substs["ACDEFINES"] = acdefines
all_defines = OrderedDict()
for k in global_defines:
all_defines[k] = config["defines"][k]
defines["ALLDEFINES"] = all_defines
self.substs._fill_group(substs)
self.defines._fill_group(defines)
def get_dependencies(self):
return ["$(wildcard %s)" % f for f in self.substs._files | self.defines._files]