Source code
Revision control
Copy as Markdown
Other Tools
# Copyright 2025 The Chromium Authors
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
import hashlib
import json
import pathlib
import os
import subprocess
import zipfile
import re
_SRC_PATH = pathlib.Path(__file__).resolve().parents[2]
_FETCH_ALL_PATH = _SRC_PATH / 'third_party/android_deps/fetch_all.py'
_HASH_LENGTH = 15
_SKIP_FILES = ('OWNERS', 'cipd.yaml')
_DEFAULT_GENERATED_DISCLAIMER = '''\
// **IMPORTANT**: build.gradle is generated and any changes would be overridden
// by the autoroller. Please update build.gradle.template
// instead.
'''
def generate_version_map_str(bom_path, with_hash=False):
"""Generate groovy code to fill the versionCache map.
Args:
bom_path: Path to bill_of_materials.json to parse.
with_hash: Whether to also return a hash of all the packages in the BoM.
"""
bom = []
version_map_lines = []
bom_hash = hashlib.sha256()
with open(bom_path) as f:
bom = json.load(f)
bom.sort(key=lambda x: (x['group'], x['name']))
for dep in bom:
group = dep['group']
name = dep['name']
version = dep['version']
bom_hash.update(f'${group}:${name}:${version}'.encode())
map_line = f"versionCache['{group}:{name}'] = '{version}'"
version_map_lines.append(map_line)
version_map_str = '\n'.join(sorted(version_map_lines))
version_hash = bom_hash.hexdigest()[:_HASH_LENGTH]
if with_hash:
return version_map_str, version_hash
return version_map_str
def fill_template(template_path, output_path, **kwargs):
"""Fills in a template.
Args:
template_path: Path to <file>.template.
output_path: Path to <file>.
**kwargs: each kwarg should be a string to replace in the template.
"""
content = pathlib.Path(template_path).read_text()
for key, value in kwargs.items():
replace_string = '{{' + key + '}}'
if not replace_string in content:
raise Exception(f'Replace text {replace_string} '
f'not found in {template_path}')
try:
content = content.replace(replace_string, value)
except Exception as e:
raise e from Exception(
f'Failed to replace {repr(replace_string)} with {repr(value)}')
content = content.replace(r'{{generated_disclaimer}}',
_DEFAULT_GENERATED_DISCLAIMER)
unreplaced_variable_re = re.compile(r'\{\{(.+)\}\}')
if matches := unreplaced_variable_re.findall(content):
unreplaced_variables = ', '.join(repr(match) for match in matches)
raise Exception('Found unreplaced variables '
f'[{unreplaced_variables}] in {template_path}')
pathlib.Path(output_path).write_text(content)
def write_cipd_yaml(package_root,
package_name,
version,
output_path,
experimental=False):
"""Writes cipd.yaml file at the passed-in path."""
root_libs_dir = package_root / 'libs'
lib_dirs = os.listdir(root_libs_dir)
if not lib_dirs:
raise Exception('No generated libraries in {}'.format(root_libs_dir))
data_files = [
'BUILD.gn',
'VERSION.txt',
'bill_of_materials.json',
'additional_readme_paths.json',
'build.gradle',
'to_commit.zip',
]
for lib_dir in lib_dirs:
abs_lib_dir: pathlib.Path = root_libs_dir / lib_dir
if not abs_lib_dir.is_dir():
continue
for lib_file in abs_lib_dir.iterdir():
if lib_file.name in _SKIP_FILES:
continue
data_files.append((abs_lib_dir / lib_file).relative_to(package_root))
if experimental:
package_name = (f'experimental/google.com/{os.getlogin()}/{package_name}')
contents = [
'# Copyright 2025 The Chromium Authors',
'# Use of this source code is governed by a BSD-style license that can be',
'# found in the LICENSE file.',
f'# version: {version}',
f'package: {package_name}',
f'description: CIPD package for {package_name}',
'data:',
]
contents.extend(f'- file: {str(f)}' for f in data_files)
with open(output_path, 'w') as out:
out.write('\n'.join(contents))
def create_to_commit_zip(output_path, package_root, dirnames,
absolute_file_map):
"""Generates a to_commit.zip from useful text files inside |package_root|.
Args:
output_path: where to output the zipfile.
package_root: path to gradle/cipd package.
dirnames: list of subdirs under |package_root| to walk.
absolute_file_map: List of files to be stored under the absolute prefix
CHROMIUM_SRC/.
"""
to_commit_paths = []
for directory in dirnames:
for root, _, files in os.walk(package_root / directory):
for filename in files:
# Avoid committing actual artifacts.
if filename.endswith(('.aar', '.jar')):
continue
# TODO(mheikal): stop outputting these from gradle since they are not
# useful.
if filename in _SKIP_FILES:
continue
file_path = pathlib.Path(root) / filename
file_path_in_zip = file_path.relative_to(package_root)
to_commit_paths.append((file_path, file_path_in_zip))
for filename, path_in_repo in absolute_file_map.items():
file_path = package_root / filename
path_in_zip = f'CHROMIUM_SRC/{path_in_repo}'
to_commit_paths.append((file_path, path_in_zip))
with zipfile.ZipFile(output_path, 'w') as zip_file:
for filename, arcname in to_commit_paths:
zip_file.write(filename, arcname=arcname)
def run_fetch_all(android_deps_dir,
extra_args,
verbose_count=0,
output_subdir=None):
fetch_all_cmd = [
_FETCH_ALL_PATH, '--android-deps-dir', android_deps_dir,
'--ignore-vulnerabilities'
] + ['-v'] * verbose_count
if output_subdir:
fetch_all_cmd += ['--output-subdir', output_subdir]
# Filter out -- from the args to pass to fetch_all.py.
fetch_all_cmd += [a for a in extra_args if a != '--']
subprocess.run(fetch_all_cmd, check=True)