Source code
Revision control
Copy as Markdown
Other Tools
#!/usr/bin/env python
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
##########################################################################
#
# This is a collection of helper tools to get stuff done in NSS.
#
import sys
import argparse
import fnmatch
import io
import subprocess
import os
import platform
import shutil
import tarfile
import tempfile
from hashlib import sha256
DEVNULL = open(os.devnull, 'wb')
cwd = os.path.dirname(os.path.abspath(__file__))
def run_tests(test, cycles="standard", env={}, silent=False):
domsuf = os.getenv('DOMSUF', "localdomain")
host = os.getenv('HOST', "localhost")
env = env.copy()
env.update({
"NSS_TESTS": test,
"NSS_CYCLES": cycles,
"DOMSUF": domsuf,
"HOST": host
})
os_env = os.environ
os_env.update(env)
command = cwd + "/tests/all.sh"
stdout = stderr = DEVNULL if silent else None
subprocess.check_call(command, env=os_env, stdout=stdout, stderr=stderr)
class cfAction(argparse.Action):
docker_command = None
restorecon = None
def __call__(self, parser, args, values, option_string=None):
self.setDockerCommand(args)
if values:
files = [os.path.relpath(os.path.abspath(x), start=cwd) for x in values]
else:
files = self.modifiedFiles()
# First check if we can run docker.
try:
with open(os.devnull, "w") as f:
subprocess.check_call(
self.docker_command + ["images"], stdout=f)
except:
self.docker_command = None
if self.docker_command is None:
print("warning: running clang-format directly, which isn't guaranteed to be correct")
command = [cwd + "/automation/clang-format/run_clang_format.sh"] + files
repr(command)
subprocess.call(command)
return
files = [os.path.join('/home/worker/nss', x) for x in files]
docker_image = 'clang-format-service:latest'
cf_docker_folder = cwd + "/automation/clang-format"
# Build the image if necessary.
if self.filesChanged(cf_docker_folder):
self.buildImage(docker_image, cf_docker_folder)
# Check if we have the docker image.
try:
command = self.docker_command + [
"image", "inspect", "clang-format-service:latest"
]
with open(os.devnull, "w") as f:
subprocess.check_call(command, stdout=f)
except:
print("I have to build the docker image first.")
self.buildImage(docker_image, cf_docker_folder)
command = self.docker_command + [
'run', '-v', cwd + ':/home/worker/nss:Z', '--rm', '-ti', docker_image
]
# The clang format script returns 1 if something's to do. We don't
# care.
subprocess.call(command + files)
if self.restorecon is not None:
subprocess.call([self.restorecon, '-R', cwd])
def filesChanged(self, path):
hash = sha256()
for dirname, dirnames, files in os.walk(path):
for file in files:
with open(os.path.join(dirname, file), "rb") as f:
hash.update(f.read())
chk_file = cwd + "/.chk"
old_chk = ""
new_chk = hash.hexdigest()
if os.path.exists(chk_file):
with open(chk_file) as f:
old_chk = f.readline()
if old_chk != new_chk:
with open(chk_file, "w+") as f:
f.write(new_chk)
return True
return False
def buildImage(self, docker_image, cf_docker_folder):
command = self.docker_command + [
"build", "-t", docker_image, cf_docker_folder
]
subprocess.check_call(command)
return
def setDockerCommand(self, args):
if platform.system() == "Linux":
self.restorecon = shutil.which("restorecon")
dcmd = shutil.which("docker")
if dcmd is not None:
self.docker_command = [dcmd]
if not args.noroot:
self.docker_command = ["sudo"] + self.docker_command
else:
self.docker_command = None
def modifiedFiles(self):
files = []
if os.path.exists(os.path.join(cwd, '.hg')):
st = subprocess.Popen(['hg', 'status', '-m', '-a'],
cwd=cwd, stdout=subprocess.PIPE, universal_newlines=True)
for line in iter(st.stdout.readline, ''):
files += [line[2:].rstrip()]
elif os.path.exists(os.path.join(cwd, '.git')):
st = subprocess.Popen(['git', 'status', '--porcelain'],
cwd=cwd, stdout=subprocess.PIPE)
for line in iter(st.stdout.readline, ''):
if line[1] == 'M' or line[1] != 'D' and \
(line[0] == 'M' or line[0] == 'A' or
line[0] == 'C' or line[0] == 'U'):
files += [line[3:].rstrip()]
elif line[0] == 'R':
files += [line[line.index(' -> ', beg=4) + 4:]]
else:
print('Warning: neither mercurial nor git detected!')
def isFormatted(x):
return x[-2:] == '.c' or x[-3:] == '.cc' or x[-2:] == '.h'
return [x for x in files if isFormatted(x)]
class buildAction(argparse.Action):
def __call__(self, parser, args, values, option_string=None):
subprocess.check_call([cwd + "/build.sh"] + values)
class testAction(argparse.Action):
def __call__(self, parser, args, values, option_string=None):
run_tests(values)
class covAction(argparse.Action):
def runSslGtests(self, outdir):
env = {
"GTESTFILTER": "*", # Prevent parallel test runs.
"ASAN_OPTIONS": "coverage=1:coverage_dir=" + outdir,
"NSS_DEFAULT_DB_TYPE": "sql",
"NSS_DISABLE_UNLOAD": "1"
}
run_tests("ssl_gtests", env=env, silent=True)
def findSanCovFile(self, outdir):
for file in os.listdir(outdir):
if fnmatch.fnmatch(file, 'ssl_gtest.*.sancov'):
return os.path.join(outdir, file)
return None
def __call__(self, parser, args, values, option_string=None):
outdir = args.outdir
print("Output directory: " + outdir)
print("\nBuild with coverage sanitizers...\n")
sancov_args = "edge,no-prune,trace-pc-guard,trace-cmp"
subprocess.check_call([
os.path.join(cwd, "build.sh"), "-c", "--clang", "--asan", "--enable-legacy-db",
"--sancov=" + sancov_args
])
print("\nRun ssl_gtests to get a coverage report...")
self.runSslGtests(outdir)
print("Done.")
sancov_file = self.findSanCovFile(outdir)
if not sancov_file:
print("Couldn't find .sancov file.")
sys.exit(1)
symcov_file = os.path.join(outdir, "ssl_gtest.symcov")
out = open(symcov_file, 'wb')
# Don't exit immediately on error
symbol_retcode = subprocess.call([
"sancov",
"-blacklist=" + os.path.join(cwd, ".sancov-blacklist"),
"-symbolize", sancov_file,
os.path.join(cwd, "../dist/Debug/bin/ssl_gtest")
], stdout=out)
out.close()
print("\nCopying ssl_gtests to artifacts...")
shutil.copyfile(os.path.join(cwd, "../dist/Debug/bin/ssl_gtest"),
os.path.join(outdir, "ssl_gtest"))
print("\nCoverage report: " + symcov_file)
if symbol_retcode > 0:
print("sancov failed to symbolize with return code {}".format(symbol_retcode))
sys.exit(symbol_retcode)
class commandsAction(argparse.Action):
commands = []
def __call__(self, parser, args, values, option_string=None):
for c in commandsAction.commands:
print(c)
def parse_arguments():
parser = argparse.ArgumentParser(
description='NSS helper script. ' +
'Make sure to separate sub-command arguments with --.')
subparsers = parser.add_subparsers()
parser_build = subparsers.add_parser(
'build', help='All arguments are passed to build.sh')
parser_build.add_argument(
'build_args', nargs='*', help="build arguments", action=buildAction)
parser_cf = subparsers.add_parser(
'clang-format',
help="""
Run clang-format.
By default this runs against any files that you have modified. If
there are no modified files, it checks everything.
""")
parser_cf.add_argument(
'--noroot',
help='On linux, suppress the use of \'sudo\' for running docker.',
action='store_true')
parser_cf.add_argument(
'<file/dir>',
nargs='*',
help="Specify files or directories to run clang-format on",
action=cfAction)
parser_test = subparsers.add_parser(
'tests', help='Run tests through tests/all.sh.')
tests = [
"cipher", "lowhash", "chains", "cert", "dbtests", "tools", "fips",
"sdr", "crmf", "smime", "ssl", "ocsp", "merge", "pkits", "ec",
"gtests", "ssl_gtests", "bogo", "interop", "policy"
]
parser_test.add_argument(
'test', choices=tests, help="Available tests", action=testAction)
parser_cov = subparsers.add_parser(
'coverage', help='Generate coverage report')
cov_modules = ["ssl_gtests"]
parser_cov.add_argument(
'--outdir', help='Output directory for coverage report data.',
default=tempfile.mkdtemp())
parser_cov.add_argument(
'module', choices=cov_modules, help="Available coverage modules",
action=covAction)
parser_commands = subparsers.add_parser(
'mach-completion',
help="list commands")
parser_commands.add_argument(
'mach-completion',
nargs='*',
action=commandsAction)
commandsAction.commands = [c for c in subparsers.choices]
return parser.parse_args()
def main():
parse_arguments()
if __name__ == '__main__':
main()