blob: c85e2f888e76424400e0ff75ae4898fb89e18ac9 [file] [log] [blame]
#!/usr/bin/env python3
# Copyright 2016 The Fuchsia Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""The Cobalt build system command-line interface."""
from __future__ import print_function
import argparse
import filecmp
import fileinput
import json
import logging
import os
import shutil
import string
import subprocess
import sys
import tempfile
import tools.clang_tidy as clang_tidy
import tools.gitfmt as gitfmt
import tools.gnlint as gnlint
import tools.golint as golint
import tools.lint_gn_rules_present as lint_gn_rules_present
import tools.lint_todos as lint_todos
import tools.lint_visibility as lint_visibility
import tools.sync_with_fuchsia as sync_with_fuchsia
import tools.test_runner as test_runner
THIS_DIR = os.path.abspath(os.path.dirname(__file__))
SYSROOT_DIR = os.path.abspath(os.path.join(THIS_DIR, 'sysroot'))
CONFIG_SUBMODULE_PATH = os.path.join(THIS_DIR, 'third_party', 'cobalt_config')
PYTHON_CMD = os.path.join(SYSROOT_DIR, 'bin', 'python3')
_logger = logging.getLogger()
_verbose_count = 0
# In Python3, the `raw_input` Built-in function was renamed to `input`.
try:
input = raw_input
except NameError:
pass
def _initLogging(verbose_count):
"""Ensures that the logger (obtained via logging.getLogger(), as usual) is
initialized, with the log level set as appropriate for |verbose_count|
instances of --verbose on the command line.
"""
assert verbose_count >= 0
if verbose_count == 0:
level = logging.WARNING
elif verbose_count == 1:
level = logging.INFO
else: # verbose_count >= 2
level = logging.DEBUG
logging.basicConfig(format='%(relativeCreated).3f:%(levelname)s:%(message)s')
logger = logging.getLogger()
logger.setLevel(level)
logger.debug(
'Initialized logging: verbose_count=%d, level=%d' % (verbose_count, level)
)
def ensureDir(dir_path):
"""Ensures that the directory at |dir_path| exists.
If not it is created.
Args: dir_path{string}: The path to a directory. If it does not exist it will
be created.
"""
if not os.path.exists(dir_path):
os.makedirs(dir_path)
def out_dir(args):
return os.path.abspath(os.path.join(THIS_DIR, args.out_dir))
def _setup(args):
subprocess.check_call(['git', 'submodule', 'init'])
subprocess.check_call(['git', 'submodule', 'sync'])
subprocess.check_call(['git', 'submodule', 'update'])
subprocess.check_call(['./setup.sh'])
def _deinit(args):
subprocess.check_call(['git', 'submodule', 'deinit', '--all', '-f'])
def _update_config(args):
print(
'./cobaltb.py update_config is deprecated. Please use ./cobaltb.py'
' sync_with_fuchsia instead'
)
def _sync_with_fuchsia(args):
sync_with_fuchsia.sync(
integration_repo=args.integration_repo,
fuchsia_repo=args.fuchsia_repo,
manifest_files=args.manifest,
to_copy_from_fuchsia=args.to_copy_from_fuchsia,
retain_paths=args.retain_paths,
prebuilts_files=args.prebuilts_files,
desired_prebuilts=args.desired_prebuilts,
make_commit=args.make_commit,
)
# Run setup.sh to ensure that any CIPD changes propagate.
subprocess.check_call(['./setup.sh'])
def _compdb(args):
# Copy the compile_commands.json to the top level for use in IDEs (CLion).
subprocess.check_call([
'cp',
'%s/compile_commands.json' % out_dir(args),
'./compile_commands.json',
])
# Remove the gomacc references that confuse IDEs.
subprocess.check_call([
'perl',
'-p',
'-i',
'-e',
's|/[/\w]+/gomacc *||',
'./compile_commands.json',
])
def _goma_login(args):
goma_dir = os.path.join(SYSROOT_DIR, 'goma')
if args.goma_dir:
goma_dir = args.goma_dir
subprocess.check_call([PYTHON_CMD, '%s/goma_auth.py' % goma_dir, 'login'])
def _build(args):
gn_args = []
use_ccache = False
goma_dir = os.path.join(SYSROOT_DIR, 'goma')
if args.goma_dir:
goma_dir = args.goma_dir
if args.ccache:
use_ccache = True
if args.no_ccache:
use_ccache = False
use_goma = os.path.exists(goma_dir)
if args.no_goma:
use_goma = False
# If goma isn't running, start it.
if use_goma:
start_goma = False
try:
if (
not subprocess.check_output(['%s/gomacc' % goma_dir, 'port'])
.strip()
.isdigit()
):
start_goma = True
except subprocess.CalledProcessError:
start_goma = True
if start_goma:
try:
if (
subprocess.check_output(
[PYTHON_CMD, '%s/goma_auth.py' % goma_dir, 'info']
).strip()
== 'Not logged in'
):
print()
print()
print(
'Goma is not logged in and will not be used. Please run:'
' `./cobaltb.py goma_login`'
)
print()
print()
use_goma = False
except subprocess.CalledProcessError:
print()
print()
print(
'Goma is not logged in and will not be used. Please run:'
' `./cobaltb.py goma_login`'
)
print()
print()
use_goma = False
if use_goma:
subprocess.check_call(
[PYTHON_CMD, '%s/goma_ctl.py' % goma_dir, 'ensure_start']
)
if args.release:
gn_args.append('is_debug=false')
if use_goma:
gn_args.append('use_goma=true')
gn_args.append('goma_dir="%s"' % goma_dir)
elif use_ccache:
gn_args.append('use_ccache=true')
if args.args != '':
gn_args.append(args.args)
if vars(args)['with']:
packages = 'extra_package_labels=['
for target in vars(args)['with']:
packages += '"%s",' % target
packages += ']'
gn_args.append(packages)
subprocess.check_call([
args.gn_path,
'gen',
out_dir(args),
'--check',
'--export-compile-commands=default',
'--args=%s' % ' '.join(gn_args),
])
subprocess.check_call([args.ninja_path, '-C', out_dir(args)])
def _check_config(args):
config_parser_bin = os.path.join(out_dir(args), 'config_parser')
if not os.path.isfile(config_parser_bin):
print(
'%s could not be found. Run \n\n%s setup\n%s build\n\nand try again.'
% (config_parser_bin, sys.argv[0], sys.argv[0])
)
return
if not args.skip_check_test_configs and not _check_test_configs(args):
return
try:
subprocess.check_call([
config_parser_bin,
'-config_dir',
args.config_dir,
'-check_only',
])
except subprocess.CalledProcessError:
sys.exit(1)
def _check_test_configs(args):
testapp_config_path = os.path.join(
args.config_dir, 'fuchsia', 'test_app2', 'metrics.yaml'
)
if not _check_config_exists(testapp_config_path):
return False
prober_config_path = os.path.join(
args.config_dir, 'fuchsia', 'prober', 'metrics.yaml'
)
if not _check_config_exists(prober_config_path):
print('Run this command and try again: ./cobaltb.py write_prober_config')
return False
_, tmp_path = tempfile.mkstemp()
_make_prober_config(testapp_config_path, tmp_path)
is_same_file = filecmp.cmp(tmp_path, prober_config_path)
os.remove(tmp_path)
if not is_same_file:
print(
'Testapp config and prober config should be identical.\n'
'Run this command and try again: ./cobaltb.py write_prober_config'
)
return is_same_file
def _write_prober_config(args):
testapp_config_path = os.path.join(
args.config_dir, 'fuchsia', 'test_app2', 'metrics.yaml'
)
if not _check_config_exists(testapp_config_path):
return False
prober_config_path = os.path.join(
args.config_dir, 'fuchsia', 'prober', 'metrics.yaml'
)
if os.path.isfile(prober_config_path):
print('This action will overwrite the file %s.' % prober_config_path)
answer = input('Continue anyway? (y/N) ')
if not _parse_bool(answer):
return
prober_dir = os.path.dirname(prober_config_path)
if not os.path.exists(prober_dir):
os.makedirs(prober_dir)
_make_prober_config(testapp_config_path, prober_config_path)
def _check_config_exists(config_path):
if not os.path.isfile(config_path):
print('Expected config at path %s' % config_path)
return False
return True
def _make_prober_config(testapp_config_path, output_path):
with open(testapp_config_path, 'r') as f:
testapp_config = f.read()
with open(output_path, 'w') as f:
f.write(testapp_config)
def _fmt(args):
gitfmt.fmt(args.staged_only, args.committed, args.all)
def _lint(args):
status = 0
failure_list = []
result = clang_tidy.main(args.directory, args.all)
failure_list.append(('clang_tidy', result))
status += result
result = golint.main(args.directory, args.all)
failure_list.append(('golint', result))
status += result
result = gnlint.main(args.directory, args.all)
failure_list.append(('gnlint', result))
status += result
result = lint_visibility.main()
failure_list.append(('lint_visibility', result))
status += result
result = lint_todos.main()
failure_list.append(('lint_todos', result))
status += result
result = lint_gn_rules_present.main()
failure_list.append(('lint_gn_rules_present', result))
status += result
if status > 0:
print('')
print('******************* SOME LINTERS FAILED *******************')
for linter, result in failure_list:
print('%s returned: %s' % (linter, result))
else:
print('All linters passed')
exit(status)
# Specifiers of subsets of tests to run
TEST_FILTERS = [
'all',
'cpp',
'nocpp',
'go',
'nogo',
'perf',
'perf',
'rust',
'other',
]
# Returns 0 if all tests pass, otherwise returns 1. Prints a failure or success
# message.
def _test(args):
# A map from positive filter specifiers to the list of test directories
# it represents. Note that 'cloud_bt' and 'perf' tests are special. They are
# not included in 'all'. They are only run if asked for explicitly.
FILTER_MAP = {
'all': ['cpp', 'go', 'other', 'rust'],
'cpp': ['cpp'],
'go': ['go'],
'perf': ['perf'],
'other': ['other'],
'rust': ['rust'],
}
# By default try each test just once.
num_times_to_try = 1
# Get the list of test directories we should run.
if args.tests.startswith('no'):
test_dirs = [
test_dir
for test_dir in FILTER_MAP['all']
if test_dir not in FILTER_MAP[args.tests[2:]]
]
else:
test_dirs = FILTER_MAP[args.tests]
failure_list = []
print(
'Will run tests in the following directories: %s.' % ', '.join(test_dirs)
)
for test_dir in test_dirs:
test_args = None
print('********************************************************')
this_failure_list = []
for attempt in range(num_times_to_try):
this_failure_list = test_runner.run_all_tests(
'tests/' + test_dir, verbose_count=_verbose_count, test_args=test_args
)
if this_failure_list and attempt < num_times_to_try - 1:
print('')
print(
'***** Attempt %i of %s failed. Retrying...'
% (attempt, this_failure_list)
)
print('')
else:
break
if this_failure_list:
failure_list.append('%s (%s)' % (test_dir, this_failure_list))
print('')
print('***** Testing Source Generator for all Projects *****')
import src.bin.config_parser.test_source_generator_for_all_projects as test_source_generator_for_all_projects
failure_list.extend(test_source_generator_for_all_projects.main())
print('')
if failure_list:
print('******************* SOME TESTS FAILED *******************')
print('failures = %s' % failure_list)
return 1
else:
print('******************* ALL TESTS PASSED *******************')
return 0
# Files and directories in the out directory to NOT delete when doing
# a partial clean.
TO_SKIP_ON_PARTIAL_CLEAN = {
'obj': {'third_party': True},
'gen': {'third_party': True},
'.ninja_deps': True,
'.ninja_log': True,
'build.ninja': True,
'rules.ninja': True,
'args.gn': True,
}
def partial_clean(current_dir, exceptions):
for f in os.listdir(current_dir):
full_path = os.path.join(current_dir, f)
if not f in exceptions:
if os.path.isfile(full_path):
os.remove(full_path)
else:
shutil.rmtree(full_path, ignore_errors=True)
elif isinstance(exceptions[f], dict):
partial_clean(full_path, exceptions[f])
else:
print('Skipping', full_path)
def _clean(args):
if args.full:
print('Deleting the out directory...')
shutil.rmtree(out_dir(args), ignore_errors=True)
else:
print('Doing a partial clean. Pass --full for a full clean.')
if not os.path.exists(out_dir(args)):
return
partial_clean(out_dir(args), TO_SKIP_ON_PARTIAL_CLEAN)
def _parse_bool(bool_string):
return bool_string.lower() in ['true', 't', 'y', 'yes', '1']
def _is_config_up_to_date():
savedDir = os.getcwd()
try:
os.chdir(CONFIG_SUBMODULE_PATH)
# Get the hash for the latest local revision.
local_hash = subprocess.check_output(['git', 'rev-parse', '@'])
# Get the hash for the latest remote revision.
remote_hash = subprocess.check_output(['git', 'rev-parse', 'origin/master'])
return local_hash == remote_hash
finally:
os.chdir(savedDir)
def main():
if not sys.platform.startswith('linux'):
print('Only linux is supported!')
return 1
# We parse the command line flags twice. The first time we are looking
# only for two particular flags, namely --production_dir and
# --cobalt_on_personal_cluster. This first pass
# will not print any help and will ignore all other flags.
parser0 = argparse.ArgumentParser(add_help=False)
parser0.add_argument('--production_dir', default='')
parser0.add_argument('-cobalt_on_personal_cluster', action='store_true')
args0, ignore = parser0.parse_known_args()
parser = argparse.ArgumentParser(
description='The Cobalt command-line interface.'
)
# Note(rudominer) A note about the handling of optional arguments here.
# We create |parent_parser| and make it a parent of all of our sub parsers.
# When we want to add a global optional argument (i.e. one that applies
# to all sub-commands such as --verbose) we add the optional argument
# to both |parent_parser| and |parser|. The reason for this is that
# that appears to be the only way to get the help string to show up both
# when the top-level command is invoked and when
# a sub-command is invoked.
#
# In other words when the user types:
#
# python cobaltb.py -h
#
# and also when the user types
#
# python cobaltb.py test -h
#
# we want to show the help for the --verbose option.
parent_parser = argparse.ArgumentParser(add_help=False)
parser.add_argument(
'--verbose',
help='Be verbose (multiple times for more)',
default=0,
dest='verbose_count',
action='count',
)
parent_parser.add_argument(
'--verbose',
help='Be verbose (multiple times for more)',
default=0,
dest='verbose_count',
action='count',
)
parser.add_argument(
'--vmodule',
help=(
'A string to use for the GLog -vmodule flag when running the Cobalt '
'processes locally. Currently only used for the end-to-end test. '
'Optional.)'
),
default='',
)
parent_parser.add_argument(
'--vmodule',
help=(
'A string to use for the GLog -vmodule flag when running the Cobalt'
'processes locally. Currently only used for the end-to-end test. '
'Optional.)'
),
default='',
)
parser.add_argument(
'--out_dir',
help='Output directory (relative to cobaltb.py)',
default='out',
)
parent_parser.add_argument(
'--out_dir',
help='Output directory (relative to cobaltb.py)',
default='out',
)
subparsers = parser.add_subparsers()
########################################################
# setup command
########################################################
sub_parser = subparsers.add_parser(
'setup', parents=[parent_parser], help='Sets up the build environment.'
)
sub_parser.set_defaults(func=_setup)
########################################################
# deinit command
########################################################
sub_parser = subparsers.add_parser(
'deinit',
parents=[parent_parser],
help=(
'Removes the submodules added by setup that '
'prevent jiri update from working.'
),
)
sub_parser.set_defaults(func=_deinit)
########################################################
# update_config command
########################################################
sub_parser = subparsers.add_parser(
'update_config',
parents=[parent_parser],
help="Pulls the current version Cobalt's config from its remote repo.",
)
sub_parser.set_defaults(func=_update_config)
sub_parser = subparsers.add_parser(
'sync_with_fuchsia',
parents=[parent_parser],
help=(
'Pulls the current version of all submodules based on the integration'
' repository provided'
),
)
sub_parser.add_argument(
'--integration_repo',
help='The integration repo to download manifests from.',
default=sync_with_fuchsia.DEFAULT_INTEGRATION_REPO,
)
sub_parser.add_argument(
'--fuchsia_repo',
help='The integration repo to download manifests from.',
default=sync_with_fuchsia.DEFAULT_FUCHSIA_REPO,
)
sub_parser.add_argument(
'--to_copy_from_fuchsia',
help=(
'Which files or directories to copy from fuchsia_repo. Can be'
' specified more than once'
),
action='append',
default=sync_with_fuchsia.DEFAULT_TO_COPY_FROM_FUCHSIA,
)
sub_parser.add_argument(
'--retain_paths',
help='Which paths to retain when syncing directories from fuchsia',
action='append',
default=sync_with_fuchsia.DEFAULT_RETAIN_PATHS,
)
sub_parser.add_argument(
'--manifest',
help='Which manifest to read from. Can be specified more than once',
action='append',
default=sync_with_fuchsia.DEFAULT_MANIFEST_FILES,
)
sub_parser.add_argument(
'--prebuilts_files',
help='Which files in the fuchsia repo to fetch prebuilts from.',
default=sync_with_fuchsia.DEFAULT_PREBUILTS_FILES,
)
sub_parser.add_argument(
'--desired_prebuilts',
help='Which prebuilts to fetch from the prebuilts file.',
action='append',
default=sync_with_fuchsia.DEFAULT_DESIRED_PREBUILTS,
)
sub_parser.add_argument(
'--make_commit',
help=(
'Should a commit be automatically created after the update takes'
' place. (default: False)'
),
action='store_true',
default=False,
)
sub_parser.set_defaults(func=_sync_with_fuchsia)
########################################################
# write_prober_config command
########################################################
sub_parser = subparsers.add_parser(
'write_prober_config',
parents=[parent_parser],
help='Copies the test_app2 config to the prober config.',
)
sub_parser.add_argument(
'--config_dir',
help=(
'Path to the configuration '
'directory which should contain the prober config. '
'Default: %s'
)
% CONFIG_SUBMODULE_PATH,
default=CONFIG_SUBMODULE_PATH,
)
sub_parser.set_defaults(func=_write_prober_config)
########################################################
# check_config command
########################################################
sub_parser = subparsers.add_parser(
'check_config',
parents=[parent_parser],
help='Check the validity of the cobalt configuration.',
)
sub_parser.add_argument(
'--config_dir',
help='Path to the configuration directory to be checked. Default: %s'
% CONFIG_SUBMODULE_PATH,
default=CONFIG_SUBMODULE_PATH,
)
sub_parser.add_argument(
'--skip_check_test_configs',
help="Don't check the test_app2/prober configs.",
action='store_true',
default=False,
)
sub_parser.set_defaults(func=_check_config)
########################################################
# build command
########################################################
sub_parser = subparsers.add_parser(
'build', parents=[parent_parser], help='Builds Cobalt.'
)
sub_parser.add_argument('--gn_path', default='gn', help='Path to GN binary')
sub_parser.add_argument(
'--ninja_path', default='ninja', help='Path to Ninja binary'
)
sub_parser.add_argument(
'--args', default='', help='Additional arguments to pass to gn'
)
sub_parser.add_argument(
'--ccache', action='store_true', help='The build should use ccache'
)
sub_parser.add_argument(
'--no-ccache', action='store_true', help='The build should not use ccache'
)
sub_parser.add_argument(
'--no-goma',
action='store_true',
help='The build should not use goma. Otherwise goma is used if found.',
)
sub_parser.add_argument(
'--goma_dir',
default='',
help='The dir where goma is installed (defaults to sysroot/goma',
)
sub_parser.add_argument(
'--release', action='store_true', help='Should build release build'
)
sub_parser.add_argument(
'--with', action='append', help='Additional packages to build'
)
sub_parser.set_defaults(func=_build)
########################################################
# goma_login command
########################################################
sub_parser = subparsers.add_parser(
'goma_login', parents=[parent_parser], help='Logs in to the goma service.'
)
sub_parser.add_argument(
'--goma_dir',
default='',
help='The dir where goma is installed (defaults to sysroot/goma',
)
sub_parser.set_defaults(func=_goma_login)
########################################################
# lint command
########################################################
sub_parser = subparsers.add_parser(
'lint',
parents=[parent_parser],
help=(
'Run language linters on some source files. By default it uses `git'
' diff` against the newest parent commit in the upstream branch (or'
' against HEAD if no such commit is found). Files that are locally'
' modified, staged or touched by any commits introduced on the local'
' branch are linted.'
),
)
sub_parser.add_argument(
'--all',
action='store_true',
default=False,
help='Run on all tracked files.',
)
sub_parser.add_argument('directory', nargs='*')
sub_parser.set_defaults(func=_lint)
########################################################
# fmt command
########################################################
sub_parser = subparsers.add_parser(
'fmt',
parents=[parent_parser],
help='Run language formatter on modified files.',
)
sub_parser.add_argument(
'--staged_only',
action='store_true',
default=False,
help='Run on staged files only.',
)
sub_parser.add_argument(
'--committed',
action='store_true',
default=False,
help='Also run on files modified in the latest commit.',
)
sub_parser.add_argument(
'--all',
action='store_true',
default=False,
help='Run on all tracked files.',
)
sub_parser.set_defaults(func=_fmt)
########################################################
# test command
########################################################
sub_parser = subparsers.add_parser(
'test',
parents=[parent_parser],
help='Runs Cobalt tests. You must build first.',
)
sub_parser.set_defaults(func=_test)
sub_parser.add_argument(
'--tests',
choices=TEST_FILTERS,
help='Specify a subset of tests to run. Default=all',
default='all',
)
########################################################
# clean command
########################################################
sub_parser = subparsers.add_parser(
'clean',
parents=[parent_parser],
help='Deletes some or all of the build products.',
)
sub_parser.set_defaults(func=_clean)
sub_parser.add_argument(
'--full', help='Delete the entire "out" directory.', action='store_true'
)
########################################################
# compdb command
########################################################
sub_parser = subparsers.add_parser(
'compdb',
parents=[parent_parser],
help=(
'Generate a compilation database for the current build configuration.'
),
)
sub_parser.set_defaults(func=_compdb)
args = parser.parse_args()
global _verbose_count
_verbose_count = args.verbose_count
_initLogging(_verbose_count)
global _vmodule
_vmodule = args.vmodule
# Add bin dirs from sysroot to the front of the path.
os.environ['PATH'] = (
'%s/bin' % SYSROOT_DIR
+ os.pathsep
+ '%s/golang/bin' % SYSROOT_DIR
+ os.pathsep
+ os.environ['PATH']
)
os.environ['LD_LIBRARY_PATH'] = '%s/lib' % SYSROOT_DIR
os.environ['GOROOT'] = '%s/golang' % SYSROOT_DIR
# Until Python3.7 adds the 'required' flag for subparsers, an error occurs
# when running without specifying a subparser on the command line:
# https://bugs.python.org/issue16308
# Work around the issue by checking whether the 'func' attribute has been
# set.
try:
a = getattr(args, 'func')
except AttributeError:
parser.print_usage()
sys.exit(0)
return args.func(args)
if __name__ == '__main__':
sys.exit(main())