blob: 23a132e1378c08417ac50ed435604ae84e064f35 [file] [log] [blame]
#!/usr/bin/env python
# Copyright 2017 The Fuchsia Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
from collections import namedtuple
import argparse
import os
import errno
import fnmatch
import shlex
import shutil
import sys
manifest_entry = namedtuple('manifest_entry', [
def format_manifest_entry(entry):
return (('' if is None else '{' + + '}') + + '=' + entry.source)
def format_manifest_file(manifest):
return ''.join(format_manifest_entry(entry) + '\n' for entry in manifest)
def read_manifest_lines(sep, lines, title, manifest_cwd, result_cwd):
for line in lines:
# Remove the trailing newline.
assert line.endswith('\n'), 'Unterminated manifest line: %r' % line
line = line[:-1]
# Grok {group}... syntax.
group = None
if line.startswith('{'):
end = line.find('}')
assert end > 0, 'Unterminated { in manifest line: %r' % line
group = line[1:end]
line = line[end + 1:]
# Grok target=source syntax.
[target_file, build_file] = line.split(sep, 1)
if manifest_cwd != result_cwd:
# Expand the path based on the cwd presumed in the manifest.
build_file = os.path.normpath(os.path.join(manifest_cwd,
# Make it relative to the cwd we want to work from.
build_file = os.path.relpath(build_file, result_cwd)
# TODO(mcgrathr): Dismal kludge to avoid pulling in asan runtime
# libraries from the zircon ulib bootfs.manifest because their source
# file names don't match the ones in the toolchain manifest.
if 'prebuilt/downloads/clang/lib/' in build_file:
yield manifest_entry(group, target_file, build_file, title)
def partition_manifest(manifest, select, selected_group, unselected_group):
selected = []
unselected = []
for entry in manifest:
if select(
return selected, unselected
def ingest_manifest_lines(sep, lines, title, in_cwd, groups, out_cwd, output_group):
groups_seen = set()
def select(group):
if isinstance(groups, bool):
return groups
return group in groups
selected, unselected = partition_manifest(
read_manifest_lines(sep, lines, title, in_cwd, out_cwd),
select, output_group, None)
return selected, unselected, groups_seen
def apply_source_rewrites(rewrites, entry):
for rewrite in rewrites:
if entry.source == rewrite.source:
return entry._replace(
return entry
def apply_rewrites(sep, rewrites, entry):
for pattern, line in rewrites:
if fnmatch.fnmatchcase(, pattern):
[new_entry] = read_manifest_lines(
[line.format(**entry._asdict()) + '\n'], entry.manifest,
entry = new_entry._replace(
return entry
def contents_entry(entry):
with open(entry.source) as file:
[line] =
return entry._replace(source=line)
class input_action_base(argparse.Action):
def __init__(self, *args, **kwargs):
super(input_action_base, self).__init__(*args, **kwargs)
def __call__(self, parser, namespace, values, option_string=None):
outputs = getattr(namespace, 'output', None)
all_selected = getattr(namespace, 'selected', None)
if all_selected is None:
all_selected = []
setattr(namespace, 'selected', all_selected)
all_unselected = getattr(namespace, 'unselected', None)
if all_unselected is None:
all_unselected = []
setattr(namespace, 'unselected', all_unselected)
if namespace.groups is None or outputs is None:
groups = False
elif namespace.groups == 'all':
groups = True
groups = set(group if group else None
for group in namespace.groups.split(','))
cwd = getattr(namespace, 'cwd', '')
if outputs is not None:
output_group = len(outputs) - 1
output_group = None
selected, unselected, groups_seen = self.get_manifest_lines(
namespace, values, cwd, groups, namespace.output_cwd, output_group)
include = getattr(namespace, 'include', [])
include_source = getattr(namespace, 'include_source', [])
exclude = getattr(namespace, 'exclude', [])
if include or exclude or include_source:
def included(entry):
def matches(file, patterns):
return any(fnmatch.fnmatch(file, pattern)
for pattern in patterns)
if matches(, exclude):
return False
if include and not matches(, include):
return False
return (not include_source or
matches(entry.source, include_source))
unselected += filter(lambda entry: not included(entry), selected)
selected = filter(included, selected)
if getattr(namespace, 'contents', False):
selected = map(contents_entry, selected);
unselected = map(contents_entry, unselected);
sep = getattr(namespace, 'separator', '=')
rewrites = [entry.split('=', 1)
for entry in getattr(namespace, 'rewrite', [])]
selected = [apply_rewrites(sep, rewrites, entry) for entry in selected]
unselected = [apply_rewrites(sep, rewrites, entry) for entry in unselected]
if not isinstance(groups, bool):
unused_groups = groups - groups_seen - set([None])
if unused_groups:
raise Exception(
'%s not found in %r; try one of: %s' %
(', '.join(map(repr, unused_groups)), values,
', '.join(map(repr, groups_seen - groups))))
all_selected += selected
all_unselected += unselected
class input_manifest_action(input_action_base):
def __init__(self, *args, **kwargs):
super(input_manifest_action, self).__init__(*args, **kwargs)
def get_manifest_lines(self, namespace, filename, *args):
all_inputs = getattr(namespace, 'manifest', None)
if all_inputs is None:
all_inputs = []
setattr(namespace, 'manifest', all_inputs)
with open(filename, 'r') as file:
return ingest_manifest_lines(getattr(namespace, 'separator', '='),
file,, *args)
class input_entry_action(input_action_base):
def __init__(self, *args, **kwargs):
super(input_entry_action, self).__init__(*args, **kwargs)
def get_manifest_lines(self, namespace, entry, *args):
return ingest_manifest_lines(
getattr(namespace, 'separator', '='),
[entry + '\n'], namespace.entry_manifest, *args)
def common_parse_args(parser):
parser.convert_arg_line_to_args = shlex.split
parser.add_argument('--output', action='append', required=True,
help='Output file')
parser.add_argument('--output-cwd', default='',
help='Emit source paths relative to DIRECTORY')
parser.add_argument('--absolute', action='store_true', default=False,
help='Output source file names as absolute paths')
parser.add_argument('--cwd', default='',
help='Input entries are relative to this directory')
parser.add_argument('--groups', default='all',
help='"all" or comma-separated groups to include')
parser.add_argument('--manifest', action=input_manifest_action,
metavar='FILE', default=[],
help='Input manifest file (must exist)')
parser.add_argument('--entry', action=input_entry_action,
help='Add a single entry as if from an input manifest')
parser.add_argument('--entry-manifest', default='<command-line --entry>',
help=('Title in lieu of manifest file name for' +
' subsequent --entry arguments'))
parser.add_argument('--include', action='append', default=[],
help='Include only input entries matching TARGET'),
parser.add_argument('--include-source', action='append', default=[],
help='Include only input entries matching SOURCE'),
action='store_const', const=[], dest='include',
help='Reset previous --include')
parser.add_argument('--exclude', action='append', default=[],
help='Ignore input entries matching TARGET'),
action='store_const', const=[], dest='exclude',
help='Reset previous --exclude')
parser.add_argument('--separator', default='=',
help='Use SEP between TARGET and SOURCE in entries')
return parser.parse_args()
def parse_args():
parser = argparse.ArgumentParser(description='Read manifest files.')
parser.add_argument('--copy-contentaddr', action='store_true', default=False,
help='Copy to content-addressed targets, not manifest')
parser.add_argument('--sources', action='store_true', default=False,
help='Write source file per line, not manifest entry')
action='store_true', default=False,
help='Replace each source file name with its contents')
action='store_false', dest='contents',
help='Reset previous --contents')
'--rewrite', action='append', default=[],
help='Replace entries whose target matches PATTERN with ENTRY,'
' which can use {source} and {target} substitutions'),
parser.add_argument('--reset-rewrite', dest='rewrite',
action='store_const', const=[],
help='Reset previous --rewrite')
action='store_true', default=False,
help='Elide duplicates even with different sources')
help='Touch FILE at the end.')
args = common_parse_args(parser)
if args.copy_contentaddr:
if args.contents:
parser.error('--copy-contentaddr is incompatible with --contents')
args.unique = True
args.sources = True
return args
def main():
args = parse_args()
output_sets = [(dict() if args.unique else set()) for file in args.output]
for entry in getattr(args, 'selected', []):
assert is not None, entry
if args.absolute:
line = os.path.abspath(entry.source)
line = entry.source
if not args.sources:
line = + args.separator + line
if args.unique:
output_sets[][] = line
for output_filename, output_set in zip(args.output, output_sets):
if args.copy_contentaddr:
created_dirs = set()
for target, source in output_set.iteritems():
target_path = os.path.join(output_filename, target)
if os.path.exists(target_path):
target_dir = os.path.dirname(target_path)
if target_dir not in created_dirs:
if not os.path.exists(target_dir):
shutil.copyfile(source, target_path)
with open(output_filename, 'w') as file:
line + '\n' for line in
(output_set.itervalues() if args.unique else output_set))))
if args.stamp:
with open(args.stamp, 'w') as file:
os.utime(, None)
return 0
if __name__ == '__main__':