blob: c39c94b97adefde0dbff4d17d386c49b6ae2422b [file] [log] [blame]
#!/usr/bin/env fuchsia-vendored-python
# Copyright 2016 The Fuchsia Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
from __future__ import print_function
import argparse
import json
import os
import paths
import re
import shutil
import subprocess
import sys
import tempfile
sys.path += [
os.path.join(paths.FUCHSIA_ROOT, 'third_party', 'pyyaml', 'src', 'lib')
]
import yaml
LICENSE_FILES = ['LICENSE', 'LICENSE.txt']
IGNORED_EXTENSIONS = [
'css', 'html', 'js', 'log', 'old', 'out', 'packages', 'snapshot', 'zip'
]
LOCAL_PACKAGES = {
'build_integration':
'//third_party/dart/pkg/build_integration',
'flutter':
'//third_party/dart-pkg/git/flutter/packages/flutter',
'flutter_test':
'//third_party/dart-pkg/git/flutter/packages/flutter_test',
'flutter_web_plugins':
'//third_party/dart-pkg/git/flutter/packages/flutter_web_plugins',
'func':
'//third_party/dart/third_party/pkg/func',
'sky_engine':
'//prebuilt/third_party/sky_engine',
'testing':
'//third_party/dart/pkg/testing',
'typed_mock':
'//third_party/dart/pkg/typed_mock',
}
FORBIDDEN_PACKAGES = ['mojo', 'mojo_services']
# This is to account for https://github.com/flutter/devtools/issues/1148
PACKAGES_WITH_NO_LIB = ['devtools']
# A list of package names that have directories that should not be included.
FORBIDDEN_DIRS = {
'characters': [
'third_party/Wikipedia' # Invalid license
]
}
def get_deps(package_name, parsed_yaml, dep_type):
if dep_type in parsed_yaml and parsed_yaml[dep_type]:
deps = parsed_yaml[dep_type]
# This is to avoid circular dependencies. See fxbug.dev/40784.
if package_name == 'built_value' and 'built_value_generator' in deps:
del deps['built_value_generator']
return deps
else:
return {}
def safe_parse_yaml(yaml_path):
""" parses a pubspec file that may be malformed """
# Some yaml files can be malformed and have an extra tab at the end
# of a line. This causes the parser to fail so we strip all tabs off
# the end of the lines.
with open(yaml_path, encoding='utf-8') as yaml_file:
yaml_data = []
for line in yaml_file.readlines():
yaml_data.append(line.rstrip('\t\n'))
yaml_doc = "\n".join(yaml_data)
parsed = yaml.safe_load(yaml_doc)
if not parsed:
raise Exception('Could not parse yaml file: %s' % yaml_file)
return parsed
def parse_min_sdk_version_and_full_dependencies(yaml_path):
""" parse the content of a pubspec.yaml """
parsed = safe_parse_yaml(yaml_path)
package_name = parsed['name']
# If a format like sdk: '>=a.b' or sdk: 'a.b' is found, we'll use a.b.
# If sdk is not specified (or 'any'), 2.8 is used.
# In all other cases 2.0 is used.
env_sdk = parsed.get('environment', {}).get('sdk', 'any')
match = re.search(r"^(>=)?((0|[1-9]\d*)\.(0|[1-9]\d*))", env_sdk)
if match:
min_sdk_version = match.group(2)
elif env_sdk == 'any':
min_sdk_version = '2.8'
else:
min_sdk_version = '2.0'
deps = get_deps(package_name, parsed, 'dependencies')
dev_deps = get_deps(package_name, parsed, 'dev_dependencies')
dep_overrides = get_deps(package_name, parsed, 'dependency_overrides')
pinned_deps = get_deps(package_name, parsed, 'pinned_dependencies')
return (
package_name, min_sdk_version, deps, dev_deps, dep_overrides,
pinned_deps)
def parse_min_sdk_and_dependencies(yaml_path):
""" parse the min sdk version and dependency map out of a pubspec.yaml """
_, min_sdk_version, deps, _, _, pinned_deps = parse_min_sdk_version_and_full_dependencies(
yaml_path)
return min_sdk_version, deps, pinned_deps
def write_build_file(
build_gn_path, package_name, name_with_version, language_version, deps,
dart_sources):
""" writes BUILD.gn file for Dart package with dependencies """
with open(build_gn_path, 'w', encoding='utf-8') as build_gn:
build_gn.write(
'''# This file is generated by package_importer.py for %s
import("//build/dart/dart_library.gni")
dart_library("%s") {
package_name = "%s"
language_version = "%s"
disable_analysis = true
deps = [
''' % (name_with_version, package_name, package_name, language_version))
for dep in deps:
if dep in LOCAL_PACKAGES:
build_gn.write(' "%s",\n' % LOCAL_PACKAGES[dep])
else:
build_gn.write(' "//third_party/dart-pkg/pub/%s",\n' % dep)
build_gn.write(''' ]
sources = [
''')
for source in sorted(dart_sources):
build_gn.write(' "%s",\n' % source)
build_gn.write(''' ]
}
''')
def read_package_versions(base):
'''Scans the packages in a given directory.'''
result = {}
for (root, dirs, files) in os.walk(base):
for dir in dirs:
spec = os.path.join(root, dir, 'pubspec.yaml')
if not os.path.exists(spec):
continue
data = safe_parse_yaml(spec)
result[data['name']] = data['version']
break
return result
def generate_package_diff(old_packages, new_packages, changelog):
'''Writes a changelog file with package version changes.'''
old = set(old_packages.items())
new = set(new_packages.items())
changed_keys = set([k for (k, _) in (old | new) - (old & new)])
if not changed_keys:
return
max_key_width = max(map(lambda k: len(k), changed_keys))
with open(changelog, 'w', encoding='utf-8') as changelog_file:
for key in sorted(changed_keys):
old = old_packages.get(key, '<none>')
new = new_packages.get(key, '<none>')
changelog_file.write(
'%s %s --> %s\n' %
(key.rjust(max_key_width), old.rjust(10), new.ljust(10)))
def valid_package_path(package_name, source_dir):
if package_name in PACKAGES_WITH_NO_LIB:
parent_dir = os.path.normpath(os.path.join(source_dir, os.pardir))
return os.path.exists(parent_dir)
else:
return os.path.exists(source_dir)
def main():
parser = argparse.ArgumentParser('Import dart packages from pub')
parser.add_argument(
'--dart', required=True, help='Path to the dart executable')
parser.add_argument(
'--pubspecs',
nargs='+',
help='Paths to packages containing pubspec.yaml files')
parser.add_argument(
'--git-pubspecs',
nargs='+',
help=
'A list of git pubspec locations formatted as DEP_NAME,GIT_URI,COMMIT_HASH,SUBDIR'
)
parser.add_argument(
'--projects',
nargs='+',
help='Paths to projects containing dependency files')
parser.add_argument(
'--output', required=True, help='Path to the output directory')
parser.add_argument(
'--changelog', help='Path to the changelog file to write', default=None)
parser.add_argument(
'--debug', help='Turns on debugging mode', action='store_true')
args = parser.parse_args()
def debug_print(message):
if args.debug:
print(message)
tempdir = tempfile.mkdtemp()
debug_print('Working directory: ' + tempdir)
try:
importer_dir = os.path.join(tempdir, 'importer')
os.mkdir(importer_dir)
# Read the requested dependencies from the canonical packages.
packages = {}
additional_deps = {}
debug_print('------------------------')
debug_print('Development dependencies')
debug_print('------------------------')
for path in args.pubspecs:
yaml_file = os.path.join(path, 'pubspec.yaml')
package_name, _, _, dev_deps, _, _ = parse_min_sdk_version_and_full_dependencies(
yaml_file)
packages[package_name] = path
additional_deps.update(dev_deps)
debug_print('# From ' + yaml_file)
for pair in sorted(dev_deps.items()):
debug_print(' - %s: %s' % pair)
# Generate a manifest containing all the dependencies we care about.
dependencies = {package_name: 'any' for package_name in packages.keys()}
for dep, version in additional_deps.items():
if dep in packages:
continue
dependencies[dep] = version
debug_print('-------------------------')
debug_print('Manually-set dependencies')
debug_print('-------------------------')
pinned_deps = None
for project in args.projects:
yaml_file = os.path.join(project, 'dart_dependencies.yaml')
_, project_deps, pinned = parse_min_sdk_and_dependencies(yaml_file)
pinned_deps = pinned if pinned else pinned_deps
debug_print('# From ' + yaml_file)
for dep, version in sorted(project_deps.items()):
dependencies[dep] = version
debug_print(' - %s: %s' % (dep, version))
overrides = {
package_name: {
'path': path,
} for package_name, path in packages.items()
}
# Start populating git-based dependencies.
if args.git_pubspecs:
debug_print('-----------------------------')
debug_print('Mannualy-set git dependencies')
debug_print('-----------------------------')
for git_pubspec in args.git_pubspecs:
dep_name, git_url, git_ref, git_path = git_pubspec.split(',')
dependencies[dep_name] = 'any'
overrides[dep_name] = {
'git': {
'url': git_url,
'ref': git_ref,
'path': git_path,
}
}
debug_print(yaml.safe_dump(dependencies, default_flow_style=False))
pubspec_filename = os.path.join(importer_dir, 'pubspec.yaml')
with open(pubspec_filename, 'w', encoding='utf-8') as pubspec:
yaml.safe_dump(
{
'name': 'importer',
'dependencies': dependencies,
'dependency_overrides': overrides,
'environment': {
'sdk': '>=2.0.0 <3.0.0'
}
},
pubspec,
default_flow_style=not args.debug)
old_packages = read_package_versions(args.output)
# Use pub to load the dependencies into a local cache.
pub_cache_dir = os.path.join(tempdir, 'pub_cache')
os.mkdir(pub_cache_dir)
env = os.environ
env['PUB_CACHE'] = pub_cache_dir
pub_get = [args.dart, 'pub', 'get']
if args.debug:
pub_get.append('-v')
subprocess.check_call(pub_get, cwd=importer_dir, env=env)
# Walk the cache and copy the packages we are interested in.
if os.path.exists(args.output):
for (root, dirs, files) in os.walk(args.output):
for dir in dirs:
if dir != '.git' and pinned_deps and dir not in pinned_deps.keys(
):
shutil.rmtree(os.path.join(root, dir))
# Only process the root of the output tree.
break
package_config_json = json.load(
open(
os.path.join(importer_dir, '.dart_tool/package_config.json'),
encoding='utf-8'))
pub_packages = package_config_json['packages']
package_config = {
'configVersion': 2,
'packages': [],
'generator': os.path.basename(__file__)
}
for package in pub_packages:
package_name = package['name']
if package_name in packages:
# Skip canonical packages.
continue
if not package['rootUri'].startswith('file://'):
continue
# We expect the package_config.json file to point to a directory called 'lib' (packageUri)
# inside the overall package, which will contain the LICENSE file
# and other potentially useful directories like 'bin'.
source_base_dir = package['rootUri'][len('file://'):]
source_dir = os.path.join(source_base_dir, package['packageUri'])
if not valid_package_path(package_name, source_dir):
continue
if not any(domain in source_dir
for domain in ['/pub.dartlang.org/', '/pub.dev/']):
print(
'Package %s not from dartlang (%s), ignoring' %
(package_name, source_dir))
continue
# Don't import packages that live canonically in the tree.
if package_name in LOCAL_PACKAGES:
continue
if package_name in FORBIDDEN_PACKAGES:
print(
'Warning: dependency on forbidden package %s' %
package_name)
continue
name_with_version = os.path.basename(source_base_dir)
has_license = any(
os.path.exists(os.path.join(source_base_dir, file_name))
for file_name in LICENSE_FILES)
if not has_license:
print(
'Could not find license file for %s, skipping' %
package_name)
continue
pubspec_path = os.path.join(source_base_dir, 'pubspec.yaml')
deps = []
min_sdk_version = '2.8'
if os.path.exists(pubspec_path):
min_sdk_version, deps, _ = parse_min_sdk_and_dependencies(
pubspec_path)
dest_dir = os.path.join(args.output, package_name)
dart_sources = []
# Add all dart files in source_dir subdirectory into dart_sources
for (path, dirs, files) in os.walk(source_dir):
for f in files:
if f.endswith('.dart'):
dart_sources.append(
os.path.relpath(os.path.join(path, f), source_dir))
if os.path.exists(dest_dir):
shutil.rmtree(dest_dir)
shutil.copytree(
source_base_dir,
dest_dir,
ignore=shutil.ignore_patterns(
*('*.' + extension for extension in IGNORED_EXTENSIONS)))
# We don't need the 'test' directory of packages we import as that
# directory exists to test that package and some of our packages
# have very heavy test directories, so nuke those.
test_path = os.path.join(dest_dir, 'test')
if os.path.exists(test_path):
shutil.rmtree(test_path)
# Check to see if this package has any forbidden directories.
if package_name in FORBIDDEN_DIRS:
for d in FORBIDDEN_DIRS[package_name]:
forbidden_dir = os.path.join(dest_dir, d)
if os.path.exists(forbidden_dir):
shutil.rmtree(forbidden_dir)
write_build_file(
os.path.join(dest_dir, 'BUILD.gn'), package_name,
name_with_version, min_sdk_version, deps, dart_sources)
# All pub packages are required to have a lib/ dir, so it's safe to
# hard-code this value.
package_config['packages'].append(
{
'name': package_name,
'rootUri': './%s/' % package_name,
'packageUri': 'lib/',
'languageVersion': min_sdk_version
})
# args.output == '//third_party/dart-pkg/pub/', so we'll try to
# serialize package_config to JSON by using json.dumps and write to
# //third_party/dart-pkg/pub/package_config.json.
with open(os.path.join(args.output, 'package_config.json'), 'w',
encoding='utf-8') as package_config_json:
package_config_json.write(
json.dumps(package_config, sort_keys=True, indent=2))
if args.changelog:
new_packages = read_package_versions(args.output)
generate_package_diff(old_packages, new_packages, args.changelog)
finally:
if not args.debug:
shutil.rmtree(tempdir)
if __name__ == '__main__':
sys.exit(main())