| #!/usr/bin/env python3 |
| |
| # Copyright 2020 The Fuchsia Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| """This script is used to roll cobalt's submodules to match those used in Fuchsia. |
| |
| It also copies some files from fuchsia and syncs the versions of prebuilts. |
| |
| With a clean checkout, updating the submodules should be as simple as: |
| |
| ./cobaltb.py sync_with_fuchsia --make_commit |
| |
| This will update all of the submodules to match those used in Fuchsia, and then |
| if there are any changes, will add them all to a commit that you can then upload |
| to Gerrit. |
| |
| If you omit the '--make_commit' flag, then the commit will not be generated, but |
| a suggested commit message will be printed out at the end of the command. |
| """ |
| |
| import os |
| import sys |
| # Ensure that this file can import from cobalt root even if run as a script. |
| sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) |
| |
| import tempfile |
| import xml.etree.ElementTree as ElementTree |
| import subprocess |
| import re |
| import shutil |
| |
| THIS_DIR = os.path.abspath(os.path.dirname(__file__)) |
| ROOT_DIR = os.path.abspath(os.path.join(THIS_DIR, '..')) |
| |
| |
| def short_hash(long_hash): |
| """Returns a shortened, still unique commit hash.""" |
| return ( |
| subprocess.check_output(['git', 'rev-parse', '--short', long_hash]) |
| .decode('utf-8') |
| .strip() |
| ) |
| |
| |
| def hash_distance(start, end): |
| """Returns the number of commits between the two provided hashes.""" |
| return len( |
| subprocess.check_output(['git', 'rev-list', '%s..%s' % (start, end)]) |
| .decode('utf-8') |
| .strip() |
| .split('\n') |
| ) |
| |
| |
| def update_from_project(project, update_report, commit_message): |
| """Attempts to update the submodule for the supplied project. |
| |
| The project is of the following format: |
| |
| <project name="third_party/abseil-cpp" |
| path="third_party/abseil-cpp" |
| revision="17b90e77683fd71c72478c2af966fe0a4c1c07be" |
| remote="https://fuchsia.googlesource.com/third_party/abseil-cpp"/> |
| """ |
| full_path = os.path.join(ROOT_DIR, project.attrib['path']) |
| if os.path.isdir(full_path): |
| savedDir = os.getcwd() |
| try: |
| os.chdir(full_path) |
| root_dir = ( |
| subprocess.check_output(['git', 'rev-parse', '--show-toplevel']) |
| .decode('utf-8') |
| .strip() |
| ) |
| if root_dir == ROOT_DIR: |
| # If the root_dir at this path is equal to the global ROOT_DIR, that means this path exists |
| # but is not actually a submodule. (for example //third_party/go) |
| return |
| |
| current_hash = ( |
| subprocess.check_output(['git', 'rev-parse', 'HEAD']) |
| .decode('utf-8') |
| .strip() |
| ) |
| if current_hash != project.attrib['revision']: |
| # The current HEAD does not match the revision specified in the manifest |
| print( |
| 'Updating %s to %s' |
| % (project.attrib['path'], project.attrib['revision']) |
| ) |
| subprocess.check_call(['git', 'fetch', 'origin']) |
| subprocess.check_call(['git', 'checkout', project.attrib['revision']]) |
| |
| # Generate a human readable submodule update message |
| num_commits = hash_distance(current_hash, project.attrib['revision']) |
| commit_count = '' |
| if num_commits == 1: |
| commit_count = ' (1 commit)' |
| raw_msg = ( |
| subprocess.check_output([ |
| 'git', |
| 'log', |
| '--format=%B', |
| '-n', |
| '1', |
| project.attrib['revision'], |
| ]) |
| .decode('utf-8') |
| .strip() |
| ) |
| # Replace commit labels (like: Change-Id: ... with Original-Change-Id: ...) |
| filtered_msg = re.sub( |
| r'^([a-zA-Z-_]*: .*$)', |
| r'Original-\1', |
| raw_msg, |
| flags=re.MULTILINE, |
| ) |
| commit_message.append('[roll] Roll %s' % filtered_msg) |
| elif num_commits > 1: |
| commit_count = ' (%d commits)' % num_commits |
| update_report.append( |
| '%s %s..%s%s' |
| % ( |
| project.attrib['name'], |
| short_hash(current_hash), |
| short_hash(project.attrib['revision']), |
| commit_count, |
| ) |
| ) |
| finally: |
| os.chdir(savedDir) |
| |
| |
| def update_from_manifest(manifest_path, commit_message): |
| """Iterates through each project in a manifest calling update_from_project for each one. |
| |
| The top level XML format is: |
| |
| <?xml version="1.0" encoding="UTF-8"?> |
| <manifest> |
| <projects> |
| <project ... /> |
| <project ... /> |
| </projects> |
| </manifest> |
| """ |
| print('Parsing %s' % manifest_path) |
| flower_tree = ElementTree.parse(manifest_path) |
| root = flower_tree.getroot() |
| update_report = [] |
| |
| print('Checking through projects...') |
| for project in root.findall('./projects/project'): |
| update_from_project(project, update_report, commit_message) |
| |
| return update_report |
| |
| |
| def sync_prebuilts(integration_dir, prebuilts_paths, desired_prebuilts): |
| print(f'Parsing {", ".join(prebuilts_paths)}') |
| with open(os.path.join(ROOT_DIR, 'cobalt.ensure'), 'w') as ensure: |
| prebuilts_trees = {} |
| roots = {} |
| for path in prebuilts_paths: |
| tree = ElementTree.parse(os.path.join(integration_dir, path)) |
| prebuilts_trees[path] = tree |
| roots[path] = tree.getroot() |
| ensure.write(""" |
| # Copyright 2017 The Fuchsia Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| # THIS FILE IS AUTO-GENERATED BY tools/sync_with_fuchsia.py DO NOT EDIT DIRECTLY. |
| |
| """) |
| current_subdir = '' |
| for prebuilt in desired_prebuilts: |
| subdir, package_name = prebuilt.split(':', 1) |
| if current_subdir != subdir: |
| current_subdir = subdir |
| ensure.write(f'\n@Subdir {subdir}\n') |
| found = False |
| for root in roots.values(): |
| for package in root.findall('./packages/package'): |
| if package.attrib['name'] in package_name: |
| ensure.write( |
| f'{package.attrib["name"]} {package.attrib["version"]}\n' |
| ) |
| found = True |
| |
| if not found: |
| sys.exit(f'Could not find {package_name} in the fuchsia prebuilts file') |
| |
| ensure.write('\n\n') |
| ensure.write('#######################################################\n') |
| ensure.write('# Below is copied from tools/cobalt.ensure.exceptions #\n') |
| ensure.write('#######################################################\n') |
| with open( |
| os.path.join(ROOT_DIR, 'tools/cobalt.ensure.exceptions'), 'r' |
| ) as ensure_others: |
| ensure.write(ensure_others.read()) |
| |
| |
| def sync_from_integration( |
| integration_repo, |
| manifest_files, |
| prebuilts_files, |
| desired_prebuilts, |
| make_commit, |
| ): |
| """Checks out an integration repository and calls update_from_manifest on each manifest_file. |
| |
| If make_commit is true, the resulting submodule change will be added to a |
| commit. |
| """ |
| with tempfile.TemporaryDirectory( |
| prefix='fuchsia-integration-' |
| ) as integration_dir: |
| print('Checking out integration repo...') |
| subprocess.check_call(['git', 'clone', integration_repo, integration_dir]) |
| print() |
| |
| sync_prebuilts(integration_dir, prebuilts_files, desired_prebuilts) |
| |
| update_report = [] |
| commit_message = [] |
| for manifest_file in manifest_files: |
| update_report.extend( |
| update_from_manifest( |
| os.path.join(integration_dir, manifest_file), commit_message |
| ) |
| ) |
| |
| commit_msg = '' |
| if len(update_report) > 1: |
| commit_msg = """[roll] Roll %d submodules |
| |
| %s""" % (len(update_report), '\n'.join(sorted(update_report))) |
| elif len(update_report) == 1: |
| if len(commit_message) == 1: |
| commit_msg = commit_message[0] |
| else: |
| commit_msg = '[roll] Roll %s' % update_report[0] |
| |
| if commit_msg != '': |
| if make_commit: |
| subprocess.check_call(['git', 'commit', '-am', commit_msg]) |
| else: |
| print() |
| print() |
| print(commit_msg) |
| print() |
| print() |
| |
| |
| def sync_files_from_fuchsia(fuchsia_repo, to_copy, retain_paths): |
| """Checks out the fuchsia repo and copies specified files.""" |
| with tempfile.TemporaryDirectory(prefix='fuchsia-repo-') as fuchsia_dir: |
| print('Checking out fuchsia repo...') |
| subprocess.check_call( |
| ['git', 'clone', '--depth', '1', fuchsia_repo, fuchsia_dir] |
| ) |
| print() |
| |
| retain_paths = [os.path.join(ROOT_DIR, path) for path in retain_paths] |
| |
| for filename in to_copy: |
| print(f'Copying {filename}') |
| f = os.path.join(fuchsia_dir, filename) |
| t = os.path.join(ROOT_DIR, filename) |
| if os.path.isdir(f): |
| for path in os.listdir(t): |
| fname = os.path.join(t, path) |
| if fname not in retain_paths: |
| if os.path.isdir(fname): |
| shutil.rmtree(fname) |
| else: |
| os.remove(fname) |
| shutil.copytree(f, t, dirs_exist_ok=True) |
| else: |
| shutil.copy2(f, t) |
| |
| |
| DEFAULT_FUCHSIA_REPO = 'http://fuchsia.googlesource.com/fuchsia' |
| DEFAULT_INTEGRATION_REPO = 'http://fuchsia.googlesource.com/integration' |
| DEFAULT_PREBUILTS_FILES = ['prebuilts', 'toolchain'] |
| |
| DEFAULT_TO_COPY_FROM_FUCHSIA = [ |
| 'third_party/googletest/BUILD.gn', |
| 'third_party/boringssl', |
| ] |
| DEFAULT_RETAIN_PATHS = ['third_party/boringssl/src'] |
| |
| DEFAULT_MANIFEST_FILES = ['third_party/flower'] |
| DEFAULT_DESIRED_PREBUILTS = [ |
| r':fuchsia/third_party/clang/${platform}', |
| r':fuchsia/third_party/rust/host/${platform}', |
| r':fuchsia/third_party/rust/target/x86_64-unknown-linux-gnu', |
| r':fuchsia/third_party/sysroot/linux', |
| r':infra/3pp/tools/cpython3/${platform}', |
| r'bin:fuchsia/third_party/ninja/${platform}', |
| r'bin:fuchsia/tools/buildidtool/${platform}', |
| r'golang:fuchsia/go/${platform}', |
| r'goma:fuchsia/third_party/goma/client/${platform}', |
| ] |
| |
| |
| def sync( |
| integration_repo=DEFAULT_INTEGRATION_REPO, |
| fuchsia_repo=DEFAULT_FUCHSIA_REPO, |
| manifest_files=DEFAULT_MANIFEST_FILES, |
| to_copy_from_fuchsia=DEFAULT_TO_COPY_FROM_FUCHSIA, |
| retain_paths=DEFAULT_RETAIN_PATHS, |
| prebuilts_files=DEFAULT_PREBUILTS_FILES, |
| desired_prebuilts=DEFAULT_DESIRED_PREBUILTS, |
| make_commit=False, |
| ): |
| sync_files_from_fuchsia(fuchsia_repo, to_copy_from_fuchsia, retain_paths) |
| sync_from_integration( |
| integration_repo, |
| manifest_files, |
| prebuilts_files, |
| desired_prebuilts, |
| make_commit, |
| ) |
| |
| |
| if __name__ == '__main__': |
| sync() |