tools/sync_with_fuchsia.py - cobalt - Git at Google

 #!/usr/bin/env python3

 # Copyright 2020 The Fuchsia Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 """This script is used to roll cobalt's submodules to match those used in Fuchsia.

 It also copies some files from fuchsia and syncs the versions of prebuilts.

 With a clean checkout, updating the submodules should be as simple as:

   ./cobaltb.py sync_with_fuchsia --make_commit

 This will update all of the submodules to match those used in Fuchsia, and then
 if there are any changes, will add them all to a commit that you can then upload
 to Gerrit.

 If you omit the '--make_commit' flag, then the commit will not be generated, but
 a suggested commit message will be printed out at the end of the command.
 """

 import sys
 import os
 # Ensure that this file can import from cobalt root even if run as a script.
 sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

 import tempfile
 import xml.etree.ElementTree as ElementTree
 import subprocess
 import re
 import shutil

 THIS_DIR = os.path.abspath(os.path.dirname(__file__))
 ROOT_DIR = os.path.abspath(os.path.join(THIS_DIR, '..'))


 def short_hash(long_hash):
   """Returns a shortened, still unique commit hash."""
   return subprocess.check_output(['git', 'rev-parse', '--short',
                                   long_hash]).decode('utf-8').strip()


 def hash_distance(start, end):
   """Returns the number of commits between the two provided hashes."""
   return len(
       subprocess.check_output(['git', 'rev-list',
                                '%s..%s' % (start, end)
                               ]).decode('utf-8').strip().split('\n'))


 def update_from_project(project, update_report, commit_message):
   """Attempts to update the submodule for the supplied project.

   The project is of the following format:

   <project name="third_party/abseil-cpp"
            path="third_party/abseil-cpp"
            revision="17b90e77683fd71c72478c2af966fe0a4c1c07be"
            remote="https://fuchsia.googlesource.com/third_party/abseil-cpp"/>
   """
   full_path = os.path.join(ROOT_DIR, project.attrib['path'])
   if os.path.isdir(full_path):
     savedDir = os.getcwd()
     try:
       os.chdir(full_path)
       root_dir = subprocess.check_output(
           ['git', 'rev-parse', '--show-toplevel']).decode('utf-8').strip()
       if root_dir == ROOT_DIR:
         # If the root_dir at this path is equal to the global ROOT_DIR, that means this path exists
         # but is not actually a submodule. (for example //third_party/go)
         return

       current_hash = subprocess.check_output(['git', 'rev-parse',
                                               'HEAD']).decode('utf-8').strip()
       if current_hash != project.attrib['revision']:
         # The current HEAD does not match the revision specified in the manifest
         print('Updating %s to %s' %
               (project.attrib['path'], project.attrib['revision']))
         subprocess.check_call(['git', 'fetch', 'origin'])
         subprocess.check_call(['git', 'checkout', project.attrib['revision']])

         # Generate a human readable submodule update message
         num_commits = hash_distance(current_hash, project.attrib['revision'])
         commit_count = ''
         if num_commits == 1:
           commit_count = ' (1 commit)'
           raw_msg = subprocess.check_output([
               'git', 'log', '--format=%B', '-n', '1', project.attrib['revision']
           ]).decode('utf-8').strip()
           # Replace commit labels (like: Change-Id: ... with Original-Change-Id: ...)
           filtered_msg = re.sub(
               r'^([a-zA-Z-_]*: .*$)',
               r'Original-\1',
               raw_msg,
               flags=re.MULTILINE)
           commit_message.append('[roll] Roll %s' % filtered_msg)
         elif num_commits > 1:
           commit_count = ' (%d commits)' % num_commits
         update_report.append(
             '%s %s..%s%s' %
             (project.attrib['name'], short_hash(current_hash),
              short_hash(project.attrib['revision']), commit_count))
     finally:
       os.chdir(savedDir)


 def update_from_manifest(manifest_path, commit_message):
   """Iterates through each project in a manifest calling update_from_project for each one.

   The top level XML format is:

   <?xml version="1.0" encoding="UTF-8"?>
   <manifest>
     <projects>
       <project ... />
       <project ... />
     </projects>
   </manifest>
   """
   print('Parsing %s' % manifest_path)
   flower_tree = ElementTree.parse(manifest_path)
   root = flower_tree.getroot()
   update_report = []

   print('Checking through projects...')
   for project in root.findall('./projects/project'):
     update_from_project(project, update_report, commit_message)

   return update_report


 def sync_prebuilts(integration_dir, prebuilts_paths, desired_prebuilts):
   print(f'Parsing {", ".join(prebuilts_paths)}')
   with open(os.path.join(ROOT_DIR, 'cobalt.ensure'), 'w') as ensure:
     prebuilts_trees = {}
     roots = {}
     for path in prebuilts_paths:
       tree = ElementTree.parse(os.path.join(integration_dir, path))
       prebuilts_trees[path] = tree
       roots[path] = tree.getroot()
     ensure.write("""
 # Copyright 2017 The Fuchsia Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.

 # THIS FILE IS AUTO-GENERATED BY tools/sync_with_fuchsia.py DO NOT EDIT DIRECTLY.

 """)
     current_subdir = ''
     for prebuilt in desired_prebuilts:
       subdir, package_name = prebuilt.split(':', 1)
       if current_subdir != subdir:
         current_subdir = subdir
         ensure.write(f'\n@Subdir {subdir}\n')
       found = False
       for root in roots.values():
         for package in root.findall('./packages/package'):
           if package.attrib['name'] in package_name:
             ensure.write(
                 f'{package.attrib["name"]} {package.attrib["version"]}\n')
             found = True

       if not found:
         sys.exit(f'Could not find {package_name} in the fuchsia prebuilts file')

     ensure.write('\n\n')
     ensure.write('#######################################################\n')
     ensure.write('# Below is copied from tools/cobalt.ensure.exceptions #\n')
     ensure.write('#######################################################\n')
     with open(os.path.join(ROOT_DIR, 'tools/cobalt.ensure.exceptions'),
               'r') as ensure_others:
       ensure.write(ensure_others.read())


 def sync_from_integration(integration_repo, manifest_files, prebuilts_files,
                           desired_prebuilts, make_commit):
   """Checks out an integration repository and calls update_from_manifest on each manifest_file.

   If make_commit is true, the resulting submodule change will be added to a
   commit.
   """
   with tempfile.TemporaryDirectory(
       prefix='fuchsia-integration-') as integration_dir:
     print('Checking out integration repo...')
     subprocess.check_call(['git', 'clone', integration_repo, integration_dir])
     print()

     sync_prebuilts(integration_dir, prebuilts_files, desired_prebuilts)

     update_report = []
     commit_message = []
     for manifest_file in manifest_files:
       update_report.extend(
           update_from_manifest(
               os.path.join(integration_dir, manifest_file), commit_message))

     commit_msg = ''
     if len(update_report) > 1:
       commit_msg = """[roll] Roll %d submodules

 %s""" % (len(update_report), '\n'.join(sorted(update_report)))
     elif len(update_report) == 1:
       if len(commit_message) == 1:
         commit_msg = commit_message[0]
       else:
         commit_msg = '[roll] Roll %s' % update_report[0]

     if commit_msg != '':
       if make_commit:
         subprocess.check_call(['git', 'commit', '-am', commit_msg])
       else:
         print()
         print()
         print(commit_msg)
         print()
         print()


 def sync_files_from_fuchsia(fuchsia_repo, to_copy, retain_paths):
   """Checks out the fuchsia repo and copies specified files."""
   with tempfile.TemporaryDirectory(prefix='fuchsia-repo-') as fuchsia_dir:
     print('Checking out fuchsia repo...')
     subprocess.check_call(
         ['git', 'clone', '--depth', '1', fuchsia_repo, fuchsia_dir])
     print()

     retain_paths = [os.path.join(ROOT_DIR, path) for path in retain_paths]

     for filename in to_copy:
       print(f'Copying {filename}')
       f = os.path.join(fuchsia_dir, filename)
       t = os.path.join(ROOT_DIR, filename)
       if os.path.isdir(f):
         for path in os.listdir(t):
           fname = os.path.join(t, path)
           if fname not in retain_paths:
             if os.path.isdir(fname):
               shutil.rmtree(fname)
             else:
               os.remove(fname)
         shutil.copytree(f, t, dirs_exist_ok=True)
       else:
         shutil.copy2(f, t)


 DEFAULT_FUCHSIA_REPO = 'http://fuchsia.googlesource.com/fuchsia'
 DEFAULT_INTEGRATION_REPO = 'http://fuchsia.googlesource.com/integration'
 DEFAULT_PREBUILTS_FILES = ['prebuilts', 'toolchain']

 DEFAULT_TO_COPY_FROM_FUCHSIA = [
     'third_party/googletest/BUILD.gn', 'third_party/boringssl'
 ]
 DEFAULT_RETAIN_PATHS = ['third_party/boringssl/src']

 DEFAULT_MANIFEST_FILES = ['third_party/flower']
 DEFAULT_DESIRED_PREBUILTS = [
     r':fuchsia/dart-sdk/${platform}',
     r':fuchsia/third_party/clang/${platform}',
     r':fuchsia/third_party/rust/${platform}',
     r':fuchsia/third_party/sysroot/linux',
     r':infra/3pp/tools/cpython3/${platform}',
     r'bin:fuchsia/third_party/ninja/${platform}',
     r'bin:fuchsia/tools/buildidtool/${platform}',
     r'golang:fuchsia/go/${platform}',
     r'goma:fuchsia/third_party/goma/client/${platform}',
 ]


 def sync(integration_repo=DEFAULT_INTEGRATION_REPO,
          fuchsia_repo=DEFAULT_FUCHSIA_REPO,
          manifest_files=DEFAULT_MANIFEST_FILES,
          to_copy_from_fuchsia=DEFAULT_TO_COPY_FROM_FUCHSIA,
          retain_paths=DEFAULT_RETAIN_PATHS,
          prebuilts_files=DEFAULT_PREBUILTS_FILES,
          desired_prebuilts=DEFAULT_DESIRED_PREBUILTS,
          make_commit=False):
   sync_files_from_fuchsia(fuchsia_repo, to_copy_from_fuchsia, retain_paths)
   sync_from_integration(integration_repo, manifest_files, prebuilts_files,
                         desired_prebuilts, make_commit)


 if __name__ == '__main__':
   sync()
	#!/usr/bin/env python3

	# Copyright 2020 The Fuchsia Authors. All rights reserved.
	# Use of this source code is governed by a BSD-style license that can be
	# found in the LICENSE file.
	"""This script is used to roll cobalt's submodules to match those used in Fuchsia.

	It also copies some files from fuchsia and syncs the versions of prebuilts.

	With a clean checkout, updating the submodules should be as simple as:

	./cobaltb.py sync_with_fuchsia --make_commit

	This will update all of the submodules to match those used in Fuchsia, and then
	if there are any changes, will add them all to a commit that you can then upload
	to Gerrit.

	If you omit the '--make_commit' flag, then the commit will not be generated, but
	a suggested commit message will be printed out at the end of the command.
	"""

	import sys
	import os
	# Ensure that this file can import from cobalt root even if run as a script.
	sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

	import tempfile
	import xml.etree.ElementTree as ElementTree
	import subprocess
	import re
	import shutil

	THIS_DIR = os.path.abspath(os.path.dirname(__file__))
	ROOT_DIR = os.path.abspath(os.path.join(THIS_DIR, '..'))


	def short_hash(long_hash):
	"""Returns a shortened, still unique commit hash."""
	return subprocess.check_output(['git', 'rev-parse', '--short',
	long_hash]).decode('utf-8').strip()


	def hash_distance(start, end):
	"""Returns the number of commits between the two provided hashes."""
	return len(
	subprocess.check_output(['git', 'rev-list',
	'%s..%s' % (start, end)
	]).decode('utf-8').strip().split('\n'))


	def update_from_project(project, update_report, commit_message):
	"""Attempts to update the submodule for the supplied project.

	The project is of the following format:

	<project name="third_party/abseil-cpp"
	path="third_party/abseil-cpp"
	revision="17b90e77683fd71c72478c2af966fe0a4c1c07be"
	remote="https://fuchsia.googlesource.com/third_party/abseil-cpp"/>
	"""
	full_path = os.path.join(ROOT_DIR, project.attrib['path'])
	if os.path.isdir(full_path):
	savedDir = os.getcwd()
	try:
	os.chdir(full_path)
	root_dir = subprocess.check_output(
	['git', 'rev-parse', '--show-toplevel']).decode('utf-8').strip()
	if root_dir == ROOT_DIR:
	# If the root_dir at this path is equal to the global ROOT_DIR, that means this path exists
	# but is not actually a submodule. (for example //third_party/go)
	return

	current_hash = subprocess.check_output(['git', 'rev-parse',
	'HEAD']).decode('utf-8').strip()
	if current_hash != project.attrib['revision']:
	# The current HEAD does not match the revision specified in the manifest
	print('Updating %s to %s' %
	(project.attrib['path'], project.attrib['revision']))
	subprocess.check_call(['git', 'fetch', 'origin'])
	subprocess.check_call(['git', 'checkout', project.attrib['revision']])

	# Generate a human readable submodule update message
	num_commits = hash_distance(current_hash, project.attrib['revision'])
	commit_count = ''
	if num_commits == 1:
	commit_count = ' (1 commit)'
	raw_msg = subprocess.check_output([
	'git', 'log', '--format=%B', '-n', '1', project.attrib['revision']
	]).decode('utf-8').strip()
	# Replace commit labels (like: Change-Id: ... with Original-Change-Id: ...)
	filtered_msg = re.sub(
	r'^([a-zA-Z-_]: .$)',
	r'Original-\1',
	raw_msg,
	flags=re.MULTILINE)
	commit_message.append('[roll] Roll %s' % filtered_msg)
	elif num_commits > 1:
	commit_count = ' (%d commits)' % num_commits
	update_report.append(
	'%s %s..%s%s' %
	(project.attrib['name'], short_hash(current_hash),
	short_hash(project.attrib['revision']), commit_count))
	finally:
	os.chdir(savedDir)


	def update_from_manifest(manifest_path, commit_message):
	"""Iterates through each project in a manifest calling update_from_project for each one.

	The top level XML format is:

	<?xml version="1.0" encoding="UTF-8"?>
	<manifest>
	<projects>
	<project ... />
	<project ... />
	</projects>
	</manifest>
	"""
	print('Parsing %s' % manifest_path)
	flower_tree = ElementTree.parse(manifest_path)
	root = flower_tree.getroot()
	update_report = []

	print('Checking through projects...')
	for project in root.findall('./projects/project'):
	update_from_project(project, update_report, commit_message)

	return update_report


	def sync_prebuilts(integration_dir, prebuilts_paths, desired_prebuilts):
	print(f'Parsing {", ".join(prebuilts_paths)}')
	with open(os.path.join(ROOT_DIR, 'cobalt.ensure'), 'w') as ensure:
	prebuilts_trees = {}
	roots = {}
	for path in prebuilts_paths:
	tree = ElementTree.parse(os.path.join(integration_dir, path))
	prebuilts_trees[path] = tree
	roots[path] = tree.getroot()
	ensure.write("""
	# Copyright 2017 The Fuchsia Authors. All rights reserved.
	# Use of this source code is governed by a BSD-style license that can be
	# found in the LICENSE file.

	# THIS FILE IS AUTO-GENERATED BY tools/sync_with_fuchsia.py DO NOT EDIT DIRECTLY.

	""")
	current_subdir = ''
	for prebuilt in desired_prebuilts:
	subdir, package_name = prebuilt.split(':', 1)
	if current_subdir != subdir:
	current_subdir = subdir
	ensure.write(f'\n@Subdir {subdir}\n')
	found = False
	for root in roots.values():
	for package in root.findall('./packages/package'):
	if package.attrib['name'] in package_name:
	ensure.write(
	f'{package.attrib["name"]} {package.attrib["version"]}\n')
	found = True

	if not found:
	sys.exit(f'Could not find {package_name} in the fuchsia prebuilts file')

	ensure.write('\n\n')
	ensure.write('#######################################################\n')
	ensure.write('# Below is copied from tools/cobalt.ensure.exceptions #\n')
	ensure.write('#######################################################\n')
	with open(os.path.join(ROOT_DIR, 'tools/cobalt.ensure.exceptions'),
	'r') as ensure_others:
	ensure.write(ensure_others.read())


	def sync_from_integration(integration_repo, manifest_files, prebuilts_files,
	desired_prebuilts, make_commit):
	"""Checks out an integration repository and calls update_from_manifest on each manifest_file.

	If make_commit is true, the resulting submodule change will be added to a
	commit.
	"""
	with tempfile.TemporaryDirectory(
	prefix='fuchsia-integration-') as integration_dir:
	print('Checking out integration repo...')
	subprocess.check_call(['git', 'clone', integration_repo, integration_dir])
	print()

	sync_prebuilts(integration_dir, prebuilts_files, desired_prebuilts)

	update_report = []
	commit_message = []
	for manifest_file in manifest_files:
	update_report.extend(
	update_from_manifest(
	os.path.join(integration_dir, manifest_file), commit_message))

	commit_msg = ''
	if len(update_report) > 1:
	commit_msg = """[roll] Roll %d submodules

	%s""" % (len(update_report), '\n'.join(sorted(update_report)))
	elif len(update_report) == 1:
	if len(commit_message) == 1:
	commit_msg = commit_message[0]
	else:
	commit_msg = '[roll] Roll %s' % update_report[0]

	if commit_msg != '':
	if make_commit:
	subprocess.check_call(['git', 'commit', '-am', commit_msg])
	else:
	print()
	print()
	print(commit_msg)
	print()
	print()


	def sync_files_from_fuchsia(fuchsia_repo, to_copy, retain_paths):
	"""Checks out the fuchsia repo and copies specified files."""
	with tempfile.TemporaryDirectory(prefix='fuchsia-repo-') as fuchsia_dir:
	print('Checking out fuchsia repo...')
	subprocess.check_call(
	['git', 'clone', '--depth', '1', fuchsia_repo, fuchsia_dir])
	print()

	retain_paths = [os.path.join(ROOT_DIR, path) for path in retain_paths]

	for filename in to_copy:
	print(f'Copying {filename}')
	f = os.path.join(fuchsia_dir, filename)
	t = os.path.join(ROOT_DIR, filename)
	if os.path.isdir(f):
	for path in os.listdir(t):
	fname = os.path.join(t, path)
	if fname not in retain_paths:
	if os.path.isdir(fname):
	shutil.rmtree(fname)
	else:
	os.remove(fname)
	shutil.copytree(f, t, dirs_exist_ok=True)
	else:
	shutil.copy2(f, t)


	DEFAULT_FUCHSIA_REPO = 'http://fuchsia.googlesource.com/fuchsia'
	DEFAULT_INTEGRATION_REPO = 'http://fuchsia.googlesource.com/integration'
	DEFAULT_PREBUILTS_FILES = ['prebuilts', 'toolchain']

	DEFAULT_TO_COPY_FROM_FUCHSIA = [
	'third_party/googletest/BUILD.gn', 'third_party/boringssl'
	]
	DEFAULT_RETAIN_PATHS = ['third_party/boringssl/src']

	DEFAULT_MANIFEST_FILES = ['third_party/flower']
	DEFAULT_DESIRED_PREBUILTS = [
	r':fuchsia/dart-sdk/${platform}',
	r':fuchsia/third_party/clang/${platform}',
	r':fuchsia/third_party/rust/${platform}',
	r':fuchsia/third_party/sysroot/linux',
	r':infra/3pp/tools/cpython3/${platform}',
	r'bin:fuchsia/third_party/ninja/${platform}',
	r'bin:fuchsia/tools/buildidtool/${platform}',
	r'golang:fuchsia/go/${platform}',
	r'goma:fuchsia/third_party/goma/client/${platform}',
	]


	def sync(integration_repo=DEFAULT_INTEGRATION_REPO,
	fuchsia_repo=DEFAULT_FUCHSIA_REPO,
	manifest_files=DEFAULT_MANIFEST_FILES,
	to_copy_from_fuchsia=DEFAULT_TO_COPY_FROM_FUCHSIA,
	retain_paths=DEFAULT_RETAIN_PATHS,
	prebuilts_files=DEFAULT_PREBUILTS_FILES,
	desired_prebuilts=DEFAULT_DESIRED_PREBUILTS,
	make_commit=False):
	sync_files_from_fuchsia(fuchsia_repo, to_copy_from_fuchsia, retain_paths)
	sync_from_integration(integration_repo, manifest_files, prebuilts_files,
	desired_prebuilts, make_commit)


	if __name__ == '__main__':
	sync()