tools/update_submodules.py

#!/usr/bin/env python3

# Copyright 2020 The Fuchsia Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""This script is used to roll cobalt's submodules to match those used in Fuchsia.

With a clean checkout, updating the submodules should be as simple as:

  ./cobaltb.py update_submodules --make_commit

This will update all of the submodules to match those used in Fuchsia, and then
if there are any changes, will add them all to a commit that you can then upload
to Gerrit.

If you omit the '--make_commit' flag, then the commit will not be generated, but
a suggested commit message will be printed out at the end of the command.
"""

import tempfile
import xml.etree.ElementTree as ElementTree
import subprocess
import os
import re

THIS_DIR = os.path.abspath(os.path.dirname(__file__))
ROOT_DIR = os.path.abspath(os.path.join(THIS_DIR, '..'))


def short_hash(long_hash):
  """Returns a shortened, still unique commit hash."""
  return subprocess.check_output(['git', 'rev-parse', '--short',
                                  long_hash]).decode('utf-8').strip()


def hash_distance(start, end):
  """Returns the number of commits between the two provided hashes."""
  return len(
      subprocess.check_output(['git', 'rev-list',
                               '%s..%s' % (start, end)
                              ]).decode('utf-8').strip().split('\n'))


def update_from_project(project, update_report, commit_message):
  """Attempts to update the submodule for the supplied project.

  The project is of the following format:

  <project name="third_party/abseil-cpp"
           path="third_party/abseil-cpp"
           revision="17b90e77683fd71c72478c2af966fe0a4c1c07be"
           remote="https://fuchsia.googlesource.com/third_party/abseil-cpp"/>
  """
  full_path = os.path.join(ROOT_DIR, project.attrib['path'])
  if os.path.isdir(full_path):
    savedDir = os.getcwd()
    try:
      os.chdir(full_path)
      root_dir = subprocess.check_output(
          ['git', 'rev-parse', '--show-toplevel']).decode('utf-8').strip()
      if root_dir == ROOT_DIR:
        # If the root_dir at this path is equal to the global ROOT_DIR, that means this path exists
        # but is not actually a submodule. (for example //third_party/go)
        return

      current_hash = subprocess.check_output(['git', 'rev-parse',
                                              'HEAD']).decode('utf-8').strip()
      if current_hash != project.attrib['revision']:
        # The current HEAD does not match the revision specified in the manifest
        print('Updating %s to %s' %
              (project.attrib['path'], project.attrib['revision']))
        subprocess.check_call(['git', 'fetch', 'origin'])
        subprocess.check_call(['git', 'checkout', project.attrib['revision']])

        # Generate a human readable submodule update message
        num_commits = hash_distance(current_hash, project.attrib['revision'])
        commit_count = ''
        if num_commits == 1:
          commit_count = ' (1 commit)'
          raw_msg = subprocess.check_output([
              'git', 'log', '--format=%B', '-n', '1', project.attrib['revision']
          ]).decode('utf-8').strip()
          # Replace commit labels (like: Change-Id: ... with Original-Change-Id: ...)
          filtered_msg = re.sub(
              r'^([a-zA-Z-_]*: .*$)', r'Original-\1', raw_msg, flags=re.MULTILINE)
          commit_message.append('[roll] Roll %s' % filtered_msg)
        elif num_commits > 1:
          commit_count = ' (%d commits)' % num_commits
        update_report.append(
            '%s %s..%s%s' %
            (project.attrib['name'], short_hash(current_hash),
             short_hash(project.attrib['revision']), commit_count))
    finally:
      os.chdir(savedDir)


def update_from_manifest(manifest_path, commit_message):
  """Iterates through each project in a manifest calling update_from_project for each one.

  The top level XML format is:

  <?xml version="1.0" encoding="UTF-8"?>
  <manifest>
    <projects>
      <project ... />
      <project ... />
    </projects>
  </manifest>
  """
  print('Parsing %s' % manifest_path)
  flower_tree = ElementTree.parse(manifest_path)
  root = flower_tree.getroot()
  update_report = []

  print('Checking through projects...')
  for project in root.findall('./projects/project'):
    update_from_project(project, update_report, commit_message)

  return update_report


def update_submodules(integration_repo, manifest_files, make_commit=False):
  """Checks out an integration repository and calls update_from_manifest on each manifest_file.

  If make_commit is true, the resulting submodule change will be added to a
  commit.
  """
  with tempfile.TemporaryDirectory(
      prefix='fuchsia-integration-') as integration_dir:
    print('Checking out integration repo...')
    subprocess.check_call(['git', 'clone', integration_repo, integration_dir])
    print()

    update_report = []
    commit_message = []
    for manifest_file in manifest_files:
      update_report.extend(
          update_from_manifest(
              os.path.join(integration_dir, manifest_file), commit_message))

    commit_msg = ''
    if len(update_report) > 1:
      commit_msg = """[roll] Roll %d submodules

%s""" % (len(update_report), '\n'.join(sorted(update_report)))
    elif len(update_report) == 1:
      if len(commit_message) == 1:
        commit_msg = commit_message[0]
      else:
        commit_msg = "[roll] Roll %s" % update_report[0]

    if commit_msg != '':
      if make_commit:
        subprocess.check_call(['git', 'commit', '-am', commit_msg])
      else:
        print()
        print()
        print(commit_msg)
        print()
        print()


if __name__ == '__main__':
  update_submodules('http://fuchsia.googlesource.com/integration',
                    ['third_party/flower', 'third_party/boringssl/boringssl'])