recipes/third_party_repo_roller.py - infra/recipes - Git at Google

 # Copyright 2021 The Fuchsia Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 """Recipe for updating a third party repo to the greatest released version.

 Searches the `import_from` repo for tags of the form "v${SEMANTIC_VERSION}", where
 SEMANTIC_VERSION looks like ${MAJOR}.${MINOR}.${PATCH}. See https://semver.org/.

 Then updates the jiri manifest `import_in` in repo `project` with the revision of
 the greatest semantic version.

 Then updates the main branch of `import_from` to that same revision.

 The dry_run property allows testing the auto-roller end-to-end. This is
 useful because now we can have an auto-roller in staging, and we can block
 updates behind 'dry_run' as a sort of feature gate. It is passed to
 api.auto_roller.attempt_roll() which handles committing changes.
 """

 from PB.recipes.fuchsia.third_party_repo_roller import InputProperties

 DEPS = [
     "fuchsia/auto_roller",
     "fuchsia/git",
     "fuchsia/gitiles",
     "fuchsia/jiri",
     "fuchsia/sso",
     "fuchsia/status_check",
     "recipe_engine/context",
     "recipe_engine/path",
     "recipe_engine/properties",
     "recipe_engine/raw_io",
     "recipe_engine/step",
 ]

 PROPERTIES = InputProperties

 COMMIT_MESSAGE = """[roll] Roll {header}
 {body}
 """

 COMMIT_HEADER = """{project} {old}..{new} ({count} commits)"""

 # These footers must be set for the roll_comment recipe to be able to comment
 # on the rolled CLs.
 COMMENT_FOOTERS = """Rolled-Repo: {repo}
 Rolled-Commits: {old}..{new}"""

 # If we're embedding the original commit message, prepend 'Original-' to lines
 # which begin with these tags.
 ESCAPE_TAGS = [
     "Reviewed-on:",
     "Bug:",
     "Fixed:",
 ]
 # If we're embedding the original commit message, remove lines which contain
 # these tags.
 FILTER_TAGS = [
     "Change-Id:",
     "Reviewed-by:",
     "Signed-off-by:",
     "Acked-by:",
     "CC:",
     "Tested-by:",
     "Commit-Queue:",
     "Testability-Review:",
     "API-Review:",
     "Auto-Submit:",
     "Fuchsia-Auto-Submit:",
 ]

 TEST_DATA_OLD_REV = "123abc"


 def RunSteps(api, props):
     with api.context(infra_steps=True):
         if props.owners:
             api.step.empty("owners", step_text=", ".join(props.owners))

         with api.step.nest("checkout"):
             api.jiri.init()
             api.jiri.import_manifest(props.manifest, props.remote, name=props.project)
             api.jiri.update(run_hooks=False, fetch_packages=False)

         manifest_project, import_from_project = api.jiri.project(
             projects=(props.project, props.import_from),
             test_data=(
                 {
                     "name": props.project,
                     "path": str(api.path["start_dir"].join(props.project)),
                     "remote": props.remote,
                 },
                 {
                     "name": props.import_from,
                     "path": str(api.path["start_dir"].join(props.import_from)),
                     "remote": "https://fuchsia.googlesource.com/%s" % props.import_from,
                     "revision": TEST_DATA_OLD_REV,
                 },
             ),
         ).json.output

         project_dir = api.path.abs_to_path(manifest_project["path"])
         with api.step.nest("edit manifest"), api.context(cwd=project_dir):
             roll_from_repo = import_from_project["remote"]
             roll_from_repo = api.sso.sso_to_https(roll_from_repo)

             old_rev = import_from_project["revision"]
             new_rev = resolve_new_revision(api, roll_from_repo)

             if old_rev == new_rev:
                 api.step.empty("nothing to roll", step_text="manifest up-to-date")
                 return api.auto_roller.nothing_to_roll()

             log = api.gitiles.log(
                 roll_from_repo, "%s..%s" % (old_rev, new_rev), step_name="log"
             )
             if not log:
                 api.step.empty(
                     "detected backwards roll",
                     step_text="expected %s to precede %s in git history"
                     % (old_rev[:7], new_rev[:7]),
                     status=api.step.INFRA_FAILURE,
                 )

             api.jiri.edit_manifest(
                 props.import_in, projects=[(props.import_from, new_rev)]
             )

         commit_message = write_commit_message(log, props.import_from, old_rev, new_rev)

         # Land the changes.
         change = api.auto_roller.attempt_roll(
             api.auto_roller.Options(
                 remote=props.remote,
                 dry_run=props.dry_run,
                 roller_owners=props.owners,
             ),
             repo_dir=project_dir,
             commit_message=commit_message,
             raise_on_failure=False,
         )

         roll_result = api.auto_roller.raw_result(change)

         # For clarity we update the main branch the same revision
         # that is pinned in the jiri manifest.
         import_from_dir = api.path.abs_to_path(import_from_project["path"])
         with api.context(cwd=import_from_dir):
             api.git.push(
                 "%s:main" % new_rev,
                 step_name="fast forward main",
                 dryrun=props.dry_run,
             )

         return roll_result


 def write_commit_message(
     log,
     import_from,
     old_rev,
     new_rev,
 ):
     commit_lines = []
     for commit in log:
         commit_label = commit["id"][:7]
         commit_lines.append(
             "{commit} {subject}".format(
                 commit=commit_label,
                 subject=commit["message"].splitlines()[0],
             )
         )

     header = COMMIT_HEADER.format(
         project=import_from,
         old=old_rev[:7],
         new=new_rev[:7],
         count=len(log),
     )
     body = "\n" + "\n".join(commit_lines)

     return COMMIT_MESSAGE.format(header=header, body=body)


 def to_semantic_version_tuple(git_ref):
     """Returns a tuple of ints: (major, minor, patch) or None."""
     ref_prefix = "refs/tags/v"
     if not git_ref.startswith(ref_prefix):
         return None
     sem_ver_str = git_ref[len(ref_prefix) :]
     sem_ver_parts = sem_ver_str.split(".")
     if len(sem_ver_parts) != 3:
         return None
     try:
         return tuple(int(x) for x in sem_ver_parts)
     except ValueError:
         return None


 def resolve_new_revision(api, repo):
     with api.step.nest("resolve new revision") as presentation:
         ls_output = api.git(
             "ls-remote",
             "ls-remote",
             repo,
             "--tags",
             "refs/tags/v*",
             stdout=api.raw_io.output_text(),
         ).stdout
         # Each line looks like:
         # <commit ID> <refs/tags/...>
         refs_commits = [
             tuple(reversed(commit_ref.split()))
             for commit_ref in ls_output.split("\n")
             if commit_ref
         ]
         semantic_versions_commits = sorted(
             (to_semantic_version_tuple(ref), commit)
             for ref, commit in refs_commits
             if to_semantic_version_tuple(ref)
         )
         assert semantic_versions_commits, "failed to resolve commit to roll"
         greatest_version, greatest_commit = semantic_versions_commits[-1]
         presentation.logs["greatest semantic version"] = "v%d.%d.%d" % greatest_version
         return greatest_commit


 def GenTests(api):
     def properties(**kwargs):
         props = dict(
             project="integration",
             manifest="minimal",
             remote="https://fuchsia.googlesource.com/integration",
             import_in="stem",
             import_from="fuchsia",
             owners=["nobody@google.com", "noreply@google.com"],
         )
         props.update(**kwargs)
         return api.properties(**props)

     def ls_remote(greatest_release_revision="123abd"):
         return api.step_data(
             "edit manifest.resolve new revision.ls-remote",
             api.raw_io.stream_output_text(
                 "\n".join(
                     [
                         "a21bf2e6466095c7a2cdb991017da9639cf496e5        refs/tags/v2.6.0",
                         "%s        refs/tags/v2.10.1" % greatest_release_revision,
                         "419c94b6e07f93610e5124a0be0a7ee7140006ed        refs/tags/v2.6.1rc1",
                         "e8ae137c96444ea313485ed1118c5e43b2099cf1        foo/bar/baz",
                         "e8ae137c96444ea313485ed1118c5e43b2099cf2        refs/tags/v1.2.3.4",
                     ]
                 )
             ),
         )

     yield (
         api.status_check.test("successful")
         + properties()
         + ls_remote()
         + api.gitiles.log("edit manifest.log", "A", n=2, add_change_id=True)
         + api.auto_roller.success()
     )

     # If the repo is already at the greatest release, the recipe should not attempt
     # a roll.
     yield (api.status_check.test("noop") + properties() + ls_remote(TEST_DATA_OLD_REV))

     # If the currently pinned revision and the main revision are not equal,
     # but the gitiles log between them is empty, that probably means that the
     # currently pinned revision is actually *ahead* of main, according to
     # `gitiles log`. This sometimes happens, possibly due to GoB
     # synchronization issues, and we should fail instead of rolling backwards.
     yield (
         api.status_check.test("backwards_roll", status="infra_failure")
         + properties()
         + ls_remote()
         + api.gitiles.log("edit manifest.log", "A", n=0)
     )
	# Copyright 2021 The Fuchsia Authors. All rights reserved.
	# Use of this source code is governed by a BSD-style license that can be
	# found in the LICENSE file.
	"""Recipe for updating a third party repo to the greatest released version.

	Searches the `import_from` repo for tags of the form "v${SEMANTIC_VERSION}", where
	SEMANTIC_VERSION looks like ${MAJOR}.${MINOR}.${PATCH}. See https://semver.org/.

	Then updates the jiri manifest `import_in` in repo `project` with the revision of
	the greatest semantic version.

	Then updates the main branch of `import_from` to that same revision.

	The dry_run property allows testing the auto-roller end-to-end. This is
	useful because now we can have an auto-roller in staging, and we can block
	updates behind 'dry_run' as a sort of feature gate. It is passed to
	api.auto_roller.attempt_roll() which handles committing changes.
	"""

	from PB.recipes.fuchsia.third_party_repo_roller import InputProperties

	DEPS = [
	"fuchsia/auto_roller",
	"fuchsia/git",
	"fuchsia/gitiles",
	"fuchsia/jiri",
	"fuchsia/sso",
	"fuchsia/status_check",
	"recipe_engine/context",
	"recipe_engine/path",
	"recipe_engine/properties",
	"recipe_engine/raw_io",
	"recipe_engine/step",
	]

	PROPERTIES = InputProperties

	COMMIT_MESSAGE = """[roll] Roll {header}
	{body}
	"""

	COMMIT_HEADER = """{project} {old}..{new} ({count} commits)"""

	# These footers must be set for the roll_comment recipe to be able to comment
	# on the rolled CLs.
	COMMENT_FOOTERS = """Rolled-Repo: {repo}
	Rolled-Commits: {old}..{new}"""

	# If we're embedding the original commit message, prepend 'Original-' to lines
	# which begin with these tags.
	ESCAPE_TAGS = [
	"Reviewed-on:",
	"Bug:",
	"Fixed:",
	]
	# If we're embedding the original commit message, remove lines which contain
	# these tags.
	FILTER_TAGS = [
	"Change-Id:",
	"Reviewed-by:",
	"Signed-off-by:",
	"Acked-by:",
	"CC:",
	"Tested-by:",
	"Commit-Queue:",
	"Testability-Review:",
	"API-Review:",
	"Auto-Submit:",
	"Fuchsia-Auto-Submit:",
	]

	TEST_DATA_OLD_REV = "123abc"


	def RunSteps(api, props):
	with api.context(infra_steps=True):
	if props.owners:
	api.step.empty("owners", step_text=", ".join(props.owners))

	with api.step.nest("checkout"):
	api.jiri.init()
	api.jiri.import_manifest(props.manifest, props.remote, name=props.project)
	api.jiri.update(run_hooks=False, fetch_packages=False)

	manifest_project, import_from_project = api.jiri.project(
	projects=(props.project, props.import_from),
	test_data=(
	{
	"name": props.project,
	"path": str(api.path["start_dir"].join(props.project)),
	"remote": props.remote,
	},
	{
	"name": props.import_from,
	"path": str(api.path["start_dir"].join(props.import_from)),
	"remote": "https://fuchsia.googlesource.com/%s" % props.import_from,
	"revision": TEST_DATA_OLD_REV,
	},
	),
	).json.output

	project_dir = api.path.abs_to_path(manifest_project["path"])
	with api.step.nest("edit manifest"), api.context(cwd=project_dir):
	roll_from_repo = import_from_project["remote"]
	roll_from_repo = api.sso.sso_to_https(roll_from_repo)

	old_rev = import_from_project["revision"]
	new_rev = resolve_new_revision(api, roll_from_repo)

	if old_rev == new_rev:
	api.step.empty("nothing to roll", step_text="manifest up-to-date")
	return api.auto_roller.nothing_to_roll()

	log = api.gitiles.log(
	roll_from_repo, "%s..%s" % (old_rev, new_rev), step_name="log"
	)
	if not log:
	api.step.empty(
	"detected backwards roll",
	step_text="expected %s to precede %s in git history"
	% (old_rev[:7], new_rev[:7]),
	status=api.step.INFRA_FAILURE,
	)

	api.jiri.edit_manifest(
	props.import_in, projects=[(props.import_from, new_rev)]
	)

	commit_message = write_commit_message(log, props.import_from, old_rev, new_rev)

	# Land the changes.
	change = api.auto_roller.attempt_roll(
	api.auto_roller.Options(
	remote=props.remote,
	dry_run=props.dry_run,
	roller_owners=props.owners,
	),
	repo_dir=project_dir,
	commit_message=commit_message,
	raise_on_failure=False,
	)

	roll_result = api.auto_roller.raw_result(change)

	# For clarity we update the main branch the same revision
	# that is pinned in the jiri manifest.
	import_from_dir = api.path.abs_to_path(import_from_project["path"])
	with api.context(cwd=import_from_dir):
	api.git.push(
	"%s:main" % new_rev,
	step_name="fast forward main",
	dryrun=props.dry_run,
	)

	return roll_result


	def write_commit_message(
	log,
	import_from,
	old_rev,
	new_rev,
	):
	commit_lines = []
	for commit in log:
	commit_label = commit["id"][:7]
	commit_lines.append(
	"{commit} {subject}".format(
	commit=commit_label,
	subject=commit["message"].splitlines()[0],
	)
	)

	header = COMMIT_HEADER.format(
	project=import_from,
	old=old_rev[:7],
	new=new_rev[:7],
	count=len(log),
	)
	body = "\n" + "\n".join(commit_lines)

	return COMMIT_MESSAGE.format(header=header, body=body)


	def to_semantic_version_tuple(git_ref):
	"""Returns a tuple of ints: (major, minor, patch) or None."""
	ref_prefix = "refs/tags/v"
	if not git_ref.startswith(ref_prefix):
	return None
	sem_ver_str = git_ref[len(ref_prefix) :]
	sem_ver_parts = sem_ver_str.split(".")
	if len(sem_ver_parts) != 3:
	return None
	try:
	return tuple(int(x) for x in sem_ver_parts)
	except ValueError:
	return None


	def resolve_new_revision(api, repo):
	with api.step.nest("resolve new revision") as presentation:
	ls_output = api.git(
	"ls-remote",
	"ls-remote",
	repo,
	"--tags",
	"refs/tags/v*",
	stdout=api.raw_io.output_text(),
	).stdout
	# Each line looks like:
	# <commit ID> <refs/tags/...>
	refs_commits = [
	tuple(reversed(commit_ref.split()))
	for commit_ref in ls_output.split("\n")
	if commit_ref
	]
	semantic_versions_commits = sorted(
	(to_semantic_version_tuple(ref), commit)
	for ref, commit in refs_commits
	if to_semantic_version_tuple(ref)
	)
	assert semantic_versions_commits, "failed to resolve commit to roll"
	greatest_version, greatest_commit = semantic_versions_commits[-1]
	presentation.logs["greatest semantic version"] = "v%d.%d.%d" % greatest_version
	return greatest_commit


	def GenTests(api):
	def properties(**kwargs):
	props = dict(
	project="integration",
	manifest="minimal",
	remote="https://fuchsia.googlesource.com/integration",
	import_in="stem",
	import_from="fuchsia",
	owners=["nobody@google.com", "noreply@google.com"],
	)
	props.update(**kwargs)
	return api.properties(**props)

	def ls_remote(greatest_release_revision="123abd"):
	return api.step_data(
	"edit manifest.resolve new revision.ls-remote",
	api.raw_io.stream_output_text(
	"\n".join(
	[
	"a21bf2e6466095c7a2cdb991017da9639cf496e5 refs/tags/v2.6.0",
	"%s refs/tags/v2.10.1" % greatest_release_revision,
	"419c94b6e07f93610e5124a0be0a7ee7140006ed refs/tags/v2.6.1rc1",
	"e8ae137c96444ea313485ed1118c5e43b2099cf1 foo/bar/baz",
	"e8ae137c96444ea313485ed1118c5e43b2099cf2 refs/tags/v1.2.3.4",
	]
	)
	),
	)

	yield (
	api.status_check.test("successful")
	+ properties()
	+ ls_remote()
	+ api.gitiles.log("edit manifest.log", "A", n=2, add_change_id=True)
	+ api.auto_roller.success()
	)

	# If the repo is already at the greatest release, the recipe should not attempt
	# a roll.
	yield (api.status_check.test("noop") + properties() + ls_remote(TEST_DATA_OLD_REV))

	# If the currently pinned revision and the main revision are not equal,
	# but the gitiles log between them is empty, that probably means that the
	# currently pinned revision is actually ahead of main, according to
	# `gitiles log`. This sometimes happens, possibly due to GoB
	# synchronization issues, and we should fail instead of rolling backwards.
	yield (
	api.status_check.test("backwards_roll", status="infra_failure")
	+ properties()
	+ ls_remote()
	+ api.gitiles.log("edit manifest.log", "A", n=0)
	)