recipes/contrib/icu_chromium_roller.py - infra/recipes - Git at Google

 # Copyright 2022 The Fuchsia Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 """Recipe that keeps the commit ID of the ICU library in sync with Chromium.

 Roughly, the recipe will check which version of Chromium is used in the Fuchsia
 tree, then figure out the commit ID of the ICU dependency that is currently used
 in that version of Chromium.

 If that commit ID is different from the one used in Fuchsia, this recipe
 auto-rolls to update the commit ID in Fuchsia to that found in use
 in Chromium.
 """

 from PB.go.chromium.org.luci.buildbucket.proto import build as build_pb2

 # See here for the input parameters to this roller. They will become properties
 # in the 'prop' parameter of RunSteps(api, props) below.
 from PB.recipes.fuchsia.contrib.icu_chromium_roller import InputProperties

 import re

 DEPS = [
     "fuchsia/auto_roller",
     "fuchsia/checkout",
     "fuchsia/gitiles",
     "fuchsia/jiri",
     "fuchsia/status_check",
     "recipe_engine/context",
     "recipe_engine/properties",
     "recipe_engine/step",
 ]

 PROPERTIES = InputProperties

 # The URL of the public Chromium repository.
 # Unlikely to change.
 CHROMIUM_REMOTE = "https://chromium.googlesource.com/chromium/src"

 # This is the commit message that the auto-roller will use to commit the
 # changes made by this builder.
 COMMIT_MESSAGE = """\
 [roll] Update {element} from {old} to {new}

 Based on: {chromium_branch}

 **NOTE**: If this change breaks your roll, to fix your roll
 just revert this change.
 """


 def RunSteps(api, props):

     manifest_repo = api.checkout.with_options(
         manifest=props.manifest,
         remote=props.roll_options.remote,
         project=props.project,
         build_input=build_pb2.Build.Input(),
         use_lock_file=True,
     )
     manifest_path = manifest_repo.join(*props.web_engine_manifest_path.split("/"))

     # The version tag you get is of the form 'version:100.1222.33.4'. So
     # we need to parse the Chromium tag out.
     with api.context(cwd=manifest_repo):
         chromium_branch = api.jiri.read_manifest_element(
             manifest=manifest_path,
             element_type="package",
             element_name=props.web_engine_element_name,
         )["version"][len("version:") :]

     with api.step.nest("find ICU commit ID in chromium"):
         deps_contents = api.gitiles.fetch(
             host="chromium.googlesource.com",
             project="chromium/src",
             path="DEPS",
             ref=chromium_branch,
             step_name="get chromium DEPS",
         ).decode("utf-8")
         with api.step.nest("parse commit ID"):
             chromium_icu_commit_id = extract_revision_from_deps(api, deps_contents)
     with api.context(cwd=manifest_repo):
         fuchsia_icu_commit_id = api.jiri.read_manifest_element(
             manifest=props.icu_manifest,
             element_type="project",
             element_name=props.icu_project_name,
         )["revision"]

     api.step.empty("chromium ICU commit ID", step_text=chromium_icu_commit_id)
     api.step.empty("fuchsia ICU commit ID", step_text=fuchsia_icu_commit_id)

     if fuchsia_icu_commit_id == chromium_icu_commit_id:
         api.step.empty("ICU commit IDs are identical: nothing to roll")
         return api.auto_roller.nothing_to_roll()

     # If there is something to roll, update the manifest with the new commit ID,
     # and commit that change.
     with api.context(cwd=manifest_repo):
         api.jiri.edit_manifest(
             manifest=props.icu_manifest,
             projects=[(props.icu_project_name, chromium_icu_commit_id)],
         )

     commit_message = COMMIT_MESSAGE.format(
         element=props.icu_project_name,
         old=fuchsia_icu_commit_id,
         new=chromium_icu_commit_id,
         chromium_branch=chromium_branch,
     )
     change = api.auto_roller.attempt_roll(
         props.roll_options,
         repo_dir=manifest_repo.join(props.project),
         commit_message=commit_message,
     )

     return api.auto_roller.raw_result(change)


 def extract_revision_from_deps(api, contents):
     """Extracts the dependency version from a DEPS file for the ICU library.

     Assumes that the DEPS file has a particular format, see below.

     Args:
       api (RecipeApi): Recipe API object.
       contents (str): String contents of a DEPS file.

     Returns:
       A String containing the revision.
     """

     # Example:
     # 'src/third_party/icu':
     #     Var('chromium_git') + '/chromium/deps/icu.git' + '@' + 'da07448619763d1cde255b361324242646f5b268',
     m = re.search(
         r"/chromium/deps/icu\.git.*\b(?P<revision>[0-9a-f]{40})\b",
         contents,
     )
     if m:
         return m.group("revision")
     raise api.step.InfraFailure("failed to find ICU revision in DEPS")


 def GenTests(api):
     def deps_contents(commit_id):
         """Returns a snippet of chromium/DEPS

         The snippet is not a complete file, but contains just the pattern that
         is likely to match, plus some additional context around it to ensure
         other code does not confuse the regex matcher.

         This is not super-robust, instead we are betting that the format of
         the DEPS file won't change significantly and frequently.
         """

         TEMPLATE = """
 ...
       'src/third_party/hunspell_dictionaries':
         Var('chromium_git') + '/chromium/deps/hunspell_dictionaries.git' + '@' + '41cdffd71c9948f63c7ad36e1fb0ff519aa7a37e',
       'src/third_party/icu':
         Var('chromium_git') + '/chromium/deps/icu.git' + '@' + '{commit_id}',
       'src/third_party/icu4j': {{
           'packages': [
               {{
                   'package': 'chromium/third_party/icu4j',
                   'version': 'e87e5bed2b4935913ee26a3ebd0b723ee2344354',
               }},
           ],
           'condition': 'checkout_android',
           'dep_type': 'cipd',
       }},
 ...
     """
         return TEMPLATE.format(commit_id=commit_id)

     def properties(**kwargs):
         """Sample values of the roller's properties."""
         props = {
             "icu_project_name": "chromium/deps/icu@test",
             "web_engine_manifest_path": "internal/third_party/chromium/nest_sd_canary",
             "project": "integration",
             "manifest": "fuchsia/minimal",
             "web_engine_element_name": "chrome_internal/fuchsia_nest_sd/web_engine/amd64/web_engine",
             "icu_manifest": "fuchsia/third_party/icu",
             "roll_options": {
                 "remote": "sso://turquoise-internal/integration",
             },
         }
         props.update(**kwargs)
         return api.properties(**props)

     def fuchsia_icu_commit_id(commit_id):
         """Returns a part of the JIRI response relevant for our decision-making."""
         return {
             "revision": commit_id,
         }

     def test_setup(**kwargs):
         return api.jiri.read_manifest_element(
             element_name="chrome_internal/fuchsia_nest_sd/web_engine/amd64/web_engine",
             test_output={"version": "version:100.0.0.42"},
         ) + properties(**kwargs)

     def fetch_chromium_with_commit_id(commit_id):
         return api.gitiles.fetch(
             "find ICU commit ID in chromium.get chromium DEPS",
             deps_contents(commit_id),
         )

     def fetch_fuchsia_with_commit_id(commit_id):
         return api.jiri.read_manifest_element(
             element_name="chromium/deps/icu@test",
             test_output=fuchsia_icu_commit_id(commit_id),
         )

     one_commit_id = "a" * 40

     another_commit_id = "b" * 40

     # When commit IDs are the same, the run stops and declares success.
     yield (
         api.status_check.test("same_commit_ids")
         + test_setup()
         + fetch_chromium_with_commit_id(one_commit_id)
         # The same commit ID as above.
         + fetch_fuchsia_with_commit_id(one_commit_id)
     )

     yield (
         api.status_check.test("different_commit_ids")
         + test_setup()
         + fetch_chromium_with_commit_id(another_commit_id)
         # A commit ID that is different from above.
         + fetch_fuchsia_with_commit_id(one_commit_id)
         + api.auto_roller.success()
     )

     yield (
         api.status_check.test("different_commit_ids_dry_run")
         + test_setup()
         + fetch_chromium_with_commit_id(another_commit_id)
         + fetch_fuchsia_with_commit_id(one_commit_id)
         + properties(
             roll_options={"dry_run": True, "remote": "foo"},
         )
         + api.auto_roller.dry_run_success()
     )

     yield (
         api.status_check.test(
             "garbled_chromium_deps",
             status="infra_failure",
         )
         + test_setup()
         + fetch_chromium_with_commit_id("not-well-formed-ID")
     )
	# Copyright 2022 The Fuchsia Authors. All rights reserved.
	# Use of this source code is governed by a BSD-style license that can be
	# found in the LICENSE file.
	"""Recipe that keeps the commit ID of the ICU library in sync with Chromium.

	Roughly, the recipe will check which version of Chromium is used in the Fuchsia
	tree, then figure out the commit ID of the ICU dependency that is currently used
	in that version of Chromium.

	If that commit ID is different from the one used in Fuchsia, this recipe
	auto-rolls to update the commit ID in Fuchsia to that found in use
	in Chromium.
	"""

	from PB.go.chromium.org.luci.buildbucket.proto import build as build_pb2

	# See here for the input parameters to this roller. They will become properties
	# in the 'prop' parameter of RunSteps(api, props) below.
	from PB.recipes.fuchsia.contrib.icu_chromium_roller import InputProperties

	import re

	DEPS = [
	"fuchsia/auto_roller",
	"fuchsia/checkout",
	"fuchsia/gitiles",
	"fuchsia/jiri",
	"fuchsia/status_check",
	"recipe_engine/context",
	"recipe_engine/properties",
	"recipe_engine/step",
	]

	PROPERTIES = InputProperties

	# The URL of the public Chromium repository.
	# Unlikely to change.
	CHROMIUM_REMOTE = "https://chromium.googlesource.com/chromium/src"

	# This is the commit message that the auto-roller will use to commit the
	# changes made by this builder.
	COMMIT_MESSAGE = """\
	[roll] Update {element} from {old} to {new}

	Based on: {chromium_branch}

	NOTE: If this change breaks your roll, to fix your roll
	just revert this change.
	"""


	def RunSteps(api, props):

	manifest_repo = api.checkout.with_options(
	manifest=props.manifest,
	remote=props.roll_options.remote,
	project=props.project,
	build_input=build_pb2.Build.Input(),
	use_lock_file=True,
	)
	manifest_path = manifest_repo.join(*props.web_engine_manifest_path.split("/"))

	# The version tag you get is of the form 'version:100.1222.33.4'. So
	# we need to parse the Chromium tag out.
	with api.context(cwd=manifest_repo):
	chromium_branch = api.jiri.read_manifest_element(
	manifest=manifest_path,
	element_type="package",
	element_name=props.web_engine_element_name,
	)["version"][len("version:") :]

	with api.step.nest("find ICU commit ID in chromium"):
	deps_contents = api.gitiles.fetch(
	host="chromium.googlesource.com",
	project="chromium/src",
	path="DEPS",
	ref=chromium_branch,
	step_name="get chromium DEPS",
	).decode("utf-8")
	with api.step.nest("parse commit ID"):
	chromium_icu_commit_id = extract_revision_from_deps(api, deps_contents)
	with api.context(cwd=manifest_repo):
	fuchsia_icu_commit_id = api.jiri.read_manifest_element(
	manifest=props.icu_manifest,
	element_type="project",
	element_name=props.icu_project_name,
	)["revision"]

	api.step.empty("chromium ICU commit ID", step_text=chromium_icu_commit_id)
	api.step.empty("fuchsia ICU commit ID", step_text=fuchsia_icu_commit_id)

	if fuchsia_icu_commit_id == chromium_icu_commit_id:
	api.step.empty("ICU commit IDs are identical: nothing to roll")
	return api.auto_roller.nothing_to_roll()

	# If there is something to roll, update the manifest with the new commit ID,
	# and commit that change.
	with api.context(cwd=manifest_repo):
	api.jiri.edit_manifest(
	manifest=props.icu_manifest,
	projects=[(props.icu_project_name, chromium_icu_commit_id)],
	)

	commit_message = COMMIT_MESSAGE.format(
	element=props.icu_project_name,
	old=fuchsia_icu_commit_id,
	new=chromium_icu_commit_id,
	chromium_branch=chromium_branch,
	)
	change = api.auto_roller.attempt_roll(
	props.roll_options,
	repo_dir=manifest_repo.join(props.project),
	commit_message=commit_message,
	)

	return api.auto_roller.raw_result(change)


	def extract_revision_from_deps(api, contents):
	"""Extracts the dependency version from a DEPS file for the ICU library.

	Assumes that the DEPS file has a particular format, see below.

	Args:
	api (RecipeApi): Recipe API object.
	contents (str): String contents of a DEPS file.

	Returns:
	A String containing the revision.
	"""

	# Example:
	# 'src/third_party/icu':
	# Var('chromium_git') + '/chromium/deps/icu.git' + '@' + 'da07448619763d1cde255b361324242646f5b268',
	m = re.search(
	r"/chromium/deps/icu\.git.*\b(?P<revision>[0-9a-f]{40})\b",
	contents,
	)
	if m:
	return m.group("revision")
	raise api.step.InfraFailure("failed to find ICU revision in DEPS")


	def GenTests(api):
	def deps_contents(commit_id):
	"""Returns a snippet of chromium/DEPS

	The snippet is not a complete file, but contains just the pattern that
	is likely to match, plus some additional context around it to ensure
	other code does not confuse the regex matcher.

	This is not super-robust, instead we are betting that the format of
	the DEPS file won't change significantly and frequently.
	"""

	TEMPLATE = """
	...
	'src/third_party/hunspell_dictionaries':
	Var('chromium_git') + '/chromium/deps/hunspell_dictionaries.git' + '@' + '41cdffd71c9948f63c7ad36e1fb0ff519aa7a37e',
	'src/third_party/icu':
	Var('chromium_git') + '/chromium/deps/icu.git' + '@' + '{commit_id}',
	'src/third_party/icu4j': {{
	'packages': [
	{{
	'package': 'chromium/third_party/icu4j',
	'version': 'e87e5bed2b4935913ee26a3ebd0b723ee2344354',
	}},
	],
	'condition': 'checkout_android',
	'dep_type': 'cipd',
	}},
	...
	"""
	return TEMPLATE.format(commit_id=commit_id)

	def properties(**kwargs):
	"""Sample values of the roller's properties."""
	props = {
	"icu_project_name": "chromium/deps/icu@test",
	"web_engine_manifest_path": "internal/third_party/chromium/nest_sd_canary",
	"project": "integration",
	"manifest": "fuchsia/minimal",
	"web_engine_element_name": "chrome_internal/fuchsia_nest_sd/web_engine/amd64/web_engine",
	"icu_manifest": "fuchsia/third_party/icu",
	"roll_options": {
	"remote": "sso://turquoise-internal/integration",
	},
	}
	props.update(**kwargs)
	return api.properties(**props)

	def fuchsia_icu_commit_id(commit_id):
	"""Returns a part of the JIRI response relevant for our decision-making."""
	return {
	"revision": commit_id,
	}

	def test_setup(**kwargs):
	return api.jiri.read_manifest_element(
	element_name="chrome_internal/fuchsia_nest_sd/web_engine/amd64/web_engine",
	test_output={"version": "version:100.0.0.42"},
	) + properties(**kwargs)

	def fetch_chromium_with_commit_id(commit_id):
	return api.gitiles.fetch(
	"find ICU commit ID in chromium.get chromium DEPS",
	deps_contents(commit_id),
	)

	def fetch_fuchsia_with_commit_id(commit_id):
	return api.jiri.read_manifest_element(
	element_name="chromium/deps/icu@test",
	test_output=fuchsia_icu_commit_id(commit_id),
	)

	one_commit_id = "a" * 40

	another_commit_id = "b" * 40

	# When commit IDs are the same, the run stops and declares success.
	yield (
	api.status_check.test("same_commit_ids")
	+ test_setup()
	+ fetch_chromium_with_commit_id(one_commit_id)
	# The same commit ID as above.
	+ fetch_fuchsia_with_commit_id(one_commit_id)
	)

	yield (
	api.status_check.test("different_commit_ids")
	+ test_setup()
	+ fetch_chromium_with_commit_id(another_commit_id)
	# A commit ID that is different from above.
	+ fetch_fuchsia_with_commit_id(one_commit_id)
	+ api.auto_roller.success()
	)

	yield (
	api.status_check.test("different_commit_ids_dry_run")
	+ test_setup()
	+ fetch_chromium_with_commit_id(another_commit_id)
	+ fetch_fuchsia_with_commit_id(one_commit_id)
	+ properties(
	roll_options={"dry_run": True, "remote": "foo"},
	)
	+ api.auto_roller.dry_run_success()
	)

	yield (
	api.status_check.test(
	"garbled_chromium_deps",
	status="infra_failure",
	)
	+ test_setup()
	+ fetch_chromium_with_commit_id("not-well-formed-ID")
	)