[roller] Create a dart module to simplify flutter roller.
Currently flutter runner is doing a lot of work, we are moving dart
functionality to its own module to simplify the flutter roller.
Bug: 36488
Bug: 36497
Change-Id: If8357c3a62da36c431f6fa96e404064590a865ad
diff --git a/recipe_modules/dart_util/__init__.py b/recipe_modules/dart_util/__init__.py
new file mode 100644
index 0000000..1191469
--- /dev/null
+++ b/recipe_modules/dart_util/__init__.py
@@ -0,0 +1,18 @@
+# Copyright 2020 The Fuchsia Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+DEPS = [
+ 'fuchsia/auto_roller',
+ 'fuchsia/buildbucket_util',
+ 'fuchsia/gerrit',
+ 'fuchsia/git',
+ 'fuchsia/jiri',
+ 'recipe_engine/buildbucket',
+ 'recipe_engine/context',
+ 'recipe_engine/file',
+ 'recipe_engine/path',
+ 'recipe_engine/python',
+ 'recipe_engine/raw_io',
+ 'recipe_engine/step',
+]
diff --git a/recipe_modules/dart_util/api.py b/recipe_modules/dart_util/api.py
new file mode 100644
index 0000000..e623563
--- /dev/null
+++ b/recipe_modules/dart_util/api.py
@@ -0,0 +1,97 @@
+# Copyright 2020 The Fuchsia Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import collections
+import os
+
+from recipe_engine import recipe_api
+
+# Used to get the remote for dart sdk
+DART_SDK_PROJECT = 'dart/sdk'
+
+COMMIT_MESSAGE = """\
+[roll] Update {deps}
+
+{logs}
+
+Cq-Cl-Tag: roller-builder:{builder}
+Cq-Cl-Tag: roller-bid:{build_id}
+CQ-Do-Not-Cancel-Tryjobs: true"""
+
+
+class DartApi(recipe_api.RecipeApi):
+ """APIs to work with Dart repo and Dart packages."""
+
+ def run_update_3p_packages(self, checkout_root):
+ """Runs update_3p_packages.py in dart 3p repo checkout.
+
+ Args:
+ checkout_root (Path): The root path to the project checkouts.
+ """
+ self.m.python(
+ 'update dart 3p packages',
+ checkout_root.join('scripts', 'dart', 'update_3p_packages.py'),
+ args=['--debug'],
+ )
+
+ def update_3p_packages(self, checkout_root, dry_run):
+ """Updates fuchsia's third-party dart packages.
+
+ Args:
+ checkout_root (Path): Root path to checkouts.
+ dry_run (bool): Whether the roll will be allowed to complete or not.
+
+ Returns:
+ A string with third_party/dart-pkg/pub latest commit hash.
+ """
+ commit_msg = '[roll] Update third-party dart packages'
+ packages_root = checkout_root.join('third_party', 'dart-pkg', 'pub')
+ with self.m.context(cwd=packages_root):
+ # Make sure third_party/dart-pkg is at origin/master before running the
+ # update script to catch any manual commits that extend past the revision at
+ # integration's HEAD.
+ self.m.git('fetch', 'origin')
+ self.m.git('checkout', 'origin/master')
+ current_hash = self.m.git.get_hash()
+ self.run_update_3p_packages(checkout_root)
+ rolled = self.m.auto_roller.attempt_roll(
+ gerrit_project='third_party/dart-pkg/pub',
+ repo_dir=packages_root,
+ commit_message=commit_msg,
+ dry_run=dry_run)
+ if rolled:
+ current_hash = self.m.git.get_hash()
+ self.m.step.active_result.presentation.logs['revision'] = [current_hash]
+ return current_hash
+
+ def update_pkg_manifest(self, path, checkout_root):
+ """Overwrites a dart third party package manifest.
+
+ Args:
+ path (Path): A path to the dart/sdk repository.
+ checkout_root (Path): A path to where the integration project
+ was checked out.
+ """
+ manifest_path = checkout_root.join('fuchsia', 'topaz',
+ 'dart_third_party_pkg')
+ self.m.python(
+ name='update %s' % self.m.path.basename(manifest_path),
+ script=path.join('tools', 'create_pkg_manifest.py'),
+ args=['-d', path.join('DEPS'), '-o', manifest_path],
+ )
+
+ def checkout(self, path, revision=None):
+ """Get dart/sdk.
+
+ Args:
+ path (Path): Location where dart source code will be checked out.
+ revision (str): The revision of the source code to check out.
+ """
+ deps_info = self.m.jiri.project([DART_SDK_PROJECT])
+ remote = deps_info.json.output[0]['remote']
+ self.m.git.checkout(
+ url=remote,
+ path=path,
+ ref=revision,
+ )
diff --git a/recipe_modules/dart_util/examples/full.expected/default.json b/recipe_modules/dart_util/examples/full.expected/default.json
new file mode 100644
index 0000000..7664ff0
--- /dev/null
+++ b/recipe_modules/dart_util/examples/full.expected/default.json
@@ -0,0 +1,505 @@
+[
+ {
+ "cmd": [],
+ "name": "ensure jiri"
+ },
+ {
+ "cmd": [
+ "cipd",
+ "ensure",
+ "-root",
+ "[START_DIR]/cipd/jiri",
+ "-ensure-file",
+ "fuchsia/tools/jiri/${platform} git_revision:4bbab8725bd3c64b56e70af3d973d526cd894b49",
+ "-json-output",
+ "/path/to/tmp/json"
+ ],
+ "infra_step": true,
+ "name": "ensure jiri.ensure_installed",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@",
+ "@@@STEP_LOG_LINE@json.output@{@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"result\": {@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"\": [@@@",
+ "@@@STEP_LOG_LINE@json.output@ {@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"instance_id\": \"resolved-instance_id-of-git_revision:4bb\", @@@",
+ "@@@STEP_LOG_LINE@json.output@ \"package\": \"fuchsia/tools/jiri/resolved-platform\"@@@",
+ "@@@STEP_LOG_LINE@json.output@ }@@@",
+ "@@@STEP_LOG_LINE@json.output@ ]@@@",
+ "@@@STEP_LOG_LINE@json.output@ }@@@",
+ "@@@STEP_LOG_LINE@json.output@}@@@",
+ "@@@STEP_LOG_END@json.output@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "[START_DIR]/cipd/jiri/jiri",
+ "project",
+ "-vv",
+ "-time",
+ "-j=50",
+ "-json-output",
+ "/path/to/tmp/json",
+ "dart/sdk"
+ ],
+ "name": "jiri project",
+ "~followup_annotations": [
+ "@@@STEP_LOG_LINE@json.output@[@@@",
+ "@@@STEP_LOG_LINE@json.output@ {@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"branches\": [@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"(HEAD detached at c22471f)\"@@@",
+ "@@@STEP_LOG_LINE@json.output@ ], @@@",
+ "@@@STEP_LOG_LINE@json.output@ \"current_branch\": \"\", @@@",
+ "@@@STEP_LOG_LINE@json.output@ \"name\": \"dart/sdk\", @@@",
+ "@@@STEP_LOG_LINE@json.output@ \"path\": \"path/to/dart/sdk\", @@@",
+ "@@@STEP_LOG_LINE@json.output@ \"remote\": \"https://fuchsia.googlesource.com/dart/sdk\", @@@",
+ "@@@STEP_LOG_LINE@json.output@ \"revision\": \"c22471f4e3f842ae18dd9adec82ed9eb78ed1127\"@@@",
+ "@@@STEP_LOG_LINE@json.output@ }@@@",
+ "@@@STEP_LOG_LINE@json.output@]@@@",
+ "@@@STEP_LOG_END@json.output@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "vpython",
+ "-u",
+ "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+ "--json-output",
+ "/path/to/tmp/json",
+ "ensure-directory",
+ "--mode",
+ "0777",
+ "[START_DIR]/dart"
+ ],
+ "infra_step": true,
+ "name": "makedirs"
+ },
+ {
+ "cmd": [
+ "git",
+ "init"
+ ],
+ "cwd": "[START_DIR]/dart",
+ "name": "git init"
+ },
+ {
+ "cmd": [
+ "git",
+ "remote",
+ "add",
+ "origin",
+ "https://fuchsia.googlesource.com/dart/sdk"
+ ],
+ "cwd": "[START_DIR]/dart",
+ "name": "git remote"
+ },
+ {
+ "cmd": [],
+ "name": "cache"
+ },
+ {
+ "cmd": [
+ "vpython",
+ "-u",
+ "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+ "--json-output",
+ "/path/to/tmp/json",
+ "ensure-directory",
+ "--mode",
+ "0777",
+ "[CACHE]/git/fuchsia.googlesource.com-dart-sdk"
+ ],
+ "cwd": "[START_DIR]/dart",
+ "infra_step": true,
+ "name": "cache.makedirs",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "git",
+ "init",
+ "--bare"
+ ],
+ "cwd": "[CACHE]/git/fuchsia.googlesource.com-dart-sdk",
+ "name": "cache.git init",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "git",
+ "config",
+ "remote.origin.url",
+ "https://fuchsia.googlesource.com/dart/sdk"
+ ],
+ "cwd": "[CACHE]/git/fuchsia.googlesource.com-dart-sdk",
+ "name": "cache.remote set-url",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "git",
+ "config",
+ "--replace-all",
+ "remote.origin.fetch",
+ "+refs/heads/*:refs/heads/*",
+ "\\+refs/heads/\\*:.*"
+ ],
+ "cwd": "[CACHE]/git/fuchsia.googlesource.com-dart-sdk",
+ "name": "cache.git config",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "git",
+ "fetch",
+ "--prune",
+ "--tags",
+ "origin"
+ ],
+ "cwd": "[CACHE]/git/fuchsia.googlesource.com-dart-sdk",
+ "name": "cache.git fetch",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "vpython",
+ "-u",
+ "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+ "--json-output",
+ "/path/to/tmp/json",
+ "ensure-directory",
+ "--mode",
+ "0777",
+ "[START_DIR]/dart/.git/objects/info"
+ ],
+ "cwd": "[START_DIR]/dart",
+ "infra_step": true,
+ "name": "cache.makedirs object/info",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "vpython",
+ "-u",
+ "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+ "--json-output",
+ "/path/to/tmp/json",
+ "copy",
+ "[CACHE]/git/fuchsia.googlesource.com-dart-sdk/objects\n",
+ "[START_DIR]/dart/.git/objects/info/alternates"
+ ],
+ "cwd": "[START_DIR]/dart",
+ "infra_step": true,
+ "name": "cache.alternates",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@",
+ "@@@STEP_LOG_LINE@alternates@[CACHE]/git/fuchsia.googlesource.com-dart-sdk/objects@@@",
+ "@@@STEP_LOG_END@alternates@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "git",
+ "fetch",
+ "--tags",
+ "origin",
+ "abc_revision"
+ ],
+ "cwd": "[START_DIR]/dart",
+ "name": "git fetch"
+ },
+ {
+ "cmd": [
+ "git",
+ "checkout",
+ "-f",
+ "FETCH_HEAD"
+ ],
+ "cwd": "[START_DIR]/dart",
+ "name": "git checkout"
+ },
+ {
+ "cmd": [
+ "git",
+ "rev-parse",
+ "HEAD"
+ ],
+ "cwd": "[START_DIR]/dart",
+ "name": "git rev-parse"
+ },
+ {
+ "cmd": [
+ "git",
+ "clean",
+ "-f",
+ "-d",
+ "-x"
+ ],
+ "cwd": "[START_DIR]/dart",
+ "name": "git clean"
+ },
+ {
+ "cmd": [],
+ "name": "submodule"
+ },
+ {
+ "cmd": [
+ "git",
+ "submodule",
+ "sync"
+ ],
+ "cwd": "[START_DIR]/dart",
+ "name": "submodule.git submodule sync",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "git",
+ "submodule",
+ "update",
+ "--init"
+ ],
+ "cwd": "[START_DIR]/dart",
+ "name": "submodule.git submodule update",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "python",
+ "-u",
+ "[START_DIR]/dart/tools/create_pkg_manifest.py",
+ "-d",
+ "[START_DIR]/dart/DEPS",
+ "-o",
+ "[START_DIR]/integration/fuchsia/topaz/dart_third_party_pkg"
+ ],
+ "name": "update dart_third_party_pkg"
+ },
+ {
+ "cmd": [
+ "git",
+ "fetch",
+ "origin"
+ ],
+ "cwd": "[START_DIR]/third_party/dart-pkg/pub",
+ "name": "git fetch (2)"
+ },
+ {
+ "cmd": [
+ "git",
+ "checkout",
+ "origin/master"
+ ],
+ "cwd": "[START_DIR]/third_party/dart-pkg/pub",
+ "name": "git checkout (2)"
+ },
+ {
+ "cmd": [
+ "git",
+ "rev-parse",
+ "HEAD"
+ ],
+ "cwd": "[START_DIR]/third_party/dart-pkg/pub",
+ "name": "git rev-parse (2)"
+ },
+ {
+ "cmd": [
+ "python",
+ "-u",
+ "[START_DIR]/scripts/dart/update_3p_packages.py",
+ "--debug"
+ ],
+ "cwd": "[START_DIR]/third_party/dart-pkg/pub",
+ "name": "update dart 3p packages"
+ },
+ {
+ "cmd": [
+ "git",
+ "ls-files",
+ "--modified",
+ "--deleted",
+ "--exclude-standard"
+ ],
+ "cwd": "[START_DIR]/third_party/dart-pkg/pub",
+ "name": "check for no-op commit",
+ "~followup_annotations": [
+ "@@@STEP_LOG_LINE@stdout@hello@@@",
+ "@@@STEP_LOG_END@stdout@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "git",
+ "diff"
+ ],
+ "cwd": "[START_DIR]/third_party/dart-pkg/pub",
+ "name": "git diff"
+ },
+ {
+ "cmd": [
+ "git",
+ "hash-object",
+ "a diff"
+ ],
+ "cwd": "[START_DIR]/third_party/dart-pkg/pub",
+ "name": "git hash-object"
+ },
+ {
+ "cmd": [
+ "git",
+ "commit",
+ "-m",
+ "[roll] Update third-party dart packages\nChange-Id: Iabc123\n",
+ "-a"
+ ],
+ "cwd": "[START_DIR]/third_party/dart-pkg/pub",
+ "name": "git commit"
+ },
+ {
+ "cmd": [
+ "git",
+ "push",
+ "origin",
+ "HEAD:refs/for/master"
+ ],
+ "cwd": "[START_DIR]/third_party/dart-pkg/pub",
+ "name": "git push",
+ "~followup_annotations": [
+ "@@@STEP_LINK@gerrit link@https://fuchsia-review.googlesource.com/q/Iabc123@@@"
+ ]
+ },
+ {
+ "cmd": [],
+ "name": "ensure gerrit"
+ },
+ {
+ "cmd": [
+ "cipd",
+ "ensure",
+ "-root",
+ "[START_DIR]/cipd/gerrit",
+ "-ensure-file",
+ "infra/tools/luci/gerrit/${platform} latest",
+ "-json-output",
+ "/path/to/tmp/json"
+ ],
+ "cwd": "[START_DIR]/third_party/dart-pkg/pub",
+ "infra_step": true,
+ "name": "ensure gerrit.ensure_installed",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@",
+ "@@@STEP_LOG_LINE@json.output@{@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"result\": {@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"\": [@@@",
+ "@@@STEP_LOG_LINE@json.output@ {@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"instance_id\": \"resolved-instance_id-of-latest----------\", @@@",
+ "@@@STEP_LOG_LINE@json.output@ \"package\": \"infra/tools/luci/gerrit/resolved-platform\"@@@",
+ "@@@STEP_LOG_LINE@json.output@ }@@@",
+ "@@@STEP_LOG_LINE@json.output@ ]@@@",
+ "@@@STEP_LOG_LINE@json.output@ }@@@",
+ "@@@STEP_LOG_LINE@json.output@}@@@",
+ "@@@STEP_LOG_END@json.output@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "[START_DIR]/cipd/gerrit/gerrit",
+ "set-review",
+ "-host",
+ "https://fuchsia-review.googlesource.com",
+ "-input",
+ "{\"change_id\": \"third_party/dart-pkg/pub~master~Iabc123\", \"input\": {\"labels\": {\"Commit-Queue\": 1}}, \"revision_id\": \"current\"}",
+ "-output",
+ "/path/to/tmp/json"
+ ],
+ "cwd": "[START_DIR]/third_party/dart-pkg/pub",
+ "name": "submit to commit queue",
+ "~followup_annotations": [
+ "@@@STEP_LOG_END@json.output (invalid)@@@",
+ "@@@STEP_LOG_LINE@json.output (exception)@No JSON object could be decoded@@@",
+ "@@@STEP_LOG_END@json.output (exception)@@@"
+ ]
+ },
+ {
+ "cmd": [],
+ "name": "check for completion"
+ },
+ {
+ "cmd": [
+ "[START_DIR]/cipd/gerrit/gerrit",
+ "change-detail",
+ "-host",
+ "https://fuchsia-review.googlesource.com",
+ "-input",
+ "{\"change_id\": \"third_party/dart-pkg/pub~master~Iabc123\"}",
+ "-output",
+ "/path/to/tmp/json"
+ ],
+ "cwd": "[START_DIR]/third_party/dart-pkg/pub",
+ "infra_step": true,
+ "name": "check for completion.check if done (0)",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@",
+ "@@@STEP_LOG_LINE@json.output@{@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"labels\": {@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"Commit-Queue\": {@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"approved\": {}@@@",
+ "@@@STEP_LOG_LINE@json.output@ }@@@",
+ "@@@STEP_LOG_LINE@json.output@ }, @@@",
+ "@@@STEP_LOG_LINE@json.output@ \"status\": \"MERGED\"@@@",
+ "@@@STEP_LOG_LINE@json.output@}@@@",
+ "@@@STEP_LOG_END@json.output@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "[START_DIR]/cipd/gerrit/gerrit",
+ "change-abandon",
+ "-host",
+ "https://fuchsia-review.googlesource.com",
+ "-input",
+ "{\"change_id\": \"third_party/dart-pkg/pub~master~Iabc123\"}",
+ "-output",
+ "/path/to/tmp/json"
+ ],
+ "cwd": "[START_DIR]/third_party/dart-pkg/pub",
+ "name": "abandon roll: dry run complete",
+ "~followup_annotations": [
+ "@@@STEP_LOG_END@json.output (invalid)@@@",
+ "@@@STEP_LOG_LINE@json.output (exception)@No JSON object could be decoded@@@",
+ "@@@STEP_LOG_END@json.output (exception)@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "git",
+ "rev-parse",
+ "HEAD"
+ ],
+ "cwd": "[START_DIR]/third_party/dart-pkg/pub",
+ "name": "git rev-parse (3)",
+ "~followup_annotations": [
+ "@@@STEP_LOG_LINE@revision@deadbeef@@@",
+ "@@@STEP_LOG_END@revision@@@"
+ ]
+ },
+ {
+ "name": "$result"
+ }
+]
\ No newline at end of file
diff --git a/recipe_modules/dart_util/examples/full.py b/recipe_modules/dart_util/examples/full.py
new file mode 100644
index 0000000..5d62f0f
--- /dev/null
+++ b/recipe_modules/dart_util/examples/full.py
@@ -0,0 +1,26 @@
+# Copyright 2020 The Fuchsia Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+DEPS = [
+ 'fuchsia/auto_roller',
+ 'fuchsia/dart_util',
+ 'recipe_engine/path',
+ 'recipe_engine/json',
+]
+
+
+def RunSteps(api):
+ """Tests for fetching and uploading debug symbols."""
+ dart_path = api.path['start_dir'].join('dart')
+ api.dart_util.checkout(path=dart_path, revision='abc_revision')
+
+ checkout_root = api.path['start_dir'].join('integration')
+ api.dart_util.update_pkg_manifest(path=dart_path, checkout_root=checkout_root)
+
+ api.dart_util.update_3p_packages(
+ checkout_root=api.path['start_dir'], dry_run=True)
+
+
+def GenTests(api):
+ yield api.test('default') + api.auto_roller.success_step_data()
