[sdk] Add skeleton SDK orchestrator
This revives a fairly old request to orchestrate builds
for SDK flavors together, such that they build at the
same revision and are assigned the same SDK ID.
This is currently a skeleton which is only responsible
for orchestrating SDK archive subbuilds, but in the
future may/should be extended to take over companion
image subbuild scheduling and publishing logic from
sdk.py.
Bug: 10223
Change-Id: I16db42cf1255adc4d987ad261e6c6b96c815c42c
diff --git a/recipes/sdk_orchestrator.expected/ci.json b/recipes/sdk_orchestrator.expected/ci.json
new file mode 100644
index 0000000..1efeee5
--- /dev/null
+++ b/recipes/sdk_orchestrator.expected/ci.json
@@ -0,0 +1,202 @@
+[
+ {
+ "cmd": [
+ "bb",
+ "batch",
+ "-host",
+ "cr-buildbucket.appspot.com"
+ ],
+ "infra_step": true,
+ "name": "schedule archive subbuilds",
+ "stdin": "{\"requests\": [{\"scheduleBuild\": {\"builder\": {\"bucket\": \"ci\", \"builder\": \"sdk-core-linux\", \"project\": \"project\"}, \"experimental\": \"NO\", \"fields\": \"builder,createTime,createdBy,critical,endTime,id,input,number,output,startTime,status,updateTime\", \"gitilesCommit\": {\"host\": \"chromium.googlesource.com\", \"id\": \"2d72510e447ab60a9728aeea2362d8be2cbd7789\", \"project\": \"project\", \"ref\": \"refs/heads/master\"}, \"properties\": {\"sdk_id\": \"8945511751514863184\"}, \"requestId\": \"8945511751514863184-00000000-0000-0000-0000-000000001337\", \"swarming\": {\"parentRunId\": \"fake-task-id\"}, \"tags\": [{\"key\": \"user_agent\", \"value\": \"recipe\"}]}}, {\"scheduleBuild\": {\"builder\": {\"bucket\": \"ci\", \"builder\": \"sdk-core-mac\", \"project\": \"project\"}, \"experimental\": \"NO\", \"fields\": \"builder,createTime,createdBy,critical,endTime,id,input,number,output,startTime,status,updateTime\", \"gitilesCommit\": {\"host\": \"chromium.googlesource.com\", \"id\": \"2d72510e447ab60a9728aeea2362d8be2cbd7789\", \"project\": \"project\", \"ref\": \"refs/heads/master\"}, \"properties\": {\"sdk_id\": \"8945511751514863184\"}, \"requestId\": \"8945511751514863184-00000000-0000-0000-0000-00000000133a\", \"swarming\": {\"parentRunId\": \"fake-task-id\"}, \"tags\": [{\"key\": \"user_agent\", \"value\": \"recipe\"}]}}]}",
+ "~followup_annotations": [
+ "@@@STEP_LOG_LINE@json.output@{@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"responses\": [@@@",
+ "@@@STEP_LOG_LINE@json.output@ {@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"scheduleBuild\": {@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"builder\": {@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"bucket\": \"ci\", @@@",
+ "@@@STEP_LOG_LINE@json.output@ \"builder\": \"sdk-core-linux\", @@@",
+ "@@@STEP_LOG_LINE@json.output@ \"project\": \"project\"@@@",
+ "@@@STEP_LOG_LINE@json.output@ }, @@@",
+ "@@@STEP_LOG_LINE@json.output@ \"id\": \"8922054662172514000\"@@@",
+ "@@@STEP_LOG_LINE@json.output@ }@@@",
+ "@@@STEP_LOG_LINE@json.output@ }, @@@",
+ "@@@STEP_LOG_LINE@json.output@ {@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"scheduleBuild\": {@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"builder\": {@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"bucket\": \"ci\", @@@",
+ "@@@STEP_LOG_LINE@json.output@ \"builder\": \"sdk-core-mac\", @@@",
+ "@@@STEP_LOG_LINE@json.output@ \"project\": \"project\"@@@",
+ "@@@STEP_LOG_LINE@json.output@ }, @@@",
+ "@@@STEP_LOG_LINE@json.output@ \"id\": \"8922054662172514001\"@@@",
+ "@@@STEP_LOG_LINE@json.output@ }@@@",
+ "@@@STEP_LOG_LINE@json.output@ }@@@",
+ "@@@STEP_LOG_LINE@json.output@ ]@@@",
+ "@@@STEP_LOG_LINE@json.output@}@@@",
+ "@@@STEP_LOG_END@json.output@@@",
+ "@@@STEP_LOG_LINE@request@{@@@",
+ "@@@STEP_LOG_LINE@request@ \"requests\": [@@@",
+ "@@@STEP_LOG_LINE@request@ {@@@",
+ "@@@STEP_LOG_LINE@request@ \"scheduleBuild\": {@@@",
+ "@@@STEP_LOG_LINE@request@ \"builder\": {@@@",
+ "@@@STEP_LOG_LINE@request@ \"bucket\": \"ci\", @@@",
+ "@@@STEP_LOG_LINE@request@ \"builder\": \"sdk-core-linux\", @@@",
+ "@@@STEP_LOG_LINE@request@ \"project\": \"project\"@@@",
+ "@@@STEP_LOG_LINE@request@ }, @@@",
+ "@@@STEP_LOG_LINE@request@ \"experimental\": \"NO\", @@@",
+ "@@@STEP_LOG_LINE@request@ \"fields\": \"builder,createTime,createdBy,critical,endTime,id,input,number,output,startTime,status,updateTime\", @@@",
+ "@@@STEP_LOG_LINE@request@ \"gitilesCommit\": {@@@",
+ "@@@STEP_LOG_LINE@request@ \"host\": \"chromium.googlesource.com\", @@@",
+ "@@@STEP_LOG_LINE@request@ \"id\": \"2d72510e447ab60a9728aeea2362d8be2cbd7789\", @@@",
+ "@@@STEP_LOG_LINE@request@ \"project\": \"project\", @@@",
+ "@@@STEP_LOG_LINE@request@ \"ref\": \"refs/heads/master\"@@@",
+ "@@@STEP_LOG_LINE@request@ }, @@@",
+ "@@@STEP_LOG_LINE@request@ \"properties\": {@@@",
+ "@@@STEP_LOG_LINE@request@ \"sdk_id\": \"8945511751514863184\"@@@",
+ "@@@STEP_LOG_LINE@request@ }, @@@",
+ "@@@STEP_LOG_LINE@request@ \"requestId\": \"8945511751514863184-00000000-0000-0000-0000-000000001337\", @@@",
+ "@@@STEP_LOG_LINE@request@ \"swarming\": {@@@",
+ "@@@STEP_LOG_LINE@request@ \"parentRunId\": \"fake-task-id\"@@@",
+ "@@@STEP_LOG_LINE@request@ }, @@@",
+ "@@@STEP_LOG_LINE@request@ \"tags\": [@@@",
+ "@@@STEP_LOG_LINE@request@ {@@@",
+ "@@@STEP_LOG_LINE@request@ \"key\": \"user_agent\", @@@",
+ "@@@STEP_LOG_LINE@request@ \"value\": \"recipe\"@@@",
+ "@@@STEP_LOG_LINE@request@ }@@@",
+ "@@@STEP_LOG_LINE@request@ ]@@@",
+ "@@@STEP_LOG_LINE@request@ }@@@",
+ "@@@STEP_LOG_LINE@request@ }, @@@",
+ "@@@STEP_LOG_LINE@request@ {@@@",
+ "@@@STEP_LOG_LINE@request@ \"scheduleBuild\": {@@@",
+ "@@@STEP_LOG_LINE@request@ \"builder\": {@@@",
+ "@@@STEP_LOG_LINE@request@ \"bucket\": \"ci\", @@@",
+ "@@@STEP_LOG_LINE@request@ \"builder\": \"sdk-core-mac\", @@@",
+ "@@@STEP_LOG_LINE@request@ \"project\": \"project\"@@@",
+ "@@@STEP_LOG_LINE@request@ }, @@@",
+ "@@@STEP_LOG_LINE@request@ \"experimental\": \"NO\", @@@",
+ "@@@STEP_LOG_LINE@request@ \"fields\": \"builder,createTime,createdBy,critical,endTime,id,input,number,output,startTime,status,updateTime\", @@@",
+ "@@@STEP_LOG_LINE@request@ \"gitilesCommit\": {@@@",
+ "@@@STEP_LOG_LINE@request@ \"host\": \"chromium.googlesource.com\", @@@",
+ "@@@STEP_LOG_LINE@request@ \"id\": \"2d72510e447ab60a9728aeea2362d8be2cbd7789\", @@@",
+ "@@@STEP_LOG_LINE@request@ \"project\": \"project\", @@@",
+ "@@@STEP_LOG_LINE@request@ \"ref\": \"refs/heads/master\"@@@",
+ "@@@STEP_LOG_LINE@request@ }, @@@",
+ "@@@STEP_LOG_LINE@request@ \"properties\": {@@@",
+ "@@@STEP_LOG_LINE@request@ \"sdk_id\": \"8945511751514863184\"@@@",
+ "@@@STEP_LOG_LINE@request@ }, @@@",
+ "@@@STEP_LOG_LINE@request@ \"requestId\": \"8945511751514863184-00000000-0000-0000-0000-00000000133a\", @@@",
+ "@@@STEP_LOG_LINE@request@ \"swarming\": {@@@",
+ "@@@STEP_LOG_LINE@request@ \"parentRunId\": \"fake-task-id\"@@@",
+ "@@@STEP_LOG_LINE@request@ }, @@@",
+ "@@@STEP_LOG_LINE@request@ \"tags\": [@@@",
+ "@@@STEP_LOG_LINE@request@ {@@@",
+ "@@@STEP_LOG_LINE@request@ \"key\": \"user_agent\", @@@",
+ "@@@STEP_LOG_LINE@request@ \"value\": \"recipe\"@@@",
+ "@@@STEP_LOG_LINE@request@ }@@@",
+ "@@@STEP_LOG_LINE@request@ ]@@@",
+ "@@@STEP_LOG_LINE@request@ }@@@",
+ "@@@STEP_LOG_LINE@request@ }@@@",
+ "@@@STEP_LOG_LINE@request@ ]@@@",
+ "@@@STEP_LOG_LINE@request@}@@@",
+ "@@@STEP_LOG_END@request@@@",
+ "@@@STEP_LINK@8922054662172514000@https://cr-buildbucket.appspot.com/build/8922054662172514000@@@",
+ "@@@STEP_LINK@8922054662172514001@https://cr-buildbucket.appspot.com/build/8922054662172514001@@@"
+ ]
+ },
+ {
+ "cmd": [],
+ "name": "collect archive subbuilds"
+ },
+ {
+ "cmd": [
+ "bb",
+ "collect",
+ "-host",
+ "cr-buildbucket.appspot.com",
+ "-interval",
+ "60s",
+ "8922054662172514000",
+ "8922054662172514001"
+ ],
+ "infra_step": true,
+ "name": "collect archive subbuilds.wait",
+ "timeout": 3600,
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "bb",
+ "batch",
+ "-host",
+ "cr-buildbucket.appspot.com"
+ ],
+ "infra_step": true,
+ "name": "collect archive subbuilds.get",
+ "stdin": "{\"requests\": [{\"getBuild\": {\"fields\": \"builder,createTime,createdBy,critical,endTime,id,input,number,output,startTime,status,updateTime\", \"id\": \"8922054662172514000\"}}, {\"getBuild\": {\"fields\": \"builder,createTime,createdBy,critical,endTime,id,input,number,output,startTime,status,updateTime\", \"id\": \"8922054662172514001\"}}]}",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@",
+ "@@@STEP_LOG_LINE@json.output@{@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"responses\": [@@@",
+ "@@@STEP_LOG_LINE@json.output@ {@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"getBuild\": {@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"id\": \"8922054662172514000\", @@@",
+ "@@@STEP_LOG_LINE@json.output@ \"status\": \"SUCCESS\"@@@",
+ "@@@STEP_LOG_LINE@json.output@ }@@@",
+ "@@@STEP_LOG_LINE@json.output@ }, @@@",
+ "@@@STEP_LOG_LINE@json.output@ {@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"getBuild\": {@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"id\": \"8922054662172514001\", @@@",
+ "@@@STEP_LOG_LINE@json.output@ \"status\": \"SUCCESS\"@@@",
+ "@@@STEP_LOG_LINE@json.output@ }@@@",
+ "@@@STEP_LOG_LINE@json.output@ }@@@",
+ "@@@STEP_LOG_LINE@json.output@ ]@@@",
+ "@@@STEP_LOG_LINE@json.output@}@@@",
+ "@@@STEP_LOG_END@json.output@@@",
+ "@@@STEP_LOG_LINE@request@{@@@",
+ "@@@STEP_LOG_LINE@request@ \"requests\": [@@@",
+ "@@@STEP_LOG_LINE@request@ {@@@",
+ "@@@STEP_LOG_LINE@request@ \"getBuild\": {@@@",
+ "@@@STEP_LOG_LINE@request@ \"fields\": \"builder,createTime,createdBy,critical,endTime,id,input,number,output,startTime,status,updateTime\", @@@",
+ "@@@STEP_LOG_LINE@request@ \"id\": \"8922054662172514000\"@@@",
+ "@@@STEP_LOG_LINE@request@ }@@@",
+ "@@@STEP_LOG_LINE@request@ }, @@@",
+ "@@@STEP_LOG_LINE@request@ {@@@",
+ "@@@STEP_LOG_LINE@request@ \"getBuild\": {@@@",
+ "@@@STEP_LOG_LINE@request@ \"fields\": \"builder,createTime,createdBy,critical,endTime,id,input,number,output,startTime,status,updateTime\", @@@",
+ "@@@STEP_LOG_LINE@request@ \"id\": \"8922054662172514001\"@@@",
+ "@@@STEP_LOG_LINE@request@ }@@@",
+ "@@@STEP_LOG_LINE@request@ }@@@",
+ "@@@STEP_LOG_LINE@request@ ]@@@",
+ "@@@STEP_LOG_LINE@request@}@@@",
+ "@@@STEP_LOG_END@request@@@",
+ "@@@STEP_LINK@8922054662172514000@https://cr-buildbucket.appspot.com/build/8922054662172514000@@@",
+ "@@@STEP_LINK@8922054662172514001@https://cr-buildbucket.appspot.com/build/8922054662172514001@@@"
+ ]
+ },
+ {
+ "cmd": [],
+ "name": "display archive subbuilds"
+ },
+ {
+ "cmd": [],
+ "name": "display archive subbuilds.",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@",
+ "@@@STEP_LINK@8922054662172514000@https://cr-buildbucket.appspot.com/build/8922054662172514000@@@"
+ ]
+ },
+ {
+ "cmd": [],
+ "name": "display archive subbuilds. (2)",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@",
+ "@@@STEP_LINK@8922054662172514001@https://cr-buildbucket.appspot.com/build/8922054662172514001@@@"
+ ]
+ },
+ {
+ "name": "$result"
+ }
+]
\ No newline at end of file
diff --git a/recipes/sdk_orchestrator.expected/cq.json b/recipes/sdk_orchestrator.expected/cq.json
new file mode 100644
index 0000000..2cee500
--- /dev/null
+++ b/recipes/sdk_orchestrator.expected/cq.json
@@ -0,0 +1,328 @@
+[
+ {
+ "cmd": [],
+ "name": "ensure gerrit"
+ },
+ {
+ "cmd": [
+ "cipd",
+ "ensure",
+ "-root",
+ "[START_DIR]/cipd/gerrit",
+ "-ensure-file",
+ "infra/tools/luci/gerrit/${platform} latest",
+ "-json-output",
+ "/path/to/tmp/json"
+ ],
+ "infra_step": true,
+ "name": "ensure gerrit.ensure_installed",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@",
+ "@@@STEP_LOG_LINE@json.output@{@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"result\": {@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"\": [@@@",
+ "@@@STEP_LOG_LINE@json.output@ {@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"instance_id\": \"resolved-instance_id-of-latest----------\", @@@",
+ "@@@STEP_LOG_LINE@json.output@ \"package\": \"infra/tools/luci/gerrit/resolved-platform\"@@@",
+ "@@@STEP_LOG_LINE@json.output@ }@@@",
+ "@@@STEP_LOG_LINE@json.output@ ]@@@",
+ "@@@STEP_LOG_LINE@json.output@ }@@@",
+ "@@@STEP_LOG_LINE@json.output@}@@@",
+ "@@@STEP_LOG_END@json.output@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "[START_DIR]/cipd/gerrit/gerrit",
+ "change-detail",
+ "-host",
+ "https://chromium-review.googlesource.com",
+ "-input",
+ "{\"change_id\": \"123456\"}",
+ "-output",
+ "/path/to/tmp/json"
+ ],
+ "infra_step": true,
+ "name": "get_gerrit_details",
+ "~followup_annotations": [
+ "@@@STEP_LOG_LINE@json.output@{@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"branch\": \"master\"@@@",
+ "@@@STEP_LOG_LINE@json.output@}@@@",
+ "@@@STEP_LOG_END@json.output@@@"
+ ]
+ },
+ {
+ "cmd": [],
+ "name": "ensure gitiles"
+ },
+ {
+ "cmd": [
+ "cipd",
+ "ensure",
+ "-root",
+ "[START_DIR]/cipd/gitiles",
+ "-ensure-file",
+ "infra/tools/luci/gitiles/${platform} latest",
+ "-json-output",
+ "/path/to/tmp/json"
+ ],
+ "infra_step": true,
+ "name": "ensure gitiles.ensure_installed",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@",
+ "@@@STEP_LOG_LINE@json.output@{@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"result\": {@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"\": [@@@",
+ "@@@STEP_LOG_LINE@json.output@ {@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"instance_id\": \"resolved-instance_id-of-latest----------\", @@@",
+ "@@@STEP_LOG_LINE@json.output@ \"package\": \"infra/tools/luci/gitiles/resolved-platform\"@@@",
+ "@@@STEP_LOG_LINE@json.output@ }@@@",
+ "@@@STEP_LOG_LINE@json.output@ ]@@@",
+ "@@@STEP_LOG_LINE@json.output@ }@@@",
+ "@@@STEP_LOG_LINE@json.output@}@@@",
+ "@@@STEP_LOG_END@json.output@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "[START_DIR]/cipd/gitiles/gitiles",
+ "refs",
+ "-json-output",
+ "/path/to/tmp/json",
+ "https://chromium.googlesource.com/project",
+ "refs/heads"
+ ],
+ "infra_step": true,
+ "name": "refs",
+ "~followup_annotations": [
+ "@@@STEP_LOG_LINE@json.output@{@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"refs/heads/master\": \"deadbeef\"@@@",
+ "@@@STEP_LOG_LINE@json.output@}@@@",
+ "@@@STEP_LOG_END@json.output@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "bb",
+ "batch",
+ "-host",
+ "cr-buildbucket.appspot.com"
+ ],
+ "infra_step": true,
+ "name": "schedule archive subbuilds",
+ "stdin": "{\"requests\": [{\"scheduleBuild\": {\"builder\": {\"bucket\": \"try\", \"builder\": \"sdk-core-linux\", \"project\": \"project\"}, \"experimental\": \"NO\", \"fields\": \"builder,createTime,createdBy,critical,endTime,id,input,number,output,startTime,status,updateTime\", \"gerritChanges\": [{\"change\": \"123456\", \"host\": \"chromium-review.googlesource.com\", \"patchset\": \"7\", \"project\": \"project\"}], \"gitilesCommit\": {\"host\": \"chromium.googlesource.com\", \"id\": \"deadbeef\", \"project\": \"project\", \"ref\": \"refs/heads/master\"}, \"properties\": {\"sdk_id\": \"8945511751514863184\"}, \"requestId\": \"8945511751514863184-00000000-0000-0000-0000-000000001337\", \"swarming\": {\"parentRunId\": \"fake-task-id\"}, \"tags\": [{\"key\": \"cq_experimental\", \"value\": \"false\"}, {\"key\": \"user_agent\", \"value\": \"recipe\"}]}}, {\"scheduleBuild\": {\"builder\": {\"bucket\": \"try\", \"builder\": \"sdk-core-mac\", \"project\": \"project\"}, \"experimental\": \"NO\", \"fields\": \"builder,createTime,createdBy,critical,endTime,id,input,number,output,startTime,status,updateTime\", \"gerritChanges\": [{\"change\": \"123456\", \"host\": \"chromium-review.googlesource.com\", \"patchset\": \"7\", \"project\": \"project\"}], \"gitilesCommit\": {\"host\": \"chromium.googlesource.com\", \"id\": \"deadbeef\", \"project\": \"project\", \"ref\": \"refs/heads/master\"}, \"properties\": {\"sdk_id\": \"8945511751514863184\"}, \"requestId\": \"8945511751514863184-00000000-0000-0000-0000-00000000133a\", \"swarming\": {\"parentRunId\": \"fake-task-id\"}, \"tags\": [{\"key\": \"cq_experimental\", \"value\": \"false\"}, {\"key\": \"user_agent\", \"value\": \"recipe\"}]}}]}",
+ "~followup_annotations": [
+ "@@@STEP_LOG_LINE@json.output@{@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"responses\": [@@@",
+ "@@@STEP_LOG_LINE@json.output@ {@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"scheduleBuild\": {@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"builder\": {@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"bucket\": \"try\", @@@",
+ "@@@STEP_LOG_LINE@json.output@ \"builder\": \"sdk-core-linux\", @@@",
+ "@@@STEP_LOG_LINE@json.output@ \"project\": \"project\"@@@",
+ "@@@STEP_LOG_LINE@json.output@ }, @@@",
+ "@@@STEP_LOG_LINE@json.output@ \"id\": \"8922054662172514000\"@@@",
+ "@@@STEP_LOG_LINE@json.output@ }@@@",
+ "@@@STEP_LOG_LINE@json.output@ }, @@@",
+ "@@@STEP_LOG_LINE@json.output@ {@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"scheduleBuild\": {@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"builder\": {@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"bucket\": \"try\", @@@",
+ "@@@STEP_LOG_LINE@json.output@ \"builder\": \"sdk-core-mac\", @@@",
+ "@@@STEP_LOG_LINE@json.output@ \"project\": \"project\"@@@",
+ "@@@STEP_LOG_LINE@json.output@ }, @@@",
+ "@@@STEP_LOG_LINE@json.output@ \"id\": \"8922054662172514001\"@@@",
+ "@@@STEP_LOG_LINE@json.output@ }@@@",
+ "@@@STEP_LOG_LINE@json.output@ }@@@",
+ "@@@STEP_LOG_LINE@json.output@ ]@@@",
+ "@@@STEP_LOG_LINE@json.output@}@@@",
+ "@@@STEP_LOG_END@json.output@@@",
+ "@@@STEP_LOG_LINE@request@{@@@",
+ "@@@STEP_LOG_LINE@request@ \"requests\": [@@@",
+ "@@@STEP_LOG_LINE@request@ {@@@",
+ "@@@STEP_LOG_LINE@request@ \"scheduleBuild\": {@@@",
+ "@@@STEP_LOG_LINE@request@ \"builder\": {@@@",
+ "@@@STEP_LOG_LINE@request@ \"bucket\": \"try\", @@@",
+ "@@@STEP_LOG_LINE@request@ \"builder\": \"sdk-core-linux\", @@@",
+ "@@@STEP_LOG_LINE@request@ \"project\": \"project\"@@@",
+ "@@@STEP_LOG_LINE@request@ }, @@@",
+ "@@@STEP_LOG_LINE@request@ \"experimental\": \"NO\", @@@",
+ "@@@STEP_LOG_LINE@request@ \"fields\": \"builder,createTime,createdBy,critical,endTime,id,input,number,output,startTime,status,updateTime\", @@@",
+ "@@@STEP_LOG_LINE@request@ \"gerritChanges\": [@@@",
+ "@@@STEP_LOG_LINE@request@ {@@@",
+ "@@@STEP_LOG_LINE@request@ \"change\": \"123456\", @@@",
+ "@@@STEP_LOG_LINE@request@ \"host\": \"chromium-review.googlesource.com\", @@@",
+ "@@@STEP_LOG_LINE@request@ \"patchset\": \"7\", @@@",
+ "@@@STEP_LOG_LINE@request@ \"project\": \"project\"@@@",
+ "@@@STEP_LOG_LINE@request@ }@@@",
+ "@@@STEP_LOG_LINE@request@ ], @@@",
+ "@@@STEP_LOG_LINE@request@ \"gitilesCommit\": {@@@",
+ "@@@STEP_LOG_LINE@request@ \"host\": \"chromium.googlesource.com\", @@@",
+ "@@@STEP_LOG_LINE@request@ \"id\": \"deadbeef\", @@@",
+ "@@@STEP_LOG_LINE@request@ \"project\": \"project\", @@@",
+ "@@@STEP_LOG_LINE@request@ \"ref\": \"refs/heads/master\"@@@",
+ "@@@STEP_LOG_LINE@request@ }, @@@",
+ "@@@STEP_LOG_LINE@request@ \"properties\": {@@@",
+ "@@@STEP_LOG_LINE@request@ \"sdk_id\": \"8945511751514863184\"@@@",
+ "@@@STEP_LOG_LINE@request@ }, @@@",
+ "@@@STEP_LOG_LINE@request@ \"requestId\": \"8945511751514863184-00000000-0000-0000-0000-000000001337\", @@@",
+ "@@@STEP_LOG_LINE@request@ \"swarming\": {@@@",
+ "@@@STEP_LOG_LINE@request@ \"parentRunId\": \"fake-task-id\"@@@",
+ "@@@STEP_LOG_LINE@request@ }, @@@",
+ "@@@STEP_LOG_LINE@request@ \"tags\": [@@@",
+ "@@@STEP_LOG_LINE@request@ {@@@",
+ "@@@STEP_LOG_LINE@request@ \"key\": \"cq_experimental\", @@@",
+ "@@@STEP_LOG_LINE@request@ \"value\": \"false\"@@@",
+ "@@@STEP_LOG_LINE@request@ }, @@@",
+ "@@@STEP_LOG_LINE@request@ {@@@",
+ "@@@STEP_LOG_LINE@request@ \"key\": \"user_agent\", @@@",
+ "@@@STEP_LOG_LINE@request@ \"value\": \"recipe\"@@@",
+ "@@@STEP_LOG_LINE@request@ }@@@",
+ "@@@STEP_LOG_LINE@request@ ]@@@",
+ "@@@STEP_LOG_LINE@request@ }@@@",
+ "@@@STEP_LOG_LINE@request@ }, @@@",
+ "@@@STEP_LOG_LINE@request@ {@@@",
+ "@@@STEP_LOG_LINE@request@ \"scheduleBuild\": {@@@",
+ "@@@STEP_LOG_LINE@request@ \"builder\": {@@@",
+ "@@@STEP_LOG_LINE@request@ \"bucket\": \"try\", @@@",
+ "@@@STEP_LOG_LINE@request@ \"builder\": \"sdk-core-mac\", @@@",
+ "@@@STEP_LOG_LINE@request@ \"project\": \"project\"@@@",
+ "@@@STEP_LOG_LINE@request@ }, @@@",
+ "@@@STEP_LOG_LINE@request@ \"experimental\": \"NO\", @@@",
+ "@@@STEP_LOG_LINE@request@ \"fields\": \"builder,createTime,createdBy,critical,endTime,id,input,number,output,startTime,status,updateTime\", @@@",
+ "@@@STEP_LOG_LINE@request@ \"gerritChanges\": [@@@",
+ "@@@STEP_LOG_LINE@request@ {@@@",
+ "@@@STEP_LOG_LINE@request@ \"change\": \"123456\", @@@",
+ "@@@STEP_LOG_LINE@request@ \"host\": \"chromium-review.googlesource.com\", @@@",
+ "@@@STEP_LOG_LINE@request@ \"patchset\": \"7\", @@@",
+ "@@@STEP_LOG_LINE@request@ \"project\": \"project\"@@@",
+ "@@@STEP_LOG_LINE@request@ }@@@",
+ "@@@STEP_LOG_LINE@request@ ], @@@",
+ "@@@STEP_LOG_LINE@request@ \"gitilesCommit\": {@@@",
+ "@@@STEP_LOG_LINE@request@ \"host\": \"chromium.googlesource.com\", @@@",
+ "@@@STEP_LOG_LINE@request@ \"id\": \"deadbeef\", @@@",
+ "@@@STEP_LOG_LINE@request@ \"project\": \"project\", @@@",
+ "@@@STEP_LOG_LINE@request@ \"ref\": \"refs/heads/master\"@@@",
+ "@@@STEP_LOG_LINE@request@ }, @@@",
+ "@@@STEP_LOG_LINE@request@ \"properties\": {@@@",
+ "@@@STEP_LOG_LINE@request@ \"sdk_id\": \"8945511751514863184\"@@@",
+ "@@@STEP_LOG_LINE@request@ }, @@@",
+ "@@@STEP_LOG_LINE@request@ \"requestId\": \"8945511751514863184-00000000-0000-0000-0000-00000000133a\", @@@",
+ "@@@STEP_LOG_LINE@request@ \"swarming\": {@@@",
+ "@@@STEP_LOG_LINE@request@ \"parentRunId\": \"fake-task-id\"@@@",
+ "@@@STEP_LOG_LINE@request@ }, @@@",
+ "@@@STEP_LOG_LINE@request@ \"tags\": [@@@",
+ "@@@STEP_LOG_LINE@request@ {@@@",
+ "@@@STEP_LOG_LINE@request@ \"key\": \"cq_experimental\", @@@",
+ "@@@STEP_LOG_LINE@request@ \"value\": \"false\"@@@",
+ "@@@STEP_LOG_LINE@request@ }, @@@",
+ "@@@STEP_LOG_LINE@request@ {@@@",
+ "@@@STEP_LOG_LINE@request@ \"key\": \"user_agent\", @@@",
+ "@@@STEP_LOG_LINE@request@ \"value\": \"recipe\"@@@",
+ "@@@STEP_LOG_LINE@request@ }@@@",
+ "@@@STEP_LOG_LINE@request@ ]@@@",
+ "@@@STEP_LOG_LINE@request@ }@@@",
+ "@@@STEP_LOG_LINE@request@ }@@@",
+ "@@@STEP_LOG_LINE@request@ ]@@@",
+ "@@@STEP_LOG_LINE@request@}@@@",
+ "@@@STEP_LOG_END@request@@@",
+ "@@@STEP_LINK@8922054662172514000@https://cr-buildbucket.appspot.com/build/8922054662172514000@@@",
+ "@@@STEP_LINK@8922054662172514001@https://cr-buildbucket.appspot.com/build/8922054662172514001@@@"
+ ]
+ },
+ {
+ "cmd": [],
+ "name": "collect archive subbuilds"
+ },
+ {
+ "cmd": [
+ "bb",
+ "collect",
+ "-host",
+ "cr-buildbucket.appspot.com",
+ "-interval",
+ "60s",
+ "8922054662172514000",
+ "8922054662172514001"
+ ],
+ "infra_step": true,
+ "name": "collect archive subbuilds.wait",
+ "timeout": 3600,
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "bb",
+ "batch",
+ "-host",
+ "cr-buildbucket.appspot.com"
+ ],
+ "infra_step": true,
+ "name": "collect archive subbuilds.get",
+ "stdin": "{\"requests\": [{\"getBuild\": {\"fields\": \"builder,createTime,createdBy,critical,endTime,id,input,number,output,startTime,status,updateTime\", \"id\": \"8922054662172514000\"}}, {\"getBuild\": {\"fields\": \"builder,createTime,createdBy,critical,endTime,id,input,number,output,startTime,status,updateTime\", \"id\": \"8922054662172514001\"}}]}",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@",
+ "@@@STEP_LOG_LINE@json.output@{@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"responses\": [@@@",
+ "@@@STEP_LOG_LINE@json.output@ {@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"getBuild\": {@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"id\": \"8922054662172514000\", @@@",
+ "@@@STEP_LOG_LINE@json.output@ \"status\": \"SUCCESS\"@@@",
+ "@@@STEP_LOG_LINE@json.output@ }@@@",
+ "@@@STEP_LOG_LINE@json.output@ }, @@@",
+ "@@@STEP_LOG_LINE@json.output@ {@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"getBuild\": {@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"id\": \"8922054662172514001\", @@@",
+ "@@@STEP_LOG_LINE@json.output@ \"status\": \"SUCCESS\"@@@",
+ "@@@STEP_LOG_LINE@json.output@ }@@@",
+ "@@@STEP_LOG_LINE@json.output@ }@@@",
+ "@@@STEP_LOG_LINE@json.output@ ]@@@",
+ "@@@STEP_LOG_LINE@json.output@}@@@",
+ "@@@STEP_LOG_END@json.output@@@",
+ "@@@STEP_LOG_LINE@request@{@@@",
+ "@@@STEP_LOG_LINE@request@ \"requests\": [@@@",
+ "@@@STEP_LOG_LINE@request@ {@@@",
+ "@@@STEP_LOG_LINE@request@ \"getBuild\": {@@@",
+ "@@@STEP_LOG_LINE@request@ \"fields\": \"builder,createTime,createdBy,critical,endTime,id,input,number,output,startTime,status,updateTime\", @@@",
+ "@@@STEP_LOG_LINE@request@ \"id\": \"8922054662172514000\"@@@",
+ "@@@STEP_LOG_LINE@request@ }@@@",
+ "@@@STEP_LOG_LINE@request@ }, @@@",
+ "@@@STEP_LOG_LINE@request@ {@@@",
+ "@@@STEP_LOG_LINE@request@ \"getBuild\": {@@@",
+ "@@@STEP_LOG_LINE@request@ \"fields\": \"builder,createTime,createdBy,critical,endTime,id,input,number,output,startTime,status,updateTime\", @@@",
+ "@@@STEP_LOG_LINE@request@ \"id\": \"8922054662172514001\"@@@",
+ "@@@STEP_LOG_LINE@request@ }@@@",
+ "@@@STEP_LOG_LINE@request@ }@@@",
+ "@@@STEP_LOG_LINE@request@ ]@@@",
+ "@@@STEP_LOG_LINE@request@}@@@",
+ "@@@STEP_LOG_END@request@@@",
+ "@@@STEP_LINK@8922054662172514000@https://cr-buildbucket.appspot.com/build/8922054662172514000@@@",
+ "@@@STEP_LINK@8922054662172514001@https://cr-buildbucket.appspot.com/build/8922054662172514001@@@"
+ ]
+ },
+ {
+ "cmd": [],
+ "name": "display archive subbuilds"
+ },
+ {
+ "cmd": [],
+ "name": "display archive subbuilds.",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@",
+ "@@@STEP_LINK@8922054662172514000@https://cr-buildbucket.appspot.com/build/8922054662172514000@@@"
+ ]
+ },
+ {
+ "cmd": [],
+ "name": "display archive subbuilds. (2)",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@",
+ "@@@STEP_LINK@8922054662172514001@https://cr-buildbucket.appspot.com/build/8922054662172514001@@@"
+ ]
+ },
+ {
+ "name": "$result"
+ }
+]
\ No newline at end of file
diff --git a/recipes/sdk_orchestrator.py b/recipes/sdk_orchestrator.py
new file mode 100644
index 0000000..221bb72
--- /dev/null
+++ b/recipes/sdk_orchestrator.py
@@ -0,0 +1,105 @@
+# Copyright 2020 The Fuchsia Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+""" sdk_orchestrator.py -- Orchestrates and publishes Fuchsia SDKs.
+
+# Execution overview
+
+## Schedule SDK archive subbuilds
+
+This recipe schedules sdk.py subbuilds, which in turn are responsible for
+generating, merging, and testing SDK archives. Each sdk.py subbuild should be
+a unique SDK flavor. The primary motivation for orchestrating the subbuilds
+is to 1) unify the SDK IDs and 2) unify the publishing logic across SDK
+flavors.
+
+## Schedule companion image subbuilds
+
+TODO: Move companion image logic here from sdk.py.
+
+This recipe will optionally schedule companion image subbuilds, which in turn
+are responsible for generating Fuchsia images. Each fuchsia/build.py subbuild
+should be a unique product.board configuration which is compatible with the
+SDK flavors from the above step.
+
+## Publish
+
+TODO: Move publish logic here from sdk.py.
+
+If publishing is enabled, this recipe will relocate the various artifacts
+generated in the above steps to Fuchsia SDK namespaces in GCS and CIPD.
+"""
+
+from recipe_engine.config import List
+from recipe_engine.recipe_api import Property
+
+DEPS = [
+ 'fuchsia/build_input_resolver',
+ 'fuchsia/buildbucket_util',
+ 'fuchsia/gitiles',
+ 'recipe_engine/buildbucket',
+ 'recipe_engine/properties',
+ 'recipe_engine/step',
+ 'recipe_engine/swarming',
+]
+
+PROPERTIES = {
+ 'subbuilders':
+ Property(
+ kind=List(basestring),
+ help='sdk.py subbuilder names to generate merged SDK archives.',
+ ),
+ 'companion_images':
+ Property(
+ kind=List(dict),
+ help='The Fuchsia images built with this version of the SDK.'
+ ' Each object should have 3 properties:'
+ ' name (str): name of the image referred to in the SDK.'
+ ' builder (str): name of the fuchsia/build.py builder.'
+ ' bucket (str): GCS bucket where the image should be installed.',
+ default=[]),
+ 'sdk_id':
+ Property(kind=str, help='The SDK version ID.', default=''),
+}
+
+
+def RunSteps(api, subbuilders, companion_images, sdk_id):
+ api.build_input_resolver.resolve(
+ default_project_url='https://fuchsia.googlesource.com/fuchsia')
+ # By default the SDK ID is set to this build's buildbucket ID. This will
+ # change when we have a real SDK versioning scheme.
+ sdk_id = sdk_id if sdk_id else api.buildbucket_util.id
+ # The same SDK ID should be shared by all sdk.py subbuilds.
+ properties = {'sdk_id': sdk_id}
+
+ # Schedule archive subbuilds.
+ archive_schedule_reqs = []
+ for subbuilder in subbuilders:
+ req = api.buildbucket.schedule_request(
+ builder=subbuilder,
+ properties=properties,
+ swarming_parent_run_id=api.swarming.task_id,
+ priority=None) # Avoid overriding priority from configs.
+ archive_schedule_reqs.append(req)
+ archive_subbuilds = api.buildbucket.schedule(
+ archive_schedule_reqs, step_name='schedule archive subbuilds')
+
+ # Collect and display archive subbuilds status.
+ archive_subbuilds = api.buildbucket.collect_builds(
+ build_ids=[subbuild.id for subbuild in archive_subbuilds],
+ step_name='collect archive subbuilds')
+ api.buildbucket_util.display_builds(
+ step_name='display archive subbuilds',
+ builds=archive_subbuilds.values(),
+ raise_on_failure=True)
+
+
+def GenTests(api):
+ subbuilders = ['sdk-core-linux', 'sdk-core-mac']
+
+ yield (api.test('ci') + api.buildbucket.ci_build() +
+ api.properties(subbuilders=subbuilders))
+ yield (api.test('cq') + api.buildbucket.try_build() +
+ api.properties(subbuilders=subbuilders) +
+ api.build_input_resolver.set_gerrit_branch() +
+ api.gitiles.refs('refs', ['refs/heads/master', 'deadbeef']))
