| [ |
| { |
| "cmd": [], |
| "name": "install fuchsia/infra/lkg" |
| }, |
| { |
| "cmd": [ |
| "vpython", |
| "-u", |
| "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", |
| "--json-output", |
| "/path/to/tmp/json", |
| "ensure-directory", |
| "--mode", |
| "0777", |
| "[CACHE]/cipd/fuchsia/infra/lkg/git_revision%3A22886bd1b229d02e28330e43b495bbeb86592ac4" |
| ], |
| "infra_step": true, |
| "name": "install fuchsia/infra/lkg.ensure package directory", |
| "~followup_annotations": [ |
| "@@@STEP_NEST_LEVEL@1@@@" |
| ] |
| }, |
| { |
| "cmd": [ |
| "cipd", |
| "ensure", |
| "-root", |
| "[CACHE]/cipd/fuchsia/infra/lkg/git_revision%3A22886bd1b229d02e28330e43b495bbeb86592ac4", |
| "-ensure-file", |
| "fuchsia/infra/lkg/${platform} git_revision:22886bd1b229d02e28330e43b495bbeb86592ac4", |
| "-max-threads", |
| "0", |
| "-json-output", |
| "/path/to/tmp/json" |
| ], |
| "infra_step": true, |
| "name": "install fuchsia/infra/lkg.ensure_installed", |
| "~followup_annotations": [ |
| "@@@STEP_NEST_LEVEL@1@@@", |
| "@@@STEP_LOG_LINE@json.output@{@@@", |
| "@@@STEP_LOG_LINE@json.output@ \"result\": {@@@", |
| "@@@STEP_LOG_LINE@json.output@ \"\": [@@@", |
| "@@@STEP_LOG_LINE@json.output@ {@@@", |
| "@@@STEP_LOG_LINE@json.output@ \"instance_id\": \"resolved-instance_id-of-git_revision:228\", @@@", |
| "@@@STEP_LOG_LINE@json.output@ \"package\": \"fuchsia/infra/lkg/resolved-platform\"@@@", |
| "@@@STEP_LOG_LINE@json.output@ }@@@", |
| "@@@STEP_LOG_LINE@json.output@ ]@@@", |
| "@@@STEP_LOG_LINE@json.output@ }@@@", |
| "@@@STEP_LOG_LINE@json.output@}@@@", |
| "@@@STEP_LOG_END@json.output@@@" |
| ] |
| }, |
| { |
| "cmd": [ |
| "[CACHE]/cipd/fuchsia/infra/lkg/git_revision%3A22886bd1b229d02e28330e43b495bbeb86592ac4/lkg", |
| "build", |
| "-builder", |
| "//" |
| ], |
| "name": "get lkg build" |
| }, |
| { |
| "cmd": [ |
| "bb", |
| "batch", |
| "-host", |
| "cr-buildbucket.appspot.com" |
| ], |
| "infra_step": true, |
| "name": "buildbucket.get", |
| "stdin": "{\"requests\": [{\"getBuild\": {\"fields\": \"id,output\", \"id\": \"889900\"}}]}", |
| "~followup_annotations": [ |
| "@@@STEP_LOG_LINE@json.output@{@@@", |
| "@@@STEP_LOG_LINE@json.output@ \"responses\": [@@@", |
| "@@@STEP_LOG_LINE@json.output@ {@@@", |
| "@@@STEP_LOG_LINE@json.output@ \"getBuild\": {@@@", |
| "@@@STEP_LOG_LINE@json.output@ \"id\": \"889900\", @@@", |
| "@@@STEP_LOG_LINE@json.output@ \"status\": \"SUCCESS\"@@@", |
| "@@@STEP_LOG_LINE@json.output@ }@@@", |
| "@@@STEP_LOG_LINE@json.output@ }@@@", |
| "@@@STEP_LOG_LINE@json.output@ ]@@@", |
| "@@@STEP_LOG_LINE@json.output@}@@@", |
| "@@@STEP_LOG_END@json.output@@@", |
| "@@@STEP_LOG_LINE@request@{@@@", |
| "@@@STEP_LOG_LINE@request@ \"requests\": [@@@", |
| "@@@STEP_LOG_LINE@request@ {@@@", |
| "@@@STEP_LOG_LINE@request@ \"getBuild\": {@@@", |
| "@@@STEP_LOG_LINE@request@ \"fields\": \"id,output\", @@@", |
| "@@@STEP_LOG_LINE@request@ \"id\": \"889900\"@@@", |
| "@@@STEP_LOG_LINE@request@ }@@@", |
| "@@@STEP_LOG_LINE@request@ }@@@", |
| "@@@STEP_LOG_LINE@request@ ]@@@", |
| "@@@STEP_LOG_LINE@request@}@@@", |
| "@@@STEP_LOG_END@request@@@", |
| "@@@STEP_LINK@889900@https://cr-buildbucket.appspot.com/build/889900@@@" |
| ] |
| }, |
| { |
| "cmd": [], |
| "name": "install fuchsia/infra/bigquery" |
| }, |
| { |
| "cmd": [ |
| "vpython", |
| "-u", |
| "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", |
| "--json-output", |
| "/path/to/tmp/json", |
| "ensure-directory", |
| "--mode", |
| "0777", |
| "[CACHE]/cipd/fuchsia/infra/bigquery/git_revision%3A315f9e290334a63cd265c67000069b82323d732f" |
| ], |
| "infra_step": true, |
| "name": "install fuchsia/infra/bigquery.ensure package directory", |
| "~followup_annotations": [ |
| "@@@STEP_NEST_LEVEL@1@@@" |
| ] |
| }, |
| { |
| "cmd": [ |
| "cipd", |
| "ensure", |
| "-root", |
| "[CACHE]/cipd/fuchsia/infra/bigquery/git_revision%3A315f9e290334a63cd265c67000069b82323d732f", |
| "-ensure-file", |
| "fuchsia/infra/bigquery/${platform} git_revision:315f9e290334a63cd265c67000069b82323d732f", |
| "-max-threads", |
| "0", |
| "-json-output", |
| "/path/to/tmp/json" |
| ], |
| "infra_step": true, |
| "name": "install fuchsia/infra/bigquery.ensure_installed", |
| "~followup_annotations": [ |
| "@@@STEP_NEST_LEVEL@1@@@", |
| "@@@STEP_LOG_LINE@json.output@{@@@", |
| "@@@STEP_LOG_LINE@json.output@ \"result\": {@@@", |
| "@@@STEP_LOG_LINE@json.output@ \"\": [@@@", |
| "@@@STEP_LOG_LINE@json.output@ {@@@", |
| "@@@STEP_LOG_LINE@json.output@ \"instance_id\": \"resolved-instance_id-of-git_revision:315\", @@@", |
| "@@@STEP_LOG_LINE@json.output@ \"package\": \"fuchsia/infra/bigquery/resolved-platform\"@@@", |
| "@@@STEP_LOG_LINE@json.output@ }@@@", |
| "@@@STEP_LOG_LINE@json.output@ ]@@@", |
| "@@@STEP_LOG_LINE@json.output@ }@@@", |
| "@@@STEP_LOG_LINE@json.output@}@@@", |
| "@@@STEP_LOG_END@json.output@@@" |
| ] |
| }, |
| { |
| "cmd": [ |
| "[CACHE]/cipd/fuchsia/infra/bigquery/git_revision%3A315f9e290334a63cd265c67000069b82323d732f/bigquery", |
| "query", |
| "-project", |
| "fuchsia-infra", |
| "-input", |
| "\nWITH\n eq_patchset_attempts AS (\n SELECT\n -- equivalent_cl_group_key is a UUID for a group of patchsets which are\n -- \"equivalent\", e.g. PS1 and PS2 will share an equivalent_cl_group_key if\n -- PS2 is a trivial rebase. It is a consistent value across multiple CQ\n -- attempts.\n -- We use this to uniquely identify a group of \"equivalent\" patchsets\n -- rather than treating every patchset as unique.\n equivalent_cl_group_key,\n cl.change,\n cl.patchset,\n cl.host,\n bb.id,\n start_time,\n status,\n COUNTIF(status = 'SUCCESS')\n OVER (PARTITION BY equivalent_cl_group_key) AS cq_success,\n FROM\n `commit-queue.fuchsia.attempts`,\n UNNEST(gerrit_changes) AS cl,\n UNNEST(builds) AS bb\n WHERE\n cl.project = \"fuchsia\"\n -- Get all attempts that were either SUCCESS or FAILURE. We'll use this\n -- later to filter out flakes.\n AND status IN (\"SUCCESS\", \"FAILURE\")\n ),\n patchset_info AS (\n SELECT\n cl.change,\n MAX(cl.patchset) AS max_patchset,\n MIN(cl.patchset) AS min_patchset,\n FROM `commit-queue.fuchsia.attempts`, UNNEST(gerrit_changes) AS cl\n WHERE cl.project = \"fuchsia\"\n GROUP BY(cl.change)\n ),\n all_details AS (\n SELECT\n equivalent_cl_group_key,\n MAX(change) AS change,\n MAX(patchset) AS patchset,\n MAX(host) AS host,\n -- This will be \"fuchsia\" if there's even one \"fuchsia\" builder.\n MIN(build_info.builder.project)\n AS project,\n EXTRACT(DATETIME FROM MIN(try.start_time)) AS start_time,\n ARRAY_AGG(build_info.builder.builder) AS failed_builders,\n ARRAY_AGG(build_info.id) AS bbids,\n FROM `cr-buildbucket.fuchsia.builds` AS build_info\n JOIN eq_patchset_attempts AS try\n ON try.id = build_info.id\n WHERE build_info.status = \"FAILURE\"\n -- Exclude any experimental tryjobs.\n AND JSON_EXTRACT(build_info.input.properties, \"$['$recipe_engine/cq'].experimental\") IS NULL\n -- Only use tryjobs with no autocorrelator findings. Findings indicate\n -- false rejection.\n AND (\n JSON_EXTRACT(build_info.output.properties, \"$.num_autocorrelator_findings\") IS NULL\n OR JSON_EXTRACT(build_info.output.properties, \"$.num_autocorrelator_findings\") = \"0.0\"\n )\n AND cq_success = 0\n GROUP BY equivalent_cl_group_key\n )\n\n-- If all equivalent patchsets failed with different builders i.e. a series of\n-- different flakes caused a failure, this query may output some flakes.\n-- Empirically however, this does not seem to be very frequent.\nSELECT\n ad.change AS change,\n 'https://fxrev.dev/c/' || ad.change || '/' || ad.patchset\n AS patchset,\n CASE\n WHEN ad.patchset > 1\n THEN\n 'https://fxrev.dev/c/' || ad.change || '/'\n || CAST(ad.patchset - 1 AS string) || '..' || ad.patchset\n ELSE NULL\n END\n AS prev_patchset,\n CASE\n WHEN ad.patchset < pi.max_patchset\n THEN\n 'https://fxrev.dev/c/' || ad.change || '/' || ad.patchset || '..'\n || CAST(ad.patchset + 1 AS string)\n ELSE NULL\n END\n AS next_patchset,\n ARRAY(SELECT DISTINCT * FROM UNNEST(failed_builders)) AS failed_builders,\n ARRAY(\n SELECT DISTINCT 'https://ci.chromium.org/b/' || bbid || ' '\n FROM UNNEST(bbids) AS bbid\n ) AS build_links,\nFROM all_details AS ad\nJOIN patchset_info AS pi\n ON ad.change = pi.change\nWHERE\n start_time >= DATETIME '2012-05-13 12:53:21'\n AND start_time <= DATETIME '2012-05-14 12:53:23'\n AND project != \"fuchsia\";\n", |
| "-json-output", |
| "[START_DIR]/sapling.json" |
| ], |
| "name": "run sapling query", |
| "~followup_annotations": [ |
| "@@@STEP_LOG_LINE@query@@@@", |
| "@@@STEP_LOG_LINE@query@WITH@@@", |
| "@@@STEP_LOG_LINE@query@ eq_patchset_attempts AS (@@@", |
| "@@@STEP_LOG_LINE@query@ SELECT@@@", |
| "@@@STEP_LOG_LINE@query@ -- equivalent_cl_group_key is a UUID for a group of patchsets which are@@@", |
| "@@@STEP_LOG_LINE@query@ -- \"equivalent\", e.g. PS1 and PS2 will share an equivalent_cl_group_key if@@@", |
| "@@@STEP_LOG_LINE@query@ -- PS2 is a trivial rebase. It is a consistent value across multiple CQ@@@", |
| "@@@STEP_LOG_LINE@query@ -- attempts.@@@", |
| "@@@STEP_LOG_LINE@query@ -- We use this to uniquely identify a group of \"equivalent\" patchsets@@@", |
| "@@@STEP_LOG_LINE@query@ -- rather than treating every patchset as unique.@@@", |
| "@@@STEP_LOG_LINE@query@ equivalent_cl_group_key,@@@", |
| "@@@STEP_LOG_LINE@query@ cl.change,@@@", |
| "@@@STEP_LOG_LINE@query@ cl.patchset,@@@", |
| "@@@STEP_LOG_LINE@query@ cl.host,@@@", |
| "@@@STEP_LOG_LINE@query@ bb.id,@@@", |
| "@@@STEP_LOG_LINE@query@ start_time,@@@", |
| "@@@STEP_LOG_LINE@query@ status,@@@", |
| "@@@STEP_LOG_LINE@query@ COUNTIF(status = 'SUCCESS')@@@", |
| "@@@STEP_LOG_LINE@query@ OVER (PARTITION BY equivalent_cl_group_key) AS cq_success,@@@", |
| "@@@STEP_LOG_LINE@query@ FROM@@@", |
| "@@@STEP_LOG_LINE@query@ `commit-queue.fuchsia.attempts`,@@@", |
| "@@@STEP_LOG_LINE@query@ UNNEST(gerrit_changes) AS cl,@@@", |
| "@@@STEP_LOG_LINE@query@ UNNEST(builds) AS bb@@@", |
| "@@@STEP_LOG_LINE@query@ WHERE@@@", |
| "@@@STEP_LOG_LINE@query@ cl.project = \"fuchsia\"@@@", |
| "@@@STEP_LOG_LINE@query@ -- Get all attempts that were either SUCCESS or FAILURE. We'll use this@@@", |
| "@@@STEP_LOG_LINE@query@ -- later to filter out flakes.@@@", |
| "@@@STEP_LOG_LINE@query@ AND status IN (\"SUCCESS\", \"FAILURE\")@@@", |
| "@@@STEP_LOG_LINE@query@ ),@@@", |
| "@@@STEP_LOG_LINE@query@ patchset_info AS (@@@", |
| "@@@STEP_LOG_LINE@query@ SELECT@@@", |
| "@@@STEP_LOG_LINE@query@ cl.change,@@@", |
| "@@@STEP_LOG_LINE@query@ MAX(cl.patchset) AS max_patchset,@@@", |
| "@@@STEP_LOG_LINE@query@ MIN(cl.patchset) AS min_patchset,@@@", |
| "@@@STEP_LOG_LINE@query@ FROM `commit-queue.fuchsia.attempts`, UNNEST(gerrit_changes) AS cl@@@", |
| "@@@STEP_LOG_LINE@query@ WHERE cl.project = \"fuchsia\"@@@", |
| "@@@STEP_LOG_LINE@query@ GROUP BY(cl.change)@@@", |
| "@@@STEP_LOG_LINE@query@ ),@@@", |
| "@@@STEP_LOG_LINE@query@ all_details AS (@@@", |
| "@@@STEP_LOG_LINE@query@ SELECT@@@", |
| "@@@STEP_LOG_LINE@query@ equivalent_cl_group_key,@@@", |
| "@@@STEP_LOG_LINE@query@ MAX(change) AS change,@@@", |
| "@@@STEP_LOG_LINE@query@ MAX(patchset) AS patchset,@@@", |
| "@@@STEP_LOG_LINE@query@ MAX(host) AS host,@@@", |
| "@@@STEP_LOG_LINE@query@ -- This will be \"fuchsia\" if there's even one \"fuchsia\" builder.@@@", |
| "@@@STEP_LOG_LINE@query@ MIN(build_info.builder.project)@@@", |
| "@@@STEP_LOG_LINE@query@ AS project,@@@", |
| "@@@STEP_LOG_LINE@query@ EXTRACT(DATETIME FROM MIN(try.start_time)) AS start_time,@@@", |
| "@@@STEP_LOG_LINE@query@ ARRAY_AGG(build_info.builder.builder) AS failed_builders,@@@", |
| "@@@STEP_LOG_LINE@query@ ARRAY_AGG(build_info.id) AS bbids,@@@", |
| "@@@STEP_LOG_LINE@query@ FROM `cr-buildbucket.fuchsia.builds` AS build_info@@@", |
| "@@@STEP_LOG_LINE@query@ JOIN eq_patchset_attempts AS try@@@", |
| "@@@STEP_LOG_LINE@query@ ON try.id = build_info.id@@@", |
| "@@@STEP_LOG_LINE@query@ WHERE build_info.status = \"FAILURE\"@@@", |
| "@@@STEP_LOG_LINE@query@ -- Exclude any experimental tryjobs.@@@", |
| "@@@STEP_LOG_LINE@query@ AND JSON_EXTRACT(build_info.input.properties, \"$['$recipe_engine/cq'].experimental\") IS NULL@@@", |
| "@@@STEP_LOG_LINE@query@ -- Only use tryjobs with no autocorrelator findings. Findings indicate@@@", |
| "@@@STEP_LOG_LINE@query@ -- false rejection.@@@", |
| "@@@STEP_LOG_LINE@query@ AND (@@@", |
| "@@@STEP_LOG_LINE@query@ JSON_EXTRACT(build_info.output.properties, \"$.num_autocorrelator_findings\") IS NULL@@@", |
| "@@@STEP_LOG_LINE@query@ OR JSON_EXTRACT(build_info.output.properties, \"$.num_autocorrelator_findings\") = \"0.0\"@@@", |
| "@@@STEP_LOG_LINE@query@ )@@@", |
| "@@@STEP_LOG_LINE@query@ AND cq_success = 0@@@", |
| "@@@STEP_LOG_LINE@query@ GROUP BY equivalent_cl_group_key@@@", |
| "@@@STEP_LOG_LINE@query@ )@@@", |
| "@@@STEP_LOG_LINE@query@@@@", |
| "@@@STEP_LOG_LINE@query@-- If all equivalent patchsets failed with different builders i.e. a series of@@@", |
| "@@@STEP_LOG_LINE@query@-- different flakes caused a failure, this query may output some flakes.@@@", |
| "@@@STEP_LOG_LINE@query@-- Empirically however, this does not seem to be very frequent.@@@", |
| "@@@STEP_LOG_LINE@query@SELECT@@@", |
| "@@@STEP_LOG_LINE@query@ ad.change AS change,@@@", |
| "@@@STEP_LOG_LINE@query@ 'https://fxrev.dev/c/' || ad.change || '/' || ad.patchset@@@", |
| "@@@STEP_LOG_LINE@query@ AS patchset,@@@", |
| "@@@STEP_LOG_LINE@query@ CASE@@@", |
| "@@@STEP_LOG_LINE@query@ WHEN ad.patchset > 1@@@", |
| "@@@STEP_LOG_LINE@query@ THEN@@@", |
| "@@@STEP_LOG_LINE@query@ 'https://fxrev.dev/c/' || ad.change || '/'@@@", |
| "@@@STEP_LOG_LINE@query@ || CAST(ad.patchset - 1 AS string) || '..' || ad.patchset@@@", |
| "@@@STEP_LOG_LINE@query@ ELSE NULL@@@", |
| "@@@STEP_LOG_LINE@query@ END@@@", |
| "@@@STEP_LOG_LINE@query@ AS prev_patchset,@@@", |
| "@@@STEP_LOG_LINE@query@ CASE@@@", |
| "@@@STEP_LOG_LINE@query@ WHEN ad.patchset < pi.max_patchset@@@", |
| "@@@STEP_LOG_LINE@query@ THEN@@@", |
| "@@@STEP_LOG_LINE@query@ 'https://fxrev.dev/c/' || ad.change || '/' || ad.patchset || '..'@@@", |
| "@@@STEP_LOG_LINE@query@ || CAST(ad.patchset + 1 AS string)@@@", |
| "@@@STEP_LOG_LINE@query@ ELSE NULL@@@", |
| "@@@STEP_LOG_LINE@query@ END@@@", |
| "@@@STEP_LOG_LINE@query@ AS next_patchset,@@@", |
| "@@@STEP_LOG_LINE@query@ ARRAY(SELECT DISTINCT * FROM UNNEST(failed_builders)) AS failed_builders,@@@", |
| "@@@STEP_LOG_LINE@query@ ARRAY(@@@", |
| "@@@STEP_LOG_LINE@query@ SELECT DISTINCT 'https://ci.chromium.org/b/' || bbid || ' '@@@", |
| "@@@STEP_LOG_LINE@query@ FROM UNNEST(bbids) AS bbid@@@", |
| "@@@STEP_LOG_LINE@query@ ) AS build_links,@@@", |
| "@@@STEP_LOG_LINE@query@FROM all_details AS ad@@@", |
| "@@@STEP_LOG_LINE@query@JOIN patchset_info AS pi@@@", |
| "@@@STEP_LOG_LINE@query@ ON ad.change = pi.change@@@", |
| "@@@STEP_LOG_LINE@query@WHERE@@@", |
| "@@@STEP_LOG_LINE@query@ start_time >= DATETIME '2012-05-13 12:53:21'@@@", |
| "@@@STEP_LOG_LINE@query@ AND start_time <= DATETIME '2012-05-14 12:53:23'@@@", |
| "@@@STEP_LOG_LINE@query@ AND project != \"fuchsia\";@@@", |
| "@@@STEP_LOG_END@query@@@", |
| "@@@SET_BUILD_PROPERTY@query_end_time@\"2012-05-14 12:53:23\"@@@", |
| "@@@SET_BUILD_PROPERTY@query_start_time@\"2012-05-13 12:53:21\"@@@" |
| ] |
| }, |
| { |
| "cmd": [ |
| "vpython", |
| "-u", |
| "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", |
| "--json-output", |
| "/path/to/tmp/json", |
| "copy", |
| "[START_DIR]/sapling.json", |
| "/path/to/tmp/" |
| ], |
| "infra_step": true, |
| "name": "read rows", |
| "~followup_annotations": [ |
| "@@@STEP_LOG_LINE@sapling.json@null@@@", |
| "@@@STEP_LOG_END@sapling.json@@@" |
| ] |
| }, |
| { |
| "name": "$result" |
| } |
| ] |
