| #!/usr/bin/env python3 |
| # Copyright 2019 The Fuchsia Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| """cq_perf_results.py prints the performance testing results of a CL. |
| |
| From a change ID (the last number in |
| https://fuchsia-review.googlesource.com/c/fuchsia/+/255116), cq_perf_results.py |
| retrieves the last CQ run and compares the its performance test results with |
| the performance test result of its parent, as found in the CI queue. |
| |
| This script probably does not work with chained changes. It assumes the usual gerrit workflow where conflicting changes are rebased instead of merged, so each commit has at most one parent. |
| """ |
| |
| import argparse |
| import json |
| import numpy |
| import operator |
| import os |
| import scipy.stats |
| import urllib.request |
| |
| # URL template for listing all tests on a given build. |
| _TEST_LIST_TMPL = 'https://logs.chromium.org/logs/fuchsia/buildbucket/cr-buildbucket.appspot.com/%s/+/steps/all_test_results/0/logs/summary.json/0?format=raw' |
| |
| # URL template for retrieving the output of a passing test on a given build. |
| _TEST_OUTPUT = 'https://logs.chromium.org/logs/fuchsia/buildbucket/cr-buildbucket.appspot.com/%s/+/steps/all_test_results/0/steps/all_passed_tests/0/steps/%s/0/logs/stdio/0?format=raw' |
| |
| # URL template to retrieve data about a change. |
| _CHANGE_URL_TMPL = 'https://fuchsia-review.googlesource.com/changes/%s?o=CURRENT_REVISION&o=CURRENT_COMMIT' |
| |
| # URL template to retrieve the CQ builds for a given change patchset. |
| _BUILD_CQ_URL_TMPL = 'https://cr-buildbucket.appspot.com/_ah/api/buildbucket/v1/search?max_builds=500&fields=builds%%28bucket%%2Cfailure_reason%%2Cid%%2Cparameters_json%%2Cresult%%2Cstatus%%2Ctags%%2Curl%%29&tag=buildset%%3Apatch%%2Fgerrit%%2Ffuchsia-review.googlesource.com%%2F%s%%2F%d' |
| |
| # URL template to retrieve the CI builds for a given committed change. |
| _BUILD_CI_URL_TMPL = 'https://cr-buildbucket.appspot.com/_ah/api/buildbucket/v1/search?max_builds=500&fields=builds%%28bucket%%2Cfailure_reason%%2Cid%%2Cparameters_json%%2Cresult%%2Cstatus%%2Ctags%%2Curl%%29&tag=buildset%%3Acommit%%2Fgit%%2F%s' |
| |
| # Default botname for performance tests. |
| _BOTNAME = 'peridot-x64-perf-dawson_canyon' |
| |
| # Get the name of all non-catapult-upload tests on a given build. |
| def _get_test_names(build_id): |
| test_list_url = _TEST_LIST_TMPL % build_id |
| test_list_request = urllib.request.urlopen(test_list_url) |
| test_list = json.loads(test_list_request.read().decode('utf-8')) |
| test_names = tuple(entry['name'] for entry in test_list['tests'] |
| if not entry['name'].endswith('.catapult_json')) |
| return test_names |
| |
| # Get a specific test result from a build. |
| def _get_perf_test_results(build_id, test_name): |
| test_result_url = _TEST_OUTPUT % (build_id, test_name) |
| test_result_request = urllib.request.urlopen(test_result_url) |
| test_result = json.loads(test_result_request.read().decode('utf-8')) |
| results = [] |
| for test in test_result: |
| name = '%s/%s' % (test['test_suite'], test['label']) |
| values = test['values'] |
| if test['split_first']: |
| values = values[1:] |
| results.append((name, values)) |
| return results |
| |
| # Get all test results for a given build ID. |
| def _get_results_for_build(build_id): |
| test_names = _get_test_names(build_id) |
| results = {} |
| for test_name in test_names: |
| result = _get_perf_test_results(build_id, test_name) |
| for label, value in result: |
| results[label] = value |
| return results |
| |
| # Get the latest build ID for a given gerrit change ID. |
| def _get_build_from_review(change_id, botname): |
| change_url = _CHANGE_URL_TMPL % change_id |
| change_request = urllib.request.urlopen(change_url) |
| change = json.loads(change_request.read().decode('utf-8')[5:]) |
| if change['status'] == 'MERGED': |
| commit_id = change['current_revision'] |
| build = _get_ci_build(commit_id, botname) |
| else: |
| patchset = change['revisions'][change['current_revision']]['_number'] |
| build = _get_cq_build(change_id, patchset, botname) |
| parent = change['revisions'][change['current_revision']]['commit']['parents'][0]['commit'] |
| |
| return build, parent, |
| |
| # Get the build ID on the CQ for a given gerrit change ID and patchset. |
| def _get_cq_build(change_id, patchset, botname): |
| build_url = _BUILD_CQ_URL_TMPL % (change_id, patchset) |
| build_request = urllib.request.urlopen(build_url).read().decode('utf-8') |
| builds = json.loads(build_request) |
| target_tag = 'builder:' + botname |
| for build in builds['builds']: |
| if target_tag in build['tags']: |
| return build['id'] |
| raise KeyError("Unable to find the target builder") |
| |
| # Get the build ID on the CI for a given commit ID. |
| def _get_ci_build(commit_id, botname): |
| build_url = _BUILD_CI_URL_TMPL % commit_id |
| build_request = urllib.request.urlopen(build_url) |
| builds = json.loads(build_request.read().decode('utf-8')) |
| target_tag = 'builder:' + botname |
| for build in builds['builds']: |
| if target_tag in build['tags']: |
| return build['id'] |
| raise KeyError("Unable to find the target builder") |
| |
| def _compute_output_format_string(target_build): |
| # We want to align test names and results, so we compute the maximum size |
| # needed from the test name length. |
| max_test_name_length = max(len(test_name) for test_name in target_build) |
| both_tests_format_string = '{:' + str(max_test_name_length) + \ |
| 's}: {:8.4f} -> {:8.4f} {:6.2f} % variation, {:5.3f} p-value' |
| single_test_format_string = '{:' + str(max_test_name_length) + \ |
| 's}: {:8.4f} (no corresponding test in base commit)' |
| return both_tests_format_string, single_test_format_string, |
| |
| def main(): |
| description="""A tool to detect performance test changes on changes. |
| |
| From a change ID (series of multiple digits, and last part of the gerrit URL: |
| https://fuchsia-review.googlesource.com/c/fuchsia/+/CHANGE_ID), this tool |
| retrieves the last CQ run and compares the its performance test results with |
| the performance test result of its parent, as found in the CI queue. |
| |
| This script probably does not work with chained changes. It assumes the usual |
| gerrit workflow where conflicting changes are rebased instead of merged, so |
| each commit has at most one parent.""" |
| epilog = """Example: |
| $> ./cq_perf_results.py --botname peridot-x64-perf-dawson_canyon 255116 |
| """ |
| argument_parser = argparse.ArgumentParser(description=description, |
| epilog=epilog, formatter_class=argparse.RawDescriptionHelpFormatter) |
| argument_parser.add_argument('change_id', default=None, |
| help="Change ID from Gerrit") |
| argument_parser.add_argument('--botname', default=_BOTNAME, |
| help="Name of the bot running the performance tests. Default: " + _BOTNAME) |
| args = argument_parser.parse_args() |
| |
| # We first get the build ID for the base change, then get its perf test |
| # results. |
| target_build_id, parent_id = _get_build_from_review(args.change_id, |
| args.botname) |
| print('Target build id', target_build_id, 'parent id', parent_id) |
| target_build = _get_results_for_build(target_build_id) |
| |
| # Get the base build ID and get its perf test results. |
| base_build_id = _get_ci_build(parent_id, args.botname) |
| print('Base build id', base_build_id) |
| base_build = _get_results_for_build(base_build_id) |
| |
| both_tests_format_string, single_test_format_string = \ |
| _compute_output_format_string(target_build) |
| for test_name, value in sorted(target_build.items(), |
| key=operator.itemgetter(0)): |
| if test_name not in base_build: |
| print(single_test_format_string.format( |
| test_name, numpy.mean(value))) |
| continue |
| base_mean = numpy.mean(base_build[test_name]) |
| target_mean = numpy.mean(value) |
| _, pvalue = scipy.stats.ttest_ind(base_build[test_name], value) |
| print(both_tests_format_string.format(test_name, |
| base_mean, |
| target_mean, |
| (target_mean-base_mean)*100.0/base_mean, |
| pvalue)) |
| |
| if __name__ == '__main__': |
| main() |