scripts/cq_perf_results.py

#!/usr/bin/env python3.8
# Copyright 2019 The Fuchsia Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""cq_perf_results.py prints the performance testing results of a CL.

From a change ID (the last number in
https://fuchsia-review.googlesource.com/c/fuchsia/+/255116), cq_perf_results.py
retrieves the last CQ run and compares the its performance test results with
the performance test result of its parent, as found in the CI queue.
"""

import argparse
import json
import numpy
import operator
import os
import scipy.stats
import urllib.request

# URL template for listing all tests on a given build.
_TEST_LIST_TMPL = 'https://logs.chromium.org/logs/fuchsia/buildbucket/cr-buildbucket.appspot.com/%s/+/steps/all_test_results/0/logs/summary.json/0?format=raw'

# URL template for retrieving the output of a passing test on a given build.
_TEST_OUTPUT = 'https://logs.chromium.org/logs/fuchsia/buildbucket/cr-buildbucket.appspot.com/%s/+/steps/all_test_results/0/steps/all_passed_tests/0/steps/%s/0/logs/stdio/0?format=raw'

# URL template to retrieve data about a change.
_CHANGE_URL_TMPL = 'https://fuchsia-review.googlesource.com/changes/%s?o=CURRENT_REVISION&o=CURRENT_COMMIT'

# URL template to retrieve the CQ builds for a given change patchset.
_BUILD_CQ_URL_TMPL = 'https://cr-buildbucket.appspot.com/_ah/api/buildbucket/v1/search?max_builds=500&fields=builds%%28bucket%%2Cfailure_reason%%2Cid%%2Cparameters_json%%2Cresult%%2Cstatus%%2Ctags%%2Curl%%29&tag=buildset%%3Apatch%%2Fgerrit%%2Ffuchsia-review.googlesource.com%%2F%s%%2F%d'

# URL template to retrieve the CI builds for a given committed change.
_BUILD_CI_URL_TMPL = 'https://cr-buildbucket.appspot.com/_ah/api/buildbucket/v1/search?max_builds=500&fields=builds%%28bucket%%2Cfailure_reason%%2Cid%%2Cparameters_json%%2Cresult%%2Cstatus%%2Ctags%%2Curl%%29&tag=buildset%%3Acommit%%2Fgit%%2F%s'

# Default botname for performance tests.
_BOTNAME = 'peridot-x64-perf-dawson_canyon'

# Get the name of all non-catapult-upload tests on a given build.
def _get_test_names(build_id):
  test_list_url = _TEST_LIST_TMPL % build_id
  test_list_request = urllib.request.urlopen(test_list_url)
  test_list = json.loads(test_list_request.read().decode('utf-8'))
  test_names = tuple(entry['name'] for entry in test_list['tests']
    if not entry['name'].endswith('.catapult_json'))
  return test_names

# Get a specific test result from a build.
def _get_perf_test_results(build_id, test_name):
  test_result_url = _TEST_OUTPUT % (build_id, test_name)
  test_result_request = urllib.request.urlopen(test_result_url)
  test_result = json.loads(test_result_request.read().decode('utf-8'))
  results = []
  for test in test_result:
    name = '%s/%s' % (test['test_suite'], test['label'])
    values = test['values']
    results.append((name, values))
  return results

# Get all test results for a given build ID.
def _get_results_for_build(build_id):
  test_names = _get_test_names(build_id)
  results = {}
  for test_name in test_names:
    result = _get_perf_test_results(build_id, test_name)
    for label, value in result:
      results[label] = value
  return results

# Get the latest build ID for a given gerrit change ID.
def _get_build_from_review(change_id, botname):
  change_url = _CHANGE_URL_TMPL % change_id
  change_request = urllib.request.urlopen(change_url)
  change = json.loads(change_request.read().decode('utf-8')[5:])
  if change['status'] == 'MERGED':
    commit_id = change['current_revision']
    build = _get_ci_build(commit_id, botname)
  else:
    patchset = change['revisions'][change['current_revision']]['_number']
    build = _get_cq_build(change_id, patchset, botname)
  parent = change['revisions'][change['current_revision']]['commit']['parents'][0]['commit']

  return build, parent,

# Get the build ID on the CQ for a given gerrit change ID and patchset.
def _get_cq_build(change_id, patchset, botname):
  build_url = _BUILD_CQ_URL_TMPL % (change_id, patchset)
  build_request = urllib.request.urlopen(build_url).read().decode('utf-8')
  builds = json.loads(build_request)
  target_tag = 'builder:' + botname
  for build in builds['builds']:
    if target_tag in build['tags']:
      return build['id']
  raise KeyError("Unable to find the target builder")

# Get the build ID on the CI for a given commit ID.
def _get_ci_build(commit_id, botname):
  build_url = _BUILD_CI_URL_TMPL % commit_id
  build_request = urllib.request.urlopen(build_url)
  builds = json.loads(build_request.read().decode('utf-8'))
  target_tag = 'builder:' + botname
  for build in builds['builds']:
    if target_tag in build['tags']:
      return build['id']
  raise KeyError("Unable to find the target builder")

def _compute_output_format_strings(target_build):
  # We want to align test names and results, so we compute the maximum size
  # needed from the test name length.
  max_test_name_length = max(len(test_name) for test_name in target_build)
  both_tests_format_string = '{:' + str(max_test_name_length) + \
    's}: {:8.4f} -> {:8.4f}  {:6.2f} % variation, {:5.3f} p-value'
  single_test_format_string = '{:' + str(max_test_name_length) + \
        's}: {:8.4f} (no corresponding test in base commit)'
  no_data_format_string = '{:' + str(max_test_name_length) + \
    's}: {:8.4f} -> {:8.4f}  {:6.2f} % variation, not enough data'
  return (both_tests_format_string, single_test_format_string,
    no_data_format_string, )

def main():
  description="""A tool to detect performance test changes on changes.

From a change ID (series of multiple digits, and last part of the gerrit URL:
https://fuchsia-review.googlesource.com/c/fuchsia/+/CHANGE_ID), this tool
retrieves the last CQ run and compares the its performance test results with
the performance test result of its parent, as found in the CI queue.

You will want to use the --base_build argument if you are doing chained
changes. This script assumes the usual gerrit workflow where conflicting
changes are rebased instead of merged, so each commit has at most one parent."""
  epilog = """Example:
$> ./cq_perf_results.py --botname peridot-x64-perf-dawson_canyon 255116
  """
  argument_parser = argparse.ArgumentParser(description=description,
    epilog=epilog, formatter_class=argparse.RawDescriptionHelpFormatter)
  argument_parser.add_argument('change_id', default=None,
    help="Change ID from Gerrit")
  argument_parser.add_argument('--botname', default=_BOTNAME,
    help="Name of the bot running the performance tests. Default: " + _BOTNAME)
  argument_parser.add_argument('--base_build', default=None,
    help="Base build to use (default: use the CI build of the base commit)")
  args = argument_parser.parse_args()

  # We first get the build ID for the base change, then get its perf test
  # results.
  target_build_id, parent_id = _get_build_from_review(args.change_id,
    args.botname)
  print('Target build id', target_build_id, 'parent id', parent_id)
  target_build = _get_results_for_build(target_build_id)

  # Get the base build ID and get its perf test results.
  if not args.base_build:
    base_build_id = _get_ci_build(parent_id, args.botname)
  else:
    base_build_id = args.base_build
  print('Base build id', base_build_id)
  base_build = _get_results_for_build(base_build_id)

  both_tests_format_string, single_test_format_string, no_data_format_string = \
      _compute_output_format_strings(target_build)
  for test_name, value in sorted(target_build.items(),
      key=operator.itemgetter(0)):
    if test_name not in base_build:
      print(single_test_format_string.format(
          test_name, numpy.mean(value)))
      continue
    if len(value) == 1:
      print(no_data_format_string.format(test_name,
        base_build[test_name][0],
        value[0],
        (value[0]-base_build[test_name][0])*100.0/base_build[test_name][0]))
      continue
    base_mean = numpy.mean(base_build[test_name][1:])
    target_mean = numpy.mean(value[1:])
    # We use a 2-sample Kolmogorov-Smirnov statistic, for the following reasons:
    # - It is non-parametric;
    # - It does not assume normality of samples.
    _, pvalue = scipy.stats.ks_2samp(base_build[test_name][1:], value[1:])
    print(both_tests_format_string.format(test_name,
      base_mean,
      target_mean,
      (target_mean-base_mean)*100.0/base_mean,
      pvalue))

if __name__ == '__main__':
  main()