tools/uiperf/trace-cmp.py

#!/usr/bin/env python3
#
# Copyright 2018 The Fuchsia Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""
"""

import argparse
import collections
import json
import sys

try:
  import matplotlib.patches
  import matplotlib.pyplot as plt
  import numpy as np
  import pandas as pd
  import scipy as sp
  import scipy.stats
  import seaborn as sns
  plt.style.use('ggplot')
except ImportError as e:
  print("""\
Looks like you didn't have one of the following libraries installed:
  * matplotlib
  * numpy
  * pandas
  * scipy
  * seaborn
Try installing them with:
  $ sudo apt install python3-matplotlib python3-numpy python3-pandas python3-scipy python3-seaborn""")
  raise e

parser = argparse.ArgumentParser(
    description="""\
Compare the distribution of event durations between two (groups of) trace files.

Example usage:
  ./topaz/tools/uiperf/trace-cmp.py \\
    --event_name 'vsync callback' \\
    --thread_names 'assistant_card_image_grid.ui' \\
    --before trace-2018-10-09T21:00:25.json trace-2018-10-09T21:08:14.json \\
    --after trace-2018-10-09T21:12:14.json trace-2018-10-09T22:03:04.json
""",
    formatter_class=argparse.RawDescriptionHelpFormatter)

parser.add_argument(
    '--event_name',
    default='vsync callback',
    nargs='?',
    help=
    'What event name to look for in each trace.  By default, this is '
    '\'vsync callback\', which is the total per frame workload on the Flutter '
    'UI thread.'
)
parser.add_argument(
    '--thread_names',
    default=[],
    nargs='*',
    help=
    'What thread names to search for |event_name| in.  Example: "dashboard.ui".'
)
parser.add_argument(
    '--before',
    default=[],
    nargs='+',
    help=
    'A list of before (a build without your changes) json trace files.  Example: "trace-2018-10-09T22:18:26.json".'
)
parser.add_argument(
    '--after',
    default=[],
    nargs='+',
    help=
    'A list of after (a build with your changes) json trace files.  Example: "trace-2018-10-09T22:25:01.json".'
)
# TODO: Add cdf as a graph option.
valid_graph_args = {'none', 'density'}
parser.add_argument(
    '--graph',
    default='none',
    nargs='?',
    help='Optionally display a density plot of the distribution of event durations.  Must be one of {}'.format(valid_graph_args)
)

args = parser.parse_args()

target_event_name = args.event_name
target_thread_names = set(args.thread_names)
before_filenames = args.before
after_filenames = args.after

if len(before_filenames) == 0:
  print('List of before tracefiles must be non-empty.')
  parser.print_usage()
  sys.exit(1)
if len(after_filenames) == 0:
  parser.print_usage()
  print('List of after tracefiles must be non-empty.')
  sys.exit(1)

if args.graph not in valid_graph_args:
  parser.print_usage()
  print('Invalid --graph argument, must be one of {}'.format(valid_graph_args))
  sys.exit(1)

# {
#   pthread:0x1234: {'Before': [e0, e1, ...], 'After': [e0, e1, ...]},
#   pthread:0x1235: {'Before': [e0, e1, ...], 'After': [e0, e1, ...]},
# }
thread_name_to_groups = collections.defaultdict(
    lambda: collections.defaultdict(list))

for group, filenames in [('Before', before_filenames), ('After',
                                                        after_filenames)]:
  for filename in filenames:
    with open(filename) as f:
      root_object = json.load(f)

    system_trace_events = root_object['systemTraceEvents']
    trace_events = root_object['traceEvents']

    # A mapping of thread ids to thread names.
    tid_to_name = {}
    for e in system_trace_events['events']:
      if e['ph'] == 't':
        tid_to_name[e['tid']] = e['name']

    # Group (events sorted by ts) by tid.
    by_tid = collections.defaultdict(list)
    for e in trace_events:
      by_tid[e['tid']].append(e)
    for k, v in by_tid.items():
      by_tid[k] = sorted(v, key=lambda e: e['ts'])

    for tid, es in by_tid.items():
      thread_name = tid_to_name[tid]
      if len(
          target_thread_names) > 0 and thread_name not in target_thread_names:
        continue

      events = [e for e in es if e['name'] == target_event_name]
      event_durations = []
      begin_stack = []
      # If the first event's phase is end, then remove it.
      if len(events) > 0 and events[0]['ph'] == 'E':
        events.pop(0)
      for e in events:
        if e['ph'] == 'B':
          begin_stack.append(e)
        elif e['ph'] == 'E':
          popped = begin_stack.pop()
          event_durations.append(e['ts'] - popped['ts'])
        else:
          assert False
      assert len(begin_stack) == 0

      if len(event_durations) == 0:
        continue

      thread_name_to_groups[thread_name][group].append(event_durations)

for thread_name, groups in thread_name_to_groups.items():
  if ('Before' in groups) ^ ('After' in groups):
    print('Found thread name "{}" only in {} trace'.format(
        thread_name, ['Before', 'After']['After' in groups]))
  df = pd.DataFrame(columns=[
      'Group',
      'Count',
      # https://en.wikipedia.org/wiki/Sample_maximum_and_minimum
      'Minimum',
      # https://en.wikipedia.org/wiki/Percentile
      '25th Percentile',
      # https://en.wikipedia.org/wiki/Median
      'Median',
      # https://en.wikipedia.org/wiki/Sample_mean_and_covariance
      'Mean',
      '75th Percentile',
      '90th Percentile',
      '95th Percentile',
      '99th Percentile',
      'Maximum',
      # https://en.wikipedia.org/wiki/Standard_deviation#Sample_standard_deviation
      'Standard Deviation',
      # https://en.wikipedia.org/wiki/Median_absolute_deviation
      'Median Absolute Deviation',
      # https://en.wikipedia.org/wiki/Outlier#Tukey's_fences
      'Tukey Outlier Count',
  ])
  yss = []
  for key in ['Before', 'After']:
    group = groups[key]
    ys = []
    for item in group:
      ys += item
    ys = np.array(ys)
    yss.append(ys)
    color = {
        'Before': 'blue',
        'After': 'orange',
    }[key]
    sns.distplot(ys, rug=True, color=color, label=key, bins=20)

    row = {}
    row['Group'] = key
    row['Count'] = ys.shape[0]
    row['Minimum'] = ys.min()
    row['25th Percentile'] = np.percentile(ys, 25)
    row['Median'] = np.percentile(ys, 50)
    row['Mean'] = ys.mean()
    row['75th Percentile'] = np.percentile(ys, 75)
    row['90th Percentile'] = np.percentile(ys, 90)
    row['95th Percentile'] = np.percentile(ys, 95)
    row['99th Percentile'] = np.percentile(ys, 99)
    row['Maximum'] = ys.max()
    row['Standard Deviation'] = ys.std()
    median = np.median(ys)
    row['Median Absolute Deviation'] = np.median(
        [np.abs(y - median) for y in ys])
    q1 = np.percentile(ys, 25)
    q3 = np.percentile(ys, 75)
    iqr = q3 - q1
    l = q1 - 1.5 * iqr
    u = q3 + 1.5 * iqr
    outliers = [y for y in ys if not (l <= y <= u)]
    row['Tukey Outlier Count'] = len(outliers)

    df = df.append(row, ignore_index=True)
  print('Results for {}:'.format(thread_name))
  print('Units: Microseconds')
  print(df.to_string(index=False, float_format='%.2f', justify='center'))

  # https://en.wikipedia.org/wiki/Mann%E2%80%93Whitney_U_test
  u_statistic, p_value = sp.stats.mannwhitneyu(
      yss[0], yss[1], alternative='two-sided')
  cutoff = 0.05
  report_result = 'DIFFERED' if p_value < cutoff else 'DID NOT DIFFER'
  print('Mann–Whitney U test:')
  print(
      'The distributions of the two groups {} significantly (Mann-Whitney U={:.2f}, P={:.2f}, cutoff={:.2f}).'
      .format(report_result, u_statistic, p_value, cutoff))

  if args.graph == 'density':
    plt.title('{}: Before/After {} Durations'.format(thread_name,
                                                     target_event_name))
    plt.xlabel('Microseconds')
    plt.ylabel('Density')
    before_patch = matplotlib.patches.Patch(color='blue', label='Before')
    after_patch = matplotlib.patches.Patch(color='orange', label='After')
    plt.legend(handles=[before_patch, after_patch])
    plt.show()

  print('')
  print('===')
  print('')