src/util/perf/u_trace_compare.py

#!/usr/bin/python3
#
# Copyright © 2024 Igalia S.L.
# SPDX-License-Identifier: MIT

# Usage:
#   u_trace_compare.py compare \
#    --results /path/to/results/ \
#    --loops-merged true \
#    --alias-a default \
#    --alias-b new-shiny-opt \
#    --event-start start_render_pass \
#    --event-end end_render_pass \
#    --filter "int(params['drawCount']) > 10"
#
# Note set --loops-merged when u_trace_gather.py was run with --loops 1
# and frame looping was done in the target application.
#
# To find where in the frame the event takes place, it could be searched
# by its index in the output of:
#   u_trace_compare.py details \
#    --results /path/to/results/ \
#    --trace-name test.rdc \
#    --alias default \
#    --event-start start_render_pass \
#    --event-end end_render_pass
#


import argparse
from collections import namedtuple
from dataclasses import dataclass
import os
import statistics
import csv

@dataclass
class GPUEventAggregate:
    idx: int
    measurements: list[int]
    workload_name: str
    event_name: str
    description: dict

    def __str__(self):
        return (
            f"[{self.workload_name}] {self.event_name} [{self.idx}] \n"
            f"\tmeasurement_0: {self.measurements[0]}\tdesc: {self.description}\n"
        )


@dataclass
class GPUEventDiff:
    event_idx: int

    mean_diff: int
    mean_diff_pct: float

    ks_statistic: float
    pvalue: float
    result_significant: bool

    workload_name: str
    event_name: str
    description_a: dict
    description_b: dict
    description_diff_a: dict
    description_diff_b: dict

    alias_a: str
    alias_b: str

    measurements_a: list[int]
    measurements_b: list[int]

    def __post_init__(self):
        for key in list(self.description_a.keys()):
            if self.description_a[key] != self.description_b[key]:
                self.description_diff_a[key] = self.description_a[key]
                self.description_diff_b[key] = self.description_b[key]
                # del self.description_a[key]

    def __str__(self):
        result = (
            f"[{self.workload_name}] {self.event_name} [{self.event_idx}] "
            f"{'Δ' if self.mean_diff_pct < 0 else '∇'} {self.mean_diff_pct:.1f}% "
            f"ks: {self.ks_statistic} pval: {self.pvalue:.4f}\n"
            f"\tmeasurements_{self.alias_a}: {*self.measurements_a,}\n"
            f"\tmeasurements_{self.alias_b}: {*self.measurements_b,}\n"
            f"\t{self.description_a}\n"
        )
        if self.description_diff_a:
            result += f"\t{self.alias_a}: {self.description_diff_a}\n"
        if self.description_diff_b:
            result += f"\t{self.alias_b}: {self.description_diff_b}\n"

        return result


def parse_with_alias(args, workload_name: str, alias: str) -> list[GPUEventAggregate]:
    ParsedEvent = namedtuple('ParsedEvent', ['timestamp', 'name', 'params'])
    events_aggregate: list[GPUEventAggregate] = []
    loop_idx = 0
    while True:
        frames = [[]]
        last_frame = 0

        file_name = f"{args.results}/{workload_name}/trace_{workload_name}_{alias}_{loop_idx}.csv"
        try:
            with open(file_name, 'r') as input_csv:
                reader = csv.reader(input_csv, delimiter=',', skipinitialspace=True)
                for row in reader:
                    if len(row) < 4 or not row[0].isdigit():
                        continue
                    frame = int(row[0])
                    if frame != last_frame:
                        frames.append([])
                        last_frame = frame
                    timestamp = row[2]
                    event_name = row[3]
                    params = {}
                    for param in row[4:]:
                        try:
                            key, value = param.split('=', 1)
                        except ValueError:
                            break
                        params[key] = value

                    frames[-1].append(ParsedEvent(timestamp, event_name, params))

        except FileNotFoundError:
            if loop_idx == 0:
                print(f"\tZero results found for workload '{workload_name}' and alias '{alias}' (\"{file_name}\")")
            break

        if args.loops_merged:
            del frames[0]
            # The last frame could be partially lost
            del frames[-1]

        event_idx = 0
        for target_frame in reversed(frames):
            if args.loops_merged:
                # We compare frames from a single run
                event_idx = 0

            start_time_ns = 0
            new = False
            for event in target_frame:
                if event.name == args.event_start:
                    # Events should be created only once
                    if event_idx == len(events_aggregate):
                        agg = GPUEventAggregate(event_idx, [], workload_name, event.name, event.params)
                        events_aggregate.append(agg)
                        new = True
                    start_time_ns = int(event.timestamp)
                if event.name == args.event_end:
                    agg = events_aggregate[event_idx]
                    if new:
                        agg.description = agg.description | event.params
                    new = False

                    duration = int(event.timestamp) - start_time_ns
                    agg.measurements.append(duration)

                    event_idx += 1

        if args.loops_merged:
            break

        loop_idx += 1

    assert (len(events_aggregate) != 0)

    if args.filter_func:
        events_aggregate = list(filter(lambda e: args.filter_func(e.description), events_aggregate))

    return events_aggregate


def parse_all_with_alias(args, alias: str) -> list[GPUEventAggregate]:
    events_aggregate: list[GPUEventAggregate] = []

    for dir_name in os.listdir(args.results):
        f = os.path.join(args.results, dir_name)
        if os.path.isdir(f):
            try:
                events_aggregate.extend(parse_with_alias(args, dir_name, alias))
            except Exception as ex:
                print(f"Invalid csv for '{dir_name}'/'{alias}'")
                raise ex

    return events_aggregate


def print_diffs_plain(args, diffs: list[GPUEventDiff]) -> None:
    diffs.sort(key=lambda x: x.mean_diff_pct)

    # TODO: configurable threshold?
    helped = list(filter(lambda d: d.result_significant and d.mean_diff_pct < -0.5, diffs))
    hurt = list(filter(lambda d: d.result_significant and d.mean_diff_pct > 0.5, diffs))

    helped_total_a = sum(statistics.mean(d.measurements_a) for d in helped)
    helped_total_b = sum(statistics.mean(d.measurements_b) for d in helped)
    if helped_total_a > 0:
        win_in_helped_pct = (helped_total_b - helped_total_a) / helped_total_a * 100.0
    else:
        win_in_helped_pct = 0

    hurt_total_a = sum(statistics.mean(d.measurements_a) for d in hurt)
    hurt_total_b = sum(statistics.mean(d.measurements_b) for d in hurt)
    if hurt_total_a > 0:
        loss_in_hurt_pct = (hurt_total_b - hurt_total_a) / hurt_total_a * 100.0
    else:
        loss_in_hurt_pct = 0

    total_a = sum(statistics.mean(d.measurements_a) for d in diffs)
    total_b = sum(statistics.mean(d.measurements_b) for d in diffs)
    total_win_pct = (total_b - total_a) / total_a * 100.0

    print("ALL:")
    for diff in diffs:
        print(diff)

    print()
    print("HELPED:")
    for diff in helped:
        print(diff)

    print()
    print("HURT:")
    for diff in hurt:
        print(diff)

    print(f"TOTAL: {len(diffs)} {'Δ' if total_win_pct < 0 else '∇'} {total_win_pct:.5f}%")
    print(
        f"TOTAL HELPED: {len(helped)} Δ {win_in_helped_pct:.1f}% (where '{args.alias_b}' is faster than '{args.alias_a}')")
    print(f"TOTAL HURT: {len(hurt)} ∇ {loss_in_hurt_pct:.1f}% (where '{args.alias_b}' is slower than '{args.alias_a}')")
    print(f"FILTER: {args.filter}")


def print_diffs_csv(args, diffs: list[GPUEventDiff]) -> None:
    description_a_keys = []
    description_diff_keys = []
    for diff in diffs:
        for key in diff.description_a:
            if key not in description_a_keys:
                description_a_keys.append(key)
        for key in diff.description_diff_a:
            if key not in description_diff_keys:
                description_diff_keys.append(key)

    with open(args.csv, 'w', newline='') as csvfile:
        field_names = ['workload', 'event_idx', 'event_name', f'mean_{args.alias_a}', f'mean_{args.alias_b}',
                       'mean_diff', 'ks_statistic', 'pvalue']
        field_names.extend(description_a_keys)
        for key in description_diff_keys:
            field_names.append(f"{key}_{args.alias_a}")
            field_names.append(f"{key}_{args.alias_b}")

        csv_writer = csv.writer(csvfile, delimiter='\t', dialect='unix')
        csv_writer.writerow(field_names)
        for diff in diffs:
            mean_a = int(statistics.mean(diff.measurements_a))
            mean_b = int(statistics.mean(diff.measurements_b))
            row = [diff.workload_name, diff.event_idx, diff.event_name, mean_a, mean_b,
                   int(diff.mean_diff), diff.ks_statistic, round(diff.pvalue, 4)]
            # Add description_a values
            for key in description_a_keys:
                row.append(diff.description_a.get(key, ''))

            # Add description_diff values
            for key in description_diff_keys:
                row.append(diff.description_diff_a.get(key, ''))
                row.append(diff.description_diff_b.get(key, ''))

            csv_writer.writerow(row)


def compare(args) -> None:
    from scipy.stats import ks_2samp

    events_a = parse_all_with_alias(args, args.alias_a)
    events_b = parse_all_with_alias(args, args.alias_b)

    assert (len(events_a) == len(events_b))

    diffs: list[GPUEventDiff] = []
    for event_a, event_b in zip(events_a, events_b):
        # GPU cannot randomly become much faster, only slower, so as
        # a trivial way to combat uncertainty - remove only the longest measurement.
        # DISABLED for now since it makes more difficult to find renderpass
        # event_a.measurements.remove(max(event_a.measurements))
        # event_b.measurements.remove(max(event_b.measurements))

        mean_a = statistics.mean(event_a.measurements)
        mean_b = statistics.mean(event_b.measurements)

        # We don't expect enough measurements to apply statistical methods
        # suitable for normal distributions.
        # Kolmogorov–Smirnov test gives us a simple metric to understand
        # whether measurements belong to different distributions,
        # if yes - we could meaningfully compare them.
        ks_stats = ks_2samp(event_a.measurements, event_b.measurements)

        diff = GPUEventDiff(
            event_idx=event_a.idx,
            mean_diff=mean_b - mean_a,
            mean_diff_pct=(mean_b - mean_a) / mean_a * 100.0,
            workload_name=event_a.workload_name,
            event_name=event_a.event_name,
            description_a=event_a.description,
            description_b=event_b.description,
            description_diff_a=dict(),
            description_diff_b=dict(),
            alias_a=args.alias_a,
            alias_b=args.alias_b,
            ks_statistic=ks_stats.statistic,
            pvalue=ks_stats.pvalue,
            # TODO: This threshold is a random guess, is it good enough?
            result_significant=ks_stats.pvalue < 0.10 and ks_stats.statistic > 0.8,
            measurements_a=event_a.measurements,
            measurements_b=event_b.measurements,
        )

        diffs.append(diff)

    if args.csv:
        print_diffs_csv(args, diffs)
    else:
        print_diffs_plain(args, diffs)


def details(args) -> None:
    args.loops_merged = True
    events = parse_with_alias(args, args.trace_name, args.alias)

    for event in events:
        print(event)


def main() -> None:
    parser = argparse.ArgumentParser()
    sub = parser.add_subparsers()

    compare_args = sub.add_parser('compare', help='Compare two results.')
    compare_args.add_argument('--results', type=str, required=True, help="Folder with results.")
    compare_args.add_argument('--loops-merged', type=bool, required=True,
                              help="true if single trace contains multiple loops for analysis.")
    compare_args.add_argument('--alias-a', type=str, required=True, help="")
    compare_args.add_argument('--alias-b', type=str, required=True, help="")
    compare_args.add_argument('--event-start', type=str, required=True, help="E.g. start_render_pass")
    compare_args.add_argument('--event-end', type=str, required=True, help="E.g. end_render_pass")
    compare_args.add_argument('--csv', type=str, required=False, help="CSV file to write the output.")
    compare_args.add_argument('--filter', type=str, required=False, help="Lambda function for filtering events.")
    compare_args.set_defaults(func=compare)

    info_args = sub.add_parser('details', help='Get info about tracepoints in a specific results log.')
    info_args.add_argument('--results', type=str, required=True, help="Folder with results.")
    info_args.add_argument('--trace-name', type=str, required=True, help="Name of the trace, e.g. test.json")
    info_args.add_argument('--alias', type=str, required=True, help="")
    info_args.add_argument('--event-start', type=str, required=True, help="E.g. start_render_pass")
    info_args.add_argument('--event-end', type=str, required=True, help="E.g. end_render_pass")
    info_args.set_defaults(func=details)

    args = parser.parse_args()

    args.filter_func = None
    if hasattr(args, 'filter') and args.filter:
        args.filter_func = eval(f"lambda params: {args.filter}")

    args.func(args)


if __name__ == "__main__":
    main()