blob: 78f2384c74f4d55db6ddcc371e8e7e1264574274 [file]
#!/usr/bin/env fuchsia-vendored-python
# Copyright 2023 The Fuchsia Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
import bisect
import collections
import dataclasses
import itertools
import logging
import sys
import typing
import weakref
from collections.abc import Collection, Mapping, MutableSequence, Sequence
from typing import (
NotRequired,
ReadOnly,
Self,
TypedDict,
cast,
)
from reporting import metrics
from trace_processing import trace_metrics, trace_model, trace_time, trace_utils
_LOGGER: logging.Logger = logging.getLogger(__name__)
_CPU_USAGE_EVENT_NAME = "cpu_usage"
_RATE_EVENT_NAME = "Rate"
# The kernel reports processing rates where 1000 represents 100% capacity.
# This constant is used as a default value when rate events are missing.
_DEFAULT_PROCESSING_RATE = 1000.0
_DEFAULT_PERCENT_CUTOFF = 0.0
_ONE_S_IN_NS = 1_000_000_000
_NS_PER_MS = 1_000_000.0
# NOTE(nathaniel): it would be nice to have these more clearly show
# that while the fields of one class are a superset of the fields of
# the other, the classes themselves aren't in a superclass-subclass
# relationship. We would be able to do this with:
#
# _PROCESS_BREAKDOWN_METRIC_FIELDS = frozendict({
# 'process_name': ReadOnly[str],
# 'cpu': ReadOnly[int],
# 'percent': ReadOnly[float],
# 'duration': ReadOnly[float],
# 'normalized_percent': NotRequired[float],
# 'normalized_duration': NotRequired[float],
# 'duration_per_rate': NotRequired[Mapping[str, float]],
# })
# _THREAD_BREAKDOWN_METRIC_FIELDS = _PROCESS_BREAKDOWN_METRIC_FIELDS | frozendict({
# 'thread_name': ReadOnly[str],
# 'tid': ReadOnly[int],
# })
# ProcessBreakdownMetric = TypedDict(
# "ProcessBreakdownMetric", _PROCESS_BREAKDOWN_METRIC_FIELDS, total=True, closed=True)
# ThreadBreakdownMetric = TypedDict(
# "ThreadBreakdownMetric", _THREAD_BREAKDOWN_METRIC_FIELDS, total=True, closed=True)
#
# ... but the formal specification gets in the way: "[the dictionary from keys
# to value type specifications] must be a dictionary display expression, not a
# variable or other expression that evaluates to a dictionary at runtime"
# (https://typing.python.org/en/latest/spec/typeddict.html#functional-syntax). 🙁
ProcessBreakdownMetric = TypedDict(
"ProcessBreakdownMetric",
{
"process_name": ReadOnly[str],
"cpu": ReadOnly[int],
"percent": ReadOnly[float],
"duration": ReadOnly[float],
"normalized_percent": NotRequired[float],
"normalized_duration": NotRequired[float],
"duration_per_rate": NotRequired[Mapping[str, float]],
},
total=True,
# TODO(https://fxbug.dev/569952695): used closed=True when able.
# closed=True,
)
ThreadBreakdownMetric = TypedDict(
"ThreadBreakdownMetric",
{
"process_name": ReadOnly[str],
"cpu": ReadOnly[int],
"percent": ReadOnly[float],
"duration": ReadOnly[float],
"normalized_percent": NotRequired[float],
"normalized_duration": NotRequired[float],
"duration_per_rate": NotRequired[Mapping[str, float]],
"thread_name": ReadOnly[str],
"tid": ReadOnly[int],
},
total=True,
# TODO(https://fxbug.dev/569952695): used closed=True when able.
# closed=True,
)
type ProcessBreakdown = Sequence[ProcessBreakdownMetric]
type ThreadBreakdown = Sequence[ThreadBreakdownMetric]
@dataclasses.dataclass(frozen=True)
class ProcessingRateSample:
"""A single processing rate sample at a given timestamp."""
timestamp_ms: float
rate: float # ranges from 0 to 1000, where 1000 represents 100% processing rate of the CPU.
@dataclasses.dataclass(frozen=True)
class VirtualProcessingSlice:
"""A slice with a constant processing rate."""
duration: float
rate: float
class CpuProcessingRateTimeline:
"""Encapsulates the processing rate shifts for a single CPU core."""
def __init__(self, rates: Collection[ProcessingRateSample]):
"""Constructor.
Args:
rates: An iterable of ProcessingRateSample objects.
"""
self.rates = sorted(rates, key=lambda r: r.timestamp_ms)
self._timestamps: list[float] = [r.timestamp_ms for r in self.rates]
self._rate_values: list[float] = [r.rate for r in self.rates]
self._rate_strs: list[str] = [str(int(r)) for r in self._rate_values]
def get_virtual_slices(
self, start_ts: float, stop_ts: float
) -> list[VirtualProcessingSlice]:
"""Splits a schedule slice into virtual slices based on rate changes.
Args:
start_ts: Start of the slice (ms).
stop_ts: End of the slice (ms).
Returns:
A list of VirtualProcessingSlice objects.
"""
if start_ts >= stop_ts:
return []
virtual_slices: list[VirtualProcessingSlice] = []
curr_ts = start_ts
timestamps = self._timestamps
rate_values = self._rate_values
n_rates = len(timestamps)
idx = bisect.bisect_right(timestamps, curr_ts)
current_rate = (
_DEFAULT_PROCESSING_RATE if idx == 0 else rate_values[idx - 1]
)
while curr_ts < stop_ts:
next_change_ts = timestamps[idx] if idx < n_rates else float("inf")
seg_end = min(stop_ts, next_change_ts)
virtual_slices.append(
VirtualProcessingSlice(
duration=seg_end - curr_ts,
rate=current_rate,
)
)
curr_ts = seg_end
if curr_ts >= next_change_ts:
current_rate = rate_values[idx]
idx += 1
return virtual_slices
@dataclasses.dataclass
class ProcessingRateStats:
"""Aggregates durations spent at different processing rates."""
duration_per_rate: dict[str, float] = dataclasses.field(
default_factory=dict
)
def add(self, rate: float, duration: float) -> None:
rate_str = str(int(rate)) # One of "1000", "798", "506", and "360"
self.duration_per_rate[rate_str] = (
self.duration_per_rate.get(rate_str, 0.0) + duration
)
@dataclasses.dataclass
class ThreadCpuStats:
"""Encapsulates CPU usage statistics for a single thread on a single CPU."""
duration: float = 0.0
normalized_duration: float = 0.0
rate_stats: ProcessingRateStats = dataclasses.field(
default_factory=ProcessingRateStats
)
class CpuMetricsProcessor(trace_metrics.MetricsProcessor):
"""Computes CPU utilization metrics, both structured and freeform.
CPU load metrics are reported as a percentage of the total load across all cores, with 100%
mapping to full utilization of every core simultaneously. Some tools will report a process using
both cores on a dual core system as taking up 200% of CPU; that is not the case here. Depending
on configuration, this module will report a time series of average utiliztion samples taken
periodically for the duration of the test, or aggregate measurements-- such as min, max,
average, and percentiles -- calculated over that data.
Freeform CPU load metrics report per-core average load for every thread in every process on the
system. These data are not suited for automated changepoint detection, and so are instead
piped to a dashboard.
"""
FREEFORM_METRICS_FILENAME = "cpu_breakdown"
def __init__(
self,
aggregates_only: bool = True,
percent_cutoff: float = _DEFAULT_PERCENT_CUTOFF,
):
"""Constructor.
Args:
aggregates_only: When True, generates CpuMin, CpuMax, CpuAverage and CpuP* (%iles).
Otherwise generates CpuLoad metric with all cpu values.
percent_cutoff: Any process that has CPU below this won't be listed in the CPU usage
breakdown reported in freeform metrics.
"""
self.aggregates_only: bool = aggregates_only
self._percent_cutoff = percent_cutoff
self._cached_rates: tuple[
weakref.ReferenceType[trace_model.Model],
Mapping[int, CpuProcessingRateTimeline],
] | None = None
def _get_cached_cpu_rates(
self, model: trace_model.Model
) -> Mapping[int, CpuProcessingRateTimeline]:
if self._cached_rates is not None and self._cached_rates[0]() is model:
return self._cached_rates[1]
timelines = _get_cpu_rates(model)
self._cached_rates = (weakref.ref(model), timelines)
return timelines
@property
@typing.override
def event_patterns(self) -> set[str]:
return {_CPU_USAGE_EVENT_NAME, f"{_RATE_EVENT_NAME}.*"}
@typing.override
def process_metrics(
self, model: trace_model.Model
) -> MutableSequence[metrics.TestCaseResult]:
cpu_starts: list[float] = []
cpu_percentages: list[float] = []
# Parse the start time and percentage for each `Event`.
for event in trace_utils.filter_events(
model.all_events(),
name=_CPU_USAGE_EVENT_NAME,
type=trace_model.CounterEvent,
):
cpu_starts.append(event.start.to_epoch_nanoseconds())
cpu_percentages.append(
event.args.get("average_cpu_percentage") or 0
)
# The suspend times calculated from https://source.corp.google.com/h/turquoise-internal/turquoise/+/main:src/cobalt/bin/system-metrics/cpu_stats_fetcher_impl.cc
# are offset by 1 event. Add that event in. It's approximately 1 s.
cpu_durations: list[float] = [_ONE_S_IN_NS] + [
curr - prev for prev, curr in itertools.pairwise(cpu_starts)
]
if len(cpu_percentages) == 0:
_LOGGER.info(
"No cpu usage measurements are present. Perhaps the trace "
"duration is too short to provide cpu usage information"
)
return []
if len(cpu_durations) != len(cpu_percentages):
_LOGGER.warning(
"The number of CPU duration segments don't match the number "
"of percentages. Data may be truncated."
)
results: list[metrics.TestCaseResult] = []
# Calculate metrics without normalization
if self.aggregates_only:
results.extend(
trace_utils.standard_metrics_set(
values=cpu_percentages,
label_prefix="Cpu",
unit=metrics.Unit.percent,
durations=cpu_durations,
)
)
else:
results.append(
metrics.TestCaseResult(
"CpuLoad", metrics.Unit.percent, cpu_percentages
)
)
# Calculate normalized metrics if Processing Rate events are present
cpu_timelines = self._get_cached_cpu_rates(model)
if cpu_timelines:
total_cpu_count = max(1, len(model.scheduling_records))
norm_percentages = _calculate_normalized_percentages(
cpu_starts=cpu_starts,
cpu_durations=cpu_durations,
cpu_percentages=cpu_percentages,
cpu_timelines=cpu_timelines,
total_cpu_count=total_cpu_count,
)
if self.aggregates_only:
results.extend(
trace_utils.standard_metrics_set(
values=norm_percentages,
label_prefix="CpuNormalized",
unit=metrics.Unit.percent,
durations=cpu_durations,
)
)
else:
results.append(
metrics.TestCaseResult(
"CpuNormalizedLoad",
metrics.Unit.percent,
norm_percentages,
)
)
return results
# TODO(https://github.com/python/mypy/issues/18176): change this return
# specification to `-> tuple[str, ThreadBreakdown]`; the type-checker
# ought understand that a ThreadBreakdown is suitable to use where a
# metrics.JSON is needed.
@typing.override
def process_freeform_metrics(
self, model: trace_model.Model
) -> tuple[str, metrics.JSON]:
"""
Given trace_model.Model, iterates through all the SchedulingRecords and calculates the
duration for each Process's Threads, and saves them by CPU.
Args:
model: The input trace model.
Returns:
str: stable identifier to use in freeform metrics file name.
Breakdown: Per-process, per-thread CPU usage breakdown.
"""
(breakdown, _) = _process_metrics_and_get_total_time(
model,
self._percent_cutoff,
cpu_timelines=self._get_cached_cpu_rates(model),
)
# TODO(https://github.com/python/mypy/issues/18176): eliminate this cast;
# the type-checker ought understand that a ThreadBreakdown is suitable
# to use where a metrics.JSON is needed.
return self.FREEFORM_METRICS_FILENAME, cast(
Sequence[Mapping[str, metrics.JSON]], breakdown
)
def _process_metrics_and_get_total_time(
model: trace_model.Model,
percent_cutoff: float,
cpu_timelines: Mapping[int, CpuProcessingRateTimeline] | None = None,
) -> tuple[ThreadBreakdown, float]:
"""
Given trace_model.Model, iterates through all the SchedulingRecords and calculates the
duration for each Process's Threads, and saves them by CPU.
Args:
model: The input trace model.
percent_cutoff: Any thread with CPU percentage below this cutoff will be omitted.
Returns:
ThreadBreakdown: Per-process, per-thread CPU usage breakdown.
float: The total duration of the trace.
"""
# Map tids to names.
tid_to_process_name: dict[int, str] = {}
tid_to_thread_name: dict[int, str] = {}
for p in model.processes:
for t in p.threads:
tid_to_process_name[t.tid] = p.name
tid_to_thread_name[t.tid] = t.name
# Parse Processing rate events for DVFS CPU normalization.
if cpu_timelines is None:
cpu_timelines = _get_cpu_rates(model)
# Calculate durations for each CPU for each tid.
durations = DurationsBreakdown.calculate(
model.scheduling_records,
tid_to_thread_name,
cpu_timelines,
)
# Calculate the percent of time the thread spent on this CPU,
# compared to the total CPU duration.
# If the percent spent is at or above our cutoff, add metric to
# breakdown.
full_breakdown: list[ThreadBreakdownMetric] = []
for tid, cpu_stats_map in durations.tid_to_stats.items():
if tid in tid_to_thread_name:
for cpu, stats in cpu_stats_map.items():
duration = stats.duration
percent = (
duration / durations.cpu_to_total_duration[cpu] * 100
if durations.cpu_to_total_duration[cpu] > 0
else 0.0
)
normalized_duration = stats.normalized_duration
normalized_percent = (
normalized_duration
/ durations.cpu_to_total_normalized_duration[cpu]
* 100
if durations.cpu_to_total_normalized_duration[cpu] > 0
else 0.0
)
if percent >= percent_cutoff:
metric: ThreadBreakdownMetric = {
"process_name": tid_to_process_name[tid],
"thread_name": tid_to_thread_name[tid],
"tid": tid,
"cpu": cpu,
"percent": percent,
"duration": duration,
"normalized_percent": normalized_percent,
"normalized_duration": normalized_duration,
"duration_per_rate": dict(
sorted(
stats.rate_stats.duration_per_rate.items(),
key=lambda item: int(
item[0]
), # Sort by rate (as int)
reverse=True,
)
),
}
full_breakdown.append(metric)
if durations.cpu_to_skipped_duration:
_LOGGER.warning(
"Possibly missing ContextSwitch record(s) in trace for these CPUs and durations: "
f"{durations.cpu_to_skipped_duration}"
)
# Sort metrics by CPU (desc) and percent (desc).
full_breakdown.sort(
key=lambda m: (m["cpu"], m["percent"]),
reverse=True,
)
return (
full_breakdown,
durations.max_timestamp - durations.min_timestamp,
)
def process_metrics_and_get_total_time(
model: trace_model.Model,
) -> tuple[ThreadBreakdown, float]:
"""
Given trace_model.Model, iterates through all the SchedulingRecords and calculates the
duration for each Process's Threads, and saves them by CPU.
Args:
model: The input trace model.
Returns:
ThreadBreakdown: Per-process, per-thread CPU usage breakdown.
float: The total duration of the trace.
"""
return _process_metrics_and_get_total_time(
model, percent_cutoff=_DEFAULT_PERCENT_CUTOFF
)
def _get_cpu_rates(
model: trace_model.Model,
) -> Mapping[int, CpuProcessingRateTimeline]:
"""Extracts DVFS processing rate events from the trace model for all CPUs.
Args:
model: The input trace model containing 'kernel:power' events.
Returns:
A dictionary keyed by cpu_index, containing CpuProcessingRateTimeline objects.
"""
rates_by_cpu: collections.defaultdict[
int, list[ProcessingRateSample]
] = collections.defaultdict(list)
for process in model.processes:
for thread in process.threads:
for event in thread.events:
if isinstance(
event, trace_model.CounterEvent
) and event.name.startswith(_RATE_EVENT_NAME):
# For "Rate" events, the CPU index is encoded in `event.id`.
# When the CPU index is 0, the `event.id` field is omitted from the trace.
cpu_idx = event.id if event.id is not None else 0
rate = event.args.get("CPU", _DEFAULT_PROCESSING_RATE)
rates_by_cpu[cpu_idx].append(
ProcessingRateSample(
timestamp_ms=float(
event.start.to_epoch_nanoseconds()
)
/ _NS_PER_MS,
rate=rate,
)
)
return {
cpu_idx: CpuProcessingRateTimeline(rates)
for cpu_idx, rates in rates_by_cpu.items()
}
def _calculate_normalized_percentages(
*,
cpu_starts: Sequence[float],
cpu_durations: Sequence[float],
cpu_percentages: Sequence[float],
cpu_timelines: Mapping[int, CpuProcessingRateTimeline],
total_cpu_count: int,
) -> list[float]:
"""Computes the normalized CPU percentages for periodic aggregate windows.
For each reporting window (e.g., 1-second buckets), this calculates the exact
normalized CPU utilization by comparing the DVFS-weighted CPU time against
the total wall-clock time of the window.
Args:
cpu_starts: Start timestamps (in ns) of the reporting windows.
cpu_durations: Durations (in ns) of the reporting windows.
cpu_percentages: The raw (unnormalized) CPU percentages for the windows.
cpu_timelines: The processing rate timelines for each CPU.
total_cpu_count: The total number of CPUs in the system.
Returns:
A list of normalized CPU percentages corresponding to each window.
"""
norm_percentages = []
cpu_count = max(1, total_cpu_count)
missing_cpus = max(0, cpu_count - len(cpu_timelines))
for start_ns, dur_ns, pct in zip(
cpu_starts, cpu_durations, cpu_percentages
):
start_ms = start_ns / _NS_PER_MS
end_ms = (start_ns + dur_ns) / _NS_PER_MS
total_dur = dur_ns / _NS_PER_MS
total_norm_dur = 0.0
if total_dur > 0 and cpu_timelines:
for timeline in cpu_timelines.values():
virtual_slices = timeline.get_virtual_slices(start_ms, end_ms)
total_norm_dur += sum(
s.duration * (s.rate / _DEFAULT_PROCESSING_RATE)
for s in virtual_slices
)
# Add full capacity duration for CPUs without rate events
total_norm_dur += missing_cpus * total_dur
avg_norm_dur = total_norm_dur / cpu_count
avg_rate_ratio = avg_norm_dur / total_dur
else:
avg_rate_ratio = 1.0
norm_percentages.append(pct * avg_rate_ratio)
return norm_percentages
class DurationsBreakdown:
def __init__(self) -> None:
# Maps TID to a dict of CPUs to stats on that CPU.
self.tid_to_stats: collections.defaultdict[
int, collections.defaultdict[int, ThreadCpuStats]
] = collections.defaultdict(
lambda: collections.defaultdict(ThreadCpuStats)
)
# Map of CPU to total duration used (ms).
self.cpu_to_total_duration: dict[int, float] = {}
# Map of CPU to total normalized duration used (ms).
self.cpu_to_total_normalized_duration: dict[int, float] = {}
self.cpu_to_skipped_duration: dict[int, float] = {}
self.min_timestamp: float = sys.float_info.max
self.max_timestamp: float = 0
def _calculate_duration_per_cpu(
self,
cpu: int,
records: list[trace_model.ContextSwitch],
tid_to_thread_name: dict[int, str],
rate_timeline: CpuProcessingRateTimeline | None,
) -> None:
"""
Calculates the total duration for each thread, on a particular CPU.
Uses a list of sorted ContextSwitch records to sum up the duration for each thread.
It's possible that consecutive records do not have matching incoming_tid and outgoing_tid.
"""
smallest_timestamp = self._timestamp_ms(records[0].start)
if smallest_timestamp < self.min_timestamp:
self.min_timestamp = smallest_timestamp
largest_timestamp = self._timestamp_ms(records[-1].start)
if largest_timestamp > self.max_timestamp:
self.max_timestamp = largest_timestamp
total_duration = largest_timestamp - smallest_timestamp
skipped_duration = 0.0
self.cpu_to_total_duration[cpu] = total_duration
if rate_timeline:
virtual_slices = rate_timeline.get_virtual_slices(
smallest_timestamp, largest_timestamp
)
self.cpu_to_total_normalized_duration[cpu] = sum(
s.duration * (s.rate / _DEFAULT_PROCESSING_RATE)
for s in virtual_slices
)
timestamps = rate_timeline._timestamps
rate_values = rate_timeline._rate_values
rate_strs = rate_timeline._rate_strs
n_rates = len(timestamps)
else:
self.cpu_to_total_normalized_duration[cpu] = (
largest_timestamp - smallest_timestamp
)
timestamps = []
rate_values = []
rate_strs = []
n_rates = 0
default_rate_str = str(int(_DEFAULT_PROCESSING_RATE))
rate_idx = 0
inf = float("inf")
for prev_record, curr_record in itertools.pairwise(records):
# Check that the previous ContextSwitch's incoming_tid ("this thread is starting work
# on this CPU") matches the current ContextSwitch's outgoing_tid ("this thread is being
# switched away from"). If so, there is a duration to calculate. Otherwise, it means
# maybe there is skipped data or something.
if prev_record.tid != curr_record.outgoing_tid:
start_ts = self._timestamp_ms(prev_record.start)
stop_ts = self._timestamp_ms(curr_record.start)
skipped_duration += stop_ts - start_ts
# Purposely skip saving idle thread durations.
elif prev_record.is_idle():
continue
else:
start_ts = self._timestamp_ms(prev_record.start)
stop_ts = self._timestamp_ms(curr_record.start)
duration = stop_ts - start_ts
assert duration >= 0
if curr_record.outgoing_tid in tid_to_thread_name:
# Add stats to the total duration for that tid and CPU.
cpu_stats = self.tid_to_stats[curr_record.outgoing_tid][cpu]
cpu_stats.duration += duration
duration_per_rate = cpu_stats.rate_stats.duration_per_rate
if rate_timeline:
if start_ts < stop_ts:
while (
rate_idx < n_rates
and timestamps[rate_idx] <= start_ts
):
rate_idx += 1
current_rate = (
_DEFAULT_PROCESSING_RATE
if rate_idx == 0
else rate_values[rate_idx - 1]
)
current_rate_str = (
default_rate_str
if rate_idx == 0
else rate_strs[rate_idx - 1]
)
curr_ts = start_ts
while curr_ts < stop_ts:
next_change_ts = (
timestamps[rate_idx]
if rate_idx < n_rates
else inf
)
seg_end = (
stop_ts
if stop_ts < next_change_ts
else next_change_ts
)
seg_dur = seg_end - curr_ts
cpu_stats.normalized_duration += seg_dur * (
current_rate / _DEFAULT_PROCESSING_RATE
)
duration_per_rate[current_rate_str] = (
duration_per_rate.get(current_rate_str, 0.0)
+ seg_dur
)
curr_ts = seg_end
if curr_ts >= next_change_ts:
current_rate = rate_values[rate_idx]
current_rate_str = rate_strs[rate_idx]
rate_idx += 1
else:
cpu_stats.normalized_duration += duration
duration_per_rate[default_rate_str] = (
duration_per_rate.get(default_rate_str, 0.0)
+ duration
)
if skipped_duration > 0:
self.cpu_to_skipped_duration[cpu] = skipped_duration
@staticmethod
def _timestamp_ms(timestamp: trace_time.TimePoint) -> float:
"""
Return timestamp in ms.
"""
return float(timestamp.to_epoch_nanoseconds()) / _NS_PER_MS
@classmethod
def calculate(
cls,
per_cpu_scheduling_records: Mapping[
int, Sequence[trace_model.SchedulingRecord]
],
tid_to_thread_name: dict[int, str],
cpu_timelines: Mapping[int, CpuProcessingRateTimeline],
) -> Self:
durations = cls()
for cpu, records in per_cpu_scheduling_records.items():
cs_records = [
r for r in records if isinstance(r, trace_model.ContextSwitch)
]
cs_records.sort(
key=lambda record: record.start.to_epoch_nanoseconds()
)
durations._calculate_duration_per_cpu(
cpu,
cs_records,
tid_to_thread_name,
cpu_timelines.get(cpu),
)
return durations
def group_by_process_name(breakdown: ThreadBreakdown) -> ProcessBreakdown:
"""
Given a breakdown, group the metrics by process_name only,
ignoring thread name.
"""
# Group metrics by (cpu, process_name)
grouped: collections.defaultdict[
tuple[int, str], list[ThreadBreakdownMetric]
] = collections.defaultdict(list)
for metric in breakdown:
grouped[(metric["cpu"], metric["process_name"])].append(metric)
return sorted(
map(_aggregate_thread_metrics, grouped.values()),
key=lambda m: (m["cpu"], m["percent"]),
reverse=True,
)
def _aggregate_thread_metrics(
metrics: Sequence[ThreadBreakdownMetric],
) -> ProcessBreakdownMetric:
"""
Combines a sequence of thread-level metrics into a single process-level metric.
The combination rules are as follows:
- Numerical fields (`percent`, `duration`, `normalized_percent`, `normalized_duration`)
are combined by adding them together.
- The `duration_per_rate` dictionaries are merged by summing their corresponding rate values.
- The returned metric retains the `process_name` and `cpu` keys from the first metric.
- Thread-specific keys (`thread_name`, `tid`) are intentionally omitted as this operates
at the process level.
Args:
metrics: A non-empty sequence of ProcessBreakdownMetrics for the same process and CPU.
Returns:
A single aggregated ProcessBreakdownMetric.
"""
first = metrics[0]
merged: ProcessBreakdownMetric = {
"process_name": first["process_name"],
"cpu": first["cpu"],
"percent": sum(m["percent"] for m in metrics),
"duration": sum(m["duration"] for m in metrics),
}
if "normalized_duration" in first:
merged["normalized_percent"] = sum(
m.get("normalized_percent", 0.0) for m in metrics
)
merged["normalized_duration"] = sum(
m.get("normalized_duration", 0.0) for m in metrics
)
if "duration_per_rate" in first:
rates: dict[str, float] = {}
for m in metrics:
for k, v in m.get("duration_per_rate", {}).items():
rates[k] = rates.get(k, 0.0) + v
merged["duration_per_rate"] = dict(
sorted(rates.items(), key=lambda x: int(x[0]), reverse=True)
)
return merged