blob: 6cffc7fd467719984dd7942f7a0fdd4b157ed320 [file]
#!/usr/bin/env fuchsia-vendored-python
# Copyright 2023 The Fuchsia Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Unit tests for trace metrics processors."""
import inspect as py_inspect
import json
import os
import pathlib
import tempfile
import typing
import unittest
from collections.abc import Collection
import trace_processing.metrics.app_render as app_render_metrics
import trace_processing.metrics.cpu as cpu_metrics
import trace_processing.metrics.fps as fps_metrics
import trace_processing.metrics.input_latency as input_latency_metrics
import trace_processing.metrics.scenic as scenic_metrics
from parameterized import param, parameterized
from reporting import metrics
from trace_processing import (
trace_importing,
trace_metrics,
trace_model,
trace_time,
trace_utils,
)
# Boilerplate-busting constants:
U = metrics.Unit
TCR = metrics.TestCaseResult
class TestCaseResultTest(unittest.TestCase):
"""Tests TestCaseResult"""
def test_to_json(self) -> None:
label = "L1"
test_suite = "bar"
result = TCR(
label=label, unit=U.bytesPerSecond, values=[0, 0.1, 23.45, 6]
)
self.assertEqual(result.label, label)
self.assertEqual(
result.to_json(test_suite=test_suite),
{
"label": label,
"test_suite": test_suite,
"unit": "bytes/second",
"values": [0, 0.1, 23.45, 6],
},
)
def test_write_fuchsia_perf_json(self) -> None:
test_suite = "ts"
results = [
TCR(label="l1", unit=U.percent, values=[1]),
TCR(label="l2", unit=U.framesPerSecond, values=[2]),
]
with tempfile.TemporaryDirectory() as tmpdir:
actual_output_path = (
pathlib.Path(tmpdir) / "actual_output.fuchsiaperf.json"
)
metrics.TestCaseResult.write_fuchsiaperf_json(
results,
test_suite=test_suite,
output_path=actual_output_path,
)
actual_output = json.loads(actual_output_path.read_text())
self.assertEqual(
actual_output, [r.to_json(test_suite) for r in results]
)
_METRICS_TEST_PARAMS = [
param(
"cpu",
processor=cpu_metrics.CpuMetricsProcessor(aggregates_only=False),
model_file="cpu_metric.json",
expected_results=[
TCR(label="CpuLoad", unit=U.percent, values=[43, 20]),
],
),
param(
"cpu_aggregates",
processor=cpu_metrics.CpuMetricsProcessor(aggregates_only=True),
model_file="cpu_metric.json",
expected_results=trace_utils.standard_metrics_set(
# ts in the json is in microseconds, durations is in ns.
[43, 20],
"Cpu",
U.percent,
durations=[1000000000, 1100000000],
),
),
param(
"fps",
processor=fps_metrics.FpsMetricsProcessor(aggregates_only=False),
model_file="fps_metric.json",
expected_results=[
TCR(
label="Fps",
unit=U.framesPerSecond,
values=[10000000.0, 5000000.0],
)
],
),
param(
"fps_aggregates",
processor=fps_metrics.FpsMetricsProcessor(aggregates_only=True),
model_file="fps_metric.json",
expected_results=trace_utils.standard_metrics_set(
[10000000.0, 5000000.0],
"Fps",
U.framesPerSecond,
),
),
param(
"scenic",
processor=scenic_metrics.ScenicMetricsProcessor(aggregates_only=False),
model_file="scenic_metric.json",
expected_results=[
TCR(
label="RenderCpu",
unit=U.milliseconds,
values=[0.09, 0.08],
),
TCR(
label="RenderTotal",
unit=U.milliseconds,
values=[0.11, 0.112],
),
],
),
param(
"scenic_aggregates",
processor=scenic_metrics.ScenicMetricsProcessor(aggregates_only=True),
model_file="scenic_metric.json",
expected_results=(
trace_utils.standard_metrics_set(
[0.09, 0.08],
"RenderCpu",
U.milliseconds,
)
+ trace_utils.standard_metrics_set(
[0.11, 0.112],
"RenderTotal",
U.milliseconds,
)
),
),
param(
"scenic_no_render_total",
processor=scenic_metrics.ScenicMetricsProcessor(
aggregates_only=False, include_render_total=False
),
model_file="scenic_metric.json",
expected_results=[
TCR(
label="RenderCpu",
unit=U.milliseconds,
values=[0.09, 0.08],
),
],
),
param(
"app_render",
processor=app_render_metrics.AppRenderLatencyMetricsProcessor(
"flatland-view-provider-example",
aggregates_only=False,
),
model_file="app_render_metric.json",
expected_results=[
TCR(
label="AppRenderVsyncLatency",
unit=U.milliseconds,
values=[0.004, 0.002, 0.001, 0.006, 0.008],
),
TCR(
label="AppFps",
unit=U.framesPerSecond,
values=[
125000.0,
111111.11111111111,
66666.66666666667,
83333.33333333333,
],
),
],
),
param(
"app_render_aggregates",
processor=app_render_metrics.AppRenderLatencyMetricsProcessor(
"flatland-view-provider-example",
aggregates_only=True,
),
model_file="app_render_metric.json",
expected_results=(
trace_utils.standard_metrics_set(
[0.004, 0.002, 0.001, 0.006, 0.008],
"AppRenderVsyncLatency",
U.milliseconds,
)
+ trace_utils.standard_metrics_set(
[
125000.0,
111111.11111111111,
66666.66666666667,
83333.33333333333,
],
"AppFps",
U.framesPerSecond,
)
),
),
param(
"input",
processor=input_latency_metrics.InputLatencyMetricsProcessor(
aggregates_only=False,
),
model_file="input_latency_metric.json",
expected_results=[
TCR(
label="total_input_latency",
unit=U.milliseconds,
values=[0.003, 0.002, 0.007, 0.009],
),
],
),
param(
"input_aggregates",
processor=input_latency_metrics.InputLatencyMetricsProcessor(
aggregates_only=True,
),
model_file="input_latency_metric.json",
expected_results=trace_utils.standard_metrics_set(
[0.003, 0.002, 0.007, 0.009],
"InputLatency",
U.milliseconds,
),
),
]
class MetricProcessorsTest(unittest.TestCase):
"""Tests for the various MetricProcessors."""
def _load_model(self, model_file_name: str) -> trace_model.Model:
# A second dirname is required to account for the .pyz archive which
# contains the test and a third one since data is a sibling of the test.
runtime_deps_path = os.path.join(
os.path.dirname(os.path.dirname(os.path.dirname(__file__))),
"runtime_deps",
)
return trace_importing.create_model_from_file_path(
os.path.join(runtime_deps_path, model_file_name)
)
def _load_fxt_model(self, fxt_file: str) -> trace_model.Model:
runtime_deps_path = os.path.join(
os.path.dirname(os.path.dirname(os.path.dirname(__file__))),
"runtime_deps",
)
return trace_importing.create_model_from_trace_file_path(
os.path.join(runtime_deps_path, fxt_file),
without_json_conversion=True,
)
@parameterized.expand(_METRICS_TEST_PARAMS)
def test_processor(
self,
_: str,
processor: trace_metrics.MetricsProcessor,
model_file: str,
expected_results: list[TCR],
) -> None:
"""Tests a processor's outputs with a given input model loaded from a json file"""
model = self._load_model(model_file)
actual_results = list(processor.process_metrics(model))
docs = processor.describe(actual_results)
self.assertEqual(
docs.get("classname", ""), processor.__class__.__name__
)
self.assertNotEqual(docs["doc"], "")
self.assertNotEqual(docs["code_path"], "")
# Improves assertEqual output when comparing lists.
self.maxDiff = 10000
self.assertEqual(actual_results, expected_results)
@parameterized.expand(_METRICS_TEST_PARAMS)
def test_processor_fxt(
self,
_: str,
processor: trace_metrics.MetricsProcessor,
model_file: str,
expected_results: list[TCR],
) -> None:
"""Tests a processor's outputs with a given input model loaded from an FXT file"""
fxt_file = model_file.replace(".json", ".fxt")
model = self._load_fxt_model(fxt_file)
actual_results = list(processor.process_metrics(model))
docs = processor.describe(actual_results)
self.assertEqual(
docs.get("classname", ""), processor.__class__.__name__
)
self.assertNotEqual(docs["doc"], "")
self.assertNotEqual(docs["code_path"], "")
self.maxDiff = 10000
self.assertEqual(actual_results, expected_results)
class TestMetricsProcessor(trace_metrics.MetricsProcessor):
"""TEST"""
@typing.override
def process_metrics(
self, model: trace_model.Model
) -> Collection[metrics.TestCaseResult]:
# If ever called, this can be implemented.
raise NotImplementedError()
def test_auto_doc(self) -> None:
docs = self.TestMetricsProcessor().describe([])
self.assertEqual(docs.get("classname", ""), "TestMetricsProcessor")
self.assertEqual(docs["doc"], "TEST")
self.assertEqual(docs["code_path"], py_inspect.getfile(self.__class__))
def test_get_thread_running_durations(self) -> None:
def make_switch(
start_us: int, incoming_tid: int, outgoing_tid: int
) -> trace_model.ContextSwitch:
return trace_model.ContextSwitch(
start=trace_time.TimePoint.from_epoch_delta(
trace_time.TimeDelta.from_microseconds(start_us)
),
incoming_tid=incoming_tid,
outgoing_tid=outgoing_tid,
incoming_prio=10,
outgoing_prio=10,
outgoing_state=trace_model.ThreadState.ZX_THREAD_STATE_RUNNING,
args={},
)
scheduling_records = {
0: [
make_switch(1000, 10, 0),
make_switch(3000, 20, 10),
make_switch(5000, 10, 20),
make_switch(6000, 0, 10),
]
}
windows = [
(
trace_time.TimePoint.from_epoch_delta(
trace_time.TimeDelta.from_microseconds(2000)
),
trace_time.TimePoint.from_epoch_delta(
trace_time.TimeDelta.from_microseconds(7000)
),
),
(
trace_time.TimePoint.from_epoch_delta(
trace_time.TimeDelta.from_microseconds(1500)
),
trace_time.TimePoint.from_epoch_delta(
trace_time.TimeDelta.from_microseconds(4000)
),
),
(
trace_time.TimePoint.from_epoch_delta(
trace_time.TimeDelta.from_microseconds(800)
),
trace_time.TimePoint.from_epoch_delta(
trace_time.TimeDelta.from_microseconds(1200)
),
),
]
durations = scenic_metrics._get_thread_running_durations(
trace_model.Model([], scheduling_records), tid=10, windows=windows
)
self.assertEqual(len(durations), 3)
self.assertEqual(durations[0].to_microseconds(), 2000)
self.assertEqual(durations[1].to_microseconds(), 1500)
self.assertEqual(durations[2].to_microseconds(), 200)