blob: cf5b4fd77fad4f7799b06ab1dee77107ca1d56a7 [file]
# Copyright 2025 The Fuchsia Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Metrics reporting helpers.
This module implements the perf test results schema.
See https://fuchsia.dev/fuchsia-src/development/performance/fuchsiaperf_format
for more details.
"""
import dataclasses
import enum
import inspect as py_inspect
import json
import logging
import pathlib
from collections.abc import (
Collection,
Mapping,
MutableMapping,
MutableSequence,
Sequence,
)
from typing import (
Any,
Callable,
NotRequired,
ReadOnly,
Self,
TypedDict,
)
_LOGGER: logging.Logger = logging.getLogger("Reporting")
type JSON = (
Mapping[str, JSON] | Sequence[JSON] | str | int | float | bool | None
)
class Direction(enum.StrEnum):
biggerIsBetter = "biggerIsBetter"
smallerIsBetter = "smallerIsBetter"
class Unit(enum.Enum):
"""The set of valid Unit constants.
This should be kept in sync with the list of supported units in the results
schema docs linked at the top of this file. These are the unit strings
accepted by catapult_converter.
"""
# Time-based units.
nanoseconds = "nanoseconds"
milliseconds = "milliseconds"
# Size-based units.
bytes = "bytes"
bytesPerSecond = "bytes/second"
# Frequency-based units.
framesPerSecond = "frames/second"
# Percentage-based units.
percent = "percent"
# Count-based units. ("count" can't be used a StrEnum)
count = "count"
# Power-based units.
watts = "W"
# Energy-based units.
joules = "J"
class MetricDescription(TypedDict):
"""Describes a single metric."""
name: ReadOnly[str]
doc: ReadOnly[str]
class MetricsProcessorDescription(TypedDict):
"""Documents a single metrics processor."""
classname: NotRequired[ReadOnly[str]]
doc: ReadOnly[str]
code_path: ReadOnly[str]
line_no: ReadOnly[int]
metrics: ReadOnly[Sequence[MetricDescription]]
@dataclasses.dataclass(frozen=True)
class TestCaseResult:
"""The results for a single test case.
See the link at the top of this file for documentation.
"""
label: str
unit: Unit
values: tuple[float, ...]
doc: str
# TODO(https://fxbug.dev/483530722) require this field be non-null
direction: Direction | None = None
def __init__(
self,
label: str,
unit: Unit,
values: Sequence[float],
doc: str = "",
direction: Direction | None = None,
):
"""Allows any Sequence to be used for values while staying hashable."""
object.__setattr__(self, "label", label)
object.__setattr__(self, "unit", unit)
object.__setattr__(self, "values", tuple(values))
object.__setattr__(self, "doc", doc)
# If unspecified, Capapult has its own notion of which units default
# to which directions which we defer to for compatibility.
if direction:
object.__setattr__(self, "direction", direction)
def to_json(self, test_suite: str) -> dict[str, Any]:
return {
"label": self.label,
"test_suite": test_suite,
"unit": str(
self.unit.value
+ (f"_{self.direction}" if self.direction else "")
),
"values": list(self.values),
}
def describe(self) -> MetricDescription:
return MetricDescription(name=self.label, doc=self.doc)
@staticmethod
def write_fuchsiaperf_json(
results: Collection[TestCaseResult],
test_suite: str,
output_path: pathlib.Path,
) -> None:
"""Writes the given TestCaseResults into a fuchsiaperf json file.
Args:
results: The results to write.
test_suite: A test suite name to embed in the json.
E.g. "fuchsia.uiperf.my_metric".
output_path: Output file path, must end with ".fuchsiaperf.json".
"""
assert output_path.name.endswith(
".fuchsiaperf.json"
), f"Expecting path that ends with '.fuchsiaperf.json' but got {output_path}"
results_json = [r.to_json(test_suite) for r in results]
with open(output_path, "w") as outfile:
json.dump(results_json, outfile, indent=4)
_LOGGER.info(f"Wrote {len(results_json)} results into {output_path}")
class SuiteDescription(TypedDict):
"""Description of a test suite class."""
test_suite: ReadOnly[str]
test_class: ReadOnly[str]
doc: ReadOnly[str]
code_path: ReadOnly[str]
line_no: ReadOnly[int]
class SuiteDocumentation(SuiteDescription):
"""Documentation for a test suite, including code that generates metrics."""
metrics_processors: ReadOnly[Sequence[MetricsProcessorDescription]]
@dataclasses.dataclass
class Report:
"""Metrics data for a test suite."""
structured: MutableSequence[TestCaseResult]
"""Named single-value (or list-of-values) metrics."""
freeform: MutableMapping[str, JSON] = dataclasses.field(
default_factory=dict
)
"""Mapping of strings to arbitrary JSON data."""
metrics_processors: list[MetricsProcessorDescription] = dataclasses.field(
default_factory=list
)
"""Descriptions of the code that captured the metrics in this report."""
def extend(self, other: Self) -> Self:
self.structured.extend(other.structured)
self.freeform.update(other.freeform)
self.metrics_processors.extend(other.metrics_processors)
return self
def generate_docs(self, desc: SuiteDescription) -> SuiteDocumentation:
return SuiteDocumentation(
**desc, metrics_processors=self.metrics_processors
)
class CallableDescription(TypedDict):
"""Documentation for a callable used to generate metrics."""
doc: ReadOnly[str]
code_path: ReadOnly[str]
line_no: ReadOnly[int]
def describe_callable(c: Callable[..., Any]) -> CallableDescription:
docstring = py_inspect.getdoc(c)
assert docstring, c
return CallableDescription(
doc=docstring,
code_path=py_inspect.getfile(c),
line_no=py_inspect.getsourcelines(c)[1],
)