| # Copyright 2023 The Fuchsia Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| """Data model and associated methods for test-list.json files. |
| |
| test-list.json is a post-processed version of tests.json that is |
| produced by //tools/test_list_tool. It provides basic execution |
| instructions and categorization for tests that are part of the |
| Fuchsia build. |
| |
| test-list.json is typically stored at the root of the output directory |
| for the Fuchsia build. |
| """ |
| |
| from dataclasses import dataclass |
| import json |
| import re |
| import typing |
| |
| from dataparse import dataparse |
| import tests_json_file |
| |
| |
| @dataparse |
| @dataclass |
| class TestListTagKV: |
| """Key/Value pair for test tags.""" |
| |
| key: str |
| value: str |
| |
| |
| @dataparse |
| @dataclass |
| class TestListExecutionEntry: |
| """The entry describing how to execute a test in test-list.json.""" |
| |
| # If set, this is the component URL to execute using ffx test. |
| component_url: str |
| |
| # If set, this is the --realm argument for ffx test. |
| realm: str | None = None |
| |
| # If set, this is the --max-severity-logs argument for ffx test. |
| max_severity_logs: str | None = None |
| |
| # If set, this is the --min-severity-logs argument for ffx test. |
| min_severity_logs: str | None = None |
| |
| |
| @dataparse |
| @dataclass |
| class TestListEntry: |
| """A single test entry stored in test-list.json.""" |
| |
| # The name of the test. Must be unique in the file for device tests, may |
| # overlap for host. |
| name: str |
| |
| # A list of tags for the test, stored as KV pairs. |
| tags: list[TestListTagKV] |
| |
| # Execution details for the test. |
| execution: TestListExecutionEntry | None = None |
| |
| def is_hermetic(self) -> bool: |
| """Determine if a test is hermetic. |
| |
| A test is hermetic only if it has a tag "hermetic" set to |
| "true". Otherwise we assume the test may be non-hermetic. |
| |
| Hermetic tests run in isolation, so knowing that a test is hermetic means |
| we can run that test in parallel with other tests safely. |
| |
| Returns: |
| bool: True if the test is definitely hermetic, False otherwise. |
| """ |
| for tag in self.tags: |
| if tag.key == "hermetic" and tag.value == "true": |
| return True |
| return False |
| |
| |
| @dataparse |
| @dataclass |
| class TestListFile: |
| """Top-level data model for test-list.json files.""" |
| |
| # The list of test list entries stored in the file. |
| data: list[TestListEntry] |
| |
| @staticmethod |
| def entries_from_file(file: str) -> dict[str, TestListEntry]: |
| """Parse the file at the given path as a test-list.json file. |
| |
| This method converts the flat list in the file into a dictionary |
| keyed by test name. |
| |
| Args: |
| file (os.PathLike): The file path to parse. |
| |
| Returns: |
| dict[str, TestListEntry]: Map from test name to entry for that test in the file. |
| |
| Raises: |
| IOError: If the file could not be read. |
| JSONDecodeError: If the file is not valid JSON. |
| DataParseError: If the data in the file does not match the data model. |
| """ |
| with open(file) as f: |
| data = json.load(f) |
| parsed: TestListFile = TestListFile.from_dict(data) # type:ignore |
| ret = dict() |
| for value in parsed.data: |
| ret[value.name] = value |
| return ret |
| |
| |
| _PACKAGE_REGEX = re.compile(r"/([\w\-_]+)#meta") |
| |
| |
| @dataclass |
| class Test: |
| """Wrapper containing data from both tests.json and test-list.json. |
| |
| tests.json contains build-specific information about tests, |
| while test-list.json provides the necessary information to |
| execute and categorize tests. Since both pieces of information |
| are needed to make sense of tests, this dataclass combines both |
| pieces for a test. |
| |
| Note that info is added lazily because we need to complete a build before |
| loading a test-list.json. Regenerating the top-level file can be very |
| expensive, so we opt to generate a test-list.json once tests.json and |
| selected tests are built only. |
| """ |
| |
| # The test as described by tests.json. |
| build: tests_json_file.TestEntry |
| |
| # The info for this test loaded from test-list.json |
| # This is lazily loaded since we need to build test packages before |
| # test-list.json is generated. |
| _maybe_info: TestListEntry | None = None |
| |
| def __hash__(self) -> int: |
| return self.build.test.name.__hash__() |
| |
| def __eq__(self, other: object) -> bool: |
| return self.build.__eq__(other) |
| |
| def needs_device(self) -> bool: |
| """Determine if this test requires a device. |
| |
| Device tests require one or more target devices to execute. |
| |
| Returns: |
| bool: True only if the test requires a device to run. |
| """ |
| return self.is_pure_device_test() or self.is_e2e_test() |
| |
| def is_host_test(self) -> bool: |
| # These are "pure" host tests, which excludes E2E tests |
| # that require a device. |
| return self.build.test.path is not None and not self.is_e2e_test() |
| |
| def join_info(self, entries: dict[str, TestListEntry]) -> None: |
| """Add info from test-list.json to this test. |
| |
| This is done is two stages to support lazily creating test-list.json |
| following the build. |
| |
| Args: |
| entries (dict[str, TestListEntry]): Result of parsing test-list.json. |
| """ |
| self._maybe_info = entries[self.name()] |
| |
| def name(self) -> str: |
| """Return the unique name of this test. |
| |
| Returns: |
| str: This tests name, which is unique among Tests. |
| """ |
| return self.build.test.name |
| |
| @property |
| def info(self) -> TestListEntry: |
| if self._maybe_info is None: |
| raise ValueError( |
| f"Test {self.name()} has not been joined with test-list.json" |
| ) |
| return self._maybe_info |
| |
| def is_e2e_test(self) -> bool: |
| """Determine if this test is an E2E test. |
| |
| E2E tests run on the host system (Linux) and also have a device_type |
| set in their environments. |
| |
| Returns: |
| bool: True only if the test is an E2E test. |
| """ |
| return self.build.test.os.lower() == "linux" and any( |
| [ |
| env.dimensions.device_type is not None |
| for env in self.build.environments or [] |
| ] |
| ) |
| |
| def is_pure_device_test(self) -> bool: |
| """Determine if this test is a pure device test. |
| |
| Pure device tests run directly on target devices. |
| |
| Returns: |
| bool: True only if the test is a pure device test. |
| """ |
| return self.build.test.package_url is not None |
| |
| def is_boot_test(self) -> bool: |
| """Determine if this test is a boot test. |
| |
| Boot tests specify a product_bundle entry and reboot the device. |
| |
| Returns: |
| bool: True only if this test is a boot test. |
| """ |
| return self.build.product_bundle is not None |
| |
| def package_name(self) -> str | None: |
| """Get the package name for this test if applicable. |
| |
| If the test in question is a host test, returns None. |
| |
| Returns: |
| str | None: Package name if this is a device test, None otherwise. |
| """ |
| if self.build.test.package_url is None: |
| return None |
| m = _PACKAGE_REGEX.findall(self.build.test.package_url) |
| return m[0] if m else None |
| |
| @classmethod |
| def augment_tests_with_info( |
| _cls, |
| test_entries: list[typing.Self], |
| test_list_entries: dict[str, TestListEntry], |
| ) -> None: |
| """Augment a list of Tests with info fields. |
| |
| Args: |
| test_entries (list[Test]): A list of Tests to augment. |
| test_list_entries (Dict[str, TestListEntry]): The contents parsed from test-list.json |
| |
| Raises: |
| ValueError: If a test from tests.json does not have a corresponding entry in test-list.json. |
| |
| Returns: |
| List[Test]: List of joined contents for all tests in tests.json. |
| """ |
| try: |
| for entry in test_entries: |
| entry.join_info(test_list_entries) |
| except KeyError as e: |
| raise ValueError( |
| f"Test '{e.args[0]} was found in " |
| + "tests.json, but not test-list.json.\nYou may need to run " |
| + "`fx build :test-list` or a full `fx build`." |
| ) |