| # Copyright 2018 The Fuchsia Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| """Recipe module that wraps the testsharder tool, which searches a Fuchsia build for |
| test specifications and groups them into shards. |
| |
| The tool assigns a unique name to each produced shard. |
| |
| Testsharder tool: |
| https://fuchsia.googlesource.com/fuchsia/+/refs/heads/master/tools/integration/testsharder/cmd/ |
| """ |
| |
| import re |
| |
| import attr |
| |
| from recipe_engine import recipe_api |
| |
| |
| # TODO(fxbug.dev/51896): The recipes shouldn't know the schema of tests.json. |
| # They should pass the JSON objects straight through with no deserialization. |
| class Test(object): |
| """Represents a test defined within the Fuchsia project. |
| |
| Args: |
| os (str): The OS on which the test is intended to run. |
| path (str): The path to the test. Only makes sense for Linux and Mac |
| tests. |
| other_fields (dict): Fields not used by the recipes that will be passed |
| straight through. |
| """ |
| |
| def __init__(self, os, path="", **other_fields): |
| self.os = os |
| self.path = path |
| self._other_fields = other_fields |
| |
| @classmethod |
| def from_jsonish(cls, jsond): |
| return cls(**jsond) |
| |
| def render_to_jsonish(self): |
| """Returns a JSON-compatible dict representing the Test. |
| |
| The format follows the format of testsharder.Test found here: |
| https://fuchsia.googlesource.com/fuchsia/+/master/tools/integration/testsharder/test.go |
| """ |
| ret = {"os": self.os, "path": self.path} |
| ret.update(self._other_fields) |
| return ret |
| |
| |
| class Shard(object): |
| """Represents a shard of several tests with one common environment.""" |
| |
| @classmethod |
| def from_jsonish(cls, jsond): |
| """Creates a new Shard from a JSON-compatible Python dict.""" |
| return cls( |
| name=jsond["name"], |
| tests=[Test.from_jsonish(test) for test in jsond["tests"]], |
| dimensions=jsond["environment"]["dimensions"], |
| service_account=jsond["environment"].get("service_account", ""), |
| deps=jsond.get("deps", ()), |
| netboot=jsond["environment"].get("netboot", False), |
| ) |
| |
| def __init__( |
| self, name, tests, dimensions, service_account="", deps=(), netboot=False |
| ): |
| """Initializes a Shard. |
| |
| Args: |
| name (str): The name of the shard. |
| tests (seq[Test]): A sequence of tests. |
| dimensions (dict[str]str): A dictionary of Swarming dimensions, mapping |
| name to value. |
| service_account (str): A service account to be attached to a swarming |
| task when this shard is run. |
| deps (seq(str)): The runtime dependencies of the tests in the shard |
| given as relative paths within the build directory. This field |
| only makes sense for Linux and Mac tests. |
| netboot (bool): Whether to netboot instead of paving before running the |
| tests. |
| """ |
| assert isinstance(name, str) |
| self._name = name |
| self._tests = tests |
| self._dimensions = {} |
| for k, v in dimensions.iteritems(): |
| assert isinstance(k, str) |
| self._dimensions[k] = str(v) |
| self._service_account = service_account |
| self._netboot = netboot |
| self._deps = tuple(deps) |
| |
| @property |
| def name(self): |
| """The name (str) of the shard.""" |
| return self._name |
| |
| @property |
| def tests(self): |
| """The list of tests (list(Test)) in the shard.""" |
| return self._tests |
| |
| @property |
| def dimensions(self): |
| """The Swarming dimensions (dict[str]str) specified by the shard.""" |
| return self._dimensions.copy() |
| |
| def __getattr__(self, dimension): |
| """Returns the value of a given dimension is set in the shard; else, None.""" |
| return self._dimensions.get(dimension) |
| |
| @property |
| def targets_fuchsia(self): |
| """Returns whether the shard of tests is meant to run on or against fuchsia.""" |
| return self.os not in ["Linux", "Mac"] |
| |
| @property |
| def service_account(self): |
| """Returns any service account (string) to be attached to a swarming task when |
| this shard is run. |
| |
| Empty if unset. |
| """ |
| return self._service_account |
| |
| @property |
| def deps(self): |
| """The runtime dependencies (tuple(str)) of the test. |
| |
| The dependencies are given as relative paths within the build |
| directory and are only specified when test's OS is Linux or Mac. |
| """ |
| return self._deps |
| |
| @property |
| def netboot(self): |
| """Return whether to netboot instead of paving before running the tests.""" |
| return self._netboot |
| |
| def render_to_jsonish(self): |
| """Returns a JSON-compatible dict representing the Shard. |
| |
| The format follows the format os testsharder.Shard found here: |
| https://fuchsia.googlesource.com/fuchsia/+/master/tools/integration/testsharder/shard.go |
| """ |
| ret = { |
| "name": self.name, |
| "tests": [test.render_to_jsonish() for test in self.tests], |
| "environment": {"dimensions": self.dimensions}, |
| "deps": self.deps, |
| } |
| if self.service_account: |
| ret["environment"]["service_account"] = self.service_account |
| if self.netboot: |
| ret["environment"]["netboot"] = self.netboot |
| return ret |
| |
| |
| @attr.s(frozen=True) |
| class TestModifier(object): |
| name = attr.ib(type=str) |
| os = attr.ib(type=str, converter=lambda x: x or "", default="") |
| total_runs = attr.ib(type=int, converter=lambda x: int(x) if x else 0, default=0) |
| affected = attr.ib(type=bool, default=False) |
| max_attempts = attr.ib(type=int, default=1) |
| |
| def render_to_jsonish(self): |
| return attr.asdict(self) |
| |
| def merge(self, mod): |
| """Returns a new TestModifier with merged fields from mod and self. |
| |
| Except for `total_runs`, the same field cannot be set to different |
| non-default values for both modifiers. For `total_runs`, we pick the |
| larger value. |
| """ |
| fields = {} |
| for field in attr.fields(TestModifier): |
| v1 = getattr(self, field.name) |
| v2 = getattr(mod, field.name) |
| if field.name in ["total_runs", "max_attempts"]: |
| # A total_runs of 0 means to run as many times as will fit in one |
| # shard whereas a negative total_runs value means to not multiply it |
| # at all. Thus, we should take the value of 0 over a negative value. |
| fields[field.name] = max(v1, v2) |
| continue |
| assert not ( |
| v1 and v2 and v1 != v2 |
| ), "multiple modifiers have conflicting values for the same test" |
| fields[field.name] = v1 or v2 |
| return TestModifier(**fields) |
| |
| |
| class TestsharderApi(recipe_api.RecipeApi): |
| """Module for interacting with the Testsharder tool. |
| |
| The testsharder tool accepts a set of test specifications and |
| produces a file containing shards of execution. |
| """ |
| |
| Test = Test |
| Shard = Shard |
| TestModifier = TestModifier |
| |
| def __init__(self, *args, **kwargs): |
| super(TestsharderApi, self).__init__(*args, **kwargs) |
| self._testsharder_path = None |
| |
| def execute( |
| self, |
| step_name, |
| testsharder_path, |
| build_dir, |
| max_shard_size=None, |
| target_duration_secs=0, |
| max_shards_per_env=0, |
| mode=None, |
| modifiers=None, |
| output_file=None, |
| tags=(), |
| use_affected_tests=False, |
| affected_tests_file=None, |
| affected_tests_multiply_threshold=0, |
| affected_tests_max_attempts=0, |
| ): |
| """Executes the testsharder tool. |
| |
| Args: |
| step_name (str): name of the step. |
| testsharder_path (Path): path to the testsharder tool. |
| build_dir (Path): path to a Fuchsia build directory root, in |
| which GN has been run (ninja need not have been executed). |
| max_shard_size (long or None): Additional shards will be created if needed |
| to keep the number of tests per shard <= this number. |
| target_duration_secs (int): If >0, testsharder will try to produce |
| shards of approximately this duration. |
| max_shards_per_env (int): Testsharder will limit each environment to |
| this many shards, regardless of the values of max_shard_size and |
| target_duration_secs. If 0, testsharder will use its hardcoded |
| default max; if <0, no max will be set. |
| mode (str or None): mode in which the testsharder may run (e.g., |
| normal or restricted); if None, the default normal mode will |
| be used. |
| See https://fuchsia.googlesource.com/fuchsia/+/master/tools/integration/testsharder/mode.go |
| for more details |
| modifiers (list(TestModifier)): the test modifiers specifying |
| tests to run multiple times or to label as affected tests; if |
| supplied, new shards will be created to run them. |
| output_file (Path): optional file path to leak output to. |
| tags (list(str)): tags on which to filter test specs. |
| use_affected_tests (bool): whether or not to pass affected tests flags to testsharder. |
| affected_tests_file (Path): path to file containing affected tests. |
| affected_tests_multiply_threshold (int): if there are <= this many affected |
| tests, they'll be considered for multiplication. |
| affected_tests_max_attempts (int): max attempts for affected tests which are |
| not multiplied. |
| |
| Returns: |
| A list of Shards, each representing one test shard. |
| """ |
| cmd = [ |
| testsharder_path, |
| "-build-dir", |
| build_dir, |
| "-output-file", |
| self.m.json.output(leak_to=output_file), |
| ] |
| if max_shard_size: |
| cmd += ["-max-shard-size", max_shard_size] |
| if target_duration_secs: |
| cmd += ["-target-duration-secs", target_duration_secs] |
| if max_shards_per_env: |
| # TODO(10404): this conditional is only necessary for backwards |
| # compatibility; by only setting this flag when we want to use a value |
| # that's not the testsharder default, we can avoid breaking builds that |
| # run against old versions of fuchsia.git from before this flag was |
| # implemented. Once backwards compatibility is no longer needed and/or |
| # recipe versioning is complete, we can remove the default flag value |
| # from testsharder into the specs, and unconditionally set this flag. |
| cmd += ["-max-shards-per-env", max_shards_per_env] |
| if modifiers: |
| input_json = [ |
| m.render_to_jsonish() for m in self._combine_modifiers(modifiers) |
| ] |
| cmd += ["-modifiers", self.m.json.input(input_json)] |
| for tag in tags: |
| cmd += ["-tag", tag] |
| if mode: |
| cmd += ["-mode", mode] |
| |
| if use_affected_tests: |
| cmd += [ |
| "-affected-tests", |
| affected_tests_file, |
| "-affected-tests-max-attempts", |
| affected_tests_max_attempts, |
| "-affected-tests-multiply-threshold", |
| affected_tests_multiply_threshold, |
| ] |
| try: |
| step = self.m.step(step_name, cmd, stdout=self.m.raw_io.output_text()) |
| finally: |
| self.m.step.active_result.presentation.step_summary_text = ( |
| self.m.step.active_result.stdout |
| ) |
| return [Shard.from_jsonish(shard) for shard in step.json.output] |
| |
| def _combine_modifiers(self, modifiers): |
| """Combine modifiers so there is one per test name.""" |
| mod_map = {} |
| for m in modifiers: |
| if m.name in mod_map: |
| mod_map[m.name] = mod_map[m.name].merge(m) |
| else: |
| mod_map[m.name] = m |
| return mod_map.itervalues() |
| |
| def affected_test_modifiers(self, tests, max_attempts): |
| """Returns TestModifiers for the given tests with the affected property set. |
| |
| The total_runs should be set to a negative number so that the test |
| doesn't get counted as a multiplier. |
| """ |
| return [ |
| TestModifier( |
| name=test, total_runs=-1, affected=True, max_attempts=max_attempts |
| ) |
| for test in tests |
| ] |
| |
| def extract_multipliers(self, commit_message): |
| """Extracts the value of the "Multiply/MULTIPLY" footer. |
| |
| We support two multiply syntaxes: |
| 1. Raw JSON: |
| Multiply: `[ |
| { |
| "name": "test_name", |
| "os": "linux", |
| "total_runs": 123 |
| } |
| ]` |
| 2. "Friendly": |
| Multiply: testsharder_tests (linux): 123, foo_tests (fuchsia): 5 |
| |
| Args: |
| commit_message (str): The commit message in which to search for the footer. |
| |
| Returns: |
| A list of `TestModifier`, which will be empty if the commit message |
| doesn't contain the footer. |
| """ |
| multipliers = [] |
| |
| prefix = r"^\s*(MULTIPLY|Multiply)\s*[:=]\s*" |
| json_multiplier_regex = prefix + r"[\s\n]*`(?P<json>(.|\n)*?)`" |
| json_match = re.search(json_multiplier_regex, commit_message, re.MULTILINE) |
| if json_match: |
| multiplier_json = self.m.jsonutil.permissive_loads(json_match.group("json")) |
| for raw_multiplier in multiplier_json: |
| try: |
| multiplier = TestModifier(**raw_multiplier) |
| except (ValueError, TypeError): |
| raise self.m.step.StepFailure( |
| "invalid test multiplier: %s" |
| % self.m.json.dumps(raw_multiplier) |
| ) |
| multipliers.append(multiplier) |
| return multipliers |
| |
| friendly_multiplier_regex = re.compile( |
| r""" |
| # Required name. It cannot contain commas, spaces, or parentheses, and |
| # must not end with a colon. |
| \s* |
| (?P<name>[^,\(\s]+)(?<!:) |
| \s* |
| # Optional "os" in parentheses |
| ( |
| \( |
| \s* |
| (?P<os>\w*) |
| \s* |
| \) |
| )? |
| \s* |
| # Optional "total_runs" integer, preceded by a colon and at least one |
| # space (to distinguish this colon from any colons in "name"). |
| ( |
| :\s+ |
| (?P<total_runs>\d+) |
| )? |
| \s* |
| """, |
| re.VERBOSE, |
| ) |
| |
| for line in commit_message.splitlines(): |
| line = line.strip() |
| footer_match = re.match(prefix, line) |
| if not footer_match: |
| continue |
| |
| raw_multipliers = line[footer_match.end() :].strip(" ,").split(",") |
| for raw_multiplier in raw_multipliers: |
| raw_multiplier = raw_multiplier.strip() |
| match = friendly_multiplier_regex.match(raw_multiplier) |
| if not match: |
| raise self.m.step.StepFailure( |
| "invalid multiplier %r" % raw_multiplier |
| ) |
| multipliers.append(TestModifier(**match.groupdict())) |
| |
| return multipliers |
