| # Copyright 2019 The Fuchsia Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| import copy |
| |
| import attr |
| |
| from recipe_engine import recipe_api |
| from RECIPE_MODULES.fuchsia.gce import api as gce_api |
| |
| # System path at which authorized SSH keys are stored. |
| AUTHORIZED_KEY_PATH = "data/ssh/authorized_keys" |
| # The path to the botanist config on the host. |
| BOTANIST_DEVICE_CONFIG = "/etc/botanist/config.json" |
| # The log level to use for botanist invocations in test tasks. Can be one of |
| # "fatal", "error", "warning", "info", "debug", or "trace", where "trace" is |
| # the most verbose, and fatal is the least. |
| BOTANIST_LOG_LEVEL = "trace" |
| # Name of image manifest produced by the build. |
| IMAGES_JSON = "images.json" |
| |
| _PRIVATE_KEY_PATH = "private_key" |
| |
| CATALYST_CIPD_REVISION = "git_revision:d0f1c217a82b83a5566b8a66427a8c80a75c0293" |
| GCSPROXY_CIPD_REVISION = "git_revision:0cc1113899f4b755d22fe878723270fb24e65af7" |
| |
| SERIAL_LOG_NAME = "serial_log.txt" |
| SYSLOG_NAME = "syslog.txt" |
| SERIAL_NAME = "serial.log" |
| SNAPSHOT_NAME = "snapshot.zip" |
| |
| |
| @attr.s |
| class TaskRequester(object): |
| """Creates requests for swarming tasks that run tests.""" |
| |
| _api = attr.ib() |
| _buildbucket_build = attr.ib() |
| _per_test_timeout_secs = attr.ib(type=int) |
| _pave = attr.ib(type=bool) |
| _pool = attr.ib(type=str) |
| _swarming_expiration_timeout_secs = attr.ib(type=int) |
| _swarming_io_timeout_secs = attr.ib(type=int) |
| _timeout_secs = attr.ib(type=int) |
| _use_runtests = attr.ib(type=bool) |
| _default_service_account = attr.ib(type=str) |
| _targets_serial = attr.ib(type=bool) |
| _catapult_dashboard_master = attr.ib(type=str) |
| _catapult_dashboard_bot = attr.ib(type=str) |
| _release_branch = attr.ib(type=str) |
| _release_version = attr.ib(type=str) |
| _zircon_args = attr.ib(type=list) |
| _gcem_cloud_project = attr.ib(type=str, default=None) |
| _gcem_host = attr.ib(type=str, default=None) |
| |
| def testrunner_request(self, shard, build_results): |
| """Returns a swarming.TaskRequest that uses `testrunner` to run tests. |
| |
| It constructs the appropriate testrunner command to run and passes that |
| as the subcommand to TaskRequester.request() which creates the task |
| request. |
| |
| Args: |
| shard (testsharder.Shard): Test shard. |
| build_results (FuchsiaBuildResults): The Fuchsia build results to test. |
| """ |
| # Copy since we modify it for each shard. |
| build_results = copy.deepcopy(build_results) |
| |
| # To freely archive files from the build directory, the source, and those we |
| # dynamically create, we create a tree of symlinks in a fresh directory and |
| # isolate that. This solves the problems of (a) finding a root directory |
| # that works for all artifacts, (b) being able to create files in that |
| # directory without fear of collision, and (c) not having to isolate |
| # extraneous files. |
| isolate_tree = self._api.file.symlink_tree( |
| root=self._api.path.mkdtemp("isolate") |
| ) |
| |
| test_manifest = "tests.json" |
| self._api.file.write_json( |
| "write test manifest", |
| isolate_tree.root.join(test_manifest), |
| shard.tests, |
| indent=2, |
| ) |
| image_manifest = "%s/%s" % (self._api.artifacts.image_url(), IMAGES_JSON) |
| cmd = [ |
| "./testrunner", |
| "-level", |
| "debug", |
| "-out-dir", |
| self._api.testing_requests.TEST_RESULTS_DIR_NAME, |
| "-snapshot-output", |
| SNAPSHOT_NAME, |
| ] |
| if self._use_runtests: |
| cmd.append("-use-runtests") |
| if self._per_test_timeout_secs: |
| cmd.extend(["-per-test-timeout", "%ds" % self._per_test_timeout_secs]) |
| cmd.append(test_manifest) |
| |
| outputs = [self._api.testing_requests.TEST_RESULTS_DIR_NAME] |
| isolate_tools = ["testrunner"] |
| is_emu_type = self._api.emu.is_emulator_type(shard.device_type) |
| test_bot_cpu = build_results.set_metadata.target_arch if is_emu_type else "x64" |
| if test_bot_cpu == "x64": |
| # Relevant for automatic symbolization of things running on host. Only |
| # the x64 variation is available in the checkout and we have nothing |
| # that runs on an arm host that needs symbolizing. |
| isolate_tools.append("llvm-symbolizer") |
| |
| # If targeting emu we include a private key corresponding to an authorized |
| # key already in the boot image; this is needed as we do not pave emu. |
| if is_emu_type and self._pave: |
| isolate_tree.register_link( |
| target=build_results.private_key, |
| linkname=isolate_tree.root.join(_PRIVATE_KEY_PATH), |
| ) |
| # Used to dynamically extend fvm.blk, which is relevant only to emu. |
| isolate_tools.append("fvm") |
| |
| return self.request( |
| shard, |
| build_results, |
| cmd, |
| isolate_tree, |
| isolate_tools, |
| image_manifest, |
| BOTANIST_LOG_LEVEL, |
| outputs, |
| ) |
| |
| def request( |
| self, |
| shard, |
| build_results, |
| subcmd, |
| isolate_tree, |
| isolate_tools, |
| image_manifest, |
| log_level, |
| subcmd_outputs=[], |
| ): |
| """Returns a swarming.TaskRequest for the specified shard. |
| |
| This function handles all the common logic for setting up test tasks |
| such as isolating the necessary tools and constructing the proper |
| botanist command depending on the details of the provided shard and |
| build_results. |
| |
| Args: |
| shard (testsharder.Shard): Test shard. |
| build_results (FuchsiaBuildResults): The Fuchsia build results to test. |
| subcmd (list(str)): The subcommand to run botanist with on the task. |
| isolate_tree (api.file.SymlinkTree): The tree into which the isolated |
| inputs for the task are linked. |
| isolate_tools (list(str)): A list of tools to add to the isolated |
| inputs. |
| image_manifest (str): The image manifest path or URL to pass to the |
| botanist command. |
| log_level (str): Passed to the botanist command. |
| subcmd_outputs (list(str)): The expected outputs of the subcmd, added |
| to the expected outputs of the task. |
| """ |
| cmd = [] |
| outputs = [] |
| ensure_file = self._api.cipd.EnsureFile() |
| dimensions = {"pool": self._pool} |
| is_emu_type = self._api.emu.is_emulator_type(shard.device_type) |
| is_gce_type = shard.device_type == "GCE" |
| # To take advantage of KVM, we execute emu-arm tasks on arm hardware. |
| test_bot_cpu = build_results.set_metadata.target_arch if is_emu_type else "x64" |
| |
| # TODO(fxbug.dev/40840): In the hardware case, we cannot yet run the GCS |
| # proxy server in the task (it must be run outside of the container on the |
| # controller). In this case we rely on provisioning logic to set |
| # $GCS_PROXY_HOST in the environment so that we may pass this to botanist. |
| # Once we can scope the server in the task, delete this codepath and do as |
| # we do in the emu case. |
| # |
| # Note: in the emu case, we pass the placeholder of "localhost" to |
| # botanist. This is because we cannot know an absolute address ahead of |
| # time. Botanist has logic to resolve this address and scope it for the |
| # host and target accordingly. |
| # |
| # In the testing on GCE case, ephemerality is still unsupported, so |
| # leave gcsproxy out. |
| if is_emu_type: |
| gcsproxy_port = "8080" |
| gcsproxy_host = "localhost:%s" % gcsproxy_port |
| elif not is_gce_type: |
| gcsproxy_host = "$GCS_PROXY_HOST" |
| |
| # This command spins up a metadata server that allows its subcommands to |
| # automagically authenticate with LUCI auth, provided the sub-exec'ed tool |
| # was written in go or dart and respectively makes use of the standard |
| # cloud.google.com/go/compute/metadata or |
| # github.com/dart-lang/googleapis_auth authentication libraries. Such |
| # libraries look for a metadata server under environment variables |
| # like $GCE_METADATA_HOST, which LUCI emulates. |
| service_account = shard.service_account or self._default_service_account |
| if service_account: |
| # TODO(fxbug.dev/37142): Find a way to use the version that LUCI is |
| # currently using, instead of 'latest'. |
| ensure_file.add_package("infra/tools/luci-auth/${platform}", "latest") |
| cmd.extend(["./luci-auth", "context", "--"]) |
| |
| if is_emu_type: |
| dimensions.update( |
| os="Debian", cpu=build_results.set_metadata.target_arch, kvm="1" |
| ) |
| elif is_gce_type: |
| # Have any GCE shards target the GCE executors, which are e2-2 |
| # machines running Linux. |
| dimensions.update(os="Linux", cores="2", gce="1") |
| else: |
| # No device -> no serial. |
| if self._targets_serial and shard.device_type: |
| dimensions["serial"] = "1" |
| dimensions.update(shard.dimensions) |
| |
| # Ensure we use GCE VMs whenever possible. |
| is_linux = not shard.os or shard.os.lower() == "linux" |
| if is_linux and not is_gce_type: |
| dimensions["kvm"] = "1" |
| if ( |
| (is_emu_type or not shard.device_type) |
| and test_bot_cpu == "x64" |
| and is_linux |
| ): |
| dimensions["gce"] = "1" |
| dimensions["cores"] = "8" |
| |
| if shard.targets_fuchsia: |
| if "bringup" not in build_results.set_metadata.product and is_emu_type: |
| # TODO(fxbug.dev/40840): As mentioned above, once this bug is resolved. |
| # we should run gcsproxy regardless of is_emu_type. |
| ensure_file.add_package( |
| "fuchsia/infra/gcsproxy/${platform}", GCSPROXY_CIPD_REVISION |
| ) |
| cmd.extend( |
| ["./gcsproxy", "-port", gcsproxy_port,] |
| ) |
| if not is_emu_type and not is_gce_type: |
| # catalyst is needed for testing on physical hardware. |
| ensure_file.add_package( |
| "fuchsia/infra/catalyst/${platform}", CATALYST_CIPD_REVISION |
| ) |
| cmd.append("./catalyst") |
| |
| cmd.extend( |
| [ |
| "./botanist", |
| "-level", |
| log_level, |
| "run", |
| "-images", |
| image_manifest, |
| "-timeout", |
| "%ds" % self._timeout_secs, |
| ] |
| ) |
| |
| # In the emulator case, serial is redirected to stdio. |
| if not is_emu_type: |
| cmd.extend(["-serial-log", SERIAL_LOG_NAME]) |
| outputs.append(SERIAL_LOG_NAME) |
| |
| if self._pave: |
| cmd.extend( |
| ["-syslog", SYSLOG_NAME,] |
| ) |
| # Testing on GCE does not yet support ephemerality. |
| if not is_gce_type: |
| cmd.extend( |
| [ |
| "-repo", |
| self._api.artifacts.package_repo_url(host=gcsproxy_host), |
| "-blobs", |
| self._api.artifacts.package_blob_url(host=gcsproxy_host), |
| ] |
| ) |
| outputs.append(SYSLOG_NAME) |
| |
| if is_emu_type: |
| cmd.extend(["-ssh", _PRIVATE_KEY_PATH]) |
| else: |
| cmd.append("-netboot") |
| |
| for arg in self._zircon_args: |
| cmd.extend(["-zircon-args", arg]) |
| |
| config = BOTANIST_DEVICE_CONFIG |
| if is_emu_type: |
| config = "./qemu.json" |
| qemu_config = [ |
| { |
| "type": shard.device_type.lower(), |
| "path": "./%s/bin" % shard.device_type.lower(), |
| "target": build_results.set_metadata.target_arch, |
| "cpu": 4, |
| "memory": self._api.emu.get_memory_for_variant(build_results), |
| "kvm": True, |
| # Is a directive to run the emu process in a way in which we can |
| # synthesize a 'serial device'. We need only do this in the bringup |
| # case, this being used for executing tests at that level; |
| # restriction to the minimal case is especially important as this |
| # mode shows tendencies to slow certain processes down. |
| "serial": not self._pave, |
| # Isolated to the root below in _isolate_build_artifacts(). |
| # Used to dynamically extend fvm.blk to fit downloaded test |
| # packages. |
| "fvm_tool": "fvm" if self._pave else "", |
| "logfile": SERIAL_NAME, |
| } |
| ] |
| outputs.append(SERIAL_NAME) |
| |
| if shard.device_type == "AEMU": |
| self._api.emu.add_aemu_to_ensure_file( |
| ensure_file, |
| checkout=build_results.checkout.root_dir, |
| subdir="aemu/bin", |
| ) |
| elif shard.device_type == "QEMU": |
| self._api.emu.add_qemu_to_ensure_file( |
| ensure_file, |
| checkout=build_results.checkout.root_dir, |
| subdir="qemu", |
| ) |
| |
| self._api.file.write_json( |
| "write qemu config", |
| isolate_tree.root.join("qemu.json"), |
| qemu_config, |
| indent=2, |
| ) |
| elif is_gce_type: |
| config = "./gce.json" |
| ensure_file.add_package( |
| gce_api.GCEM_CLIENT_CIPD_PATH, gce_api.GCEM_CLIENT_CIPD_REVISION, |
| ) |
| self._api.gce.create_botanist_config( |
| self._gcem_host, |
| self._gcem_cloud_project, |
| self._api.buildbucket.build.infra.swarming.parent_run_id, |
| isolate_tree.root.join("gce.json"), |
| ) |
| cmd.extend(["-config", config]) |
| |
| cmd.extend(subcmd) |
| |
| outputs.extend(subcmd_outputs) |
| |
| isolated_hash = self._api.testing_requests._isolate_build_artifacts( |
| isolate_tree, |
| build_results, |
| shard=shard, |
| test_bot_cpu=test_bot_cpu, |
| extra_tools=isolate_tools, |
| ) |
| |
| tags = self._api.testing_requests.test_task_tags( |
| self._buildbucket_build, |
| build_results, |
| env_name="%s-%s" |
| % (shard.device_type or shard.os, build_results.set_metadata.target_arch), |
| task_name=shard.name, |
| ) |
| |
| request = ( |
| self._api.swarming.task_request() |
| .with_name(shard.name) |
| .with_tags(self._api.testing_requests.create_swarming_tags(tags)) |
| ) |
| if service_account: |
| request = request.with_service_account(service_account) |
| return request.with_slice( |
| 0, |
| request[0] |
| .with_command(cmd) |
| .with_isolated(isolated_hash) |
| .with_dimensions(**dimensions) |
| .with_expiration_secs(self._swarming_expiration_timeout_secs) |
| .with_io_timeout_secs(self._swarming_io_timeout_secs) |
| .with_execution_timeout_secs(self._timeout_secs) |
| .with_outputs(outputs) |
| .with_cipd_ensure_file(ensure_file) |
| .with_env_vars( |
| **self._api.testing_requests.test_task_env_vars( |
| self._buildbucket_build, |
| shard.device_type, |
| build_results, |
| catapult_dashboard_master=self._catapult_dashboard_master, |
| catapult_dashboard_bot=self._catapult_dashboard_bot, |
| release_branch=self._release_branch, |
| release_version=self._release_version, |
| image_manifest=image_manifest, |
| ) |
| ), |
| ) |
| |
| |
| class TestingRequestsApi(recipe_api.RecipeApi): |
| """APIs for constructing Swarming task requests to test Fuchsia.""" |
| |
| SERIAL_LOG_NAME = SERIAL_LOG_NAME |
| TEST_RESULTS_DIR_NAME = "out" |
| |
| # The name of the tag to set on every task request that contains the name |
| # of the shard's environment (device type/OS and architecture). |
| TEST_ENVIRONMENT_TAG_NAME = "test_environment_name" |
| |
| def task_requests( |
| self, |
| build_results, |
| buildbucket_build, |
| per_test_timeout_secs, |
| pool, |
| shards, |
| swarming_expiration_timeout_secs, |
| swarming_io_timeout_secs, |
| use_runtests, |
| timeout_secs, |
| default_service_account=None, |
| pave=True, |
| targets_serial=False, |
| catapult_dashboard_master=None, |
| catapult_dashboard_bot=None, |
| release_branch=None, |
| release_version=None, |
| zircon_args=(), |
| gcem_host=None, |
| gcem_cloud_project=None, |
| ): |
| """Returns a swarming.TaskRequest for each shard in build_artifact.shards. |
| |
| Args: |
| build_results (FuchsiaBuildResults): The Fuchsia build results to test. |
| buildbucket_build (build_pb2.Build): The buildbucket build that is going |
| to orchestrate testing. |
| per_test_timeout_secs (int): Any test that executes for longer than this |
| will be considered failed. |
| pool (str): The Swarming pool to schedule test tasks in. |
| shards (list of testsharder.Shard): Test shards. |
| use_runtests (bool): Whether to use runtests (or else run_test_component) |
| when executing tests on target. |
| timeout_secs (int): The amount of seconds to wait for the tests to execute |
| before giving up. |
| default_service_account (str or None): The default service account to run the |
| test task with. This is required for fetching images from GCS. |
| pave (bool): Whether to pave (or else 'netboot') the system; this is |
| effectively equivalent to "not bringup" and is treated as such (even for |
| QEMU). |
| targets_serial (bool): Whether the task should target a bot with serial |
| enabled. |
| release_branches (str or None): The release branch corresponding to |
| the checkout. Passed as a task environment variable. |
| release_version (str or None): The release version that we checked out. |
| Passed as a task environment variable. |
| zircon_args (list(str)): Kernel command-line arguments to pass on boot. |
| gcem_host (str): The endpoint the GCE Mediator can be found at. |
| gcem_cloud_project (str): The cloud project the Mediator should create VMs in. |
| """ |
| # Embed the authorized key into the appropriate ZBI. This enabled us to SSH |
| # into QEMU, in which case we are unable to supply the key at pave-time (as |
| # QEMU instances are not paved.) |
| has_emu_shard = any( |
| self.m.emu.is_emulator_type(shard.device_type) for shard in shards |
| ) |
| if pave and has_emu_shard: |
| self.m.zbi.zbi_path = build_results.zbi |
| zbi_path = build_results.build_dir.join(self._zbi_path(build_results)) |
| |
| self.m.zbi.copy_and_extend( |
| step_name="embed authorized key", |
| input_image=zbi_path, |
| output_image=zbi_path, |
| manifest={AUTHORIZED_KEY_PATH: build_results.authorized_key}, |
| ) |
| |
| task_requester = self.get_task_requester( |
| buildbucket_build=buildbucket_build, |
| per_test_timeout_secs=per_test_timeout_secs, |
| pave=pave, |
| pool=pool, |
| swarming_expiration_timeout_secs=swarming_expiration_timeout_secs, |
| swarming_io_timeout_secs=swarming_io_timeout_secs, |
| timeout_secs=timeout_secs, |
| use_runtests=use_runtests, |
| default_service_account=default_service_account, |
| targets_serial=targets_serial, |
| catapult_dashboard_master=catapult_dashboard_master, |
| catapult_dashboard_bot=catapult_dashboard_bot, |
| release_branch=release_branch, |
| release_version=release_version, |
| zircon_args=zircon_args, |
| gcem_host=gcem_host, |
| gcem_cloud_project=gcem_cloud_project, |
| ) |
| task_requests = [] |
| for s in shards: |
| if s.device_type == "GCE" and not self.m.experimental.test_on_gce: |
| # Do not generate a task request for GCE shards if the |
| # test_on_gce flag is disabled. |
| continue |
| with self.m.step.nest("shard %s" % s.name): |
| task_requests.append( |
| task_requester.testrunner_request(s, build_results) |
| ) |
| return task_requests |
| |
| def get_task_requester(self, **kwargs): |
| """Returns a TaskRequester with self.m as the api. |
| |
| This allows all dependencies of TaskRequester to only need to be |
| imported into the testing_requests module. |
| """ |
| return TaskRequester(self.m, **kwargs) |
| |
| def test_task_env_vars( |
| self, |
| build, |
| device_type, |
| build_results, |
| image_manifest, |
| catapult_dashboard_master=None, |
| catapult_dashboard_bot=None, |
| release_branch=None, |
| release_version=None, |
| ): |
| """Returns the environment variables to be set for the test task. |
| |
| Returns: |
| A dict mapping string env var names to string values. |
| """ |
| # build = self.m.buildbucket.build |
| commit = build.input.gitiles_commit |
| llvm_symbolizer = self.m.path.basename(build_results.llvm_symbolizer) |
| env_vars = dict( |
| # `${ISOLATED_OUTDIR}` is a magic string that Swarming will replace |
| # with a temporary directory into which files will be automatically |
| # collected upon exit of a task. |
| FUCHSIA_TEST_OUTDIR="${ISOLATED_OUTDIR}", |
| # Don't set BUILDBUCKET_ID for builds run using led. |
| BUILDBUCKET_ID=str(build.id) if build.id else None, |
| BUILD_BOARD=build_results.set_metadata.board, |
| BUILD_TYPE=build_results.set_metadata.optimize, |
| BUILD_PRODUCT=build_results.set_metadata.product, |
| BUILD_TARGET=build_results.set_metadata.target_arch, |
| BUILDBUCKET_BUCKET=build.builder.bucket, |
| # Used for symbolization: |
| ASAN_SYMBOLIZER_PATH=llvm_symbolizer, |
| UBSAN_SYMBOLIZER_PATH=llvm_symbolizer, |
| LSAN_SYMBOLIZER_PATH=llvm_symbolizer, |
| # Used for generating data to upload to the Catapult performance |
| # dashboard. |
| # TODO(fxb/50210): Don't fall back to time.time() once led starts |
| # setting create_time again. |
| BUILD_CREATE_TIME=str(build.create_time.seconds or int(self.m.time.time())), |
| BUILDER_NAME=build.builder.builder, |
| FUCHSIA_DEVICE_TYPE=device_type, |
| INPUT_COMMIT_HOST=commit.host, |
| INPUT_COMMIT_PROJECT=commit.project, |
| INPUT_COMMIT_REF=commit.ref, |
| RELEASE_BRANCH=release_branch, |
| RELEASE_VERSION=release_version, |
| BOOTSERVER_PATH="./" |
| + self.m.path.basename(build_results.tool("bootserver_new")), |
| IMAGE_MANIFEST_PATH=image_manifest, |
| SWARMING_BOT_FILE="${SWARMING_BOT_FILE}", |
| ) |
| env_vars.update( |
| self.get_catapult_dashboard_env_vars( |
| catapult_dashboard_master, catapult_dashboard_bot, commit.ref |
| ) |
| ) |
| # For some reason, empty string environment variables sent to the swarming |
| # API get interpreted as null and rejected. So don't bother sending them to |
| # avoid breaking the task request. |
| # TODO(olivernewman): Figure out whether this logic should be moved into |
| # the upstream swarming module (or obviated by fixing the "" -> null |
| # behavior). |
| return {k: v for k, v in env_vars.iteritems() if v} |
| |
| def test_task_tags(self, buildbucket_build, build_results, env_name, task_name): |
| return { |
| "board": ( |
| build_results.set_metadata.board |
| or build_results.set_metadata.target_arch |
| ), |
| "build_type": build_results.set_metadata.optimize, |
| "buildbucket_bucket": buildbucket_build.builder.bucket, |
| "buildbucket_builder": buildbucket_build.builder.builder, |
| "product": build_results.set_metadata.product, |
| "role": "tester", |
| "task_name": task_name, |
| # Consumed by google3 results uploader, and by the orchestrator |
| # when uploading to resultdb. |
| self.TEST_ENVIRONMENT_TAG_NAME: env_name, |
| "variants": build_results.set_metadata.variants, |
| } |
| |
| def _isolate_build_artifacts( |
| self, |
| isolate_tree, |
| build_results, |
| shard=None, |
| test_bot_cpu="x64", |
| extra_tools=[], |
| ): |
| """Populates a tree with build artifacts and isolates it. |
| |
| Specifically, the following is linked into or created within the tree: |
| - The images in the build are linked in and manifest of them is created |
| in the root, if targeting a fuchsia device; |
| - The Linux/Mac tests in the shard and their runtime dependencies. |
| |
| Args: |
| isolate_tree (api.file.SymlinkTree): A tree into which artifacts may be |
| linked. |
| build_results (FuchsiaBuildResults): The result of a fuchsia build. |
| shard (api.testsharder.Shard or None): A test shard. |
| test_bot_cpu (str or None): The host cpu of the bot running the test task. |
| extra_tools (list(str)): A list of extra tools to isolate. |
| |
| Returns: |
| The isolated hash that may be used to reference and download the |
| artifacts. |
| """ |
| if shard: |
| for dep in shard.deps: |
| # Prepare a symlink of a relative path within the build |
| # directory to the tree. |
| isolate_tree.register_link( |
| target=build_results.build_dir.join(dep), |
| linkname=isolate_tree.root.join(dep), |
| ) |
| |
| # TODO(fxb/38517): s/bootserver_new/bootserver. |
| tools = ["botanist", "bootserver_new"] + extra_tools |
| |
| for tool_name in tools: |
| tool = build_results.tool(tool_name, cpu=test_bot_cpu) |
| isolate_tree.register_link( |
| target=tool, linkname=isolate_tree.root.join(tool_name) |
| ) |
| |
| isolate_tree.create_links("create tree of build artifacts") |
| isolated = self.m.isolated.isolated(isolate_tree.root) |
| isolated.add_dir(isolate_tree.root) |
| return isolated.archive("isolate build artifacts") |
| |
| def _zbi_path(self, build_results, pave=True): |
| zbi = None |
| bootserver_type = "bootserver_%s" % ("pave" if pave else "netboot") |
| for image in build_results.images: |
| if image["type"] == "zbi": |
| if "--boot" in image.get(bootserver_type, []): |
| zbi = image |
| break |
| elif image["name"] == "zircon-a": # pragma: no cover |
| zbi = image |
| |
| assert zbi, "No ZBI image found in image manifest" |
| return zbi["path"] |
| |
| def create_swarming_tags(self, tags): |
| """Creates a properly formatted tags dict to pass to a swarming task. |
| |
| Args: |
| tags (dict): A dictionary of key-value pairs of the desired tags. |
| |
| Returns: |
| A dictionary where the keys are strings and the values are lists of |
| strings. |
| """ |
| swarming_tags = {} |
| for k, val in tags.items(): |
| if not val: |
| val = [] |
| elif isinstance(val, basestring): |
| val = [val] |
| swarming_tags[str(k)] = [str(i) for i in val] |
| return swarming_tags |
| |
| # This is included in the API just so that it can be unit-tested. |
| def get_catapult_dashboard_env_vars(self, master_name, bot_name, commit_ref): |
| if not master_name and not bot_name: |
| # Uploading to Catapult is disabled. |
| return {} |
| if not (master_name and bot_name): |
| raise ValueError( |
| "Catapult master and bot names not set consistently: %r, %r" |
| % (master_name, bot_name) |
| ) |
| |
| prefix = "refs/heads/releases/" |
| if commit_ref.startswith(prefix): |
| branch_name = commit_ref[len(prefix) :] |
| master_name += "." + branch_name |
| elif commit_ref != "refs/heads/master": |
| # Unrecognized Git branch/tag name. Disable uploading to Catapult |
| # by not setting the env vars. |
| return {} |
| |
| return dict( |
| CATAPULT_DASHBOARD_MASTER=master_name, CATAPULT_DASHBOARD_BOT=bot_name |
| ) |
