| # Copyright 2019 The Fuchsia Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| import copy |
| import os |
| from collections import namedtuple |
| |
| # Test output for the step that reads the images.json file produced and isolated by the |
| # "builder" build. |
| TEST_IMAGES_JSON = { |
| 'zircon-a': { |
| 'path': 'fuchsia.zbi', |
| 'type': 'zbi', |
| 'name': 'zircon-a', |
| 'bootserver_pave': ['--boot', '--zircona'] |
| }, |
| 'netboot': { |
| 'path': 'netboot.zbi', |
| 'bootserver_netboot': ['--boot'], |
| 'type': 'zbi', |
| 'name': 'netboot' |
| } |
| } |
| |
| # Test output for the step that reads the args.json file produced and isolated by the |
| # "builder" build. |
| TEST_BUILD_ARGS_JSON = { |
| 'board': 'boards/x64.gni', |
| 'build_type': 'build_type', |
| 'product': 'products/core.gni', |
| 'target': 'x64', |
| 'variants': [], |
| } |
| |
| |
| class BuildArtifacts(object): |
| """The outputs of a Fuchsia build needed for testing. |
| |
| This object contains a collection of read-only paths to artifacts, tools, and metadata |
| produced by a Fuchsia build. It is used to isolate and download this data for Tester |
| builds. |
| """ |
| |
| # IsolatedLayout describes the directory structure to use when isolating |
| # BuildArtifacts. If an IsolatedLayout is used to isolate BuildArtifacts, that |
| # same IsolatedLayout should be used to download the inputs. |
| IsolatedLayout = namedtuple( |
| 'IsolatedLayout', |
| ('fuchsia_build_dir', 'symbolize_tool', 'llvm_symbolizer', |
| 'llvm_profdata', 'llvm_cov', 'minfs', 'zbi', 'covargs', 'botanist_x64', |
| 'testrunner_x64', 'bootserver', 'botanist_arm64', 'testrunner_arm64', |
| 'args', 'images', 'shards', 'ids', 'authorized_key', 'private_key', |
| 'secret_specs'), |
| ) |
| |
| # The default layout to use when isolating BuildArtifacts. This results in the following |
| # structure: |
| # ./ |
| # fuchsia/ |
| # symbolize_tool |
| # llvm_symbolizer |
| # llvm_profdata |
| # llvm_cov |
| # minfs |
| # bootserver |
| # zbi |
| # covargs |
| # x64/ |
| # botanist |
| # testrunner |
| # arm64/ |
| # botanist |
| # testrunner |
| # args.json |
| # images.json |
| # shards.json |
| # ids.txt |
| # authorized_key |
| # private_key |
| # secret_specs/ |
| DEFAULT_ISOLATED_LAYOUT = IsolatedLayout( |
| fuchsia_build_dir='fuchsia', |
| symbolize_tool='symbolize_tool', |
| llvm_symbolizer='llvm_symbolizer', |
| llvm_profdata='llvm_profdata', |
| llvm_cov='llvm_cov', |
| minfs='minfs', |
| bootserver='bootserver', |
| zbi='zbi', |
| covargs='covargs', |
| botanist_x64=os.path.join('x64', 'botanist'), |
| testrunner_x64=os.path.join('x64', 'testrunner'), |
| botanist_arm64=os.path.join('arm64', 'botanist'), |
| testrunner_arm64=os.path.join('arm64', 'testrunner'), |
| args='args.json', |
| images='images.json', |
| shards='shards.json', |
| ids='ids.txt', |
| authorized_key='authorized_key', |
| private_key='private_key', |
| secret_specs='secret_specs', |
| ) |
| |
| def __init__(self, fuchsia_build_dir, symbolize_tool, llvm_symbolizer, |
| llvm_profdata, llvm_cov, minfs, bootserver, zbi, covargs, |
| botanist_x64, testrunner_x64, botanist_arm64, testrunner_arm64, |
| target, board, product, variants, build_type, images, shards, |
| ids, authorized_key, private_key, secret_specs): |
| self._fuchsia_build_dir = fuchsia_build_dir |
| self._symbolize_tool = symbolize_tool |
| self._llvm_symbolizer = llvm_symbolizer |
| self._llvm_profdata = llvm_profdata |
| self._llvm_cov = llvm_cov |
| self._minfs = minfs |
| self._bootserver = bootserver |
| self._zbi = zbi |
| self._covargs = covargs |
| self._botanist_x64 = botanist_x64 |
| self._testrunner_x64 = testrunner_x64 |
| self._botanist_arm64 = botanist_arm64 |
| self._testrunner_arm64 = testrunner_arm64 |
| self._target = target |
| self._board = board |
| self._product = product |
| self._variants = variants |
| self._build_type = build_type |
| self._images = images |
| self._shards = shards |
| self._ids = ids |
| self._authorized_key = authorized_key |
| self._private_key = private_key |
| self._secret_specs = secret_specs |
| self._args_file = None |
| self._images_file = None |
| self._shards_file = None |
| |
| def __deepcopy__(self, memodict): |
| # Shallow copy first. |
| new = copy.copy(self) |
| # Only images needs to be a real deep copy. |
| new._images = copy.deepcopy(new._images, memodict) |
| return new |
| |
| @staticmethod |
| def download(api, isolated_hash): |
| """Restores BuildArtifacts from an isolated. |
| |
| Args: |
| api (RecipeApi): The Recipe API. |
| isolated_hash (string): The isolated hash to fetch. |
| |
| Returns: |
| A BuildArtifacts object using the given layout. |
| """ |
| with api.step.nest('download build artifacts'): |
| return _Isolator( |
| api, BuildArtifacts.DEFAULT_ISOLATED_LAYOUT).download(isolated_hash) |
| |
| def isolate(self, api): |
| """Isolates these BuildArtifacts. |
| |
| Args: |
| api (RecipeApi): The Recipe API. |
| |
| Returns: |
| A string isolated ID hash. |
| """ |
| layout = BuildArtifacts.DEFAULT_ISOLATED_LAYOUT |
| args = { |
| 'board': self.board, |
| 'build_type': self.build_type, |
| 'product': self.product, |
| 'target': self.target, |
| # Convert repeated proto field to list so it can be serialized to JSON. |
| 'variants': list(self.variants), |
| } |
| with api.step.nest('isolate build artifacts'): |
| self._args_file = api.path['start_dir'].join(layout.args) |
| api.file.write_json( |
| 'write %s' % layout.args, data=args, dest=self._args_file, indent=2) |
| |
| self._images_file = api.path['start_dir'].join(layout.images) |
| api.file.write_json( |
| 'write %s' % layout.images, |
| data=self._images, |
| dest=self._images_file, |
| indent=2) |
| |
| if not self._shards: |
| self._shards = [] |
| self._shards_file = api.path['start_dir'].join(layout.shards) |
| api.file.write_json( |
| 'write %s' % layout.shards, |
| data=[s.render_to_jsonish() for s in self._shards], |
| dest=self._shards_file, |
| indent=2) |
| |
| return _Isolator(api, layout).isolate(self) |
| |
| @property |
| def fuchsia_build_dir(self): |
| """The directory where Fuchsia build inputs may be found.""" |
| return self._fuchsia_build_dir |
| |
| @property |
| def symbolize_tool(self): |
| """The path to the symbolize tool.""" |
| return self._symbolize_tool |
| |
| @property |
| def llvm_symbolizer(self): |
| """The path to the llvm_symbolizer tool.""" |
| return self._llvm_symbolizer |
| |
| @property |
| def llvm_profdata(self): |
| """The path to the llvm_profdata tool.""" |
| return self._llvm_profdata |
| |
| @property |
| def llvm_cov(self): |
| """The path to the llvm_cov tool.""" |
| return self._llvm_cov |
| |
| @property |
| def ids(self): |
| """The path to the ids.txt file.""" |
| return self._ids |
| |
| @property |
| def authorized_key(self): |
| """The path to the authorized SSH key from a fuchsia checkout.""" |
| return self._authorized_key |
| |
| @property |
| def private_key(self): |
| """The path to the private SSH key from a fuchsia checkout.""" |
| return self._private_key |
| |
| @property |
| def secret_specs(self): |
| """The path to the secret_specs directory.""" |
| return self._secret_specs |
| |
| @property |
| def minfs(self): |
| """The path to the minfs tool.""" |
| return self._minfs |
| |
| @property |
| def bootserver(self): |
| """The path to the bootserver tool.""" |
| return self._bootserver |
| |
| @property |
| def zbi(self): |
| """The path to the zbi tool.""" |
| return self._zbi |
| |
| @property |
| def covargs(self): |
| """The path to the covargs tool.""" |
| return self._covargs |
| |
| def botanist(self, cpu): |
| """The path to the botanist tool. |
| |
| Args: |
| cpu (str): The cpu for which the tool was built. |
| """ |
| return self._botanist_x64 if cpu == 'x64' else self._botanist_arm64 |
| |
| def testrunner(self, cpu): |
| """The path to the testrunner tool. |
| |
| Args: |
| cpu (str): The cpu for which the tool was built. |
| """ |
| return self._testrunner_x64 if cpu == 'x64' else self._testrunner_arm64 |
| |
| @property |
| def target(self): |
| """The target of this build.""" |
| return self._target |
| |
| @property |
| def board(self): |
| """The board to build.""" |
| return self._board |
| |
| @property |
| def product(self): |
| """The product to build.""" |
| return self._product |
| |
| @property |
| def variants(self): |
| """The variants of this build.""" |
| return self._variants |
| |
| @property |
| def build_type(self): |
| """The build_type of this build.""" |
| return self._build_type |
| |
| @property |
| def images(self): |
| """The images of this build.""" |
| return self._images |
| |
| @property |
| def shards(self): |
| """The shards needed by tester.""" |
| return self._shards |
| |
| |
| class _Isolator(object): |
| """Isolator isolates and downloads Fuchsia BuildArtifacts""" |
| |
| def __init__(self, api, layout): |
| self._api = api |
| self._layout = layout |
| |
| def isolate(self, inputs): |
| """Isolates BuildArtifacts according to this Isolator's layout. |
| |
| Args: |
| inputs (BuildArtifacts): The BuildArtifacts to isolate. |
| |
| Returns: |
| A string isolated ID hash. |
| """ |
| root = self._api.path.mkdtemp('build_artifacts') |
| symlink_tree = self._api.file.symlink_tree(root) |
| self._symlink(symlink_tree, self._layout, inputs, root) |
| symlink_tree.create_links('create_links') |
| isolated = self._api.isolated.isolated(root) |
| isolated.add_dir(root) |
| return isolated.archive('isolate') |
| |
| def download(self, isolated_hash): |
| """Downloads an isolated containing BuildArtifacts |
| |
| Args: |
| isolated_hash (string): The isolated hash to fetch. |
| |
| Returns: |
| A BuildArtifacts object using the given layout. |
| """ |
| build_artifacts_dir = self._api.path.mkdtemp('build-artifacts') |
| self._api.isolated.download( |
| 'download build_artifacts', |
| isolated_hash=isolated_hash, |
| output_dir=build_artifacts_dir) |
| |
| args_file = build_artifacts_dir.join(self._layout.args) |
| args = self._api.json.read( |
| 'read %s' % self._layout.args, |
| args_file, |
| step_test_data=lambda: self._api.json.test_api.output( |
| TEST_BUILD_ARGS_JSON)).json.output |
| |
| images_file = build_artifacts_dir.join(self._layout.images) |
| images = self._api.json.read( |
| 'read %s' % self._layout.images, |
| images_file, |
| step_test_data=lambda: self._api.json.test_api.output(TEST_IMAGES_JSON) |
| ).json.output |
| |
| shards_file = build_artifacts_dir.join(self._layout.shards) |
| shards_in_json = self._api.json.read('load test shards', |
| shards_file).json.output |
| shards_in_json = shards_in_json or [] |
| shards = [ |
| self._api.testsharder.Shard.from_jsonish(s) for s in shards_in_json |
| ] |
| |
| # mac builders don't have arm64 versions |
| botanist_arm64 = None |
| testrunner_arm64 = None |
| botanist_arm64_path = build_artifacts_dir.join(self._layout.botanist_arm64) |
| testrunner_arm64_path = build_artifacts_dir.join( |
| self._layout.testrunner_arm64) |
| self._api.path.mock_add_paths(botanist_arm64_path) |
| self._api.path.mock_add_paths(testrunner_arm64_path) |
| if self._api.path.exists(botanist_arm64_path): |
| botanist_arm64 = botanist_arm64_path |
| if self._api.path.exists(testrunner_arm64_path): |
| testrunner_arm64 = testrunner_arm64_path |
| |
| return BuildArtifacts( |
| fuchsia_build_dir=build_artifacts_dir, |
| symbolize_tool=build_artifacts_dir.join(self._layout.symbolize_tool), |
| llvm_symbolizer=build_artifacts_dir.join(self._layout.llvm_symbolizer), |
| llvm_profdata=build_artifacts_dir.join(self._layout.llvm_profdata), |
| llvm_cov=build_artifacts_dir.join(self._layout.llvm_cov), |
| minfs=build_artifacts_dir.join(self._layout.minfs), |
| bootserver=build_artifacts_dir.join(self._layout.bootserver), |
| zbi=build_artifacts_dir.join(self._layout.zbi), |
| covargs=build_artifacts_dir.join(self._layout.covargs), |
| botanist_x64=build_artifacts_dir.join(self._layout.botanist_x64), |
| testrunner_x64=build_artifacts_dir.join(self._layout.testrunner_x64), |
| botanist_arm64=botanist_arm64, |
| testrunner_arm64=testrunner_arm64, |
| target=args['target'], |
| board=args['board'], |
| product=args['product'], |
| variants=args['variants'], |
| build_type=args['build_type'], |
| images=images, |
| shards=shards, |
| ids=build_artifacts_dir.join(self._layout.ids), |
| authorized_key=build_artifacts_dir.join(self._layout.authorized_key), |
| private_key=build_artifacts_dir.join(self._layout.private_key), |
| secret_specs=build_artifacts_dir.join(self._layout.secret_specs), |
| ) |
| |
| def _symlink(self, symlink_tree, layout, inputs, root): |
| """Adds these BuildArtifacts to a SymlinkTree. |
| |
| Args: |
| symlink_tree (SymlinkTree): The tree of symlinks to write to. |
| layout (_Layout): The layout to use for the symlinks. |
| inputs (BuildArtifacts): buildArtifacts provide paths to add to |
| symlink tree. |
| root (Path): Root of symlink_tree |
| """ |
| symlink_tree.register_link( |
| linkname=root.join(layout.symbolize_tool), |
| target=inputs.symbolize_tool, |
| ) |
| symlink_tree.register_link( |
| linkname=root.join(layout.llvm_symbolizer), |
| target=inputs.llvm_symbolizer, |
| ) |
| symlink_tree.register_link( |
| linkname=root.join(layout.llvm_profdata), |
| target=inputs.llvm_profdata, |
| ) |
| symlink_tree.register_link( |
| linkname=root.join(layout.llvm_cov), |
| target=inputs.llvm_cov, |
| ) |
| symlink_tree.register_link( |
| linkname=root.join(layout.minfs), |
| target=inputs.minfs, |
| ) |
| symlink_tree.register_link( |
| linkname=root.join(layout.bootserver), |
| target=inputs.bootserver, |
| ) |
| symlink_tree.register_link( |
| linkname=root.join(layout.zbi), |
| target=inputs.zbi, |
| ) |
| symlink_tree.register_link( |
| linkname=root.join(layout.covargs), |
| target=inputs.covargs, |
| ) |
| symlink_tree.register_link( |
| linkname=root.join(layout.botanist_x64), |
| target=inputs.botanist('x64'), |
| ) |
| symlink_tree.register_link( |
| linkname=root.join(layout.testrunner_x64), |
| target=inputs.testrunner('x64'), |
| ) |
| if inputs.botanist('arm64'): |
| symlink_tree.register_link( |
| linkname=root.join(layout.botanist_arm64), |
| target=inputs.botanist('arm64'), |
| ) |
| if inputs.testrunner('arm64'): |
| symlink_tree.register_link( |
| linkname=root.join(layout.testrunner_arm64), |
| target=inputs.testrunner('arm64'), |
| ) |
| symlink_tree.register_link( |
| linkname=root.join(layout.args), |
| target=inputs._args_file, |
| ) |
| symlink_tree.register_link( |
| linkname=root.join(layout.images), |
| target=inputs._images_file, |
| ) |
| for image in inputs.images.values(): |
| symlink_tree.register_link( |
| linkname=root.join(image['path']), |
| target=inputs._fuchsia_build_dir.join(image['path'])) |
| |
| symlink_tree.register_link( |
| linkname=root.join(layout.shards), |
| target=inputs._shards_file, |
| ) |
| for shard in inputs.shards: |
| for test in shard.tests: |
| if test.os in ['linux', 'mac']: |
| symlink_tree.register_link( |
| linkname=root.join(test.path), |
| target=inputs._fuchsia_build_dir.join(test.path), |
| ) |
| for dep in shard.deps: |
| symlink_tree.register_link( |
| linkname=root.join(dep), |
| target=inputs._fuchsia_build_dir.join(dep), |
| ) |
| |
| symlink_tree.register_link( |
| linkname=root.join(layout.ids), |
| target=inputs._ids, |
| ) |
| symlink_tree.register_link( |
| linkname=root.join(layout.authorized_key), |
| target=inputs.authorized_key, |
| ) |
| symlink_tree.register_link( |
| linkname=root.join(layout.private_key), |
| target=inputs.private_key, |
| ) |
| if self._api.path.exists(inputs._secret_specs): |
| symlink_tree.register_link( |
| linkname=root.join(layout.secret_specs), |
| target=inputs._secret_specs, |
| ) |
