blob: 7ff0b0f5d384a4193dcaa629dfa153384f491168 [file] [log] [blame]
# Copyright 2019 The Fuchsia Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
import collections
import copy
import itertools
import os
from collections import namedtuple
# Test output for the step that reads the images.json file produced and isolated by the
# "builder" build.
TEST_IMAGES_JSON = {
'zircon-a': {
'path': 'fuchsia.zbi',
'type': 'zbi',
'name': 'zircon-a',
'bootserver_pave': ['--boot', '--zircona']
},
'netboot': {
'path': 'netboot.zbi',
'bootserver_netboot': ['--boot'],
'type': 'zbi',
'name': 'netboot'
}
}
# Test output for the step that reads the args.json file produced and isolated by the
# "builder" build.
TEST_BUILD_ARGS_JSON = {
'board': 'boards/x64.gni',
'build_type': 'build_type',
'product': 'products/core.gni',
'target': 'x64',
'variants': [],
}
# A simple container for associating a shard with a swarming task request. We
# need this to be able to match up the summary.json produced by the task with
# the tests.json manifest for the shard, so that the "name" and "gn_label"
# fields in summary.json can be correctly populated using tests.json for QEMU
# shards.
# TODO(40190): Once done, maybe we can remove the shards and just keep the task
# requests.
#
# Attributes:
# shard (api.testsharder.Shard): The input shard containing a list of tests
# to run.
# task_request (api.swarming.TaskRequest): The swarming request
# corresponding to the shard.
ShardTaskRequest = collections.namedtuple(
'ShardTaskRequest', field_names=('shard', 'task_request'))
class TestOrchestrationInputs(object):
"""Data used to orchestrate testing."""
# Constant shared between recipes that produce and consume.
# Property that holds the isolate hash of the test orchestration inputs.
HASH_PROPERTY = 'test_orchestration_inputs_hash'
# IsolatedLayout describes the directory structure to use when isolating
# TestOrchestrationInputs. Equivalent layouts must be used to isolate
# and download.
IsolatedLayout = namedtuple(
'IsolatedLayout',
(
'llvm_symbolizer',
'minfs',
'shards',
'symbolize_tool',
'task_requests',
# TODO(garymm,joshuaseaton): Once deprecated testing codepath is eliminated we
# can remove the tests file from this class.
'tests'),
)
DEFAULT_ISOLATED_LAYOUT = IsolatedLayout(
llvm_symbolizer='llvm-symbolizer',
minfs='minfs',
shards='shards.json',
symbolize_tool='symbolize_tool',
task_requests='task_requests.json',
tests='tests.json',
)
def __init__(self, llvm_symbolizer, minfs, symbolize_tool, shard_requests,
tests_file):
# TODO(garymm): Take in an API object here rather than in the
# member functions, and hide this constructor behind a function
# in build/api.py. That will prevent the calling recipes from
# needing to depend on all the transitive dependencies of this
# code. See comment in docstring of TestOrchestrationInputs.isolate()
# for more on that.
self.llvm_symbolizer = llvm_symbolizer
self.minfs = minfs
self.symbolize_tool = symbolize_tool
self.shard_requests = shard_requests
self.tests_file = tests_file
self._layout = self.DEFAULT_ISOLATED_LAYOUT
self._shards_file = None
self._task_requests_file = None
@staticmethod
def download(api, isolated_hash):
"""Downloads an isolated containing TestOrchestrationInputs
Args:
isolated_hash (string): The isolated hash to fetch.
Returns:
A TestOrchestrationInputs object.
"""
with api.step.nest('download test orchestration inputs'):
return TestOrchestrationInputs._Isolator(
api, TestOrchestrationInputs.DEFAULT_ISOLATED_LAYOUT).download(
isolated_hash)
def isolate(self, api):
"""Uploads to isolate.
Args:
api: magic recipe engine API object.
Note: because of the way DEPs stuff works, the api object gets the
deps of the calling recipe, NOT the deps listed in
build/__init__.py.
Returns:
(str) Isolated hash.
"""
with api.step.nest('isolate test orchestration inputs'):
shards_data = []
task_requests_data = []
for shard_request in self.shard_requests:
shards_data.append(shard_request.shard.render_to_jsonish())
task_requests_data.append(shard_request.task_request.to_jsonish())
temp_dir = api.path.mkdtemp('test-orchestration-inputs')
self._shards_file = temp_dir.join(self._layout.shards)
self._task_requests_file = temp_dir.join(self._layout.task_requests)
api.file.write_json(
'write %s' % self._layout.shards,
data=shards_data,
dest=self._shards_file,
indent=2)
api.file.write_json(
'write %s' % self._layout.task_requests,
data=task_requests_data,
dest=self._task_requests_file,
indent=2)
return self._Isolator(api, self._layout).isolate(self)
class _Isolator(object):
"""Isolator isolates and downloads TestOrchestrationInputs"""
def __init__(self, api, layout):
self._api = api
self._layout = layout
def download(self, isolated_hash):
"""Downloads an isolated containing TestOrchestrationInputs
Args:
isolated_hash (string): The isolated hash to fetch.
Returns:
A TestOrchestrationInputs object.
"""
download_dir = self._api.path.mkdtemp('test-orchestration-inputs')
self._api.isolated.download(
'download test orchestration inputs',
isolated_hash=isolated_hash,
output_dir=download_dir)
shards_file = download_dir.join(self._layout.shards)
shards_in_json = self._api.json.read('load test shards',
shards_file).json.output
shards_in_json = shards_in_json or []
task_requests_file = download_dir.join(self._layout.task_requests)
task_requests_in_json = self._api.json.read(
'load task requests', task_requests_file).json.output
task_requests_in_json = task_requests_in_json or []
assert len(shards_in_json) == len(task_requests_in_json), '%d vs %d' % (
len(shards_in_json), len(task_requests_in_json))
shard_requests = []
for shard_json, task_request_json in itertools.izip(
shards_in_json, task_requests_in_json):
shard_requests.append(
ShardTaskRequest(
self._api.testsharder.Shard.from_jsonish(shard_json),
self._api.swarming.task_request_from_jsonish(
task_request_json)))
return TestOrchestrationInputs(
download_dir.join(self._layout.llvm_symbolizer),
download_dir.join(self._layout.minfs),
download_dir.join(self._layout.symbolize_tool), shard_requests,
download_dir.join(self._layout.tests))
def isolate(self, inputs):
"""Isolates BuildArtifacts according to this Isolator's layout.
Args:
inputs (TestOrchestrationInputs):
The TestOrchestrationInputs to isolate.
Returns:
A string isolated ID hash.
"""
root = self._api.path.mkdtemp('test_orchestration_inputs')
symlink_tree = self._api.file.symlink_tree(root)
self._symlink(symlink_tree, self._layout, inputs, root)
symlink_tree.create_links('create_links')
isolated = self._api.isolated.isolated(root)
isolated.add_dir(root)
return isolated.archive('isolate')
def _symlink(self, symlink_tree, layout, inputs, root):
"""Adds TestOrchestrationInputs to a SymlinkTree.
Args:
symlink_tree (SymlinkTree): The tree of symlinks to write to.
layout (_Layout): The layout to use for the symlinks.
inputs (TestOrchestrationInputs): provide paths to add to symlink tree.
root (Path): Root of symlink_tree
"""
symlink_tree.register_link(
linkname=root.join(layout.llvm_symbolizer),
target=inputs.llvm_symbolizer,
)
symlink_tree.register_link(
linkname=root.join(layout.minfs),
target=inputs.minfs,
)
symlink_tree.register_link(
linkname=root.join(layout.shards),
target=inputs._shards_file,
)
symlink_tree.register_link(
linkname=root.join(layout.symbolize_tool),
target=inputs.symbolize_tool,
)
symlink_tree.register_link(
linkname=root.join(layout.task_requests),
target=inputs._task_requests_file,
)
symlink_tree.register_link(
linkname=root.join(layout.tests),
target=inputs.tests_file,
)
class BuildArtifacts(object):
"""The outputs of a Fuchsia build needed for testing.
This object contains a collection of read-only paths to artifacts, tools, and metadata
produced by a Fuchsia build. It is used to isolate and download this data for Tester
builds.
"""
# IsolatedLayout describes the directory structure to use when isolating
# BuildArtifacts. If an IsolatedLayout is used to isolate BuildArtifacts, that
# same IsolatedLayout should be used to download the inputs.
IsolatedLayout = namedtuple(
'IsolatedLayout',
('fuchsia_build_dir', 'symbolize_tool', 'llvm_symbolizer',
'llvm_profdata', 'llvm_cov', 'minfs', 'zbi', 'covargs', 'botanist_x64',
'testrunner_x64', 'bootserver', 'botanist_arm64', 'testrunner_arm64',
'args', 'images', 'shards', 'tests', 'ids', 'authorized_key',
'private_key', 'secret_specs'),
)
# The default layout to use when isolating BuildArtifacts. This results in the following
# structure:
# ./
# fuchsia/
# symbolize_tool
# llvm_symbolizer
# llvm_profdata
# llvm_cov
# minfs
# bootserver
# zbi
# covargs
# x64/
# botanist
# testrunner
# arm64/
# botanist
# testrunner
# args.json
# images.json
# shards.json
# tests.json
# ids.txt
# authorized_key
# private_key
# secret_specs/
DEFAULT_ISOLATED_LAYOUT = IsolatedLayout(
fuchsia_build_dir='fuchsia',
symbolize_tool='symbolize_tool',
llvm_symbolizer='llvm-symbolizer',
llvm_profdata='llvm_profdata',
llvm_cov='llvm_cov',
minfs='minfs',
bootserver='bootserver',
zbi='zbi',
covargs='covargs',
botanist_x64=os.path.join('x64', 'botanist'),
testrunner_x64=os.path.join('x64', 'testrunner'),
botanist_arm64=os.path.join('arm64', 'botanist'),
testrunner_arm64=os.path.join('arm64', 'testrunner'),
args='args.json',
images='images.json',
shards='shards.json',
tests='tests.json',
ids='ids.txt',
authorized_key='authorized_key',
private_key='private_key',
secret_specs='secret_specs',
)
def __init__(self, fuchsia_build_dir, symbolize_tool, llvm_symbolizer,
llvm_profdata, llvm_cov, minfs, bootserver, zbi, covargs,
botanist_x64, testrunner_x64, botanist_arm64, testrunner_arm64,
target, board, product, variants, build_type, images, shards,
tests_file, ids, authorized_key, private_key, secret_specs):
self._fuchsia_build_dir = fuchsia_build_dir
self._symbolize_tool = symbolize_tool
self._llvm_symbolizer = llvm_symbolizer
self._llvm_profdata = llvm_profdata
self._llvm_cov = llvm_cov
self._minfs = minfs
self._bootserver = bootserver
self._zbi = zbi
self._covargs = covargs
self._botanist_x64 = botanist_x64
self._testrunner_x64 = testrunner_x64
self._botanist_arm64 = botanist_arm64
self._testrunner_arm64 = testrunner_arm64
self._target = target
self._board = board
self._product = product
self._variants = variants
self._build_type = build_type
self._images = images
self._shards = shards
self._tests_file = tests_file
self._ids = ids
self._authorized_key = authorized_key
self._private_key = private_key
self._secret_specs = secret_specs
self._args_file = None
self._images_file = None
self._shards_file = None
def __deepcopy__(self, memodict):
# Shallow copy first.
new = copy.copy(self)
# Only images needs to be a real deep copy.
new._images = copy.deepcopy(new._images, memodict)
return new
@staticmethod
def download(api, isolated_hash):
"""Restores BuildArtifacts from an isolated.
Args:
api (RecipeApi): The Recipe API.
isolated_hash (string): The isolated hash to fetch.
Returns:
A BuildArtifacts object using the given layout.
"""
with api.step.nest('download build artifacts'):
return BuildArtifacts._Isolator(
api, BuildArtifacts.DEFAULT_ISOLATED_LAYOUT).download(isolated_hash)
def isolate(self, api):
"""Isolates these BuildArtifacts.
Args:
api (RecipeApi): The Recipe API.
Returns:
A string isolated ID hash.
"""
layout = BuildArtifacts.DEFAULT_ISOLATED_LAYOUT
args = {
'board': self.board,
'build_type': self.build_type,
'product': self.product,
'target': self.target,
# Convert repeated proto field to list so it can be serialized to JSON.
'variants': list(self.variants),
}
with api.step.nest('isolate build artifacts'):
self._args_file = api.path['start_dir'].join(layout.args)
api.file.write_json(
'write %s' % layout.args, data=args, dest=self._args_file, indent=2)
self._images_file = api.path['start_dir'].join(layout.images)
api.file.write_json(
'write %s' % layout.images,
data=self._images,
dest=self._images_file,
indent=2)
if not self._shards:
self._shards = []
self._shards_file = api.path['start_dir'].join(layout.shards)
api.file.write_json(
'write %s' % layout.shards,
data=[s.render_to_jsonish() for s in self._shards],
dest=self._shards_file,
indent=2)
return BuildArtifacts._Isolator(api, layout).isolate(self)
@property
def fuchsia_build_dir(self):
"""The directory where Fuchsia build inputs may be found."""
return self._fuchsia_build_dir
@property
def symbolize_tool(self):
"""The path to the symbolize tool."""
return self._symbolize_tool
@property
def llvm_symbolizer(self):
"""The path to the llvm_symbolizer tool."""
return self._llvm_symbolizer
@property
def llvm_profdata(self):
"""The path to the llvm_profdata tool."""
return self._llvm_profdata
@property
def llvm_cov(self):
"""The path to the llvm_cov tool."""
return self._llvm_cov
@property
def ids(self):
"""The path to the ids.txt file."""
return self._ids
@property
def tests_file(self):
"""The path to the tests.json file."""
return self._tests_file
@property
def authorized_key(self):
"""The path to the authorized SSH key from a fuchsia checkout."""
return self._authorized_key
@property
def private_key(self):
"""The path to the private SSH key from a fuchsia checkout."""
return self._private_key
@property
def secret_specs(self):
"""The path to the secret_specs directory."""
return self._secret_specs
@property
def minfs(self):
"""The path to the minfs tool."""
return self._minfs
@property
def bootserver(self):
"""The path to the bootserver tool."""
return self._bootserver
@property
def zbi(self):
"""The path to the zbi tool."""
return self._zbi
@property
def covargs(self):
"""The path to the covargs tool."""
return self._covargs
def botanist(self, cpu):
"""The path to the botanist tool.
Args:
cpu (str): The cpu for which the tool was built.
"""
return self._botanist_x64 if cpu == 'x64' else self._botanist_arm64
def testrunner(self, cpu):
"""The path to the testrunner tool.
Args:
cpu (str): The cpu for which the tool was built.
"""
return self._testrunner_x64 if cpu == 'x64' else self._testrunner_arm64
@property
def target(self):
"""The target of this build."""
return self._target
@property
def board(self):
"""The board to build."""
return self._board
@property
def product(self):
"""The product to build."""
return self._product
@property
def variants(self):
"""The variants of this build."""
return self._variants
@property
def build_type(self):
"""The build_type of this build."""
return self._build_type
@property
def images(self):
"""The images of this build."""
return self._images
@property
def shards(self):
"""The shards needed by tester."""
return self._shards
class _Isolator(object):
"""Isolator isolates and downloads Fuchsia BuildArtifacts"""
def __init__(self, api, layout):
self._api = api
self._layout = layout
def isolate(self, inputs):
"""Isolates BuildArtifacts according to this Isolator's layout.
Args:
inputs (BuildArtifacts): The BuildArtifacts to isolate.
Returns:
A string isolated ID hash.
"""
root = self._api.path.mkdtemp('build_artifacts')
symlink_tree = self._api.file.symlink_tree(root)
self._symlink(symlink_tree, self._layout, inputs, root)
symlink_tree.create_links('create_links')
isolated = self._api.isolated.isolated(root)
isolated.add_dir(root)
return isolated.archive('isolate')
def download(self, isolated_hash):
"""Downloads an isolated containing BuildArtifacts
Args:
isolated_hash (string): The isolated hash to fetch.
Returns:
A BuildArtifacts object using the given layout.
"""
build_artifacts_dir = self._api.path.mkdtemp('build-artifacts')
self._api.isolated.download(
'download build_artifacts',
isolated_hash=isolated_hash,
output_dir=build_artifacts_dir)
args_file = build_artifacts_dir.join(self._layout.args)
args = self._api.json.read(
'read %s' % self._layout.args,
args_file,
step_test_data=lambda: self._api.json.test_api.output(
TEST_BUILD_ARGS_JSON)).json.output
images_file = build_artifacts_dir.join(self._layout.images)
images = self._api.json.read(
'read %s' % self._layout.images,
images_file,
step_test_data=lambda: self._api.json.test_api.output(TEST_IMAGES_JSON
)).json.output
shards_file = build_artifacts_dir.join(self._layout.shards)
shards_in_json = self._api.json.read('load test shards',
shards_file).json.output
shards_in_json = shards_in_json or []
shards = [
self._api.testsharder.Shard.from_jsonish(s) for s in shards_in_json
]
# mac builders don't have arm64 versions
botanist_arm64 = None
testrunner_arm64 = None
botanist_arm64_path = build_artifacts_dir.join(
self._layout.botanist_arm64)
testrunner_arm64_path = build_artifacts_dir.join(
self._layout.testrunner_arm64)
self._api.path.mock_add_paths(botanist_arm64_path)
self._api.path.mock_add_paths(testrunner_arm64_path)
if self._api.path.exists(botanist_arm64_path):
botanist_arm64 = botanist_arm64_path
if self._api.path.exists(testrunner_arm64_path):
testrunner_arm64 = testrunner_arm64_path
return BuildArtifacts(
fuchsia_build_dir=build_artifacts_dir,
symbolize_tool=build_artifacts_dir.join(self._layout.symbolize_tool),
llvm_symbolizer=build_artifacts_dir.join(
self._layout.llvm_symbolizer),
llvm_profdata=build_artifacts_dir.join(self._layout.llvm_profdata),
llvm_cov=build_artifacts_dir.join(self._layout.llvm_cov),
minfs=build_artifacts_dir.join(self._layout.minfs),
bootserver=build_artifacts_dir.join(self._layout.bootserver),
zbi=build_artifacts_dir.join(self._layout.zbi),
covargs=build_artifacts_dir.join(self._layout.covargs),
botanist_x64=build_artifacts_dir.join(self._layout.botanist_x64),
testrunner_x64=build_artifacts_dir.join(self._layout.testrunner_x64),
botanist_arm64=botanist_arm64,
testrunner_arm64=testrunner_arm64,
target=args['target'],
board=args['board'],
product=args['product'],
variants=args['variants'],
build_type=args['build_type'],
images=images,
shards=shards,
tests_file=build_artifacts_dir.join(self._layout.tests),
ids=build_artifacts_dir.join(self._layout.ids),
authorized_key=build_artifacts_dir.join(self._layout.authorized_key),
private_key=build_artifacts_dir.join(self._layout.private_key),
secret_specs=build_artifacts_dir.join(self._layout.secret_specs),
)
def _symlink(self, symlink_tree, layout, inputs, root):
"""Adds these BuildArtifacts to a SymlinkTree.
Args:
symlink_tree (SymlinkTree): The tree of symlinks to write to.
layout (_Layout): The layout to use for the symlinks.
inputs (BuildArtifacts): buildArtifacts provide paths to add to
symlink tree.
root (Path): Root of symlink_tree
"""
symlink_tree.register_link(
linkname=root.join(layout.symbolize_tool),
target=inputs.symbolize_tool,
)
symlink_tree.register_link(
linkname=root.join(layout.llvm_symbolizer),
target=inputs.llvm_symbolizer,
)
symlink_tree.register_link(
linkname=root.join(layout.llvm_profdata),
target=inputs.llvm_profdata,
)
symlink_tree.register_link(
linkname=root.join(layout.llvm_cov),
target=inputs.llvm_cov,
)
symlink_tree.register_link(
linkname=root.join(layout.minfs),
target=inputs.minfs,
)
symlink_tree.register_link(
linkname=root.join(layout.bootserver),
target=inputs.bootserver,
)
symlink_tree.register_link(
linkname=root.join(layout.zbi),
target=inputs.zbi,
)
symlink_tree.register_link(
linkname=root.join(layout.covargs),
target=inputs.covargs,
)
symlink_tree.register_link(
linkname=root.join(layout.botanist_x64),
target=inputs.botanist('x64'),
)
symlink_tree.register_link(
linkname=root.join(layout.testrunner_x64),
target=inputs.testrunner('x64'),
)
if inputs.botanist('arm64'):
symlink_tree.register_link(
linkname=root.join(layout.botanist_arm64),
target=inputs.botanist('arm64'),
)
if inputs.testrunner('arm64'):
symlink_tree.register_link(
linkname=root.join(layout.testrunner_arm64),
target=inputs.testrunner('arm64'),
)
symlink_tree.register_link(
linkname=root.join(layout.args),
target=inputs._args_file,
)
symlink_tree.register_link(
linkname=root.join(layout.images),
target=inputs._images_file,
)
for image in inputs.images.values():
symlink_tree.register_link(
linkname=root.join(image['path']),
target=inputs._fuchsia_build_dir.join(image['path']))
symlink_tree.register_link(
linkname=root.join(layout.shards),
target=inputs._shards_file,
)
for shard in inputs.shards:
for test in shard.tests:
if test.os in ['linux', 'mac']:
symlink_tree.register_link(
linkname=root.join(test.path),
target=inputs._fuchsia_build_dir.join(test.path),
)
for dep in shard.deps:
symlink_tree.register_link(
linkname=root.join(dep),
target=inputs._fuchsia_build_dir.join(dep),
)
symlink_tree.register_link(
linkname=root.join(layout.tests),
target=inputs.tests_file,
)
symlink_tree.register_link(
linkname=root.join(layout.ids),
target=inputs._ids,
)
symlink_tree.register_link(
linkname=root.join(layout.authorized_key),
target=inputs.authorized_key,
)
symlink_tree.register_link(
linkname=root.join(layout.private_key),
target=inputs.private_key,
)
if self._api.path.exists(inputs._secret_specs):
symlink_tree.register_link(
linkname=root.join(layout.secret_specs),
target=inputs._secret_specs,
)