blob: d29956d53627e41ea292ef69fd6348ca89b34b67 [file] [log] [blame]
# Copyright 2015 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
import json
from contextlib import contextmanager
from recipe_engine import recipe_api
class GomaApi(recipe_api.RecipeApi):
"""GomaApi contains helper functions for using goma."""
def __init__(self, luci_context, goma_properties, *args, **kwargs):
super(GomaApi, self).__init__(*args, **kwargs)
self._luci_context = luci_context
self._goma_context = None
self._goma_started = False
self._goma_ctl_env = {}
self._is_local = 'goma_dir' in goma_properties
self._enable_arbritrary_toolchains = goma_properties.get(
'enable_arbritrary_toolchains', False)
self._goma_dir = goma_properties.get('goma_dir', None)
self._jobs = goma_properties.get('jobs', None)
self._server = goma_properties.get('server', None)
self._recommended_jobs = None
self._jsonstatus = None
self._deps_cache = goma_properties.get('deps_cache', True)
self._local_output_cache = goma_properties.get('local_output_cache', False)
@property
def json_path(self):
assert self._goma_dir
return self.m.path.join(self._goma_dir, 'jsonstatus')
@property
def stats_path(self):
assert self._goma_dir
return self.m.path.join(self._goma_dir, 'goma_stats.json')
@property
def jsonstatus(self): # pragma: no cover
return self._jsonstatus
@property
def jobs(self):
"""Returns number of jobs for parallel build using Goma.
Uses value from property "$infra/goma:{\"jobs\": JOBS}" if configured
(typically in cr-buildbucket.cfg), else defaults to `recommended_goma_jobs`.
"""
return self._jobs or self.recommended_goma_jobs
@property
def recommended_goma_jobs(self):
"""Return the recommended number of jobs for parallel build using Goma.
Prefer to use just `goma.jobs` and configure it through default builder
properties in cr-buildbucket.cfg.
This function caches the _recommended_jobs.
"""
if self._recommended_jobs is None:
# When goma is used, 10 * self.m.platform.cpu_count is basically good in
# various situations according to our measurement. Build speed won't
# be improved if -j is larger than that.
#
# For safety, we'd like to set the upper limit to 1000.
self._recommended_jobs = min(10 * self.m.platform.cpu_count, 1000)
return self._recommended_jobs
@property
def goma_ctl(self):
return self.m.path.join(self._goma_dir, 'goma_ctl.py')
@property
def goma_dir(self):
assert self._goma_dir
return self._goma_dir
def ensure(self, canary=False):
if self._is_local:
return self._goma_dir
with self.m.step.nest('ensure goma') as step_result:
if canary:
step_result.presentation.step_text = 'using canary goma client'
step_result.presentation.status = self.m.step.WARNING
with self.m.context(infra_steps=True):
pkgs = self.m.cipd.EnsureFile()
ref = 'release'
if canary:
ref = 'candidate'
pkgs.add_package('infra_internal/goma/client/${platform}', ref)
self._goma_dir = self.m.path['cache'].join('goma', 'client')
self.m.cipd.ensure(self._goma_dir, pkgs)
return self._goma_dir
def _run_jsonstatus(self):
with self.m.context(env=self._goma_ctl_env):
jsonstatus_result = self.m.python(
name='goma_jsonstatus',
script=self.goma_ctl,
args=['jsonstatus',
self.m.json.output(leak_to=self.json_path)],
step_test_data=lambda: self.m.json.test_api.output(
data={
'notice': [{
'infra_status': {
'ping_status_code': 200,
'num_user_error': 0,
}
}]
}))
self._jsonstatus = jsonstatus_result.json.output
if self._jsonstatus is None:
jsonstatus_result.presentation.status = self.m.step.WARNING
def _upload_goma_stats(self):
test_data = {}
json_obj = self.m.file.read_json(
'read goma_stats.json', self.stats_path, test_data=test_data)
if not (self.m.buildbucket.builder_name and self.m.buildbucket.build.id):
# Skip the upload if it does not have build input information.
return
build_dict = {
'build_id': self.m.buildbucket.build.id,
'builder': self.m.buildbucket.builder_name,
'time_stamp': str(self.m.time.utcnow()),
'time_stamp_int': self.m.time.ms_since_epoch()
}
json_obj['build_info'] = build_dict
self.m.file.write_json('write goma_stats.json', self.stats_path, json_obj)
self.m.step.active_result.presentation.logs['json.output'] = \
json.dumps(json_obj, indent=4).splitlines()
self.m.bqupload.insert(
step_name='upload goma_stats_to BQ: fuchsia-infra/artifacts/builds_beta_goma',
project='fuchsia-infra',
dataset='artifacts',
table='builds_beta_goma',
data_file=self.stats_path,
)
def start(self, env=None, **kwargs):
"""Start goma compiler_proxy.
A user MUST execute ensure_goma beforehand.
It is user's responsibility to handle failure of starting compiler_proxy.
"""
assert self._goma_dir
assert not self._goma_started
# TODO(olivernewman): Determine if the default value can be removed for this
# parameter. If not, add a test to cover the case where it's not passed.
env = env or {}
with self.m.step.nest('pre_goma') as nested_result:
if 'GOMA_CACHE_DIR' not in env:
self._goma_ctl_env['GOMA_CACHE_DIR'] = self.m.path['cache'].join('goma')
if self._deps_cache:
if 'GOMA_DEPS_CACHE_FILE' not in env:
self._goma_ctl_env['GOMA_DEPS_CACHE_FILE'] = 'goma_deps_cache'
if self._local_output_cache:
if 'GOMA_LOCAL_OUTPUT_CACHE_DIR' not in env:
self._goma_ctl_env['GOMA_LOCAL_OUTPUT_CACHE_DIR'] = (
self.m.path['cache'].join('goma', 'localoutputcache'))
if self._server:
if 'GOMA_SERVER_HOST' not in env:
self._goma_ctl_env['GOMA_SERVER_HOST'] = self._server
if self._enable_arbritrary_toolchains:
if 'GOMA_ARBITRARY_TOOLCHAIN_SUPPORT' not in env:
self._goma_ctl_env[
'GOMA_ARBITRARY_TOOLCHAIN_SUPPORT'] = self._enable_arbritrary_toolchains
if self._luci_context:
if not self._goma_context:
step_result = self.m.json.read(
'read context',
self._luci_context,
add_json_log=False,
step_test_data=lambda: self.m.json.test_api.output({
'local_auth': {
'accounts': [{
'id': 'test',
'email': 'some@example.com'
}],
'default_account_id': 'test',
}
}))
ctx = step_result.json.output.copy()
if 'local_auth' not in ctx: # pragma: no cover
raise self.m.step.InfraFailure('local_auth missing in LUCI_CONTEXT')
ctx['local_auth']['default_account_id'] = 'system'
self._goma_context = self.m.path.mkstemp('luci_context.')
self.m.file.write_text('write context', self._goma_context,
self.m.json.dumps(ctx))
self._goma_ctl_env['LUCI_CONTEXT'] = self._goma_context
# GLOG_log_dir should not be set.
assert 'GLOG_log_dir' not in self.m.context.env, (
'GLOG_log_dir must not be set in env during goma.start()')
self._goma_ctl_env['GOMA_DUMP_STATS_FILE'] = self.stats_path
goma_ctl_env = self._goma_ctl_env.copy()
goma_ctl_env.update(env)
try:
with self.m.context(env=goma_ctl_env):
self.m.python(
name='start_goma',
script=self.goma_ctl,
args=['restart'],
infra_step=True,
**kwargs)
self._goma_started = True
except self.m.step.InfraFailure as e: # pragma: no cover
with self.m.step.defer_results():
self._run_jsonstatus()
with self.m.context(env=self._goma_ctl_env):
self.m.python(
name='stop_goma (start failure)',
script=self.goma_ctl,
args=['stop'],
**kwargs)
nested_result.presentation.status = self.m.step.EXCEPTION
raise e
def stop(self, **kwargs):
"""Stop goma compiler_proxy.
A user MUST execute start beforehand.
It is user's responsibility to handle failure of stopping compiler_proxy.
Raises:
StepFailure if it fails to stop goma.
"""
assert self._goma_dir
assert self._goma_started
with self.m.step.nest('post_goma') as nested_result:
try:
with self.m.step.defer_results():
self._run_jsonstatus()
with self.m.context(env=self._goma_ctl_env):
self.m.python(
name='goma_stats',
script=self.goma_ctl,
args=['stat'],
**kwargs)
self.m.python(
name='stop_goma', script=self.goma_ctl, args=['stop'], **kwargs)
self._goma_started = False
# Upload stats to BigQuery
self._upload_goma_stats()
except self.m.step.StepFailure:
nested_result.presentation.status = self.m.step.EXCEPTION
raise
@contextmanager
def build_with_goma(self, env=None):
"""Make context wrapping goma start/stop.
Raises:
StepFailure or InfraFailure if it fails to build.
"""
env = env or {}
self.start(env)
try:
yield
finally:
self.stop()