#!/usr/bin/env python | |
# Copyright 2017 The Fuchsia Authors | |
# | |
# Licensed under the Apache License, Version 2.0 (the "License"); | |
# you may not use this file except in compliance with the License. | |
# You may obtain a copy of the License at | |
# | |
# http://www.apache.org/licenses/LICENSE-2.0 | |
# | |
# Unless required by applicable law or agreed to in writing, software | |
# distributed under the License is distributed on an "AS IS" BASIS, | |
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
# See the License for the specific language governing permissions and | |
# limitations under the License. | |
"""A library with functions to start each of the Cobalt processes locally.""" | |
import os | |
import shutil | |
import subprocess | |
THIS_DIR = os.path.dirname(__file__) | |
SRC_ROOT_DIR = os.path.abspath(os.path.join(THIS_DIR, os.pardir)) | |
OUT_DIR = os.path.join(SRC_ROOT_DIR, 'out') | |
SYS_ROOT_DIR = os.path.join(SRC_ROOT_DIR, 'sysroot') | |
DEMO_CONFIG_DIR = os.path.abspath(os.path.join(SRC_ROOT_DIR, 'config', | |
'demo')) | |
PRODUCTION_CONFIG_DIR = os.path.abspath(os.path.join(SRC_ROOT_DIR, 'config', | |
'production')) | |
SHUFFLER_DEMO_CONFIG_FILE = os.path.abspath(os.path.join(SRC_ROOT_DIR, | |
'shuffler', 'src', 'config', 'config_demo.txt')) | |
SHUFFLER_DB_DIR = os.path.join("/tmp/cobalt_shuffler") | |
SHUFFLER_CONFIG_DIR = os.path.abspath(os.path.join(SRC_ROOT_DIR, 'shuffler', | |
'src', 'shuffler_config')) | |
SHUFFLER_CONFIG_FILE = os.path.join(SHUFFLER_CONFIG_DIR, 'config_v0.txt') | |
SHUFFLER_DEMO_CONFIG_FILE = os.path.join(SHUFFLER_CONFIG_DIR, 'config_demo.txt') | |
SHUFFLER_TMP_DB_DIR = os.path.join("/tmp/cobalt_shuffler") | |
DEFAULT_SHUFFLER_PORT=5001 | |
DEFAULT_ANALYZER_SERVICE_PORT=6001 | |
DEFAULT_REPORT_MASTER_PORT=7001 | |
DEFAULT_ANALYZER_PUBLIC_KEY_PEM=os.path.join(SRC_ROOT_DIR, | |
"analyzer_public.pem") | |
ANALYZER_PRIVATE_KEY_PEM_NAME="analyzer_private.pem" | |
DEFAULT_ANALYZER_PRIVATE_KEY_PEM=os.path.join(SRC_ROOT_DIR, | |
ANALYZER_PRIVATE_KEY_PEM_NAME) | |
DEFAULT_SHUFFLER_PUBLIC_KEY_PEM=os.path.join(SRC_ROOT_DIR, | |
"shuffler_public.pem") | |
SHUFFLER_PRIVATE_KEY_PEM_NAME="shuffler_private.pem" | |
DEFAULT_SHUFFLER_PRIVATE_KEY_PEM=os.path.join(SRC_ROOT_DIR, | |
SHUFFLER_PRIVATE_KEY_PEM_NAME) | |
LOCALHOST_TLS_CERT_FILE=os.path.join(SRC_ROOT_DIR, "end_to_end_tests", | |
"localhost.crt") | |
LOCALHOST_TLS_KEY_FILE=os.path.join(SRC_ROOT_DIR, "end_to_end_tests", | |
"localhost.key") | |
def kill_process(process, name): | |
""" Kills the given process if it is running and waits for it to terminate. | |
Args: | |
process {Popen} A representation of the process to be killed. | |
May be None. | |
name {String} Name of the process for use in a user-facing message. | |
""" | |
if process and process.poll() is None: | |
print "Killing %s..." % name | |
process.terminate() | |
process.kill() | |
process.wait() | |
def execute_command(cmd, wait): | |
""" Executes the given command and optionally waits for it to complete. | |
command {list of strings} will be passed to Popen(). | |
wait {bool} If true we will wait for the command to complete and return | |
the result code. If false we will return immediately and return an | |
instance of Popen. | |
Returns: | |
An instance of Popen if wait is false or an integer return code if | |
wait is true. | |
""" | |
p = subprocess.Popen(cmd) | |
if not wait: | |
return p | |
return_code = p.wait() | |
if return_code < 0: | |
print "****** WARNING Process terminated by signal %d" % (- return_code) | |
return return_code | |
def start_bigtable_emulator(wait=True): | |
# Note(rudominer) We can pass -port=n to cbtemulator to run on a different | |
# port. | |
print "Starting the Cloud Bigtable Emulator..." | |
path = os.path.abspath(os.path.join(SYS_ROOT_DIR, 'gcloud', | |
'google-cloud-sdk', 'platform', 'bigtable-emulator', 'cbtemulator')) | |
cmd = [path] | |
return execute_command(cmd, wait) | |
SHUFFLER_PATH = os.path.abspath(os.path.join(OUT_DIR, 'shuffler', 'shuffler')) | |
def start_shuffler(port=DEFAULT_SHUFFLER_PORT, | |
analyzer_uri='localhost:%d' % DEFAULT_ANALYZER_SERVICE_PORT, | |
use_memstore=False, erase_db=True, db_dir=SHUFFLER_TMP_DB_DIR, | |
config_file=SHUFFLER_DEMO_CONFIG_FILE, | |
private_key_pem_file=DEFAULT_SHUFFLER_PRIVATE_KEY_PEM, | |
use_tls=False, | |
tls_cert_file=LOCALHOST_TLS_CERT_FILE, | |
tls_key_file=LOCALHOST_TLS_KEY_FILE, | |
verbose_count=0, wait=True): | |
"""Starts the Shuffler. | |
Args: | |
port {int} The port on which the Shuffler should listen. | |
analyzer_uri {string} The URI of the Analyzer Service | |
use_memstore {bool} If false the Shuffler will use the LevelDB store | |
emtpy_db {bool} When using the LevelDB store, should the store be | |
erased before the shuffler starts? | |
config_file {string} The path to the Shuffler's config file. | |
""" | |
cmd = [SHUFFLER_PATH, | |
"-port", str(port), | |
"-private_key_pem_file", private_key_pem_file, | |
"-analyzer_uri", analyzer_uri, | |
"-config_file", config_file, | |
"-logtostderr"] | |
if use_tls: | |
cmd.append("-tls") | |
cmd.append("-cert_file=%s"%tls_cert_file) | |
cmd.append("-key_file=%s"%tls_key_file) | |
if verbose_count > 0: | |
cmd.append("-v=%d"%verbose_count) | |
if use_memstore: | |
cmd.append("-use_memstore") | |
else: | |
cmd = cmd + ["-db_dir", db_dir] | |
if erase_db: | |
print "Erasing Shuffler's LevelDB store at %s." % db_dir | |
shutil.rmtree(db_dir, ignore_errors=True) | |
print "Starting the shuffler..." | |
return execute_command(cmd, wait) | |
ANALYZER_SERVICE_PATH = os.path.abspath(os.path.join(OUT_DIR, 'analyzer', | |
'analyzer_service', 'analyzer_service')) | |
def start_analyzer_service(port=DEFAULT_ANALYZER_SERVICE_PORT, | |
bigtable_project_name='', bigtable_instance_id='', | |
private_key_pem_file=DEFAULT_ANALYZER_PRIVATE_KEY_PEM, | |
verbose_count=0, vmodule=None, | |
wait=True): | |
print "Starting the analyzer service..." | |
cmd = [ANALYZER_SERVICE_PATH, | |
"-port", str(port), | |
"-private_key_pem_file", private_key_pem_file, | |
"-logtostderr"] | |
if bigtable_project_name != '' and bigtable_instance_id != '': | |
cmd = cmd + [ | |
"-bigtable_project_name", | |
bigtable_project_name, | |
"-bigtable_instance_id", | |
bigtable_instance_id, | |
] | |
else: | |
print "Will connect to a local Bigtable Emulator instance." | |
cmd.append("-for_testing_only_use_bigtable_emulator") | |
if verbose_count > 0: | |
cmd.append("-v=%d"%verbose_count) | |
if vmodule: | |
cmd.append("-vmodule=%s"%vmodule) | |
return execute_command(cmd, wait) | |
REPORT_MASTER_PATH = os.path.abspath(os.path.join(OUT_DIR, 'analyzer', | |
'report_master', 'analyzer_report_master')) | |
def start_report_master(port=DEFAULT_REPORT_MASTER_PORT, | |
bigtable_project_name='', bigtable_instance_id='', | |
cobalt_config_dir=DEMO_CONFIG_DIR, | |
use_tls=False, | |
tls_cert_file=LOCALHOST_TLS_CERT_FILE, | |
tls_key_file=LOCALHOST_TLS_KEY_FILE, | |
verbose_count=0, vmodule=None, | |
wait=True): | |
print "Starting the analyzer ReportMaster service..." | |
cmd = [REPORT_MASTER_PATH, | |
"-port", str(port), | |
"-cobalt_config_dir", cobalt_config_dir, | |
"-logtostderr"] | |
if use_tls: | |
cmd.append("-use_tls") | |
cmd.append("-tls_cert_file=%s"%tls_cert_file) | |
cmd.append("-tls_key_file=%s"%tls_key_file) | |
if bigtable_project_name != '' and bigtable_instance_id != '': | |
cmd = cmd + [ | |
"-bigtable_project_name", | |
bigtable_project_name, | |
"-bigtable_instance_id", | |
bigtable_instance_id, | |
] | |
else: | |
print "Will connect to a local Bigtable Emulator instance." | |
cmd.append("-for_testing_only_use_bigtable_emulator") | |
if verbose_count > 0: | |
cmd.append("-v=%d"%verbose_count) | |
if vmodule: | |
cmd.append("-vmodule=%s"%vmodule) | |
return execute_command(cmd, wait) | |
TEST_APP_PATH = os.path.abspath(os.path.join(OUT_DIR, 'tools', 'test_app', | |
'cobalt_test_app')) | |
def start_test_app(shuffler_uri='', analyzer_uri='', use_tls=False, | |
root_certs_pem_file='', | |
analyzer_pk_pem_file=DEFAULT_ANALYZER_PUBLIC_KEY_PEM, | |
shuffler_pk_pem_file=DEFAULT_SHUFFLER_PUBLIC_KEY_PEM, | |
cobalt_config_dir=DEMO_CONFIG_DIR, | |
project_id=1, | |
verbose_count=0, wait=True): | |
cmd = [TEST_APP_PATH, | |
"-shuffler_uri", shuffler_uri, | |
"-analyzer_uri", analyzer_uri, | |
"-analyzer_pk_pem_file", analyzer_pk_pem_file, | |
"-shuffler_pk_pem_file", shuffler_pk_pem_file, | |
"-registry", cobalt_config_dir, | |
"-project", str(project_id), | |
"-logtostderr"] | |
if (use_tls): | |
cmd.append("-use_tls") | |
if (root_certs_pem_file): | |
cmd.append("-root_certs_pem_file") | |
cmd.append(root_certs_pem_file) | |
if verbose_count > 0: | |
cmd.append("-v=%d"%verbose_count) | |
return execute_command(cmd, wait) | |
def start_report_client(report_master_uri='', | |
use_tls=False, | |
root_certs_pem_file=LOCALHOST_TLS_CERT_FILE, | |
project_id=1, | |
verbose_count=0, wait=True): | |
path = os.path.abspath(os.path.join(OUT_DIR, 'tools', 'report_client')) | |
cmd = [path, | |
"-report_master_uri", report_master_uri, | |
"-project_id", str(project_id), | |
"-logtostderr"] | |
if (use_tls): | |
cmd.append("-tls") | |
if (root_certs_pem_file): | |
cmd.append("-ca_file") | |
cmd.append(root_certs_pem_file) | |
if verbose_count > 0: | |
cmd.append("-v=%d"%verbose_count) | |
return execute_command(cmd, wait) | |
OBSERVATION_QUERIER_PATH = os.path.abspath(os.path.join(OUT_DIR, 'tools', | |
'observation_querier', | |
'query_observations')) | |
def start_observation_querier(bigtable_project_name='', | |
bigtable_instance_id='', | |
verbose_count=0): | |
cmd = [OBSERVATION_QUERIER_PATH, | |
"-logtostderr"] | |
if not bigtable_project_name or not bigtable_instance_id: | |
cmd.append("-for_testing_only_use_bigtable_emulator") | |
else: | |
cmd = cmd + ["-bigtable_project_name", bigtable_project_name, | |
"-bigtable_instance_id", bigtable_instance_id] | |
if verbose_count > 0: | |
cmd.append("-v=%d"%verbose_count) | |
return execute_command(cmd, wait=True) | |