Hooks up encryption to the Shuffler.
- Encryption to the Shuffler is now used in the end-to-end test.
- If you create a shuffler_public.pem and shuffler_private.pem file
in your root directory (using ./cobaltb.py keygen) then Encryption
to the shuffler will be used in a local demo of Cobalt.
- After you run ./cobaltb.py deploy upload_secret_keys then encryption
to the shuffler will be used in the demo or the end-to-end test run
against your Cloud instance of Cobalt.
Change-Id: Ib3e8be1879d4208b95b84ab170031346752e1a3a
diff --git a/README.md b/README.md
index 63ea8fc..af55d7a 100644
--- a/README.md
+++ b/README.md
@@ -54,7 +54,7 @@
## Generating PEM Files
Cobalt uses a custom public-key encryption scheme in which the Encoder encrypts
-Observations to the public-key of the Analyzer before sending them to the
+Observations to the public key of the Analyzer before sending them to the
Shuffler. This is a key part of the design of Cobalt and we refer to it
via the slogan "The Shuffler shuffles sealed envelopes" meaning that the
Shuffler does not get to see the data that it is shuffling. In order for this
@@ -63,7 +63,7 @@
PEM files located in the *end_to_end_tests* directory named
*analyzer_private_key.pem.e2e_test* and
*analyzer_public_key.pem.e2e_test*. But for running Cobalt in any other
-environment we do not want to check-in a private key into source control
+environment we do not want to check in a private key into source control
and so we ask each developer to generate thier own key pair.
`./cobaltb.py keygen`
@@ -74,6 +74,19 @@
steps including running the demo manually and deploying to Google Container
Engine.
+### Encryption to the Shuffler
+In addition to the encryption to the Analyzer mentioned above there is a second
+layer of encryption in which *Envelopes* are encrypted to the public key of the
+*Shuffler*. The purpose of this layer of encryption is that TLS between the
+Encoder and the Shuffler may be terminated prior to reaching the Shuffler in
+some load-balanced environments. We need a second public/private key pair for
+this encryption. The end-to-end test uses the PEM files located in the
+*end_to_end_tests* directory named *shuffler_private_key.pem.e2e_test* and
+*shuffler_public_key.pem.e2e_test*. But for running Cobalt in any other
+environment follow the instructions above for generating *analyzer_public.pem*
+and *analyzer_private.pem* but this time create two new files named
+*shuffler_public.pem* and *shuffler_private.pem*
+
## Running the Demo Manually
You can run a complete Cobalt system locally (for example in order to give a
@@ -405,12 +418,13 @@
Run this one time in order associate your computer with your GKE cluster
and set up authentication.
-`./cobaltb.py deploy upload_secret_key`
-Run this one time in order to upload the PEM file containing the Analyzer's
-private key. This is the file *analyzer_private.pem* that was created in
-the section **Generating PEM Files** above. To upload a different private key,
-first delete any previously upload secret key by running
-`./cobaltb.py deploy delete_secret_key`
+`./cobaltb.py deploy upload_secret_keys`
+Run this one time in order to upload the PEM files containing the Analyzer's
+and Shuffler's private keys. These are the files *analyzer_private.pem* and
+*shuffler_private.pem* that were created in the section
+**Generating PEM Files** above. To upload different private keys,
+first delete any previously upload secret keys by running
+`./cobaltb.py deploy delete_secret_keys`
`./cobaltb.py deploy build`
Run this to build Docker containers for the Shuffler, Analyzer Service and
diff --git a/cobaltb.py b/cobaltb.py
index 2250430..3c72e01 100755
--- a/cobaltb.py
+++ b/cobaltb.py
@@ -30,10 +30,12 @@
import tools.test_runner as test_runner
from tools.test_runner import E2E_TEST_ANALYZER_PUBLIC_KEY_PEM
+from tools.test_runner import E2E_TEST_SHUFFLER_PUBLIC_KEY_PEM
-from tools.process_starter import DEFAULT_SHUFFLER_PORT
from tools.process_starter import DEFAULT_ANALYZER_PUBLIC_KEY_PEM
+from tools.process_starter import DEFAULT_SHUFFLER_PUBLIC_KEY_PEM
from tools.process_starter import DEFAULT_ANALYZER_SERVICE_PORT
+from tools.process_starter import DEFAULT_SHUFFLER_PORT
from tools.process_starter import DEFAULT_REPORT_MASTER_PORT
from tools.process_starter import DEMO_CONFIG_DIR
from tools.process_starter import SHUFFLER_DEMO_CONFIG_FILE
@@ -165,12 +167,14 @@
analyzer_pk_pem_file=E2E_TEST_ANALYZER_PUBLIC_KEY_PEM
analyzer_uri = "localhost:%d" % DEFAULT_ANALYZER_SERVICE_PORT
report_master_uri = "localhost:%d" % DEFAULT_REPORT_MASTER_PORT
+ shuffler_pk_pem_file=E2E_TEST_SHUFFLER_PUBLIC_KEY_PEM
shuffler_uri = "localhost:%d" % DEFAULT_SHUFFLER_PORT
if args.cobalt_on_gke:
public_uris = container_util.get_public_uris()
analyzer_pk_pem_file=DEFAULT_ANALYZER_PUBLIC_KEY_PEM
analyzer_uri = public_uris["analyzer"]
report_master_uri = public_uris["report_master"]
+ shuffler_pk_pem_file=DEFAULT_SHUFFLER_PUBLIC_KEY_PEM
shuffler_uri = public_uris["shuffler"]
if args.use_cloud_bt:
# use_cloud_bt means to use local instances of the Cobalt processes
@@ -184,6 +188,7 @@
"-analyzer_uri=%s" % analyzer_uri,
"-analyzer_pk_pem_file=%s" % analyzer_pk_pem_file,
"-shuffler_uri=%s" % shuffler_uri,
+ "-shuffler_pk_pem_file=%s" % shuffler_pk_pem_file,
"-report_master_uri=%s" % report_master_uri,
("-observation_querier_path=%s" %
process_starter.OBSERVATION_QUERIER_PATH),
@@ -383,11 +388,13 @@
print('Unknown job "%s". I only know how to stop "shuffler", '
'"analyzer-service" and "report-master".' % args.job)
-def _deploy_upload_secret_key(args):
+def _deploy_upload_secret_keys(args):
container_util.create_analyzer_private_key_secret()
+ container_util.create_shuffler_private_key_secret()
-def _deploy_delete_secret_key(args):
+def _deploy_delete_secret_keys(args):
container_util.delete_analyzer_private_key_secret()
+ container_util.delete_shuffler_private_key_secret()
def main():
personal_cluster_settings = {
@@ -750,20 +757,21 @@
help='The job you wish to stop. Valid choices are "shuffler", '
'"analyzer-service", "report-master". Required.')
- sub_parser = deploy_subparsers.add_parser('upload_secret_key',
- parents=[parent_parser], help='Creates a |secret| object in the '
- 'cluster to store a private key for the Analyzer. The private key must '
- 'first be generated using the "generate_keys" command. This must be done '
- 'at least once before starting the Analyzer Service. To replace the '
- 'key first delete the old one using the "deploy delete_secret_key" '
- 'command.')
- sub_parser.set_defaults(func=_deploy_upload_secret_key)
+ sub_parser = deploy_subparsers.add_parser('upload_secret_keys',
+ parents=[parent_parser], help='Creates |secret| objects in the '
+ 'cluster to store the private keys for the Analyzer and the Shuffler. '
+ 'The private keys must first be generated using the "generate_keys" '
+ 'command (once for the Analyzer and once for the Shuffler). This must be '
+ 'done at least once before starting the Analyzer Service or the '
+ 'Shuffler. To replace the keys first delete the old ones using the '
+ '"deploy delete_secret_keys" command.')
+ sub_parser.set_defaults(func=_deploy_upload_secret_keys)
- sub_parser = deploy_subparsers.add_parser('delete_secret_key',
- parents=[parent_parser], help='Deletes a |secret| object in the '
- 'cluster that was created using the "deploy upload_secret_key" '
+ sub_parser = deploy_subparsers.add_parser('delete_secret_keys',
+ parents=[parent_parser], help='Deletes the |secret| objects in the '
+ 'cluster that were created using the "deploy upload_secret_keys" '
'command.')
- sub_parser.set_defaults(func=_deploy_delete_secret_key)
+ sub_parser.set_defaults(func=_deploy_delete_secret_keys)
args = parser.parse_args()
diff --git a/end_to_end_tests/shuffler_private_key.pem.e2e_test b/end_to_end_tests/shuffler_private_key.pem.e2e_test
new file mode 100644
index 0000000..675c43a
--- /dev/null
+++ b/end_to_end_tests/shuffler_private_key.pem.e2e_test
@@ -0,0 +1,5 @@
+-----BEGIN PRIVATE KEY-----
+MIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQg1kZxvT81qrRWg2Y8
+g/M7YNtiHaC14/fbevhy/hgXcByhRANCAASkbLO+7iLLaPayYIr3YVmY0jkbwalG
+sOB9Tf3R8TR7Ow43cHlGjX3HALV1z4Lxs1v2K13yeegBJF8lU88cdAqY
+-----END PRIVATE KEY-----
diff --git a/end_to_end_tests/shuffler_public_key.pem.e2e_test b/end_to_end_tests/shuffler_public_key.pem.e2e_test
new file mode 100644
index 0000000..666f108
--- /dev/null
+++ b/end_to_end_tests/shuffler_public_key.pem.e2e_test
@@ -0,0 +1,4 @@
+-----BEGIN PUBLIC KEY-----
+MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEpGyzvu4iy2j2smCK92FZmNI5G8Gp
+RrDgfU390fE0ezsON3B5Ro19xwC1dc+C8bNb9itd8nnoASRfJVPPHHQKmA==
+-----END PUBLIC KEY-----
diff --git a/kubernetes/shuffler/shuffler_deployment.yaml b/kubernetes/shuffler/shuffler_deployment.yaml
index 41ad058..263346d 100644
--- a/kubernetes/shuffler/shuffler_deployment.yaml
+++ b/kubernetes/shuffler/shuffler_deployment.yaml
@@ -51,8 +51,13 @@
# This is the path to where the file is copied in Dockerfile.
- '/etc/cobalt/shuffler_config.txt'
- '-db_dir'
- # This path must match mountPath below.
- - '/var/lib/cobalt'
+ # This path must match mountPath for shuffler-persistent-storage
+ # below.
+ - '/var/lib/cobalt/db'
+ - '-private_key_pem_file'
+ # The directory path must match mountPath for shuffler-key-storage
+ # below.
+ - '/var/lib/cobalt/key/$$SHUFFLER_PRIVATE_PEM_NAME$$'
- '-danger_danger_delete_all_data_at_startup=$$DANGER_DANGER_DELETE_ALL_DATA_AT_STARTUP$$'
# TODO(rudominer) Eventually remove this.
- '-logtostderr'
@@ -60,15 +65,24 @@
ports:
- containerPort: 5001
volumeMounts:
- # This name must match the volumes.name below.
+ # This name must match the volumes.name for
+ # shuffler-persistent-storage below.
- name: shuffler-persistent-storage
- mountPath: /var/lib/cobalt
+ mountPath: /var/lib/cobalt/db
+ # This name must match the volumes.name for shuffler-key-storage
+ # below.
+ - name: shuffler-key-storage
+ mountPath: /var/lib/cobalt/key
+ readOnly: true
volumes:
- name: shuffler-persistent-storage
gcePersistentDisk:
- # The name of a GCe persistent disk that has already been created.
+ # The name of a GCE persistent disk that has already been created.
pdName: $$GCE_PERSISTENT_DISK_NAME$$
fsType: ext4
+ - name: shuffler-key-storage
+ secret:
+ secretName: $$SHUFFLER_PRIVATE_KEY_SECRET_NAME$$
---
# The defintion of the Service
apiVersion: v1
diff --git a/shuffler/CMakeLists.txt b/shuffler/CMakeLists.txt
index 3757816..5032035 100644
--- a/shuffler/CMakeLists.txt
+++ b/shuffler/CMakeLists.txt
@@ -60,7 +60,7 @@
set(LEVELDB_STORE_ITER_SRC "${CMAKE_CURRENT_SOURCE_DIR}/src/storage/leveldb_store_iter.go")
add_custom_command(OUTPUT ${SHUFFLER_BIN}
# Compiles shuffler_main and all its dependencies
- COMMAND ${GO_BIN} build -o ${SHUFFLER_BIN} shuffler_main.go
+ COMMAND ${GO_BIN} build -o ${SHUFFLER_BIN}
DEPENDS ${SHUFFLER_PB_GO}
DEPENDS ${SHUFFLER_DB_PB_GO}
DEPENDS ${ANALYZER_PB_GO}
diff --git a/shuffler/src/receiver/receiver.go b/shuffler/src/receiver/receiver.go
index 9c9d515..4580d58 100644
--- a/shuffler/src/receiver/receiver.go
+++ b/shuffler/src/receiver/receiver.go
@@ -30,7 +30,6 @@
"time"
"github.com/golang/glog"
- "github.com/golang/protobuf/proto"
"github.com/golang/protobuf/ptypes/empty"
"golang.org/x/net/context"
"google.golang.org/grpc"
@@ -40,14 +39,16 @@
"cobalt"
"shuffler"
"storage"
+ "util"
)
-var shufflerServer *ShufflerServer
+var shufflerServerSingleton *ShufflerServer
// ShufflerServer implements the Shufffler service.
type ShufflerServer struct {
- store storage.Store
- config ServerConfig
+ store storage.Store
+ config ServerConfig
+ decrypter *util.MessageDecrypter
}
// ServerConfig specifies the configuration options for setting up a Grpc
@@ -61,6 +62,11 @@
KeyFile string
// The server port
Port int
+ // A PEM encoding of the Shuffler's private key for use in Cobalt's custom
+ // hybrid encryption scheme.
+ // TODO(rudominer) Support key rotation: Rather than a single private key
+ // this should be a set of (public-key-hash, private-key) pairs.
+ PrivateKeyPem string
}
// Process processes the incoming encoder requests and persists them locally in
@@ -69,10 +75,9 @@
func (s *ShufflerServer) Process(ctx context.Context,
encryptedMessage *cobalt.EncryptedMessage) (*empty.Empty, error) {
glog.V(4).Infoln("Process() is invoked.")
- envelope, err := decryptEnvelope(encryptedMessage)
-
+ envelope, err := s.decryptEnvelope(encryptedMessage)
if err != nil {
- return nil, grpc.Errorf(codes.InvalidArgument, "Failed to decrypt encrypted message: %v", err)
+ return nil, err
}
// TODO(ukode): Some notes here for future development:
@@ -108,16 +113,17 @@
glog.Fatal("Invalid server config, exiting.")
}
- if shufflerServer != nil {
+ if shufflerServerSingleton != nil {
glog.Fatal("Run() must not be invoked twice, exiting.")
}
// Start shuffler service
- shufflerServer := &ShufflerServer{
- store: dataStore,
- config: *config,
+ shufflerServerSingleton = &ShufflerServer{
+ store: dataStore,
+ config: *config,
+ decrypter: util.NewMessageDecrypter(config.PrivateKeyPem),
}
- shufflerServer.startServer()
+ shufflerServerSingleton.startServer()
}
// startServer sets up and starts the grpc server using configuration from
@@ -144,14 +150,15 @@
grpcServer.Serve(lis)
}
-// decryptEnvelope decrypts the incoming encoder message and returns the sealed
-// envelope or an error.
-func decryptEnvelope(encryptedMessage *cobalt.EncryptedMessage) (*cobalt.Envelope, error) {
- ciphertext := encryptedMessage.Ciphertext
-
- // TODO: Perform PKE decryption on ciphertext and then unmarshall the
- // decrypted bytes.
- envelope := &cobalt.Envelope{}
- err := proto.Unmarshal([]byte(ciphertext), envelope)
- return envelope, err
+// decryptEnvelope decrypts the incoming EncryptedMessage and returns an Envelope or an error.
+func (s *ShufflerServer) decryptEnvelope(encryptedMessage *cobalt.EncryptedMessage) (*cobalt.Envelope, error) {
+ if s.decrypter == nil {
+ return nil, grpc.Errorf(codes.Internal, "s.decrypter is nil")
+ }
+ envelope := new(cobalt.Envelope)
+ if err := s.decrypter.DecryptMessage(encryptedMessage, envelope); err != nil {
+ glog.Errorf("Decryption failed: %v", err)
+ return nil, err
+ }
+ return envelope, nil
}
diff --git a/shuffler/src/receiver/receiver_test.go b/shuffler/src/receiver/receiver_test.go
index 4239bc4..5ff8d74 100644
--- a/shuffler/src/receiver/receiver_test.go
+++ b/shuffler/src/receiver/receiver_test.go
@@ -23,6 +23,7 @@
shufflerpb "cobalt"
"storage"
+ "util"
)
// makeEnvelope creates an Envelope containing |numBatches| ObservationBatches
@@ -85,6 +86,7 @@
KeyFile: "",
Port: 0,
},
+ decrypter: util.NewMessageDecrypter(""),
}
_, err = shuffler.Process(context.Background(), eMsg)
diff --git a/shuffler/src/shuffler_main.go b/shuffler/src/shuffler_main.go
index 0bd32b1..2e161fd 100644
--- a/shuffler/src/shuffler_main.go
+++ b/shuffler/src/shuffler_main.go
@@ -18,6 +18,7 @@
"config"
"dispatcher"
"flag"
+ "io/ioutil"
"path/filepath"
"receiver"
"shuffler"
@@ -36,6 +37,11 @@
keyFile = flag.String("key_file", "", "The TLS key file")
port = flag.Int("port", 50051, "The server port")
+ privateKeyPemFile = flag.String("private_key_pem_file", "",
+ "Path to a file containing a PEM encoding of the private key of "+
+ "the Shuffler used for Cobalt's internal encryption scheme. If "+
+ "not specified then the Shuffler will not support encrypted Envelopes.")
+
// shuffler client configuration flags to connect to analyzer
caFile = flag.String("ca_file", "", "The file containing the CA root certificate")
timeout = flag.Int("timeout", 30, "Grpc connection timeout in seconds")
@@ -75,6 +81,20 @@
}
}
+ // Read the private key PEM file
+ privateKeyPem := ""
+ if *privateKeyPemFile != "" {
+ if fileContents, err := ioutil.ReadFile(*privateKeyPemFile); err != nil {
+ glog.Errorf("Error attempting to read private key PEM file %s: %v. "+
+ "The shuffler will not be able to decrypt EncryptedMessages.", *privateKeyPemFile, err)
+ } else {
+ glog.Infof("Successfully read private key PEM file %s.", *privateKeyPemFile)
+ privateKeyPem = string(fileContents)
+ }
+ } else {
+ glog.Warning("The flag -private_key_pem_file was not provided. The shuffler will not be able to decrypt EncryptedMessages.")
+ }
+
// Initialize Shuffler data store
var store storage.Store
if *useMemStore {
@@ -117,9 +137,10 @@
// Start listening on receiver for incoming requests from Encoder
receiver.Run(store, &receiver.ServerConfig{
- EnableTLS: *tls,
- CertFile: *certFile,
- KeyFile: *keyFile,
- Port: *port,
+ EnableTLS: *tls,
+ CertFile: *certFile,
+ KeyFile: *keyFile,
+ Port: *port,
+ PrivateKeyPem: privateKeyPem,
})
}
diff --git a/shuffler/src/util/encrypted_message_util.go b/shuffler/src/util/encrypted_message_util.go
index 66cb0ca..1730acf 100644
--- a/shuffler/src/util/encrypted_message_util.go
+++ b/shuffler/src/util/encrypted_message_util.go
@@ -174,7 +174,7 @@
return grpc.Errorf(codes.InvalidArgument, "Unrecognized encryption scheme specified in EncryptedMessage: %v", encryptedMessage.Scheme)
}
if m.hybridCipher == nil {
- return grpc.Errorf(codes.Internal, "m.hybridCipher is nil")
+ return grpc.Errorf(codes.Internal, "Cannot decrypt: Decryption was not successfully initialized.")
}
recoveredText, err := m.hybridCipher.Decrypt(encryptedMessage.Ciphertext)
if err != nil {
diff --git a/tools/container_util.py b/tools/container_util.py
index cd41ba9..41f73df 100755
--- a/tools/container_util.py
+++ b/tools/container_util.py
@@ -31,6 +31,8 @@
from process_starter import DEFAULT_SHUFFLER_PORT
from process_starter import REGISTERED_CONFIG_DIR
from process_starter import REPORT_MASTER_PATH
+from process_starter import SHUFFLER_PRIVATE_KEY_PEM_NAME
+from process_starter import DEFAULT_SHUFFLER_PRIVATE_KEY_PEM
from process_starter import SHUFFLER_CONFIG_FILE
from process_starter import SHUFFLER_PATH
@@ -108,6 +110,7 @@
]]
ANALYZER_PRIVATE_KEY_SECRET_NAME = "analyzer-private-key"
+SHUFFLER_PRIVATE_KEY_SECRET_NAME = "shuffler-private-key"
def _ensure_dir(dir_path):
"""Ensures that the directory at |dir_path| exists. If not it is created.
@@ -226,6 +229,12 @@
ANALYZER_PRIVATE_KEY_PEM_NAME,
path_to_pem)
+def create_shuffler_private_key_secret(
+ path_to_pem=DEFAULT_SHUFFLER_PRIVATE_KEY_PEM):
+ _create_secret_from_file(SHUFFLER_PRIVATE_KEY_SECRET_NAME,
+ SHUFFLER_PRIVATE_KEY_PEM_NAME,
+ path_to_pem)
+
def delete_analyzer_private_key_secret():
_delete_secret(ANALYZER_PRIVATE_KEY_SECRET_NAME)
@@ -314,6 +323,8 @@
delete_all_data = 'true'
token_substitutions = {'$$SHUFFLER_IMAGE_URI$$' : image_uri,
'$$GCE_PERSISTENT_DISK_NAME$$' : gce_pd_name,
+ '$$SHUFFLER_PRIVATE_PEM_NAME$$' : SHUFFLER_PRIVATE_KEY_PEM_NAME,
+ '$$SHUFFLER_PRIVATE_KEY_SECRET_NAME$$' : SHUFFLER_PRIVATE_KEY_SECRET_NAME,
'$$DANGER_DANGER_DELETE_ALL_DATA_AT_STARTUP$$' : delete_all_data}
_start_gke_service(SHUFFLER_DEPLOYMENT_TEMPLATE_FILE,
SHUFFLER_DEPLOYMENT_FILE,
diff --git a/tools/process_starter.py b/tools/process_starter.py
index 399752b..e7d7f6e 100644
--- a/tools/process_starter.py
+++ b/tools/process_starter.py
@@ -50,8 +50,9 @@
ANALYZER_PRIVATE_KEY_PEM_NAME)
DEFAULT_SHUFFLER_PUBLIC_KEY_PEM=os.path.join(SRC_ROOT_DIR,
"shuffler_public.pem")
+SHUFFLER_PRIVATE_KEY_PEM_NAME="shuffler_private.pem"
DEFAULT_SHUFFLER_PRIVATE_KEY_PEM=os.path.join(SRC_ROOT_DIR,
- "shuffler_private.pem")
+ SHUFFLER_PRIVATE_KEY_PEM_NAME)
def kill_process(process, name):
@@ -108,6 +109,7 @@
analyzer_uri='localhost:%d' % DEFAULT_ANALYZER_SERVICE_PORT,
use_memstore=False, erase_db=True, db_dir=SHUFFLER_TMP_DB_DIR,
config_file=SHUFFLER_DEMO_CONFIG_FILE,
+ private_key_pem_file=DEFAULT_SHUFFLER_PRIVATE_KEY_PEM,
verbose_count=0, wait=True):
"""Starts the Shuffler.
@@ -122,6 +124,7 @@
print
cmd = [SHUFFLER_PATH,
"-port", str(port),
+ "-private_key_pem_file", private_key_pem_file,
"-analyzer_uri", analyzer_uri,
"-config_file", config_file,
"-logtostderr"]
diff --git a/tools/test_runner.py b/tools/test_runner.py
index baeda25..eea18b9 100644
--- a/tools/test_runner.py
+++ b/tools/test_runner.py
@@ -32,6 +32,10 @@
"analyzer_private_key.pem.e2e_test")
E2E_TEST_ANALYZER_PUBLIC_KEY_PEM = os.path.join(E2E_DIR,
"analyzer_public_key.pem.e2e_test")
+E2E_TEST_SHUFFLER_PRIVATE_KEY_PEM = os.path.join(E2E_DIR,
+ "shuffler_private_key.pem.e2e_test")
+E2E_TEST_SHUFFLER_PUBLIC_KEY_PEM = os.path.join(E2E_DIR,
+ "shuffler_public_key.pem.e2e_test")
_logger = logging.getLogger()
@@ -99,7 +103,9 @@
bigtable_project_name=bigtable_project_name,
verbose_count=verbose_count, wait=False)
time.sleep(1)
- shuffler_process=process_starter.start_shuffler(wait=False)
+ shuffler_process=process_starter.start_shuffler(
+ private_key_pem_file=E2E_TEST_SHUFFLER_PRIVATE_KEY_PEM,
+ verbose_count=verbose_count, wait=False)
print "Running %s..." % test_executable
path = os.path.abspath(os.path.join(tdir, test_executable))
command = [path] + test_args
