Improve the protobuf build.
This CL makes no changes to any behavior but improves the way protocol
buffers are compiled in Cobalt. The motivation for doing this is that in
a different CL I wanted to add a reference from a proto message in the
report_master directory to a proto message in the config directory and it
was impossible to make this compile in both C++ and Go the way our
protobufs were being compiled.
- All protobufs are now compiled to C++ using the root directory as the
only element of the .proto search path. This means that all of the
generated code thinks of itself as being located at a path which is the
relative location of the .proto file. This provides a consistent and
uniform way for all the generated C++ code to reference all the other
generated C++ code. All generated code is referenced using its full path
and all .proto files import other .proto files using their full path.
- In order to make the above happen it was necessary to stop using the
convenient CMake macro protobuf_generate_cpp and instead directly
embed the full invocation of the protobuf compiler into the CMake file.
The macro was not expressive enough for the change we wanted to make.
- The generated Go code for the .proto files located in the config
directory is now in the "config" package. This makes it consistent with
the way the other generated go code was.
- In order to make the above happen it was necessary to change the name
of the Shuffler's config package from "config" to "shuffler_config".
Change-Id: I4e23cd29fd07e8e373951efa9f4649a2ef4f10a4
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 7b1f080..72ce061 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -91,7 +91,7 @@
COMMAND ${GO_PROTOC}
${CMAKE_CURRENT_SOURCE_DIR}/encrypted_message.proto
${CMAKE_CURRENT_SOURCE_DIR}/observation.proto
- -I ${CMAKE_SOURCE_DIR}:${PROTOBUF_IMPORT_DIRS}
+ -I ${CMAKE_SOURCE_DIR}
--go_out=plugins=Mobservation.proto=cobalt,Mencrypted_message.proto=cobalt:${GO_PROTO_GEN_SRC_DIR}/cobalt
DEPENDS ${CMAKE_SOURCE_DIR}/encrypted_message.proto
DEPENDS ${CMAKE_SOURCE_DIR}/observation.proto
diff --git a/analyzer/analyzer_service/CMakeLists.txt b/analyzer/analyzer_service/CMakeLists.txt
index 2fd5920..12d9480 100644
--- a/analyzer/analyzer_service/CMakeLists.txt
+++ b/analyzer/analyzer_service/CMakeLists.txt
@@ -21,12 +21,16 @@
set(PROTO_GRPC "${CMAKE_CURRENT_BINARY_DIR}/analyzer.grpc.pb.cc")
add_custom_command(OUTPUT ${PROTO_GRPC}
COMMAND protoc ${CMAKE_CURRENT_SOURCE_DIR}/analyzer.proto
- -I ${CMAKE_CURRENT_SOURCE_DIR}:${PROTOBUF_IMPORT_DIRS}
- --grpc_out=.
+ -I ${CMAKE_SOURCE_DIR}
+ --grpc_out=${CMAKE_BINARY_DIR}
--plugin=protoc-gen-grpc=`which grpc_cpp_plugin`
DEPENDS analyzer.proto
)
+add_custom_target(build_analyzer_protos
+ DEPENDS ${PROTO_SRCS}
+ DEPENDS ${PROTO_GRPC})
+
# Generate the Go bindings for Analyzer service.
set(ANALYZER_PB_GO "${GO_PROTO_GEN_SRC_DIR}/analyzer/analyzer_service/analyzer.pb.go")
@@ -36,7 +40,7 @@
add_custom_command(OUTPUT "${ANALYZER_PB_GO}"
COMMAND ${GO_PROTOC}
${CMAKE_CURRENT_SOURCE_DIR}/analyzer.proto
- -I ${CMAKE_SOURCE_DIR}:${PROTOBUF_IMPORT_DIRS}
+ -I ${CMAKE_SOURCE_DIR}
--go_out=plugins=grpc,Mobservation.proto=cobalt,Mencrypted_message.proto=cobalt:${GO_PROTO_GEN_SRC_DIR}
DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/analyzer.proto
DEPENDS ${OBSERVATION_PB_GO}
diff --git a/analyzer/report_master/CMakeLists.txt b/analyzer/report_master/CMakeLists.txt
index 4581580..ad25488 100644
--- a/analyzer/report_master/CMakeLists.txt
+++ b/analyzer/report_master/CMakeLists.txt
@@ -13,15 +13,28 @@
# limitations under the License.
# Compile report_master.proto and report_internal.proto
-protobuf_generate_cpp(REPORT_PROTO_SRCS REPORT_PROTO_HDRS
- report_master.proto report_internal.proto)
+set(REPORT_PROTO_SRCS
+ "${CMAKE_CURRENT_BINARY_DIR}/report_master.pb.cc"
+ "${CMAKE_CURRENT_BINARY_DIR}/report_internal.pb.cc")
+set(REPORT_PROTO_HDRS
+ "${CMAKE_CURRENT_BINARY_DIR}/report_master.pb.h"
+ "${CMAKE_CURRENT_BINARY_DIR}/report_internal.pb.h")
+add_custom_command(OUTPUT ${REPORT_PROTO_HDRS}
+ OUTPUT ${REPORT_PROTO_SRCS}
+ COMMAND protoc ${CMAKE_CURRENT_SOURCE_DIR}/report_master.proto
+ ${CMAKE_CURRENT_SOURCE_DIR}/report_internal.proto
+ -I ${CMAKE_SOURCE_DIR}
+ --cpp_out=${CMAKE_BINARY_DIR}
+ DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/report_master.proto
+ DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/report_internal.proto
+)
# generate the gRPC service. CMake doesn't support it by default.
set(PROTO_GRPC "${CMAKE_CURRENT_BINARY_DIR}/report_master.grpc.pb.cc")
add_custom_command(OUTPUT ${PROTO_GRPC}
COMMAND protoc ${CMAKE_CURRENT_SOURCE_DIR}/report_master.proto
- -I ${CMAKE_CURRENT_SOURCE_DIR}:${PROTOBUF_IMPORT_DIRS}
- --grpc_out=.
+ -I ${CMAKE_SOURCE_DIR}
+ --grpc_out=${CMAKE_BINARY_DIR}
--plugin=protoc-gen-grpc=`which grpc_cpp_plugin`
DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/report_master.proto
)
@@ -34,17 +47,21 @@
# Generate the Go bindings for ReportMaster gRPC service.
set(REPORT_MASTER_PB_GO "${GO_PROTO_GEN_SRC_DIR}/analyzer/report_master/report_master.pb.go")
+set(ENCODINGS_PB_GO "${GO_PROTO_GEN_SRC_DIR}/config/encodings.pb.go")
-# This allows us to indicate below that report_master.pb.go depends on observation.pb.go
+# This allows us to indicate below that report_master.pb.go depends on
+# observation.pb.go and encodings.pb.go
set_source_files_properties(${OBSERVATION_PB_GO} PROPERTIES GENERATED TRUE)
+set_source_files_properties(${ENCODINGS_PB_GO} PROPERTIES GENERATED TRUE)
add_custom_command(OUTPUT "${REPORT_MASTER_PB_GO}"
COMMAND ${GO_PROTOC}
${CMAKE_CURRENT_SOURCE_DIR}/report_master.proto
- -I ${CMAKE_SOURCE_DIR}:${PROTOBUF_IMPORT_DIRS}
+ -I ${CMAKE_SOURCE_DIR}
--go_out=plugins=grpc,Mobservation.proto=cobalt,Mencrypted_message.proto=cobalt:${GO_PROTO_GEN_SRC_DIR}
DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/report_master.proto
DEPENDS ${OBSERVATION_PB_GO}
+ DEPENDS ${ENCODINGS_PB_GO}
)
add_custom_target(build_report_master_pb_go
diff --git a/analyzer/report_master/report_internal.proto b/analyzer/report_master/report_internal.proto
index 84d5180..4609647 100644
--- a/analyzer/report_master/report_internal.proto
+++ b/analyzer/report_master/report_internal.proto
@@ -15,7 +15,9 @@
package cobalt.analyzer;
-import "report_master.proto";
+import "analyzer/report_master/report_master.proto";
+
+option go_package = "config";
/////////////////////////////////////////////////////////////////////////////
// This file contains report-related proto messages that are used internally
diff --git a/config/CMakeLists.txt b/config/CMakeLists.txt
index d761fba..1b70c53 100644
--- a/config/CMakeLists.txt
+++ b/config/CMakeLists.txt
@@ -12,9 +12,40 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-set(PROTOBUF_IMPORT_DIRS ${CMAKE_SOURCE_DIR})
-protobuf_generate_cpp(CONFIG_PROTO_SRCS CONFIG_PROTO_HDRS
- encodings.proto metrics.proto report_configs.proto)
+add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/encodings.pb.cc
+ OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/encodings.pb.h
+ OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/metrics.pb.cc
+ OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/metrics.pb.h
+ OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/report_configs.pb.cc
+ OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/report_configs.pb.h
+ COMMAND protoc ${CMAKE_CURRENT_SOURCE_DIR}/encodings.proto
+ ${CMAKE_CURRENT_SOURCE_DIR}/metrics.proto
+ ${CMAKE_CURRENT_SOURCE_DIR}/report_configs.proto
+ -I ${CMAKE_SOURCE_DIR}
+ --cpp_out=${CMAKE_BINARY_DIR}
+ DEPENDS encodings.proto
+ DEPENDS metrics.proto
+ DEPENDS report_configs.proto
+)
+
+# Generate Go bindings for the config .proto files.
+set(ENCODINGS_PB_GO "${GO_PROTO_GEN_SRC_DIR}/config/encodings.pb.go")
+set(METRICS_PB_GO "${GO_PROTO_GEN_SRC_DIR}/config/metrics.pb.go")
+set(REPORT_CONFIGS_PB_GO "${GO_PROTO_GEN_SRC_DIR}/config/report_configs.pb.go")
+
+add_custom_command(OUTPUT ${ENCODINGS_PB_GO} ${METRICS_PB_GO} ${REPORT_CONFIGS_PB_GO}
+ COMMAND ${GO_PROTOC}
+ ${CMAKE_CURRENT_SOURCE_DIR}/encodings.proto
+ ${CMAKE_CURRENT_SOURCE_DIR}/metrics.proto
+ ${CMAKE_CURRENT_SOURCE_DIR}/report_configs.proto
+ -I ${CMAKE_SOURCE_DIR}
+ --go_out=plugins=Mencodings.proto=config,Mmetrics.proto=config,Mreport_configs.proto=config:${GO_PROTO_GEN_SRC_DIR}
+ DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/encodings.proto
+ DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/metrics.proto
+ DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/report_configs.proto)
+
+add_custom_target(build_config_go_protos
+ DEPENDS ${REPORT_CONFIGS_PB_GO})
# Build the analyzer_config library
add_library(analyzer_config
diff --git a/config/encodings.proto b/config/encodings.proto
index 6eea32e..596f238 100644
--- a/config/encodings.proto
+++ b/config/encodings.proto
@@ -15,6 +15,8 @@
package cobalt;
+option go_package = "config";
+
// A Cobalt Epoch is a contiguous sequence of days used to aggregate
// observations. Each Report analyzes a set of observations from one epoch.
// Some encodings, such as Forculus, use the current epoch as a parameter.
diff --git a/config/metrics.proto b/config/metrics.proto
index 1a81d67..e6e740d 100644
--- a/config/metrics.proto
+++ b/config/metrics.proto
@@ -15,6 +15,8 @@
package cobalt;
+option go_package = "config";
+
// A |Metric| is composed of one or more |MetricParts|. A |MetricPart| has a
// name which is specified as the key in the |parts| field of |Metric|, a
// description and a data type which specifies the type of a |ValuePart| that
diff --git a/config/report_configs.proto b/config/report_configs.proto
index cf38927..cac8b3f 100644
--- a/config/report_configs.proto
+++ b/config/report_configs.proto
@@ -13,10 +13,12 @@
// limitations under the License.
syntax = "proto3";
-import "encodings.proto";
-
package cobalt;
+import "config/encodings.proto";
+
+option go_package = "config";
+
message DifferentialPrivacyConfig {
float epsilon = 1;
float delta = 2;
diff --git a/observation.proto b/observation.proto
index 931181a..9efa129 100644
--- a/observation.proto
+++ b/observation.proto
@@ -79,7 +79,7 @@
// The encoding_config_id, along with the customer_id and project_id
// (specified in the containing Observation) form the primary key into the
// "EncodingConfigs" table in the Cobalt configuration database. The value
- // column is an "EncodingConfg" proto message describing how this
+ // column is an "EncodingConfig" proto message describing how this
// ObservationPart is encoded.
uint32 encoding_config_id = 5; // e.g. 114=(Forculus with threshold=50)
}
diff --git a/shuffler/CMakeLists.txt b/shuffler/CMakeLists.txt
index 5032035..22bc219 100644
--- a/shuffler/CMakeLists.txt
+++ b/shuffler/CMakeLists.txt
@@ -50,7 +50,7 @@
set(ELLIPTIC_UTIL_SRC "${CMAKE_CURRENT_SOURCE_DIR}/src/util/elliptic_util.go")
set(ENCRYPTED_MESSAGE_UTIL_SRC "${CMAKE_CURRENT_SOURCE_DIR}/src/util/encrypted_message_util.go")
set(RAND_UTIL_SRC "${CMAKE_CURRENT_SOURCE_DIR}/src/util/rand_util.go")
-set(CONFIG_SRC "${CMAKE_CURRENT_SOURCE_DIR}/src/config/config_util.go")
+set(CONFIG_SRC "${CMAKE_CURRENT_SOURCE_DIR}/src/shuffler_config/config_util.go")
set(STORE_SRC "${CMAKE_CURRENT_SOURCE_DIR}/src/storage/store.go")
set(MEM_STORE_SRC "${CMAKE_CURRENT_SOURCE_DIR}/src/storage/mem_store.go")
set(LEVELDB_STORE_SRC "${CMAKE_CURRENT_SOURCE_DIR}/src/storage/leveldb_store.go")
@@ -130,7 +130,7 @@
)
# Build config unit tests
-set(CONFIG_TEST_SRC "${CMAKE_CURRENT_SOURCE_DIR}/src/config/config_util_test.go")
+set(CONFIG_TEST_SRC "${CMAKE_CURRENT_SOURCE_DIR}/src/shuffler_config/config_util_test.go")
set(CONFIG_TEST_BIN ${GO_TESTS}/config_test)
add_custom_command(OUTPUT ${CONFIG_TEST_BIN}
COMMAND ${GO_BIN} test -c -o ${CONFIG_TEST_BIN} ${CONFIG_TEST_SRC} ${CONFIG_SRC}
diff --git a/shuffler/src/config/config_demo.txt b/shuffler/src/shuffler_config/config_demo.txt
similarity index 100%
rename from shuffler/src/config/config_demo.txt
rename to shuffler/src/shuffler_config/config_demo.txt
diff --git a/shuffler/src/config/config_util.go b/shuffler/src/shuffler_config/config_util.go
similarity index 98%
rename from shuffler/src/config/config_util.go
rename to shuffler/src/shuffler_config/config_util.go
index a6b4ee9..450c8d7 100644
--- a/shuffler/src/config/config_util.go
+++ b/shuffler/src/shuffler_config/config_util.go
@@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
-package config
+package shuffler_config
import (
"bufio"
diff --git a/shuffler/src/config/config_util_test.go b/shuffler/src/shuffler_config/config_util_test.go
similarity index 98%
rename from shuffler/src/config/config_util_test.go
rename to shuffler/src/shuffler_config/config_util_test.go
index 28bfd06..c419ac6 100644
--- a/shuffler/src/config/config_util_test.go
+++ b/shuffler/src/shuffler_config/config_util_test.go
@@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
-package config
+package shuffler_config
import (
"os"
@@ -26,7 +26,7 @@
"shuffler"
)
-var configDir = "shuffler/src/config"
+var configDir = "shuffler/src/shuffler_config"
func getTmpFile() string {
tmpFileSuffix := uint32(time.Now().UnixNano() + int64(os.Getpid()))
diff --git a/shuffler/src/config/config_v0.txt b/shuffler/src/shuffler_config/config_v0.txt
similarity index 100%
rename from shuffler/src/config/config_v0.txt
rename to shuffler/src/shuffler_config/config_v0.txt
diff --git a/shuffler/src/config/testdata/config_invalid.txt b/shuffler/src/shuffler_config/testdata/config_invalid.txt
similarity index 100%
rename from shuffler/src/config/testdata/config_invalid.txt
rename to shuffler/src/shuffler_config/testdata/config_invalid.txt
diff --git a/shuffler/src/config/testdata/config_valid.txt b/shuffler/src/shuffler_config/testdata/config_valid.txt
similarity index 100%
rename from shuffler/src/config/testdata/config_valid.txt
rename to shuffler/src/shuffler_config/testdata/config_valid.txt
diff --git a/shuffler/src/shuffler_main.go b/shuffler/src/shuffler_main.go
index 2e161fd..d9f7d81 100644
--- a/shuffler/src/shuffler_main.go
+++ b/shuffler/src/shuffler_main.go
@@ -15,16 +15,17 @@
package main
import (
- "config"
- "dispatcher"
"flag"
"io/ioutil"
"path/filepath"
"receiver"
- "shuffler"
- "storage"
"time"
+ "dispatcher"
+ "shuffler"
+ "shuffler_config"
+ "storage"
+
"github.com/golang/glog"
)
@@ -76,7 +77,7 @@
DisposalAgeDays: 4,
}
} else {
- if sConfig, err = config.LoadConfig(*configFile); err != nil {
+ if sConfig, err = shuffler_config.LoadConfig(*configFile); err != nil {
glog.Fatal("Error loading shuffler config file: [", *configFile, "]: ", err)
}
}
diff --git a/shuffler/src/src b/shuffler/src/src
new file mode 100755
index 0000000..bb9014a
--- /dev/null
+++ b/shuffler/src/src
Binary files differ
diff --git a/tools/process_starter.py b/tools/process_starter.py
index e7d7f6e..bb05c36 100644
--- a/tools/process_starter.py
+++ b/tools/process_starter.py
@@ -34,7 +34,7 @@
SHUFFLER_DB_DIR = os.path.join("/tmp/cobalt_shuffler")
SHUFFLER_CONFIG_DIR = os.path.abspath(os.path.join(SRC_ROOT_DIR, 'shuffler',
- 'src', 'config'))
+ 'src', 'shuffler_config'))
SHUFFLER_CONFIG_FILE = os.path.join(SHUFFLER_CONFIG_DIR, 'config_v0.txt')
SHUFFLER_DEMO_CONFIG_FILE = os.path.join(SHUFFLER_CONFIG_DIR, 'config_demo.txt')
SHUFFLER_TMP_DB_DIR = os.path.join("/tmp/cobalt_shuffler")
