| // Copyright 2018 The Fuchsia Authors. All rights reserved. |
| // Use of this source code is governed by a BSD-style license that can be |
| // found in the LICENSE file. |
| |
| #include "src/logger/encoder.h" |
| |
| #include <memory> |
| #include <string> |
| |
| #include "src/algorithms/rappor/rappor_config_helper.h" |
| #include "src/algorithms/rappor/rappor_encoder.h" |
| #include "src/lib/crypto_util/hash.h" |
| #include "src/logger/project_context.h" |
| #include "src/logging.h" |
| #include "src/pb/observation2.pb.h" |
| #include "src/registry/buckets_config.h" |
| #include "src/registry/packed_event_codes.h" |
| #include "src/tracing.h" |
| |
| namespace cobalt::logger { |
| |
| using ::cobalt::config::IntegerBucketConfig; |
| using ::cobalt::crypto::byte; |
| using ::cobalt::crypto::hash::DIGEST_SIZE; |
| using ::cobalt::rappor::BasicRapporEncoder; |
| using ::cobalt::rappor::RapporConfigHelper; |
| using ::cobalt::system_data::ClientSecret; |
| using ::cobalt::system_data::SystemDataInterface; |
| using ::google::protobuf::RepeatedField; |
| |
| namespace { |
| // Populates |*hash_out| with the SHA256 of |component|, unless |component| |
| // is empty in which case *hash_out is set to the empty string also. An |
| // empty string indicates that the component_name feature is not being used. |
| // We expect this to be a common case and in this case there is no point |
| // in using 32 bytes to represent the empty string. Returns true on success |
| // and false on failure (unexpected). |
| bool HashComponentNameIfNotEmpty(const std::string& component, std::string* hash_out) { |
| CHECK(hash_out); |
| if (component.empty()) { |
| hash_out->resize(0); |
| return true; |
| } |
| hash_out->resize(DIGEST_SIZE); |
| return cobalt::crypto::hash::Hash(reinterpret_cast<const byte*>(component.data()), |
| component.size(), reinterpret_cast<byte*>(&hash_out->front())); |
| } |
| |
| // Translates a rappor::Status |status| into a logger::Status and prints a debug |
| // message if |status| is not kOK. |
| Status TranslateBasicRapporEncoderStatus(MetricRef metric, const ReportDefinition* report, |
| const rappor::Status& status) { |
| switch (status) { |
| case rappor::kOK: |
| return kOK; |
| case rappor::kInvalidConfig: |
| LOG(ERROR) << "BasicRapporEncoder returned kInvalidConfig for: Report " |
| << report->report_name() << " for metric " << metric.metric_name() |
| << " in project " << metric.ProjectDebugString() << "."; |
| return kInvalidConfig; |
| case rappor::kInvalidInput: |
| LOG(ERROR) << "BasicRapporEncoder returned kInvalidInput for: Report " |
| << report->report_name() << " for metric " << metric.metric_name() |
| << " in project " << metric.ProjectDebugString() << "."; |
| return kInvalidArguments; |
| } |
| } |
| |
| } // namespace |
| |
| Encoder::Encoder(ClientSecret client_secret, const system_data::SystemDataInterface* system_data) |
| : client_secret_(std::move(client_secret)), system_data_(system_data) {} |
| |
| Encoder::Result Encoder::EncodeBasicRapporObservation(MetricRef metric, |
| const ReportDefinition* report, |
| uint32_t day_index, uint32_t value_index, |
| uint32_t num_categories) const { |
| TRACE_DURATION("cobalt_core", "Encoder::EncodeBasicRapporObservation"); |
| |
| auto result = MakeObservation(metric, report, day_index); |
| auto* observation = result.observation.get(); |
| auto* basic_rappor_observation = observation->mutable_basic_rappor(); |
| |
| rappor::BasicRapporConfig basic_rappor_config; |
| basic_rappor_config.prob_rr = RapporConfigHelper::kProbRR; |
| basic_rappor_config.categories.set_indexed(num_categories); |
| float prob_bit_flip = RapporConfigHelper::ProbBitFlip(*report, metric.FullyQualifiedName()); |
| basic_rappor_config.prob_0_becomes_1 = prob_bit_flip; |
| basic_rappor_config.prob_1_stays_1 = 1.0f - prob_bit_flip; |
| |
| // TODO(rudominer) Stop copying the client_secret_ on each Encode*() |
| // operation. |
| BasicRapporEncoder basic_rappor_encoder(basic_rappor_config, client_secret_); |
| ValuePart index_value; |
| index_value.set_index_value(value_index); |
| result.status = TranslateBasicRapporEncoderStatus( |
| metric, report, basic_rappor_encoder.Encode(index_value, basic_rappor_observation)); |
| return result; |
| } |
| |
| Encoder::Result Encoder::EncodeIntegerEventObservation( |
| MetricRef metric, const ReportDefinition* report, uint32_t day_index, |
| const RepeatedField<uint32_t>& event_codes, const std::string& component, int64_t value) const { |
| auto result = MakeObservation(metric, report, day_index); |
| auto* observation = result.observation.get(); |
| auto* integer_event_observation = observation->mutable_numeric_event(); |
| integer_event_observation->set_event_code(config::PackEventCodes(event_codes)); |
| if (!HashComponentNameIfNotEmpty(component, |
| integer_event_observation->mutable_component_name_hash())) { |
| LOG(ERROR) << "Hashing the component name failed for: Report " << report->report_name() |
| << " for metric " << metric.metric_name() << " in project " |
| << metric.ProjectDebugString() << "."; |
| result.status = kOther; |
| } |
| integer_event_observation->set_value(value); |
| return result; |
| } |
| |
| Encoder::Result Encoder::EncodeHistogramObservation(MetricRef metric, |
| const ReportDefinition* report, |
| uint32_t day_index, |
| const RepeatedField<uint32_t>& event_codes, |
| const std::string& component, |
| HistogramPtr histogram) const { |
| auto result = MakeObservation(metric, report, day_index); |
| auto* observation = result.observation.get(); |
| auto* histogram_observation = observation->mutable_histogram(); |
| histogram_observation->set_event_code(config::PackEventCodes(event_codes)); |
| if (!HashComponentNameIfNotEmpty(component, |
| histogram_observation->mutable_component_name_hash())) { |
| LOG(ERROR) << "Hashing the component name failed for: Report " << report->report_name() |
| << " for metric " << metric.metric_name() << " in project " |
| << metric.ProjectDebugString() << "."; |
| result.status = kOther; |
| } |
| histogram_observation->mutable_buckets()->Swap(histogram.get()); |
| return result; |
| } |
| |
| Encoder::Result Encoder::EncodeCustomObservation(MetricRef metric, const ReportDefinition* report, |
| uint32_t day_index, |
| EventValuesPtr event_values) const { |
| auto result = MakeObservation(metric, report, day_index); |
| auto* observation = result.observation.get(); |
| auto* custom_observation = observation->mutable_custom(); |
| custom_observation->mutable_values()->swap(*event_values); |
| return result; |
| } |
| |
| Encoder::Result Encoder::EncodeUniqueActivesObservation( |
| MetricRef metric, const ReportDefinition* report, uint32_t day_index, uint32_t event_code, |
| bool was_active, const OnDeviceAggregationWindow& aggregation_window) const { |
| auto result = MakeObservation(metric, report, day_index); |
| Encoder::Result basic_rappor_result; |
| if (was_active) { |
| // Encode a single 1 bit |
| basic_rappor_result = EncodeBasicRapporObservation(metric, report, day_index, 0u, 1u); |
| } else { |
| // Encode a single 0 bit |
| basic_rappor_result = EncodeNullBasicRapporObservation(metric, report, day_index, 1u); |
| } |
| if (basic_rappor_result.status != kOK) { |
| result.status = basic_rappor_result.status; |
| return result; |
| } |
| auto* activity_observation = result.observation->mutable_unique_actives(); |
| *activity_observation->mutable_aggregation_window() = aggregation_window; |
| activity_observation->set_event_code(event_code); |
| activity_observation->mutable_basic_rappor_obs()->mutable_data()->swap( |
| *(basic_rappor_result.observation->mutable_basic_rappor()->mutable_data())); |
| |
| return result; |
| } |
| |
| Encoder::Result Encoder::EncodePerDeviceNumericObservation( |
| MetricRef metric, const ReportDefinition* report, uint32_t day_index, |
| const std::string& component, const RepeatedField<uint32_t>& event_codes, int64_t value, |
| const OnDeviceAggregationWindow& aggregation_window) const { |
| auto result = |
| EncodeIntegerEventObservation(metric, report, day_index, event_codes, component, value); |
| auto* integer_event_observation = result.observation->release_numeric_event(); |
| auto* per_device_observation = result.observation->mutable_per_device_numeric(); |
| per_device_observation->set_allocated_integer_event_obs(integer_event_observation); |
| *per_device_observation->mutable_aggregation_window() = aggregation_window; |
| return result; |
| } |
| |
| Encoder::Result Encoder::EncodePerDeviceHistogramObservation( |
| MetricRef metric, const ReportDefinition* report, uint32_t day_index, |
| const std::string& component, const RepeatedField<uint32_t>& event_codes, int64_t value, |
| const OnDeviceAggregationWindow& aggregation_window) const { |
| auto result = MakeObservation(metric, report, day_index); |
| auto* per_device_histogram_obs = result.observation->mutable_per_device_histogram(); |
| *per_device_histogram_obs->mutable_aggregation_window() = aggregation_window; |
| auto* histogram_observation = per_device_histogram_obs->mutable_histogram(); |
| histogram_observation->set_event_code(config::PackEventCodes(event_codes)); |
| if (!HashComponentNameIfNotEmpty(component, |
| histogram_observation->mutable_component_name_hash())) { |
| LOG(ERROR) << "Hashing the component name failed for: Report " << report->report_name() |
| << " for metric " << metric.metric_name() << " in project " |
| << metric.ProjectDebugString() << "."; |
| result.status = kOther; |
| } |
| |
| auto integer_bucket_config = IntegerBucketConfig::CreateFromProto(report->int_buckets()); |
| if (integer_bucket_config == nullptr) { |
| LOG(ERROR) << "Invalid IntBucketConfig for: Report " << report->report_name() << " for metric " |
| << metric.metric_name() << " in project " << metric.ProjectDebugString() << "."; |
| result.status = kOther; |
| return result; |
| } |
| auto bucket = histogram_observation->add_buckets(); |
| bucket->set_index(integer_bucket_config->BucketIndex(value)); |
| bucket->set_count(1); |
| |
| return result; |
| } |
| |
| Encoder::Result Encoder::EncodeReportParticipationObservation(MetricRef metric, |
| const ReportDefinition* report, |
| uint32_t day_index) const { |
| auto result = MakeObservation(metric, report, day_index); |
| auto* observation = result.observation.get(); |
| observation->mutable_report_participation(); |
| return result; |
| } |
| |
| Encoder::Result Encoder::EncodeNullBasicRapporObservation(MetricRef metric, |
| const ReportDefinition* report, |
| uint32_t day_index, |
| uint32_t num_categories) const { |
| auto result = MakeObservation(metric, report, day_index); |
| auto* observation = result.observation.get(); |
| auto* basic_rappor_observation = observation->mutable_basic_rappor(); |
| |
| rappor::BasicRapporConfig basic_rappor_config; |
| basic_rappor_config.prob_rr = RapporConfigHelper::kProbRR; |
| basic_rappor_config.categories.set_indexed(num_categories); |
| float prob_bit_flip = RapporConfigHelper::ProbBitFlip(*report, metric.FullyQualifiedName()); |
| basic_rappor_config.prob_0_becomes_1 = prob_bit_flip; |
| basic_rappor_config.prob_1_stays_1 = 1.0f - prob_bit_flip; |
| |
| // TODO(rudominer) Stop copying the client_secret_ on each Encode*() |
| // operation. |
| BasicRapporEncoder basic_rappor_encoder(basic_rappor_config, client_secret_); |
| result.status = TranslateBasicRapporEncoderStatus( |
| metric, report, basic_rappor_encoder.EncodeNullObservation(basic_rappor_observation)); |
| return result; |
| } |
| |
| Encoder::Result Encoder::MakeObservation(MetricRef metric, const ReportDefinition* report, |
| uint32_t day_index) const { |
| Result result; |
| result.status = kOK; |
| result.observation = std::make_unique<Observation2>(); |
| auto* observation = result.observation.get(); |
| result.metadata = std::make_unique<ObservationMetadata>(); |
| auto* metadata = result.metadata.get(); |
| |
| // Generate the random_id field. Currently we use 8 bytes but our |
| // infrastructure allows us to change that in the future if we wish to. The |
| // random_id is used by the Analyzer Service as part of a unique row key |
| // for the observation in the Observation Store. |
| static const size_t kNumRandomBytes = 8; |
| observation->set_allocated_random_id(new std::string(kNumRandomBytes, 0)); |
| random_.RandomString(observation->mutable_random_id()); |
| |
| metadata->set_customer_id(metric.project().customer_id()); |
| metadata->set_project_id(metric.project().project_id()); |
| metadata->set_metric_id(metric.metric_id()); |
| metadata->set_report_id(report->id()); |
| metadata->set_day_index(day_index); |
| |
| if (system_data_) { |
| const auto& profile = system_data_->system_profile(); |
| for (const auto& field : report->system_profile_field()) { |
| switch (field) { |
| case SystemProfileField::OS: |
| metadata->mutable_system_profile()->set_os(profile.os()); |
| break; |
| case SystemProfileField::ARCH: |
| metadata->mutable_system_profile()->set_arch(profile.arch()); |
| break; |
| case SystemProfileField::BOARD_NAME: |
| metadata->mutable_system_profile()->set_board_name(profile.board_name()); |
| break; |
| case SystemProfileField::PRODUCT_NAME: |
| metadata->mutable_system_profile()->set_product_name(profile.product_name()); |
| break; |
| case SystemProfileField::SYSTEM_VERSION: |
| metadata->mutable_system_profile()->set_system_version(profile.system_version()); |
| break; |
| case SystemProfileField::CHANNEL: |
| metadata->mutable_system_profile()->set_channel(profile.channel()); |
| break; |
| } |
| } |
| } |
| |
| return result; |
| } |
| |
| } // namespace cobalt::logger |