| // Copyright 2018 The Fuchsia Authors. All rights reserved. |
| // Use of this source code is governed by a BSD-style license that can be |
| // found in the LICENSE file. |
| |
| #include "src/logger/encoder.h" |
| |
| #include <memory> |
| #include <string> |
| |
| #include "src/algorithms/rappor/rappor_config_helper.h" |
| #include "src/algorithms/rappor/rappor_encoder.h" |
| #include "src/lib/client/cpp/buckets_config.h" |
| #include "src/lib/crypto_util/hash.h" |
| #include "src/lib/util/datetime_util.h" |
| #include "src/lib/util/status_builder.h" |
| #include "src/logger/project_context.h" |
| #include "src/logging.h" |
| #include "src/pb/metadata_builder.h" |
| #include "src/pb/observation.pb.h" |
| #include "src/registry/packed_event_codes.h" |
| #include "src/tracing.h" |
| |
| namespace cobalt::logger { |
| |
| using ::cobalt::config::IntegerBucketConfig; |
| using ::cobalt::rappor::BasicRapporEncoder; |
| using ::cobalt::rappor::RapporConfigHelper; |
| using ::cobalt::system_data::ClientSecret; |
| using ::google::protobuf::RepeatedField; |
| |
| namespace { |
| // Populates |*hash_out| with the SHA256 of |component|, unless |component| |
| // is empty in which case *hash_out is set to the empty string also. An |
| // empty string indicates that the component_name feature is not being used. |
| // We expect this to be a common case and in this case there is no point |
| // in using 32 bytes to represent the empty string. Returns true on success |
| // and false on failure (unexpected). |
| bool HashComponentNameIfNotEmpty(const std::string& component, std::string& hash_out) { |
| if (component.empty()) { |
| hash_out.resize(0); |
| return true; |
| } |
| return cobalt::crypto::hash::Hash(component, hash_out); |
| } |
| |
| Status EncoderError(util::StatusBuilder builder, MetricRef metric, const ReportDefinition* report) { |
| return std::move(builder).WithContexts(*report, metric).LogError().Build(); |
| } |
| |
| } // namespace |
| |
| Encoder::Encoder(ClientSecret client_secret, MetadataBuilder& metadata_builder) |
| : client_secret_(std::move(client_secret)), metadata_builder_(metadata_builder) {} |
| |
| Encoder::Result Encoder::EncodeBasicRapporObservation( |
| MetricRef metric, const ReportDefinition* report, |
| // TODO(fxbug.dev/85571): NOLINTNEXTLINE(bugprone-easily-swappable-parameters) |
| uint32_t day_index, uint32_t value_index, uint32_t num_categories) const { |
| TRACE_DURATION("cobalt_core", "Encoder::EncodeBasicRapporObservation"); |
| |
| Encoder::Result result = NewObservationWithMetadata(metric, report, day_index); |
| Observation* observation = result.observation.get(); |
| BasicRapporObservation* basic_rappor_observation = observation->mutable_basic_rappor(); |
| |
| rappor::BasicRapporConfig basic_rappor_config; |
| basic_rappor_config.prob_rr = RapporConfigHelper::kProbRR; |
| basic_rappor_config.categories.set_indexed(num_categories); |
| float prob_bit_flip = RapporConfigHelper::ProbBitFlip(*report, metric.FullyQualifiedName()); |
| basic_rappor_config.prob_0_becomes_1 = Prob0Becomes1(prob_bit_flip); |
| basic_rappor_config.prob_1_stays_1 = Prob1Stays1(1.0f - prob_bit_flip); |
| |
| // TODO(fxbug.dev/87143): Stop copying the client_secret_ on each Encode*() |
| // operation. |
| BasicRapporEncoder basic_rappor_encoder(basic_rappor_config, client_secret_); |
| ValuePart index_value; |
| index_value.set_index_value(value_index); |
| result.status = EncoderError( |
| util::StatusBuilder(basic_rappor_encoder.Encode(index_value, basic_rappor_observation)), |
| metric, report); |
| return result; |
| } |
| |
| Encoder::Result Encoder::EncodeIntegerEventObservation( |
| MetricRef metric, const ReportDefinition* report, uint32_t day_index, |
| const RepeatedField<uint32_t>& event_codes, const std::string& component, int64_t value) const { |
| Encoder::Result result = NewObservationWithMetadata(metric, report, day_index); |
| Observation* observation = result.observation.get(); |
| IntegerEventObservation* integer_event_observation = observation->mutable_numeric_event(); |
| integer_event_observation->set_event_code(config::PackEventCodes(event_codes)); |
| if (!HashComponentNameIfNotEmpty(component, |
| *integer_event_observation->mutable_component_name_hash())) { |
| result.status = |
| EncoderError(util::StatusBuilder(StatusCode::INTERNAL, "Hashing the component name failed"), |
| metric, report); |
| } |
| integer_event_observation->set_value(value); |
| return result; |
| } |
| |
| Encoder::Result Encoder::EncodeHistogramObservation(MetricRef metric, |
| const ReportDefinition* report, |
| uint32_t day_index, |
| const RepeatedField<uint32_t>& event_codes, |
| const std::string& component, |
| HistogramPtr histogram) const { |
| Encoder::Result result = NewObservationWithMetadata(metric, report, day_index); |
| Observation* observation = result.observation.get(); |
| HistogramObservation* histogram_observation = observation->mutable_histogram(); |
| histogram_observation->set_event_code(config::PackEventCodes(event_codes)); |
| if (!HashComponentNameIfNotEmpty(component, |
| *histogram_observation->mutable_component_name_hash())) { |
| result.status = |
| EncoderError(util::StatusBuilder(StatusCode::INTERNAL, "Hashing the component name failed"), |
| metric, report); |
| } |
| histogram_observation->mutable_buckets()->Swap(histogram.get()); |
| return result; |
| } |
| |
| Encoder::Result Encoder::EncodeCustomObservation(MetricRef metric, const ReportDefinition* report, |
| uint32_t day_index, |
| EventValuesPtr event_values) const { |
| Encoder::Result result = NewObservationWithMetadata(metric, report, day_index); |
| Observation* observation = result.observation.get(); |
| CustomObservation* custom_observation = observation->mutable_custom(); |
| custom_observation->mutable_values()->swap(*event_values); |
| return result; |
| } |
| |
| Encoder::Result Encoder::EncodeSerializedCustomObservation( |
| MetricRef metric, const ReportDefinition* report, uint32_t day_index, |
| std::unique_ptr<std::string> serialized_proto) const { |
| Encoder::Result result = NewObservationWithMetadata(metric, report, day_index); |
| Observation* observation = result.observation.get(); |
| CustomObservation* custom_observation = observation->mutable_custom(); |
| custom_observation->mutable_serialized_proto()->swap(*serialized_proto); |
| return result; |
| } |
| |
| Encoder::Result Encoder::EncodeUniqueActivesObservation( |
| // TODO(fxbug.dev/85571): NOLINTNEXTLINE(bugprone-easily-swappable-parameters) |
| MetricRef metric, const ReportDefinition* report, uint32_t day_index, uint32_t event_code, |
| bool was_active, const OnDeviceAggregationWindow& aggregation_window) const { |
| Encoder::Result result = NewObservationWithMetadata(metric, report, day_index); |
| Encoder::Result basic_rappor_result; |
| if (was_active) { |
| // Encode a single 1 bit |
| basic_rappor_result = EncodeBasicRapporObservation(metric, report, day_index, 0u, 1u); |
| } else { |
| // Encode a single 0 bit |
| basic_rappor_result = EncodeNullBasicRapporObservation(metric, report, day_index, 1u); |
| } |
| if (!basic_rappor_result.status.ok()) { |
| result.status = basic_rappor_result.status; |
| return result; |
| } |
| UniqueActivesObservation* activity_observation = result.observation->mutable_unique_actives(); |
| activity_observation->mutable_aggregation_window()->CopyFrom(aggregation_window); |
| activity_observation->set_event_code(event_code); |
| activity_observation->mutable_basic_rappor_obs()->mutable_data()->swap( |
| *(basic_rappor_result.observation->mutable_basic_rappor()->mutable_data())); |
| |
| return result; |
| } |
| |
| Encoder::Result Encoder::EncodePerDeviceNumericObservation( |
| MetricRef metric, const ReportDefinition* report, uint32_t day_index, |
| const std::string& component, const RepeatedField<uint32_t>& event_codes, int64_t value, |
| const OnDeviceAggregationWindow& aggregation_window) const { |
| Encoder::Result result = |
| EncodeIntegerEventObservation(metric, report, day_index, event_codes, component, value); |
| IntegerEventObservation* integer_event_observation = result.observation->release_numeric_event(); |
| PerDeviceNumericObservation* per_device_observation = |
| result.observation->mutable_per_device_numeric(); |
| per_device_observation->set_allocated_integer_event_obs(integer_event_observation); |
| per_device_observation->mutable_aggregation_window()->CopyFrom(aggregation_window); |
| return result; |
| } |
| |
| Encoder::Result Encoder::EncodePerDeviceHistogramObservation( |
| MetricRef metric, const ReportDefinition* report, uint32_t day_index, |
| const std::string& component, const RepeatedField<uint32_t>& event_codes, int64_t value, |
| const OnDeviceAggregationWindow& aggregation_window) const { |
| Encoder::Result result = NewObservationWithMetadata(metric, report, day_index); |
| PerDeviceHistogramObservation* per_device_histogram_obs = |
| result.observation->mutable_per_device_histogram(); |
| per_device_histogram_obs->mutable_aggregation_window()->CopyFrom(aggregation_window); |
| HistogramObservation* histogram_observation = per_device_histogram_obs->mutable_histogram(); |
| histogram_observation->set_event_code(config::PackEventCodes(event_codes)); |
| if (!HashComponentNameIfNotEmpty(component, |
| *histogram_observation->mutable_component_name_hash())) { |
| result.status = |
| EncoderError(util::StatusBuilder(StatusCode::INTERNAL, "Hashing the component name failed"), |
| metric, report); |
| return result; |
| } |
| |
| std::unique_ptr<IntegerBucketConfig> integer_bucket_config = |
| IntegerBucketConfig::CreateFromProto(report->int_buckets()); |
| if (integer_bucket_config == nullptr) { |
| result.status = |
| EncoderError(util::StatusBuilder(StatusCode::INVALID_ARGUMENT, "Invalid IntBucketConfig"), |
| metric, report); |
| return result; |
| } |
| HistogramBucket* bucket = histogram_observation->add_buckets(); |
| bucket->set_index(integer_bucket_config->BucketIndex(value)); |
| bucket->set_count(1); |
| |
| return result; |
| } |
| |
| Encoder::Result Encoder::EncodeReportParticipationObservation(MetricRef metric, |
| const ReportDefinition* report, |
| uint32_t day_index) const { |
| Encoder::Result result = NewObservationWithMetadata(metric, report, day_index); |
| Observation* observation = result.observation.get(); |
| observation->mutable_report_participation(); |
| return result; |
| } |
| |
| lib::statusor::StatusOr<std::unique_ptr<Observation>> Encoder::EncodeIntegerObservation( |
| const std::vector<std::tuple<std::vector<uint32_t>, int64_t>>& data) { |
| auto observation = std::make_unique<Observation>(); |
| |
| IntegerObservation* integer_observation = observation->mutable_integer(); |
| |
| for (const auto& [event_codes, value] : data) { |
| IntegerObservation_Value* value_proto = integer_observation->add_values(); |
| for (uint32_t code : event_codes) { |
| value_proto->add_event_codes(code); |
| } |
| value_proto->set_value(value); |
| } |
| |
| return observation; |
| } |
| |
| lib::statusor::StatusOr<std::unique_ptr<Observation>> Encoder::EncodeSumAndCountObservation( |
| const std::vector<std::tuple<std::vector<uint32_t>, int64_t, uint32_t>>& data) { |
| auto observation = std::make_unique<Observation>(); |
| |
| SumAndCountObservation* sum_and_count_observation = observation->mutable_sum_and_count(); |
| |
| for (const auto& [event_codes, sum, count] : data) { |
| SumAndCountObservation_SumAndCount* sum_and_count = |
| sum_and_count_observation->add_sums_and_counts(); |
| for (uint32_t code : event_codes) { |
| sum_and_count->add_event_codes(code); |
| } |
| sum_and_count->set_sum(sum); |
| sum_and_count->set_count(count); |
| } |
| |
| return observation; |
| } |
| |
| lib::statusor::StatusOr<std::unique_ptr<Observation>> Encoder::EncodeIndexHistogramObservation( |
| const std::vector< |
| std::tuple<std::vector<uint32_t>, std::vector<std::tuple<uint32_t, int64_t>>>>& data) { |
| auto observation = std::make_unique<Observation>(); |
| |
| IndexHistogramObservation* index_histogram_observation = observation->mutable_index_histogram(); |
| |
| for (const auto& [event_codes, histogram] : data) { |
| IndexHistogram* index_histogram = index_histogram_observation->add_index_histograms(); |
| for (uint32_t code : event_codes) { |
| index_histogram->add_event_codes(code); |
| } |
| for (const auto& [index, count] : histogram) { |
| index_histogram->add_bucket_indices(index); |
| index_histogram->add_bucket_counts(count); |
| } |
| } |
| |
| return observation; |
| } |
| |
| lib::statusor::StatusOr<std::unique_ptr<Observation>> Encoder::EncodeStringHistogramObservation( |
| const std::vector<std::string>& hashes, |
| const std::vector<std::tuple<EventCodes, Histogram>>& data) { |
| auto observation = std::make_unique<Observation>(); |
| |
| StringHistogramObservation* string_histogram_observation = |
| observation->mutable_string_histogram(); |
| |
| for (const std::string& hash : hashes) { |
| string_histogram_observation->add_string_hashes(hash); |
| } |
| |
| for (const auto& [event_codes, histogram] : data) { |
| IndexHistogram* index_histogram = string_histogram_observation->add_string_histograms(); |
| for (uint32_t code : event_codes) { |
| index_histogram->add_event_codes(code); |
| } |
| for (const auto& [index, count] : histogram) { |
| index_histogram->add_bucket_indices(index); |
| index_histogram->add_bucket_counts(count); |
| } |
| } |
| |
| return observation; |
| } |
| |
| Encoder::Result Encoder::EncodeNullBasicRapporObservation( |
| MetricRef metric, const ReportDefinition* report, |
| // TODO(fxbug.dev/85571): NOLINTNEXTLINE(bugprone-easily-swappable-parameters) |
| uint32_t day_index, uint32_t num_categories) const { |
| Encoder::Result result = NewObservationWithMetadata(metric, report, day_index); |
| Observation* observation = result.observation.get(); |
| BasicRapporObservation* basic_rappor_observation = observation->mutable_basic_rappor(); |
| |
| rappor::BasicRapporConfig basic_rappor_config; |
| basic_rappor_config.prob_rr = RapporConfigHelper::kProbRR; |
| basic_rappor_config.categories.set_indexed(num_categories); |
| float prob_bit_flip = RapporConfigHelper::ProbBitFlip(*report, metric.FullyQualifiedName()); |
| basic_rappor_config.prob_0_becomes_1 = Prob0Becomes1(prob_bit_flip); |
| basic_rappor_config.prob_1_stays_1 = Prob1Stays1(1.0f - prob_bit_flip); |
| |
| // TODO(fxbug.dev/87143): Stop copying the client_secret_ on each Encode*() |
| // operation. |
| BasicRapporEncoder basic_rappor_encoder(basic_rappor_config, client_secret_); |
| result.status = EncoderError( |
| util::StatusBuilder(basic_rappor_encoder.EncodeNullObservation(basic_rappor_observation)), |
| metric, report); |
| return result; |
| } |
| |
| Encoder::Result Encoder::NewObservationWithMetadata(MetricRef metric, |
| const ReportDefinition* report, |
| uint32_t day_index) const { |
| Result result; |
| result.status = Status::OK; |
| result.observation = std::make_unique<Observation>(); |
| result.metadata = |
| metadata_builder_.Build(metric, *report, util::TimeInfo::FromDayIndex(day_index)); |
| |
| return result; |
| } |
| |
| } // namespace cobalt::logger |