blob: 33ff9379d02fbe492669ed7166579f4a730dd391 [file] [log] [blame]
// Copyright 2018 The Fuchsia Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#include "src/logger/encoder.h"
#include <memory>
#include <string>
#include "src/algorithms/rappor/rappor_config_helper.h"
#include "src/algorithms/rappor/rappor_encoder.h"
#include "src/lib/client/cpp/buckets_config.h"
#include "src/lib/crypto_util/hash.h"
#include "src/lib/util/datetime_util.h"
#include "src/lib/util/status_builder.h"
#include "src/logger/project_context.h"
#include "src/logging.h"
#include "src/pb/metadata_builder.h"
#include "src/pb/observation.pb.h"
#include "src/registry/packed_event_codes.h"
#include "src/tracing.h"
namespace cobalt::logger {
using ::cobalt::config::IntegerBucketConfig;
using ::cobalt::rappor::BasicRapporEncoder;
using ::cobalt::rappor::RapporConfigHelper;
using ::cobalt::system_data::ClientSecret;
using ::google::protobuf::RepeatedField;
namespace {
// Populates |*hash_out| with the SHA256 of |component|, unless |component|
// is empty in which case *hash_out is set to the empty string also. An
// empty string indicates that the component_name feature is not being used.
// We expect this to be a common case and in this case there is no point
// in using 32 bytes to represent the empty string. Returns true on success
// and false on failure (unexpected).
bool HashComponentNameIfNotEmpty(const std::string& component, std::string& hash_out) {
if (component.empty()) {
hash_out.resize(0);
return true;
}
return cobalt::crypto::hash::Hash(component, hash_out);
}
Status EncoderError(util::StatusBuilder builder, MetricRef metric, const ReportDefinition* report) {
return std::move(builder).WithContexts(*report, metric).LogError().Build();
}
} // namespace
Encoder::Encoder(ClientSecret client_secret, MetadataBuilder& metadata_builder)
: client_secret_(std::move(client_secret)), metadata_builder_(metadata_builder) {}
Encoder::Result Encoder::EncodeBasicRapporObservation(
MetricRef metric, const ReportDefinition* report,
// TODO(fxbug.dev/85571): NOLINTNEXTLINE(bugprone-easily-swappable-parameters)
uint32_t day_index, uint32_t value_index, uint32_t num_categories) const {
TRACE_DURATION("cobalt_core", "Encoder::EncodeBasicRapporObservation");
Encoder::Result result = NewObservationWithMetadata(metric, report, day_index);
Observation* observation = result.observation.get();
BasicRapporObservation* basic_rappor_observation = observation->mutable_basic_rappor();
rappor::BasicRapporConfig basic_rappor_config;
basic_rappor_config.prob_rr = RapporConfigHelper::kProbRR;
basic_rappor_config.categories.set_indexed(num_categories);
float prob_bit_flip = RapporConfigHelper::ProbBitFlip(*report, metric.FullyQualifiedName());
basic_rappor_config.prob_0_becomes_1 = Prob0Becomes1(prob_bit_flip);
basic_rappor_config.prob_1_stays_1 = Prob1Stays1(1.0f - prob_bit_flip);
// TODO(fxbug.dev/87143): Stop copying the client_secret_ on each Encode*()
// operation.
BasicRapporEncoder basic_rappor_encoder(basic_rappor_config, client_secret_);
ValuePart index_value;
index_value.set_index_value(value_index);
result.status = EncoderError(
util::StatusBuilder(basic_rappor_encoder.Encode(index_value, basic_rappor_observation)),
metric, report);
return result;
}
Encoder::Result Encoder::EncodeIntegerEventObservation(
MetricRef metric, const ReportDefinition* report, uint32_t day_index,
const RepeatedField<uint32_t>& event_codes, const std::string& component, int64_t value) const {
Encoder::Result result = NewObservationWithMetadata(metric, report, day_index);
Observation* observation = result.observation.get();
IntegerEventObservation* integer_event_observation = observation->mutable_numeric_event();
integer_event_observation->set_event_code(config::PackEventCodes(event_codes));
if (!HashComponentNameIfNotEmpty(component,
*integer_event_observation->mutable_component_name_hash())) {
result.status =
EncoderError(util::StatusBuilder(StatusCode::INTERNAL, "Hashing the component name failed"),
metric, report);
}
integer_event_observation->set_value(value);
return result;
}
Encoder::Result Encoder::EncodeHistogramObservation(MetricRef metric,
const ReportDefinition* report,
uint32_t day_index,
const RepeatedField<uint32_t>& event_codes,
const std::string& component,
HistogramPtr histogram) const {
Encoder::Result result = NewObservationWithMetadata(metric, report, day_index);
Observation* observation = result.observation.get();
HistogramObservation* histogram_observation = observation->mutable_histogram();
histogram_observation->set_event_code(config::PackEventCodes(event_codes));
if (!HashComponentNameIfNotEmpty(component,
*histogram_observation->mutable_component_name_hash())) {
result.status =
EncoderError(util::StatusBuilder(StatusCode::INTERNAL, "Hashing the component name failed"),
metric, report);
}
histogram_observation->mutable_buckets()->Swap(histogram.get());
return result;
}
Encoder::Result Encoder::EncodeCustomObservation(MetricRef metric, const ReportDefinition* report,
uint32_t day_index,
EventValuesPtr event_values) const {
Encoder::Result result = NewObservationWithMetadata(metric, report, day_index);
Observation* observation = result.observation.get();
CustomObservation* custom_observation = observation->mutable_custom();
custom_observation->mutable_values()->swap(*event_values);
return result;
}
Encoder::Result Encoder::EncodeSerializedCustomObservation(
MetricRef metric, const ReportDefinition* report, uint32_t day_index,
std::unique_ptr<std::string> serialized_proto) const {
Encoder::Result result = NewObservationWithMetadata(metric, report, day_index);
Observation* observation = result.observation.get();
CustomObservation* custom_observation = observation->mutable_custom();
custom_observation->mutable_serialized_proto()->swap(*serialized_proto);
return result;
}
Encoder::Result Encoder::EncodeUniqueActivesObservation(
// TODO(fxbug.dev/85571): NOLINTNEXTLINE(bugprone-easily-swappable-parameters)
MetricRef metric, const ReportDefinition* report, uint32_t day_index, uint32_t event_code,
bool was_active, const OnDeviceAggregationWindow& aggregation_window) const {
Encoder::Result result = NewObservationWithMetadata(metric, report, day_index);
Encoder::Result basic_rappor_result;
if (was_active) {
// Encode a single 1 bit
basic_rappor_result = EncodeBasicRapporObservation(metric, report, day_index, 0u, 1u);
} else {
// Encode a single 0 bit
basic_rappor_result = EncodeNullBasicRapporObservation(metric, report, day_index, 1u);
}
if (!basic_rappor_result.status.ok()) {
result.status = basic_rappor_result.status;
return result;
}
UniqueActivesObservation* activity_observation = result.observation->mutable_unique_actives();
activity_observation->mutable_aggregation_window()->CopyFrom(aggregation_window);
activity_observation->set_event_code(event_code);
activity_observation->mutable_basic_rappor_obs()->mutable_data()->swap(
*(basic_rappor_result.observation->mutable_basic_rappor()->mutable_data()));
return result;
}
Encoder::Result Encoder::EncodePerDeviceNumericObservation(
MetricRef metric, const ReportDefinition* report, uint32_t day_index,
const std::string& component, const RepeatedField<uint32_t>& event_codes, int64_t value,
const OnDeviceAggregationWindow& aggregation_window) const {
Encoder::Result result =
EncodeIntegerEventObservation(metric, report, day_index, event_codes, component, value);
IntegerEventObservation* integer_event_observation = result.observation->release_numeric_event();
PerDeviceNumericObservation* per_device_observation =
result.observation->mutable_per_device_numeric();
per_device_observation->set_allocated_integer_event_obs(integer_event_observation);
per_device_observation->mutable_aggregation_window()->CopyFrom(aggregation_window);
return result;
}
Encoder::Result Encoder::EncodePerDeviceHistogramObservation(
MetricRef metric, const ReportDefinition* report, uint32_t day_index,
const std::string& component, const RepeatedField<uint32_t>& event_codes, int64_t value,
const OnDeviceAggregationWindow& aggregation_window) const {
Encoder::Result result = NewObservationWithMetadata(metric, report, day_index);
PerDeviceHistogramObservation* per_device_histogram_obs =
result.observation->mutable_per_device_histogram();
per_device_histogram_obs->mutable_aggregation_window()->CopyFrom(aggregation_window);
HistogramObservation* histogram_observation = per_device_histogram_obs->mutable_histogram();
histogram_observation->set_event_code(config::PackEventCodes(event_codes));
if (!HashComponentNameIfNotEmpty(component,
*histogram_observation->mutable_component_name_hash())) {
result.status =
EncoderError(util::StatusBuilder(StatusCode::INTERNAL, "Hashing the component name failed"),
metric, report);
return result;
}
std::unique_ptr<IntegerBucketConfig> integer_bucket_config =
IntegerBucketConfig::CreateFromProto(report->int_buckets());
if (integer_bucket_config == nullptr) {
result.status =
EncoderError(util::StatusBuilder(StatusCode::INVALID_ARGUMENT, "Invalid IntBucketConfig"),
metric, report);
return result;
}
HistogramBucket* bucket = histogram_observation->add_buckets();
bucket->set_index(integer_bucket_config->BucketIndex(value));
bucket->set_count(1);
return result;
}
Encoder::Result Encoder::EncodeReportParticipationObservation(MetricRef metric,
const ReportDefinition* report,
uint32_t day_index) const {
Encoder::Result result = NewObservationWithMetadata(metric, report, day_index);
Observation* observation = result.observation.get();
observation->mutable_report_participation();
return result;
}
lib::statusor::StatusOr<std::unique_ptr<Observation>> Encoder::EncodeIntegerObservation(
const std::vector<std::tuple<std::vector<uint32_t>, int64_t>>& data) {
auto observation = std::make_unique<Observation>();
IntegerObservation* integer_observation = observation->mutable_integer();
for (const auto& [event_codes, value] : data) {
IntegerObservation_Value* value_proto = integer_observation->add_values();
for (uint32_t code : event_codes) {
value_proto->add_event_codes(code);
}
value_proto->set_value(value);
}
return observation;
}
lib::statusor::StatusOr<std::unique_ptr<Observation>> Encoder::EncodeSumAndCountObservation(
const std::vector<std::tuple<std::vector<uint32_t>, int64_t, uint32_t>>& data) {
auto observation = std::make_unique<Observation>();
SumAndCountObservation* sum_and_count_observation = observation->mutable_sum_and_count();
for (const auto& [event_codes, sum, count] : data) {
SumAndCountObservation_SumAndCount* sum_and_count =
sum_and_count_observation->add_sums_and_counts();
for (uint32_t code : event_codes) {
sum_and_count->add_event_codes(code);
}
sum_and_count->set_sum(sum);
sum_and_count->set_count(count);
}
return observation;
}
lib::statusor::StatusOr<std::unique_ptr<Observation>> Encoder::EncodeIndexHistogramObservation(
const std::vector<
std::tuple<std::vector<uint32_t>, std::vector<std::tuple<uint32_t, int64_t>>>>& data) {
auto observation = std::make_unique<Observation>();
IndexHistogramObservation* index_histogram_observation = observation->mutable_index_histogram();
for (const auto& [event_codes, histogram] : data) {
IndexHistogram* index_histogram = index_histogram_observation->add_index_histograms();
for (uint32_t code : event_codes) {
index_histogram->add_event_codes(code);
}
for (const auto& [index, count] : histogram) {
index_histogram->add_bucket_indices(index);
index_histogram->add_bucket_counts(count);
}
}
return observation;
}
lib::statusor::StatusOr<std::unique_ptr<Observation>> Encoder::EncodeStringHistogramObservation(
const std::vector<std::string>& hashes,
const std::vector<std::tuple<EventCodes, Histogram>>& data) {
auto observation = std::make_unique<Observation>();
StringHistogramObservation* string_histogram_observation =
observation->mutable_string_histogram();
for (const std::string& hash : hashes) {
string_histogram_observation->add_string_hashes(hash);
}
for (const auto& [event_codes, histogram] : data) {
IndexHistogram* index_histogram = string_histogram_observation->add_string_histograms();
for (uint32_t code : event_codes) {
index_histogram->add_event_codes(code);
}
for (const auto& [index, count] : histogram) {
index_histogram->add_bucket_indices(index);
index_histogram->add_bucket_counts(count);
}
}
return observation;
}
Encoder::Result Encoder::EncodeNullBasicRapporObservation(
MetricRef metric, const ReportDefinition* report,
// TODO(fxbug.dev/85571): NOLINTNEXTLINE(bugprone-easily-swappable-parameters)
uint32_t day_index, uint32_t num_categories) const {
Encoder::Result result = NewObservationWithMetadata(metric, report, day_index);
Observation* observation = result.observation.get();
BasicRapporObservation* basic_rappor_observation = observation->mutable_basic_rappor();
rappor::BasicRapporConfig basic_rappor_config;
basic_rappor_config.prob_rr = RapporConfigHelper::kProbRR;
basic_rappor_config.categories.set_indexed(num_categories);
float prob_bit_flip = RapporConfigHelper::ProbBitFlip(*report, metric.FullyQualifiedName());
basic_rappor_config.prob_0_becomes_1 = Prob0Becomes1(prob_bit_flip);
basic_rappor_config.prob_1_stays_1 = Prob1Stays1(1.0f - prob_bit_flip);
// TODO(fxbug.dev/87143): Stop copying the client_secret_ on each Encode*()
// operation.
BasicRapporEncoder basic_rappor_encoder(basic_rappor_config, client_secret_);
result.status = EncoderError(
util::StatusBuilder(basic_rappor_encoder.EncodeNullObservation(basic_rappor_observation)),
metric, report);
return result;
}
Encoder::Result Encoder::NewObservationWithMetadata(MetricRef metric,
const ReportDefinition* report,
uint32_t day_index) const {
Result result;
result.status = Status::OK;
result.observation = std::make_unique<Observation>();
result.metadata =
metadata_builder_.Build(metric, *report, util::TimeInfo::FromDayIndex(day_index));
return result;
}
} // namespace cobalt::logger