[cobalt][LocalAggregationRedesign] Create an AggregateStore
- Moves non-EventAggregator/EventAggregatorManager functionality outside of the EventAggregator and into an AggregateStore
- The AggregateStore contains the functionality of AggregateStore, MemoryManager, ObservationGenerator
- Does not change the functionality of the methods to reduce the size and complexity of this change. Code style improvements and and new methods will be added in following CLs
- Divides the event_aggregator_tests.cc into two files. All methods that test more than the EventAggregator methods are moved to the aggregate_store_tests.cc
- Removes unused methods in the event_aggregator_tests.cc
- The tests will be reevaluated to match the current class layout in following CLs
- New tests will be added in following CLs
For more information see go/cobalt-local-aggregation-redesign
Bug: 40853
Change-Id: Id6c5639b15cfd1495e52fc5ad6637caaa4c24d3f
diff --git a/src/local_aggregation/BUILD.gn b/src/local_aggregation/BUILD.gn
index a148730..a73a40c 100644
--- a/src/local_aggregation/BUILD.gn
+++ b/src/local_aggregation/BUILD.gn
@@ -71,17 +71,40 @@
]
public_deps = [
+ ":aggregate_store",
+ "$cobalt_root/src:logging",
+ "$cobalt_root/src/lib/util:clock",
+ "$cobalt_root/src/lib/util:datetime_util",
+ "$cobalt_root/src/lib/util:proto_util",
+ "$cobalt_root/src/logger:encoder",
+ "$cobalt_root/src/logger:event_record",
+ "$cobalt_root/src/logger:observation_writer",
+ "$cobalt_root/src/logger:status",
+ "$cobalt_root/src/pb",
+ "$cobalt_root/src/registry:packed_event_codes",
+ ]
+}
+
+source_set("aggregate_store") {
+ sources = [
+ "aggregate_store.cc",
+ "aggregate_store.h",
+ ]
+
+ public_configs = [
+ "$cobalt_root:cobalt_config",
+ "$cobalt_root/src/registry:proto_config",
+ ]
+
+ public_deps = [
":aggregation_utils",
":cobalt_local_aggregation_proto",
- "$cobalt_root/src:logging",
"$cobalt_root/src/algorithms/rappor:rappor_encoder",
- "$cobalt_root/src/lib/util:clock",
"$cobalt_root/src/lib/util:consistent_proto_store",
"$cobalt_root/src/lib/util:datetime_util",
"$cobalt_root/src/lib/util:protected_fields",
"$cobalt_root/src/lib/util:proto_util",
"$cobalt_root/src/logger:encoder",
- "$cobalt_root/src/logger:event_record",
"$cobalt_root/src/logger:observation_writer",
"$cobalt_root/src/logger:status",
"$cobalt_root/src/pb",
@@ -121,6 +144,21 @@
]
}
+source_set("aggregate_store_test") {
+ testonly = true
+ sources = [
+ "aggregate_store_test.cc",
+ ]
+ public_deps = [
+ ":aggregate_store",
+ ":aggregation_utils",
+ ":event_aggregator",
+ "$cobalt_root/src/logger:logger_test_utils",
+ "$cobalt_root/src/logger:testing_constants",
+ "//third_party/googletest:gtest",
+ ]
+}
+
source_set("event_aggregator_mgr_test") {
testonly = true
sources = [
@@ -138,6 +176,7 @@
testonly = true
deps = [
+ ":aggregate_store_test",
":aggregation_utils_test",
":event_aggregator_mgr_test",
":event_aggregator_test",
diff --git a/src/local_aggregation/aggregate_store.cc b/src/local_aggregation/aggregate_store.cc
new file mode 100644
index 0000000..c7eee52
--- /dev/null
+++ b/src/local_aggregation/aggregate_store.cc
@@ -0,0 +1,965 @@
+// Copyright 2018 The Fuchsia Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "src/local_aggregation/aggregate_store.h"
+
+#include <algorithm>
+#include <map>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "src/algorithms/rappor/rappor_config_helper.h"
+#include "src/lib/util/datetime_util.h"
+#include "src/lib/util/proto_util.h"
+#include "src/lib/util/status.h"
+#include "src/local_aggregation/aggregation_utils.h"
+#include "src/registry/packed_event_codes.h"
+
+namespace cobalt::local_aggregation {
+
+using google::protobuf::RepeatedField;
+using logger::Encoder;
+using logger::kInvalidArguments;
+using logger::kOK;
+using logger::kOther;
+using logger::MetricRef;
+using logger::ObservationWriter;
+using logger::ProjectContext;
+using logger::Status;
+using rappor::RapporConfigHelper;
+using util::ConsistentProtoStore;
+using util::SerializeToBase64;
+using util::StatusCode;
+
+namespace {
+
+////// General helper functions.
+
+// Populates a ReportAggregationKey proto message and then populates a string
+// with the base64 encoding of the serialized proto.
+bool PopulateReportKey(uint32_t customer_id, uint32_t project_id, uint32_t metric_id,
+ uint32_t report_id, std::string* key) {
+ ReportAggregationKey key_data;
+ key_data.set_customer_id(customer_id);
+ key_data.set_project_id(project_id);
+ key_data.set_metric_id(metric_id);
+ key_data.set_report_id(report_id);
+ return SerializeToBase64(key_data, key);
+}
+
+////// Helper functions used by the constructor and UpdateAggregationConfigs().
+
+// Gets and validates the window sizes and/or aggregation windows from a ReportDefinition, converts
+// window sizes to daily aggregation windows, sorts the aggregation windows in increasing order, and
+// adds them to an AggregationConfig.
+//
+// TODO(pesk): Stop looking at the window_size field of |report| once all reports have been updated
+// to have OnDeviceAggregationWindows only.
+bool GetSortedAggregationWindowsFromReport(const ReportDefinition& report,
+ AggregationConfig* aggregation_config) {
+ if (report.window_size_size() == 0 && report.aggregation_window_size() == 0) {
+ LOG(ERROR) << "Report must have at least one window size or aggregation window.";
+ return false;
+ }
+ std::vector<uint32_t> aggregation_days;
+ std::vector<uint32_t> aggregation_hours;
+ for (const uint32_t window_size : report.window_size()) {
+ if (window_size == 0 || window_size > kMaxAllowedAggregationDays) {
+ LOG(ERROR) << "Window size must be positive and cannot exceed " << kMaxAllowedAggregationDays;
+ return false;
+ }
+ aggregation_days.push_back(window_size);
+ }
+ for (const auto& window : report.aggregation_window()) {
+ switch (window.units_case()) {
+ case OnDeviceAggregationWindow::kDays: {
+ uint32_t num_days = window.days();
+ if (num_days == 0 || num_days > kMaxAllowedAggregationDays) {
+ LOG(ERROR) << "Daily windows must contain at least 1 and no more than "
+ << kMaxAllowedAggregationDays << " days";
+ return false;
+ }
+ aggregation_days.push_back(num_days);
+ break;
+ }
+ case OnDeviceAggregationWindow::kHours: {
+ uint32_t num_hours = window.hours();
+ if (num_hours == 0 || num_hours > kMaxAllowedAggregationHours) {
+ LOG(ERROR) << "Hourly windows must contain at least 1 and no more than "
+ << kMaxAllowedAggregationHours << " hours";
+ return false;
+ }
+ aggregation_hours.push_back(num_hours);
+ break;
+ }
+ default:
+ LOG(ERROR) << "Invalid OnDeviceAggregationWindow type " << window.units_case();
+ }
+ }
+ std::sort(aggregation_hours.begin(), aggregation_hours.end());
+ std::sort(aggregation_days.begin(), aggregation_days.end());
+ for (auto num_hours : aggregation_hours) {
+ *aggregation_config->add_aggregation_window() = MakeHourWindow(num_hours);
+ }
+ for (auto num_days : aggregation_days) {
+ *aggregation_config->add_aggregation_window() = MakeDayWindow(num_days);
+ }
+ return true;
+}
+
+// Creates an AggregationConfig from a ProjectContext, MetricDefinition, and
+// ReportDefinition and populates the aggregation_config field of a specified
+// ReportAggregates. Also sets the type of the ReportAggregates based on the
+// ReportDefinition's type.
+//
+// Accepts ReportDefinitions with either at least one WindowSize, or at least one
+// OnDeviceAggregationWindow with units in days.
+bool PopulateReportAggregates(const ProjectContext& project_context, const MetricDefinition& metric,
+ const ReportDefinition& report, ReportAggregates* report_aggregates) {
+ if (report.window_size_size() == 0 && report.aggregation_window_size() == 0) {
+ }
+ AggregationConfig* aggregation_config = report_aggregates->mutable_aggregation_config();
+ *aggregation_config->mutable_project() = project_context.project();
+ *aggregation_config->mutable_metric() = *project_context.GetMetric(metric.id());
+ *aggregation_config->mutable_report() = report;
+ if (!GetSortedAggregationWindowsFromReport(report, aggregation_config)) {
+ return false;
+ }
+ switch (report.report_type()) {
+ case ReportDefinition::UNIQUE_N_DAY_ACTIVES: {
+ report_aggregates->set_allocated_unique_actives_aggregates(new UniqueActivesReportAggregates);
+ return true;
+ }
+ case ReportDefinition::PER_DEVICE_NUMERIC_STATS:
+ case ReportDefinition::PER_DEVICE_HISTOGRAM: {
+ report_aggregates->set_allocated_numeric_aggregates(new PerDeviceNumericAggregates);
+ return true;
+ }
+ default:
+ return false;
+ }
+}
+
+// Move all items from the |window_size| field to the |aggregation_window| field
+// of each AggregationConfig, preserving the order of the items. The |aggregation_window| field
+// should be empty if the |window_size| field is nonempty. If for some reason this is not true, log
+// an error and discard the contents of |aggregation_window| and replace them with the migrated
+// |window_size| values.
+void ConvertWindowSizesToAggregationDays(LocalAggregateStore* store) {
+ for (auto [key, aggregates] : store->by_report_key()) {
+ auto config = (*store->mutable_by_report_key())[key].mutable_aggregation_config();
+ if (config->window_size_size() > 0 && config->aggregation_window_size() > 0) {
+ LOG(ERROR) << "Config has both a window_size and an aggregation_window; discarding all "
+ "aggregation_windows";
+ config->clear_aggregation_window();
+ }
+ for (auto window_size : config->window_size()) {
+ *config->add_aggregation_window() = MakeDayWindow(window_size);
+ }
+ config->clear_window_size();
+ }
+}
+
+// Upgrades the LocalAggregateStore from version 0 to |kCurrentLocalAggregateStoreVersion|.
+Status UpgradeLocalAggregateStoreFromVersion0(LocalAggregateStore* store) {
+ ConvertWindowSizesToAggregationDays(store);
+ store->set_version(kCurrentLocalAggregateStoreVersion);
+ return kOK;
+}
+
+} // namespace
+
+AggregateStore::AggregateStore(const Encoder* encoder, const ObservationWriter* observation_writer,
+ ConsistentProtoStore* local_aggregate_proto_store,
+ ConsistentProtoStore* obs_history_proto_store,
+ const size_t backfill_days)
+ : encoder_(encoder),
+ observation_writer_(observation_writer),
+ local_aggregate_proto_store_(local_aggregate_proto_store),
+ obs_history_proto_store_(obs_history_proto_store) {
+ CHECK_LE(backfill_days, kMaxAllowedBackfillDays)
+ << "backfill_days must be less than or equal to " << kMaxAllowedBackfillDays;
+ backfill_days_ = backfill_days;
+ auto locked = protected_aggregate_store_.lock();
+ auto restore_aggregates_status =
+ local_aggregate_proto_store_->Read(&(locked->local_aggregate_store));
+ switch (restore_aggregates_status.error_code()) {
+ case StatusCode::OK: {
+ VLOG(4) << "Read LocalAggregateStore from disk.";
+ break;
+ }
+ case StatusCode::NOT_FOUND: {
+ VLOG(4) << "No file found for local_aggregate_proto_store. Proceeding "
+ "with empty LocalAggregateStore. File will be created on "
+ "first snapshot of the LocalAggregateStore.";
+ locked->local_aggregate_store = MakeNewLocalAggregateStore();
+ break;
+ }
+ default: {
+ LOG(ERROR) << "Read to local_aggregate_proto_store failed with status code: "
+ << restore_aggregates_status.error_code()
+ << "\nError message: " << restore_aggregates_status.error_message()
+ << "\nError details: " << restore_aggregates_status.error_details()
+ << "\nProceeding with empty LocalAggregateStore.";
+ locked->local_aggregate_store = MakeNewLocalAggregateStore();
+ }
+ }
+ if (auto status = MaybeUpgradeLocalAggregateStore(&(locked->local_aggregate_store));
+ status != kOK) {
+ LOG(ERROR) << "Failed to upgrade LocalAggregateStore to current version with status " << status
+ << ".\nProceeding with empty "
+ "LocalAggregateStore.";
+ locked->local_aggregate_store = MakeNewLocalAggregateStore();
+ }
+
+ auto restore_history_status = obs_history_proto_store_->Read(&obs_history_);
+ switch (restore_history_status.error_code()) {
+ case StatusCode::OK: {
+ VLOG(4) << "Read AggregatedObservationHistoryStore from disk.";
+ break;
+ }
+ case StatusCode::NOT_FOUND: {
+ VLOG(4) << "No file found for obs_history_proto_store. Proceeding "
+ "with empty AggregatedObservationHistoryStore. File will be "
+ "created on first snapshot of the AggregatedObservationHistoryStore.";
+ break;
+ }
+ default: {
+ LOG(ERROR) << "Read to obs_history_proto_store failed with status code: "
+ << restore_history_status.error_code()
+ << "\nError message: " << restore_history_status.error_message()
+ << "\nError details: " << restore_history_status.error_details()
+ << "\nProceeding with empty AggregatedObservationHistoryStore.";
+ obs_history_ = MakeNewObservationHistoryStore();
+ }
+ }
+ if (auto status = MaybeUpgradeObservationHistoryStore(&obs_history_); status != kOK) {
+ LOG(ERROR)
+ << "Failed to upgrade AggregatedObservationHistoryStore to current version with status "
+ << status << ".\nProceeding with empty AggregatedObservationHistoryStore.";
+ obs_history_ = MakeNewObservationHistoryStore();
+ }
+}
+
+// Given a ProjectContext, MetricDefinition, and ReportDefinition and a pointer
+// to the LocalAggregateStore, checks whether a key with the same customer,
+// project, metric, and report ID already exists in the LocalAggregateStore. If
+// not, creates and inserts a new key and value. Returns kInvalidArguments if
+// creation of the key or value fails, and kOK otherwise. The caller should hold
+// the mutex protecting the LocalAggregateStore.
+Status AggregateStore::MaybeInsertReportConfigLocked(const ProjectContext& project_context,
+ const MetricDefinition& metric,
+ const ReportDefinition& report,
+ LocalAggregateStore* store) {
+ std::string key;
+ if (!PopulateReportKey(project_context.project().customer_id(),
+ project_context.project().project_id(), metric.id(), report.id(), &key)) {
+ return kInvalidArguments;
+ }
+ ReportAggregates report_aggregates;
+ if (store->by_report_key().count(key) == 0) {
+ if (!PopulateReportAggregates(project_context, metric, report, &report_aggregates)) {
+ return kInvalidArguments;
+ }
+ (*store->mutable_by_report_key())[key] = report_aggregates;
+ }
+ return kOK;
+}
+
+RepeatedField<uint32_t> UnpackEventCodesProto(uint64_t packed_event_codes) {
+ RepeatedField<uint32_t> fields;
+ for (auto code : config::UnpackEventCodes(packed_event_codes)) {
+ *fields.Add() = code;
+ }
+ return fields;
+}
+
+Status AggregateStore::BackUpLocalAggregateStore() {
+ // Lock, copy the LocalAggregateStore, and release the lock. Write the copy
+ // to |local_aggregate_proto_store_|.
+ auto local_aggregate_store = CopyLocalAggregateStore();
+ auto status = local_aggregate_proto_store_->Write(local_aggregate_store);
+ if (!status.ok()) {
+ LOG(ERROR) << "Failed to back up the LocalAggregateStore with error code: "
+ << status.error_code() << "\nError message: " << status.error_message()
+ << "\nError details: " << status.error_details();
+ return kOther;
+ }
+ return kOK;
+}
+
+Status AggregateStore::BackUpObservationHistory() {
+ auto status = obs_history_proto_store_->Write(obs_history_);
+ if (!status.ok()) {
+ LOG(ERROR) << "Failed to back up the AggregatedObservationHistoryStore. "
+ "::cobalt::util::Status error code: "
+ << status.error_code() << "\nError message: " << status.error_message()
+ << "\nError details: " << status.error_details();
+ return kOther;
+ }
+ return kOK;
+}
+
+////////////////////// GarbageCollect and helper functions //////////////////
+
+namespace {
+
+void GarbageCollectUniqueActivesReportAggregates(uint32_t day_index, uint32_t max_aggregation_days,
+ uint32_t backfill_days,
+ UniqueActivesReportAggregates* report_aggregates) {
+ auto map_by_event_code = report_aggregates->mutable_by_event_code();
+ for (auto event_code = map_by_event_code->begin(); event_code != map_by_event_code->end();) {
+ auto map_by_day = event_code->second.mutable_by_day_index();
+ for (auto day = map_by_day->begin(); day != map_by_day->end();) {
+ if (day->first <= day_index - backfill_days - max_aggregation_days) {
+ day = map_by_day->erase(day);
+ } else {
+ ++day;
+ }
+ }
+ if (map_by_day->empty()) {
+ event_code = map_by_event_code->erase(event_code);
+ } else {
+ ++event_code;
+ }
+ }
+}
+
+void GarbageCollectNumericReportAggregates(uint32_t day_index, uint32_t max_aggregation_days,
+ uint32_t backfill_days,
+ PerDeviceNumericAggregates* report_aggregates) {
+ auto map_by_component = report_aggregates->mutable_by_component();
+ for (auto component = map_by_component->begin(); component != map_by_component->end();) {
+ auto map_by_event_code = component->second.mutable_by_event_code();
+ for (auto event_code = map_by_event_code->begin(); event_code != map_by_event_code->end();) {
+ auto map_by_day = event_code->second.mutable_by_day_index();
+ for (auto day = map_by_day->begin(); day != map_by_day->end();) {
+ if (day->first <= day_index - backfill_days - max_aggregation_days) {
+ day = map_by_day->erase(day);
+ } else {
+ ++day;
+ }
+ }
+ if (map_by_day->empty()) {
+ event_code = map_by_event_code->erase(event_code);
+ } else {
+ ++event_code;
+ }
+ }
+ if (map_by_event_code->empty()) {
+ component = map_by_component->erase(component);
+ } else {
+ ++component;
+ }
+ }
+}
+
+} // namespace
+
+Status AggregateStore::GarbageCollect(uint32_t day_index_utc, uint32_t day_index_local) {
+ if (day_index_local == 0u) {
+ day_index_local = day_index_utc;
+ }
+ CHECK_LT(day_index_utc, UINT32_MAX);
+ CHECK_LT(day_index_local, UINT32_MAX);
+ CHECK_GE(day_index_utc, kMaxAllowedAggregationDays + backfill_days_);
+ CHECK_GE(day_index_local, kMaxAllowedAggregationDays + backfill_days_);
+
+ auto locked = protected_aggregate_store_.lock();
+ for (const auto& [report_key, aggregates] : locked->local_aggregate_store.by_report_key()) {
+ uint32_t day_index;
+ const auto& config = aggregates.aggregation_config();
+ switch (config.metric().time_zone_policy()) {
+ case MetricDefinition::UTC: {
+ day_index = day_index_utc;
+ break;
+ }
+ case MetricDefinition::LOCAL: {
+ day_index = day_index_local;
+ break;
+ }
+ default:
+ LOG_FIRST_N(ERROR, 10) << "The TimeZonePolicy of this MetricDefinition is invalid.";
+ continue;
+ }
+ if (aggregates.aggregation_config().aggregation_window_size() == 0) {
+ LOG_FIRST_N(ERROR, 10) << "This ReportDefinition does not have an aggregation window.";
+ continue;
+ }
+ // PopulateReportAggregates ensured that aggregation_window has at least one element, that all
+ // aggregation windows are <= kMaxAllowedAggregationDays, and that config.aggregation_window()
+ // is sorted in increasing order.
+ uint32_t max_aggregation_days = 1u;
+ const OnDeviceAggregationWindow& largest_window =
+ config.aggregation_window(config.aggregation_window_size() - 1);
+ if (largest_window.units_case() == OnDeviceAggregationWindow::kDays) {
+ max_aggregation_days = largest_window.days();
+ }
+ if (max_aggregation_days == 0u || max_aggregation_days > day_index) {
+ LOG_FIRST_N(ERROR, 10) << "The maximum number of aggregation days " << max_aggregation_days
+ << " of this ReportDefinition is out of range.";
+ continue;
+ }
+ // For each ReportAggregates, descend to and iterate over the sub-map of
+ // local aggregates keyed by day index. Keep buckets with day indices
+ // greater than |day_index| - |backfill_days_| - |max_aggregation_days|, and
+ // remove all buckets with smaller day indices.
+ switch (aggregates.type_case()) {
+ case ReportAggregates::kUniqueActivesAggregates: {
+ GarbageCollectUniqueActivesReportAggregates(
+ day_index, max_aggregation_days, backfill_days_,
+ locked->local_aggregate_store.mutable_by_report_key()
+ ->at(report_key)
+ .mutable_unique_actives_aggregates());
+ break;
+ }
+ case ReportAggregates::kNumericAggregates: {
+ GarbageCollectNumericReportAggregates(day_index, max_aggregation_days, backfill_days_,
+ locked->local_aggregate_store.mutable_by_report_key()
+ ->at(report_key)
+ .mutable_numeric_aggregates());
+ break;
+ }
+ default:
+ continue;
+ }
+ }
+ return kOK;
+}
+
+Status AggregateStore::GenerateObservations(uint32_t final_day_index_utc,
+ uint32_t final_day_index_local) {
+ if (final_day_index_local == 0u) {
+ final_day_index_local = final_day_index_utc;
+ }
+ CHECK_LT(final_day_index_utc, UINT32_MAX);
+ CHECK_LT(final_day_index_local, UINT32_MAX);
+ CHECK_GE(final_day_index_utc, kMaxAllowedAggregationDays + backfill_days_);
+ CHECK_GE(final_day_index_local, kMaxAllowedAggregationDays + backfill_days_);
+
+ // Lock, copy the LocalAggregateStore, and release the lock. Use the copy to
+ // generate observations.
+ auto local_aggregate_store = CopyLocalAggregateStore();
+ for (const auto& [report_key, aggregates] : local_aggregate_store.by_report_key()) {
+ const auto& config = aggregates.aggregation_config();
+
+ const auto& metric = config.metric();
+ auto metric_ref = MetricRef(&config.project(), &metric);
+ uint32_t final_day_index;
+ switch (metric.time_zone_policy()) {
+ case MetricDefinition::UTC: {
+ final_day_index = final_day_index_utc;
+ break;
+ }
+ case MetricDefinition::LOCAL: {
+ final_day_index = final_day_index_local;
+ break;
+ }
+ default:
+ LOG_FIRST_N(ERROR, 10) << "The TimeZonePolicy of this MetricDefinition is invalid.";
+ continue;
+ }
+
+ const auto& report = config.report();
+ // PopulateReportAggregates ensured that aggregation_window has at least one element, that all
+ // aggregation windows are <= kMaxAllowedAggregationDays, and that config.aggregation_window()
+ // is sorted in increasing order.
+ if (config.aggregation_window_size() == 0u) {
+ LOG_FIRST_N(ERROR, 10) << "No aggregation_window found for this report.";
+ continue;
+ }
+ uint32_t max_aggregation_days = 1u;
+ const OnDeviceAggregationWindow& largest_window =
+ config.aggregation_window(config.aggregation_window_size() - 1);
+ if (largest_window.units_case() == OnDeviceAggregationWindow::kDays) {
+ max_aggregation_days = largest_window.days();
+ }
+ if (max_aggregation_days == 0u || max_aggregation_days > final_day_index) {
+ LOG_FIRST_N(ERROR, 10) << "The maximum number of aggregation days " << max_aggregation_days
+ << " of this ReportDefinition is out of range.";
+ continue;
+ }
+ switch (metric.metric_type()) {
+ case MetricDefinition::EVENT_OCCURRED: {
+ auto num_event_codes = RapporConfigHelper::BasicRapporNumCategories(metric);
+
+ switch (report.report_type()) {
+ case ReportDefinition::UNIQUE_N_DAY_ACTIVES: {
+ auto status = GenerateUniqueActivesObservations(metric_ref, report_key, aggregates,
+ num_event_codes, final_day_index);
+ if (status != kOK) {
+ return status;
+ }
+ break;
+ }
+ default:
+ continue;
+ }
+ break;
+ }
+ case MetricDefinition::EVENT_COUNT:
+ case MetricDefinition::ELAPSED_TIME:
+ case MetricDefinition::FRAME_RATE:
+ case MetricDefinition::MEMORY_USAGE: {
+ switch (report.report_type()) {
+ case ReportDefinition::PER_DEVICE_NUMERIC_STATS:
+ case ReportDefinition::PER_DEVICE_HISTOGRAM: {
+ auto status = GenerateObsFromNumericAggregates(metric_ref, report_key, aggregates,
+ final_day_index);
+ if (status != kOK) {
+ return status;
+ }
+ break;
+ }
+ default:
+ continue;
+ }
+ break;
+ }
+ default:
+ continue;
+ }
+ }
+ return kOK;
+}
+
+////////// GenerateUniqueActivesObservations and helper methods ////////////////
+
+namespace {
+
+// Given the set of daily aggregates for a fixed event code, and the size and
+// end date of an aggregation window, returns the first day index within that
+// window on which the event code occurred. Returns 0 if the event code did
+// not occur within the window.
+uint32_t FirstActiveDayIndexInWindow(const DailyAggregates& daily_aggregates,
+ uint32_t obs_day_index, uint32_t aggregation_days) {
+ for (uint32_t day_index = obs_day_index - aggregation_days + 1; day_index <= obs_day_index;
+ day_index++) {
+ auto day_aggregate = daily_aggregates.by_day_index().find(day_index);
+ if (day_aggregate != daily_aggregates.by_day_index().end() &&
+ day_aggregate->second.activity_daily_aggregate().activity_indicator()) {
+ return day_index;
+ }
+ }
+ return 0u;
+}
+
+// Given the day index of an event occurrence and the size and end date
+// of an aggregation window, returns true if the occurrence falls within
+// the window and false if not.
+bool IsActivityInWindow(uint32_t active_day_index, uint32_t obs_day_index,
+ uint32_t aggregation_days) {
+ return (active_day_index <= obs_day_index && active_day_index > obs_day_index - aggregation_days);
+}
+
+} // namespace
+
+uint32_t AggregateStore::UniqueActivesLastGeneratedDayIndex(const std::string& report_key,
+ uint32_t event_code,
+ uint32_t aggregation_days) const {
+ auto report_history = obs_history_.by_report_key().find(report_key);
+ if (report_history == obs_history_.by_report_key().end()) {
+ return 0u;
+ }
+ auto event_code_history =
+ report_history->second.unique_actives_history().by_event_code().find(event_code);
+ if (event_code_history == report_history->second.unique_actives_history().by_event_code().end()) {
+ return 0u;
+ }
+ auto window_history = event_code_history->second.by_window_size().find(aggregation_days);
+ if (window_history == event_code_history->second.by_window_size().end()) {
+ return 0u;
+ }
+ return window_history->second;
+}
+
+Status AggregateStore::GenerateSingleUniqueActivesObservation(
+ const MetricRef metric_ref, const ReportDefinition* report, uint32_t obs_day_index,
+ uint32_t event_code, const OnDeviceAggregationWindow& window, bool was_active) const {
+ auto encoder_result = encoder_->EncodeUniqueActivesObservation(metric_ref, report, obs_day_index,
+ event_code, was_active, window);
+ if (encoder_result.status != kOK) {
+ return encoder_result.status;
+ }
+ if (encoder_result.observation == nullptr || encoder_result.metadata == nullptr) {
+ LOG(ERROR) << "Failed to encode UniqueActivesObservation";
+ return kOther;
+ }
+
+ auto writer_status = observation_writer_->WriteObservation(*encoder_result.observation,
+ std::move(encoder_result.metadata));
+ if (writer_status != kOK) {
+ return writer_status;
+ }
+ return kOK;
+}
+
+Status AggregateStore::GenerateUniqueActivesObservations(const MetricRef metric_ref,
+ const std::string& report_key,
+ const ReportAggregates& report_aggregates,
+ uint32_t num_event_codes,
+ uint32_t final_day_index) {
+ CHECK_GT(final_day_index, backfill_days_);
+ // The earliest day index for which we might need to generate an
+ // Observation.
+ auto backfill_period_start = uint32_t(final_day_index - backfill_days_);
+
+ for (uint32_t event_code = 0; event_code < num_event_codes; event_code++) {
+ auto daily_aggregates =
+ report_aggregates.unique_actives_aggregates().by_event_code().find(event_code);
+ // Have any events ever been logged for this report and event code?
+ bool found_event_code =
+ (daily_aggregates != report_aggregates.unique_actives_aggregates().by_event_code().end());
+ for (const auto& window : report_aggregates.aggregation_config().aggregation_window()) {
+ // Skip all hourly windows, and all daily windows which are larger than
+ // kMaxAllowedAggregationDays.
+ //
+ // TODO(pesk): Generate observations for hourly windows.
+ if (window.units_case() != OnDeviceAggregationWindow::kDays) {
+ LOG(INFO) << "Skipping unsupported aggregation window.";
+ continue;
+ }
+ if (window.days() > kMaxAllowedAggregationDays) {
+ LOG(WARNING) << "GenerateUniqueActivesObservations ignoring a window "
+ "size exceeding the maximum allowed value";
+ continue;
+ }
+ // Find the earliest day index for which an Observation has not yet
+ // been generated for this report, event code, and window size. If
+ // that day index is later than |final_day_index|, no Observation is
+ // generated on this invocation.
+ auto last_gen = UniqueActivesLastGeneratedDayIndex(report_key, event_code, window.days());
+ auto first_day_index = std::max(last_gen + 1, backfill_period_start);
+ // The latest day index on which |event_type| is known to have
+ // occurred, so far. This value will be updated as we search
+ // forward from the earliest day index belonging to a window of
+ // interest.
+ uint32_t active_day_index = 0u;
+ // Iterate over the day indices |obs_day_index| for which we need
+ // to generate Observations. On each iteration, generate an
+ // Observation for the |window| ending on |obs_day_index|.
+ for (uint32_t obs_day_index = first_day_index; obs_day_index <= final_day_index;
+ obs_day_index++) {
+ bool was_active = false;
+ if (found_event_code) {
+ // If the current value of |active_day_index| falls within the
+ // window, generate an Observation of activity. If not, search
+ // forward in the window, update |active_day_index|, and generate an
+ // Observation of activity or inactivity depending on the result of
+ // the search.
+ if (IsActivityInWindow(active_day_index, obs_day_index, window.days())) {
+ was_active = true;
+ } else {
+ active_day_index =
+ FirstActiveDayIndexInWindow(daily_aggregates->second, obs_day_index, window.days());
+ was_active = IsActivityInWindow(active_day_index, obs_day_index, window.days());
+ }
+ }
+ auto status = GenerateSingleUniqueActivesObservation(
+ metric_ref, &report_aggregates.aggregation_config().report(), obs_day_index, event_code,
+ window, was_active);
+ if (status != kOK) {
+ return status;
+ }
+ // Update |obs_history_| with the latest date of Observation
+ // generation for this report, event code, and window size.
+ (*(*(*obs_history_.mutable_by_report_key())[report_key]
+ .mutable_unique_actives_history()
+ ->mutable_by_event_code())[event_code]
+ .mutable_by_window_size())[window.days()] = obs_day_index;
+ }
+ }
+ }
+ return kOK;
+}
+
+////////// GenerateObsFromNumericAggregates and helper methods /////////////
+
+uint32_t AggregateStore::PerDeviceNumericLastGeneratedDayIndex(const std::string& report_key,
+ const std::string& component,
+ uint32_t event_code,
+ uint32_t aggregation_days) const {
+ const auto& report_history = obs_history_.by_report_key().find(report_key);
+ if (report_history == obs_history_.by_report_key().end()) {
+ return 0u;
+ }
+ if (!report_history->second.has_per_device_numeric_history()) {
+ return 0u;
+ }
+ const auto& component_history =
+ report_history->second.per_device_numeric_history().by_component().find(component);
+ if (component_history ==
+ report_history->second.per_device_numeric_history().by_component().end()) {
+ return 0u;
+ }
+ const auto& event_code_history = component_history->second.by_event_code().find(event_code);
+ if (event_code_history == component_history->second.by_event_code().end()) {
+ return 0u;
+ }
+ const auto& window_history = event_code_history->second.by_window_size().find(aggregation_days);
+ if (window_history == event_code_history->second.by_window_size().end()) {
+ return 0u;
+ }
+ return window_history->second;
+}
+
+uint32_t AggregateStore::ReportParticipationLastGeneratedDayIndex(
+ const std::string& report_key) const {
+ const auto& report_history = obs_history_.by_report_key().find(report_key);
+ if (report_history == obs_history_.by_report_key().end()) {
+ return 0u;
+ }
+ return report_history->second.report_participation_history().last_generated();
+}
+
+Status AggregateStore::GenerateSinglePerDeviceNumericObservation(
+ const MetricRef metric_ref, const ReportDefinition* report, uint32_t obs_day_index,
+ const std::string& component, uint32_t event_code, const OnDeviceAggregationWindow& window,
+ int64_t value) const {
+ Encoder::Result encoder_result =
+ encoder_->EncodePerDeviceNumericObservation(metric_ref, report, obs_day_index, component,
+ UnpackEventCodesProto(event_code), value, window);
+ if (encoder_result.status != kOK) {
+ return encoder_result.status;
+ }
+ if (encoder_result.observation == nullptr || encoder_result.metadata == nullptr) {
+ LOG(ERROR) << "Failed to encode PerDeviceNumericObservation";
+ return kOther;
+ }
+
+ const auto& writer_status = observation_writer_->WriteObservation(
+ *encoder_result.observation, std::move(encoder_result.metadata));
+ if (writer_status != kOK) {
+ return writer_status;
+ }
+ return kOK;
+}
+
+Status AggregateStore::GenerateSinglePerDeviceHistogramObservation(
+ const MetricRef metric_ref, const ReportDefinition* report, uint32_t obs_day_index,
+ const std::string& component, uint32_t event_code, const OnDeviceAggregationWindow& window,
+ int64_t value) const {
+ Encoder::Result encoder_result = encoder_->EncodePerDeviceHistogramObservation(
+ metric_ref, report, obs_day_index, component, UnpackEventCodesProto(event_code), value,
+ window);
+
+ if (encoder_result.status != kOK) {
+ return encoder_result.status;
+ }
+ if (encoder_result.observation == nullptr || encoder_result.metadata == nullptr) {
+ LOG(ERROR) << "Failed to encode PerDeviceNumericObservation";
+ return kOther;
+ }
+
+ const auto& writer_status = observation_writer_->WriteObservation(
+ *encoder_result.observation, std::move(encoder_result.metadata));
+ if (writer_status != kOK) {
+ return writer_status;
+ }
+ return kOK;
+}
+
+Status AggregateStore::GenerateSingleReportParticipationObservation(const MetricRef metric_ref,
+ const ReportDefinition* report,
+ uint32_t obs_day_index) const {
+ auto encoder_result =
+ encoder_->EncodeReportParticipationObservation(metric_ref, report, obs_day_index);
+ if (encoder_result.status != kOK) {
+ return encoder_result.status;
+ }
+ if (encoder_result.observation == nullptr || encoder_result.metadata == nullptr) {
+ LOG(ERROR) << "Failed to encode ReportParticipationObservation";
+ return kOther;
+ }
+
+ const auto& writer_status = observation_writer_->WriteObservation(
+ *encoder_result.observation, std::move(encoder_result.metadata));
+ if (writer_status != kOK) {
+ return writer_status;
+ }
+ return kOK;
+}
+
+Status AggregateStore::GenerateObsFromNumericAggregates(const MetricRef metric_ref,
+ const std::string& report_key,
+ const ReportAggregates& report_aggregates,
+ uint32_t final_day_index) {
+ CHECK_GT(final_day_index, backfill_days_);
+ // The first day index for which we might have to generate an Observation.
+ auto backfill_period_start = uint32_t(final_day_index - backfill_days_);
+
+ // Generate any necessary PerDeviceNumericObservations for this report.
+ for (const auto& [component, event_code_aggregates] :
+ report_aggregates.numeric_aggregates().by_component()) {
+ for (const auto& [event_code, daily_aggregates] : event_code_aggregates.by_event_code()) {
+ // Populate a helper map keyed by day indices which belong to the range
+ // [|backfill_period_start|, |final_day_index|]. The value at a day
+ // index is the list of windows, in increasing size order, for which an
+ // Observation should be generated for that day index.
+ std::map<uint32_t, std::vector<OnDeviceAggregationWindow>> windows_by_obs_day;
+ for (const auto& window : report_aggregates.aggregation_config().aggregation_window()) {
+ if (window.units_case() != OnDeviceAggregationWindow::kDays) {
+ LOG(INFO) << "Skipping unsupported aggregation window.";
+ continue;
+ }
+ auto last_gen =
+ PerDeviceNumericLastGeneratedDayIndex(report_key, component, event_code, window.days());
+ auto first_day_index = std::max(last_gen + 1, backfill_period_start);
+ for (auto obs_day_index = first_day_index; obs_day_index <= final_day_index;
+ obs_day_index++) {
+ windows_by_obs_day[obs_day_index].push_back(window);
+ }
+ }
+ // Iterate over the day indices |obs_day_index| for which we might need
+ // to generate an Observation. For each day index, generate an
+ // Observation for each needed window.
+ //
+ // More precisely, for each needed window, iterate over the days within that window. If at
+ // least one numeric event was logged during the window, compute the aggregate of the numeric
+ // values over the window and generate a PerDeviceNumericObservation. Whether or not a numeric
+ // event was found, update the AggregatedObservationHistory for this report, component, event
+ // code, and window size with |obs_day_index| as the most recent date of Observation
+ // generation. This reflects the fact that all needed Observations were generated for the
+ // window ending on that date.
+ for (auto obs_day_index = backfill_period_start; obs_day_index <= final_day_index;
+ obs_day_index++) {
+ const auto& windows = windows_by_obs_day.find(obs_day_index);
+ if (windows == windows_by_obs_day.end()) {
+ continue;
+ }
+ bool found_value_for_window = false;
+ int64_t window_aggregate = 0;
+ uint32_t num_days = 0;
+ for (const auto& window : windows->second) {
+ while (num_days < window.days()) {
+ bool found_value_for_day = false;
+ const auto& day_aggregates =
+ daily_aggregates.by_day_index().find(obs_day_index - num_days);
+ if (day_aggregates != daily_aggregates.by_day_index().end()) {
+ found_value_for_day = true;
+ }
+ const auto& aggregation_type =
+ report_aggregates.aggregation_config().report().aggregation_type();
+ switch (aggregation_type) {
+ case ReportDefinition::SUM:
+ if (found_value_for_day) {
+ window_aggregate += day_aggregates->second.numeric_daily_aggregate().value();
+ found_value_for_window = true;
+ }
+ break;
+ case ReportDefinition::MAX:
+ if (found_value_for_day) {
+ window_aggregate = std::max(
+ window_aggregate, day_aggregates->second.numeric_daily_aggregate().value());
+ found_value_for_window = true;
+ }
+ break;
+ case ReportDefinition::MIN:
+ if (found_value_for_day && !found_value_for_window) {
+ window_aggregate = day_aggregates->second.numeric_daily_aggregate().value();
+ found_value_for_window = true;
+ } else if (found_value_for_day) {
+ window_aggregate = std::min(
+ window_aggregate, day_aggregates->second.numeric_daily_aggregate().value());
+ }
+ break;
+ default:
+ LOG(ERROR) << "Unexpected aggregation type " << aggregation_type;
+ return kInvalidArguments;
+ }
+ num_days++;
+ }
+ if (found_value_for_window) {
+ Status status;
+ const ReportDefinition* report = &report_aggregates.aggregation_config().report();
+ switch (report->report_type()) {
+ case ReportDefinition::PER_DEVICE_NUMERIC_STATS: {
+ status = GenerateSinglePerDeviceNumericObservation(
+ metric_ref, report, obs_day_index, component, event_code, window,
+ window_aggregate);
+ if (status != kOK) {
+ return status;
+ }
+ break;
+ }
+ case ReportDefinition::PER_DEVICE_HISTOGRAM: {
+ auto status = GenerateSinglePerDeviceHistogramObservation(
+ metric_ref, report, obs_day_index, component, event_code, window,
+ window_aggregate);
+ if (status != kOK) {
+ return status;
+ }
+ break;
+ }
+ default:
+ LOG(ERROR) << "Unexpected report type " << report->report_type();
+ return kInvalidArguments;
+ }
+ }
+ // Update |obs_history_| with the latest date of Observation
+ // generation for this report, component, event code, and window.
+ (*(*(*(*obs_history_.mutable_by_report_key())[report_key]
+ .mutable_per_device_numeric_history()
+ ->mutable_by_component())[component]
+ .mutable_by_event_code())[event_code]
+ .mutable_by_window_size())[window.days()] = obs_day_index;
+ }
+ }
+ }
+ }
+ // Generate any necessary ReportParticipationObservations for this report.
+ auto participation_last_gen = ReportParticipationLastGeneratedDayIndex(report_key);
+ auto participation_first_day_index = std::max(participation_last_gen + 1, backfill_period_start);
+ for (auto obs_day_index = participation_first_day_index; obs_day_index <= final_day_index;
+ obs_day_index++) {
+ GenerateSingleReportParticipationObservation(
+ metric_ref, &report_aggregates.aggregation_config().report(), obs_day_index);
+ (*obs_history_.mutable_by_report_key())[report_key]
+ .mutable_report_participation_history()
+ ->set_last_generated(obs_day_index);
+ }
+ return kOK;
+}
+
+LocalAggregateStore AggregateStore::MakeNewLocalAggregateStore(uint32_t version) {
+ LocalAggregateStore store;
+ store.set_version(version);
+ return store;
+}
+
+AggregatedObservationHistoryStore AggregateStore::MakeNewObservationHistoryStore(uint32_t version) {
+ AggregatedObservationHistoryStore store;
+ store.set_version(version);
+ return store;
+}
+
+// We can upgrade from v0, but no other versions.
+Status AggregateStore::MaybeUpgradeLocalAggregateStore(LocalAggregateStore* store) {
+ uint32_t version = store->version();
+ if (version == kCurrentLocalAggregateStoreVersion) {
+ return kOK;
+ }
+ VLOG(4) << "Attempting to upgrade LocalAggregateStore from version " << version;
+ switch (version) {
+ case 0u:
+ return UpgradeLocalAggregateStoreFromVersion0(store);
+ default:
+ LOG(ERROR) << "Cannot upgrade LocalAggregateStore from version " << version;
+ return kInvalidArguments;
+ }
+}
+
+// The current version is the earliest version, so no other versions are accepted.
+Status AggregateStore::MaybeUpgradeObservationHistoryStore(
+ AggregatedObservationHistoryStore* store) {
+ uint32_t version = store->version();
+ if (version == kCurrentObservationHistoryStoreVersion) {
+ return kOK;
+ }
+ LOG(ERROR) << "Cannot upgrade AggregatedObservationHistoryStore from version " << version;
+ return kInvalidArguments;
+}
+
+} // namespace cobalt::local_aggregation
diff --git a/src/local_aggregation/aggregate_store.h b/src/local_aggregation/aggregate_store.h
new file mode 100644
index 0000000..67e3fb7
--- /dev/null
+++ b/src/local_aggregation/aggregate_store.h
@@ -0,0 +1,295 @@
+// Copyright 2018 The Fuchsia Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef COBALT_SRC_LOCAL_AGGREGATION_AGGREGATE_STORE_H_
+#define COBALT_SRC_LOCAL_AGGREGATION_AGGREGATE_STORE_H_
+
+#include <condition_variable>
+#include <memory>
+#include <string>
+#include <thread>
+
+#include "src/lib/util/consistent_proto_store.h"
+#include "src/lib/util/protected_fields.h"
+#include "src/local_aggregation/local_aggregation.pb.h"
+#include "src/logger/encoder.h"
+#include "src/logger/observation_writer.h"
+#include "src/logger/project_context.h"
+#include "src/logger/status.h"
+
+namespace cobalt {
+
+// Forward declaration used for friend tests. This will be removed.
+// TODO(ninai): remove this
+namespace logger {
+
+class TestEventAggregator;
+
+} // namespace logger
+
+namespace local_aggregation {
+
+const std::chrono::hours kOneDay(24);
+
+// Maximum value of |backfill_days| allowed by the constructor.
+constexpr size_t kMaxAllowedBackfillDays = 1000;
+// All aggregation windows larger than this number of days are ignored.
+constexpr uint32_t kMaxAllowedAggregationDays = 365;
+// All hourly aggregation windows larger than this number of hours are ignored.
+constexpr uint32_t kMaxAllowedAggregationHours = 23;
+
+// The current version number of the LocalAggregateStore.
+constexpr uint32_t kCurrentLocalAggregateStoreVersion = 1;
+// The current version number of the AggregatedObservationHistoryStore.
+constexpr uint32_t kCurrentObservationHistoryStoreVersion = 0;
+
+// The AggregateStore manages an in-memory store of aggregated Event values, indexed by report,
+// day index, and other dimensions specific to the report type (e.g. event code).
+//
+// When GenerateObservations() is called, this data is used to generate Observations representing
+// aggregates of Event values over a day, week, month, etc.
+//
+// This class also exposes a GarbageCollect*() and Backup*() functionality which deletes
+// unnecessary data and backs up the store respectively.
+class AggregateStore {
+ public:
+ // Constructs an AggregateStore.
+ //
+ // An AggregateStore maintains daily aggregates of Events in a
+ // LocalAggregateStore, uses an Encoder to generate Observations for rolling
+ // windows ending on a specified day index, and writes the Observations to
+ // an ObservationStore using an ObservationWriter.
+ //
+ // encoder: the singleton instance of an Encoder on the system.
+ //
+ // local_aggregate_proto_store: A ConsistentProtoStore to be used by the
+ // AggregateStore to store snapshots of its in-memory store of event
+ // aggregates.
+ //
+ // obs_history_proto_store: A ConsistentProtoStore to be used by the
+ // AggregateStore to store snapshots of its in-memory history of generated
+ // Observations.
+ //
+ // backfill_days: the number of past days for which the AggregateStore
+ // generates and sends Observations, in addition to a requested day index.
+ // See the comment above GenerateObservations for more detail. The constructor CHECK-fails if a
+ // value larger than |kMaxAllowedBackfillDays| is passed.
+ AggregateStore(const logger::Encoder* encoder,
+ const logger::ObservationWriter* observation_writer,
+ util::ConsistentProtoStore* local_aggregate_proto_store,
+ util::ConsistentProtoStore* obs_history_proto_store, size_t backfill_days = 0);
+
+ // Given a ProjectContext, MetricDefinition, and ReportDefinition and a pointer
+ // to the LocalAggregateStore, checks whether a key with the same customer,
+ // project, metric, and report ID already exists in the LocalAggregateStore. If
+ // not, creates and inserts a new key and value. Returns kInvalidArguments if
+ // creation of the key or value fails, and kOK otherwise. The caller should hold
+ // the mutex protecting the LocalAggregateStore.
+ logger::Status MaybeInsertReportConfigLocked(const logger::ProjectContext& project_context,
+ const MetricDefinition& metric,
+ const ReportDefinition& report,
+ LocalAggregateStore* store);
+
+ // Writes a snapshot of the LocalAggregateStore to
+ // |local_aggregate_proto_store_|.
+ logger::Status BackUpLocalAggregateStore();
+
+ // Writes a snapshot of |obs_history_|to |obs_history_proto_store_|.
+ logger::Status BackUpObservationHistory();
+
+ // Removes from the LocalAggregateStore all daily aggregates that are too
+ // old to contribute to their parent report's largest rolling window on the
+ // day which is |backfill_days| before |day_index_utc| (if the parent
+ // MetricDefinitions' time zone policy is UTC) or which is |backfill_days|
+ // before |day_index_local| (if the parent MetricDefinition's time zone policy
+ // is LOCAL). If |day_index_local| is 0, then we set |day_index_local| =
+ // |day_index_utc|.
+ //
+ // If the time zone policy of a report's parent metric is UTC (resp., LOCAL)
+ // and if day_index is the largest value of the |day_index_utc| (resp.,
+ // |day_index_local|) argument with which GarbageCollect() has been called,
+ // then the LocalAggregateStore contains the data needed to generate
+ // Observations for that report for day index (day_index + k) for any k >= 0.
+ logger::Status GarbageCollect(uint32_t day_index_utc, uint32_t day_index_local = 0u);
+
+ // Generates one or more Observations for all of the registered locally
+ // aggregated reports known to this AggregateStore, for rolling windows
+ // ending on specified day indices.
+ //
+ // Each MetricDefinition specifies a time zone policy, which determines the
+ // day index for which an Event associated with that MetricDefinition is
+ // logged. For all MetricDefinitions whose Events are logged with respect to
+ // UTC, this method generates Observations for rolling windows ending on
+ // |final_day_index_utc|. For all MetricDefinitions whose Events are logged
+ // with respect to local time, this method generates Observations for rolling
+ // windows ending on |final_day_index_local|. If |final_day_index_local| is
+ // 0, then we set |final_day_index_local| = |final_day_index_utc|.
+ //
+ // The generated Observations are written to the |observation_writer| passed
+ // to the constructor.
+ //
+ // This class maintains a history of generated Observations and this method
+ // additionally performs backfill: Observations are also generated for
+ // rolling windows ending on any day in the interval [final_day_index -
+ // backfill_days, final_day_index] (where final_day_index is either
+ // final_day_index_utc or final_day_index_local, depending on the time zone
+ // policy of the associated MetricDefinition), if this history indicates they
+ // were not previously generated. Does not generate duplicate Observations
+ // when called multiple times with the same day index.
+ //
+ // Observations are not generated for aggregation windows larger than
+ // |kMaxAllowedAggregationDays|. Hourly windows are not yet supported.
+ logger::Status GenerateObservations(uint32_t final_day_index_utc,
+ uint32_t final_day_index_local = 0u);
+
+ private:
+ friend class EventAggregator; // used for transition during redesign.
+ friend class AggregateStoreTest;
+ friend class EventAggregatorTest;
+ friend class EventAggregatorManagerTest;
+ friend class logger::TestEventAggregator;
+
+ // Make a LocalAggregateStore which is empty except that its version number is set to |version|.
+ LocalAggregateStore MakeNewLocalAggregateStore(
+ uint32_t version = kCurrentLocalAggregateStoreVersion);
+
+ // Make an AggregatedObservationHistoryStore which is empty except that its version number is set
+ // to |version|.
+ AggregatedObservationHistoryStore MakeNewObservationHistoryStore(
+ uint32_t version = kCurrentObservationHistoryStoreVersion);
+
+ // The LocalAggregateStore or AggregatedObservationHistoryStore may need to be changed in ways
+ // which are structurally but not semantically backwards-compatible. In other words, the meaning
+ // to the AggregateStore of a field in the LocalAggregateStore might change. An example is that
+ // we might deprecate one field while introducing a new one.
+ //
+ // The MaybeUpgrade*Store methods allow the AggregateStore to update the contents of its stored
+ // protos from previously meaningful values to currently meaningful values. (For example, a
+ // possible implementation would move the contents of a deprecated field to the replacement
+ // field.)
+ //
+ // These methods are called by the AggregateStore constructor immediately after reading in stored
+ // protos from disk in order to ensure that proto contents have the expected semantics.
+ //
+ // The method first checks the version number of the store. If the version number is equal to
+ // |kCurrentLocalAggregateStoreVersion| or |kCurrentObservationHistoryStoreVersion|
+ // (respectively), returns an OK status. Otherwise, if it is possible to upgrade the store to the
+ // current version, does so and returns an OK status. If not, logs an error and returns
+ // kInvalidArguments. If a non-OK status is returned, the caller should discard the contents of
+ // |store| and replace it with an empty store at the current version. The MakeNew*Store() methods
+ // may be used to create the new store.
+ logger::Status MaybeUpgradeLocalAggregateStore(LocalAggregateStore* store);
+ logger::Status MaybeUpgradeObservationHistoryStore(AggregatedObservationHistoryStore* store);
+
+ // Returns the most recent day index for which an Observation was generated
+ // for a given UNIQUE_N_DAY_ACTIVES report, event code, and day-based aggregation window,
+ // according to |obs_history_|. Returns 0 if no Observation has been generated
+ // for the given arguments.
+ uint32_t UniqueActivesLastGeneratedDayIndex(const std::string& report_key, uint32_t event_code,
+ uint32_t aggregation_days) const;
+
+ // Returns the most recent day index for which an Observation was generated for a given
+ // PER_DEVICE_NUMERIC_STATS or PER_DEVICE_HISTOGRAM report, component, event code, and day-based
+ // aggregation window, according to |obs_history_|. Returns 0 if no Observation has been generated
+ // for the given arguments.
+ uint32_t PerDeviceNumericLastGeneratedDayIndex(const std::string& report_key,
+ const std::string& component, uint32_t event_code,
+ uint32_t aggregation_days) const;
+
+ // Returns the most recent day index for which a
+ // ReportParticipationObservation was generated for a given report, according
+ // to |obs_history_|. Returns 0 if no Observation has been generated for the
+ // given arguments.
+ uint32_t ReportParticipationLastGeneratedDayIndex(const std::string& report_key) const;
+
+ // For a fixed report of type UNIQUE_N_DAY_ACTIVES, generates an Observation
+ // for each event code of the parent metric, for each day-based aggregation window of the
+ // report ending on |final_day_index|, unless an Observation with those parameters was generated
+ // in the past. Also generates Observations for days in the backfill period if needed. Writes the
+ // Observations to an ObservationStore via the ObservationWriter that was passed to the
+ // constructor.
+ //
+ // Observations are not generated for aggregation windows larger than
+ // |kMaxAllowedAggregationDays|. Hourly windows are not yet supported.
+ logger::Status GenerateUniqueActivesObservations(logger::MetricRef metric_ref,
+ const std::string& report_key,
+ const ReportAggregates& report_aggregates,
+ uint32_t num_event_codes,
+ uint32_t final_day_index);
+
+ // Helper method called by GenerateUniqueActivesObservations() to generate
+ // and write a single Observation.
+ logger::Status GenerateSingleUniqueActivesObservation(logger::MetricRef metric_ref,
+ const ReportDefinition* report,
+ uint32_t obs_day_index, uint32_t event_code,
+ const OnDeviceAggregationWindow& window,
+ bool was_active) const;
+
+ // For a fixed report of type PER_DEVICE_NUMERIC_STATS or PER_DEVICE_HISTOGRAM, generates a
+ // PerDeviceNumericObservation and PerDeviceHistogramObservation respectively for each
+ // tuple (component, event code, aggregation_window) for which a numeric event was logged for that
+ // event code and component during the window of that size ending on |final_day_index|, unless an
+ // Observation with those parameters has been generated in the past. The value of the Observation
+ // is the sum, max, or min (depending on the aggregation_type field of the report definition) of
+ // all numeric events logged for that report during the window. Also generates observations for
+ // days in the backfill period if needed.
+ //
+ // In addition to PerDeviceNumericObservations or PerDeviceHistogramObservation , generates
+ // a ReportParticipationObservation for |final_day_index| and any needed days in the backfill
+ // period. These ReportParticipationObservations are used by the report generators to infer the
+ // fleet-wide number of devices for which the sum of numeric events associated to each tuple
+ // (component, event code, window size) was zero.
+ //
+ // Observations are not generated for aggregation windows larger than
+ // |kMaxAllowedAggregationWindowSize|.
+ logger::Status GenerateObsFromNumericAggregates(logger::MetricRef metric_ref,
+ const std::string& report_key,
+ const ReportAggregates& report_aggregates,
+ uint32_t final_day_index);
+
+ // Helper method called by GenerateObsFromNumericAggregates() to generate and write a single
+ // Observation with value |value|. The method will produce a PerDeviceNumericObservation or
+ // PerDeviceHistogramObservation depending on whether the report type is
+ // PER_DEVICE_NUMERIC_STATS or PER_DEVICE_HISTOGRAM respectively.
+ logger::Status GenerateSinglePerDeviceNumericObservation(
+ logger::MetricRef metric_ref, const ReportDefinition* report, uint32_t obs_day_index,
+ const std::string& component, uint32_t event_code, const OnDeviceAggregationWindow& window,
+ int64_t value) const;
+
+ // Helper method called by GenerateObsFromNumericAggregates() to generate and write a single
+ // Observation with value |value|.
+ logger::Status GenerateSinglePerDeviceHistogramObservation(
+ logger::MetricRef metric_ref, const ReportDefinition* report, uint32_t obs_day_index,
+ const std::string& component, uint32_t event_code, const OnDeviceAggregationWindow& window,
+ int64_t value) const;
+
+ // Helper method called by GenerateObsFromNumericAggregates() to generate and write a single
+ // ReportParticipationObservation.
+ logger::Status GenerateSingleReportParticipationObservation(logger::MetricRef metric_ref,
+ const ReportDefinition* report,
+ uint32_t obs_day_index) const;
+
+ LocalAggregateStore CopyLocalAggregateStore() {
+ auto local_aggregate_store = protected_aggregate_store_.lock()->local_aggregate_store;
+ return local_aggregate_store;
+ }
+
+ struct AggregateStoreFields {
+ LocalAggregateStore local_aggregate_store;
+ };
+
+ const logger::Encoder* encoder_;
+ const logger::ObservationWriter* observation_writer_;
+ util::ConsistentProtoStore* local_aggregate_proto_store_;
+ util::ConsistentProtoStore* obs_history_proto_store_;
+ util::ProtectedFields<AggregateStoreFields> protected_aggregate_store_;
+ // Not protected by a mutex. Should only be accessed by the Event Aggregator's |worker_thread_|.
+ AggregatedObservationHistoryStore obs_history_;
+ size_t backfill_days_ = 0;
+};
+
+} // namespace local_aggregation
+} // namespace cobalt
+
+#endif // COBALT_SRC_LOCAL_AGGREGATION_AGGREGATE_STORE_H_
diff --git a/src/local_aggregation/aggregate_store_test.cc b/src/local_aggregation/aggregate_store_test.cc
new file mode 100644
index 0000000..fc3425f
--- /dev/null
+++ b/src/local_aggregation/aggregate_store_test.cc
@@ -0,0 +1,3357 @@
+// Copyright 2019 The Fuchsia Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "src/local_aggregation/aggregate_store.h"
+
+#include <algorithm>
+#include <map>
+#include <memory>
+#include <set>
+#include <utility>
+#include <vector>
+
+#include <google/protobuf/io/coded_stream.h>
+#include <google/protobuf/io/zero_copy_stream_impl_lite.h>
+
+#include "src/lib/util/clock.h"
+#include "src/lib/util/datetime_util.h"
+#include "src/lib/util/proto_util.h"
+#include "src/local_aggregation/aggregation_utils.h"
+#include "src/local_aggregation/event_aggregator.h"
+#include "src/logger/logger_test_utils.h"
+#include "src/logger/testing_constants.h"
+#include "src/pb/event.pb.h"
+#include "src/registry/packed_event_codes.h"
+#include "src/registry/project_configs.h"
+#include "third_party/googletest/googletest/include/gtest/gtest.h"
+
+namespace cobalt {
+
+using config::PackEventCodes;
+using encoder::ClientSecret;
+using encoder::SystemDataInterface;
+using logger::Encoder;
+using logger::EventRecord;
+using logger::kInvalidArguments;
+using logger::kOK;
+using logger::MetricReportId;
+using logger::ObservationWriter;
+using logger::ProjectContext;
+using logger::Status;
+using logger::testing::CheckPerDeviceNumericObservations;
+using logger::testing::CheckUniqueActivesObservations;
+using logger::testing::ExpectedAggregationParams;
+using logger::testing::ExpectedPerDeviceNumericObservations;
+using logger::testing::ExpectedReportParticipationObservations;
+using logger::testing::ExpectedUniqueActivesObservations;
+using logger::testing::FakeObservationStore;
+using logger::testing::FetchAggregatedObservations;
+using logger::testing::GetTestProject;
+using logger::testing::MakeAggregationKey;
+using logger::testing::MakeExpectedReportParticipationObservations;
+using logger::testing::MakeNullExpectedUniqueActivesObservations;
+using logger::testing::MockConsistentProtoStore;
+using logger::testing::TestUpdateRecipient;
+using util::EncryptedMessageMaker;
+using util::IncrementingSteadyClock;
+using util::IncrementingSystemClock;
+using util::SerializeToBase64;
+using util::TimeToDayIndex;
+
+namespace local_aggregation {
+
+namespace {
+// Number of seconds in a day
+constexpr int kDay = 60 * 60 * 24;
+// Number of seconds in an ideal year
+constexpr int kYear = kDay * 365;
+
+template <typename T>
+std::string SerializeAsStringDeterministic(const T& message) {
+ std::string s;
+ {
+ google::protobuf::io::StringOutputStream output(&s);
+ google::protobuf::io::CodedOutputStream out(&output);
+ out.SetSerializationDeterministic(true);
+ message.SerializePartialToCodedStream(&out);
+ }
+ return s;
+}
+
+// Filenames for constructors of ConsistentProtoStores
+constexpr char kAggregateStoreFilename[] = "local_aggregate_store_backup";
+constexpr char kObsHistoryFilename[] = "obs_history_backup";
+
+// A map keyed by base64-encoded, serialized ReportAggregationKeys. The value at
+// a key is a map of event codes to sets of day indices. Used in tests as
+// a record, external to the LocalAggregateStore, of the activity logged for
+// UNIQUE_N_DAY_ACTIVES reports.
+using LoggedActivity = std::map<std::string, std::map<uint32_t, std::set<uint32_t>>>;
+
+// A map used in tests as a record, external to the LocalAggregateStore, of the
+// activity logged for PER_DEVICE_NUMERIC_STATS reports. The keys are, in
+// descending order, serialized ReportAggregationKeys, components, event codes,
+// and day indices. Each day index maps to a vector of numeric values that were
+// logged for that day index..
+using LoggedValues =
+ std::map<std::string,
+ std::map<std::string, std::map<uint32_t, std::map<uint32_t, std::vector<int64_t>>>>>;
+
+} // namespace
+
+// AggregateStoreTest creates an EventAggregator which sends its Observations
+// to a FakeObservationStore. The EventAggregator is not pre-populated with
+// aggregation configurations.
+class AggregateStoreTest : public ::testing::Test {
+ protected:
+ void SetUp() override {
+ observation_store_ = std::make_unique<FakeObservationStore>();
+ update_recipient_ = std::make_unique<TestUpdateRecipient>();
+ observation_encrypter_ = EncryptedMessageMaker::MakeUnencrypted();
+ observation_writer_ = std::make_unique<ObservationWriter>(
+ observation_store_.get(), update_recipient_.get(), observation_encrypter_.get());
+ encoder_ = std::make_unique<Encoder>(ClientSecret::GenerateNewSecret(), system_data_.get());
+ local_aggregate_proto_store_ =
+ std::make_unique<MockConsistentProtoStore>(kAggregateStoreFilename);
+ obs_history_proto_store_ = std::make_unique<MockConsistentProtoStore>(kObsHistoryFilename);
+ ResetEventAggregator();
+ }
+
+ void ResetEventAggregator() {
+ event_aggregator_ = std::make_unique<EventAggregator>(encoder_.get(), observation_writer_.get(),
+ local_aggregate_proto_store_.get(),
+ obs_history_proto_store_.get());
+ // Pass this clock to the EventAggregator::Start method, if it is called.
+ test_clock_ = std::make_unique<IncrementingSystemClock>(std::chrono::system_clock::duration(0));
+ // Initilize it to 10 years after the beginning of time.
+ test_clock_->set_time(std::chrono::system_clock::time_point(std::chrono::seconds(10 * kYear)));
+ // Use this to advance the clock in the tests.
+ unowned_test_clock_ = test_clock_.get();
+ day_store_created_ = CurrentDayIndex();
+ test_steady_clock_ = new IncrementingSteadyClock(std::chrono::system_clock::duration(0));
+ event_aggregator_->SetSteadyClock(test_steady_clock_);
+ }
+
+ // Destruct the EventAggregator (thus calling EventAggregator::ShutDown())
+ // before destructing the objects which the EventAggregator points to but does
+ // not own.
+ void TearDown() override { event_aggregator_.reset(); }
+
+ // Advances |test_clock_| by |num_seconds| seconds.
+ void AdvanceClock(int num_seconds) {
+ unowned_test_clock_->increment_by(std::chrono::seconds(num_seconds));
+ test_steady_clock_->increment_by(std::chrono::seconds(num_seconds));
+ }
+
+ // Returns the day index of the current day according to |test_clock_|, in
+ // |time_zone|, without incrementing the clock.
+ uint32_t CurrentDayIndex(MetricDefinition::TimeZonePolicy time_zone = MetricDefinition::UTC) {
+ return TimeToDayIndex(std::chrono::system_clock::to_time_t(unowned_test_clock_->peek_now()),
+ time_zone);
+ }
+
+ size_t GetBackfillDays() { return event_aggregator_->aggregate_store_->backfill_days_; }
+
+ void SetBackfillDays(size_t num_days) {
+ event_aggregator_->aggregate_store_->backfill_days_ = num_days;
+ }
+
+ Status BackUpLocalAggregateStore() {
+ return event_aggregator_->aggregate_store_->BackUpLocalAggregateStore();
+ }
+
+ Status BackUpObservationHistory() {
+ return event_aggregator_->aggregate_store_->BackUpObservationHistory();
+ }
+
+ LocalAggregateStore MakeNewLocalAggregateStore(
+ uint32_t version = kCurrentLocalAggregateStoreVersion) {
+ return event_aggregator_->aggregate_store_->MakeNewLocalAggregateStore(version);
+ }
+
+ AggregatedObservationHistoryStore MakeNewObservationHistoryStore(
+ uint32_t version = kCurrentObservationHistoryStoreVersion) {
+ return event_aggregator_->aggregate_store_->MakeNewObservationHistoryStore(version);
+ }
+
+ Status MaybeUpgradeLocalAggregateStore(LocalAggregateStore* store) {
+ return event_aggregator_->aggregate_store_->MaybeUpgradeLocalAggregateStore(store);
+ }
+
+ Status MaybeUpgradeObservationHistoryStore(AggregatedObservationHistoryStore* store) {
+ return event_aggregator_->aggregate_store_->MaybeUpgradeObservationHistoryStore(store);
+ }
+
+ LocalAggregateStore CopyLocalAggregateStore() {
+ return event_aggregator_->aggregate_store_->CopyLocalAggregateStore();
+ }
+
+ Status GenerateObservations(uint32_t final_day_index_utc, uint32_t final_day_index_local = 0u) {
+ return event_aggregator_->GenerateObservationsNoWorker(final_day_index_utc,
+ final_day_index_local);
+ }
+
+ Status GarbageCollect(uint32_t day_index_utc, uint32_t day_index_local = 0u) {
+ return event_aggregator_->aggregate_store_->GarbageCollect(day_index_utc, day_index_local);
+ }
+
+ void DoScheduledTasksNow() {
+ // Steady values don't matter, just tell DoScheduledTasks to run everything.
+ auto steady_time = std::chrono::steady_clock::now();
+ event_aggregator_->next_generate_obs_ = steady_time;
+ event_aggregator_->next_gc_ = steady_time;
+ event_aggregator_->DoScheduledTasks(unowned_test_clock_->now(), steady_time);
+ }
+
+ // Clears the FakeObservationStore and resets the counts of Observations
+ // received by the FakeObservationStore and the TestUpdateRecipient.
+ void ResetObservationStore() {
+ observation_store_->messages_received.clear();
+ observation_store_->metadata_received.clear();
+ observation_store_->ResetObservationCounter();
+ update_recipient_->invocation_count = 0;
+ }
+
+ // Given a ProjectContext |project_context| and the MetricReportId of a
+ // UNIQUE_N_DAY_ACTIVES report in |project_context|, as well as a day index
+ // and an event code, logs an OccurrenceEvent to the EventAggregator for
+ // that report, day index, and event code. If a non-null LoggedActivity map is
+ // provided, updates the map with information about the logged Event.
+ Status LogUniqueActivesEvent(std::shared_ptr<const ProjectContext> project_context,
+ const MetricReportId& metric_report_id, uint32_t day_index,
+ uint32_t event_code, LoggedActivity* logged_activity = nullptr) {
+ EventRecord event_record(std::move(project_context), metric_report_id.first);
+ event_record.event()->set_day_index(day_index);
+ event_record.event()->mutable_occurrence_event()->set_event_code(event_code);
+ auto status = event_aggregator_->LogUniqueActivesEvent(metric_report_id.second, event_record);
+ if (logged_activity == nullptr) {
+ return status;
+ }
+ std::string key;
+ if (!SerializeToBase64(MakeAggregationKey(*event_record.project_context(), metric_report_id),
+ &key)) {
+ return kInvalidArguments;
+ }
+ (*logged_activity)[key][event_code].insert(day_index);
+ return status;
+ }
+
+ // Given a ProjectContext |project_context| and the MetricReportId of an
+ // EVENT_COUNT metric with a PER_DEVICE_NUMERIC_STATS report in
+ // |project_context|, as well as a day index, a component string, and an event
+ // code, logs a CountEvent to the EventAggregator for that report, day
+ // index, component, and event code. If a non-null LoggedValues map is
+ // provided, updates the map with information about the logged Event.
+ Status LogPerDeviceCountEvent(std::shared_ptr<const ProjectContext> project_context,
+ const MetricReportId& metric_report_id, uint32_t day_index,
+ const std::string& component, uint32_t event_code, int64_t count,
+ LoggedValues* logged_values = nullptr) {
+ EventRecord event_record(std::move(project_context), metric_report_id.first);
+ event_record.event()->set_day_index(day_index);
+ auto count_event = event_record.event()->mutable_count_event();
+ count_event->set_component(component);
+ count_event->add_event_code(event_code);
+ count_event->set_count(count);
+ auto status = event_aggregator_->LogCountEvent(metric_report_id.second, event_record);
+ if (logged_values == nullptr) {
+ return status;
+ }
+ std::string key;
+ if (!SerializeToBase64(MakeAggregationKey(*event_record.project_context(), metric_report_id),
+ &key)) {
+ return kInvalidArguments;
+ }
+ (*logged_values)[key][component][event_code][day_index].push_back(count);
+ return status;
+ }
+
+ // Given a ProjectContext |project_context| and the MetricReportId of an
+ // ELAPSED_TIME metric with a PER_DEVICE_NUMERIC_STATS report in
+ // |project_context|, as well as a day index, a component string, and an event
+ // code, logs an ElapsedTimeEvent to the EventAggregator for that report, day
+ // index, component, and event code. If a non-null LoggedValues map is
+ // provided, updates the map with information about the logged Event.
+ Status LogPerDeviceElapsedTimeEvent(std::shared_ptr<const ProjectContext> project_context,
+ const MetricReportId& metric_report_id, uint32_t day_index,
+ const std::string& component, uint32_t event_code,
+ int64_t micros, LoggedValues* logged_values = nullptr) {
+ EventRecord event_record(std::move(project_context), metric_report_id.first);
+ event_record.event()->set_day_index(day_index);
+ auto elapsed_time_event = event_record.event()->mutable_elapsed_time_event();
+ elapsed_time_event->set_component(component);
+ elapsed_time_event->add_event_code(event_code);
+ elapsed_time_event->set_elapsed_micros(micros);
+ auto status = event_aggregator_->LogElapsedTimeEvent(metric_report_id.second, event_record);
+ if (logged_values == nullptr) {
+ return status;
+ }
+ std::string key;
+ if (!SerializeToBase64(MakeAggregationKey(*event_record.project_context(), metric_report_id),
+ &key)) {
+ return kInvalidArguments;
+ }
+ (*logged_values)[key][component][event_code][day_index].push_back(micros);
+ return status;
+ }
+
+ // Given a ProjectContext |project_context| and the MetricReportId of a
+ // FRAME_RATE metric with a PER_DEVICE_NUMERIC_STATS report in
+ // |project_context|, as well as a day index, a component string, and an event
+ // code, logs a FrameRateEvent to the EventAggregator for that report, day
+ // index, component, and event code. If a non-null LoggedValues map is
+ // provided, updates the map with information about the logged Event.
+ Status LogPerDeviceFrameRateEvent(std::shared_ptr<const ProjectContext> project_context,
+ const MetricReportId& metric_report_id, uint32_t day_index,
+ const std::string& component, uint32_t event_code, float fps,
+ LoggedValues* logged_values = nullptr) {
+ EventRecord event_record(std::move(project_context), metric_report_id.first);
+ event_record.event()->set_day_index(day_index);
+ auto frame_rate_event = event_record.event()->mutable_frame_rate_event();
+ frame_rate_event->set_component(component);
+ frame_rate_event->add_event_code(event_code);
+ int64_t frames_per_1000_seconds = std::round(fps * 1000.0);
+ frame_rate_event->set_frames_per_1000_seconds(frames_per_1000_seconds);
+ auto status = event_aggregator_->LogFrameRateEvent(metric_report_id.second, event_record);
+ if (logged_values == nullptr) {
+ return status;
+ }
+ std::string key;
+ if (!SerializeToBase64(MakeAggregationKey(*event_record.project_context(), metric_report_id),
+ &key)) {
+ return kInvalidArguments;
+ }
+ (*logged_values)[key][component][event_code][day_index].push_back(frames_per_1000_seconds);
+ return status;
+ }
+
+ // Given a ProjectContext |project_context| and the MetricReportId of a
+ // MEMORY_USAGE metric with a PER_DEVICE_NUMERIC_STATS report in
+ // |project_context|, as well as a day index, a component string, and an event
+ // code, logs a MemoryUsageEvent to the EventAggregator for that report, day
+ // index, component, and event code. If a non-null LoggedValues map is
+ // provided, updates the map with information about the logged Event.
+ Status LogPerDeviceMemoryUsageEvent(std::shared_ptr<const ProjectContext> project_context,
+ const MetricReportId& metric_report_id, uint32_t day_index,
+ const std::string& component,
+ const std::vector<uint32_t>& event_codes, int64_t bytes,
+ LoggedValues* logged_values = nullptr) {
+ EventRecord event_record(std::move(project_context), metric_report_id.first);
+ event_record.event()->set_day_index(day_index);
+ auto memory_usage_event = event_record.event()->mutable_memory_usage_event();
+ memory_usage_event->set_component(component);
+ for (auto event_code : event_codes) {
+ memory_usage_event->add_event_code(event_code);
+ }
+ memory_usage_event->set_bytes(bytes);
+ auto status = event_aggregator_->LogMemoryUsageEvent(metric_report_id.second, event_record);
+ if (logged_values == nullptr) {
+ return status;
+ }
+ std::string key;
+ if (!SerializeToBase64(MakeAggregationKey(*event_record.project_context(), metric_report_id),
+ &key)) {
+ return kInvalidArguments;
+ }
+ (*logged_values)[key][component][PackEventCodes(event_codes)][day_index].push_back(bytes);
+ return status;
+ }
+
+ // Given a LoggedActivity map describing the events that have been logged
+ // to the EventAggregator, checks whether the contents of the
+ // LocalAggregateStore are as expected, accounting for any garbage
+ // collection.
+ //
+ // logged_activity: a LoggedActivity representing event occurrences
+ // since the LocalAggregateStore was created. All day indices should be
+ // greater than or equal to |day_store_created_| and less than or equal to
+ // |current_day_index|.
+ //
+ // current_day_index: The day index of the current day in the test's frame
+ // of reference.
+ bool CheckUniqueActivesAggregates(const LoggedActivity& logged_activity,
+ uint32_t /*current_day_index*/) {
+ auto local_aggregate_store = event_aggregator_->aggregate_store_->CopyLocalAggregateStore();
+ // Check that the LocalAggregateStore contains no more UniqueActives
+ // aggregates than |logged_activity| and |day_last_garbage_collected_|
+ // should imply.
+ for (const auto& report_pair : local_aggregate_store.by_report_key()) {
+ const auto& aggregates = report_pair.second;
+ if (aggregates.type_case() != ReportAggregates::kUniqueActivesAggregates) {
+ continue;
+ }
+ const auto& report_key = report_pair.first;
+ // Check whether this ReportAggregationKey is in |logged_activity|. If
+ // not, expect that its by_event_code map is empty.
+ auto report_activity = logged_activity.find(report_key);
+ if (report_activity == logged_activity.end()) {
+ EXPECT_TRUE(aggregates.unique_actives_aggregates().by_event_code().empty());
+ if (!aggregates.unique_actives_aggregates().by_event_code().empty()) {
+ return false;
+ }
+ break;
+ }
+ auto expected_events = report_activity->second;
+ for (const auto& event_pair : aggregates.unique_actives_aggregates().by_event_code()) {
+ // Check that this event code is in |logged_activity| under this
+ // ReportAggregationKey.
+ auto event_code = event_pair.first;
+ auto event_activity = expected_events.find(event_code);
+ EXPECT_NE(event_activity, expected_events.end());
+ if (event_activity == expected_events.end()) {
+ return false;
+ }
+ const auto& expected_days = event_activity->second;
+ for (const auto& day_pair : event_pair.second.by_day_index()) {
+ // Check that this day index is in |logged_activity| under this
+ // ReportAggregationKey and event code.
+ const auto& day_index = day_pair.first;
+ auto day_activity = expected_days.find(day_index);
+ EXPECT_NE(day_activity, expected_days.end());
+ if (day_activity == expected_days.end()) {
+ return false;
+ }
+ // Check that the day index is no earlier than is implied by the
+ // dates of store creation and garbage collection.
+ EXPECT_GE(day_index, EarliestAllowedDayIndex(aggregates.aggregation_config()));
+ if (day_index < EarliestAllowedDayIndex(aggregates.aggregation_config())) {
+ return false;
+ }
+ }
+ }
+ }
+
+ // Check that the LocalAggregateStore contains aggregates for all events in
+ // |logged_activity|, as long as they are recent enough to have survived any
+ // garbage collection.
+ for (const auto& logged_pair : logged_activity) {
+ const auto& logged_key = logged_pair.first;
+ const auto& logged_event_map = logged_pair.second;
+ // Check that this ReportAggregationKey is in the LocalAggregateStore, and
+ // that the aggregates are of the expected type.
+ auto report_aggregates = local_aggregate_store.by_report_key().find(logged_key);
+ EXPECT_NE(report_aggregates, local_aggregate_store.by_report_key().end());
+ if (report_aggregates == local_aggregate_store.by_report_key().end()) {
+ return false;
+ }
+ if (report_aggregates->second.type_case() != ReportAggregates::kUniqueActivesAggregates) {
+ return false;
+ }
+ // Compute the earliest day index that should appear among the aggregates
+ // for this report.
+ auto earliest_allowed =
+ EarliestAllowedDayIndex(report_aggregates->second.aggregation_config());
+ for (const auto& logged_event_pair : logged_event_map) {
+ const auto& logged_event_code = logged_event_pair.first;
+ const auto& logged_days = logged_event_pair.second;
+ // Check whether this event code is in the LocalAggregateStore
+ // under this ReportAggregationKey. If not, check that all day indices
+ // for this event code are smaller than the day index of the earliest
+ // allowed aggregate.
+ auto event_code_aggregates =
+ report_aggregates->second.unique_actives_aggregates().by_event_code().find(
+ logged_event_code);
+ if (event_code_aggregates ==
+ report_aggregates->second.unique_actives_aggregates().by_event_code().end()) {
+ for (auto day_index : logged_days) {
+ EXPECT_LT(day_index, earliest_allowed);
+ if (day_index >= earliest_allowed) {
+ return false;
+ }
+ }
+ break;
+ }
+ // Check that all of the day indices in |logged_activity| under this
+ // ReportAggregationKey and event code are in the
+ // LocalAggregateStore, as long as they are recent enough to have
+ // survived any garbage collection. Check that each aggregate has its
+ // activity field set to true.
+ for (const auto& logged_day_index : logged_days) {
+ auto day_aggregate = event_code_aggregates->second.by_day_index().find(logged_day_index);
+ if (logged_day_index >= earliest_allowed) {
+ EXPECT_NE(day_aggregate, event_code_aggregates->second.by_day_index().end());
+ if (day_aggregate == event_code_aggregates->second.by_day_index().end()) {
+ return false;
+ }
+ EXPECT_TRUE(day_aggregate->second.activity_daily_aggregate().activity_indicator());
+ if (!day_aggregate->second.activity_daily_aggregate().activity_indicator()) {
+ return false;
+ }
+ }
+ }
+ }
+ }
+ return true;
+ }
+
+ bool CheckPerDeviceNumericAggregates(const LoggedValues& logged_values,
+ uint32_t /*current_day_index*/) {
+ auto local_aggregate_store = event_aggregator_->aggregate_store_->CopyLocalAggregateStore();
+ // Check that the LocalAggregateStore contains no more PerDeviceNumeric
+ // aggregates than |logged_values| and |day_last_garbage_collected_| should
+ // imply.
+ for (const auto& report_pair : local_aggregate_store.by_report_key()) {
+ const auto& aggregates = report_pair.second;
+ if (aggregates.type_case() != ReportAggregates::kNumericAggregates) {
+ continue;
+ }
+ const auto& report_key = report_pair.first;
+ // Check whether this ReportAggregationKey is in |logged_values|. If not,
+ // expect that its by_component map is empty.
+ auto report_values = logged_values.find(report_key);
+ if (report_values == logged_values.end()) {
+ EXPECT_TRUE(aggregates.numeric_aggregates().by_component().empty());
+ if (!aggregates.numeric_aggregates().by_component().empty()) {
+ return false;
+ }
+ break;
+ }
+ auto expected_components = report_values->second;
+ for (const auto& component_pair : aggregates.numeric_aggregates().by_component()) {
+ // Check that this component is in |logged_values| under this
+ // ReportAggregationKey.
+ auto component = component_pair.first;
+ auto component_values = expected_components.find(component);
+ EXPECT_NE(component_values, expected_components.end());
+ if (component_values == expected_components.end()) {
+ return false;
+ }
+ const auto& expected_events = component_values->second;
+ for (const auto& event_pair : component_pair.second.by_event_code()) {
+ // Check that this event code is in |logged_values| under this
+ // ReportAggregationKey and component.
+ const auto& event_code = event_pair.first;
+ auto event_values = expected_events.find(event_code);
+ EXPECT_NE(event_values, expected_events.end());
+ if (event_values == expected_events.end()) {
+ return false;
+ }
+ const auto& expected_days = event_values->second;
+ for (const auto& day_pair : event_pair.second.by_day_index()) {
+ // Check that this day index is in |logged_values| under this
+ // ReportAggregationKey, component, and event code.
+ const auto& day_index = day_pair.first;
+ auto day_value = expected_days.find(day_index);
+ EXPECT_NE(day_value, expected_days.end());
+ if (day_value == expected_days.end()) {
+ return false;
+ }
+ // Check that the day index is no earlier than is implied by the
+ // dates of store creation and garbage collection.
+ EXPECT_GE(day_index, EarliestAllowedDayIndex(aggregates.aggregation_config()));
+ if (day_index < EarliestAllowedDayIndex(aggregates.aggregation_config())) {
+ return false;
+ }
+ }
+ }
+ }
+ }
+
+ // Check that the LocalAggregateStore contains aggregates for all values in
+ // |logged_values|, as long as they are recent enough to have survived any
+ // garbage collection.
+ for (const auto& logged_pair : logged_values) {
+ const auto& logged_key = logged_pair.first;
+ const auto& logged_component_map = logged_pair.second;
+ // Check that this ReportAggregationKey is in the LocalAggregateStore, and
+ // that the aggregates are of the expected type.
+ auto report_aggregates = local_aggregate_store.by_report_key().find(logged_key);
+ EXPECT_NE(report_aggregates, local_aggregate_store.by_report_key().end());
+ if (report_aggregates == local_aggregate_store.by_report_key().end()) {
+ return false;
+ }
+ if (report_aggregates->second.type_case() != ReportAggregates::kNumericAggregates) {
+ return false;
+ }
+ const auto& aggregation_type =
+ report_aggregates->second.aggregation_config().report().aggregation_type();
+ // Compute the earliest day index that should appear among the aggregates
+ // for this report.
+ auto earliest_allowed =
+ EarliestAllowedDayIndex(report_aggregates->second.aggregation_config());
+ for (const auto& logged_component_pair : logged_component_map) {
+ const auto& logged_component = logged_component_pair.first;
+ const auto& logged_event_code_map = logged_component_pair.second;
+ // Check whether this component is in the LocalAggregateStore under this
+ // ReportAggregationKey. If not, check that all day indices for all
+ // entries in |logged_values| under this component are smaller than the
+ // day index of the earliest allowed aggregate.
+ bool component_found = false;
+ auto component_aggregates =
+ report_aggregates->second.numeric_aggregates().by_component().find(logged_component);
+ if (component_aggregates !=
+ report_aggregates->second.numeric_aggregates().by_component().end()) {
+ component_found = true;
+ }
+ for (const auto& logged_event_pair : logged_event_code_map) {
+ const auto& logged_event_code = logged_event_pair.first;
+ const auto& logged_day_map = logged_event_pair.second;
+ // Check whether this event code is in the LocalAggregateStore under
+ // this ReportAggregationKey. If not, check that all day indices in
+ // |logged_values| under this component are smaller than the day index
+ // of the earliest allowed aggregate.
+ bool event_code_found = false;
+ if (component_found) {
+ auto event_code_aggregates =
+ component_aggregates->second.by_event_code().find(logged_event_code);
+ if (event_code_aggregates != component_aggregates->second.by_event_code().end()) {
+ event_code_found = true;
+ }
+ if (event_code_found) {
+ // Check that all of the day indices in |logged_values| under this
+ // ReportAggregationKey, component, and event code are in the
+ // LocalAggregateStore, as long as they are recent enough to have
+ // survived any garbage collection. Check that each aggregate has
+ // the expected value.
+ for (const auto& logged_day_pair : logged_day_map) {
+ auto logged_day_index = logged_day_pair.first;
+ auto logged_values = logged_day_pair.second;
+ auto day_aggregate =
+ event_code_aggregates->second.by_day_index().find(logged_day_index);
+ if (logged_day_index >= earliest_allowed) {
+ EXPECT_NE(day_aggregate, event_code_aggregates->second.by_day_index().end());
+ if (day_aggregate == event_code_aggregates->second.by_day_index().end()) {
+ return false;
+ }
+ int64_t aggregate_from_logged_values = 0;
+ for (size_t index = 0; index < logged_values.size(); index++) {
+ switch (aggregation_type) {
+ case ReportDefinition::SUM:
+ aggregate_from_logged_values += logged_values[index];
+ break;
+ case ReportDefinition::MAX:
+ aggregate_from_logged_values =
+ std::max(aggregate_from_logged_values, logged_values[index]);
+ break;
+ case ReportDefinition::MIN:
+ if (index == 0) {
+ aggregate_from_logged_values = logged_values[0];
+ }
+ aggregate_from_logged_values =
+ std::min(aggregate_from_logged_values, logged_values[index]);
+ break;
+ default:
+ return false;
+ }
+ }
+ EXPECT_EQ(day_aggregate->second.numeric_daily_aggregate().value(),
+ aggregate_from_logged_values);
+ if (day_aggregate->second.numeric_daily_aggregate().value() !=
+ aggregate_from_logged_values) {
+ return false;
+ }
+ }
+ }
+ }
+ }
+ if (!component_found | !event_code_found) {
+ for (const auto& logged_day_pair : logged_day_map) {
+ auto logged_day_index = logged_day_pair.first;
+ EXPECT_LT(logged_day_index, earliest_allowed);
+ if (logged_day_index >= earliest_allowed) {
+ return false;
+ }
+ }
+ break;
+ }
+ }
+ }
+ }
+
+ return true;
+ }
+
+ // Given the AggregationConfig of a locally aggregated report, returns the
+ // earliest (smallest) day index for which an aggregate may exist in the
+ // LocalAggregateStore for that report, accounting for garbage
+ // collection and the number of backfill days.
+ uint32_t EarliestAllowedDayIndex(const AggregationConfig& config) {
+ // If the LocalAggregateStore has never been garbage-collected, then the
+ // earliest allowed day index is just the day when the store was created,
+ // minus the number of backfill days.
+ auto backfill_days = GetBackfillDays();
+ EXPECT_GE(day_store_created_, backfill_days)
+ << "The day index of store creation must be larger than the number "
+ "of backfill days.";
+ if (day_last_garbage_collected_ == 0u) {
+ return day_store_created_ - backfill_days;
+ }
+ uint32_t max_aggregation_days = 1;
+ for (const auto& window : config.aggregation_window()) {
+ if (window.units_case() == OnDeviceAggregationWindow::kDays &&
+ window.days() > max_aggregation_days) {
+ max_aggregation_days = window.days();
+ }
+ }
+ // Otherwise, it is the later of:
+ // (a) The day index on which the store was created minus the number
+ // of backfill days.
+ // (b) The day index for which the store was last garbage-collected
+ // minus the number of backfill days, minus the largest window size in
+ // the report associated to |config|, plus 1.
+ EXPECT_GE(day_last_garbage_collected_, backfill_days)
+ << "The day index of last garbage collection must be larger than "
+ "the number of backfill days.";
+
+ if (day_last_garbage_collected_ - backfill_days < (max_aggregation_days + 1)) {
+ return day_store_created_ - backfill_days;
+ }
+ return (day_store_created_ < (day_last_garbage_collected_ - max_aggregation_days + 1))
+ ? (day_last_garbage_collected_ - backfill_days - max_aggregation_days + 1)
+ : day_store_created_ - backfill_days;
+ }
+
+ std::unique_ptr<EventAggregator> event_aggregator_;
+ std::unique_ptr<MockConsistentProtoStore> local_aggregate_proto_store_;
+ std::unique_ptr<MockConsistentProtoStore> obs_history_proto_store_;
+ std::unique_ptr<ObservationWriter> observation_writer_;
+ std::unique_ptr<Encoder> encoder_;
+ std::unique_ptr<EncryptedMessageMaker> observation_encrypter_;
+ std::unique_ptr<TestUpdateRecipient> update_recipient_;
+ std::unique_ptr<FakeObservationStore> observation_store_;
+ std::unique_ptr<IncrementingSystemClock> test_clock_;
+ IncrementingSystemClock* unowned_test_clock_;
+ IncrementingSteadyClock* test_steady_clock_;
+ // The day index on which the LocalAggregateStore was last
+ // garbage-collected. A value of 0 indicates that the store has never been
+ // garbage-collected.
+ uint32_t day_last_garbage_collected_ = 0u;
+ // The day index on which the LocalAggregateStore was created.
+ uint32_t day_store_created_ = 0u;
+
+ private:
+ std::unique_ptr<SystemDataInterface> system_data_;
+};
+
+// Creates an EventAggregator and provides it with a ProjectContext generated
+// from a registry.
+class AggregateStoreTestWithProjectContext : public AggregateStoreTest {
+ protected:
+ explicit AggregateStoreTestWithProjectContext(const std::string& registry_var_name) {
+ project_context_ = GetTestProject(registry_var_name);
+ }
+
+ void SetUp() override {
+ AggregateStoreTest::SetUp();
+ event_aggregator_->UpdateAggregationConfigs(*project_context_);
+ }
+
+ // Logs an OccurrenceEvent for the MetricReportId of a locally
+ // aggregated report of the ProjectContext. Overrides the method
+ // AggregateStoreTest::LogUniqueActivesEvent.
+ Status LogUniqueActivesEvent(const MetricReportId& metric_report_id, uint32_t day_index,
+ uint32_t event_code, LoggedActivity* logged_activity = nullptr) {
+ return AggregateStoreTest::LogUniqueActivesEvent(project_context_, metric_report_id, day_index,
+ event_code, logged_activity);
+ }
+
+ // Logs a CountEvent for the MetricReportId of a locally
+ // aggregated report of the ProjectContext. Overrides the method
+ // AggregateStoreTest::LogPerDeviceCountEvent.
+ Status LogPerDeviceCountEvent(const MetricReportId& metric_report_id, uint32_t day_index,
+ const std::string& component, uint32_t event_code, int64_t count,
+ LoggedValues* logged_values = nullptr) {
+ return AggregateStoreTest::LogPerDeviceCountEvent(project_context_, metric_report_id, day_index,
+ component, event_code, count, logged_values);
+ }
+
+ // Logs an ElapsedTimeEvent for the MetricReportId of a locally
+ // aggregated report of the ProjectContext. Overrides the method
+ // AggregateStoreTest::LogPerDeviceElapsedTimeEvent.
+ Status LogPerDeviceElapsedTimeEvent(const MetricReportId& metric_report_id, uint32_t day_index,
+ const std::string& component, uint32_t event_code,
+ int64_t micros, LoggedValues* logged_values = nullptr) {
+ return AggregateStoreTest::LogPerDeviceElapsedTimeEvent(project_context_, metric_report_id,
+ day_index, component, event_code,
+ micros, logged_values);
+ }
+
+ // Logs a FrameRateEvent for the MetricReportId of a locally
+ // aggregated report of the ProjectContext. Overrides the method
+ // AggregateStoreTest::LogPerDeviceFrameRateEvent.
+ Status LogPerDeviceFrameRateEvent(const MetricReportId& metric_report_id, uint32_t day_index,
+ const std::string& component, uint32_t event_code, float fps,
+ LoggedValues* logged_values = nullptr) {
+ return AggregateStoreTest::LogPerDeviceFrameRateEvent(
+ project_context_, metric_report_id, day_index, component, event_code, fps, logged_values);
+ }
+
+ // Logs a MemoryUsageEvent for the MetricReportId of a locally
+ // aggregated report of the ProjectContext. Overrides the method
+ // AggregateStoreTest::LogPerDeviceMemoryUsageEvent.
+ Status LogPerDeviceMemoryUsageEvent(const MetricReportId& metric_report_id, uint32_t day_index,
+ const std::string& component,
+ const std::vector<uint32_t>& event_codes, int64_t bytes,
+ LoggedValues* logged_values = nullptr) {
+ return AggregateStoreTest::LogPerDeviceMemoryUsageEvent(project_context_, metric_report_id,
+ day_index, component, event_codes,
+ bytes, logged_values);
+ }
+
+ private:
+ // A ProjectContext wrapping the MetricDefinitions passed to the
+ // constructor in |metric_string|.
+ std::shared_ptr<ProjectContext> project_context_;
+};
+
+// Creates an EventAggregator and provides it with a ProjectContext generated
+// from test_registries/unique_actives_test_registry.yaml. All metrics in this
+// registry are of type EVENT_OCCURRED and have a UNIQUE_N_DAY_ACTIVES report.
+class UniqueActivesAggregateStoreTest : public AggregateStoreTestWithProjectContext {
+ protected:
+ UniqueActivesAggregateStoreTest()
+ : AggregateStoreTestWithProjectContext(
+ logger::testing::unique_actives::kCobaltRegistryBase64) {}
+};
+
+// Creates an EventAggregator and provides it with a ProjectContext generated
+// from test_registries/unique_actives_noise_free_test_registry.yaml. All
+// metrics in this registry are of type EVENT_OCCURRED and have a
+// UNIQUE_N_DAY_ACTIVES report with local_privacy_noise_level NONE.
+class UniqueActivesNoiseFreeAggregateStoreTest : public AggregateStoreTestWithProjectContext {
+ protected:
+ UniqueActivesNoiseFreeAggregateStoreTest()
+ : AggregateStoreTestWithProjectContext(
+ logger::testing::unique_actives_noise_free::kCobaltRegistryBase64) {}
+};
+
+// Creates an EventAggregator and provides it with a ProjectContext generated
+// from test_registries/per_device_numeric_stats_test_registry.yaml. All metrics
+// in this registry are of type EVENT_COUNT and have a PER_DEVICE_NUMERIC_STATS
+// report.
+class PerDeviceNumericAggregateStoreTest : public AggregateStoreTestWithProjectContext {
+ protected:
+ PerDeviceNumericAggregateStoreTest()
+ : AggregateStoreTestWithProjectContext(
+ logger::testing::per_device_numeric_stats::kCobaltRegistryBase64) {}
+};
+
+// Creates an EventAggregator and provides it with a ProjectContext generated
+// from test_registries/mixed_time_zone_test_registry.yaml. This registry
+// contains multiple MetricDefinitions with different time zone policies.
+class NoiseFreeMixedTimeZoneAggregateStoreTest : public AggregateStoreTestWithProjectContext {
+ protected:
+ NoiseFreeMixedTimeZoneAggregateStoreTest()
+ : AggregateStoreTestWithProjectContext(
+ logger::testing::mixed_time_zone::kCobaltRegistryBase64) {}
+};
+
+class PerDeviceHistogramAggregateStoreTest : public AggregateStoreTestWithProjectContext {
+ protected:
+ PerDeviceHistogramAggregateStoreTest()
+ : AggregateStoreTestWithProjectContext(
+ logger::testing::per_device_histogram::kCobaltRegistryBase64) {}
+};
+
+class AggregateStoreWorkerTest : public AggregateStoreTest {
+ protected:
+ void SetUp() override { AggregateStoreTest::SetUp(); }
+
+ void ShutDownWorkerThread() { event_aggregator_->ShutDown(); }
+
+ bool in_shutdown_state() { return (shutdown_flag_set() && !worker_joinable()); }
+
+ bool in_run_state() { return (!shutdown_flag_set() && worker_joinable()); }
+
+ bool shutdown_flag_set() {
+ return event_aggregator_->protected_worker_thread_controller_.const_lock()->shut_down;
+ }
+
+ bool worker_joinable() { return event_aggregator_->worker_thread_.joinable(); }
+};
+
+// Tests that the Read() method of each ConsistentProtoStore is called once
+// during construction of the EventAggregator.
+TEST_F(AggregateStoreTest, ReadProtosFromFiles) {
+ EXPECT_EQ(1, local_aggregate_proto_store_->read_count_);
+ EXPECT_EQ(1, obs_history_proto_store_->read_count_);
+}
+
+// Tests that the BackUp*() methods return a positive status, and checks that
+// the Write() method of a ConsistentProtoStore is called once when its
+// respective BackUp*() method is called.
+TEST_F(AggregateStoreTest, BackUpProtos) {
+ EXPECT_EQ(kOK, BackUpLocalAggregateStore());
+ EXPECT_EQ(kOK, BackUpObservationHistory());
+ EXPECT_EQ(1, local_aggregate_proto_store_->write_count_);
+ EXPECT_EQ(1, obs_history_proto_store_->write_count_);
+}
+
+// MaybeUpgradeLocalAggregateStore should return an OK status if the version is current. The store
+// should not change.
+TEST_F(AggregateStoreTest, MaybeUpgradeLocalAggregateStoreCurrent) {
+ auto store = MakeNewLocalAggregateStore();
+ std::string store_before = SerializeAsStringDeterministic(store);
+ ASSERT_EQ(kCurrentLocalAggregateStoreVersion, store.version());
+ EXPECT_EQ(kOK, MaybeUpgradeLocalAggregateStore(&store));
+ EXPECT_EQ(store_before, SerializeAsStringDeterministic(store));
+}
+
+// MaybeUpgradeLocalAggregateStore should return kInvalidArguments if it is not possible to upgrade
+// to the current version.
+TEST_F(AggregateStoreTest, MaybeUpgradeLocalAggregateStoreUnsupported) {
+ const uint32_t kFutureVersion = kCurrentLocalAggregateStoreVersion + 1;
+ auto store = MakeNewLocalAggregateStore(kFutureVersion);
+ ASSERT_EQ(kFutureVersion, store.version());
+ EXPECT_EQ(kInvalidArguments, MaybeUpgradeLocalAggregateStore(&store));
+}
+
+// It should be possible to upgrade the LocalAggregateStore from v0 to the current version. The
+// version number should be updated and the contents of window_size in each AggregationConfigs
+// should be moved to aggregation_window, preserving their order.
+TEST_F(AggregateStoreTest, MaybeUpgradeLocalAggregateStoreFromV0) {
+ const uint32_t kVersionZero = 0;
+ const std::vector<uint32_t> kWindowSizes = {1, 7, 30};
+ const std::string kKey = "some_report_key";
+
+ // Make a v0 LocalAggregateStore with one report.
+ auto store = MakeNewLocalAggregateStore(kVersionZero);
+ ReportAggregates report_aggregates;
+ for (auto window_size : kWindowSizes) {
+ report_aggregates.mutable_aggregation_config()->add_window_size(window_size);
+ }
+ (*store.mutable_by_report_key())[kKey] = report_aggregates;
+
+ // Make the expected upgraded store.
+ auto expected_store = MakeNewLocalAggregateStore(kCurrentLocalAggregateStoreVersion);
+ ReportAggregates expected_report_aggregates;
+ for (auto window_size : kWindowSizes) {
+ *expected_report_aggregates.mutable_aggregation_config()->add_aggregation_window() =
+ MakeDayWindow(window_size);
+ }
+ (*expected_store.mutable_by_report_key())[kKey] = expected_report_aggregates;
+
+ // Upgrade and check that the upgraded store is as expected.
+ EXPECT_EQ(kOK, MaybeUpgradeLocalAggregateStore(&store));
+ EXPECT_EQ(SerializeAsStringDeterministic(expected_store), SerializeAsStringDeterministic(store));
+}
+
+// MaybeUpgradeObservationHistoryStore should return an OK status if the version is current. The
+// store should not change.
+TEST_F(AggregateStoreTest, MaybeUpgradeObservationHistoryStoreCurrent) {
+ auto store = MakeNewObservationHistoryStore();
+ std::string store_before = SerializeAsStringDeterministic(store);
+ ASSERT_EQ(kCurrentObservationHistoryStoreVersion, store.version());
+ EXPECT_EQ(kOK, MaybeUpgradeObservationHistoryStore(&store));
+ EXPECT_EQ(store_before, SerializeAsStringDeterministic(store));
+}
+
+// MaybeUpgradeObservationHistoryStore should return kInvalidArguments if it is not possible to
+// upgrade to the current version.
+TEST_F(AggregateStoreTest, MaybeUpgradeObservationHistoryStoreUnsupported) {
+ const uint32_t kFutureVersion = kCurrentObservationHistoryStoreVersion + 1;
+ auto store = MakeNewObservationHistoryStore(kFutureVersion);
+ ASSERT_EQ(kFutureVersion, store.version());
+ EXPECT_EQ(kInvalidArguments, MaybeUpgradeObservationHistoryStore(&store));
+}
+
+// Tests that EventAggregator::GenerateObservations() returns a positive
+// status and that the expected number of Observations is generated when no
+// Events have been logged to the EventAggregator.
+TEST_F(AggregateStoreTest, GenerateObservationsNoEvents) {
+ // Provide the all_report_types test registry to the EventAggregator.
+ auto project_context = GetTestProject(logger::testing::all_report_types::kCobaltRegistryBase64);
+ EXPECT_EQ(kOK, event_aggregator_->UpdateAggregationConfigs(*project_context));
+ // Generate locally aggregated Observations for the current day index.
+ EXPECT_EQ(kOK, GenerateObservations(CurrentDayIndex()));
+ std::vector<Observation2> observations(0);
+ EXPECT_TRUE(FetchAggregatedObservations(
+ &observations, logger::testing::all_report_types::kExpectedAggregationParams,
+ observation_store_.get(), update_recipient_.get()));
+}
+
+// Tests that EventAggregator::GenerateObservations() only generates
+// Observations the first time it is called for a given day index.
+TEST_F(AggregateStoreTest, GenerateObservationsTwice) {
+ // Provide the all_report_types test registry to the EventAggregator.
+ auto project_context = GetTestProject(logger::testing::all_report_types::kCobaltRegistryBase64);
+ EXPECT_EQ(kOK, event_aggregator_->UpdateAggregationConfigs(*project_context));
+ // Check that Observations are generated when GenerateObservations is called
+ // for the current day index for the first time.
+ auto current_day_index = CurrentDayIndex();
+ EXPECT_EQ(kOK, GenerateObservations(current_day_index));
+ std::vector<Observation2> observations(0);
+ EXPECT_TRUE(FetchAggregatedObservations(
+ &observations, logger::testing::all_report_types::kExpectedAggregationParams,
+ observation_store_.get(), update_recipient_.get()));
+ // Check that no Observations are generated when GenerateObservations is
+ // called for the currentday index for the second time.
+ ResetObservationStore();
+ EXPECT_EQ(kOK, GenerateObservations(current_day_index));
+ EXPECT_EQ(0u, observation_store_->messages_received.size());
+}
+
+// When the LocalAggregateStore contains one ReportAggregates proto and that
+// proto is empty, GenerateObservations should return success but generate no
+// observations.
+TEST_F(AggregateStoreTest, GenerateObservationsFromBadStore) {
+ auto bad_store = std::make_unique<LocalAggregateStore>();
+ (*bad_store->mutable_by_report_key())["some_key"] = ReportAggregates();
+ local_aggregate_proto_store_->set_stored_proto(std::move(bad_store));
+ // Read the bad store in to the EventAggregator.
+ ResetEventAggregator();
+ EXPECT_EQ(kOK, GenerateObservations(CurrentDayIndex()));
+ EXPECT_EQ(0u, observation_store_->messages_received.size());
+}
+
+// When the LocalAggregateStore contains one empty ReportAggregates proto and
+// some valid ReportAggregates, GenerateObservations should produce observations
+// for the valid ReportAggregates.
+TEST_F(AggregateStoreTest, GenerateObservationsFromBadStoreMultiReport) {
+ auto bad_store = std::make_unique<LocalAggregateStore>();
+ (*bad_store->mutable_by_report_key())["some_key"] = ReportAggregates();
+ local_aggregate_proto_store_->set_stored_proto(std::move(bad_store));
+ // Read the bad store in to the EventAggregator.
+ ResetEventAggregator();
+ // Provide the all_report_types test registry to the EventAggregator.
+ auto project_context = GetTestProject(logger::testing::all_report_types::kCobaltRegistryBase64);
+ EXPECT_EQ(kOK, event_aggregator_->UpdateAggregationConfigs(*project_context));
+ EXPECT_EQ(kOK, GenerateObservations(CurrentDayIndex()));
+ std::vector<Observation2> observations(0);
+ EXPECT_TRUE(FetchAggregatedObservations(
+ &observations, logger::testing::all_report_types::kExpectedAggregationParams,
+ observation_store_.get(), update_recipient_.get()));
+}
+
+// When the LocalAggregateStore contains one ReportAggregates proto and that
+// proto is empty, GarbageCollect should return success.
+TEST_F(AggregateStoreTest, GarbageCollectBadStore) {
+ auto bad_store = std::make_unique<LocalAggregateStore>();
+ (*bad_store->mutable_by_report_key())["some_key"] = ReportAggregates();
+ local_aggregate_proto_store_->set_stored_proto(std::move(bad_store));
+ // Read the bad store in to the EventAggregator.
+ ResetEventAggregator();
+ EXPECT_EQ(kOK, GarbageCollect(CurrentDayIndex()));
+}
+
+// Tests GarbageCollect() for UniqueActivesReportAggregates.
+//
+// For each value of N in the range [0, 34], logs some UniqueActivesEvents
+// each day for N consecutive days and then garbage-collects the
+// LocalAggregateStore. After garbage collection, verifies the contents of
+// the LocalAggregateStore.
+TEST_F(UniqueActivesAggregateStoreTest, GarbageCollect) {
+ uint32_t max_days_before_gc = 35;
+ for (uint32_t days_before_gc = 0; days_before_gc < max_days_before_gc; days_before_gc++) {
+ SetUp();
+ day_last_garbage_collected_ = 0u;
+ LoggedActivity logged_activity;
+ for (uint32_t offset = 0; offset < days_before_gc; offset++) {
+ auto day_index = CurrentDayIndex();
+ for (const auto& metric_report_id :
+ logger::testing::unique_actives::kExpectedAggregationParams.metric_report_ids) {
+ // Log 2 events with event code 0.
+ EXPECT_EQ(kOK, LogUniqueActivesEvent(metric_report_id, day_index, 0u, &logged_activity));
+ EXPECT_EQ(kOK, LogUniqueActivesEvent(metric_report_id, day_index, 0u, &logged_activity));
+ if (offset < 3) {
+ // Log 1 event with event code 1.
+ EXPECT_EQ(kOK, LogUniqueActivesEvent(metric_report_id, day_index, 1u, &logged_activity));
+ }
+ }
+ AdvanceClock(kDay);
+ }
+ auto end_day_index = CurrentDayIndex();
+ EXPECT_EQ(kOK, GarbageCollect(end_day_index));
+ day_last_garbage_collected_ = end_day_index;
+ EXPECT_TRUE(CheckUniqueActivesAggregates(logged_activity, end_day_index));
+ TearDown();
+ }
+}
+
+// Tests that EventAggregator::GenerateObservations() returns a positive
+// status and that the expected number of Observations is generated after
+// some UniqueActivesEvents have been logged, without any garbage
+// collection.
+//
+// For 35 days, logs 2 events each day for the NetworkActivity_UniqueDevices
+// reports and 2 events for the FeaturesActive_UniqueDevices report, all
+// with event code 0.
+//
+// Each day, calls GenerateObservations() with the day index of the previous
+// day. Checks that a positive status is returned and that the
+// FakeObservationStore has received the expected number of new observations
+// for each locally aggregated report ID in the unique_actives registry.
+TEST_F(UniqueActivesAggregateStoreTest, GenerateObservations) {
+ int num_days = 35;
+ std::vector<Observation2> observations(0);
+ for (int offset = 0; offset < num_days; offset++) {
+ auto day_index = CurrentDayIndex();
+ observations.clear();
+ ResetObservationStore();
+ EXPECT_EQ(kOK, GenerateObservations(day_index - 1));
+ EXPECT_TRUE(FetchAggregatedObservations(
+ &observations, logger::testing::unique_actives::kExpectedAggregationParams,
+ observation_store_.get(), update_recipient_.get()));
+ for (int i = 0; i < 2; i++) {
+ EXPECT_EQ(kOK, LogUniqueActivesEvent(
+ logger::testing::unique_actives::kNetworkActivityWindowSizeMetricReportId,
+ day_index, 0u));
+ EXPECT_EQ(
+ kOK, LogUniqueActivesEvent(
+ logger::testing::unique_actives::kNetworkActivityAggregationWindowMetricReportId,
+ day_index, 0u));
+ EXPECT_EQ(
+ kOK, LogUniqueActivesEvent(logger::testing::unique_actives::kFeaturesActiveMetricReportId,
+ day_index, 0u));
+ }
+ AdvanceClock(kDay);
+ }
+ observations.clear();
+ ResetObservationStore();
+ EXPECT_EQ(kOK, GenerateObservations(CurrentDayIndex() - 1));
+ EXPECT_TRUE(FetchAggregatedObservations(
+ &observations, logger::testing::unique_actives::kExpectedAggregationParams,
+ observation_store_.get(), update_recipient_.get()));
+}
+
+// Tests that GenerateObservations() returns a positive status and that the
+// expected number of Observations is generated each day when Events are
+// logged for UNIQUE_N_DAY_ACTIVES reports over multiple days, and when the
+// LocalAggregateStore is garbage-collected each day.
+//
+// For 35 days, logs 2 events each day for the NetworkActivity_UniqueDevices
+// reports and 2 events for the FeaturesActive_UniqueDevices report, all
+// with event code 0.
+//
+// Each day following the first day, calls GenerateObservations() and then
+// GarbageCollect() with the day index of the current day. Checks that
+// positive statuses are returned and that the FakeObservationStore has
+// received the expected number of new observations for each locally
+// aggregated report ID in the unique_actives registry.
+TEST_F(UniqueActivesAggregateStoreTest, GenerateObservationsWithGc) {
+ int num_days = 35;
+ std::vector<Observation2> observations(0);
+ for (int offset = 0; offset < num_days; offset++) {
+ auto day_index = CurrentDayIndex();
+ observations.clear();
+ ResetObservationStore();
+ EXPECT_EQ(kOK, GenerateObservations(day_index - 1));
+ EXPECT_TRUE(FetchAggregatedObservations(
+ &observations, logger::testing::unique_actives::kExpectedAggregationParams,
+ observation_store_.get(), update_recipient_.get()));
+ EXPECT_EQ(kOK, GarbageCollect(day_index));
+ for (int i = 0; i < 2; i++) {
+ EXPECT_EQ(kOK, LogUniqueActivesEvent(
+ logger::testing::unique_actives::kNetworkActivityWindowSizeMetricReportId,
+ day_index, 0u));
+ EXPECT_EQ(
+ kOK, LogUniqueActivesEvent(
+ logger::testing::unique_actives::kNetworkActivityAggregationWindowMetricReportId,
+ day_index, 0u));
+ EXPECT_EQ(
+ kOK, LogUniqueActivesEvent(logger::testing::unique_actives::kFeaturesActiveMetricReportId,
+ day_index, 0u));
+ }
+ AdvanceClock(kDay);
+ }
+ observations.clear();
+ ResetObservationStore();
+ auto day_index = CurrentDayIndex();
+ EXPECT_EQ(kOK, GenerateObservations(day_index - 1));
+ EXPECT_TRUE(FetchAggregatedObservations(
+ &observations, logger::testing::unique_actives::kExpectedAggregationParams,
+ observation_store_.get(), update_recipient_.get()));
+ EXPECT_EQ(kOK, GarbageCollect(day_index));
+}
+
+// Tests that GenerateObservations() returns a positive status and that the
+// expected number of Observations is generated when events are logged over
+// multiple days and some of those days' Observations are backfilled, without
+// any garbage collection of the LocalAggregateStore.
+//
+// Sets the |backfill_days_| field of the EventAggregator to 3.
+//
+// Logging pattern:
+// For 35 days, logs 2 events each day for the
+// NetworkActivity_UniqueDevices reports and 2 events for the
+// FeaturesActive_UniqueDevices report, all with event code 0.
+//
+// Observation generation pattern:
+// Calls GenerateObservations() on the 1st through 5th and the 7th out of
+// every 10 days, for 35 days.
+//
+// Expected numbers of Observations:
+// It is expected that 4 days' worth of Observations are generated on
+// the first day of every 10 (the day index for which GenerateObservations()
+// was called, plus 3 days of backfill), that 1 day's worth of Observations
+// are generated on the 2nd through 5th day of every 10, that 2 days'
+// worth of Observations are generated on the 7th day of every 10 (the
+// day index for which GenerateObservations() was called, plus 1 day of
+// backfill), and that no Observations are generated on the remaining days.
+TEST_F(UniqueActivesAggregateStoreTest, GenerateObservationsWithBackfill) {
+ // Set |backfill_days_| to 3.
+ size_t backfill_days = 3;
+ SetBackfillDays(backfill_days);
+ // Log 2 events each day for 35 days. Call GenerateObservations() on the
+ // first 5 day indices, and the 7th, out of every 10.
+ for (int offset = 0; offset < 35; offset++) {
+ auto day_index = CurrentDayIndex();
+ for (int i = 0; i < 2; i++) {
+ EXPECT_EQ(kOK, LogUniqueActivesEvent(
+ logger::testing::unique_actives::kNetworkActivityWindowSizeMetricReportId,
+ day_index, 0u));
+ EXPECT_EQ(
+ kOK, LogUniqueActivesEvent(
+ logger::testing::unique_actives::kNetworkActivityAggregationWindowMetricReportId,
+ day_index, 0u));
+ EXPECT_EQ(
+ kOK, LogUniqueActivesEvent(logger::testing::unique_actives::kFeaturesActiveMetricReportId,
+ day_index, 0u));
+ }
+ observation_store_->ResetObservationCounter();
+ if (offset % 10 < 5 || offset % 10 == 6) {
+ EXPECT_EQ(kOK, GenerateObservations(day_index));
+ }
+ auto num_new_obs = observation_store_->num_observations_added();
+ EXPECT_GE(num_new_obs, 0u);
+ // Check that the expected daily number of Observations was generated.
+ switch (offset % 10) {
+ case 0:
+ EXPECT_EQ(logger::testing::unique_actives::kExpectedAggregationParams.daily_num_obs *
+ (backfill_days + 1),
+ num_new_obs);
+ break;
+ case 1:
+ case 2:
+ case 3:
+ case 4:
+ EXPECT_EQ(logger::testing::unique_actives::kExpectedAggregationParams.daily_num_obs,
+ num_new_obs);
+ break;
+ case 6:
+ EXPECT_EQ(logger::testing::unique_actives::kExpectedAggregationParams.daily_num_obs * 2,
+ num_new_obs);
+ break;
+ default:
+ EXPECT_EQ(0u, num_new_obs);
+ }
+ AdvanceClock(kDay);
+ }
+}
+
+// Tests that GenerateObservations() returns a positive status and that the
+// expected number of Observations is generated when events are logged over
+// multiple days and some of those days' Observations are backfilled, and when
+// the LocalAggregateStore is garbage-collected after each call to
+// GenerateObservations().
+//
+// Sets the |backfill_days_| field of the EventAggregator to 3.
+//
+// Logging pattern:
+// For 35 days, logs 2 events each day for the
+// NetworkActivity_UniqueDevices reports and 2 events for the
+// FeaturesActive_Unique_Devices report, all with event code 0.
+//
+// Observation generation pattern:
+// Calls GenerateObservations() on the 1st through 5th and the 7th out of
+// every 10 days, for 35 days. Garbage-collects the LocalAggregateStore after
+// each call.
+//
+// Expected numbers of Observations:
+// It is expected that 4 days' worth of Observations are generated on
+// the first day of every 10 (the day index for which GenerateObservations()
+// was called, plus 3 days of backfill), that 1 day's worth of Observations
+// are generated on the 2nd through 5th day of every 10, that 2 days'
+// worth of Observations are generated on the 7th day of every 10 (the
+// day index for which GenerateObservations() was called, plus 1 day of
+// backfill), and that no Observations are generated on the remaining days.
+TEST_F(UniqueActivesAggregateStoreTest, GenerateObservationsWithBackfillAndGc) {
+ int num_days = 35;
+ // Set |backfill_days_| to 3.
+ size_t backfill_days = 3;
+ SetBackfillDays(backfill_days);
+ // Log 2 events each day for 35 days. Call GenerateObservations() on the
+ // first 5 day indices, and the 7th, out of every 10.
+ for (int offset = 0; offset < num_days; offset++) {
+ auto day_index = CurrentDayIndex();
+ for (int i = 0; i < 2; i++) {
+ EXPECT_EQ(kOK, LogUniqueActivesEvent(
+ logger::testing::unique_actives::kNetworkActivityWindowSizeMetricReportId,
+ day_index, 0u));
+ EXPECT_EQ(
+ kOK, LogUniqueActivesEvent(
+ logger::testing::unique_actives::kNetworkActivityAggregationWindowMetricReportId,
+ day_index, 0u));
+ EXPECT_EQ(
+ kOK, LogUniqueActivesEvent(logger::testing::unique_actives::kFeaturesActiveMetricReportId,
+ day_index, 0u));
+ }
+ observation_store_->ResetObservationCounter();
+ if (offset % 10 < 5 || offset % 10 == 6) {
+ EXPECT_EQ(kOK, GenerateObservations(day_index));
+ EXPECT_EQ(kOK, GarbageCollect(day_index));
+ }
+ auto num_new_obs = observation_store_->num_observations_added();
+ EXPECT_GE(num_new_obs, 0u);
+ // Check that the expected daily number of Observations was generated.
+ // This expected number is some multiple of the daily_num_obs field of
+ // |kUniqueActivesExpectedParams|, depending on the number of days which
+ // should have been backfilled when GenerateObservations() was called.
+ switch (offset % 10) {
+ case 0:
+ EXPECT_EQ(logger::testing::unique_actives::kExpectedAggregationParams.daily_num_obs *
+ (backfill_days + 1),
+ num_new_obs);
+ break;
+ case 1:
+ case 2:
+ case 3:
+ case 4:
+ EXPECT_EQ(logger::testing::unique_actives::kExpectedAggregationParams.daily_num_obs,
+ num_new_obs);
+ break;
+ case 6:
+ EXPECT_EQ(logger::testing::unique_actives::kExpectedAggregationParams.daily_num_obs * 2,
+ num_new_obs);
+ break;
+ default:
+ EXPECT_EQ(0u, num_new_obs);
+ }
+ AdvanceClock(kDay);
+ }
+}
+
+// Checks that UniqueActivesObservations with the expected values (i.e.,
+// non-active for all UNIQUE_N_DAY_ACTIVES reports, for all window sizes and
+// event codes) are generated when no Events have been logged to the
+// EventAggregator.
+TEST_F(UniqueActivesNoiseFreeAggregateStoreTest, CheckObservationValuesNoEvents) {
+ auto current_day_index = CurrentDayIndex();
+ EXPECT_EQ(kOK, GenerateObservations(current_day_index));
+ auto expected_obs = MakeNullExpectedUniqueActivesObservations(
+ logger::testing::unique_actives_noise_free::kExpectedAggregationParams, current_day_index);
+ EXPECT_TRUE(CheckUniqueActivesObservations(expected_obs, observation_store_.get(),
+ update_recipient_.get()));
+}
+
+// Checks that UniqueActivesObservations with the expected values are
+// generated when GenerateObservations() is called for a single day index
+// after logging some events for UNIQUE_N_DAY_ACTIVES reports for that day
+// index, without any garbage collection or backfill.
+//
+// Logging pattern:
+// Logs 2 occurrences of event code 0 for the FeaturesActives_UniqueDevices
+// report, and 1 occurrence of event code 1 for the
+// EventsOccurred_UniqueDevices report, all on the same day.
+//
+// Observation generation pattern:
+// Calls GenerateObservations() after logging all events.
+//
+// Expected numbers of Observations:
+// The expected number of Observations is the daily_num_obs field of
+// |logger::testing::unique_actives_noise_free::kExpectedAggregationParams|.
+//
+// Expected Observation values:
+// All Observations should be labeled with the day index on which the events
+// were logged.
+//
+// For the FeaturesActive_UniqueDevices report, expect activity indicators:
+//
+// window size active for event codes
+// ------------------------------------------
+// 1 0
+// 7 0
+// 30 0
+//
+// For the EventsOccurred_UniqueDevices report, expected activity indicators:
+// window size active for event codes
+// ------------------------------------------
+// 1 1
+// 7 1
+//
+// All other Observations should be of inactivity.
+TEST_F(UniqueActivesNoiseFreeAggregateStoreTest, CheckObservationValuesSingleDay) {
+ auto day_index = CurrentDayIndex();
+ // Log several events on |day_index|.
+ EXPECT_EQ(kOK, LogUniqueActivesEvent(
+ logger::testing::unique_actives_noise_free::kFeaturesActiveMetricReportId,
+ day_index, 0u));
+ EXPECT_EQ(kOK, LogUniqueActivesEvent(
+ logger::testing::unique_actives_noise_free::kFeaturesActiveMetricReportId,
+ day_index, 0u));
+ EXPECT_EQ(kOK, LogUniqueActivesEvent(
+ logger::testing::unique_actives_noise_free::kEventsOccurredMetricReportId,
+ day_index, 1u));
+ // Generate locally aggregated Observations for |day_index|.
+ EXPECT_EQ(kOK, GenerateObservations(day_index));
+
+ // Form the expected observations.
+ auto expected_obs = MakeNullExpectedUniqueActivesObservations(
+ logger::testing::unique_actives_noise_free::kExpectedAggregationParams, day_index);
+ expected_obs[{logger::testing::unique_actives_noise_free::kFeaturesActiveMetricReportId,
+ day_index}] = {{1, {true, false, false, false, false}},
+ {7, {true, false, false, false, false}},
+ {30, {true, false, false, false, false}}};
+ expected_obs[{logger::testing::unique_actives_noise_free::kEventsOccurredMetricReportId,
+ day_index}] = {{1, {false, true, false, false, false}},
+ {7, {false, true, false, false, false}}};
+
+ // Check the contents of the FakeObservationStore.
+ EXPECT_TRUE(CheckUniqueActivesObservations(expected_obs, observation_store_.get(),
+ update_recipient_.get()));
+}
+
+// Checks that UniqueActivesObservations with the expected values are
+// generated when some events have been logged for a UNIQUE_N_DAY_ACTIVES
+// report over multiple days and GenerateObservations() is called each
+// day, without garbage collection or backfill.
+//
+// Logging pattern:
+// Logs events for the EventsOccurred_UniqueDevices report (whose parent
+// metric has max_event_code = 4) for 10 days, according to the following
+// pattern:
+//
+// * Never log event code 0.
+// * On the i-th day (0-indexed) of logging, log an event for event code k,
+// 1 <= k < 5, if 3*k divides i.
+//
+// Observation generation pattern:
+// Each day following the first day, generates Observations for the previous
+// day index.
+//
+// Expected number of Observations:
+// Each call to GenerateObservations should generate a number of Observations
+// equal to the daily_num_obs field of
+// |testing::unique_actives_noise_free::kExpectedAggregationParams|.
+//
+// Expected Observation values:
+// The EventsOccurred_UniqueDevices report has window sizes 1 and 7, and
+// the expected activity indicators of Observations for that report on the
+// i-th day are:
+//
+// (i, window size) active for event codes
+// ------------------------------------------------------
+// (0, 1) 1, 2, 3, 4
+// (0, 7) 1, 2, 3, 4
+// (1, 1) ---
+// (1, 7) 1, 2, 3, 4
+// (2, 1) ---
+// (2, 7) 1, 2, 3, 4
+// (3, 1) 1
+// (3, 7) 1, 2, 3, 4
+// (4, 1) ---
+// (4, 7) 1, 2, 3, 4
+// (5, 1) ---
+// (5, 7) 1, 2, 3, 4
+// (6, 1) 1, 2
+// (6, 7) 1, 2, 3, 4
+// (7, 1) ---
+// (7, 7) 1, 2
+// (8, 1) ---
+// (8, 7) 1, 2
+// (9, 1) 1, 3
+// (9, 7) 1, 2, 3
+//
+// All Observations for all other locally aggregated reports should be
+// observations of non-occurrence.
+TEST_F(UniqueActivesNoiseFreeAggregateStoreTest, CheckObservationValuesMultiDay) {
+ auto start_day_index = CurrentDayIndex();
+ // Form expected Obsevations for the 10 days of logging.
+ uint32_t num_days = 10;
+ std::vector<ExpectedUniqueActivesObservations> expected_obs(num_days);
+ const auto& expected_id =
+ logger::testing::unique_actives_noise_free::kEventsOccurredMetricReportId;
+ for (uint32_t offset = 0; offset < num_days; offset++) {
+ expected_obs[offset] = MakeNullExpectedUniqueActivesObservations(
+ logger::testing::unique_actives_noise_free::kExpectedAggregationParams,
+ start_day_index + offset);
+ }
+ expected_obs[0][{expected_id, start_day_index}] = {{1, {false, true, true, true, true}},
+ {7, {false, true, true, true, true}}};
+ expected_obs[1][{expected_id, start_day_index + 1}] = {{1, {false, false, false, false, false}},
+ {7, {false, true, true, true, true}}};
+ expected_obs[2][{expected_id, start_day_index + 2}] = {{1, {false, false, false, false, false}},
+ {7, {false, true, true, true, true}}};
+ expected_obs[3][{expected_id, start_day_index + 3}] = {{1, {false, true, false, false, false}},
+ {7, {false, true, true, true, true}}};
+ expected_obs[4][{expected_id, start_day_index + 4}] = {{1, {false, false, false, false, false}},
+ {7, {false, true, true, true, true}}};
+ expected_obs[5][{expected_id, start_day_index + 5}] = {{1, {false, false, false, false, false}},
+ {7, {false, true, true, true, true}}};
+ expected_obs[6][{expected_id, start_day_index + 6}] = {{1, {false, true, true, false, false}},
+ {7, {false, true, true, true, true}}};
+ expected_obs[7][{expected_id, start_day_index + 7}] = {{1, {false, false, false, false, false}},
+ {7, {false, true, true, false, false}}};
+ expected_obs[8][{expected_id, start_day_index + 8}] = {{1, {false, false, false, false, false}},
+ {7, {false, true, true, false, false}}};
+ expected_obs[9][{expected_id, start_day_index + 9}] = {{1, {false, true, false, true, false}},
+ {7, {false, true, true, true, false}}};
+
+ for (uint32_t offset = 0; offset < num_days; offset++) {
+ auto day_index = CurrentDayIndex();
+ for (uint32_t event_code = 1;
+ event_code <
+ logger::testing::unique_actives_noise_free::kExpectedAggregationParams.num_event_codes.at(
+ expected_id);
+ event_code++) {
+ if (offset % (3 * event_code) == 0) {
+ EXPECT_EQ(kOK, LogUniqueActivesEvent(expected_id, day_index, event_code));
+ }
+ }
+ // Clear the FakeObservationStore.
+ ResetObservationStore();
+ // Generate locally aggregated Observations.
+ EXPECT_EQ(kOK, GenerateObservations(day_index));
+ // Check the generated Observations against the expectation.
+ EXPECT_TRUE(CheckUniqueActivesObservations(expected_obs[offset], observation_store_.get(),
+ update_recipient_.get()));
+ AdvanceClock(kDay);
+ }
+}
+
+// Checks that UniqueActivesObservations with the expected values are
+// generated when some events have been logged for a UNIQUE_N_DAY_ACTIVES
+// report over multiple days and GenerateObservations() is called each
+// day, and when the LocalAggregateStore is garbage-collected after each call
+// to GenerateObservations().
+//
+// Logging pattern:
+// Logs events for the EventsOccurred_UniqueDevices report (whose parent
+// metric has max_event_code = 4) for 10 days, according to the following
+// pattern:
+//
+// * Never log event code 0.
+// * On the i-th day (0-indexed) of logging, log an event for event code k,
+// 1 <= k < 5, if 3*k divides i.
+//
+// Observation generation pattern:
+// Each day following the first day, generates Observations for the previous
+// day index.
+//
+// Expected number of Observations:
+// Each call to GenerateObservations should generate a number of Observations
+// equal to the daily_num_obs field of
+// |logger::testing::unique_actives_noise_free::kExpectedAggregationParams|.
+//
+// Expected Observation values:
+// The EventsOccurred_UniqueDevices report has window sizes 1 and 7, and
+// the expected activity indicators of Observations for that report on the
+// i-th day are:
+//
+// (i, window size) active for event codes
+// ------------------------------------------------------
+// (0, 1) 1, 2, 3, 4
+// (0, 7) 1, 2, 3, 4
+// (1, 1) ---
+// (1, 7) 1, 2, 3, 4
+// (2, 1) ---
+// (2, 7) 1, 2, 3, 4
+// (3, 1) 1
+// (3, 7) 1, 2, 3, 4
+// (4, 1) ---
+// (4, 7) 1, 2, 3, 4
+// (5, 1) ---
+// (5, 7) 1, 2, 3, 4
+// (6, 1) 1, 2
+// (6, 7) 1, 2, 3, 4
+// (7, 1) ---
+// (7, 7) 1, 2
+// (8, 1) ---
+// (8, 7) 1, 2
+// (9, 1) 1, 3
+// (9, 7) 1, 2, 3
+//
+// All Observations for all other locally aggregated reports should be
+// observations of non-occurrence.
+TEST_F(UniqueActivesNoiseFreeAggregateStoreTest,
+ CheckObservationValuesMultiDayWithGarbageCollection) {
+ auto start_day_index = CurrentDayIndex();
+ // Form expected Observations for the 10 days of logging.
+ uint32_t num_days = 10;
+ std::vector<ExpectedUniqueActivesObservations> expected_obs(num_days);
+ const auto& expected_id =
+ logger::testing::unique_actives_noise_free::kEventsOccurredMetricReportId;
+
+ for (uint32_t offset = 0; offset < num_days; offset++) {
+ expected_obs[offset] = MakeNullExpectedUniqueActivesObservations(
+ logger::testing::unique_actives_noise_free::kExpectedAggregationParams,
+ start_day_index + offset);
+ }
+ expected_obs[0][{expected_id, start_day_index}] = {{1, {false, true, true, true, true}},
+ {7, {false, true, true, true, true}}};
+ expected_obs[1][{expected_id, start_day_index + 1}] = {{1, {false, false, false, false, false}},
+ {7, {false, true, true, true, true}}};
+ expected_obs[2][{expected_id, start_day_index + 2}] = {{1, {false, false, false, false, false}},
+ {7, {false, true, true, true, true}}};
+ expected_obs[3][{expected_id, start_day_index + 3}] = {{1, {false, true, false, false, false}},
+ {7, {false, true, true, true, true}}};
+ expected_obs[4][{expected_id, start_day_index + 4}] = {{1, {false, false, false, false, false}},
+ {7, {false, true, true, true, true}}};
+ expected_obs[5][{expected_id, start_day_index + 5}] = {{1, {false, false, false, false, false}},
+ {7, {false, true, true, true, true}}};
+ expected_obs[6][{expected_id, start_day_index + 6}] = {{1, {false, true, true, false, false}},
+ {7, {false, true, true, true, true}}};
+ expected_obs[7][{expected_id, start_day_index + 7}] = {{1, {false, false, false, false, false}},
+ {7, {false, true, true, false, false}}};
+ expected_obs[8][{expected_id, start_day_index + 8}] = {{1, {false, false, false, false, false}},
+ {7, {false, true, true, false, false}}};
+ expected_obs[9][{expected_id, start_day_index + 9}] = {{1, {false, true, false, true, false}},
+ {7, {false, true, true, true, false}}};
+
+ for (uint32_t offset = 0; offset < num_days; offset++) {
+ auto day_index = CurrentDayIndex();
+ for (uint32_t event_code = 1;
+ event_code <
+ logger::testing::unique_actives_noise_free::kExpectedAggregationParams.num_event_codes.at(
+ expected_id);
+ event_code++) {
+ if (offset % (3 * event_code) == 0) {
+ EXPECT_EQ(kOK, LogUniqueActivesEvent(expected_id, day_index, event_code));
+ }
+ }
+ // Advance |test_clock_| by 1 day.
+ AdvanceClock(kDay);
+ // Clear the FakeObservationStore.
+ ResetObservationStore();
+ // Generate locally aggregated Observations and garbage-collect the
+ // LocalAggregateStore, both for the previous day as measured by
+ // |test_clock_|. Back up the LocalAggregateStore and
+ // AggregatedObservationHistoryStore.
+ DoScheduledTasksNow();
+ // Check the generated Observations against the expectation.
+ EXPECT_TRUE(CheckUniqueActivesObservations(expected_obs[offset], observation_store_.get(),
+ update_recipient_.get()));
+ }
+}
+
+// Tests that the expected UniqueActivesObservations are generated when events
+// are logged over multiple days and when Observations are backfilled for some
+// days during that period, without any garbage-collection of the
+// LocalAggregateStore.
+//
+// The test sets the number of backfill days to 3.
+//
+// Logging pattern:
+// Events for the EventsOccurred_UniqueDevices report are logged over the days
+// |start_day_index| to |start_day_index + 8| according to the following
+// pattern:
+//
+// * For i = 0 to i = 4, log an event with event code i on day
+// |start_day_index + i| and |start_day_index + 2*i|.
+//
+// Observation generation pattern:
+// The test calls GenerateObservations() on day |start_day_index + i| for i =
+// 0 through i = 5 and for i = 8, skipping the days |start_day_index + 6| and
+// |start_day_index + 7|.
+//
+// Expected numbers of Observations:
+// It is expected that 4 days' worth of Observations are generated on the
+// first day (the day index for which GenerateObservations() was called, plus
+// 3 days of backfill), that 1 day's worth of Observations is generated on the
+// 2nd through 6th day, that 3 days' worth of Observations are generated on
+// the 9th day (the day index for which GenerateObservations() was called,
+// plus 2 days of backfill), and that no Observations are generated on the
+// remaining days.
+//
+// Expected Observation values:
+// The expected activity indicators of Observations for the
+// EventsOccurred_UniqueDevices report for the i-th day of logging are:
+//
+// (i, window size) active for event codes
+// -------------------------------------------------------------------------
+// (0, 1) 0
+// (0, 7) 0
+// (1, 1) 1
+// (1, 7) 0, 1
+// (2, 1) 1, 2
+// (2, 7) 0, 1, 2
+// (3, 1) 3
+// (3, 7) 0, 1, 2, 3
+// (4, 1) 2, 4
+// (4, 7) 0, 1, 2, 3, 4
+// (5, 1) ---
+// (5, 7) 0, 1, 2, 3, 4
+// (6, 1) 3
+// (6, 7) 0, 1, 2, 3, 4
+// (7, 1) ---
+// (7, 7) 1, 2, 3, 4
+// (8, 1) 4
+// (8, 7) 1, 2, 3, 4
+//
+// All other Observations should be of non-activity.
+TEST_F(UniqueActivesNoiseFreeAggregateStoreTest, CheckObservationValuesWithBackfill) {
+ auto start_day_index = CurrentDayIndex();
+ // Set |backfill_days_| to 3.
+ size_t backfill_days = 3;
+ SetBackfillDays(backfill_days);
+ const auto& expected_id =
+ logger::testing::unique_actives_noise_free::kEventsOccurredMetricReportId;
+ const auto& expected_params =
+ logger::testing::unique_actives_noise_free::kExpectedAggregationParams;
+ // Log events for 9 days. Call GenerateObservations() on the first 6 day
+ // indices, and the 9th.
+ for (uint32_t offset = 0; offset < 9; offset++) {
+ auto day_index = CurrentDayIndex();
+ ResetObservationStore();
+ for (uint32_t event_code = 0; event_code < expected_params.num_event_codes.at(expected_id);
+ event_code++) {
+ if (event_code == offset || (2 * event_code) == offset) {
+ EXPECT_EQ(kOK, LogUniqueActivesEvent(expected_id, day_index, event_code));
+ }
+ }
+ if (offset < 6 || offset == 8) {
+ EXPECT_EQ(kOK, GenerateObservations(day_index));
+ }
+ // Make the set of Observations which are expected to be generated on
+ // |start_day_index + offset| and check it against the contents of the
+ // FakeObservationStore.
+ ExpectedUniqueActivesObservations expected_obs;
+ switch (offset) {
+ case 0: {
+ for (uint32_t day_index = start_day_index - backfill_days; day_index <= start_day_index;
+ day_index++) {
+ for (const auto& pair :
+ MakeNullExpectedUniqueActivesObservations(expected_params, day_index)) {
+ expected_obs.insert(pair);
+ }
+ }
+ expected_obs[{expected_id, start_day_index}] = {{1, {true, false, false, false, false}},
+ {7, {true, false, false, false, false}}};
+ EXPECT_TRUE(CheckUniqueActivesObservations(expected_obs, observation_store_.get(),
+ update_recipient_.get()));
+ break;
+ }
+ case 1: {
+ expected_obs =
+ MakeNullExpectedUniqueActivesObservations(expected_params, start_day_index + 1);
+ expected_obs[{expected_id, start_day_index + 1}] = {{1, {false, true, false, false, false}},
+ {7, {true, true, false, false, false}}};
+ EXPECT_TRUE(CheckUniqueActivesObservations(expected_obs, observation_store_.get(),
+ update_recipient_.get()));
+ break;
+ }
+ case 2: {
+ expected_obs =
+ MakeNullExpectedUniqueActivesObservations(expected_params, start_day_index + 2);
+ expected_obs[{expected_id, start_day_index + 2}] = {{1, {false, true, true, false, false}},
+ {7, {true, true, true, false, false}}};
+ EXPECT_TRUE(CheckUniqueActivesObservations(expected_obs, observation_store_.get(),
+ update_recipient_.get()));
+ break;
+ }
+ case 3: {
+ expected_obs =
+ MakeNullExpectedUniqueActivesObservations(expected_params, start_day_index + 3);
+ expected_obs[{expected_id, start_day_index + 3}] = {{1, {false, false, false, true, false}},
+ {7, {true, true, true, true, false}}};
+ EXPECT_TRUE(CheckUniqueActivesObservations(expected_obs, observation_store_.get(),
+ update_recipient_.get()));
+ break;
+ }
+ case 4: {
+ expected_obs =
+ MakeNullExpectedUniqueActivesObservations(expected_params, start_day_index + 4);
+ expected_obs[{expected_id, start_day_index + 4}] = {{1, {false, false, true, false, true}},
+ {7, {true, true, true, true, true}}};
+ EXPECT_TRUE(CheckUniqueActivesObservations(expected_obs, observation_store_.get(),
+ update_recipient_.get()));
+ break;
+ }
+ case 5: {
+ expected_obs =
+ MakeNullExpectedUniqueActivesObservations(expected_params, start_day_index + 5);
+ expected_obs[{expected_id, start_day_index + 5}] = {
+ {1, {false, false, false, false, false}}, {7, {true, true, true, true, true}}};
+ EXPECT_TRUE(CheckUniqueActivesObservations(expected_obs, observation_store_.get(),
+ update_recipient_.get()));
+ break;
+ }
+ case 8: {
+ for (uint32_t day_index = start_day_index + 6; day_index <= start_day_index + 8;
+ day_index++) {
+ for (const auto& pair :
+ MakeNullExpectedUniqueActivesObservations(expected_params, day_index)) {
+ expected_obs.insert(pair);
+ }
+ }
+ expected_obs[{expected_id, start_day_index + 6}] = {{1, {false, false, false, true, false}},
+ {7, {true, true, true, true, true}}};
+ expected_obs[{expected_id, start_day_index + 7}] = {
+ {1, {false, false, false, false, false}}, {7, {false, true, true, true, true}}};
+ expected_obs[{expected_id, start_day_index + 8}] = {{1, {false, false, false, false, true}},
+ {7, {false, true, true, true, true}}};
+ EXPECT_TRUE(CheckUniqueActivesObservations(expected_obs, observation_store_.get(),
+ update_recipient_.get()));
+ break;
+ }
+ default:
+ EXPECT_TRUE(CheckUniqueActivesObservations(expected_obs, observation_store_.get(),
+ update_recipient_.get()));
+ }
+ AdvanceClock(kDay);
+ }
+}
+
+// Tests that the expected UniqueActivesObservations are generated when events
+// are logged over multiple days and when Observations are backfilled for some
+// days during that period, and when the LocalAggregateStore is
+// garbage-collected after each call to GenerateObservations().
+//
+// The test sets the number of backfill days to 3.
+//
+// Logging pattern:
+// Events for the EventsOccurred_UniqueDevices report are logged over the days
+// |start_day_index| to |start_day_index + 8| according to the following
+// pattern:
+//
+// * For i = 0 to i = 4, log an event with event code i on day
+// |start_day_index + i| and |start_day_index + 2*i|.
+//
+// Observation generation pattern:
+// The test calls GenerateObservations() on day |start_day_index + i| for i =
+// 0 through i = 5 and for i = 8, skipping the days |start_day_index + 6| and
+// |start_day_index + 7|.
+//
+// Expected numbers of Observations:
+// It is expected that 4 days' worth of Observations are generated on the
+// first day (the day index for which GenerateObservations() was called, plus
+// 3 days of backfill), that 1 day's worth of Observations is generated on the
+// 2nd through 6th day, that 3 days' worth of Observations are generated on
+// the 9th day (the day index for which GenerateObservations() was called,
+// plus 2 days of backfill), and that no Observations are generated on the
+// remaining days.
+//
+// Expected Observation values:
+// The expected activity indicators of Observations for the
+// EventsOccurred_UniqueDevices report for the i-th day of logging are:
+//
+// (i, window size) active for event codes
+// -------------------------------------------------------------------------
+// (0, 1) 0
+// (0, 7) 0
+// (1, 1) 1
+// (1, 7) 0, 1
+// (2, 1) 1, 2
+// (2, 7) 0, 1, 2
+// (3, 1) 3
+// (3, 7) 0, 1, 2, 3
+// (4, 1) 2, 4
+// (4, 7) 0, 1, 2, 3, 4
+// (5, 1) ---
+// (5, 7) 0, 1, 2, 3, 4
+// (6, 1) 3
+// (6, 7) 0, 1, 2, 3, 4
+// (7, 1) ---
+// (7, 7) 1, 2, 3, 4
+// (8, 1) 4
+// (8, 7) 1, 2, 3, 4
+//
+// All other Observations should be of non-activity.
+TEST_F(UniqueActivesNoiseFreeAggregateStoreTest, CheckObservationValuesWithBackfillAndGc) {
+ auto start_day_index = CurrentDayIndex();
+ // Set |backfill_days_| to 3.
+ size_t backfill_days = 3;
+ SetBackfillDays(backfill_days);
+
+ const auto& expected_id =
+ logger::testing::unique_actives_noise_free::kEventsOccurredMetricReportId;
+ const auto& expected_params =
+ logger::testing::unique_actives_noise_free::kExpectedAggregationParams;
+
+ // Log events for 9 days. Call GenerateObservations() on the first 6 day
+ // indices, and the 9th.
+ for (uint32_t offset = 0; offset < 8; offset++) {
+ auto day_index = CurrentDayIndex();
+ ResetObservationStore();
+ for (uint32_t event_code = 0; event_code < expected_params.num_event_codes.at(expected_id);
+ event_code++) {
+ if (event_code == offset || (2 * event_code) == offset) {
+ EXPECT_EQ(kOK, LogUniqueActivesEvent(expected_id, day_index, event_code));
+ }
+ }
+ // Advance |test_clock_| by 1 day.
+ AdvanceClock(kDay);
+ if (offset < 6 || offset == 9) {
+ // Generate Observations and garbage-collect, both for the previous day
+ // index according to |test_clock_|. Back up the LocalAggregateStore and
+ // the AggregatedObservationHistoryStore.
+ DoScheduledTasksNow();
+ }
+ // Make the set of Observations which are expected to be generated on
+ // |start_day_index + offset| and check it against the contents of the
+ // FakeObservationStore.
+ ExpectedUniqueActivesObservations expected_obs;
+ switch (offset) {
+ case 0: {
+ for (uint32_t day_index = start_day_index - backfill_days; day_index <= start_day_index;
+ day_index++) {
+ for (const auto& pair :
+ MakeNullExpectedUniqueActivesObservations(expected_params, day_index)) {
+ expected_obs.insert(pair);
+ }
+ }
+ expected_obs[{expected_id, start_day_index}] = {{1, {true, false, false, false, false}},
+ {7, {true, false, false, false, false}}};
+ EXPECT_TRUE(CheckUniqueActivesObservations(expected_obs, observation_store_.get(),
+ update_recipient_.get()));
+ break;
+ }
+ case 1: {
+ expected_obs =
+ MakeNullExpectedUniqueActivesObservations(expected_params, start_day_index + 1);
+ expected_obs[{expected_id, start_day_index + 1}] = {{1, {false, true, false, false, false}},
+ {7, {true, true, false, false, false}}};
+ EXPECT_TRUE(CheckUniqueActivesObservations(expected_obs, observation_store_.get(),
+ update_recipient_.get()));
+ break;
+ }
+ case 2: {
+ expected_obs =
+ MakeNullExpectedUniqueActivesObservations(expected_params, start_day_index + 2);
+ expected_obs[{expected_id, start_day_index + 2}] = {{1, {false, true, true, false, false}},
+ {7, {true, true, true, false, false}}};
+ EXPECT_TRUE(CheckUniqueActivesObservations(expected_obs, observation_store_.get(),
+ update_recipient_.get()));
+ break;
+ }
+ case 3: {
+ expected_obs =
+ MakeNullExpectedUniqueActivesObservations(expected_params, start_day_index + 3);
+ expected_obs[{expected_id, start_day_index + 3}] = {{1, {false, false, false, true, false}},
+ {7, {true, true, true, true, false}}};
+ EXPECT_TRUE(CheckUniqueActivesObservations(expected_obs, observation_store_.get(),
+ update_recipient_.get()));
+ break;
+ }
+ case 4: {
+ expected_obs =
+ MakeNullExpectedUniqueActivesObservations(expected_params, start_day_index + 4);
+ expected_obs[{expected_id, start_day_index + 4}] = {{1, {false, false, true, false, true}},
+ {7, {true, true, true, true, true}}};
+ EXPECT_TRUE(CheckUniqueActivesObservations(expected_obs, observation_store_.get(),
+ update_recipient_.get()));
+ break;
+ }
+ case 5: {
+ expected_obs =
+ MakeNullExpectedUniqueActivesObservations(expected_params, start_day_index + 5);
+ expected_obs[{expected_id, start_day_index + 5}] = {
+ {1, {false, false, false, false, false}}, {7, {true, true, true, true, true}}};
+ EXPECT_TRUE(CheckUniqueActivesObservations(expected_obs, observation_store_.get(),
+ update_recipient_.get()));
+ break;
+ }
+ case 8: {
+ for (uint32_t day_index = start_day_index + 6; day_index <= start_day_index + 8;
+ day_index++) {
+ for (const auto& pair :
+ MakeNullExpectedUniqueActivesObservations(expected_params, day_index)) {
+ expected_obs.insert(pair);
+ }
+ }
+ expected_obs[{expected_id, start_day_index + 6}] = {{1, {false, false, false, true, false}},
+ {7, {true, true, true, true, true}}};
+ expected_obs[{expected_id, start_day_index + 7}] = {
+ {1, {false, false, false, false, false}}, {7, {false, true, true, true, true}}};
+ expected_obs[{expected_id, start_day_index + 8}] = {{1, {false, false, false, false, true}},
+ {7, {false, true, true, true, true}}};
+ EXPECT_TRUE(CheckUniqueActivesObservations(expected_obs, observation_store_.get(),
+ update_recipient_.get()));
+ break;
+ }
+ default:
+ EXPECT_TRUE(CheckUniqueActivesObservations(expected_obs, observation_store_.get(),
+ update_recipient_.get()));
+ }
+ }
+}
+
+// Tests GarbageCollect() for PerDeviceNumericReportAggregates.
+//
+// For each value of N in the range [0, 34], logs some events for
+// PerDeviceNumeric reports each day for N consecutive days, and then
+// garbage-collects the LocalAggregateStore. After garbage collection, verifies
+// the contents of the LocalAggregateStore.
+TEST_F(PerDeviceNumericAggregateStoreTest, GarbageCollect) {
+ uint32_t max_days_before_gc = 35;
+ for (uint32_t days_before_gc = 0; days_before_gc < max_days_before_gc; days_before_gc++) {
+ SetUp();
+ day_last_garbage_collected_ = 0u;
+ LoggedValues logged_values;
+ std::vector<MetricReportId> count_metric_report_ids = {
+ logger::testing::per_device_numeric_stats::kSettingsChangedWindowSizeMetricReportId,
+ logger::testing::per_device_numeric_stats::kSettingsChangedAggregationWindowMetricReportId,
+ logger::testing::per_device_numeric_stats::kConnectionFailuresMetricReportId};
+ std::vector<MetricReportId> elapsed_time_metric_report_ids = {
+ logger::testing::per_device_numeric_stats::kStreamingTimeTotalMetricReportId,
+ logger::testing::per_device_numeric_stats::kStreamingTimeMinMetricReportId,
+ logger::testing::per_device_numeric_stats::kStreamingTimeMaxMetricReportId};
+ MetricReportId frame_rate_metric_report_id =
+ logger::testing::per_device_numeric_stats::kLoginModuleFrameRateMinMetricReportId;
+ MetricReportId memory_usage_metric_report_id =
+ logger::testing::per_device_numeric_stats::kLedgerMemoryUsageMaxMetricReportId;
+ for (uint32_t offset = 0; offset < days_before_gc; offset++) {
+ auto day_index = CurrentDayIndex();
+ for (const auto& id : count_metric_report_ids) {
+ for (const auto& component : {"component_A", "component_B", "component_C"}) {
+ // Log 2 events with event code 0, for each component A, B, C.
+ EXPECT_EQ(kOK, LogPerDeviceCountEvent(id, day_index, component, 0u, 2, &logged_values));
+ EXPECT_EQ(kOK, LogPerDeviceCountEvent(id, day_index, component, 0u, 3, &logged_values));
+ }
+ if (offset < 3) {
+ // Log 1 event for component D and event code 1.
+ EXPECT_EQ(kOK,
+ LogPerDeviceCountEvent(id, day_index, "component_D", 1u, 4, &logged_values));
+ }
+ }
+ for (const auto& id : elapsed_time_metric_report_ids) {
+ for (const auto& component : {"component_A", "component_B", "component_C"}) {
+ // Log 2 events with event code 0, for each component A, B, C.
+ EXPECT_EQ(kOK,
+ LogPerDeviceElapsedTimeEvent(id, day_index, component, 0u, 2, &logged_values));
+ EXPECT_EQ(kOK,
+ LogPerDeviceElapsedTimeEvent(id, day_index, component, 0u, 3, &logged_values));
+ }
+ if (offset < 3) {
+ // Log 1 event for component D and event code 1.
+ EXPECT_EQ(kOK, LogPerDeviceElapsedTimeEvent(id, day_index, "component_D", 1u, 4,
+ &logged_values));
+ }
+ }
+ for (const auto& component : {"component_A", "component_B"}) {
+ EXPECT_EQ(kOK, LogPerDeviceFrameRateEvent(frame_rate_metric_report_id, day_index, component,
+ 0u, 2.25, &logged_values));
+ EXPECT_EQ(kOK, LogPerDeviceFrameRateEvent(frame_rate_metric_report_id, day_index, component,
+ 0u, 1.75, &logged_values));
+ EXPECT_EQ(kOK,
+ LogPerDeviceMemoryUsageEvent(memory_usage_metric_report_id, day_index, component,
+ std::vector<uint32_t>{0u, 0u}, 300, &logged_values));
+ EXPECT_EQ(kOK,
+ LogPerDeviceMemoryUsageEvent(memory_usage_metric_report_id, day_index, component,
+ std::vector<uint32_t>{1u, 0u}, 300, &logged_values));
+ }
+ AdvanceClock(kDay);
+ }
+ auto end_day_index = CurrentDayIndex();
+ EXPECT_EQ(kOK, GarbageCollect(end_day_index));
+ day_last_garbage_collected_ = end_day_index;
+ EXPECT_TRUE(CheckPerDeviceNumericAggregates(logged_values, end_day_index));
+ TearDown();
+ }
+}
+
+// Tests that EventAggregator::GenerateObservations() returns a positive
+// status and that the expected number of Observations is generated after
+// some CountEvents have been logged for PerDeviceNumericStats reports, without
+// any garbage collection.
+//
+// For 35 days, logs a positive number of events each day for the
+// ConnectionFailures_PerDeviceNumericStats report with "component_A" and for
+// the SettingsChanged_PerDeviceNumericStats reports with "component_B", all with
+// event code 0.
+//
+// Each day, calls GenerateObservations() with the day index of the previous
+// day. Checks that a positive status is returned and that the
+// FakeObservationStore has received the expected number of new observations
+// for each locally aggregated report ID in the per_device_numeric_stats test
+// registry.
+TEST_F(PerDeviceNumericAggregateStoreTest, GenerateObservations) {
+ int num_days = 1;
+ std::vector<Observation2> observations(0);
+ ExpectedAggregationParams expected_params =
+ logger::testing::per_device_numeric_stats::kExpectedAggregationParams;
+ for (int offset = 0; offset < num_days; offset++) {
+ auto day_index = CurrentDayIndex();
+ observations.clear();
+ ResetObservationStore();
+ EXPECT_EQ(kOK, GenerateObservations(day_index - 1));
+ EXPECT_TRUE(FetchAggregatedObservations(&observations, expected_params,
+ observation_store_.get(), update_recipient_.get()));
+ for (int i = 0; i < 2; i++) {
+ EXPECT_EQ(kOK,
+ LogPerDeviceCountEvent(
+ logger::testing::per_device_numeric_stats::kConnectionFailuresMetricReportId,
+ day_index, "component_A", 0u, 1));
+ EXPECT_EQ(kOK,
+ LogPerDeviceCountEvent(
+ logger::testing::per_device_numeric_stats::kConnectionFailuresMetricReportId,
+ day_index, "component_A", 0u, 1));
+ EXPECT_EQ(
+ kOK,
+ LogPerDeviceCountEvent(
+ logger::testing::per_device_numeric_stats::kSettingsChangedWindowSizeMetricReportId,
+ day_index, "component_B", 0u, 5));
+ EXPECT_EQ(kOK, LogPerDeviceCountEvent(logger::testing::per_device_numeric_stats::
+ kSettingsChangedAggregationWindowMetricReportId,
+ day_index, "component_B", 0u, 5));
+ }
+ // If this is the first time we're logging events, update the expected
+ // numbers of generated Observations to account for the logged events.
+ // For each report, for each aggregation window, expect 1 Observation more than if
+ // no events had been logged.
+ if (offset == 0) {
+ expected_params.daily_num_obs += 5;
+ expected_params.num_obs_per_report
+ [logger::testing::per_device_numeric_stats::kConnectionFailuresMetricReportId] += 1;
+ expected_params.num_obs_per_report
+ [logger::testing::per_device_numeric_stats::kSettingsChangedWindowSizeMetricReportId] +=
+ 2;
+ expected_params.num_obs_per_report[logger::testing::per_device_numeric_stats::
+ kSettingsChangedAggregationWindowMetricReportId] += 2;
+ }
+ AdvanceClock(kDay);
+ }
+ observations.clear();
+ ResetObservationStore();
+ EXPECT_EQ(kOK, GenerateObservations(CurrentDayIndex() - 1));
+ EXPECT_TRUE(FetchAggregatedObservations(&observations, expected_params, observation_store_.get(),
+ update_recipient_.get()));
+}
+
+// Tests that EventAggregator::GenerateObservations() returns a positive
+// status and that the expected number of Observations is generated after
+// some CountEvents have been logged for PerDeviceNumeric reports over multiple
+// days, and when the LocalAggregateStore is garbage-collected each day.
+//
+// For 35 days, logs a positive number of events each day for the
+// ConnectionFailures_PerDeviceNumeric report with "component_A" and for
+// the SettingsChanged_PerDeviceNumeric report with "component_B", all with
+// event code 0.
+//
+// Each day, calls GenerateObservations() with the day index of the previous
+// day. Checks that a positive status is returned and that the
+// FakeObservationStore has received the expected number of new observations
+// for each locally aggregated report ID in the per_device_numeric_stats test
+// registry.
+TEST_F(PerDeviceNumericAggregateStoreTest, GenerateObservationsWithGc) {
+ int num_days = 35;
+ std::vector<Observation2> observations(0);
+ ExpectedAggregationParams expected_params =
+ logger::testing::per_device_numeric_stats::kExpectedAggregationParams;
+ for (int offset = 0; offset < num_days; offset++) {
+ auto day_index = CurrentDayIndex();
+ observations.clear();
+ ResetObservationStore();
+ EXPECT_EQ(kOK, GenerateObservations(day_index - 1));
+ EXPECT_TRUE(FetchAggregatedObservations(&observations, expected_params,
+ observation_store_.get(), update_recipient_.get()));
+ EXPECT_EQ(kOK, GarbageCollect(day_index));
+ for (int i = 0; i < 2; i++) {
+ EXPECT_EQ(kOK,
+ LogPerDeviceCountEvent(
+ logger::testing::per_device_numeric_stats::kConnectionFailuresMetricReportId,
+ day_index, "component_A", 0u, 1));
+ EXPECT_EQ(kOK,
+ LogPerDeviceCountEvent(
+ logger::testing::per_device_numeric_stats::kConnectionFailuresMetricReportId,
+ day_index, "component_A", 0u, 1));
+ EXPECT_EQ(
+ kOK,
+ LogPerDeviceCountEvent(
+ logger::testing::per_device_numeric_stats::kSettingsChangedWindowSizeMetricReportId,
+ day_index, "component_B", 0u, 5));
+ EXPECT_EQ(kOK, LogPerDeviceCountEvent(logger::testing::per_device_numeric_stats::
+ kSettingsChangedAggregationWindowMetricReportId,
+ day_index, "component_B", 0u, 5));
+ }
+ // If this is the first time we're logging events, update the expected
+ // numbers of generated Observations to account for the logged events.
+ // For each report, for each window size, expect 1 Observation more than if
+ // no events had been logged.
+ if (offset == 0) {
+ expected_params.daily_num_obs += 5;
+ expected_params.num_obs_per_report
+ [logger::testing::per_device_numeric_stats::kConnectionFailuresMetricReportId] += 1;
+ expected_params.num_obs_per_report
+ [logger::testing::per_device_numeric_stats::kSettingsChangedWindowSizeMetricReportId] +=
+ 2;
+ expected_params.num_obs_per_report[logger::testing::per_device_numeric_stats::
+ kSettingsChangedAggregationWindowMetricReportId] += 2;
+ }
+ AdvanceClock(kDay);
+ }
+ observations.clear();
+ ResetObservationStore();
+ auto day_index = CurrentDayIndex();
+ EXPECT_EQ(kOK, GenerateObservations(day_index - 1));
+ EXPECT_TRUE(FetchAggregatedObservations(&observations, expected_params, observation_store_.get(),
+ update_recipient_.get()));
+ EXPECT_EQ(kOK, GarbageCollect(day_index));
+}
+
+// Tests that GenerateObservations() returns a positive status and that the
+// expected number of Observations is generated when events are logged over
+// multiple days and some of those days' Observations are backfilled, without
+// any garbage collection of the LocalAggregateStore.
+//
+// Sets the |backfill_days_| field of the EventAggregator to 3.
+//
+// Logging pattern:
+// For 35 days, logs 2 events each day for the
+// ConnectionFailures_PerDeviceCount report and 2 events for the
+// SettingsChanged_PerDeviceCount report, all with event code 0.
+//
+// Observation generation pattern:
+// Calls GenerateObservations() on the 1st through 5th and the 7th out of
+// every 10 days, for 35 days.
+//
+// Expected numbers of Observations:
+// It is expected that 4 days' worth of Observations are generated on
+// the first day of every 10 (the day index for which GenerateObservations()
+// was called, plus 3 days of backfill), that 1 day's worth of Observations
+// are generated on the 2nd through 5th day of every 10, that 2 days'
+// worth of Observations are generated on the 7th day of every 10 (the
+// day index for which GenerateObservations() was called, plus 1 day of
+// backfill), and that no Observations are generated on the remaining days.
+TEST_F(PerDeviceNumericAggregateStoreTest, GenerateObservationsWithBackfill) {
+ const auto& expected_params =
+ logger::testing::per_device_numeric_stats::kExpectedAggregationParams;
+ // Set |backfill_days_| to 3.
+ size_t backfill_days = 3;
+ SetBackfillDays(backfill_days);
+ // Log 2 events each day for 35 days. Call GenerateObservations() on the
+ // first 5 day indices, and the 7th, out of every 10.
+ for (int offset = 0; offset < 35; offset++) {
+ auto day_index = CurrentDayIndex();
+ for (int i = 0; i < 2; i++) {
+ EXPECT_EQ(kOK,
+ LogPerDeviceCountEvent(
+ logger::testing::per_device_numeric_stats::kConnectionFailuresMetricReportId,
+ day_index, "component_A", 0u, 1));
+ EXPECT_EQ(kOK,
+ LogPerDeviceCountEvent(
+ logger::testing::per_device_numeric_stats::kConnectionFailuresMetricReportId,
+ day_index, "component_A", 0u, 1));
+ EXPECT_EQ(
+ kOK,
+ LogPerDeviceCountEvent(
+ logger::testing::per_device_numeric_stats::kSettingsChangedWindowSizeMetricReportId,
+ day_index, "component_B", 0u, 5));
+ EXPECT_EQ(kOK, LogPerDeviceCountEvent(logger::testing::per_device_numeric_stats::
+ kSettingsChangedAggregationWindowMetricReportId,
+ day_index, "component_B", 0u, 5));
+ }
+ auto num_obs_before = observation_store_->messages_received.size();
+ if (offset % 10 < 5 || offset % 10 == 6) {
+ EXPECT_EQ(kOK, GenerateObservations(day_index));
+ }
+ auto num_obs_after = observation_store_->messages_received.size();
+ EXPECT_GE(num_obs_after, num_obs_before);
+ // Check that the expected daily number of Observations was generated.
+ switch (offset % 10) {
+ case 0:
+ // If this is the first day of logging, expect 3 Observations for each
+ // day in the backfill period and 8 Observations for the current day.
+ if (offset == 0) {
+ EXPECT_EQ(
+ (expected_params.daily_num_obs * backfill_days) + expected_params.daily_num_obs + 5,
+ num_obs_after - num_obs_before);
+ } else {
+ // If this is another day whose offset is a multiple of 10, expect 8
+ // Observations for each day in the backfill period as well as the
+ // current day.
+ EXPECT_EQ((expected_params.daily_num_obs + 5) * (backfill_days + 1),
+ num_obs_after - num_obs_before);
+ }
+ break;
+ case 1:
+ case 2:
+ case 3:
+ case 4:
+ // Expect 8 Observations for this day.
+ EXPECT_EQ(expected_params.daily_num_obs + 5, num_obs_after - num_obs_before);
+ break;
+ case 6:
+ // Expect 8 Observations for each of today and yesterday.
+ EXPECT_EQ((expected_params.daily_num_obs + 5) * 2, num_obs_after - num_obs_before);
+ break;
+ default:
+ EXPECT_EQ(num_obs_after, num_obs_before);
+ }
+ AdvanceClock(kDay);
+ }
+}
+
+// Tests that GenerateObservations() returns a positive status and that the
+// expected number of Observations is generated when events are logged over
+// multiple days and some of those days' Observations are backfilled, and when
+// the LocalAggregateStore is garbage-collected after each call to
+// GenerateObservations().
+//
+// Sets the |backfill_days_| field of the EventAggregator to 3.
+//
+// Logging pattern:
+// For 35 days, logs 2 events each day for the
+// ConnectionFailures_PerDeviceNumeric report with "component_A" and 2 events
+// for the SettingsChanged_PerDeviceNumeric reports with "component_B", all with
+// event code 0.
+//
+// Observation generation pattern:
+// Calls GenerateObservations() on the 1st through 5th and the 7th out of
+// every 10 days, for 35 days. Garbage-collects the LocalAggregateStore after
+// each call.
+//
+// Expected numbers of Observations:
+// It is expected that 4 days' worth of Observations are generated on
+// the first day of every 10 (the day index for which GenerateObservations()
+// was called, plus 3 days of backfill), that 1 day's worth of Observations
+// are generated on the 2nd through 5th day of every 10, that 2 days'
+// worth of Observations are generated on the 7th day of every 10 (the
+// day index for which GenerateObservations() was called, plus 1 day of
+// backfill), and that no Observations are generated on the remaining days.
+TEST_F(PerDeviceNumericAggregateStoreTest, GenerateObservationsWithBackfillAndGc) {
+ int num_days = 35;
+ const auto& expected_params =
+ logger::testing::per_device_numeric_stats::kExpectedAggregationParams;
+ // Set |backfill_days_| to 3.
+ size_t backfill_days = 3;
+ SetBackfillDays(backfill_days);
+ // Log 2 events each day for 35 days. Call GenerateObservations() on the
+ // first 5 day indices, and the 7th, out of every 10.
+ for (int offset = 0; offset < num_days; offset++) {
+ auto day_index = CurrentDayIndex();
+ for (int i = 0; i < 2; i++) {
+ EXPECT_EQ(kOK,
+ LogPerDeviceCountEvent(
+ logger::testing::per_device_numeric_stats::kConnectionFailuresMetricReportId,
+ day_index, "component_A", 0u, 1));
+ EXPECT_EQ(
+ kOK,
+ LogPerDeviceCountEvent(
+ logger::testing::per_device_numeric_stats::kSettingsChangedWindowSizeMetricReportId,
+ day_index, "component_B", 0u, 5));
+ EXPECT_EQ(kOK, LogPerDeviceCountEvent(logger::testing::per_device_numeric_stats::
+ kSettingsChangedAggregationWindowMetricReportId,
+ day_index, "component_B", 0u, 5));
+ }
+ auto num_obs_before = observation_store_->messages_received.size();
+ if (offset % 10 < 5 || offset % 10 == 6) {
+ EXPECT_EQ(kOK, GenerateObservations(day_index));
+ EXPECT_EQ(kOK, GarbageCollect(day_index));
+ }
+ auto num_obs_after = observation_store_->messages_received.size();
+ EXPECT_GE(num_obs_after, num_obs_before);
+ // Check that the expected daily number of Observations was generated.
+ switch (offset % 10) {
+ case 0:
+ // If this is the first day of logging, expect 3 Observations for each
+ // day in the backfill period and 8 Observations for the current day.
+ if (offset == 0) {
+ EXPECT_EQ(
+ (expected_params.daily_num_obs * backfill_days) + expected_params.daily_num_obs + 5,
+ num_obs_after - num_obs_before);
+ } else {
+ // If this is another day whose offset is a multiple of 10, expect 8
+ // Observations for each day in the backfill period as well as the
+ // current day.
+ EXPECT_EQ((expected_params.daily_num_obs + 5) * (backfill_days + 1),
+ num_obs_after - num_obs_before);
+ }
+ break;
+ case 1:
+ case 2:
+ case 3:
+ case 4:
+ // Expect 8 Observations for this day.
+ EXPECT_EQ(expected_params.daily_num_obs + 5, num_obs_after - num_obs_before);
+ break;
+ case 6:
+ // Expect 6 Observations for each of today and yesterday.
+ EXPECT_EQ((expected_params.daily_num_obs + 5) * 2, num_obs_after - num_obs_before);
+ break;
+ default:
+ EXPECT_EQ(num_obs_after, num_obs_before);
+ }
+ AdvanceClock(kDay);
+ }
+}
+
+// Generate Observations without logging any events, and check that the
+// resulting Observations are as expected: 1 ReportParticipationObservation for
+// each PER_DEVICE_NUMERIC_STATS report in the config, and no
+// PerDeviceNumericObservations.
+TEST_F(PerDeviceNumericAggregateStoreTest, CheckObservationValuesNoEvents) {
+ const auto current_day_index = CurrentDayIndex();
+ EXPECT_EQ(kOK, GenerateObservations(current_day_index));
+ const auto& expected_report_participation_obs = MakeExpectedReportParticipationObservations(
+ logger::testing::per_device_numeric_stats::kExpectedAggregationParams, current_day_index);
+ EXPECT_TRUE(CheckPerDeviceNumericObservations({}, expected_report_participation_obs,
+ observation_store_.get(), update_recipient_.get()));
+}
+
+// Check that the expected PerDeviceNumericObservations and
+// ReportParticipationObservations are generated when GenerateObservations() is
+// called after logging some CountEvents and ElapsedTimeEvents for
+// PER_DEVICE_NUMERIC_STATS reports over a single day index.
+TEST_F(PerDeviceNumericAggregateStoreTest, CheckObservationValuesSingleDay) {
+ const auto day_index = CurrentDayIndex();
+ // Log several events on |day_index|.
+ EXPECT_EQ(kOK, LogPerDeviceCountEvent(
+ logger::testing::per_device_numeric_stats::kConnectionFailuresMetricReportId,
+ day_index, "component_A", 0u, 5));
+ EXPECT_EQ(kOK, LogPerDeviceCountEvent(
+ logger::testing::per_device_numeric_stats::kConnectionFailuresMetricReportId,
+ day_index, "component_B", 0u, 5));
+ EXPECT_EQ(kOK, LogPerDeviceCountEvent(
+ logger::testing::per_device_numeric_stats::kConnectionFailuresMetricReportId,
+ day_index, "component_A", 0u, 5));
+ EXPECT_EQ(kOK, LogPerDeviceCountEvent(
+ logger::testing::per_device_numeric_stats::kConnectionFailuresMetricReportId,
+ day_index, "component_A", 1u, 5));
+ EXPECT_EQ(kOK,
+ LogPerDeviceCountEvent(
+ logger::testing::per_device_numeric_stats::kSettingsChangedWindowSizeMetricReportId,
+ day_index, "component_C", 0u, 5));
+ EXPECT_EQ(kOK,
+ LogPerDeviceCountEvent(
+ logger::testing::per_device_numeric_stats::kSettingsChangedWindowSizeMetricReportId,
+ day_index, "component_C", 0u, 5));
+ EXPECT_EQ(kOK, LogPerDeviceCountEvent(logger::testing::per_device_numeric_stats::
+ kSettingsChangedAggregationWindowMetricReportId,
+ day_index, "component_C", 0u, 5));
+ EXPECT_EQ(kOK, LogPerDeviceCountEvent(logger::testing::per_device_numeric_stats::
+ kSettingsChangedAggregationWindowMetricReportId,
+ day_index, "component_C", 0u, 5));
+
+ std::vector<MetricReportId> streaming_time_ids = {
+ logger::testing::per_device_numeric_stats::kStreamingTimeTotalMetricReportId,
+ logger::testing::per_device_numeric_stats::kStreamingTimeMinMetricReportId,
+ logger::testing::per_device_numeric_stats::kStreamingTimeMaxMetricReportId};
+ for (const auto& id : streaming_time_ids) {
+ EXPECT_EQ(kOK, LogPerDeviceElapsedTimeEvent(id, day_index, "component_D", 0u, 15));
+ EXPECT_EQ(kOK, LogPerDeviceElapsedTimeEvent(id, day_index, "component_D", 1u, 5));
+ EXPECT_EQ(kOK, LogPerDeviceElapsedTimeEvent(id, day_index, "component_D", 0u, 10));
+ }
+ // Generate locally aggregated Observations for |day_index|.
+ EXPECT_EQ(kOK, GenerateObservations(day_index));
+
+ // Form the expected Observations.
+ auto expected_report_participation_obs = MakeExpectedReportParticipationObservations(
+ logger::testing::per_device_numeric_stats::kExpectedAggregationParams, day_index);
+ ExpectedPerDeviceNumericObservations expected_per_device_numeric_obs;
+ expected_per_device_numeric_obs[{
+ logger::testing::per_device_numeric_stats::kConnectionFailuresMetricReportId, day_index}][1] =
+ {{"component_A", 0u, 10}, {"component_A", 1u, 5}, {"component_B", 0u, 5}};
+ expected_per_device_numeric_obs[{
+ logger::testing::per_device_numeric_stats::kSettingsChangedWindowSizeMetricReportId,
+ day_index}][7] = {{"component_C", 0u, 10}};
+ expected_per_device_numeric_obs[{
+ logger::testing::per_device_numeric_stats::kSettingsChangedWindowSizeMetricReportId,
+ day_index}][30] = {{"component_C", 0u, 10}};
+ expected_per_device_numeric_obs[{
+ logger::testing::per_device_numeric_stats::kSettingsChangedAggregationWindowMetricReportId,
+ day_index}][7] = {{"component_C", 0u, 10}};
+ expected_per_device_numeric_obs[{
+ logger::testing::per_device_numeric_stats::kSettingsChangedAggregationWindowMetricReportId,
+ day_index}][30] = {{"component_C", 0u, 10}};
+ expected_per_device_numeric_obs[{
+ logger::testing::per_device_numeric_stats::kStreamingTimeTotalMetricReportId, day_index}][1] =
+ {{"component_D", 0u, 25}, {"component_D", 1u, 5}};
+ expected_per_device_numeric_obs[{
+ logger::testing::per_device_numeric_stats::kStreamingTimeTotalMetricReportId, day_index}][7] =
+ {{"component_D", 0u, 25}, {"component_D", 1u, 5}};
+ // The 7-day minimum value for the StreamingTime metric is 0 for all event
+ // codes and components, so we don't expect a PerDeviceNumericObservation with
+ // a 7-day window for the StreamingTime_PerDeviceMin report.
+ expected_per_device_numeric_obs[{
+ logger::testing::per_device_numeric_stats::kStreamingTimeMinMetricReportId, day_index}][1] = {
+ {"component_D", 0u, 10}, {"component_D", 1u, 5}};
+ expected_per_device_numeric_obs[{
+ logger::testing::per_device_numeric_stats::kStreamingTimeMinMetricReportId, day_index}][7] = {
+ {"component_D", 0u, 10}, {"component_D", 1u, 5}};
+ expected_per_device_numeric_obs[{
+ logger::testing::per_device_numeric_stats::kStreamingTimeMaxMetricReportId, day_index}][1] = {
+ {"component_D", 0u, 15}, {"component_D", 1u, 5}};
+ expected_per_device_numeric_obs[{
+ logger::testing::per_device_numeric_stats::kStreamingTimeMaxMetricReportId, day_index}][7] = {
+ {"component_D", 0u, 15}, {"component_D", 1u, 5}};
+
+ EXPECT_TRUE(CheckPerDeviceNumericObservations(expected_per_device_numeric_obs,
+ expected_report_participation_obs,
+ observation_store_.get(), update_recipient_.get()));
+}
+
+// Checks that PerDeviceNumericObservations with the expected values are
+// generated when some events have been logged for an EVENT_COUNT metric with
+// a PER_DEVICE_NUMERIC_STATS report over multiple days and
+// GenerateObservations() is called each day, without garbage collection or
+// backfill.
+//
+// Logged events for the SettingsChanged_PerDeviceCount report on the i-th
+// day:
+//
+// i (component, event code, count)
+// -----------------------------------------------------------------------
+// 0
+// 1 ("A", 1, 3)
+// 2 ("A", 1, 3), ("A", 2, 3), ("B", 1, 2)
+// 3 ("A", 1, 3)
+// 4 ("A", 1, 3), ("A", 2, 3), ("B", 1, 2), ("B", 2, 2)
+// 5 ("A", 1, 3)
+// 6 ("A", 1, 3), ("A", 2, 3), ("B", 1, 2)
+// 7 ("A", 1, 3)
+// 8 ("A", 1, 3), ("A", 2, 3), ("B", 1, 2), ("B", 2, 2)
+// 9 ("A", 1, 3)
+//
+// Expected PerDeviceNumericObservations for the
+// SettingsChanged_PerDeviceNumeric report on the i-th day:
+//
+// (i, window size) (component, event code, count)
+// -----------------------------------------------------------------------
+// (0, 7)
+// (0, 30)
+// (1, 7) ("A", 1, 3)
+// (1, 30) ("A", 1, 3)
+// (2, 7) ("A", 1, 6), ("A", 2, 3), ("B", 1, 2)
+// (2, 30) ("A", 1, 6), ("A", 2, 3), ("B", 1, 2)
+// (3, 7) ("A", 1, 9), ("A", 2, 3), ("B", 1, 2)
+// (3, 30) ("A", 1, 9), ("A", 2, 3), ("B", 1, 2)
+// (4, 7) ("A", 1, 12), ("A", 2, 6), ("B", 1, 4), ("B", 2, 2)
+// (4, 30) ("A", 1, 12), ("A", 2, 6), ("B", 1, 4), ("B", 2, 2)
+// (5, 7) ("A", 1, 15), ("A", 2, 6), ("B", 1, 4), ("B", 2, 2)
+// (5, 30) ("A", 1, 15), ("A", 2, 6), ("B", 1, 4), ("B", 2, 2)
+// (6, 7) ("A", 1, 18), ("A", 2, 9), ("B", 1, 6), ("B", 2, 2)
+// (6, 30) ("A", 1, 18), ("A", 2, 9), ("B", 1, 6), ("B", 2, 2)
+// (7, 7) ("A", 1, 21), ("A", 2, 9), ("B", 1, 6), ("B", 2, 2)
+// (7, 30) ("A", 1, 21), ("A", 2, 9), ("B", 1, 6), ("B", 2, 2)
+// (8, 7) ("A", 1, 21), ("A", 2, 12), ("B", 1, 8), ("B", 2, 4)
+// (8, 30) ("A", 1, 24), ("A", 2, 12), ("B", 1, 8), ("B", 2, 4)
+// (9, 7) ("A", 1, 21), ("A", 2, 9), ("B", 1, 6), ("B", 2, 4)
+// (9, 30) ("A", 1, 27), ("A", 2, 12), ("B", 1, 8), ("B", 2, 4)
+//
+// In addition, expect 1 ReportParticipationObservation each day for each of
+// the reports in the registry.
+TEST_F(PerDeviceNumericAggregateStoreTest, CheckObservationValuesMultiDay) {
+ auto start_day_index = CurrentDayIndex();
+ const auto& expected_id =
+ logger::testing::per_device_numeric_stats::kSettingsChangedAggregationWindowMetricReportId;
+ const auto& expected_params =
+ logger::testing::per_device_numeric_stats::kExpectedAggregationParams;
+ // Form expected Observations for the 10 days of logging.
+ uint32_t num_days = 10;
+ std::vector<ExpectedPerDeviceNumericObservations> expected_per_device_numeric_obs(num_days);
+ std::vector<ExpectedReportParticipationObservations> expected_report_participation_obs(num_days);
+ for (uint32_t offset = 0; offset < num_days; offset++) {
+ expected_report_participation_obs[offset] =
+ MakeExpectedReportParticipationObservations(expected_params, start_day_index + offset);
+ }
+ expected_per_device_numeric_obs[0] = {};
+ expected_per_device_numeric_obs[1][{expected_id, start_day_index + 1}] = {{7, {{"A", 1u, 3}}},
+ {30, {{"A", 1u, 3}}}};
+ expected_per_device_numeric_obs[2][{expected_id, start_day_index + 2}] = {
+ {7, {{"A", 1u, 6}, {"A", 2u, 3}, {"B", 1u, 2}}},
+ {30, {{"A", 1u, 6}, {"A", 2u, 3}, {"B", 1u, 2}}}};
+ expected_per_device_numeric_obs[3][{expected_id, start_day_index + 3}] = {
+ {7, {{"A", 1u, 9}, {"A", 2u, 3}, {"B", 1u, 2}}},
+ {30, {{"A", 1u, 9}, {"A", 2u, 3}, {"B", 1u, 2}}}};
+ expected_per_device_numeric_obs[4][{expected_id, start_day_index + 4}] = {
+ {7, {{"A", 1u, 12}, {"A", 2u, 6}, {"B", 1u, 4}, {"B", 2u, 2}}},
+ {30, {{"A", 1u, 12}, {"A", 2u, 6}, {"B", 1u, 4}, {"B", 2u, 2}}}};
+ expected_per_device_numeric_obs[5][{expected_id, start_day_index + 5}] = {
+ {7, {{"A", 1u, 15}, {"A", 2u, 6}, {"B", 1u, 4}, {"B", 2u, 2}}},
+ {30, {{"A", 1u, 15}, {"A", 2u, 6}, {"B", 1u, 4}, {"B", 2u, 2}}}};
+ expected_per_device_numeric_obs[6][{expected_id, start_day_index + 6}] = {
+ {7, {{"A", 1u, 18}, {"A", 2u, 9}, {"B", 1u, 6}, {"B", 2u, 2}}},
+ {30, {{"A", 1u, 18}, {"A", 2u, 9}, {"B", 1u, 6}, {"B", 2u, 2}}}};
+ expected_per_device_numeric_obs[7][{expected_id, start_day_index + 7}] = {
+ {7, {{"A", 1u, 21}, {"A", 2u, 9}, {"B", 1u, 6}, {"B", 2u, 2}}},
+ {30, {{"A", 1u, 21}, {"A", 2u, 9}, {"B", 1u, 6}, {"B", 2u, 2}}}};
+ expected_per_device_numeric_obs[8][{expected_id, start_day_index + 8}] = {
+ {7, {{"A", 1u, 21}, {"A", 2u, 12}, {"B", 1u, 8}, {"B", 2u, 4}}},
+ {30, {{"A", 1u, 24}, {"A", 2u, 12}, {"B", 1u, 8}, {"B", 2u, 4}}}};
+ expected_per_device_numeric_obs[9][{expected_id, start_day_index + 9}] = {
+ {7, {{"A", 1u, 21}, {"A", 2u, 9}, {"B", 1u, 6}, {"B", 2u, 4}}},
+ {30, {{"A", 1u, 27}, {"A", 2u, 12}, {"B", 1u, 8}, {"B", 2u, 4}}}};
+
+ for (uint32_t offset = 0; offset < 1; offset++) {
+ auto day_index = CurrentDayIndex();
+ for (uint32_t event_code = 1; event_code < 3; event_code++) {
+ if (offset > 0 && offset % event_code == 0) {
+ EXPECT_EQ(kOK, LogPerDeviceCountEvent(expected_id, day_index, "A", event_code, 3));
+ }
+ if (offset > 0 && offset % (2 * event_code) == 0) {
+ EXPECT_EQ(kOK, LogPerDeviceCountEvent(expected_id, day_index, "B", event_code, 2));
+ }
+ }
+ // Clear the FakeObservationStore.
+ ResetObservationStore();
+ // Generate locally aggregated Observations.
+ EXPECT_EQ(kOK, GenerateObservations(day_index));
+ EXPECT_TRUE(CheckPerDeviceNumericObservations(
+ expected_per_device_numeric_obs[offset], expected_report_participation_obs[offset],
+ observation_store_.get(), update_recipient_.get()))
+ << "offset = " << offset;
+ AdvanceClock(kDay);
+ }
+}
+
+// Repeat the CheckObservationValuesMultiDay test, this time calling
+// GarbageCollect() after each call to GenerateObservations.
+//
+// The logging pattern and set of Observations for each day index is the same
+// as in PerDeviceNumericAggregateStoreTest::CheckObservationValuesMultiDay.
+// See that test for documentation.
+TEST_F(PerDeviceNumericAggregateStoreTest, CheckObservationValuesMultiDayWithGarbageCollection) {
+ auto start_day_index = CurrentDayIndex();
+ const auto& expected_id =
+ logger::testing::per_device_numeric_stats::kSettingsChangedAggregationWindowMetricReportId;
+ const auto& expected_params =
+ logger::testing::per_device_numeric_stats::kExpectedAggregationParams;
+ // Form expected Observations for the 10 days of logging.
+ uint32_t num_days = 10;
+ std::vector<ExpectedPerDeviceNumericObservations> expected_per_device_numeric_obs(num_days);
+ std::vector<ExpectedReportParticipationObservations> expected_report_participation_obs(num_days);
+ for (uint32_t offset = 0; offset < num_days; offset++) {
+ expected_report_participation_obs[offset] =
+ MakeExpectedReportParticipationObservations(expected_params, start_day_index + offset);
+ }
+ expected_per_device_numeric_obs[0] = {};
+ expected_per_device_numeric_obs[1][{expected_id, start_day_index + 1}] = {{7, {{"A", 1u, 3}}},
+ {30, {{"A", 1u, 3}}}};
+ expected_per_device_numeric_obs[2][{expected_id, start_day_index + 2}] = {
+ {7, {{"A", 1u, 6}, {"A", 2u, 3}, {"B", 1u, 2}}},
+ {30, {{"A", 1u, 6}, {"A", 2u, 3}, {"B", 1u, 2}}}};
+ expected_per_device_numeric_obs[3][{expected_id, start_day_index + 3}] = {
+ {7, {{"A", 1u, 9}, {"A", 2u, 3}, {"B", 1u, 2}}},
+ {30, {{"A", 1u, 9}, {"A", 2u, 3}, {"B", 1u, 2}}}};
+ expected_per_device_numeric_obs[4][{expected_id, start_day_index + 4}] = {
+ {7, {{"A", 1u, 12}, {"A", 2u, 6}, {"B", 1u, 4}, {"B", 2u, 2}}},
+ {30, {{"A", 1u, 12}, {"A", 2u, 6}, {"B", 1u, 4}, {"B", 2u, 2}}}};
+ expected_per_device_numeric_obs[5][{expected_id, start_day_index + 5}] = {
+ {7, {{"A", 1u, 15}, {"A", 2u, 6}, {"B", 1u, 4}, {"B", 2u, 2}}},
+ {30, {{"A", 1u, 15}, {"A", 2u, 6}, {"B", 1u, 4}, {"B", 2u, 2}}}};
+ expected_per_device_numeric_obs[6][{expected_id, start_day_index + 6}] = {
+ {7, {{"A", 1u, 18}, {"A", 2u, 9}, {"B", 1u, 6}, {"B", 2u, 2}}},
+ {30, {{"A", 1u, 18}, {"A", 2u, 9}, {"B", 1u, 6}, {"B", 2u, 2}}}};
+ expected_per_device_numeric_obs[7][{expected_id, start_day_index + 7}] = {
+ {7, {{"A", 1u, 21}, {"A", 2u, 9}, {"B", 1u, 6}, {"B", 2u, 2}}},
+ {30, {{"A", 1u, 21}, {"A", 2u, 9}, {"B", 1u, 6}, {"B", 2u, 2}}}};
+ expected_per_device_numeric_obs[8][{expected_id, start_day_index + 8}] = {
+ {7, {{"A", 1u, 21}, {"A", 2u, 12}, {"B", 1u, 8}, {"B", 2u, 4}}},
+ {30, {{"A", 1u, 24}, {"A", 2u, 12}, {"B", 1u, 8}, {"B", 2u, 4}}}};
+ expected_per_device_numeric_obs[9][{expected_id, start_day_index + 9}] = {
+ {7, {{"A", 1u, 21}, {"A", 2u, 9}, {"B", 1u, 6}, {"B", 2u, 4}}},
+ {30, {{"A", 1u, 27}, {"A", 2u, 12}, {"B", 1u, 8}, {"B", 2u, 4}}}};
+
+ for (uint32_t offset = 0; offset < 10; offset++) {
+ auto day_index = CurrentDayIndex();
+ for (uint32_t event_code = 1; event_code < 3; event_code++) {
+ if (offset > 0 && offset % event_code == 0) {
+ EXPECT_EQ(kOK, LogPerDeviceCountEvent(expected_id, day_index, "A", event_code, 3));
+ }
+ if (offset > 0 && offset % (2 * event_code) == 0) {
+ EXPECT_EQ(kOK, LogPerDeviceCountEvent(expected_id, day_index, "B", event_code, 2));
+ }
+ }
+ // Advance |test_clock_| by 1 day.
+ AdvanceClock(kDay);
+ // Clear the FakeObservationStore.
+ ResetObservationStore();
+ // Generate locally aggregated Observations and garbage-collect the
+ // LocalAggregateStore, both for the previous day as measured by
+ // |test_clock_|. Back up the LocalAggregateStore and
+ // AggregatedObservationHistoryStore.
+ DoScheduledTasksNow();
+ EXPECT_TRUE(CheckPerDeviceNumericObservations(
+ expected_per_device_numeric_obs[offset], expected_report_participation_obs[offset],
+ observation_store_.get(), update_recipient_.get()));
+ }
+}
+
+// Tests that the expected PerDeviceNumericObservations are generated when
+// events are logged over multiple days for an EVENT_COUNT
+// metric with a PER_DEVICE_NUMERIC_STATS report, when Observations are
+// backfilled for some days during that period, without any garbage-collection
+// of the LocalAggregateStore.
+//
+// The logging pattern and set of Observations for each day index is the same
+// as in PerDeviceNumericAggregateStoreTest::CheckObservationValuesMultiDay.
+// See that test for documentation.
+TEST_F(PerDeviceNumericAggregateStoreTest, CheckObservationValuesWithBackfill) {
+ auto start_day_index = CurrentDayIndex();
+ const auto& expected_id =
+ logger::testing::per_device_numeric_stats::kSettingsChangedAggregationWindowMetricReportId;
+ const auto& expected_params =
+ logger::testing::per_device_numeric_stats::kExpectedAggregationParams;
+ // Set |backfill_days_| to 3.
+ size_t backfill_days = 3;
+ SetBackfillDays(backfill_days);
+ // Log events for 9 days. Call GenerateObservations() on the first 6 day
+ // indices, and the 9th.
+ uint32_t num_days = 9;
+ for (uint32_t offset = 0; offset < num_days; offset++) {
+ auto day_index = CurrentDayIndex();
+ ResetObservationStore();
+ for (uint32_t event_code = 1; event_code < 3; event_code++) {
+ if (offset > 0 && (offset % event_code == 0)) {
+ EXPECT_EQ(kOK, LogPerDeviceCountEvent(expected_id, day_index, "A", event_code, 3));
+ }
+ if (offset > 0 && offset % (2 * event_code) == 0) {
+ EXPECT_EQ(kOK, LogPerDeviceCountEvent(expected_id, day_index, "B", event_code, 2));
+ }
+ }
+ if (offset < 6 || offset == 8) {
+ EXPECT_EQ(kOK, GenerateObservations(day_index));
+ }
+ // Make the set of Observations which are expected to be generated on
+ // |start_day_index + offset| and check it against the contents of the
+ // FakeObservationStore.
+ ExpectedPerDeviceNumericObservations expected_per_device_numeric_obs;
+ ExpectedReportParticipationObservations expected_report_participation_obs;
+ switch (offset) {
+ case 0: {
+ for (uint32_t day_index = start_day_index - backfill_days; day_index <= start_day_index;
+ day_index++) {
+ for (const auto& pair :
+ MakeExpectedReportParticipationObservations(expected_params, day_index)) {
+ expected_report_participation_obs.insert(pair);
+ }
+ }
+ EXPECT_TRUE(CheckPerDeviceNumericObservations(
+ expected_per_device_numeric_obs, expected_report_participation_obs,
+ observation_store_.get(), update_recipient_.get()));
+ break;
+ }
+ case 1: {
+ expected_per_device_numeric_obs[{expected_id, day_index}] = {{7, {{"A", 1u, 3}}},
+ {30, {{"A", 1u, 3}}}};
+ expected_report_participation_obs =
+ MakeExpectedReportParticipationObservations(expected_params, day_index);
+ EXPECT_TRUE(CheckPerDeviceNumericObservations(
+ expected_per_device_numeric_obs, expected_report_participation_obs,
+ observation_store_.get(), update_recipient_.get()));
+ break;
+ }
+ case 2: {
+ expected_per_device_numeric_obs[{expected_id, day_index}] = {
+ {7, {{"A", 1u, 6}, {"A", 2u, 3}, {"B", 1u, 2}}},
+ {30, {{"A", 1u, 6}, {"A", 2u, 3}, {"B", 1u, 2}}}};
+ expected_report_participation_obs =
+ MakeExpectedReportParticipationObservations(expected_params, day_index);
+ EXPECT_TRUE(CheckPerDeviceNumericObservations(
+ expected_per_device_numeric_obs, expected_report_participation_obs,
+ observation_store_.get(), update_recipient_.get()));
+ break;
+ }
+ case 3: {
+ expected_per_device_numeric_obs[{expected_id, day_index}] = {
+ {7, {{"A", 1u, 9}, {"A", 2u, 3}, {"B", 1u, 2}}},
+ {30, {{"A", 1u, 9}, {"A", 2u, 3}, {"B", 1u, 2}}}};
+ expected_report_participation_obs =
+ MakeExpectedReportParticipationObservations(expected_params, day_index);
+ EXPECT_TRUE(CheckPerDeviceNumericObservations(
+ expected_per_device_numeric_obs, expected_report_participation_obs,
+ observation_store_.get(), update_recipient_.get()));
+ break;
+ }
+ case 4: {
+ expected_per_device_numeric_obs[{expected_id, day_index}] = {
+ {7, {{"A", 1u, 12}, {"A", 2u, 6}, {"B", 1u, 4}, {"B", 2u, 2}}},
+ {30, {{"A", 1u, 12}, {"A", 2u, 6}, {"B", 1u, 4}, {"B", 2u, 2}}}};
+ expected_report_participation_obs =
+ MakeExpectedReportParticipationObservations(expected_params, day_index);
+ EXPECT_TRUE(CheckPerDeviceNumericObservations(
+ expected_per_device_numeric_obs, expected_report_participation_obs,
+ observation_store_.get(), update_recipient_.get()));
+ break;
+ }
+ case 5: {
+ expected_per_device_numeric_obs[{expected_id, day_index}] = {
+ {7, {{"A", 1u, 15}, {"A", 2u, 6}, {"B", 1u, 4}, {"B", 2u, 2}}},
+ {30, {{"A", 1u, 15}, {"A", 2u, 6}, {"B", 1u, 4}, {"B", 2u, 2}}}};
+ expected_report_participation_obs =
+ MakeExpectedReportParticipationObservations(expected_params, day_index);
+ EXPECT_TRUE(CheckPerDeviceNumericObservations(
+ expected_per_device_numeric_obs, expected_report_participation_obs,
+ observation_store_.get(), update_recipient_.get()));
+ break;
+ }
+ case 8: {
+ expected_per_device_numeric_obs[{expected_id, start_day_index + 6}] = {
+ {7, {{"A", 1u, 18}, {"A", 2u, 9}, {"B", 1u, 6}, {"B", 2u, 2}}},
+ {30, {{"A", 1u, 18}, {"A", 2u, 9}, {"B", 1u, 6}, {"B", 2u, 2}}}};
+ expected_per_device_numeric_obs[{expected_id, start_day_index + 7}] = {
+ {7, {{"A", 1u, 21}, {"A", 2u, 9}, {"B", 1u, 6}, {"B", 2u, 2}}},
+ {30, {{"A", 1u, 21}, {"A", 2u, 9}, {"B", 1u, 6}, {"B", 2u, 2}}}};
+ expected_per_device_numeric_obs[{expected_id, start_day_index + 8}] = {
+ {7, {{"A", 1u, 21}, {"A", 2u, 12}, {"B", 1u, 8}, {"B", 2u, 4}}},
+ {30, {{"A", 1u, 24}, {"A", 2u, 12}, {"B", 1u, 8}, {"B", 2u, 4}}}};
+ for (uint32_t day_index = start_day_index + 6; day_index <= start_day_index + 8;
+ day_index++) {
+ for (const auto& pair :
+ MakeExpectedReportParticipationObservations(expected_params, day_index)) {
+ expected_report_participation_obs.insert(pair);
+ }
+ }
+ EXPECT_TRUE(CheckPerDeviceNumericObservations(
+ expected_per_device_numeric_obs, expected_report_participation_obs,
+ observation_store_.get(), update_recipient_.get()));
+ break;
+ }
+ default:
+ EXPECT_TRUE(CheckPerDeviceNumericObservations(
+ expected_per_device_numeric_obs, expected_report_participation_obs,
+ observation_store_.get(), update_recipient_.get()));
+ }
+ AdvanceClock(kDay);
+ }
+}
+
+// Tests that the expected Observations are generated for
+// PerDeviceNumericStats reports when events are logged for over multiple days
+// for an EVENT_COUNT metric with a PER_DEVICE_NUMERIC_STATS report, when
+// Observations are backfilled for some days during that period, and when the
+// LocalAggregatedStore is garbage-collected after each call to
+// GenerateObservations().
+//
+// The logging pattern and set of Observations for each day index is the same
+// as in PerDeviceNumericAggregateStoreTest::CheckObservationValuesMultiDay.
+// See that test for documentation.
+TEST_F(PerDeviceNumericAggregateStoreTest, EventCountCheckObservationValuesWithBackfillAndGc) {
+ auto start_day_index = CurrentDayIndex();
+ const auto& expected_id =
+ logger::testing::per_device_numeric_stats::kSettingsChangedAggregationWindowMetricReportId;
+ const auto& expected_params =
+ logger::testing::per_device_numeric_stats::kExpectedAggregationParams;
+ // Set |backfill_days_| to 3.
+ size_t backfill_days = 3;
+ SetBackfillDays(backfill_days);
+ // Log events for 9 days. Call GenerateObservations() on the first 6 day
+ // indices, and the 9th.
+ uint32_t num_days = 9;
+ for (uint32_t offset = 0; offset < num_days; offset++) {
+ auto day_index = CurrentDayIndex();
+ ResetObservationStore();
+ for (uint32_t event_code = 1; event_code < 3; event_code++) {
+ if (offset > 0 && (offset % event_code == 0)) {
+ EXPECT_EQ(kOK, LogPerDeviceCountEvent(expected_id, day_index, "A", event_code, 3));
+ }
+ if (offset > 0 && offset % (2 * event_code) == 0) {
+ EXPECT_EQ(kOK, LogPerDeviceCountEvent(expected_id, day_index, "B", event_code, 2));
+ }
+ }
+ // Advance |test_clock_| by 1 day.
+ AdvanceClock(kDay);
+ if (offset < 6 || offset == 8) {
+ // Generate Observations and garbage-collect, both for the previous day
+ // index according to |test_clock_|. Back up the LocalAggregateStore and
+ // the AggregatedObservationHistoryStore.
+ DoScheduledTasksNow();
+ }
+ // Make the set of Observations which are expected to be generated on
+ // |start_day_index + offset| and check it against the contents of the
+ // FakeObservationStore.
+ ExpectedPerDeviceNumericObservations expected_per_device_numeric_obs;
+ ExpectedReportParticipationObservations expected_report_participation_obs;
+ switch (offset) {
+ case 0: {
+ for (uint32_t day_index = start_day_index - backfill_days; day_index <= start_day_index;
+ day_index++) {
+ for (const auto& pair :
+ MakeExpectedReportParticipationObservations(expected_params, day_index)) {
+ expected_report_participation_obs.insert(pair);
+ }
+ }
+ EXPECT_TRUE(CheckPerDeviceNumericObservations(
+ expected_per_device_numeric_obs, expected_report_participation_obs,
+ observation_store_.get(), update_recipient_.get()));
+ break;
+ }
+ case 1: {
+ expected_per_device_numeric_obs[{expected_id, day_index}] = {{7, {{"A", 1u, 3}}},
+ {30, {{"A", 1u, 3}}}};
+ expected_report_participation_obs =
+ MakeExpectedReportParticipationObservations(expected_params, day_index);
+ EXPECT_TRUE(CheckPerDeviceNumericObservations(
+ expected_per_device_numeric_obs, expected_report_participation_obs,
+ observation_store_.get(), update_recipient_.get()));
+ break;
+ }
+ case 2: {
+ expected_per_device_numeric_obs[{expected_id, day_index}] = {
+ {7, {{"A", 1u, 6}, {"A", 2u, 3}, {"B", 1u, 2}}},
+ {30, {{"A", 1u, 6}, {"A", 2u, 3}, {"B", 1u, 2}}}};
+ expected_report_participation_obs =
+ MakeExpectedReportParticipationObservations(expected_params, day_index);
+ EXPECT_TRUE(CheckPerDeviceNumericObservations(
+ expected_per_device_numeric_obs, expected_report_participation_obs,
+ observation_store_.get(), update_recipient_.get()));
+ break;
+ }
+ case 3: {
+ expected_per_device_numeric_obs[{expected_id, day_index}] = {
+ {7, {{"A", 1u, 9}, {"A", 2u, 3}, {"B", 1u, 2}}},
+ {30, {{"A", 1u, 9}, {"A", 2u, 3}, {"B", 1u, 2}}}};
+ expected_report_participation_obs =
+ MakeExpectedReportParticipationObservations(expected_params, day_index);
+ EXPECT_TRUE(CheckPerDeviceNumericObservations(
+ expected_per_device_numeric_obs, expected_report_participation_obs,
+ observation_store_.get(), update_recipient_.get()));
+ break;
+ }
+ case 4: {
+ expected_per_device_numeric_obs[{expected_id, day_index}] = {
+ {7, {{"A", 1u, 12}, {"A", 2u, 6}, {"B", 1u, 4}, {"B", 2u, 2}}},
+ {30, {{"A", 1u, 12}, {"A", 2u, 6}, {"B", 1u, 4}, {"B", 2u, 2}}}};
+ expected_report_participation_obs =
+ MakeExpectedReportParticipationObservations(expected_params, day_index);
+ EXPECT_TRUE(CheckPerDeviceNumericObservations(
+ expected_per_device_numeric_obs, expected_report_participation_obs,
+ observation_store_.get(), update_recipient_.get()));
+ break;
+ }
+ case 5: {
+ expected_per_device_numeric_obs[{expected_id, day_index}] = {
+ {7, {{"A", 1u, 15}, {"A", 2u, 6}, {"B", 1u, 4}, {"B", 2u, 2}}},
+ {30, {{"A", 1u, 15}, {"A", 2u, 6}, {"B", 1u, 4}, {"B", 2u, 2}}}};
+ expected_report_participation_obs =
+ MakeExpectedReportParticipationObservations(expected_params, day_index);
+ EXPECT_TRUE(CheckPerDeviceNumericObservations(
+ expected_per_device_numeric_obs, expected_report_participation_obs,
+ observation_store_.get(), update_recipient_.get()));
+ break;
+ }
+ case 8: {
+ expected_per_device_numeric_obs[{expected_id, start_day_index + 6}] = {
+ {7, {{"A", 1u, 18}, {"A", 2u, 9}, {"B", 1u, 6}, {"B", 2u, 2}}},
+ {30, {{"A", 1u, 18}, {"A", 2u, 9}, {"B", 1u, 6}, {"B", 2u, 2}}}};
+ expected_per_device_numeric_obs[{expected_id, start_day_index + 7}] = {
+ {7, {{"A", 1u, 21}, {"A", 2u, 9}, {"B", 1u, 6}, {"B", 2u, 2}}},
+ {30, {{"A", 1u, 21}, {"A", 2u, 9}, {"B", 1u, 6}, {"B", 2u, 2}}}};
+ expected_per_device_numeric_obs[{expected_id, start_day_index + 8}] = {
+ {7, {{"A", 1u, 21}, {"A", 2u, 12}, {"B", 1u, 8}, {"B", 2u, 4}}},
+ {30, {{"A", 1u, 24}, {"A", 2u, 12}, {"B", 1u, 8}, {"B", 2u, 4}}}};
+ for (uint32_t day_index = start_day_index + 6; day_index <= start_day_index + 8;
+ day_index++) {
+ for (const auto& pair :
+ MakeExpectedReportParticipationObservations(expected_params, day_index)) {
+ expected_report_participation_obs.insert(pair);
+ }
+ }
+ EXPECT_TRUE(CheckPerDeviceNumericObservations(
+ expected_per_device_numeric_obs, expected_report_participation_obs,
+ observation_store_.get(), update_recipient_.get()));
+ break;
+ }
+ default:
+ EXPECT_TRUE(CheckPerDeviceNumericObservations(
+ expected_per_device_numeric_obs, expected_report_participation_obs,
+ observation_store_.get(), update_recipient_.get()));
+ }
+ }
+}
+
+// Tests that the expected Observations are generated for
+// PerDeviceNumericStats reports when events are logged for over multiple days
+// for an ELAPSED_TIME metric with PER_DEVICE_NUMERIC_STATS reports with
+// multiple aggregation types, when Observations are backfilled for some days
+// during that period, and when the LocalAggregatedStore is garbage-collected
+// after each call to GenerateObservations().
+//
+// Logged events for the StreamingTime_PerDevice{Total, Min, Max} reports on the
+// i-th day:
+//
+// i (component, event code, count)
+// -----------------------------------------------------------------------
+// 0
+// 1 ("A", 1, 3)
+// 2 ("A", 1, 3), ("A", 2, 3), ("B", 1, 2)
+// 3 ("A", 1, 3)
+// 4 ("A", 1, 3), ("A", 2, 3), ("B", 1, 2), ("B", 2, 2)
+// 5 ("A", 1, 3)
+// 6 ("A", 1, 3), ("A", 2, 3), ("B", 1, 2)
+// 7 ("A", 1, 3)
+// 8 ("A", 1, 3), ("A", 2, 3), ("B", 1, 2), ("B", 2, 2)
+//
+// Expected PerDeviceNumericObservations for the
+// StreamingTime_PerDeviceTotal report on the i-th day:
+//
+// (day, window size) (event code, component, total)
+// ---------------------------------------------------------------------------
+// (0, 1)
+// (0, 7)
+// (1, 1) ("A", 1, 3)
+// (1, 7) ("A", 1, 3)
+// (2, 1) ("A", 1, 3), ("A", 2, 3), ("B", 1, 2)
+// (2, 7) ("A", 1, 6), ("A", 2, 3), ("B", 1, 2)
+// (3, 1) ("A", 1, 3)
+// (3, 7) ("A", 1, 9), ("A", 2, 3), ("B", 1, 2)
+// (4, 1) ("A", 1, 3), ("A", 2, 3), ("B", 1, 2), ("B", 2, 2)
+// (4, 7) ("A", 1, 12), ("A", 2, 6), ("B", 1, 4), ("B", 2, 2)
+// (5, 1) ("A", 1, 3)
+// (5, 7) ("A", 1, 15), ("A", 2, 6), ("B", 1, 4), ("B", 2, 2)
+// (6, 1) ("A", 1, 3), ("A", 2, 3), ("B", 1, 2)
+// (6, 7) ("A", 1, 18), ("A", 2, 9), ("B", 1, 6), ("B", 2, 2)
+// (7, 1) ("A", 1, 3)
+// (7, 7) ("A", 1, 21), ("A", 2, 9), ("B", 1, 6), ("B", 2, 2)
+// (8, 1) ("A", 1, 3), ("A", 2, 3), ("B", 1, 2), ("B", 2, 2)
+// (8, 7) ("A", 1, 21), ("A", 2, 12), ("B", 1, 8), ("B", 2, 4)
+//
+// Expected PerDeviceNumericObservations for the
+// StreamingTime_PerDeviceMin report on the i-th day:
+//
+// (day, window size) (event code, component, total)
+// ---------------------------------------------------------------------------
+// (0, 1)
+// (0. 7)
+// (1, 1) ("A", 1, 3)
+// (1, 7) ("A", 1, 3)
+// (2, 1) ("A", 1, 3), ("A", 2, 3), ("B", 1, 2)
+// (2, 7) ("A", 1, 3), ("A", 2, 3), ("B", 1, 2)
+// (3, 1) ("A", 1, 3)
+// (3, 7) ("A", 1, 3), ("A", 2, 3), ("B", 1, 2)
+// (4, 1) ("A", 1, 3), ("A", 2, 3), ("B", 1, 2), ("B", 2, 2)
+// (4, 7) ("A", 1, 3), ("A", 2, 3), ("B", 1, 2), ("B", 2, 2)
+// (5, 1) ("A", 1, 3)
+// (5, 7) ("A", 1, 3), ("A", 2, 3), ("B", 1, 2), ("B", 2, 2)
+// (6, 1) ("A", 1, 3), ("A", 2, 3), ("B", 1, 2)
+// (6, 7) ("A", 1, 3), ("A", 2, 3), ("B", 1, 2), ("B", 2, 2)
+// (7, 1) ("A", 1, 3)
+// (7, 7) ("A", 1, 3), ("A", 2, 3), ("B", 1, 2), ("B", 2, 2)
+// (8, 1) ("A", 1, 3), ("A", 2, 3), ("B", 1, 2), ("B", 2, 2)
+// (8, 7) ("A", 1, 3), ("A", 2, 3), ("B", 1, 2), ("B", 2, 2)
+//
+// Expected PerDeviceNumericObservations for the
+// StreamingTime_PerDeviceMax report on the i-th day:
+//
+// (day, window size) (event code, component, total)
+// ---------------------------------------------------------------------------
+// (0, 1)
+// (0. 7)
+// (1, 1) ("A", 1, 3)
+// (1, 7) ("A", 1, 3)
+// (2, 1) ("A", 1, 3), ("A", 2, 3), ("B", 1, 2)
+// (2, 7) ("A", 1, 3), ("A", 2, 3), ("B", 1, 2)
+// (3, 1) ("A", 1, 3)
+// (3, 7) ("A", 1, 3), ("A", 2, 3), ("B", 1, 2)
+// (4, 1) ("A", 1, 3), ("A", 2, 3), ("B", 1, 2), ("B", 2, 2)
+// (4, 7) ("A", 1, 3), ("A", 2, 3), ("B", 1, 2), ("B", 2, 2)
+// (5, 1) ("A", 1, 3)
+// (5, 7) ("A", 1, 3), ("A", 2, 3), ("B", 1, 2), ("B", 2, 2)
+// (6, 1) ("A", 1, 3), ("A", 2, 3), ("B", 1, 2)
+// (6, 7) ("A", 1, 3), ("A", 2, 3), ("B", 1, 2), ("B", 2, 2)
+// (7, 1) ("A", 1, 3)
+// (7, 7) ("A", 1, 3), ("A", 2, 3), ("B", 1, 2), ("B", 2, 2)
+// (8, 1) ("A", 1, 3), ("A", 2, 3), ("B", 1, 2), ("B", 2, 2)
+// (8, 7) ("A", 1, 3), ("A", 2, 3), ("B", 1, 2), ("B", 2, 2)
+//
+// In addition, expect 1 ReportParticipationObservation each day for each
+// report in the registry.
+TEST_F(PerDeviceNumericAggregateStoreTest, ElapsedTimeCheckObservationValuesWithBackfillAndGc) {
+ auto start_day_index = CurrentDayIndex();
+ const auto& total_report_id =
+ logger::testing::per_device_numeric_stats::kStreamingTimeTotalMetricReportId;
+ const auto& min_report_id =
+ logger::testing::per_device_numeric_stats::kStreamingTimeMinMetricReportId;
+ const auto& max_report_id =
+ logger::testing::per_device_numeric_stats::kStreamingTimeMaxMetricReportId;
+ std::vector<MetricReportId> streaming_time_ids = {total_report_id, min_report_id, max_report_id};
+ const auto& expected_params =
+ logger::testing::per_device_numeric_stats::kExpectedAggregationParams;
+ // Set |backfill_days_| to 3.
+ size_t backfill_days = 3;
+ SetBackfillDays(backfill_days);
+ // Log events for 9 days. Call GenerateObservations() on the first 6 day
+ // indices, and the 9th.
+ uint32_t num_days = 9;
+ for (uint32_t offset = 0; offset < num_days; offset++) {
+ auto day_index = CurrentDayIndex();
+ ResetObservationStore();
+ for (uint32_t event_code = 1; event_code < 3; event_code++) {
+ for (const auto& report_id : streaming_time_ids) {
+ if (offset > 0 && (offset % event_code == 0)) {
+ EXPECT_EQ(kOK, LogPerDeviceElapsedTimeEvent(report_id, day_index, "A", event_code, 3));
+ }
+ if (offset > 0 && offset % (2 * event_code) == 0) {
+ EXPECT_EQ(kOK, LogPerDeviceElapsedTimeEvent(report_id, day_index, "B", event_code, 2));
+ }
+ }
+ }
+
+ // Advance |test_clock_| by 1 day.
+ AdvanceClock(kDay);
+ if (offset < 6 || offset == 8) {
+ // Generate Observations and garbage-collect, both for the previous day
+ // index according to |test_clock_|. Back up the LocalAggregateStore and
+ // the AggregatedObservationHistoryStore.
+ DoScheduledTasksNow();
+ }
+ // Make the set of Observations which are expected to be generated on
+ // |start_day_index + offset| and check it against the contents of the
+ // FakeObservationStore.
+ ExpectedPerDeviceNumericObservations expected_per_device_numeric_obs;
+ ExpectedReportParticipationObservations expected_report_participation_obs;
+ switch (offset) {
+ case 0: {
+ for (uint32_t day_index = start_day_index - backfill_days; day_index <= start_day_index;
+ day_index++) {
+ for (const auto& pair :
+ MakeExpectedReportParticipationObservations(expected_params, day_index)) {
+ expected_report_participation_obs.insert(pair);
+ }
+ }
+ EXPECT_TRUE(CheckPerDeviceNumericObservations(
+ expected_per_device_numeric_obs, expected_report_participation_obs,
+ observation_store_.get(), update_recipient_.get()));
+ break;
+ }
+ case 1: {
+ expected_per_device_numeric_obs[{total_report_id, day_index}] = {{1, {{"A", 1u, 3}}},
+ {7, {{"A", 1u, 3}}}};
+ expected_per_device_numeric_obs[{min_report_id, day_index}] = {{1, {{"A", 1u, 3}}},
+ {7, {{"A", 1u, 3}}}};
+ expected_per_device_numeric_obs[{max_report_id, day_index}] = {{1, {{"A", 1u, 3}}},
+ {7, {{"A", 1u, 3}}}};
+ expected_report_participation_obs =
+ MakeExpectedReportParticipationObservations(expected_params, day_index);
+ EXPECT_TRUE(CheckPerDeviceNumericObservations(
+ expected_per_device_numeric_obs, expected_report_participation_obs,
+ observation_store_.get(), update_recipient_.get()))
+ << "day 1";
+ break;
+ }
+ case 2: {
+ expected_per_device_numeric_obs[{total_report_id, day_index}] = {
+ {1, {{"A", 1u, 3}, {"A", 2u, 3}, {"B", 1u, 2}}},
+ {7, {{"A", 1u, 6}, {"A", 2u, 3}, {"B", 1u, 2}}}};
+ expected_per_device_numeric_obs[{min_report_id, day_index}] = {
+ {1, {{"A", 1u, 3}, {"A", 2u, 3}, {"B", 1u, 2}}},
+ {7, {{"A", 1u, 3}, {"A", 2u, 3}, {"B", 1u, 2}}}};
+ expected_per_device_numeric_obs[{max_report_id, day_index}] = {
+ {1, {{"A", 1u, 3}, {"A", 2u, 3}, {"B", 1u, 2}}},
+ {7, {{"A", 1u, 3}, {"A", 2u, 3}, {"B", 1u, 2}}}};
+ expected_report_participation_obs =
+ MakeExpectedReportParticipationObservations(expected_params, day_index);
+ EXPECT_TRUE(CheckPerDeviceNumericObservations(
+ expected_per_device_numeric_obs, expected_report_participation_obs,
+ observation_store_.get(), update_recipient_.get()));
+ break;
+ }
+ case 3: {
+ expected_per_device_numeric_obs[{total_report_id, day_index}] = {
+ {1, {{"A", 1u, 3}}}, {7, {{"A", 1u, 9}, {"A", 2u, 3}, {"B", 1u, 2}}}};
+ expected_per_device_numeric_obs[{min_report_id, day_index}] = {
+ {1, {{"A", 1u, 3}}}, {7, {{"A", 1u, 3}, {"A", 2u, 3}, {"B", 1u, 2}}}};
+ expected_per_device_numeric_obs[{max_report_id, day_index}] = {
+ {1, {{"A", 1u, 3}}}, {7, {{"A", 1u, 3}, {"A", 2u, 3}, {"B", 1u, 2}}}};
+ expected_report_participation_obs =
+ MakeExpectedReportParticipationObservations(expected_params, day_index);
+ EXPECT_TRUE(CheckPerDeviceNumericObservations(
+ expected_per_device_numeric_obs, expected_report_participation_obs,
+ observation_store_.get(), update_recipient_.get()));
+ break;
+ }
+ case 4: {
+ expected_per_device_numeric_obs[{total_report_id, day_index}] = {
+ {1, {{"A", 1u, 3}, {"A", 2u, 3}, {"B", 1u, 2}, {"B", 2u, 2}}},
+ {7, {{"A", 1u, 12}, {"A", 2u, 6}, {"B", 1u, 4}, {"B", 2u, 2}}}};
+ expected_per_device_numeric_obs[{min_report_id, day_index}] = {
+ {1, {{"A", 1u, 3}, {"A", 2u, 3}, {"B", 1u, 2}, {"B", 2u, 2}}},
+ {7, {{"A", 1u, 3}, {"A", 2u, 3}, {"B", 1u, 2}, {"B", 2u, 2}}}};
+ expected_per_device_numeric_obs[{max_report_id, day_index}] = {
+ {1, {{"A", 1u, 3}, {"A", 2u, 3}, {"B", 1u, 2}, {"B", 2u, 2}}},
+ {7, {{"A", 1u, 3}, {"A", 2u, 3}, {"B", 1u, 2}, {"B", 2u, 2}}}};
+ expected_report_participation_obs =
+ MakeExpectedReportParticipationObservations(expected_params, day_index);
+ EXPECT_TRUE(CheckPerDeviceNumericObservations(
+ expected_per_device_numeric_obs, expected_report_participation_obs,
+ observation_store_.get(), update_recipient_.get()));
+ break;
+ }
+ case 5: {
+ expected_per_device_numeric_obs[{total_report_id, day_index}] = {
+ {1, {{"A", 1u, 3}}}, {7, {{"A", 1u, 15}, {"A", 2u, 6}, {"B", 1u, 4}, {"B", 2u, 2}}}};
+ expected_per_device_numeric_obs[{min_report_id, day_index}] = {
+ {1, {{"A", 1u, 3}}}, {7, {{"A", 1u, 3}, {"A", 2u, 3}, {"B", 1u, 2}, {"B", 2u, 2}}}};
+ expected_per_device_numeric_obs[{max_report_id, day_index}] = {
+ {1, {{"A", 1u, 3}}}, {7, {{"A", 1u, 3}, {"A", 2u, 3}, {"B", 1u, 2}, {"B", 2u, 2}}}};
+ expected_report_participation_obs =
+ MakeExpectedReportParticipationObservations(expected_params, day_index);
+ EXPECT_TRUE(CheckPerDeviceNumericObservations(
+ expected_per_device_numeric_obs, expected_report_participation_obs,
+ observation_store_.get(), update_recipient_.get()));
+ break;
+ }
+ case 8: {
+ expected_per_device_numeric_obs[{total_report_id, start_day_index + 6}] = {
+ {1, {{"A", 1u, 3}, {"A", 2u, 3}, {"B", 1u, 2}}},
+ {7, {{"A", 1u, 18}, {"A", 2u, 9}, {"B", 1u, 6}, {"B", 2u, 2}}}};
+ expected_per_device_numeric_obs[{total_report_id, start_day_index + 7}] = {
+ {1, {{"A", 1u, 3}}}, {7, {{"A", 1u, 21}, {"A", 2u, 9}, {"B", 1u, 6}, {"B", 2u, 2}}}};
+ expected_per_device_numeric_obs[{total_report_id, start_day_index + 8}] = {
+ {1, {{"A", 1u, 3}, {"A", 2u, 3}, {"B", 1u, 2}, {"B", 2u, 2}}},
+ {7, {{"A", 1u, 21}, {"A", 2u, 12}, {"B", 1u, 8}, {"B", 2u, 4}}}};
+
+ expected_per_device_numeric_obs[{min_report_id, start_day_index + 6}] = {
+ {1, {{"A", 1u, 3}, {"A", 2u, 3}, {"B", 1u, 2}}},
+ {7, {{"A", 1u, 3}, {"A", 2u, 3}, {"B", 1u, 2}, {"B", 2u, 2}}}};
+ expected_per_device_numeric_obs[{min_report_id, start_day_index + 7}] = {
+ {1, {{"A", 1u, 3}}}, {7, {{"A", 1u, 3}, {"A", 2u, 3}, {"B", 1u, 2}, {"B", 2u, 2}}}};
+ expected_per_device_numeric_obs[{min_report_id, start_day_index + 8}] = {
+ {1, {{"A", 1u, 3}, {"A", 2u, 3}, {"B", 1u, 2}, {"B", 2u, 2}}},
+ {7, {{"A", 1u, 3}, {"A", 2u, 3}, {"B", 1u, 2}, {"B", 2u, 2}}}};
+
+ expected_per_device_numeric_obs[{max_report_id, start_day_index + 6}] = {
+ {1, {{"A", 1u, 3}, {"A", 2u, 3}, {"B", 1u, 2}}},
+ {7, {{"A", 1u, 3}, {"A", 2u, 3}, {"B", 1u, 2}, {"B", 2u, 2}}}};
+ expected_per_device_numeric_obs[{max_report_id, start_day_index + 7}] = {
+ {1, {{"A", 1u, 3}}}, {7, {{"A", 1u, 3}, {"A", 2u, 3}, {"B", 1u, 2}, {"B", 2u, 2}}}};
+ expected_per_device_numeric_obs[{max_report_id, start_day_index + 8}] = {
+ {1, {{"A", 1u, 3}, {"A", 2u, 3}, {"B", 1u, 2}, {"B", 2u, 2}}},
+ {7, {{"A", 1u, 3}, {"A", 2u, 3}, {"B", 1u, 2}, {"B", 2u, 2}}}};
+
+ for (uint32_t day_index = start_day_index + 6; day_index <= start_day_index + 8;
+ day_index++) {
+ for (const auto& pair :
+ MakeExpectedReportParticipationObservations(expected_params, day_index)) {
+ expected_report_participation_obs.insert(pair);
+ }
+ }
+ EXPECT_TRUE(CheckPerDeviceNumericObservations(
+ expected_per_device_numeric_obs, expected_report_participation_obs,
+ observation_store_.get(), update_recipient_.get()));
+ break;
+ }
+ default:
+ EXPECT_TRUE(CheckPerDeviceNumericObservations(
+ expected_per_device_numeric_obs, expected_report_participation_obs,
+ observation_store_.get(), update_recipient_.get()));
+ }
+ }
+}
+
+// Check that GenerateObservations returns an OK status after some events have been logged for a
+// PerDeviceHistogram report.
+TEST_F(PerDeviceHistogramAggregateStoreTest, GenerateObservations) {
+ const auto day_index = CurrentDayIndex();
+ // Log several events on |day_index|.
+ EXPECT_EQ(kOK, LogPerDeviceCountEvent(
+ logger::testing::per_device_histogram::kSettingsChangedMetricReportId,
+ day_index, "component_C", 0u, 5));
+ EXPECT_EQ(kOK, LogPerDeviceCountEvent(
+ logger::testing::per_device_histogram::kSettingsChangedMetricReportId,
+ day_index, "component_C", 0u, 5));
+
+ // Generate locally aggregated Observations for |day_index|.
+ EXPECT_EQ(kOK, GenerateObservations(day_index));
+}
+
+// Tests GenerateObservations() and GarbageCollect() in the case where the
+// LocalAggregateStore contains aggregates for metrics with both UTC and LOCAL
+// time zone policies, and where the day index in local time may be less than
+// the day index in UTC.
+TEST_F(NoiseFreeMixedTimeZoneAggregateStoreTest, LocalBeforeUTC) {
+ std::vector<ExpectedUniqueActivesObservations> expected_obs(3);
+ // Begin at a time when the current day index is the same in both UTC and
+ // local time. Log 1 event for event code 0 for each of the 2 reports, then
+ // generate Observations and garbage-collect for the previous day index in
+ // each of UTC and local time.
+ auto start_day_index = CurrentDayIndex();
+ LogUniqueActivesEvent(logger::testing::mixed_time_zone::kDeviceBootsMetricReportId,
+ start_day_index, 0u);
+ LogUniqueActivesEvent(logger::testing::mixed_time_zone::kFeaturesActiveMetricReportId,
+ start_day_index, 0u);
+ GenerateObservations(start_day_index - 1, start_day_index - 1);
+ GarbageCollect(start_day_index - 1, start_day_index - 1);
+ // Form the expected contents of the FakeObservationStore.
+ // Since no events were logged on the previous day and no Observations have
+ // been generated for that day yet, expect Observations of non-activity for
+ // all event codes, for both reports.
+ expected_obs[0] = MakeNullExpectedUniqueActivesObservations(
+ logger::testing::mixed_time_zone::kExpectedAggregationParams, start_day_index - 1);
+ EXPECT_TRUE(CheckUniqueActivesObservations(expected_obs[0], observation_store_.get(),
+ update_recipient_.get()));
+ ResetObservationStore();
+ // Advance the day index in UTC, but not in local time, and log 1 event for
+ // event code 1 for each of the 2 reports. Generate Observations and
+ // garbage-collect for the previous day in each of UTC and local time.
+ LogUniqueActivesEvent(logger::testing::mixed_time_zone::kDeviceBootsMetricReportId,
+ start_day_index, 1u);
+ LogUniqueActivesEvent(logger::testing::mixed_time_zone::kFeaturesActiveMetricReportId,
+ start_day_index + 1, 1u);
+ GenerateObservations(start_day_index, start_day_index - 1);
+ GarbageCollect(start_day_index, start_day_index - 1);
+ // Form the expected contents of the FakeObservationStore. Since
+ // Observations have already been generated for the
+ // DeviceBoots_UniqueDevices report for |start_day_index - 1|, expect no
+ // Observations for that report.
+ expected_obs[1][{logger::testing::mixed_time_zone::kFeaturesActiveMetricReportId,
+ start_day_index}] = {{1, {true, false, false}}};
+ EXPECT_TRUE(CheckUniqueActivesObservations(expected_obs[1], observation_store_.get(),
+ update_recipient_.get()));
+ ResetObservationStore();
+ // Advance the day index in local time so that it is equal to the day index
+ // in UTC. Log 1 event for event code 2 for each of the 2 reports, then
+ // generate Observations and garbage-collect for the previous day in each of
+ // UTC and local time.
+ LogUniqueActivesEvent(logger::testing::mixed_time_zone::kDeviceBootsMetricReportId,
+ start_day_index + 1, 2u);
+ LogUniqueActivesEvent(logger::testing::mixed_time_zone::kFeaturesActiveMetricReportId,
+ start_day_index + 1, 2u);
+ GenerateObservations(start_day_index, start_day_index);
+ GarbageCollect(start_day_index, start_day_index);
+ // Form the expected contents of the FakeObservationStore. Since
+ // Observations have already been generated for the
+ // FeaturesActive_UniqueDevices report for day |start_day_index|, expect no
+ // Observations for that report.
+ expected_obs[2][{logger::testing::mixed_time_zone::kDeviceBootsMetricReportId, start_day_index}] =
+ {{1, {true, true, false}}};
+ EXPECT_TRUE(CheckUniqueActivesObservations(expected_obs[2], observation_store_.get(),
+ update_recipient_.get()));
+}
+
+// Tests GenerateObservations() and GarbageCollect() in the case where the
+// LocalAggregateStore contains aggregates for metrics with both UTC and LOCAL
+// time zone policies, and where the day index in UTC may be less than
+// the day index in local time.
+TEST_F(NoiseFreeMixedTimeZoneAggregateStoreTest, LocalAfterUTC) {
+ std::vector<ExpectedUniqueActivesObservations> expected_obs(3);
+ // Begin at a time when the current day index is the same in both UTC and
+ // local time. Log 1 event for event code 0 for each of the 2 reports, then
+ // generate Observations and garbage-collect for the previous day index in
+ // each of UTC and local time.
+ auto start_day_index = CurrentDayIndex();
+ LogUniqueActivesEvent(logger::testing::mixed_time_zone::kDeviceBootsMetricReportId,
+ start_day_index, 0u);
+ LogUniqueActivesEvent(logger::testing::mixed_time_zone::kFeaturesActiveMetricReportId,
+ start_day_index, 0u);
+ GenerateObservations(start_day_index - 1, start_day_index - 1);
+ GarbageCollect(start_day_index - 1, start_day_index - 1);
+ // Form the expected contents of the FakeObservationStore.
+ // Since no events were logged on the previous day and no Observations have
+ // been generated for that day yet, expect Observations of non-activity for
+ // all event codes, for both reports.
+ expected_obs[0] = MakeNullExpectedUniqueActivesObservations(
+ logger::testing::mixed_time_zone::kExpectedAggregationParams, start_day_index - 1);
+ EXPECT_TRUE(CheckUniqueActivesObservations(expected_obs[0], observation_store_.get(),
+ update_recipient_.get()));
+ ResetObservationStore();
+ // Advance the day index in local time, but not in UTC, and log 1 event for
+ // event code 1 for each of the 2 reports. Generate Observations and
+ // garbage-collect for the previous day in each of UTC and local time.
+ LogUniqueActivesEvent(logger::testing::mixed_time_zone::kDeviceBootsMetricReportId,
+ start_day_index + 1, 1u);
+ LogUniqueActivesEvent(logger::testing::mixed_time_zone::kFeaturesActiveMetricReportId,
+ start_day_index, 1u);
+ GenerateObservations(start_day_index - 1, start_day_index);
+ GarbageCollect(start_day_index - 1, start_day_index);
+ // Form the expected contents of the FakeObservationStore. Since
+ // Observations have already been generated for the
+ // FeaturesActive_UniqueDevices report for |start_day_index - 1|, expect no
+ // Observations for that report.
+ expected_obs[1][{logger::testing::mixed_time_zone::kDeviceBootsMetricReportId, start_day_index}] =
+ {{1, {true, false, false}}};
+ EXPECT_TRUE(CheckUniqueActivesObservations(expected_obs[1], observation_store_.get(),
+ update_recipient_.get()));
+ ResetObservationStore();
+ // Advance the day index in UTC so that it is equal to the day index in
+ // local time. Log 1 event for event code 2 for each of the 2 reports, then
+ // generate Observations and garbage-collect for the previous day in each of
+ // UTC and local time.
+ LogUniqueActivesEvent(logger::testing::mixed_time_zone::kDeviceBootsMetricReportId,
+ start_day_index + 1, 2u);
+ LogUniqueActivesEvent(logger::testing::mixed_time_zone::kFeaturesActiveMetricReportId,
+ start_day_index + 1, 2u);
+ GenerateObservations(start_day_index, start_day_index);
+ GarbageCollect(start_day_index, start_day_index);
+ // Form the expected contents of the FakeObservationStore. Since
+ // Observations have already been generated for the
+ // DeviceBoots_UniqueDevices report for day |start_day_index|, expect no
+ // Observations for that report.
+ expected_obs[2][{logger::testing::mixed_time_zone::kFeaturesActiveMetricReportId,
+ start_day_index}] = {{1, {true, true, false}}};
+ EXPECT_TRUE(CheckUniqueActivesObservations(expected_obs[2], observation_store_.get(),
+ update_recipient_.get()));
+}
+} // namespace local_aggregation
+} // namespace cobalt
diff --git a/src/local_aggregation/event_aggregator.cc b/src/local_aggregation/event_aggregator.cc
index f556f58..c126127 100644
--- a/src/local_aggregation/event_aggregator.cc
+++ b/src/local_aggregation/event_aggregator.cc
@@ -10,38 +10,27 @@
#include <utility>
#include <vector>
-#include "src/algorithms/rappor/rappor_config_helper.h"
#include "src/lib/util/datetime_util.h"
#include "src/lib/util/proto_util.h"
-#include "src/lib/util/status.h"
-#include "src/local_aggregation/aggregation_utils.h"
-#include "src/logger/project_context.h"
-#include "src/registry/metric_definition.pb.h"
#include "src/registry/packed_event_codes.h"
namespace cobalt::local_aggregation {
-using google::protobuf::RepeatedField;
using logger::Encoder;
using logger::EventRecord;
using logger::kInvalidArguments;
using logger::kOK;
using logger::kOther;
-using logger::MetricRef;
using logger::ObservationWriter;
using logger::ProjectContext;
using logger::Status;
-using rappor::RapporConfigHelper;
using util::ConsistentProtoStore;
using util::SerializeToBase64;
-using util::StatusCode;
using util::SteadyClock;
using util::TimeToDayIndex;
namespace {
-////// General helper functions.
-
// Populates a ReportAggregationKey proto message and then populates a string
// with the base64 encoding of the serialized proto.
bool PopulateReportKey(uint32_t customer_id, uint32_t project_id, uint32_t metric_id,
@@ -54,201 +43,8 @@
return SerializeToBase64(key_data, key);
}
-////// Helper functions used by the constructor and UpdateAggregationConfigs().
-
-// Gets and validates the window sizes and/or aggregation windows from a ReportDefinition, converts
-// window sizes to daily aggregation windows, sorts the aggregation windows in increasing order, and
-// adds them to an AggregationConfig.
-//
-// TODO(pesk): Stop looking at the window_size field of |report| once all reports have been updated
-// to have OnDeviceAggregationWindows only.
-bool GetSortedAggregationWindowsFromReport(const ReportDefinition& report,
- AggregationConfig* aggregation_config) {
- if (report.window_size_size() == 0 && report.aggregation_window_size() == 0) {
- LOG(ERROR) << "Report must have at least one window size or aggregation window.";
- return false;
- }
- std::vector<uint32_t> aggregation_days;
- std::vector<uint32_t> aggregation_hours;
- for (const uint32_t window_size : report.window_size()) {
- if (window_size == 0 || window_size > EventAggregator::kMaxAllowedAggregationDays) {
- LOG(ERROR) << "Window size must be positive and cannot exceed "
- << EventAggregator::kMaxAllowedAggregationDays;
- return false;
- }
- aggregation_days.push_back(window_size);
- }
- for (const auto& window : report.aggregation_window()) {
- switch (window.units_case()) {
- case OnDeviceAggregationWindow::kDays: {
- uint32_t num_days = window.days();
- if (num_days == 0 || num_days > EventAggregator::kMaxAllowedAggregationDays) {
- LOG(ERROR) << "Daily windows must contain at least 1 and no more than "
- << EventAggregator::kMaxAllowedAggregationDays << " days";
- return false;
- }
- aggregation_days.push_back(num_days);
- break;
- }
- case OnDeviceAggregationWindow::kHours: {
- uint32_t num_hours = window.hours();
- if (num_hours == 0 || num_hours > EventAggregator::kMaxAllowedAggregationHours) {
- LOG(ERROR) << "Hourly windows must contain at least 1 and no more than "
- << EventAggregator::kMaxAllowedAggregationHours << " hours";
- return false;
- }
- aggregation_hours.push_back(num_hours);
- break;
- }
- default:
- LOG(ERROR) << "Invalid OnDeviceAggregationWindow type " << window.units_case();
- }
- }
- std::sort(aggregation_hours.begin(), aggregation_hours.end());
- std::sort(aggregation_days.begin(), aggregation_days.end());
- for (auto num_hours : aggregation_hours) {
- *aggregation_config->add_aggregation_window() = MakeHourWindow(num_hours);
- }
- for (auto num_days : aggregation_days) {
- *aggregation_config->add_aggregation_window() = MakeDayWindow(num_days);
- }
- return true;
-}
-
-// Creates an AggregationConfig from a ProjectContext, MetricDefinition, and
-// ReportDefinition and populates the aggregation_config field of a specified
-// ReportAggregates. Also sets the type of the ReportAggregates based on the
-// ReportDefinition's type.
-//
-// Accepts ReportDefinitions with either at least one WindowSize, or at least one
-// OnDeviceAggregationWindow with units in days.
-bool PopulateReportAggregates(const ProjectContext& project_context, const MetricDefinition& metric,
- const ReportDefinition& report, ReportAggregates* report_aggregates) {
- if (report.window_size_size() == 0 && report.aggregation_window_size() == 0) {
- }
- AggregationConfig* aggregation_config = report_aggregates->mutable_aggregation_config();
- *aggregation_config->mutable_project() = project_context.project();
- *aggregation_config->mutable_metric() = *project_context.GetMetric(metric.id());
- *aggregation_config->mutable_report() = report;
- if (!GetSortedAggregationWindowsFromReport(report, aggregation_config)) {
- return false;
- }
- switch (report.report_type()) {
- case ReportDefinition::UNIQUE_N_DAY_ACTIVES: {
- report_aggregates->set_allocated_unique_actives_aggregates(new UniqueActivesReportAggregates);
- return true;
- }
- case ReportDefinition::PER_DEVICE_NUMERIC_STATS:
- case ReportDefinition::PER_DEVICE_HISTOGRAM: {
- report_aggregates->set_allocated_numeric_aggregates(new PerDeviceNumericAggregates);
- return true;
- }
- default:
- return false;
- }
-}
-
-// Given a ProjectContext, MetricDefinition, and ReportDefinition and a pointer
-// to the LocalAggregateStore, checks whether a key with the same customer,
-// project, metric, and report ID already exists in the LocalAggregateStore. If
-// not, creates and inserts a new key and value. Returns kInvalidArguments if
-// creation of the key or value fails, and kOK otherwise. The caller should hold
-// the mutex protecting the LocalAggregateStore.
-Status MaybeInsertReportConfigLocked(const ProjectContext& project_context,
- const MetricDefinition& metric, const ReportDefinition& report,
- LocalAggregateStore* store) {
- std::string key;
- if (!PopulateReportKey(project_context.project().customer_id(),
- project_context.project().project_id(), metric.id(), report.id(), &key)) {
- return kInvalidArguments;
- }
- ReportAggregates report_aggregates;
- if (store->by_report_key().count(key) == 0) {
- if (!PopulateReportAggregates(project_context, metric, report, &report_aggregates)) {
- return kInvalidArguments;
- }
- (*store->mutable_by_report_key())[key] = report_aggregates;
- }
- return kOK;
-}
-
-RepeatedField<uint32_t> UnpackEventCodesProto(uint64_t packed_event_codes) {
- RepeatedField<uint32_t> fields;
- for (auto code : config::UnpackEventCodes(packed_event_codes)) {
- *fields.Add() = code;
- }
- return fields;
-}
-
-// Move all items from the |window_size| field to the |aggregation_window| field
-// of each AggregationConfig, preserving the order of the items. The |aggregation_window| field
-// should be empty if the |window_size| field is nonempty. If for some reason this is not true, log
-// an error and discard the contents of |aggregation_window| and replace them with the migrated
-// |window_size| values.
-void ConvertWindowSizesToAggregationDays(LocalAggregateStore* store) {
- for (auto [key, aggregates] : store->by_report_key()) {
- auto config = (*store->mutable_by_report_key())[key].mutable_aggregation_config();
- if (config->window_size_size() > 0 && config->aggregation_window_size() > 0) {
- LOG(ERROR) << "Config has both a window_size and an aggregation_window; discarding all "
- "aggregation_windows";
- config->clear_aggregation_window();
- }
- for (auto window_size : config->window_size()) {
- *config->add_aggregation_window() = MakeDayWindow(window_size);
- }
- config->clear_window_size();
- }
-}
-
-// Upgrades the LocalAggregateStore from version 0 to |kCurrentLocalAggregateStoreVersion|.
-Status UpgradeLocalAggregateStoreFromVersion0(LocalAggregateStore* store) {
- ConvertWindowSizesToAggregationDays(store);
- store->set_version(EventAggregator::kCurrentLocalAggregateStoreVersion);
- return kOK;
-}
-
} // namespace
-LocalAggregateStore EventAggregator::MakeNewLocalAggregateStore(uint32_t version) {
- LocalAggregateStore store;
- store.set_version(version);
- return store;
-}
-
-AggregatedObservationHistoryStore EventAggregator::MakeNewObservationHistoryStore(
- uint32_t version) {
- AggregatedObservationHistoryStore store;
- store.set_version(version);
- return store;
-}
-
-// We can upgrade from v0, but no other versions.
-Status EventAggregator::MaybeUpgradeLocalAggregateStore(LocalAggregateStore* store) {
- uint32_t version = store->version();
- if (version == kCurrentLocalAggregateStoreVersion) {
- return kOK;
- }
- VLOG(4) << "Attempting to upgrade LocalAggregateStore from version " << version;
- switch (version) {
- case 0u:
- return UpgradeLocalAggregateStoreFromVersion0(store);
- default:
- LOG(ERROR) << "Cannot upgrade LocalAggregateStore from version " << version;
- return kInvalidArguments;
- }
-}
-
-// The current version is the earliest version, so no other versions are accepted.
-Status EventAggregator::MaybeUpgradeObservationHistoryStore(
- AggregatedObservationHistoryStore* store) {
- uint32_t version = store->version();
- if (version == kCurrentObservationHistoryStoreVersion) {
- return kOK;
- }
- LOG(ERROR) << "Cannot upgrade AggregatedObservationHistoryStore from version " << version;
- return kInvalidArguments;
-}
-
EventAggregator::EventAggregator(const Encoder* encoder,
const ObservationWriter* observation_writer,
ConsistentProtoStore* local_aggregate_proto_store,
@@ -256,81 +52,19 @@
const size_t backfill_days,
const std::chrono::seconds aggregate_backup_interval,
const std::chrono::seconds generate_obs_interval,
- const std::chrono::seconds gc_interval)
- : encoder_(encoder),
- observation_writer_(observation_writer),
- local_aggregate_proto_store_(local_aggregate_proto_store),
- obs_history_proto_store_(obs_history_proto_store) {
+ const std::chrono::seconds gc_interval) {
CHECK_LE(aggregate_backup_interval.count(), generate_obs_interval.count())
<< "aggregate_backup_interval must be less than or equal to "
"generate_obs_interval";
CHECK_LE(aggregate_backup_interval.count(), gc_interval.count())
<< "aggregate_backup_interval must be less than or equal to gc_interval";
- CHECK_LE(backfill_days, kMaxAllowedBackfillDays)
- << "backfill_days must be less than or equal to " << kMaxAllowedBackfillDays;
aggregate_backup_interval_ = aggregate_backup_interval;
generate_obs_interval_ = generate_obs_interval;
gc_interval_ = gc_interval;
- backfill_days_ = backfill_days;
- auto locked = protected_aggregate_store_.lock();
- auto restore_aggregates_status =
- local_aggregate_proto_store_->Read(&(locked->local_aggregate_store));
- switch (restore_aggregates_status.error_code()) {
- case StatusCode::OK: {
- VLOG(4) << "Read LocalAggregateStore from disk.";
- break;
- }
- case StatusCode::NOT_FOUND: {
- VLOG(4) << "No file found for local_aggregate_proto_store. Proceeding "
- "with empty LocalAggregateStore. File will be created on "
- "first snapshot of the LocalAggregateStore.";
- locked->local_aggregate_store = MakeNewLocalAggregateStore();
- break;
- }
- default: {
- LOG(ERROR) << "Read to local_aggregate_proto_store failed with status code: "
- << restore_aggregates_status.error_code()
- << "\nError message: " << restore_aggregates_status.error_message()
- << "\nError details: " << restore_aggregates_status.error_details()
- << "\nProceeding with empty LocalAggregateStore.";
- locked->local_aggregate_store = MakeNewLocalAggregateStore();
- }
- }
- if (auto status = MaybeUpgradeLocalAggregateStore(&(locked->local_aggregate_store));
- status != kOK) {
- LOG(ERROR) << "Failed to upgrade LocalAggregateStore to current version with status " << status
- << ".\nProceeding with empty "
- "LocalAggregateStore.";
- locked->local_aggregate_store = MakeNewLocalAggregateStore();
- }
- auto restore_history_status = obs_history_proto_store_->Read(&obs_history_);
- switch (restore_history_status.error_code()) {
- case StatusCode::OK: {
- VLOG(4) << "Read AggregatedObservationHistoryStore from disk.";
- break;
- }
- case StatusCode::NOT_FOUND: {
- VLOG(4) << "No file found for obs_history_proto_store. Proceeding "
- "with empty AggregatedObservationHistoryStore. File will be "
- "created on first snapshot of the AggregatedObservationHistoryStore.";
- break;
- }
- default: {
- LOG(ERROR) << "Read to obs_history_proto_store failed with status code: "
- << restore_history_status.error_code()
- << "\nError message: " << restore_history_status.error_message()
- << "\nError details: " << restore_history_status.error_details()
- << "\nProceeding with empty AggregatedObservationHistoryStore.";
- obs_history_ = MakeNewObservationHistoryStore();
- }
- }
- if (auto status = MaybeUpgradeObservationHistoryStore(&obs_history_); status != kOK) {
- LOG(ERROR)
- << "Failed to upgrade AggregatedObservationHistoryStore to current version with status "
- << status << ".\nProceeding with empty AggregatedObservationHistoryStore.";
- obs_history_ = MakeNewObservationHistoryStore();
- }
+ aggregate_store_ =
+ std::make_unique<AggregateStore>(encoder, observation_writer, local_aggregate_proto_store,
+ obs_history_proto_store, backfill_days);
steady_clock_ = std::make_unique<SteadyClock>();
}
@@ -347,7 +81,7 @@
// TODO(pesk): update the EventAggregator's view of a Metric
// or ReportDefinition when appropriate.
Status EventAggregator::UpdateAggregationConfigs(const ProjectContext& project_context) {
- auto locked = protected_aggregate_store_.lock();
+ auto locked = aggregate_store_->protected_aggregate_store_.lock();
Status status;
for (const auto& metric : project_context.metrics()) {
switch (metric.metric_type()) {
@@ -355,8 +89,8 @@
for (const auto& report : metric.reports()) {
switch (report.report_type()) {
case ReportDefinition::UNIQUE_N_DAY_ACTIVES: {
- status = MaybeInsertReportConfigLocked(project_context, metric, report,
- &(locked->local_aggregate_store));
+ status = aggregate_store_->MaybeInsertReportConfigLocked(
+ project_context, metric, report, &(locked->local_aggregate_store));
if (status != kOK) {
return status;
}
@@ -374,8 +108,8 @@
switch (report.report_type()) {
case ReportDefinition::PER_DEVICE_NUMERIC_STATS:
case ReportDefinition::PER_DEVICE_HISTOGRAM: {
- status = MaybeInsertReportConfigLocked(project_context, metric, report,
- &(locked->local_aggregate_store));
+ status = aggregate_store_->MaybeInsertReportConfigLocked(
+ project_context, metric, report, &(locked->local_aggregate_store));
if (status != kOK) {
return status;
}
@@ -419,7 +153,7 @@
&key)) {
return kInvalidArguments;
}
- auto locked = protected_aggregate_store_.lock();
+ auto locked = aggregate_store_->protected_aggregate_store_.lock();
auto aggregates = locked->local_aggregate_store.mutable_by_report_key()->find(key);
if (aggregates == locked->local_aggregate_store.mutable_by_report_key()->end()) {
LOG(ERROR) << "The Local Aggregate Store received an unexpected key.";
@@ -508,7 +242,7 @@
Status EventAggregator::LogNumericEvent(const std::string& report_key, uint32_t day_index,
const std::string& component, uint64_t event_code,
int64_t value) {
- auto locked = protected_aggregate_store_.lock();
+ auto locked = aggregate_store_->protected_aggregate_store_.lock();
auto aggregates = locked->local_aggregate_store.mutable_by_report_key()->find(report_key);
if (aggregates == locked->local_aggregate_store.mutable_by_report_key()->end()) {
LOG(ERROR) << "The Local Aggregate Store received an unexpected key.";
@@ -554,33 +288,7 @@
"worker thread was running.";
return kOther;
}
- return GenerateObservations(final_day_index_utc, final_day_index_local);
-}
-
-Status EventAggregator::BackUpLocalAggregateStore() {
- // Lock, copy the LocalAggregateStore, and release the lock. Write the copy
- // to |local_aggregate_proto_store_|.
- auto local_aggregate_store = CopyLocalAggregateStore();
- auto status = local_aggregate_proto_store_->Write(local_aggregate_store);
- if (!status.ok()) {
- LOG(ERROR) << "Failed to back up the LocalAggregateStore with error code: "
- << status.error_code() << "\nError message: " << status.error_message()
- << "\nError details: " << status.error_details();
- return kOther;
- }
- return kOK;
-}
-
-Status EventAggregator::BackUpObservationHistory() {
- auto status = obs_history_proto_store_->Write(obs_history_);
- if (!status.ok()) {
- LOG(ERROR) << "Failed to back up the AggregatedObservationHistoryStore. "
- "::cobalt::util::Status error code: "
- << status.error_code() << "\nError message: " << status.error_message()
- << "\nError details: " << status.error_details();
- return kOther;
- }
- return kOK;
+ return aggregate_store_->GenerateObservations(final_day_index_utc, final_day_index_local);
}
void EventAggregator::ShutDown() {
@@ -608,7 +316,7 @@
// If shutdown has been requested, back up the LocalAggregateStore and
// exit.
if (locked->shut_down) {
- BackUpLocalAggregateStore();
+ aggregate_store_->BackUpLocalAggregateStore();
return;
}
// Sleep until the next scheduled backup of the LocalAggregateStore or
@@ -621,7 +329,7 @@
}
return locked->shut_down;
});
- BackUpLocalAggregateStore();
+ aggregate_store_->BackUpLocalAggregateStore();
// If the worker thread was woken up by a shutdown request, exit.
// Otherwise, complete any scheduled Observation generation and garbage
// collection.
@@ -642,7 +350,7 @@
auto yesterday_local_time = TimeToDayIndex(current_time_t, MetricDefinition::LOCAL) - 1;
// Skip the tasks (but do schedule a retry) if either day index is too small.
- uint32_t min_allowed_day_index = kMaxAllowedAggregationDays + backfill_days_;
+ uint32_t min_allowed_day_index = kMaxAllowedAggregationDays + aggregate_store_->backfill_days_;
bool skip_tasks =
(yesterday_utc < min_allowed_day_index || yesterday_local_time < min_allowed_day_index);
if (steady_time >= next_generate_obs_) {
@@ -651,9 +359,9 @@
LOG_FIRST_N(ERROR, 10) << "EventAggregator is skipping Observation generation because the "
"current day index is too small.";
} else {
- auto obs_status = GenerateObservations(yesterday_utc, yesterday_local_time);
+ auto obs_status = aggregate_store_->GenerateObservations(yesterday_utc, yesterday_local_time);
if (obs_status == kOK) {
- BackUpObservationHistory();
+ aggregate_store_->BackUpObservationHistory();
} else {
LOG(ERROR) << "GenerateObservations failed with status: " << obs_status;
}
@@ -665,9 +373,9 @@
LOG_FIRST_N(ERROR, 10) << "EventAggregator is skipping garbage collection because the "
"current day index is too small.";
} else {
- auto gc_status = GarbageCollect(yesterday_utc, yesterday_local_time);
+ auto gc_status = aggregate_store_->GarbageCollect(yesterday_utc, yesterday_local_time);
if (gc_status == kOK) {
- BackUpLocalAggregateStore();
+ aggregate_store_->BackUpLocalAggregateStore();
} else {
LOG(ERROR) << "GarbageCollect failed with status: " << gc_status;
}
@@ -675,625 +383,4 @@
}
}
-////////////////////// GarbageCollect and helper functions //////////////////
-
-namespace {
-
-void GarbageCollectUniqueActivesReportAggregates(uint32_t day_index, uint32_t max_aggregation_days,
- uint32_t backfill_days,
- UniqueActivesReportAggregates* report_aggregates) {
- auto map_by_event_code = report_aggregates->mutable_by_event_code();
- for (auto event_code = map_by_event_code->begin(); event_code != map_by_event_code->end();) {
- auto map_by_day = event_code->second.mutable_by_day_index();
- for (auto day = map_by_day->begin(); day != map_by_day->end();) {
- if (day->first <= day_index - backfill_days - max_aggregation_days) {
- day = map_by_day->erase(day);
- } else {
- ++day;
- }
- }
- if (map_by_day->empty()) {
- event_code = map_by_event_code->erase(event_code);
- } else {
- ++event_code;
- }
- }
-}
-
-void GarbageCollectNumericReportAggregates(uint32_t day_index, uint32_t max_aggregation_days,
- uint32_t backfill_days,
- PerDeviceNumericAggregates* report_aggregates) {
- auto map_by_component = report_aggregates->mutable_by_component();
- for (auto component = map_by_component->begin(); component != map_by_component->end();) {
- auto map_by_event_code = component->second.mutable_by_event_code();
- for (auto event_code = map_by_event_code->begin(); event_code != map_by_event_code->end();) {
- auto map_by_day = event_code->second.mutable_by_day_index();
- for (auto day = map_by_day->begin(); day != map_by_day->end();) {
- if (day->first <= day_index - backfill_days - max_aggregation_days) {
- day = map_by_day->erase(day);
- } else {
- ++day;
- }
- }
- if (map_by_day->empty()) {
- event_code = map_by_event_code->erase(event_code);
- } else {
- ++event_code;
- }
- }
- if (map_by_event_code->empty()) {
- component = map_by_component->erase(component);
- } else {
- ++component;
- }
- }
-}
-
-} // namespace
-
-Status EventAggregator::GarbageCollect(uint32_t day_index_utc, uint32_t day_index_local) {
- if (day_index_local == 0u) {
- day_index_local = day_index_utc;
- }
- CHECK_LT(day_index_utc, UINT32_MAX);
- CHECK_LT(day_index_local, UINT32_MAX);
- CHECK_GE(day_index_utc, kMaxAllowedAggregationDays + backfill_days_);
- CHECK_GE(day_index_local, kMaxAllowedAggregationDays + backfill_days_);
-
- auto locked = protected_aggregate_store_.lock();
- for (const auto& [report_key, aggregates] : locked->local_aggregate_store.by_report_key()) {
- uint32_t day_index;
- const auto& config = aggregates.aggregation_config();
- switch (config.metric().time_zone_policy()) {
- case MetricDefinition::UTC: {
- day_index = day_index_utc;
- break;
- }
- case MetricDefinition::LOCAL: {
- day_index = day_index_local;
- break;
- }
- default:
- LOG_FIRST_N(ERROR, 10) << "The TimeZonePolicy of this MetricDefinition is invalid.";
- continue;
- }
- if (aggregates.aggregation_config().aggregation_window_size() == 0) {
- LOG_FIRST_N(ERROR, 10) << "This ReportDefinition does not have an aggregation window.";
- continue;
- }
- // PopulateReportAggregates ensured that aggregation_window has at least one element, that all
- // aggregation windows are <= kMaxAllowedAggregationDays, and that config.aggregation_window()
- // is sorted in increasing order.
- uint32_t max_aggregation_days = 1u;
- const OnDeviceAggregationWindow& largest_window =
- config.aggregation_window(config.aggregation_window_size() - 1);
- if (largest_window.units_case() == OnDeviceAggregationWindow::kDays) {
- max_aggregation_days = largest_window.days();
- }
- if (max_aggregation_days == 0u || max_aggregation_days > day_index) {
- LOG_FIRST_N(ERROR, 10) << "The maximum number of aggregation days " << max_aggregation_days
- << " of this ReportDefinition is out of range.";
- continue;
- }
- // For each ReportAggregates, descend to and iterate over the sub-map of
- // local aggregates keyed by day index. Keep buckets with day indices
- // greater than |day_index| - |backfill_days_| - |max_aggregation_days|, and
- // remove all buckets with smaller day indices.
- switch (aggregates.type_case()) {
- case ReportAggregates::kUniqueActivesAggregates: {
- GarbageCollectUniqueActivesReportAggregates(
- day_index, max_aggregation_days, backfill_days_,
- locked->local_aggregate_store.mutable_by_report_key()
- ->at(report_key)
- .mutable_unique_actives_aggregates());
- break;
- }
- case ReportAggregates::kNumericAggregates: {
- GarbageCollectNumericReportAggregates(day_index, max_aggregation_days, backfill_days_,
- locked->local_aggregate_store.mutable_by_report_key()
- ->at(report_key)
- .mutable_numeric_aggregates());
- break;
- }
- default:
- continue;
- }
- }
- return kOK;
-}
-
-Status EventAggregator::GenerateObservations(uint32_t final_day_index_utc,
- uint32_t final_day_index_local) {
- if (final_day_index_local == 0u) {
- final_day_index_local = final_day_index_utc;
- }
- CHECK_LT(final_day_index_utc, UINT32_MAX);
- CHECK_LT(final_day_index_local, UINT32_MAX);
- CHECK_GE(final_day_index_utc, kMaxAllowedAggregationDays + backfill_days_);
- CHECK_GE(final_day_index_local, kMaxAllowedAggregationDays + backfill_days_);
-
- // Lock, copy the LocalAggregateStore, and release the lock. Use the copy to
- // generate observations.
- auto local_aggregate_store = CopyLocalAggregateStore();
- for (const auto& [report_key, aggregates] : local_aggregate_store.by_report_key()) {
- const auto& config = aggregates.aggregation_config();
-
- const auto& metric = config.metric();
- auto metric_ref = MetricRef(&config.project(), &metric);
- uint32_t final_day_index;
- switch (metric.time_zone_policy()) {
- case MetricDefinition::UTC: {
- final_day_index = final_day_index_utc;
- break;
- }
- case MetricDefinition::LOCAL: {
- final_day_index = final_day_index_local;
- break;
- }
- default:
- LOG_FIRST_N(ERROR, 10) << "The TimeZonePolicy of this MetricDefinition is invalid.";
- continue;
- }
-
- const auto& report = config.report();
- // PopulateReportAggregates ensured that aggregation_window has at least one element, that all
- // aggregation windows are <= kMaxAllowedAggregationDays, and that config.aggregation_window()
- // is sorted in increasing order.
- if (config.aggregation_window_size() == 0u) {
- LOG_FIRST_N(ERROR, 10) << "No aggregation_window found for this report.";
- continue;
- }
- uint32_t max_aggregation_days = 1u;
- const OnDeviceAggregationWindow& largest_window =
- config.aggregation_window(config.aggregation_window_size() - 1);
- if (largest_window.units_case() == OnDeviceAggregationWindow::kDays) {
- max_aggregation_days = largest_window.days();
- }
- if (max_aggregation_days == 0u || max_aggregation_days > final_day_index) {
- LOG_FIRST_N(ERROR, 10) << "The maximum number of aggregation days " << max_aggregation_days
- << " of this ReportDefinition is out of range.";
- continue;
- }
- switch (metric.metric_type()) {
- case MetricDefinition::EVENT_OCCURRED: {
- auto num_event_codes = RapporConfigHelper::BasicRapporNumCategories(metric);
-
- switch (report.report_type()) {
- case ReportDefinition::UNIQUE_N_DAY_ACTIVES: {
- auto status = GenerateUniqueActivesObservations(metric_ref, report_key, aggregates,
- num_event_codes, final_day_index);
- if (status != kOK) {
- return status;
- }
- break;
- }
- default:
- continue;
- }
- break;
- }
- case MetricDefinition::EVENT_COUNT:
- case MetricDefinition::ELAPSED_TIME:
- case MetricDefinition::FRAME_RATE:
- case MetricDefinition::MEMORY_USAGE: {
- switch (report.report_type()) {
- case ReportDefinition::PER_DEVICE_NUMERIC_STATS:
- case ReportDefinition::PER_DEVICE_HISTOGRAM: {
- auto status = GenerateObsFromNumericAggregates(metric_ref, report_key, aggregates,
- final_day_index);
- if (status != kOK) {
- return status;
- }
- break;
- }
- default:
- continue;
- }
- break;
- }
- default:
- continue;
- }
- }
- return kOK;
-}
-
-////////// GenerateUniqueActivesObservations and helper methods ////////////////
-
-namespace {
-
-// Given the set of daily aggregates for a fixed event code, and the size and
-// end date of an aggregation window, returns the first day index within that
-// window on which the event code occurred. Returns 0 if the event code did
-// not occur within the window.
-uint32_t FirstActiveDayIndexInWindow(const DailyAggregates& daily_aggregates,
- uint32_t obs_day_index, uint32_t aggregation_days) {
- for (uint32_t day_index = obs_day_index - aggregation_days + 1; day_index <= obs_day_index;
- day_index++) {
- auto day_aggregate = daily_aggregates.by_day_index().find(day_index);
- if (day_aggregate != daily_aggregates.by_day_index().end() &&
- day_aggregate->second.activity_daily_aggregate().activity_indicator()) {
- return day_index;
- }
- }
- return 0u;
-}
-
-// Given the day index of an event occurrence and the size and end date
-// of an aggregation window, returns true if the occurrence falls within
-// the window and false if not.
-bool IsActivityInWindow(uint32_t active_day_index, uint32_t obs_day_index,
- uint32_t aggregation_days) {
- return (active_day_index <= obs_day_index && active_day_index > obs_day_index - aggregation_days);
-}
-
-} // namespace
-
-uint32_t EventAggregator::UniqueActivesLastGeneratedDayIndex(const std::string& report_key,
- uint32_t event_code,
- uint32_t aggregation_days) const {
- auto report_history = obs_history_.by_report_key().find(report_key);
- if (report_history == obs_history_.by_report_key().end()) {
- return 0u;
- }
- auto event_code_history =
- report_history->second.unique_actives_history().by_event_code().find(event_code);
- if (event_code_history == report_history->second.unique_actives_history().by_event_code().end()) {
- return 0u;
- }
- auto window_history = event_code_history->second.by_window_size().find(aggregation_days);
- if (window_history == event_code_history->second.by_window_size().end()) {
- return 0u;
- }
- return window_history->second;
-}
-
-Status EventAggregator::GenerateSingleUniqueActivesObservation(
- const MetricRef metric_ref, const ReportDefinition* report, uint32_t obs_day_index,
- uint32_t event_code, const OnDeviceAggregationWindow& window, bool was_active) const {
- auto encoder_result = encoder_->EncodeUniqueActivesObservation(metric_ref, report, obs_day_index,
- event_code, was_active, window);
- if (encoder_result.status != kOK) {
- return encoder_result.status;
- }
- if (encoder_result.observation == nullptr || encoder_result.metadata == nullptr) {
- LOG(ERROR) << "Failed to encode UniqueActivesObservation";
- return kOther;
- }
-
- auto writer_status = observation_writer_->WriteObservation(*encoder_result.observation,
- std::move(encoder_result.metadata));
- if (writer_status != kOK) {
- return writer_status;
- }
- return kOK;
-}
-
-Status EventAggregator::GenerateUniqueActivesObservations(const MetricRef metric_ref,
- const std::string& report_key,
- const ReportAggregates& report_aggregates,
- uint32_t num_event_codes,
- uint32_t final_day_index) {
- CHECK_GT(final_day_index, backfill_days_);
- // The earliest day index for which we might need to generate an
- // Observation.
- auto backfill_period_start = uint32_t(final_day_index - backfill_days_);
-
- for (uint32_t event_code = 0; event_code < num_event_codes; event_code++) {
- auto daily_aggregates =
- report_aggregates.unique_actives_aggregates().by_event_code().find(event_code);
- // Have any events ever been logged for this report and event code?
- bool found_event_code =
- (daily_aggregates != report_aggregates.unique_actives_aggregates().by_event_code().end());
- for (const auto& window : report_aggregates.aggregation_config().aggregation_window()) {
- // Skip all hourly windows, and all daily windows which are larger than
- // kMaxAllowedAggregationDays.
- //
- // TODO(pesk): Generate observations for hourly windows.
- if (window.units_case() != OnDeviceAggregationWindow::kDays) {
- LOG(INFO) << "Skipping unsupported aggregation window.";
- continue;
- }
- if (window.days() > kMaxAllowedAggregationDays) {
- LOG(WARNING) << "GenerateUniqueActivesObservations ignoring a window "
- "size exceeding the maximum allowed value";
- continue;
- }
- // Find the earliest day index for which an Observation has not yet
- // been generated for this report, event code, and window size. If
- // that day index is later than |final_day_index|, no Observation is
- // generated on this invocation.
- auto last_gen = UniqueActivesLastGeneratedDayIndex(report_key, event_code, window.days());
- auto first_day_index = std::max(last_gen + 1, backfill_period_start);
- // The latest day index on which |event_type| is known to have
- // occurred, so far. This value will be updated as we search
- // forward from the earliest day index belonging to a window of
- // interest.
- uint32_t active_day_index = 0u;
- // Iterate over the day indices |obs_day_index| for which we need
- // to generate Observations. On each iteration, generate an
- // Observation for the |window| ending on |obs_day_index|.
- for (uint32_t obs_day_index = first_day_index; obs_day_index <= final_day_index;
- obs_day_index++) {
- bool was_active = false;
- if (found_event_code) {
- // If the current value of |active_day_index| falls within the
- // window, generate an Observation of activity. If not, search
- // forward in the window, update |active_day_index|, and generate an
- // Observation of activity or inactivity depending on the result of
- // the search.
- if (IsActivityInWindow(active_day_index, obs_day_index, window.days())) {
- was_active = true;
- } else {
- active_day_index =
- FirstActiveDayIndexInWindow(daily_aggregates->second, obs_day_index, window.days());
- was_active = IsActivityInWindow(active_day_index, obs_day_index, window.days());
- }
- }
- auto status = GenerateSingleUniqueActivesObservation(
- metric_ref, &report_aggregates.aggregation_config().report(), obs_day_index, event_code,
- window, was_active);
- if (status != kOK) {
- return status;
- }
- // Update |obs_history_| with the latest date of Observation
- // generation for this report, event code, and window size.
- (*(*(*obs_history_.mutable_by_report_key())[report_key]
- .mutable_unique_actives_history()
- ->mutable_by_event_code())[event_code]
- .mutable_by_window_size())[window.days()] = obs_day_index;
- }
- }
- }
- return kOK;
-}
-
-////////// GenerateObsFromNumericAggregates and helper methods /////////////
-
-uint32_t EventAggregator::PerDeviceNumericLastGeneratedDayIndex(const std::string& report_key,
- const std::string& component,
- uint32_t event_code,
- uint32_t aggregation_days) const {
- const auto& report_history = obs_history_.by_report_key().find(report_key);
- if (report_history == obs_history_.by_report_key().end()) {
- return 0u;
- }
- if (!report_history->second.has_per_device_numeric_history()) {
- return 0u;
- }
- const auto& component_history =
- report_history->second.per_device_numeric_history().by_component().find(component);
- if (component_history ==
- report_history->second.per_device_numeric_history().by_component().end()) {
- return 0u;
- }
- const auto& event_code_history = component_history->second.by_event_code().find(event_code);
- if (event_code_history == component_history->second.by_event_code().end()) {
- return 0u;
- }
- const auto& window_history = event_code_history->second.by_window_size().find(aggregation_days);
- if (window_history == event_code_history->second.by_window_size().end()) {
- return 0u;
- }
- return window_history->second;
-}
-
-uint32_t EventAggregator::ReportParticipationLastGeneratedDayIndex(
- const std::string& report_key) const {
- const auto& report_history = obs_history_.by_report_key().find(report_key);
- if (report_history == obs_history_.by_report_key().end()) {
- return 0u;
- }
- return report_history->second.report_participation_history().last_generated();
-}
-
-Status EventAggregator::GenerateSinglePerDeviceNumericObservation(
- const MetricRef metric_ref, const ReportDefinition* report, uint32_t obs_day_index,
- const std::string& component, uint32_t event_code, const OnDeviceAggregationWindow& window,
- int64_t value) const {
- Encoder::Result encoder_result =
- encoder_->EncodePerDeviceNumericObservation(metric_ref, report, obs_day_index, component,
- UnpackEventCodesProto(event_code), value, window);
- if (encoder_result.status != kOK) {
- return encoder_result.status;
- }
- if (encoder_result.observation == nullptr || encoder_result.metadata == nullptr) {
- LOG(ERROR) << "Failed to encode PerDeviceNumericObservation";
- return kOther;
- }
-
- const auto& writer_status = observation_writer_->WriteObservation(
- *encoder_result.observation, std::move(encoder_result.metadata));
- if (writer_status != kOK) {
- return writer_status;
- }
- return kOK;
-}
-
-Status EventAggregator::GenerateSinglePerDeviceHistogramObservation(
- const MetricRef metric_ref, const ReportDefinition* report, uint32_t obs_day_index,
- const std::string& component, uint32_t event_code, const OnDeviceAggregationWindow& window,
- int64_t value) const {
- Encoder::Result encoder_result = encoder_->EncodePerDeviceHistogramObservation(
- metric_ref, report, obs_day_index, component, UnpackEventCodesProto(event_code), value,
- window);
-
- if (encoder_result.status != kOK) {
- return encoder_result.status;
- }
- if (encoder_result.observation == nullptr || encoder_result.metadata == nullptr) {
- LOG(ERROR) << "Failed to encode PerDeviceNumericObservation";
- return kOther;
- }
-
- const auto& writer_status = observation_writer_->WriteObservation(
- *encoder_result.observation, std::move(encoder_result.metadata));
- if (writer_status != kOK) {
- return writer_status;
- }
- return kOK;
-}
-
-Status EventAggregator::GenerateSingleReportParticipationObservation(const MetricRef metric_ref,
- const ReportDefinition* report,
- uint32_t obs_day_index) const {
- auto encoder_result =
- encoder_->EncodeReportParticipationObservation(metric_ref, report, obs_day_index);
- if (encoder_result.status != kOK) {
- return encoder_result.status;
- }
- if (encoder_result.observation == nullptr || encoder_result.metadata == nullptr) {
- LOG(ERROR) << "Failed to encode ReportParticipationObservation";
- return kOther;
- }
-
- const auto& writer_status = observation_writer_->WriteObservation(
- *encoder_result.observation, std::move(encoder_result.metadata));
- if (writer_status != kOK) {
- return writer_status;
- }
- return kOK;
-}
-
-Status EventAggregator::GenerateObsFromNumericAggregates(const MetricRef metric_ref,
- const std::string& report_key,
- const ReportAggregates& report_aggregates,
- uint32_t final_day_index) {
- CHECK_GT(final_day_index, backfill_days_);
- // The first day index for which we might have to generate an Observation.
- auto backfill_period_start = uint32_t(final_day_index - backfill_days_);
-
- // Generate any necessary PerDeviceNumericObservations for this report.
- for (const auto& [component, event_code_aggregates] :
- report_aggregates.numeric_aggregates().by_component()) {
- for (const auto& [event_code, daily_aggregates] : event_code_aggregates.by_event_code()) {
- // Populate a helper map keyed by day indices which belong to the range
- // [|backfill_period_start|, |final_day_index|]. The value at a day
- // index is the list of windows, in increasing size order, for which an
- // Observation should be generated for that day index.
- std::map<uint32_t, std::vector<OnDeviceAggregationWindow>> windows_by_obs_day;
- for (const auto& window : report_aggregates.aggregation_config().aggregation_window()) {
- if (window.units_case() != OnDeviceAggregationWindow::kDays) {
- LOG(INFO) << "Skipping unsupported aggregation window.";
- continue;
- }
- auto last_gen =
- PerDeviceNumericLastGeneratedDayIndex(report_key, component, event_code, window.days());
- auto first_day_index = std::max(last_gen + 1, backfill_period_start);
- for (auto obs_day_index = first_day_index; obs_day_index <= final_day_index;
- obs_day_index++) {
- windows_by_obs_day[obs_day_index].push_back(window);
- }
- }
- // Iterate over the day indices |obs_day_index| for which we might need
- // to generate an Observation. For each day index, generate an
- // Observation for each needed window.
- //
- // More precisely, for each needed window, iterate over the days within that window. If at
- // least one numeric event was logged during the window, compute the aggregate of the numeric
- // values over the window and generate a PerDeviceNumericObservation. Whether or not a numeric
- // event was found, update the AggregatedObservationHistory for this report, component, event
- // code, and window size with |obs_day_index| as the most recent date of Observation
- // generation. This reflects the fact that all needed Observations were generated for the
- // window ending on that date.
- for (auto obs_day_index = backfill_period_start; obs_day_index <= final_day_index;
- obs_day_index++) {
- const auto& windows = windows_by_obs_day.find(obs_day_index);
- if (windows == windows_by_obs_day.end()) {
- continue;
- }
- bool found_value_for_window = false;
- int64_t window_aggregate = 0;
- uint32_t num_days = 0;
- for (const auto& window : windows->second) {
- while (num_days < window.days()) {
- bool found_value_for_day = false;
- const auto& day_aggregates =
- daily_aggregates.by_day_index().find(obs_day_index - num_days);
- if (day_aggregates != daily_aggregates.by_day_index().end()) {
- found_value_for_day = true;
- }
- const auto& aggregation_type =
- report_aggregates.aggregation_config().report().aggregation_type();
- switch (aggregation_type) {
- case ReportDefinition::SUM:
- if (found_value_for_day) {
- window_aggregate += day_aggregates->second.numeric_daily_aggregate().value();
- found_value_for_window = true;
- }
- break;
- case ReportDefinition::MAX:
- if (found_value_for_day) {
- window_aggregate = std::max(
- window_aggregate, day_aggregates->second.numeric_daily_aggregate().value());
- found_value_for_window = true;
- }
- break;
- case ReportDefinition::MIN:
- if (found_value_for_day && !found_value_for_window) {
- window_aggregate = day_aggregates->second.numeric_daily_aggregate().value();
- found_value_for_window = true;
- } else if (found_value_for_day) {
- window_aggregate = std::min(
- window_aggregate, day_aggregates->second.numeric_daily_aggregate().value());
- }
- break;
- default:
- LOG(ERROR) << "Unexpected aggregation type " << aggregation_type;
- return kInvalidArguments;
- }
- num_days++;
- }
- if (found_value_for_window) {
- Status status;
- const ReportDefinition* report = &report_aggregates.aggregation_config().report();
- switch (report->report_type()) {
- case ReportDefinition::PER_DEVICE_NUMERIC_STATS: {
- status = GenerateSinglePerDeviceNumericObservation(
- metric_ref, report, obs_day_index, component, event_code, window,
- window_aggregate);
- if (status != kOK) {
- return status;
- }
- break;
- }
- case ReportDefinition::PER_DEVICE_HISTOGRAM: {
- auto status = GenerateSinglePerDeviceHistogramObservation(
- metric_ref, report, obs_day_index, component, event_code, window,
- window_aggregate);
- if (status != kOK) {
- return status;
- }
- break;
- }
- default:
- LOG(ERROR) << "Unexpected report type " << report->report_type();
- return kInvalidArguments;
- }
- }
- // Update |obs_history_| with the latest date of Observation
- // generation for this report, component, event code, and window.
- (*(*(*(*obs_history_.mutable_by_report_key())[report_key]
- .mutable_per_device_numeric_history()
- ->mutable_by_component())[component]
- .mutable_by_event_code())[event_code]
- .mutable_by_window_size())[window.days()] = obs_day_index;
- }
- }
- }
- }
- // Generate any necessary ReportParticipationObservations for this report.
- auto participation_last_gen = ReportParticipationLastGeneratedDayIndex(report_key);
- auto participation_first_day_index = std::max(participation_last_gen + 1, backfill_period_start);
- for (auto obs_day_index = participation_first_day_index; obs_day_index <= final_day_index;
- obs_day_index++) {
- GenerateSingleReportParticipationObservation(
- metric_ref, &report_aggregates.aggregation_config().report(), obs_day_index);
- (*obs_history_.mutable_by_report_key())[report_key]
- .mutable_report_participation_history()
- ->set_last_generated(obs_day_index);
- }
- return kOK;
-}
-
} // namespace cobalt::local_aggregation
diff --git a/src/local_aggregation/event_aggregator.h b/src/local_aggregation/event_aggregator.h
index 6a6009c..0c08041 100644
--- a/src/local_aggregation/event_aggregator.h
+++ b/src/local_aggregation/event_aggregator.h
@@ -12,17 +12,11 @@
#include "src/lib/util/clock.h"
#include "src/lib/util/consistent_proto_store.h"
-#include "src/lib/util/protected_fields.h"
-#include "src/local_aggregation/local_aggregation.pb.h"
+#include "src/local_aggregation/aggregate_store.h"
#include "src/logger/encoder.h"
#include "src/logger/event_record.h"
#include "src/logger/observation_writer.h"
#include "src/logger/status.h"
-#include "src/logging.h"
-#include "src/pb/event.pb.h"
-#include "src/pb/observation2.pb.h"
-#include "src/registry/metric_definition.pb.h"
-#include "src/registry/report_definition.pb.h"
namespace cobalt {
@@ -38,65 +32,42 @@
namespace local_aggregation {
-const std::chrono::hours kOneDay(24);
-
-// The EventAggregator manages an in-memory store of aggregated Event values,
-// indexed by report, day index, and other dimensions specific to the report
-// type (e.g. event code). Periodically, this data is used to generate
-// Observations representing aggregates of Event values over a day, week, month,
-// etc.
+// The EventAggregator manages the Loggers' interactions with the local aggregation.
//
// Each Logger interacts with the EventAggregator in the following way:
// (1) When the Logger is created, it calls UpdateAggregationConfigs() to
// provide the EventAggregator with its view of Cobalt's metric and report
// registry.
// (2) When logging an Event for a locally aggregated report, a Logger
-// calls an Update*Aggregation() method with the Event and the
-// ReportAggregationKey of the report.
+// calls a Log*() method with the EventRecord and the report id for the event.
//
-// A worker thread does the following tasks at intervals specified in the
-// EventAggregator's constructor:
-// (1) Backs up the EventAggregator's state to the file system.
-// (2) Calls GenerateObservations() with the previous day's day index to
-// generate all Observations for rolling windows ending on that day index,
-// as well as any missing Observations for a specified number of days in the
-// past.
-// (3) Calls GarbageCollect() to delete daily aggregates which are not
-// needed to compute aggregates for any windows of interest in the future.
+// Functionality that the EventAggregator curently has but will be moved to the
+// EventAggregatorManager:
+// A worker thread calls on AggregateStore methods to do the following tasks at intervals
+// specified in the EventAggregator's constructor:
+// (1) Calls BackUp*() to back up the EventAggregator's state to the file system.
+// (2) Calls GenerateObservations() with the previous day's day index to generate all Observations
+// for rolling windows ending on that day index, as well as any missing Observations for a specified
+// number of days in the past.
+// (3) Calls GarbageCollect() to delete daily aggregates which are not needed to compute aggregates
+// for any windows of interest in the future.
class EventAggregator {
public:
- // Maximum value of |backfill_days| allowed by the constructor.
- static const size_t kMaxAllowedBackfillDays = 1000;
- // All aggregation windows larger than this number of days are ignored.
- static const uint32_t kMaxAllowedAggregationDays = 365;
- // All hourly aggregation windows larger than this number of hours are ignored.
- static const uint32_t kMaxAllowedAggregationHours = 23;
-
- // The current version number of the LocalAggregateStore.
- static const uint32_t kCurrentLocalAggregateStoreVersion = 1;
- // The current version number of the AggregatedObservationHistoryStore.
- static const uint32_t kCurrentObservationHistoryStoreVersion = 0;
-
// Constructs an EventAggregator.
//
- // An EventAggregator maintains daily aggregates of Events in a
- // LocalAggregateStore, uses an Encoder to generate Observations for rolling
- // windows ending on a specified day index, and writes the Observations to
- // an ObservationStore using an ObservationWriter.
- //
// encoder: the singleton instance of an Encoder on the system.
//
// local_aggregate_proto_store: A ConsistentProtoStore to be used by the
- // EventAggregator to store snapshots of its in-memory store of event
+ // AggregateStore to store snapshots of its in-memory store of event
// aggregates.
//
// obs_history_proto_store: A ConsistentProtoStore to be used by the
- // EventAggregator to store snapshots of its in-memory history of generated
+ // AggregateStore to store snapshots of its in-memory history of generated
// Observations.
//
- // backfill_days: the number of past days for which the EventAggregator
+ // backfill_days: the number of past days for which the AggregateStore
// generates and sends Observations, in addition to a requested day index.
- // See the comment above EventAggregator::GenerateObservations for more
+ // See the comment above AggreateStoe::GenerateObservations for more
// detail. The constructor CHECK-fails if a value larger than
// |kEventAggregatorMaxAllowedBackfillDays| is passed.
//
@@ -148,8 +119,8 @@
// ReportDefinition differ from those of the existing report.
logger::Status UpdateAggregationConfigs(const logger::ProjectContext& project_context);
- // Logs an Event associated to a report of type UNIQUE_N_DAY_ACTIVES to the
- // EventAggregator.
+ // Adds an Event associated to a report of type UNIQUE_N_DAY_ACTIVES to the
+ // AggregateStore.
//
// report_id: the ID of the report associated to the logged Event.
//
@@ -170,10 +141,11 @@
// together with valid aggregates when EventAggregator::GarbageCollect() is
// called.
logger::Status LogUniqueActivesEvent(uint32_t report_id, const logger::EventRecord& event_record);
+
// LogCountEvent, LogElapsedTimeEvent:
//
- // Logs an Event associated to a report of type PER_DEVICE_NUMERIC_STATS to
- // the EventAggregator.
+ // Adds an Event associated to a report of type PER_DEVICE_NUMERIC_STATS to
+ // the AggregateStore.
//
// report_id: the ID of the report associated to the logged Event.
//
@@ -194,9 +166,9 @@
// LogMemoryUsageEvent: |event_record| should wrap a MemoryUsageEvent.
logger::Status LogMemoryUsageEvent(uint32_t report_id, const logger::EventRecord& event_record);
- // Checks that the worker thread is shut down, and if so, calls the private
- // method GenerateObservations() and returns its result. Returns kOther if the
- // worker thread is joinable. See the documentation on GenerateObservations()
+ // Checks that the worker thread is shut down, and if so, calls
+ // AggregateStore::GenerateObservations() and returns its result. Returns kOther if the
+ // worker thread is joinable. See the documentation on AggregateStore::GenerateObservations()
// for a description of the parameters.
//
// This method is intended for use in tests which require a single thread to
@@ -206,43 +178,13 @@
private:
friend class EventAggregatorManager; // used for transition during redesign.
- friend class EventAggregatorManagerTest;
- friend class logger::UniqueActivesLoggerTest;
+ friend class AggregateStoreTest;
+ friend class AggregateStoreWorkerTest;
friend class EventAggregatorTest;
+ friend class EventAggregatorManagerTest;
friend class EventAggregatorWorkerTest;
friend class logger::TestEventAggregator;
-
- // Make a LocalAggregateStore which is empty except that its version number is set to |version|.
- LocalAggregateStore MakeNewLocalAggregateStore(
- uint32_t version = kCurrentLocalAggregateStoreVersion);
-
- // Make an AggregatedObservationHistoryStore which is empty except that its version number is set
- // to |version|.
- AggregatedObservationHistoryStore MakeNewObservationHistoryStore(
- uint32_t version = kCurrentObservationHistoryStoreVersion);
-
- // The LocalAggregateStore or AggregatedObservationHistoryStore may need to be changed in ways
- // which are structurally but not semantically backwards-compatible. In other words, the meaning
- // to the EventAggregator of a field in the LocalAggregateStore might change. An example is that
- // we might deprecate one field while introducing a new one.
- //
- // The MaybeUpgrade*Store methods allow the EventAggregator to update the contents of its stored
- // protos from previously meaningful values to currently meaningful values. (For example, a
- // possible implementation would move the contents of a deprecated field to the replacement
- // field.)
- //
- // These methods are called by the EventAggregator constructor immediately after reading in stored
- // protos from disk in order to ensure that proto contents have the expected semantics.
- //
- // The method first checks the version number of the store. If the version number is equal to
- // |kCurrentLocalAggregateStoreVersion| or |kCurrentObservationHistoryStoreVersion|
- // (respectively), returns an OK status. Otherwise, if it is possible to upgrade the store to the
- // current version, does so and returns an OK status. If not, logs an error and returns
- // kInvalidArguments. If a non-OK status is returned, the caller should discard the contents of
- // |store| and replace it with an empty store at the current version. The MakeNew*Store() methods
- // may be used to create the new store.
- logger::Status MaybeUpgradeLocalAggregateStore(LocalAggregateStore* store);
- logger::Status MaybeUpgradeObservationHistoryStore(AggregatedObservationHistoryStore* store);
+ friend class logger::UniqueActivesLoggerTest;
// Request that the worker thread shut down and wait for it to exit. The
// worker thread backs up the LocalAggregateStore before exiting.
@@ -255,69 +197,16 @@
// tasks.
void Run(std::unique_ptr<util::SystemClockInterface> system_clock);
- // Helper method called by Run(). If |next_generate_obs_| is less than or
- // equal to |steady_time|, calls GenerateObservations() with the day index of
- // the previous day from |system_time| in each of UTC and local time, and then
- // backs up the history of generated Observations. If |next_gc_| is less than
- // or equal to |steady_time|, calls GarbageCollect() with the day index of the
- // previous day from |system_time| in each of UTC and local time and then
- // backs up the LocalAggregateStore. In each case, an error is logged and
- // execution continues if the operation fails.
+ // Helper method called by Run(). If |next_generate_obs_| is less than or equal to |steady_time|,
+ // calls AggregateStore::GenerateObservations() with the day index of the previous day from
+ // |system_time| in each of UTC and local time, and then backs up the history of generated
+ // Observations. If |next_gc_| is less than or equal to |steady_time|, calls
+ // AggregateStore::GarbageCollect() with the day index of the previous day from |system_time| in
+ // each of UTC and local time and then backs up the LocalAggregateStore. In each case, an error is
+ // logged and execution continues if the operation fails.
void DoScheduledTasks(std::chrono::system_clock::time_point system_time,
std::chrono::steady_clock::time_point steady_time);
- // Writes a snapshot of the LocalAggregateStore to
- // |local_aggregate_proto_store_|.
- logger::Status BackUpLocalAggregateStore();
-
- // Writes a snapshot of |obs_history_|to |obs_history_proto_store_|.
- logger::Status BackUpObservationHistory();
-
- // Removes from the LocalAggregateStore all daily aggregates that are too
- // old to contribute to their parent report's largest rolling window on the
- // day which is |backfill_days| before |day_index_utc| (if the parent
- // MetricDefinitions' time zone policy is UTC) or which is |backfill_days|
- // before |day_index_local| (if the parent MetricDefinition's time zone policy
- // is LOCAL). If |day_index_local| is 0, then we set |day_index_local| =
- // |day_index_utc|.
- //
- // If the time zone policy of a report's parent metric is UTC (resp., LOCAL)
- // and if day_index is the largest value of the |day_index_utc| (resp.,
- // |day_index_local|) argument with which GarbageCollect() has been called,
- // then the LocalAggregateStore contains the data needed to generate
- // Observations for that report for day index (day_index + k) for any k >= 0.
- logger::Status GarbageCollect(uint32_t day_index_utc, uint32_t day_index_local = 0u);
-
- // Generates one or more Observations for all of the registered locally
- // aggregated reports known to this EventAggregator, for rolling windows
- // ending on specified day indices.
- //
- // Each MetricDefinition specifies a time zone policy, which determines the
- // day index for which an Event associated with that MetricDefinition is
- // logged. For all MetricDefinitions whose Events are logged with respect to
- // UTC, this method generates Observations for rolling windows ending on
- // |final_day_index_utc|. For all MetricDefinitions whose Events are logged
- // with respect to local time, this method generates Observations for rolling
- // windows ending on |final_day_index_local|. If |final_day_index_local| is
- // 0, then we set |final_day_index_local| = |final_day_index_utc|.
- //
- // The generated Observations are written to the |observation_writer| passed
- // to the constructor.
- //
- // This class maintains a history of generated Observations and this method
- // additionally performs backfill: Observations are also generated for
- // rolling windows ending on any day in the interval [final_day_index -
- // backfill_days, final_day_index] (where final_day_index is either
- // final_day_index_utc or final_day_index_local, depending on the time zone
- // policy of the associated MetricDefinition), if this history indicates they
- // were not previously generated. Does not generate duplicate Observations
- // when called multiple times with the same day index.
- //
- // Observations are not generated for aggregation windows larger than
- // |kMaxAllowedAggregationDays|. Hourly windows are not yet supported.
- logger::Status GenerateObservations(uint32_t final_day_index_utc,
- uint32_t final_day_index_local = 0u);
-
// Logs a numeric value to the LocalAggregateStore by adding |value| to the
// current daily aggregate in the bucket indexed by |report_key|, |day_index|,
// |component|, and |event_code|. This is a helper method called by
@@ -325,103 +214,6 @@
logger::Status LogNumericEvent(const std::string& report_key, uint32_t day_index,
const std::string& component, uint64_t event_code, int64_t value);
- // Returns the most recent day index for which an Observation was generated
- // for a given UNIQUE_N_DAY_ACTIVES report, event code, and day-based aggregation window,
- // according to |obs_history_|. Returns 0 if no Observation has been generated
- // for the given arguments.
- uint32_t UniqueActivesLastGeneratedDayIndex(const std::string& report_key, uint32_t event_code,
- uint32_t aggregation_days) const;
-
- // Returns the most recent day index for which an Observation was generated for a given
- // PER_DEVICE_NUMERIC_STATS or PER_DEVICE_HISTOGRAM report, component, event code, and day-based
- // aggregation window, according to |obs_history_|. Returns 0 if no Observation has been generated
- // for the given arguments.
- uint32_t PerDeviceNumericLastGeneratedDayIndex(const std::string& report_key,
- const std::string& component, uint32_t event_code,
- uint32_t aggregation_days) const;
-
- // Returns the most recent day index for which a
- // ReportParticipationObservation was generated for a given report, according
- // to |obs_history_|. Returns 0 if no Observation has been generated for the
- // given arguments.
- uint32_t ReportParticipationLastGeneratedDayIndex(const std::string& report_key) const;
-
- // For a fixed report of type UNIQUE_N_DAY_ACTIVES, generates an Observation
- // for each event code of the parent metric, for each day-based aggregation window of the
- // report ending on |final_day_index|, unless an Observation with those parameters was generated
- // in the past. Also generates Observations for days in the backfill period if needed. Writes the
- // Observations to an ObservationStore via the ObservationWriter that was passed to the
- // constructor.
- //
- // Observations are not generated for aggregation windows larger than
- // |kMaxAllowedAggregationDays|. Hourly windows are not yet supported.
- logger::Status GenerateUniqueActivesObservations(logger::MetricRef metric_ref,
- const std::string& report_key,
- const ReportAggregates& report_aggregates,
- uint32_t num_event_codes,
- uint32_t final_day_index);
-
- // Helper method called by GenerateUniqueActivesObservations() to generate
- // and write a single Observation.
- logger::Status GenerateSingleUniqueActivesObservation(logger::MetricRef metric_ref,
- const ReportDefinition* report,
- uint32_t obs_day_index, uint32_t event_code,
- const OnDeviceAggregationWindow& window,
- bool was_active) const;
-
- // For a fixed report of type PER_DEVICE_NUMERIC_STATS or PER_DEVICE_HISTOGRAM, generates a
- // PerDeviceNumericObservation and PerDeviceHistogramObservation respectively for each
- // tuple (component, event code, aggregation_window) for which a numeric event was logged for that
- // event code and component during the window of that size ending on |final_day_index|, unless an
- // Observation with those parameters has been generated in the past. The value of the Observation
- // is the sum, max, or min (depending on the aggregation_type field of the report definition) of
- // all numeric events logged for that report during the window. Also generates observations for
- // days in the backfill period if needed.
- //
- // In addition to PerDeviceNumericObservations or PerDeviceHistogramObservation , generates
- // a ReportParticipationObservation for |final_day_index| and any needed days in the backfill
- // period. These ReportParticipationObservations are used by the report generators to infer the
- // fleet-wide number of devices for which the sum of numeric events associated to each tuple
- // (component, event code, window size) was zero.
- //
- // Observations are not generated for aggregation windows larger than
- // |kMaxAllowedAggregationWindowSize|.
- logger::Status GenerateObsFromNumericAggregates(logger::MetricRef metric_ref,
- const std::string& report_key,
- const ReportAggregates& report_aggregates,
- uint32_t final_day_index);
-
- // Helper method called by GenerateObsFromNumericAggregates() to generate and write a single
- // Observation with value |value|. The method will produce a PerDeviceNumericObservation or
- // PerDeviceHistogramObservation depending on whether the report type is
- // PER_DEVICE_NUMERIC_STATS or PER_DEVICE_HISTOGRAM respectively.
- logger::Status GenerateSinglePerDeviceNumericObservation(
- logger::MetricRef metric_ref, const ReportDefinition* report, uint32_t obs_day_index,
- const std::string& component, uint32_t event_code, const OnDeviceAggregationWindow& window,
- int64_t value) const;
-
- // Helper method called by GenerateObsFromNumericAggregates() to generate and write a single
- // Observation with value |value|.
- logger::Status GenerateSinglePerDeviceHistogramObservation(
- logger::MetricRef metric_ref, const ReportDefinition* report, uint32_t obs_day_index,
- const std::string& component, uint32_t event_code, const OnDeviceAggregationWindow& window,
- int64_t value) const;
-
- // Helper method called by GenerateObsFromNumericAggregates() to generate and write a single
- // ReportParticipationObservation.
- logger::Status GenerateSingleReportParticipationObservation(logger::MetricRef metric_ref,
- const ReportDefinition* report,
- uint32_t obs_day_index) const;
-
- LocalAggregateStore CopyLocalAggregateStore() {
- auto local_aggregate_store = protected_aggregate_store_.lock()->local_aggregate_store;
- return local_aggregate_store;
- }
-
- struct AggregateStoreFields {
- LocalAggregateStore local_aggregate_store;
- };
-
// Sets the EventAggregator's SteadyClockInterface. Only for use in tests.
void SetSteadyClock(util::SteadyClockInterface* clock) { steady_clock_.reset(clock); }
@@ -447,14 +239,7 @@
std::condition_variable_any shutdown_notifier;
};
- const logger::Encoder* encoder_;
- const logger::ObservationWriter* observation_writer_;
- util::ConsistentProtoStore* local_aggregate_proto_store_;
- util::ConsistentProtoStore* obs_history_proto_store_;
- util::ProtectedFields<AggregateStoreFields> protected_aggregate_store_;
- // Not protected by a mutex. Should only be accessed by |worker_thread_|.
- AggregatedObservationHistoryStore obs_history_;
- size_t backfill_days_ = 0;
+ std::unique_ptr<AggregateStore> aggregate_store_;
std::thread worker_thread_;
util::ProtectedFields<WorkerThreadController> protected_worker_thread_controller_;
diff --git a/src/local_aggregation/event_aggregator_mgr_test.cc b/src/local_aggregation/event_aggregator_mgr_test.cc
index 7bdee94..099e708 100644
--- a/src/local_aggregation/event_aggregator_mgr_test.cc
+++ b/src/local_aggregation/event_aggregator_mgr_test.cc
@@ -122,7 +122,7 @@
uint32_t NumberOfKVPairsInStore(EventAggregatorManager* event_aggregator_mgr) {
return event_aggregator_mgr->GetEventAggregator()
- ->CopyLocalAggregateStore()
+ ->aggregate_store_->CopyLocalAggregateStore()
.by_report_key()
.size();
}
@@ -145,8 +145,7 @@
uint32_t GetNumberOfUniqueActivesAggregates(EventAggregatorManager* event_aggregator_mgr) {
auto local_aggregate_store =
- event_aggregator_mgr->GetEventAggregator()->CopyLocalAggregateStore();
-
+ event_aggregator_mgr->GetEventAggregator()->aggregate_store_->CopyLocalAggregateStore();
uint32_t num_aggregates = 0;
for (const auto& [report_key, aggregates] : local_aggregate_store.by_report_key()) {
if (aggregates.type_case() != ReportAggregates::kUniqueActivesAggregates) {
@@ -166,8 +165,7 @@
const std::shared_ptr<const ProjectContext>& project_context,
const MetricReportId& metric_report_id, uint32_t day_index, uint32_t event_code) {
auto local_aggregate_store =
- event_aggregator_mgr->GetEventAggregator()->CopyLocalAggregateStore();
-
+ event_aggregator_mgr->GetEventAggregator()->aggregate_store_->CopyLocalAggregateStore();
std::string key;
if (!SerializeToBase64(MakeAggregationKey(*project_context, metric_report_id), &key)) {
return AssertionFailure() << "Could not serialize key with metric id "
diff --git a/src/local_aggregation/event_aggregator_test.cc b/src/local_aggregation/event_aggregator_test.cc
index a6635f0..d5c8653 100644
--- a/src/local_aggregation/event_aggregator_test.cc
+++ b/src/local_aggregation/event_aggregator_test.cc
@@ -25,7 +25,7 @@
#include "src/registry/project_configs.h"
#include "third_party/googletest/googletest/include/gtest/gtest.h"
-namespace cobalt {
+namespace cobalt::local_aggregation {
using config::PackEventCodes;
using logger::Encoder;
@@ -36,18 +36,12 @@
using logger::ObservationWriter;
using logger::ProjectContext;
using logger::Status;
-using logger::testing::CheckPerDeviceNumericObservations;
using logger::testing::CheckUniqueActivesObservations;
-using logger::testing::ExpectedAggregationParams;
-using logger::testing::ExpectedPerDeviceNumericObservations;
-using logger::testing::ExpectedReportParticipationObservations;
using logger::testing::ExpectedUniqueActivesObservations;
using logger::testing::FakeObservationStore;
-using logger::testing::FetchAggregatedObservations;
using logger::testing::GetTestProject;
using logger::testing::MakeAggregationConfig;
using logger::testing::MakeAggregationKey;
-using logger::testing::MakeExpectedReportParticipationObservations;
using logger::testing::MakeNullExpectedUniqueActivesObservations;
using logger::testing::MockConsistentProtoStore;
using logger::testing::TestUpdateRecipient;
@@ -59,8 +53,6 @@
using util::SerializeToBase64;
using util::TimeToDayIndex;
-namespace local_aggregation {
-
namespace {
// Number of seconds in a day
constexpr int kDay = 60 * 60 * 24;
@@ -105,12 +97,6 @@
// aggregation configurations.
class EventAggregatorTest : public ::testing::Test {
protected:
- const uint32_t kCurrentLocalAggregateStoreVersion =
- EventAggregator::kCurrentLocalAggregateStoreVersion;
-
- const uint32_t kCurrentObservationHistoryStoreVersion =
- EventAggregator::kCurrentObservationHistoryStoreVersion;
-
void SetUp() override {
observation_store_ = std::make_unique<FakeObservationStore>();
update_recipient_ = std::make_unique<TestUpdateRecipient>();
@@ -158,43 +144,10 @@
time_zone);
}
- size_t GetBackfillDays() { return event_aggregator_->backfill_days_; }
-
- void SetBackfillDays(size_t num_days) { event_aggregator_->backfill_days_ = num_days; }
-
- Status BackUpLocalAggregateStore() { return event_aggregator_->BackUpLocalAggregateStore(); }
-
- Status BackUpObservationHistory() { return event_aggregator_->BackUpObservationHistory(); }
-
- LocalAggregateStore MakeNewLocalAggregateStore(
- uint32_t version = EventAggregator::kCurrentLocalAggregateStoreVersion) {
- return event_aggregator_->MakeNewLocalAggregateStore(version);
- }
-
- AggregatedObservationHistoryStore MakeNewObservationHistoryStore(
- uint32_t version = EventAggregator::kCurrentObservationHistoryStoreVersion) {
- return event_aggregator_->MakeNewObservationHistoryStore(version);
- }
-
- Status MaybeUpgradeLocalAggregateStore(LocalAggregateStore* store) {
- return event_aggregator_->MaybeUpgradeLocalAggregateStore(store);
- }
-
- Status MaybeUpgradeObservationHistoryStore(AggregatedObservationHistoryStore* store) {
- return event_aggregator_->MaybeUpgradeObservationHistoryStore(store);
- }
+ size_t GetBackfillDays() { return event_aggregator_->aggregate_store_->backfill_days_; }
LocalAggregateStore CopyLocalAggregateStore() {
- return event_aggregator_->CopyLocalAggregateStore();
- }
-
- Status GenerateObservations(uint32_t final_day_index_utc, uint32_t final_day_index_local = 0u) {
- return event_aggregator_->GenerateObservationsNoWorker(final_day_index_utc,
- final_day_index_local);
- }
-
- Status GarbageCollect(uint32_t day_index_utc, uint32_t day_index_local = 0u) {
- return event_aggregator_->GarbageCollect(day_index_utc, day_index_local);
+ return event_aggregator_->aggregate_store_->CopyLocalAggregateStore();
}
void TriggerAndWaitForDoScheduledTasks() {
@@ -214,14 +167,6 @@
}
}
- void DoScheduledTasksNow() {
- // Steady values don't matter, just tell DoScheduledTasks to run everything.
- auto steady_time = std::chrono::steady_clock::now();
- event_aggregator_->next_generate_obs_ = steady_time;
- event_aggregator_->next_gc_ = steady_time;
- event_aggregator_->DoScheduledTasks(unowned_test_clock_->now(), steady_time);
- }
-
// Clears the FakeObservationStore and resets the counts of Observations
// received by the FakeObservationStore and the TestUpdateRecipient.
void ResetObservationStore() {
@@ -389,7 +334,7 @@
// of reference.
bool CheckUniqueActivesAggregates(const LoggedActivity& logged_activity,
uint32_t /*current_day_index*/) {
- auto local_aggregate_store = event_aggregator_->CopyLocalAggregateStore();
+ auto local_aggregate_store = event_aggregator_->aggregate_store_->CopyLocalAggregateStore();
// Check that the LocalAggregateStore contains no more UniqueActives
// aggregates than |logged_activity| and |day_last_garbage_collected_|
// should imply.
@@ -504,7 +449,7 @@
bool CheckPerDeviceNumericAggregates(const LoggedValues& logged_values,
uint32_t /*current_day_index*/) {
- auto local_aggregate_store = event_aggregator_->CopyLocalAggregateStore();
+ auto local_aggregate_store = event_aggregator_->aggregate_store_->CopyLocalAggregateStore();
// Check that the LocalAggregateStore contains no more PerDeviceNumeric
// aggregates than |logged_values| and |day_last_garbage_collected_| should
// imply.
@@ -844,16 +789,6 @@
logger::testing::per_device_numeric_stats::kCobaltRegistryBase64) {}
};
-// Creates an EventAggregator and provides it with a ProjectContext generated
-// from test_registries/mixed_time_zone_test_registry.yaml. This registry
-// contains multiple MetricDefinitions with different time zone policies.
-class NoiseFreeMixedTimeZoneEventAggregatorTest : public EventAggregatorTestWithProjectContext {
- protected:
- NoiseFreeMixedTimeZoneEventAggregatorTest()
- : EventAggregatorTestWithProjectContext(
- logger::testing::mixed_time_zone::kCobaltRegistryBase64) {}
-};
-
class PerDeviceHistogramEventAggregatorTest : public EventAggregatorTestWithProjectContext {
protected:
PerDeviceHistogramEventAggregatorTest()
@@ -878,92 +813,6 @@
bool worker_joinable() { return event_aggregator_->worker_thread_.joinable(); }
};
-// Tests that the Read() method of each ConsistentProtoStore is called once
-// during construction of the EventAggregator.
-TEST_F(EventAggregatorTest, ReadProtosFromFiles) {
- EXPECT_EQ(1, local_aggregate_proto_store_->read_count_);
- EXPECT_EQ(1, obs_history_proto_store_->read_count_);
-}
-
-// Tests that the BackUp*() methods return a positive status, and checks that
-// the Write() method of a ConsistentProtoStore is called once when its
-// respective BackUp*() method is called.
-TEST_F(EventAggregatorTest, BackUpProtos) {
- EXPECT_EQ(kOK, BackUpLocalAggregateStore());
- EXPECT_EQ(kOK, BackUpObservationHistory());
- EXPECT_EQ(1, local_aggregate_proto_store_->write_count_);
- EXPECT_EQ(1, obs_history_proto_store_->write_count_);
-}
-
-// MaybeUpgradeLocalAggregateStore should return an OK status if the version is current. The store
-// should not change.
-TEST_F(EventAggregatorTest, MaybeUpgradeLocalAggregateStoreCurrent) {
- auto store = MakeNewLocalAggregateStore();
- std::string store_before = SerializeAsStringDeterministic(store);
- ASSERT_EQ(kCurrentLocalAggregateStoreVersion, store.version());
- EXPECT_EQ(kOK, MaybeUpgradeLocalAggregateStore(&store));
- EXPECT_EQ(store_before, SerializeAsStringDeterministic(store));
-}
-
-// MaybeUpgradeLocalAggregateStore should return kInvalidArguments if it is not possible to upgrade
-// to the current version.
-TEST_F(EventAggregatorTest, MaybeUpgradeLocalAggregateStoreUnsupported) {
- const uint32_t kFutureVersion = kCurrentLocalAggregateStoreVersion + 1;
- auto store = MakeNewLocalAggregateStore(kFutureVersion);
- ASSERT_EQ(kFutureVersion, store.version());
- EXPECT_EQ(kInvalidArguments, MaybeUpgradeLocalAggregateStore(&store));
-}
-
-// It should be possible to upgrade the LocalAggregateStore from v0 to the current version. The
-// version number should be updated and the contents of window_size in each AggregationConfigs
-// should be moved to aggregation_window, preserving their order.
-TEST_F(EventAggregatorTest, MaybeUpgradeLocalAggregateStoreFromV0) {
- const uint32_t kVersionZero = 0;
- const std::vector<uint32_t> kWindowSizes = {1, 7, 30};
- const std::string kKey = "some_report_key";
-
- // Make a v0 LocalAggregateStore with one report.
- auto store = MakeNewLocalAggregateStore(kVersionZero);
- ReportAggregates report_aggregates;
- for (auto window_size : kWindowSizes) {
- report_aggregates.mutable_aggregation_config()->add_window_size(window_size);
- }
- (*store.mutable_by_report_key())[kKey] = report_aggregates;
-
- // Make the expected upgraded store.
- auto expected_store =
- MakeNewLocalAggregateStore(EventAggregator::kCurrentLocalAggregateStoreVersion);
- ReportAggregates expected_report_aggregates;
- for (auto window_size : kWindowSizes) {
- *expected_report_aggregates.mutable_aggregation_config()->add_aggregation_window() =
- MakeDayWindow(window_size);
- }
- (*expected_store.mutable_by_report_key())[kKey] = expected_report_aggregates;
-
- // Upgrade and check that the upgraded store is as expected.
- EXPECT_EQ(kOK, MaybeUpgradeLocalAggregateStore(&store));
- EXPECT_EQ(SerializeAsStringDeterministic(expected_store), SerializeAsStringDeterministic(store));
-}
-
-// MaybeUpgradeObservationHistoryStore should return an OK status if the version is current. The
-// store should not change.
-TEST_F(EventAggregatorTest, MaybeUpgradeObservationHistoryStoreCurrent) {
- auto store = MakeNewObservationHistoryStore();
- std::string store_before = SerializeAsStringDeterministic(store);
- ASSERT_EQ(kCurrentObservationHistoryStoreVersion, store.version());
- EXPECT_EQ(kOK, MaybeUpgradeObservationHistoryStore(&store));
- EXPECT_EQ(store_before, SerializeAsStringDeterministic(store));
-}
-
-// MaybeUpgradeObservationHistoryStore should return kInvalidArguments if it is not possible to
-// upgrade to the current version.
-TEST_F(EventAggregatorTest, MaybeUpgradeObservationHistoryStoreUnsupported) {
- const uint32_t kFutureVersion = kCurrentObservationHistoryStoreVersion + 1;
- auto store = MakeNewObservationHistoryStore(kFutureVersion);
- ASSERT_EQ(kFutureVersion, store.version());
- EXPECT_EQ(kInvalidArguments, MaybeUpgradeObservationHistoryStore(&store));
-}
-
// Tests that an empty LocalAggregateStore is updated with
// ReportAggregationKeys and AggregationConfigs as expected when
// EventAggregator::UpdateAggregationConfigs is called with a ProjectContext
@@ -1101,85 +950,6 @@
bad_event_record3));
}
-// Tests that EventAggregator::GenerateObservations() returns a positive
-// status and that the expected number of Observations is generated when no
-// Events have been logged to the EventAggregator.
-TEST_F(EventAggregatorTest, GenerateObservationsNoEvents) {
- // Provide the all_report_types test registry to the EventAggregator.
- auto project_context = GetTestProject(logger::testing::all_report_types::kCobaltRegistryBase64);
- EXPECT_EQ(kOK, event_aggregator_->UpdateAggregationConfigs(*project_context));
- // Generate locally aggregated Observations for the current day index.
- EXPECT_EQ(kOK, GenerateObservations(CurrentDayIndex()));
- std::vector<Observation2> observations(0);
- EXPECT_TRUE(FetchAggregatedObservations(
- &observations, logger::testing::all_report_types::kExpectedAggregationParams,
- observation_store_.get(), update_recipient_.get()));
-}
-
-// Tests that EventAggregator::GenerateObservations() only generates
-// Observations the first time it is called for a given day index.
-TEST_F(EventAggregatorTest, GenerateObservationsTwice) {
- // Provide the all_report_types test registry to the EventAggregator.
- auto project_context = GetTestProject(logger::testing::all_report_types::kCobaltRegistryBase64);
- EXPECT_EQ(kOK, event_aggregator_->UpdateAggregationConfigs(*project_context));
- // Check that Observations are generated when GenerateObservations is called
- // for the current day index for the first time.
- auto current_day_index = CurrentDayIndex();
- EXPECT_EQ(kOK, GenerateObservations(current_day_index));
- std::vector<Observation2> observations(0);
- EXPECT_TRUE(FetchAggregatedObservations(
- &observations, logger::testing::all_report_types::kExpectedAggregationParams,
- observation_store_.get(), update_recipient_.get()));
- // Check that no Observations are generated when GenerateObservations is
- // called for the currentday index for the second time.
- ResetObservationStore();
- EXPECT_EQ(kOK, GenerateObservations(current_day_index));
- EXPECT_EQ(0u, observation_store_->messages_received.size());
-}
-
-// When the LocalAggregateStore contains one ReportAggregates proto and that
-// proto is empty, GenerateObservations should return success but generate no
-// observations.
-TEST_F(EventAggregatorTest, GenerateObservationsFromBadStore) {
- auto bad_store = std::make_unique<LocalAggregateStore>();
- (*bad_store->mutable_by_report_key())["some_key"] = ReportAggregates();
- local_aggregate_proto_store_->set_stored_proto(std::move(bad_store));
- // Read the bad store in to the EventAggregator.
- ResetEventAggregator();
- EXPECT_EQ(kOK, GenerateObservations(CurrentDayIndex()));
- EXPECT_EQ(0u, observation_store_->messages_received.size());
-}
-
-// When the LocalAggregateStore contains one empty ReportAggregates proto and
-// some valid ReportAggregates, GenerateObservations should produce observations
-// for the valid ReportAggregates.
-TEST_F(EventAggregatorTest, GenerateObservationsFromBadStoreMultiReport) {
- auto bad_store = std::make_unique<LocalAggregateStore>();
- (*bad_store->mutable_by_report_key())["some_key"] = ReportAggregates();
- local_aggregate_proto_store_->set_stored_proto(std::move(bad_store));
- // Read the bad store in to the EventAggregator.
- ResetEventAggregator();
- // Provide the all_report_types test registry to the EventAggregator.
- auto project_context = GetTestProject(logger::testing::all_report_types::kCobaltRegistryBase64);
- EXPECT_EQ(kOK, event_aggregator_->UpdateAggregationConfigs(*project_context));
- EXPECT_EQ(kOK, GenerateObservations(CurrentDayIndex()));
- std::vector<Observation2> observations(0);
- EXPECT_TRUE(FetchAggregatedObservations(
- &observations, logger::testing::all_report_types::kExpectedAggregationParams,
- observation_store_.get(), update_recipient_.get()));
-}
-
-// When the LocalAggregateStore contains one ReportAggregates proto and that
-// proto is empty, GarbageCollect should return success.
-TEST_F(EventAggregatorTest, GarbageCollectBadStore) {
- auto bad_store = std::make_unique<LocalAggregateStore>();
- (*bad_store->mutable_by_report_key())["some_key"] = ReportAggregates();
- local_aggregate_proto_store_->set_stored_proto(std::move(bad_store));
- // Read the bad store in to the EventAggregator.
- ResetEventAggregator();
- EXPECT_EQ(kOK, GarbageCollect(CurrentDayIndex()));
-}
-
// Tests that the LocalAggregateStore is updated as expected when
// EventAggregator::LogUniqueActivesEvent() is called with valid arguments;
// i.e., with a report ID associated to an existing key of the
@@ -1225,938 +995,6 @@
}
}
-// Tests GarbageCollect() for UniqueActivesReportAggregates.
-//
-// For each value of N in the range [0, 34], logs some UniqueActivesEvents
-// each day for N consecutive days and then garbage-collects the
-// LocalAggregateStore. After garbage collection, verifies the contents of
-// the LocalAggregateStore.
-TEST_F(UniqueActivesEventAggregatorTest, GarbageCollect) {
- uint32_t max_days_before_gc = 35;
- for (uint32_t days_before_gc = 0; days_before_gc < max_days_before_gc; days_before_gc++) {
- SetUp();
- day_last_garbage_collected_ = 0u;
- LoggedActivity logged_activity;
- for (uint32_t offset = 0; offset < days_before_gc; offset++) {
- auto day_index = CurrentDayIndex();
- for (const auto& metric_report_id :
- logger::testing::unique_actives::kExpectedAggregationParams.metric_report_ids) {
- // Log 2 events with event code 0.
- EXPECT_EQ(kOK, LogUniqueActivesEvent(metric_report_id, day_index, 0u, &logged_activity));
- EXPECT_EQ(kOK, LogUniqueActivesEvent(metric_report_id, day_index, 0u, &logged_activity));
- if (offset < 3) {
- // Log 1 event with event code 1.
- EXPECT_EQ(kOK, LogUniqueActivesEvent(metric_report_id, day_index, 1u, &logged_activity));
- }
- }
- AdvanceClock(kDay);
- }
- auto end_day_index = CurrentDayIndex();
- EXPECT_EQ(kOK, GarbageCollect(end_day_index));
- day_last_garbage_collected_ = end_day_index;
- EXPECT_TRUE(CheckUniqueActivesAggregates(logged_activity, end_day_index));
- TearDown();
- }
-}
-
-// Tests that EventAggregator::GenerateObservations() returns a positive
-// status and that the expected number of Observations is generated after
-// some UniqueActivesEvents have been logged, without any garbage
-// collection.
-//
-// For 35 days, logs 2 events each day for the NetworkActivity_UniqueDevices
-// reports and 2 events for the FeaturesActive_UniqueDevices report, all
-// with event code 0.
-//
-// Each day, calls GenerateObservations() with the day index of the previous
-// day. Checks that a positive status is returned and that the
-// FakeObservationStore has received the expected number of new observations
-// for each locally aggregated report ID in the unique_actives registry.
-TEST_F(UniqueActivesEventAggregatorTest, GenerateObservations) {
- int num_days = 35;
- std::vector<Observation2> observations(0);
- for (int offset = 0; offset < num_days; offset++) {
- auto day_index = CurrentDayIndex();
- observations.clear();
- ResetObservationStore();
- EXPECT_EQ(kOK, GenerateObservations(day_index - 1));
- EXPECT_TRUE(FetchAggregatedObservations(
- &observations, logger::testing::unique_actives::kExpectedAggregationParams,
- observation_store_.get(), update_recipient_.get()));
- for (int i = 0; i < 2; i++) {
- EXPECT_EQ(kOK, LogUniqueActivesEvent(
- logger::testing::unique_actives::kNetworkActivityWindowSizeMetricReportId,
- day_index, 0u));
- EXPECT_EQ(
- kOK, LogUniqueActivesEvent(
- logger::testing::unique_actives::kNetworkActivityAggregationWindowMetricReportId,
- day_index, 0u));
- EXPECT_EQ(
- kOK, LogUniqueActivesEvent(logger::testing::unique_actives::kFeaturesActiveMetricReportId,
- day_index, 0u));
- }
- AdvanceClock(kDay);
- }
- observations.clear();
- ResetObservationStore();
- EXPECT_EQ(kOK, GenerateObservations(CurrentDayIndex() - 1));
- EXPECT_TRUE(FetchAggregatedObservations(
- &observations, logger::testing::unique_actives::kExpectedAggregationParams,
- observation_store_.get(), update_recipient_.get()));
-}
-
-// Tests that GenerateObservations() returns a positive status and that the
-// expected number of Observations is generated each day when Events are
-// logged for UNIQUE_N_DAY_ACTIVES reports over multiple days, and when the
-// LocalAggregateStore is garbage-collected each day.
-//
-// For 35 days, logs 2 events each day for the NetworkActivity_UniqueDevices
-// reports and 2 events for the FeaturesActive_UniqueDevices report, all
-// with event code 0.
-//
-// Each day following the first day, calls GenerateObservations() and then
-// GarbageCollect() with the day index of the current day. Checks that
-// positive statuses are returned and that the FakeObservationStore has
-// received the expected number of new observations for each locally
-// aggregated report ID in the unique_actives registry.
-TEST_F(UniqueActivesEventAggregatorTest, GenerateObservationsWithGc) {
- int num_days = 35;
- std::vector<Observation2> observations(0);
- for (int offset = 0; offset < num_days; offset++) {
- auto day_index = CurrentDayIndex();
- observations.clear();
- ResetObservationStore();
- EXPECT_EQ(kOK, GenerateObservations(day_index - 1));
- EXPECT_TRUE(FetchAggregatedObservations(
- &observations, logger::testing::unique_actives::kExpectedAggregationParams,
- observation_store_.get(), update_recipient_.get()));
- EXPECT_EQ(kOK, GarbageCollect(day_index));
- for (int i = 0; i < 2; i++) {
- EXPECT_EQ(kOK, LogUniqueActivesEvent(
- logger::testing::unique_actives::kNetworkActivityWindowSizeMetricReportId,
- day_index, 0u));
- EXPECT_EQ(
- kOK, LogUniqueActivesEvent(
- logger::testing::unique_actives::kNetworkActivityAggregationWindowMetricReportId,
- day_index, 0u));
- EXPECT_EQ(
- kOK, LogUniqueActivesEvent(logger::testing::unique_actives::kFeaturesActiveMetricReportId,
- day_index, 0u));
- }
- AdvanceClock(kDay);
- }
- observations.clear();
- ResetObservationStore();
- auto day_index = CurrentDayIndex();
- EXPECT_EQ(kOK, GenerateObservations(day_index - 1));
- EXPECT_TRUE(FetchAggregatedObservations(
- &observations, logger::testing::unique_actives::kExpectedAggregationParams,
- observation_store_.get(), update_recipient_.get()));
- EXPECT_EQ(kOK, GarbageCollect(day_index));
-}
-
-// Tests that GenerateObservations() returns a positive status and that the
-// expected number of Observations is generated when events are logged over
-// multiple days and some of those days' Observations are backfilled, without
-// any garbage collection of the LocalAggregateStore.
-//
-// Sets the |backfill_days_| field of the EventAggregator to 3.
-//
-// Logging pattern:
-// For 35 days, logs 2 events each day for the
-// NetworkActivity_UniqueDevices reports and 2 events for the
-// FeaturesActive_UniqueDevices report, all with event code 0.
-//
-// Observation generation pattern:
-// Calls GenerateObservations() on the 1st through 5th and the 7th out of
-// every 10 days, for 35 days.
-//
-// Expected numbers of Observations:
-// It is expected that 4 days' worth of Observations are generated on
-// the first day of every 10 (the day index for which GenerateObservations()
-// was called, plus 3 days of backfill), that 1 day's worth of Observations
-// are generated on the 2nd through 5th day of every 10, that 2 days'
-// worth of Observations are generated on the 7th day of every 10 (the
-// day index for which GenerateObservations() was called, plus 1 day of
-// backfill), and that no Observations are generated on the remaining days.
-TEST_F(UniqueActivesEventAggregatorTest, GenerateObservationsWithBackfill) {
- // Set |backfill_days_| to 3.
- size_t backfill_days = 3;
- SetBackfillDays(backfill_days);
- // Log 2 events each day for 35 days. Call GenerateObservations() on the
- // first 5 day indices, and the 7th, out of every 10.
- for (int offset = 0; offset < 35; offset++) {
- auto day_index = CurrentDayIndex();
- for (int i = 0; i < 2; i++) {
- EXPECT_EQ(kOK, LogUniqueActivesEvent(
- logger::testing::unique_actives::kNetworkActivityWindowSizeMetricReportId,
- day_index, 0u));
- EXPECT_EQ(
- kOK, LogUniqueActivesEvent(
- logger::testing::unique_actives::kNetworkActivityAggregationWindowMetricReportId,
- day_index, 0u));
- EXPECT_EQ(
- kOK, LogUniqueActivesEvent(logger::testing::unique_actives::kFeaturesActiveMetricReportId,
- day_index, 0u));
- }
- observation_store_->ResetObservationCounter();
- if (offset % 10 < 5 || offset % 10 == 6) {
- EXPECT_EQ(kOK, GenerateObservations(day_index));
- }
- auto num_new_obs = observation_store_->num_observations_added();
- EXPECT_GE(num_new_obs, 0u);
- // Check that the expected daily number of Observations was generated.
- switch (offset % 10) {
- case 0:
- EXPECT_EQ(logger::testing::unique_actives::kExpectedAggregationParams.daily_num_obs *
- (backfill_days + 1),
- num_new_obs);
- break;
- case 1:
- case 2:
- case 3:
- case 4:
- EXPECT_EQ(logger::testing::unique_actives::kExpectedAggregationParams.daily_num_obs,
- num_new_obs);
- break;
- case 6:
- EXPECT_EQ(logger::testing::unique_actives::kExpectedAggregationParams.daily_num_obs * 2,
- num_new_obs);
- break;
- default:
- EXPECT_EQ(0u, num_new_obs);
- }
- AdvanceClock(kDay);
- }
-}
-
-// Tests that GenerateObservations() returns a positive status and that the
-// expected number of Observations is generated when events are logged over
-// multiple days and some of those days' Observations are backfilled, and when
-// the LocalAggregateStore is garbage-collected after each call to
-// GenerateObservations().
-//
-// Sets the |backfill_days_| field of the EventAggregator to 3.
-//
-// Logging pattern:
-// For 35 days, logs 2 events each day for the
-// NetworkActivity_UniqueDevices reports and 2 events for the
-// FeaturesActive_Unique_Devices report, all with event code 0.
-//
-// Observation generation pattern:
-// Calls GenerateObservations() on the 1st through 5th and the 7th out of
-// every 10 days, for 35 days. Garbage-collects the LocalAggregateStore after
-// each call.
-//
-// Expected numbers of Observations:
-// It is expected that 4 days' worth of Observations are generated on
-// the first day of every 10 (the day index for which GenerateObservations()
-// was called, plus 3 days of backfill), that 1 day's worth of Observations
-// are generated on the 2nd through 5th day of every 10, that 2 days'
-// worth of Observations are generated on the 7th day of every 10 (the
-// day index for which GenerateObservations() was called, plus 1 day of
-// backfill), and that no Observations are generated on the remaining days.
-TEST_F(UniqueActivesEventAggregatorTest, GenerateObservationsWithBackfillAndGc) {
- int num_days = 35;
- // Set |backfill_days_| to 3.
- size_t backfill_days = 3;
- SetBackfillDays(backfill_days);
- // Log 2 events each day for 35 days. Call GenerateObservations() on the
- // first 5 day indices, and the 7th, out of every 10.
- for (int offset = 0; offset < num_days; offset++) {
- auto day_index = CurrentDayIndex();
- for (int i = 0; i < 2; i++) {
- EXPECT_EQ(kOK, LogUniqueActivesEvent(
- logger::testing::unique_actives::kNetworkActivityWindowSizeMetricReportId,
- day_index, 0u));
- EXPECT_EQ(
- kOK, LogUniqueActivesEvent(
- logger::testing::unique_actives::kNetworkActivityAggregationWindowMetricReportId,
- day_index, 0u));
- EXPECT_EQ(
- kOK, LogUniqueActivesEvent(logger::testing::unique_actives::kFeaturesActiveMetricReportId,
- day_index, 0u));
- }
- observation_store_->ResetObservationCounter();
- if (offset % 10 < 5 || offset % 10 == 6) {
- EXPECT_EQ(kOK, GenerateObservations(day_index));
- EXPECT_EQ(kOK, GarbageCollect(day_index));
- }
- auto num_new_obs = observation_store_->num_observations_added();
- EXPECT_GE(num_new_obs, 0u);
- // Check that the expected daily number of Observations was generated.
- // This expected number is some multiple of the daily_num_obs field of
- // |kUniqueActivesExpectedParams|, depending on the number of days which
- // should have been backfilled when GenerateObservations() was called.
- switch (offset % 10) {
- case 0:
- EXPECT_EQ(logger::testing::unique_actives::kExpectedAggregationParams.daily_num_obs *
- (backfill_days + 1),
- num_new_obs);
- break;
- case 1:
- case 2:
- case 3:
- case 4:
- EXPECT_EQ(logger::testing::unique_actives::kExpectedAggregationParams.daily_num_obs,
- num_new_obs);
- break;
- case 6:
- EXPECT_EQ(logger::testing::unique_actives::kExpectedAggregationParams.daily_num_obs * 2,
- num_new_obs);
- break;
- default:
- EXPECT_EQ(0u, num_new_obs);
- }
- AdvanceClock(kDay);
- }
-}
-
-// Checks that UniqueActivesObservations with the expected values (i.e.,
-// non-active for all UNIQUE_N_DAY_ACTIVES reports, for all window sizes and
-// event codes) are generated when no Events have been logged to the
-// EventAggregator.
-TEST_F(UniqueActivesNoiseFreeEventAggregatorTest, CheckObservationValuesNoEvents) {
- auto current_day_index = CurrentDayIndex();
- EXPECT_EQ(kOK, GenerateObservations(current_day_index));
- auto expected_obs = MakeNullExpectedUniqueActivesObservations(
- logger::testing::unique_actives_noise_free::kExpectedAggregationParams, current_day_index);
- EXPECT_TRUE(CheckUniqueActivesObservations(expected_obs, observation_store_.get(),
- update_recipient_.get()));
-}
-
-// Checks that UniqueActivesObservations with the expected values are
-// generated when GenerateObservations() is called for a single day index
-// after logging some events for UNIQUE_N_DAY_ACTIVES reports for that day
-// index, without any garbage collection or backfill.
-//
-// Logging pattern:
-// Logs 2 occurrences of event code 0 for the FeaturesActives_UniqueDevices
-// report, and 1 occurrence of event code 1 for the
-// EventsOccurred_UniqueDevices report, all on the same day.
-//
-// Observation generation pattern:
-// Calls GenerateObservations() after logging all events.
-//
-// Expected numbers of Observations:
-// The expected number of Observations is the daily_num_obs field of
-// |logger::testing::unique_actives_noise_free::kExpectedAggregationParams|.
-//
-// Expected Observation values:
-// All Observations should be labeled with the day index on which the events
-// were logged.
-//
-// For the FeaturesActive_UniqueDevices report, expect activity indicators:
-//
-// window size active for event codes
-// ------------------------------------------
-// 1 0
-// 7 0
-// 30 0
-//
-// For the EventsOccurred_UniqueDevices report, expected activity indicators:
-// window size active for event codes
-// ------------------------------------------
-// 1 1
-// 7 1
-//
-// All other Observations should be of inactivity.
-TEST_F(UniqueActivesNoiseFreeEventAggregatorTest, CheckObservationValuesSingleDay) {
- auto day_index = CurrentDayIndex();
- // Log several events on |day_index|.
- EXPECT_EQ(kOK, LogUniqueActivesEvent(
- logger::testing::unique_actives_noise_free::kFeaturesActiveMetricReportId,
- day_index, 0u));
- EXPECT_EQ(kOK, LogUniqueActivesEvent(
- logger::testing::unique_actives_noise_free::kFeaturesActiveMetricReportId,
- day_index, 0u));
- EXPECT_EQ(kOK, LogUniqueActivesEvent(
- logger::testing::unique_actives_noise_free::kEventsOccurredMetricReportId,
- day_index, 1u));
- // Generate locally aggregated Observations for |day_index|.
- EXPECT_EQ(kOK, GenerateObservations(day_index));
-
- // Form the expected observations.
- auto expected_obs = MakeNullExpectedUniqueActivesObservations(
- logger::testing::unique_actives_noise_free::kExpectedAggregationParams, day_index);
- expected_obs[{logger::testing::unique_actives_noise_free::kFeaturesActiveMetricReportId,
- day_index}] = {{1, {true, false, false, false, false}},
- {7, {true, false, false, false, false}},
- {30, {true, false, false, false, false}}};
- expected_obs[{logger::testing::unique_actives_noise_free::kEventsOccurredMetricReportId,
- day_index}] = {{1, {false, true, false, false, false}},
- {7, {false, true, false, false, false}}};
-
- // Check the contents of the FakeObservationStore.
- EXPECT_TRUE(CheckUniqueActivesObservations(expected_obs, observation_store_.get(),
- update_recipient_.get()));
-}
-
-// Checks that UniqueActivesObservations with the expected values are
-// generated when some events have been logged for a UNIQUE_N_DAY_ACTIVES
-// report over multiple days and GenerateObservations() is called each
-// day, without garbage collection or backfill.
-//
-// Logging pattern:
-// Logs events for the EventsOccurred_UniqueDevices report (whose parent
-// metric has max_event_code = 4) for 10 days, according to the following
-// pattern:
-//
-// * Never log event code 0.
-// * On the i-th day (0-indexed) of logging, log an event for event code k,
-// 1 <= k < 5, if 3*k divides i.
-//
-// Observation generation pattern:
-// Each day following the first day, generates Observations for the previous
-// day index.
-//
-// Expected number of Observations:
-// Each call to GenerateObservations should generate a number of Observations
-// equal to the daily_num_obs field of
-// |testing::unique_actives_noise_free::kExpectedAggregationParams|.
-//
-// Expected Observation values:
-// The EventsOccurred_UniqueDevices report has window sizes 1 and 7, and
-// the expected activity indicators of Observations for that report on the
-// i-th day are:
-//
-// (i, window size) active for event codes
-// ------------------------------------------------------
-// (0, 1) 1, 2, 3, 4
-// (0, 7) 1, 2, 3, 4
-// (1, 1) ---
-// (1, 7) 1, 2, 3, 4
-// (2, 1) ---
-// (2, 7) 1, 2, 3, 4
-// (3, 1) 1
-// (3, 7) 1, 2, 3, 4
-// (4, 1) ---
-// (4, 7) 1, 2, 3, 4
-// (5, 1) ---
-// (5, 7) 1, 2, 3, 4
-// (6, 1) 1, 2
-// (6, 7) 1, 2, 3, 4
-// (7, 1) ---
-// (7, 7) 1, 2
-// (8, 1) ---
-// (8, 7) 1, 2
-// (9, 1) 1, 3
-// (9, 7) 1, 2, 3
-//
-// All Observations for all other locally aggregated reports should be
-// observations of non-occurrence.
-TEST_F(UniqueActivesNoiseFreeEventAggregatorTest, CheckObservationValuesMultiDay) {
- auto start_day_index = CurrentDayIndex();
- // Form expected Obsevations for the 10 days of logging.
- uint32_t num_days = 10;
- std::vector<ExpectedUniqueActivesObservations> expected_obs(num_days);
- const auto& expected_id =
- logger::testing::unique_actives_noise_free::kEventsOccurredMetricReportId;
- for (uint32_t offset = 0; offset < num_days; offset++) {
- expected_obs[offset] = MakeNullExpectedUniqueActivesObservations(
- logger::testing::unique_actives_noise_free::kExpectedAggregationParams,
- start_day_index + offset);
- }
- expected_obs[0][{expected_id, start_day_index}] = {{1, {false, true, true, true, true}},
- {7, {false, true, true, true, true}}};
- expected_obs[1][{expected_id, start_day_index + 1}] = {{1, {false, false, false, false, false}},
- {7, {false, true, true, true, true}}};
- expected_obs[2][{expected_id, start_day_index + 2}] = {{1, {false, false, false, false, false}},
- {7, {false, true, true, true, true}}};
- expected_obs[3][{expected_id, start_day_index + 3}] = {{1, {false, true, false, false, false}},
- {7, {false, true, true, true, true}}};
- expected_obs[4][{expected_id, start_day_index + 4}] = {{1, {false, false, false, false, false}},
- {7, {false, true, true, true, true}}};
- expected_obs[5][{expected_id, start_day_index + 5}] = {{1, {false, false, false, false, false}},
- {7, {false, true, true, true, true}}};
- expected_obs[6][{expected_id, start_day_index + 6}] = {{1, {false, true, true, false, false}},
- {7, {false, true, true, true, true}}};
- expected_obs[7][{expected_id, start_day_index + 7}] = {{1, {false, false, false, false, false}},
- {7, {false, true, true, false, false}}};
- expected_obs[8][{expected_id, start_day_index + 8}] = {{1, {false, false, false, false, false}},
- {7, {false, true, true, false, false}}};
- expected_obs[9][{expected_id, start_day_index + 9}] = {{1, {false, true, false, true, false}},
- {7, {false, true, true, true, false}}};
-
- for (uint32_t offset = 0; offset < num_days; offset++) {
- auto day_index = CurrentDayIndex();
- for (uint32_t event_code = 1;
- event_code <
- logger::testing::unique_actives_noise_free::kExpectedAggregationParams.num_event_codes.at(
- expected_id);
- event_code++) {
- if (offset % (3 * event_code) == 0) {
- EXPECT_EQ(kOK, LogUniqueActivesEvent(expected_id, day_index, event_code));
- }
- }
- // Clear the FakeObservationStore.
- ResetObservationStore();
- // Generate locally aggregated Observations.
- EXPECT_EQ(kOK, GenerateObservations(day_index));
- // Check the generated Observations against the expectation.
- EXPECT_TRUE(CheckUniqueActivesObservations(expected_obs[offset], observation_store_.get(),
- update_recipient_.get()));
- AdvanceClock(kDay);
- }
-}
-
-// Checks that UniqueActivesObservations with the expected values are
-// generated when some events have been logged for a UNIQUE_N_DAY_ACTIVES
-// report over multiple days and GenerateObservations() is called each
-// day, and when the LocalAggregateStore is garbage-collected after each call
-// to GenerateObservations().
-//
-// Logging pattern:
-// Logs events for the EventsOccurred_UniqueDevices report (whose parent
-// metric has max_event_code = 4) for 10 days, according to the following
-// pattern:
-//
-// * Never log event code 0.
-// * On the i-th day (0-indexed) of logging, log an event for event code k,
-// 1 <= k < 5, if 3*k divides i.
-//
-// Observation generation pattern:
-// Each day following the first day, generates Observations for the previous
-// day index.
-//
-// Expected number of Observations:
-// Each call to GenerateObservations should generate a number of Observations
-// equal to the daily_num_obs field of
-// |logger::testing::unique_actives_noise_free::kExpectedAggregationParams|.
-//
-// Expected Observation values:
-// The EventsOccurred_UniqueDevices report has window sizes 1 and 7, and
-// the expected activity indicators of Observations for that report on the
-// i-th day are:
-//
-// (i, window size) active for event codes
-// ------------------------------------------------------
-// (0, 1) 1, 2, 3, 4
-// (0, 7) 1, 2, 3, 4
-// (1, 1) ---
-// (1, 7) 1, 2, 3, 4
-// (2, 1) ---
-// (2, 7) 1, 2, 3, 4
-// (3, 1) 1
-// (3, 7) 1, 2, 3, 4
-// (4, 1) ---
-// (4, 7) 1, 2, 3, 4
-// (5, 1) ---
-// (5, 7) 1, 2, 3, 4
-// (6, 1) 1, 2
-// (6, 7) 1, 2, 3, 4
-// (7, 1) ---
-// (7, 7) 1, 2
-// (8, 1) ---
-// (8, 7) 1, 2
-// (9, 1) 1, 3
-// (9, 7) 1, 2, 3
-//
-// All Observations for all other locally aggregated reports should be
-// observations of non-occurrence.
-TEST_F(UniqueActivesNoiseFreeEventAggregatorTest,
- CheckObservationValuesMultiDayWithGarbageCollection) {
- auto start_day_index = CurrentDayIndex();
- // Form expected Observations for the 10 days of logging.
- uint32_t num_days = 10;
- std::vector<ExpectedUniqueActivesObservations> expected_obs(num_days);
- const auto& expected_id =
- logger::testing::unique_actives_noise_free::kEventsOccurredMetricReportId;
-
- for (uint32_t offset = 0; offset < num_days; offset++) {
- expected_obs[offset] = MakeNullExpectedUniqueActivesObservations(
- logger::testing::unique_actives_noise_free::kExpectedAggregationParams,
- start_day_index + offset);
- }
- expected_obs[0][{expected_id, start_day_index}] = {{1, {false, true, true, true, true}},
- {7, {false, true, true, true, true}}};
- expected_obs[1][{expected_id, start_day_index + 1}] = {{1, {false, false, false, false, false}},
- {7, {false, true, true, true, true}}};
- expected_obs[2][{expected_id, start_day_index + 2}] = {{1, {false, false, false, false, false}},
- {7, {false, true, true, true, true}}};
- expected_obs[3][{expected_id, start_day_index + 3}] = {{1, {false, true, false, false, false}},
- {7, {false, true, true, true, true}}};
- expected_obs[4][{expected_id, start_day_index + 4}] = {{1, {false, false, false, false, false}},
- {7, {false, true, true, true, true}}};
- expected_obs[5][{expected_id, start_day_index + 5}] = {{1, {false, false, false, false, false}},
- {7, {false, true, true, true, true}}};
- expected_obs[6][{expected_id, start_day_index + 6}] = {{1, {false, true, true, false, false}},
- {7, {false, true, true, true, true}}};
- expected_obs[7][{expected_id, start_day_index + 7}] = {{1, {false, false, false, false, false}},
- {7, {false, true, true, false, false}}};
- expected_obs[8][{expected_id, start_day_index + 8}] = {{1, {false, false, false, false, false}},
- {7, {false, true, true, false, false}}};
- expected_obs[9][{expected_id, start_day_index + 9}] = {{1, {false, true, false, true, false}},
- {7, {false, true, true, true, false}}};
-
- for (uint32_t offset = 0; offset < num_days; offset++) {
- auto day_index = CurrentDayIndex();
- for (uint32_t event_code = 1;
- event_code <
- logger::testing::unique_actives_noise_free::kExpectedAggregationParams.num_event_codes.at(
- expected_id);
- event_code++) {
- if (offset % (3 * event_code) == 0) {
- EXPECT_EQ(kOK, LogUniqueActivesEvent(expected_id, day_index, event_code));
- }
- }
- // Advance |test_clock_| by 1 day.
- AdvanceClock(kDay);
- // Clear the FakeObservationStore.
- ResetObservationStore();
- // Generate locally aggregated Observations and garbage-collect the
- // LocalAggregateStore, both for the previous day as measured by
- // |test_clock_|. Back up the LocalAggregateStore and
- // AggregatedObservationHistoryStore.
- DoScheduledTasksNow();
- // Check the generated Observations against the expectation.
- EXPECT_TRUE(CheckUniqueActivesObservations(expected_obs[offset], observation_store_.get(),
- update_recipient_.get()));
- }
-}
-
-// Tests that the expected UniqueActivesObservations are generated when events
-// are logged over multiple days and when Observations are backfilled for some
-// days during that period, without any garbage-collection of the
-// LocalAggregateStore.
-//
-// The test sets the number of backfill days to 3.
-//
-// Logging pattern:
-// Events for the EventsOccurred_UniqueDevices report are logged over the days
-// |start_day_index| to |start_day_index + 8| according to the following
-// pattern:
-//
-// * For i = 0 to i = 4, log an event with event code i on day
-// |start_day_index + i| and |start_day_index + 2*i|.
-//
-// Observation generation pattern:
-// The test calls GenerateObservations() on day |start_day_index + i| for i =
-// 0 through i = 5 and for i = 8, skipping the days |start_day_index + 6| and
-// |start_day_index + 7|.
-//
-// Expected numbers of Observations:
-// It is expected that 4 days' worth of Observations are generated on the
-// first day (the day index for which GenerateObservations() was called, plus
-// 3 days of backfill), that 1 day's worth of Observations is generated on the
-// 2nd through 6th day, that 3 days' worth of Observations are generated on
-// the 9th day (the day index for which GenerateObservations() was called,
-// plus 2 days of backfill), and that no Observations are generated on the
-// remaining days.
-//
-// Expected Observation values:
-// The expected activity indicators of Observations for the
-// EventsOccurred_UniqueDevices report for the i-th day of logging are:
-//
-// (i, window size) active for event codes
-// -------------------------------------------------------------------------
-// (0, 1) 0
-// (0, 7) 0
-// (1, 1) 1
-// (1, 7) 0, 1
-// (2, 1) 1, 2
-// (2, 7) 0, 1, 2
-// (3, 1) 3
-// (3, 7) 0, 1, 2, 3
-// (4, 1) 2, 4
-// (4, 7) 0, 1, 2, 3, 4
-// (5, 1) ---
-// (5, 7) 0, 1, 2, 3, 4
-// (6, 1) 3
-// (6, 7) 0, 1, 2, 3, 4
-// (7, 1) ---
-// (7, 7) 1, 2, 3, 4
-// (8, 1) 4
-// (8, 7) 1, 2, 3, 4
-//
-// All other Observations should be of non-activity.
-TEST_F(UniqueActivesNoiseFreeEventAggregatorTest, CheckObservationValuesWithBackfill) {
- auto start_day_index = CurrentDayIndex();
- // Set |backfill_days_| to 3.
- size_t backfill_days = 3;
- SetBackfillDays(backfill_days);
- const auto& expected_id =
- logger::testing::unique_actives_noise_free::kEventsOccurredMetricReportId;
- const auto& expected_params =
- logger::testing::unique_actives_noise_free::kExpectedAggregationParams;
- // Log events for 9 days. Call GenerateObservations() on the first 6 day
- // indices, and the 9th.
- for (uint32_t offset = 0; offset < 9; offset++) {
- auto day_index = CurrentDayIndex();
- ResetObservationStore();
- for (uint32_t event_code = 0; event_code < expected_params.num_event_codes.at(expected_id);
- event_code++) {
- if (event_code == offset || (2 * event_code) == offset) {
- EXPECT_EQ(kOK, LogUniqueActivesEvent(expected_id, day_index, event_code));
- }
- }
- if (offset < 6 || offset == 8) {
- EXPECT_EQ(kOK, GenerateObservations(day_index));
- }
- // Make the set of Observations which are expected to be generated on
- // |start_day_index + offset| and check it against the contents of the
- // FakeObservationStore.
- ExpectedUniqueActivesObservations expected_obs;
- switch (offset) {
- case 0: {
- for (uint32_t day_index = start_day_index - backfill_days; day_index <= start_day_index;
- day_index++) {
- for (const auto& pair :
- MakeNullExpectedUniqueActivesObservations(expected_params, day_index)) {
- expected_obs.insert(pair);
- }
- }
- expected_obs[{expected_id, start_day_index}] = {{1, {true, false, false, false, false}},
- {7, {true, false, false, false, false}}};
- EXPECT_TRUE(CheckUniqueActivesObservations(expected_obs, observation_store_.get(),
- update_recipient_.get()));
- break;
- }
- case 1: {
- expected_obs =
- MakeNullExpectedUniqueActivesObservations(expected_params, start_day_index + 1);
- expected_obs[{expected_id, start_day_index + 1}] = {{1, {false, true, false, false, false}},
- {7, {true, true, false, false, false}}};
- EXPECT_TRUE(CheckUniqueActivesObservations(expected_obs, observation_store_.get(),
- update_recipient_.get()));
- break;
- }
- case 2: {
- expected_obs =
- MakeNullExpectedUniqueActivesObservations(expected_params, start_day_index + 2);
- expected_obs[{expected_id, start_day_index + 2}] = {{1, {false, true, true, false, false}},
- {7, {true, true, true, false, false}}};
- EXPECT_TRUE(CheckUniqueActivesObservations(expected_obs, observation_store_.get(),
- update_recipient_.get()));
- break;
- }
- case 3: {
- expected_obs =
- MakeNullExpectedUniqueActivesObservations(expected_params, start_day_index + 3);
- expected_obs[{expected_id, start_day_index + 3}] = {{1, {false, false, false, true, false}},
- {7, {true, true, true, true, false}}};
- EXPECT_TRUE(CheckUniqueActivesObservations(expected_obs, observation_store_.get(),
- update_recipient_.get()));
- break;
- }
- case 4: {
- expected_obs =
- MakeNullExpectedUniqueActivesObservations(expected_params, start_day_index + 4);
- expected_obs[{expected_id, start_day_index + 4}] = {{1, {false, false, true, false, true}},
- {7, {true, true, true, true, true}}};
- EXPECT_TRUE(CheckUniqueActivesObservations(expected_obs, observation_store_.get(),
- update_recipient_.get()));
- break;
- }
- case 5: {
- expected_obs =
- MakeNullExpectedUniqueActivesObservations(expected_params, start_day_index + 5);
- expected_obs[{expected_id, start_day_index + 5}] = {
- {1, {false, false, false, false, false}}, {7, {true, true, true, true, true}}};
- EXPECT_TRUE(CheckUniqueActivesObservations(expected_obs, observation_store_.get(),
- update_recipient_.get()));
- break;
- }
- case 8: {
- for (uint32_t day_index = start_day_index + 6; day_index <= start_day_index + 8;
- day_index++) {
- for (const auto& pair :
- MakeNullExpectedUniqueActivesObservations(expected_params, day_index)) {
- expected_obs.insert(pair);
- }
- }
- expected_obs[{expected_id, start_day_index + 6}] = {{1, {false, false, false, true, false}},
- {7, {true, true, true, true, true}}};
- expected_obs[{expected_id, start_day_index + 7}] = {
- {1, {false, false, false, false, false}}, {7, {false, true, true, true, true}}};
- expected_obs[{expected_id, start_day_index + 8}] = {{1, {false, false, false, false, true}},
- {7, {false, true, true, true, true}}};
- EXPECT_TRUE(CheckUniqueActivesObservations(expected_obs, observation_store_.get(),
- update_recipient_.get()));
- break;
- }
- default:
- EXPECT_TRUE(CheckUniqueActivesObservations(expected_obs, observation_store_.get(),
- update_recipient_.get()));
- }
- AdvanceClock(kDay);
- }
-}
-
-// Tests that the expected UniqueActivesObservations are generated when events
-// are logged over multiple days and when Observations are backfilled for some
-// days during that period, and when the LocalAggregateStore is
-// garbage-collected after each call to GenerateObservations().
-//
-// The test sets the number of backfill days to 3.
-//
-// Logging pattern:
-// Events for the EventsOccurred_UniqueDevices report are logged over the days
-// |start_day_index| to |start_day_index + 8| according to the following
-// pattern:
-//
-// * For i = 0 to i = 4, log an event with event code i on day
-// |start_day_index + i| and |start_day_index + 2*i|.
-//
-// Observation generation pattern:
-// The test calls GenerateObservations() on day |start_day_index + i| for i =
-// 0 through i = 5 and for i = 8, skipping the days |start_day_index + 6| and
-// |start_day_index + 7|.
-//
-// Expected numbers of Observations:
-// It is expected that 4 days' worth of Observations are generated on the
-// first day (the day index for which GenerateObservations() was called, plus
-// 3 days of backfill), that 1 day's worth of Observations is generated on the
-// 2nd through 6th day, that 3 days' worth of Observations are generated on
-// the 9th day (the day index for which GenerateObservations() was called,
-// plus 2 days of backfill), and that no Observations are generated on the
-// remaining days.
-//
-// Expected Observation values:
-// The expected activity indicators of Observations for the
-// EventsOccurred_UniqueDevices report for the i-th day of logging are:
-//
-// (i, window size) active for event codes
-// -------------------------------------------------------------------------
-// (0, 1) 0
-// (0, 7) 0
-// (1, 1) 1
-// (1, 7) 0, 1
-// (2, 1) 1, 2
-// (2, 7) 0, 1, 2
-// (3, 1) 3
-// (3, 7) 0, 1, 2, 3
-// (4, 1) 2, 4
-// (4, 7) 0, 1, 2, 3, 4
-// (5, 1) ---
-// (5, 7) 0, 1, 2, 3, 4
-// (6, 1) 3
-// (6, 7) 0, 1, 2, 3, 4
-// (7, 1) ---
-// (7, 7) 1, 2, 3, 4
-// (8, 1) 4
-// (8, 7) 1, 2, 3, 4
-//
-// All other Observations should be of non-activity.
-TEST_F(UniqueActivesNoiseFreeEventAggregatorTest, CheckObservationValuesWithBackfillAndGc) {
- auto start_day_index = CurrentDayIndex();
- // Set |backfill_days_| to 3.
- size_t backfill_days = 3;
- SetBackfillDays(backfill_days);
-
- const auto& expected_id =
- logger::testing::unique_actives_noise_free::kEventsOccurredMetricReportId;
- const auto& expected_params =
- logger::testing::unique_actives_noise_free::kExpectedAggregationParams;
-
- // Log events for 9 days. Call GenerateObservations() on the first 6 day
- // indices, and the 9th.
- for (uint32_t offset = 0; offset < 8; offset++) {
- auto day_index = CurrentDayIndex();
- ResetObservationStore();
- for (uint32_t event_code = 0; event_code < expected_params.num_event_codes.at(expected_id);
- event_code++) {
- if (event_code == offset || (2 * event_code) == offset) {
- EXPECT_EQ(kOK, LogUniqueActivesEvent(expected_id, day_index, event_code));
- }
- }
- // Advance |test_clock_| by 1 day.
- AdvanceClock(kDay);
- if (offset < 6 || offset == 9) {
- // Generate Observations and garbage-collect, both for the previous day
- // index according to |test_clock_|. Back up the LocalAggregateStore and
- // the AggregatedObservationHistoryStore.
- DoScheduledTasksNow();
- }
- // Make the set of Observations which are expected to be generated on
- // |start_day_index + offset| and check it against the contents of the
- // FakeObservationStore.
- ExpectedUniqueActivesObservations expected_obs;
- switch (offset) {
- case 0: {
- for (uint32_t day_index = start_day_index - backfill_days; day_index <= start_day_index;
- day_index++) {
- for (const auto& pair :
- MakeNullExpectedUniqueActivesObservations(expected_params, day_index)) {
- expected_obs.insert(pair);
- }
- }
- expected_obs[{expected_id, start_day_index}] = {{1, {true, false, false, false, false}},
- {7, {true, false, false, false, false}}};
- EXPECT_TRUE(CheckUniqueActivesObservations(expected_obs, observation_store_.get(),
- update_recipient_.get()));
- break;
- }
- case 1: {
- expected_obs =
- MakeNullExpectedUniqueActivesObservations(expected_params, start_day_index + 1);
- expected_obs[{expected_id, start_day_index + 1}] = {{1, {false, true, false, false, false}},
- {7, {true, true, false, false, false}}};
- EXPECT_TRUE(CheckUniqueActivesObservations(expected_obs, observation_store_.get(),
- update_recipient_.get()));
- break;
- }
- case 2: {
- expected_obs =
- MakeNullExpectedUniqueActivesObservations(expected_params, start_day_index + 2);
- expected_obs[{expected_id, start_day_index + 2}] = {{1, {false, true, true, false, false}},
- {7, {true, true, true, false, false}}};
- EXPECT_TRUE(CheckUniqueActivesObservations(expected_obs, observation_store_.get(),
- update_recipient_.get()));
- break;
- }
- case 3: {
- expected_obs =
- MakeNullExpectedUniqueActivesObservations(expected_params, start_day_index + 3);
- expected_obs[{expected_id, start_day_index + 3}] = {{1, {false, false, false, true, false}},
- {7, {true, true, true, true, false}}};
- EXPECT_TRUE(CheckUniqueActivesObservations(expected_obs, observation_store_.get(),
- update_recipient_.get()));
- break;
- }
- case 4: {
- expected_obs =
- MakeNullExpectedUniqueActivesObservations(expected_params, start_day_index + 4);
- expected_obs[{expected_id, start_day_index + 4}] = {{1, {false, false, true, false, true}},
- {7, {true, true, true, true, true}}};
- EXPECT_TRUE(CheckUniqueActivesObservations(expected_obs, observation_store_.get(),
- update_recipient_.get()));
- break;
- }
- case 5: {
- expected_obs =
- MakeNullExpectedUniqueActivesObservations(expected_params, start_day_index + 5);
- expected_obs[{expected_id, start_day_index + 5}] = {
- {1, {false, false, false, false, false}}, {7, {true, true, true, true, true}}};
- EXPECT_TRUE(CheckUniqueActivesObservations(expected_obs, observation_store_.get(),
- update_recipient_.get()));
- break;
- }
- case 8: {
- for (uint32_t day_index = start_day_index + 6; day_index <= start_day_index + 8;
- day_index++) {
- for (const auto& pair :
- MakeNullExpectedUniqueActivesObservations(expected_params, day_index)) {
- expected_obs.insert(pair);
- }
- }
- expected_obs[{expected_id, start_day_index + 6}] = {{1, {false, false, false, true, false}},
- {7, {true, true, true, true, true}}};
- expected_obs[{expected_id, start_day_index + 7}] = {
- {1, {false, false, false, false, false}}, {7, {false, true, true, true, true}}};
- expected_obs[{expected_id, start_day_index + 8}] = {{1, {false, false, false, false, true}},
- {7, {false, true, true, true, true}}};
- EXPECT_TRUE(CheckUniqueActivesObservations(expected_obs, observation_store_.get(),
- update_recipient_.get()));
- break;
- }
- default:
- EXPECT_TRUE(CheckUniqueActivesObservations(expected_obs, observation_store_.get(),
- update_recipient_.get()));
- }
- }
-}
-
// Checks that UniqueActivesObservations with the expected values are
// generated by the the scheduled Observation generation when some events
// have been logged for a UNIQUE_N_DAY_ACTIVES.
@@ -2287,1260 +1125,6 @@
EXPECT_TRUE(CheckPerDeviceNumericAggregates(logged_values, day_index));
AdvanceClock(kDay);
}
-} // namespace local_aggregation
-
-// Tests GarbageCollect() for PerDeviceNumericReportAggregates.
-//
-// For each value of N in the range [0, 34], logs some events for
-// PerDeviceNumeric reports each day for N consecutive days, and then
-// garbage-collects the LocalAggregateStore. After garbage collection, verifies
-// the contents of the LocalAggregateStore.
-TEST_F(PerDeviceNumericEventAggregatorTest, GarbageCollect) {
- uint32_t max_days_before_gc = 35;
- for (uint32_t days_before_gc = 0; days_before_gc < max_days_before_gc; days_before_gc++) {
- SetUp();
- day_last_garbage_collected_ = 0u;
- LoggedValues logged_values;
- std::vector<MetricReportId> count_metric_report_ids = {
- logger::testing::per_device_numeric_stats::kSettingsChangedWindowSizeMetricReportId,
- logger::testing::per_device_numeric_stats::kSettingsChangedAggregationWindowMetricReportId,
- logger::testing::per_device_numeric_stats::kConnectionFailuresMetricReportId};
- std::vector<MetricReportId> elapsed_time_metric_report_ids = {
- logger::testing::per_device_numeric_stats::kStreamingTimeTotalMetricReportId,
- logger::testing::per_device_numeric_stats::kStreamingTimeMinMetricReportId,
- logger::testing::per_device_numeric_stats::kStreamingTimeMaxMetricReportId};
- MetricReportId frame_rate_metric_report_id =
- logger::testing::per_device_numeric_stats::kLoginModuleFrameRateMinMetricReportId;
- MetricReportId memory_usage_metric_report_id =
- logger::testing::per_device_numeric_stats::kLedgerMemoryUsageMaxMetricReportId;
- for (uint32_t offset = 0; offset < days_before_gc; offset++) {
- auto day_index = CurrentDayIndex();
- for (const auto& id : count_metric_report_ids) {
- for (const auto& component : {"component_A", "component_B", "component_C"}) {
- // Log 2 events with event code 0, for each component A, B, C.
- EXPECT_EQ(kOK, LogPerDeviceCountEvent(id, day_index, component, 0u, 2, &logged_values));
- EXPECT_EQ(kOK, LogPerDeviceCountEvent(id, day_index, component, 0u, 3, &logged_values));
- }
- if (offset < 3) {
- // Log 1 event for component D and event code 1.
- EXPECT_EQ(kOK,
- LogPerDeviceCountEvent(id, day_index, "component_D", 1u, 4, &logged_values));
- }
- }
- for (const auto& id : elapsed_time_metric_report_ids) {
- for (const auto& component : {"component_A", "component_B", "component_C"}) {
- // Log 2 events with event code 0, for each component A, B, C.
- EXPECT_EQ(kOK,
- LogPerDeviceElapsedTimeEvent(id, day_index, component, 0u, 2, &logged_values));
- EXPECT_EQ(kOK,
- LogPerDeviceElapsedTimeEvent(id, day_index, component, 0u, 3, &logged_values));
- }
- if (offset < 3) {
- // Log 1 event for component D and event code 1.
- EXPECT_EQ(kOK, LogPerDeviceElapsedTimeEvent(id, day_index, "component_D", 1u, 4,
- &logged_values));
- }
- }
- for (const auto& component : {"component_A", "component_B"}) {
- EXPECT_EQ(kOK, LogPerDeviceFrameRateEvent(frame_rate_metric_report_id, day_index, component,
- 0u, 2.25, &logged_values));
- EXPECT_EQ(kOK, LogPerDeviceFrameRateEvent(frame_rate_metric_report_id, day_index, component,
- 0u, 1.75, &logged_values));
- EXPECT_EQ(kOK,
- LogPerDeviceMemoryUsageEvent(memory_usage_metric_report_id, day_index, component,
- std::vector<uint32_t>{0u, 0u}, 300, &logged_values));
- EXPECT_EQ(kOK,
- LogPerDeviceMemoryUsageEvent(memory_usage_metric_report_id, day_index, component,
- std::vector<uint32_t>{1u, 0u}, 300, &logged_values));
- }
- AdvanceClock(kDay);
- }
- auto end_day_index = CurrentDayIndex();
- EXPECT_EQ(kOK, GarbageCollect(end_day_index));
- day_last_garbage_collected_ = end_day_index;
- EXPECT_TRUE(CheckPerDeviceNumericAggregates(logged_values, end_day_index));
- TearDown();
- }
-}
-
-// Tests that EventAggregator::GenerateObservations() returns a positive
-// status and that the expected number of Observations is generated after
-// some CountEvents have been logged for PerDeviceNumericStats reports, without
-// any garbage collection.
-//
-// For 35 days, logs a positive number of events each day for the
-// ConnectionFailures_PerDeviceNumericStats report with "component_A" and for
-// the SettingsChanged_PerDeviceNumericStats reports with "component_B", all with
-// event code 0.
-//
-// Each day, calls GenerateObservations() with the day index of the previous
-// day. Checks that a positive status is returned and that the
-// FakeObservationStore has received the expected number of new observations
-// for each locally aggregated report ID in the per_device_numeric_stats test
-// registry.
-TEST_F(PerDeviceNumericEventAggregatorTest, GenerateObservations) {
- int num_days = 1;
- std::vector<Observation2> observations(0);
- ExpectedAggregationParams expected_params =
- logger::testing::per_device_numeric_stats::kExpectedAggregationParams;
- for (int offset = 0; offset < num_days; offset++) {
- auto day_index = CurrentDayIndex();
- observations.clear();
- ResetObservationStore();
- EXPECT_EQ(kOK, GenerateObservations(day_index - 1));
- EXPECT_TRUE(FetchAggregatedObservations(&observations, expected_params,
- observation_store_.get(), update_recipient_.get()));
- for (int i = 0; i < 2; i++) {
- EXPECT_EQ(kOK,
- LogPerDeviceCountEvent(
- logger::testing::per_device_numeric_stats::kConnectionFailuresMetricReportId,
- day_index, "component_A", 0u, 1));
- EXPECT_EQ(kOK,
- LogPerDeviceCountEvent(
- logger::testing::per_device_numeric_stats::kConnectionFailuresMetricReportId,
- day_index, "component_A", 0u, 1));
- EXPECT_EQ(
- kOK,
- LogPerDeviceCountEvent(
- logger::testing::per_device_numeric_stats::kSettingsChangedWindowSizeMetricReportId,
- day_index, "component_B", 0u, 5));
- EXPECT_EQ(kOK, LogPerDeviceCountEvent(logger::testing::per_device_numeric_stats::
- kSettingsChangedAggregationWindowMetricReportId,
- day_index, "component_B", 0u, 5));
- }
- // If this is the first time we're logging events, update the expected
- // numbers of generated Observations to account for the logged events.
- // For each report, for each aggregation window, expect 1 Observation more than if
- // no events had been logged.
- if (offset == 0) {
- expected_params.daily_num_obs += 5;
- expected_params.num_obs_per_report
- [logger::testing::per_device_numeric_stats::kConnectionFailuresMetricReportId] += 1;
- expected_params.num_obs_per_report
- [logger::testing::per_device_numeric_stats::kSettingsChangedWindowSizeMetricReportId] +=
- 2;
- expected_params.num_obs_per_report[logger::testing::per_device_numeric_stats::
- kSettingsChangedAggregationWindowMetricReportId] += 2;
- }
- AdvanceClock(kDay);
- }
- observations.clear();
- ResetObservationStore();
- EXPECT_EQ(kOK, GenerateObservations(CurrentDayIndex() - 1));
- EXPECT_TRUE(FetchAggregatedObservations(&observations, expected_params, observation_store_.get(),
- update_recipient_.get()));
-}
-
-// Tests that EventAggregator::GenerateObservations() returns a positive
-// status and that the expected number of Observations is generated after
-// some CountEvents have been logged for PerDeviceNumeric reports over multiple
-// days, and when the LocalAggregateStore is garbage-collected each day.
-//
-// For 35 days, logs a positive number of events each day for the
-// ConnectionFailures_PerDeviceNumeric report with "component_A" and for
-// the SettingsChanged_PerDeviceNumeric report with "component_B", all with
-// event code 0.
-//
-// Each day, calls GenerateObservations() with the day index of the previous
-// day. Checks that a positive status is returned and that the
-// FakeObservationStore has received the expected number of new observations
-// for each locally aggregated report ID in the per_device_numeric_stats test
-// registry.
-TEST_F(PerDeviceNumericEventAggregatorTest, GenerateObservationsWithGc) {
- int num_days = 35;
- std::vector<Observation2> observations(0);
- ExpectedAggregationParams expected_params =
- logger::testing::per_device_numeric_stats::kExpectedAggregationParams;
- for (int offset = 0; offset < num_days; offset++) {
- auto day_index = CurrentDayIndex();
- observations.clear();
- ResetObservationStore();
- EXPECT_EQ(kOK, GenerateObservations(day_index - 1));
- EXPECT_TRUE(FetchAggregatedObservations(&observations, expected_params,
- observation_store_.get(), update_recipient_.get()));
- EXPECT_EQ(kOK, GarbageCollect(day_index));
- for (int i = 0; i < 2; i++) {
- EXPECT_EQ(kOK,
- LogPerDeviceCountEvent(
- logger::testing::per_device_numeric_stats::kConnectionFailuresMetricReportId,
- day_index, "component_A", 0u, 1));
- EXPECT_EQ(kOK,
- LogPerDeviceCountEvent(
- logger::testing::per_device_numeric_stats::kConnectionFailuresMetricReportId,
- day_index, "component_A", 0u, 1));
- EXPECT_EQ(
- kOK,
- LogPerDeviceCountEvent(
- logger::testing::per_device_numeric_stats::kSettingsChangedWindowSizeMetricReportId,
- day_index, "component_B", 0u, 5));
- EXPECT_EQ(kOK, LogPerDeviceCountEvent(logger::testing::per_device_numeric_stats::
- kSettingsChangedAggregationWindowMetricReportId,
- day_index, "component_B", 0u, 5));
- }
- // If this is the first time we're logging events, update the expected
- // numbers of generated Observations to account for the logged events.
- // For each report, for each window size, expect 1 Observation more than if
- // no events had been logged.
- if (offset == 0) {
- expected_params.daily_num_obs += 5;
- expected_params.num_obs_per_report
- [logger::testing::per_device_numeric_stats::kConnectionFailuresMetricReportId] += 1;
- expected_params.num_obs_per_report
- [logger::testing::per_device_numeric_stats::kSettingsChangedWindowSizeMetricReportId] +=
- 2;
- expected_params.num_obs_per_report[logger::testing::per_device_numeric_stats::
- kSettingsChangedAggregationWindowMetricReportId] += 2;
- }
- AdvanceClock(kDay);
- }
- observations.clear();
- ResetObservationStore();
- auto day_index = CurrentDayIndex();
- EXPECT_EQ(kOK, GenerateObservations(day_index - 1));
- EXPECT_TRUE(FetchAggregatedObservations(&observations, expected_params, observation_store_.get(),
- update_recipient_.get()));
- EXPECT_EQ(kOK, GarbageCollect(day_index));
-}
-
-// Tests that GenerateObservations() returns a positive status and that the
-// expected number of Observations is generated when events are logged over
-// multiple days and some of those days' Observations are backfilled, without
-// any garbage collection of the LocalAggregateStore.
-//
-// Sets the |backfill_days_| field of the EventAggregator to 3.
-//
-// Logging pattern:
-// For 35 days, logs 2 events each day for the
-// ConnectionFailures_PerDeviceCount report and 2 events for the
-// SettingsChanged_PerDeviceCount report, all with event code 0.
-//
-// Observation generation pattern:
-// Calls GenerateObservations() on the 1st through 5th and the 7th out of
-// every 10 days, for 35 days.
-//
-// Expected numbers of Observations:
-// It is expected that 4 days' worth of Observations are generated on
-// the first day of every 10 (the day index for which GenerateObservations()
-// was called, plus 3 days of backfill), that 1 day's worth of Observations
-// are generated on the 2nd through 5th day of every 10, that 2 days'
-// worth of Observations are generated on the 7th day of every 10 (the
-// day index for which GenerateObservations() was called, plus 1 day of
-// backfill), and that no Observations are generated on the remaining days.
-TEST_F(PerDeviceNumericEventAggregatorTest, GenerateObservationsWithBackfill) {
- const auto& expected_params =
- logger::testing::per_device_numeric_stats::kExpectedAggregationParams;
- // Set |backfill_days_| to 3.
- size_t backfill_days = 3;
- SetBackfillDays(backfill_days);
- // Log 2 events each day for 35 days. Call GenerateObservations() on the
- // first 5 day indices, and the 7th, out of every 10.
- for (int offset = 0; offset < 35; offset++) {
- auto day_index = CurrentDayIndex();
- for (int i = 0; i < 2; i++) {
- EXPECT_EQ(kOK,
- LogPerDeviceCountEvent(
- logger::testing::per_device_numeric_stats::kConnectionFailuresMetricReportId,
- day_index, "component_A", 0u, 1));
- EXPECT_EQ(kOK,
- LogPerDeviceCountEvent(
- logger::testing::per_device_numeric_stats::kConnectionFailuresMetricReportId,
- day_index, "component_A", 0u, 1));
- EXPECT_EQ(
- kOK,
- LogPerDeviceCountEvent(
- logger::testing::per_device_numeric_stats::kSettingsChangedWindowSizeMetricReportId,
- day_index, "component_B", 0u, 5));
- EXPECT_EQ(kOK, LogPerDeviceCountEvent(logger::testing::per_device_numeric_stats::
- kSettingsChangedAggregationWindowMetricReportId,
- day_index, "component_B", 0u, 5));
- }
- auto num_obs_before = observation_store_->messages_received.size();
- if (offset % 10 < 5 || offset % 10 == 6) {
- EXPECT_EQ(kOK, GenerateObservations(day_index));
- }
- auto num_obs_after = observation_store_->messages_received.size();
- EXPECT_GE(num_obs_after, num_obs_before);
- // Check that the expected daily number of Observations was generated.
- switch (offset % 10) {
- case 0:
- // If this is the first day of logging, expect 3 Observations for each
- // day in the backfill period and 8 Observations for the current day.
- if (offset == 0) {
- EXPECT_EQ(
- (expected_params.daily_num_obs * backfill_days) + expected_params.daily_num_obs + 5,
- num_obs_after - num_obs_before);
- } else {
- // If this is another day whose offset is a multiple of 10, expect 8
- // Observations for each day in the backfill period as well as the
- // current day.
- EXPECT_EQ((expected_params.daily_num_obs + 5) * (backfill_days + 1),
- num_obs_after - num_obs_before);
- }
- break;
- case 1:
- case 2:
- case 3:
- case 4:
- // Expect 8 Observations for this day.
- EXPECT_EQ(expected_params.daily_num_obs + 5, num_obs_after - num_obs_before);
- break;
- case 6:
- // Expect 8 Observations for each of today and yesterday.
- EXPECT_EQ((expected_params.daily_num_obs + 5) * 2, num_obs_after - num_obs_before);
- break;
- default:
- EXPECT_EQ(num_obs_after, num_obs_before);
- }
- AdvanceClock(kDay);
- }
-}
-
-// Tests that GenerateObservations() returns a positive status and that the
-// expected number of Observations is generated when events are logged over
-// multiple days and some of those days' Observations are backfilled, and when
-// the LocalAggregateStore is garbage-collected after each call to
-// GenerateObservations().
-//
-// Sets the |backfill_days_| field of the EventAggregator to 3.
-//
-// Logging pattern:
-// For 35 days, logs 2 events each day for the
-// ConnectionFailures_PerDeviceNumeric report with "component_A" and 2 events
-// for the SettingsChanged_PerDeviceNumeric reports with "component_B", all with
-// event code 0.
-//
-// Observation generation pattern:
-// Calls GenerateObservations() on the 1st through 5th and the 7th out of
-// every 10 days, for 35 days. Garbage-collects the LocalAggregateStore after
-// each call.
-//
-// Expected numbers of Observations:
-// It is expected that 4 days' worth of Observations are generated on
-// the first day of every 10 (the day index for which GenerateObservations()
-// was called, plus 3 days of backfill), that 1 day's worth of Observations
-// are generated on the 2nd through 5th day of every 10, that 2 days'
-// worth of Observations are generated on the 7th day of every 10 (the
-// day index for which GenerateObservations() was called, plus 1 day of
-// backfill), and that no Observations are generated on the remaining days.
-TEST_F(PerDeviceNumericEventAggregatorTest, GenerateObservationsWithBackfillAndGc) {
- int num_days = 35;
- const auto& expected_params =
- logger::testing::per_device_numeric_stats::kExpectedAggregationParams;
- // Set |backfill_days_| to 3.
- size_t backfill_days = 3;
- SetBackfillDays(backfill_days);
- // Log 2 events each day for 35 days. Call GenerateObservations() on the
- // first 5 day indices, and the 7th, out of every 10.
- for (int offset = 0; offset < num_days; offset++) {
- auto day_index = CurrentDayIndex();
- for (int i = 0; i < 2; i++) {
- EXPECT_EQ(kOK,
- LogPerDeviceCountEvent(
- logger::testing::per_device_numeric_stats::kConnectionFailuresMetricReportId,
- day_index, "component_A", 0u, 1));
- EXPECT_EQ(
- kOK,
- LogPerDeviceCountEvent(
- logger::testing::per_device_numeric_stats::kSettingsChangedWindowSizeMetricReportId,
- day_index, "component_B", 0u, 5));
- EXPECT_EQ(kOK, LogPerDeviceCountEvent(logger::testing::per_device_numeric_stats::
- kSettingsChangedAggregationWindowMetricReportId,
- day_index, "component_B", 0u, 5));
- }
- auto num_obs_before = observation_store_->messages_received.size();
- if (offset % 10 < 5 || offset % 10 == 6) {
- EXPECT_EQ(kOK, GenerateObservations(day_index));
- EXPECT_EQ(kOK, GarbageCollect(day_index));
- }
- auto num_obs_after = observation_store_->messages_received.size();
- EXPECT_GE(num_obs_after, num_obs_before);
- // Check that the expected daily number of Observations was generated.
- switch (offset % 10) {
- case 0:
- // If this is the first day of logging, expect 3 Observations for each
- // day in the backfill period and 8 Observations for the current day.
- if (offset == 0) {
- EXPECT_EQ(
- (expected_params.daily_num_obs * backfill_days) + expected_params.daily_num_obs + 5,
- num_obs_after - num_obs_before);
- } else {
- // If this is another day whose offset is a multiple of 10, expect 8
- // Observations for each day in the backfill period as well as the
- // current day.
- EXPECT_EQ((expected_params.daily_num_obs + 5) * (backfill_days + 1),
- num_obs_after - num_obs_before);
- }
- break;
- case 1:
- case 2:
- case 3:
- case 4:
- // Expect 8 Observations for this day.
- EXPECT_EQ(expected_params.daily_num_obs + 5, num_obs_after - num_obs_before);
- break;
- case 6:
- // Expect 6 Observations for each of today and yesterday.
- EXPECT_EQ((expected_params.daily_num_obs + 5) * 2, num_obs_after - num_obs_before);
- break;
- default:
- EXPECT_EQ(num_obs_after, num_obs_before);
- }
- AdvanceClock(kDay);
- }
-}
-
-// Generate Observations without logging any events, and check that the
-// resulting Observations are as expected: 1 ReportParticipationObservation for
-// each PER_DEVICE_NUMERIC_STATS report in the config, and no
-// PerDeviceNumericObservations.
-TEST_F(PerDeviceNumericEventAggregatorTest, CheckObservationValuesNoEvents) {
- const auto current_day_index = CurrentDayIndex();
- EXPECT_EQ(kOK, GenerateObservations(current_day_index));
- const auto& expected_report_participation_obs = MakeExpectedReportParticipationObservations(
- logger::testing::per_device_numeric_stats::kExpectedAggregationParams, current_day_index);
- EXPECT_TRUE(CheckPerDeviceNumericObservations({}, expected_report_participation_obs,
- observation_store_.get(), update_recipient_.get()));
-}
-
-// Check that the expected PerDeviceNumericObservations and
-// ReportParticipationObservations are generated when GenerateObservations() is
-// called after logging some CountEvents and ElapsedTimeEvents for
-// PER_DEVICE_NUMERIC_STATS reports over a single day index.
-TEST_F(PerDeviceNumericEventAggregatorTest, CheckObservationValuesSingleDay) {
- const auto day_index = CurrentDayIndex();
- // Log several events on |day_index|.
- EXPECT_EQ(kOK, LogPerDeviceCountEvent(
- logger::testing::per_device_numeric_stats::kConnectionFailuresMetricReportId,
- day_index, "component_A", 0u, 5));
- EXPECT_EQ(kOK, LogPerDeviceCountEvent(
- logger::testing::per_device_numeric_stats::kConnectionFailuresMetricReportId,
- day_index, "component_B", 0u, 5));
- EXPECT_EQ(kOK, LogPerDeviceCountEvent(
- logger::testing::per_device_numeric_stats::kConnectionFailuresMetricReportId,
- day_index, "component_A", 0u, 5));
- EXPECT_EQ(kOK, LogPerDeviceCountEvent(
- logger::testing::per_device_numeric_stats::kConnectionFailuresMetricReportId,
- day_index, "component_A", 1u, 5));
- EXPECT_EQ(kOK,
- LogPerDeviceCountEvent(
- logger::testing::per_device_numeric_stats::kSettingsChangedWindowSizeMetricReportId,
- day_index, "component_C", 0u, 5));
- EXPECT_EQ(kOK,
- LogPerDeviceCountEvent(
- logger::testing::per_device_numeric_stats::kSettingsChangedWindowSizeMetricReportId,
- day_index, "component_C", 0u, 5));
- EXPECT_EQ(kOK, LogPerDeviceCountEvent(logger::testing::per_device_numeric_stats::
- kSettingsChangedAggregationWindowMetricReportId,
- day_index, "component_C", 0u, 5));
- EXPECT_EQ(kOK, LogPerDeviceCountEvent(logger::testing::per_device_numeric_stats::
- kSettingsChangedAggregationWindowMetricReportId,
- day_index, "component_C", 0u, 5));
-
- std::vector<MetricReportId> streaming_time_ids = {
- logger::testing::per_device_numeric_stats::kStreamingTimeTotalMetricReportId,
- logger::testing::per_device_numeric_stats::kStreamingTimeMinMetricReportId,
- logger::testing::per_device_numeric_stats::kStreamingTimeMaxMetricReportId};
- for (const auto& id : streaming_time_ids) {
- EXPECT_EQ(kOK, LogPerDeviceElapsedTimeEvent(id, day_index, "component_D", 0u, 15));
- EXPECT_EQ(kOK, LogPerDeviceElapsedTimeEvent(id, day_index, "component_D", 1u, 5));
- EXPECT_EQ(kOK, LogPerDeviceElapsedTimeEvent(id, day_index, "component_D", 0u, 10));
- }
- // Generate locally aggregated Observations for |day_index|.
- EXPECT_EQ(kOK, GenerateObservations(day_index));
-
- // Form the expected Observations.
- auto expected_report_participation_obs = MakeExpectedReportParticipationObservations(
- logger::testing::per_device_numeric_stats::kExpectedAggregationParams, day_index);
- ExpectedPerDeviceNumericObservations expected_per_device_numeric_obs;
- expected_per_device_numeric_obs[{
- logger::testing::per_device_numeric_stats::kConnectionFailuresMetricReportId, day_index}][1] =
- {{"component_A", 0u, 10}, {"component_A", 1u, 5}, {"component_B", 0u, 5}};
- expected_per_device_numeric_obs[{
- logger::testing::per_device_numeric_stats::kSettingsChangedWindowSizeMetricReportId,
- day_index}][7] = {{"component_C", 0u, 10}};
- expected_per_device_numeric_obs[{
- logger::testing::per_device_numeric_stats::kSettingsChangedWindowSizeMetricReportId,
- day_index}][30] = {{"component_C", 0u, 10}};
- expected_per_device_numeric_obs[{
- logger::testing::per_device_numeric_stats::kSettingsChangedAggregationWindowMetricReportId,
- day_index}][7] = {{"component_C", 0u, 10}};
- expected_per_device_numeric_obs[{
- logger::testing::per_device_numeric_stats::kSettingsChangedAggregationWindowMetricReportId,
- day_index}][30] = {{"component_C", 0u, 10}};
- expected_per_device_numeric_obs[{
- logger::testing::per_device_numeric_stats::kStreamingTimeTotalMetricReportId, day_index}][1] =
- {{"component_D", 0u, 25}, {"component_D", 1u, 5}};
- expected_per_device_numeric_obs[{
- logger::testing::per_device_numeric_stats::kStreamingTimeTotalMetricReportId, day_index}][7] =
- {{"component_D", 0u, 25}, {"component_D", 1u, 5}};
- // The 7-day minimum value for the StreamingTime metric is 0 for all event
- // codes and components, so we don't expect a PerDeviceNumericObservation with
- // a 7-day window for the StreamingTime_PerDeviceMin report.
- expected_per_device_numeric_obs[{
- logger::testing::per_device_numeric_stats::kStreamingTimeMinMetricReportId, day_index}][1] = {
- {"component_D", 0u, 10}, {"component_D", 1u, 5}};
- expected_per_device_numeric_obs[{
- logger::testing::per_device_numeric_stats::kStreamingTimeMinMetricReportId, day_index}][7] = {
- {"component_D", 0u, 10}, {"component_D", 1u, 5}};
- expected_per_device_numeric_obs[{
- logger::testing::per_device_numeric_stats::kStreamingTimeMaxMetricReportId, day_index}][1] = {
- {"component_D", 0u, 15}, {"component_D", 1u, 5}};
- expected_per_device_numeric_obs[{
- logger::testing::per_device_numeric_stats::kStreamingTimeMaxMetricReportId, day_index}][7] = {
- {"component_D", 0u, 15}, {"component_D", 1u, 5}};
-
- EXPECT_TRUE(CheckPerDeviceNumericObservations(expected_per_device_numeric_obs,
- expected_report_participation_obs,
- observation_store_.get(), update_recipient_.get()));
-}
-
-// Checks that PerDeviceNumericObservations with the expected values are
-// generated when some events have been logged for an EVENT_COUNT metric with
-// a PER_DEVICE_NUMERIC_STATS report over multiple days and
-// GenerateObservations() is called each day, without garbage collection or
-// backfill.
-//
-// Logged events for the SettingsChanged_PerDeviceCount report on the i-th
-// day:
-//
-// i (component, event code, count)
-// -----------------------------------------------------------------------
-// 0
-// 1 ("A", 1, 3)
-// 2 ("A", 1, 3), ("A", 2, 3), ("B", 1, 2)
-// 3 ("A", 1, 3)
-// 4 ("A", 1, 3), ("A", 2, 3), ("B", 1, 2), ("B", 2, 2)
-// 5 ("A", 1, 3)
-// 6 ("A", 1, 3), ("A", 2, 3), ("B", 1, 2)
-// 7 ("A", 1, 3)
-// 8 ("A", 1, 3), ("A", 2, 3), ("B", 1, 2), ("B", 2, 2)
-// 9 ("A", 1, 3)
-//
-// Expected PerDeviceNumericObservations for the
-// SettingsChanged_PerDeviceNumeric report on the i-th day:
-//
-// (i, window size) (component, event code, count)
-// -----------------------------------------------------------------------
-// (0, 7)
-// (0, 30)
-// (1, 7) ("A", 1, 3)
-// (1, 30) ("A", 1, 3)
-// (2, 7) ("A", 1, 6), ("A", 2, 3), ("B", 1, 2)
-// (2, 30) ("A", 1, 6), ("A", 2, 3), ("B", 1, 2)
-// (3, 7) ("A", 1, 9), ("A", 2, 3), ("B", 1, 2)
-// (3, 30) ("A", 1, 9), ("A", 2, 3), ("B", 1, 2)
-// (4, 7) ("A", 1, 12), ("A", 2, 6), ("B", 1, 4), ("B", 2, 2)
-// (4, 30) ("A", 1, 12), ("A", 2, 6), ("B", 1, 4), ("B", 2, 2)
-// (5, 7) ("A", 1, 15), ("A", 2, 6), ("B", 1, 4), ("B", 2, 2)
-// (5, 30) ("A", 1, 15), ("A", 2, 6), ("B", 1, 4), ("B", 2, 2)
-// (6, 7) ("A", 1, 18), ("A", 2, 9), ("B", 1, 6), ("B", 2, 2)
-// (6, 30) ("A", 1, 18), ("A", 2, 9), ("B", 1, 6), ("B", 2, 2)
-// (7, 7) ("A", 1, 21), ("A", 2, 9), ("B", 1, 6), ("B", 2, 2)
-// (7, 30) ("A", 1, 21), ("A", 2, 9), ("B", 1, 6), ("B", 2, 2)
-// (8, 7) ("A", 1, 21), ("A", 2, 12), ("B", 1, 8), ("B", 2, 4)
-// (8, 30) ("A", 1, 24), ("A", 2, 12), ("B", 1, 8), ("B", 2, 4)
-// (9, 7) ("A", 1, 21), ("A", 2, 9), ("B", 1, 6), ("B", 2, 4)
-// (9, 30) ("A", 1, 27), ("A", 2, 12), ("B", 1, 8), ("B", 2, 4)
-//
-// In addition, expect 1 ReportParticipationObservation each day for each of
-// the reports in the registry.
-TEST_F(PerDeviceNumericEventAggregatorTest, CheckObservationValuesMultiDay) {
- auto start_day_index = CurrentDayIndex();
- const auto& expected_id =
- logger::testing::per_device_numeric_stats::kSettingsChangedAggregationWindowMetricReportId;
- const auto& expected_params =
- logger::testing::per_device_numeric_stats::kExpectedAggregationParams;
- // Form expected Observations for the 10 days of logging.
- uint32_t num_days = 10;
- std::vector<ExpectedPerDeviceNumericObservations> expected_per_device_numeric_obs(num_days);
- std::vector<ExpectedReportParticipationObservations> expected_report_participation_obs(num_days);
- for (uint32_t offset = 0; offset < num_days; offset++) {
- expected_report_participation_obs[offset] =
- MakeExpectedReportParticipationObservations(expected_params, start_day_index + offset);
- }
- expected_per_device_numeric_obs[0] = {};
- expected_per_device_numeric_obs[1][{expected_id, start_day_index + 1}] = {{7, {{"A", 1u, 3}}},
- {30, {{"A", 1u, 3}}}};
- expected_per_device_numeric_obs[2][{expected_id, start_day_index + 2}] = {
- {7, {{"A", 1u, 6}, {"A", 2u, 3}, {"B", 1u, 2}}},
- {30, {{"A", 1u, 6}, {"A", 2u, 3}, {"B", 1u, 2}}}};
- expected_per_device_numeric_obs[3][{expected_id, start_day_index + 3}] = {
- {7, {{"A", 1u, 9}, {"A", 2u, 3}, {"B", 1u, 2}}},
- {30, {{"A", 1u, 9}, {"A", 2u, 3}, {"B", 1u, 2}}}};
- expected_per_device_numeric_obs[4][{expected_id, start_day_index + 4}] = {
- {7, {{"A", 1u, 12}, {"A", 2u, 6}, {"B", 1u, 4}, {"B", 2u, 2}}},
- {30, {{"A", 1u, 12}, {"A", 2u, 6}, {"B", 1u, 4}, {"B", 2u, 2}}}};
- expected_per_device_numeric_obs[5][{expected_id, start_day_index + 5}] = {
- {7, {{"A", 1u, 15}, {"A", 2u, 6}, {"B", 1u, 4}, {"B", 2u, 2}}},
- {30, {{"A", 1u, 15}, {"A", 2u, 6}, {"B", 1u, 4}, {"B", 2u, 2}}}};
- expected_per_device_numeric_obs[6][{expected_id, start_day_index + 6}] = {
- {7, {{"A", 1u, 18}, {"A", 2u, 9}, {"B", 1u, 6}, {"B", 2u, 2}}},
- {30, {{"A", 1u, 18}, {"A", 2u, 9}, {"B", 1u, 6}, {"B", 2u, 2}}}};
- expected_per_device_numeric_obs[7][{expected_id, start_day_index + 7}] = {
- {7, {{"A", 1u, 21}, {"A", 2u, 9}, {"B", 1u, 6}, {"B", 2u, 2}}},
- {30, {{"A", 1u, 21}, {"A", 2u, 9}, {"B", 1u, 6}, {"B", 2u, 2}}}};
- expected_per_device_numeric_obs[8][{expected_id, start_day_index + 8}] = {
- {7, {{"A", 1u, 21}, {"A", 2u, 12}, {"B", 1u, 8}, {"B", 2u, 4}}},
- {30, {{"A", 1u, 24}, {"A", 2u, 12}, {"B", 1u, 8}, {"B", 2u, 4}}}};
- expected_per_device_numeric_obs[9][{expected_id, start_day_index + 9}] = {
- {7, {{"A", 1u, 21}, {"A", 2u, 9}, {"B", 1u, 6}, {"B", 2u, 4}}},
- {30, {{"A", 1u, 27}, {"A", 2u, 12}, {"B", 1u, 8}, {"B", 2u, 4}}}};
-
- for (uint32_t offset = 0; offset < 1; offset++) {
- auto day_index = CurrentDayIndex();
- for (uint32_t event_code = 1; event_code < 3; event_code++) {
- if (offset > 0 && offset % event_code == 0) {
- EXPECT_EQ(kOK, LogPerDeviceCountEvent(expected_id, day_index, "A", event_code, 3));
- }
- if (offset > 0 && offset % (2 * event_code) == 0) {
- EXPECT_EQ(kOK, LogPerDeviceCountEvent(expected_id, day_index, "B", event_code, 2));
- }
- }
- // Clear the FakeObservationStore.
- ResetObservationStore();
- // Generate locally aggregated Observations.
- EXPECT_EQ(kOK, GenerateObservations(day_index));
- EXPECT_TRUE(CheckPerDeviceNumericObservations(
- expected_per_device_numeric_obs[offset], expected_report_participation_obs[offset],
- observation_store_.get(), update_recipient_.get()))
- << "offset = " << offset;
- AdvanceClock(kDay);
- }
-}
-
-// Repeat the CheckObservationValuesMultiDay test, this time calling
-// GarbageCollect() after each call to GenerateObservations.
-//
-// The logging pattern and set of Observations for each day index is the same
-// as in PerDeviceNumericEventAggregatorTest::CheckObservationValuesMultiDay.
-// See that test for documentation.
-TEST_F(PerDeviceNumericEventAggregatorTest, CheckObservationValuesMultiDayWithGarbageCollection) {
- auto start_day_index = CurrentDayIndex();
- const auto& expected_id =
- logger::testing::per_device_numeric_stats::kSettingsChangedAggregationWindowMetricReportId;
- const auto& expected_params =
- logger::testing::per_device_numeric_stats::kExpectedAggregationParams;
- // Form expected Observations for the 10 days of logging.
- uint32_t num_days = 10;
- std::vector<ExpectedPerDeviceNumericObservations> expected_per_device_numeric_obs(num_days);
- std::vector<ExpectedReportParticipationObservations> expected_report_participation_obs(num_days);
- for (uint32_t offset = 0; offset < num_days; offset++) {
- expected_report_participation_obs[offset] =
- MakeExpectedReportParticipationObservations(expected_params, start_day_index + offset);
- }
- expected_per_device_numeric_obs[0] = {};
- expected_per_device_numeric_obs[1][{expected_id, start_day_index + 1}] = {{7, {{"A", 1u, 3}}},
- {30, {{"A", 1u, 3}}}};
- expected_per_device_numeric_obs[2][{expected_id, start_day_index + 2}] = {
- {7, {{"A", 1u, 6}, {"A", 2u, 3}, {"B", 1u, 2}}},
- {30, {{"A", 1u, 6}, {"A", 2u, 3}, {"B", 1u, 2}}}};
- expected_per_device_numeric_obs[3][{expected_id, start_day_index + 3}] = {
- {7, {{"A", 1u, 9}, {"A", 2u, 3}, {"B", 1u, 2}}},
- {30, {{"A", 1u, 9}, {"A", 2u, 3}, {"B", 1u, 2}}}};
- expected_per_device_numeric_obs[4][{expected_id, start_day_index + 4}] = {
- {7, {{"A", 1u, 12}, {"A", 2u, 6}, {"B", 1u, 4}, {"B", 2u, 2}}},
- {30, {{"A", 1u, 12}, {"A", 2u, 6}, {"B", 1u, 4}, {"B", 2u, 2}}}};
- expected_per_device_numeric_obs[5][{expected_id, start_day_index + 5}] = {
- {7, {{"A", 1u, 15}, {"A", 2u, 6}, {"B", 1u, 4}, {"B", 2u, 2}}},
- {30, {{"A", 1u, 15}, {"A", 2u, 6}, {"B", 1u, 4}, {"B", 2u, 2}}}};
- expected_per_device_numeric_obs[6][{expected_id, start_day_index + 6}] = {
- {7, {{"A", 1u, 18}, {"A", 2u, 9}, {"B", 1u, 6}, {"B", 2u, 2}}},
- {30, {{"A", 1u, 18}, {"A", 2u, 9}, {"B", 1u, 6}, {"B", 2u, 2}}}};
- expected_per_device_numeric_obs[7][{expected_id, start_day_index + 7}] = {
- {7, {{"A", 1u, 21}, {"A", 2u, 9}, {"B", 1u, 6}, {"B", 2u, 2}}},
- {30, {{"A", 1u, 21}, {"A", 2u, 9}, {"B", 1u, 6}, {"B", 2u, 2}}}};
- expected_per_device_numeric_obs[8][{expected_id, start_day_index + 8}] = {
- {7, {{"A", 1u, 21}, {"A", 2u, 12}, {"B", 1u, 8}, {"B", 2u, 4}}},
- {30, {{"A", 1u, 24}, {"A", 2u, 12}, {"B", 1u, 8}, {"B", 2u, 4}}}};
- expected_per_device_numeric_obs[9][{expected_id, start_day_index + 9}] = {
- {7, {{"A", 1u, 21}, {"A", 2u, 9}, {"B", 1u, 6}, {"B", 2u, 4}}},
- {30, {{"A", 1u, 27}, {"A", 2u, 12}, {"B", 1u, 8}, {"B", 2u, 4}}}};
-
- for (uint32_t offset = 0; offset < 10; offset++) {
- auto day_index = CurrentDayIndex();
- for (uint32_t event_code = 1; event_code < 3; event_code++) {
- if (offset > 0 && offset % event_code == 0) {
- EXPECT_EQ(kOK, LogPerDeviceCountEvent(expected_id, day_index, "A", event_code, 3));
- }
- if (offset > 0 && offset % (2 * event_code) == 0) {
- EXPECT_EQ(kOK, LogPerDeviceCountEvent(expected_id, day_index, "B", event_code, 2));
- }
- }
- // Advance |test_clock_| by 1 day.
- AdvanceClock(kDay);
- // Clear the FakeObservationStore.
- ResetObservationStore();
- // Generate locally aggregated Observations and garbage-collect the
- // LocalAggregateStore, both for the previous day as measured by
- // |test_clock_|. Back up the LocalAggregateStore and
- // AggregatedObservationHistoryStore.
- DoScheduledTasksNow();
- EXPECT_TRUE(CheckPerDeviceNumericObservations(
- expected_per_device_numeric_obs[offset], expected_report_participation_obs[offset],
- observation_store_.get(), update_recipient_.get()));
- }
-}
-
-// Tests that the expected PerDeviceNumericObservations are generated when
-// events are logged over multiple days for an EVENT_COUNT
-// metric with a PER_DEVICE_NUMERIC_STATS report, when Observations are
-// backfilled for some days during that period, without any garbage-collection
-// of the LocalAggregateStore.
-//
-// The logging pattern and set of Observations for each day index is the same
-// as in PerDeviceNumericEventAggregatorTest::CheckObservationValuesMultiDay.
-// See that test for documentation.
-TEST_F(PerDeviceNumericEventAggregatorTest, CheckObservationValuesWithBackfill) {
- auto start_day_index = CurrentDayIndex();
- const auto& expected_id =
- logger::testing::per_device_numeric_stats::kSettingsChangedAggregationWindowMetricReportId;
- const auto& expected_params =
- logger::testing::per_device_numeric_stats::kExpectedAggregationParams;
- // Set |backfill_days_| to 3.
- size_t backfill_days = 3;
- SetBackfillDays(backfill_days);
- // Log events for 9 days. Call GenerateObservations() on the first 6 day
- // indices, and the 9th.
- uint32_t num_days = 9;
- for (uint32_t offset = 0; offset < num_days; offset++) {
- auto day_index = CurrentDayIndex();
- ResetObservationStore();
- for (uint32_t event_code = 1; event_code < 3; event_code++) {
- if (offset > 0 && (offset % event_code == 0)) {
- EXPECT_EQ(kOK, LogPerDeviceCountEvent(expected_id, day_index, "A", event_code, 3));
- }
- if (offset > 0 && offset % (2 * event_code) == 0) {
- EXPECT_EQ(kOK, LogPerDeviceCountEvent(expected_id, day_index, "B", event_code, 2));
- }
- }
- if (offset < 6 || offset == 8) {
- EXPECT_EQ(kOK, GenerateObservations(day_index));
- }
- // Make the set of Observations which are expected to be generated on
- // |start_day_index + offset| and check it against the contents of the
- // FakeObservationStore.
- ExpectedPerDeviceNumericObservations expected_per_device_numeric_obs;
- ExpectedReportParticipationObservations expected_report_participation_obs;
- switch (offset) {
- case 0: {
- for (uint32_t day_index = start_day_index - backfill_days; day_index <= start_day_index;
- day_index++) {
- for (const auto& pair :
- MakeExpectedReportParticipationObservations(expected_params, day_index)) {
- expected_report_participation_obs.insert(pair);
- }
- }
- EXPECT_TRUE(CheckPerDeviceNumericObservations(
- expected_per_device_numeric_obs, expected_report_participation_obs,
- observation_store_.get(), update_recipient_.get()));
- break;
- }
- case 1: {
- expected_per_device_numeric_obs[{expected_id, day_index}] = {{7, {{"A", 1u, 3}}},
- {30, {{"A", 1u, 3}}}};
- expected_report_participation_obs =
- MakeExpectedReportParticipationObservations(expected_params, day_index);
- EXPECT_TRUE(CheckPerDeviceNumericObservations(
- expected_per_device_numeric_obs, expected_report_participation_obs,
- observation_store_.get(), update_recipient_.get()));
- break;
- }
- case 2: {
- expected_per_device_numeric_obs[{expected_id, day_index}] = {
- {7, {{"A", 1u, 6}, {"A", 2u, 3}, {"B", 1u, 2}}},
- {30, {{"A", 1u, 6}, {"A", 2u, 3}, {"B", 1u, 2}}}};
- expected_report_participation_obs =
- MakeExpectedReportParticipationObservations(expected_params, day_index);
- EXPECT_TRUE(CheckPerDeviceNumericObservations(
- expected_per_device_numeric_obs, expected_report_participation_obs,
- observation_store_.get(), update_recipient_.get()));
- break;
- }
- case 3: {
- expected_per_device_numeric_obs[{expected_id, day_index}] = {
- {7, {{"A", 1u, 9}, {"A", 2u, 3}, {"B", 1u, 2}}},
- {30, {{"A", 1u, 9}, {"A", 2u, 3}, {"B", 1u, 2}}}};
- expected_report_participation_obs =
- MakeExpectedReportParticipationObservations(expected_params, day_index);
- EXPECT_TRUE(CheckPerDeviceNumericObservations(
- expected_per_device_numeric_obs, expected_report_participation_obs,
- observation_store_.get(), update_recipient_.get()));
- break;
- }
- case 4: {
- expected_per_device_numeric_obs[{expected_id, day_index}] = {
- {7, {{"A", 1u, 12}, {"A", 2u, 6}, {"B", 1u, 4}, {"B", 2u, 2}}},
- {30, {{"A", 1u, 12}, {"A", 2u, 6}, {"B", 1u, 4}, {"B", 2u, 2}}}};
- expected_report_participation_obs =
- MakeExpectedReportParticipationObservations(expected_params, day_index);
- EXPECT_TRUE(CheckPerDeviceNumericObservations(
- expected_per_device_numeric_obs, expected_report_participation_obs,
- observation_store_.get(), update_recipient_.get()));
- break;
- }
- case 5: {
- expected_per_device_numeric_obs[{expected_id, day_index}] = {
- {7, {{"A", 1u, 15}, {"A", 2u, 6}, {"B", 1u, 4}, {"B", 2u, 2}}},
- {30, {{"A", 1u, 15}, {"A", 2u, 6}, {"B", 1u, 4}, {"B", 2u, 2}}}};
- expected_report_participation_obs =
- MakeExpectedReportParticipationObservations(expected_params, day_index);
- EXPECT_TRUE(CheckPerDeviceNumericObservations(
- expected_per_device_numeric_obs, expected_report_participation_obs,
- observation_store_.get(), update_recipient_.get()));
- break;
- }
- case 8: {
- expected_per_device_numeric_obs[{expected_id, start_day_index + 6}] = {
- {7, {{"A", 1u, 18}, {"A", 2u, 9}, {"B", 1u, 6}, {"B", 2u, 2}}},
- {30, {{"A", 1u, 18}, {"A", 2u, 9}, {"B", 1u, 6}, {"B", 2u, 2}}}};
- expected_per_device_numeric_obs[{expected_id, start_day_index + 7}] = {
- {7, {{"A", 1u, 21}, {"A", 2u, 9}, {"B", 1u, 6}, {"B", 2u, 2}}},
- {30, {{"A", 1u, 21}, {"A", 2u, 9}, {"B", 1u, 6}, {"B", 2u, 2}}}};
- expected_per_device_numeric_obs[{expected_id, start_day_index + 8}] = {
- {7, {{"A", 1u, 21}, {"A", 2u, 12}, {"B", 1u, 8}, {"B", 2u, 4}}},
- {30, {{"A", 1u, 24}, {"A", 2u, 12}, {"B", 1u, 8}, {"B", 2u, 4}}}};
- for (uint32_t day_index = start_day_index + 6; day_index <= start_day_index + 8;
- day_index++) {
- for (const auto& pair :
- MakeExpectedReportParticipationObservations(expected_params, day_index)) {
- expected_report_participation_obs.insert(pair);
- }
- }
- EXPECT_TRUE(CheckPerDeviceNumericObservations(
- expected_per_device_numeric_obs, expected_report_participation_obs,
- observation_store_.get(), update_recipient_.get()));
- break;
- }
- default:
- EXPECT_TRUE(CheckPerDeviceNumericObservations(
- expected_per_device_numeric_obs, expected_report_participation_obs,
- observation_store_.get(), update_recipient_.get()));
- }
- AdvanceClock(kDay);
- }
-}
-
-// Tests that the expected Observations are generated for
-// PerDeviceNumericStats reports when events are logged for over multiple days
-// for an EVENT_COUNT metric with a PER_DEVICE_NUMERIC_STATS report, when
-// Observations are backfilled for some days during that period, and when the
-// LocalAggregatedStore is garbage-collected after each call to
-// GenerateObservations().
-//
-// The logging pattern and set of Observations for each day index is the same
-// as in PerDeviceNumericEventAggregatorTest::CheckObservationValuesMultiDay.
-// See that test for documentation.
-TEST_F(PerDeviceNumericEventAggregatorTest, EventCountCheckObservationValuesWithBackfillAndGc) {
- auto start_day_index = CurrentDayIndex();
- const auto& expected_id =
- logger::testing::per_device_numeric_stats::kSettingsChangedAggregationWindowMetricReportId;
- const auto& expected_params =
- logger::testing::per_device_numeric_stats::kExpectedAggregationParams;
- // Set |backfill_days_| to 3.
- size_t backfill_days = 3;
- SetBackfillDays(backfill_days);
- // Log events for 9 days. Call GenerateObservations() on the first 6 day
- // indices, and the 9th.
- uint32_t num_days = 9;
- for (uint32_t offset = 0; offset < num_days; offset++) {
- auto day_index = CurrentDayIndex();
- ResetObservationStore();
- for (uint32_t event_code = 1; event_code < 3; event_code++) {
- if (offset > 0 && (offset % event_code == 0)) {
- EXPECT_EQ(kOK, LogPerDeviceCountEvent(expected_id, day_index, "A", event_code, 3));
- }
- if (offset > 0 && offset % (2 * event_code) == 0) {
- EXPECT_EQ(kOK, LogPerDeviceCountEvent(expected_id, day_index, "B", event_code, 2));
- }
- }
- // Advance |test_clock_| by 1 day.
- AdvanceClock(kDay);
- if (offset < 6 || offset == 8) {
- // Generate Observations and garbage-collect, both for the previous day
- // index according to |test_clock_|. Back up the LocalAggregateStore and
- // the AggregatedObservationHistoryStore.
- DoScheduledTasksNow();
- }
- // Make the set of Observations which are expected to be generated on
- // |start_day_index + offset| and check it against the contents of the
- // FakeObservationStore.
- ExpectedPerDeviceNumericObservations expected_per_device_numeric_obs;
- ExpectedReportParticipationObservations expected_report_participation_obs;
- switch (offset) {
- case 0: {
- for (uint32_t day_index = start_day_index - backfill_days; day_index <= start_day_index;
- day_index++) {
- for (const auto& pair :
- MakeExpectedReportParticipationObservations(expected_params, day_index)) {
- expected_report_participation_obs.insert(pair);
- }
- }
- EXPECT_TRUE(CheckPerDeviceNumericObservations(
- expected_per_device_numeric_obs, expected_report_participation_obs,
- observation_store_.get(), update_recipient_.get()));
- break;
- }
- case 1: {
- expected_per_device_numeric_obs[{expected_id, day_index}] = {{7, {{"A", 1u, 3}}},
- {30, {{"A", 1u, 3}}}};
- expected_report_participation_obs =
- MakeExpectedReportParticipationObservations(expected_params, day_index);
- EXPECT_TRUE(CheckPerDeviceNumericObservations(
- expected_per_device_numeric_obs, expected_report_participation_obs,
- observation_store_.get(), update_recipient_.get()));
- break;
- }
- case 2: {
- expected_per_device_numeric_obs[{expected_id, day_index}] = {
- {7, {{"A", 1u, 6}, {"A", 2u, 3}, {"B", 1u, 2}}},
- {30, {{"A", 1u, 6}, {"A", 2u, 3}, {"B", 1u, 2}}}};
- expected_report_participation_obs =
- MakeExpectedReportParticipationObservations(expected_params, day_index);
- EXPECT_TRUE(CheckPerDeviceNumericObservations(
- expected_per_device_numeric_obs, expected_report_participation_obs,
- observation_store_.get(), update_recipient_.get()));
- break;
- }
- case 3: {
- expected_per_device_numeric_obs[{expected_id, day_index}] = {
- {7, {{"A", 1u, 9}, {"A", 2u, 3}, {"B", 1u, 2}}},
- {30, {{"A", 1u, 9}, {"A", 2u, 3}, {"B", 1u, 2}}}};
- expected_report_participation_obs =
- MakeExpectedReportParticipationObservations(expected_params, day_index);
- EXPECT_TRUE(CheckPerDeviceNumericObservations(
- expected_per_device_numeric_obs, expected_report_participation_obs,
- observation_store_.get(), update_recipient_.get()));
- break;
- }
- case 4: {
- expected_per_device_numeric_obs[{expected_id, day_index}] = {
- {7, {{"A", 1u, 12}, {"A", 2u, 6}, {"B", 1u, 4}, {"B", 2u, 2}}},
- {30, {{"A", 1u, 12}, {"A", 2u, 6}, {"B", 1u, 4}, {"B", 2u, 2}}}};
- expected_report_participation_obs =
- MakeExpectedReportParticipationObservations(expected_params, day_index);
- EXPECT_TRUE(CheckPerDeviceNumericObservations(
- expected_per_device_numeric_obs, expected_report_participation_obs,
- observation_store_.get(), update_recipient_.get()));
- break;
- }
- case 5: {
- expected_per_device_numeric_obs[{expected_id, day_index}] = {
- {7, {{"A", 1u, 15}, {"A", 2u, 6}, {"B", 1u, 4}, {"B", 2u, 2}}},
- {30, {{"A", 1u, 15}, {"A", 2u, 6}, {"B", 1u, 4}, {"B", 2u, 2}}}};
- expected_report_participation_obs =
- MakeExpectedReportParticipationObservations(expected_params, day_index);
- EXPECT_TRUE(CheckPerDeviceNumericObservations(
- expected_per_device_numeric_obs, expected_report_participation_obs,
- observation_store_.get(), update_recipient_.get()));
- break;
- }
- case 8: {
- expected_per_device_numeric_obs[{expected_id, start_day_index + 6}] = {
- {7, {{"A", 1u, 18}, {"A", 2u, 9}, {"B", 1u, 6}, {"B", 2u, 2}}},
- {30, {{"A", 1u, 18}, {"A", 2u, 9}, {"B", 1u, 6}, {"B", 2u, 2}}}};
- expected_per_device_numeric_obs[{expected_id, start_day_index + 7}] = {
- {7, {{"A", 1u, 21}, {"A", 2u, 9}, {"B", 1u, 6}, {"B", 2u, 2}}},
- {30, {{"A", 1u, 21}, {"A", 2u, 9}, {"B", 1u, 6}, {"B", 2u, 2}}}};
- expected_per_device_numeric_obs[{expected_id, start_day_index + 8}] = {
- {7, {{"A", 1u, 21}, {"A", 2u, 12}, {"B", 1u, 8}, {"B", 2u, 4}}},
- {30, {{"A", 1u, 24}, {"A", 2u, 12}, {"B", 1u, 8}, {"B", 2u, 4}}}};
- for (uint32_t day_index = start_day_index + 6; day_index <= start_day_index + 8;
- day_index++) {
- for (const auto& pair :
- MakeExpectedReportParticipationObservations(expected_params, day_index)) {
- expected_report_participation_obs.insert(pair);
- }
- }
- EXPECT_TRUE(CheckPerDeviceNumericObservations(
- expected_per_device_numeric_obs, expected_report_participation_obs,
- observation_store_.get(), update_recipient_.get()));
- break;
- }
- default:
- EXPECT_TRUE(CheckPerDeviceNumericObservations(
- expected_per_device_numeric_obs, expected_report_participation_obs,
- observation_store_.get(), update_recipient_.get()));
- }
- }
-}
-
-// Tests that the expected Observations are generated for
-// PerDeviceNumericStats reports when events are logged for over multiple days
-// for an ELAPSED_TIME metric with PER_DEVICE_NUMERIC_STATS reports with
-// multiple aggregation types, when Observations are backfilled for some days
-// during that period, and when the LocalAggregatedStore is garbage-collected
-// after each call to GenerateObservations().
-//
-// Logged events for the StreamingTime_PerDevice{Total, Min, Max} reports on the
-// i-th day:
-//
-// i (component, event code, count)
-// -----------------------------------------------------------------------
-// 0
-// 1 ("A", 1, 3)
-// 2 ("A", 1, 3), ("A", 2, 3), ("B", 1, 2)
-// 3 ("A", 1, 3)
-// 4 ("A", 1, 3), ("A", 2, 3), ("B", 1, 2), ("B", 2, 2)
-// 5 ("A", 1, 3)
-// 6 ("A", 1, 3), ("A", 2, 3), ("B", 1, 2)
-// 7 ("A", 1, 3)
-// 8 ("A", 1, 3), ("A", 2, 3), ("B", 1, 2), ("B", 2, 2)
-//
-// Expected PerDeviceNumericObservations for the
-// StreamingTime_PerDeviceTotal report on the i-th day:
-//
-// (day, window size) (event code, component, total)
-// ---------------------------------------------------------------------------
-// (0, 1)
-// (0, 7)
-// (1, 1) ("A", 1, 3)
-// (1, 7) ("A", 1, 3)
-// (2, 1) ("A", 1, 3), ("A", 2, 3), ("B", 1, 2)
-// (2, 7) ("A", 1, 6), ("A", 2, 3), ("B", 1, 2)
-// (3, 1) ("A", 1, 3)
-// (3, 7) ("A", 1, 9), ("A", 2, 3), ("B", 1, 2)
-// (4, 1) ("A", 1, 3), ("A", 2, 3), ("B", 1, 2), ("B", 2, 2)
-// (4, 7) ("A", 1, 12), ("A", 2, 6), ("B", 1, 4), ("B", 2, 2)
-// (5, 1) ("A", 1, 3)
-// (5, 7) ("A", 1, 15), ("A", 2, 6), ("B", 1, 4), ("B", 2, 2)
-// (6, 1) ("A", 1, 3), ("A", 2, 3), ("B", 1, 2)
-// (6, 7) ("A", 1, 18), ("A", 2, 9), ("B", 1, 6), ("B", 2, 2)
-// (7, 1) ("A", 1, 3)
-// (7, 7) ("A", 1, 21), ("A", 2, 9), ("B", 1, 6), ("B", 2, 2)
-// (8, 1) ("A", 1, 3), ("A", 2, 3), ("B", 1, 2), ("B", 2, 2)
-// (8, 7) ("A", 1, 21), ("A", 2, 12), ("B", 1, 8), ("B", 2, 4)
-//
-// Expected PerDeviceNumericObservations for the
-// StreamingTime_PerDeviceMin report on the i-th day:
-//
-// (day, window size) (event code, component, total)
-// ---------------------------------------------------------------------------
-// (0, 1)
-// (0. 7)
-// (1, 1) ("A", 1, 3)
-// (1, 7) ("A", 1, 3)
-// (2, 1) ("A", 1, 3), ("A", 2, 3), ("B", 1, 2)
-// (2, 7) ("A", 1, 3), ("A", 2, 3), ("B", 1, 2)
-// (3, 1) ("A", 1, 3)
-// (3, 7) ("A", 1, 3), ("A", 2, 3), ("B", 1, 2)
-// (4, 1) ("A", 1, 3), ("A", 2, 3), ("B", 1, 2), ("B", 2, 2)
-// (4, 7) ("A", 1, 3), ("A", 2, 3), ("B", 1, 2), ("B", 2, 2)
-// (5, 1) ("A", 1, 3)
-// (5, 7) ("A", 1, 3), ("A", 2, 3), ("B", 1, 2), ("B", 2, 2)
-// (6, 1) ("A", 1, 3), ("A", 2, 3), ("B", 1, 2)
-// (6, 7) ("A", 1, 3), ("A", 2, 3), ("B", 1, 2), ("B", 2, 2)
-// (7, 1) ("A", 1, 3)
-// (7, 7) ("A", 1, 3), ("A", 2, 3), ("B", 1, 2), ("B", 2, 2)
-// (8, 1) ("A", 1, 3), ("A", 2, 3), ("B", 1, 2), ("B", 2, 2)
-// (8, 7) ("A", 1, 3), ("A", 2, 3), ("B", 1, 2), ("B", 2, 2)
-//
-// Expected PerDeviceNumericObservations for the
-// StreamingTime_PerDeviceMax report on the i-th day:
-//
-// (day, window size) (event code, component, total)
-// ---------------------------------------------------------------------------
-// (0, 1)
-// (0. 7)
-// (1, 1) ("A", 1, 3)
-// (1, 7) ("A", 1, 3)
-// (2, 1) ("A", 1, 3), ("A", 2, 3), ("B", 1, 2)
-// (2, 7) ("A", 1, 3), ("A", 2, 3), ("B", 1, 2)
-// (3, 1) ("A", 1, 3)
-// (3, 7) ("A", 1, 3), ("A", 2, 3), ("B", 1, 2)
-// (4, 1) ("A", 1, 3), ("A", 2, 3), ("B", 1, 2), ("B", 2, 2)
-// (4, 7) ("A", 1, 3), ("A", 2, 3), ("B", 1, 2), ("B", 2, 2)
-// (5, 1) ("A", 1, 3)
-// (5, 7) ("A", 1, 3), ("A", 2, 3), ("B", 1, 2), ("B", 2, 2)
-// (6, 1) ("A", 1, 3), ("A", 2, 3), ("B", 1, 2)
-// (6, 7) ("A", 1, 3), ("A", 2, 3), ("B", 1, 2), ("B", 2, 2)
-// (7, 1) ("A", 1, 3)
-// (7, 7) ("A", 1, 3), ("A", 2, 3), ("B", 1, 2), ("B", 2, 2)
-// (8, 1) ("A", 1, 3), ("A", 2, 3), ("B", 1, 2), ("B", 2, 2)
-// (8, 7) ("A", 1, 3), ("A", 2, 3), ("B", 1, 2), ("B", 2, 2)
-//
-// In addition, expect 1 ReportParticipationObservation each day for each
-// report in the registry.
-TEST_F(PerDeviceNumericEventAggregatorTest, ElapsedTimeCheckObservationValuesWithBackfillAndGc) {
- auto start_day_index = CurrentDayIndex();
- const auto& total_report_id =
- logger::testing::per_device_numeric_stats::kStreamingTimeTotalMetricReportId;
- const auto& min_report_id =
- logger::testing::per_device_numeric_stats::kStreamingTimeMinMetricReportId;
- const auto& max_report_id =
- logger::testing::per_device_numeric_stats::kStreamingTimeMaxMetricReportId;
- std::vector<MetricReportId> streaming_time_ids = {total_report_id, min_report_id, max_report_id};
- const auto& expected_params =
- logger::testing::per_device_numeric_stats::kExpectedAggregationParams;
- // Set |backfill_days_| to 3.
- size_t backfill_days = 3;
- SetBackfillDays(backfill_days);
- // Log events for 9 days. Call GenerateObservations() on the first 6 day
- // indices, and the 9th.
- uint32_t num_days = 9;
- for (uint32_t offset = 0; offset < num_days; offset++) {
- auto day_index = CurrentDayIndex();
- ResetObservationStore();
- for (uint32_t event_code = 1; event_code < 3; event_code++) {
- for (const auto& report_id : streaming_time_ids) {
- if (offset > 0 && (offset % event_code == 0)) {
- EXPECT_EQ(kOK, LogPerDeviceElapsedTimeEvent(report_id, day_index, "A", event_code, 3));
- }
- if (offset > 0 && offset % (2 * event_code) == 0) {
- EXPECT_EQ(kOK, LogPerDeviceElapsedTimeEvent(report_id, day_index, "B", event_code, 2));
- }
- }
- }
-
- // Advance |test_clock_| by 1 day.
- AdvanceClock(kDay);
- if (offset < 6 || offset == 8) {
- // Generate Observations and garbage-collect, both for the previous day
- // index according to |test_clock_|. Back up the LocalAggregateStore and
- // the AggregatedObservationHistoryStore.
- DoScheduledTasksNow();
- }
- // Make the set of Observations which are expected to be generated on
- // |start_day_index + offset| and check it against the contents of the
- // FakeObservationStore.
- ExpectedPerDeviceNumericObservations expected_per_device_numeric_obs;
- ExpectedReportParticipationObservations expected_report_participation_obs;
- switch (offset) {
- case 0: {
- for (uint32_t day_index = start_day_index - backfill_days; day_index <= start_day_index;
- day_index++) {
- for (const auto& pair :
- MakeExpectedReportParticipationObservations(expected_params, day_index)) {
- expected_report_participation_obs.insert(pair);
- }
- }
- EXPECT_TRUE(CheckPerDeviceNumericObservations(
- expected_per_device_numeric_obs, expected_report_participation_obs,
- observation_store_.get(), update_recipient_.get()));
- break;
- }
- case 1: {
- expected_per_device_numeric_obs[{total_report_id, day_index}] = {{1, {{"A", 1u, 3}}},
- {7, {{"A", 1u, 3}}}};
- expected_per_device_numeric_obs[{min_report_id, day_index}] = {{1, {{"A", 1u, 3}}},
- {7, {{"A", 1u, 3}}}};
- expected_per_device_numeric_obs[{max_report_id, day_index}] = {{1, {{"A", 1u, 3}}},
- {7, {{"A", 1u, 3}}}};
- expected_report_participation_obs =
- MakeExpectedReportParticipationObservations(expected_params, day_index);
- EXPECT_TRUE(CheckPerDeviceNumericObservations(
- expected_per_device_numeric_obs, expected_report_participation_obs,
- observation_store_.get(), update_recipient_.get()))
- << "day 1";
- break;
- }
- case 2: {
- expected_per_device_numeric_obs[{total_report_id, day_index}] = {
- {1, {{"A", 1u, 3}, {"A", 2u, 3}, {"B", 1u, 2}}},
- {7, {{"A", 1u, 6}, {"A", 2u, 3}, {"B", 1u, 2}}}};
- expected_per_device_numeric_obs[{min_report_id, day_index}] = {
- {1, {{"A", 1u, 3}, {"A", 2u, 3}, {"B", 1u, 2}}},
- {7, {{"A", 1u, 3}, {"A", 2u, 3}, {"B", 1u, 2}}}};
- expected_per_device_numeric_obs[{max_report_id, day_index}] = {
- {1, {{"A", 1u, 3}, {"A", 2u, 3}, {"B", 1u, 2}}},
- {7, {{"A", 1u, 3}, {"A", 2u, 3}, {"B", 1u, 2}}}};
- expected_report_participation_obs =
- MakeExpectedReportParticipationObservations(expected_params, day_index);
- EXPECT_TRUE(CheckPerDeviceNumericObservations(
- expected_per_device_numeric_obs, expected_report_participation_obs,
- observation_store_.get(), update_recipient_.get()));
- break;
- }
- case 3: {
- expected_per_device_numeric_obs[{total_report_id, day_index}] = {
- {1, {{"A", 1u, 3}}}, {7, {{"A", 1u, 9}, {"A", 2u, 3}, {"B", 1u, 2}}}};
- expected_per_device_numeric_obs[{min_report_id, day_index}] = {
- {1, {{"A", 1u, 3}}}, {7, {{"A", 1u, 3}, {"A", 2u, 3}, {"B", 1u, 2}}}};
- expected_per_device_numeric_obs[{max_report_id, day_index}] = {
- {1, {{"A", 1u, 3}}}, {7, {{"A", 1u, 3}, {"A", 2u, 3}, {"B", 1u, 2}}}};
- expected_report_participation_obs =
- MakeExpectedReportParticipationObservations(expected_params, day_index);
- EXPECT_TRUE(CheckPerDeviceNumericObservations(
- expected_per_device_numeric_obs, expected_report_participation_obs,
- observation_store_.get(), update_recipient_.get()));
- break;
- }
- case 4: {
- expected_per_device_numeric_obs[{total_report_id, day_index}] = {
- {1, {{"A", 1u, 3}, {"A", 2u, 3}, {"B", 1u, 2}, {"B", 2u, 2}}},
- {7, {{"A", 1u, 12}, {"A", 2u, 6}, {"B", 1u, 4}, {"B", 2u, 2}}}};
- expected_per_device_numeric_obs[{min_report_id, day_index}] = {
- {1, {{"A", 1u, 3}, {"A", 2u, 3}, {"B", 1u, 2}, {"B", 2u, 2}}},
- {7, {{"A", 1u, 3}, {"A", 2u, 3}, {"B", 1u, 2}, {"B", 2u, 2}}}};
- expected_per_device_numeric_obs[{max_report_id, day_index}] = {
- {1, {{"A", 1u, 3}, {"A", 2u, 3}, {"B", 1u, 2}, {"B", 2u, 2}}},
- {7, {{"A", 1u, 3}, {"A", 2u, 3}, {"B", 1u, 2}, {"B", 2u, 2}}}};
- expected_report_participation_obs =
- MakeExpectedReportParticipationObservations(expected_params, day_index);
- EXPECT_TRUE(CheckPerDeviceNumericObservations(
- expected_per_device_numeric_obs, expected_report_participation_obs,
- observation_store_.get(), update_recipient_.get()));
- break;
- }
- case 5: {
- expected_per_device_numeric_obs[{total_report_id, day_index}] = {
- {1, {{"A", 1u, 3}}}, {7, {{"A", 1u, 15}, {"A", 2u, 6}, {"B", 1u, 4}, {"B", 2u, 2}}}};
- expected_per_device_numeric_obs[{min_report_id, day_index}] = {
- {1, {{"A", 1u, 3}}}, {7, {{"A", 1u, 3}, {"A", 2u, 3}, {"B", 1u, 2}, {"B", 2u, 2}}}};
- expected_per_device_numeric_obs[{max_report_id, day_index}] = {
- {1, {{"A", 1u, 3}}}, {7, {{"A", 1u, 3}, {"A", 2u, 3}, {"B", 1u, 2}, {"B", 2u, 2}}}};
- expected_report_participation_obs =
- MakeExpectedReportParticipationObservations(expected_params, day_index);
- EXPECT_TRUE(CheckPerDeviceNumericObservations(
- expected_per_device_numeric_obs, expected_report_participation_obs,
- observation_store_.get(), update_recipient_.get()));
- break;
- }
- case 8: {
- expected_per_device_numeric_obs[{total_report_id, start_day_index + 6}] = {
- {1, {{"A", 1u, 3}, {"A", 2u, 3}, {"B", 1u, 2}}},
- {7, {{"A", 1u, 18}, {"A", 2u, 9}, {"B", 1u, 6}, {"B", 2u, 2}}}};
- expected_per_device_numeric_obs[{total_report_id, start_day_index + 7}] = {
- {1, {{"A", 1u, 3}}}, {7, {{"A", 1u, 21}, {"A", 2u, 9}, {"B", 1u, 6}, {"B", 2u, 2}}}};
- expected_per_device_numeric_obs[{total_report_id, start_day_index + 8}] = {
- {1, {{"A", 1u, 3}, {"A", 2u, 3}, {"B", 1u, 2}, {"B", 2u, 2}}},
- {7, {{"A", 1u, 21}, {"A", 2u, 12}, {"B", 1u, 8}, {"B", 2u, 4}}}};
-
- expected_per_device_numeric_obs[{min_report_id, start_day_index + 6}] = {
- {1, {{"A", 1u, 3}, {"A", 2u, 3}, {"B", 1u, 2}}},
- {7, {{"A", 1u, 3}, {"A", 2u, 3}, {"B", 1u, 2}, {"B", 2u, 2}}}};
- expected_per_device_numeric_obs[{min_report_id, start_day_index + 7}] = {
- {1, {{"A", 1u, 3}}}, {7, {{"A", 1u, 3}, {"A", 2u, 3}, {"B", 1u, 2}, {"B", 2u, 2}}}};
- expected_per_device_numeric_obs[{min_report_id, start_day_index + 8}] = {
- {1, {{"A", 1u, 3}, {"A", 2u, 3}, {"B", 1u, 2}, {"B", 2u, 2}}},
- {7, {{"A", 1u, 3}, {"A", 2u, 3}, {"B", 1u, 2}, {"B", 2u, 2}}}};
-
- expected_per_device_numeric_obs[{max_report_id, start_day_index + 6}] = {
- {1, {{"A", 1u, 3}, {"A", 2u, 3}, {"B", 1u, 2}}},
- {7, {{"A", 1u, 3}, {"A", 2u, 3}, {"B", 1u, 2}, {"B", 2u, 2}}}};
- expected_per_device_numeric_obs[{max_report_id, start_day_index + 7}] = {
- {1, {{"A", 1u, 3}}}, {7, {{"A", 1u, 3}, {"A", 2u, 3}, {"B", 1u, 2}, {"B", 2u, 2}}}};
- expected_per_device_numeric_obs[{max_report_id, start_day_index + 8}] = {
- {1, {{"A", 1u, 3}, {"A", 2u, 3}, {"B", 1u, 2}, {"B", 2u, 2}}},
- {7, {{"A", 1u, 3}, {"A", 2u, 3}, {"B", 1u, 2}, {"B", 2u, 2}}}};
-
- for (uint32_t day_index = start_day_index + 6; day_index <= start_day_index + 8;
- day_index++) {
- for (const auto& pair :
- MakeExpectedReportParticipationObservations(expected_params, day_index)) {
- expected_report_participation_obs.insert(pair);
- }
- }
- EXPECT_TRUE(CheckPerDeviceNumericObservations(
- expected_per_device_numeric_obs, expected_report_participation_obs,
- observation_store_.get(), update_recipient_.get()));
- break;
- }
- default:
- EXPECT_TRUE(CheckPerDeviceNumericObservations(
- expected_per_device_numeric_obs, expected_report_participation_obs,
- observation_store_.get(), update_recipient_.get()));
- }
- }
}
// Tests that the LocalAggregateStore is updated as expected when
@@ -3607,150 +1191,6 @@
}
}
-// Check that GenerateObservations returns an OK status after some events have been logged for a
-// PerDeviceHistogram report.
-TEST_F(PerDeviceHistogramEventAggregatorTest, GenerateObservations) {
- const auto day_index = CurrentDayIndex();
- // Log several events on |day_index|.
- EXPECT_EQ(kOK, LogPerDeviceCountEvent(
- logger::testing::per_device_histogram::kSettingsChangedMetricReportId,
- day_index, "component_C", 0u, 5));
- EXPECT_EQ(kOK, LogPerDeviceCountEvent(
- logger::testing::per_device_histogram::kSettingsChangedMetricReportId,
- day_index, "component_C", 0u, 5));
-
- // Generate locally aggregated Observations for |day_index|.
- EXPECT_EQ(kOK, GenerateObservations(day_index));
-}
-
-// Tests GenerateObservations() and GarbageCollect() in the case where the
-// LocalAggregateStore contains aggregates for metrics with both UTC and LOCAL
-// time zone policies, and where the day index in local time may be less than
-// the day index in UTC.
-TEST_F(NoiseFreeMixedTimeZoneEventAggregatorTest, LocalBeforeUTC) {
- std::vector<ExpectedUniqueActivesObservations> expected_obs(3);
- // Begin at a time when the current day index is the same in both UTC and
- // local time. Log 1 event for event code 0 for each of the 2 reports, then
- // generate Observations and garbage-collect for the previous day index in
- // each of UTC and local time.
- auto start_day_index = CurrentDayIndex();
- LogUniqueActivesEvent(logger::testing::mixed_time_zone::kDeviceBootsMetricReportId,
- start_day_index, 0u);
- LogUniqueActivesEvent(logger::testing::mixed_time_zone::kFeaturesActiveMetricReportId,
- start_day_index, 0u);
- GenerateObservations(start_day_index - 1, start_day_index - 1);
- GarbageCollect(start_day_index - 1, start_day_index - 1);
- // Form the expected contents of the FakeObservationStore.
- // Since no events were logged on the previous day and no Observations have
- // been generated for that day yet, expect Observations of non-activity for
- // all event codes, for both reports.
- expected_obs[0] = MakeNullExpectedUniqueActivesObservations(
- logger::testing::mixed_time_zone::kExpectedAggregationParams, start_day_index - 1);
- EXPECT_TRUE(CheckUniqueActivesObservations(expected_obs[0], observation_store_.get(),
- update_recipient_.get()));
- ResetObservationStore();
- // Advance the day index in UTC, but not in local time, and log 1 event for
- // event code 1 for each of the 2 reports. Generate Observations and
- // garbage-collect for the previous day in each of UTC and local time.
- LogUniqueActivesEvent(logger::testing::mixed_time_zone::kDeviceBootsMetricReportId,
- start_day_index, 1u);
- LogUniqueActivesEvent(logger::testing::mixed_time_zone::kFeaturesActiveMetricReportId,
- start_day_index + 1, 1u);
- GenerateObservations(start_day_index, start_day_index - 1);
- GarbageCollect(start_day_index, start_day_index - 1);
- // Form the expected contents of the FakeObservationStore. Since
- // Observations have already been generated for the
- // DeviceBoots_UniqueDevices report for |start_day_index - 1|, expect no
- // Observations for that report.
- expected_obs[1][{logger::testing::mixed_time_zone::kFeaturesActiveMetricReportId,
- start_day_index}] = {{1, {true, false, false}}};
- EXPECT_TRUE(CheckUniqueActivesObservations(expected_obs[1], observation_store_.get(),
- update_recipient_.get()));
- ResetObservationStore();
- // Advance the day index in local time so that it is equal to the day index
- // in UTC. Log 1 event for event code 2 for each of the 2 reports, then
- // generate Observations and garbage-collect for the previous day in each of
- // UTC and local time.
- LogUniqueActivesEvent(logger::testing::mixed_time_zone::kDeviceBootsMetricReportId,
- start_day_index + 1, 2u);
- LogUniqueActivesEvent(logger::testing::mixed_time_zone::kFeaturesActiveMetricReportId,
- start_day_index + 1, 2u);
- GenerateObservations(start_day_index, start_day_index);
- GarbageCollect(start_day_index, start_day_index);
- // Form the expected contents of the FakeObservationStore. Since
- // Observations have already been generated for the
- // FeaturesActive_UniqueDevices report for day |start_day_index|, expect no
- // Observations for that report.
- expected_obs[2][{logger::testing::mixed_time_zone::kDeviceBootsMetricReportId, start_day_index}] =
- {{1, {true, true, false}}};
- EXPECT_TRUE(CheckUniqueActivesObservations(expected_obs[2], observation_store_.get(),
- update_recipient_.get()));
-}
-
-// Tests GenerateObservations() and GarbageCollect() in the case where the
-// LocalAggregateStore contains aggregates for metrics with both UTC and LOCAL
-// time zone policies, and where the day index in UTC may be less than
-// the day index in local time.
-TEST_F(NoiseFreeMixedTimeZoneEventAggregatorTest, LocalAfterUTC) {
- std::vector<ExpectedUniqueActivesObservations> expected_obs(3);
- // Begin at a time when the current day index is the same in both UTC and
- // local time. Log 1 event for event code 0 for each of the 2 reports, then
- // generate Observations and garbage-collect for the previous day index in
- // each of UTC and local time.
- auto start_day_index = CurrentDayIndex();
- LogUniqueActivesEvent(logger::testing::mixed_time_zone::kDeviceBootsMetricReportId,
- start_day_index, 0u);
- LogUniqueActivesEvent(logger::testing::mixed_time_zone::kFeaturesActiveMetricReportId,
- start_day_index, 0u);
- GenerateObservations(start_day_index - 1, start_day_index - 1);
- GarbageCollect(start_day_index - 1, start_day_index - 1);
- // Form the expected contents of the FakeObservationStore.
- // Since no events were logged on the previous day and no Observations have
- // been generated for that day yet, expect Observations of non-activity for
- // all event codes, for both reports.
- expected_obs[0] = MakeNullExpectedUniqueActivesObservations(
- logger::testing::mixed_time_zone::kExpectedAggregationParams, start_day_index - 1);
- EXPECT_TRUE(CheckUniqueActivesObservations(expected_obs[0], observation_store_.get(),
- update_recipient_.get()));
- ResetObservationStore();
- // Advance the day index in local time, but not in UTC, and log 1 event for
- // event code 1 for each of the 2 reports. Generate Observations and
- // garbage-collect for the previous day in each of UTC and local time.
- LogUniqueActivesEvent(logger::testing::mixed_time_zone::kDeviceBootsMetricReportId,
- start_day_index + 1, 1u);
- LogUniqueActivesEvent(logger::testing::mixed_time_zone::kFeaturesActiveMetricReportId,
- start_day_index, 1u);
- GenerateObservations(start_day_index - 1, start_day_index);
- GarbageCollect(start_day_index - 1, start_day_index);
- // Form the expected contents of the FakeObservationStore. Since
- // Observations have already been generated for the
- // FeaturesActive_UniqueDevices report for |start_day_index - 1|, expect no
- // Observations for that report.
- expected_obs[1][{logger::testing::mixed_time_zone::kDeviceBootsMetricReportId, start_day_index}] =
- {{1, {true, false, false}}};
- EXPECT_TRUE(CheckUniqueActivesObservations(expected_obs[1], observation_store_.get(),
- update_recipient_.get()));
- ResetObservationStore();
- // Advance the day index in UTC so that it is equal to the day index in
- // local time. Log 1 event for event code 2 for each of the 2 reports, then
- // generate Observations and garbage-collect for the previous day in each of
- // UTC and local time.
- LogUniqueActivesEvent(logger::testing::mixed_time_zone::kDeviceBootsMetricReportId,
- start_day_index + 1, 2u);
- LogUniqueActivesEvent(logger::testing::mixed_time_zone::kFeaturesActiveMetricReportId,
- start_day_index + 1, 2u);
- GenerateObservations(start_day_index, start_day_index);
- GarbageCollect(start_day_index, start_day_index);
- // Form the expected contents of the FakeObservationStore. Since
- // Observations have already been generated for the
- // DeviceBoots_UniqueDevices report for day |start_day_index|, expect no
- // Observations for that report.
- expected_obs[2][{logger::testing::mixed_time_zone::kFeaturesActiveMetricReportId,
- start_day_index}] = {{1, {true, true, false}}};
- EXPECT_TRUE(CheckUniqueActivesObservations(expected_obs[2], observation_store_.get(),
- update_recipient_.get()));
-}
-
// Starts the worker thread, and destructs the EventAggregator without
// explicitly shutting down the worker thread. Checks that the shutdown flag
// and worker thread are in the expected states before and after the thread is
@@ -3832,5 +1272,4 @@
EXPECT_GE(local_aggregate_proto_store_->write_count_, 1);
}
-} // namespace local_aggregation
-} // namespace cobalt
+} // namespace cobalt::local_aggregation
diff --git a/src/logger/event_loggers_test.cc b/src/logger/event_loggers_test.cc
index 7cc7268..e02cc16 100644
--- a/src/logger/event_loggers_test.cc
+++ b/src/logger/event_loggers_test.cc
@@ -303,7 +303,7 @@
}
Status GarbageCollectAggregateStore(uint32_t day_index) {
- return event_aggregator_->GarbageCollect(day_index);
+ return event_aggregator_->aggregate_store_->GarbageCollect(day_index);
}
};
@@ -1356,8 +1356,8 @@
uint32_t NumPerDeviceNumericAggregatesInStore() {
int count = 0;
- for (const auto& aggregates :
- protected_aggregate_store_.lock()->local_aggregate_store.by_report_key()) {
+ for (const auto& aggregates : aggregate_store_->protected_aggregate_store_.lock()
+ ->local_aggregate_store.by_report_key()) {
if (aggregates.second.has_numeric_aggregates()) {
count += aggregates.second.numeric_aggregates().by_component().size();
}
