blob: 198540196f37c8276596f161fa2a5393cf0f5407 [file] [log] [blame]
// Copyright 2018 The Fuchsia Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
syntax = "proto3";
package cobalt;
import "src/registry/aggregation_window.proto";
// An Observation is a piece of data sent from a Cobalt client to the Cobalt
// server as input to a Report.
//
// Observations are associated with a specific Metric. They are built on a
// Fuchsia client based on the Events logged for that Metric. Observations are
// associated with a specific Report. They are built for the purpose of
// generating that Report. They are transmitted from the Fuchsia client to
// the Cobalt server where they are aggregated and analyzed in order to generate
// the Report.
//
// Observations may store their data using special privacy-preserving encodings.
// The specification of how to do this is part of the definition of a Report.
// Observations are built for a particular Report and so use an encoding
// appropriate for that Report.
//
// There are different types of Observations that are appropriate for different
// types of Metrics and different types of Reports.
//
// An Observation is always transmitted and stored along with
// some ObservationMetadata that describes, among other things, which Metric
// and which Report it is for and on which day the Observation was formed.
//
// Observations are transmitted from the client to the server encrypted, inside
// the |ciphertext| field of an EncryptedMessage. Many encrypted Observations
// are transmitted together inside of an ObservationBatch.
message Observation {
// Next observation_type ID: 9
// Next general ID: 1002;
reserved 4, 5;
// An Observation has one of the following types.
oneof observation_type {
IntegerEventObservation numeric_event = 1;
HistogramObservation histogram = 2;
BasicRapporObservation basic_rappor = 3;
UniqueActivesObservation unique_actives = 6;
PerDeviceNumericObservation per_device_numeric = 7;
PerDeviceHistogramObservation per_device_histogram = 8;
SumAndCountObservation sum_and_count = 9;
IntegerObservation integer = 10;
IndexHistogramObservation index_histogram = 11;
StringHistogramObservation string_histogram = 12;
PrivateIndexObservation private_index = 13;
CustomObservation custom = 1000;
ReportParticipationObservation report_participation = 10000;
}
// A quasi-unique identifier for this observation. This is randomly generated
// on the client and used on the server as part of a fully-unique identifier.
// This field allows the operation of sending an Observation to the Cobalt
// server to be idempotent: If the same Observation is transmitted twice then
// the server will store the observation only once.
bytes random_id = 1001;
}
// Cobalt 1.1
// Observations of type SumAndCountObservation contain one or more triples of
// event vectors, a signed integer recording a sum and an unsigned integer
// recording a count.
//
// This type of observation is used only for the FLEETWIDE_MEANS report type.
message SumAndCountObservation {
message SumAndCount {
// Event vector to which the sum and count are associated.
repeated uint32 event_codes = 1;
// The device-level sum.
sint64 sum = 2;
// the device-level count.
uint64 count = 3;
}
repeated SumAndCount sums_and_counts = 1;
}
// Cobalt 1.1
// Observations of type IntegerObservation contain one or more pairs of
// event vectors and signed integers.
//
// This type of observation is used for the following report types:
// FLEETWIDE_OCCURRENCE_COUNTS
// HOURLY_VALUE_NUMERIC_STATS
// HOURLY_VALUE_HISTOGRAMS
// UNIQUE_DEVICE_COUNTS
// UNIQUE_DEVICE_NUMERIC_STATS
// UNIQUE_DEVICE_HISTOGRAMS
message IntegerObservation {
message Value {
// Event vector to which the value is associated.
repeated uint32 event_codes = 1;
// The value depends upon the report type.
//
// FLEETWIDE_OCCURRENCE_COUNTS
// Device-level count of events with the associated event vector.
// Value is between 0 and the configured maximum count (inclusive).
//
// UNIQUE_DEVICE_COUNTS
// 1 if the associated event vector was observed.
//
// HOURLY_VALUE_NUMERIC_STATS
// UNIQUE_DEVICE_NUMERIC_STATS
// HOURLY_VALUE_HISTOGRAMS
// UNIQUE_DEVICE_HISTOGRAMS
// Device-level aggregate value for the associated event vector.
sint64 value = 2;
}
repeated Value values = 1;
}
// IndexHistogram represents a list of counts for each bucket of a histogram
// associated with a particular event vector.
message IndexHistogram {
// Event vector to which the histogram is associated.
repeated uint32 event_codes = 1;
// bucket_counts[i] is the count for the bucket_indices[i]th index in the
// histogram.
repeated uint32 bucket_indices = 2;
repeated sint64 bucket_counts = 3;
}
// Cobalt 1.1
// Observations of type IndexHistogramObservation contain one or more pairs of
// event vectors and lists of bucket counts.
//
// This type of observation is used with the FLEETWIDE_HISTOGRAMS report type.
message IndexHistogramObservation {
repeated IndexHistogram index_histograms = 1;
}
// Cobalt 1.1
// Observations of the type StringHistogramObservation contain a list of strings
// mapping string hashes to indices and one or more pairs of event vectors
// and lists of bucket counts.
//
// This type of observation is used only with the STRING_COUNTS report type.
message StringHistogramObservation {
// List of hashes of strings (hashed using Farmhash Fingerprint128).
// The string that hashes to the bytes value in the ith position in
// |string_hashes| corresponds to the bucket with index i in each of the
// |bucket_indices| values in |string_histograms|.
repeated bytes string_hashes = 1;
repeated IndexHistogram string_histograms = 2;
}
// Cobalt 1.1
// An observation of type PrivateIndexObservation contains a single integer.
// Observations of this type are produced by the PrivacyEncoder: given a single
// Observation of some other type, the PrivacyEncoder outputs one or more
// PrivateIndexObservations. In Cobalt 1.1, all data for reports with a nontrivial
// privacy level will be transported from client to server in the form of
// PrivateIndexObservations.
//
// The field |index| of a PrivateIndexObservation is an index into an enumeration
// of the set of possible Observations of another type (after bucketing some
// parameters, as specified in the ReportDefinition for which the Observation was
// generated.) The details of this enumeration depend on the type of Observation
// which was given to the PrivacyEncoder.
message PrivateIndexObservation {
uint64 index = 1;
}
// Observations of type IntegerEventObservation contain an event type specified
// by its index, a component specified by its hash, and an integer value.
// This type of Observation is used in the following cases:
//
// (a) A Metric of type EVENT_COUNT and a Report of type
// EVENT_COMPONENT_OCCURRENCE_COUNT.
// In this case the Observation is *immediate* meaning it is generated
// directly from a single Event as soon as the Event is logged. The |value|
// field contains the count. Note that the |period_duration_micros| from the
// Event is not used. It's purpose is to compute event *rates* which are not
// relevant to the EVENT_COMPONENT_OCCURRENCE_COUNT Report type.
//
// (b) A Metric of type ELAPSED_TIME and a Report of type NUMERIC_AGGREGATION
// or INT_RANGE_HISTOGRAM. In this case the Observation again is immediate.
// The |value| field contains the duration in microseconds.
//
// (c) A Metric of type FrameRateMetric and a Report of type
// NUMERIC_AGGREGATION or INT_RANGE_HISTOGRAM. In this case the Observation
// again is immediate. The |value| field contains round(fps * 1000).
//
// (d) A Metric of type MemoryUseMetric and a Report of type
// NUMERIC_AGGREGATION or INT_RANGE_HISTOGRAM. In this case the Observation
// again is immediate. The |value| field contains the number of bytes.
message IntegerEventObservation {
// The event codes for each of the metric_dimensions of this observation.
// Multiple event codes are packed together into the bits of this value.
//
// See config/packed_event_codes.h for a description of the bit layout.
uint64 event_code = 1;
// A hash of the component name. If component_name is empty then this field
// will be empty. Otherwise this field will contain the 32-byte SHA256
// hash of component_name.
bytes component_name_hash = 2;
int64 value = 3;
}
// A pair consisting of a bucket index and a count. Each bucket is
// an integer range. The definition of the buckets and their indices
// is given in the MetricDefinition.
message HistogramBucket {
// The index of one of the buckets.
uint32 index = 1;
// The count for that bucket.
uint64 count = 2;
};
// Observations of this type contain an encoding of an element of an
// an indexed set. The encoding uses Basic RAPPOR to provide local differential
// privacy.
//
// This type of Observation is used in the following case:
//
// A MetricDefinition of type EVENT_OCCURRED and Report of type
// SIMPLE_OCCURRENCE_COUNT. In this case the Observation is *immediate* meaning
// it is generated directly from a single EVENT_OCCURRED Event as soon as
// the Event is logged. The Event contains a single index, the index of the
// event-type that occurred. The |data| field in this Observation contains
// an encoding of that index. It contains a bit vector obtained by representing
// the index as a one-hot bit vector and then adding random noise by
// flipping bits.
message BasicRapporObservation {
// A bit vector containing the encoding of the index.
bytes data = 1;
}
// Observations of type HistogramObservation contain an event type specified
// by its index, a component specified by its hash, and a histogram of
// integer-range buckets. This type of Observation is used in the
// following case:
//
// A MetricDefinition of type INT_HISTOGRAM and a Report of type
// INT_RANGE_HISTOGRAM. In this case the Observation is *immediate* meaning
// it is generated directly from a single INT_HISTOGRAM Event as soon as
// the Event is logged.
message HistogramObservation {
// The event codes for each of the metric_dimensions of this observation.
// Multiple event codes are packed together into the bits of this value.
uint64 event_code = 1;
// A hash of the component name. If component_name is empty then this field
// will be empty. Otherwise this field will contain the 32-byte SHA256
// hash of component_name.
bytes component_name_hash = 2;
// The set of bucket indices and their corresponding counts.
repeated HistogramBucket buckets = 3;
};
// Observations of type UniqueActivesObservation are used when a
// MetricDefinition of type EVENT_OCCURRED is reported with a
// ReportDefinition of type UNIQUE_N_DAY_ACTIVES. An Observation
// of this type is created by aggregating Events on-device over
// a rolling window of 1, 7, or 30 days.
//
// A UniqueActivesObservation is generated each day for each event code of the
// associated MetricDefinition, for each window size in the associated
// ReportDefinition. A UniqueActivesObservation generated for a given day, event
// code, and window size encodes the occurrence or non-occurrence of at least
// one Event with that event code during the window of that size ending on that
// day.
message UniqueActivesObservation {
// A BasicRapporObservation encoding a single bit representing the occurrence
// or non-occurrence of |event_code| during the window associated with this
// Observation.
BasicRapporObservation basic_rappor_obs = 1;
// The size in days of the rolling window associated with this Observation (deprecated).
uint32 window_size = 2;
// The size in days or hours of the rolling window associated with this Observation.
OnDeviceAggregationWindow aggregation_window = 4;
// The event codes for each of the metric_dimensions of this observation.
// Multiple event codes are packed together into the bits of this value.
uint64 event_code = 3;
};
// Observations of type PerDeviceNumericObservation are used when a
// MetricDefinition of type EVENT_COUNT, ELAPSED_TIME, MEMORY_USAGE, or
// FRAME_RATE is reported with a ReportDefinition of type
// PER_DEVICE_NUMERIC_STATS. An Observation of this type is created by
// aggregating Events on-device over a rolling window of 1, 7, or 30 days.
//
// A PerDeviceNumericObservation is generated each day, for each aggregation
// window size of the ReportDefinition, for each pair (component, event code)
// for which an event was logged during the window. In addition, each day and
// for each window size, each device generates one
// ReportParticipationObservation for that report. The number
// of those Observations is used by the ReportGenerator to compute the number of
// devices on which each pair (component, event code) was *not* logged during
// the window.
message PerDeviceNumericObservation {
// An IntegerEventObservation containing the event code, component,
// and occurrence count associated with this Observation.
IntegerEventObservation integer_event_obs = 1;
// The size in days of the rolling window associated with this Observation (deprecated).
uint32 window_size = 2;
// The size in days or hours of the rolling window associated with this Observation.
OnDeviceAggregationWindow aggregation_window = 3;
};
// Observations of type PerDeviceHistogramObservation are used when a MetricDefinition of type
// EVENT_COUNT, ELAPSED_TIME, MEMORY_USAGE, or FRAME_RATE is reported with a ReportDefinition of
// type PER_DEVICE_HISTOGRAM. An Observation of this type is created by aggregating Events on-device
// and then creating a histogram to represent the aggregation. Currently all supported aggregations
// produce one number (example: SUM) and therefore all Observations of this type will represent one
// value.
//
// A PerDeviceHistogramObservation is generated each day, for each aggregation window size of
// the ReportDefinition, for each pair (component, event code) for which an event was logged during
// the window. In addition, for each day and for each window size, each device generates one
// ReportParticipationObservation for that report. The number of those Observations is used by the
// ReportGenerator to compute the number of devices on which each pair (component, event code) was
// *not* logged during the window.
message PerDeviceHistogramObservation {
// A histogram of the aggregated value(s) over the aggregation_window. This can represent a single
// or multiple values depending on the aggregation type.
HistogramObservation histogram = 1;
// The size in days or hours of the rolling window associated with this Observation.
OnDeviceAggregationWindow aggregation_window = 2;
};
// A value for a single dimension of a Cobalt custom metric. Both CustomEvent
// (the object logged to the Cobalt client on the device) and CustomObservation
// (transmitted from the Cobalt client device to the server) contain a
// CustomDimension values.
message CustomDimensionValue {
oneof data {
// A human-readable, UTF8 string. Maps to a proto string.
string string_value = 1;
// An integer. Maps to a proto int64.
int64 int_value = 2;
// An uninterpreted sequence of bytes. Maps to proto bytes.
bytes blob_value = 3;
// The index of an enumerated value, specified in the metric configuration.
// Usually used for the index of an enum value defined in the associated
// .proto file.
uint32 index_value = 4;
// A double-precision floating point value. Maps to a proto double.
double double_value = 5;
}
}
// Observations of this type contain a custom number of parts each with a
// custom name and a custom type.
//
// This type of Observation is used in the following case:
//
// A MetricDefinition of type CUSTOM and Report of type
// CUSTOM_RAW_DUMP. In this case the Observation is *immediate* meaning it is
// generated directly from a single CUSTOM Event as soon as the Event is
// logged. The Event represents the contents of a protocol buffer either as a
// list of (name, value) pairs corresponding to the fields in the protocol
// buffer, or the serialized protocol buffer itself.
message CustomObservation {
// Only one of values or serialized_proto must be set.
// The keys are the names of the metric dimensions to which each
// CustomDimensionValue is associated.
map<string, CustomDimensionValue> values = 1;
// The serialized protocol buffer.
bytes serialized_proto = 2;
}
// Observations of this type are used to signal that a given device was
// collecting data for a given report, over some window of time. This
// Observation type has no required fields, although the size of an
// aggregation window may be provided. Observations are generated and
// rate-limited by the EventAggregator.
//
// Cobalt 1.0 usage:
// ReportParticipationObservations are produced by the EventAggregator and
// consumed by the ReportGenerator for reports of type PER_DEVICE_COUNT_STATS.
// For each report of that type, each device sends one Observation of this type
// per day, along with a PerDeviceCountObservation for tuple (component, event
// code, window size) for which at least one event with that code occurred
// during the window of that size, with that component. The ReportGenerator
// uses the number of each of the two kinds of Observations to compute the
// number of devices in the fleet on which a given pair (component, event
// code) did *not* occur during the aggregation window.
//
// Cobalt 1.1 usage:
// ReportParticipationObservations are produced by the PrivacyEncoder and consumed by the
// ReportGenerator for reports that use local differential privacy.
message ReportParticipationObservation {}
// TODO(azani) Remove this message. This is a vestige of Cobalt 0.1.
// Currently it is being used in the interface to the Basic Rappor Encoder
// so that needs to be refactored before we can remove this.
message ValuePart {
oneof data {
// A human-readable, UTF8 string.
string string_value = 1;
// An integer.
int64 int_value = 2;
// An uninterpreted sequence of bytes.
bytes blob_value = 3;
// A zero-based index into some enumerated set that is specified outside
// of Cobalt's configuration. See comments on the INDEX DataType in
// metrics.proto for more about this.
uint32 index_value = 4;
// A double-precision floating point value.
double double_value = 5;
}
}