[PrivacyEncoder] Decode without rounding for remaining report types
Adds methods for decoding private indices without
rounding to the nearest integer, for all of the
remaining report types with numeric observations.
(This was done for FleetwideOccurrenceCount in a
previous change.) Switching the server-side privacy
decoder to use these methods will eliminate
unnecessary error from aggregated reports.
In a follow-up, the original decoding methods
will be deleted.
Bug: 95956
Change-Id: Ie853236e3576bc324b11f76f13adb70b6349a8bb
Reviewed-on: https://fuchsia-review.googlesource.com/c/cobalt/+/660348
Reviewed-by: Alexandre Zani <azani@google.com>
Commit-Queue: Laura Peskin <pesk@google.com>
diff --git a/src/algorithms/privacy/numeric_encoding.cc b/src/algorithms/privacy/numeric_encoding.cc
index 9967083..9ebde43 100644
--- a/src/algorithms/privacy/numeric_encoding.cc
+++ b/src/algorithms/privacy/numeric_encoding.cc
@@ -59,6 +59,11 @@
static_cast<int64_t>(max_count), num_index_points));
}
+double CountAsDoubleFromIndex(uint64_t index, uint64_t max_count, uint64_t num_index_points) {
+ return DoubleFromIndex(index - num_index_points, 0, static_cast<double>(max_count),
+ num_index_points);
+}
+
// TODO(fxbug.dev/85571): NOLINTNEXTLINE(bugprone-easily-swappable-parameters)
void HistogramBucketAndCountFromIndex(uint64_t index, uint64_t max_count, uint64_t num_index_points,
uint32_t* bucket_index, uint64_t* bucket_count) {
@@ -68,6 +73,16 @@
IntegerFromIndex(numeric_index, 0, static_cast<int64_t>(max_count), num_index_points));
}
+// TODO(fxbug.dev/85571): NOLINTNEXTLINE(bugprone-easily-swappable-parameters)
+void HistogramBucketAndCountAsDoubleFromIndex(uint64_t index, uint64_t max_count,
+ uint64_t num_index_points, uint32_t* bucket_index,
+ double* bucket_count) {
+ *bucket_index = static_cast<uint32_t>(index / num_index_points);
+ uint64_t numeric_index = index - (*bucket_index * num_index_points);
+ *bucket_count =
+ DoubleFromIndex(numeric_index, 0, static_cast<double>(max_count), num_index_points);
+}
+
uint64_t ValueAndEventVectorIndicesToIndex(uint64_t value_index, uint64_t event_vector_index,
uint64_t max_event_vector_index) {
return value_index * (max_event_vector_index + 1) + event_vector_index;
diff --git a/src/algorithms/privacy/numeric_encoding.h b/src/algorithms/privacy/numeric_encoding.h
index 91c17c2..d530e74 100644
--- a/src/algorithms/privacy/numeric_encoding.h
+++ b/src/algorithms/privacy/numeric_encoding.h
@@ -48,10 +48,20 @@
// Decodes an |index| that was encoded with CountToIndex.
uint64_t CountFromIndex(uint64_t index, uint64_t max_count, uint64_t num_index_points);
+// Decodes an |index| that was encoded with CountToIndex, as a double representing an approximate
+// count.
+double CountAsDoubleFromIndex(uint64_t index, uint64_t max_count, uint64_t num_index_points);
+
// Decodes an |index| that was encoded with HistogramBucketAndCountToIndex.
void HistogramBucketAndCountFromIndex(uint64_t index, uint64_t max_count, uint64_t num_index_points,
uint32_t* bucket_index, uint64_t* bucket_count);
+// Decodes an |index| that was encoded with HistogramBucketAndCountToIndex, decoding the count value
+// as a double.
+void HistogramBucketAndCountAsDoubleFromIndex(uint64_t index, uint64_t max_count,
+ uint64_t num_index_points, uint32_t* bucket_index,
+ double* bucket_count);
+
// ValueAndEventVectorIndicesToIndex uniquely maps a |value_index|, |event_vector_index| pair to a
// single index.
//
diff --git a/src/algorithms/privacy/numeric_encoding_test.cc b/src/algorithms/privacy/numeric_encoding_test.cc
index 0f561e6..1afc181 100644
--- a/src/algorithms/privacy/numeric_encoding_test.cc
+++ b/src/algorithms/privacy/numeric_encoding_test.cc
@@ -100,6 +100,7 @@
}
TEST(NumericEncodingTest, DoubleFromIndex) {
+ // The approximate numeric values are all integers.
double min_value = -4.0;
double max_value = 6.0;
uint64_t num_index_points = 6u;
@@ -109,6 +110,14 @@
EXPECT_THAT(DoubleFromIndex(index, min_value, max_value, num_index_points), DoubleEq(expected));
expected += 2.0;
}
+
+ // The approximate numeric values are not all integers.
+ num_index_points = 5u;
+ expected = min_value;
+ for (uint64_t index = 0u; index <= num_index_points; index++) {
+ EXPECT_THAT(DoubleFromIndex(index, min_value, max_value, num_index_points), DoubleEq(expected));
+ expected += 2.5;
+ }
}
TEST(NumericEncodingTest, IntegerFromIndex) {
@@ -135,6 +144,27 @@
}
}
+TEST(NumericEncodingTest, CountAsDoubleFromIndex) {
+ // The approximate counts are all integers.
+ uint64_t max_count = 10u;
+ uint64_t num_index_points = 6u;
+ double expected = 0.0;
+
+ for (uint64_t index = num_index_points; index < 2 * num_index_points; index++) {
+ EXPECT_THAT(CountAsDoubleFromIndex(index, max_count, num_index_points), DoubleEq(expected));
+ expected += 2.0;
+ }
+
+ // The approximate counts are not all integers.
+ num_index_points = 5u;
+ expected = 0.0;
+
+ for (uint64_t index = num_index_points; index < 2 * num_index_points; index++) {
+ EXPECT_THAT(CountAsDoubleFromIndex(index, max_count, num_index_points), DoubleEq(expected));
+ expected += 2.5;
+ }
+}
+
TEST(NumericEncodingTest, HistogramBucketAndCountFromIndex) {
uint64_t max_count = 10u;
uint64_t num_index_points = 6u;
@@ -160,6 +190,53 @@
}
}
+TEST(NumericEncodingTest, HistogramBucketAndCountAsDoubleFromIndex) {
+ // The approximate counts are all integers.
+ uint64_t max_count = 10u;
+ uint64_t num_index_points = 6u;
+ uint64_t num_buckets = 10u;
+
+ double expected_bucket_count = 0.0;
+ uint64_t expected_bucket_index = 0u;
+
+ for (uint64_t index = 0u; index < num_index_points * num_buckets; ++index) {
+ double bucket_count;
+ uint32_t bucket_index;
+
+ HistogramBucketAndCountAsDoubleFromIndex(index, max_count, num_index_points, &bucket_index,
+ &bucket_count);
+ EXPECT_EQ(bucket_index, expected_bucket_index);
+ EXPECT_THAT(bucket_count, DoubleEq(expected_bucket_count));
+
+ expected_bucket_count += 2.0;
+ if (expected_bucket_count > static_cast<double>(max_count)) {
+ expected_bucket_count = 0.0;
+ ++expected_bucket_index;
+ }
+ }
+
+ // The approximate counts are not all integers.
+ num_index_points = 5u;
+ expected_bucket_count = 0.0;
+ expected_bucket_index = 0u;
+
+ for (uint64_t index = 0u; index < num_index_points * num_buckets; ++index) {
+ double bucket_count;
+ uint32_t bucket_index;
+
+ HistogramBucketAndCountAsDoubleFromIndex(index, max_count, num_index_points, &bucket_index,
+ &bucket_count);
+ EXPECT_EQ(bucket_index, expected_bucket_index);
+ EXPECT_THAT(bucket_count, DoubleEq(expected_bucket_count));
+
+ expected_bucket_count += 2.5;
+ if (expected_bucket_count > static_cast<double>(max_count)) {
+ expected_bucket_count = 0.0;
+ ++expected_bucket_index;
+ }
+ }
+}
+
TEST(NumericEncodingTest, ValueAndEventVectorIndicesToIndex) {
uint64_t num_index_points = 6;
uint64_t max_event_vector_index = 9;
diff --git a/src/lib/privacy/private_index_decoding.cc b/src/lib/privacy/private_index_decoding.cc
index 81b3124..f3f9e78 100644
--- a/src/lib/privacy/private_index_decoding.cc
+++ b/src/lib/privacy/private_index_decoding.cc
@@ -122,6 +122,42 @@
return Status::OkStatus();
}
+cobalt::Status DecodePrivateIndexAsSumOrCountDouble(
+ uint64_t index,
+ const google::protobuf::RepeatedPtrField<MetricDefinition::MetricDimension>& metric_dimensions,
+ // TODO(fxbug.dev/85571): NOLINTNEXTLINE(bugprone-easily-swappable-parameters)
+ int64_t min_value, int64_t max_value, uint64_t max_count, uint64_t num_index_points,
+ std::vector<uint32_t>* event_vector, SumOrCountDouble* sum_or_count) {
+ uint64_t event_vector_index = 0;
+ uint64_t value_index = 0;
+ ValueAndEventVectorIndicesFromIndex(index, logger::GetNumEventVectors(metric_dimensions) - 1,
+ &value_index, &event_vector_index);
+ if (Status decode_event_vector_index =
+ DecodePrivateIndexAsEventVector(event_vector_index, metric_dimensions, event_vector);
+ !decode_event_vector_index.ok()) {
+ return decode_event_vector_index;
+ }
+
+ if (IsCountIndex(value_index, num_index_points)) {
+ if (Status validate_value_index = ValidateIndexAsCount(value_index, num_index_points);
+ !validate_value_index.ok()) {
+ return validate_value_index;
+ }
+ (*sum_or_count).type = SumOrCountDouble::COUNT;
+ (*sum_or_count).count = CountAsDoubleFromIndex(value_index, max_count, num_index_points);
+ return Status::OkStatus();
+ }
+
+ if (Status validate_value_index = ValidateIndexAsNumericValue(value_index, num_index_points);
+ !validate_value_index.ok()) {
+ return validate_value_index;
+ }
+ (*sum_or_count).type = SumOrCountDouble::SUM;
+ (*sum_or_count).sum = DoubleFromIndex(value_index, static_cast<double>(min_value),
+ static_cast<double>(max_value), num_index_points);
+ return Status::OkStatus();
+}
+
Status DecodePrivateIndexAsHistogramBucketIndex(
uint64_t index,
const google::protobuf::RepeatedPtrField<MetricDefinition::MetricDimension>& metric_dimensions,
@@ -173,4 +209,31 @@
return Status::OkStatus();
}
+Status DecodePrivateIndexAsHistogramBucketIndexAndCountDouble(
+ uint64_t index,
+ const google::protobuf::RepeatedPtrField<MetricDefinition::MetricDimension>& metric_dimensions,
+ // TODO(fxbug.dev/85571): NOLINTNEXTLINE(bugprone-easily-swappable-parameters)
+ uint32_t max_bucket_index, uint64_t max_count, uint64_t num_index_points,
+ std::vector<uint32_t>* event_vector, uint32_t* bucket_index, double* bucket_count) {
+ uint64_t event_vector_index = 0;
+ uint64_t value_index = 0;
+ ValueAndEventVectorIndicesFromIndex(index, logger::GetNumEventVectors(metric_dimensions) - 1,
+ &value_index, &event_vector_index);
+ if (Status decode_event_vector_index =
+ DecodePrivateIndexAsEventVector(event_vector_index, metric_dimensions, event_vector);
+ !decode_event_vector_index.ok()) {
+ return decode_event_vector_index;
+ }
+
+ HistogramBucketAndCountAsDoubleFromIndex(value_index, max_count, num_index_points, bucket_index,
+ bucket_count);
+ if (Status validate_bucket_index =
+ ValidateIndexAsHistogramBucketIndex(*bucket_index, max_bucket_index);
+ !validate_bucket_index.ok()) {
+ return validate_bucket_index;
+ }
+
+ return Status::OkStatus();
+}
+
} // namespace cobalt
diff --git a/src/lib/privacy/private_index_decoding.h b/src/lib/privacy/private_index_decoding.h
index 9be33bb..926b6b9 100644
--- a/src/lib/privacy/private_index_decoding.h
+++ b/src/lib/privacy/private_index_decoding.h
@@ -12,8 +12,8 @@
namespace cobalt {
-// A value which represents either a sum or a count. Used when decoding observations for
-// FleetwideMeans reports.
+// A value which represents either a sum or a count, as an integer. Used when decoding observations
+// for FleetwideMeans reports.
struct SumOrCount {
enum SumOrCountType { SUM, COUNT };
SumOrCountType type;
@@ -21,6 +21,15 @@
int64_t sum = 0;
uint64_t count = 0;
};
+// A value which represents either a sum or a count, as a double. Used when decoding observations
+// for FleetwideMeans reports.
+struct SumOrCountDouble {
+ enum SumOrCountType { SUM, COUNT };
+ SumOrCountType type;
+
+ double sum = 0.0;
+ double count = 0.0;
+};
// Populates |event_vector| with the event vector which corresponds to |index| according to the
// contents of |metric_dimensions| and returns an OK status, or returns an error status if |index|
@@ -65,6 +74,21 @@
int64_t min_value, int64_t max_value, uint64_t max_count, uint64_t num_index_points,
std::vector<uint32_t>* event_vector, SumOrCount* sum_or_count);
+// Populates |event_vector| and |sum_or_count| with the event vector and SumAndCount which
+// correspond to |index| according to |metric_dimensions|, |min_value|, |max_value|, |max_count| and
+// |num_index_points|, or returns an error status if |index| does not represent a valid
+// (event_vector, sum) or (event_vector, count) pair.
+//
+// After a successful call, |sum_or_count| will be a SumOrCountDouble struct whose |type| field is
+// set to either SUM or COUNT. If the |type| is SUM, then the |sum| field is populated with a double
+// representing a sum; if the |type| is COUNT, then the |count| field is populated with an double
+// representing a count.
+Status DecodePrivateIndexAsSumOrCountDouble(
+ uint64_t index,
+ const google::protobuf::RepeatedPtrField<MetricDefinition::MetricDimension>& metric_dimensions,
+ int64_t min_value, int64_t max_value, uint64_t max_count, uint64_t num_index_points,
+ std::vector<uint32_t>* event_vector, SumOrCountDouble* sum_or_count);
+
// Populates |event_vector| and |bucket_index| with the event vector and histogram bucket index
// which correspond to |index| according to |metric_dimensions| and |max_bucket_index|, or returns
// an error status if |index| does not represent a valid (event_vector, bucket_index) pair.
@@ -83,6 +107,17 @@
uint32_t max_bucket_index, uint64_t max_count, uint64_t num_index_points,
std::vector<uint32_t>* event_vector, uint32_t* bucket_index, uint64_t* bucket_count);
+// Populates |event_vector|, |bucket_index|, and |bucket_count| with the event vector, histogram
+// bucket index, and bucket count which correspond to |index| according to |metric_dimensions|,
+// |max_bucket_index|, |max_count|, and |num_index_points|, or else returns an error status if
+// |index| does not represent a valid (event_vector, bucket_index, bucket_count) tuple.
+// The bucket count is decoded as a double, without rounding.
+Status DecodePrivateIndexAsHistogramBucketIndexAndCountDouble(
+ uint64_t index,
+ const google::protobuf::RepeatedPtrField<MetricDefinition::MetricDimension>& metric_dimensions,
+ uint32_t max_bucket_index, uint64_t max_count, uint64_t num_index_points,
+ std::vector<uint32_t>* event_vector, uint32_t* bucket_index, double* bucket_count);
+
} // namespace cobalt
#endif // COBALT_SRC_LIB_PRIVACY_PRIVATE_INDEX_DECODING_H_
diff --git a/src/lib/privacy/private_index_decoding_test.cc b/src/lib/privacy/private_index_decoding_test.cc
index bd64e7c..c326af4 100644
--- a/src/lib/privacy/private_index_decoding_test.cc
+++ b/src/lib/privacy/private_index_decoding_test.cc
@@ -5,6 +5,8 @@
#include "src/logger/event_vector_index.h"
+using ::testing::DoubleEq;
+
namespace cobalt {
namespace {
@@ -120,7 +122,7 @@
kNumIndexPoints, &event_vector, &double_value);
ASSERT_TRUE(status.ok());
EXPECT_EQ(event_vector, std::vector<uint32_t>({0, 0, 0}));
- EXPECT_EQ(double_value, static_cast<double>(kMinValue));
+ EXPECT_THAT(double_value, DoubleEq(static_cast<double>(kMinValue)));
// Check that an intermediate private index is decoded correctly.
// The private index (MaxEventVectorIndex() + 1) should be the first index whose |double_value| is
@@ -130,16 +132,15 @@
kMaxValue, kNumIndexPoints, &event_vector, &double_value);
ASSERT_TRUE(status.ok());
EXPECT_EQ(event_vector, std::vector<uint32_t>({0, 0, 0}));
- EXPECT_THAT(double_value,
- testing::DoubleEq(kMinValue + static_cast<double>(kMaxValue - kMinValue) /
- (kNumIndexPoints - 1)));
+ EXPECT_THAT(double_value, DoubleEq(kMinValue + static_cast<double>(kMaxValue - kMinValue) /
+ (kNumIndexPoints - 1)));
// Check that the maximum private index is decoded correctly.
status = DecodePrivateIndexAsDouble(MaxIndexForNumericValue(), metric_dimensions_, kMinValue,
kMaxValue, kNumIndexPoints, &event_vector, &double_value);
ASSERT_TRUE(status.ok());
EXPECT_EQ(event_vector, std::vector<uint32_t>({2, 300, 10}));
- EXPECT_EQ(double_value, static_cast<double>(kMaxValue));
+ EXPECT_THAT(double_value, DoubleEq(static_cast<double>(kMaxValue)));
}
TEST_F(PrivateIndexDecodingTest, DecodeAsDoubleInvalid) {
@@ -205,6 +206,60 @@
EXPECT_FALSE(status.ok());
}
+TEST_F(PrivateIndexDecodingTest, DecodeAsSumOrCountDoubleValidSum) {
+ std::vector<uint32_t> event_vector;
+ SumOrCountDouble sum_or_count;
+ // Check that the minimum private index that corresponds to a sum is decoded correctly.
+ Status status =
+ DecodePrivateIndexAsSumOrCountDouble(0, metric_dimensions_, kMinValue, kMaxValue, kMaxCount,
+ kNumIndexPoints, &event_vector, &sum_or_count);
+ ASSERT_TRUE(status.ok());
+ EXPECT_EQ(event_vector, std::vector<uint32_t>({0, 0, 0}));
+ EXPECT_EQ(sum_or_count.type, SumOrCountDouble::SUM);
+ EXPECT_THAT(sum_or_count.sum, DoubleEq(static_cast<double>(kMinValue)));
+
+ // Check that the maximum private index that corresponds to a sum is decoded correctly.
+ status = DecodePrivateIndexAsSumOrCountDouble(MaxIndexForNumericValue(), metric_dimensions_,
+ kMinValue, kMaxValue, kMaxCount, kNumIndexPoints,
+ &event_vector, &sum_or_count);
+ ASSERT_TRUE(status.ok());
+ EXPECT_EQ(event_vector, std::vector<uint32_t>({2, 300, 10}));
+ EXPECT_EQ(sum_or_count.type, SumOrCountDouble::SUM);
+ EXPECT_THAT(sum_or_count.sum, DoubleEq(static_cast<double>(kMaxValue)));
+}
+
+TEST_F(PrivateIndexDecodingTest, DecodeAsSumOrCountDoubleValidCount) {
+ std::vector<uint32_t> event_vector;
+ SumOrCountDouble sum_or_count;
+ // Check that the minimum private index that corresponds to a count is decoded correctly.
+ Status status = DecodePrivateIndexAsSumOrCountDouble(
+ MinIndexForCount(), metric_dimensions_, kMinValue, kMaxValue, kMaxCount, kNumIndexPoints,
+ &event_vector, &sum_or_count);
+ ASSERT_TRUE(status.ok());
+ EXPECT_EQ(event_vector, std::vector<uint32_t>({0, 0, 0}));
+ EXPECT_EQ(sum_or_count.type, SumOrCountDouble::COUNT);
+ EXPECT_THAT(sum_or_count.count, DoubleEq(0.0));
+
+ // Check that the maximum private index that corresponds to a count is decoded correctly.
+ status = DecodePrivateIndexAsSumOrCountDouble(MaxIndexForCount(), metric_dimensions_, kMinValue,
+ kMaxValue, kMaxCount, kNumIndexPoints,
+ &event_vector, &sum_or_count);
+ ASSERT_TRUE(status.ok());
+ EXPECT_EQ(event_vector, std::vector<uint32_t>({2, 300, 10}));
+ EXPECT_EQ(sum_or_count.type, SumOrCountDouble::COUNT);
+ EXPECT_THAT(sum_or_count.count, DoubleEq(static_cast<double>(kMaxCount)));
+}
+
+TEST_F(PrivateIndexDecodingTest, DecodeAsSumOrCountDoubleInvalid) {
+ std::vector<uint32_t> event_vector;
+ SumOrCountDouble sum_or_count;
+
+ Status status = DecodePrivateIndexAsSumOrCountDouble(
+ MaxIndexForCount() + 1, metric_dimensions_, kMinValue, kMaxValue, kMaxCount, kNumIndexPoints,
+ &event_vector, &sum_or_count);
+ EXPECT_FALSE(status.ok());
+}
+
TEST_F(PrivateIndexDecodingTest, DecodeAsHistogramBucketIndexValid) {
std::vector<uint32_t> event_vector;
uint32_t bucket_index;
@@ -268,4 +323,38 @@
ASSERT_FALSE(status.ok());
}
+TEST_F(PrivateIndexDecodingTest, DecodeAsHistogramBucketIndexAndCountDoubleValid) {
+ std::vector<uint32_t> event_vector;
+ uint32_t bucket_index;
+ double bucket_count;
+ // Check that the minimum private index is decoded correctly.
+ Status status = DecodePrivateIndexAsHistogramBucketIndexAndCountDouble(
+ 0, metric_dimensions_, kMaxBucketIndex, kMaxCount, kNumIndexPoints, &event_vector,
+ &bucket_index, &bucket_count);
+ ASSERT_TRUE(status.ok());
+ EXPECT_EQ(event_vector, std::vector<uint32_t>({0, 0, 0}));
+ EXPECT_EQ(bucket_index, 0u);
+ EXPECT_THAT(bucket_count, DoubleEq(0.0));
+
+ // Check that the maximum private index is decoded correctly.
+ status = DecodePrivateIndexAsHistogramBucketIndexAndCountDouble(
+ MaxIndexForHistogramBucketAndCount(), metric_dimensions_, kMaxBucketIndex, kMaxCount,
+ kNumIndexPoints, &event_vector, &bucket_index, &bucket_count);
+ ASSERT_TRUE(status.ok());
+ EXPECT_EQ(event_vector, std::vector<uint32_t>({2, 300, 10}));
+ EXPECT_EQ(bucket_index, kMaxBucketIndex);
+ EXPECT_THAT(bucket_count, DoubleEq(static_cast<double>(kMaxCount)));
+}
+
+TEST_F(PrivateIndexDecodingTest, DecodeAsHistogramBucketIndexAndCountDoubleInvalid) {
+ std::vector<uint32_t> event_vector;
+ uint32_t bucket_index;
+ double bucket_count;
+
+ Status status = DecodePrivateIndexAsHistogramBucketIndexAndCountDouble(
+ MaxIndexForHistogramBucketAndCount() + 1, metric_dimensions_, kMaxBucketIndex, kMaxCount,
+ kNumIndexPoints, &event_vector, &bucket_index, &bucket_count);
+ ASSERT_FALSE(status.ok());
+}
+
} // namespace cobalt
