blob: 49174a9e42351af2346319b6a20e2806b250d248 [file] [log] [blame]
// Copyright 2020 The Fuchsia Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
package privacy
import (
"config"
"testing"
"github.com/golang/glog"
)
// The try-bots expect glog to be imported, but we do not use it.
var _ = glog.Info
func TestEstimateWithEpsilon(t *testing.T) {
privacyParamsCalculator, err := NewPrivacyEncodingParamsCalculatorForTesting(testParamRecords)
if err != nil {
t.Fatal("Failed to create PrivacyEncodingParamsCalculator")
}
errorCalculator := NewErrorCalculator(*privacyParamsCalculator)
testMetric := config.MetricDefinition{
MetricName: "Occurrence",
MetricType: config.MetricDefinition_OCCURRENCE,
}
testMetricWithBufferMax := config.MetricDefinition{
MetricName: "OccurrenceWithBufferMax",
MetricType: config.MetricDefinition_OCCURRENCE,
EventCodeBufferMax: 5,
}
uniqueDeviceCount := config.ReportDefinition{
ReportName: "UniqueDeviceCounts",
ReportType: config.ReportDefinition_UNIQUE_DEVICE_COUNTS,
LocalAggregationProcedure: config.ReportDefinition_SELECT_FIRST,
}
uniqueDeviceHistogram := config.ReportDefinition{
ReportName: "UniqueDeviceHistograms",
ReportType: config.ReportDefinition_UNIQUE_DEVICE_HISTOGRAMS,
}
hourlyValueHistogram := config.ReportDefinition{
ReportName: "HourlyValueHistograms",
ReportType: config.ReportDefinition_HOURLY_VALUE_HISTOGRAMS,
}
fleetwideOccurrenceCount := config.ReportDefinition{
ReportName: "FleetwideOccurrenceCounts",
ReportType: config.ReportDefinition_FLEETWIDE_OCCURRENCE_COUNTS,
MaxValue: 5,
MinValue: 1,
}
fleetwideOccurrenceCountHighMax := config.ReportDefinition{
ReportName: "FleetwideOccurrenceCountsWithHighMaxValue",
ReportType: config.ReportDefinition_FLEETWIDE_OCCURRENCE_COUNTS,
MaxValue: 20,
MinValue: 1,
}
fleetwideOccurrenceCountNegativeValue := config.ReportDefinition{
ReportName: "FleetwideOccurrenceCountsNegativeValue",
ReportType: config.ReportDefinition_FLEETWIDE_OCCURRENCE_COUNTS,
MaxValue: -1,
MinValue: -5,
}
fleetwideHistogram := config.ReportDefinition{
ReportName: "FleetwideHistograms",
ReportType: config.ReportDefinition_FLEETWIDE_HISTOGRAMS,
MaxCount: 5,
}
fleetwideHistogramHighMax := config.ReportDefinition{
ReportName: "FleetwideHistogramsWithHighMaxCount",
ReportType: config.ReportDefinition_FLEETWIDE_HISTOGRAMS,
MaxCount: 20,
}
uniqueDeviceNumericStats := config.ReportDefinition{
ReportName: "UniqueDeviceNumericStats",
ReportType: config.ReportDefinition_UNIQUE_DEVICE_NUMERIC_STATS,
MinValue: 0,
MaxValue: 1,
}
hourlyValueNumericStats := config.ReportDefinition{
ReportName: "HourlyValueNumericStats",
ReportType: config.ReportDefinition_HOURLY_VALUE_NUMERIC_STATS,
MinValue: 0,
MaxValue: 1,
}
uniqueDeviceNumericStatsMissingMaxValue := config.ReportDefinition{
ReportName: "UniqueDeviceNumericStatsMissingMaxValue",
ReportType: config.ReportDefinition_UNIQUE_DEVICE_NUMERIC_STATS,
}
fleetwideMeans := config.ReportDefinition{
ReportName: "FleetwideMeans",
ReportType: config.ReportDefinition_FLEETWIDE_MEANS,
MinValue: 0,
MaxValue: 5,
MaxCount: 5,
}
fleetwideMeansHighMaxCount := config.ReportDefinition{
ReportName: "FleetwideMeansHighMaxCount",
ReportType: config.ReportDefinition_FLEETWIDE_MEANS,
MinValue: 0,
MaxValue: 5,
MaxCount: 20,
}
fleetwideMeansMissingMaxValue := config.ReportDefinition{
ReportName: "FleetwideMeansMissingMaxValue",
ReportType: config.ReportDefinition_FLEETWIDE_MEANS,
}
fleetwideHistogramMissingMaxCount := config.ReportDefinition{
ReportName: "FleetwideHistogramMissingMaxCount",
ReportType: config.ReportDefinition_FLEETWIDE_HISTOGRAMS,
}
fleetwideOccurrenceCountMissingMaxValue := config.ReportDefinition{
ReportName: "FleetwideOccurrenceCounts",
ReportType: config.ReportDefinition_FLEETWIDE_OCCURRENCE_COUNTS,
}
unsupportedReport := config.ReportDefinition{
ReportName: "UnsupportedReport",
}
type args struct {
metric *config.MetricDefinition
report *config.ReportDefinition
epsilon float64
population uint64
minDenominatorEstimate uint64
}
var tests = []struct {
input args
valid bool
expected float64
}{
// Single contribution reports
{args{&testMetric, &uniqueDeviceCount, 1, 10000, 0}, true, 4.935511431266572},
{args{&testMetric, &uniqueDeviceCount, 10, 10000, 0}, true, 3.014414887122711},
{args{&testMetric, &uniqueDeviceCount, 1, 20000, 0}, true, 5.1971495884363295},
{args{&testMetric, &uniqueDeviceHistogram, 1, 10000, 0}, true, 4.935511431266572},
{args{&testMetric, &uniqueDeviceHistogram, 10, 10000, 0}, true, 3.014414887122711},
{args{&testMetric, &uniqueDeviceHistogram, 1, 20000, 0}, true, 5.1971495884363295},
{args{&testMetric, &hourlyValueHistogram, 1, 10000, 0}, true, 24.178969252553177},
{args{&testMetric, &hourlyValueHistogram, 10, 10000, 0}, true, 14.767556692999985},
{args{&testMetric, &hourlyValueHistogram, 1, 20000, 0}, true, 25.460729217169213},
// Multi-contribution reports
{args{&testMetric, &fleetwideOccurrenceCount, 1, 10000, 0}, true, 143.45282815225397},
{args{&testMetric, &fleetwideOccurrenceCount, 10, 10000, 0}, true, 87.61530528378074},
{args{&testMetric, &fleetwideOccurrenceCount, 1, 20000, 0}, true, 151.0574572005781},
{args{&testMetric, &fleetwideOccurrenceCountHighMax, 1, 10000, 0}, true, 1004.9807533333975},
{args{&testMetric, &fleetwideOccurrenceCountNegativeValue, 1, 10000, 0}, true, 143.45282815225397},
{args{&testMetric, &fleetwideHistogram, 1, 10000, 0}, true, 192.21055516554554},
{args{&testMetric, &fleetwideHistogram, 10, 10000, 0}, true, 117.39459365499897},
{args{&testMetric, &fleetwideHistogram, 1, 20000, 0}, true, 202.39989747432912},
{args{&testMetric, &fleetwideHistogramHighMax, 1, 10000, 0}, true, 1057.87447719305},
// Mean reports
{args{&testMetric, &uniqueDeviceNumericStats, 1, 10000, 500}, true, 0.0036523819410084948},
{args{&testMetric, &uniqueDeviceNumericStats, 10, 10000, 500}, true, 0.0013409958467188899},
{args{&testMetric, &uniqueDeviceNumericStats, 1, 20000, 500}, true, 0.004058060014483685},
{args{&testMetric, &uniqueDeviceNumericStats, 1, 10000, 2000}, true, 0.00022148418420625318},
{args{&testMetric, &hourlyValueNumericStats, 1, 10000, 2000}, true, 0.00552775088765882},
{args{&testMetric, &hourlyValueNumericStats, 10, 10000, 2000}, true, 0.002022364644709206},
{args{&testMetric, &hourlyValueNumericStats, 1, 20000, 2000}, true, 0.0061455080639499894},
{args{&testMetric, &hourlyValueNumericStats, 1, 10000, 5000}, true, 0.0008584921943317022},
{args{&testMetric, &fleetwideMeans, 1, 10000, 500}, true, 467.3427023401392},
{args{&testMetric, &fleetwideMeans, 10, 10000, 500}, true, 28.078743913512465},
{args{&testMetric, &fleetwideMeans, 1, 20000, 500}, true, 551.1417346736127},
{args{&testMetric, &fleetwideMeans, 1, 10000, 2000}, true, 0.9722845785198999},
{args{&testMetric, &fleetwideMeansHighMaxCount, 1, 10000, 500}, true, 2370.2707572142904},
{args{&testMetric, &fleetwideMeansHighMaxCount, 10, 10000, 500}, true, 2238.0931858912145},
{args{&testMetric, &fleetwideMeansHighMaxCount, 1, 20000, 500}, true, 2404.5027616065695},
{args{&testMetric, &fleetwideMeansHighMaxCount, 1, 10000, 2000}, true, 4098.32133059633},
{args{&testMetric, &uniqueDeviceCount, 0, 100000, 0}, true, 0},
// Metric with EventBufferMax set.
{args{&testMetricWithBufferMax, &uniqueDeviceCount, 1, 10000, 0}, true, 4.935511431266572},
{args{&testMetricWithBufferMax, &fleetwideOccurrenceCount, 1, 10000, 0}, true, 386.4795472694349},
{args{&testMetricWithBufferMax, &uniqueDeviceNumericStats, 1, 10000, 500}, true, 0.03372677729859206},
{args{&testMetricWithBufferMax, &fleetwideMeans, 1, 10000, 500}, true, 2117.7764297912554},
// Invalid input
{args{&testMetric, &uniqueDeviceNumericStats, 1, 10000, 0}, false, 0},
{args{&testMetric, &hourlyValueNumericStats, 1, 10000, 0}, false, 0},
{args{&testMetric, &uniqueDeviceNumericStatsMissingMaxValue, 1, 10000, 500}, false, 0},
{args{&testMetric, &fleetwideMeansMissingMaxValue, 1, 10000, 500}, false, 0},
{args{&testMetric, &fleetwideHistogramMissingMaxCount, 1, 10000, 500}, false, 0},
{args{&testMetric, &fleetwideOccurrenceCountMissingMaxValue, 1, 10000, 500}, false, 0},
// This report type is not currently supported.
{args{&testMetric, &unsupportedReport, 1, 10000, 0}, false, 0},
}
for _, test := range tests {
input := test.input
result, err := errorCalculator.Estimate(input.metric, input.report, input.epsilon, input.population, input.minDenominatorEstimate)
if test.valid && err != nil {
t.Errorf("Estimate failed for report %v: %v", input.report.ReportName, err)
} else if !test.valid && err == nil {
t.Errorf("Estimate accepted invalid report: %v", input.report.ReportName)
} else if test.valid && result != test.expected {
t.Errorf("Estimate for report %v (epsilon: %v, population: %v, minDenominator: %v): expected %v, got %v", input.report.ReportName, input.epsilon, input.population, input.minDenominatorEstimate, test.expected, result)
}
}
}