src/bin/config_parser/src/privacy/error_calculator_test.go

// Copyright 2020 The Fuchsia Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

package privacy

import (
	"config"
	"testing"

	"github.com/golang/glog"
)

// The try-bots expect glog to be imported, but we do not use it.
var _ = glog.Info

var metric = config.MetricDefinition{
	EventCodeBufferMax: 5,
}

func TestEstimateWithEpsilon(t *testing.T) {
	privacyParamsCalculator, err := NewPrivacyEncodingParamsCalculatorForTesting(testParamRecords)
	if err != nil {
		t.Fatal("Failed to create PrivacyEncodingParamsCalculator")
	}
	errorCalculator := NewErrorCalculator(*privacyParamsCalculator)

	testMetric := config.MetricDefinition{
		MetricName: "Occurrence",
		MetricType: config.MetricDefinition_OCCURRENCE,
	}
	uniqueDeviceCount := config.ReportDefinition{
		ReportName:                "UniqueDeviceCounts",
		ReportType:                config.ReportDefinition_UNIQUE_DEVICE_COUNTS,
		LocalAggregationProcedure: config.ReportDefinition_SELECT_FIRST,
	}
	uniqueDeviceHistogram := config.ReportDefinition{
		ReportName:                "UniqueDeviceHistograms",
		ReportType:                config.ReportDefinition_UNIQUE_DEVICE_HISTOGRAMS,
		LocalAggregationProcedure: config.ReportDefinition_SELECT_FIRST,
	}
	hourlyValueHistogram := config.ReportDefinition{
		ReportName:                "HourlyValueHistograms",
		ReportType:                config.ReportDefinition_HOURLY_VALUE_HISTOGRAMS,
		LocalAggregationProcedure: config.ReportDefinition_SELECT_FIRST,
	}
	fleetwideOccurrenceCount := config.ReportDefinition{
		ReportName:                "FleetwideOccurrenceCounts",
		ReportType:                config.ReportDefinition_FLEETWIDE_OCCURRENCE_COUNTS,
		LocalAggregationProcedure: config.ReportDefinition_SELECT_FIRST,
		MaxCount:                  1,
	}
	fleetwideOccurrenceCountHighMax := config.ReportDefinition{
		ReportName:                "FleetwideOccurrenceCountsWithHighMaxCount",
		ReportType:                config.ReportDefinition_FLEETWIDE_OCCURRENCE_COUNTS,
		LocalAggregationProcedure: config.ReportDefinition_SELECT_FIRST,
		MaxCount:                  4,
	}
	fleetwideHistogram := config.ReportDefinition{
		ReportName:                "FleetwideHistograms",
		ReportType:                config.ReportDefinition_FLEETWIDE_HISTOGRAMS,
		LocalAggregationProcedure: config.ReportDefinition_SELECT_FIRST,
	}
	uniqueDeviceNumericStats := config.ReportDefinition{
		ReportName: "UniqueDeviceNumericStats",
		ReportType: config.ReportDefinition_UNIQUE_DEVICE_NUMERIC_STATS,
		MaxValue:   1,
	}
	hourlyValueNumericStats := config.ReportDefinition{
		ReportName: "HourlyDeviceNumericStats",
		ReportType: config.ReportDefinition_HOURLY_VALUE_NUMERIC_STATS,
		MaxValue:   1,
	}
	uniqueDeviceNumericStatsMissingMaxValue := config.ReportDefinition{
		ReportName: "UniqueDeviceNumericStatsMissingMaxValue",
		ReportType: config.ReportDefinition_UNIQUE_DEVICE_NUMERIC_STATS,
	}
	fleetwideMeans := config.ReportDefinition{
		ReportName: "FleetwideMeans",
		ReportType: config.ReportDefinition_FLEETWIDE_MEANS,
		MaxValue:   1,
		MaxCount:   1,
	}
	fleetwideMeansHighMaxCount := config.ReportDefinition{
		ReportName: "FleetwideMeansHighMaxCount",
		ReportType: config.ReportDefinition_FLEETWIDE_MEANS,
		MaxValue:   1,
		MaxCount:   4,
	}
	fleetwideMeansMissingMaxValue := config.ReportDefinition{
		ReportName: "FleetwideMeansMissingMaxValue",
		ReportType: config.ReportDefinition_FLEETWIDE_MEANS,
	}
	unsupportedReport := config.ReportDefinition{
		ReportName: "UnsupportedReport",
	}

	type args struct {
		metric                 *config.MetricDefinition
		report                 *config.ReportDefinition
		epsilon                float64
		population             uint64
		minDenominatorEstimate uint64
	}
	var tests = []struct {
		input    args
		valid    bool
		expected float64
	}{
		// Single contribution reports
		{args{&testMetric, &uniqueDeviceCount, 1, 10000, 0}, true, 4.935511431266572},
		{args{&testMetric, &uniqueDeviceCount, 10, 10000, 0}, true, 3.014414887122711},
		{args{&testMetric, &uniqueDeviceCount, 1, 20000, 0}, true, 6.979867203344631},
		{args{&testMetric, &uniqueDeviceHistogram, 1, 10000, 0}, true, 4.935511431266572},
		{args{&testMetric, &uniqueDeviceHistogram, 10, 10000, 0}, true, 3.014414887122711},
		{args{&testMetric, &uniqueDeviceHistogram, 1, 20000, 0}, true, 6.979867203344631},
		{args{&testMetric, &hourlyValueHistogram, 1, 10000, 0}, true, 24.178969252553177},
		{args{&testMetric, &hourlyValueHistogram, 10, 10000, 0}, true, 14.767556692999985},
		{args{&testMetric, &hourlyValueHistogram, 1, 20000, 0}, true, 34.19422624116276},

		// Multi-contribution reports
		{args{&testMetric, &fleetwideOccurrenceCount, 1, 10000, 0}, true, 52.893723859652496},
		{args{&testMetric, &fleetwideOccurrenceCount, 10, 10000, 0}, true, 37.430720721998675},
		{args{&testMetric, &fleetwideOccurrenceCount, 1, 20000, 0}, true, 74.80302164673793},
		{args{&testMetric, &fleetwideOccurrenceCountHighMax, 1, 10000, 0}, true, 211.57489543860999},
		{args{&testMetric, &fleetwideHistogram, 1, 10000, 0}, true, 52.893723859652496},
		{args{&testMetric, &fleetwideHistogram, 10, 10000, 0}, true, 37.430720721998675},
		{args{&testMetric, &fleetwideHistogram, 1, 20000, 0}, true, 74.80302164673793},

		// Mean reports
		{args{&testMetric, &uniqueDeviceNumericStats, 1, 10000, 500}, true, 0.027891803930152177},
		{args{&testMetric, &uniqueDeviceNumericStats, 10, 10000, 500}, true, 0.018650077285590205},
		{args{&testMetric, &uniqueDeviceNumericStats, 1, 20000, 500}, true, 0.039654432133364455},
		{args{&testMetric, &uniqueDeviceNumericStats, 1, 10000, 2000}, true, 0.006907368191507632},
		{args{&testMetric, &hourlyValueNumericStats, 1, 10000, 500}, true, 0.14403990551821316},
		{args{&testMetric, &hourlyValueNumericStats, 10, 10000, 500}, true, 0.09424308127635954},
		{args{&testMetric, &hourlyValueNumericStats, 1, 20000, 500}, true, 0.20996453488325503},
		{args{&testMetric, &hourlyValueNumericStats, 1, 10000, 2000}, true, 0.03425744055252169},
		{args{&testMetric, &fleetwideMeans, 1, 10000, 500}, true, 0.14403990551821316},
		{args{&testMetric, &fleetwideMeans, 10, 10000, 500}, true, 0.09424308127635954},
		{args{&testMetric, &fleetwideMeans, 1, 20000, 500}, true, 0.20996453488325503},
		{args{&testMetric, &fleetwideMeans, 1, 10000, 2000}, true, 0.03425744055252169},
		{args{&testMetric, &fleetwideMeansHighMaxCount, 1, 10000, 500}, true, 14.67564563395404},
		{args{&testMetric, &fleetwideMeansHighMaxCount, 10, 10000, 500}, true, 0.3612570115350953},
		{args{&testMetric, &fleetwideMeansHighMaxCount, 1, 20000, 500}, true, 86.18119817858194},
		{args{&testMetric, &fleetwideMeansHighMaxCount, 1, 10000, 2000}, true, 0.11189363618734707},

		// Invalid input
		// Mean Report missing minDenominatorEstimate
		{args{&testMetric, &uniqueDeviceNumericStats, 1, 10000, 0}, false, 0},
		{args{&testMetric, &hourlyValueNumericStats, 1, 10000, 0}, false, 0},
		{args{&testMetric, &uniqueDeviceNumericStatsMissingMaxValue, 1, 10000, 500}, false, 0},
		{args{&testMetric, &fleetwideMeansMissingMaxValue, 1, 10000, 500}, false, 0},

		// This report type is not currently supported.
		{args{&testMetric, &unsupportedReport, 1, 10000, 0}, false, 0},
	}

	for _, test := range tests {
		input := test.input
		result, err := errorCalculator.Estimate(input.metric, input.report, input.epsilon, input.population, input.minDenominatorEstimate)
		if test.valid && err != nil {
			t.Errorf("Estimate failed for report %v: %v", input.report.ReportName, err)
		} else if !test.valid && err == nil {
			t.Errorf("Estimate accepted invalid report: %v", input.report.ReportName)
		} else if test.valid && result != test.expected {
			t.Errorf("Estimate for report %v (epsilon: %v, population: %v, minDenominator: %v): expected %v, got %v", input.report.ReportName, input.epsilon, input.population, input.minDenominatorEstimate, test.expected, result)
		}
	}
}