| # |
| # Copyright 2020 Google LLC |
| # |
| # Licensed under the Apache License, Version 2.0 (the "License"); |
| # you may not use this file except in compliance with the License. |
| # You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| # See the License for the specific language governing permissions and |
| # limitations under the License. |
| |
| # This collection of test cases is intended to statistically evaluate the shape |
| # of the distribution of the Gaussian noise generated by the DP library. |
| |
| # The test cases and validation parameters are calibrated such that: |
| # - A distribution closeness test accepts with a probability of at least 0.9 if |
| # two identical Gaussian distributions are compared. |
| # - A distribution closeness test rejects with a probability of at least 0.9 if |
| # two Gaussian distributions are compared whose means are shifted by the |
| # respective standard deviation times 0.0375 or whose variances differs by |
| # 1.0625 in scale or if the shape of one of the distributions is Laplace |
| # instead of Gaussian. |
| validity_parameters { |
| shift_specificity: 0.0375 |
| scale_specificity: 1.0625 |
| } |
| |
| # Taking the majority vote over 9 repeated runs of a particular test case |
| # increases the accept and reject probabilities to 0.99911 or more. |
| voting_parameters { |
| number_of_votes: 9 |
| } |
| |
| distribution_closeness_test_case { |
| name: "Default parameters" |
| closeness_test_parameters { |
| mean: 0.0 |
| variance: 11.73597717285156 |
| l2_tolerance: 0.00065 |
| granularity: 0.0078125 |
| } |
| noise_sampling_parameters { |
| number_of_samples: 1000000 |
| l0_sensitivity: 1 |
| linf_sensitivity: 1.0 |
| epsilon: 1.09861228866810969140 # = log(3) |
| delta: 0.00001 |
| raw_input: 0.0 |
| } |
| } |
| |
| distribution_closeness_test_case { |
| name: "Low l_inf sensitivity" |
| closeness_test_parameters { |
| mean: 0.0 |
| variance: 0.029339942932129 |
| l2_tolerance: 0.00075 |
| granularity: 0.0009765625 |
| } |
| noise_sampling_parameters { |
| number_of_samples: 1000000 |
| l0_sensitivity: 1 |
| linf_sensitivity: 0.05 |
| epsilon: 1.09861228866810969140 # = log(3) |
| delta: 0.00001 |
| raw_input: 0.0 |
| } |
| } |
| |
| distribution_closeness_test_case { |
| name: "Very low l_inf sensitivity" |
| closeness_test_parameters { |
| mean: 0.0 |
| variance: 0.000002933994293 |
| l2_tolerance: 0.0016 |
| granularity: 0.00006103515625 |
| } |
| noise_sampling_parameters { |
| number_of_samples: 1000000 |
| l0_sensitivity: 1 |
| linf_sensitivity: 0.0005 |
| epsilon: 1.09861228866810969140 # = log(3) |
| delta: 0.00001 |
| raw_input: 0.0 |
| } |
| } |
| |
| distribution_closeness_test_case { |
| name: "High l_inf sensitivity" |
| closeness_test_parameters { |
| mean: 0.0 |
| variance: 46.94390869140625 |
| l2_tolerance: 0.00065 |
| granularity: 0.015625 |
| } |
| noise_sampling_parameters { |
| number_of_samples: 1000000 |
| l0_sensitivity: 1 |
| linf_sensitivity: 2.0 |
| epsilon: 1.09861228866810969140 # = log(3) |
| delta: 0.00001 |
| raw_input: 0.0 |
| } |
| } |
| |
| distribution_closeness_test_case { |
| name: "Very high l_inf sensitivity" |
| closeness_test_parameters { |
| mean: 0.0 |
| variance: 733498.5733032227 |
| l2_tolerance: 0.00055 |
| granularity: 2.0 |
| } |
| noise_sampling_parameters { |
| number_of_samples: 1000000 |
| l0_sensitivity: 1 |
| linf_sensitivity: 250.0 |
| epsilon: 1.09861228866810969140 # = log(3) |
| delta: 0.00001 |
| raw_input: 0.0 |
| } |
| } |
| |
| distribution_closeness_test_case { |
| name: "High l_0 sensitivity" |
| closeness_test_parameters { |
| mean: 0.0 |
| variance: 23.47195434570314 |
| l2_tolerance: 0.00065 |
| granularity: 0.015625 |
| } |
| noise_sampling_parameters { |
| number_of_samples: 1000000 |
| l0_sensitivity: 2 |
| linf_sensitivity: 1.0 |
| epsilon: 1.09861228866810969140 # = log(3) |
| delta: 0.00001 |
| raw_input: 0.0 |
| } |
| } |
| |
| distribution_closeness_test_case { |
| name: "Very high l_0 sensitivity" |
| closeness_test_parameters { |
| mean: 0.0 |
| variance: 11735.97717285156 |
| l2_tolerance: 0.00055 |
| granularity: 0.25 |
| } |
| noise_sampling_parameters { |
| number_of_samples: 1000000 |
| l0_sensitivity: 1000 |
| linf_sensitivity: 1.0 |
| epsilon: 1.09861228866810969140 # = log(3) |
| delta: 0.00001 |
| raw_input: 0.0 |
| } |
| } |
| |
| distribution_closeness_test_case { |
| name: "Small epsilon" |
| closeness_test_parameters { |
| mean: 0.0 |
| variance: 59475.015625 |
| l2_tolerance: 0.0007 |
| granularity: 1.0 |
| } |
| noise_sampling_parameters { |
| number_of_samples: 1000000 |
| l0_sensitivity: 1 |
| linf_sensitivity: 1.0 |
| epsilon: 0.01 |
| delta: 0.00001 |
| raw_input: 0.0 |
| } |
| } |
| |
| distribution_closeness_test_case { |
| name: "Very small epsilon" |
| closeness_test_parameters { |
| mean: 0.0 |
| variance: 1447650304.0 |
| l2_tolerance: 0.005 |
| granularity: 16384.0 |
| } |
| noise_sampling_parameters { |
| number_of_samples: 1000000 |
| l0_sensitivity: 1 |
| linf_sensitivity: 1.0 |
| epsilon: 0.000001 |
| delta: 0.00001 |
| raw_input: 0.0 |
| } |
| } |
| |
| distribution_closeness_test_case { |
| name: "Large epsilon" |
| closeness_test_parameters { |
| mean: 0.0 |
| variance: 3.363498687744140 |
| l2_tolerance: 0.00065 |
| granularity: 0.00390625 |
| } |
| noise_sampling_parameters { |
| number_of_samples: 1000000 |
| l0_sensitivity: 1 |
| linf_sensitivity: 1.0 |
| epsilon: 2.19722457733621938279 # = 2log(3) |
| delta: 0.00001 |
| raw_input: 0.0 |
| } |
| } |
| |
| distribution_closeness_test_case { |
| name: "Very large epsilon" |
| closeness_test_parameters { |
| mean: 0.0 |
| variance: 0.795824289321899 |
| l2_tolerance: 0.00065 |
| granularity: 0.001953125 |
| } |
| noise_sampling_parameters { |
| number_of_samples: 1000000 |
| l0_sensitivity: 1 |
| linf_sensitivity: 1.0 |
| epsilon: 5.0 |
| delta: 0.00001 |
| raw_input: 0.0 |
| } |
| } |
| |
| distribution_closeness_test_case { |
| name: "Small delta" |
| closeness_test_parameters { |
| mean: 0.0 |
| variance: 15.00049209594727 |
| l2_tolerance: 0.00065 |
| granularity: 0.015625 |
| } |
| noise_sampling_parameters { |
| number_of_samples: 1000000 |
| l0_sensitivity: 1 |
| linf_sensitivity: 1.0 |
| epsilon: 1.09861228866810969140 # = log(3) |
| delta: 0.000001 |
| raw_input: 0.0 |
| } |
| } |
| |
| distribution_closeness_test_case { |
| name: "Very small delta" |
| closeness_test_parameters { |
| mean: 0.0 |
| variance: 28.76478576660156 |
| l2_tolerance: 0.00075 |
| granularity: 0.015625 |
| } |
| noise_sampling_parameters { |
| number_of_samples: 1000000 |
| l0_sensitivity: 1 |
| linf_sensitivity: 1.0 |
| epsilon: 1.09861228866810969140 # = log(3) |
| delta: 0.0000000001 |
| raw_input: 0.0 |
| } |
| } |
| |
| distribution_closeness_test_case { |
| name: "Large delta" |
| closeness_test_parameters { |
| mean: 0.0 |
| variance: 8.605972290039062 |
| l2_tolerance: 0.00055 |
| granularity: 0.0078125 |
| } |
| noise_sampling_parameters { |
| number_of_samples: 1000000 |
| l0_sensitivity: 1 |
| linf_sensitivity: 1.0 |
| epsilon: 1.09861228866810969140 # = log(3) |
| delta: 0.0001 |
| raw_input: 0.0 |
| } |
| } |
| |
| distribution_closeness_test_case { |
| name: "Very large delta" |
| closeness_test_parameters { |
| mean: 0.0 |
| variance: 5.668491363525391 |
| l2_tolerance: 0.00065 |
| granularity: 0.0078125 |
| } |
| noise_sampling_parameters { |
| number_of_samples: 1000000 |
| l0_sensitivity: 1 |
| linf_sensitivity: 1.0 |
| epsilon: 1.09861228866810969140 # = log(3) |
| delta: 0.001 |
| raw_input: 0.0 |
| } |
| } |
| |
| distribution_closeness_test_case { |
| name: "Small positive raw input " |
| closeness_test_parameters { |
| mean: 1.0 |
| variance: 11.73597717285156 |
| l2_tolerance: 0.00065 |
| granularity: 0.0078125 |
| } |
| noise_sampling_parameters { |
| number_of_samples: 1000000 |
| l0_sensitivity: 1 |
| linf_sensitivity: 1.0 |
| epsilon: 1.09861228866810969140 # = log(3) |
| delta: 0.00001 |
| raw_input: 1.0 |
| } |
| } |
| |
| distribution_closeness_test_case { |
| name: "Large positive raw input " |
| closeness_test_parameters { |
| mean: 1000000.0 |
| variance: 11.73597717285156 |
| l2_tolerance: 0.00065 |
| granularity: 0.0078125 |
| } |
| noise_sampling_parameters { |
| number_of_samples: 1000000 |
| l0_sensitivity: 1 |
| linf_sensitivity: 1.0 |
| epsilon: 1.09861228866810969140 # = log(3) |
| delta: 0.00001 |
| raw_input: 1000000.0 |
| } |
| } |
| |
| distribution_closeness_test_case { |
| name: "Very large positive raw input " |
| closeness_test_parameters { |
| mean: 1000000000000.0 |
| variance: 11.73597717285156 |
| l2_tolerance: 0.00065 |
| granularity: 0.0078125 |
| } |
| noise_sampling_parameters { |
| number_of_samples: 1000000 |
| l0_sensitivity: 1 |
| linf_sensitivity: 1.0 |
| epsilon: 1.09861228866810969140 # = log(3) |
| delta: 0.00001 |
| raw_input: 1000000000000.0 |
| } |
| } |
| |
| distribution_closeness_test_case { |
| name: "Small negative raw input " |
| closeness_test_parameters { |
| mean: -1.0 |
| variance: 11.73597717285156 |
| l2_tolerance: 0.00065 |
| granularity: 0.0078125 |
| } |
| noise_sampling_parameters { |
| number_of_samples: 1000000 |
| l0_sensitivity: 1 |
| linf_sensitivity: 1.0 |
| epsilon: 1.09861228866810969140 # = log(3) |
| delta: 0.00001 |
| raw_input: -1.0 |
| } |
| } |
| |
| distribution_closeness_test_case { |
| name: "Large negative raw input " |
| closeness_test_parameters { |
| mean: -1000000.0 |
| variance: 11.73597717285156 |
| l2_tolerance: 0.00065 |
| granularity: 0.0078125 |
| } |
| noise_sampling_parameters { |
| number_of_samples: 1000000 |
| l0_sensitivity: 1 |
| linf_sensitivity: 1.0 |
| epsilon: 1.09861228866810969140 # = log(3) |
| delta: 0.00001 |
| raw_input: -1000000.0 |
| } |
| } |
| |
| distribution_closeness_test_case { |
| name: "Very large negative raw input " |
| closeness_test_parameters { |
| mean: -1000000000000.0 |
| variance: 11.73597717285156 |
| l2_tolerance: 0.00065 |
| granularity: 0.0078125 |
| } |
| noise_sampling_parameters { |
| number_of_samples: 1000000 |
| l0_sensitivity: 1 |
| linf_sensitivity: 1.0 |
| epsilon: 1.09861228866810969140 # = log(3) |
| delta: 0.00001 |
| raw_input: -1000000000000.0 |
| } |
| } |
| |
| distribution_closeness_test_case { |
| name: "Fractional positive raw input " |
| closeness_test_parameters { |
| mean: 6.48074069840786023096 |
| variance: 11.73597717285156 |
| l2_tolerance: 0.00065 |
| granularity: 0.0078125 |
| } |
| noise_sampling_parameters { |
| number_of_samples: 1000000 |
| l0_sensitivity: 1 |
| linf_sensitivity: 1.0 |
| epsilon: 1.09861228866810969140 # = log(3) |
| delta: 0.00001 |
| raw_input: 6.48074069840786023096 |
| } |
| } |
| |
| distribution_closeness_test_case { |
| name: "Fractional negative raw input " |
| closeness_test_parameters { |
| mean: -125773.181179024284936 |
| variance: 11.73597717285156 |
| l2_tolerance: 0.00065 |
| granularity: 0.0078125 |
| } |
| noise_sampling_parameters { |
| number_of_samples: 1000000 |
| l0_sensitivity: 1 |
| linf_sensitivity: 1.0 |
| epsilon: 1.09861228866810969140 # = log(3) |
| delta: 0.00001 |
| raw_input: -125773.181179024284936 |
| } |
| } |
