cmd/autogardener/blamelist_test.go

// Copyright 2022 The Fuchsia Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

package main

import (
	"testing"

	"github.com/google/go-cmp/cmp"
)

func TestCalculateBlamelistDistances(t *testing.T) {
	const pass = "PASS"
	const failure = "FAIL"

	// Mapping from builder name to the status of the test in question at each
	// commit position.
	allBuilds := map[string]map[int]string{
		"bar": {
			499: pass,
			502: pass,
			503: pass,
			504: failure,
			506: failure,
			511: failure,
		},
		"foo": {
			500: pass,
			505: failure,
			510: pass,
			515: failure,
		},
		"allpass": {
			499: pass,
			502: pass,
			511: pass,
		},
	}
	// Mapping from commit (by position) to expected CI builder distances for
	// that commit.
	expected := map[int]map[string]int{
		502: {
			"bar": 2,
			"foo": 0,
		},
		503: {
			"bar": 1,
			"foo": 0,
		},
		504: {
			"bar": 0,
			"foo": 0,
		},
		505: {
			"bar": -1,
			"foo": 0,
		},
	}

	var results []nearbyTestResult
	for builder, builds := range allBuilds {
		for position, status := range builds {
			results = append(results, nearbyTestResult{
				Builder:        builder,
				CommitPosition: position,
				Failed:         status == failure,
			})
		}
	}

	var suspects []suspectCommit
	for pos := range expected {
		suspects = append(suspects, suspectCommit{
			CommitPosition:     pos,
			BlamelistDistances: make(map[string]int),
		})
	}

	calculateBlamelistDistances(results, suspects)

	got := make(map[int]map[string]int)
	for _, suspect := range suspects {
		got[suspect.CommitPosition] = suspect.BlamelistDistances
	}

	if diff := cmp.Diff(expected, got); diff != "" {
		t.Errorf("Diff:\n%s", diff)
	}
}

func TestScoreBlamelistDistances(t *testing.T) {
	tests := []struct {
		name      string
		distances []int
		// min/max are the inclusive range of allowed scores for the given CI
		// blamelist distances. We don't require equality because the exact
		// numbers don't matter that much; we just care that the scores follow a
		// certain descending pattern as changes get further and further away
		// from the first CI failure.
		min int
		max int
	}{
		{
			// If a change is in the first-failure blamelist of many builders,
			// it should have a very high score.
			name:      "in the blamelist for many builders",
			distances: []int{0, 0, 0, 0, 0},
			min:       100,
			max:       100,
		},
		{
			// Fewer blamelists should lower the score slightly since there are
			// fewer data points.
			name:      "in the blamelist for two builders",
			distances: []int{0, 0},
			min:       90,
			max:       99,
		},
		{
			// If there's only one data point then the score should be a bit
			// lower to reflect lack of confidence.
			name:      "in the blamelist for one builder",
			distances: []int{0},
			min:       80,
			max:       90,
		},
		{
			// The culprit may have landed *before* the first failure for a few
			// builders if the failure mode is flaky. The confidence level
			// should be lower but still reasonably high.
			name:      "in earlier blamelists",
			distances: []int{0, 1, 2},
			min:       60,
			max:       70,
		},
		{
			name:      "pretty far away from first failures",
			distances: []int{6, 10, 5},
			min:       15,
			max:       25,
		},
		{
			// A negative distance indicates that the change landed *after* the
			// first failed build, which makes the change very unlikely to be
			// the culprit unless it was a pre-existing failure mode that was
			// made worse by the change.
			name:      "change landed after first failure",
			distances: []int{0, -1},
			min:       0,
			max:       30,
		},
		{
			name:      "no blamelist data",
			distances: nil,
			min:       0,
			max:       0,
		},
	}

	for _, test := range tests {
		t.Run(test.name, func(t *testing.T) {
			score := scoreBlamelistDistances(test.distances)
			if score < test.min || score > test.max {
				t.Errorf("scoreBlamelistDistances(%d) = %d is outside range [%d, %d]",
					test.distances, score, test.min, test.max,
				)
			} else {
				t.Logf("scoreBlamelistDistances(%d) = %d", test.distances, score)
			}
		})
	}
}