cmd/autocorrelator/check_ci_test.go

// Copyright 2021 The Fuchsia Authors.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

package main

import (
	"testing"

	buildbucketpb "go.chromium.org/luci/buildbucket/proto"
	"go.chromium.org/luci/common/proto/git"
	"go.fuchsia.dev/infra/cmd/autocorrelator/findings"

	"github.com/google/go-cmp/cmp"
)

func TestCheckCI(t *testing.T) {
	t.Parallel()

	var tests = []struct {
		log             []*git.Commit
		builds          []*buildbucketpb.Build
		status          buildbucketpb.Status
		summaryMarkdown string
		expected        *findings.SummarySimilarity
	}{
		{
			log: []*git.Commit{
				{
					Id: "D",
				},
				{
					Id: "C",
				},
				{
					Id: "B",
				},
				{
					Id: "A",
				},
			},
			// SUCCESS > FAILURE case, where we are looking for FAILURE.
			builds: []*buildbucketpb.Build{
				// This build's status is SUCCESS. We should return a finding
				// as soon as we see this, indicating that CI does not appear
				// broken.
				{
					Id: int64(999999),
					Input: &buildbucketpb.Build_Input{
						GitilesCommit: &buildbucketpb.GitilesCommit{
							Id: "C",
						},
					},
					Status:          buildbucketpb.Status_SUCCESS,
					SummaryMarkdown: "",
				},
				// Even though this build's status is FAILURE, it should be
				// ignored, since the above build should have been caught first.
				{
					Id: int64(999998),
					Input: &buildbucketpb.Build_Input{
						GitilesCommit: &buildbucketpb.GitilesCommit{
							Id: "A",
						},
					},
					Status:          buildbucketpb.Status_FAILURE,
					SummaryMarkdown: "ERROR",
				},
			},
			status:          buildbucketpb.Status_FAILURE,
			summaryMarkdown: "ERROR",
			expected: &findings.SummarySimilarity{
				Score:   0.0,
				BuildId: "999999",
				// The caught build has commit ID C which is one commit away
				// from the front of the log.
				CommitDist: 1,
				IsGreen:    true,
			},
		},
		{
			log: []*git.Commit{
				{
					Id: "D",
				},
				{
					Id: "C",
				},
				{
					Id: "B",
				},
				{
					Id: "A",
				},
			},
			// SUCCESS > FAILURE case, where we are looking for FAILURE.
			builds: []*buildbucketpb.Build{
				// This build's commit ID is not in the log, and should be
				// ignored, even though the build's status matches the input
				// status.
				{
					Id: int64(999999),
					Input: &buildbucketpb.Build_Input{
						GitilesCommit: &buildbucketpb.GitilesCommit{
							Id: "E",
						},
					},
					Status:          buildbucketpb.Status_FAILURE,
					SummaryMarkdown: "ERROR",
				},
				// This build's status is FAILURE, which matches the input
				// status. We should catch this build.
				{
					Id: int64(999998),
					Input: &buildbucketpb.Build_Input{
						GitilesCommit: &buildbucketpb.GitilesCommit{
							Id: "C",
						},
					},
					Status:          buildbucketpb.Status_FAILURE,
					SummaryMarkdown: "ERROR",
				},
				// Even though this build's status is FAILURE, it should be
				// ignored, since the above build should have been caught first.
				{
					Id: int64(999997),
					Input: &buildbucketpb.Build_Input{
						GitilesCommit: &buildbucketpb.GitilesCommit{
							Id: "A",
						},
					},
					Status:          buildbucketpb.Status_FAILURE,
					SummaryMarkdown: "ERROR",
				},
			},
			status:          buildbucketpb.Status_FAILURE,
			summaryMarkdown: "ERROR",
			expected: &findings.SummarySimilarity{
				Score:   1.0,
				BuildId: "999998",
				// The caught build has commit ID C which is one commit away
				// from the front of the log.
				CommitDist: 1,
				IsGreen:    false,
			},
		},
		{
			log: []*git.Commit{
				{
					Id: "D",
				},
				{
					Id: "C",
				},
				{
					Id: "B",
				},
				{
					Id: "A",
				},
			},
			// FAILURE > INFRA_FAILURE case, where we are looking for
			// INFRA_FAILURE.
			builds: []*buildbucketpb.Build{
				// This build's status is FAILURE, which does not match the
				// input status, and should be ignored.
				{
					Id: int64(999999),
					Input: &buildbucketpb.Build_Input{
						GitilesCommit: &buildbucketpb.GitilesCommit{
							Id: "C",
						},
					},
					Status:          buildbucketpb.Status_FAILURE,
					SummaryMarkdown: "ERROR",
				},
				// This build's status is INFRA_FAILURE, which does match the
				// input status, and should be caught.
				{
					Id: int64(999998),
					Input: &buildbucketpb.Build_Input{
						GitilesCommit: &buildbucketpb.GitilesCommit{
							Id: "B",
						},
					},
					Status:          buildbucketpb.Status_INFRA_FAILURE,
					SummaryMarkdown: "ERROR: checkout failed",
				},
			},
			status:          buildbucketpb.Status_INFRA_FAILURE,
			summaryMarkdown: "ERROR: checkout failed",
			expected: &findings.SummarySimilarity{
				Score:   1.0,
				BuildId: "999998",
				// The caught build has commit ID B which is three commits away
				// from the front of the log.
				CommitDist: 2,
				IsGreen:    false,
			},
		},
		{
			log: []*git.Commit{
				{
					Id: "D",
				},
				{
					Id: "C",
				},
				{
					Id: "B",
				},
				{
					Id: "A",
				},
			},
			// INFRA_FAILURE > FAILURE case, where we are looking for FAILURE.
			builds: []*buildbucketpb.Build{
				// This build's status is INFRA_FAILURE, which does not match
				// the input status, and should be ignored.
				{
					Id: int64(999999),
					Input: &buildbucketpb.Build_Input{
						GitilesCommit: &buildbucketpb.GitilesCommit{
							Id: "C",
						},
					},
					Status:          buildbucketpb.Status_INFRA_FAILURE,
					SummaryMarkdown: "ERROR: checkout failed",
				},
				// This build's status is FAILURE, which does match the input
				// status, and should be caught.
				{
					Id: int64(999998),
					Input: &buildbucketpb.Build_Input{
						GitilesCommit: &buildbucketpb.GitilesCommit{
							Id: "A",
						},
					},
					Status:          buildbucketpb.Status_FAILURE,
					SummaryMarkdown: "ERROR",
				},
			},
			status:          buildbucketpb.Status_FAILURE,
			summaryMarkdown: "ERROR",
			expected: &findings.SummarySimilarity{
				Score:   1.0,
				BuildId: "999998",
				// The caught build has commit ID A which is three commits away
				// from the front of the log.
				CommitDist: 3,
				IsGreen:    false,
			},
		},
		// Build exhaustion, where we are looking for FAILURE.
		{
			log: []*git.Commit{
				{
					Id: "D",
				},
				{
					Id: "C",
				},
				{
					Id: "B",
				},
				{
					Id: "A",
				},
			},
			builds: []*buildbucketpb.Build{
				// This build's status is INFRA_FAILURE, which does not match
				// the input status, and should be ignored.
				{
					Id: int64(999999),
					Input: &buildbucketpb.Build_Input{
						GitilesCommit: &buildbucketpb.GitilesCommit{
							Id: "C",
						},
					},
					Status:          buildbucketpb.Status_INFRA_FAILURE,
					SummaryMarkdown: "ERROR: checkout failed",
				},
			},
			status:          buildbucketpb.Status_FAILURE,
			summaryMarkdown: "ERROR",
			// We should have exhausted the list of builds without having found
			// anything.
			expected: nil,
		},
	}

	for _, test := range tests {
		comparator := mockComparator{}
		ss := checkCI(test.log, test.builds, test.status, test.summaryMarkdown, comparator)
		if diff := cmp.Diff(test.expected, ss); diff != "" {
			t.Fatalf("different (-want +got):\n%s", diff)
		}
	}
}