| // Copyright 2023 The Fuchsia Authors. All rights reserved. | |
| // Use of this source code is governed by a BSD-style license that can be | |
| // found in the LICENSE file. | |
| package resultdb | |
| import ( | |
| "encoding/json" | |
| "flag" | |
| "fmt" | |
| "io" | |
| "net" | |
| "net/http" | |
| "net/http/httptest" | |
| "os" | |
| "path/filepath" | |
| "sort" | |
| "strings" | |
| "sync" | |
| "sync/atomic" | |
| "testing" | |
| "time" | |
| "github.com/google/go-cmp/cmp" | |
| resultpb "go.chromium.org/luci/resultdb/proto/v1" | |
| sinkpb "go.chromium.org/luci/resultdb/sink/proto/v1" | |
| "google.golang.org/protobuf/encoding/protojson" | |
| "google.golang.org/protobuf/testing/protocmp" | |
| "go.fuchsia.dev/fuchsia/tools/testing/runtests" | |
| ) | |
| var testDataDir = flag.String("test_data_dir", "testdata", "Path to testdata/; only used in GN build") | |
| func TestGetLUCICtx(t *testing.T) { | |
| old := os.Getenv("LUCI_CONTEXT") | |
| defer os.Setenv("LUCI_CONTEXT", old) | |
| os.Setenv("LUCI_CONTEXT", filepath.Join(*testDataDir, "lucictx.json")) | |
| client, err := NewClient() | |
| if err != nil { | |
| t.Errorf("Cannot parse LUCI_CONTEXT: %v", err) | |
| } | |
| if client.resultSink.ResultSinkAddr != "result.sink" { | |
| t.Errorf("Incorrect value parsed for result_sink address. Got %s", client.resultSink.ResultSinkAddr) | |
| } | |
| if client.resultSink.AuthToken != "token" { | |
| t.Errorf("Incorrect value parsed for result_sink auth_token field. Got %s", client.resultSink.AuthToken) | |
| } | |
| } | |
| func TestParse2Summary(t *testing.T) { | |
| t.Parallel() | |
| for _, name := range []string{"summary.json", "summary2.json"} { | |
| summary, err := ParseSummaryTests(filepath.Join(*testDataDir, name)) | |
| if err != nil { | |
| t.Fatal(err) | |
| } | |
| testResults, _, skipped := SummaryToResultSink(summary, []*resultpb.StringPair{}, name) | |
| for _, testResult := range testResults { | |
| if len(testResult.TestId) == 0 { | |
| t.Errorf("Empty testId is not allowed.") | |
| } | |
| } | |
| if len(skipped) != 0 { | |
| t.Errorf("Tests got skipped %v, expect no skip", skipped) | |
| } | |
| } | |
| } | |
| func TestFailWithLongTestName(t *testing.T) { | |
| summary, err := ParseSummaryTests(filepath.Join(*testDataDir, "summary_long_name.json")) | |
| if err != nil { | |
| t.Fatal(err) | |
| } | |
| _, _, testsSkipped := SummaryToResultSink(summary, []*resultpb.StringPair{}, "") | |
| skippedTestName := "fuchsia-pkg://fuchsia.com/netstack-integration-tests#meta/netstack-inspect-integration-test.cm/:fuchsia_fuchsia_fuchsia_fuchsia_fuchsia_fuchsia_fuchsia_fuchsia_fuchsia_fuchsia_inspect_dhcp_netdevice::_multiple_invalid_port_and_single_invalid_trans_proto_vec_packetattributes_ip_proto_packet_formats_ip_ipv4proto_proto_packet_formats_ip_ipproto_udp_port_invalid_port_packetattributes_ip_proto_packet_formats_ip_ipv4proto_proto_packet_formats_ip_ipproto_udp_port_invalid_port_packetattributes_ip_proto_packet_formats_ip_ipv4proto_proto_packet_formats_ip_ipproto_tcp_port_dhcp_client_port_" | |
| if len(testsSkipped) != 1 { | |
| t.Errorf("Incorrect number of skipped tests got: %d want %d", len(testsSkipped), 1) | |
| } | |
| if testsSkipped[0] != skippedTestName { | |
| t.Errorf("Incorrect skipped test, got: %s want %s", testsSkipped[0], skippedTestName) | |
| } | |
| } | |
| func TestParseExoneratedSummary(t *testing.T) { | |
| // 1. Load and map the custom summary containing exonerations | |
| summary, err := ParseSummaryTests(filepath.Join(*testDataDir, "summary_exonerated.json")) | |
| if err != nil { | |
| t.Fatal(err) | |
| } | |
| // Assert total count of target items in summary returned by ParseSummaryTests | |
| if len(summary.Tests) != 5 { | |
| t.Fatalf("Unexpected parsed tests count inside summary: got %d, want 5", len(summary.Tests)) | |
| } | |
| results, exonerations, skipped := SummaryToResultSink(summary, []*resultpb.StringPair{}, "summary_exonerated.json") | |
| // Expect exactly 6 results: | |
| // - 1 from prefix/exonerated_target_only (exonerated target is mapped as FAILED) | |
| // - 3 from prefix/failed_target_with_exonerated_case (1 exonerated case (FAILED) + 1 passed case (PASSED) + 1 failed target details (FAILED)) | |
| // - 1 from prefix/passed_target (passed target itself) | |
| // - 1 from prefix/skipped_target (skipped target itself) | |
| // - 0 from prefix/unloaded_target_exceeding_limit_... (skipped upload target due to name length exceeds 512 limit) | |
| if len(results) != 6 { | |
| t.Fatalf("Unexpected number of mapped results: got %d, want 6", len(results)) | |
| } | |
| // Expect exactly 2 exonerations: | |
| // - 1 target exoneration from prefix/exonerated_target_only | |
| // - 1 case exoneration from prefix/failed_target_with_exonerated_case/SuiteX:CaseExonerated | |
| if len(exonerations) != 2 { | |
| t.Fatalf("Unexpected number of mapped exonerations: got %d, want 2", len(exonerations)) | |
| } | |
| // Expect exactly 1 skipped target run exceeding the 512-byte limit | |
| if len(skipped) != 1 { | |
| t.Fatalf("Got %d skipped tests, want 1", len(skipped)) | |
| } | |
| expectedSkippedName := "prefix/unloaded_target_exceeding_limit_fuchsia_fuchsia_fuchsia_fuchsia_fuchsia_fuchsia_fuchsia_fuchsia_fuchsia_fuchsia_fuchsia_fuchsia_fuchsia_fuchsia_fuchsia_fuchsia_fuchsia_fuchsia_fuchsia_fuchsia_fuchsia_fuchsia_fuchsia_fuchsia_fuchsia_fuchsia_fuchsia_fuchsia_fuchsia_fuchsia_fuchsia_fuchsia_fuchsia_fuchsia_fuchsia_fuchsia_fuchsia_fuchsia_fuchsia_fuchsia_fuchsia_fuchsia_fuchsia_fuchsia_fuchsia_fuchsia_fuchsia_fuchsia_fuchsia_fuchsia_fuchsia_fuchsia_fuchsia_fuchsia_fuchsia_fuchsia_fuchsia_fuchsia_fuchsia_fuchsia_fuchsia_fuchsia" | |
| if skipped[0] != expectedSkippedName { | |
| t.Errorf("Unexpected skipped target returned: got %q, want %q", skipped[0], expectedSkippedName) | |
| } | |
| // 2. Verify mapped payload details | |
| // results[0]: prefix/exonerated_target_only (FAILED) | |
| if results[0].TestId != "prefix/exonerated_target_only" || results[0].StatusV2 != resultpb.TestResult_FAILED { | |
| t.Errorf("Unexpected mapped result 0: %+v", results[0]) | |
| } | |
| // results[1]: prefix/failed_target_with_exonerated_case/SuiteX:CaseExonerated (FAILED) | |
| if results[1].TestId != "prefix/failed_target_with_exonerated_case/SuiteX:CaseExonerated" || results[1].StatusV2 != resultpb.TestResult_FAILED { | |
| t.Errorf("Unexpected mapped result 1: %+v", results[1]) | |
| } | |
| if results[1].FailureReason == nil || len(results[1].FailureReason.Errors) == 0 || results[1].FailureReason.Errors[0].Message != "Flaky run instance" { | |
| t.Errorf("Unexpected FailureReason for result 1: %+v", results[1].FailureReason) | |
| } | |
| // results[2]: prefix/failed_target_with_exonerated_case/SuiteX:CasePassed (PASSED) | |
| if results[2].TestId != "prefix/failed_target_with_exonerated_case/SuiteX:CasePassed" || results[2].StatusV2 != resultpb.TestResult_PASSED { | |
| t.Errorf("Unexpected mapped result 2: %+v", results[2]) | |
| } | |
| // results[3]: prefix/failed_target_with_exonerated_case (FAILED) | |
| if results[3].TestId != "prefix/failed_target_with_exonerated_case" || results[3].StatusV2 != resultpb.TestResult_FAILED { | |
| t.Errorf("Unexpected mapped result 3: %+v", results[3]) | |
| } | |
| expectedErrorMessage := "CaseExonerated: Flaky run instance" | |
| if results[3].FailureReason == nil || len(results[3].FailureReason.Errors) == 0 || results[3].FailureReason.Errors[0].Message != expectedErrorMessage { | |
| t.Errorf("Target failure reason was not populated with the failed cases. got err: %q", results[3].FailureReason.GetErrors()[0].GetMessage()) | |
| } | |
| // results[4]: prefix/passed_target (PASSED) | |
| if results[4].TestId != "prefix/passed_target" || results[4].StatusV2 != resultpb.TestResult_PASSED { | |
| t.Errorf("Unexpected mapped result 4: %+v", results[4]) | |
| } | |
| // results[5]: prefix/skipped_target (SKIPPED) | |
| if results[5].TestId != "prefix/skipped_target" || results[5].StatusV2 != resultpb.TestResult_SKIPPED { | |
| t.Errorf("Unexpected mapped result 5: %+v", results[5]) | |
| } | |
| if results[5].SkippedReason == nil || results[5].SkippedReason.ReasonMessage != "skipped because unaffected" { | |
| t.Errorf("Skipped target mapped incorrect skipped reason, got: %+v", results[5].SkippedReason) | |
| } | |
| // Expect the exonerations mapped to: | |
| // - Case exoneration: prefix/failed_target_with_exonerated_case/SuiteX:CaseExonerated | |
| // - Target exoneration: prefix/exonerated_target_only | |
| exoneratedIDs := []string{exonerations[0].TestId, exonerations[1].TestId} | |
| sort.Strings(exoneratedIDs) | |
| expectedIDs := []string{ | |
| "prefix/exonerated_target_only", | |
| "prefix/failed_target_with_exonerated_case/SuiteX:CaseExonerated", | |
| } | |
| if diff := cmp.Diff(exoneratedIDs, expectedIDs); diff != "" { | |
| t.Errorf("Exoneration IDs differ (-got +want):\n%s", diff) | |
| } | |
| } | |
| func TestSendData(t *testing.T) { | |
| var receivedBody string | |
| var receivedAuthHeader string | |
| var receivedPath string | |
| server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { | |
| receivedAuthHeader = r.Header.Get("Authorization") | |
| receivedPath = r.URL.Path | |
| b, err := io.ReadAll(r.Body) | |
| if err != nil { | |
| w.WriteHeader(http.StatusInternalServerError) | |
| return | |
| } | |
| receivedBody = string(b) | |
| w.WriteHeader(http.StatusOK) | |
| w.Write([]byte("{}")) | |
| })) | |
| defer server.Close() | |
| client := &Client{ | |
| resultSink: &resultSink{ | |
| AuthToken: "test-token", | |
| ResultSinkAddr: server.Listener.Addr().String(), | |
| }, | |
| httpClient: &http.Client{}, | |
| semaphore: make(chan struct{}, 1), | |
| } | |
| client.semaphore <- struct{}{} | |
| payload := `{"testResults":[{"testId":"foo"}]}` | |
| if err := client.SendData([]byte(payload), "ReportTestResults"); err != nil { | |
| t.Fatalf("SendData failed: %v", err) | |
| } | |
| if receivedAuthHeader != "ResultSink test-token" { | |
| t.Errorf("got auth header %q, want %q", receivedAuthHeader, "ResultSink test-token") | |
| } | |
| if receivedPath != "/prpc/luci.resultsink.v1.Sink/ReportTestResults" { | |
| t.Errorf("got path %q, want %q", receivedPath, "/prpc/luci.resultsink.v1.Sink/ReportTestResults") | |
| } | |
| if receivedBody != payload { | |
| t.Errorf("got body %q, want %q", receivedBody, payload) | |
| } | |
| } | |
| func TestSendDataError(t *testing.T) { | |
| server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { | |
| w.WriteHeader(http.StatusBadRequest) | |
| })) | |
| defer server.Close() | |
| client := &Client{ | |
| resultSink: &resultSink{ | |
| AuthToken: "test-token", | |
| ResultSinkAddr: server.Listener.Addr().String(), | |
| }, | |
| httpClient: &http.Client{}, | |
| semaphore: make(chan struct{}, 1), | |
| } | |
| client.semaphore <- struct{}{} | |
| if err := client.SendData([]byte(`{}`), "ReportTestResults"); err == nil { | |
| t.Errorf("expected error for HTTP 400, got nil") | |
| } | |
| } | |
| func TestReportSummariesMatchesSummaryToResultSink(t *testing.T) { | |
| summaries := []string{ | |
| filepath.Join(*testDataDir, "summary.json"), | |
| filepath.Join(*testDataDir, "summary2.json"), | |
| filepath.Join(*testDataDir, "summary_exonerated.json"), | |
| } | |
| tags := []*resultpb.StringPair{ | |
| {Key: "builder", Value: "core.x64-release"}, | |
| } | |
| var wantResults []*sinkpb.TestResult | |
| var wantExonerations []*sinkpb.TestExoneration | |
| var wantSkipped []string | |
| for _, summaryFile := range summaries { | |
| summary, err := ParseSummaryTests(summaryFile) | |
| if err != nil { | |
| t.Fatalf("ParseSummaryTests(%s) failed: %s", summaryFile, err) | |
| } | |
| results, exonerations, skipped := SummaryToResultSink(summary, tags, *testDataDir) | |
| wantResults = append(wantResults, results...) | |
| wantExonerations = append(wantExonerations, exonerations...) | |
| wantSkipped = append(wantSkipped, skipped...) | |
| } | |
| var mu sync.Mutex | |
| var gotResults []*sinkpb.TestResult | |
| var gotExonerations []*sinkpb.TestExoneration | |
| srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { | |
| body, err := io.ReadAll(r.Body) | |
| if err != nil { | |
| t.Errorf("failed reading request body: %s", err) | |
| w.WriteHeader(http.StatusInternalServerError) | |
| return | |
| } | |
| mu.Lock() | |
| defer mu.Unlock() | |
| switch { | |
| case strings.HasSuffix(r.URL.Path, "/ReportTestResults"): | |
| var req sinkpb.ReportTestResultsRequest | |
| if err := protojson.Unmarshal(body, &req); err != nil { | |
| t.Errorf("failed unmarshaling ReportTestResultsRequest: %s", err) | |
| } | |
| if len(req.TestResults) > MaxBatchSize { | |
| t.Errorf("batch size %d exceeds MaxBatchSize %d", len(req.TestResults), MaxBatchSize) | |
| } | |
| gotResults = append(gotResults, req.TestResults...) | |
| case strings.HasSuffix(r.URL.Path, "/ReportTestExonerations"): | |
| var req sinkpb.ReportTestExonerationsRequest | |
| if err := protojson.Unmarshal(body, &req); err != nil { | |
| t.Errorf("failed unmarshaling ReportTestExonerationsRequest: %s", err) | |
| } | |
| if len(req.TestExonerations) > MaxBatchSize { | |
| t.Errorf("batch size %d exceeds MaxBatchSize %d", len(req.TestExonerations), MaxBatchSize) | |
| } | |
| gotExonerations = append(gotExonerations, req.TestExonerations...) | |
| default: | |
| t.Errorf("unexpected endpoint %s", r.URL.Path) | |
| } | |
| w.WriteHeader(http.StatusOK) | |
| _, _ = w.Write([]byte("{}")) | |
| })) | |
| defer srv.Close() | |
| client := &Client{ | |
| resultSink: &resultSink{ | |
| AuthToken: "token", | |
| ResultSinkAddr: strings.TrimPrefix(srv.URL, "http://"), | |
| }, | |
| httpClient: srv.Client(), | |
| semaphore: make(chan struct{}, 64), | |
| } | |
| for i := 0; i < cap(client.semaphore); i++ { | |
| client.semaphore <- struct{}{} | |
| } | |
| gotSkipped, err := client.ReportSummaries(summaries, tags, *testDataDir) | |
| if err != nil { | |
| t.Fatalf("ReportSummaries failed: %s", err) | |
| } | |
| if diff := cmp.Diff(wantSkipped, gotSkipped); diff != "" { | |
| t.Errorf("skipped tests differ (-want +got):\n%s", diff) | |
| } | |
| sortTags := protocmp.SortRepeatedFields(&sinkpb.TestResult{}, "tags") | |
| if diff := cmp.Diff(wantResults, gotResults, protocmp.Transform(), sortTags); diff != "" { | |
| t.Errorf("reported results differ (-want +got):\n%s", diff) | |
| } | |
| if diff := cmp.Diff(wantExonerations, gotExonerations, protocmp.Transform()); diff != "" { | |
| t.Errorf("reported exonerations differ (-want +got):\n%s", diff) | |
| } | |
| } | |
| func TestReportSummariesBatching(t *testing.T) { | |
| // Generate a summary with > 2*MaxBatchSize results and exonerations to | |
| // exercise multi-batch flushing and final tail flushing. | |
| const caseCount = MaxBatchSize*2 + 10 | |
| cases := make([]runtests.TestCaseResult, caseCount) | |
| for i := range cases { | |
| cases[i] = runtests.TestCaseResult{ | |
| DisplayName: fmt.Sprintf("suite/case_%d", i), | |
| SuiteName: "suite", | |
| CaseName: fmt.Sprintf("case_%d", i), | |
| Status: runtests.TestExonerated, | |
| Format: "Rust", | |
| } | |
| } | |
| summary := runtests.TestSummary{ | |
| Outputs: map[string]string{"syslog.txt": "syslog.txt"}, | |
| Tests: []runtests.TestDetails{ | |
| { | |
| Name: "host_x64/multiplied_test", | |
| SourceLabel: "//src/foo:multiplied_test", | |
| Status: runtests.TestFailure, | |
| StartTime: time.Unix(1700000000, 0), | |
| TestResult: runtests.TestResult{Cases: cases}, | |
| }, | |
| }, | |
| } | |
| summaryBytes, err := json.Marshal(summary) | |
| if err != nil { | |
| t.Fatalf("failed marshaling summary: %s", err) | |
| } | |
| summaryPath := filepath.Join(t.TempDir(), "summary.json") | |
| if err := os.WriteFile(summaryPath, summaryBytes, 0o600); err != nil { | |
| t.Fatalf("failed writing summary: %s", err) | |
| } | |
| var resultBatchSizes []int | |
| var exonerationBatchSizes []int | |
| srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { | |
| body, err := io.ReadAll(r.Body) | |
| if err != nil { | |
| t.Errorf("failed reading body: %s", err) | |
| w.WriteHeader(http.StatusInternalServerError) | |
| return | |
| } | |
| switch { | |
| case strings.HasSuffix(r.URL.Path, "/ReportTestResults"): | |
| var req sinkpb.ReportTestResultsRequest | |
| if err := protojson.Unmarshal(body, &req); err != nil { | |
| t.Errorf("failed unmarshaling ReportTestResultsRequest: %s", err) | |
| } | |
| resultBatchSizes = append(resultBatchSizes, len(req.TestResults)) | |
| case strings.HasSuffix(r.URL.Path, "/ReportTestExonerations"): | |
| var req sinkpb.ReportTestExonerationsRequest | |
| if err := protojson.Unmarshal(body, &req); err != nil { | |
| t.Errorf("failed unmarshaling ReportTestExonerationsRequest: %s", err) | |
| } | |
| exonerationBatchSizes = append(exonerationBatchSizes, len(req.TestExonerations)) | |
| } | |
| w.WriteHeader(http.StatusOK) | |
| _, _ = w.Write([]byte("{}")) | |
| })) | |
| defer srv.Close() | |
| client := &Client{ | |
| resultSink: &resultSink{ | |
| AuthToken: "token", | |
| ResultSinkAddr: strings.TrimPrefix(srv.URL, "http://"), | |
| }, | |
| httpClient: srv.Client(), | |
| semaphore: make(chan struct{}, 64), | |
| } | |
| for i := 0; i < cap(client.semaphore); i++ { | |
| client.semaphore <- struct{}{} | |
| } | |
| skipped, err := client.ReportSummaries([]string{summaryPath}, nil, t.TempDir()) | |
| if err != nil { | |
| t.Fatalf("ReportSummaries failed: %s", err) | |
| } | |
| if len(skipped) != 0 { | |
| t.Errorf("got skipped %v, want empty", skipped) | |
| } | |
| // caseCount cases + 1 top-level test result. | |
| wantResultBatches := []int{MaxBatchSize, MaxBatchSize, 11} | |
| if diff := cmp.Diff(wantResultBatches, resultBatchSizes); diff != "" { | |
| t.Errorf("result batch sizes differ (-want +got):\n%s", diff) | |
| } | |
| // caseCount exonerated cases (top-level test has Status=TestFailure, not TestExonerated). | |
| wantExonerationBatches := []int{MaxBatchSize, MaxBatchSize, 10} | |
| if diff := cmp.Diff(wantExonerationBatches, exonerationBatchSizes); diff != "" { | |
| t.Errorf("exoneration batch sizes differ (-want +got):\n%s", diff) | |
| } | |
| } | |
| func TestClientKeepAlive(t *testing.T) { | |
| var newConns atomic.Int64 | |
| srv := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { | |
| _, _ = io.Copy(io.Discard, r.Body) | |
| w.WriteHeader(http.StatusOK) | |
| _, _ = w.Write([]byte(")]}'\n{}\n")) | |
| })) | |
| srv.Config.ConnState = func(_ net.Conn, state http.ConnState) { | |
| if state == http.StateNew { | |
| newConns.Add(1) | |
| } | |
| } | |
| srv.Start() | |
| defer srv.Close() | |
| lucictxFile := filepath.Join(t.TempDir(), "lucictx.json") | |
| if err := os.WriteFile(lucictxFile, []byte(fmt.Sprintf(`{ | |
| "result_sink": { | |
| "address": "%s", | |
| "auth_token": "test-token" | |
| } | |
| }`, srv.Listener.Addr().String())), 0o600); err != nil { | |
| t.Fatal(err) | |
| } | |
| t.Setenv("LUCI_CONTEXT", lucictxFile) | |
| client, err := NewClient() | |
| if err != nil { | |
| t.Fatalf("NewClient failed: %s", err) | |
| } | |
| for i := 0; i < 20; i++ { | |
| if err := client.SendData([]byte(`{}`), "ReportTestResults"); err != nil { | |
| t.Fatalf("SendData(%d) failed: %s", i, err) | |
| } | |
| } | |
| if got := newConns.Load(); got != 1 { | |
| t.Errorf("sequential SendData opened %d TCP connections, want 1 reused keep-alive connection", got) | |
| } | |
| } |