| // Copyright 2020 The Fuchsia Authors. All rights reserved. | |
| // Use of this source code is governed by a BSD-style license that can be | |
| // found in the LICENSE file. | |
| package resultdb | |
| import ( | |
| "cmp" | |
| "encoding/json" | |
| "fmt" | |
| "log" | |
| "os" | |
| "path" | |
| "path/filepath" | |
| "regexp" | |
| "slices" | |
| "strings" | |
| "time" | |
| resultpb "go.chromium.org/luci/resultdb/proto/v1" | |
| sinkpb "go.chromium.org/luci/resultdb/sink/proto/v1" | |
| "google.golang.org/protobuf/encoding/protojson" | |
| "google.golang.org/protobuf/proto" | |
| "google.golang.org/protobuf/types/known/durationpb" | |
| "google.golang.org/protobuf/types/known/structpb" | |
| "google.golang.org/protobuf/types/known/timestamppb" | |
| "go.fuchsia.dev/fuchsia/tools/build" | |
| "go.fuchsia.dev/fuchsia/tools/testing/runtests" | |
| ) | |
| const ( | |
| // Test ID is limited to 512 bytes. | |
| // https://source.chromium.org/chromium/infra/infra_superproject/+/main:infra/go/src/go.chromium.org/luci/resultdb/pbutil/test_id.go;l=508;drc=14e57c2183912ea2a9c93cd19dbe6eb7347283e8 | |
| MaxTestIDLength = 512 | |
| // Failure reason is limited to 1024 bytes. | |
| // https://source.chromium.org/chromium/infra/infra_superproject/+/main:infra/go/src/go.chromium.org/luci/resultdb/pbutil/test_result.go;l=44;drc=bfb50731e1b97d7ca771fb2d31dc7338c3db40f5 | |
| MaxFailureReasonLength = 1024 | |
| // MaxFailureReasonTotalSize is the maximum total protobuf size of a failure reason (16384 bytes). | |
| MaxFailureReasonTotalSize = 16384 | |
| // MaxExplanationHTMLLength is the maximum size of a test exoneration's explanation_html (4096 bytes). | |
| MaxExplanationHTMLLength = 4096 | |
| // MaxBatchSize is the maximum number of items (results or exonerations) reported in a single request. | |
| MaxBatchSize = 250 | |
| // DefaultRepo is the Gitiles repository URL for the main Fuchsia repository. | |
| DefaultRepo = "https://fuchsia.googlesource.com/fuchsia" | |
| // VendorGoogleRepo is the Gitiles repository URL for the Google vendor repository. | |
| VendorGoogleRepo = "https://turquoise-internal.googlesource.com/vendor/google" | |
| // VendorGooglePathPrefix is the path prefix for the Google vendor repository. | |
| VendorGooglePathPrefix = "//vendor/google" | |
| // MaxLocationRepoLength is the maximum length in bytes for a test location repository URL (256 bytes). | |
| // //third_party/luci-go/resultdb/proto/v1/test_metadata.proto:106 | |
| // https://source.chromium.org/chromium/infra/infra_superproject/+/main:infra/go/src/go.chromium.org/luci/resultdb/proto/v1/test_metadata.proto;l=107 | |
| MaxLocationRepoLength = 256 | |
| // MaxLocationFileNameLength is the maximum length in bytes for a test location file name (512 bytes). | |
| // //third_party/luci-go/resultdb/proto/v1/test_metadata.proto:113 | |
| // https://source.chromium.org/chromium/infra/infra_superproject/+/main:infra/go/src/go.chromium.org/luci/resultdb/proto/v1/test_metadata.proto;l=114 | |
| MaxLocationFileNameLength = 512 | |
| // MaxPropertiesSize is the maximum serialized size of a test result's | |
| // properties. ResultDB enforces it in ValidateTestResultProperties through | |
| // MaxSizeTestResultProperties; the 8 KB mentioned in the comment of the | |
| // vendored test_result.proto is stale. | |
| // https://source.chromium.org/chromium/infra/infra_superproject/+/main:infra/go/src/go.chromium.org/luci/resultdb/pbutil/common.go;l=53;drc=4ea817bdb4c03f2a48c98fc85a3151edf1c0cb86 | |
| MaxPropertiesSize = 20 * 1024 | |
| ) | |
| // UploadRequest represents a serialized payload to be posted to ResultSink. | |
| type UploadRequest struct { | |
| Endpoint string `json:"endpoint"` | |
| Data json.RawMessage `json:"data"` | |
| } | |
| // ParseSummaryTests unmarshals the "tests" array of a summary.json file into | |
| // a runtests.TestSummary. Other top-level fields, such as Outputs, are left | |
| // empty. | |
| func ParseSummaryTests(filePath string) (*runtests.TestSummary, error) { | |
| var summary runtests.TestSummary | |
| if err := forEachSummaryTest(filePath, func(test *runtests.TestDetails) error { | |
| summary.Tests = append(summary.Tests, *test) | |
| return nil | |
| }); err != nil { | |
| return nil, err | |
| } | |
| return &summary, nil | |
| } | |
| // forEachSummaryTest decodes the "tests" array in a summary.json file one | |
| // TestDetails element at a time and calls fn on each element, avoiding holding | |
| // all TestDetails (and their nested TestCaseResult slices) in memory at once. | |
| func forEachSummaryTest(filePath string, fn func(*runtests.TestDetails) error) error { | |
| f, err := os.Open(filePath) | |
| if err != nil { | |
| return err | |
| } | |
| defer f.Close() | |
| dec := json.NewDecoder(f) | |
| tok, err := dec.Token() | |
| if err != nil { | |
| return err | |
| } | |
| if tok != json.Delim('{') { | |
| return fmt.Errorf("expected '{' at start of %s, got %v", filePath, tok) | |
| } | |
| for dec.More() { | |
| keyTok, err := dec.Token() | |
| if err != nil { | |
| return err | |
| } | |
| key, ok := keyTok.(string) | |
| if !ok { | |
| return fmt.Errorf("expected string key in %s, got %v", filePath, keyTok) | |
| } | |
| if key != "tests" { | |
| var skip json.RawMessage | |
| if err := dec.Decode(&skip); err != nil { | |
| return err | |
| } | |
| continue | |
| } | |
| arrTok, err := dec.Token() | |
| if err != nil { | |
| return err | |
| } | |
| if arrTok == nil { | |
| continue | |
| } | |
| if arrTok != json.Delim('[') { | |
| return fmt.Errorf("expected '[' for tests field in %s, got %v", filePath, arrTok) | |
| } | |
| for dec.More() { | |
| var test runtests.TestDetails | |
| if err := dec.Decode(&test); err != nil { | |
| return err | |
| } | |
| if err := fn(&test); err != nil { | |
| return err | |
| } | |
| } | |
| if _, err := dec.Token(); err != nil { | |
| return err | |
| } | |
| } | |
| if _, err := dec.Token(); err != nil { | |
| return err | |
| } | |
| return nil | |
| } | |
| // TODO(b/527958872): Remove legacy flat test-id and summary batch conversion logic | |
| // once Fuchsia test results fully transition to structured test identifiers. | |
| // Major functions and helpers to be removed: | |
| // - SummaryToResultSink | |
| // - testToResultSink | |
| // - MaxTestIDLength checks and flat test_id formatting in testCaseToResultSink | |
| // and testDetailsToResultSink (along with testsSkipped return plumbing) | |
| // - unreachable legacy scheme switch branches in testDetailsToResultSink | |
| // | |
| // SummaryToResultSink converts runtests.TestSummary data into an array of result_sink TestResult and TestExoneration. | |
| func SummaryToResultSink(s *runtests.TestSummary, tags []*resultpb.StringPair, outputRoot string) ([]*sinkpb.TestResult, []*sinkpb.TestExoneration, []string) { | |
| if len(outputRoot) == 0 { | |
| outputRoot, _ = os.Getwd() | |
| } | |
| rootPath, _ := filepath.Abs(outputRoot) | |
| var r []*sinkpb.TestResult | |
| var exonerations []*sinkpb.TestExoneration | |
| var ts []string | |
| for _, test := range s.Tests { | |
| testResults, testExonerations, testsSkipped := testToResultSink(&test, tags, rootPath) | |
| r = append(r, testResults...) | |
| exonerations = append(exonerations, testExonerations...) | |
| ts = append(ts, testsSkipped...) | |
| } | |
| return r, exonerations, ts | |
| } | |
| // testToResultSink converts a single runtests.TestDetails and its test cases into an array of | |
| // result_sink TestResult and TestExoneration. | |
| func testToResultSink(test *runtests.TestDetails, tags []*resultpb.StringPair, rootPath string) ([]*sinkpb.TestResult, []*sinkpb.TestExoneration, []string) { | |
| var r []*sinkpb.TestResult | |
| var exonerations []*sinkpb.TestExoneration | |
| var ts []string | |
| if len(test.Cases) > 0 { | |
| r, exonerations, ts = testCaseToResultSink(test.Cases, tags, test, rootPath) | |
| } | |
| // TODO(b/502613208): If a top-level test passes but has a nested test case that | |
| // is exonerated, it currently reports both a "PASS" status and an exoneration. | |
| // This is redundant since ResultDB generally ignores exonerations for passing tests. | |
| if testResult, testExoneration, testSkipped, err := testDetailsToResultSink(tags, test, rootPath); err == nil { | |
| if testResult == nil { | |
| panic("testResult shouldn't be nil when err is nil") | |
| } | |
| r = append(r, testResult) | |
| if testExoneration != nil { | |
| exonerations = append(exonerations, testExoneration) | |
| } | |
| if testSkipped != "" { | |
| panic(fmt.Sprintf("testSkipped should be empty when err is nil, got: %q", testSkipped)) | |
| } | |
| } else { | |
| if testSkipped != "" { | |
| ts = append(ts, testSkipped) | |
| } | |
| } | |
| // TODO(b/527958852): Report test cases blocked by a fixture setup failure as | |
| // PRECLUDED once runtests.TestStatus has a dedicated not-run status. | |
| // This must run after the test case and top-level results are collected so | |
| // that the *fixture is not reported twice under the same fine_name. | |
| r, ts = appendFixtureResult(r, ts, test) | |
| return r, exonerations, ts | |
| } | |
| // DetermineScheme inspects a test's test cases to determine the appropriate | |
| // ResultDB module scheme (e.g. "rust", "go", "mobly", "gtest", "flat", or "single"). | |
| // Returns "single" if the test has no cases (indicating a standalone script/binary test). | |
| // If a test has cases with conflicting formats, it logs a warning and falls back to "flat". | |
| func DetermineScheme(test *runtests.TestDetails) string { | |
| if len(test.Cases) == 0 { | |
| return "single" | |
| } | |
| var format string | |
| for _, tc := range test.Cases { | |
| if tc.Format == "" { | |
| continue | |
| } | |
| if format == "" { | |
| format = tc.Format | |
| } else if !strings.EqualFold(tc.Format, format) { | |
| log.Printf("[Warn] Test %q has cases with conflicting formats (%q vs %q); falling back to default 'flat' scheme", test.Name, format, tc.Format) | |
| return "flat" | |
| } | |
| } | |
| switch strings.ToLower(format) { | |
| case "ftf", "flat", "shac": | |
| return "flat" | |
| case "googletest", "gtest": | |
| return "gtest" | |
| case "rust": | |
| return "rust" | |
| case "go": | |
| return "go" | |
| case "mobly": | |
| return "mobly" | |
| case "junit": | |
| return "junit" | |
| case "": | |
| log.Printf("[Info] Falling back to default 'flat' scheme for test %q with %d cases", test.Name, len(test.Cases)) | |
| return "flat" | |
| default: | |
| log.Printf("[Info] Unknown format %q for test %q, falling back to 'flat' scheme", format, test.Name) | |
| return "flat" | |
| } | |
| } | |
| // parentInvocationLevelArtifacts creates resultdb artifacts for parent invocation-level files | |
| // with the "parent/" key prefix to be sent to ResultDB sub-invocations. | |
| func parentInvocationLevelArtifacts(outputRoot string, invocationArtifacts []string) map[string]*sinkpb.Artifact { | |
| artifacts := InvocationLevelArtifacts(outputRoot, invocationArtifacts) | |
| parentArtifacts := make(map[string]*sinkpb.Artifact, len(artifacts)) | |
| for k, v := range artifacts { | |
| parentArtifacts[path.Join("parent", k)] = v | |
| } | |
| return parentArtifacts | |
| } | |
| // TestToUploadRequests converts a single runtests.TestDetails into serialized UploadRequests | |
| // suitable for sending to ResultSink endpoints. | |
| func TestToUploadRequests(test *runtests.TestDetails, tags []*resultpb.StringPair, outputRoot string, parentArtifacts []string) ([]UploadRequest, []string, error) { | |
| if len(outputRoot) == 0 { | |
| outputRoot, _ = os.Getwd() | |
| } | |
| rootPath, _ := filepath.Abs(outputRoot) | |
| testResults, exonerations, testsSkipped := testToResultSink(test, tags, rootPath) | |
| artifacts := parentInvocationLevelArtifacts(rootPath, parentArtifacts) | |
| if len(test.Cases) > 0 { | |
| for _, of := range test.OutputFiles { | |
| outputFile := filepath.Join(rootPath, test.OutputDir, of) | |
| if isReadable(outputFile) { | |
| artifacts[artifactName(of)] = &sinkpb.Artifact{ | |
| Body: &sinkpb.Artifact_FilePath{FilePath: outputFile}, | |
| } | |
| } else { | |
| log.Printf("[Warn] outputFile: %s is not readable, skip.", outputFile) | |
| } | |
| } | |
| } | |
| var uploadRequests []UploadRequest | |
| for i := 0; i < len(testResults); i += MaxBatchSize { | |
| end := min(i+MaxBatchSize, len(testResults)) | |
| data, err := protojson.Marshal(&sinkpb.ReportTestResultsRequest{ | |
| TestResults: testResults[i:end], | |
| }) | |
| if err != nil { | |
| return nil, nil, fmt.Errorf("failed to marshal ReportTestResultsRequest: %w", err) | |
| } | |
| uploadRequests = append(uploadRequests, UploadRequest{ | |
| Endpoint: "ReportTestResults", | |
| Data: data, | |
| }) | |
| } | |
| for i := 0; i < len(exonerations); i += MaxBatchSize { | |
| end := min(i+MaxBatchSize, len(exonerations)) | |
| data, err := protojson.Marshal(&sinkpb.ReportTestExonerationsRequest{ | |
| TestExonerations: exonerations[i:end], | |
| }) | |
| if err != nil { | |
| return nil, nil, fmt.Errorf("failed to marshal ReportTestExonerationsRequest: %w", err) | |
| } | |
| uploadRequests = append(uploadRequests, UploadRequest{ | |
| Endpoint: "ReportTestExonerations", | |
| Data: data, | |
| }) | |
| } | |
| if len(artifacts) > 0 { | |
| data, err := protojson.Marshal(&sinkpb.ReportInvocationLevelArtifactsRequest{ | |
| Artifacts: artifacts, | |
| }) | |
| if err != nil { | |
| return nil, nil, fmt.Errorf("failed to marshal ReportInvocationLevelArtifactsRequest: %w", err) | |
| } | |
| uploadRequests = append(uploadRequests, UploadRequest{ | |
| Endpoint: "ReportInvocationLevelArtifacts", | |
| Data: data, | |
| }) | |
| } | |
| return uploadRequests, testsSkipped, nil | |
| } | |
| // InvocationLevelArtifacts creates resultdb artifacts for invocation-level files to be sent to ResultDB. | |
| func InvocationLevelArtifacts(outputRoot string, invocationArtifacts []string) map[string]*sinkpb.Artifact { | |
| if len(outputRoot) == 0 { | |
| outputRoot, _ = os.Getwd() | |
| } | |
| rootPath, _ := filepath.Abs(outputRoot) | |
| artifacts := map[string]*sinkpb.Artifact{} | |
| // TODO(ihuh): Remove once these are passed in through recipes. | |
| if len(invocationArtifacts) == 0 { | |
| invocationArtifacts = []string{ | |
| "infra_and_test_std_and_klog.txt", | |
| "serial_log.txt", | |
| "syslog.txt", | |
| "triage_output", | |
| } | |
| } | |
| for _, invocationArtifact := range invocationArtifacts { | |
| artifactFile := filepath.Join(rootPath, invocationArtifact) | |
| if isReadable(artifactFile) { | |
| artifacts[invocationArtifact] = &sinkpb.Artifact{ | |
| Body: &sinkpb.Artifact_FilePath{FilePath: artifactFile}, | |
| ContentType: "text/plain", | |
| } | |
| } | |
| } | |
| return artifacts | |
| } | |
| var invalidArtifactNameChars = regexp.MustCompile(`[^a-zA-Z0-9-_.\/]`) | |
| // artifactName returns a unique name to correspond to the file which | |
| // will be uploaded as a resultDB artifact. | |
| func artifactName(file string) string { | |
| return invalidArtifactNameChars.ReplaceAllString(file, "_") | |
| } | |
| // testMetadataFields holds the BugComponent and Location shared across all | |
| // results within a single TestDetails. | |
| type testMetadataFields struct { | |
| bugComponent *resultpb.BugComponent | |
| location *resultpb.TestLocation | |
| } | |
| func resolveTestMetadataFields(testDetail *runtests.TestDetails) testMetadataFields { | |
| var m testMetadataFields | |
| if testDetail.Metadata.ComponentID > 0 { | |
| m.bugComponent = &resultpb.BugComponent{ | |
| System: &resultpb.BugComponent_IssueTracker{ | |
| IssueTracker: &resultpb.IssueTrackerComponent{ | |
| ComponentId: int64(testDetail.Metadata.ComponentID), | |
| }, | |
| }, | |
| } | |
| } | |
| label := testDetail.SourceLabel | |
| if label == "" { | |
| label = testDetail.GNLabel | |
| } | |
| if len(label) > 0 { | |
| loc, err := parseTestLocation(label) | |
| if err != nil { | |
| log.Printf("[Warn] Failed to parse test location for %q: %v", label, err) | |
| } else { | |
| m.location = loc | |
| } | |
| } | |
| return m | |
| } | |
| func (m testMetadataFields) apply(r *sinkpb.TestResult, displayName string) { | |
| r.TestMetadata = &resultpb.TestMetadata{ | |
| Name: displayName, | |
| BugComponent: m.bugComponent, | |
| Location: m.location, | |
| } | |
| } | |
| // set the TestMetadata on TestResult | |
| func setTestMetadata(r *sinkpb.TestResult, testDetail *runtests.TestDetails, displayName string) { | |
| resolveTestMetadataFields(testDetail).apply(r, displayName) | |
| } | |
| // parseTestLocation parses a test's source label and resolves | |
| // the source code directory or file path within the repository, returning a | |
| // resultpb.TestLocation suitable for ResultDB / Milo Gitiles source linking. | |
| func parseTestLocation(label string) (*resultpb.TestLocation, error) { | |
| // Strip toolchain suffix (e.g. "(//build/toolchain:x64)") from label. | |
| cleaned := strings.TrimSpace(label) | |
| if cleaned == "" { | |
| return nil, fmt.Errorf("label is empty") | |
| } | |
| if idx := strings.LastIndex(cleaned, "("); idx != -1 { | |
| candidate := cleaned[idx:] | |
| if strings.HasPrefix(candidate, "(//") || strings.HasPrefix(candidate, "(@//") || strings.HasPrefix(candidate, "(@@//") { | |
| if !strings.HasSuffix(candidate, ")") { | |
| return nil, fmt.Errorf("malformed toolchain in label %q", label) | |
| } | |
| cleaned = strings.TrimSpace(cleaned[:idx]) | |
| if cleaned == "" { | |
| return nil, fmt.Errorf("label %q is empty after stripping toolchain", label) | |
| } | |
| } else if strings.HasSuffix(cleaned, ")") { | |
| // If the label ends with a closing parenthesis but does not look like a valid toolchain, | |
| // treat it as a malformed toolchain or unmatched parenthesis. | |
| return nil, fmt.Errorf("malformed toolchain in label %q", label) | |
| } | |
| } else if strings.HasSuffix(cleaned, ")") { | |
| return nil, fmt.Errorf("malformed toolchain in label %q", label) | |
| } | |
| // Validate label does not contain whitespace. | |
| if strings.ContainsAny(cleaned, " \t\n\r") { | |
| return nil, fmt.Errorf("label %q contains whitespace", label) | |
| } | |
| // Normalize Bazel/GN prefixes so the label starts with "//". | |
| for _, prefix := range []string{"@@//", "@//"} { | |
| if strings.HasPrefix(cleaned, prefix) { | |
| cleaned = "//" + strings.TrimPrefix(cleaned, prefix) | |
| break | |
| } | |
| } | |
| if !strings.HasPrefix(cleaned, "//") { | |
| // Support and normalize single leading slash (e.g. "/src/...") to "//" defensively | |
| // to tolerate tool or input variations, although GN and Bazel labels standardly begin with "//". | |
| if strings.HasPrefix(cleaned, "/") { | |
| cleaned = "/" + cleaned | |
| } else { | |
| return nil, fmt.Errorf("label %q must start with '//'", label) | |
| } | |
| } | |
| // Extract the source file or directory path from the cleaned label. | |
| parts := strings.SplitN(cleaned, ":", 2) | |
| dirPart := strings.TrimRight(parts[0], "/") | |
| if dirPart == "" { | |
| dirPart = "//" | |
| } | |
| fileName := dirPart | |
| if len(parts) == 2 { | |
| targetPart := parts[1] | |
| ext := path.Ext(targetPart) | |
| // If the target has a file extension and is not a compiled component manifest (.cm), | |
| // treat it as a file inside the package directory. We link to the directory for | |
| // .cm targets because the manifest source (usually .cml) resides in that package | |
| // directory, whereas the .cm itself is a build artifact. | |
| if ext != "" && ext != ".cm" { | |
| if dirPart == "//" { | |
| fileName = "//" + targetPart | |
| } else { | |
| fileName = fmt.Sprintf("%s/%s", dirPart, targetPart) | |
| } | |
| } | |
| } | |
| // Clean path using the path package (not filepath, since build labels always use forward slashes). | |
| // Strip leading slashes before cleaning so that path.Clean treats it as an unrooted relative path. | |
| // This preserves any leading ".." components so we can detect paths that escape the repository root, | |
| // and avoids Go's path.Clean collapsing double leading slashes into a single slash. | |
| cleanRel := path.Clean(strings.TrimLeft(fileName, "/")) | |
| if cleanRel == ".." || strings.HasPrefix(cleanRel, "../") { | |
| return nil, fmt.Errorf("label path %q escapes repository root: %s", label, cleanRel) | |
| } | |
| repoPath := "//" | |
| if cleanRel != "." && cleanRel != "" { | |
| repoPath = "//" + cleanRel | |
| } | |
| // For vendor/google, route to turquoise-internal.googlesource.com/vendor/google | |
| // and make path relative to that repository root. | |
| repo := DefaultRepo | |
| filePath := repoPath | |
| if strings.HasPrefix(repoPath, VendorGooglePathPrefix+"/") { | |
| repo = VendorGoogleRepo | |
| filePath = "//" + strings.TrimPrefix(repoPath, VendorGooglePathPrefix+"/") | |
| } else if repoPath == VendorGooglePathPrefix { | |
| repo = VendorGoogleRepo | |
| filePath = "//" | |
| } | |
| // Validate field lengths according to ResultDB TestLocation specifications. | |
| if len(repo) > MaxLocationRepoLength { | |
| return nil, fmt.Errorf("repo %q exceeds max length %d bytes (%d bytes)", repo, MaxLocationRepoLength, len(repo)) | |
| } | |
| if len(filePath) > MaxLocationFileNameLength { | |
| return nil, fmt.Errorf("file path %q exceeds max length %d bytes (%d bytes)", filePath, MaxLocationFileNameLength, len(filePath)) | |
| } | |
| // Keeping Line defaulting to 0 denotes an unknown or unspecified line number in ResultDB (which uses 1-based line numbers). | |
| return &resultpb.TestLocation{ | |
| Repo: repo, | |
| FileName: filePath, | |
| }, nil | |
| } | |
| // testCaseToResultSink converts TestCaseResult defined in //tools/testing/runtests/runtests.go | |
| // to ResultSink's TestResult. A testcase will not be converted if test result cannot be | |
| // mapped to result_sink.Status. | |
| func testCaseToResultSink(testCases []runtests.TestCaseResult, tags []*resultpb.StringPair, testDetail *runtests.TestDetails, outputRoot string) ([]*sinkpb.TestResult, []*sinkpb.TestExoneration, []string) { | |
| testResults := make([]*sinkpb.TestResult, 0, len(testCases)+2) | |
| var testExonerations []*sinkpb.TestExoneration | |
| var testsSkipped []string | |
| // Ignore the failure reason kind and error. We only check the top-level test status | |
| // to see if it passed, which would mean that a failed result for a test case is | |
| // expected and thus should be reported as a passed result. | |
| testStatus, _, _ := resultDBStatus(testDetail.Status) | |
| scheme := DetermineScheme(testDetail) | |
| testIdentifier := caseIdentifierResolver(scheme, testDetail) | |
| metadataFields := resolveTestMetadataFields(testDetail) | |
| startTime := timestamppb.New(testDetail.StartTime) | |
| // Within a single TestDetails, almost all test cases share only 1-2 unique | |
| // (Format, Tags) combinations (for example, Fuchsia target tests attach | |
| // test_outcome and use_ffx tags to every case), so their Properties and | |
| // Tags are identical. Cache a small slice of unique combinations to avoid | |
| // rebuilding a structpb.Struct and calling proto.Size on every test case. | |
| type cachedPropsAndTags struct { | |
| format string | |
| customTags []build.TestTag | |
| properties *structpb.Struct | |
| tags []*resultpb.StringPair | |
| } | |
| const maxCachedPropsEntries = 8 | |
| var propCache []cachedPropsAndTags | |
| for _, testCase := range testCases { | |
| testID := LegacyTestCaseID(testDetail.Name, testCase.SuiteName, testCase.CaseName) | |
| if len(testID) > MaxTestIDLength { | |
| log.Printf("[ERROR] Skip uploading to ResultDB due to test_id exceeding %d bytes max limit: %q", MaxTestIDLength, testID) | |
| testsSkipped = append(testsSkipped, testID) | |
| continue | |
| } | |
| var properties *structpb.Struct | |
| var testCaseTags []*resultpb.StringPair | |
| foundCached := false | |
| for i := range propCache { | |
| if propCache[i].format == testCase.Format && slices.Equal(propCache[i].customTags, testCase.Tags) { | |
| properties, testCaseTags = propCache[i].properties, propCache[i].tags | |
| foundCached = true | |
| break | |
| } | |
| } | |
| if !foundCached { | |
| properties, testCaseTags = testCaseProperties(testCase.Format, testCase.Tags, testDetail, tags) | |
| if len(propCache) < maxCachedPropsEntries { | |
| propCache = append(propCache, cachedPropsAndTags{ | |
| format: testCase.Format, | |
| customTags: slices.Clone(testCase.Tags), | |
| properties: properties, | |
| tags: slices.Clip(testCaseTags), | |
| }) | |
| } | |
| } | |
| fineName, caseName := structuredCaseIdentifier(testIdentifier, &testCase) | |
| testIDStructured := &sinkpb.TestIdentifier{ | |
| FineName: fineName, | |
| CaseNameComponents: []string{caseName}, | |
| } | |
| r := sinkpb.TestResult{ | |
| TestId: testID, | |
| TestIdStructured: testIDStructured, | |
| Tags: testCaseTags, | |
| Properties: properties, | |
| } | |
| testCaseStatus, testCaseFailureReasonKind, err := resultDBStatus(testCase.Status) | |
| if err != nil { | |
| log.Printf("[Warn] Skip uploading testcase: %s to ResultDB due to error: %v", testID, err) | |
| continue | |
| } | |
| if testStatus != resultpb.TestResult_PASSED && testCaseStatus == resultpb.TestResult_FAILED { | |
| r.FailureReason = toResultDBFailureReason(testCase.FailureReason, testCaseFailureReasonKind) | |
| } else if testCaseStatus == resultpb.TestResult_SKIPPED { | |
| r.SkippedReason = &resultpb.SkippedReason{Kind: resultpb.SkippedReason_DISABLED_AT_DECLARATION} | |
| } else if testStatus == resultpb.TestResult_PASSED { | |
| testCaseStatus = testStatus | |
| } | |
| r.StatusV2 = testCaseStatus | |
| r.StartTime = startTime | |
| if testCase.Duration > 0 { | |
| r.Duration = durationpb.New(testCase.Duration) | |
| } | |
| if len(testCase.OutputFiles) > 0 { | |
| r.Artifacts = make(map[string]*sinkpb.Artifact, len(testCase.OutputFiles)) | |
| for _, of := range testCase.OutputFiles { | |
| outputFile := filepath.Join(outputRoot, testDetail.OutputDir, of) | |
| if isReadable(outputFile) { | |
| r.Artifacts[artifactName(of)] = &sinkpb.Artifact{ | |
| Body: &sinkpb.Artifact_FilePath{FilePath: outputFile}, | |
| } | |
| } else { | |
| log.Printf("[Warn] outputFile: %s is not readable, skip.", outputFile) | |
| } | |
| } | |
| } | |
| metadataFields.apply(&r, testCase.DisplayName) | |
| testResults = append(testResults, &r) | |
| if testCase.Status == runtests.TestExonerated { | |
| testExonerations = append(testExonerations, &sinkpb.TestExoneration{ | |
| TestId: testID, | |
| TestIdStructured: testIDStructured, | |
| ExplanationHtml: truncateEscapedHTML( | |
| cmp.Or(testCase.ExonerationExplanation, fmt.Sprintf("Test case %s was exonerated in the test summary.", testCase.CaseName)), | |
| MaxExplanationHTMLLength, | |
| ), | |
| Reason: resultpb.ExonerationReason_OCCURS_ON_OTHER_CLS, | |
| }) | |
| } | |
| } | |
| return testResults, testExonerations, testsSkipped | |
| } | |
| // testIdentifierFromGo returns the fineName and caseName for Go tests. | |
| // In Go, ResultDB scheme "go" requires fine_name to be the Go package name. | |
| // Go tests are package-level functions with optional dynamic subtest arguments, | |
| // not a static directory hierarchy. Using DisplayName (e.g. "TestParent/subtest" | |
| // or "TestFoo") as a single opaque case component avoids artificial splitting on "/", | |
| // preserving subtest paths, regexes, and URLs verbatim. | |
| func testIdentifierFromGo(testCase *runtests.TestCaseResult, goPkg string) (string, string) { | |
| caseName := testCase.DisplayName | |
| if caseName == "" { | |
| caseName = testCase.CaseName | |
| } | |
| return goPkg, caseName | |
| } | |
| // rustCrateRootModule is the fine_name of Rust tests defined at the crate root, | |
| // using "crate", the official Rust keyword for the crate root module. | |
| const rustCrateRootModule = "crate" | |
| // testIdentifierFromRust returns the fineName and caseName for Rust tests. | |
| // In Rust, test functions defined at the crate root outside of a submodule have an empty | |
| // SuiteName. ResultDB scheme "rust" requires a fine_name (Rust Module). | |
| // Use "crate", the official Rust keyword for the crate root module. | |
| func testIdentifierFromRust(testCase *runtests.TestCaseResult) (string, string) { | |
| fineName := testCase.SuiteName | |
| if fineName == "" { | |
| fineName = rustCrateRootModule | |
| } | |
| return fineName, testCase.CaseName | |
| } | |
| // testIdentifierFromCase returns the fineName and caseName using the default mapping. | |
| func testIdentifierFromCase(testCase *runtests.TestCaseResult) (string, string) { | |
| return testCase.SuiteName, testCase.CaseName | |
| } | |
| // testIdentifierFromFlat returns the fineName and caseName for tests under the "flat" scheme. | |
| // In ResultDB, scheme "flat" defines only the test case level; coarse_name and | |
| // fine_name must be empty. | |
| func testIdentifierFromFlat(testCase *runtests.TestCaseResult) (string, string) { | |
| return "", testCase.CaseName | |
| } | |
| // ResultDB reserves characters <= ',' (ASCII U+0020 to U+002C, such as ' ', | |
| // '!', '"', '#', etc.) as leading characters for fine_name and case_name | |
| // components, rejecting them with 400 Bad Request. Wrapping any component | |
| // starting with a reserved character in brackets preserves the original name | |
| // verbatim while satisfying ResultDB constraints. | |
| func sanitizeIdentifier(s string) string { | |
| if len(s) > 0 && s[0] <= ',' { | |
| return fmt.Sprintf("[%s]", s) | |
| } | |
| return s | |
| } | |
| func caseIdentifierResolver(scheme string, testDetail *runtests.TestDetails) func(*runtests.TestCaseResult) (string, string) { | |
| switch scheme { | |
| case "flat": | |
| return testIdentifierFromFlat | |
| case "go": | |
| label := testDetail.SourceLabel | |
| if label == "" { | |
| label = testDetail.GNLabel | |
| } | |
| if idx := strings.Index(label, "("); idx != -1 { | |
| label = label[:idx] | |
| } | |
| label = strings.TrimLeft(label, "@/") | |
| dir, _, _ := strings.Cut(label, ":") | |
| goPkg := testDetail.Name | |
| if dir != "" && dir != "." { | |
| goPkg = dir | |
| } | |
| return func(testCase *runtests.TestCaseResult) (string, string) { | |
| return testIdentifierFromGo(testCase, goPkg) | |
| } | |
| case "rust": | |
| return testIdentifierFromRust | |
| default: | |
| return testIdentifierFromCase | |
| } | |
| } | |
| func structuredCaseIdentifier(resolver func(*runtests.TestCaseResult) (string, string), testCase *runtests.TestCaseResult) (string, string) { | |
| fineName, caseName := resolver(testCase) | |
| return sanitizeIdentifier(fineName), sanitizeIdentifier(caseName) | |
| } | |
| // StructuredCaseIdentifier returns the sanitized structured test identifier | |
| // (fine_name, case_name) for testCase within testDetail under scheme. | |
| func StructuredCaseIdentifier(scheme string, testDetail *runtests.TestDetails, testCase *runtests.TestCaseResult) (string, string) { | |
| return structuredCaseIdentifier(caseIdentifierResolver(scheme, testDetail), testCase) | |
| } | |
| // StructuredDetailIdentifier returns the structured test identifier | |
| // (fine_name, case_name) for a caseless top-level test under scheme. | |
| func StructuredDetailIdentifier(scheme string) (string, string) { | |
| switch scheme { | |
| case "single": | |
| return "", FixtureCaseName | |
| case "flat": | |
| return "", "case" | |
| default: | |
| return "test", "case" | |
| } | |
| } | |
| // LegacyTestCaseID returns the legacy flat ResultDB test_id for a test case. | |
| // TODO(b/527958872): Remove once Fuchsia test results complete the migration to | |
| // structured test identifiers. | |
| func LegacyTestCaseID(testName, suiteName, caseName string) string { | |
| return fmt.Sprintf("%s/%s:%s", testName, suiteName, caseName) | |
| } | |
| // testDetailsToResultSink converts TestDetail defined in /tools/testing/runtests/runtests.go | |
| // to ResultSink's TestResult. Returns an error if a test result cannot be mapped to | |
| // result_sink.Status | |
| func testDetailsToResultSink(tags []*resultpb.StringPair, testDetail *runtests.TestDetails, outputRoot string) (*sinkpb.TestResult, *sinkpb.TestExoneration, string, error) { | |
| if len(testDetail.Name) > MaxTestIDLength { | |
| log.Printf("[ERROR] Skip uploading to ResultDB due to test_id exceeding %d bytes max limit: %q", MaxTestIDLength, testDetail.Name) | |
| return nil, nil, testDetail.Name, fmt.Errorf("The test name exceeds %d bytes max limit: %q ", MaxTestIDLength, testDetail.Name) | |
| } | |
| properties, testTags := testDetailProperties(testDetail, tags) | |
| var testIDStructured *sinkpb.TestIdentifier | |
| if len(testDetail.Cases) == 0 { | |
| fineName, caseName := StructuredDetailIdentifier(DetermineScheme(testDetail)) | |
| testIDStructured = &sinkpb.TestIdentifier{ | |
| FineName: fineName, | |
| CaseNameComponents: []string{caseName}, | |
| } | |
| } | |
| r := sinkpb.TestResult{ | |
| TestId: testDetail.Name, | |
| TestIdStructured: testIDStructured, | |
| Tags: testTags, | |
| Properties: properties, | |
| } | |
| testStatus, failureReasonKind, err := resultDBStatus(testDetail.Status) | |
| if err != nil { | |
| log.Printf("[Warn] Skip uploading test target: %s to ResultDB due to error: %v", testDetail.Name, err) | |
| return nil, nil, "", err | |
| } | |
| r.StatusV2 = testStatus | |
| if testStatus == resultpb.TestResult_FAILED { | |
| r.FailureReason = toResultDBFailureReason(testDetail.FailureReason, failureReasonKind) | |
| } else if testStatus == resultpb.TestResult_SKIPPED { | |
| r.SkippedReason = &resultpb.SkippedReason{Kind: resultpb.SkippedReason_OTHER, ReasonMessage: "skipped because unaffected"} | |
| } | |
| r.StartTime = timestamppb.New(testDetail.StartTime) | |
| if testDetail.DurationMillis > 0 { | |
| r.Duration = durationpb.New(time.Duration(testDetail.DurationMillis) * time.Millisecond) | |
| } | |
| r.Artifacts = make(map[string]*sinkpb.Artifact) | |
| for _, of := range testDetail.OutputFiles { | |
| outputFile := filepath.Join(outputRoot, testDetail.OutputDir, of) | |
| if isReadable(outputFile) { | |
| r.Artifacts[artifactName(of)] = &sinkpb.Artifact{ | |
| Body: &sinkpb.Artifact_FilePath{FilePath: outputFile}, | |
| } | |
| } else { | |
| log.Printf("[Warn] outputFile: %s is not readable, skip.", outputFile) | |
| } | |
| } | |
| setTestMetadata(&r, testDetail, "") | |
| if testDetail.Status == runtests.TestExonerated { | |
| return &r, &sinkpb.TestExoneration{ | |
| TestId: testDetail.Name, | |
| TestIdStructured: testIDStructured, | |
| ExplanationHtml: truncateEscapedHTML( | |
| cmp.Or(testDetail.ExonerationExplanation, fmt.Sprintf("Test target %s was exonerated in the test summary.", testDetail.Name)), | |
| MaxExplanationHTMLLength, | |
| ), | |
| Reason: resultpb.ExonerationReason_OCCURS_ON_OTHER_CLS, | |
| }, "", nil | |
| } | |
| return &r, nil, "", nil | |
| } | |
| // testDetailProperties returns the properties and the legacy tags of a | |
| // top-level test result. | |
| // | |
| // buildTags is the build and environment metadata that applies to every test | |
| // result, such as the builder and the board. | |
| func testDetailProperties(testDetail *runtests.TestDetails, buildTags []*resultpb.StringPair) (*structpb.Struct, []*resultpb.StringPair) { | |
| fields := map[string]any{ | |
| "gn_label": testDetail.GNLabel, | |
| // Most consumers should use `source_label` rather than `gn_label` | |
| // since it better corresponds to the location of the test's source | |
| // code for Bazel tests. | |
| "source_label": testDetail.SourceLabel, | |
| "test_case_count": len(testDetail.Cases), | |
| "affected": testDetail.Affected, | |
| } | |
| // ResultDB already models the suite and test case hierarchy, so | |
| // is_top_level_test is reported as a legacy tag only. See b/527958757, | |
| // which stops reporting suite-level results altogether. | |
| tags := []*resultpb.StringPair{ | |
| {Key: "is_top_level_test", Value: "true"}, | |
| } | |
| // The tags are an unordered list, so the properties can be flattened into | |
| // them in map order. | |
| for key, value := range fields { | |
| tags = append(tags, &resultpb.StringPair{Key: key, Value: fmt.Sprint(value)}) | |
| } | |
| addCommonProperties(fields, &tags, testDetail.Metadata.Owners, testDetail.Tags, buildTags) | |
| return newProperties(fields), tags | |
| } | |
| // testCaseProperties returns the properties and the legacy tags of a test case | |
| // result. Accepting only format and caseTags (rather than the full | |
| // TestCaseResult) ensures callers can safely cache results by (format, | |
| // caseTags) within a TestDetails. | |
| // | |
| // buildTags is the build and environment metadata that applies to every test | |
| // result, such as the builder and the board. | |
| func testCaseProperties(format string, caseTags []build.TestTag, testDetail *runtests.TestDetails, buildTags []*resultpb.StringPair) (*structpb.Struct, []*resultpb.StringPair) { | |
| fields := map[string]any{ | |
| "gn_label": testDetail.GNLabel, | |
| // Most consumers should use `source_label` rather than `gn_label` | |
| // since it better corresponds to the location of the test's source | |
| // code for Bazel tests. | |
| "source_label": testDetail.SourceLabel, | |
| "affected": testDetail.Affected, | |
| "format": format, | |
| } | |
| // See the comment on is_top_level_test in testDetailProperties. | |
| tags := []*resultpb.StringPair{ | |
| {Key: "is_test_case", Value: "true"}, | |
| } | |
| for key, value := range fields { | |
| tags = append(tags, &resultpb.StringPair{Key: key, Value: fmt.Sprint(value)}) | |
| } | |
| customTags := make([]build.TestTag, 0, len(testDetail.Tags)+len(caseTags)) | |
| for _, tag := range testDetail.Tags { | |
| // Suite-level stability exoneration tags describe the aggregate suite | |
| // outcome; failing test cases carry their own case-level tags in | |
| // caseTags, and passing/skipped test cases should not inherit them. | |
| if !strings.HasPrefix(tag.Key, "luci_exoneration_v2_") { | |
| customTags = append(customTags, tag) | |
| } | |
| } | |
| customTags = append(customTags, caseTags...) | |
| addCommonProperties(fields, &tags, testDetail.Metadata.Owners, customTags, buildTags) | |
| return newProperties(fields), tags | |
| } | |
| // addCommonProperties adds the properties and the legacy tags shared by test | |
| // and test case results to fields and tags. | |
| // | |
| // TODO(b/527958920): Remove the tags once downstream consumers read the | |
| // equivalent values from the test result properties instead. The known | |
| // consumers are the PLX queries over fuchsia-infra.resultdb.ci, which read the | |
| // gn_label tag. | |
| func addCommonProperties(fields map[string]any, tags *[]*resultpb.StringPair, owners []string, customTags []build.TestTag, buildTags []*resultpb.StringPair) { | |
| // The properties hold every owner, whereas the legacy tag holds at most 5 | |
| // of them, comma-joined. The overall properties size limit already bounds | |
| // the payload. | |
| if len(owners) > 0 { | |
| ownerValues := make([]any, 0, len(owners)) | |
| for _, owner := range owners { | |
| ownerValues = append(ownerValues, owner) | |
| } | |
| fields["owners"] = ownerValues | |
| truncatedOwners := owners | |
| if len(truncatedOwners) > 5 { | |
| truncatedOwners = truncatedOwners[:5] | |
| } | |
| *tags = append(*tags, &resultpb.StringPair{Key: "owners", Value: strings.Join(truncatedOwners, ",")}) | |
| } | |
| // The build metadata and the custom test tags hold free-form keys that are | |
| // not known ahead of time. They are nested under their own key in the | |
| // properties to keep them from colliding with the properties that this tool | |
| // controls, and flattened into the legacy tags. | |
| if len(buildTags) > 0 { | |
| buildFields := make(map[string]any, len(buildTags)) | |
| for _, tag := range buildTags { | |
| if tag.Key == "" { | |
| log.Printf("[Warn] Skip reporting build tag with an empty key, value: %q", tag.Value) | |
| continue | |
| } | |
| buildFields[tag.Key] = tag.Value | |
| } | |
| if len(buildFields) > 0 { | |
| fields["build"] = buildFields | |
| } | |
| *tags = append(*tags, buildTags...) | |
| } | |
| if len(customTags) > 0 { | |
| tagFields := make(map[string]any, len(customTags)) | |
| for _, tag := range customTags { | |
| if tag.Key == "" { | |
| log.Printf("[Warn] Skip reporting test tag with an empty key, value: %q", tag.Value) | |
| continue | |
| } | |
| // A JSON object can only hold a single value per key, so the last | |
| // value of a repeated key wins, whereas the tags keep every value. | |
| tagFields[tag.Key] = tag.Value | |
| *tags = append(*tags, &resultpb.StringPair{Key: tag.Key, Value: tag.Value}) | |
| } | |
| if len(tagFields) > 0 { | |
| fields["tags"] = tagFields | |
| } | |
| } | |
| } | |
| // newProperties converts fields into the test result properties, and enforces | |
| // ResultDB's size limit. | |
| func newProperties(fields map[string]any) *structpb.Struct { | |
| properties, err := structpb.NewStruct(fields) | |
| if err != nil { | |
| log.Printf("[Warn] Skip reporting properties %v due to error: %v", fields, err) | |
| return nil | |
| } | |
| if proto.Size(properties) > MaxPropertiesSize { | |
| log.Printf("[ERROR] Skip reporting properties due to exceeding the %d bytes max limit.", MaxPropertiesSize) | |
| return nil | |
| } | |
| return properties | |
| } | |
| func resultDBStatus(result runtests.TestStatus) (resultpb.TestResult_Status, resultpb.FailureReason_Kind, error) { | |
| switch result { | |
| case runtests.TestSuccess: | |
| return resultpb.TestResult_PASSED, resultpb.FailureReason_KIND_UNSPECIFIED, nil | |
| case runtests.TestFailure: | |
| return resultpb.TestResult_FAILED, resultpb.FailureReason_ORDINARY, nil | |
| case runtests.TestSkipped: | |
| return resultpb.TestResult_SKIPPED, resultpb.FailureReason_KIND_UNSPECIFIED, nil | |
| case runtests.TestAborted: | |
| return resultpb.TestResult_FAILED, resultpb.FailureReason_TIMEOUT, nil | |
| case runtests.TestInfraFailure: | |
| return resultpb.TestResult_FAILED, resultpb.FailureReason_CRASH, nil | |
| case runtests.TestExonerated: | |
| return resultpb.TestResult_FAILED, resultpb.FailureReason_ORDINARY, nil | |
| } | |
| return resultpb.TestResult_STATUS_UNSPECIFIED, resultpb.FailureReason_KIND_UNSPECIFIED, fmt.Errorf("cannot map Result: %s to result_sink test_result status", result) | |
| } | |
| func toResultDBFailureReason(fr *runtests.FailureReason, defaultKind resultpb.FailureReason_Kind) *resultpb.FailureReason { | |
| res := &resultpb.FailureReason{ | |
| Kind: defaultKind, | |
| } | |
| if fr == nil || len(fr.Errors) == 0 { | |
| return res | |
| } | |
| // Copy over errors to the resultpb.FailureReason. | |
| res.Errors = make([]*resultpb.FailureReason_Error, 0, len(fr.Errors)) | |
| for i, e := range fr.Errors { | |
| msg := truncateString(e.Message, MaxFailureReasonLength) | |
| if msg == "" { | |
| continue | |
| } | |
| pbErr := &resultpb.FailureReason_Error{ | |
| Message: msg, | |
| } | |
| res.Errors = append(res.Errors, pbErr) | |
| if proto.Size(res) > MaxFailureReasonTotalSize { | |
| res.Errors = res.Errors[:len(res.Errors)-1] | |
| res.TruncatedErrorsCount = int32(len(fr.Errors) - i) | |
| break | |
| } | |
| } | |
| return res | |
| } | |
| func isReadable(p string) bool { | |
| if len(p) == 0 { | |
| return false | |
| } | |
| info, err := os.Stat(p) | |
| if err != nil { | |
| return false | |
| } | |
| if info.IsDir() { | |
| return false | |
| } | |
| f, err := os.Open(p) | |
| if err != nil { | |
| return false | |
| } | |
| _ = f.Close() | |
| return true | |
| } | |
| func truncateString(str string, maxLength int) string { | |
| if len(str) <= maxLength { | |
| return str | |
| } | |
| // We want to append "..." to maxLength, which takes up 3 spaces. If maxLength is less than that, just return empty. | |
| if maxLength <= 3 { | |
| return "" | |
| } | |
| runes := []rune(str) | |
| byteCount := 0 | |
| for _, char := range runes { | |
| if byteCount+len(string(char)) > (maxLength - 3) { | |
| if byteCount == 0 { | |
| return "" | |
| } | |
| return str[:byteCount] + "..." | |
| } | |
| byteCount = byteCount + len(string(char)) | |
| } | |
| return str | |
| } |