blob: e00f8b8a6405b889045202f1519903fbd4b04c50 [file]
// Copyright 2023 The Fuchsia Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
package resultdb
import (
"bytes"
"encoding/json"
"fmt"
"io"
"net/http"
"os"
"path/filepath"
resultpb "go.chromium.org/luci/resultdb/proto/v1"
sinkpb "go.chromium.org/luci/resultdb/sink/proto/v1"
"google.golang.org/protobuf/encoding/protojson"
"google.golang.org/protobuf/proto"
"go.fuchsia.dev/fuchsia/tools/testing/runtests"
)
// luciContext corresponds to the schema of the file identified by the
// LUCI_CONTEXT env var. See
// https://crsrc.org/i/go/src/go.chromium.org/luci/lucictx/sections.proto for
// the whole structure.
type luciContext struct {
ResultDB resultDB `json:"resultdb"`
ResultSink resultSink `json:"result_sink"`
}
// resultSink holds the result_sink information parsed from LUCI_CONTEXT.
type resultSink struct {
AuthToken string `json:"auth_token"`
ResultSinkAddr string `json:"address"`
}
type Client struct {
resultSink *resultSink
httpClient *http.Client
semaphore chan struct{}
}
type resultDB struct {
CurrentInvocation resultDBInvocation `json:"current_invocation"`
}
type resultDBInvocation struct {
Name string `json:"name"`
}
func (c *Client) sendMessage(req proto.Message, endpoint string) error {
data, err := protojson.Marshal(req)
if err != nil {
return err
}
return c.SendData(data, endpoint)
}
// ReportSummaries parses each summary.json file and streams test results and
// exonerations to ResultSink in batches of MaxBatchSize. Streaming per test
// entry avoids holding all converted TestResult protobufs and unmarshaled
// TestCaseResult slices in memory at once for large multiplied test runs.
func (c *Client) ReportSummaries(summaries []string, tags []*resultpb.StringPair, outputRoot string) ([]string, error) {
if len(outputRoot) == 0 {
outputRoot, _ = os.Getwd()
}
rootPath, _ := filepath.Abs(outputRoot)
var pendingResults []*sinkpb.TestResult
var pendingExonerations []*sinkpb.TestExoneration
var allTestsSkipped []string
flushResults := func(batch []*sinkpb.TestResult) error {
return c.sendMessage(&sinkpb.ReportTestResultsRequest{TestResults: batch}, "ReportTestResults")
}
flushExonerations := func(batch []*sinkpb.TestExoneration) error {
return c.sendMessage(&sinkpb.ReportTestExonerationsRequest{TestExonerations: batch}, "ReportTestExonerations")
}
for _, summaryFile := range summaries {
if err := forEachSummaryTest(summaryFile, func(test *runtests.TestDetails) error {
results, exonerations, skipped := testToResultSink(test, tags, rootPath)
allTestsSkipped = append(allTestsSkipped, skipped...)
pendingResults = append(pendingResults, results...)
for len(pendingResults) >= MaxBatchSize {
if err := flushResults(pendingResults[:MaxBatchSize:MaxBatchSize]); err != nil {
return err
}
clear(pendingResults[:MaxBatchSize])
pendingResults = pendingResults[MaxBatchSize:]
}
pendingExonerations = append(pendingExonerations, exonerations...)
for len(pendingExonerations) >= MaxBatchSize {
if err := flushExonerations(pendingExonerations[:MaxBatchSize:MaxBatchSize]); err != nil {
return err
}
clear(pendingExonerations[:MaxBatchSize])
pendingExonerations = pendingExonerations[MaxBatchSize:]
}
return nil
}); err != nil {
return nil, err
}
}
if len(pendingResults) > 0 {
if err := flushResults(pendingResults); err != nil {
return nil, err
}
}
if len(pendingExonerations) > 0 {
if err := flushExonerations(pendingExonerations); err != nil {
return nil, err
}
}
return allTestsSkipped, nil
}
func (c *Client) ReportInvocationLevelArtifacts(outputRoot string, invocationArtifacts []string) error {
invocationRequest := &sinkpb.ReportInvocationLevelArtifactsRequest{
Artifacts: InvocationLevelArtifacts(outputRoot, invocationArtifacts),
}
return c.sendMessage(invocationRequest, "ReportInvocationLevelArtifacts")
}
// SendData sends a raw JSON payload to the specified endpoint on ResultSink.
func (c *Client) SendData(data []byte, endpoint string) error {
<-c.semaphore
defer func() { c.semaphore <- struct{}{} }()
url := fmt.Sprintf("http://%s/prpc/luci.resultsink.v1.Sink/%s", c.resultSink.ResultSinkAddr, endpoint)
req, err := http.NewRequest("POST", url, bytes.NewReader(data))
if err != nil {
return err
}
// ResultSink HTTP authorization scheme is documented at
// https://fuchsia.googlesource.com/third_party/luci-go/+/HEAD/resultdb/sink/proto/v1/sink.proto#29
req.Header.Set("Authorization", "ResultSink "+c.resultSink.AuthToken)
req.Header.Set("Accept", "application/json")
req.Header.Set("Content-Type", "application/json")
resp, err := c.httpClient.Do(req)
if err != nil {
return err
}
// Drain the response body before closing so net/http reuses the persistent
// loopback TCP connection across batches.
defer func() {
_, _ = io.Copy(io.Discard, resp.Body)
_ = resp.Body.Close()
}()
if resp.StatusCode != http.StatusOK {
return fmt.Errorf("ResultDB Http Request errored with status code %s (%d)", http.StatusText(resp.StatusCode), resp.StatusCode)
}
return nil
}
func NewClient() (*Client, error) {
b, err := os.ReadFile(os.Getenv("LUCI_CONTEXT"))
if err != nil {
return nil, err
}
var ctx luciContext
if err = json.Unmarshal(b, &ctx); err != nil {
return nil, err
}
// We are clearly running inside a LUCI_CONTEXT luciexe environment but rdb
// stream was not started.
if ctx.ResultSink.AuthToken == "" || ctx.ResultSink.ResultSinkAddr == "" {
return nil, fmt.Errorf("resultdb is enabled but not resultsink for invocation. Make sure swarming is run under \"rdb stream\"")
}
client := &Client{
resultSink: &ctx.ResultSink,
httpClient: &http.Client{},
semaphore: make(chan struct{}, 64),
}
for i := 0; i < cap(client.semaphore); i++ {
client.semaphore <- struct{}{}
}
return client, nil
}