[infra][ftx] Actually launch swarming task on the launch task step.
Bug: b/258456267
Change-Id: If23f200ca71ac03c7b25f065a5af797689fd1feb
Reviewed-on: https://fuchsia-review.googlesource.com/c/infra/infra/+/864317
Commit-Queue: Vinicius Felizardo <felizardo@google.com>
Reviewed-by: Rahul Bangar <rahulbn@google.com>
diff --git a/cmd/ftxtest/common.go b/cmd/ftxtest/common.go
new file mode 100644
index 0000000..77c3e59
--- /dev/null
+++ b/cmd/ftxtest/common.go
@@ -0,0 +1,34 @@
+// Copyright 2023 The Fuchsia Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package main
+
+import (
+ "github.com/maruel/subcommands"
+
+ "go.chromium.org/luci/auth"
+ "go.chromium.org/luci/auth/client/authcli"
+)
+
+type commonFlags struct {
+ subcommands.CommandRunBase
+ authFlags authcli.Flags
+ project string
+
+ parsedAuthOpts auth.Options
+}
+
+func (c *commonFlags) Init(authOpts auth.Options) {
+ c.authFlags = authcli.Flags{}
+ c.authFlags.Register(&c.Flags, authOpts)
+}
+
+func (c *commonFlags) Parse() error {
+ var err error
+ c.parsedAuthOpts, err = c.authFlags.Options()
+ if err != nil {
+ return err
+ }
+ return nil
+}
diff --git a/cmd/ftxtest/main.go b/cmd/ftxtest/main.go
index 15e5688..bd19bd0 100644
--- a/cmd/ftxtest/main.go
+++ b/cmd/ftxtest/main.go
@@ -1,43 +1,44 @@
-// Copyright 2023 The Fuchsia Authors. All rights reserved
-// Use of this source code is governed by a BSD-style license that can
+// Copyright 2023 The Fuchsia Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
package main
import (
- "context"
- "fmt"
+ "log"
+ "os"
- "github.com/golang/protobuf/ptypes/any"
- "go.chromium.org/luci/luciexe/build"
- ftxproto "go.fuchsia.dev/infra/cmd/ftxtest/proto"
+ "github.com/maruel/subcommands"
+ "go.chromium.org/luci/auth"
+ "go.chromium.org/luci/auth/client/authcli"
+ "go.chromium.org/luci/client/versioncli"
+ "go.chromium.org/luci/hardcoded/chromeinfra"
)
-func main() {
- input := &ftxproto.InputProperties{}
- var writeOutputProps func(*any.Any)
- build.Main(input, &writeOutputProps, nil, func(ctx context.Context, extraArgs []string, state *build.State) error {
- _, _, err := LaunchTaskStep(ctx)
- if err != nil {
- return fmt.Errorf("LaunchTaskStep: %v", err)
- }
- return nil
- })
+const (
+ // Version must be updated on functional change (behavior, arguments, supported commands).
+ version = "0.0.1"
+)
+
+func getApplication(defaultAuthOpts auth.Options) *subcommands.DefaultApplication {
+ defaultAuthOpts.Scopes = []string{
+ "https://www.googleapis.com/auth/userinfo.email",
+ }
+ return &subcommands.DefaultApplication{
+ Name: "ftx-test",
+ Title: "Generic test execution for buildbucket.",
+ Commands: []*subcommands.Command{
+ cmdRun(defaultAuthOpts),
+ authcli.SubcommandInfo(defaultAuthOpts, "whoami", false),
+ authcli.SubcommandLogin(defaultAuthOpts, "login", false),
+ authcli.SubcommandLogout(defaultAuthOpts, "logout", false),
+ versioncli.CmdVersion(version),
+ subcommands.CmdHelp,
+ },
+ }
}
-func LaunchTaskStep(ctx context.Context) (*Swarming, string, error) {
- step, ctx := build.StartStep(ctx, "Launch Swarming Task")
- swarming, err := NewSwarming(ctx)
- if err != nil {
- step.End(err)
- return nil, "", fmt.Errorf("NewSwarming: %v", err)
- }
- taskId, err := swarming.LaunchTask()
- if err != nil {
- step.End(err)
- return nil, "", fmt.Errorf("LaunchTask: %v", err)
- }
- md := fmt.Sprintf("* [swarming task](https://chrome-swarming.appspot.com/task?id=%s)", taskId)
- step.SetSummaryMarkdown(md)
- step.End(nil)
- return swarming, taskId, nil
+func main() {
+ log.SetFlags(log.Lmicroseconds)
+ app := getApplication(chromeinfra.DefaultAuthOptions())
+ os.Exit(subcommands.Run(app, nil))
}
diff --git a/cmd/ftxtest/proto/input.pb.go b/cmd/ftxtest/proto/input.pb.go
index b0017d2..816d45e 100644
--- a/cmd/ftxtest/proto/input.pb.go
+++ b/cmd/ftxtest/proto/input.pb.go
@@ -25,6 +25,10 @@
sizeCache protoimpl.SizeCache
unknownFields protoimpl.UnknownFields
+ // Name of the test being run.
+ Name string `protobuf:"bytes,6,opt,name=name,proto3" json:"name,omitempty"`
+ // Whether this test should run in the Google internal or external infrastructure.
+ External bool `protobuf:"varint,7,opt,name=external,proto3" json:"external,omitempty"`
// CAS digest containing all of the inputs needed to both prepare targets
// and run tests.
InputArtifactsDigest string `protobuf:"bytes,1,opt,name=input_artifacts_digest,json=inputArtifactsDigest,proto3" json:"input_artifacts_digest,omitempty"`
@@ -71,6 +75,20 @@
return file_proto_input_proto_rawDescGZIP(), []int{0}
}
+func (x *InputProperties) GetName() string {
+ if x != nil {
+ return x.Name
+ }
+ return ""
+}
+
+func (x *InputProperties) GetExternal() bool {
+ if x != nil {
+ return x.External
+ }
+ return false
+}
+
func (x *InputProperties) GetInputArtifactsDigest() string {
if x != nil {
return x.InputArtifactsDigest
@@ -244,8 +262,11 @@
var file_proto_input_proto_rawDesc = []byte{
0x0a, 0x11, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x2e, 0x70, 0x72,
- 0x6f, 0x74, 0x6f, 0x22, 0xf8, 0x02, 0x0a, 0x0f, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x50, 0x72, 0x6f,
- 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x12, 0x34, 0x0a, 0x16, 0x69, 0x6e, 0x70, 0x75, 0x74,
+ 0x6f, 0x74, 0x6f, 0x22, 0xa8, 0x03, 0x0a, 0x0f, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x50, 0x72, 0x6f,
+ 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18,
+ 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x65,
+ 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x18, 0x07, 0x20, 0x01, 0x28, 0x08, 0x52, 0x08, 0x65,
+ 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x12, 0x34, 0x0a, 0x16, 0x69, 0x6e, 0x70, 0x75, 0x74,
0x5f, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x73, 0x5f, 0x64, 0x69, 0x67, 0x65, 0x73,
0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x14, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x41, 0x72,
0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x73, 0x44, 0x69, 0x67, 0x65, 0x73, 0x74, 0x12, 0x31, 0x0a,
diff --git a/cmd/ftxtest/proto/input.proto b/cmd/ftxtest/proto/input.proto
index 9116bfd..076ccf8 100644
--- a/cmd/ftxtest/proto/input.proto
+++ b/cmd/ftxtest/proto/input.proto
@@ -3,6 +3,10 @@
option go_package = "go.fuchsia.dev/infra/cmd/ftxtest/proto";
message InputProperties {
+ // Name of the test being run.
+ string name = 6;
+ // Whether this test should run in the Google internal or external infrastructure.
+ bool external = 7;
// CAS digest containing all of the inputs needed to both prepare targets
// and run tests.
string input_artifacts_digest = 1;
diff --git a/cmd/ftxtest/run.go b/cmd/ftxtest/run.go
new file mode 100644
index 0000000..c0a8589
--- /dev/null
+++ b/cmd/ftxtest/run.go
@@ -0,0 +1,101 @@
+// Copyright 2023 The Fuchsia Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+package main
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "net/http"
+ "os"
+ "strings"
+
+ "github.com/golang/protobuf/ptypes/any"
+ "github.com/maruel/subcommands"
+ "go.chromium.org/luci/auth"
+ "go.chromium.org/luci/luciexe/build"
+ ftxproto "go.fuchsia.dev/infra/cmd/ftxtest/proto"
+)
+
+func cmdRun(authOpts auth.Options) *subcommands.Command {
+ return &subcommands.Command{
+ UsageLine: "run",
+ ShortDesc: "runs test",
+ LongDesc: "Runs test based on luciexe protocol.",
+ CommandRun: func() subcommands.CommandRun {
+ r := &runImpl{}
+ r.Init(authOpts)
+ return r
+ },
+ }
+}
+
+type runImpl struct {
+ commonFlags
+
+ subcommands.CommandRunBase
+}
+
+func (r *runImpl) Init(defaultAuthOpts auth.Options) {
+ r.commonFlags.Init(defaultAuthOpts)
+}
+
+func (r *runImpl) Run(a subcommands.Application, args []string, env subcommands.Env) int {
+ if err := r.commonFlags.Parse(); err != nil {
+ fmt.Fprintf(os.Stderr, "Error parsing common flags: %v\n", err)
+ return 1
+ }
+ r.luciexeInit()
+ // luciexe uses os.exit to exit.
+ return 0
+}
+
+func (r *runImpl) luciexeInit() {
+ buildInput := &ftxproto.InputProperties{}
+ var writeOutputProps func(*any.Any)
+ build.Main(buildInput, &writeOutputProps, nil, func(ctx context.Context, extraArgs []string, state *build.State) error {
+ if len(buildInput.Name) == 0 {
+ return errors.New("Name is required.")
+ }
+ authenticator := auth.NewAuthenticator(ctx, auth.SilentLogin, r.parsedAuthOpts)
+ httpClient, err := authenticator.Client()
+ if err != nil {
+ fmt.Fprintf(os.Stderr, "You need to login first by running:\n")
+ fmt.Fprintf(os.Stderr, " luci-auth login -scopes %q\n", strings.Join(r.parsedAuthOpts.Scopes, " "))
+ return errors.New("Not logged in.")
+ }
+ _, _, err = LaunchTaskStep(ctx, httpClient, buildInput)
+ if err != nil {
+ return fmt.Errorf("LaunchTaskStep: %v", err)
+ }
+ return nil
+ })
+}
+
+func LaunchTaskStep(ctx context.Context, httpClient *http.Client, buildInput *ftxproto.InputProperties) (*Swarming, string, error) {
+ step, ctx := build.StartStep(ctx, "Launch Swarming Task")
+ instance := instance(buildInput)
+ swarming, err := NewSwarming(ctx, httpClient, instance)
+ if err != nil {
+ step.End(err)
+ return nil, "", fmt.Errorf("NewSwarming: %v", err)
+ }
+ task, err := swarming.LaunchTask(buildInput)
+ if err != nil {
+ step.End(err)
+ return nil, "", fmt.Errorf("LaunchTask: %v", err)
+ }
+ md := fmt.Sprintf("* [swarming task](https://%s.appspot.com/task?id=%s)", instance, task.TaskId)
+ step.SetSummaryMarkdown(md)
+ step.End(nil)
+ return swarming, task.TaskId, nil
+}
+
+func instance(buildInput *ftxproto.InputProperties) string {
+ if buildInput.External {
+ return "chromium-swarm"
+ } else {
+ return "chrome-swarming"
+ }
+}
diff --git a/cmd/ftxtest/run.sh b/cmd/ftxtest/run.sh
index 82befe3..484743f 100755
--- a/cmd/ftxtest/run.sh
+++ b/cmd/ftxtest/run.sh
@@ -3,4 +3,4 @@
set -x
./gen.sh
-LUCIEXE_FAKEBUILD=test/build.json go run -mod=vendor *.go -- -working-dir=$PWD
+LUCIEXE_FAKEBUILD=test/build.json go run -mod=vendor *.go run
diff --git a/cmd/ftxtest/swarming.go b/cmd/ftxtest/swarming.go
index bfcdf60..0f4f61c 100644
--- a/cmd/ftxtest/swarming.go
+++ b/cmd/ftxtest/swarming.go
@@ -6,32 +6,88 @@
import (
"context"
"fmt"
+ "net/http"
+ "strconv"
+ "strings"
+ "time"
- "go.chromium.org/luci/auth"
"go.chromium.org/luci/common/api/swarming/swarming/v1"
+ ftxproto "go.fuchsia.dev/infra/cmd/ftxtest/proto"
)
type Swarming struct {
- client *swarming.Service
+ instance string
+ service *swarming.Service
}
-func NewSwarming(ctx context.Context) (*Swarming, error) {
- authenticator := auth.NewAuthenticator(ctx, auth.SilentLogin, auth.Options{})
- httpClient, err := authenticator.Client()
- if err != nil {
- return nil, fmt.Errorf("authenticator.Client: %v", err)
- }
- swarmingClient, err := swarming.New(httpClient)
+const (
+ taskPriority = 200
+ taskExpiration = 5 * time.Hour
+ taskExecutionTimeout = 2 * time.Hour
+)
+
+func NewSwarming(ctx context.Context, httpClient *http.Client, instance string) (*Swarming, error) {
+ swarmingService, err := swarming.New(httpClient)
+ swarmingService.BasePath = fmt.Sprintf("https://%s.appspot.com/_ah/api/swarming/v1/", instance)
if err != nil {
return nil, fmt.Errorf("swarming.New: %v", err)
}
swarming := &Swarming{
- client: swarmingClient,
+ service: swarmingService,
+ instance: instance,
}
return swarming, nil
}
-func (s *Swarming) LaunchTask() (string, error) {
- // TODO(b/258456267): Launch swarming task...
- return "foo", nil
+func (s *Swarming) LaunchTask(buildInput *ftxproto.InputProperties) (*swarming.SwarmingRpcsTaskRequestMetadata, error) {
+ casInput, err := s.casInput(buildInput)
+ if err != nil {
+ return nil, fmt.Errorf("casInput: %v", err)
+ }
+ return s.service.Tasks.New(&swarming.SwarmingRpcsNewTaskRequest{
+ Name: buildInput.Name,
+ ExpirationSecs: int64(taskExpiration.Seconds()),
+ Priority: taskPriority,
+ Realm: realm(buildInput),
+ Properties: &swarming.SwarmingRpcsTaskProperties{
+ ExecutionTimeoutSecs: int64(taskExpiration.Seconds()),
+ Command: []string{buildInput.TestCommand},
+ CasInputRoot: casInput,
+ Dimensions: dimensions(buildInput),
+ },
+ }).Do()
+}
+
+func (s *Swarming) casInput(buildInput *ftxproto.InputProperties) (*swarming.SwarmingRpcsCASReference, error) {
+ digestSplit := strings.Split(buildInput.InputArtifactsDigest, "/")
+ sizeBytes, err := strconv.ParseInt(digestSplit[1], 10, 64)
+ if err != nil {
+ return nil, fmt.Errorf("sizeBytes: %v", err)
+ }
+ return &swarming.SwarmingRpcsCASReference{
+ CasInstance: fmt.Sprintf("projects/%s/instances/default_instance", s.instance),
+ Digest: &swarming.SwarmingRpcsDigest{
+ Hash: digestSplit[0],
+ SizeBytes: sizeBytes,
+ },
+ }, nil
+}
+
+func dimensions(buildInput *ftxproto.InputProperties) []*swarming.SwarmingRpcsStringPair {
+ result := []*swarming.SwarmingRpcsStringPair{}
+ for key, value := range buildInput.TargetDimensions {
+ result = append(result, &swarming.SwarmingRpcsStringPair{
+ Key: key,
+ Value: value,
+ })
+ }
+ return result
+}
+
+func realm(buildInput *ftxproto.InputProperties) string {
+ if buildInput.External {
+ return "fuchsia:try"
+ } else {
+ return "turquoise:global.try"
+ }
}
diff --git a/cmd/ftxtest/test/build.json b/cmd/ftxtest/test/build.json
index 7eb02d4..a3dff20 100644
--- a/cmd/ftxtest/test/build.json
+++ b/cmd/ftxtest/test/build.json
@@ -1,11 +1,12 @@
{
"input": {
"properties": {
- "input_artifacts_digest": "0123456789abcdef",
- "test_command": "some/command",
+ "name": "hello_swarming_test",
+ "external": false,
+ "input_artifacts_digest": "11ab3e1a0ba7bc9fbd6957c57222331f340dc981ec2bcb5edeffc05014e7a2ba/83",
+ "test_command": "turquoise/infra/foundation/go/ftxclient/examples/hello_swarming/hello_swarming.par",
"target_dimensions": {
- "d1": "v1",
- "d2": "v2"
+ "pool": "fuchsia.dev.tests"
}
}
}
