pkg/gce/gce.go - third_party/syzkaller - Git at Google

 // Copyright 2016 syzkaller project authors. All rights reserved.
 // Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.

 // Package gce provides wrappers around Google Compute Engine (GCE) APIs.
 // It is assumed that the program itself also runs on GCE as APIs operate on the current project/zone.
 //
 // See https://cloud.google.com/compute/docs for details.
 // In particular, API reference:
 // https://cloud.google.com/compute/docs/reference/latest
 // and Go API wrappers:
 // https://godoc.org/google.golang.org/api/compute/v1
 package gce

 import (
 	"errors"
 	"fmt"
 	"io"
 	"math/rand"
 	"net/http"
 	"regexp"
 	"strings"
 	"time"

 	"golang.org/x/net/context"
 	"golang.org/x/oauth2"
 	"golang.org/x/oauth2/google"
 	"google.golang.org/api/compute/v1"
 	"google.golang.org/api/googleapi"
 )

 type Context struct {
 	ProjectID  string
 	ZoneID     string
 	RegionID   string
 	Instance   string
 	InternalIP string
 	ExternalIP string
 	Network    string
 	Subnetwork string

 	computeService *compute.Service

 	// apiCallTicker ticks regularly, preventing us from accidentally making
 	// GCE API calls too quickly. Our quota is 20 QPS, but we limit ourselves
 	// to less than that because several independent programs can do API calls.
 	apiRateGate <-chan time.Time
 }

 type CreateArgs struct {
 	Preemptible   bool
 	DisplayDevice bool
 }

 func NewContext(customZoneID string) (*Context, error) {
 	ctx := &Context{
 		apiRateGate: time.NewTicker(time.Second).C,
 	}
 	background := context.Background()
 	tokenSource, err := google.DefaultTokenSource(background, compute.CloudPlatformScope)
 	if err != nil {
 		return nil, fmt.Errorf("failed to get a token source: %w", err)
 	}
 	httpClient := oauth2.NewClient(background, tokenSource)
 	// nolint
 	// compute.New is deprecated: please use NewService instead.
 	// To provide a custom HTTP client, use option.WithHTTPClient.
 	// If you are using google.golang.org/api/googleapis/transport.APIKey,
 	// use option.WithAPIKey with NewService instead.
 	ctx.computeService, _ = compute.New(httpClient)
 	// Obtain project name, zone and current instance IP address.
 	ctx.ProjectID, err = ctx.getMeta("project/project-id")
 	if err != nil {
 		return nil, fmt.Errorf("failed to query gce project-id: %w", err)
 	}
 	myZoneID, err := ctx.getMeta("instance/zone")
 	if err != nil {
 		return nil, fmt.Errorf("failed to query gce zone: %w", err)
 	}
 	if i := strings.LastIndexByte(myZoneID, '/'); i != -1 {
 		myZoneID = myZoneID[i+1:] // the query returns some nonsense prefix
 	}
 	if customZoneID != "" {
 		ctx.ZoneID = customZoneID
 	} else {
 		ctx.ZoneID = myZoneID
 	}
 	if !validateZone(ctx.ZoneID) {
 		return nil, fmt.Errorf("%q is not a valid zone name", ctx.ZoneID)
 	}
 	ctx.RegionID = zoneToRegion(ctx.ZoneID)
 	if ctx.RegionID == "" {
 		return nil, fmt.Errorf("failed to extract region id from %s", ctx.ZoneID)
 	}
 	ctx.Instance, err = ctx.getMeta("instance/name")
 	if err != nil {
 		return nil, fmt.Errorf("failed to query gce instance name: %w", err)
 	}
 	inst, err := ctx.computeService.Instances.Get(ctx.ProjectID, myZoneID, ctx.Instance).Do()
 	if err != nil {
 		return nil, fmt.Errorf("error getting instance info: %w", err)
 	}
 	for _, iface := range inst.NetworkInterfaces {
 		if strings.HasPrefix(iface.NetworkIP, "10.") {
 			ctx.InternalIP = iface.NetworkIP
 		}
 		for _, ac := range iface.AccessConfigs {
 			if ac.NatIP != "" {
 				ctx.ExternalIP = ac.NatIP
 			}
 		}
 		ctx.Network = iface.Network
 		ctx.Subnetwork = iface.Subnetwork
 	}
 	if ctx.InternalIP == "" {
 		return nil, fmt.Errorf("failed to get current instance internal IP")
 	}
 	return ctx, nil
 }

 func (ctx *Context) CreateInstance(name, machineType, image, sshkey string,
 	preemptible, displayDevice bool) (string, error) {
 	prefix := "https://www.googleapis.com/compute/v1/projects/" + ctx.ProjectID
 	sshkeyAttr := "syzkaller:" + sshkey
 	oneAttr := "1"
 	falseAttr := false
 	instance := &compute.Instance{
 		Name:        name,
 		Description: "syzkaller worker",
 		MachineType: prefix + "/zones/" + ctx.ZoneID + "/machineTypes/" + machineType,
 		Disks: []*compute.AttachedDisk{
 			{
 				AutoDelete: true,
 				Boot:       true,
 				Type:       "PERSISTENT",
 				InitializeParams: &compute.AttachedDiskInitializeParams{
 					DiskName:    name,
 					SourceImage: prefix + "/global/images/" + image,
 				},
 			},
 		},
 		Metadata: &compute.Metadata{
 			Items: []*compute.MetadataItems{
 				{
 					Key:   "ssh-keys",
 					Value: &sshkeyAttr,
 				},
 				{
 					Key:   "serial-port-enable",
 					Value: &oneAttr,
 				},
 			},
 		},
 		NetworkInterfaces: []*compute.NetworkInterface{
 			{
 				Network:    ctx.Network,
 				Subnetwork: ctx.Subnetwork,
 			},
 		},
 		Scheduling: &compute.Scheduling{
 			AutomaticRestart:  &falseAttr,
 			Preemptible:       preemptible,
 			OnHostMaintenance: "TERMINATE",
 		},
 		DisplayDevice: &compute.DisplayDevice{
 			EnableDisplay: displayDevice,
 		},
 	}
 retry:
 	if !instance.Scheduling.Preemptible && strings.HasPrefix(machineType, "e2-") {
 		// Otherwise we get "Error 400: Efficient instances do not support
 		// onHostMaintenance=TERMINATE unless they are preemptible".
 		instance.Scheduling.OnHostMaintenance = "MIGRATE"
 	}
 	var op *compute.Operation
 	err := ctx.apiCall(func() (err error) {
 		op, err = ctx.computeService.Instances.Insert(ctx.ProjectID, ctx.ZoneID, instance).Do()
 		return
 	})
 	if err != nil {
 		return "", fmt.Errorf("failed to create instance: %w", err)
 	}
 	if err := ctx.waitForCompletion("zone", "create instance", op.Name, false); err != nil {
 		var resourcePoolExhaustedError resourcePoolExhaustedError
 		if errors.As(err, &resourcePoolExhaustedError) && instance.Scheduling.Preemptible {
 			instance.Scheduling.Preemptible = false
 			goto retry
 		}
 		return "", err
 	}

 	var inst *compute.Instance
 	err = ctx.apiCall(func() (err error) {
 		inst, err = ctx.computeService.Instances.Get(ctx.ProjectID, ctx.ZoneID, name).Do()
 		return
 	})
 	if err != nil {
 		return "", fmt.Errorf("error getting instance %s details after creation: %w", name, err)
 	}

 	// Finds its internal IP.
 	ip := ""
 	for _, iface := range inst.NetworkInterfaces {
 		if strings.HasPrefix(iface.NetworkIP, "10.") {
 			ip = iface.NetworkIP
 			break
 		}
 	}
 	if ip == "" {
 		return "", fmt.Errorf("didn't find instance internal IP address")
 	}
 	return ip, nil
 }

 func (ctx *Context) DeleteInstance(name string, wait bool) error {
 	var op *compute.Operation
 	err := ctx.apiCall(func() (err error) {
 		op, err = ctx.computeService.Instances.Delete(ctx.ProjectID, ctx.ZoneID, name).Do()
 		return
 	})
 	var apiErr *googleapi.Error
 	if errors.As(err, &apiErr) && apiErr.Code == 404 {
 		return nil
 	}
 	if err != nil {
 		return fmt.Errorf("failed to delete instance: %w", err)
 	}
 	if wait {
 		if err := ctx.waitForCompletion("zone", "delete image", op.Name, true); err != nil {
 			return err
 		}
 	}
 	return nil
 }

 func (ctx *Context) IsInstanceRunning(name string) bool {
 	var inst *compute.Instance
 	err := ctx.apiCall(func() (err error) {
 		inst, err = ctx.computeService.Instances.Get(ctx.ProjectID, ctx.ZoneID, name).Do()
 		return
 	})
 	if err != nil {
 		return false
 	}
 	return inst.Status == "RUNNING"
 }

 func (ctx *Context) CreateImage(imageName, gcsFile string) error {
 	image := &compute.Image{
 		Name: imageName,
 		RawDisk: &compute.ImageRawDisk{
 			Source: "https://storage.googleapis.com/" + gcsFile,
 		},
 		Licenses: []string{
 			"https://www.googleapis.com/compute/v1/projects/vm-options/global/licenses/enable-vmx",
 		},
 	}
 	var op *compute.Operation
 	err := ctx.apiCall(func() (err error) {
 		op, err = ctx.computeService.Images.Insert(ctx.ProjectID, image).Do()
 		return
 	})
 	if err != nil {
 		// Try again without the vmx license in case it is not supported.
 		image.Licenses = nil
 		err := ctx.apiCall(func() (err error) {
 			op, err = ctx.computeService.Images.Insert(ctx.ProjectID, image).Do()
 			return
 		})
 		if err != nil {
 			return fmt.Errorf("failed to create image: %w", err)
 		}
 	}
 	if err := ctx.waitForCompletion("global", "create image", op.Name, false); err != nil {
 		return err
 	}
 	return nil
 }

 func (ctx *Context) DeleteImage(imageName string) error {
 	var op *compute.Operation
 	err := ctx.apiCall(func() (err error) {
 		op, err = ctx.computeService.Images.Delete(ctx.ProjectID, imageName).Do()
 		return
 	})
 	var apiErr *googleapi.Error
 	if errors.As(err, &apiErr) && apiErr.Code == 404 {
 		return nil
 	}
 	if err != nil {
 		return fmt.Errorf("failed to delete image: %w", err)
 	}
 	if err := ctx.waitForCompletion("global", "delete image", op.Name, true); err != nil {
 		return err
 	}
 	return nil
 }

 type resourcePoolExhaustedError string

 func (err resourcePoolExhaustedError) Error() string {
 	return string(err)
 }

 func (ctx *Context) waitForCompletion(typ, desc, opName string, ignoreNotFound bool) error {
 	time.Sleep(3 * time.Second)
 	for {
 		time.Sleep(3 * time.Second)
 		var op *compute.Operation
 		err := ctx.apiCall(func() (err error) {
 			switch typ {
 			case "global":
 				op, err = ctx.computeService.GlobalOperations.Get(ctx.ProjectID, opName).Do()
 			case "zone":
 				op, err = ctx.computeService.ZoneOperations.Get(ctx.ProjectID, ctx.ZoneID, opName).Do()
 			default:
 				panic("unknown operation type: " + typ)
 			}
 			return
 		})
 		if err != nil {
 			return fmt.Errorf("failed to get %v operation %v: %w", desc, opName, err)
 		}
 		switch op.Status {
 		case "PENDING", "RUNNING":
 			continue
 		case "DONE":
 			if op.Error != nil {
 				reason := ""
 				for _, operr := range op.Error.Errors {
 					if operr.Code == "ZONE_RESOURCE_POOL_EXHAUSTED" ||
 						operr.Code == "ZONE_RESOURCE_POOL_EXHAUSTED_WITH_DETAILS" {
 						return resourcePoolExhaustedError(fmt.Sprintf("%+v", operr))
 					}
 					if ignoreNotFound && operr.Code == "RESOURCE_NOT_FOUND" {
 						return nil
 					}
 					reason += fmt.Sprintf("%+v.", operr)
 				}
 				return fmt.Errorf("%v operation failed: %v", desc, reason)
 			}
 			return nil
 		default:
 			return fmt.Errorf("unknown %v operation status %q: %+v", desc, op.Status, op)
 		}
 	}
 }

 func (ctx *Context) getMeta(path string) (string, error) {
 	req, err := http.NewRequest("GET", "http://metadata.google.internal/computeMetadata/v1/"+path, nil)
 	if err != nil {
 		return "", err
 	}
 	req.Header.Add("Metadata-Flavor", "Google")
 	resp, err := http.DefaultClient.Do(req)
 	if err != nil {
 		return "", err
 	}
 	defer resp.Body.Close()
 	body, err := io.ReadAll(resp.Body)
 	if err != nil {
 		return "", err
 	}
 	return string(body), nil
 }

 func (ctx *Context) apiCall(fn func() error) error {
 	rateLimited := 0
 	for {
 		<-ctx.apiRateGate
 		err := fn()
 		if err != nil {
 			if strings.Contains(err.Error(), "Rate Limit Exceeded") ||
 				strings.Contains(err.Error(), "rateLimitExceeded") {
 				rateLimited++
 				backoff := time.Duration(float64(rateLimited) * 1e9 * (rand.Float64() + 1))
 				time.Sleep(backoff)
 				if rateLimited < 20 {
 					continue
 				}
 			}
 		}
 		return err
 	}
 }

 var zoneNameRe = regexp.MustCompile("^[a-zA-Z0-9]*-[a-zA-Z0-9]*[-][a-zA-Z0-9]*$")

 func validateZone(zone string) bool {
 	return zoneNameRe.MatchString(zone)
 }

 var regionNameRe = regexp.MustCompile("^[a-zA-Z0-9]*-[a-zA-Z0-9]*")

 func zoneToRegion(zone string) string {
 	return regionNameRe.FindString(zone)
 }
	// Copyright 2016 syzkaller project authors. All rights reserved.
	// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.

	// Package gce provides wrappers around Google Compute Engine (GCE) APIs.
	// It is assumed that the program itself also runs on GCE as APIs operate on the current project/zone.
	//
	// See https://cloud.google.com/compute/docs for details.
	// In particular, API reference:
	// https://cloud.google.com/compute/docs/reference/latest
	// and Go API wrappers:
	// https://godoc.org/google.golang.org/api/compute/v1
	package gce

	import (
	"errors"
	"fmt"
	"io"
	"math/rand"
	"net/http"
	"regexp"
	"strings"
	"time"

	"golang.org/x/net/context"
	"golang.org/x/oauth2"
	"golang.org/x/oauth2/google"
	"google.golang.org/api/compute/v1"
	"google.golang.org/api/googleapi"
	)

	type Context struct {
	ProjectID string
	ZoneID string
	RegionID string
	Instance string
	InternalIP string
	ExternalIP string
	Network string
	Subnetwork string

	computeService *compute.Service

	// apiCallTicker ticks regularly, preventing us from accidentally making
	// GCE API calls too quickly. Our quota is 20 QPS, but we limit ourselves
	// to less than that because several independent programs can do API calls.
	apiRateGate <-chan time.Time
	}

	type CreateArgs struct {
	Preemptible bool
	DisplayDevice bool
	}

	func NewContext(customZoneID string) (*Context, error) {
	ctx := &Context{
	apiRateGate: time.NewTicker(time.Second).C,
	}
	background := context.Background()
	tokenSource, err := google.DefaultTokenSource(background, compute.CloudPlatformScope)
	if err != nil {
	return nil, fmt.Errorf("failed to get a token source: %w", err)
	}
	httpClient := oauth2.NewClient(background, tokenSource)
	// nolint
	// compute.New is deprecated: please use NewService instead.
	// To provide a custom HTTP client, use option.WithHTTPClient.
	// If you are using google.golang.org/api/googleapis/transport.APIKey,
	// use option.WithAPIKey with NewService instead.
	ctx.computeService, _ = compute.New(httpClient)
	// Obtain project name, zone and current instance IP address.
	ctx.ProjectID, err = ctx.getMeta("project/project-id")
	if err != nil {
	return nil, fmt.Errorf("failed to query gce project-id: %w", err)
	}
	myZoneID, err := ctx.getMeta("instance/zone")
	if err != nil {
	return nil, fmt.Errorf("failed to query gce zone: %w", err)
	}
	if i := strings.LastIndexByte(myZoneID, '/'); i != -1 {
	myZoneID = myZoneID[i+1:] // the query returns some nonsense prefix
	}
	if customZoneID != "" {
	ctx.ZoneID = customZoneID
	} else {
	ctx.ZoneID = myZoneID
	}
	if !validateZone(ctx.ZoneID) {
	return nil, fmt.Errorf("%q is not a valid zone name", ctx.ZoneID)
	}
	ctx.RegionID = zoneToRegion(ctx.ZoneID)
	if ctx.RegionID == "" {
	return nil, fmt.Errorf("failed to extract region id from %s", ctx.ZoneID)
	}
	ctx.Instance, err = ctx.getMeta("instance/name")
	if err != nil {
	return nil, fmt.Errorf("failed to query gce instance name: %w", err)
	}
	inst, err := ctx.computeService.Instances.Get(ctx.ProjectID, myZoneID, ctx.Instance).Do()
	if err != nil {
	return nil, fmt.Errorf("error getting instance info: %w", err)
	}
	for _, iface := range inst.NetworkInterfaces {
	if strings.HasPrefix(iface.NetworkIP, "10.") {
	ctx.InternalIP = iface.NetworkIP
	}
	for _, ac := range iface.AccessConfigs {
	if ac.NatIP != "" {
	ctx.ExternalIP = ac.NatIP
	}
	}
	ctx.Network = iface.Network
	ctx.Subnetwork = iface.Subnetwork
	}
	if ctx.InternalIP == "" {
	return nil, fmt.Errorf("failed to get current instance internal IP")
	}
	return ctx, nil
	}

	func (ctx *Context) CreateInstance(name, machineType, image, sshkey string,
	preemptible, displayDevice bool) (string, error) {
	prefix := "https://www.googleapis.com/compute/v1/projects/" + ctx.ProjectID
	sshkeyAttr := "syzkaller:" + sshkey
	oneAttr := "1"
	falseAttr := false
	instance := &compute.Instance{
	Name: name,
	Description: "syzkaller worker",
	MachineType: prefix + "/zones/" + ctx.ZoneID + "/machineTypes/" + machineType,
	Disks: []*compute.AttachedDisk{
	{
	AutoDelete: true,
	Boot: true,
	Type: "PERSISTENT",
	InitializeParams: &compute.AttachedDiskInitializeParams{
	DiskName: name,
	SourceImage: prefix + "/global/images/" + image,
	},
	},
	},
	Metadata: &compute.Metadata{
	Items: []*compute.MetadataItems{
	{
	Key: "ssh-keys",
	Value: &sshkeyAttr,
	},
	{
	Key: "serial-port-enable",
	Value: &oneAttr,
	},
	},
	},
	NetworkInterfaces: []*compute.NetworkInterface{
	{
	Network: ctx.Network,
	Subnetwork: ctx.Subnetwork,
	},
	},
	Scheduling: &compute.Scheduling{
	AutomaticRestart: &falseAttr,
	Preemptible: preemptible,
	OnHostMaintenance: "TERMINATE",
	},
	DisplayDevice: &compute.DisplayDevice{
	EnableDisplay: displayDevice,
	},
	}
	retry:
	if !instance.Scheduling.Preemptible && strings.HasPrefix(machineType, "e2-") {
	// Otherwise we get "Error 400: Efficient instances do not support
	// onHostMaintenance=TERMINATE unless they are preemptible".
	instance.Scheduling.OnHostMaintenance = "MIGRATE"
	}
	var op *compute.Operation
	err := ctx.apiCall(func() (err error) {
	op, err = ctx.computeService.Instances.Insert(ctx.ProjectID, ctx.ZoneID, instance).Do()
	return
	})
	if err != nil {
	return "", fmt.Errorf("failed to create instance: %w", err)
	}
	if err := ctx.waitForCompletion("zone", "create instance", op.Name, false); err != nil {
	var resourcePoolExhaustedError resourcePoolExhaustedError
	if errors.As(err, &resourcePoolExhaustedError) && instance.Scheduling.Preemptible {
	instance.Scheduling.Preemptible = false
	goto retry
	}
	return "", err
	}

	var inst *compute.Instance
	err = ctx.apiCall(func() (err error) {
	inst, err = ctx.computeService.Instances.Get(ctx.ProjectID, ctx.ZoneID, name).Do()
	return
	})
	if err != nil {
	return "", fmt.Errorf("error getting instance %s details after creation: %w", name, err)
	}

	// Finds its internal IP.
	ip := ""
	for _, iface := range inst.NetworkInterfaces {
	if strings.HasPrefix(iface.NetworkIP, "10.") {
	ip = iface.NetworkIP
	break
	}
	}
	if ip == "" {
	return "", fmt.Errorf("didn't find instance internal IP address")
	}
	return ip, nil
	}

	func (ctx *Context) DeleteInstance(name string, wait bool) error {
	var op *compute.Operation
	err := ctx.apiCall(func() (err error) {
	op, err = ctx.computeService.Instances.Delete(ctx.ProjectID, ctx.ZoneID, name).Do()
	return
	})
	var apiErr *googleapi.Error
	if errors.As(err, &apiErr) && apiErr.Code == 404 {
	return nil
	}
	if err != nil {
	return fmt.Errorf("failed to delete instance: %w", err)
	}
	if wait {
	if err := ctx.waitForCompletion("zone", "delete image", op.Name, true); err != nil {
	return err
	}
	}
	return nil
	}

	func (ctx *Context) IsInstanceRunning(name string) bool {
	var inst *compute.Instance
	err := ctx.apiCall(func() (err error) {
	inst, err = ctx.computeService.Instances.Get(ctx.ProjectID, ctx.ZoneID, name).Do()
	return
	})
	if err != nil {
	return false
	}
	return inst.Status == "RUNNING"
	}

	func (ctx *Context) CreateImage(imageName, gcsFile string) error {
	image := &compute.Image{
	Name: imageName,
	RawDisk: &compute.ImageRawDisk{
	Source: "https://storage.googleapis.com/" + gcsFile,
	},
	Licenses: []string{
	"https://www.googleapis.com/compute/v1/projects/vm-options/global/licenses/enable-vmx",
	},
	}
	var op *compute.Operation
	err := ctx.apiCall(func() (err error) {
	op, err = ctx.computeService.Images.Insert(ctx.ProjectID, image).Do()
	return
	})
	if err != nil {
	// Try again without the vmx license in case it is not supported.
	image.Licenses = nil
	err := ctx.apiCall(func() (err error) {
	op, err = ctx.computeService.Images.Insert(ctx.ProjectID, image).Do()
	return
	})
	if err != nil {
	return fmt.Errorf("failed to create image: %w", err)
	}
	}
	if err := ctx.waitForCompletion("global", "create image", op.Name, false); err != nil {
	return err
	}
	return nil
	}

	func (ctx *Context) DeleteImage(imageName string) error {
	var op *compute.Operation
	err := ctx.apiCall(func() (err error) {
	op, err = ctx.computeService.Images.Delete(ctx.ProjectID, imageName).Do()
	return
	})
	var apiErr *googleapi.Error
	if errors.As(err, &apiErr) && apiErr.Code == 404 {
	return nil
	}
	if err != nil {
	return fmt.Errorf("failed to delete image: %w", err)
	}
	if err := ctx.waitForCompletion("global", "delete image", op.Name, true); err != nil {
	return err
	}
	return nil
	}

	type resourcePoolExhaustedError string

	func (err resourcePoolExhaustedError) Error() string {
	return string(err)
	}

	func (ctx *Context) waitForCompletion(typ, desc, opName string, ignoreNotFound bool) error {
	time.Sleep(3 * time.Second)
	for {
	time.Sleep(3 * time.Second)
	var op *compute.Operation
	err := ctx.apiCall(func() (err error) {
	switch typ {
	case "global":
	op, err = ctx.computeService.GlobalOperations.Get(ctx.ProjectID, opName).Do()
	case "zone":
	op, err = ctx.computeService.ZoneOperations.Get(ctx.ProjectID, ctx.ZoneID, opName).Do()
	default:
	panic("unknown operation type: " + typ)
	}
	return
	})
	if err != nil {
	return fmt.Errorf("failed to get %v operation %v: %w", desc, opName, err)
	}
	switch op.Status {
	case "PENDING", "RUNNING":
	continue
	case "DONE":
	if op.Error != nil {
	reason := ""
	for _, operr := range op.Error.Errors {
	if operr.Code == "ZONE_RESOURCE_POOL_EXHAUSTED" \|\|
	operr.Code == "ZONE_RESOURCE_POOL_EXHAUSTED_WITH_DETAILS" {
	return resourcePoolExhaustedError(fmt.Sprintf("%+v", operr))
	}
	if ignoreNotFound && operr.Code == "RESOURCE_NOT_FOUND" {
	return nil
	}
	reason += fmt.Sprintf("%+v.", operr)
	}
	return fmt.Errorf("%v operation failed: %v", desc, reason)
	}
	return nil
	default:
	return fmt.Errorf("unknown %v operation status %q: %+v", desc, op.Status, op)
	}
	}
	}

	func (ctx *Context) getMeta(path string) (string, error) {
	req, err := http.NewRequest("GET", "http://metadata.google.internal/computeMetadata/v1/"+path, nil)
	if err != nil {
	return "", err
	}
	req.Header.Add("Metadata-Flavor", "Google")
	resp, err := http.DefaultClient.Do(req)
	if err != nil {
	return "", err
	}
	defer resp.Body.Close()
	body, err := io.ReadAll(resp.Body)
	if err != nil {
	return "", err
	}
	return string(body), nil
	}

	func (ctx *Context) apiCall(fn func() error) error {
	rateLimited := 0
	for {
	<-ctx.apiRateGate
	err := fn()
	if err != nil {
	if strings.Contains(err.Error(), "Rate Limit Exceeded") \|\|
	strings.Contains(err.Error(), "rateLimitExceeded") {
	rateLimited++
	backoff := time.Duration(float64(rateLimited) * 1e9 * (rand.Float64() + 1))
	time.Sleep(backoff)
	if rateLimited < 20 {
	continue
	}
	}
	}
	return err
	}
	}

	var zoneNameRe = regexp.MustCompile("^[a-zA-Z0-9]-[a-zA-Z0-9][-][a-zA-Z0-9]*$")

	func validateZone(zone string) bool {
	return zoneNameRe.MatchString(zone)
	}

	var regionNameRe = regexp.MustCompile("^[a-zA-Z0-9]-[a-zA-Z0-9]")

	func zoneToRegion(zone string) string {
	return regionNameRe.FindString(zone)
	}