Merge pull request #42061 from thaJeztah/20.10_backport_bump_buildkit
[20.10 backport] vendor: github.com/moby/buildkit v0.8.2
diff --git a/builder/builder-next/adapters/localinlinecache/inlinecache.go b/builder/builder-next/adapters/localinlinecache/inlinecache.go
index 0c51e20..c5e19ce 100644
--- a/builder/builder-next/adapters/localinlinecache/inlinecache.go
+++ b/builder/builder-next/adapters/localinlinecache/inlinecache.go
@@ -22,6 +22,11 @@
"github.com/pkg/errors"
)
+func init() {
+ // See https://github.com/moby/buildkit/pull/1993.
+ v1.EmptyLayerRemovalSupported = false
+}
+
// ResolveCacheImporterFunc returns a resolver function for local inline cache
func ResolveCacheImporterFunc(sm *session.Manager, resolverFunc docker.RegistryHosts, cs content.Store, rs reference.Store, is imagestore.Store) remotecache.ResolveCacheImporterFunc {
diff --git a/vendor.conf b/vendor.conf
index e507095..68b4f86 100644
--- a/vendor.conf
+++ b/vendor.conf
@@ -33,7 +33,7 @@
golang.org/x/sync cd5d95a43a6e21273425c7ae415d3df9ea832eeb
# buildkit
-github.com/moby/buildkit 68bb095353c65bc3993fd534c26cf77fe05e61b1 # v0.8 branch
+github.com/moby/buildkit 9065b18ba4633c75862befca8188de4338d9f94a # v0.8.2
github.com/tonistiigi/fsutil 0834f99b7b85462efb69b4f571a4fa3ca7da5ac9
github.com/tonistiigi/units 6950e57a87eaf136bbe44ef2ec8e75b9e3569de2
github.com/grpc-ecosystem/grpc-opentracing 8e809c8a86450a29b90dcc9efbf062d0fe6d9746
diff --git a/vendor/github.com/moby/buildkit/cache/remotecache/v1/utils.go b/vendor/github.com/moby/buildkit/cache/remotecache/v1/utils.go
index fc494aa..9531675 100644
--- a/vendor/github.com/moby/buildkit/cache/remotecache/v1/utils.go
+++ b/vendor/github.com/moby/buildkit/cache/remotecache/v1/utils.go
@@ -10,6 +10,10 @@
"github.com/pkg/errors"
)
+// EmptyLayerRemovalSupported defines if implementation supports removal of empty layers. Buildkit image exporter
+// removes empty layers, but moby layerstore based implementation does not.
+var EmptyLayerRemovalSupported = true
+
// sortConfig sorts the config structure to make sure it is deterministic
func sortConfig(cc *CacheConfig) {
type indexedLayer struct {
@@ -239,7 +243,7 @@
}
desc := r.Descriptors[len(r.Descriptors)-1]
- if desc.Digest == exptypes.EmptyGZLayer {
+ if desc.Digest == exptypes.EmptyGZLayer && EmptyLayerRemovalSupported {
return parentID
}
diff --git a/vendor/github.com/moby/buildkit/frontend/dockerfile/instructions/bflag.go b/vendor/github.com/moby/buildkit/frontend/dockerfile/instructions/bflag.go
index d8bf747..66c53ad 100644
--- a/vendor/github.com/moby/buildkit/frontend/dockerfile/instructions/bflag.go
+++ b/vendor/github.com/moby/buildkit/frontend/dockerfile/instructions/bflag.go
@@ -108,6 +108,15 @@
return false
}
+// Used returns a slice of flag names that are set
+func (bf *BFlags) Used() []string {
+ used := make([]string, 0, len(bf.used))
+ for f := range bf.used {
+ used = append(used, f)
+ }
+ return used
+}
+
// IsTrue checks if a bool flag is true
func (fl *Flag) IsTrue() bool {
if fl.flagType != boolType {
diff --git a/vendor/github.com/moby/buildkit/frontend/dockerfile/instructions/commands.go b/vendor/github.com/moby/buildkit/frontend/dockerfile/instructions/commands.go
index e602744..b900ad7 100644
--- a/vendor/github.com/moby/buildkit/frontend/dockerfile/instructions/commands.go
+++ b/vendor/github.com/moby/buildkit/frontend/dockerfile/instructions/commands.go
@@ -269,6 +269,7 @@
withNameAndCode
withExternalData
ShellDependantCmdLine
+ FlagsUsed []string
}
// CmdCommand : CMD foo
diff --git a/vendor/github.com/moby/buildkit/frontend/dockerfile/instructions/parse.go b/vendor/github.com/moby/buildkit/frontend/dockerfile/instructions/parse.go
index 83cab66..18e7272 100644
--- a/vendor/github.com/moby/buildkit/frontend/dockerfile/instructions/parse.go
+++ b/vendor/github.com/moby/buildkit/frontend/dockerfile/instructions/parse.go
@@ -375,7 +375,7 @@
if err := req.flags.Parse(); err != nil {
return nil, err
}
-
+ cmd.FlagsUsed = req.flags.Used()
cmd.ShellDependantCmdLine = parseShellDependentCommand(req, false)
cmd.withNameAndCode = newWithNameAndCode(req)
diff --git a/vendor/github.com/moby/buildkit/frontend/gateway/container.go b/vendor/github.com/moby/buildkit/frontend/gateway/container.go
index 1f79bb6..e124566 100644
--- a/vendor/github.com/moby/buildkit/frontend/gateway/container.go
+++ b/vendor/github.com/moby/buildkit/frontend/gateway/container.go
@@ -127,6 +127,7 @@
type MountMutableRef struct {
Ref cache.MutableRef
MountIndex int
+ NoCommit bool
}
type MakeMutable func(m *opspb.Mount, ref cache.ImmutableRef) (cache.MutableRef, error)
@@ -196,6 +197,7 @@
p.Actives = append(p.Actives, MountMutableRef{
MountIndex: i,
Ref: active,
+ NoCommit: true,
})
if m.Output != opspb.SkipOutput && ref != nil {
p.OutputRefs = append(p.OutputRefs, MountRef{
diff --git a/vendor/github.com/moby/buildkit/solver/llbsolver/errdefs/exec.go b/vendor/github.com/moby/buildkit/solver/llbsolver/errdefs/exec.go
index f607571..ed3c0d4 100644
--- a/vendor/github.com/moby/buildkit/solver/llbsolver/errdefs/exec.go
+++ b/vendor/github.com/moby/buildkit/solver/llbsolver/errdefs/exec.go
@@ -21,10 +21,15 @@
}
func (e *ExecError) EachRef(fn func(solver.Result) error) (err error) {
+ m := map[solver.Result]struct{}{}
for _, res := range e.Inputs {
if res == nil {
continue
}
+ if _, ok := m[res]; ok {
+ continue
+ }
+ m[res] = struct{}{}
if err1 := fn(res); err1 != nil && err == nil {
err = err1
}
@@ -33,6 +38,10 @@
if res == nil {
continue
}
+ if _, ok := m[res]; ok {
+ continue
+ }
+ m[res] = struct{}{}
if err1 := fn(res); err1 != nil && err == nil {
err = err1
}
diff --git a/vendor/github.com/moby/buildkit/solver/llbsolver/ops/exec.go b/vendor/github.com/moby/buildkit/solver/llbsolver/ops/exec.go
index 5ec45fa..24aa46a 100644
--- a/vendor/github.com/moby/buildkit/solver/llbsolver/ops/exec.go
+++ b/vendor/github.com/moby/buildkit/solver/llbsolver/ops/exec.go
@@ -235,7 +235,7 @@
if m.Input == -1 {
continue
}
- execInputs[i] = inputs[m.Input]
+ execInputs[i] = inputs[m.Input].Clone()
}
execMounts := make([]solver.Result, len(e.op.Mounts))
copy(execMounts, execInputs)
@@ -243,12 +243,16 @@
execMounts[p.OutputRefs[i].MountIndex] = res
}
for _, active := range p.Actives {
- ref, cerr := active.Ref.Commit(ctx)
- if cerr != nil {
- err = errors.Wrapf(err, "error committing %s: %s", active.Ref.ID(), cerr)
- continue
+ if active.NoCommit {
+ active.Ref.Release(context.TODO())
+ } else {
+ ref, cerr := active.Ref.Commit(ctx)
+ if cerr != nil {
+ err = errors.Wrapf(err, "error committing %s: %s", active.Ref.ID(), cerr)
+ continue
+ }
+ execMounts[active.MountIndex] = worker.NewWorkerRefResult(ref, e.w)
}
- execMounts[active.MountIndex] = worker.NewWorkerRefResult(ref, e.w)
}
err = errdefs.WithExecError(err, execInputs, execMounts)
} else {
diff --git a/vendor/github.com/moby/buildkit/solver/llbsolver/ops/exec_binfmt.go b/vendor/github.com/moby/buildkit/solver/llbsolver/ops/exec_binfmt.go
index 0603cf5..707c5f6 100644
--- a/vendor/github.com/moby/buildkit/solver/llbsolver/ops/exec_binfmt.go
+++ b/vendor/github.com/moby/buildkit/solver/llbsolver/ops/exec_binfmt.go
@@ -27,6 +27,7 @@
"arm": "arm",
"s390x": "s390x",
"ppc64le": "ppc64le",
+ "386": "i386",
}
type emulator struct {
diff --git a/vendor/github.com/moby/buildkit/solver/llbsolver/ops/file.go b/vendor/github.com/moby/buildkit/solver/llbsolver/ops/file.go
index 1111cdb..554532c 100644
--- a/vendor/github.com/moby/buildkit/solver/llbsolver/ops/file.go
+++ b/vendor/github.com/moby/buildkit/solver/llbsolver/ops/file.go
@@ -61,6 +61,8 @@
}
}
+ indexes := make([][]int, 0, len(f.op.Actions))
+
for _, action := range f.op.Actions {
var dt []byte
var err error
@@ -103,14 +105,21 @@
}
actions = append(actions, dt)
+ indexes = append(indexes, []int{int(action.Input), int(action.SecondaryInput), int(action.Output)})
+ }
+
+ if isDefaultIndexes(indexes) {
+ indexes = nil
}
dt, err := json.Marshal(struct {
Type string
Actions [][]byte
+ Indexes [][]int `json:"indexes,omitempty"`
}{
Type: fileCacheType,
Actions: actions,
+ Indexes: indexes,
})
if err != nil {
return nil, false, err
@@ -421,7 +430,6 @@
if cerr == nil {
outputRes[idx-len(inputs)] = worker.NewWorkerRefResult(ref.(cache.ImmutableRef), s.w)
}
- inpMount.Release(context.TODO())
}
// If the action has a secondary input, commit it and set the ref on
@@ -611,3 +619,39 @@
}
return inp.(input), err
}
+
+func isDefaultIndexes(idxs [][]int) bool {
+ // Older version of checksum did not contain indexes for actions resulting in possibility for a wrong cache match.
+ // We detect the most common pattern for indexes and maintain old checksum for that case to minimize cache misses on upgrade.
+ // If a future change causes braking changes in instruction cache consider removing this exception.
+ if len(idxs) == 0 {
+ return false
+ }
+
+ for i, idx := range idxs {
+ if len(idx) != 3 {
+ return false
+ }
+ // input for first action is first input
+ if i == 0 && idx[0] != 0 {
+ return false
+ }
+ // input for other actions is previous action
+ if i != 0 && idx[0] != len(idxs)+(i-1) {
+ return false
+ }
+ // secondary input is second input or -1
+ if idx[1] != -1 && idx[1] != 1 {
+ return false
+ }
+ // last action creates output
+ if i == len(idxs)-1 && idx[2] != 0 {
+ return false
+ }
+ // other actions do not create an output
+ if i != len(idxs)-1 && idx[2] != -1 {
+ return false
+ }
+ }
+ return true
+}
diff --git a/vendor/github.com/moby/buildkit/solver/result.go b/vendor/github.com/moby/buildkit/solver/result.go
index 75b378c..b35c74b 100644
--- a/vendor/github.com/moby/buildkit/solver/result.go
+++ b/vendor/github.com/moby/buildkit/solver/result.go
@@ -47,7 +47,7 @@
func (r *splitResult) Release(ctx context.Context) error {
if atomic.AddInt64(&r.released, 1) > 1 {
- err := errors.Errorf("releasing already released reference")
+ err := errors.Errorf("releasing already released reference %+v", r.Result.ID())
logrus.Error(err)
return err
}
@@ -78,10 +78,14 @@
}
}
-func (r *SharedCachedResult) Clone() CachedResult {
+func (r *SharedCachedResult) CloneCachedResult() CachedResult {
return &clonedCachedResult{Result: r.SharedResult.Clone(), cr: r.CachedResult}
}
+func (r *SharedCachedResult) Clone() Result {
+ return r.CloneCachedResult()
+}
+
func (r *SharedCachedResult) Release(ctx context.Context) error {
return r.SharedResult.Release(ctx)
}
diff --git a/vendor/github.com/moby/buildkit/solver/scheduler.go b/vendor/github.com/moby/buildkit/solver/scheduler.go
index 5b598bd..bcad4dc 100644
--- a/vendor/github.com/moby/buildkit/solver/scheduler.go
+++ b/vendor/github.com/moby/buildkit/solver/scheduler.go
@@ -244,7 +244,7 @@
if err := p.Receiver.Status().Err; err != nil {
return nil, err
}
- return p.Receiver.Status().Value.(*edgeState).result.Clone(), nil
+ return p.Receiver.Status().Value.(*edgeState).result.CloneCachedResult(), nil
}
// newPipe creates a new request pipe between two edges
diff --git a/vendor/github.com/moby/buildkit/solver/types.go b/vendor/github.com/moby/buildkit/solver/types.go
index 76e4c91..f9de6a9 100644
--- a/vendor/github.com/moby/buildkit/solver/types.go
+++ b/vendor/github.com/moby/buildkit/solver/types.go
@@ -61,6 +61,7 @@
ID() string
Release(context.Context) error
Sys() interface{}
+ Clone() Result
}
// CachedResult is a result connected with its cache key
diff --git a/vendor/github.com/moby/buildkit/source/git/gitsource.go b/vendor/github.com/moby/buildkit/source/git/gitsource.go
index 7cef764..3d1bfe2 100644
--- a/vendor/github.com/moby/buildkit/source/git/gitsource.go
+++ b/vendor/github.com/moby/buildkit/source/git/gitsource.go
@@ -231,7 +231,7 @@
if s.token {
dt = []byte("basic " + base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("x-access-token:%s", dt))))
}
- gs.auth = []string{"-c", "http.extraheader=Authorization: " + string(dt)}
+ gs.auth = []string{"-c", "http." + tokenScope(gs.src.Remote) + ".extraheader=Authorization: " + string(dt)}
break
}
return nil
@@ -631,3 +631,14 @@
}
return out
}
+
+func tokenScope(remote string) string {
+ // generally we can only use the token for fetching main remote but in case of github.com we do best effort
+ // to try reuse same token for all github.com remotes. This is the same behavior actions/checkout uses
+ for _, pfx := range []string{"https://github.com/", "https://www.github.com/"} {
+ if strings.HasPrefix(remote, pfx) {
+ return pfx
+ }
+ }
+ return remote
+}
diff --git a/vendor/github.com/moby/buildkit/util/resolver/authorizer.go b/vendor/github.com/moby/buildkit/util/resolver/authorizer.go
index 80e2a35..e58038c 100644
--- a/vendor/github.com/moby/buildkit/util/resolver/authorizer.go
+++ b/vendor/github.com/moby/buildkit/util/resolver/authorizer.go
@@ -220,15 +220,13 @@
// authResult is used to control limit rate.
type authResult struct {
- sync.WaitGroup
token string
- err error
expires time.Time
}
// authHandler is used to handle auth request per registry server.
type authHandler struct {
- sync.Mutex
+ g flightcontrol.Group
client *http.Client
@@ -240,7 +238,8 @@
// scopedTokens caches token indexed by scopes, which used in
// bearer auth case
- scopedTokens map[string]*authResult
+ scopedTokens map[string]*authResult
+ scopedTokensMu sync.Mutex
lastUsed time.Time
@@ -292,46 +291,44 @@
// Docs: https://docs.docker.com/registry/spec/auth/scope
scoped := strings.Join(to.Scopes, " ")
- ah.Lock()
- for {
+ res, err := ah.g.Do(ctx, scoped, func(ctx context.Context) (interface{}, error) {
+ ah.scopedTokensMu.Lock()
r, exist := ah.scopedTokens[scoped]
- if !exist {
- // no entry cached
- break
- }
- ah.Unlock()
- r.Wait()
- if r.err != nil {
- select {
- case <-ctx.Done():
- return "", r.err
- default:
+ ah.scopedTokensMu.Unlock()
+ if exist {
+ if r.expires.IsZero() || r.expires.After(time.Now()) {
+ return r, nil
}
}
- if !errors.Is(r.err, context.Canceled) &&
- (r.expires.IsZero() || r.expires.After(time.Now())) {
- return r.token, r.err
+ r, err := ah.fetchToken(ctx, sm, g, to)
+ if err != nil {
+ return nil, err
}
- // r.err is canceled or token expired. Get rid of it and try again
- ah.Lock()
- r2, exist := ah.scopedTokens[scoped]
- if exist && r == r2 {
- delete(ah.scopedTokens, scoped)
- }
+ ah.scopedTokensMu.Lock()
+ ah.scopedTokens[scoped] = r
+ ah.scopedTokensMu.Unlock()
+ return r, nil
+ })
+
+ if err != nil || res == nil {
+ return "", err
}
+ r := res.(*authResult)
+ if r == nil {
+ return "", nil
+ }
+ return r.token, nil
+}
- // only one fetch token job
- r := new(authResult)
- r.Add(1)
- ah.scopedTokens[scoped] = r
- ah.Unlock()
-
+func (ah *authHandler) fetchToken(ctx context.Context, sm *session.Manager, g session.Group, to auth.TokenOptions) (r *authResult, err error) {
var issuedAt time.Time
var expires int
+ var token string
defer func() {
token = fmt.Sprintf("Bearer %s", token)
- r.token, r.err = token, err
+
if err == nil {
+ r = &authResult{token: token}
if issuedAt.IsZero() {
issuedAt = time.Now()
}
@@ -339,7 +336,6 @@
r.expires = exp
}
}
- r.Done()
}()
if ah.authority != nil {
@@ -351,10 +347,11 @@
Scopes: to.Scopes,
}, sm, g)
if err != nil {
- return "", err
+ return nil, err
}
issuedAt, expires = time.Unix(resp.IssuedAt, 0), int(resp.ExpiresIn)
- return resp.Token, nil
+ token = resp.Token
+ return nil, nil
}
// fetch token for the resource scope
@@ -374,29 +371,32 @@
if (errStatus.StatusCode == 405 && to.Username != "") || errStatus.StatusCode == 404 || errStatus.StatusCode == 401 {
resp, err := auth.FetchTokenWithOAuth(ctx, ah.client, nil, "buildkit-client", to)
if err != nil {
- return "", err
+ return nil, err
}
issuedAt, expires = resp.IssuedAt, resp.ExpiresIn
- return resp.AccessToken, nil
+ token = resp.AccessToken
+ return nil, nil
}
log.G(ctx).WithFields(logrus.Fields{
"status": errStatus.Status,
"body": string(errStatus.Body),
}).Debugf("token request failed")
}
- return "", err
+ return nil, err
}
issuedAt, expires = resp.IssuedAt, resp.ExpiresIn
- return resp.Token, nil
+ token = resp.Token
+ return nil, nil
}
// do request anonymously
resp, err := auth.FetchToken(ctx, ah.client, nil, to)
if err != nil {
- return "", errors.Wrap(err, "failed to fetch anonymous token")
+ return nil, errors.Wrap(err, "failed to fetch anonymous token")
}
issuedAt, expires = resp.IssuedAt, resp.ExpiresIn
- return resp.Token, nil
+ token = resp.Token
+ return nil, nil
}
func invalidAuthorization(c auth.Challenge, responses []*http.Response) error {
diff --git a/vendor/github.com/moby/buildkit/worker/result.go b/vendor/github.com/moby/buildkit/worker/result.go
index e178ef3..9f4cdc5 100644
--- a/vendor/github.com/moby/buildkit/worker/result.go
+++ b/vendor/github.com/moby/buildkit/worker/result.go
@@ -52,3 +52,11 @@
func (r *workerRefResult) Sys() interface{} {
return r.WorkerRef
}
+
+func (r *workerRefResult) Clone() solver.Result {
+ r2 := *r
+ if r.ImmutableRef != nil {
+ r.ImmutableRef = r.ImmutableRef.Clone()
+ }
+ return &r2
+}
