Merge pull request #41793 from tiborvass/bump_buildkit_0.8.1

Vendor buildkit to v0.8.1

vendor.conf

@@ -33,7 +33,7 @@ github.com/imdario/mergo                            1afb36080aec31e0d1528973ebe6
 golang.org/x/sync                                   cd5d95a43a6e21273425c7ae415d3df9ea832eeb
 
 # buildkit
-github.com/moby/buildkit                            950603da215ae03b843f3f66fbe86c4876a6f5a1
+github.com/moby/buildkit                            8142d66b5ebde79846b869fba30d9d30633e74aa # v0.8.1
 github.com/tonistiigi/fsutil                        0834f99b7b85462efb69b4f571a4fa3ca7da5ac9
 github.com/tonistiigi/units                         6950e57a87eaf136bbe44ef2ec8e75b9e3569de2
 github.com/grpc-ecosystem/grpc-opentracing          8e809c8a86450a29b90dcc9efbf062d0fe6d9746

vendor/github.com/moby/buildkit/README.md

@@ -3,7 +3,7 @@
 # BuildKit
 
 [![GoDoc](https://godoc.org/github.com/moby/buildkit?status.svg)](https://godoc.org/github.com/moby/buildkit/client/llb)
-[![Build Status](https://travis-ci.com/moby/buildkit.svg?branch=master)](https://travis-ci.com/moby/buildkit)
+[![Build Status](https://github.com/moby/buildkit/workflows/build/badge.svg)](https://github.com/moby/buildkit/actions?query=workflow%3Abuild)
 [![Go Report Card](https://goreportcard.com/badge/github.com/moby/buildkit)](https://goreportcard.com/report/github.com/moby/buildkit)
 [![codecov](https://codecov.io/gh/moby/buildkit/branch/master/graph/badge.svg)](https://codecov.io/gh/moby/buildkit)
 
@@ -28,7 +28,7 @@ Introductory blog post https://blog.mobyproject.org/introducing-buildkit-17e056c
 
 Join `#buildkit` channel on [Docker Community Slack](http://dockr.ly/slack)
 
-:information_source: If you are visiting this repo for the usage of experimental Dockerfile features like `RUN --mount=type=(bind|cache|tmpfs|secret|ssh)`, please refer to [`frontend/dockerfile/docs/experimental.md`](frontend/dockerfile/docs/experimental.md).
+:information_source: If you are visiting this repo for the usage of BuildKit-only Dockerfile features like `RUN --mount=type=(bind|cache|tmpfs|secret|ssh)`, please refer to [`frontend/dockerfile/docs/syntax.md`](frontend/dockerfile/docs/syntax.md).
 
 :information_source: [BuildKit has been integrated to `docker build` since Docker 18.06 .](https://docs.docker.com/develop/develop-images/build_enhancements/)
 You don't need to read this document unless you want to use the full-featured standalone version of BuildKit.
@@ -178,7 +178,7 @@ buildctl build \
 
 #### Building a Dockerfile using external frontend:
 
-External versions of the Dockerfile frontend are pushed to https://hub.docker.com/r/docker/dockerfile-upstream and https://hub.docker.com/r/docker/dockerfile and can be used with the gateway frontend. The source for the external frontend is currently located in `./frontend/dockerfile/cmd/dockerfile-frontend` but will move out of this repository in the future ([#163](https://github.com/moby/buildkit/issues/163)). For automatic build from master branch of this repository `docker/dockerfile-upsteam:master` or `docker/dockerfile-upstream:master-experimental` image can be used.
+External versions of the Dockerfile frontend are pushed to https://hub.docker.com/r/docker/dockerfile-upstream and https://hub.docker.com/r/docker/dockerfile and can be used with the gateway frontend. The source for the external frontend is currently located in `./frontend/dockerfile/cmd/dockerfile-frontend` but will move out of this repository in the future ([#163](https://github.com/moby/buildkit/issues/163)). For automatic build from master branch of this repository `docker/dockerfile-upstream:master` or `docker/dockerfile-upstream:master-labs` image can be used.
 
 ```bash
 buildctl build \
@@ -435,7 +435,7 @@ For Kubernetes deployments, see [`examples/kubernetes`](./examples/kubernetes).
 
 ### Daemonless
 
-To run client and an ephemeral daemon in a single container ("daemonless mode"):
+To run the client and an ephemeral daemon in a single container ("daemonless mode"):
 
 ```bash
 docker run \

vendor/github.com/moby/buildkit/client/llb/fileop.go

@@ -661,6 +661,14 @@ func (f *FileOp) Marshal(ctx context.Context, c *Constraints) (digest.Digest, []
 
 	pfo := &pb.FileOp{}
 
+	if f.constraints.Platform == nil {
+		p, err := getPlatform(*f.action.state)(ctx)
+		if err != nil {
+			return "", nil, nil, nil, err
+		}
+		f.constraints.Platform = p
+	}
+
 	pop, md := MarshalConstraints(c, &f.constraints)
 	pop.Op = &pb.Op_File{
 		File: pfo,

vendor/github.com/moby/buildkit/client/llb/source.go

@@ -5,7 +5,6 @@ import (
 	_ "crypto/sha256" // for opencontainers/go-digest
 	"encoding/json"
 	"os"
-	"regexp"
 	"strconv"
 	"strings"
 
@@ -207,8 +206,6 @@ const (
 	gitProtocolUnknown
 )
 
-var gitSSHRegex = regexp.MustCompile("^([a-z0-9]+@)?[^:]+:.*$")
-
 func getGitProtocol(remote string) (string, int) {
 	prefixes := map[string]int{
 		"http://":  gitProtocolHTTP,
@@ -224,7 +221,7 @@ func getGitProtocol(remote string) (string, int) {
 		}
 	}
 
-	if protocolType == gitProtocolUnknown && gitSSHRegex.MatchString(remote) {
+	if protocolType == gitProtocolUnknown && sshutil.IsSSHTransport(remote) {
 		protocolType = gitProtocolSSH
 	}
 
@@ -254,6 +251,9 @@ func Git(remote, ref string, opts ...GitOption) State {
 			remote = parts[0] + "/" + parts[1]
 		}
 	}
+	if protocolType == gitProtocolUnknown {
+		url = "https://" + url
+	}
 
 	id := remote
 

vendor/github.com/moby/buildkit/solver/edge.go

@@ -595,6 +595,7 @@ func (e *edge) recalcCurrentState() {
 	stHigh := edgeStatusCacheSlow // maximum possible state
 	if e.cacheMap != nil {
 		for _, dep := range e.deps {
+			isSlowCacheIncomplete := e.slowCacheFunc(dep) != nil && (dep.state == edgeStatusCacheSlow || (dep.state == edgeStatusComplete && !dep.slowCacheComplete))
 			isSlowIncomplete := (e.slowCacheFunc(dep) != nil || e.preprocessFunc(dep) != nil) && (dep.state == edgeStatusCacheSlow || (dep.state == edgeStatusComplete && !dep.slowCacheComplete))
 
 			if dep.state > stLow && len(dep.keyMap) == 0 && !isSlowIncomplete {
@@ -604,10 +605,10 @@ func (e *edge) recalcCurrentState() {
 				}
 			}
 			effectiveState := dep.state
-			if dep.state == edgeStatusCacheSlow && isSlowIncomplete {
+			if dep.state == edgeStatusCacheSlow && isSlowCacheIncomplete {
 				effectiveState = edgeStatusCacheFast
 			}
-			if dep.state == edgeStatusComplete && isSlowIncomplete {
+			if dep.state == edgeStatusComplete && isSlowCacheIncomplete {
 				effectiveState = edgeStatusCacheFast
 			}
 			if effectiveState < stHigh {
@@ -619,7 +620,7 @@ func (e *edge) recalcCurrentState() {
 			if dep.state < edgeStatusCacheFast {
 				allDepsCompletedCacheFast = false
 			}
-			if isSlowIncomplete || dep.state < edgeStatusCacheSlow {
+			if isSlowCacheIncomplete || dep.state < edgeStatusCacheSlow {
 				allDepsCompletedCacheSlow = false
 			}
 			if dep.state < edgeStatusCacheSlow && len(dep.keyMap) == 0 {

vendor/github.com/moby/buildkit/solver/jobs.go

@@ -527,7 +527,13 @@ func (j *Job) Discard() error {
 		st.mu.Unlock()
 	}
 
-	delete(j.list.jobs, j.id)
+	go func() {
+		// don't clear job right away. there might still be a status request coming to read progress
+		time.Sleep(10 * time.Second)
+		j.list.mu.Lock()
+		defer j.list.mu.Unlock()
+		delete(j.list.jobs, j.id)
+	}()
 	return nil
 }
 

vendor/github.com/moby/buildkit/solver/llbsolver/solver.go

@@ -131,6 +131,10 @@ func (s *Solver) Solve(ctx context.Context, id string, sessionID string, req fro
 		}
 	}
 
+	if res == nil {
+		res = &frontend.Result{}
+	}
+
 	defer func() {
 		res.EachRef(func(ref solver.ResultProxy) error {
 			go ref.Release(context.TODO())

vendor/github.com/moby/buildkit/source/gitidentifier.go

@@ -4,6 +4,7 @@ import (
 	"net/url"
 	"strings"
 
+	"github.com/moby/buildkit/util/sshutil"
 	"github.com/pkg/errors"
 )
 
@@ -58,7 +59,7 @@ func (i *GitIdentifier) ID() string {
 // isGitTransport returns true if the provided str is a git transport by inspecting
 // the prefix of the string for known protocols used in git.
 func isGitTransport(str string) bool {
-	return strings.HasPrefix(str, "http://") || strings.HasPrefix(str, "https://") || strings.HasPrefix(str, "git://") || strings.HasPrefix(str, "git@")
+	return strings.HasPrefix(str, "http://") || strings.HasPrefix(str, "https://") || strings.HasPrefix(str, "git://") || sshutil.IsSSHTransport(str)
 }
 
 func getRefAndSubdir(fragment string) (ref string, subdir string) {

vendor/github.com/moby/buildkit/source/identifier.go

@@ -102,6 +102,9 @@ func FromLLB(op *pb.Op_Source, platform *pb.Platform) (Identifier, error) {
 					id.KeepGitDir = true
 				}
 			case pb.AttrFullRemoteURL:
+				if !isGitTransport(v) {
+					v = "https://" + v
+				}
 				id.Remote = v
 			case pb.AttrAuthHeaderSecret:
 				id.AuthHeaderSecret = v

vendor/github.com/moby/buildkit/util/compression/compression.go

@@ -99,17 +99,21 @@ func detectCompressionType(cr io.Reader) (Type, error) {
 }
 
 var toDockerLayerType = map[string]string{
-	ocispec.MediaTypeImageLayer:            images.MediaTypeDockerSchema2Layer,
-	images.MediaTypeDockerSchema2Layer:     images.MediaTypeDockerSchema2Layer,
-	ocispec.MediaTypeImageLayerGzip:        images.MediaTypeDockerSchema2LayerGzip,
-	images.MediaTypeDockerSchema2LayerGzip: images.MediaTypeDockerSchema2LayerGzip,
+	ocispec.MediaTypeImageLayer:                   images.MediaTypeDockerSchema2Layer,
+	images.MediaTypeDockerSchema2Layer:            images.MediaTypeDockerSchema2Layer,
+	ocispec.MediaTypeImageLayerGzip:               images.MediaTypeDockerSchema2LayerGzip,
+	images.MediaTypeDockerSchema2LayerGzip:        images.MediaTypeDockerSchema2LayerGzip,
+	images.MediaTypeDockerSchema2LayerForeign:     images.MediaTypeDockerSchema2Layer,
+	images.MediaTypeDockerSchema2LayerForeignGzip: images.MediaTypeDockerSchema2LayerGzip,
 }
 
 var toOCILayerType = map[string]string{
-	ocispec.MediaTypeImageLayer:            ocispec.MediaTypeImageLayer,
-	images.MediaTypeDockerSchema2Layer:     ocispec.MediaTypeImageLayer,
-	ocispec.MediaTypeImageLayerGzip:        ocispec.MediaTypeImageLayerGzip,
-	images.MediaTypeDockerSchema2LayerGzip: ocispec.MediaTypeImageLayerGzip,
+	ocispec.MediaTypeImageLayer:                   ocispec.MediaTypeImageLayer,
+	images.MediaTypeDockerSchema2Layer:            ocispec.MediaTypeImageLayer,
+	ocispec.MediaTypeImageLayerGzip:               ocispec.MediaTypeImageLayerGzip,
+	images.MediaTypeDockerSchema2LayerGzip:        ocispec.MediaTypeImageLayerGzip,
+	images.MediaTypeDockerSchema2LayerForeign:     ocispec.MediaTypeImageLayer,
+	images.MediaTypeDockerSchema2LayerForeignGzip: ocispec.MediaTypeImageLayerGzip,
 }
 
 func convertLayerMediaType(mediaType string, oci bool) string {

vendor/github.com/moby/buildkit/util/flightcontrol/flightcontrol.go

@@ -130,7 +130,11 @@ func (c *call) wait(ctx context.Context) (v interface{}, err error) {
 	c.mu.Lock()
 	// detect case where caller has just returned, let it clean up before
 	select {
-	case <-c.ready: // could return if no error
+	case <-c.ready:
+		c.mu.Unlock()
+		<-c.cleaned
+		return nil, errRetry
+	case <-c.ctx.done: // could return if no error
 		c.mu.Unlock()
 		<-c.cleaned
 		return nil, errRetry
@@ -141,6 +145,10 @@ func (c *call) wait(ctx context.Context) (v interface{}, err error) {
 	if ok {
 		c.progressState.add(pw)
 	}
+
+	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
+
 	c.ctxs = append(c.ctxs, ctx)
 
 	c.mu.Unlock()
@@ -149,18 +157,16 @@ func (c *call) wait(ctx context.Context) (v interface{}, err error) {
 
 	select {
 	case <-ctx.Done():
-		select {
-		case <-c.ctx.Done():
+		if c.ctx.checkDone() {
 			// if this cancelled the last context, then wait for function to shut down
 			// and don't accept any more callers
 			<-c.ready
 			return c.result, c.err
-		default:
-			if ok {
-				c.progressState.close(pw)
-			}
-			return nil, ctx.Err()
 		}
+		if ok {
+			c.progressState.close(pw)
+		}
+		return nil, ctx.Err()
 	case <-c.ready:
 		return c.result, c.err // shared not implemented yet
 	}
@@ -183,9 +189,6 @@ func (c *call) Deadline() (deadline time.Time, ok bool) {
 }
 
 func (c *call) Done() <-chan struct{} {
-	c.mu.Lock()
-	c.ctx.signal()
-	c.mu.Unlock()
 	return c.ctx.done
 }
 
@@ -238,23 +241,28 @@ func newContext(c *call) *sharedContext {
 	return &sharedContext{call: c, done: make(chan struct{})}
 }
 
-// call with lock
-func (c *sharedContext) signal() {
+func (sc *sharedContext) checkDone() bool {
+	sc.mu.Lock()
 	select {
-	case <-c.done:
+	case <-sc.done:
+		sc.mu.Unlock()
+		return true
 	default:
-		var err error
-		for _, ctx := range c.ctxs {
-			select {
-			case <-ctx.Done():
-				err = ctx.Err()
-			default:
-				return
-			}
+	}
+	var err error
+	for _, ctx := range sc.ctxs {
+		select {
+		case <-ctx.Done():
+			err = ctx.Err()
+		default:
+			sc.mu.Unlock()
+			return false
 		}
-		c.err = err
-		close(c.done)
 	}
+	sc.err = err
+	close(sc.done)
+	sc.mu.Unlock()
+	return true
 }
 
 type rawProgressWriter interface {

vendor/github.com/moby/buildkit/util/sshutil/transport_validation.go

@@ -0,0 +1,11 @@
+package sshutil
+
+import (
+	"regexp"
+)
+
+var gitSSHRegex = regexp.MustCompile("^[a-zA-Z0-9-_]+@[a-zA-Z0-9-.]+:.*$")
+
+func IsSSHTransport(s string) bool {
+	return gitSSHRegex.MatchString(s)
+}