Merge pull request #373 from tonistiigi/19.03-buildkit

[19.03] vendor: update buildkit for 19.03

vendor.conf

@@ -27,8 +27,8 @@ github.com/imdario/mergo                            7c29201646fa3de8506f70121347
 golang.org/x/sync                                   e225da77a7e68af35c70ccbf71af2b83e6acac3c
 
 # buildkit
-github.com/moby/buildkit                            588c73e1e4f0f3d7d3738abaaa7cf8026064b33e
+github.com/moby/buildkit                            ae10b292fefb00e0fbf9fecd1419c5f252e58895
-github.com/tonistiigi/fsutil                        3bbb99cdbd76619ab717299830c60f6f2a533a6b
+github.com/tonistiigi/fsutil                        3d2716dd0a4d06ff854241c7e8b6f3f904e1719f
 github.com/grpc-ecosystem/grpc-opentracing          8e809c8a86450a29b90dcc9efbf062d0fe6d9746
 github.com/opentracing/opentracing-go               1361b9cd60be79c4c3a7fa9841b3c132e40066a7
 github.com/google/shlex                             6f45313302b9c56850fc17f99e40caebce98c716
@@ -125,7 +125,7 @@ github.com/containerd/fifo                          a9fb20d87448d386e6d50b1f2e1f
 github.com/containerd/continuity                    aaeac12a7ffcd198ae25440a9dff125c2e2703a7
 github.com/containerd/cgroups                       4994991857f9b0ae8dc439551e8bebdbb4bf66c1
 github.com/containerd/console                       0650fd9eeb50bab4fc99dceb9f2e14cf58f36e7f
-github.com/containerd/go-runc                       7d11b49dc0769f6dbb0d1b19f3d48524d1bad9ad
+github.com/containerd/go-runc                       e029b79d8cda8374981c64eba71f28ec38e5526f
 github.com/containerd/typeurl                       2a93cfde8c20b23de8eb84a5adbc234ddf7a9e8d
 github.com/containerd/ttrpc                         92c8520ef9f86600c650dd540266a007bf03670f
 github.com/gogo/googleapis                          d31c731455cb061f42baff3bda55bad0118b126b # v1.2.0

vendor/github.com/containerd/go-runc/command_linux.go

@@ -20,6 +20,7 @@ import (
 	"context"
 	"os"
 	"os/exec"
+	"strings"
 	"syscall"
 )
 
@@ -32,10 +33,24 @@ func (r *Runc) command(context context.Context, args ...string) *exec.Cmd {
 	cmd.SysProcAttr = &syscall.SysProcAttr{
 		Setpgid: r.Setpgid,
 	}
-	cmd.Env = os.Environ()
+	cmd.Env = filterEnv(os.Environ(), "NOTIFY_SOCKET") // NOTIFY_SOCKET introduces a special behavior in runc but should only be set if invoked from systemd
 	if r.PdeathSignal != 0 {
 		cmd.SysProcAttr.Pdeathsig = r.PdeathSignal
 	}
 
 	return cmd
 }
+
+func filterEnv(in []string, names ...string) []string {
+	out := make([]string, 0, len(in))
+loop0:
+	for _, v := range in {
+		for _, k := range names {
+			if strings.HasPrefix(v, k+"=") {
+				continue loop0
+			}
+		}
+		out = append(out, v)
+	}
+	return out
+}

vendor/github.com/containerd/go-runc/runc.go

@@ -275,7 +275,11 @@ func (r *Runc) Run(context context.Context, id, bundle string, opts *CreateOpts)
 	if err != nil {
 		return -1, err
 	}
-	return Monitor.Wait(cmd, ec)
+	status, err := Monitor.Wait(cmd, ec)
+	if err == nil && status != 0 {
+		err = fmt.Errorf("%s did not terminate sucessfully", cmd.Args[0])
+	}
+	return status, err
 }
 
 type DeleteOpts struct {
@@ -570,7 +574,11 @@ func (r *Runc) Restore(context context.Context, id, bundle string, opts *Restore
 			}
 		}
 	}
-	return Monitor.Wait(cmd, ec)
+	status, err := Monitor.Wait(cmd, ec)
+	if err == nil && status != 0 {
+		err = fmt.Errorf("%s did not terminate sucessfully", cmd.Args[0])
+	}
+	return status, err
 }
 
 // Update updates the current container with the provided resource spec

vendor/github.com/moby/buildkit/executor/runcexecutor/executor.go

@@ -293,12 +293,11 @@ func (w *runcExecutor) Exec(ctx context.Context, meta executor.Meta, root cache.
 		NoPivot: w.noPivot,
 	})
 	close(done)
-	if err != nil {
-		return err
-	}
 
-	if status != 0 {
+	if status != 0 || err != nil {
-		err := errors.Errorf("exit code: %d", status)
+		if err == nil {
+			err = errors.Errorf("exit code: %d", status)
+		}
 		select {
 		case <-ctx.Done():
 			return errors.Wrapf(ctx.Err(), err.Error())

vendor/github.com/moby/buildkit/go.mod

@@ -10,10 +10,10 @@ require (
 	github.com/containerd/cgroups v0.0.0-20190226200435-dbea6f2bd416 // indirect
 	github.com/containerd/console v0.0.0-20181022165439-0650fd9eeb50
 	github.com/containerd/containerd v1.3.0-0.20190507210959-7c1e88399ec0
-	github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc
+	github.com/containerd/continuity v0.0.0-20190827140505-75bee3e2ccb6
 	github.com/containerd/fifo v0.0.0-20180307165137-3d5202aec260 // indirect
 	github.com/containerd/go-cni v0.0.0-20190610170741-5a4663dad645
-	github.com/containerd/go-runc v0.0.0-20180907222934-5a6d9f37cfa3
+	github.com/containerd/go-runc v0.0.0-20190911050354-e029b79d8cda
 	github.com/containerd/ttrpc v0.0.0-20190411181408-699c4e40d1e7 // indirect
 	github.com/containerd/typeurl v0.0.0-20180627222232-a93fcdb778cd // indirect
 	github.com/containernetworking/cni v0.6.1-0.20180218032124-142cde0c766c // indirect
@@ -53,7 +53,7 @@ require (
 	github.com/sirupsen/logrus v1.3.0
 	github.com/stretchr/testify v1.3.0
 	github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2 // indirect
-	github.com/tonistiigi/fsutil v0.0.0-20190327153851-3bbb99cdbd76
+	github.com/tonistiigi/fsutil v0.0.0-20190819224149-3d2716dd0a4d
 	github.com/tonistiigi/units v0.0.0-20180711220420-6950e57a87ea
 	github.com/uber/jaeger-client-go v0.0.0-20180103221425-e02c85f9069e
 	github.com/uber/jaeger-lib v1.2.1 // indirect

vendor/github.com/moby/buildkit/session/sshforward/copy.go

@@ -9,17 +9,17 @@ import (
 	"google.golang.org/grpc"
 )
 
-func Copy(ctx context.Context, conn io.ReadWriteCloser, stream grpc.Stream) error {
+func Copy(ctx context.Context, conn io.ReadWriteCloser, stream grpc.Stream, closeStream func() error) error {
 	g, ctx := errgroup.WithContext(ctx)
 
 	g.Go(func() (retErr error) {
 		p := &BytesMessage{}
 		for {
 			if err := stream.RecvMsg(p); err != nil {
+				conn.Close()
 				if err == io.EOF {
 					return nil
 				}
-				conn.Close()
 				return errors.WithStack(err)
 			}
 			select {
@@ -42,6 +42,9 @@ func Copy(ctx context.Context, conn io.ReadWriteCloser, stream grpc.Stream) erro
 			n, err := conn.Read(buf)
 			switch {
 			case err == io.EOF:
+				if closeStream != nil {
+					closeStream()
+				}
 				return nil
 			case err != nil:
 				return errors.WithStack(err)

vendor/github.com/moby/buildkit/session/sshforward/ssh.go

@@ -49,7 +49,7 @@ func (s *server) run(ctx context.Context, l net.Listener, id string) error {
 				return err
 			}
 
-			go Copy(ctx, conn, stream)
+			go Copy(ctx, conn, stream, stream.CloseSend)
 		}
 	})
 

vendor/github.com/moby/buildkit/solver/edge.go

@@ -177,6 +177,9 @@ func (e *edge) finishIncoming(req pipe.Sender) {
 
 // updateIncoming updates the current value of incoming pipe request
 func (e *edge) updateIncoming(req pipe.Sender) {
+	if debugScheduler {
+		logrus.Debugf("updateIncoming %s %#v desired=%s", e.edge.Vertex.Name(), e.edgeState, req.Request().Payload.(*edgeRequest).desiredState)
+	}
 	req.Update(&e.edgeState)
 }
 

vendor/github.com/moby/buildkit/solver/internal/pipe/pipe.go

@@ -11,11 +11,15 @@ import (
 type channel struct {
 	OnSendCompletion func()
 	value            atomic.Value
-	lastValue        interface{}
+	lastValue        *wrappedValue
+}
+
+type wrappedValue struct {
+	value interface{}
 }
 
 func (c *channel) Send(v interface{}) {
-	c.value.Store(v)
+	c.value.Store(&wrappedValue{value: v})
 	if c.OnSendCompletion != nil {
 		c.OnSendCompletion()
 	}
@@ -23,11 +27,11 @@ func (c *channel) Send(v interface{}) {
 
 func (c *channel) Receive() (interface{}, bool) {
 	v := c.value.Load()
-	if c.lastValue == v {
+	if v == nil || v.(*wrappedValue) == c.lastValue {
 		return nil, false
 	}
-	c.lastValue = v
+	c.lastValue = v.(*wrappedValue)
-	return v, true
+	return v.(*wrappedValue).value, true
 }
 
 type Pipe struct {

vendor/github.com/moby/buildkit/solver/llbsolver/bridge.go

@@ -54,7 +54,7 @@ func (b *llbBridge) Solve(ctx context.Context, req frontend.SolveRequest) (res *
 			}
 		}
 		if prevCm, ok := b.cms[cmId]; !ok {
-			func(cmId string) {
+			func(cmId string, im gw.CacheOptionsEntry) {
 				cm = newLazyCacheManager(cmId, func() (solver.CacheManager, error) {
 					var cmNew solver.CacheManager
 					if err := inVertexContext(b.builder.Context(ctx), "importing cache manifest from "+cmId, "", func(ctx context.Context) error {
@@ -74,7 +74,7 @@ func (b *llbBridge) Solve(ctx context.Context, req frontend.SolveRequest) (res *
 					}
 					return cmNew, nil
 				})
-			}(cmId)
+			}(cmId, im)
 			b.cms[cmId] = cm
 		} else {
 			cm = prevCm

vendor/github.com/moby/buildkit/source/http/httpsource.go

@@ -144,6 +144,11 @@ func (hs *httpSourceHandler) CacheKey(ctx context.Context, index int) (string, b
 	req = req.WithContext(ctx)
 	m := map[string]*metadata.StorageItem{}
 
+	// If we request a single ETag in 'If-None-Match', some servers omit the
+	// unambiguous ETag in their response.
+	// See: https://github.com/moby/buildkit/issues/905
+	var onlyETag string
+
 	if len(sis) > 0 {
 		for _, si := range sis {
 			// if metaDigest := getMetaDigest(si); metaDigest == hs.formatCacheKey("") {
@@ -160,6 +165,10 @@ func (hs *httpSourceHandler) CacheKey(ctx context.Context, index int) (string, b
 				etags = append(etags, t)
 			}
 			req.Header.Add("If-None-Match", strings.Join(etags, ", "))
+
+			if len(etags) == 1 {
+				onlyETag = etags[0]
+			}
 		}
 	}
 
@@ -172,6 +181,12 @@ func (hs *httpSourceHandler) CacheKey(ctx context.Context, index int) (string, b
 		if err == nil {
 			if resp.StatusCode == http.StatusOK || resp.StatusCode == http.StatusNotModified {
 				respETag := resp.Header.Get("ETag")
+
+				// If a 304 is returned without an ETag and we had only sent one ETag,
+				// the response refers to the ETag we asked about.
+				if respETag == "" && onlyETag != "" && resp.StatusCode == http.StatusNotModified {
+					respETag = onlyETag
+				}
 				si, ok := m[respETag]
 				if ok {
 					hs.refID = si.ID()
@@ -197,6 +212,13 @@ func (hs *httpSourceHandler) CacheKey(ctx context.Context, index int) (string, b
 	}
 	if resp.StatusCode == http.StatusNotModified {
 		respETag := resp.Header.Get("ETag")
+		if respETag == "" && onlyETag != "" {
+			respETag = onlyETag
+
+			// Set the missing ETag header on the response so that it's available
+			// to .save()
+			resp.Header.Set("ETag", onlyETag)
+		}
 		si, ok := m[respETag]
 		if !ok {
 			return "", false, errors.Errorf("invalid not-modified ETag: %v", respETag)

vendor/github.com/tonistiigi/fsutil/copy/copy.go

@@ -329,21 +329,6 @@ func ensureEmptyFileTarget(dst string) error {
 	return os.Remove(dst)
 }
 
-func copyFile(source, target string) error {
-	src, err := os.Open(source)
-	if err != nil {
-		return errors.Wrapf(err, "failed to open source %s", source)
-	}
-	defer src.Close()
-	tgt, err := os.Create(target)
-	if err != nil {
-		return errors.Wrapf(err, "failed to open target %s", target)
-	}
-	defer tgt.Close()
-
-	return copyFileContent(tgt, src)
-}
-
 func containsWildcards(name string) bool {
 	isWindows := runtime.GOOS == "windows"
 	for i := 0; i < len(name); i++ {

vendor/github.com/tonistiigi/fsutil/copy/copy_darwin.go

@@ -0,0 +1,84 @@
+// +build darwin
+
+package fs
+
+import (
+	"io"
+	"os"
+	"syscall"
+	"unsafe"
+
+	"github.com/pkg/errors"
+	"golang.org/x/sys/unix"
+)
+
+// <sys/clonefile.h
+// int clonefileat(int, const char *, int, const char *, uint32_t) __OSX_AVAILABLE(10.12) __IOS_AVAILABLE(10.0) __TVOS_AVAILABLE(10.0) __WATCHOS_AVAILABLE(3.0);
+
+const CLONE_NOFOLLOW = 0x0001    /* Don't follow symbolic links */
+const CLONE_NOOWNERCOPY = 0x0002 /* Don't copy ownership information from */
+
+func copyFile(source, target string) error {
+	if err := clonefile(source, target); err != nil {
+		if err != unix.EINVAL {
+			return err
+		}
+	} else {
+		return nil
+	}
+
+	src, err := os.Open(source)
+	if err != nil {
+		return errors.Wrapf(err, "failed to open source %s", source)
+	}
+	defer src.Close()
+	tgt, err := os.Create(target)
+	if err != nil {
+		return errors.Wrapf(err, "failed to open target %s", target)
+	}
+	defer tgt.Close()
+
+	return copyFileContent(tgt, src)
+}
+
+func copyFileContent(dst, src *os.File) error {
+	buf := bufferPool.Get().(*[]byte)
+	_, err := io.CopyBuffer(dst, src, *buf)
+	bufferPool.Put(buf)
+
+	return err
+}
+
+// errnoErr returns common boxed Errno values, to prevent
+// allocations at runtime.
+func errnoErr(e syscall.Errno) error {
+	switch e {
+	case 0:
+		return nil
+	case unix.EAGAIN:
+		return syscall.EAGAIN
+	case unix.EINVAL:
+		return syscall.EINVAL
+	case unix.ENOENT:
+		return syscall.ENOENT
+	}
+	return e
+}
+
+func clonefile(src, dst string) (err error) {
+	var _p0, _p1 *byte
+	_p0, err = unix.BytePtrFromString(src)
+	if err != nil {
+		return
+	}
+	_p1, err = unix.BytePtrFromString(dst)
+	if err != nil {
+		return
+	}
+	fdcwd := unix.AT_FDCWD
+	_, _, e1 := unix.Syscall6(unix.SYS_CLONEFILEAT, uintptr(fdcwd), uintptr(unsafe.Pointer(_p0)), uintptr(fdcwd), uintptr(unsafe.Pointer(_p1)), uintptr(CLONE_NOFOLLOW), 0)
+	if e1 != 0 {
+		err = errnoErr(e1)
+	}
+	return
+}

vendor/github.com/tonistiigi/fsutil/copy/copy_linux.go

@@ -52,6 +52,21 @@ func (c *copier) copyFileInfo(fi os.FileInfo, name string) error {
 	return nil
 }
 
+func copyFile(source, target string) error {
+	src, err := os.Open(source)
+	if err != nil {
+		return errors.Wrapf(err, "failed to open source %s", source)
+	}
+	defer src.Close()
+	tgt, err := os.Create(target)
+	if err != nil {
+		return errors.Wrapf(err, "failed to open target %s", target)
+	}
+	defer tgt.Close()
+
+	return copyFileContent(tgt, src)
+}
+
 func copyFileContent(dst, src *os.File) error {
 	st, err := src.Stat()
 	if err != nil {

vendor/github.com/tonistiigi/fsutil/copy/copy_unix.go

@@ -3,7 +3,6 @@
 package fs
 
 import (
-	"io"
 	"os"
 	"syscall"
 
@@ -51,14 +50,6 @@ func (c *copier) copyFileInfo(fi os.FileInfo, name string) error {
 	return nil
 }
 
-func copyFileContent(dst, src *os.File) error {
-	buf := bufferPool.Get().(*[]byte)
-	_, err := io.CopyBuffer(dst, src, *buf)
-	bufferPool.Put(buf)
-
-	return err
-}
-
 func copyDevice(dst string, fi os.FileInfo) error {
 	st, ok := fi.Sys().(*syscall.Stat_t)
 	if !ok {

vendor/github.com/tonistiigi/fsutil/copy/copy_windows.go

@@ -17,6 +17,21 @@ func (c *copier) copyFileInfo(fi os.FileInfo, name string) error {
 	return nil
 }
 
+func copyFile(source, target string) error {
+	src, err := os.Open(source)
+	if err != nil {
+		return errors.Wrapf(err, "failed to open source %s", source)
+	}
+	defer src.Close()
+	tgt, err := os.Create(target)
+	if err != nil {
+		return errors.Wrapf(err, "failed to open target %s", target)
+	}
+	defer tgt.Close()
+
+	return copyFileContent(tgt, src)
+}
+
 func copyFileContent(dst, src *os.File) error {
 	buf := bufferPool.Get().(*[]byte)
 	_, err := io.CopyBuffer(dst, src, *buf)

vendor/github.com/tonistiigi/fsutil/stat.go

@@ -49,6 +49,9 @@ func mkstat(path, relpath string, fi os.FileInfo, inodemap map[uint64]string) (*
 		stat.Mode = noPermPart | permPart
 	}
 
+	// Clear the socket bit since archive/tar.FileInfoHeader does not handle it
+	stat.Mode &^= uint32(os.ModeSocket)
+
 	return stat, nil
 }