vendor: github.com/tonistiigi/fsutil d72af97c0eaf93c1d20360e3cb9c63c223675b83

full diff: https://github.com/tonistiigi/fsutil/compare/0834f99b7b85462efb69b4f571a4fa3ca7da5ac9...d72af97c0eaf93c1d20360e3cb9c63c223675b83

Signed-off-by: Sebastiaan van Stijn <github@gone.nl>

vendor.conf

@@ -33,7 +33,7 @@ golang.org/x/sync                                   036812b2e83c0ddf193dd5a34e03
 
 # buildkit
 github.com/moby/buildkit                            7e03277b32d4f0150bed0e081d4253b3a8557f13 https://github.com/cpuguy83/buildkit.git # v0.8.3-3-g244e8cde + libnetwork changes
-github.com/tonistiigi/fsutil                        0834f99b7b85462efb69b4f571a4fa3ca7da5ac9
+github.com/tonistiigi/fsutil                        d72af97c0eaf93c1d20360e3cb9c63c223675b83
 github.com/tonistiigi/units                         6950e57a87eaf136bbe44ef2ec8e75b9e3569de2
 github.com/grpc-ecosystem/grpc-opentracing          8e809c8a86450a29b90dcc9efbf062d0fe6d9746
 github.com/opentracing/opentracing-go               d34af3eaa63c4d08ab54863a4bdd0daa45212e12 # v1.2.0

vendor/github.com/tonistiigi/fsutil/copy/copy.go

@@ -11,6 +11,7 @@ import (
 	"time"
 
 	"github.com/containerd/continuity/fs"
+	"github.com/docker/docker/pkg/fileutils"
 	"github.com/pkg/errors"
 )
 
@@ -86,7 +87,10 @@ func Copy(ctx context.Context, srcRoot, src, dstRoot, dst string, opts ...Opt) e
 		return err
 	}
 
-	c := newCopier(ci.Chown, ci.Utime, ci.Mode, ci.XAttrErrorHandler)
+	c, err := newCopier(ci.Chown, ci.Utime, ci.Mode, ci.XAttrErrorHandler, ci.IncludePatterns, ci.ExcludePatterns)
+	if err != nil {
+		return err
+	}
 	srcs := []string{src}
 
 	if ci.AllowWildcards {
@@ -109,7 +113,8 @@ func Copy(ctx context.Context, srcRoot, src, dstRoot, dst string, opts ...Opt) e
 		if err != nil {
 			return err
 		}
-		if err := c.copy(ctx, srcFollowed, dst, false); err != nil {
+		skipIncludePatterns := c.includePatternMatcher == nil
+		if err := c.copy(ctx, srcFollowed, "", dst, false, skipIncludePatterns); err != nil {
 			return err
 		}
 	}
@@ -162,6 +167,10 @@ type CopyInfo struct {
 	XAttrErrorHandler XAttrErrorHandler
 	CopyDirContents   bool
 	FollowLinks       bool
+	// Include only files/dirs matching at least one of these patterns
+	IncludePatterns []string
+	// Exclude files/dir matching any of these patterns (even if they match an include pattern)
+	ExcludePatterns []string
 }
 
 type Opt func(*CopyInfo)
@@ -197,36 +206,112 @@ func AllowXAttrErrors(ci *CopyInfo) {
 	WithXAttrErrorHandler(h)(ci)
 }
 
+func WithIncludePattern(includePattern string) Opt {
+	return func(ci *CopyInfo) {
+		ci.IncludePatterns = append(ci.IncludePatterns, includePattern)
+	}
+}
+
+func WithExcludePattern(excludePattern string) Opt {
+	return func(ci *CopyInfo) {
+		ci.ExcludePatterns = append(ci.ExcludePatterns, excludePattern)
+	}
+}
+
 type copier struct {
-	chown             Chowner
-	utime             *time.Time
-	mode              *int
-	inodes            map[uint64]string
-	xattrErrorHandler XAttrErrorHandler
+	chown                 Chowner
+	utime                 *time.Time
+	mode                  *int
+	inodes                map[uint64]string
+	xattrErrorHandler     XAttrErrorHandler
+	includePatternMatcher *fileutils.PatternMatcher
+	excludePatternMatcher *fileutils.PatternMatcher
+	parentDirs            []parentDir
+}
+
+type parentDir struct {
+	srcPath string
+	dstPath string
+	copied  bool
 }
 
-func newCopier(chown Chowner, tm *time.Time, mode *int, xeh XAttrErrorHandler) *copier {
+func newCopier(chown Chowner, tm *time.Time, mode *int, xeh XAttrErrorHandler, includePatterns, excludePatterns []string) (*copier, error) {
 	if xeh == nil {
 		xeh = func(dst, src, key string, err error) error {
 			return err
 		}
 	}
-	return &copier{inodes: map[uint64]string{}, chown: chown, utime: tm, xattrErrorHandler: xeh, mode: mode}
+
+	var includePatternMatcher *fileutils.PatternMatcher
+	if len(includePatterns) != 0 {
+		var err error
+		includePatternMatcher, err = fileutils.NewPatternMatcher(includePatterns)
+		if err != nil {
+			return nil, errors.Wrapf(err, "invalid includepatterns: %s", includePatterns)
+		}
+	}
+
+	var excludePatternMatcher *fileutils.PatternMatcher
+	if len(excludePatterns) != 0 {
+		var err error
+		excludePatternMatcher, err = fileutils.NewPatternMatcher(excludePatterns)
+		if err != nil {
+			return nil, errors.Wrapf(err, "invalid excludepatterns: %s", excludePatterns)
+		}
+	}
+
+	return &copier{
+		inodes:                map[uint64]string{},
+		chown:                 chown,
+		utime:                 tm,
+		xattrErrorHandler:     xeh,
+		mode:                  mode,
+		includePatternMatcher: includePatternMatcher,
+		excludePatternMatcher: excludePatternMatcher,
+	}, nil
 }
 
 // dest is always clean
-func (c *copier) copy(ctx context.Context, src, target string, overwriteTargetMetadata bool) error {
+func (c *copier) copy(ctx context.Context, src, srcComponents, target string, overwriteTargetMetadata, skipIncludePatterns bool) error {
 	select {
 	case <-ctx.Done():
 		return ctx.Err()
 	default:
 	}
+
 	fi, err := os.Lstat(src)
 	if err != nil {
 		return errors.Wrapf(err, "failed to stat %s", src)
 	}
 
+	include := true
+	if srcComponents != "" {
+		if !skipIncludePatterns {
+			include, err = c.include(srcComponents, fi)
+			if err != nil {
+				return err
+			}
+		}
+		exclude, err := c.exclude(srcComponents, fi)
+		if err != nil {
+			return err
+		}
+		if exclude {
+			include = false
+		}
+	}
+
+	if include {
+		if err := c.createParentDirs(src, srcComponents, target, overwriteTargetMetadata); err != nil {
+			return err
+		}
+	}
+
 	if !fi.IsDir() {
+		if !include {
+			return nil
+		}
+
 		if err := ensureEmptyFileTarget(target); err != nil {
 			return err
 		}
@@ -236,9 +321,9 @@ func (c *copier) copy(ctx context.Context, src, target string, overwriteTargetMe
 
 	switch {
 	case fi.IsDir():
-		if created, err := c.copyDirectory(ctx, src, target, fi, overwriteTargetMetadata); err != nil {
+		if created, err := c.copyDirectory(ctx, src, srcComponents, target, fi, overwriteTargetMetadata, skipIncludePatterns, include); err != nil {
 			return err
-		} else if !overwriteTargetMetadata {
+		} else if !overwriteTargetMetadata || !skipIncludePatterns {
 			copyFileInfo = created
 		}
 	case (fi.Mode() & os.ModeType) == 0:
@@ -282,36 +367,101 @@ func (c *copier) copy(ctx context.Context, src, target string, overwriteTargetMe
 	return nil
 }
 
-func (c *copier) copyDirectory(ctx context.Context, src, dst string, stat os.FileInfo, overwriteTargetMetadata bool) (bool, error) {
+func (c *copier) include(path string, fi os.FileInfo) (bool, error) {
+	if c.includePatternMatcher == nil {
+		return false, nil
+	}
+
+	m, err := c.includePatternMatcher.Matches(path)
+	if err != nil {
+		return false, errors.Wrap(err, "failed to match includepatterns")
+	}
+	return m, nil
+}
+
+func (c *copier) exclude(path string, fi os.FileInfo) (bool, error) {
+	if c.excludePatternMatcher == nil {
+		return false, nil
+	}
+
+	m, err := c.excludePatternMatcher.Matches(path)
+	if err != nil {
+		return false, errors.Wrap(err, "failed to match excludepatterns")
+	}
+	return m, nil
+}
+
+// Delayed creation of parent directories when a file or dir matches an include
+// pattern.
+func (c *copier) createParentDirs(src, srcComponents, target string, overwriteTargetMetadata bool) error {
+	for i, parentDir := range c.parentDirs {
+		if parentDir.copied {
+			continue
+		}
+
+		fi, err := os.Stat(parentDir.srcPath)
+		if err != nil {
+			return errors.Wrapf(err, "failed to stat %s", src)
+		}
+		if !fi.IsDir() {
+			return errors.Errorf("%s is not a directory", parentDir.srcPath)
+		}
+
+		created, err := copyDirectoryOnly(parentDir.srcPath, parentDir.dstPath, fi, overwriteTargetMetadata)
+		if err != nil {
+			return err
+		}
+		if created {
+			if err := c.copyFileInfo(fi, parentDir.dstPath); err != nil {
+				return errors.Wrap(err, "failed to copy file info")
+			}
+
+			if err := copyXAttrs(parentDir.dstPath, parentDir.srcPath, c.xattrErrorHandler); err != nil {
+				return errors.Wrap(err, "failed to copy xattrs")
+			}
+		}
+
+		c.parentDirs[i].copied = true
+	}
+	return nil
+}
+
+func (c *copier) copyDirectory(ctx context.Context, src, srcComponents, dst string, stat os.FileInfo, overwriteTargetMetadata, skipIncludePatterns, matchedExactly bool) (bool, error) {
 	if !stat.IsDir() {
 		return false, errors.Errorf("source is not directory")
 	}
 
 	created := false
 
-	if st, err := os.Lstat(dst); err != nil {
-		if !os.IsNotExist(err) {
-			return false, err
-		}
-		created = true
-		if err := os.Mkdir(dst, stat.Mode()); err != nil {
-			return created, errors.Wrapf(err, "failed to mkdir %s", dst)
-		}
-	} else if !st.IsDir() {
-		return false, errors.Errorf("cannot copy to non-directory: %s", dst)
-	} else if overwriteTargetMetadata {
-		if err := os.Chmod(dst, stat.Mode()); err != nil {
-			return false, errors.Wrapf(err, "failed to chmod on %s", dst)
+	// If there are no include patterns or this directory matched an include
+	// pattern exactly, go ahead and create the directory. Otherwise, delay to
+	// handle include patterns like a/*/c where we do not want to create a/b
+	// until we encounter a/b/c.
+	if matchedExactly || skipIncludePatterns {
+		var err error
+		created, err = copyDirectoryOnly(src, dst, stat, overwriteTargetMetadata)
+		if err != nil {
+			return created, err
 		}
 	}
 
+	c.parentDirs = append(c.parentDirs, parentDir{
+		srcPath: src,
+		dstPath: dst,
+		copied:  skipIncludePatterns,
+	})
+
+	defer func() {
+		c.parentDirs = c.parentDirs[:len(c.parentDirs)-1]
+	}()
+
 	fis, err := ioutil.ReadDir(src)
 	if err != nil {
 		return false, errors.Wrapf(err, "failed to read %s", src)
 	}
 
 	for _, fi := range fis {
-		if err := c.copy(ctx, filepath.Join(src, fi.Name()), filepath.Join(dst, fi.Name()), true); err != nil {
+		if err := c.copy(ctx, filepath.Join(src, fi.Name()), filepath.Join(srcComponents, fi.Name()), filepath.Join(dst, fi.Name()), true, skipIncludePatterns); err != nil {
 			return false, err
 		}
 	}
@@ -319,6 +469,25 @@ func (c *copier) copyDirectory(ctx context.Context, src, dst string, stat os.Fil
 	return created, nil
 }
 
+func copyDirectoryOnly(src, dst string, stat os.FileInfo, overwriteTargetMetadata bool) (bool, error) {
+	if st, err := os.Lstat(dst); err != nil {
+		if !os.IsNotExist(err) {
+			return false, err
+		}
+		if err := os.Mkdir(dst, stat.Mode()); err != nil {
+			return false, errors.Wrapf(err, "failed to mkdir %s", dst)
+		}
+		return true, nil
+	} else if !st.IsDir() {
+		return false, errors.Errorf("cannot copy to non-directory: %s", dst)
+	} else if overwriteTargetMetadata {
+		if err := os.Chmod(dst, stat.Mode()); err != nil {
+			return false, errors.Wrapf(err, "failed to chmod on %s", dst)
+		}
+	}
+	return false, nil
+}
+
 func ensureEmptyFileTarget(dst string) error {
 	fi, err := os.Lstat(dst)
 	if err != nil {

vendor/github.com/tonistiigi/fsutil/copy/copy_linux.go

@@ -89,7 +89,8 @@ func copyFileContent(dst, src *os.File) error {
 
 		n, err := unix.CopyFileRange(int(src.Fd()), nil, int(dst.Fd()), nil, desired, 0)
 		if err != nil {
-			if (err != unix.ENOSYS && err != unix.EXDEV && err != unix.EPERM) || !first {
+			// matches go/src/internal/poll/copy_file_range_linux.go
+			if (err != unix.ENOSYS && err != unix.EXDEV && err != unix.EPERM && err != syscall.EIO && err != unix.EOPNOTSUPP && err != syscall.EINVAL) || !first {
 				return errors.Wrap(err, "copy file range failed")
 			}
 

vendor/github.com/tonistiigi/fsutil/diff_containerd.go

@@ -1,7 +1,9 @@
 package fsutil
 
 import (
+	"bytes"
 	"context"
+	"io"
 	"os"
 	"strings"
 
@@ -35,6 +37,8 @@ const (
 // computed during a directory changes calculation.
 type ChangeFunc func(ChangeKind, string, os.FileInfo, error) error
 
+const compareChunkSize = 32 * 1024
+
 type currentPath struct {
 	path string
 	stat *types.Stat
@@ -42,7 +46,7 @@ type currentPath struct {
 }
 
 // doubleWalkDiff walks both directories to create a diff
-func doubleWalkDiff(ctx context.Context, changeFn ChangeFunc, a, b walkerFn, filter FilterFunc) (err error) {
+func doubleWalkDiff(ctx context.Context, changeFn ChangeFunc, a, b walkerFn, filter FilterFunc, differ DiffType) (err error) {
 	g, ctx := errgroup.WithContext(ctx)
 
 	var (
@@ -116,7 +120,7 @@ func doubleWalkDiff(ctx context.Context, changeFn ChangeFunc, a, b walkerFn, fil
 				}
 				f1 = nil
 			case ChangeKindModify:
-				same, err := sameFile(f1, f2copy)
+				same, err := sameFile(f1, f2copy, differ)
 				if err != nil {
 					return err
 				}
@@ -165,7 +169,10 @@ func pathChange(lower, upper *currentPath) (ChangeKind, string) {
 	}
 }
 
-func sameFile(f1, f2 *currentPath) (same bool, retErr error) {
+func sameFile(f1, f2 *currentPath, differ DiffType) (same bool, retErr error) {
+	if differ == DiffNone {
+		return false, nil
+	}
 	// If not a directory also check size, modtime, and content
 	if !f1.stat.IsDir() {
 		if f1.stat.Size_ != f2.stat.Size_ {
@@ -177,7 +184,43 @@ func sameFile(f1, f2 *currentPath) (same bool, retErr error) {
 		}
 	}
 
-	return compareStat(f1.stat, f2.stat)
+	same, err := compareStat(f1.stat, f2.stat)
+	if err != nil || !same || differ == DiffMetadata {
+		return same, err
+	}
+	return compareFileContent(f1.path, f2.path)
+}
+
+func compareFileContent(p1, p2 string) (bool, error) {
+	f1, err := os.Open(p1)
+	if err != nil {
+		return false, err
+	}
+	defer f1.Close()
+	f2, err := os.Open(p2)
+	if err != nil {
+		return false, err
+	}
+	defer f2.Close()
+
+	b1 := make([]byte, compareChunkSize)
+	b2 := make([]byte, compareChunkSize)
+	for {
+		n1, err1 := f1.Read(b1)
+		if err1 != nil && err1 != io.EOF {
+			return false, err1
+		}
+		n2, err2 := f2.Read(b2)
+		if err2 != nil && err2 != io.EOF {
+			return false, err2
+		}
+		if n1 != n2 || !bytes.Equal(b1[:n1], b2[:n2]) {
+			return false, nil
+		}
+		if err1 == io.EOF && err2 == io.EOF {
+			return true, nil
+		}
+	}
 }
 
 // compareStat returns whether the stats are equivalent,

vendor/github.com/tonistiigi/fsutil/go.mod

@@ -3,18 +3,18 @@ module github.com/tonistiigi/fsutil
 go 1.13
 
 require (
-	github.com/Microsoft/hcsshim v0.8.9 // indirect
-	github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc
-	github.com/docker/docker v0.0.0-20200511152416-a93e9eb0e95c
-	github.com/gogo/protobuf v1.3.1
-	github.com/moby/sys/mount v0.1.0 // indirect
-	github.com/moby/sys/mountinfo v0.1.3 // indirect
+	github.com/containerd/continuity v0.1.0
+	github.com/docker/docker v20.10.3-0.20210609071616-4c2ec79bf2a8+incompatible // master (v21.xx-dev)
+	github.com/gogo/protobuf v1.3.2
+	github.com/golang/protobuf v1.4.3 // indirect
+	github.com/google/go-cmp v0.5.2 // indirect
+	github.com/kr/pretty v0.2.0 // indirect
 	github.com/opencontainers/go-digest v1.0.0
-	github.com/opencontainers/image-spec v1.0.1 // indirect
-	github.com/opencontainers/runc v1.0.0-rc10 // indirect
 	github.com/pkg/errors v0.9.1
-	github.com/stretchr/testify v1.5.1
-	golang.org/x/sync v0.0.0-20190423024810-112230192c58
-	golang.org/x/sys v0.0.0-20200917073148-efd3b9a0ff20
-	gotest.tools/v3 v3.0.2 // indirect
+	github.com/stretchr/testify v1.7.0
+	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
+	golang.org/x/sys v0.0.0-20210313202042-bd2e13477e9c
+	google.golang.org/protobuf v1.25.0 // indirect
+	gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect
+	gotest.tools/v3 v3.0.3 // indirect
 )

vendor/github.com/tonistiigi/fsutil/prefix/match.go

@@ -0,0 +1,45 @@
+package prefix
+
+import (
+	"path"
+	"path/filepath"
+	"strings"
+)
+
+// Match matches a path against a pattern. It returns m = true if the path
+// matches the pattern, and partial = true if the pattern has more separators
+// than the path and the common components match (for example, name = foo and
+// pattern = foo/bar/*). slashSeparator determines whether the path and pattern
+// are '/' delimited (true) or use the native path separator (false).
+func Match(pattern, name string, slashSeparator bool) (m bool, partial bool) {
+	separator := filepath.Separator
+	if slashSeparator {
+		separator = '/'
+	}
+	count := strings.Count(name, string(separator))
+	if strings.Count(pattern, string(separator)) > count {
+		pattern = trimUntilIndex(pattern, string(separator), count)
+		partial = true
+	}
+	if slashSeparator {
+		m, _ = path.Match(pattern, name)
+	} else {
+		m, _ = filepath.Match(pattern, name)
+	}
+	return m, partial
+}
+
+func trimUntilIndex(str, sep string, count int) string {
+	s := str
+	i := 0
+	c := 0
+	for {
+		idx := strings.Index(s, sep)
+		s = s[idx+len(sep):]
+		i += idx + len(sep)
+		c++
+		if c > count {
+			return str[:i-len(sep)]
+		}
+	}
+}

vendor/github.com/tonistiigi/fsutil/receive.go

@@ -11,12 +11,21 @@ import (
 	"golang.org/x/sync/errgroup"
 )
 
+type DiffType int
+
+const (
+	DiffMetadata DiffType = iota
+	DiffNone
+	DiffContent
+)
+
 type ReceiveOpt struct {
 	NotifyHashed  ChangeFunc
 	ContentHasher ContentHasher
 	ProgressCb    func(int, bool)
 	Merge         bool
 	Filter        FilterFunc
+	Differ        DiffType
 }
 
 func Receive(ctx context.Context, conn Stream, dest string, opt ReceiveOpt) error {
@@ -33,6 +42,7 @@ func Receive(ctx context.Context, conn Stream, dest string, opt ReceiveOpt) erro
 		progressCb:    opt.ProgressCb,
 		merge:         opt.Merge,
 		filter:        opt.Filter,
+		differ:        opt.Differ,
 	}
 	return r.run(ctx)
 }
@@ -47,6 +57,7 @@ type receiver struct {
 	progressCb func(int, bool)
 	merge      bool
 	filter     FilterFunc
+	differ     DiffType
 
 	notifyHashed   ChangeFunc
 	contentHasher  ContentHasher
@@ -132,7 +143,7 @@ func (r *receiver) run(ctx context.Context) error {
 		if !r.merge {
 			destWalker = getWalkerFn(r.dest)
 		}
-		err := doubleWalkDiff(ctx, dw.HandleChange, destWalker, w.fill, r.filter)
+		err := doubleWalkDiff(ctx, dw.HandleChange, destWalker, w.fill, r.filter, r.differ)
 		if err != nil {
 			return err
 		}

vendor/github.com/tonistiigi/fsutil/walker.go

@@ -10,6 +10,7 @@ import (
 
 	"github.com/docker/docker/pkg/fileutils"
 	"github.com/pkg/errors"
+	"github.com/tonistiigi/fsutil/prefix"
 	"github.com/tonistiigi/fsutil/types"
 )
 
@@ -96,8 +97,8 @@ func Walk(ctx context.Context, p string, opt *WalkOpt, fn filepath.WalkFunc) err
 				if !skip {
 					matched := false
 					partial := true
-					for _, p := range includePatterns {
-						if ok, p := matchPrefix(p, path); ok {
+					for _, pattern := range includePatterns {
+						if ok, p := prefix.Match(pattern, path, false); ok {
 							matched = true
 							if !p {
 								partial = false
@@ -190,32 +191,6 @@ func (s *StatInfo) Sys() interface{} {
 	return s.Stat
 }
 
-func matchPrefix(pattern, name string) (bool, bool) {
-	count := strings.Count(name, string(filepath.Separator))
-	partial := false
-	if strings.Count(pattern, string(filepath.Separator)) > count {
-		pattern = trimUntilIndex(pattern, string(filepath.Separator), count)
-		partial = true
-	}
-	m, _ := filepath.Match(pattern, name)
-	return m, partial
-}
-
-func trimUntilIndex(str, sep string, count int) string {
-	s := str
-	i := 0
-	c := 0
-	for {
-		idx := strings.Index(s, sep)
-		s = s[idx+len(sep):]
-		i += idx + len(sep)
-		c++
-		if c > count {
-			return str[:i-len(sep)]
-		}
-	}
-}
-
 func isNotExist(err error) bool {
 	return errors.Is(err, os.ErrNotExist) || errors.Is(err, syscall.ENOTDIR)
 }