diff --git a/vendor.conf b/vendor.conf index 517d80e12c..f081c02018 100644 --- a/vendor.conf +++ b/vendor.conf @@ -33,7 +33,7 @@ golang.org/x/sync 036812b2e83c0ddf193dd5a34e03 # buildkit github.com/moby/buildkit 7e03277b32d4f0150bed0e081d4253b3a8557f13 https://github.com/cpuguy83/buildkit.git # v0.8.3-3-g244e8cde + libnetwork changes -github.com/tonistiigi/fsutil 0834f99b7b85462efb69b4f571a4fa3ca7da5ac9 +github.com/tonistiigi/fsutil d72af97c0eaf93c1d20360e3cb9c63c223675b83 github.com/tonistiigi/units 6950e57a87eaf136bbe44ef2ec8e75b9e3569de2 github.com/grpc-ecosystem/grpc-opentracing 8e809c8a86450a29b90dcc9efbf062d0fe6d9746 github.com/opentracing/opentracing-go d34af3eaa63c4d08ab54863a4bdd0daa45212e12 # v1.2.0 diff --git a/vendor/github.com/tonistiigi/fsutil/copy/copy.go b/vendor/github.com/tonistiigi/fsutil/copy/copy.go index 02b3dc9ef1..569b4ed885 100644 --- a/vendor/github.com/tonistiigi/fsutil/copy/copy.go +++ b/vendor/github.com/tonistiigi/fsutil/copy/copy.go @@ -11,6 +11,7 @@ import ( "time" "github.com/containerd/continuity/fs" + "github.com/docker/docker/pkg/fileutils" "github.com/pkg/errors" ) @@ -86,7 +87,10 @@ func Copy(ctx context.Context, srcRoot, src, dstRoot, dst string, opts ...Opt) e return err } - c := newCopier(ci.Chown, ci.Utime, ci.Mode, ci.XAttrErrorHandler) + c, err := newCopier(ci.Chown, ci.Utime, ci.Mode, ci.XAttrErrorHandler, ci.IncludePatterns, ci.ExcludePatterns) + if err != nil { + return err + } srcs := []string{src} if ci.AllowWildcards { @@ -109,7 +113,8 @@ func Copy(ctx context.Context, srcRoot, src, dstRoot, dst string, opts ...Opt) e if err != nil { return err } - if err := c.copy(ctx, srcFollowed, dst, false); err != nil { + skipIncludePatterns := c.includePatternMatcher == nil + if err := c.copy(ctx, srcFollowed, "", dst, false, skipIncludePatterns); err != nil { return err } } @@ -162,6 +167,10 @@ type CopyInfo struct { XAttrErrorHandler XAttrErrorHandler CopyDirContents bool FollowLinks bool + // Include only files/dirs matching at least one of these patterns + IncludePatterns []string + // Exclude files/dir matching any of these patterns (even if they match an include pattern) + ExcludePatterns []string } type Opt func(*CopyInfo) @@ -197,36 +206,112 @@ func AllowXAttrErrors(ci *CopyInfo) { WithXAttrErrorHandler(h)(ci) } -type copier struct { - chown Chowner - utime *time.Time - mode *int - inodes map[uint64]string - xattrErrorHandler XAttrErrorHandler +func WithIncludePattern(includePattern string) Opt { + return func(ci *CopyInfo) { + ci.IncludePatterns = append(ci.IncludePatterns, includePattern) + } } -func newCopier(chown Chowner, tm *time.Time, mode *int, xeh XAttrErrorHandler) *copier { +func WithExcludePattern(excludePattern string) Opt { + return func(ci *CopyInfo) { + ci.ExcludePatterns = append(ci.ExcludePatterns, excludePattern) + } +} + +type copier struct { + chown Chowner + utime *time.Time + mode *int + inodes map[uint64]string + xattrErrorHandler XAttrErrorHandler + includePatternMatcher *fileutils.PatternMatcher + excludePatternMatcher *fileutils.PatternMatcher + parentDirs []parentDir +} + +type parentDir struct { + srcPath string + dstPath string + copied bool +} + +func newCopier(chown Chowner, tm *time.Time, mode *int, xeh XAttrErrorHandler, includePatterns, excludePatterns []string) (*copier, error) { if xeh == nil { xeh = func(dst, src, key string, err error) error { return err } } - return &copier{inodes: map[uint64]string{}, chown: chown, utime: tm, xattrErrorHandler: xeh, mode: mode} + + var includePatternMatcher *fileutils.PatternMatcher + if len(includePatterns) != 0 { + var err error + includePatternMatcher, err = fileutils.NewPatternMatcher(includePatterns) + if err != nil { + return nil, errors.Wrapf(err, "invalid includepatterns: %s", includePatterns) + } + } + + var excludePatternMatcher *fileutils.PatternMatcher + if len(excludePatterns) != 0 { + var err error + excludePatternMatcher, err = fileutils.NewPatternMatcher(excludePatterns) + if err != nil { + return nil, errors.Wrapf(err, "invalid excludepatterns: %s", excludePatterns) + } + } + + return &copier{ + inodes: map[uint64]string{}, + chown: chown, + utime: tm, + xattrErrorHandler: xeh, + mode: mode, + includePatternMatcher: includePatternMatcher, + excludePatternMatcher: excludePatternMatcher, + }, nil } // dest is always clean -func (c *copier) copy(ctx context.Context, src, target string, overwriteTargetMetadata bool) error { +func (c *copier) copy(ctx context.Context, src, srcComponents, target string, overwriteTargetMetadata, skipIncludePatterns bool) error { select { case <-ctx.Done(): return ctx.Err() default: } + fi, err := os.Lstat(src) if err != nil { return errors.Wrapf(err, "failed to stat %s", src) } + include := true + if srcComponents != "" { + if !skipIncludePatterns { + include, err = c.include(srcComponents, fi) + if err != nil { + return err + } + } + exclude, err := c.exclude(srcComponents, fi) + if err != nil { + return err + } + if exclude { + include = false + } + } + + if include { + if err := c.createParentDirs(src, srcComponents, target, overwriteTargetMetadata); err != nil { + return err + } + } + if !fi.IsDir() { + if !include { + return nil + } + if err := ensureEmptyFileTarget(target); err != nil { return err } @@ -236,9 +321,9 @@ func (c *copier) copy(ctx context.Context, src, target string, overwriteTargetMe switch { case fi.IsDir(): - if created, err := c.copyDirectory(ctx, src, target, fi, overwriteTargetMetadata); err != nil { + if created, err := c.copyDirectory(ctx, src, srcComponents, target, fi, overwriteTargetMetadata, skipIncludePatterns, include); err != nil { return err - } else if !overwriteTargetMetadata { + } else if !overwriteTargetMetadata || !skipIncludePatterns { copyFileInfo = created } case (fi.Mode() & os.ModeType) == 0: @@ -282,36 +367,101 @@ func (c *copier) copy(ctx context.Context, src, target string, overwriteTargetMe return nil } -func (c *copier) copyDirectory(ctx context.Context, src, dst string, stat os.FileInfo, overwriteTargetMetadata bool) (bool, error) { +func (c *copier) include(path string, fi os.FileInfo) (bool, error) { + if c.includePatternMatcher == nil { + return false, nil + } + + m, err := c.includePatternMatcher.Matches(path) + if err != nil { + return false, errors.Wrap(err, "failed to match includepatterns") + } + return m, nil +} + +func (c *copier) exclude(path string, fi os.FileInfo) (bool, error) { + if c.excludePatternMatcher == nil { + return false, nil + } + + m, err := c.excludePatternMatcher.Matches(path) + if err != nil { + return false, errors.Wrap(err, "failed to match excludepatterns") + } + return m, nil +} + +// Delayed creation of parent directories when a file or dir matches an include +// pattern. +func (c *copier) createParentDirs(src, srcComponents, target string, overwriteTargetMetadata bool) error { + for i, parentDir := range c.parentDirs { + if parentDir.copied { + continue + } + + fi, err := os.Stat(parentDir.srcPath) + if err != nil { + return errors.Wrapf(err, "failed to stat %s", src) + } + if !fi.IsDir() { + return errors.Errorf("%s is not a directory", parentDir.srcPath) + } + + created, err := copyDirectoryOnly(parentDir.srcPath, parentDir.dstPath, fi, overwriteTargetMetadata) + if err != nil { + return err + } + if created { + if err := c.copyFileInfo(fi, parentDir.dstPath); err != nil { + return errors.Wrap(err, "failed to copy file info") + } + + if err := copyXAttrs(parentDir.dstPath, parentDir.srcPath, c.xattrErrorHandler); err != nil { + return errors.Wrap(err, "failed to copy xattrs") + } + } + + c.parentDirs[i].copied = true + } + return nil +} + +func (c *copier) copyDirectory(ctx context.Context, src, srcComponents, dst string, stat os.FileInfo, overwriteTargetMetadata, skipIncludePatterns, matchedExactly bool) (bool, error) { if !stat.IsDir() { return false, errors.Errorf("source is not directory") } created := false - if st, err := os.Lstat(dst); err != nil { - if !os.IsNotExist(err) { - return false, err - } - created = true - if err := os.Mkdir(dst, stat.Mode()); err != nil { - return created, errors.Wrapf(err, "failed to mkdir %s", dst) - } - } else if !st.IsDir() { - return false, errors.Errorf("cannot copy to non-directory: %s", dst) - } else if overwriteTargetMetadata { - if err := os.Chmod(dst, stat.Mode()); err != nil { - return false, errors.Wrapf(err, "failed to chmod on %s", dst) + // If there are no include patterns or this directory matched an include + // pattern exactly, go ahead and create the directory. Otherwise, delay to + // handle include patterns like a/*/c where we do not want to create a/b + // until we encounter a/b/c. + if matchedExactly || skipIncludePatterns { + var err error + created, err = copyDirectoryOnly(src, dst, stat, overwriteTargetMetadata) + if err != nil { + return created, err } } + c.parentDirs = append(c.parentDirs, parentDir{ + srcPath: src, + dstPath: dst, + copied: skipIncludePatterns, + }) + + defer func() { + c.parentDirs = c.parentDirs[:len(c.parentDirs)-1] + }() + fis, err := ioutil.ReadDir(src) if err != nil { return false, errors.Wrapf(err, "failed to read %s", src) } for _, fi := range fis { - if err := c.copy(ctx, filepath.Join(src, fi.Name()), filepath.Join(dst, fi.Name()), true); err != nil { + if err := c.copy(ctx, filepath.Join(src, fi.Name()), filepath.Join(srcComponents, fi.Name()), filepath.Join(dst, fi.Name()), true, skipIncludePatterns); err != nil { return false, err } } @@ -319,6 +469,25 @@ func (c *copier) copyDirectory(ctx context.Context, src, dst string, stat os.Fil return created, nil } +func copyDirectoryOnly(src, dst string, stat os.FileInfo, overwriteTargetMetadata bool) (bool, error) { + if st, err := os.Lstat(dst); err != nil { + if !os.IsNotExist(err) { + return false, err + } + if err := os.Mkdir(dst, stat.Mode()); err != nil { + return false, errors.Wrapf(err, "failed to mkdir %s", dst) + } + return true, nil + } else if !st.IsDir() { + return false, errors.Errorf("cannot copy to non-directory: %s", dst) + } else if overwriteTargetMetadata { + if err := os.Chmod(dst, stat.Mode()); err != nil { + return false, errors.Wrapf(err, "failed to chmod on %s", dst) + } + } + return false, nil +} + func ensureEmptyFileTarget(dst string) error { fi, err := os.Lstat(dst) if err != nil { diff --git a/vendor/github.com/tonistiigi/fsutil/copy/copy_linux.go b/vendor/github.com/tonistiigi/fsutil/copy/copy_linux.go index 4e6b93c166..326b7ee76b 100644 --- a/vendor/github.com/tonistiigi/fsutil/copy/copy_linux.go +++ b/vendor/github.com/tonistiigi/fsutil/copy/copy_linux.go @@ -89,7 +89,8 @@ func copyFileContent(dst, src *os.File) error { n, err := unix.CopyFileRange(int(src.Fd()), nil, int(dst.Fd()), nil, desired, 0) if err != nil { - if (err != unix.ENOSYS && err != unix.EXDEV && err != unix.EPERM) || !first { + // matches go/src/internal/poll/copy_file_range_linux.go + if (err != unix.ENOSYS && err != unix.EXDEV && err != unix.EPERM && err != syscall.EIO && err != unix.EOPNOTSUPP && err != syscall.EINVAL) || !first { return errors.Wrap(err, "copy file range failed") } diff --git a/vendor/github.com/tonistiigi/fsutil/diff_containerd.go b/vendor/github.com/tonistiigi/fsutil/diff_containerd.go index 370271523b..d72f4e64fe 100644 --- a/vendor/github.com/tonistiigi/fsutil/diff_containerd.go +++ b/vendor/github.com/tonistiigi/fsutil/diff_containerd.go @@ -1,7 +1,9 @@ package fsutil import ( + "bytes" "context" + "io" "os" "strings" @@ -35,6 +37,8 @@ const ( // computed during a directory changes calculation. type ChangeFunc func(ChangeKind, string, os.FileInfo, error) error +const compareChunkSize = 32 * 1024 + type currentPath struct { path string stat *types.Stat @@ -42,7 +46,7 @@ type currentPath struct { } // doubleWalkDiff walks both directories to create a diff -func doubleWalkDiff(ctx context.Context, changeFn ChangeFunc, a, b walkerFn, filter FilterFunc) (err error) { +func doubleWalkDiff(ctx context.Context, changeFn ChangeFunc, a, b walkerFn, filter FilterFunc, differ DiffType) (err error) { g, ctx := errgroup.WithContext(ctx) var ( @@ -116,7 +120,7 @@ func doubleWalkDiff(ctx context.Context, changeFn ChangeFunc, a, b walkerFn, fil } f1 = nil case ChangeKindModify: - same, err := sameFile(f1, f2copy) + same, err := sameFile(f1, f2copy, differ) if err != nil { return err } @@ -165,7 +169,10 @@ func pathChange(lower, upper *currentPath) (ChangeKind, string) { } } -func sameFile(f1, f2 *currentPath) (same bool, retErr error) { +func sameFile(f1, f2 *currentPath, differ DiffType) (same bool, retErr error) { + if differ == DiffNone { + return false, nil + } // If not a directory also check size, modtime, and content if !f1.stat.IsDir() { if f1.stat.Size_ != f2.stat.Size_ { @@ -177,7 +184,43 @@ func sameFile(f1, f2 *currentPath) (same bool, retErr error) { } } - return compareStat(f1.stat, f2.stat) + same, err := compareStat(f1.stat, f2.stat) + if err != nil || !same || differ == DiffMetadata { + return same, err + } + return compareFileContent(f1.path, f2.path) +} + +func compareFileContent(p1, p2 string) (bool, error) { + f1, err := os.Open(p1) + if err != nil { + return false, err + } + defer f1.Close() + f2, err := os.Open(p2) + if err != nil { + return false, err + } + defer f2.Close() + + b1 := make([]byte, compareChunkSize) + b2 := make([]byte, compareChunkSize) + for { + n1, err1 := f1.Read(b1) + if err1 != nil && err1 != io.EOF { + return false, err1 + } + n2, err2 := f2.Read(b2) + if err2 != nil && err2 != io.EOF { + return false, err2 + } + if n1 != n2 || !bytes.Equal(b1[:n1], b2[:n2]) { + return false, nil + } + if err1 == io.EOF && err2 == io.EOF { + return true, nil + } + } } // compareStat returns whether the stats are equivalent, diff --git a/vendor/github.com/tonistiigi/fsutil/go.mod b/vendor/github.com/tonistiigi/fsutil/go.mod index 075742a89f..fc4cdb0014 100644 --- a/vendor/github.com/tonistiigi/fsutil/go.mod +++ b/vendor/github.com/tonistiigi/fsutil/go.mod @@ -3,18 +3,18 @@ module github.com/tonistiigi/fsutil go 1.13 require ( - github.com/Microsoft/hcsshim v0.8.9 // indirect - github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc - github.com/docker/docker v0.0.0-20200511152416-a93e9eb0e95c - github.com/gogo/protobuf v1.3.1 - github.com/moby/sys/mount v0.1.0 // indirect - github.com/moby/sys/mountinfo v0.1.3 // indirect + github.com/containerd/continuity v0.1.0 + github.com/docker/docker v20.10.3-0.20210609071616-4c2ec79bf2a8+incompatible // master (v21.xx-dev) + github.com/gogo/protobuf v1.3.2 + github.com/golang/protobuf v1.4.3 // indirect + github.com/google/go-cmp v0.5.2 // indirect + github.com/kr/pretty v0.2.0 // indirect github.com/opencontainers/go-digest v1.0.0 - github.com/opencontainers/image-spec v1.0.1 // indirect - github.com/opencontainers/runc v1.0.0-rc10 // indirect github.com/pkg/errors v0.9.1 - github.com/stretchr/testify v1.5.1 - golang.org/x/sync v0.0.0-20190423024810-112230192c58 - golang.org/x/sys v0.0.0-20200917073148-efd3b9a0ff20 - gotest.tools/v3 v3.0.2 // indirect + github.com/stretchr/testify v1.7.0 + golang.org/x/sync v0.0.0-20210220032951-036812b2e83c + golang.org/x/sys v0.0.0-20210313202042-bd2e13477e9c + google.golang.org/protobuf v1.25.0 // indirect + gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect + gotest.tools/v3 v3.0.3 // indirect ) diff --git a/vendor/github.com/tonistiigi/fsutil/prefix/match.go b/vendor/github.com/tonistiigi/fsutil/prefix/match.go new file mode 100644 index 0000000000..57e745c681 --- /dev/null +++ b/vendor/github.com/tonistiigi/fsutil/prefix/match.go @@ -0,0 +1,45 @@ +package prefix + +import ( + "path" + "path/filepath" + "strings" +) + +// Match matches a path against a pattern. It returns m = true if the path +// matches the pattern, and partial = true if the pattern has more separators +// than the path and the common components match (for example, name = foo and +// pattern = foo/bar/*). slashSeparator determines whether the path and pattern +// are '/' delimited (true) or use the native path separator (false). +func Match(pattern, name string, slashSeparator bool) (m bool, partial bool) { + separator := filepath.Separator + if slashSeparator { + separator = '/' + } + count := strings.Count(name, string(separator)) + if strings.Count(pattern, string(separator)) > count { + pattern = trimUntilIndex(pattern, string(separator), count) + partial = true + } + if slashSeparator { + m, _ = path.Match(pattern, name) + } else { + m, _ = filepath.Match(pattern, name) + } + return m, partial +} + +func trimUntilIndex(str, sep string, count int) string { + s := str + i := 0 + c := 0 + for { + idx := strings.Index(s, sep) + s = s[idx+len(sep):] + i += idx + len(sep) + c++ + if c > count { + return str[:i-len(sep)] + } + } +} diff --git a/vendor/github.com/tonistiigi/fsutil/receive.go b/vendor/github.com/tonistiigi/fsutil/receive.go index 5c6a486978..209d1d2faf 100644 --- a/vendor/github.com/tonistiigi/fsutil/receive.go +++ b/vendor/github.com/tonistiigi/fsutil/receive.go @@ -11,12 +11,21 @@ import ( "golang.org/x/sync/errgroup" ) +type DiffType int + +const ( + DiffMetadata DiffType = iota + DiffNone + DiffContent +) + type ReceiveOpt struct { NotifyHashed ChangeFunc ContentHasher ContentHasher ProgressCb func(int, bool) Merge bool Filter FilterFunc + Differ DiffType } func Receive(ctx context.Context, conn Stream, dest string, opt ReceiveOpt) error { @@ -33,6 +42,7 @@ func Receive(ctx context.Context, conn Stream, dest string, opt ReceiveOpt) erro progressCb: opt.ProgressCb, merge: opt.Merge, filter: opt.Filter, + differ: opt.Differ, } return r.run(ctx) } @@ -47,6 +57,7 @@ type receiver struct { progressCb func(int, bool) merge bool filter FilterFunc + differ DiffType notifyHashed ChangeFunc contentHasher ContentHasher @@ -132,7 +143,7 @@ func (r *receiver) run(ctx context.Context) error { if !r.merge { destWalker = getWalkerFn(r.dest) } - err := doubleWalkDiff(ctx, dw.HandleChange, destWalker, w.fill, r.filter) + err := doubleWalkDiff(ctx, dw.HandleChange, destWalker, w.fill, r.filter, r.differ) if err != nil { return err } diff --git a/vendor/github.com/tonistiigi/fsutil/walker.go b/vendor/github.com/tonistiigi/fsutil/walker.go index b10383e4c5..cc6143db49 100644 --- a/vendor/github.com/tonistiigi/fsutil/walker.go +++ b/vendor/github.com/tonistiigi/fsutil/walker.go @@ -10,6 +10,7 @@ import ( "github.com/docker/docker/pkg/fileutils" "github.com/pkg/errors" + "github.com/tonistiigi/fsutil/prefix" "github.com/tonistiigi/fsutil/types" ) @@ -96,8 +97,8 @@ func Walk(ctx context.Context, p string, opt *WalkOpt, fn filepath.WalkFunc) err if !skip { matched := false partial := true - for _, p := range includePatterns { - if ok, p := matchPrefix(p, path); ok { + for _, pattern := range includePatterns { + if ok, p := prefix.Match(pattern, path, false); ok { matched = true if !p { partial = false @@ -190,32 +191,6 @@ func (s *StatInfo) Sys() interface{} { return s.Stat } -func matchPrefix(pattern, name string) (bool, bool) { - count := strings.Count(name, string(filepath.Separator)) - partial := false - if strings.Count(pattern, string(filepath.Separator)) > count { - pattern = trimUntilIndex(pattern, string(filepath.Separator), count) - partial = true - } - m, _ := filepath.Match(pattern, name) - return m, partial -} - -func trimUntilIndex(str, sep string, count int) string { - s := str - i := 0 - c := 0 - for { - idx := strings.Index(s, sep) - s = s[idx+len(sep):] - i += idx + len(sep) - c++ - if c > count { - return str[:i-len(sep)] - } - } -} - func isNotExist(err error) bool { return errors.Is(err, os.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) }