|
@@ -1,3 +1,19 @@
|
|
|
+/*
|
|
|
+ Copyright The containerd Authors.
|
|
|
+
|
|
|
+ Licensed under the Apache License, Version 2.0 (the "License");
|
|
|
+ you may not use this file except in compliance with the License.
|
|
|
+ You may obtain a copy of the License at
|
|
|
+
|
|
|
+ http://www.apache.org/licenses/LICENSE-2.0
|
|
|
+
|
|
|
+ Unless required by applicable law or agreed to in writing, software
|
|
|
+ distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
+ See the License for the specific language governing permissions and
|
|
|
+ limitations under the License.
|
|
|
+*/
|
|
|
+
|
|
|
package archive
|
|
|
|
|
|
import (
|
|
@@ -13,19 +29,21 @@ import (
|
|
|
"syscall"
|
|
|
"time"
|
|
|
|
|
|
- "github.com/containerd/containerd/fs"
|
|
|
"github.com/containerd/containerd/log"
|
|
|
+ "github.com/containerd/continuity/fs"
|
|
|
"github.com/dmcgowan/go-tar"
|
|
|
"github.com/pkg/errors"
|
|
|
)
|
|
|
|
|
|
-var bufferPool = &sync.Pool{
|
|
|
+var bufPool = &sync.Pool{
|
|
|
New: func() interface{} {
|
|
|
buffer := make([]byte, 32*1024)
|
|
|
return &buffer
|
|
|
},
|
|
|
}
|
|
|
|
|
|
+var errInvalidArchive = errors.New("invalid archive")
|
|
|
+
|
|
|
// Diff returns a tar stream of the computed filesystem
|
|
|
// difference between the provided directories.
|
|
|
//
|
|
@@ -87,12 +105,23 @@ const (
|
|
|
|
|
|
// Apply applies a tar stream of an OCI style diff tar.
|
|
|
// See https://github.com/opencontainers/image-spec/blob/master/layer.md#applying-changesets
|
|
|
-func Apply(ctx context.Context, root string, r io.Reader) (int64, error) {
|
|
|
+func Apply(ctx context.Context, root string, r io.Reader, opts ...ApplyOpt) (int64, error) {
|
|
|
root = filepath.Clean(root)
|
|
|
|
|
|
+ var options ApplyOptions
|
|
|
+ for _, opt := range opts {
|
|
|
+ if err := opt(&options); err != nil {
|
|
|
+ return 0, errors.Wrap(err, "failed to apply option")
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ return apply(ctx, root, tar.NewReader(r), options)
|
|
|
+}
|
|
|
+
|
|
|
+// applyNaive applies a tar stream of an OCI style diff tar.
|
|
|
+// See https://github.com/opencontainers/image-spec/blob/master/layer.md#applying-changesets
|
|
|
+func applyNaive(ctx context.Context, root string, tr *tar.Reader, options ApplyOptions) (size int64, err error) {
|
|
|
var (
|
|
|
- tr = tar.NewReader(r)
|
|
|
- size int64
|
|
|
dirs []*tar.Header
|
|
|
|
|
|
// Used for handling opaque directory markers which
|
|
@@ -220,6 +249,15 @@ func Apply(ctx context.Context, root string, r io.Reader) (int64, error) {
|
|
|
|
|
|
originalBase := base[len(whiteoutPrefix):]
|
|
|
originalPath := filepath.Join(dir, originalBase)
|
|
|
+
|
|
|
+ // Ensure originalPath is under dir
|
|
|
+ if dir[len(dir)-1] != filepath.Separator {
|
|
|
+ dir += string(filepath.Separator)
|
|
|
+ }
|
|
|
+ if !strings.HasPrefix(originalPath, dir) {
|
|
|
+ return 0, errors.Wrapf(errInvalidArchive, "invalid whiteout name: %v", base)
|
|
|
+ }
|
|
|
+
|
|
|
if err := os.RemoveAll(originalPath); err != nil {
|
|
|
return 0, err
|
|
|
}
|
|
@@ -285,12 +323,106 @@ func Apply(ctx context.Context, root string, r io.Reader) (int64, error) {
|
|
|
return size, nil
|
|
|
}
|
|
|
|
|
|
+func createTarFile(ctx context.Context, path, extractDir string, hdr *tar.Header, reader io.Reader) error {
|
|
|
+ // hdr.Mode is in linux format, which we can use for syscalls,
|
|
|
+ // but for os.Foo() calls we need the mode converted to os.FileMode,
|
|
|
+ // so use hdrInfo.Mode() (they differ for e.g. setuid bits)
|
|
|
+ hdrInfo := hdr.FileInfo()
|
|
|
+
|
|
|
+ switch hdr.Typeflag {
|
|
|
+ case tar.TypeDir:
|
|
|
+ // Create directory unless it exists as a directory already.
|
|
|
+ // In that case we just want to merge the two
|
|
|
+ if fi, err := os.Lstat(path); !(err == nil && fi.IsDir()) {
|
|
|
+ if err := mkdir(path, hdrInfo.Mode()); err != nil {
|
|
|
+ return err
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ case tar.TypeReg, tar.TypeRegA:
|
|
|
+ file, err := openFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, hdrInfo.Mode())
|
|
|
+ if err != nil {
|
|
|
+ return err
|
|
|
+ }
|
|
|
+
|
|
|
+ _, err = copyBuffered(ctx, file, reader)
|
|
|
+ if err1 := file.Close(); err == nil {
|
|
|
+ err = err1
|
|
|
+ }
|
|
|
+ if err != nil {
|
|
|
+ return err
|
|
|
+ }
|
|
|
+
|
|
|
+ case tar.TypeBlock, tar.TypeChar:
|
|
|
+ // Handle this is an OS-specific way
|
|
|
+ if err := handleTarTypeBlockCharFifo(hdr, path); err != nil {
|
|
|
+ return err
|
|
|
+ }
|
|
|
+
|
|
|
+ case tar.TypeFifo:
|
|
|
+ // Handle this is an OS-specific way
|
|
|
+ if err := handleTarTypeBlockCharFifo(hdr, path); err != nil {
|
|
|
+ return err
|
|
|
+ }
|
|
|
+
|
|
|
+ case tar.TypeLink:
|
|
|
+ targetPath, err := fs.RootPath(extractDir, hdr.Linkname)
|
|
|
+ if err != nil {
|
|
|
+ return err
|
|
|
+ }
|
|
|
+ if err := os.Link(targetPath, path); err != nil {
|
|
|
+ return err
|
|
|
+ }
|
|
|
+
|
|
|
+ case tar.TypeSymlink:
|
|
|
+ if err := os.Symlink(hdr.Linkname, path); err != nil {
|
|
|
+ return err
|
|
|
+ }
|
|
|
+
|
|
|
+ case tar.TypeXGlobalHeader:
|
|
|
+ log.G(ctx).Debug("PAX Global Extended Headers found and ignored")
|
|
|
+ return nil
|
|
|
+
|
|
|
+ default:
|
|
|
+ return errors.Errorf("unhandled tar header type %d\n", hdr.Typeflag)
|
|
|
+ }
|
|
|
+
|
|
|
+ // Lchown is not supported on Windows.
|
|
|
+ if runtime.GOOS != "windows" {
|
|
|
+ if err := os.Lchown(path, hdr.Uid, hdr.Gid); err != nil {
|
|
|
+ return err
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ for key, value := range hdr.PAXRecords {
|
|
|
+ if strings.HasPrefix(key, paxSchilyXattr) {
|
|
|
+ key = key[len(paxSchilyXattr):]
|
|
|
+ if err := setxattr(path, key, value); err != nil {
|
|
|
+ if errors.Cause(err) == syscall.ENOTSUP {
|
|
|
+ log.G(ctx).WithError(err).Warnf("ignored xattr %s in archive", key)
|
|
|
+ continue
|
|
|
+ }
|
|
|
+ return err
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ // There is no LChmod, so ignore mode for symlink. Also, this
|
|
|
+ // must happen after chown, as that can modify the file mode
|
|
|
+ if err := handleLChmod(hdr, path, hdrInfo); err != nil {
|
|
|
+ return err
|
|
|
+ }
|
|
|
+
|
|
|
+ return chtimes(path, boundTime(latestTime(hdr.AccessTime, hdr.ModTime)), boundTime(hdr.ModTime))
|
|
|
+}
|
|
|
+
|
|
|
type changeWriter struct {
|
|
|
tw *tar.Writer
|
|
|
source string
|
|
|
whiteoutT time.Time
|
|
|
inodeSrc map[uint64]string
|
|
|
inodeRefs map[uint64][]string
|
|
|
+ addedDirs map[string]struct{}
|
|
|
}
|
|
|
|
|
|
func newChangeWriter(w io.Writer, source string) *changeWriter {
|
|
@@ -300,6 +432,7 @@ func newChangeWriter(w io.Writer, source string) *changeWriter {
|
|
|
whiteoutT: time.Now(),
|
|
|
inodeSrc: map[uint64]string{},
|
|
|
inodeRefs: map[uint64][]string{},
|
|
|
+ addedDirs: map[string]struct{}{},
|
|
|
}
|
|
|
}
|
|
|
|
|
@@ -312,12 +445,16 @@ func (cw *changeWriter) HandleChange(k fs.ChangeKind, p string, f os.FileInfo, e
|
|
|
whiteOutBase := filepath.Base(p)
|
|
|
whiteOut := filepath.Join(whiteOutDir, whiteoutPrefix+whiteOutBase)
|
|
|
hdr := &tar.Header{
|
|
|
+ Typeflag: tar.TypeReg,
|
|
|
Name: whiteOut[1:],
|
|
|
Size: 0,
|
|
|
ModTime: cw.whiteoutT,
|
|
|
AccessTime: cw.whiteoutT,
|
|
|
ChangeTime: cw.whiteoutT,
|
|
|
}
|
|
|
+ if err := cw.includeParents(hdr); err != nil {
|
|
|
+ return err
|
|
|
+ }
|
|
|
if err := cw.tw.WriteHeader(hdr); err != nil {
|
|
|
return errors.Wrap(err, "failed to write whiteout header")
|
|
|
}
|
|
@@ -381,10 +518,8 @@ func (cw *changeWriter) HandleChange(k fs.ChangeKind, p string, f os.FileInfo, e
|
|
|
additionalLinks = cw.inodeRefs[inode]
|
|
|
delete(cw.inodeRefs, inode)
|
|
|
}
|
|
|
- } else if k == fs.ChangeKindUnmodified && !f.IsDir() {
|
|
|
+ } else if k == fs.ChangeKindUnmodified {
|
|
|
// Nothing to write to diff
|
|
|
- // Unmodified directories should still be written to keep
|
|
|
- // directory permissions correct on direct unpack
|
|
|
return nil
|
|
|
}
|
|
|
|
|
@@ -397,6 +532,9 @@ func (cw *changeWriter) HandleChange(k fs.ChangeKind, p string, f os.FileInfo, e
|
|
|
hdr.PAXRecords[paxSchilyXattr+"security.capability"] = string(capability)
|
|
|
}
|
|
|
|
|
|
+ if err := cw.includeParents(hdr); err != nil {
|
|
|
+ return err
|
|
|
+ }
|
|
|
if err := cw.tw.WriteHeader(hdr); err != nil {
|
|
|
return errors.Wrap(err, "failed to write file header")
|
|
|
}
|
|
@@ -408,9 +546,7 @@ func (cw *changeWriter) HandleChange(k fs.ChangeKind, p string, f os.FileInfo, e
|
|
|
}
|
|
|
defer file.Close()
|
|
|
|
|
|
- buf := bufferPool.Get().(*[]byte)
|
|
|
- n, err := io.CopyBuffer(cw.tw, file, *buf)
|
|
|
- bufferPool.Put(buf)
|
|
|
+ n, err := copyBuffered(context.TODO(), cw.tw, file)
|
|
|
if err != nil {
|
|
|
return errors.Wrap(err, "failed to copy")
|
|
|
}
|
|
@@ -426,6 +562,10 @@ func (cw *changeWriter) HandleChange(k fs.ChangeKind, p string, f os.FileInfo, e
|
|
|
hdr.Typeflag = tar.TypeLink
|
|
|
hdr.Linkname = source
|
|
|
hdr.Size = 0
|
|
|
+
|
|
|
+ if err := cw.includeParents(hdr); err != nil {
|
|
|
+ return err
|
|
|
+ }
|
|
|
if err := cw.tw.WriteHeader(hdr); err != nil {
|
|
|
return errors.Wrap(err, "failed to write file header")
|
|
|
}
|
|
@@ -442,102 +582,35 @@ func (cw *changeWriter) Close() error {
|
|
|
return nil
|
|
|
}
|
|
|
|
|
|
-func createTarFile(ctx context.Context, path, extractDir string, hdr *tar.Header, reader io.Reader) error {
|
|
|
- // hdr.Mode is in linux format, which we can use for syscalls,
|
|
|
- // but for os.Foo() calls we need the mode converted to os.FileMode,
|
|
|
- // so use hdrInfo.Mode() (they differ for e.g. setuid bits)
|
|
|
- hdrInfo := hdr.FileInfo()
|
|
|
-
|
|
|
- switch hdr.Typeflag {
|
|
|
- case tar.TypeDir:
|
|
|
- // Create directory unless it exists as a directory already.
|
|
|
- // In that case we just want to merge the two
|
|
|
- if fi, err := os.Lstat(path); !(err == nil && fi.IsDir()) {
|
|
|
- if err := mkdir(path, hdrInfo.Mode()); err != nil {
|
|
|
+func (cw *changeWriter) includeParents(hdr *tar.Header) error {
|
|
|
+ name := strings.TrimRight(hdr.Name, "/")
|
|
|
+ fname := filepath.Join(cw.source, name)
|
|
|
+ parent := filepath.Dir(name)
|
|
|
+ pname := filepath.Join(cw.source, parent)
|
|
|
+
|
|
|
+ // Do not include root directory as parent
|
|
|
+ if fname != cw.source && pname != cw.source {
|
|
|
+ _, ok := cw.addedDirs[parent]
|
|
|
+ if !ok {
|
|
|
+ cw.addedDirs[parent] = struct{}{}
|
|
|
+ fi, err := os.Stat(pname)
|
|
|
+ if err != nil {
|
|
|
return err
|
|
|
}
|
|
|
- }
|
|
|
-
|
|
|
- case tar.TypeReg, tar.TypeRegA:
|
|
|
- file, err := openFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, hdrInfo.Mode())
|
|
|
- if err != nil {
|
|
|
- return err
|
|
|
- }
|
|
|
-
|
|
|
- _, err = copyBuffered(ctx, file, reader)
|
|
|
- if err1 := file.Close(); err == nil {
|
|
|
- err = err1
|
|
|
- }
|
|
|
- if err != nil {
|
|
|
- return err
|
|
|
- }
|
|
|
-
|
|
|
- case tar.TypeBlock, tar.TypeChar:
|
|
|
- // Handle this is an OS-specific way
|
|
|
- if err := handleTarTypeBlockCharFifo(hdr, path); err != nil {
|
|
|
- return err
|
|
|
- }
|
|
|
-
|
|
|
- case tar.TypeFifo:
|
|
|
- // Handle this is an OS-specific way
|
|
|
- if err := handleTarTypeBlockCharFifo(hdr, path); err != nil {
|
|
|
- return err
|
|
|
- }
|
|
|
-
|
|
|
- case tar.TypeLink:
|
|
|
- targetPath, err := fs.RootPath(extractDir, hdr.Linkname)
|
|
|
- if err != nil {
|
|
|
- return err
|
|
|
- }
|
|
|
- if err := os.Link(targetPath, path); err != nil {
|
|
|
- return err
|
|
|
- }
|
|
|
-
|
|
|
- case tar.TypeSymlink:
|
|
|
- if err := os.Symlink(hdr.Linkname, path); err != nil {
|
|
|
- return err
|
|
|
- }
|
|
|
-
|
|
|
- case tar.TypeXGlobalHeader:
|
|
|
- log.G(ctx).Debug("PAX Global Extended Headers found and ignored")
|
|
|
- return nil
|
|
|
-
|
|
|
- default:
|
|
|
- return errors.Errorf("unhandled tar header type %d\n", hdr.Typeflag)
|
|
|
- }
|
|
|
-
|
|
|
- // Lchown is not supported on Windows.
|
|
|
- if runtime.GOOS != "windows" {
|
|
|
- if err := os.Lchown(path, hdr.Uid, hdr.Gid); err != nil {
|
|
|
- return err
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
- for key, value := range hdr.PAXRecords {
|
|
|
- if strings.HasPrefix(key, paxSchilyXattr) {
|
|
|
- key = key[len(paxSchilyXattr):]
|
|
|
- if err := setxattr(path, key, value); err != nil {
|
|
|
- if errors.Cause(err) == syscall.ENOTSUP {
|
|
|
- log.G(ctx).WithError(err).Warnf("ignored xattr %s in archive", key)
|
|
|
- continue
|
|
|
- }
|
|
|
+ if err := cw.HandleChange(fs.ChangeKindModify, parent, fi, nil); err != nil {
|
|
|
return err
|
|
|
}
|
|
|
}
|
|
|
}
|
|
|
-
|
|
|
- // There is no LChmod, so ignore mode for symlink. Also, this
|
|
|
- // must happen after chown, as that can modify the file mode
|
|
|
- if err := handleLChmod(hdr, path, hdrInfo); err != nil {
|
|
|
- return err
|
|
|
+ if hdr.Typeflag == tar.TypeDir {
|
|
|
+ cw.addedDirs[name] = struct{}{}
|
|
|
}
|
|
|
-
|
|
|
- return chtimes(path, boundTime(latestTime(hdr.AccessTime, hdr.ModTime)), boundTime(hdr.ModTime))
|
|
|
+ return nil
|
|
|
}
|
|
|
|
|
|
func copyBuffered(ctx context.Context, dst io.Writer, src io.Reader) (written int64, err error) {
|
|
|
- buf := bufferPool.Get().(*[]byte)
|
|
|
- defer bufferPool.Put(buf)
|
|
|
+ buf := bufPool.Get().(*[]byte)
|
|
|
+ defer bufPool.Put(buf)
|
|
|
|
|
|
for {
|
|
|
select {
|