Merge pull request #15040 from vbatts/vbatts-double-decompress-fix

archive, graphdriver: double decompress fix

daemon/graphdriver/aufs/aufs.go

@@ -331,7 +331,7 @@ func (a *Driver) Diff(id, parent string) (archive.Archive, error) {
 }
 
 func (a *Driver) applyDiff(id string, diff archive.ArchiveReader) error {
-	return chrootarchive.Untar(diff, path.Join(a.rootPath(), "diff", id), nil)
+	return chrootarchive.UntarUncompressed(diff, path.Join(a.rootPath(), "diff", id), nil)
 }
 
 // DiffSize calculates the changes between the specified id

daemon/graphdriver/driver.go

@@ -77,6 +77,7 @@ type Driver interface {
 	// ApplyDiff extracts the changeset from the given diff into the
 	// layer with the specified id and parent, returning the size of the
 	// new layer in bytes.
+	// The archive.ArchiveReader must be an uncompressed stream.
 	ApplyDiff(id, parent string, diff archive.ArchiveReader) (size int64, err error)
 	// DiffSize calculates the changes between the specified id
 	// and its parent and returns the size in bytes of the changes

daemon/graphdriver/fsdiff.go

@@ -121,7 +121,7 @@ func (gdw *naiveDiffDriver) ApplyDiff(id, parent string, diff archive.ArchiveRea
 
 	start := time.Now().UTC()
 	logrus.Debugf("Start untar layer")
-	if size, err = chrootarchive.ApplyLayer(layerFs, diff); err != nil {
+	if size, err = chrootarchive.ApplyUncompressedLayer(layerFs, diff); err != nil {
 		return
 	}
 	logrus.Debugf("Untar time: %vs", time.Now().UTC().Sub(start).Seconds())

daemon/graphdriver/overlay/overlay.go

@@ -411,7 +411,7 @@ func (d *Driver) ApplyDiff(id string, parent string, diff archive.ArchiveReader)
 		return 0, err
 	}
 
-	if size, err = chrootarchive.ApplyLayer(tmpRootDir, diff); err != nil {
+	if size, err = chrootarchive.ApplyUncompressedLayer(tmpRootDir, diff); err != nil {
 		return 0, err
 	}
 

pkg/archive/archive.go

@@ -633,8 +633,20 @@ loop:
 // The archive may be compressed with one of the following algorithms:
 //  identity (uncompressed), gzip, bzip2, xz.
 // FIXME: specify behavior when target path exists vs. doesn't exist.
-func Untar(archive io.Reader, dest string, options *TarOptions) error {
-	if archive == nil {
+func Untar(tarArchive io.Reader, dest string, options *TarOptions) error {
+	return untarHandler(tarArchive, dest, options, true)
+}
+
+// Untar reads a stream of bytes from `archive`, parses it as a tar archive,
+// and unpacks it into the directory at `dest`.
+// The archive must be an uncompressed stream.
+func UntarUncompressed(tarArchive io.Reader, dest string, options *TarOptions) error {
+	return untarHandler(tarArchive, dest, options, false)
+}
+
+// Handler for teasing out the automatic decompression
+func untarHandler(tarArchive io.Reader, dest string, options *TarOptions, decompress bool) error {
+	if tarArchive == nil {
 		return fmt.Errorf("Empty archive")
 	}
 	dest = filepath.Clean(dest)
@@ -644,12 +656,18 @@ func Untar(archive io.Reader, dest string, options *TarOptions) error {
 	if options.ExcludePatterns == nil {
 		options.ExcludePatterns = []string{}
 	}
-	decompressedArchive, err := DecompressStream(archive)
-	if err != nil {
-		return err
+
+	var r io.Reader = tarArchive
+	if decompress {
+		decompressedArchive, err := DecompressStream(tarArchive)
+		if err != nil {
+			return err
+		}
+		defer decompressedArchive.Close()
+		r = decompressedArchive
 	}
-	defer decompressedArchive.Close()
-	return Unpack(decompressedArchive, dest, options)
+
+	return Unpack(r, dest, options)
 }
 
 func (archiver *Archiver) TarUntar(src, dst string) error {

pkg/archive/diff.go

@@ -173,10 +173,24 @@ func UnpackLayer(dest string, layer ArchiveReader) (size int64, err error) {
 	return size, nil
 }
 
-// ApplyLayer parses a diff in the standard layer format from `layer`, and
-// applies it to the directory `dest`. Returns the size in bytes of the
-// contents of the layer.
+// ApplyLayer parses a diff in the standard layer format from `layer`,
+// and applies it to the directory `dest`. The stream `layer` can be
+// compressed or uncompressed.
+// Returns the size in bytes of the contents of the layer.
 func ApplyLayer(dest string, layer ArchiveReader) (int64, error) {
+	return applyLayerHandler(dest, layer, true)
+}
+
+// ApplyUncompressedLayer parses a diff in the standard layer format from
+// `layer`, and applies it to the directory `dest`. The stream `layer`
+// can only be uncompressed.
+// Returns the size in bytes of the contents of the layer.
+func ApplyUncompressedLayer(dest string, layer ArchiveReader) (int64, error) {
+	return applyLayerHandler(dest, layer, false)
+}
+
+// do the bulk load of ApplyLayer, but allow for not calling DecompressStream
+func applyLayerHandler(dest string, layer ArchiveReader, decompress bool) (int64, error) {
 	dest = filepath.Clean(dest)
 
 	// We need to be able to set any perms
@@ -186,9 +200,11 @@ func ApplyLayer(dest string, layer ArchiveReader) (int64, error) {
 	}
 	defer system.Umask(oldmask) // ignore err, ErrNotSupportedPlatform
 
-	layer, err = DecompressStream(layer)
-	if err != nil {
-		return 0, err
+	if decompress {
+		layer, err = DecompressStream(layer)
+		if err != nil {
+			return 0, err
+		}
 	}
 	return UnpackLayer(dest, layer)
 }

pkg/chrootarchive/archive.go

@@ -3,6 +3,7 @@ package chrootarchive
 import (
 	"fmt"
 	"io"
+	"io/ioutil"
 	"os"
 	"path/filepath"
 
@@ -17,6 +18,18 @@ var chrootArchiver = &archive.Archiver{Untar: Untar}
 // The archive may be compressed with one of the following algorithms:
 //  identity (uncompressed), gzip, bzip2, xz.
 func Untar(tarArchive io.Reader, dest string, options *archive.TarOptions) error {
+	return untarHandler(tarArchive, dest, options, true)
+}
+
+// UntarUncompressed reads a stream of bytes from `archive`, parses it as a tar archive,
+// and unpacks it into the directory at `dest`.
+// The archive must be an uncompressed stream.
+func UntarUncompressed(tarArchive io.Reader, dest string, options *archive.TarOptions) error {
+	return untarHandler(tarArchive, dest, options, false)
+}
+
+// Handler for teasing out the automatic decompression
+func untarHandler(tarArchive io.Reader, dest string, options *archive.TarOptions, decompress bool) error {
 
 	if tarArchive == nil {
 		return fmt.Errorf("Empty archive")
@@ -35,13 +48,17 @@ func Untar(tarArchive io.Reader, dest string, options *archive.TarOptions) error
 		}
 	}
 
-	decompressedArchive, err := archive.DecompressStream(tarArchive)
-	if err != nil {
-		return err
+	r := ioutil.NopCloser(tarArchive)
+	if decompress {
+		decompressedArchive, err := archive.DecompressStream(tarArchive)
+		if err != nil {
+			return err
+		}
+		defer decompressedArchive.Close()
+		r = decompressedArchive
 	}
-	defer decompressedArchive.Close()
 
-	return invokeUnpack(decompressedArchive, dest, options)
+	return invokeUnpack(r, dest, options)
 }
 
 // TarUntar is a convenience function which calls Tar and Untar, with the output of one piped into the other.

pkg/chrootarchive/archive_unix.go

@@ -49,7 +49,7 @@ func untar() {
 	os.Exit(0)
 }
 
-func invokeUnpack(decompressedArchive io.ReadCloser, dest string, options *archive.TarOptions) error {
+func invokeUnpack(decompressedArchive io.Reader, dest string, options *archive.TarOptions) error {
 
 	// We can't pass a potentially large exclude list directly via cmd line
 	// because we easily overrun the kernel's max argument/environment size

pkg/chrootarchive/diff_unix.go

@@ -65,20 +65,36 @@ func applyLayer() {
 	os.Exit(0)
 }
 
-// ApplyLayer parses a diff in the standard layer format from `layer`, and
-// applies it to the directory `dest`. Returns the size in bytes of the
-// contents of the layer.
+// ApplyLayer parses a diff in the standard layer format from `layer`,
+// and applies it to the directory `dest`. The stream `layer` can only be
+// uncompressed.
+// Returns the size in bytes of the contents of the layer.
 func ApplyLayer(dest string, layer archive.ArchiveReader) (size int64, err error) {
+	return applyLayerHandler(dest, layer, true)
+}
+
+// ApplyUncompressedLayer parses a diff in the standard layer format from
+// `layer`, and applies it to the directory `dest`. The stream `layer`
+// can only be uncompressed.
+// Returns the size in bytes of the contents of the layer.
+func ApplyUncompressedLayer(dest string, layer archive.ArchiveReader) (int64, error) {
+	return applyLayerHandler(dest, layer, false)
+}
+
+func applyLayerHandler(dest string, layer archive.ArchiveReader, decompress bool) (size int64, err error) {
 	dest = filepath.Clean(dest)
-	decompressed, err := archive.DecompressStream(layer)
-	if err != nil {
-		return 0, err
+	if decompress {
+		decompressed, err := archive.DecompressStream(layer)
+		if err != nil {
+			return 0, err
+		}
+		defer decompressed.Close()
+
+		layer = decompressed
 	}
 
-	defer decompressed.Close()
-
 	cmd := reexec.Command("docker-applyLayer", dest)
-	cmd.Stdin = decompressed
+	cmd.Stdin = layer
 
 	outBuf, errBuf := new(bytes.Buffer), new(bytes.Buffer)
 	cmd.Stdout, cmd.Stderr = outBuf, errBuf