Merge pull request #3682 from alexlarsson/implement-tar

Implement tar in Go

archive/archive.go

@@ -23,10 +23,7 @@ type Compression int
 
 type TarOptions struct {
 	Includes    []string
-	Excludes    []string
-	Recursive   bool
 	Compression Compression
-	CreateFiles []string
 }
 
 const (
@@ -66,7 +63,7 @@ func DetectCompression(source []byte) Compression {
 func xzDecompress(archive io.Reader) (io.Reader, error) {
 	args := []string{"xz", "-d", "-c", "-q"}
 
-	return CmdStream(exec.Command(args[0], args[1:]...), archive, nil)
+	return CmdStream(exec.Command(args[0], args[1:]...), archive)
 }
 
 func DecompressStream(archive io.Reader) (io.Reader, error) {
@@ -100,16 +97,20 @@ func DecompressStream(archive io.Reader) (io.Reader, error) {
 	}
 }
 
-func (compression *Compression) Flag() string {
-	switch *compression {
-	case Bzip2:
-		return "j"
+func CompressStream(dest io.WriteCloser, compression Compression) (io.WriteCloser, error) {
+
+	switch compression {
+	case Uncompressed:
+		return dest, nil
 	case Gzip:
-		return "z"
-	case Xz:
-		return "J"
+		return gzip.NewWriter(dest), nil
+	case Bzip2, Xz:
+		// archive/bzip2 does not support writing, and there is no xz support at all
+		// However, this is not a problem as docker only currently generates gzipped tars
+		return nil, fmt.Errorf("Unsupported compression format %s", (&compression).Extension())
+	default:
+		return nil, fmt.Errorf("Unsupported compression format %s", (&compression).Extension())
 	}
-	return ""
 }
 
 func (compression *Compression) Extension() string {
@@ -126,6 +127,59 @@ func (compression *Compression) Extension() string {
 	return ""
 }
 
+func addTarFile(path, name string, tw *tar.Writer) error {
+	fi, err := os.Lstat(path)
+	if err != nil {
+		return err
+	}
+
+	link := ""
+	if fi.Mode()&os.ModeSymlink != 0 {
+		if link, err = os.Readlink(path); err != nil {
+			return err
+		}
+	}
+
+	hdr, err := tar.FileInfoHeader(fi, link)
+	if err != nil {
+		return err
+	}
+
+	if fi.IsDir() && !strings.HasSuffix(name, "/") {
+		name = name + "/"
+	}
+
+	hdr.Name = name
+
+	stat, ok := fi.Sys().(*syscall.Stat_t)
+	if ok {
+		// Currently go does not fill in the major/minors
+		if stat.Mode&syscall.S_IFBLK == syscall.S_IFBLK ||
+			stat.Mode&syscall.S_IFCHR == syscall.S_IFCHR {
+			hdr.Devmajor = int64(major(uint64(stat.Rdev)))
+			hdr.Devminor = int64(minor(uint64(stat.Rdev)))
+		}
+	}
+
+	if err := tw.WriteHeader(hdr); err != nil {
+		return err
+	}
+
+	if hdr.Typeflag == tar.TypeReg {
+		if file, err := os.Open(path); err != nil {
+			return err
+		} else {
+			_, err := io.Copy(tw, file)
+			if err != nil {
+				return err
+			}
+			file.Close()
+		}
+	}
+
+	return nil
+}
+
 func createTarFile(path, extractDir string, hdr *tar.Header, reader *tar.Reader) error {
 	switch hdr.Typeflag {
 	case tar.TypeDir:
@@ -207,7 +261,7 @@ func createTarFile(path, extractDir string, hdr *tar.Header, reader *tar.Reader)
 // Tar creates an archive from the directory at `path`, and returns it as a
 // stream of bytes.
 func Tar(path string, compression Compression) (io.Reader, error) {
-	return TarFilter(path, &TarOptions{Recursive: true, Compression: compression})
+	return TarFilter(path, &TarOptions{Compression: compression})
 }
 
 func escapeName(name string) string {
@@ -228,57 +282,55 @@ func escapeName(name string) string {
 
 // Tar creates an archive from the directory at `path`, only including files whose relative
 // paths are included in `filter`. If `filter` is nil, then all files are included.
-func TarFilter(path string, options *TarOptions) (io.Reader, error) {
-	args := []string{"tar", "--numeric-owner", "-f", "-", "-C", path, "-T", "-"}
-	if options.Includes == nil {
-		options.Includes = []string{"."}
-	}
-	args = append(args, "-c"+options.Compression.Flag())
+func TarFilter(srcPath string, options *TarOptions) (io.Reader, error) {
+	pipeReader, pipeWriter := io.Pipe()
 
-	for _, exclude := range options.Excludes {
-		args = append(args, fmt.Sprintf("--exclude=%s", exclude))
+	compressWriter, err := CompressStream(pipeWriter, options.Compression)
+	if err != nil {
+		return nil, err
 	}
 
-	if !options.Recursive {
-		args = append(args, "--no-recursion")
-	}
+	tw := tar.NewWriter(compressWriter)
 
-	files := ""
-	for _, f := range options.Includes {
-		files = files + escapeName(f) + "\n"
-	}
-
-	tmpDir := ""
+	go func() {
+		// In general we log errors here but ignore them because
+		// during e.g. a diff operation the container can continue
+		// mutating the filesystem and we can see transient errors
+		// from this
 
-	if options.CreateFiles != nil {
-		var err error // Can't use := here or we override the outer tmpDir
-		tmpDir, err = ioutil.TempDir("", "docker-tar")
-		if err != nil {
-			return nil, err
+		if options.Includes == nil {
+			options.Includes = []string{"."}
 		}
 
-		files = files + "-C" + tmpDir + "\n"
-		for _, f := range options.CreateFiles {
-			path := filepath.Join(tmpDir, f)
-			err := os.MkdirAll(filepath.Dir(path), 0600)
-			if err != nil {
-				return nil, err
-			}
+		for _, include := range options.Includes {
+			filepath.Walk(filepath.Join(srcPath, include), func(filePath string, f os.FileInfo, err error) error {
+				if err != nil {
+					utils.Debugf("Tar: Can't stat file %s to tar: %s\n", srcPath, err)
+					return nil
+				}
 
-			if file, err := os.OpenFile(path, os.O_CREATE, 0600); err != nil {
-				return nil, err
-			} else {
-				file.Close()
-			}
-			files = files + escapeName(f) + "\n"
+				relFilePath, err := filepath.Rel(srcPath, filePath)
+				if err != nil {
+					return nil
+				}
+
+				if err := addTarFile(filePath, relFilePath, tw); err != nil {
+					utils.Debugf("Can't add file %s to tar: %s\n", srcPath, err)
+				}
+				return nil
+			})
 		}
-	}
 
-	return CmdStream(exec.Command(args[0], args[1:]...), bytes.NewBufferString(files), func() {
-		if tmpDir != "" {
-			_ = os.RemoveAll(tmpDir)
+		// Make sure to check the error on Close.
+		if err := tw.Close(); err != nil {
+			utils.Debugf("Can't close tar writer: %s\n", err)
 		}
-	})
+		if err := compressWriter.Close(); err != nil {
+			utils.Debugf("Can't close compress writer: %s\n", err)
+		}
+	}()
+
+	return pipeReader, nil
 }
 
 // Untar reads a stream of bytes from `archive`, parses it as a tar archive,
@@ -311,19 +363,6 @@ func Untar(archive io.Reader, dest string, options *TarOptions) error {
 			return err
 		}
 
-		if options != nil {
-			excludeFile := false
-			for _, exclude := range options.Excludes {
-				if strings.HasPrefix(hdr.Name, exclude) {
-					excludeFile = true
-					break
-				}
-			}
-			if excludeFile {
-				continue
-			}
-		}
-
 		// Normalize name, for safety and for a simple is-root check
 		hdr.Name = filepath.Clean(hdr.Name)
 
@@ -378,9 +417,9 @@ func Untar(archive io.Reader, dest string, options *TarOptions) error {
 // TarUntar is a convenience function which calls Tar and Untar, with
 // the output of one piped into the other. If either Tar or Untar fails,
 // TarUntar aborts and returns the error.
-func TarUntar(src string, filter []string, dst string) error {
-	utils.Debugf("TarUntar(%s %s %s)", src, filter, dst)
-	archive, err := TarFilter(src, &TarOptions{Compression: Uncompressed, Includes: filter, Recursive: true})
+func TarUntar(src string, dst string) error {
+	utils.Debugf("TarUntar(%s %s)", src, dst)
+	archive, err := TarFilter(src, &TarOptions{Compression: Uncompressed})
 	if err != nil {
 		return err
 	}
@@ -417,7 +456,7 @@ func CopyWithTar(src, dst string) error {
 		return err
 	}
 	utils.Debugf("Calling TarUntar(%s, %s)", src, dst)
-	return TarUntar(src, nil, dst)
+	return TarUntar(src, dst)
 }
 
 // CopyFileWithTar emulates the behavior of the 'cp' command-line
@@ -480,13 +519,10 @@ func CopyFileWithTar(src, dst string) (err error) {
 // CmdStream executes a command, and returns its stdout as a stream.
 // If the command fails to run or doesn't complete successfully, an error
 // will be returned, including anything written on stderr.
-func CmdStream(cmd *exec.Cmd, input io.Reader, atEnd func()) (io.Reader, error) {
+func CmdStream(cmd *exec.Cmd, input io.Reader) (io.Reader, error) {
 	if input != nil {
 		stdin, err := cmd.StdinPipe()
 		if err != nil {
-			if atEnd != nil {
-				atEnd()
-			}
 			return nil, err
 		}
 		// Write stdin if any
@@ -497,16 +533,10 @@ func CmdStream(cmd *exec.Cmd, input io.Reader, atEnd func()) (io.Reader, error)
 	}
 	stdout, err := cmd.StdoutPipe()
 	if err != nil {
-		if atEnd != nil {
-			atEnd()
-		}
 		return nil, err
 	}
 	stderr, err := cmd.StderrPipe()
 	if err != nil {
-		if atEnd != nil {
-			atEnd()
-		}
 		return nil, err
 	}
 	pipeR, pipeW := io.Pipe()
@@ -531,9 +561,6 @@ func CmdStream(cmd *exec.Cmd, input io.Reader, atEnd func()) (io.Reader, error)
 		} else {
 			pipeW.Close()
 		}
-		if atEnd != nil {
-			atEnd()
-		}
 	}()
 	// Run the command and return the pipe
 	if err := cmd.Start(); err != nil {

archive/archive_test.go

@@ -14,7 +14,7 @@ import (
 
 func TestCmdStreamLargeStderr(t *testing.T) {
 	cmd := exec.Command("/bin/sh", "-c", "dd if=/dev/zero bs=1k count=1000 of=/dev/stderr; echo hello")
-	out, err := CmdStream(cmd, nil, nil)
+	out, err := CmdStream(cmd, nil)
 	if err != nil {
 		t.Fatalf("Failed to start command: %s", err)
 	}
@@ -35,7 +35,7 @@ func TestCmdStreamLargeStderr(t *testing.T) {
 
 func TestCmdStreamBad(t *testing.T) {
 	badCmd := exec.Command("/bin/sh", "-c", "echo hello; echo >&2 error couldn\\'t reverse the phase pulser; exit 1")
-	out, err := CmdStream(badCmd, nil, nil)
+	out, err := CmdStream(badCmd, nil)
 	if err != nil {
 		t.Fatalf("Failed to start command: %s", err)
 	}
@@ -50,7 +50,7 @@ func TestCmdStreamBad(t *testing.T) {
 
 func TestCmdStreamGood(t *testing.T) {
 	cmd := exec.Command("/bin/sh", "-c", "echo hello; exit 0")
-	out, err := CmdStream(cmd, nil, nil)
+	out, err := CmdStream(cmd, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -89,6 +89,16 @@ func tarUntar(t *testing.T, origin string, compression Compression) error {
 	if _, err := os.Stat(tmp); err != nil {
 		return err
 	}
+
+	changes, err := ChangesDirs(origin, tmp)
+	if err != nil {
+		return err
+	}
+
+	if len(changes) != 0 {
+		t.Fatalf("Unexpected differences after tarUntar: %v", changes)
+	}
+
 	return nil
 }
 
@@ -108,8 +118,6 @@ func TestTarUntar(t *testing.T) {
 	for _, c := range []Compression{
 		Uncompressed,
 		Gzip,
-		Bzip2,
-		Xz,
 	} {
 		if err := tarUntar(t, origin, c); err != nil {
 			t.Fatalf("Error tar/untar for compression %s: %s", c.Extension(), err)

archive/changes.go

@@ -347,70 +347,12 @@ func ExportChanges(dir string, changes []Change) (Archive, error) {
 				}
 			} else {
 				path := filepath.Join(dir, change.Path)
-
-				var stat syscall.Stat_t
-				if err := syscall.Lstat(path, &stat); err != nil {
-					utils.Debugf("Can't stat source file: %s\n", err)
-					continue
-				}
-
-				mtim := getLastModification(&stat)
-				atim := getLastAccess(&stat)
-				hdr := &tar.Header{
-					Name:       change.Path[1:],
-					Mode:       int64(stat.Mode & 07777),
-					Uid:        int(stat.Uid),
-					Gid:        int(stat.Gid),
-					ModTime:    time.Unix(int64(mtim.Sec), int64(mtim.Nsec)),
-					AccessTime: time.Unix(int64(atim.Sec), int64(atim.Nsec)),
-				}
-
-				if stat.Mode&syscall.S_IFDIR == syscall.S_IFDIR {
-					hdr.Typeflag = tar.TypeDir
-				} else if stat.Mode&syscall.S_IFLNK == syscall.S_IFLNK {
-					hdr.Typeflag = tar.TypeSymlink
-					if link, err := os.Readlink(path); err != nil {
-						utils.Debugf("Can't readlink source file: %s\n", err)
-						continue
-					} else {
-						hdr.Linkname = link
-					}
-				} else if stat.Mode&syscall.S_IFBLK == syscall.S_IFBLK ||
-					stat.Mode&syscall.S_IFCHR == syscall.S_IFCHR {
-					if stat.Mode&syscall.S_IFBLK == syscall.S_IFBLK {
-						hdr.Typeflag = tar.TypeBlock
-					} else {
-						hdr.Typeflag = tar.TypeChar
-					}
-					hdr.Devmajor = int64(major(uint64(stat.Rdev)))
-					hdr.Devminor = int64(minor(uint64(stat.Rdev)))
-				} else if stat.Mode&syscall.S_IFIFO == syscall.S_IFIFO ||
-					stat.Mode&syscall.S_IFSOCK == syscall.S_IFSOCK {
-					hdr.Typeflag = tar.TypeFifo
-				} else if stat.Mode&syscall.S_IFREG == syscall.S_IFREG {
-					hdr.Typeflag = tar.TypeReg
-					hdr.Size = stat.Size
-				} else {
-					utils.Debugf("Unknown file type: %s\n", path)
-					continue
-				}
-
-				if err := tw.WriteHeader(hdr); err != nil {
-					utils.Debugf("Can't write tar header: %s\n", err)
-				}
-				if hdr.Typeflag == tar.TypeReg {
-					if file, err := os.Open(path); err != nil {
-						utils.Debugf("Can't open file: %s\n", err)
-					} else {
-						_, err := io.Copy(tw, file)
-						if err != nil {
-							utils.Debugf("Can't copy file: %s\n", err)
-						}
-						file.Close()
-					}
+				if err := addTarFile(path, change.Path[1:], tw); err != nil {
+					utils.Debugf("Can't add file %s to tar: %s\n", path, err)
 				}
 			}
 		}
+
 		// Make sure to check the error on Close.
 		if err := tw.Close(); err != nil {
 			utils.Debugf("Can't close layer: %s\n", err)

container.go

@@ -1473,7 +1473,6 @@ func (container *Container) Copy(resource string) (archive.Archive, error) {
 	return archive.TarFilter(basePath, &archive.TarOptions{
 		Compression: archive.Uncompressed,
 		Includes:    filter,
-		Recursive:   true,
 	})
 }
 

graphdriver/aufs/aufs.go

@@ -225,7 +225,6 @@ func (a *Driver) Get(id string) (string, error) {
 // Returns an archive of the contents for the id
 func (a *Driver) Diff(id string) (archive.Archive, error) {
 	return archive.TarFilter(path.Join(a.rootPath(), "diff", id), &archive.TarOptions{
-		Recursive:   true,
 		Compression: archive.Uncompressed,
 	})
 }

hack/PACKAGERS.md

@@ -120,7 +120,6 @@ The test suite will also download a small test container, so you will need inter
 
 To run properly, docker needs the following software to be installed at runtime:
 
-* GNU Tar version 1.26 or later
 * iproute2 version 3.5 or later (build after 2012-05-21), and specifically the "ip" utility
 * iptables version 1.4 or later
 * The LXC utility scripts (http://lxc.sourceforge.net) version 0.8 or later

integration/api_test.go

@@ -374,7 +374,7 @@ func TestGetContainersExport(t *testing.T) {
 			}
 			t.Fatal(err)
 		}
-		if h.Name == "./test" {
+		if h.Name == "test" {
 			found = true
 			break
 		}