archive: Implement ApplyLayer directly

Rather than calling out to tar we use the golang tar parser
to directly extract the tar files. This has two major advantages:

1) We're able to replace an existing directory with a file in the
   new layer. This currently breaks with the external tar, since
   it refuses to recursively remove the destination directory in
   this case, and there are no options to make it do that.

2) We avoid extracting the whiteout files just to later remove them.

archive/diff.go

@@ -1,6 +1,9 @@
 package archive
 
 import (
+	"archive/tar"
+	"github.com/dotcloud/docker/utils"
+	"io"
 	"os"
 	"path/filepath"
 	"strings"
@@ -8,87 +11,159 @@ import (
 	"time"
 )
 
+// Linux device nodes are a bit weird due to backwards compat with 16 bit device nodes
+// The lower 8 bit is the lower 8 bit in the minor, the following 12 bits are the major,
+// and then there is the top 12 bits of then minor
+func mkdev(major int64, minor int64) uint32 {
+	return uint32(((minor & 0xfff00) << 12) | ((major & 0xfff) << 8) | (minor & 0xff))
+}
+func timeToTimespec(time time.Time) (ts syscall.Timespec) {
+	if time.IsZero() {
+		// Return UTIME_OMIT special value
+		ts.Sec = 0
+		ts.Nsec = ((1 << 30) - 2)
+		return
+	}
+	return syscall.NsecToTimespec(time.UnixNano())
+}
+
 // ApplyLayer parses a diff in the standard layer format from `layer`, and
 // applies it to the directory `dest`.
 func ApplyLayer(dest string, layer Archive) error {
-	// Poor man's diff applyer in 2 steps:
+	// We need to be able to set any perms
+	oldmask := syscall.Umask(0)
+	defer syscall.Umask(oldmask)
 
-	// Step 1: untar everything in place
-	if err := Untar(layer, dest, nil); err != nil {
-		return err
-	}
+	tr := tar.NewReader(layer)
 
-	modifiedDirs := make(map[string]*syscall.Stat_t)
-	addDir := func(file string) {
-		d := filepath.Dir(file)
-		if _, exists := modifiedDirs[d]; !exists {
-			if s, err := os.Lstat(d); err == nil {
-				if sys := s.Sys(); sys != nil {
-					if stat, ok := sys.(*syscall.Stat_t); ok {
-						modifiedDirs[d] = stat
-					}
-				}
-			}
-		}
-	}
+	var dirs []*tar.Header
 
-	// Step 2: walk for whiteouts and apply them, removing them in the process
-	err := filepath.Walk(dest, func(fullPath string, f os.FileInfo, err error) error {
+	// Iterate through the files in the archive.
+	for {
+		hdr, err := tr.Next()
+		if err == io.EOF {
+			// end of tar archive
+			break
+		}
 		if err != nil {
-			if os.IsNotExist(err) {
-				// This happens in the case of whiteouts in parent dir removing a directory
-				// We just ignore it
-				return filepath.SkipDir
-			}
 			return err
 		}
 
-		// Rebase path
-		path, err := filepath.Rel(dest, fullPath)
-		if err != nil {
-			return err
+		// Skip AUFS metadata dirs
+		if strings.HasPrefix(hdr.Name, ".wh..wh.") {
+			continue
 		}
-		path = filepath.Join("/", path)
 
-		// Skip AUFS metadata
-		if matched, err := filepath.Match("/.wh..wh.*", path); err != nil {
-			return err
-		} else if matched {
-			addDir(fullPath)
-			if err := os.RemoveAll(fullPath); err != nil {
+		path := filepath.Join(dest, hdr.Name)
+		base := filepath.Base(path)
+		if strings.HasPrefix(base, ".wh.") {
+			originalBase := base[len(".wh."):]
+			originalPath := filepath.Join(filepath.Dir(path), originalBase)
+			if err := os.RemoveAll(originalPath); err != nil {
 				return err
 			}
-		}
+		} else {
+			// If path exits we almost always just want to remove and replace it
+			// The only exception is when it is a directory *and* the file from
+			// the layer is also a directory. Then we want to merge them (i.e.
+			// just apply the metadata from the layer).
+			hasDir := false
+			if fi, err := os.Lstat(path); err == nil {
+				if fi.IsDir() && hdr.Typeflag == tar.TypeDir {
+					hasDir = true
+				} else {
+					if err := os.RemoveAll(path); err != nil {
+						return err
+					}
+				}
+			}
 
-		filename := filepath.Base(path)
-		if strings.HasPrefix(filename, ".wh.") {
-			rmTargetName := filename[len(".wh."):]
-			rmTargetPath := filepath.Join(filepath.Dir(fullPath), rmTargetName)
+			switch hdr.Typeflag {
+			case tar.TypeDir:
+				if !hasDir {
+					err = os.Mkdir(path, os.FileMode(hdr.Mode))
+					if err != nil {
+						return err
+					}
+				}
+				dirs = append(dirs, hdr)
 
-			// Remove the file targeted by the whiteout
-			addDir(rmTargetPath)
-			if err := os.RemoveAll(rmTargetPath); err != nil {
-				return err
+			case tar.TypeReg, tar.TypeRegA:
+				// Source is regular file
+				file, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY, os.FileMode(hdr.Mode))
+				if err != nil {
+					return err
+				}
+				if _, err := io.Copy(file, tr); err != nil {
+					file.Close()
+					return err
+				}
+				file.Close()
+
+			case tar.TypeBlock, tar.TypeChar, tar.TypeFifo:
+				mode := uint32(hdr.Mode & 07777)
+				switch hdr.Typeflag {
+				case tar.TypeBlock:
+					mode |= syscall.S_IFBLK
+				case tar.TypeChar:
+					mode |= syscall.S_IFCHR
+				case tar.TypeFifo:
+					mode |= syscall.S_IFIFO
+				}
+
+				if err := syscall.Mknod(path, mode, int(mkdev(hdr.Devmajor, hdr.Devminor))); err != nil {
+					return err
+				}
+
+			case tar.TypeLink:
+				if err := os.Link(filepath.Join(dest, hdr.Linkname), path); err != nil {
+					return err
+				}
+
+			case tar.TypeSymlink:
+				if err := os.Symlink(hdr.Linkname, path); err != nil {
+					return err
+				}
+
+			default:
+				utils.Debugf("unhandled type %d\n", hdr.Typeflag)
 			}
-			// Remove the whiteout itself
-			addDir(fullPath)
-			if err := os.RemoveAll(fullPath); err != nil {
+
+			if err = syscall.Lchown(path, hdr.Uid, hdr.Gid); err != nil {
 				return err
 			}
+
+			// There is no LChmod, so ignore mode for symlink.  Also, this
+			// must happen after chown, as that can modify the file mode
+			if hdr.Typeflag != tar.TypeSymlink {
+				err = syscall.Chmod(path, uint32(hdr.Mode&07777))
+				if err != nil {
+					return err
+				}
+			}
+
+			// Directories must be handled at the end to avoid further
+			// file creation in them to modify the mtime
+			if hdr.Typeflag != tar.TypeDir {
+				ts := []syscall.Timespec{timeToTimespec(hdr.AccessTime), timeToTimespec(hdr.ModTime)}
+				// syscall.UtimesNano doesn't support a NOFOLLOW flag atm, and
+				if hdr.Typeflag != tar.TypeSymlink {
+					if err := syscall.UtimesNano(path, ts); err != nil {
+						return err
+					}
+				} else {
+					if err := LUtimesNano(path, ts); err != nil {
+						return err
+					}
+				}
+			}
 		}
-		return nil
-	})
-	if err != nil {
-		return err
 	}
 
-	for k, v := range modifiedDirs {
-		lastAccess := getLastAccess(v)
-		lastModification := getLastModification(v)
-		aTime := time.Unix(lastAccess.Unix())
-		mTime := time.Unix(lastModification.Unix())
-
-		if err := os.Chtimes(k, aTime, mTime); err != nil {
+	for _, hdr := range dirs {
+		path := filepath.Join(dest, hdr.Name)
+		ts := []syscall.Timespec{timeToTimespec(hdr.AccessTime), timeToTimespec(hdr.ModTime)}
+		if err := syscall.UtimesNano(path, ts); err != nil {
 			return err
 		}
 	}

archive/stat_darwin.go

@@ -9,3 +9,7 @@ func getLastAccess(stat *syscall.Stat_t) syscall.Timespec {
 func getLastModification(stat *syscall.Stat_t) syscall.Timespec {
 	return stat.Mtimespec
 }
+
+func LUtimesNano(path string, ts []syscall.Timespec) error {
+	return nil
+}

archive/stat_linux.go

@@ -1,6 +1,9 @@
 package archive
 
-import "syscall"
+import (
+	"syscall"
+	"unsafe"
+)
 
 func getLastAccess(stat *syscall.Stat_t) syscall.Timespec {
 	return stat.Atim
@@ -9,3 +12,21 @@ func getLastAccess(stat *syscall.Stat_t) syscall.Timespec {
 func getLastModification(stat *syscall.Stat_t) syscall.Timespec {
 	return stat.Mtim
 }
+
+func LUtimesNano(path string, ts []syscall.Timespec) error {
+	// These are not currently availible in syscall
+	AT_FDCWD := -100
+	AT_SYMLINK_NOFOLLOW := 0x100
+
+	var _path *byte
+	_path, err := syscall.BytePtrFromString(path)
+	if err != nil {
+		return err
+	}
+
+	if _, _, err := syscall.Syscall6(syscall.SYS_UTIMENSAT, uintptr(AT_FDCWD), uintptr(unsafe.Pointer(_path)), uintptr(unsafe.Pointer(&ts[0])), uintptr(AT_SYMLINK_NOFOLLOW), 0, 0); err != 0 && err != syscall.ENOSYS {
+		return err
+	}
+
+	return nil
+}