Merge pull request #21774 from Microsoft/jstarks/support_non_base_layered_images

Windows: support non-base-layered images

daemon/daemon_windows.go

@@ -373,8 +373,7 @@ func restoreCustomImage(is image.Store, ls layer.Store, rs reference.Store) erro
 		}
 		// layer is intentionally not released
 
-		rootFS := image.NewRootFS()
-		rootFS.BaseLayer = filepath.Base(info.Path)
+		rootFS := image.NewRootFSWithBaseLayer(filepath.Base(info.Path))
 
 		// Create history for base layer
 		config, err := json.Marshal(&image.Image{

daemon/graphdriver/windows/windows.go

@@ -26,6 +26,7 @@ import (
 	"github.com/docker/docker/pkg/chrootarchive"
 	"github.com/docker/docker/pkg/idtools"
 	"github.com/docker/docker/pkg/ioutils"
+	"github.com/docker/docker/pkg/longpath"
 	"github.com/vbatts/tar-split/tar/storage"
 )
 
@@ -319,10 +320,10 @@ func (d *Driver) Changes(id, parent string) ([]archive.Change, error) {
 		}
 		name = filepath.ToSlash(name)
 		if fileInfo == nil {
-			changes = append(changes, archive.Change{name, archive.ChangeDelete})
+			changes = append(changes, archive.Change{Path: name, Kind: archive.ChangeDelete})
 		} else {
 			// Currently there is no way to tell between an add and a modify.
-			changes = append(changes, archive.Change{name, archive.ChangeModify})
+			changes = append(changes, archive.Change{Path: name, Kind: archive.ChangeModify})
 		}
 	}
 	return changes, nil
@@ -332,45 +333,49 @@ func (d *Driver) Changes(id, parent string) ([]archive.Change, error) {
 // layer with the specified id and parent, returning the size of the
 // new layer in bytes.
 // The layer should not be mounted when calling this function
-func (d *Driver) ApplyDiff(id, parent string, diff archive.Reader) (size int64, err error) {
-	rPId, err := d.resolveID(parent)
-	if err != nil {
-		return
-	}
-
+func (d *Driver) ApplyDiff(id, parent string, diff archive.Reader) (int64, error) {
 	if d.info.Flavour == diffDriver {
 		start := time.Now().UTC()
 		logrus.Debugf("WindowsGraphDriver ApplyDiff: Start untar layer")
 		destination := d.dir(id)
 		destination = filepath.Dir(destination)
-		if size, err = chrootarchive.ApplyUncompressedLayer(destination, diff, nil); err != nil {
-			return
+		size, err := chrootarchive.ApplyUncompressedLayer(destination, diff, nil)
+		if err != nil {
+			return 0, err
 		}
 		logrus.Debugf("WindowsGraphDriver ApplyDiff: Untar time: %vs", time.Now().UTC().Sub(start).Seconds())
 
-		return
+		return size, nil
 	}
 
-	parentChain, err := d.getLayerChain(rPId)
-	if err != nil {
-		return
-	}
-	parentPath, err := hcsshim.GetLayerMountPath(d.info, rPId)
-	if err != nil {
-		return
+	var layerChain []string
+	if parent != "" {
+		rPId, err := d.resolveID(parent)
+		if err != nil {
+			return 0, err
+		}
+		parentChain, err := d.getLayerChain(rPId)
+		if err != nil {
+			return 0, err
+		}
+		parentPath, err := hcsshim.GetLayerMountPath(d.info, rPId)
+		if err != nil {
+			return 0, err
+		}
+		layerChain = append(layerChain, parentPath)
+		layerChain = append(layerChain, parentChain...)
 	}
-	layerChain := []string{parentPath}
-	layerChain = append(layerChain, parentChain...)
 
-	if size, err = d.importLayer(id, diff, layerChain); err != nil {
-		return
+	size, err := d.importLayer(id, diff, layerChain)
+	if err != nil {
+		return 0, err
 	}
 
 	if err = d.setLayerChain(id, layerChain); err != nil {
-		return
+		return 0, err
 	}
 
-	return
+	return size, nil
 }
 
 // DiffSize calculates the changes between the specified layer
@@ -539,6 +544,12 @@ func writeLayerFromTar(r archive.Reader, w hcsshim.LayerWriter) (int64, error) {
 				return 0, err
 			}
 			hdr, err = t.Next()
+		} else if hdr.Typeflag == tar.TypeLink {
+			err = w.AddLink(filepath.FromSlash(hdr.Name), filepath.FromSlash(hdr.Linkname))
+			if err != nil {
+				return 0, err
+			}
+			hdr, err = t.Next()
 		} else {
 			var (
 				name     string
@@ -575,7 +586,6 @@ func (d *Driver) importLayer(id string, layerData archive.Reader, parentLayerPat
 	if err != nil {
 		return
 	}
-
 	size, err = writeLayerFromTar(layerData, w)
 	if err != nil {
 		w.Close()
@@ -653,7 +663,7 @@ func (fg *fileGetCloserWithBackupPrivileges) Get(filename string) (io.ReadCloser
 	// file can be opened even if the caller does not actually have access to it according
 	// to the security descriptor.
 	err := winio.RunWithPrivilege(winio.SeBackupPrivilege, func() error {
-		path := filepath.Join(fg.path, filename)
+		path := longpath.AddPrefix(filepath.Join(fg.path, filename))
 		p, err := syscall.UTF16FromString(path)
 		if err != nil {
 			return err

daemon/oci_windows.go

@@ -5,6 +5,7 @@ import (
 	"syscall"
 
 	"github.com/docker/docker/container"
+	"github.com/docker/docker/image"
 	"github.com/docker/docker/layer"
 	"github.com/docker/docker/libcontainerd"
 	"github.com/docker/docker/libcontainerd/windowsoci"
@@ -88,9 +89,15 @@ func (daemon *Daemon) createSpec(c *container.Container) (*libcontainerd.Spec, e
 
 	// s.Windows.LayerPaths
 	var layerPaths []string
-	if img.RootFS != nil && img.RootFS.Type == "layers+base" {
+	if img.RootFS != nil && (img.RootFS.Type == image.TypeLayers || img.RootFS.Type == image.TypeLayersWithBase) {
+		// Get the layer path for each layer.
+		start := 1
+		if img.RootFS.Type == image.TypeLayersWithBase {
+			// Include an empty slice to get the base layer ID.
+			start = 0
+		}
 		max := len(img.RootFS.DiffIDs)
-		for i := 0; i <= max; i++ {
+		for i := start; i <= max; i++ {
 			img.RootFS.DiffIDs = img.RootFS.DiffIDs[:i]
 			path, err := layer.GetLayerPath(daemon.layerStore, img.RootFS.ChainID())
 			if err != nil {

distribution/pull_v2_windows.go

@@ -20,8 +20,9 @@ func detectBaseLayer(is image.Store, m *schema1.Manifest, rootFS *image.RootFS)
 	}
 	// There must be an image that already references the baselayer.
 	for _, img := range is.Map() {
-		if img.RootFS.BaseLayerID() == v1img.Parent {
+		if img.RootFS.Type == image.TypeLayersWithBase && img.RootFS.BaseLayerID() == v1img.Parent {
 			rootFS.BaseLayer = img.RootFS.BaseLayer
+			rootFS.Type = image.TypeLayersWithBase
 			return nil
 		}
 	}

hack/vendor.sh

@@ -7,8 +7,8 @@ source 'hack/.vendor-helpers.sh'
 
 # the following lines are in sorted order, FYI
 clone git github.com/Azure/go-ansiterm 70b2c90b260171e829f1ebd7c17f600c11858dbe
-clone git github.com/Microsoft/hcsshim v0.1.0
-clone git github.com/Microsoft/go-winio v0.1.0
+clone git github.com/Microsoft/hcsshim v0.2.0
+clone git github.com/Microsoft/go-winio v0.3.0
 clone git github.com/Sirupsen/logrus v0.9.0 # logrus is a common dependency among multiple deps
 clone git github.com/docker/libtrust 9cbd2a1374f46905c68a4eb3694a130610adc62a
 clone git github.com/go-check/check a625211d932a2a643d0d17352095f03fb7774663 https://github.com/cpuguy83/check.git

image/rootfs.go

@@ -2,6 +2,14 @@ package image
 
 import "github.com/docker/docker/layer"
 
+// TypeLayers is used for RootFS.Type for filesystems organized into layers.
+const TypeLayers = "layers"
+
+// NewRootFS returns empty RootFS struct
+func NewRootFS() *RootFS {
+	return &RootFS{Type: TypeLayers}
+}
+
 // Append appends a new diffID to rootfs
 func (r *RootFS) Append(id layer.DiffID) {
 	r.DiffIDs = append(r.DiffIDs, id)

image/rootfs_unix.go

@@ -16,8 +16,3 @@ type RootFS struct {
 func (r *RootFS) ChainID() layer.ChainID {
 	return layer.CreateChainID(r.DiffIDs)
 }
-
-// NewRootFS returns empty RootFS struct
-func NewRootFS() *RootFS {
-	return &RootFS{Type: "layers"}
-}

image/rootfs_windows.go

@@ -10,6 +10,9 @@ import (
 	"github.com/docker/docker/layer"
 )
 
+// TypeLayersWithBase is used for RootFS.Type for Windows filesystems that have layers and a centrally-stored base layer.
+const TypeLayersWithBase = "layers+base"
+
 // RootFS describes images root filesystem
 // This is currently a placeholder that only supports layers. In the future
 // this can be made into an interface that supports different implementations.
@@ -21,17 +24,25 @@ type RootFS struct {
 
 // BaseLayerID returns the 64 byte hex ID for the baselayer name.
 func (r *RootFS) BaseLayerID() string {
+	if r.Type != TypeLayersWithBase {
+		panic("tried to get base layer ID without a base layer")
+	}
 	baseID := sha512.Sum384([]byte(r.BaseLayer))
 	return fmt.Sprintf("%x", baseID[:32])
 }
 
 // ChainID returns the ChainID for the top layer in RootFS.
 func (r *RootFS) ChainID() layer.ChainID {
-	baseDiffID := digest.FromBytes([]byte(r.BaseLayerID()))
-	return layer.CreateChainID(append([]layer.DiffID{layer.DiffID(baseDiffID)}, r.DiffIDs...))
+	ids := r.DiffIDs
+	if r.Type == TypeLayersWithBase {
+		// Add an extra ID for the base.
+		baseDiffID := layer.DiffID(digest.FromBytes([]byte(r.BaseLayerID())))
+		ids = append([]layer.DiffID{baseDiffID}, ids...)
+	}
+	return layer.CreateChainID(ids)
 }
 
-// NewRootFS returns empty RootFS struct
-func NewRootFS() *RootFS {
-	return &RootFS{Type: "layers+base"}
+// NewRootFSWithBaseLayer returns a RootFS struct with a base layer
+func NewRootFSWithBaseLayer(baseLayer string) *RootFS {
+	return &RootFS{Type: TypeLayersWithBase, BaseLayer: baseLayer}
 }

vendor/src/github.com/Microsoft/go-winio/archive/tar/common.go

@@ -44,22 +44,23 @@ const (
 // A Header represents a single header in a tar archive.
 // Some fields may not be populated.
 type Header struct {
-	Name       string    // name of header file entry
-	Mode       int64     // permission and mode bits
-	Uid        int       // user id of owner
-	Gid        int       // group id of owner
-	Size       int64     // length in bytes
-	ModTime    time.Time // modified time
-	Typeflag   byte      // type of header entry
-	Linkname   string    // target name of link
-	Uname      string    // user name of owner
-	Gname      string    // group name of owner
-	Devmajor   int64     // major number of character or block device
-	Devminor   int64     // minor number of character or block device
-	AccessTime time.Time // access time
-	ChangeTime time.Time // status change time
-	Xattrs     map[string]string
-	Winheaders map[string]string
+	Name         string    // name of header file entry
+	Mode         int64     // permission and mode bits
+	Uid          int       // user id of owner
+	Gid          int       // group id of owner
+	Size         int64     // length in bytes
+	ModTime      time.Time // modified time
+	Typeflag     byte      // type of header entry
+	Linkname     string    // target name of link
+	Uname        string    // user name of owner
+	Gname        string    // group name of owner
+	Devmajor     int64     // major number of character or block device
+	Devminor     int64     // minor number of character or block device
+	AccessTime   time.Time // access time
+	ChangeTime   time.Time // status change time
+	CreationTime time.Time // creation time
+	Xattrs       map[string]string
+	Winheaders   map[string]string
 }
 
 // File name constants from the tar spec.
@@ -180,21 +181,22 @@ const (
 
 // Keywords for the PAX Extended Header
 const (
-	paxAtime    = "atime"
-	paxCharset  = "charset"
-	paxComment  = "comment"
-	paxCtime    = "ctime" // please note that ctime is not a valid pax header.
-	paxGid      = "gid"
-	paxGname    = "gname"
-	paxLinkpath = "linkpath"
-	paxMtime    = "mtime"
-	paxPath     = "path"
-	paxSize     = "size"
-	paxUid      = "uid"
-	paxUname    = "uname"
-	paxXattr    = "SCHILY.xattr."
-	paxWindows  = "MSWINDOWS."
-	paxNone     = ""
+	paxAtime        = "atime"
+	paxCharset      = "charset"
+	paxComment      = "comment"
+	paxCtime        = "ctime" // please note that ctime is not a valid pax header.
+	paxCreationTime = "LIBARCHIVE.creationtime"
+	paxGid          = "gid"
+	paxGname        = "gname"
+	paxLinkpath     = "linkpath"
+	paxMtime        = "mtime"
+	paxPath         = "path"
+	paxSize         = "size"
+	paxUid          = "uid"
+	paxUname        = "uname"
+	paxXattr        = "SCHILY.xattr."
+	paxWindows      = "MSWINDOWS."
+	paxNone         = ""
 )
 
 // FileInfoHeader creates a partially-populated Header from fi.

vendor/src/github.com/Microsoft/go-winio/archive/tar/reader.go

@@ -302,6 +302,12 @@ func mergePAX(hdr *Header, headers map[string]string) error {
 				return err
 			}
 			hdr.ChangeTime = t
+		case paxCreationTime:
+			t, err := parsePAXTime(v)
+			if err != nil {
+				return err
+			}
+			hdr.CreationTime = t
 		case paxSize:
 			size, err := strconv.ParseInt(v, 10, 0)
 			if err != nil {

vendor/src/github.com/Microsoft/go-winio/archive/tar/writer.go

@@ -47,7 +47,7 @@ type formatter struct {
 }
 
 // NewWriter creates a new Writer writing to w.
-func NewWriter(w io.Writer) *Writer { return &Writer{w: w} }
+func NewWriter(w io.Writer) *Writer { return &Writer{w: w, preferPax: true} }
 
 // Flush finishes writing the current file (optional).
 func (tw *Writer) Flush() error {
@@ -201,23 +201,29 @@ func (tw *Writer) writeHeader(hdr *Header, allowPax bool) error {
 		tw.usedBinary = true
 		f.formatNumeric(b, x)
 	}
+	var formatTime = func(b []byte, t time.Time, paxKeyword string) {
+		var unixTime int64
+		if !t.Before(minTime) && !t.After(maxTime) {
+			unixTime = t.Unix()
+		}
+		formatNumeric(b, unixTime, paxNone)
+
+		// Write a PAX header if the time didn't fit precisely.
+		if paxKeyword != "" && tw.preferPax && allowPax && (t.Nanosecond() != 0 || !t.Before(minTime) || !t.After(maxTime)) {
+			paxHeaders[paxKeyword] = formatPAXTime(t)
+		}
+	}
 
 	// keep a reference to the filename to allow to overwrite it later if we detect that we can use ustar longnames instead of pax
 	pathHeaderBytes := s.next(fileNameSize)
 
 	formatString(pathHeaderBytes, hdr.Name, paxPath)
 
-	// Handle out of range ModTime carefully.
-	var modTime int64
-	if !hdr.ModTime.Before(minTime) && !hdr.ModTime.After(maxTime) {
-		modTime = hdr.ModTime.Unix()
-	}
-
 	f.formatOctal(s.next(8), hdr.Mode)               // 100:108
 	formatNumeric(s.next(8), int64(hdr.Uid), paxUid) // 108:116
 	formatNumeric(s.next(8), int64(hdr.Gid), paxGid) // 116:124
 	formatNumeric(s.next(12), hdr.Size, paxSize)     // 124:136
-	formatNumeric(s.next(12), modTime, paxNone)      // 136:148 --- consider using pax for finer granularity
+	formatTime(s.next(12), hdr.ModTime, paxMtime)    // 136:148
 	s.next(8)                                        // chksum (148:156)
 	s.next(1)[0] = hdr.Typeflag                      // 156:157
 
@@ -265,6 +271,15 @@ func (tw *Writer) writeHeader(hdr *Header, allowPax bool) error {
 	}
 
 	if allowPax {
+		if !hdr.AccessTime.IsZero() {
+			paxHeaders[paxAtime] = formatPAXTime(hdr.AccessTime)
+		}
+		if !hdr.ChangeTime.IsZero() {
+			paxHeaders[paxCtime] = formatPAXTime(hdr.ChangeTime)
+		}
+		if !hdr.CreationTime.IsZero() {
+			paxHeaders[paxCreationTime] = formatPAXTime(hdr.CreationTime)
+		}
 		for k, v := range hdr.Xattrs {
 			paxHeaders[paxXattr+k] = v
 		}
@@ -288,6 +303,16 @@ func (tw *Writer) writeHeader(hdr *Header, allowPax bool) error {
 	return tw.err
 }
 
+func formatPAXTime(t time.Time) string {
+	sec := t.Unix()
+	usec := t.Nanosecond()
+	s := strconv.FormatInt(sec, 10)
+	if usec != 0 {
+		s = fmt.Sprintf("%s.%09d", s, usec)
+	}
+	return s
+}
+
 // splitUSTARPath splits a path according to USTAR prefix and suffix rules.
 // If the path is not splittable, then it will return ("", "", false).
 func splitUSTARPath(name string) (prefix, suffix string, ok bool) {

vendor/src/github.com/Microsoft/go-winio/backup.go

@@ -26,10 +26,18 @@ const (
 	BackupReparseData
 	BackupSparseBlock
 	BackupTxfsData
+)
 
+const (
 	StreamSparseAttributes = uint32(8)
 )
 
+const (
+	WRITE_DAC              = 0x40000
+	WRITE_OWNER            = 0x80000
+	ACCESS_SYSTEM_SECURITY = 0x1000000
+)
+
 // BackupHeader represents a backup stream of a file.
 type BackupHeader struct {
 	Id         uint32 // The backup stream ID
@@ -239,3 +247,20 @@ func (w *BackupFileWriter) Close() error {
 	}
 	return nil
 }
+
+// OpenForBackup opens a file or directory, potentially skipping access checks if the backup
+// or restore privileges have been acquired.
+//
+// If the file opened was a directory, it cannot be used with Readdir().
+func OpenForBackup(path string, access uint32, share uint32, createmode uint32) (*os.File, error) {
+	winPath, err := syscall.UTF16FromString(path)
+	if err != nil {
+		return nil, err
+	}
+	h, err := syscall.CreateFile(&winPath[0], access, share, nil, createmode, syscall.FILE_FLAG_BACKUP_SEMANTICS, 0)
+	if err != nil {
+		err = &os.PathError{Op: "open", Path: path, Err: err}
+		return nil, err
+	}
+	return os.NewFile(uintptr(h), path), nil
+}

vendor/src/github.com/Microsoft/go-winio/backuptar/tar.go

@@ -30,10 +30,6 @@ const (
 
 const (
 	hdrFileAttributes     = "fileattr"
-	hdrAccessTime         = "accesstime"
-	hdrChangeTime         = "changetime"
-	hdrCreateTime         = "createtime"
-	hdrWriteTime          = "writetime"
 	hdrSecurityDescriptor = "sd"
 	hdrMountPoint         = "mountpoint"
 )
@@ -82,21 +78,29 @@ func copySparse(t *tar.Writer, br *winio.BackupStreamReader) error {
 	return nil
 }
 
-func win32TimeFromTar(key string, hdrs map[string]string, unixTime time.Time) syscall.Filetime {
-	if s, ok := hdrs[key]; ok {
-		n, err := strconv.ParseUint(s, 10, 64)
-		if err == nil {
-			return syscall.Filetime{uint32(n & 0xffffffff), uint32(n >> 32)}
-		}
+// BasicInfoHeader creates a tar header from basic file information.
+func BasicInfoHeader(name string, size int64, fileInfo *winio.FileBasicInfo) *tar.Header {
+	hdr := &tar.Header{
+		Name:         filepath.ToSlash(name),
+		Size:         size,
+		Typeflag:     tar.TypeReg,
+		ModTime:      time.Unix(0, fileInfo.LastWriteTime.Nanoseconds()),
+		ChangeTime:   time.Unix(0, fileInfo.ChangeTime.Nanoseconds()),
+		AccessTime:   time.Unix(0, fileInfo.LastAccessTime.Nanoseconds()),
+		CreationTime: time.Unix(0, fileInfo.CreationTime.Nanoseconds()),
+		Winheaders:   make(map[string]string),
 	}
-	return syscall.NsecToFiletime(unixTime.UnixNano())
-}
+	hdr.Winheaders[hdrFileAttributes] = fmt.Sprintf("%d", fileInfo.FileAttributes)
 
-func win32TimeToTar(ft syscall.Filetime) (string, time.Time) {
-	return fmt.Sprintf("%d", uint64(ft.LowDateTime)+(uint64(ft.HighDateTime)<<32)), time.Unix(0, ft.Nanoseconds())
+	if (fileInfo.FileAttributes & syscall.FILE_ATTRIBUTE_DIRECTORY) != 0 {
+		hdr.Mode |= c_ISDIR
+		hdr.Size = 0
+		hdr.Typeflag = tar.TypeDir
+	}
+	return hdr
 }
 
-// Writes a file to a tar writer using data from a Win32 backup stream.
+// WriteTarFileFromBackupStream writes a file to a tar writer using data from a Win32 backup stream.
 //
 // This encodes Win32 metadata as tar pax vendor extensions starting with MSWINDOWS.
 //
@@ -104,37 +108,12 @@ func win32TimeToTar(ft syscall.Filetime) (string, time.Time) {
 //
 // MSWINDOWS.fileattr: The Win32 file attributes, as a decimal value
 //
-// MSWINDOWS.accesstime: The last access time, as a Filetime expressed as a 64-bit decimal value.
-//
-// MSWINDOWS.createtime: The creation time, as a Filetime expressed as a 64-bit decimal value.
-//
-// MSWINDOWS.changetime: The creation time, as a Filetime expressed as a 64-bit decimal value.
-//
-// MSWINDOWS.writetime: The creation time, as a Filetime expressed as a 64-bit decimal value.
-//
 // MSWINDOWS.sd: The Win32 security descriptor, in SDDL (string) format
 //
 // MSWINDOWS.mountpoint: If present, this is a mount point and not a symlink, even though the type is '2' (symlink)
 func WriteTarFileFromBackupStream(t *tar.Writer, r io.Reader, name string, size int64, fileInfo *winio.FileBasicInfo) error {
 	name = filepath.ToSlash(name)
-	hdr := &tar.Header{
-		Name:       name,
-		Size:       size,
-		Typeflag:   tar.TypeReg,
-		Winheaders: make(map[string]string),
-	}
-	hdr.Winheaders[hdrFileAttributes] = fmt.Sprintf("%d", fileInfo.FileAttributes)
-	hdr.Winheaders[hdrAccessTime], hdr.AccessTime = win32TimeToTar(fileInfo.LastAccessTime)
-	hdr.Winheaders[hdrChangeTime], hdr.ChangeTime = win32TimeToTar(fileInfo.ChangeTime)
-	hdr.Winheaders[hdrCreateTime], _ = win32TimeToTar(fileInfo.CreationTime)
-	hdr.Winheaders[hdrWriteTime], hdr.ModTime = win32TimeToTar(fileInfo.LastWriteTime)
-
-	if (fileInfo.FileAttributes & syscall.FILE_ATTRIBUTE_DIRECTORY) != 0 {
-		hdr.Mode |= c_ISDIR
-		hdr.Size = 0
-		hdr.Typeflag = tar.TypeDir
-	}
-
+	hdr := BasicInfoHeader(name, size, fileInfo)
 	br := winio.NewBackupStreamReader(r)
 	var dataHdr *winio.BackupHeader
 	for dataHdr == nil {
@@ -252,7 +231,7 @@ func WriteTarFileFromBackupStream(t *tar.Writer, r io.Reader, name string, size
 	return nil
 }
 
-// Retrieves basic Win32 file information from a tar header, using the additional metadata written by
+// FileInfoFromHeader retrieves basic Win32 file information from a tar header, using the additional metadata written by
 // WriteTarFileFromBackupStream.
 func FileInfoFromHeader(hdr *tar.Header) (name string, size int64, fileInfo *winio.FileBasicInfo, err error) {
 	name = hdr.Name
@@ -260,10 +239,10 @@ func FileInfoFromHeader(hdr *tar.Header) (name string, size int64, fileInfo *win
 		size = hdr.Size
 	}
 	fileInfo = &winio.FileBasicInfo{
-		LastAccessTime: win32TimeFromTar(hdrAccessTime, hdr.Winheaders, hdr.AccessTime),
-		LastWriteTime:  win32TimeFromTar(hdrWriteTime, hdr.Winheaders, hdr.ModTime),
-		ChangeTime:     win32TimeFromTar(hdrChangeTime, hdr.Winheaders, hdr.ChangeTime),
-		CreationTime:   win32TimeFromTar(hdrCreateTime, hdr.Winheaders, hdr.ModTime),
+		LastAccessTime: syscall.NsecToFiletime(hdr.AccessTime.UnixNano()),
+		LastWriteTime:  syscall.NsecToFiletime(hdr.ModTime.UnixNano()),
+		ChangeTime:     syscall.NsecToFiletime(hdr.ChangeTime.UnixNano()),
+		CreationTime:   syscall.NsecToFiletime(hdr.CreationTime.UnixNano()),
 	}
 	if attrStr, ok := hdr.Winheaders[hdrFileAttributes]; ok {
 		attr, err := strconv.ParseUint(attrStr, 10, 32)
@@ -279,7 +258,7 @@ func FileInfoFromHeader(hdr *tar.Header) (name string, size int64, fileInfo *win
 	return
 }
 
-// Writes a Win32 backup stream from the current tar file. Since this function may process multiple
+// WriteBackupStreamFromTarFile writes a Win32 backup stream from the current tar file. Since this function may process multiple
 // tar file entries in order to collect all the alternate data streams for the file, it returns the next
 // tar file that was not processed, or io.EOF is there are no more.
 func WriteBackupStreamFromTarFile(w io.Writer, t *tar.Reader, hdr *tar.Header) (*tar.Header, error) {

vendor/src/github.com/Microsoft/go-winio/fileinfo.go

@@ -9,22 +9,46 @@ import (
 //sys getFileInformationByHandleEx(h syscall.Handle, class uint32, buffer *byte, size uint32) (err error) = GetFileInformationByHandleEx
 //sys setFileInformationByHandle(h syscall.Handle, class uint32, buffer *byte, size uint32) (err error) = SetFileInformationByHandle
 
+const (
+	fileBasicInfo = 0
+	fileIDInfo    = 0x12
+)
+
+// FileBasicInfo contains file access time and file attributes information.
 type FileBasicInfo struct {
 	CreationTime, LastAccessTime, LastWriteTime, ChangeTime syscall.Filetime
 	FileAttributes                                          uintptr // includes padding
 }
 
+// GetFileBasicInfo retrieves times and attributes for a file.
 func GetFileBasicInfo(f *os.File) (*FileBasicInfo, error) {
 	bi := &FileBasicInfo{}
-	if err := getFileInformationByHandleEx(syscall.Handle(f.Fd()), 0, (*byte)(unsafe.Pointer(bi)), uint32(unsafe.Sizeof(*bi))); err != nil {
-		return nil, &os.PathError{"GetFileInformationByHandleEx", f.Name(), err}
+	if err := getFileInformationByHandleEx(syscall.Handle(f.Fd()), fileBasicInfo, (*byte)(unsafe.Pointer(bi)), uint32(unsafe.Sizeof(*bi))); err != nil {
+		return nil, &os.PathError{Op: "GetFileInformationByHandleEx", Path: f.Name(), Err: err}
 	}
 	return bi, nil
 }
 
+// SetFileBasicInfo sets times and attributes for a file.
 func SetFileBasicInfo(f *os.File, bi *FileBasicInfo) error {
-	if err := setFileInformationByHandle(syscall.Handle(f.Fd()), 0, (*byte)(unsafe.Pointer(bi)), uint32(unsafe.Sizeof(*bi))); err != nil {
-		return &os.PathError{"SetFileInformationByHandle", f.Name(), err}
+	if err := setFileInformationByHandle(syscall.Handle(f.Fd()), fileBasicInfo, (*byte)(unsafe.Pointer(bi)), uint32(unsafe.Sizeof(*bi))); err != nil {
+		return &os.PathError{Op: "SetFileInformationByHandle", Path: f.Name(), Err: err}
 	}
 	return nil
 }
+
+// FileIDInfo contains the volume serial number and file ID for a file. This pair should be
+// unique on a system.
+type FileIDInfo struct {
+	VolumeSerialNumber uint64
+	FileID             [16]byte
+}
+
+// GetFileID retrieves the unique (volume, file ID) pair for a file.
+func GetFileID(f *os.File) (*FileIDInfo, error) {
+	fileID := &FileIDInfo{}
+	if err := getFileInformationByHandleEx(syscall.Handle(f.Fd()), fileIDInfo, (*byte)(unsafe.Pointer(fileID)), uint32(unsafe.Sizeof(*fileID))); err != nil {
+		return nil, &os.PathError{Op: "GetFileInformationByHandleEx", Path: f.Name(), Err: err}
+	}
+	return fileID, nil
+}

vendor/src/github.com/Microsoft/go-winio/reparse.go

@@ -43,8 +43,12 @@ func (e *UnsupportedReparsePointError) Error() string {
 // DecodeReparsePoint decodes a Win32 REPARSE_DATA_BUFFER structure containing either a symlink
 // or a mount point.
 func DecodeReparsePoint(b []byte) (*ReparsePoint, error) {
-	isMountPoint := false
 	tag := binary.LittleEndian.Uint32(b[0:4])
+	return DecodeReparsePointData(tag, b[8:])
+}
+
+func DecodeReparsePointData(tag uint32, b []byte) (*ReparsePoint, error) {
+	isMountPoint := false
 	switch tag {
 	case reparseTagMountPoint:
 		isMountPoint = true
@@ -52,11 +56,11 @@ func DecodeReparsePoint(b []byte) (*ReparsePoint, error) {
 	default:
 		return nil, &UnsupportedReparsePointError{tag}
 	}
-	nameOffset := 16 + binary.LittleEndian.Uint16(b[12:14])
+	nameOffset := 8 + binary.LittleEndian.Uint16(b[4:6])
 	if !isMountPoint {
 		nameOffset += 4
 	}
-	nameLength := binary.LittleEndian.Uint16(b[14:16])
+	nameLength := binary.LittleEndian.Uint16(b[6:8])
 	name := make([]uint16, nameLength/2)
 	err := binary.Read(bytes.NewReader(b[nameOffset:nameOffset+nameLength]), binary.LittleEndian, &name)
 	if err != nil {

vendor/src/github.com/Microsoft/hcsshim/baselayer.go

@@ -0,0 +1,144 @@
+package hcsshim
+
+import (
+	"errors"
+	"os"
+	"path/filepath"
+	"syscall"
+
+	"github.com/Microsoft/go-winio"
+)
+
+type baseLayerWriter struct {
+	root string
+	f    *os.File
+	bw   *winio.BackupFileWriter
+	err  error
+}
+
+func (w *baseLayerWriter) closeCurrentFile() error {
+	if w.f != nil {
+		err := w.bw.Close()
+		err2 := w.f.Close()
+		w.f = nil
+		w.bw = nil
+		if err != nil {
+			return err
+		}
+		if err2 != nil {
+			return err2
+		}
+	}
+	return nil
+}
+
+func (w *baseLayerWriter) Add(name string, fileInfo *winio.FileBasicInfo) (err error) {
+	defer func() {
+		if err != nil {
+			w.err = err
+		}
+	}()
+
+	err = w.closeCurrentFile()
+	if err != nil {
+		return err
+	}
+
+	path := filepath.Join(w.root, name)
+	path, err = makeLongAbsPath(path)
+	if err != nil {
+		return err
+	}
+
+	var f *os.File
+	defer func() {
+		if f != nil {
+			f.Close()
+		}
+	}()
+
+	err = winio.RunWithPrivileges([]string{winio.SeBackupPrivilege, winio.SeRestorePrivilege}, func() (err error) {
+		createmode := uint32(syscall.CREATE_NEW)
+		if fileInfo.FileAttributes&syscall.FILE_ATTRIBUTE_DIRECTORY != 0 {
+			err := os.Mkdir(path, 0)
+			if err != nil && !os.IsExist(err) {
+				return err
+			}
+			createmode = syscall.OPEN_EXISTING
+		}
+
+		mode := uint32(syscall.GENERIC_READ | syscall.GENERIC_WRITE | winio.WRITE_DAC | winio.WRITE_OWNER | winio.ACCESS_SYSTEM_SECURITY)
+		f, err = winio.OpenForBackup(path, mode, syscall.FILE_SHARE_READ, createmode)
+		return
+	})
+	if err != nil {
+		return err
+	}
+
+	err = winio.SetFileBasicInfo(f, fileInfo)
+	if err != nil {
+		return err
+	}
+
+	w.f = f
+	w.bw = winio.NewBackupFileWriter(f, true)
+	f = nil
+	return nil
+}
+
+func (w *baseLayerWriter) AddLink(name string, target string) (err error) {
+	defer func() {
+		if err != nil {
+			w.err = err
+		}
+	}()
+
+	err = w.closeCurrentFile()
+	if err != nil {
+		return err
+	}
+
+	linkpath, err := makeLongAbsPath(filepath.Join(w.root, name))
+	if err != nil {
+		return err
+	}
+
+	linktarget, err := makeLongAbsPath(filepath.Join(w.root, target))
+	if err != nil {
+		return err
+	}
+
+	return winio.RunWithPrivileges([]string{winio.SeBackupPrivilege, winio.SeRestorePrivilege}, func() (err error) {
+		return os.Link(linktarget, linkpath)
+	})
+}
+
+func (w *baseLayerWriter) Remove(name string) error {
+	return errors.New("base layer cannot have tombstones")
+}
+
+func (w *baseLayerWriter) Write(b []byte) (int, error) {
+	var n int
+	err := winio.RunWithPrivileges([]string{winio.SeBackupPrivilege, winio.SeRestorePrivilege}, func() (err error) {
+		n, err = w.bw.Write(b)
+		return
+	})
+	if err != nil {
+		w.err = err
+	}
+	return n, err
+}
+
+func (w *baseLayerWriter) Close() error {
+	err := w.closeCurrentFile()
+	if err != nil {
+		return err
+	}
+	if w.err == nil {
+		err = ProcessBaseLayer(w.root)
+		if err != nil {
+			return err
+		}
+	}
+	return w.err
+}

vendor/src/github.com/Microsoft/hcsshim/exportlayer.go

@@ -125,7 +125,7 @@ func NewLayerReader(info DriverInfo, layerId string, parentLayerPaths []string)
 			os.RemoveAll(path)
 			return nil, err
 		}
-		return &legacyLayerReaderWrapper{NewLegacyLayerReader(path)}, nil
+		return &legacyLayerReaderWrapper{newLegacyLayerReader(path)}, nil
 	}
 
 	layers, err := layerPathsToDescriptors(parentLayerPaths)
@@ -146,11 +146,11 @@ func NewLayerReader(info DriverInfo, layerId string, parentLayerPaths []string)
 }
 
 type legacyLayerReaderWrapper struct {
-	*LegacyLayerReader
+	*legacyLayerReader
 }
 
 func (r *legacyLayerReaderWrapper) Close() error {
-	err := r.LegacyLayerReader.Close()
+	err := r.legacyLayerReader.Close()
 	os.RemoveAll(r.root)
 	return err
 }

vendor/src/github.com/Microsoft/hcsshim/importlayer.go

@@ -1,6 +1,7 @@
 package hcsshim
 
 import (
+	"errors"
 	"io/ioutil"
 	"os"
 	"path/filepath"
@@ -14,9 +15,9 @@ import (
 // that into a layer with the id layerId.  Note that in order to correctly populate
 // the layer and interperet the transport format, all parent layers must already
 // be present on the system at the paths provided in parentLayerPaths.
-func ImportLayer(info DriverInfo, layerId string, importFolderPath string, parentLayerPaths []string) error {
+func ImportLayer(info DriverInfo, layerID string, importFolderPath string, parentLayerPaths []string) error {
 	title := "hcsshim::ImportLayer "
-	logrus.Debugf(title+"flavour %d layerId %s folder %s", info.Flavour, layerId, importFolderPath)
+	logrus.Debugf(title+"flavour %d layerId %s folder %s", info.Flavour, layerID, importFolderPath)
 
 	// Generate layer descriptors
 	layers, err := layerPathsToDescriptors(parentLayerPaths)
@@ -31,21 +32,29 @@ func ImportLayer(info DriverInfo, layerId string, importFolderPath string, paren
 		return err
 	}
 
-	err = importLayer(&infop, layerId, importFolderPath, layers)
+	err = importLayer(&infop, layerID, importFolderPath, layers)
 	if err != nil {
-		err = makeErrorf(err, title, "layerId=%s flavour=%d folder=%s", layerId, info.Flavour, importFolderPath)
+		err = makeErrorf(err, title, "layerId=%s flavour=%d folder=%s", layerID, info.Flavour, importFolderPath)
 		logrus.Error(err)
 		return err
 	}
 
-	logrus.Debugf(title+"succeeded flavour=%d layerId=%s folder=%s", info.Flavour, layerId, importFolderPath)
+	logrus.Debugf(title+"succeeded flavour=%d layerId=%s folder=%s", info.Flavour, layerID, importFolderPath)
 	return nil
 }
 
+// LayerWriter is an interface that supports writing a new container image layer.
 type LayerWriter interface {
+	// Add adds a file to the layer with given metadata.
 	Add(name string, fileInfo *winio.FileBasicInfo) error
+	// AddLink adds a hard link to the layer. The target must already have been added.
+	AddLink(name string, target string) error
+	// Remove removes a file that was present in a parent layer from the layer.
 	Remove(name string) error
+	// Write writes data to the current file. The data must be in the format of a Win32
+	// backup stream.
 	Write(b []byte) (int, error)
+	// Close finishes the layer writing process and releases any resources.
 	Close() error
 }
 
@@ -70,6 +79,11 @@ func (w *FilterLayerWriter) Add(name string, fileInfo *winio.FileBasicInfo) erro
 	return nil
 }
 
+// AddLink adds a hard link to the layer. The target of the link must have already been added.
+func (w *FilterLayerWriter) AddLink(name string, target string) error {
+	return errors.New("hard links not yet supported")
+}
+
 // Remove removes a file from the layer. The file must have been present in the parent layer.
 //
 // name contains the file's relative path.
@@ -108,21 +122,22 @@ func (w *FilterLayerWriter) Close() (err error) {
 }
 
 type legacyLayerWriterWrapper struct {
-	*LegacyLayerWriter
+	*legacyLayerWriter
 	info             DriverInfo
-	layerId          string
+	layerID          string
 	path             string
 	parentLayerPaths []string
 }
 
 func (r *legacyLayerWriterWrapper) Close() error {
-	err := r.LegacyLayerWriter.Close()
+	err := r.legacyLayerWriter.Close()
 	if err == nil {
+		var fullPath string
 		// Use the original path here because ImportLayer does not support long paths for the source in TP5.
 		// But do use a long path for the destination to work around another bug with directories
 		// with MAX_PATH - 12 < length < MAX_PATH.
 		info := r.info
-		fullPath, err := makeLongAbsPath(filepath.Join(info.HomeDir, r.layerId))
+		fullPath, err = makeLongAbsPath(filepath.Join(info.HomeDir, r.layerID))
 		if err == nil {
 			info.HomeDir = ""
 			err = ImportLayer(info, fullPath, r.path, r.parentLayerPaths)
@@ -133,7 +148,14 @@ func (r *legacyLayerWriterWrapper) Close() error {
 }
 
 // NewLayerWriter returns a new layer writer for creating a layer on disk.
-func NewLayerWriter(info DriverInfo, layerId string, parentLayerPaths []string) (LayerWriter, error) {
+func NewLayerWriter(info DriverInfo, layerID string, parentLayerPaths []string) (LayerWriter, error) {
+	if len(parentLayerPaths) == 0 {
+		// This is a base layer. It gets imported differently.
+		return &baseLayerWriter{
+			root: filepath.Join(info.HomeDir, layerID),
+		}, nil
+	}
+
 	if procImportLayerBegin.Find() != nil {
 		// The new layer reader is not available on this Windows build. Fall back to the
 		// legacy export code path.
@@ -142,9 +164,9 @@ func NewLayerWriter(info DriverInfo, layerId string, parentLayerPaths []string)
 			return nil, err
 		}
 		return &legacyLayerWriterWrapper{
-			LegacyLayerWriter: NewLegacyLayerWriter(path),
+			legacyLayerWriter: newLegacyLayerWriter(path),
 			info:              info,
-			layerId:           layerId,
+			layerID:           layerID,
 			path:              path,
 			parentLayerPaths:  parentLayerPaths,
 		}, nil
@@ -160,7 +182,7 @@ func NewLayerWriter(info DriverInfo, layerId string, parentLayerPaths []string)
 	}
 
 	w := &FilterLayerWriter{}
-	err = importLayerBegin(&infop, layerId, layers, &w.context)
+	err = importLayerBegin(&infop, layerID, layers, &w.context)
 	if err != nil {
 		return nil, makeError(err, "ImportLayerStart", "")
 	}

vendor/src/github.com/Microsoft/hcsshim/legacy.go

@@ -16,17 +16,7 @@ import (
 var errorIterationCanceled = errors.New("")
 
 func openFileOrDir(path string, mode uint32, createDisposition uint32) (file *os.File, err error) {
-	winPath, err := syscall.UTF16FromString(path)
-	if err != nil {
-		return
-	}
-	h, err := syscall.CreateFile(&winPath[0], mode, syscall.FILE_SHARE_READ, nil, createDisposition, syscall.FILE_FLAG_BACKUP_SEMANTICS, 0)
-	if err != nil {
-		err = &os.PathError{"open", path, err}
-		return
-	}
-	file = os.NewFile(uintptr(h), path)
-	return
+	return winio.OpenForBackup(path, mode, syscall.FILE_SHARE_READ, createDisposition)
 }
 
 func makeLongAbsPath(path string) (string, error) {
@@ -52,7 +42,7 @@ type fileEntry struct {
 	err  error
 }
 
-type LegacyLayerReader struct {
+type legacyLayerReader struct {
 	root         string
 	result       chan *fileEntry
 	proceed      chan bool
@@ -61,10 +51,10 @@ type LegacyLayerReader struct {
 	isTP4Format  bool
 }
 
-// NewLegacyLayerReader returns a new LayerReader that can read the Windows
+// newLegacyLayerReader returns a new LayerReader that can read the Windows
 // TP4 transport format from disk.
-func NewLegacyLayerReader(root string) *LegacyLayerReader {
-	r := &LegacyLayerReader{
+func newLegacyLayerReader(root string) *legacyLayerReader {
+	r := &legacyLayerReader{
 		root:        root,
 		result:      make(chan *fileEntry),
 		proceed:     make(chan bool),
@@ -98,7 +88,7 @@ func readTombstones(path string) (map[string]([]string), error) {
 	return ts, nil
 }
 
-func (r *LegacyLayerReader) walkUntilCancelled() error {
+func (r *legacyLayerReader) walkUntilCancelled() error {
 	root, err := makeLongAbsPath(r.root)
 	if err != nil {
 		return err
@@ -148,7 +138,7 @@ func (r *LegacyLayerReader) walkUntilCancelled() error {
 	return err
 }
 
-func (r *LegacyLayerReader) walk() {
+func (r *legacyLayerReader) walk() {
 	defer close(r.result)
 	if !<-r.proceed {
 		return
@@ -165,7 +155,7 @@ func (r *LegacyLayerReader) walk() {
 	}
 }
 
-func (r *LegacyLayerReader) reset() {
+func (r *legacyLayerReader) reset() {
 	if r.backupReader != nil {
 		r.backupReader.Close()
 		r.backupReader = nil
@@ -192,7 +182,7 @@ func findBackupStreamSize(r io.Reader) (int64, error) {
 	}
 }
 
-func (r *LegacyLayerReader) Next() (path string, size int64, fileInfo *winio.FileBasicInfo, err error) {
+func (r *legacyLayerReader) Next() (path string, size int64, fileInfo *winio.FileBasicInfo, err error) {
 	r.reset()
 	r.proceed <- true
 	fe := <-r.result
@@ -275,7 +265,7 @@ func (r *LegacyLayerReader) Next() (path string, size int64, fileInfo *winio.Fil
 		if !fe.fi.IsDir() {
 			size, err = findBackupStreamSize(f)
 			if err != nil {
-				err = &os.PathError{"findBackupStreamSize", fe.path, err}
+				err = &os.PathError{Op: "findBackupStreamSize", Path: fe.path, Err: err}
 				return
 			}
 		}
@@ -292,7 +282,7 @@ func (r *LegacyLayerReader) Next() (path string, size int64, fileInfo *winio.Fil
 	return
 }
 
-func (r *LegacyLayerReader) Read(b []byte) (int, error) {
+func (r *legacyLayerReader) Read(b []byte) (int, error) {
 	if r.backupReader == nil {
 		if r.currentFile == nil {
 			return 0, io.EOF
@@ -302,14 +292,14 @@ func (r *LegacyLayerReader) Read(b []byte) (int, error) {
 	return r.backupReader.Read(b)
 }
 
-func (r *LegacyLayerReader) Close() error {
+func (r *legacyLayerReader) Close() error {
 	r.proceed <- false
 	<-r.result
 	r.reset()
 	return nil
 }
 
-type LegacyLayerWriter struct {
+type legacyLayerWriter struct {
 	root         string
 	currentFile  *os.File
 	backupWriter *winio.BackupFileWriter
@@ -318,16 +308,16 @@ type LegacyLayerWriter struct {
 	pathFixed    bool
 }
 
-// NewLegacyLayerWriter returns a LayerWriter that can write the TP4 transport format
+// newLegacyLayerWriter returns a LayerWriter that can write the TP4 transport format
 // to disk.
-func NewLegacyLayerWriter(root string) *LegacyLayerWriter {
-	return &LegacyLayerWriter{
+func newLegacyLayerWriter(root string) *legacyLayerWriter {
+	return &legacyLayerWriter{
 		root:        root,
 		isTP4Format: IsTP4(),
 	}
 }
 
-func (w *LegacyLayerWriter) init() error {
+func (w *legacyLayerWriter) init() error {
 	if !w.pathFixed {
 		path, err := makeLongAbsPath(w.root)
 		if err != nil {
@@ -339,7 +329,7 @@ func (w *LegacyLayerWriter) init() error {
 	return nil
 }
 
-func (w *LegacyLayerWriter) reset() {
+func (w *legacyLayerWriter) reset() {
 	if w.backupWriter != nil {
 		w.backupWriter.Close()
 		w.backupWriter = nil
@@ -350,7 +340,7 @@ func (w *LegacyLayerWriter) reset() {
 	}
 }
 
-func (w *LegacyLayerWriter) Add(name string, fileInfo *winio.FileBasicInfo) error {
+func (w *legacyLayerWriter) Add(name string, fileInfo *winio.FileBasicInfo) error {
 	w.reset()
 	err := w.init()
 	if err != nil {
@@ -402,12 +392,16 @@ func (w *LegacyLayerWriter) Add(name string, fileInfo *winio.FileBasicInfo) erro
 	return nil
 }
 
-func (w *LegacyLayerWriter) Remove(name string) error {
+func (w *legacyLayerWriter) AddLink(name string, target string) error {
+	return errors.New("hard links not supported with legacy writer")
+}
+
+func (w *legacyLayerWriter) Remove(name string) error {
 	w.tombstones = append(w.tombstones, name)
 	return nil
 }
 
-func (w *LegacyLayerWriter) Write(b []byte) (int, error) {
+func (w *legacyLayerWriter) Write(b []byte) (int, error) {
 	if w.backupWriter == nil {
 		if w.currentFile == nil {
 			return 0, errors.New("closed")
@@ -417,7 +411,7 @@ func (w *LegacyLayerWriter) Write(b []byte) (int, error) {
 	return w.backupWriter.Write(b)
 }
 
-func (w *LegacyLayerWriter) Close() error {
+func (w *legacyLayerWriter) Close() error {
 	w.reset()
 	err := w.init()
 	if err != nil {