Merge pull request #27038 from jstarks/non_base_utilityvm

Windows: support Windows servicing layers

hack/vendor.sh

@@ -48,7 +48,7 @@ esac
 
 # the following lines are in sorted order, FYI
 clone git github.com/Azure/go-ansiterm 388960b655244e76e24c75f48631564eaefade62
-clone git github.com/Microsoft/hcsshim v0.4.3
+clone git github.com/Microsoft/hcsshim v0.4.5
 clone git github.com/Microsoft/go-winio v0.3.4
 clone git github.com/Sirupsen/logrus v0.10.0 # logrus is a common dependency among multiple deps
 clone git github.com/docker/libtrust 9cbd2a1374f46905c68a4eb3694a130610adc62a

libcontainerd/client_windows.go

@@ -162,12 +162,25 @@ func (clnt *client) Create(containerID string, checkpoint string, checkpointDir
 	}
 
 	if configuration.HvPartition {
-		// Make sure the Utility VM image is present in the base layer directory.
+		// Find the upper-most utility VM image, since the utility VM does not
+		// use layering in RS1.
 		// TODO @swernli/jhowardmsft at some point post RS1 this may be re-locatable.
-		configuration.HvRuntime = &hcsshim.HvRuntime{ImagePath: filepath.Join(layerOpt.LayerPaths[len(layerOpt.LayerPaths)-1], "UtilityVM")}
-		if _, err := os.Stat(configuration.HvRuntime.ImagePath); os.IsNotExist(err) {
-			return fmt.Errorf("utility VM image '%s' could not be found", configuration.HvRuntime.ImagePath)
+		var uvmImagePath string
+		for _, path := range layerOpt.LayerPaths {
+			fullPath := filepath.Join(path, "UtilityVM")
+			_, err := os.Stat(fullPath)
+			if err == nil {
+				uvmImagePath = fullPath
+				break
+			}
+			if !os.IsNotExist(err) {
+				return err
+			}
 		}
+		if uvmImagePath == "" {
+			return errors.New("utility VM image could not be found")
+		}
+		configuration.HvRuntime = &hcsshim.HvRuntime{ImagePath: uvmImagePath}
 	} else {
 		configuration.VolumePath = spec.Root.Path
 		configuration.LayerFolderPath = layerOpt.LayerFolderPath

vendor/src/github.com/Microsoft/hcsshim/baselayer.go

@@ -23,6 +23,26 @@ type dirInfo struct {
 	fileInfo winio.FileBasicInfo
 }
 
+// reapplyDirectoryTimes reapplies directory modification, creation, etc. times
+// after processing of the directory tree has completed. The times are expected
+// to be ordered such that parent directories come before child directories.
+func reapplyDirectoryTimes(dis []dirInfo) error {
+	for i := range dis {
+		di := &dis[len(dis)-i-1] // reverse order: process child directories first
+		f, err := winio.OpenForBackup(di.path, syscall.GENERIC_READ|syscall.GENERIC_WRITE, syscall.FILE_SHARE_READ, syscall.OPEN_EXISTING)
+		if err != nil {
+			return err
+		}
+
+		err = winio.SetFileBasicInfo(f, &di.fileInfo)
+		f.Close()
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 func (w *baseLayerWriter) closeCurrentFile() error {
 	if w.f != nil {
 		err := w.bw.Close()
@@ -142,18 +162,9 @@ func (w *baseLayerWriter) Close() error {
 	if w.err == nil {
 		// Restore the file times of all the directories, since they may have
 		// been modified by creating child directories.
-		for i := range w.dirInfo {
-			di := &w.dirInfo[len(w.dirInfo)-i-1]
-			f, err := winio.OpenForBackup(di.path, uint32(syscall.GENERIC_READ|syscall.GENERIC_WRITE), syscall.FILE_SHARE_READ, syscall.OPEN_EXISTING)
-			if err != nil {
-				return makeError(err, "Failed to OpenForBackup", di.path)
-			}
-
-			err = winio.SetFileBasicInfo(f, &di.fileInfo)
-			f.Close()
-			if err != nil {
-				return makeError(err, "Failed to SetFileBasicInfo", di.path)
-			}
+		err = reapplyDirectoryTimes(w.dirInfo)
+		if err != nil {
+			return err
 		}
 
 		err = ProcessBaseLayer(w.root)

vendor/src/github.com/Microsoft/hcsshim/importlayer.go

@@ -130,21 +130,42 @@ type legacyLayerWriterWrapper struct {
 }
 
 func (r *legacyLayerWriterWrapper) Close() error {
+	defer os.RemoveAll(r.root)
 	err := r.legacyLayerWriter.Close()
-	if err == nil {
-		var fullPath string
-		// Use the original path here because ImportLayer does not support long paths for the source in TP5.
-		// But do use a long path for the destination to work around another bug with directories
-		// with MAX_PATH - 12 < length < MAX_PATH.
-		info := r.info
-		fullPath, err = makeLongAbsPath(filepath.Join(info.HomeDir, r.layerID))
-		if err == nil {
-			info.HomeDir = ""
-			err = ImportLayer(info, fullPath, r.path, r.parentLayerPaths)
+	if err != nil {
+		return err
+	}
+
+	// Use the original path here because ImportLayer does not support long paths for the source in TP5.
+	// But do use a long path for the destination to work around another bug with directories
+	// with MAX_PATH - 12 < length < MAX_PATH.
+	info := r.info
+	fullPath, err := makeLongAbsPath(filepath.Join(info.HomeDir, r.layerID))
+	if err != nil {
+		return err
+	}
+
+	info.HomeDir = ""
+	if err = ImportLayer(info, fullPath, r.path, r.parentLayerPaths); err != nil {
+		return err
+	}
+	// Add any hard links that were collected.
+	for _, lnk := range r.PendingLinks {
+		if err = os.Remove(lnk.Path); err != nil && !os.IsNotExist(err) {
+			return err
+		}
+		if err = os.Link(lnk.Target, lnk.Path); err != nil {
+			return err
 		}
 	}
-	os.RemoveAll(r.root)
-	return err
+	// Prepare the utility VM for use if one is present in the layer.
+	if r.HasUtilityVM {
+		err = ProcessUtilityVMImage(filepath.Join(fullPath, "UtilityVM"))
+		if err != nil {
+			return err
+		}
+	}
+	return nil
 }
 
 // NewLayerWriter returns a new layer writer for creating a layer on disk.
@@ -166,7 +187,7 @@ func NewLayerWriter(info DriverInfo, layerID string, parentLayerPaths []string)
 			return nil, err
 		}
 		return &legacyLayerWriterWrapper{
-			legacyLayerWriter: newLegacyLayerWriter(path),
+			legacyLayerWriter: newLegacyLayerWriter(path, parentLayerPaths, filepath.Join(info.HomeDir, layerID)),
 			info:              info,
 			layerID:           layerID,
 			path:              path,

vendor/src/github.com/Microsoft/hcsshim/legacy.go

@@ -16,6 +16,13 @@ import (
 
 var errorIterationCanceled = errors.New("")
 
+var mutatedUtilityVMFiles = map[string]bool{
+	`EFI\Microsoft\Boot\BCD`:      true,
+	`EFI\Microsoft\Boot\BCD.LOG`:  true,
+	`EFI\Microsoft\Boot\BCD.LOG1`: true,
+	`EFI\Microsoft\Boot\BCD.LOG2`: true,
+}
+
 func openFileOrDir(path string, mode uint32, createDisposition uint32) (file *os.File, err error) {
 	return winio.OpenForBackup(path, mode, syscall.FILE_SHARE_READ, createDisposition)
 }
@@ -49,17 +56,15 @@ type legacyLayerReader struct {
 	proceed      chan bool
 	currentFile  *os.File
 	backupReader *winio.BackupFileReader
-	isTP4Format  bool
 }
 
 // newLegacyLayerReader returns a new LayerReader that can read the Windows
-// TP4 transport format from disk.
+// container layer transport format from disk.
 func newLegacyLayerReader(root string) *legacyLayerReader {
 	r := &legacyLayerReader{
-		root:        root,
-		result:      make(chan *fileEntry),
-		proceed:     make(chan bool),
-		isTP4Format: IsTP4(),
+		root:    root,
+		result:  make(chan *fileEntry),
+		proceed: make(chan bool),
 	}
 	go r.walk()
 	return r
@@ -251,17 +256,14 @@ func (r *legacyLayerReader) Next() (path string, size int64, fileInfo *winio.Fil
 		fileInfo.LastAccessTime = fileInfo.LastWriteTime
 
 	} else {
-		beginning := int64(0)
-		if !r.isTP4Format {
-			// In TP5, the file attributes were added before the backup stream
-			var attr uint32
-			err = binary.Read(f, binary.LittleEndian, &attr)
-			if err != nil {
-				return
-			}
-			fileInfo.FileAttributes = uintptr(attr)
-			beginning = 4
+		// The file attributes are written before the backup stream.
+		var attr uint32
+		err = binary.Read(f, binary.LittleEndian, &attr)
+		if err != nil {
+			return
 		}
+		fileInfo.FileAttributes = uintptr(attr)
+		beginning := int64(4)
 
 		// Find the accurate file size.
 		if !fe.fi.IsDir() {
@@ -301,21 +303,32 @@ func (r *legacyLayerReader) Close() error {
 	return nil
 }
 
+type pendingLink struct {
+	Path, Target string
+}
+
 type legacyLayerWriter struct {
 	root         string
+	parentRoots  []string
+	destRoot     string
 	currentFile  *os.File
 	backupWriter *winio.BackupFileWriter
 	tombstones   []string
-	isTP4Format  bool
 	pathFixed    bool
+	HasUtilityVM bool
+	uvmDi        []dirInfo
+	addedFiles   map[string]bool
+	PendingLinks []pendingLink
 }
 
-// newLegacyLayerWriter returns a LayerWriter that can write the TP4 transport format
-// to disk.
-func newLegacyLayerWriter(root string) *legacyLayerWriter {
+// newLegacyLayerWriter returns a LayerWriter that can write the contaler layer
+// transport format to disk.
+func newLegacyLayerWriter(root string, parentRoots []string, destRoot string) *legacyLayerWriter {
 	return &legacyLayerWriter{
 		root:        root,
-		isTP4Format: IsTP4(),
+		parentRoots: parentRoots,
+		destRoot:    destRoot,
+		addedFiles:  make(map[string]bool),
 	}
 }
 
@@ -325,12 +338,42 @@ func (w *legacyLayerWriter) init() error {
 		if err != nil {
 			return err
 		}
+		for i, p := range w.parentRoots {
+			w.parentRoots[i], err = makeLongAbsPath(p)
+			if err != nil {
+				return err
+			}
+		}
+		destPath, err := makeLongAbsPath(w.destRoot)
+		if err != nil {
+			return err
+		}
 		w.root = path
+		w.destRoot = destPath
 		w.pathFixed = true
 	}
 	return nil
 }
 
+func (w *legacyLayerWriter) initUtilityVM() error {
+	if !w.HasUtilityVM {
+		err := os.Mkdir(filepath.Join(w.destRoot, `UtilityVM`), 0)
+		if err != nil {
+			return err
+		}
+		// Server 2016 does not support multiple layers for the utility VM, so
+		// clone the utility VM from the parent layer into this layer. Use hard
+		// links to avoid unnecessary copying, since most of the files are
+		// immutable.
+		err = cloneTree(filepath.Join(w.parentRoots[0], `UtilityVM\Files`), filepath.Join(w.destRoot, `UtilityVM\Files`), mutatedUtilityVMFiles)
+		if err != nil {
+			return fmt.Errorf("cloning the parent utility VM image failed: %s", err)
+		}
+		w.HasUtilityVM = true
+	}
+	return nil
+}
+
 func (w *legacyLayerWriter) reset() {
 	if w.backupWriter != nil {
 		w.backupWriter.Close()
@@ -342,15 +385,180 @@ func (w *legacyLayerWriter) reset() {
 	}
 }
 
+// copyFileWithMetadata copies a file using the backup/restore APIs in order to preserve metadata
+func copyFileWithMetadata(srcPath, destPath string, isDir bool) (fileInfo *winio.FileBasicInfo, err error) {
+	createDisposition := uint32(syscall.CREATE_NEW)
+	if isDir {
+		err = os.Mkdir(destPath, 0)
+		if err != nil {
+			return nil, err
+		}
+		createDisposition = syscall.OPEN_EXISTING
+	}
+
+	src, err := openFileOrDir(srcPath, syscall.GENERIC_READ|winio.ACCESS_SYSTEM_SECURITY, syscall.OPEN_EXISTING)
+	if err != nil {
+		return nil, err
+	}
+	defer src.Close()
+	srcr := winio.NewBackupFileReader(src, true)
+	defer srcr.Close()
+
+	fileInfo, err = winio.GetFileBasicInfo(src)
+	if err != nil {
+		return nil, err
+	}
+
+	dest, err := openFileOrDir(destPath, syscall.GENERIC_READ|syscall.GENERIC_WRITE|winio.WRITE_DAC|winio.WRITE_OWNER|winio.ACCESS_SYSTEM_SECURITY, createDisposition)
+	if err != nil {
+		return nil, err
+	}
+	defer dest.Close()
+
+	err = winio.SetFileBasicInfo(dest, fileInfo)
+	if err != nil {
+		return nil, err
+	}
+
+	destw := winio.NewBackupFileWriter(dest, true)
+	defer func() {
+		cerr := destw.Close()
+		if err == nil {
+			err = cerr
+		}
+	}()
+
+	_, err = io.Copy(destw, srcr)
+	if err != nil {
+		return nil, err
+	}
+
+	return fileInfo, nil
+}
+
+// cloneTree clones a directory tree using hard links. It skips hard links for
+// the file names in the provided map and just copies those files.
+func cloneTree(srcPath, destPath string, mutatedFiles map[string]bool) error {
+	var di []dirInfo
+	err := filepath.Walk(srcPath, func(srcFilePath string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+
+		relPath, err := filepath.Rel(srcPath, srcFilePath)
+		if err != nil {
+			return err
+		}
+		destFilePath := filepath.Join(destPath, relPath)
+
+		// Directories, reparse points, and files that will be mutated during
+		// utility VM import must be copied. All other files can be hard linked.
+		isReparsePoint := info.Sys().(*syscall.Win32FileAttributeData).FileAttributes&syscall.FILE_ATTRIBUTE_REPARSE_POINT != 0
+		if info.IsDir() || isReparsePoint || mutatedFiles[relPath] {
+			fi, err := copyFileWithMetadata(srcFilePath, destFilePath, info.IsDir())
+			if err != nil {
+				return err
+			}
+			if info.IsDir() && !isReparsePoint {
+				di = append(di, dirInfo{path: destFilePath, fileInfo: *fi})
+			}
+		} else {
+			err = os.Link(srcFilePath, destFilePath)
+			if err != nil {
+				return err
+			}
+		}
+
+		// Don't recurse on reparse points.
+		if info.IsDir() && isReparsePoint {
+			return filepath.SkipDir
+		}
+
+		return nil
+	})
+	if err != nil {
+		return err
+	}
+
+	return reapplyDirectoryTimes(di)
+}
+
 func (w *legacyLayerWriter) Add(name string, fileInfo *winio.FileBasicInfo) error {
 	w.reset()
 	err := w.init()
 	if err != nil {
 		return err
 	}
-	path := filepath.Join(w.root, name)
 
-	createDisposition := uint32(syscall.CREATE_NEW)
+	if name == `UtilityVM` {
+		return w.initUtilityVM()
+	}
+
+	if strings.HasPrefix(name, `UtilityVM\`) {
+		if !w.HasUtilityVM {
+			return errors.New("missing UtilityVM directory")
+		}
+		if !strings.HasPrefix(name, `UtilityVM\Files\`) && name != `UtilityVM\Files` {
+			return errors.New("invalid UtilityVM layer")
+		}
+		path := filepath.Join(w.destRoot, name)
+		createDisposition := uint32(syscall.OPEN_EXISTING)
+		if (fileInfo.FileAttributes & syscall.FILE_ATTRIBUTE_DIRECTORY) != 0 {
+			st, err := os.Lstat(path)
+			if err != nil && !os.IsNotExist(err) {
+				return err
+			}
+			if st != nil {
+				// Delete the existing file/directory if it is not the same type as this directory.
+				existingAttr := st.Sys().(*syscall.Win32FileAttributeData).FileAttributes
+				if (uint32(fileInfo.FileAttributes)^existingAttr)&(syscall.FILE_ATTRIBUTE_DIRECTORY|syscall.FILE_ATTRIBUTE_REPARSE_POINT) != 0 {
+					if err = os.RemoveAll(path); err != nil {
+						return err
+					}
+					st = nil
+				}
+			}
+			if st == nil {
+				if err = os.Mkdir(path, 0); err != nil {
+					return err
+				}
+			}
+			if fileInfo.FileAttributes&syscall.FILE_ATTRIBUTE_REPARSE_POINT == 0 {
+				w.uvmDi = append(w.uvmDi, dirInfo{path: path, fileInfo: *fileInfo})
+			}
+		} else {
+			// Overwrite any existing hard link.
+			err = os.Remove(path)
+			if err != nil && !os.IsNotExist(err) {
+				return err
+			}
+			createDisposition = syscall.CREATE_NEW
+		}
+
+		f, err := openFileOrDir(path, syscall.GENERIC_READ|syscall.GENERIC_WRITE|winio.WRITE_DAC|winio.WRITE_OWNER|winio.ACCESS_SYSTEM_SECURITY, createDisposition)
+		if err != nil {
+			return err
+		}
+		defer func() {
+			if f != nil {
+				f.Close()
+				os.Remove(path)
+			}
+		}()
+
+		err = winio.SetFileBasicInfo(f, fileInfo)
+		if err != nil {
+			return err
+		}
+
+		w.backupWriter = winio.NewBackupFileWriter(f, true)
+		w.currentFile = f
+		w.addedFiles[name] = true
+		f = nil
+		return nil
+	}
+
+	path := filepath.Join(w.root, name)
 	if (fileInfo.FileAttributes & syscall.FILE_ATTRIBUTE_DIRECTORY) != 0 {
 		err := os.Mkdir(path, 0)
 		if err != nil {
@@ -359,7 +567,7 @@ func (w *legacyLayerWriter) Add(name string, fileInfo *winio.FileBasicInfo) erro
 		path += ".$wcidirs$"
 	}
 
-	f, err := openFileOrDir(path, syscall.GENERIC_READ|syscall.GENERIC_WRITE, createDisposition)
+	f, err := openFileOrDir(path, syscall.GENERIC_READ|syscall.GENERIC_WRITE, syscall.CREATE_NEW)
 	if err != nil {
 		return err
 	}
@@ -380,29 +588,97 @@ func (w *legacyLayerWriter) Add(name string, fileInfo *winio.FileBasicInfo) erro
 	if strings.HasPrefix(name, `Hives\`) {
 		w.backupWriter = winio.NewBackupFileWriter(f, false)
 	} else {
-		if !w.isTP4Format {
-			// In TP5, the file attributes were added to the header
-			err = binary.Write(f, binary.LittleEndian, uint32(fileInfo.FileAttributes))
-			if err != nil {
-				return err
-			}
+		// The file attributes are written before the stream.
+		err = binary.Write(f, binary.LittleEndian, uint32(fileInfo.FileAttributes))
+		if err != nil {
+			return err
 		}
 	}
 
 	w.currentFile = f
+	w.addedFiles[name] = true
 	f = nil
 	return nil
 }
 
 func (w *legacyLayerWriter) AddLink(name string, target string) error {
-	return errors.New("hard links not supported with legacy writer")
+	w.reset()
+	err := w.init()
+	if err != nil {
+		return err
+	}
+
+	var requiredPrefix string
+	var roots []string
+	if prefix := `Files\`; strings.HasPrefix(name, prefix) {
+		requiredPrefix = prefix
+		// Look for cross-layer hard link targets in the parent layers, since
+		// nothing is in the destination path yet.
+		roots = w.parentRoots
+	} else if prefix := `UtilityVM\Files\`; strings.HasPrefix(name, prefix) {
+		requiredPrefix = prefix
+		// Since the utility VM is fully cloned into the destination path
+		// already, look for cross-layer hard link targets directly in the
+		// destination path.
+		roots = []string{w.destRoot}
+	}
+
+	if requiredPrefix == "" || !strings.HasPrefix(target, requiredPrefix) {
+		return errors.New("invalid hard link in layer")
+	}
+
+	// Find to try the target of the link in a previously added file. If that
+	// fails, search in parent layers.
+	var selectedRoot string
+	if _, ok := w.addedFiles[target]; ok {
+		selectedRoot = w.destRoot
+	} else {
+		for _, r := range roots {
+			if _, err = os.Lstat(filepath.Join(r, target)); err != nil {
+				if !os.IsNotExist(err) {
+					return err
+				}
+			} else {
+				selectedRoot = r
+				break
+			}
+		}
+		if selectedRoot == "" {
+			return fmt.Errorf("failed to find link target for '%s' -> '%s'", name, target)
+		}
+	}
+	// The link can't be written until after the ImportLayer call.
+	w.PendingLinks = append(w.PendingLinks, pendingLink{
+		Path:   filepath.Join(w.destRoot, name),
+		Target: filepath.Join(selectedRoot, target),
+	})
+	w.addedFiles[name] = true
+	return nil
 }
 
 func (w *legacyLayerWriter) Remove(name string) error {
-	if !strings.HasPrefix(name, `Files\`) {
+	if strings.HasPrefix(name, `Files\`) {
+		w.tombstones = append(w.tombstones, name[len(`Files\`):])
+	} else if strings.HasPrefix(name, `UtilityVM\Files\`) {
+		err := w.initUtilityVM()
+		if err != nil {
+			return err
+		}
+		// Make sure the path exists; os.RemoveAll will not fail if the file is
+		// already gone, and this needs to be a fatal error for diagnostics
+		// purposes.
+		path := filepath.Join(w.destRoot, name)
+		if _, err := os.Lstat(path); err != nil {
+			return err
+		}
+		err = os.RemoveAll(path)
+		if err != nil {
+			return err
+		}
+	} else {
 		return fmt.Errorf("invalid tombstone %s", name)
 	}
-	w.tombstones = append(w.tombstones, name[len(`Files\`):])
+
 	return nil
 }
 
@@ -437,5 +713,11 @@ func (w *legacyLayerWriter) Close() error {
 			return err
 		}
 	}
+	if w.HasUtilityVM {
+		err = reapplyDirectoryTimes(w.uvmDi)
+		if err != nil {
+			return err
+		}
+	}
 	return nil
 }