Merge pull request #26746 from Microsoft/jjh/hvruntime

Windows: OCI HVRuntime and LayerPaths to options

daemon/oci_windows.go

@@ -1,16 +1,10 @@
 package daemon
 
 import (
-	"errors"
-	"fmt"
-	"os"
-	"path/filepath"
 	"syscall"
 
 	containertypes "github.com/docker/docker/api/types/container"
 	"github.com/docker/docker/container"
-	"github.com/docker/docker/image"
-	"github.com/docker/docker/layer"
 	"github.com/docker/docker/libcontainerd"
 	"github.com/docker/docker/libcontainerd/windowsoci"
 	"github.com/docker/docker/oci"
@@ -30,11 +24,6 @@ func (daemon *Daemon) createSpec(c *container.Container) (*libcontainerd.Spec, e
 		return nil, err
 	}
 
-	img, err := daemon.imageStore.Get(c.ImageID)
-	if err != nil {
-		return nil, fmt.Errorf("Failed to graph.Get on ImageID %s - %s", c.ImageID, err)
-	}
-
 	// In base spec
 	s.Hostname = c.FullHostname()
 
@@ -80,60 +69,6 @@ func (daemon *Daemon) createSpec(c *container.Container) (*libcontainerd.Spec, e
 	s.Root.Path = c.BaseFS
 	s.Root.Readonly = c.HostConfig.ReadonlyRootfs
 
-	// s.Windows.LayerFolder.
-	m, err := c.RWLayer.Metadata()
-	if err != nil {
-		return nil, fmt.Errorf("Failed to get layer metadata - %s", err)
-	}
-	s.Windows.LayerFolder = m["dir"]
-
-	// s.Windows.LayerPaths
-	var layerPaths []string
-	if img.RootFS != nil && img.RootFS.Type == image.TypeLayers {
-		// Get the layer path for each layer.
-		max := len(img.RootFS.DiffIDs)
-		for i := 1; i <= max; i++ {
-			img.RootFS.DiffIDs = img.RootFS.DiffIDs[:i]
-			path, err := layer.GetLayerPath(daemon.layerStore, img.RootFS.ChainID())
-			if err != nil {
-				return nil, fmt.Errorf("Failed to get layer path from graphdriver %s for ImageID %s - %s", daemon.layerStore, img.RootFS.ChainID(), err)
-			}
-			// Reverse order, expecting parent most first
-			layerPaths = append([]string{path}, layerPaths...)
-		}
-	}
-	s.Windows.LayerPaths = layerPaths
-
-	// Are we going to run as a Hyper-V container?
-	hv := false
-	if c.HostConfig.Isolation.IsDefault() {
-		// Container is set to use the default, so take the default from the daemon configuration
-		hv = daemon.defaultIsolation.IsHyperV()
-	} else {
-		// Container is requesting an isolation mode. Honour it.
-		hv = c.HostConfig.Isolation.IsHyperV()
-	}
-	if hv {
-		hvr := &windowsoci.WindowsHvRuntime{}
-		if img.RootFS != nil && img.RootFS.Type == image.TypeLayers {
-			// For TP5, the utility VM is part of the base layer.
-			// TODO-jstarks: Add support for separate utility VM images
-			// once it is decided how they can be stored.
-			uvmpath := filepath.Join(layerPaths[len(layerPaths)-1], "UtilityVM")
-			_, err = os.Stat(uvmpath)
-			if err != nil {
-				if os.IsNotExist(err) {
-					err = errors.New("container image does not contain a utility VM")
-				}
-				return nil, err
-			}
-
-			hvr.ImagePath = uvmpath
-		}
-
-		s.Windows.HvRuntime = hvr
-	}
-
 	// In s.Windows.Networking
 	// Connect all the libnetwork allocated networks to the container
 	var epList []string

daemon/start_windows.go

@@ -1,12 +1,60 @@
 package daemon
 
 import (
+	"fmt"
+	"path/filepath"
+
 	"github.com/docker/docker/container"
+	"github.com/docker/docker/layer"
 	"github.com/docker/docker/libcontainerd"
 )
 
 func (daemon *Daemon) getLibcontainerdCreateOptions(container *container.Container) (*[]libcontainerd.CreateOption, error) {
 	createOptions := []libcontainerd.CreateOption{}
+
+	// Are we going to run as a Hyper-V container?
+	hvOpts := &libcontainerd.HyperVIsolationOption{}
+	if container.HostConfig.Isolation.IsDefault() {
+		// Container is set to use the default, so take the default from the daemon configuration
+		hvOpts.IsHyperV = daemon.defaultIsolation.IsHyperV()
+	} else {
+		// Container is requesting an isolation mode. Honour it.
+		hvOpts.IsHyperV = container.HostConfig.Isolation.IsHyperV()
+	}
+
+	// Generate the layer folder of the layer options
+	layerOpts := &libcontainerd.LayerOption{}
+	m, err := container.RWLayer.Metadata()
+	if err != nil {
+		return nil, fmt.Errorf("failed to get layer metadata - %s", err)
+	}
+	if hvOpts.IsHyperV {
+		hvOpts.SandboxPath = filepath.Dir(m["dir"])
+	} else {
+		layerOpts.LayerFolderPath = m["dir"]
+	}
+
+	// Generate the layer paths of the layer options
+	img, err := daemon.imageStore.Get(container.ImageID)
+	if err != nil {
+		return nil, fmt.Errorf("failed to graph.Get on ImageID %s - %s", container.ImageID, err)
+	}
+	// Get the layer path for each layer.
+	max := len(img.RootFS.DiffIDs)
+	for i := 1; i <= max; i++ {
+		img.RootFS.DiffIDs = img.RootFS.DiffIDs[:i]
+		layerPath, err := layer.GetLayerPath(daemon.layerStore, img.RootFS.ChainID())
+		if err != nil {
+			return nil, fmt.Errorf("failed to get layer path from graphdriver %s for ImageID %s - %s", daemon.layerStore, img.RootFS.ChainID(), err)
+		}
+		// Reverse order, expecting parent most first
+		layerOpts.LayerPaths = append([]string{layerPath}, layerOpts.LayerPaths...)
+	}
+
+	// Now build the full set of options
 	createOptions = append(createOptions, &libcontainerd.FlushOption{IgnoreFlushesDuringBoot: !container.HasBeenStartedBefore})
+	createOptions = append(createOptions, hvOpts)
+	createOptions = append(createOptions, layerOpts)
+
 	return &createOptions, nil
 }

libcontainerd/client_windows.go

@@ -4,6 +4,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"os"
 	"path/filepath"
 	"strings"
 	"syscall"
@@ -36,20 +37,73 @@ const (
 const defaultOwner = "docker"
 
 // Create is the entrypoint to create a container from a spec, and if successfully
-// created, start it too.
+// created, start it too. Table below shows the fields required for HCS JSON calling parameters,
+// where if not populated, is omitted.
+// +-----------------+--------------------------------------------+--------------------------------------------+
+// |                 | Isolation=Process                          | Isolation=Hyper-V                          |
+// +-----------------+--------------------------------------------+--------------------------------------------+
+// | VolumePath      | \\?\\Volume{GUIDa}                         |                                            |
+// | LayerFolderPath | %root%\windowsfilter\containerID           |                                            |
+// | Layers[]        | ID=GUIDb;Path=%root%\windowsfilter\layerID | ID=GUIDb;Path=%root%\windowsfilter\layerID |
+// | SandboxPath     |                                            | %root%\windowsfilter                       |
+// | HvRuntime       |                                            | ImagePath=%root%\BaseLayerID\UtilityVM     |
+// +-----------------+--------------------------------------------+--------------------------------------------+
+//
+// Isolation=Process example:
+//
+// {
+//	"SystemType": "Container",
+//	"Name": "5e0055c814a6005b8e57ac59f9a522066e0af12b48b3c26a9416e23907698776",
+//	"Owner": "docker",
+//	"IsDummy": false,
+//	"VolumePath": "\\\\\\\\?\\\\Volume{66d1ef4c-7a00-11e6-8948-00155ddbef9d}",
+//	"IgnoreFlushesDuringBoot": true,
+//	"LayerFolderPath": "C:\\\\control\\\\windowsfilter\\\\5e0055c814a6005b8e57ac59f9a522066e0af12b48b3c26a9416e23907698776",
+//	"Layers": [{
+//		"ID": "18955d65-d45a-557b-bf1c-49d6dfefc526",
+//		"Path": "C:\\\\control\\\\windowsfilter\\\\65bf96e5760a09edf1790cb229e2dfb2dbd0fcdc0bf7451bae099106bfbfea0c"
+//	}],
+//	"HostName": "5e0055c814a6",
+//	"MappedDirectories": [],
+//	"HvPartition": false,
+//	"EndpointList": ["eef2649d-bb17-4d53-9937-295a8efe6f2c"],
+//	"Servicing": false
+//}
+//
+// Isolation=Hyper-V example:
+//
+//{
+//	"SystemType": "Container",
+//	"Name": "475c2c58933b72687a88a441e7e0ca4bd72d76413c5f9d5031fee83b98f6045d",
+//	"Owner": "docker",
+//	"IsDummy": false,
+//	"IgnoreFlushesDuringBoot": true,
+//	"Layers": [{
+//		"ID": "18955d65-d45a-557b-bf1c-49d6dfefc526",
+//		"Path": "C:\\\\control\\\\windowsfilter\\\\65bf96e5760a09edf1790cb229e2dfb2dbd0fcdc0bf7451bae099106bfbfea0c"
+//	}],
+//	"HostName": "475c2c58933b",
+//	"MappedDirectories": [],
+//	"SandboxPath": "C:\\\\control\\\\windowsfilter",
+//	"HvPartition": true,
+//	"EndpointList": ["e1bb1e61-d56f-405e-b75d-fd520cefa0cb"],
+//	"HvRuntime": {
+//		"ImagePath": "C:\\\\control\\\\windowsfilter\\\\65bf96e5760a09edf1790cb229e2dfb2dbd0fcdc0bf7451bae099106bfbfea0c\\\\UtilityVM"
+//	},
+//	"Servicing": false
+//}
 func (clnt *client) Create(containerID string, checkpoint string, checkpointDir string, spec Spec, options ...CreateOption) error {
 	clnt.lock(containerID)
 	defer clnt.unlock(containerID)
 	logrus.Debugln("libcontainerd: client.Create() with spec", spec)
 
 	configuration := &hcsshim.ContainerConfig{
-		SystemType:              "Container",
+		SystemType: "Container",
-		Name:                    containerID,
+		Name:       containerID,
-		Owner:                   defaultOwner,
+		Owner:      defaultOwner,
-		VolumePath:              spec.Root.Path,
 		IgnoreFlushesDuringBoot: false,
-		LayerFolderPath:         spec.Windows.LayerFolder,
 		HostName:                spec.Hostname,
+		HvPartition:             false,
 	}
 
 	if spec.Windows.Networking != nil {
@@ -80,33 +134,45 @@ func (clnt *client) Create(containerID string, checkpoint string, checkpointDir
 		}
 	}
 
-	if spec.Windows.HvRuntime != nil {
+	var layerOpt *LayerOption
-		configuration.VolumePath = "" // Always empty for Hyper-V containers
-		configuration.HvPartition = true
-		configuration.HvRuntime = &hcsshim.HvRuntime{
-			ImagePath: spec.Windows.HvRuntime.ImagePath,
-		}
-	}
-
-	if configuration.HvPartition {
-		configuration.SandboxPath = filepath.Dir(spec.Windows.LayerFolder)
-	} else {
-		configuration.VolumePath = spec.Root.Path
-		configuration.LayerFolderPath = spec.Windows.LayerFolder
-	}
-
 	for _, option := range options {
 		if s, ok := option.(*ServicingOption); ok {
 			configuration.Servicing = s.IsServicing
 			continue
 		}
-		if s, ok := option.(*FlushOption); ok {
+		if f, ok := option.(*FlushOption); ok {
-			configuration.IgnoreFlushesDuringBoot = s.IgnoreFlushesDuringBoot
+			configuration.IgnoreFlushesDuringBoot = f.IgnoreFlushesDuringBoot
+			continue
+		}
+		if h, ok := option.(*HyperVIsolationOption); ok {
+			configuration.HvPartition = h.IsHyperV
+			configuration.SandboxPath = h.SandboxPath
+			continue
+		}
+		if l, ok := option.(*LayerOption); ok {
+			layerOpt = l
 			continue
 		}
 	}
 
-	for _, layerPath := range spec.Windows.LayerPaths {
+	// We must have a layer option with at least one path
+	if layerOpt == nil || layerOpt.LayerPaths == nil {
+		return fmt.Errorf("no layer option or paths were supplied to the runtime")
+	}
+
+	if configuration.HvPartition {
+		// Make sure the Utility VM image is present in the base layer directory.
+		// TODO @swernli/jhowardmsft at some point post RS1 this may be re-locatable.
+		configuration.HvRuntime = &hcsshim.HvRuntime{ImagePath: filepath.Join(layerOpt.LayerPaths[len(layerOpt.LayerPaths)-1], "UtilityVM")}
+		if _, err := os.Stat(configuration.HvRuntime.ImagePath); os.IsNotExist(err) {
+			return fmt.Errorf("utility VM image '%s' could not be found", configuration.HvRuntime.ImagePath)
+		}
+	} else {
+		configuration.VolumePath = spec.Root.Path
+		configuration.LayerFolderPath = layerOpt.LayerFolderPath
+	}
+
+	for _, layerPath := range layerOpt.LayerPaths {
 		_, filename := filepath.Split(layerPath)
 		g, err := hcsshim.NameToGuid(filename)
 		if err != nil {

libcontainerd/types_windows.go

@@ -45,6 +45,22 @@ type FlushOption struct {
 	IgnoreFlushesDuringBoot bool
 }
 
+// HyperVIsolationOption is a CreateOption that indicates whether the runtime
+// should start the container as a Hyper-V container, and if so, the sandbox path.
+type HyperVIsolationOption struct {
+	IsHyperV    bool
+	SandboxPath string `json:",omitempty"`
+}
+
+// LayerOption is a CreateOption that indicates to the runtime the layer folder
+// and layer paths for a container.
+type LayerOption struct {
+	// LayerFolder is the path to the current layer folder. Empty for Hyper-V containers.
+	LayerFolderPath string `json:",omitempty"`
+	// Layer paths of the parent layers
+	LayerPaths []string
+}
+
 // Checkpoint holds the details of a checkpoint (not supported in windows)
 type Checkpoint struct {
 	Name string

libcontainerd/utils_windows.go

@@ -21,6 +21,16 @@ func (s *ServicingOption) Apply(interface{}) error {
 }
 
 // Apply for the flush option is a no-op.
-func (s *FlushOption) Apply(interface{}) error {
+func (f *FlushOption) Apply(interface{}) error {
+	return nil
+}
+
+// Apply for the hypervisolation option is a no-op.
+func (h *HyperVIsolationOption) Apply(interface{}) error {
+	return nil
+}
+
+// Apply for the layer option is a no-op.
+func (h *LayerOption) Apply(interface{}) error {
 	return nil
 }

libcontainerd/windowsoci/oci_windows.go

@@ -39,12 +39,6 @@ type Windows struct {
 	Resources *WindowsResources `json:"resources,omitempty"`
 	// Networking contains the platform specific network settings for the container.
 	Networking *WindowsNetworking `json:"networking,omitempty"`
-	// LayerFolder is the path to the current layer folder
-	LayerFolder string `json:"layer_folder,omitempty"`
-	// Layer paths of the parent layers
-	LayerPaths []string `json:"layer_paths,omitempty"`
-	// HvRuntime contains settings specific to Hyper-V containers, omitted if not using Hyper-V isolation
-	HvRuntime *WindowsHvRuntime `json:"hv_runtime,omitempty"`
 }
 
 // Process contains information to start a specific application inside the container.
@@ -122,12 +116,6 @@ type Mount struct {
 	Options []string `json:"options,omitempty"`
 }
 
-// WindowsHvRuntime contains settings specific to Hyper-V containers
-type WindowsHvRuntime struct {
-	// ImagePath is the path to the Utility VM image for this container
-	ImagePath string `json:"image_path,omitempty"`
-}
-
 // WindowsNetworking contains the platform specific network settings for the container
 type WindowsNetworking struct {
 	// List of endpoints to be attached to the container