LCOW: OCI Spec and Environment for container start

Signed-off-by: John Howard <jhoward@microsoft.com>

builder/dockerfile/dispatchers_test.go

@@ -203,7 +203,7 @@ func TestFromScratch(t *testing.T) {
 	assert.True(t, req.state.hasFromImage())
 	assert.Equal(t, "", req.state.imageID)
 	// Windows does not set the default path. TODO @jhowardmsft LCOW support. This will need revisiting as we get further into the implementation
-	expected := "PATH=" + system.DefaultPathEnv
+	expected := "PATH=" + system.DefaultPathEnv(runtime.GOOS)
 	if runtime.GOOS == "windows" {
 		expected = ""
 	}

builder/dockerfile/evaluator.go

@@ -22,6 +22,7 @@ package dockerfile
 import (
 	"bytes"
 	"fmt"
+	"runtime"
 	"strings"
 
 	"github.com/docker/docker/api/types/container"
@@ -228,14 +229,19 @@ func (s *dispatchState) beginStage(stageName string, image builder.Image) {
 }
 
 // Add the default PATH to runConfig.ENV if one exists for the platform and there
-// is no PATH set. Note that windows won't have one as it's set by HCS
+// is no PATH set. Note that Windows containers on Windows won't have one as it's set by HCS
 func (s *dispatchState) setDefaultPath() {
-	if system.DefaultPathEnv == "" {
+	// TODO @jhowardmsft LCOW Support - This will need revisiting later
+	platform := runtime.GOOS
+	if platform == "windows" && system.LCOWSupported() {
+		platform = "linux"
+	}
+	if system.DefaultPathEnv(platform) == "" {
 		return
 	}
 	envMap := opts.ConvertKVStringsToMap(s.runConfig.Env)
 	if _, ok := envMap["PATH"]; !ok {
-		s.runConfig.Env = append(s.runConfig.Env, "PATH="+system.DefaultPathEnv)
+		s.runConfig.Env = append(s.runConfig.Env, "PATH="+system.DefaultPathEnv(platform))
 	}
 }
 

container/container.go

@@ -34,6 +34,7 @@ import (
 	"github.com/docker/docker/pkg/ioutils"
 	"github.com/docker/docker/pkg/signal"
 	"github.com/docker/docker/pkg/symlink"
+	"github.com/docker/docker/pkg/system"
 	"github.com/docker/docker/restartmanager"
 	"github.com/docker/docker/runconfig"
 	"github.com/docker/docker/volume"
@@ -1004,3 +1005,31 @@ func (container *Container) ConfigsDirPath() string {
 func (container *Container) ConfigFilePath(configRef swarmtypes.ConfigReference) string {
 	return filepath.Join(container.ConfigsDirPath(), configRef.ConfigID)
 }
+
+// CreateDaemonEnvironment creates a new environment variable slice for this container.
+func (container *Container) CreateDaemonEnvironment(tty bool, linkedEnv []string) []string {
+	// Setup environment
+	// TODO @jhowardmsft LCOW Support. This will need revisiting later.
+	platform := container.Platform
+	if platform == "" {
+		platform = runtime.GOOS
+	}
+	env := []string{}
+	if runtime.GOOS != "windows" || (runtime.GOOS == "windows" && system.LCOWSupported() && platform == "linux") {
+		env = []string{
+			"PATH=" + system.DefaultPathEnv(platform),
+			"HOSTNAME=" + container.Config.Hostname,
+		}
+		if tty {
+			env = append(env, "TERM=xterm")
+		}
+		env = append(env, linkedEnv...)
+	}
+
+	// because the env on the container can override certain default values
+	// we need to replace the 'env' keys where they match and append anything
+	// else.
+	//return ReplaceOrAppendEnvValues(linkedEnv, container.Config.Env)
+	foo := ReplaceOrAppendEnvValues(env, container.Config.Env)
+	return foo
+}

container/container_unix.go

@@ -35,27 +35,6 @@ type ExitStatus struct {
 	OOMKilled bool
 }
 
-// CreateDaemonEnvironment returns the list of all environment variables given the list of
-// environment variables related to links.
-// Sets PATH, HOSTNAME and if container.Config.Tty is set: TERM.
-// The defaults set here do not override the values in container.Config.Env
-func (container *Container) CreateDaemonEnvironment(tty bool, linkedEnv []string) []string {
-	// Setup environment
-	env := []string{
-		"PATH=" + system.DefaultPathEnv,
-		"HOSTNAME=" + container.Config.Hostname,
-	}
-	if tty {
-		env = append(env, "TERM=xterm")
-	}
-	env = append(env, linkedEnv...)
-	// because the env on the container can override certain default values
-	// we need to replace the 'env' keys where they match and append anything
-	// else.
-	env = ReplaceOrAppendEnvValues(env, container.Config.Env)
-	return env
-}
-
 // TrySetNetworkMount attempts to set the network mounts given a provided destination and
 // the path to use for it; return true if the given destination was a network mount file
 func (container *Container) TrySetNetworkMount(destination string, path string) bool {

container/container_windows.go

@@ -23,14 +23,6 @@ type ExitStatus struct {
 	ExitCode int
 }
 
-// CreateDaemonEnvironment creates a new environment variable slice for this container.
-func (container *Container) CreateDaemonEnvironment(_ bool, linkedEnv []string) []string {
-	// because the env on the container can override certain default values
-	// we need to replace the 'env' keys where they match and append anything
-	// else.
-	return ReplaceOrAppendEnvValues(linkedEnv, container.Config.Env)
-}
-
 // UnmountIpcMounts unmounts Ipc related mounts.
 // This is a NOOP on windows.
 func (container *Container) UnmountIpcMounts(unmount func(pth string) error) {

daemon/oci_windows.go

@@ -7,11 +7,17 @@ import (
 	"github.com/docker/docker/container"
 	"github.com/docker/docker/oci"
 	"github.com/docker/docker/pkg/sysinfo"
+	"github.com/docker/docker/pkg/system"
 	"github.com/opencontainers/runtime-spec/specs-go"
 )
 
 func (daemon *Daemon) createSpec(c *container.Container) (*specs.Spec, error) {
-	s := oci.DefaultSpec()
+	img, err := daemon.GetImage(string(c.ImageID))
+	if err != nil {
+		return nil, err
+	}
+
+	s := oci.DefaultOSSpec(img.OS)
 
 	linkedEnv, err := daemon.setupLinkedContainers(c)
 	if err != nil {
@@ -95,7 +101,30 @@ func (daemon *Daemon) createSpec(c *container.Container) (*specs.Spec, error) {
 	if !c.Config.ArgsEscaped {
 		s.Process.Args = escapeArgs(s.Process.Args)
 	}
+
 	s.Process.Cwd = c.Config.WorkingDir
+	s.Process.Env = c.CreateDaemonEnvironment(c.Config.Tty, linkedEnv)
+	if c.Config.Tty {
+		s.Process.Terminal = c.Config.Tty
+		s.Process.ConsoleSize.Height = c.HostConfig.ConsoleSize[0]
+		s.Process.ConsoleSize.Width = c.HostConfig.ConsoleSize[1]
+	}
+	s.Process.User.Username = c.Config.User
+
+	if img.OS == "windows" {
+		daemon.createSpecWindowsFields(c, &s, isHyperV)
+	} else {
+		// TODO @jhowardmsft LCOW Support. Modify this check when running in dual-mode
+		if system.LCOWSupported() && img.OS == "linux" {
+			daemon.createSpecLinuxFields(c, &s)
+		}
+	}
+
+	return (*specs.Spec)(&s), nil
+}
+
+// Sets the Windows-specific fields of the OCI spec
+func (daemon *Daemon) createSpecWindowsFields(c *container.Container, s *specs.Spec, isHyperV bool) {
 	if len(s.Process.Cwd) == 0 {
 		// We default to C:\ to workaround the oddity of the case that the
 		// default directory for cmd running as LocalSystem (or
@@ -106,17 +135,11 @@ func (daemon *Daemon) createSpec(c *container.Container) (*specs.Spec, error) {
 		// as c:\. Hence, setting it to default of c:\ makes for consistency.
 		s.Process.Cwd = `C:\`
 	}
-	s.Process.Env = c.CreateDaemonEnvironment(c.Config.Tty, linkedEnv)
-	s.Process.ConsoleSize.Height = c.HostConfig.ConsoleSize[0]
-	s.Process.ConsoleSize.Width = c.HostConfig.ConsoleSize[1]
-	s.Process.Terminal = c.Config.Tty
-	s.Process.User.Username = c.Config.User
 
-	// In spec.Root. This is not set for Hyper-V containers
+	s.Root.Readonly = false // Windows does not support a read-only root filesystem
 	if !isHyperV {
-		s.Root.Path = c.BaseFS
+		s.Root.Path = c.BaseFS // This is not set for Hyper-V containers
 	}
-	s.Root.Readonly = false // Windows does not support a read-only root filesystem
 
 	// In s.Windows.Resources
 	cpuShares := uint16(c.HostConfig.CPUShares)
@@ -157,7 +180,17 @@ func (daemon *Daemon) createSpec(c *container.Container) (*specs.Spec, error) {
 			Iops: &c.HostConfig.IOMaximumIOps,
 		},
 	}
-	return (*specs.Spec)(&s), nil
+}
+
+// Sets the Linux-specific fields of the OCI spec
+// TODO: @jhowardmsft LCOW Support. We need to do a lot more pulling in what can
+// be pulled in from oci_linux.go.
+func (daemon *Daemon) createSpecLinuxFields(c *container.Container, s *specs.Spec) {
+	if len(s.Process.Cwd) == 0 {
+		s.Process.Cwd = `/`
+	}
+	s.Root.Path = "rootfs"
+	s.Root.Readonly = c.HostConfig.ReadonlyRootfs
 }
 
 func escapeArgs(args []string) []string {

libcontainerd/client_windows.go

@@ -1,6 +1,7 @@
 package libcontainerd
 
 import (
+	"encoding/json"
 	"errors"
 	"fmt"
 	"io"
@@ -96,8 +97,17 @@ const defaultOwner = "docker"
 func (clnt *client) Create(containerID string, checkpoint string, checkpointDir string, spec specs.Spec, attachStdio StdioCallback, options ...CreateOption) error {
 	clnt.lock(containerID)
 	defer clnt.unlock(containerID)
-	logrus.Debugln("libcontainerd: client.Create() with spec", spec)
+	if b, err := json.Marshal(spec); err == nil {
+		logrus.Debugln("libcontainerd: client.Create() with spec", string(b))
+	}
+	osName := spec.Platform.OS
+	if osName == "windows" {
+		return clnt.createWindows(containerID, checkpoint, checkpointDir, spec, attachStdio, options...)
+	}
+	return clnt.createLinux(containerID, checkpoint, checkpointDir, spec, attachStdio, options...)
+}
 
+func (clnt *client) createWindows(containerID string, checkpoint string, checkpointDir string, spec specs.Spec, attachStdio StdioCallback, options ...CreateOption) error {
 	configuration := &hcsshim.ContainerConfig{
 		SystemType: "Container",
 		Name:       containerID,
@@ -265,17 +275,100 @@ func (clnt *client) Create(containerID string, checkpoint string, checkpointDir
 	// Call start, and if it fails, delete the container from our
 	// internal structure, start will keep HCS in sync by deleting the
 	// container there.
-	logrus.Debugf("libcontainerd: Create() id=%s, Calling start()", containerID)
+	logrus.Debugf("libcontainerd: createWindows() id=%s, Calling start()", containerID)
 	if err := container.start(attachStdio); err != nil {
 		clnt.deleteContainer(containerID)
 		return err
 	}
 
-	logrus.Debugf("libcontainerd: Create() id=%s completed successfully", containerID)
+	logrus.Debugf("libcontainerd: createWindows() id=%s completed successfully", containerID)
 	return nil
 
 }
 
+func (clnt *client) createLinux(containerID string, checkpoint string, checkpointDir string, spec specs.Spec, attachStdio StdioCallback, options ...CreateOption) error {
+	logrus.Debugf("libcontainerd: createLinux(): containerId %s ", containerID)
+
+	// TODO @jhowardmsft LCOW Support: This needs to be configurable, not hard-coded.
+	// However, good-enough for the LCOW bring-up.
+	configuration := &hcsshim.ContainerConfig{
+		HvPartition:                 true,
+		Name:                        containerID,
+		SystemType:                  "container",
+		ContainerType:               "linux",
+		TerminateOnLastHandleClosed: true,
+		HvRuntime: &hcsshim.HvRuntime{
+			ImagePath: `c:\program files\lcow`,
+		},
+	}
+
+	var layerOpt *LayerOption
+	for _, option := range options {
+		if l, ok := option.(*LayerOption); ok {
+			layerOpt = l
+		}
+	}
+
+	// We must have a layer option with at least one path
+	if layerOpt == nil || layerOpt.LayerPaths == nil {
+		return fmt.Errorf("no layer option or paths were supplied to the runtime")
+	}
+
+	// LayerFolderPath (writeable layer) + Layers (Guid + path)
+	configuration.LayerFolderPath = layerOpt.LayerFolderPath
+	for _, layerPath := range layerOpt.LayerPaths {
+		_, filename := filepath.Split(layerPath)
+		g, err := hcsshim.NameToGuid(filename)
+		if err != nil {
+			return err
+		}
+		configuration.Layers = append(configuration.Layers, hcsshim.Layer{
+			ID:   g.ToString(),
+			Path: filepath.Join(layerPath, "layer.vhd"),
+		})
+	}
+
+	hcsContainer, err := hcsshim.CreateContainer(containerID, configuration)
+	if err != nil {
+		return err
+	}
+
+	// Construct a container object for calling start on it.
+	container := &container{
+		containerCommon: containerCommon{
+			process: process{
+				processCommon: processCommon{
+					containerID:  containerID,
+					client:       clnt,
+					friendlyName: InitFriendlyName,
+				},
+			},
+			processes: make(map[string]*process),
+		},
+		ociSpec:      spec,
+		hcsContainer: hcsContainer,
+	}
+
+	container.options = options
+	for _, option := range options {
+		if err := option.Apply(container); err != nil {
+			logrus.Errorf("libcontainerd: createLinux() %v", err)
+		}
+	}
+
+	// Call start, and if it fails, delete the container from our
+	// internal structure, start will keep HCS in sync by deleting the
+	// container there.
+	logrus.Debugf("libcontainerd: createLinux() id=%s, Calling start()", containerID)
+	if err := container.start(attachStdio); err != nil {
+		clnt.deleteContainer(containerID)
+		return err
+	}
+
+	logrus.Debugf("libcontainerd: createLinux() id=%s completed successfully", containerID)
+	return nil
+}
+
 // AddProcess is the handler for adding a process to an already running
 // container. It's called through docker exec. It returns the system pid of the
 // exec'd process.
@@ -292,13 +385,15 @@ func (clnt *client) AddProcess(ctx context.Context, containerID, processFriendly
 	// create stdin, even if it's not used - it will be closed shortly. Stderr
 	// is only created if it we're not -t.
 	createProcessParms := hcsshim.ProcessConfig{
-		EmulateConsole:   procToAdd.Terminal,
 		CreateStdInPipe:  true,
 		CreateStdOutPipe: true,
 		CreateStdErrPipe: !procToAdd.Terminal,
 	}
-	createProcessParms.ConsoleSize[0] = uint(procToAdd.ConsoleSize.Height)
-	createProcessParms.ConsoleSize[1] = uint(procToAdd.ConsoleSize.Width)
+	if procToAdd.Terminal {
+		createProcessParms.EmulateConsole = true
+		createProcessParms.ConsoleSize[0] = uint(procToAdd.ConsoleSize.Height)
+		createProcessParms.ConsoleSize[1] = uint(procToAdd.ConsoleSize.Width)
+	}
 
 	// Take working directory from the process to add if it is defined,
 	// otherwise take from the first process.

libcontainerd/container_windows.go

@@ -1,6 +1,7 @@
 package libcontainerd
 
 import (
+	"encoding/json"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -10,6 +11,7 @@ import (
 
 	"github.com/Microsoft/hcsshim"
 	"github.com/Sirupsen/logrus"
+	"github.com/docker/docker/pkg/system"
 	"github.com/opencontainers/runtime-spec/specs-go"
 )
 
@@ -83,6 +85,16 @@ func (ctr *container) start(attachStdio StdioCallback) error {
 	createProcessParms.CommandLine = strings.Join(ctr.ociSpec.Process.Args, " ")
 	createProcessParms.User = ctr.ociSpec.Process.User.Username
 
+	// LCOW requires the raw OCI spec passed through HCS and onwards to GCS for the utility VM.
+	if system.LCOWSupported() && ctr.ociSpec.Platform.OS == "linux" {
+		ociBuf, err := json.Marshal(ctr.ociSpec)
+		if err != nil {
+			return err
+		}
+		ociRaw := json.RawMessage(ociBuf)
+		createProcessParms.OCISpecification = &ociRaw
+	}
+
 	// Start the command running in the container.
 	newProcess, err := ctr.hcsContainer.CreateProcess(createProcessParms)
 	if err != nil {
@@ -228,11 +240,14 @@ func (ctr *container) waitExit(process *process, isFirstProcessToStart bool) err
 	if !isFirstProcessToStart {
 		si.State = StateExitProcess
 	} else {
-		updatePending, err := ctr.hcsContainer.HasPendingUpdates()
-		if err != nil {
-			logrus.Warnf("libcontainerd: HasPendingUpdates() failed (container may have been killed): %s", err)
-		} else {
-			si.UpdatePending = updatePending
+		// Pending updates is only applicable for WCOW
+		if ctr.ociSpec.Platform.OS == "windows" {
+			updatePending, err := ctr.hcsContainer.HasPendingUpdates()
+			if err != nil {
+				logrus.Warnf("libcontainerd: HasPendingUpdates() failed (container may have been killed): %s", err)
+			} else {
+				si.UpdatePending = updatePending
+			}
 		}
 
 		logrus.Debugf("libcontainerd: shutting down container %s", ctr.containerID)

oci/defaults_linux.go → oci/defaults.go

@@ -30,14 +30,55 @@ func defaultCapabilities() []string {
 	}
 }
 
-// DefaultSpec returns default oci spec used by docker.
+// DefaultSpec returns the default spec used by docker for the current Platform
 func DefaultSpec() specs.Spec {
-	s := specs.Spec{
+	return DefaultOSSpec(runtime.GOOS)
+}
+
+// DefaultOSSpec returns the spec for a given OS
+func DefaultOSSpec(osName string) specs.Spec {
+	if osName == "windows" {
+		return DefaultWindowsSpec()
+	} else if osName == "solaris" {
+		return DefaultSolarisSpec()
+	} else {
+		return DefaultLinuxSpec()
+	}
+}
+
+// DefaultWindowsSpec create a default spec for running Windows containers
+func DefaultWindowsSpec() specs.Spec {
+	return specs.Spec{
 		Version: specs.Version,
 		Platform: specs.Platform{
 			OS:   runtime.GOOS,
 			Arch: runtime.GOARCH,
 		},
+		Windows: &specs.Windows{},
+	}
+}
+
+// DefaultSolarisSpec create a default spec for running Solaris containers
+func DefaultSolarisSpec() specs.Spec {
+	s := specs.Spec{
+		Version: "0.6.0",
+		Platform: specs.Platform{
+			OS:   "SunOS",
+			Arch: runtime.GOARCH,
+		},
+	}
+	s.Solaris = &specs.Solaris{}
+	return s
+}
+
+// DefaultLinuxSpec create a default spec for running Linux containers
+func DefaultLinuxSpec() specs.Spec {
+	s := specs.Spec{
+		Version: specs.Version,
+		Platform: specs.Platform{
+			OS:   "linux",
+			Arch: runtime.GOARCH,
+		},
 	}
 	s.Mounts = []specs.Mount{
 		{
@@ -91,7 +132,6 @@ func DefaultSpec() specs.Spec {
 			"/proc/timer_list",
 			"/proc/timer_stats",
 			"/proc/sched_debug",
-			"/sys/firmware",
 		},
 		ReadonlyPaths: []string{
 			"/proc/asound",
@@ -172,5 +212,10 @@ func DefaultSpec() specs.Spec {
 		},
 	}
 
+	// For LCOW support, don't mask /sys/firmware
+	if runtime.GOOS != "windows" {
+		s.Linux.MaskedPaths = append(s.Linux.MaskedPaths, "/sys/firmware")
+	}
+
 	return s
 }

oci/defaults_solaris.go

@@ -1,20 +0,0 @@
-package oci
-
-import (
-	"runtime"
-
-	"github.com/opencontainers/runtime-spec/specs-go"
-)
-
-// DefaultSpec returns default oci spec used by docker.
-func DefaultSpec() specs.Spec {
-	s := specs.Spec{
-		Version: "0.6.0",
-		Platform: specs.Platform{
-			OS:   "SunOS",
-			Arch: runtime.GOARCH,
-		},
-	}
-	s.Solaris = &specs.Solaris{}
-	return s
-}

oci/defaults_windows.go

@@ -1,19 +0,0 @@
-package oci
-
-import (
-	"runtime"
-
-	"github.com/opencontainers/runtime-spec/specs-go"
-)
-
-// DefaultSpec returns default spec used by docker.
-func DefaultSpec() specs.Spec {
-	return specs.Spec{
-		Version: specs.Version,
-		Platform: specs.Platform{
-			OS:   runtime.GOOS,
-			Arch: runtime.GOARCH,
-		},
-		Windows: &specs.Windows{},
-	}
-}

pkg/system/path.go

@@ -0,0 +1,21 @@
+package system
+
+import "runtime"
+
+const defaultUnixPathEnv = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
+
+// DefaultPathEnv is unix style list of directories to search for
+// executables. Each directory is separated from the next by a colon
+// ':' character .
+func DefaultPathEnv(platform string) string {
+	if runtime.GOOS == "windows" {
+		if platform != runtime.GOOS && LCOWSupported() {
+			return defaultUnixPathEnv
+		}
+		// Deliberately empty on Windows containers on Windows as the default path will be set by
+		// the container. Docker has no context of what the default path should be.
+		return ""
+	}
+	return defaultUnixPathEnv
+
+}

pkg/system/path_unix.go

@@ -2,11 +2,6 @@
 
 package system
 
-// DefaultPathEnv is unix style list of directories to search for
-// executables. Each directory is separated from the next by a colon
-// ':' character .
-const DefaultPathEnv = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
-
 // CheckSystemDriveAndRemoveDriveLetter verifies that a path, if it includes a drive letter,
 // is the system drive. This is a no-op on Linux.
 func CheckSystemDriveAndRemoveDriveLetter(path string) (string, error) {

pkg/system/path_windows.go

@@ -8,10 +8,6 @@ import (
 	"strings"
 )
 
-// DefaultPathEnv is deliberately empty on Windows as the default path will be set by
-// the container. Docker has no context of what the default path should be.
-const DefaultPathEnv = ""
-
 // CheckSystemDriveAndRemoveDriveLetter verifies and manipulates a Windows path.
 // This is used, for example, when validating a user provided path in docker cp.
 // If a drive letter is supplied, it must be the system drive. The drive letter

plugin/v2/plugin_linux.go

@@ -5,6 +5,7 @@ package v2
 import (
 	"os"
 	"path/filepath"
+	"runtime"
 	"strings"
 
 	"github.com/docker/docker/api/types"
@@ -108,7 +109,7 @@ func (p *Plugin) InitSpec(execRoot string) (*specs.Spec, error) {
 	}
 
 	envs := make([]string, 1, len(p.PluginObj.Settings.Env)+1)
-	envs[0] = "PATH=" + system.DefaultPathEnv
+	envs[0] = "PATH=" + system.DefaultPathEnv(runtime.GOOS)
 	envs = append(envs, p.PluginObj.Settings.Env...)
 
 	args := append(p.PluginObj.Config.Entrypoint, p.PluginObj.Settings.Args...)