From 85ad4b16c11425eaeace532bfce9029cfdcabe4b Mon Sep 17 00:00:00 2001 From: John Howard Date: Tue, 8 Jan 2019 14:30:52 -0800 Subject: [PATCH] Windows: Experimental: Allow containerd for runtime Signed-off-by: John Howard This is the first step in refactoring moby (dockerd) to use containerd on Windows. Similar to the current model in Linux, this adds the option to enable it for runtime. It does not switch the graphdriver to containerd snapshotters. - Refactors libcontainerd to a series of subpackages so that either a "local" containerd (1) or a "remote" (2) containerd can be loaded as opposed to conditional compile as "local" for Windows and "remote" for Linux. - Updates libcontainerd such that Windows has an option to allow the use of a "remote" containerd. Here, it communicates over a named pipe using GRPC. This is currently guarded behind the experimental flag, an environment variable, and the providing of a pipename to connect to containerd. - Infrastructure pieces such as under pkg/system to have helper functions for determining whether containerd is being used. (1) "local" containerd is what the daemon on Windows has used since inception. It's not really containerd at all - it's simply local invocation of HCS APIs directly in-process from the daemon through the Microsoft/hcsshim library. (2) "remote" containerd is what docker on Linux uses for it's runtime. It means that there is a separate containerd service running, and docker communicates over GRPC to it. To try this out, you will need to start with something like the following: Window 1: containerd --log-level debug Window 2: $env:DOCKER_WINDOWS_CONTAINERD=1 dockerd --experimental -D --containerd \\.\pipe\containerd-containerd You will need the following binary from github.com/containerd/containerd in your path: - containerd.exe You will need the following binaries from github.com/Microsoft/hcsshim in your path: - runhcs.exe - containerd-shim-runhcs-v1.exe For LCOW, it will require and initrd.img and kernel in `C:\Program Files\Linux Containers`. This is no different to the current requirements. However, you may need updated binaries, particularly initrd.img built from Microsoft/opengcs as (at the time of writing), Linuxkit binaries are somewhat out of date. Note that containerd and hcsshim for HCS v2 APIs do not yet support all the required functionality needed for docker. This will come in time - this is a baby (although large) step to migrating Docker on Windows to containerd. Note that the HCS v2 APIs are only called on RS5+ builds. RS1..RS4 will still use HCS v1 APIs as the v2 APIs were not fully developed enough on these builds to be usable. This abstraction is done in HCSShim. (Referring specifically to runtime) Note the LCOW graphdriver still uses HCS v1 APIs regardless. Note also that this does not migrate docker to use containerd snapshotters rather than graphdrivers. This needs to be done in conjunction with Linux also doing the same switch. --- cmd/dockerd/daemon.go | 28 +-- cmd/dockerd/daemon_unix.go | 31 ++++ cmd/dockerd/daemon_windows.go | 7 + daemon/daemon.go | 10 +- daemon/daemon_windows.go | 2 + daemon/kill.go | 4 +- daemon/monitor.go | 14 +- daemon/oci_windows.go | 51 ++++-- daemon/resize.go | 4 +- daemon/start_windows.go | 12 ++ daemon/update_linux.go | 6 +- daemon/update_windows.go | 4 +- daemon/util_test.go | 16 +- libcontainerd/errors.go | 13 -- libcontainerd/libcontainerd_linux.go | 14 ++ libcontainerd/libcontainerd_windows.go | 19 ++ libcontainerd/{ => local}/process_windows.go | 2 +- libcontainerd/{ => local}/utils_windows.go | 7 +- .../{ => local}/utils_windows_test.go | 2 +- .../windows.go} | 167 +++++++++--------- libcontainerd/{ => queue}/queue.go | 8 +- libcontainerd/{ => queue}/queue_test.go | 10 +- .../{client_daemon.go => remote/client.go} | 153 ++++++++-------- libcontainerd/remote/client_io_windows.go | 161 +++++++++++++++++ .../client_linux.go} | 17 +- .../client_windows.go} | 33 +++- libcontainerd/{ => types}/types.go | 5 +- libcontainerd/{ => types}/types_linux.go | 5 +- libcontainerd/{ => types}/types_windows.go | 11 +- pkg/system/init_unix.go | 5 + pkg/system/init_windows.go | 32 +++- plugin/executor/containerd/containerd.go | 19 +- plugin/executor/containerd/containerd_test.go | 14 +- 33 files changed, 598 insertions(+), 288 deletions(-) delete mode 100644 libcontainerd/errors.go create mode 100644 libcontainerd/libcontainerd_linux.go create mode 100644 libcontainerd/libcontainerd_windows.go rename libcontainerd/{ => local}/process_windows.go (92%) rename libcontainerd/{ => local}/utils_windows.go (83%) rename libcontainerd/{ => local}/utils_windows_test.go (81%) rename libcontainerd/{client_local_windows.go => local/windows.go} (88%) rename libcontainerd/{ => queue}/queue.go (60%) rename libcontainerd/{ => queue}/queue_test.go (70%) rename libcontainerd/{client_daemon.go => remote/client.go} (81%) create mode 100644 libcontainerd/remote/client_io_windows.go rename libcontainerd/{client_daemon_linux.go => remote/client_linux.go} (81%) rename libcontainerd/{client_daemon_windows.go => remote/client_windows.go} (57%) rename libcontainerd/{ => types}/types.go (95%) rename libcontainerd/{ => types}/types_linux.go (72%) rename libcontainerd/{ => types}/types_windows.go (68%) diff --git a/cmd/dockerd/daemon.go b/cmd/dockerd/daemon.go index 362d43666f..a86c0a576a 100644 --- a/cmd/dockerd/daemon.go +++ b/cmd/dockerd/daemon.go @@ -163,31 +163,9 @@ func (cli *DaemonCli) start(opts *daemonOptions) (err error) { } ctx, cancel := context.WithCancel(context.Background()) - if cli.Config.ContainerdAddr == "" && runtime.GOOS != "windows" { - systemContainerdAddr, ok, err := systemContainerdRunning(cli.Config.IsRootless()) - if err != nil { - cancel() - return errors.Wrap(err, "could not determine whether the system containerd is running") - } - if !ok { - opts, err := cli.getContainerdDaemonOpts() - if err != nil { - cancel() - return errors.Wrap(err, "failed to generate containerd options") - } - - r, err := supervisor.Start(ctx, filepath.Join(cli.Config.Root, "containerd"), filepath.Join(cli.Config.ExecRoot, "containerd"), opts...) - if err != nil { - cancel() - return errors.Wrap(err, "failed to start containerd") - } - cli.Config.ContainerdAddr = r.Address() - - // Try to wait for containerd to shutdown - defer r.WaitTimeout(10 * time.Second) - } else { - cli.Config.ContainerdAddr = systemContainerdAddr - } + if err := cli.initContainerD(ctx); err != nil { + cancel() + return err } defer cancel() diff --git a/cmd/dockerd/daemon_unix.go b/cmd/dockerd/daemon_unix.go index f6ada580aa..6ee042632b 100644 --- a/cmd/dockerd/daemon_unix.go +++ b/cmd/dockerd/daemon_unix.go @@ -3,12 +3,14 @@ package main import ( + "context" "fmt" "net" "os" "os/signal" "path/filepath" "strconv" + "time" "github.com/containerd/containerd/runtime/v1/linux" "github.com/docker/docker/cmd/dockerd/hack" @@ -18,6 +20,7 @@ import ( "github.com/docker/docker/pkg/homedir" "github.com/docker/docker/rootless" "github.com/docker/libnetwork/portallocator" + "github.com/pkg/errors" "golang.org/x/sys/unix" ) @@ -145,3 +148,31 @@ func newCgroupParent(config *config.Config) string { } return cgroupParent } + +func (cli *DaemonCli) initContainerD(ctx context.Context) error { + if cli.Config.ContainerdAddr == "" { + systemContainerdAddr, ok, err := systemContainerdRunning(cli.Config.IsRootless()) + if err != nil { + return errors.Wrap(err, "could not determine whether the system containerd is running") + } + if !ok { + opts, err := cli.getContainerdDaemonOpts() + if err != nil { + return errors.Wrap(err, "failed to generate containerd options") + } + + r, err := supervisor.Start(ctx, filepath.Join(cli.Config.Root, "containerd"), filepath.Join(cli.Config.ExecRoot, "containerd"), opts...) + if err != nil { + return errors.Wrap(err, "failed to start containerd") + } + cli.Config.ContainerdAddr = r.Address() + + // Try to wait for containerd to shutdown + defer r.WaitTimeout(10 * time.Second) + } else { + cli.Config.ContainerdAddr = systemContainerdAddr + } + } + + return nil +} diff --git a/cmd/dockerd/daemon_windows.go b/cmd/dockerd/daemon_windows.go index f4d213da96..e03b928eaf 100644 --- a/cmd/dockerd/daemon_windows.go +++ b/cmd/dockerd/daemon_windows.go @@ -1,6 +1,7 @@ package main import ( + "context" "fmt" "net" "os" @@ -8,6 +9,7 @@ import ( "github.com/docker/docker/daemon/config" "github.com/docker/docker/libcontainerd/supervisor" + "github.com/docker/docker/pkg/system" "github.com/sirupsen/logrus" "golang.org/x/sys/windows" ) @@ -90,3 +92,8 @@ func wrapListeners(proto string, ls []net.Listener) []net.Listener { func newCgroupParent(config *config.Config) string { return "" } + +func (cli *DaemonCli) initContainerD(_ context.Context) error { + system.InitContainerdRuntime(cli.Config.Experimental, cli.Config.ContainerdAddr) + return nil +} diff --git a/daemon/daemon.go b/daemon/daemon.go index 428da80846..9edfc63c7a 100644 --- a/daemon/daemon.go +++ b/daemon/daemon.go @@ -42,6 +42,7 @@ import ( "github.com/moby/buildkit/util/resolver" "github.com/moby/buildkit/util/tracing" "github.com/sirupsen/logrus" + // register graph drivers _ "github.com/docker/docker/daemon/graphdriver/register" "github.com/docker/docker/daemon/stats" @@ -50,6 +51,7 @@ import ( "github.com/docker/docker/image" "github.com/docker/docker/layer" "github.com/docker/docker/libcontainerd" + libcontainerdtypes "github.com/docker/docker/libcontainerd/types" "github.com/docker/docker/pkg/idtools" "github.com/docker/docker/pkg/locker" "github.com/docker/docker/pkg/plugingetter" @@ -105,7 +107,7 @@ type Daemon struct { pluginManager *plugin.Manager linkIndex *linkIndex containerdCli *containerd.Client - containerd libcontainerd.Client + containerd libcontainerdtypes.Client defaultIsolation containertypes.Isolation // Default isolation mode on Windows clusterProvider cluster.Provider cluster Cluster @@ -351,11 +353,11 @@ func (daemon *Daemon) restore() error { logrus.WithField("container", c.ID).WithField("state", s). Info("restored container paused") switch s { - case libcontainerd.StatusPaused, libcontainerd.StatusPausing: + case libcontainerdtypes.StatusPaused, libcontainerdtypes.StatusPausing: // nothing to do - case libcontainerd.StatusStopped: + case libcontainerdtypes.StatusStopped: alive = false - case libcontainerd.StatusUnknown: + case libcontainerdtypes.StatusUnknown: logrus.WithField("container", c.ID). Error("Unknown status for container during restore") default: diff --git a/daemon/daemon_windows.go b/daemon/daemon_windows.go index 5603afd52d..f6d0f8c6ce 100644 --- a/daemon/daemon_windows.go +++ b/daemon/daemon_windows.go @@ -502,6 +502,7 @@ func (daemon *Daemon) runAsHyperVContainer(hostConfig *containertypes.HostConfig // conditionalMountOnStart is a platform specific helper function during the // container start to call mount. func (daemon *Daemon) conditionalMountOnStart(container *container.Container) error { + // Bail out now for Linux containers. We cannot mount the containers filesystem on the // host as it is a non-Windows filesystem. if system.LCOWSupported() && container.OS != "windows" { @@ -519,6 +520,7 @@ func (daemon *Daemon) conditionalMountOnStart(container *container.Container) er // conditionalUnmountOnCleanup is a platform specific helper function called // during the cleanup of a container to unmount. func (daemon *Daemon) conditionalUnmountOnCleanup(container *container.Container) error { + // Bail out now for Linux containers if system.LCOWSupported() && container.OS != "windows" { return nil diff --git a/daemon/kill.go b/daemon/kill.go index 017f885f9b..3e97158a0c 100644 --- a/daemon/kill.go +++ b/daemon/kill.go @@ -9,7 +9,7 @@ import ( containerpkg "github.com/docker/docker/container" "github.com/docker/docker/errdefs" - "github.com/docker/docker/libcontainerd" + libcontainerdtypes "github.com/docker/docker/libcontainerd/types" "github.com/docker/docker/pkg/signal" "github.com/pkg/errors" "github.com/sirupsen/logrus" @@ -177,5 +177,5 @@ func (daemon *Daemon) killPossiblyDeadProcess(container *containerpkg.Container, } func (daemon *Daemon) kill(c *containerpkg.Container, sig int) error { - return daemon.containerd.SignalProcess(context.Background(), c.ID, libcontainerd.InitProcessName, sig) + return daemon.containerd.SignalProcess(context.Background(), c.ID, libcontainerdtypes.InitProcessName, sig) } diff --git a/daemon/monitor.go b/daemon/monitor.go index 9b4452d7ef..8c67d2ce25 100644 --- a/daemon/monitor.go +++ b/daemon/monitor.go @@ -10,7 +10,7 @@ import ( "github.com/docker/docker/api/types" "github.com/docker/docker/container" - "github.com/docker/docker/libcontainerd" + libcontainerdtypes "github.com/docker/docker/libcontainerd/types" "github.com/docker/docker/restartmanager" "github.com/sirupsen/logrus" ) @@ -27,14 +27,14 @@ func (daemon *Daemon) setStateCounter(c *container.Container) { } // ProcessEvent is called by libcontainerd whenever an event occurs -func (daemon *Daemon) ProcessEvent(id string, e libcontainerd.EventType, ei libcontainerd.EventInfo) error { +func (daemon *Daemon) ProcessEvent(id string, e libcontainerdtypes.EventType, ei libcontainerdtypes.EventInfo) error { c, err := daemon.GetContainer(id) if c == nil || err != nil { return fmt.Errorf("no such container: %s", id) } switch e { - case libcontainerd.EventOOM: + case libcontainerdtypes.EventOOM: // StateOOM is Linux specific and should never be hit on Windows if runtime.GOOS == "windows" { return errors.New("received StateOOM from libcontainerd on Windows. This should never happen") @@ -48,7 +48,7 @@ func (daemon *Daemon) ProcessEvent(id string, e libcontainerd.EventType, ei libc } daemon.LogContainerEvent(c, "oom") - case libcontainerd.EventExit: + case libcontainerdtypes.EventExit: if int(ei.Pid) == c.Pid { c.Lock() _, _, err := daemon.containerd.DeleteTask(context.Background(), c.ID) @@ -140,7 +140,7 @@ func (daemon *Daemon) ProcessEvent(id string, e libcontainerd.EventType, ei libc "exec-pid": ei.Pid, }).Warn("Ignoring Exit Event, no such exec command found") } - case libcontainerd.EventStart: + case libcontainerdtypes.EventStart: c.Lock() defer c.Unlock() @@ -159,7 +159,7 @@ func (daemon *Daemon) ProcessEvent(id string, e libcontainerd.EventType, ei libc daemon.LogContainerEvent(c, "start") } - case libcontainerd.EventPaused: + case libcontainerdtypes.EventPaused: c.Lock() defer c.Unlock() @@ -172,7 +172,7 @@ func (daemon *Daemon) ProcessEvent(id string, e libcontainerd.EventType, ei libc } daemon.LogContainerEvent(c, "pause") } - case libcontainerd.EventResumed: + case libcontainerdtypes.EventResumed: c.Lock() defer c.Unlock() diff --git a/daemon/oci_windows.go b/daemon/oci_windows.go index 215d5b6d3a..01452d45bb 100644 --- a/daemon/oci_windows.go +++ b/daemon/oci_windows.go @@ -1,6 +1,7 @@ package daemon // import "github.com/docker/docker/daemon" import ( + "encoding/json" "fmt" "io/ioutil" "path/filepath" @@ -15,7 +16,7 @@ import ( "github.com/docker/docker/pkg/system" "github.com/opencontainers/runtime-spec/specs-go" "github.com/pkg/errors" - "golang.org/x/sys/windows" + "github.com/sirupsen/logrus" "golang.org/x/sys/windows/registry" ) @@ -25,6 +26,7 @@ const ( ) func (daemon *Daemon) createSpec(c *container.Container) (*specs.Spec, error) { + img, err := daemon.imageService.GetImage(string(c.ImageID)) if err != nil { return nil, err @@ -219,11 +221,18 @@ func (daemon *Daemon) createSpec(c *container.Container) (*specs.Spec, error) { return nil, fmt.Errorf("Unsupported platform %q", img.OS) } + if logrus.IsLevelEnabled(logrus.DebugLevel) { + if b, err := json.Marshal(&s); err == nil { + logrus.Debugf("Generated spec: %s", string(b)) + } + } + return (*specs.Spec)(&s), nil } // Sets the Windows-specific fields of the OCI spec func (daemon *Daemon) createSpecWindowsFields(c *container.Container, s *specs.Spec, isHyperV bool) error { + if len(s.Process.Cwd) == 0 { // We default to C:\ to workaround the oddity of the case that the // default directory for cmd running as LocalSystem (or @@ -396,31 +405,39 @@ func setResourcesInSpec(c *container.Container, s *specs.Spec, isHyperV bool) { } } } - memoryLimit := uint64(c.HostConfig.Memory) - s.Windows.Resources = &specs.WindowsResources{ - CPU: &specs.WindowsCPUResources{ + + if cpuMaximum != 0 || cpuShares != 0 || cpuCount != 0 { + if s.Windows.Resources == nil { + s.Windows.Resources = &specs.WindowsResources{} + } + s.Windows.Resources.CPU = &specs.WindowsCPUResources{ Maximum: &cpuMaximum, Shares: &cpuShares, Count: &cpuCount, - }, - Memory: &specs.WindowsMemoryResources{ + } + } + + memoryLimit := uint64(c.HostConfig.Memory) + if memoryLimit != 0 { + if s.Windows.Resources == nil { + s.Windows.Resources = &specs.WindowsResources{} + } + s.Windows.Resources.Memory = &specs.WindowsMemoryResources{ Limit: &memoryLimit, - }, - Storage: &specs.WindowsStorageResources{ + } + } + + if c.HostConfig.IOMaximumBandwidth != 0 || c.HostConfig.IOMaximumIOps != 0 { + if s.Windows.Resources == nil { + s.Windows.Resources = &specs.WindowsResources{} + } + s.Windows.Resources.Storage = &specs.WindowsStorageResources{ Bps: &c.HostConfig.IOMaximumBandwidth, Iops: &c.HostConfig.IOMaximumIOps, - }, + } } } -func escapeArgs(args []string) []string { - escapedArgs := make([]string, len(args)) - for i, a := range args { - escapedArgs[i] = windows.EscapeArg(a) - } - return escapedArgs -} - // mergeUlimits merge the Ulimits from HostConfig with daemon defaults, and update HostConfig // It will do nothing on non-Linux platform func (daemon *Daemon) mergeUlimits(c *containertypes.HostConfig) { diff --git a/daemon/resize.go b/daemon/resize.go index 21240650f8..623c73e87d 100644 --- a/daemon/resize.go +++ b/daemon/resize.go @@ -5,7 +5,7 @@ import ( "fmt" "time" - "github.com/docker/docker/libcontainerd" + libcontainerdtypes "github.com/docker/docker/libcontainerd/types" ) // ContainerResize changes the size of the TTY of the process running @@ -20,7 +20,7 @@ func (daemon *Daemon) ContainerResize(name string, height, width int) error { return errNotRunning(container.ID) } - if err = daemon.containerd.ResizeTerminal(context.Background(), container.ID, libcontainerd.InitProcessName, width, height); err == nil { + if err = daemon.containerd.ResizeTerminal(context.Background(), container.ID, libcontainerdtypes.InitProcessName, width, height); err == nil { attributes := map[string]string{ "height": fmt.Sprintf("%d", height), "width": fmt.Sprintf("%d", width), diff --git a/daemon/start_windows.go b/daemon/start_windows.go index f4606f7a60..ded058496e 100644 --- a/daemon/start_windows.go +++ b/daemon/start_windows.go @@ -1,11 +1,23 @@ package daemon // import "github.com/docker/docker/daemon" import ( + "github.com/Microsoft/hcsshim/cmd/containerd-shim-runhcs-v1/options" "github.com/Microsoft/opengcs/client" "github.com/docker/docker/container" + "github.com/docker/docker/pkg/system" ) func (daemon *Daemon) getLibcontainerdCreateOptions(container *container.Container) (interface{}, error) { + + // Set the runtime options to debug regardless of current logging level. + if system.ContainerdRuntimeSupported() { + opts := &options.Options{Debug: true} + return opts, nil + } + + // TODO @jhowardmsft (containerd) - Probably need to revisit LCOW options here + // rather than blindly ignoring them. + // LCOW options. if container.OS == "linux" { config := &client.Config{} diff --git a/daemon/update_linux.go b/daemon/update_linux.go index 7d61ec381c..7dc1f33156 100644 --- a/daemon/update_linux.go +++ b/daemon/update_linux.go @@ -4,12 +4,12 @@ import ( "time" "github.com/docker/docker/api/types/container" - "github.com/docker/docker/libcontainerd" + libcontainerdtypes "github.com/docker/docker/libcontainerd/types" "github.com/opencontainers/runtime-spec/specs-go" ) -func toContainerdResources(resources container.Resources) *libcontainerd.Resources { - var r libcontainerd.Resources +func toContainerdResources(resources container.Resources) *libcontainerdtypes.Resources { + var r libcontainerdtypes.Resources r.BlockIO = &specs.LinuxBlockIO{ Weight: &resources.BlkioWeight, diff --git a/daemon/update_windows.go b/daemon/update_windows.go index fada3c1c0b..cefbe919a5 100644 --- a/daemon/update_windows.go +++ b/daemon/update_windows.go @@ -2,10 +2,10 @@ package daemon // import "github.com/docker/docker/daemon" import ( "github.com/docker/docker/api/types/container" - "github.com/docker/docker/libcontainerd" + libcontainerdtypes "github.com/docker/docker/libcontainerd/types" ) -func toContainerdResources(resources container.Resources) *libcontainerd.Resources { +func toContainerdResources(resources container.Resources) *libcontainerdtypes.Resources { // We don't support update, so do nothing return nil } diff --git a/daemon/util_test.go b/daemon/util_test.go index b2c464f737..46a7337893 100644 --- a/daemon/util_test.go +++ b/daemon/util_test.go @@ -7,7 +7,7 @@ import ( "time" "github.com/containerd/containerd" - "github.com/docker/docker/libcontainerd" + libcontainerdtypes "github.com/docker/docker/libcontainerd/types" specs "github.com/opencontainers/runtime-spec/specs-go" ) @@ -18,19 +18,19 @@ type MockContainerdClient struct { func (c *MockContainerdClient) Version(ctx context.Context) (containerd.Version, error) { return containerd.Version{}, nil } -func (c *MockContainerdClient) Restore(ctx context.Context, containerID string, attachStdio libcontainerd.StdioCallback) (alive bool, pid int, err error) { +func (c *MockContainerdClient) Restore(ctx context.Context, containerID string, attachStdio libcontainerdtypes.StdioCallback) (alive bool, pid int, err error) { return false, 0, nil } func (c *MockContainerdClient) Create(ctx context.Context, containerID string, spec *specs.Spec, runtimeOptions interface{}) error { return nil } -func (c *MockContainerdClient) Start(ctx context.Context, containerID, checkpointDir string, withStdin bool, attachStdio libcontainerd.StdioCallback) (pid int, err error) { +func (c *MockContainerdClient) Start(ctx context.Context, containerID, checkpointDir string, withStdin bool, attachStdio libcontainerdtypes.StdioCallback) (pid int, err error) { return 0, nil } func (c *MockContainerdClient) SignalProcess(ctx context.Context, containerID, processID string, signal int) error { return nil } -func (c *MockContainerdClient) Exec(ctx context.Context, containerID, processID string, spec *specs.Process, withStdin bool, attachStdio libcontainerd.StdioCallback) (int, error) { +func (c *MockContainerdClient) Exec(ctx context.Context, containerID, processID string, spec *specs.Process, withStdin bool, attachStdio libcontainerdtypes.StdioCallback) (int, error) { return 0, nil } func (c *MockContainerdClient) ResizeTerminal(ctx context.Context, containerID, processID string, width, height int) error { @@ -41,23 +41,23 @@ func (c *MockContainerdClient) CloseStdin(ctx context.Context, containerID, proc } func (c *MockContainerdClient) Pause(ctx context.Context, containerID string) error { return nil } func (c *MockContainerdClient) Resume(ctx context.Context, containerID string) error { return nil } -func (c *MockContainerdClient) Stats(ctx context.Context, containerID string) (*libcontainerd.Stats, error) { +func (c *MockContainerdClient) Stats(ctx context.Context, containerID string) (*libcontainerdtypes.Stats, error) { return nil, nil } func (c *MockContainerdClient) ListPids(ctx context.Context, containerID string) ([]uint32, error) { return nil, nil } -func (c *MockContainerdClient) Summary(ctx context.Context, containerID string) ([]libcontainerd.Summary, error) { +func (c *MockContainerdClient) Summary(ctx context.Context, containerID string) ([]libcontainerdtypes.Summary, error) { return nil, nil } func (c *MockContainerdClient) DeleteTask(ctx context.Context, containerID string) (uint32, time.Time, error) { return 0, time.Time{}, nil } func (c *MockContainerdClient) Delete(ctx context.Context, containerID string) error { return nil } -func (c *MockContainerdClient) Status(ctx context.Context, containerID string) (libcontainerd.Status, error) { +func (c *MockContainerdClient) Status(ctx context.Context, containerID string) (libcontainerdtypes.Status, error) { return "null", nil } -func (c *MockContainerdClient) UpdateResources(ctx context.Context, containerID string, resources *libcontainerd.Resources) error { +func (c *MockContainerdClient) UpdateResources(ctx context.Context, containerID string, resources *libcontainerdtypes.Resources) error { return nil } func (c *MockContainerdClient) CreateCheckpoint(ctx context.Context, containerID, checkpointDir string, exit bool) error { diff --git a/libcontainerd/errors.go b/libcontainerd/errors.go deleted file mode 100644 index bdc26715bc..0000000000 --- a/libcontainerd/errors.go +++ /dev/null @@ -1,13 +0,0 @@ -package libcontainerd // import "github.com/docker/docker/libcontainerd" - -import ( - "errors" - - "github.com/docker/docker/errdefs" -) - -func newNotFoundError(err string) error { return errdefs.NotFound(errors.New(err)) } - -func newInvalidParameterError(err string) error { return errdefs.InvalidParameter(errors.New(err)) } - -func newConflictError(err string) error { return errdefs.Conflict(errors.New(err)) } diff --git a/libcontainerd/libcontainerd_linux.go b/libcontainerd/libcontainerd_linux.go new file mode 100644 index 0000000000..ec195a7905 --- /dev/null +++ b/libcontainerd/libcontainerd_linux.go @@ -0,0 +1,14 @@ +package libcontainerd // import "github.com/docker/docker/libcontainerd" + +import ( + "context" + + "github.com/containerd/containerd" + "github.com/docker/docker/libcontainerd/remote" + libcontainerdtypes "github.com/docker/docker/libcontainerd/types" +) + +// NewClient creates a new libcontainerd client from a containerd client +func NewClient(ctx context.Context, cli *containerd.Client, stateDir, ns string, b libcontainerdtypes.Backend) (libcontainerdtypes.Client, error) { + return remote.NewClient(ctx, cli, stateDir, ns, b) +} diff --git a/libcontainerd/libcontainerd_windows.go b/libcontainerd/libcontainerd_windows.go new file mode 100644 index 0000000000..61f19ba087 --- /dev/null +++ b/libcontainerd/libcontainerd_windows.go @@ -0,0 +1,19 @@ +package libcontainerd // import "github.com/docker/docker/libcontainerd" + +import ( + "context" + + "github.com/containerd/containerd" + "github.com/docker/docker/libcontainerd/local" + "github.com/docker/docker/libcontainerd/remote" + libcontainerdtypes "github.com/docker/docker/libcontainerd/types" + "github.com/docker/docker/pkg/system" +) + +// NewClient creates a new libcontainerd client from a containerd client +func NewClient(ctx context.Context, cli *containerd.Client, stateDir, ns string, b libcontainerdtypes.Backend) (libcontainerdtypes.Client, error) { + if !system.ContainerdRuntimeSupported() { + return local.NewClient(ctx, cli, stateDir, ns, b) + } + return remote.NewClient(ctx, cli, stateDir, ns, b) +} diff --git a/libcontainerd/process_windows.go b/libcontainerd/local/process_windows.go similarity index 92% rename from libcontainerd/process_windows.go rename to libcontainerd/local/process_windows.go index 8cdf1daca8..6ff9f7e83e 100644 --- a/libcontainerd/process_windows.go +++ b/libcontainerd/local/process_windows.go @@ -1,4 +1,4 @@ -package libcontainerd // import "github.com/docker/docker/libcontainerd" +package local // import "github.com/docker/docker/libcontainerd/local" import ( "io" diff --git a/libcontainerd/utils_windows.go b/libcontainerd/local/utils_windows.go similarity index 83% rename from libcontainerd/utils_windows.go rename to libcontainerd/local/utils_windows.go index aabb9aeaaa..ac3eca1c49 100644 --- a/libcontainerd/utils_windows.go +++ b/libcontainerd/local/utils_windows.go @@ -1,4 +1,4 @@ -package libcontainerd // import "github.com/docker/docker/libcontainerd" +package local // import "github.com/docker/docker/libcontainerd/local" import ( "strings" @@ -19,6 +19,11 @@ func setupEnvironmentVariables(a []string) map[string]string { return r } +// LCOWOption is a CreateOption required for LCOW configuration +type LCOWOption struct { + Config *opengcs.Config +} + // Apply for the LCOW option is a no-op. func (s *LCOWOption) Apply(interface{}) error { return nil diff --git a/libcontainerd/utils_windows_test.go b/libcontainerd/local/utils_windows_test.go similarity index 81% rename from libcontainerd/utils_windows_test.go rename to libcontainerd/local/utils_windows_test.go index 2e0c260eca..4fc837e326 100644 --- a/libcontainerd/utils_windows_test.go +++ b/libcontainerd/local/utils_windows_test.go @@ -1,4 +1,4 @@ -package libcontainerd // import "github.com/docker/docker/libcontainerd" +package local // import "github.com/docker/docker/libcontainerd/local" import ( "testing" diff --git a/libcontainerd/client_local_windows.go b/libcontainerd/local/windows.go similarity index 88% rename from libcontainerd/client_local_windows.go rename to libcontainerd/local/windows.go index 767cc5d196..f25ec22177 100644 --- a/libcontainerd/client_local_windows.go +++ b/libcontainerd/local/windows.go @@ -1,4 +1,7 @@ -package libcontainerd // import "github.com/docker/docker/libcontainerd" +package local // import "github.com/docker/docker/libcontainerd/local" + +// This package contains the legacy in-proc calls in HCS using the v1 schema +// for Windows runtime purposes. import ( "context" @@ -18,6 +21,10 @@ import ( opengcs "github.com/Microsoft/opengcs/client" "github.com/containerd/containerd" "github.com/containerd/containerd/cio" + + "github.com/docker/docker/errdefs" + "github.com/docker/docker/libcontainerd/queue" + libcontainerdtypes "github.com/docker/docker/libcontainerd/types" "github.com/docker/docker/pkg/sysinfo" "github.com/docker/docker/pkg/system" specs "github.com/opencontainers/runtime-spec/specs-go" @@ -26,8 +33,6 @@ import ( "golang.org/x/sys/windows" ) -const InitProcessName = "init" - type process struct { id string pid int @@ -46,7 +51,7 @@ type container struct { hcsContainer hcsshim.Container id string - status Status + status libcontainerdtypes.Status exitedAt time.Time exitCode uint32 waitCh chan struct{} @@ -74,14 +79,14 @@ type client struct { sync.Mutex stateDir string - backend Backend + backend libcontainerdtypes.Backend logger *logrus.Entry - eventQ queue + eventQ queue.Queue containers map[string]*container } // NewClient creates a new local executor for windows -func NewClient(ctx context.Context, cli *containerd.Client, stateDir, ns string, b Backend) (Client, error) { +func NewClient(ctx context.Context, cli *containerd.Client, stateDir, ns string, b libcontainerdtypes.Backend) (libcontainerdtypes.Client, error) { c := &client{ stateDir: stateDir, backend: b, @@ -149,7 +154,7 @@ func (c *client) Version(ctx context.Context) (containerd.Version, error) { //} func (c *client) Create(_ context.Context, id string, spec *specs.Spec, runtimeOptions interface{}) error { if ctr := c.getContainer(id); ctr != nil { - return errors.WithStack(newConflictError("id already in use")) + return errors.WithStack(errdefs.Conflict(errors.New("id already in use"))) } // spec.Linux must be nil for Windows containers, but spec.Windows @@ -328,7 +333,7 @@ func (c *client) createWindows(id string, spec *specs.Spec, runtimeOptions inter isWindows: true, ociSpec: spec, hcsContainer: hcsContainer, - status: StatusCreated, + status: libcontainerdtypes.StatusCreated, waitCh: make(chan struct{}), } @@ -532,7 +537,7 @@ func (c *client) createLinux(id string, spec *specs.Spec, runtimeOptions interfa isWindows: false, ociSpec: spec, hcsContainer: hcsContainer, - status: StatusCreated, + status: libcontainerdtypes.StatusCreated, waitCh: make(chan struct{}), } @@ -556,19 +561,19 @@ func (c *client) createLinux(id string, spec *specs.Spec, runtimeOptions interfa c.containers[id] = ctr c.Unlock() - c.eventQ.append(id, func() { - ei := EventInfo{ + c.eventQ.Append(id, func() { + ei := libcontainerdtypes.EventInfo{ ContainerID: id, } c.logger.WithFields(logrus.Fields{ "container": ctr.id, - "event": EventCreate, + "event": libcontainerdtypes.EventCreate, }).Info("sending event") - err := c.backend.ProcessEvent(id, EventCreate, ei) + err := c.backend.ProcessEvent(id, libcontainerdtypes.EventCreate, ei) if err != nil { c.logger.WithError(err).WithFields(logrus.Fields{ "container": id, - "event": EventCreate, + "event": libcontainerdtypes.EventCreate, }).Error("failed to process event") } }) @@ -607,13 +612,13 @@ func (c *client) extractResourcesFromSpec(spec *specs.Spec, configuration *hcssh } } -func (c *client) Start(_ context.Context, id, _ string, withStdin bool, attachStdio StdioCallback) (int, error) { +func (c *client) Start(_ context.Context, id, _ string, withStdin bool, attachStdio libcontainerdtypes.StdioCallback) (int, error) { ctr := c.getContainer(id) switch { case ctr == nil: - return -1, errors.WithStack(newNotFoundError("no such container")) + return -1, errors.WithStack(errdefs.NotFound(errors.New("no such container"))) case ctr.init != nil: - return -1, errors.WithStack(newConflictError("container already started")) + return -1, errors.WithStack(errdefs.Conflict(errors.New("container already started"))) } logger := c.logger.WithField("container", id) @@ -691,7 +696,7 @@ func (c *client) Start(_ context.Context, id, _ string, withStdin bool, attachSt }() p := &process{ hcsProcess: newProcess, - id: InitProcessName, + id: libcontainerdtypes.InitProcessName, pid: newProcess.Pid(), } logger.WithField("pid", p.pid).Debug("init process started") @@ -706,29 +711,29 @@ func (c *client) Start(_ context.Context, id, _ string, withStdin bool, attachSt logger.WithError(err).Error("failed to attache stdio") return -1, err } - ctr.status = StatusRunning + ctr.status = libcontainerdtypes.StatusRunning ctr.init = p // Spin up a go routine waiting for exit to handle cleanup go c.reapProcess(ctr, p) // Generate the associated event - c.eventQ.append(id, func() { - ei := EventInfo{ + c.eventQ.Append(id, func() { + ei := libcontainerdtypes.EventInfo{ ContainerID: id, - ProcessID: InitProcessName, + ProcessID: libcontainerdtypes.InitProcessName, Pid: uint32(p.pid), } c.logger.WithFields(logrus.Fields{ "container": ctr.id, - "event": EventStart, + "event": libcontainerdtypes.EventStart, "event-info": ei, }).Info("sending event") - err := c.backend.ProcessEvent(ei.ContainerID, EventStart, ei) + err := c.backend.ProcessEvent(ei.ContainerID, libcontainerdtypes.EventStart, ei) if err != nil { c.logger.WithError(err).WithFields(logrus.Fields{ "container": id, - "event": EventStart, + "event": libcontainerdtypes.EventStart, "event-info": ei, }).Error("failed to process event") } @@ -756,15 +761,15 @@ func newIOFromProcess(newProcess hcsshim.Process, terminal bool) (*cio.DirectIO, } // Exec adds a process in an running container -func (c *client) Exec(ctx context.Context, containerID, processID string, spec *specs.Process, withStdin bool, attachStdio StdioCallback) (int, error) { +func (c *client) Exec(ctx context.Context, containerID, processID string, spec *specs.Process, withStdin bool, attachStdio libcontainerdtypes.StdioCallback) (int, error) { ctr := c.getContainer(containerID) switch { case ctr == nil: - return -1, errors.WithStack(newNotFoundError("no such container")) + return -1, errors.WithStack(errdefs.NotFound(errors.New("no such container"))) case ctr.hcsContainer == nil: - return -1, errors.WithStack(newInvalidParameterError("container is not running")) + return -1, errors.WithStack(errdefs.InvalidParameter(errors.New("container is not running"))) case ctr.execs != nil && ctr.execs[processID] != nil: - return -1, errors.WithStack(newConflictError("id already in use")) + return -1, errors.WithStack(errdefs.Conflict(errors.New("id already in use"))) } logger := c.logger.WithFields(logrus.Fields{ "container": containerID, @@ -856,30 +861,30 @@ func (c *client) Exec(ctx context.Context, containerID, processID string, spec * // Spin up a go routine waiting for exit to handle cleanup go c.reapProcess(ctr, p) - c.eventQ.append(ctr.id, func() { - ei := EventInfo{ + c.eventQ.Append(ctr.id, func() { + ei := libcontainerdtypes.EventInfo{ ContainerID: ctr.id, ProcessID: p.id, Pid: uint32(p.pid), } c.logger.WithFields(logrus.Fields{ "container": ctr.id, - "event": EventExecAdded, + "event": libcontainerdtypes.EventExecAdded, "event-info": ei, }).Info("sending event") - err := c.backend.ProcessEvent(ctr.id, EventExecAdded, ei) + err := c.backend.ProcessEvent(ctr.id, libcontainerdtypes.EventExecAdded, ei) if err != nil { c.logger.WithError(err).WithFields(logrus.Fields{ "container": ctr.id, - "event": EventExecAdded, + "event": libcontainerdtypes.EventExecAdded, "event-info": ei, }).Error("failed to process event") } - err = c.backend.ProcessEvent(ctr.id, EventExecStarted, ei) + err = c.backend.ProcessEvent(ctr.id, libcontainerdtypes.EventExecStarted, ei) if err != nil { c.logger.WithError(err).WithFields(logrus.Fields{ "container": ctr.id, - "event": EventExecStarted, + "event": libcontainerdtypes.EventExecStarted, "event-info": ei, }).Error("failed to process event") } @@ -905,7 +910,7 @@ func (c *client) SignalProcess(_ context.Context, containerID, processID string, }) logger.Debug("Signal()") - if processID == InitProcessName { + if processID == libcontainerdtypes.InitProcessName { if syscall.Signal(signal) == syscall.SIGKILL { // Terminate the compute system ctr.Lock() @@ -961,7 +966,7 @@ func (c *client) CloseStdin(_ context.Context, containerID, processID string) er // Pause handles pause requests for containers func (c *client) Pause(_ context.Context, containerID string) error { - ctr, _, err := c.getProcess(containerID, InitProcessName) + ctr, _, err := c.getProcess(containerID, libcontainerdtypes.InitProcessName) if err != nil { return err } @@ -977,21 +982,21 @@ func (c *client) Pause(_ context.Context, containerID string) error { return err } - ctr.status = StatusPaused + ctr.status = libcontainerdtypes.StatusPaused - c.eventQ.append(containerID, func() { - err := c.backend.ProcessEvent(containerID, EventPaused, EventInfo{ + c.eventQ.Append(containerID, func() { + err := c.backend.ProcessEvent(containerID, libcontainerdtypes.EventPaused, libcontainerdtypes.EventInfo{ ContainerID: containerID, - ProcessID: InitProcessName, + ProcessID: libcontainerdtypes.InitProcessName, }) c.logger.WithFields(logrus.Fields{ "container": ctr.id, - "event": EventPaused, + "event": libcontainerdtypes.EventPaused, }).Info("sending event") if err != nil { c.logger.WithError(err).WithFields(logrus.Fields{ "container": containerID, - "event": EventPaused, + "event": libcontainerdtypes.EventPaused, }).Error("failed to process event") } }) @@ -1001,7 +1006,7 @@ func (c *client) Pause(_ context.Context, containerID string) error { // Resume handles resume requests for containers func (c *client) Resume(_ context.Context, containerID string) error { - ctr, _, err := c.getProcess(containerID, InitProcessName) + ctr, _, err := c.getProcess(containerID, libcontainerdtypes.InitProcessName) if err != nil { return err } @@ -1017,21 +1022,21 @@ func (c *client) Resume(_ context.Context, containerID string) error { return err } - ctr.status = StatusRunning + ctr.status = libcontainerdtypes.StatusRunning - c.eventQ.append(containerID, func() { - err := c.backend.ProcessEvent(containerID, EventResumed, EventInfo{ + c.eventQ.Append(containerID, func() { + err := c.backend.ProcessEvent(containerID, libcontainerdtypes.EventResumed, libcontainerdtypes.EventInfo{ ContainerID: containerID, - ProcessID: InitProcessName, + ProcessID: libcontainerdtypes.InitProcessName, }) c.logger.WithFields(logrus.Fields{ "container": ctr.id, - "event": EventResumed, + "event": libcontainerdtypes.EventResumed, }).Info("sending event") if err != nil { c.logger.WithError(err).WithFields(logrus.Fields{ "container": containerID, - "event": EventResumed, + "event": libcontainerdtypes.EventResumed, }).Error("failed to process event") } }) @@ -1040,8 +1045,8 @@ func (c *client) Resume(_ context.Context, containerID string) error { } // Stats handles stats requests for containers -func (c *client) Stats(_ context.Context, containerID string) (*Stats, error) { - ctr, _, err := c.getProcess(containerID, InitProcessName) +func (c *client) Stats(_ context.Context, containerID string) (*libcontainerdtypes.Stats, error) { + ctr, _, err := c.getProcess(containerID, libcontainerdtypes.InitProcessName) if err != nil { return nil, err } @@ -1051,14 +1056,14 @@ func (c *client) Stats(_ context.Context, containerID string) (*Stats, error) { if err != nil { return nil, err } - return &Stats{ + return &libcontainerdtypes.Stats{ Read: readAt, HCSStats: &s, }, nil } // Restore is the handler for restoring a container -func (c *client) Restore(ctx context.Context, id string, attachStdio StdioCallback) (bool, int, error) { +func (c *client) Restore(ctx context.Context, id string, attachStdio libcontainerdtypes.StdioCallback) (bool, int, error) { c.logger.WithField("container", id).Debug("restore()") // TODO Windows: On RS1, a re-attach isn't possible. @@ -1098,8 +1103,8 @@ func (c *client) ListPids(_ context.Context, _ string) ([]uint32, error) { // the containers could be Hyper-V containers, they would not be // visible on the container host. However, libcontainerd does have // that information. -func (c *client) Summary(_ context.Context, containerID string) ([]Summary, error) { - ctr, _, err := c.getProcess(containerID, InitProcessName) +func (c *client) Summary(_ context.Context, containerID string) ([]libcontainerdtypes.Summary, error) { + ctr, _, err := c.getProcess(containerID, libcontainerdtypes.InitProcessName) if err != nil { return nil, err } @@ -1109,9 +1114,9 @@ func (c *client) Summary(_ context.Context, containerID string) ([]Summary, erro return nil, err } - pl := make([]Summary, len(p)) + pl := make([]libcontainerdtypes.Summary, len(p)) for i := range p { - pl[i] = Summary(p[i]) + pl[i] = libcontainerdtypes.Summary(p[i]) } return pl, nil } @@ -1120,7 +1125,7 @@ func (c *client) DeleteTask(ctx context.Context, containerID string) (uint32, ti ec := -1 ctr := c.getContainer(containerID) if ctr == nil { - return uint32(ec), time.Now(), errors.WithStack(newNotFoundError("no such container")) + return uint32(ec), time.Now(), errors.WithStack(errdefs.NotFound(errors.New("no such container"))) } select { @@ -1141,32 +1146,32 @@ func (c *client) Delete(_ context.Context, containerID string) error { defer c.Unlock() ctr := c.containers[containerID] if ctr == nil { - return errors.WithStack(newNotFoundError("no such container")) + return errors.WithStack(errdefs.NotFound(errors.New("no such container"))) } ctr.Lock() defer ctr.Unlock() switch ctr.status { - case StatusCreated: + case libcontainerdtypes.StatusCreated: if err := c.shutdownContainer(ctr); err != nil { return err } fallthrough - case StatusStopped: + case libcontainerdtypes.StatusStopped: delete(c.containers, containerID) return nil } - return errors.WithStack(newInvalidParameterError("container is not stopped")) + return errors.WithStack(errdefs.InvalidParameter(errors.New("container is not stopped"))) } -func (c *client) Status(ctx context.Context, containerID string) (Status, error) { +func (c *client) Status(ctx context.Context, containerID string) (libcontainerdtypes.Status, error) { c.Lock() defer c.Unlock() ctr := c.containers[containerID] if ctr == nil { - return StatusUnknown, errors.WithStack(newNotFoundError("no such container")) + return libcontainerdtypes.StatusUnknown, errors.WithStack(errdefs.NotFound(errors.New("no such container"))) } ctr.Lock() @@ -1174,7 +1179,7 @@ func (c *client) Status(ctx context.Context, containerID string) (Status, error) return ctr.status, nil } -func (c *client) UpdateResources(ctx context.Context, containerID string, resources *Resources) error { +func (c *client) UpdateResources(ctx context.Context, containerID string, resources *libcontainerdtypes.Resources) error { // Updating resource isn't supported on Windows // but we should return nil for enabling updating container return nil @@ -1196,22 +1201,22 @@ func (c *client) getProcess(containerID, processID string) (*container, *process ctr := c.getContainer(containerID) switch { case ctr == nil: - return nil, nil, errors.WithStack(newNotFoundError("no such container")) + return nil, nil, errors.WithStack(errdefs.NotFound(errors.New("no such container"))) case ctr.init == nil: - return nil, nil, errors.WithStack(newNotFoundError("container is not running")) - case processID == InitProcessName: + return nil, nil, errors.WithStack(errdefs.NotFound(errors.New("container is not running"))) + case processID == libcontainerdtypes.InitProcessName: return ctr, ctr.init, nil default: ctr.Lock() defer ctr.Unlock() if ctr.execs == nil { - return nil, nil, errors.WithStack(newNotFoundError("no execs")) + return nil, nil, errors.WithStack(errdefs.NotFound(errors.New("no execs"))) } } p := ctr.execs[processID] if p == nil { - return nil, nil, errors.WithStack(newNotFoundError("no such exec")) + return nil, nil, errors.WithStack(errdefs.NotFound(errors.New("no such exec"))) } return ctr, p, nil @@ -1309,10 +1314,10 @@ func (c *client) reapProcess(ctr *container, p *process) int { eventErr = fmt.Errorf("hcsProcess.Close() failed %s", err) } - if p.id == InitProcessName { + if p.id == libcontainerdtypes.InitProcessName { // Update container status ctr.Lock() - ctr.status = StatusStopped + ctr.status = libcontainerdtypes.StatusStopped ctr.exitedAt = exitedAt ctr.exitCode = uint32(exitCode) close(ctr.waitCh) @@ -1343,8 +1348,8 @@ func (c *client) reapProcess(ctr *container, p *process) int { } } - c.eventQ.append(ctr.id, func() { - ei := EventInfo{ + c.eventQ.Append(ctr.id, func() { + ei := libcontainerdtypes.EventInfo{ ContainerID: ctr.id, ProcessID: p.id, Pid: uint32(p.pid), @@ -1354,18 +1359,18 @@ func (c *client) reapProcess(ctr *container, p *process) int { } c.logger.WithFields(logrus.Fields{ "container": ctr.id, - "event": EventExit, + "event": libcontainerdtypes.EventExit, "event-info": ei, }).Info("sending event") - err := c.backend.ProcessEvent(ctr.id, EventExit, ei) + err := c.backend.ProcessEvent(ctr.id, libcontainerdtypes.EventExit, ei) if err != nil { c.logger.WithError(err).WithFields(logrus.Fields{ "container": ctr.id, - "event": EventExit, + "event": libcontainerdtypes.EventExit, "event-info": ei, }).Error("failed to process event") } - if p.id != InitProcessName { + if p.id != libcontainerdtypes.InitProcessName { ctr.Lock() delete(ctr.execs, p.id) ctr.Unlock() diff --git a/libcontainerd/queue.go b/libcontainerd/queue/queue.go similarity index 60% rename from libcontainerd/queue.go rename to libcontainerd/queue/queue.go index 207722c441..71ff0c79cd 100644 --- a/libcontainerd/queue.go +++ b/libcontainerd/queue/queue.go @@ -1,13 +1,15 @@ -package libcontainerd // import "github.com/docker/docker/libcontainerd" +package queue // import "github.com/docker/docker/libcontainerd/queue" import "sync" -type queue struct { +// Queue is the structure used for holding functions in a queue. +type Queue struct { sync.Mutex fns map[string]chan struct{} } -func (q *queue) append(id string, f func()) { +// Append adds an item to a queue. +func (q *Queue) Append(id string, f func()) { q.Lock() defer q.Unlock() diff --git a/libcontainerd/queue_test.go b/libcontainerd/queue/queue_test.go similarity index 70% rename from libcontainerd/queue_test.go rename to libcontainerd/queue/queue_test.go index e13afca89a..e0163f3626 100644 --- a/libcontainerd/queue_test.go +++ b/libcontainerd/queue/queue_test.go @@ -1,4 +1,4 @@ -package libcontainerd // import "github.com/docker/docker/libcontainerd" +package queue // import "github.com/docker/docker/libcontainerd/queue" import ( "testing" @@ -9,21 +9,21 @@ import ( func TestSerialization(t *testing.T) { var ( - q queue + q Queue serialization = 1 ) - q.append("aaa", func() { + q.Append("aaa", func() { //simulate a long time task time.Sleep(10 * time.Millisecond) assert.Equal(t, serialization, 1) serialization = 2 }) - q.append("aaa", func() { + q.Append("aaa", func() { assert.Equal(t, serialization, 2) serialization = 3 }) - q.append("aaa", func() { + q.Append("aaa", func() { assert.Equal(t, serialization, 3) serialization = 4 }) diff --git a/libcontainerd/client_daemon.go b/libcontainerd/remote/client.go similarity index 81% rename from libcontainerd/client_daemon.go rename to libcontainerd/remote/client.go index d59907c570..07999b8ab0 100644 --- a/libcontainerd/client_daemon.go +++ b/libcontainerd/remote/client.go @@ -1,11 +1,8 @@ -// +build !windows - -package libcontainerd // import "github.com/docker/docker/libcontainerd" +package remote // import "github.com/docker/docker/libcontainerd/remote" import ( "context" "encoding/json" - "fmt" "io" "os" "path/filepath" @@ -16,6 +13,7 @@ import ( "syscall" "time" + "github.com/Microsoft/hcsshim/cmd/containerd-shim-runhcs-v1/options" "github.com/containerd/containerd" apievents "github.com/containerd/containerd/api/events" "github.com/containerd/containerd/api/types" @@ -28,6 +26,9 @@ import ( "github.com/containerd/containerd/runtime/linux/runctypes" "github.com/containerd/typeurl" "github.com/docker/docker/errdefs" + "github.com/docker/docker/libcontainerd/queue" + libcontainerdtypes "github.com/docker/docker/libcontainerd/types" + "github.com/docker/docker/pkg/ioutils" v1 "github.com/opencontainers/image-spec/specs-go/v1" specs "github.com/opencontainers/runtime-spec/specs-go" @@ -37,10 +38,6 @@ import ( "google.golang.org/grpc/status" ) -// InitProcessName is the name given to the first process of a -// container -const InitProcessName = "init" - type container struct { mu sync.Mutex @@ -107,13 +104,13 @@ type client struct { logger *logrus.Entry ns string - backend Backend - eventQ queue + backend libcontainerdtypes.Backend + eventQ queue.Queue containers map[string]*container } // NewClient creates a new libcontainerd client from a containerd client -func NewClient(ctx context.Context, cli *containerd.Client, stateDir, ns string, b Backend) (Client, error) { +func NewClient(ctx context.Context, cli *containerd.Client, stateDir, ns string, b libcontainerdtypes.Backend) (libcontainerdtypes.Client, error) { c := &client{ client: cli, stateDir: stateDir, @@ -134,12 +131,12 @@ func (c *client) Version(ctx context.Context) (containerd.Version, error) { // Restore loads the containerd container. // It should not be called concurrently with any other operation for the given ID. -func (c *client) Restore(ctx context.Context, id string, attachStdio StdioCallback) (alive bool, pid int, err error) { +func (c *client) Restore(ctx context.Context, id string, attachStdio libcontainerdtypes.StdioCallback) (alive bool, pid int, err error) { c.Lock() _, ok := c.containers[id] if ok { c.Unlock() - return false, 0, errors.WithStack(newConflictError("id already in use")) + return false, 0, errors.WithStack(errdefs.Conflict(errors.New("id already in use"))) } cntr := &container{} @@ -174,7 +171,8 @@ func (c *client) Restore(ctx context.Context, id string, attachStdio StdioCallba attachIO := func(fifos *cio.FIFOSet) (cio.IO, error) { // dio must be assigned to the previously defined dio for the defer above // to handle cleanup - dio, err = cio.NewDirectIO(ctx, fifos) + + dio, err = c.newDirectIO(ctx, fifos) if err != nil { return nil, err } @@ -211,7 +209,7 @@ func (c *client) Restore(ctx context.Context, id string, attachStdio StdioCallba func (c *client) Create(ctx context.Context, id string, ociSpec *specs.Spec, runtimeOptions interface{}) error { if ctr := c.getContainer(id); ctr != nil { - return errors.WithStack(newConflictError("id already in use")) + return errors.WithStack(errdefs.Conflict(errors.New("id already in use"))) } bdir, err := prepareBundleDir(filepath.Join(c.stateDir, id), ociSpec) @@ -223,8 +221,7 @@ func (c *client) Create(ctx context.Context, id string, ociSpec *specs.Spec, run cdCtr, err := c.client.NewContainer(ctx, id, containerd.WithSpec(ociSpec), - // TODO(mlaventure): when containerd support lcow, revisit runtime value - containerd.WithRuntime(fmt.Sprintf("io.containerd.runtime.v1.%s", runtime.GOOS), runtimeOptions)) + containerd.WithRuntime(runtimeName, runtimeOptions)) if err != nil { return wrapError(err) } @@ -240,13 +237,13 @@ func (c *client) Create(ctx context.Context, id string, ociSpec *specs.Spec, run } // Start create and start a task for the specified containerd id -func (c *client) Start(ctx context.Context, id, checkpointDir string, withStdin bool, attachStdio StdioCallback) (int, error) { +func (c *client) Start(ctx context.Context, id, checkpointDir string, withStdin bool, attachStdio libcontainerdtypes.StdioCallback) (int, error) { ctr := c.getContainer(id) if ctr == nil { - return -1, errors.WithStack(newNotFoundError("no such container")) + return -1, errors.WithStack(errdefs.NotFound(errors.New("no such container"))) } if t := ctr.getTask(); t != nil { - return -1, errors.WithStack(newConflictError("container already started")) + return -1, errors.WithStack(errdefs.Conflict(errors.New("container already started"))) } var ( @@ -288,17 +285,24 @@ func (c *client) Start(ctx context.Context, id, checkpointDir string, withStdin uid, gid := getSpecUser(spec) t, err = ctr.ctr.NewTask(ctx, func(id string) (cio.IO, error) { - fifos := newFIFOSet(ctr.bundleDir, InitProcessName, withStdin, spec.Process.Terminal) + fifos := newFIFOSet(ctr.bundleDir, libcontainerdtypes.InitProcessName, withStdin, spec.Process.Terminal) - rio, err = c.createIO(fifos, id, InitProcessName, stdinCloseSync, attachStdio) + rio, err = c.createIO(fifos, id, libcontainerdtypes.InitProcessName, stdinCloseSync, attachStdio) return rio, err }, func(_ context.Context, _ *containerd.Client, info *containerd.TaskInfo) error { info.Checkpoint = cp - info.Options = &runctypes.CreateOptions{ - IoUid: uint32(uid), - IoGid: uint32(gid), - NoPivotRoot: os.Getenv("DOCKER_RAMDISK") != "", + if runtime.GOOS != "windows" { + info.Options = &runctypes.CreateOptions{ + IoUid: uint32(uid), + IoGid: uint32(gid), + NoPivotRoot: os.Getenv("DOCKER_RAMDISK") != "", + } + } else { + // Make sure we set the runhcs options to debug if we are at debug level. + if c.logger.Level == logrus.DebugLevel { + info.Options = &options.Options{Debug: true} + } } return nil }) @@ -335,18 +339,18 @@ func (c *client) Start(ctx context.Context, id, checkpointDir string, withStdin // for the container main process, the stdin fifo will be created in Create not // the Start call. stdinCloseSync channel should be closed after Start exec // process. -func (c *client) Exec(ctx context.Context, containerID, processID string, spec *specs.Process, withStdin bool, attachStdio StdioCallback) (int, error) { +func (c *client) Exec(ctx context.Context, containerID, processID string, spec *specs.Process, withStdin bool, attachStdio libcontainerdtypes.StdioCallback) (int, error) { ctr := c.getContainer(containerID) if ctr == nil { - return -1, errors.WithStack(newNotFoundError("no such container")) + return -1, errors.WithStack(errdefs.NotFound(errors.New("no such container"))) } t := ctr.getTask() if t == nil { - return -1, errors.WithStack(newInvalidParameterError("container is not running")) + return -1, errors.WithStack(errdefs.InvalidParameter(errors.New("container is not running"))) } if p := ctr.getProcess(processID); p != nil { - return -1, errors.WithStack(newConflictError("id already in use")) + return -1, errors.WithStack(errdefs.Conflict(errors.New("id already in use"))) } var ( @@ -424,7 +428,7 @@ func (c *client) CloseStdin(ctx context.Context, containerID, processID string) } func (c *client) Pause(ctx context.Context, containerID string) error { - p, err := c.getProcess(containerID, InitProcessName) + p, err := c.getProcess(containerID, libcontainerdtypes.InitProcessName) if err != nil { return err } @@ -433,7 +437,7 @@ func (c *client) Pause(ctx context.Context, containerID string) error { } func (c *client) Resume(ctx context.Context, containerID string) error { - p, err := c.getProcess(containerID, InitProcessName) + p, err := c.getProcess(containerID, libcontainerdtypes.InitProcessName) if err != nil { return err } @@ -441,8 +445,8 @@ func (c *client) Resume(ctx context.Context, containerID string) error { return p.(containerd.Task).Resume(ctx) } -func (c *client) Stats(ctx context.Context, containerID string) (*Stats, error) { - p, err := c.getProcess(containerID, InitProcessName) +func (c *client) Stats(ctx context.Context, containerID string) (*libcontainerdtypes.Stats, error) { + p, err := c.getProcess(containerID, libcontainerdtypes.InitProcessName) if err != nil { return nil, err } @@ -456,11 +460,11 @@ func (c *client) Stats(ctx context.Context, containerID string) (*Stats, error) if err != nil { return nil, err } - return interfaceToStats(m.Timestamp, v), nil + return libcontainerdtypes.InterfaceToStats(m.Timestamp, v), nil } func (c *client) ListPids(ctx context.Context, containerID string) ([]uint32, error) { - p, err := c.getProcess(containerID, InitProcessName) + p, err := c.getProcess(containerID, libcontainerdtypes.InitProcessName) if err != nil { return nil, err } @@ -478,8 +482,8 @@ func (c *client) ListPids(ctx context.Context, containerID string) ([]uint32, er return pids, nil } -func (c *client) Summary(ctx context.Context, containerID string) ([]Summary, error) { - p, err := c.getProcess(containerID, InitProcessName) +func (c *client) Summary(ctx context.Context, containerID string) ([]libcontainerdtypes.Summary, error) { + p, err := c.getProcess(containerID, libcontainerdtypes.InitProcessName) if err != nil { return nil, err } @@ -489,7 +493,7 @@ func (c *client) Summary(ctx context.Context, containerID string) ([]Summary, er return nil, err } - var infos []Summary + var infos []libcontainerdtypes.Summary for _, pi := range pis { i, err := typeurl.UnmarshalAny(pi.Info) if err != nil { @@ -506,7 +510,7 @@ func (c *client) Summary(ctx context.Context, containerID string) ([]Summary, er } func (c *client) DeleteTask(ctx context.Context, containerID string) (uint32, time.Time, error) { - p, err := c.getProcess(containerID, InitProcessName) + p, err := c.getProcess(containerID, libcontainerdtypes.InitProcessName) if err != nil { return 255, time.Now(), nil } @@ -525,7 +529,7 @@ func (c *client) DeleteTask(ctx context.Context, containerID string) (uint32, ti func (c *client) Delete(ctx context.Context, containerID string) error { ctr := c.getContainer(containerID) if ctr == nil { - return errors.WithStack(newNotFoundError("no such container")) + return errors.WithStack(errdefs.NotFound(errors.New("no such container"))) } if err := ctr.ctr.Delete(ctx); err != nil { @@ -546,27 +550,27 @@ func (c *client) Delete(ctx context.Context, containerID string) error { return nil } -func (c *client) Status(ctx context.Context, containerID string) (Status, error) { +func (c *client) Status(ctx context.Context, containerID string) (libcontainerdtypes.Status, error) { ctr := c.getContainer(containerID) if ctr == nil { - return StatusUnknown, errors.WithStack(newNotFoundError("no such container")) + return libcontainerdtypes.StatusUnknown, errors.WithStack(errdefs.NotFound(errors.New("no such container"))) } t := ctr.getTask() if t == nil { - return StatusUnknown, errors.WithStack(newNotFoundError("no such task")) + return libcontainerdtypes.StatusUnknown, errors.WithStack(errdefs.NotFound(errors.New("no such task"))) } s, err := t.Status(ctx) if err != nil { - return StatusUnknown, wrapError(err) + return libcontainerdtypes.StatusUnknown, wrapError(err) } - return Status(s.Status), nil + return libcontainerdtypes.Status(s.Status), nil } func (c *client) CreateCheckpoint(ctx context.Context, containerID, checkpointDir string, exit bool) error { - p, err := c.getProcess(containerID, InitProcessName) + p, err := c.getProcess(containerID, libcontainerdtypes.InitProcessName) if err != nil { return err } @@ -648,33 +652,32 @@ func (c *client) removeContainer(id string) { func (c *client) getProcess(containerID, processID string) (containerd.Process, error) { ctr := c.getContainer(containerID) if ctr == nil { - return nil, errors.WithStack(newNotFoundError("no such container")) + return nil, errors.WithStack(errdefs.NotFound(errors.New("no such container"))) } t := ctr.getTask() if t == nil { - return nil, errors.WithStack(newNotFoundError("container is not running")) + return nil, errors.WithStack(errdefs.NotFound(errors.New("container is not running"))) } - if processID == InitProcessName { + if processID == libcontainerdtypes.InitProcessName { return t, nil } p := ctr.getProcess(processID) if p == nil { - return nil, errors.WithStack(newNotFoundError("no such exec")) + return nil, errors.WithStack(errdefs.NotFound(errors.New("no such exec"))) } return p, nil } // createIO creates the io to be used by a process // This needs to get a pointer to interface as upon closure the process may not have yet been registered -func (c *client) createIO(fifos *cio.FIFOSet, containerID, processID string, stdinCloseSync chan struct{}, attachStdio StdioCallback) (cio.IO, error) { +func (c *client) createIO(fifos *cio.FIFOSet, containerID, processID string, stdinCloseSync chan struct{}, attachStdio libcontainerdtypes.StdioCallback) (cio.IO, error) { var ( io *cio.DirectIO err error ) - - io, err = cio.NewDirectIO(context.Background(), fifos) + io, err = c.newDirectIO(context.Background(), fifos) if err != nil { return nil, err } @@ -713,8 +716,8 @@ func (c *client) createIO(fifos *cio.FIFOSet, containerID, processID string, std return rio, err } -func (c *client) processEvent(ctr *container, et EventType, ei EventInfo) { - c.eventQ.append(ei.ContainerID, func() { +func (c *client) processEvent(ctr *container, et libcontainerdtypes.EventType, ei libcontainerdtypes.EventInfo) { + c.eventQ.Append(ei.ContainerID, func() { err := c.backend.ProcessEvent(ei.ContainerID, et, ei) if err != nil { c.logger.WithError(err).WithFields(logrus.Fields{ @@ -724,7 +727,7 @@ func (c *client) processEvent(ctr *container, et EventType, ei EventInfo) { }).Error("failed to process event") } - if et == EventExit && ei.ProcessID != ei.ContainerID { + if et == libcontainerdtypes.EventExit && ei.ProcessID != ei.ContainerID { p := ctr.getProcess(ei.ProcessID) if p == nil { c.logger.WithError(errors.New("no such process")). @@ -759,8 +762,8 @@ func (c *client) processEventStream(ctx context.Context, ns string) { var ( err error ev *events.Envelope - et EventType - ei EventInfo + et libcontainerdtypes.EventType + ei libcontainerdtypes.EventInfo ctr *container ) @@ -800,22 +803,22 @@ func (c *client) processEventStream(ctx context.Context, ns string) { switch t := v.(type) { case *apievents.TaskCreate: - et = EventCreate - ei = EventInfo{ + et = libcontainerdtypes.EventCreate + ei = libcontainerdtypes.EventInfo{ ContainerID: t.ContainerID, ProcessID: t.ContainerID, Pid: t.Pid, } case *apievents.TaskStart: - et = EventStart - ei = EventInfo{ + et = libcontainerdtypes.EventStart + ei = libcontainerdtypes.EventInfo{ ContainerID: t.ContainerID, ProcessID: t.ContainerID, Pid: t.Pid, } case *apievents.TaskExit: - et = EventExit - ei = EventInfo{ + et = libcontainerdtypes.EventExit + ei = libcontainerdtypes.EventInfo{ ContainerID: t.ContainerID, ProcessID: t.ID, Pid: t.Pid, @@ -823,33 +826,33 @@ func (c *client) processEventStream(ctx context.Context, ns string) { ExitedAt: t.ExitedAt, } case *apievents.TaskOOM: - et = EventOOM - ei = EventInfo{ + et = libcontainerdtypes.EventOOM + ei = libcontainerdtypes.EventInfo{ ContainerID: t.ContainerID, OOMKilled: true, } oomKilled = true case *apievents.TaskExecAdded: - et = EventExecAdded - ei = EventInfo{ + et = libcontainerdtypes.EventExecAdded + ei = libcontainerdtypes.EventInfo{ ContainerID: t.ContainerID, ProcessID: t.ExecID, } case *apievents.TaskExecStarted: - et = EventExecStarted - ei = EventInfo{ + et = libcontainerdtypes.EventExecStarted + ei = libcontainerdtypes.EventInfo{ ContainerID: t.ContainerID, ProcessID: t.ExecID, Pid: t.Pid, } case *apievents.TaskPaused: - et = EventPaused - ei = EventInfo{ + et = libcontainerdtypes.EventPaused + ei = libcontainerdtypes.EventInfo{ ContainerID: t.ContainerID, } case *apievents.TaskResumed: - et = EventResumed - ei = EventInfo{ + et = libcontainerdtypes.EventResumed + ei = libcontainerdtypes.EventInfo{ ContainerID: t.ContainerID, } default: diff --git a/libcontainerd/remote/client_io_windows.go b/libcontainerd/remote/client_io_windows.go new file mode 100644 index 0000000000..b437fb6898 --- /dev/null +++ b/libcontainerd/remote/client_io_windows.go @@ -0,0 +1,161 @@ +package remote // import "github.com/docker/docker/libcontainerd/remote" + +import ( + "io" + "net" + "sync" + + winio "github.com/Microsoft/go-winio" + "github.com/containerd/containerd/cio" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + // "golang.org/x/net/context" +) + +type delayedConnection struct { + l net.Listener + con net.Conn + wg sync.WaitGroup + once sync.Once +} + +func (dc *delayedConnection) Write(p []byte) (int, error) { + dc.wg.Wait() + if dc.con != nil { + return dc.con.Write(p) + } + return 0, errors.New("use of closed network connection") +} + +func (dc *delayedConnection) Read(p []byte) (int, error) { + dc.wg.Wait() + if dc.con != nil { + return dc.con.Read(p) + } + return 0, errors.New("use of closed network connection") +} + +func (dc *delayedConnection) unblockConnectionWaiters() { + defer dc.once.Do(func() { + dc.wg.Done() + }) +} + +func (dc *delayedConnection) Close() error { + dc.l.Close() + if dc.con != nil { + return dc.con.Close() + } + dc.unblockConnectionWaiters() + return nil +} + +type stdioPipes struct { + stdin io.WriteCloser + stdout io.ReadCloser + stderr io.ReadCloser +} + +// newStdioPipes creates actual fifos for stdio. +func (c *client) newStdioPipes(fifos *cio.FIFOSet) (_ *stdioPipes, err error) { + p := &stdioPipes{} + if fifos.Stdin != "" { + c.logger.WithFields(logrus.Fields{"stdin": fifos.Stdin}).Debug("listen") + l, err := winio.ListenPipe(fifos.Stdin, nil) + if err != nil { + return nil, errors.Wrapf(err, "failed to create stdin pipe %s", fifos.Stdin) + } + dc := &delayedConnection{ + l: l, + } + dc.wg.Add(1) + defer func() { + if err != nil { + dc.Close() + } + }() + p.stdin = dc + + go func() { + c.logger.WithFields(logrus.Fields{"stdin": fifos.Stdin}).Debug("accept") + conn, err := l.Accept() + if err != nil { + dc.Close() + if err != winio.ErrPipeListenerClosed { + c.logger.WithError(err).Errorf("failed to accept stdin connection on %s", fifos.Stdin) + } + return + } + c.logger.WithFields(logrus.Fields{"stdin": fifos.Stdin}).Debug("connected") + dc.con = conn + dc.unblockConnectionWaiters() + }() + } + + if fifos.Stdout != "" { + c.logger.WithFields(logrus.Fields{"stdout": fifos.Stdout}).Debug("listen") + l, err := winio.ListenPipe(fifos.Stdout, nil) + if err != nil { + return nil, errors.Wrapf(err, "failed to create stdout pipe %s", fifos.Stdout) + } + dc := &delayedConnection{ + l: l, + } + dc.wg.Add(1) + defer func() { + if err != nil { + dc.Close() + } + }() + p.stdout = dc + + go func() { + c.logger.WithFields(logrus.Fields{"stdout": fifos.Stdout}).Debug("accept") + conn, err := l.Accept() + if err != nil { + dc.Close() + if err != winio.ErrPipeListenerClosed { + c.logger.WithError(err).Errorf("failed to accept stdout connection on %s", fifos.Stdout) + } + return + } + c.logger.WithFields(logrus.Fields{"stdout": fifos.Stdout}).Debug("connected") + dc.con = conn + dc.unblockConnectionWaiters() + }() + } + + if fifos.Stderr != "" { + c.logger.WithFields(logrus.Fields{"stderr": fifos.Stderr}).Debug("listen") + l, err := winio.ListenPipe(fifos.Stderr, nil) + if err != nil { + return nil, errors.Wrapf(err, "failed to create stderr pipe %s", fifos.Stderr) + } + dc := &delayedConnection{ + l: l, + } + dc.wg.Add(1) + defer func() { + if err != nil { + dc.Close() + } + }() + p.stderr = dc + + go func() { + c.logger.WithFields(logrus.Fields{"stderr": fifos.Stderr}).Debug("accept") + conn, err := l.Accept() + if err != nil { + dc.Close() + if err != winio.ErrPipeListenerClosed { + c.logger.WithError(err).Errorf("failed to accept stderr connection on %s", fifos.Stderr) + } + return + } + c.logger.WithFields(logrus.Fields{"stderr": fifos.Stderr}).Debug("connected") + dc.con = conn + dc.unblockConnectionWaiters() + }() + } + return p, nil +} diff --git a/libcontainerd/client_daemon_linux.go b/libcontainerd/remote/client_linux.go similarity index 81% rename from libcontainerd/client_daemon_linux.go rename to libcontainerd/remote/client_linux.go index bebe5f7ae8..22e764fbd1 100644 --- a/libcontainerd/client_daemon_linux.go +++ b/libcontainerd/remote/client_linux.go @@ -1,4 +1,4 @@ -package libcontainerd // import "github.com/docker/docker/libcontainerd" +package remote // import "github.com/docker/docker/libcontainerd/remote" import ( "context" @@ -9,17 +9,20 @@ import ( "github.com/containerd/containerd" "github.com/containerd/containerd/cio" + libcontainerdtypes "github.com/docker/docker/libcontainerd/types" "github.com/docker/docker/pkg/idtools" "github.com/opencontainers/runtime-spec/specs-go" "github.com/sirupsen/logrus" ) -func summaryFromInterface(i interface{}) (*Summary, error) { - return &Summary{}, nil +const runtimeName = "io.containerd.runtime.v1.linux" + +func summaryFromInterface(i interface{}) (*libcontainerdtypes.Summary, error) { + return &libcontainerdtypes.Summary{}, nil } -func (c *client) UpdateResources(ctx context.Context, containerID string, resources *Resources) error { - p, err := c.getProcess(containerID, InitProcessName) +func (c *client) UpdateResources(ctx context.Context, containerID string, resources *libcontainerdtypes.Resources) error { + p, err := c.getProcess(containerID, libcontainerdtypes.InitProcessName) if err != nil { return err } @@ -106,3 +109,7 @@ func newFIFOSet(bundleDir, processID string, withStdin, withTerminal bool) *cio. return cio.NewFIFOSet(config, closer) } + +func (c *client) newDirectIO(ctx context.Context, fifos *cio.FIFOSet) (*cio.DirectIO, error) { + return cio.NewDirectIO(ctx, fifos) +} diff --git a/libcontainerd/client_daemon_windows.go b/libcontainerd/remote/client_windows.go similarity index 57% rename from libcontainerd/client_daemon_windows.go rename to libcontainerd/remote/client_windows.go index 4aba33e18c..2d9cdf9fdc 100644 --- a/libcontainerd/client_daemon_windows.go +++ b/libcontainerd/remote/client_windows.go @@ -1,19 +1,24 @@ -package libcontainerd // import "github.com/docker/docker/libcontainerd" +package remote // import "github.com/docker/docker/libcontainerd/remote" import ( + "context" "fmt" + "os" "path/filepath" "github.com/containerd/containerd/cio" "github.com/containerd/containerd/windows/hcsshimtypes" + libcontainerdtypes "github.com/docker/docker/libcontainerd/types" specs "github.com/opencontainers/runtime-spec/specs-go" "github.com/pkg/errors" ) -func summaryFromInterface(i interface{}) (*Summary, error) { +const runtimeName = "io.containerd.runhcs.v1" + +func summaryFromInterface(i interface{}) (*libcontainerdtypes.Summary, error) { switch pd := i.(type) { case *hcsshimtypes.ProcessDetails: - return &Summary{ + return &libcontainerdtypes.Summary{ CreateTimestamp: pd.CreatedAt, ImageName: pd.ImageName, KernelTime100ns: pd.KernelTime_100Ns, @@ -29,7 +34,8 @@ func summaryFromInterface(i interface{}) (*Summary, error) { } func prepareBundleDir(bundleDir string, ociSpec *specs.Spec) (string, error) { - return bundleDir, nil + // TODO: (containerd) Determine if we need to use system.MkdirAllWithACL here + return bundleDir, os.MkdirAll(bundleDir, 0755) } func pipeName(containerID, processID, name string) string { @@ -53,3 +59,22 @@ func newFIFOSet(bundleDir, processID string, withStdin, withTerminal bool) *cio. return cio.NewFIFOSet(config, nil) } + +func (c *client) newDirectIO(ctx context.Context, fifos *cio.FIFOSet) (*cio.DirectIO, error) { + pipes, err := c.newStdioPipes(fifos) + if err != nil { + return nil, err + } + return cio.NewDirectIOFromFIFOSet(ctx, pipes.stdin, pipes.stdout, pipes.stderr, fifos), nil +} + +func (c *client) UpdateResources(ctx context.Context, containerID string, resources *libcontainerdtypes.Resources) error { + // TODO: (containerd): Not implemented, but don't error. + return nil +} + +func getSpecUser(ociSpec *specs.Spec) (int, int) { + // TODO: (containerd): Not implemented, but don't error. + // Not clear if we can even do this for LCOW. + return 0, 0 +} diff --git a/libcontainerd/types.go b/libcontainerd/types/types.go similarity index 95% rename from libcontainerd/types.go rename to libcontainerd/types/types.go index c4de5e674d..5b59a089d8 100644 --- a/libcontainerd/types.go +++ b/libcontainerd/types/types.go @@ -1,4 +1,4 @@ -package libcontainerd // import "github.com/docker/docker/libcontainerd" +package types // import "github.com/docker/docker/libcontainerd/types" import ( "context" @@ -89,3 +89,6 @@ type Client interface { // StdioCallback is called to connect a container or process stdio. type StdioCallback func(io *cio.DirectIO) (cio.IO, error) + +// InitProcessName is the name given to the first process of a container +const InitProcessName = "init" diff --git a/libcontainerd/types_linux.go b/libcontainerd/types/types_linux.go similarity index 72% rename from libcontainerd/types_linux.go rename to libcontainerd/types/types_linux.go index 943382b9b0..0a2daf5777 100644 --- a/libcontainerd/types_linux.go +++ b/libcontainerd/types/types_linux.go @@ -1,4 +1,4 @@ -package libcontainerd // import "github.com/docker/docker/libcontainerd" +package types // import "github.com/docker/docker/libcontainerd/types" import ( "time" @@ -16,7 +16,8 @@ type Stats struct { Metrics *cgroups.Metrics } -func interfaceToStats(read time.Time, v interface{}) *Stats { +// InterfaceToStats returns a stats object from the platform-specific interface. +func InterfaceToStats(read time.Time, v interface{}) *Stats { return &Stats{ Metrics: v.(*cgroups.Metrics), Read: read, diff --git a/libcontainerd/types_windows.go b/libcontainerd/types/types_windows.go similarity index 68% rename from libcontainerd/types_windows.go rename to libcontainerd/types/types_windows.go index 9041a2e8d5..8f8982275b 100644 --- a/libcontainerd/types_windows.go +++ b/libcontainerd/types/types_windows.go @@ -1,10 +1,9 @@ -package libcontainerd // import "github.com/docker/docker/libcontainerd" +package types // import "github.com/docker/docker/libcontainerd/types" import ( "time" "github.com/Microsoft/hcsshim" - opengcs "github.com/Microsoft/opengcs/client" ) // Summary contains a ProcessList item from HCS to support `top` @@ -16,7 +15,8 @@ type Stats struct { HCSStats *hcsshim.Statistics } -func interfaceToStats(read time.Time, v interface{}) *Stats { +// InterfaceToStats returns a stats object from the platform-specific interface. +func InterfaceToStats(read time.Time, v interface{}) *Stats { return &Stats{ HCSStats: v.(*hcsshim.Statistics), Read: read, @@ -26,11 +26,6 @@ func interfaceToStats(read time.Time, v interface{}) *Stats { // Resources defines updatable container resource values. type Resources struct{} -// LCOWOption is a CreateOption required for LCOW configuration -type LCOWOption struct { - Config *opengcs.Config -} - // Checkpoint holds the details of a checkpoint (not supported in windows) type Checkpoint struct { Name string diff --git a/pkg/system/init_unix.go b/pkg/system/init_unix.go index 4996a67c12..c2bb0f4cc4 100644 --- a/pkg/system/init_unix.go +++ b/pkg/system/init_unix.go @@ -5,3 +5,8 @@ package system // import "github.com/docker/docker/pkg/system" // InitLCOW does nothing since LCOW is a windows only feature func InitLCOW(experimental bool) { } + +// ContainerdRuntimeSupported returns true if the use of ContainerD runtime is supported. +func ContainerdRuntimeSupported(_ bool, _ string) bool { + return true +} diff --git a/pkg/system/init_windows.go b/pkg/system/init_windows.go index 4910ff69d6..7f67501285 100644 --- a/pkg/system/init_windows.go +++ b/pkg/system/init_windows.go @@ -1,7 +1,19 @@ package system // import "github.com/docker/docker/pkg/system" -// lcowSupported determines if Linux Containers on Windows are supported. -var lcowSupported = false +import ( + "os" + + "github.com/sirupsen/logrus" +) + +var ( + // lcowSupported determines if Linux Containers on Windows are supported. + lcowSupported = false + + // containerdRuntimeSupported determines if ContainerD should be the runtime. + // As of March 2019, this is an experimental feature. + containerdRuntimeSupported = false +) // InitLCOW sets whether LCOW is supported or not func InitLCOW(experimental bool) { @@ -10,3 +22,19 @@ func InitLCOW(experimental bool) { lcowSupported = true } } + +// InitContainerdRuntime sets whether to use ContainerD for runtime +// on Windows. This is an experimental feature still in development, and +// also requires an environment variable to be set (so as not to turn the +// feature on from simply experimental which would also mean LCOW. +func InitContainerdRuntime(experimental bool, cdPath string) { + if experimental && len(cdPath) > 0 && len(os.Getenv("DOCKER_WINDOWS_CONTAINERD_RUNTIME")) > 0 { + logrus.Warnf("Using ContainerD runtime. This feature is experimental") + containerdRuntimeSupported = true + } +} + +// ContainerdRuntimeSupported returns true if the use of ContainerD runtime is supported. +func ContainerdRuntimeSupported() bool { + return containerdRuntimeSupported +} diff --git a/plugin/executor/containerd/containerd.go b/plugin/executor/containerd/containerd.go index a3401dce79..2c6368fec8 100644 --- a/plugin/executor/containerd/containerd.go +++ b/plugin/executor/containerd/containerd.go @@ -12,6 +12,7 @@ import ( "github.com/containerd/containerd/runtime/linux/runctypes" "github.com/docker/docker/errdefs" "github.com/docker/docker/libcontainerd" + libcontainerdtypes "github.com/docker/docker/libcontainerd/types" "github.com/opencontainers/runtime-spec/specs-go" "github.com/pkg/errors" "github.com/sirupsen/logrus" @@ -30,11 +31,11 @@ type ExitHandler interface { // However right now this whole package is tied to github.com/docker/docker/libcontainerd type Client interface { Create(ctx context.Context, containerID string, spec *specs.Spec, runtimeOptions interface{}) error - Restore(ctx context.Context, containerID string, attachStdio libcontainerd.StdioCallback) (alive bool, pid int, err error) - Status(ctx context.Context, containerID string) (libcontainerd.Status, error) + Restore(ctx context.Context, containerID string, attachStdio libcontainerdtypes.StdioCallback) (alive bool, pid int, err error) + Status(ctx context.Context, containerID string) (libcontainerdtypes.Status, error) Delete(ctx context.Context, containerID string) error DeleteTask(ctx context.Context, containerID string) (uint32, time.Time, error) - Start(ctx context.Context, containerID, checkpointDir string, withStdin bool, attachStdio libcontainerd.StdioCallback) (pid int, err error) + Start(ctx context.Context, containerID, checkpointDir string, withStdin bool, attachStdio libcontainerdtypes.StdioCallback) (pid int, err error) SignalProcess(ctx context.Context, containerID, processID string, signal int) error } @@ -87,7 +88,7 @@ func (e *Executor) Create(id string, spec specs.Spec, stdout, stderr io.WriteClo logrus.WithError(err2).WithField("id", id).Warn("Received an error while attempting to read plugin status") } } else { - if status != libcontainerd.StatusRunning && status != libcontainerd.StatusUnknown { + if status != libcontainerdtypes.StatusRunning && status != libcontainerdtypes.StatusUnknown { if err2 := e.client.Delete(ctx, id); err2 != nil && !errdefs.IsNotFound(err2) { logrus.WithError(err2).WithField("plugin", id).Error("Error cleaning up containerd container") } @@ -122,19 +123,19 @@ func (e *Executor) Restore(id string, stdout, stderr io.WriteCloser) (bool, erro // IsRunning returns if the container with the given id is running func (e *Executor) IsRunning(id string) (bool, error) { status, err := e.client.Status(context.Background(), id) - return status == libcontainerd.StatusRunning, err + return status == libcontainerdtypes.StatusRunning, err } // Signal sends the specified signal to the container func (e *Executor) Signal(id string, signal int) error { - return e.client.SignalProcess(context.Background(), id, libcontainerd.InitProcessName, signal) + return e.client.SignalProcess(context.Background(), id, libcontainerdtypes.InitProcessName, signal) } // ProcessEvent handles events from containerd // All events are ignored except the exit event, which is sent of to the stored handler -func (e *Executor) ProcessEvent(id string, et libcontainerd.EventType, ei libcontainerd.EventInfo) error { +func (e *Executor) ProcessEvent(id string, et libcontainerdtypes.EventType, ei libcontainerdtypes.EventInfo) error { switch et { - case libcontainerd.EventExit: + case libcontainerdtypes.EventExit: deleteTaskAndContainer(context.Background(), e.client, id) return e.exitHandler.HandleExitEvent(ei.ContainerID) } @@ -152,7 +153,7 @@ func (c *rio) Wait() { c.IO.Wait() } -func attachStreamsFunc(stdout, stderr io.WriteCloser) libcontainerd.StdioCallback { +func attachStreamsFunc(stdout, stderr io.WriteCloser) libcontainerdtypes.StdioCallback { return func(iop *cio.DirectIO) (cio.IO, error) { if iop.Stdin != nil { iop.Stdin.Close() diff --git a/plugin/executor/containerd/containerd_test.go b/plugin/executor/containerd/containerd_test.go index 3fb45981f3..7443cb9e8e 100644 --- a/plugin/executor/containerd/containerd_test.go +++ b/plugin/executor/containerd/containerd_test.go @@ -8,7 +8,7 @@ import ( "testing" "time" - "github.com/docker/docker/libcontainerd" + libcontainerdtypes "github.com/docker/docker/libcontainerd/types" "github.com/opencontainers/runtime-spec/specs-go" "github.com/pkg/errors" "gotest.tools/assert" @@ -82,22 +82,22 @@ func (c *mockClient) Create(ctx context.Context, id string, _ *specs.Spec, _ int return nil } -func (c *mockClient) Restore(ctx context.Context, id string, attachStdio libcontainerd.StdioCallback) (alive bool, pid int, err error) { +func (c *mockClient) Restore(ctx context.Context, id string, attachStdio libcontainerdtypes.StdioCallback) (alive bool, pid int, err error) { return false, 0, nil } -func (c *mockClient) Status(ctx context.Context, id string) (libcontainerd.Status, error) { +func (c *mockClient) Status(ctx context.Context, id string) (libcontainerdtypes.Status, error) { c.mu.Lock() defer c.mu.Unlock() running, ok := c.containers[id] if !ok { - return libcontainerd.StatusUnknown, errors.New("not found") + return libcontainerdtypes.StatusUnknown, errors.New("not found") } if running { - return libcontainerd.StatusRunning, nil + return libcontainerdtypes.StatusRunning, nil } - return libcontainerd.StatusStopped, nil + return libcontainerdtypes.StatusStopped, nil } func (c *mockClient) Delete(ctx context.Context, id string) error { @@ -111,7 +111,7 @@ func (c *mockClient) DeleteTask(ctx context.Context, id string) (uint32, time.Ti return 0, time.Time{}, nil } -func (c *mockClient) Start(ctx context.Context, id, checkpointDir string, withStdin bool, attachStdio libcontainerd.StdioCallback) (pid int, err error) { +func (c *mockClient) Start(ctx context.Context, id, checkpointDir string, withStdin bool, attachStdio libcontainerdtypes.StdioCallback) (pid int, err error) { c.mu.Lock() defer c.mu.Unlock()