Merge pull request #44804 from rumpl/containerd-run

containerd integration: `docker run`

daemon/containerd/image_snapshot.go

@@ -0,0 +1,59 @@
+package containerd
+
+import (
+	"context"
+
+	containerdimages "github.com/containerd/containerd/images"
+	"github.com/containerd/containerd/leases"
+	"github.com/containerd/containerd/platforms"
+	"github.com/opencontainers/image-spec/identity"
+	v1 "github.com/opencontainers/image-spec/specs-go/v1"
+)
+
+// PrepareSnapshot prepares a snapshot from a parent image for a container
+func (i *ImageService) PrepareSnapshot(ctx context.Context, id string, parentImage string, platform *v1.Platform) error {
+	desc, err := i.resolveDescriptor(ctx, parentImage)
+	if err != nil {
+		return err
+	}
+
+	cs := i.client.ContentStore()
+
+	matcher := platforms.Default()
+	if platform != nil {
+		matcher = platforms.Only(*platform)
+	}
+
+	desc, err = containerdimages.Config(ctx, cs, desc, matcher)
+	if err != nil {
+		return err
+	}
+
+	diffIDs, err := containerdimages.RootFS(ctx, cs, desc)
+	if err != nil {
+		return err
+	}
+
+	parent := identity.ChainID(diffIDs).String()
+
+	// Add a lease so that containerd doesn't garbage collect our snapshot
+	ls := i.client.LeasesService()
+	lease, err := ls.Create(ctx, leases.WithID(id))
+	if err != nil {
+		return err
+	}
+
+	if err := ls.AddResource(ctx, lease, leases.Resource{
+		ID:   id,
+		Type: "snapshots/" + i.StorageDriver(),
+	}); err != nil {
+		return err
+	}
+
+	s := i.client.SnapshotService(i.StorageDriver())
+	if _, err := s.Prepare(ctx, id, parent); err == nil {
+		return err
+	}
+
+	return nil
+}

daemon/containerd/mount.go

@@ -0,0 +1,52 @@
+package containerd
+
+import (
+	"context"
+	"fmt"
+	"os"
+
+	"github.com/containerd/containerd/mount"
+	"github.com/docker/docker/container"
+	"github.com/sirupsen/logrus"
+)
+
+// Mount mounts the container filesystem in a temporary location, use defer imageService.Unmount
+// to unmount the filesystem when calling this
+func (i *ImageService) Mount(ctx context.Context, container *container.Container) error {
+	snapshotter := i.client.SnapshotService(i.snapshotter)
+	mounts, err := snapshotter.Mounts(ctx, container.ID)
+	if err != nil {
+		return err
+	}
+
+	// The temporary location will be under /var/lib/docker/... because
+	// we set the `TMPDIR`
+	root, err := os.MkdirTemp("", fmt.Sprintf("%s_rootfs-mount", container.ID))
+	if err != nil {
+		return fmt.Errorf("failed to create temp dir: %w", err)
+	}
+
+	if err := mount.All(mounts, root); err != nil {
+		return fmt.Errorf("failed to mount %s: %w", root, err)
+	}
+
+	container.BaseFS = root
+	return nil
+}
+
+// Unmount unmounts the container base filesystem
+func (i *ImageService) Unmount(ctx context.Context, container *container.Container) error {
+	root := container.BaseFS
+
+	if err := mount.UnmountAll(root, 0); err != nil {
+		return fmt.Errorf("failed to unmount %s: %w", root, err)
+	}
+
+	if err := os.Remove(root); err != nil {
+		logrus.WithError(err).WithField("dir", root).Error("failed to remove mount temp dir")
+	}
+
+	container.BaseFS = ""
+
+	return nil
+}

daemon/create.go

@@ -170,12 +170,18 @@ func (daemon *Daemon) create(ctx context.Context, opts createOpts) (retC *contai
 
 	ctr.HostConfig.StorageOpt = opts.params.HostConfig.StorageOpt
 
-	// Set RWLayer for container after mount labels have been set
-	rwLayer, err := daemon.imageService.CreateLayer(ctr, setupInitLayer(daemon.idMapping))
-	if err != nil {
-		return nil, errdefs.System(err)
+	if daemon.UsesSnapshotter() {
+		if err := daemon.imageService.PrepareSnapshot(ctx, ctr.ID, opts.params.Config.Image, opts.params.Platform); err != nil {
+			return nil, err
+		}
+	} else {
+		// Set RWLayer for container after mount labels have been set
+		rwLayer, err := daemon.imageService.CreateLayer(ctr, setupInitLayer(daemon.idMapping))
+		if err != nil {
+			return nil, errdefs.System(err)
+		}
+		ctr.RWLayer = rwLayer
 	}
-	ctr.RWLayer = rwLayer
 
 	current := idtools.CurrentIdentity()
 	if err := idtools.MkdirAndChown(ctr.Root, 0710, idtools.Identity{UID: current.UID, GID: daemon.IdentityMapping().RootPair().GID}); err != nil {

daemon/daemon.go

@@ -1259,42 +1259,13 @@ func (daemon *Daemon) Shutdown(ctx context.Context) error {
 }
 
 // Mount sets container.BaseFS
-// (is it not set coming in? why is it unset?)
 func (daemon *Daemon) Mount(container *container.Container) error {
-	if container.RWLayer == nil {
-		return errors.New("RWLayer of container " + container.ID + " is unexpectedly nil")
-	}
-	dir, err := container.RWLayer.Mount(container.GetMountLabel())
-	if err != nil {
-		return err
-	}
-	logrus.WithField("container", container.ID).Debugf("container mounted via layerStore: %v", dir)
-
-	if container.BaseFS != "" && container.BaseFS != dir {
-		// The mount path reported by the graph driver should always be trusted on Windows, since the
-		// volume path for a given mounted layer may change over time.  This should only be an error
-		// on non-Windows operating systems.
-		if runtime.GOOS != "windows" {
-			daemon.Unmount(container)
-			return fmt.Errorf("driver %s is returning inconsistent paths for container %s ('%s' then '%s')",
-				container.Driver, container.ID, container.BaseFS, dir)
-		}
-	}
-	container.BaseFS = dir // TODO: combine these fields
-	return nil
+	return daemon.imageService.Mount(context.Background(), container)
 }
 
 // Unmount unsets the container base filesystem
 func (daemon *Daemon) Unmount(container *container.Container) error {
-	if container.RWLayer == nil {
-		return errors.New("RWLayer of container " + container.ID + " is unexpectedly nil")
-	}
-	if err := container.RWLayer.Unmount(); err != nil {
-		logrus.WithField("container", container.ID).WithError(err).Error("error unmounting container")
-		return err
-	}
-
-	return nil
+	return daemon.imageService.Unmount(context.Background(), container)
 }
 
 // Subnets return the IPv4 and IPv6 subnets of networks that are manager by Docker.

daemon/daemon_unix.go

@@ -1394,13 +1394,19 @@ func (daemon *Daemon) registerLinks(container *container.Container, hostConfig *
 // conditionalMountOnStart is a platform specific helper function during the
 // container start to call mount.
 func (daemon *Daemon) conditionalMountOnStart(container *container.Container) error {
-	return daemon.Mount(container)
+	if !daemon.UsesSnapshotter() {
+		return daemon.Mount(container)
+	}
+	return nil
 }
 
 // conditionalUnmountOnCleanup is a platform specific helper function called
 // during the cleanup of a container to unmount.
 func (daemon *Daemon) conditionalUnmountOnCleanup(container *container.Container) error {
-	return daemon.Unmount(container)
+	if !daemon.UsesSnapshotter() {
+		return daemon.Unmount(container)
+	}
+	return nil
 }
 
 func copyBlkioEntry(entries []*statsV1.BlkIOEntry) []types.BlkioStatEntry {

daemon/delete.go

@@ -8,6 +8,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/containerd/containerd/leases"
 	"github.com/docker/docker/api/types"
 	containertypes "github.com/docker/docker/api/types/container"
 	"github.com/docker/docker/container"
@@ -136,6 +137,17 @@ func (daemon *Daemon) cleanupContainer(container *container.Container, config ty
 			return err
 		}
 		container.RWLayer = nil
+	} else {
+		if daemon.UsesSnapshotter() {
+			ls := daemon.containerdCli.LeasesService()
+			lease := leases.Lease{
+				ID: container.ID,
+			}
+			if err := ls.Delete(context.Background(), lease, leases.SynchronousDelete); err != nil {
+				container.SetRemovalError(err)
+				return err
+			}
+		}
 	}
 
 	// Hold the container lock while deleting the container root directory

daemon/image_service.go

@@ -44,6 +44,10 @@ type ImageService interface {
 	CommitImage(ctx context.Context, c backend.CommitConfig) (image.ID, error)
 	SquashImage(id, parent string) (string, error)
 
+	// Containerd related methods
+
+	PrepareSnapshot(ctx context.Context, id string, image string, platform *v1.Platform) error
+
 	// Layers
 
 	GetImageAndReleasableLayer(ctx context.Context, refOrID string, opts backend.GetImageAndLayerOptions) (builder.Image, builder.ROLayer, error)
@@ -54,6 +58,8 @@ type ImageService interface {
 	ReleaseLayer(rwlayer layer.RWLayer) error
 	LayerDiskUsage(ctx context.Context) (int64, error)
 	GetContainerLayerSize(containerID string) (int64, int64)
+	Mount(ctx context.Context, container *container.Container) error
+	Unmount(ctx context.Context, container *container.Container) error
 
 	// Windows specific
 

daemon/images/image.go

@@ -18,6 +18,7 @@ import (
 	"github.com/docker/docker/layer"
 	"github.com/opencontainers/go-digest"
 	specs "github.com/opencontainers/image-spec/specs-go/v1"
+	v1 "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 )
@@ -46,6 +47,11 @@ type manifest struct {
 	Config specs.Descriptor `json:"config"`
 }
 
+func (i *ImageService) PrepareSnapshot(ctx context.Context, id string, image string, platform *v1.Platform) error {
+	// Only makes sense when conatinerd image store is used
+	panic("not implemented")
+}
+
 func (i *ImageService) manifestMatchesPlatform(ctx context.Context, img *image.Image, platform specs.Platform) (bool, error) {
 	logger := logrus.WithField("image", img.ID).WithField("desiredPlatform", platforms.Format(platform))
 

daemon/images/mount.go

@@ -0,0 +1,50 @@
+package images
+
+import (
+	"context"
+	"fmt"
+	"runtime"
+
+	"github.com/docker/docker/container"
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+)
+
+// Mount sets container.BaseFS
+// (is it not set coming in? why is it unset?)
+func (i *ImageService) Mount(ctx context.Context, container *container.Container) error {
+	if container.RWLayer == nil {
+		return errors.New("RWLayer of container " + container.ID + " is unexpectedly nil")
+	}
+	dir, err := container.RWLayer.Mount(container.GetMountLabel())
+	if err != nil {
+		return err
+	}
+	logrus.WithField("container", container.ID).Debugf("container mounted via layerStore: %v", dir)
+
+	if container.BaseFS != "" && container.BaseFS != dir {
+		// The mount path reported by the graph driver should always be trusted on Windows, since the
+		// volume path for a given mounted layer may change over time.  This should only be an error
+		// on non-Windows operating systems.
+		if runtime.GOOS != "windows" {
+			i.Unmount(ctx, container)
+			return fmt.Errorf("Error: driver %s is returning inconsistent paths for container %s ('%s' then '%s')",
+				i.StorageDriver(), container.ID, container.BaseFS, dir)
+		}
+	}
+	container.BaseFS = dir // TODO: combine these fields
+	return nil
+}
+
+// Unmount unsets the container base filesystem
+func (i *ImageService) Unmount(ctx context.Context, container *container.Container) error {
+	if container.RWLayer == nil {
+		return errors.New("RWLayer of container " + container.ID + " is unexpectedly nil")
+	}
+	if err := container.RWLayer.Unmount(); err != nil {
+		logrus.WithField("container", container.ID).WithError(err).Error("error unmounting container")
+		return err
+	}
+
+	return nil
+}

daemon/oci_linux.go

@@ -731,9 +731,9 @@ func WithCommonOptions(daemon *Daemon, c *container.Container) coci.SpecOpts {
 				Path:     c.BaseFS,
 				Readonly: c.HostConfig.ReadonlyRootfs,
 			}
-		}
-		if err := c.SetupWorkingDirectory(daemon.idMapping.RootPair()); err != nil {
-			return err
+			if err := c.SetupWorkingDirectory(daemon.idMapping.RootPair()); err != nil {
+				return err
+			}
 		}
 		cwd := c.Config.WorkingDir
 		if len(cwd) == 0 {
@@ -1017,7 +1017,6 @@ func (daemon *Daemon) createSpec(ctx context.Context, c *container.Container) (r
 		WithResources(c),
 		WithSysctls(c),
 		WithDevices(daemon, c),
-		WithUser(c),
 		WithRlimits(daemon, c),
 		WithNamespaces(daemon, c),
 		WithCapabilities(c),
@@ -1028,6 +1027,20 @@ func (daemon *Daemon) createSpec(ctx context.Context, c *container.Container) (r
 		WithSelinux(c),
 		WithOOMScore(&c.HostConfig.OomScoreAdj),
 	)
+	if daemon.UsesSnapshotter() {
+		s.Root = &specs.Root{
+			Path: "rootfs",
+		}
+		if c.Config.User != "" {
+			opts = append(opts, coci.WithUser(c.Config.User))
+		}
+		if c.Config.WorkingDir != "" {
+			opts = append(opts, coci.WithProcessCwd(c.Config.WorkingDir))
+		}
+	} else {
+		opts = append(opts, WithUser(c))
+	}
+
 	if c.NoNewPrivileges {
 		opts = append(opts, coci.WithNoNewPrivileges)
 	}
@@ -1051,7 +1064,7 @@ func (daemon *Daemon) createSpec(ctx context.Context, c *container.Container) (r
 		snapshotKey = c.ID
 	}
 
-	return &s, coci.ApplyOpts(ctx, nil, &containers.Container{
+	return &s, coci.ApplyOpts(ctx, daemon.containerdCli, &containers.Container{
 		ID:          c.ID,
 		Snapshotter: snapshotter,
 		SnapshotKey: snapshotKey,

daemon/start.go

@@ -5,6 +5,7 @@ import (
 	"runtime"
 	"time"
 
+	"github.com/containerd/containerd"
 	"github.com/docker/docker/api/types"
 	containertypes "github.com/docker/docker/api/types/container"
 	"github.com/docker/docker/container"
@@ -177,7 +178,13 @@ func (daemon *Daemon) containerStart(ctx context.Context, container *container.C
 		return err
 	}
 
-	ctr, err := libcontainerd.ReplaceContainer(ctx, daemon.containerd, container.ID, spec, shim, createOptions)
+	newContainerOpts := []containerd.NewContainerOpts{}
+	if daemon.UsesSnapshotter() {
+		newContainerOpts = append(newContainerOpts, containerd.WithSnapshotter(container.Driver))
+		newContainerOpts = append(newContainerOpts, containerd.WithSnapshot(container.ID))
+	}
+
+	ctr, err := libcontainerd.ReplaceContainer(ctx, daemon.containerd, container.ID, spec, shim, createOptions, newContainerOpts...)
 	if err != nil {
 		return setExitCodeFromError(container.SetExitCode, err)
 	}