Implement run using the containerd snapshotter

Signed-off-by: Djordje Lukic <djordje.lukic@docker.com>

c8d/daemon: Mount root and fill BaseFS

This fixes things that were broken due to nil BaseFS like `docker cp`
and running a container with workdir override.

This is more of a temporary hack than a real solution.
The correct fix would be to refactor the code to make BaseFS and LayerRW
an implementation detail of the old image store implementation and use
the temporary mounts for the c8d implementation instead.
That requires more work though.

Signed-off-by: Paweł Gronowski <pawel.gronowski@docker.com>

daemon/images: Don't unset BaseFS

Signed-off-by: Paweł Gronowski <pawel.gronowski@docker.com>

daemon/containerd/image_snapshot.go

@@ -0,0 +1,59 @@
+package containerd
+
+import (
+	"context"
+
+	containerdimages "github.com/containerd/containerd/images"
+	"github.com/containerd/containerd/leases"
+	"github.com/containerd/containerd/platforms"
+	"github.com/opencontainers/image-spec/identity"
+	v1 "github.com/opencontainers/image-spec/specs-go/v1"
+)
+
+// PrepareSnapshot prepares a snapshot from a parent image for a container
+func (i *ImageService) PrepareSnapshot(ctx context.Context, id string, parentImage string, platform *v1.Platform) error {
+	desc, err := i.resolveDescriptor(ctx, parentImage)
+	if err != nil {
+		return err
+	}
+
+	cs := i.client.ContentStore()
+
+	matcher := platforms.Default()
+	if platform != nil {
+		matcher = platforms.Only(*platform)
+	}
+
+	desc, err = containerdimages.Config(ctx, cs, desc, matcher)
+	if err != nil {
+		return err
+	}
+
+	diffIDs, err := containerdimages.RootFS(ctx, cs, desc)
+	if err != nil {
+		return err
+	}
+
+	parent := identity.ChainID(diffIDs).String()
+
+	// Add a lease so that containerd doesn't garbage collect our snapshot
+	ls := i.client.LeasesService()
+	lease, err := ls.Create(ctx, leases.WithID(id))
+	if err != nil {
+		return err
+	}
+
+	if err := ls.AddResource(ctx, lease, leases.Resource{
+		ID:   id,
+		Type: "snapshots/" + i.StorageDriver(),
+	}); err != nil {
+		return err
+	}
+
+	s := i.client.SnapshotService(i.StorageDriver())
+	if _, err := s.Prepare(ctx, id, parent); err == nil {
+		return err
+	}
+
+	return nil
+}

daemon/containerd/mount.go

@@ -0,0 +1,52 @@
+package containerd
+
+import (
+	"context"
+	"fmt"
+	"os"
+
+	"github.com/containerd/containerd/mount"
+	"github.com/docker/docker/container"
+	"github.com/sirupsen/logrus"
+)
+
+// Mount mounts the container filesystem in a temporary location, use defer imageService.Unmount
+// to unmount the filesystem when calling this
+func (i *ImageService) Mount(ctx context.Context, container *container.Container) error {
+	snapshotter := i.client.SnapshotService(i.snapshotter)
+	mounts, err := snapshotter.Mounts(ctx, container.ID)
+	if err != nil {
+		return err
+	}
+
+	// The temporary location will be under /var/lib/docker/... because
+	// we set the `TMPDIR`
+	root, err := os.MkdirTemp("", fmt.Sprintf("%s_rootfs-mount", container.ID))
+	if err != nil {
+		return fmt.Errorf("failed to create temp dir: %w", err)
+	}
+
+	if err := mount.All(mounts, root); err != nil {
+		return fmt.Errorf("failed to mount %s: %w", root, err)
+	}
+
+	container.BaseFS = root
+	return nil
+}
+
+// Unmount unmounts the container base filesystem
+func (i *ImageService) Unmount(ctx context.Context, container *container.Container) error {
+	root := container.BaseFS
+
+	if err := mount.UnmountAll(root, 0); err != nil {
+		return fmt.Errorf("failed to unmount %s: %w", root, err)
+	}
+
+	if err := os.Remove(root); err != nil {
+		logrus.WithError(err).WithField("dir", root).Error("failed to remove mount temp dir")
+	}
+
+	container.BaseFS = ""
+
+	return nil
+}

daemon/create.go

@@ -170,12 +170,18 @@ func (daemon *Daemon) create(ctx context.Context, opts createOpts) (retC *contai
 
 	ctr.HostConfig.StorageOpt = opts.params.HostConfig.StorageOpt
 
-	// Set RWLayer for container after mount labels have been set
-	rwLayer, err := daemon.imageService.CreateLayer(ctr, setupInitLayer(daemon.idMapping))
-	if err != nil {
-		return nil, errdefs.System(err)
+	if daemon.UsesSnapshotter() {
+		if err := daemon.imageService.PrepareSnapshot(ctx, ctr.ID, opts.params.Config.Image, opts.params.Platform); err != nil {
+			return nil, err
+		}
+	} else {
+		// Set RWLayer for container after mount labels have been set
+		rwLayer, err := daemon.imageService.CreateLayer(ctr, setupInitLayer(daemon.idMapping))
+		if err != nil {
+			return nil, errdefs.System(err)
+		}
+		ctr.RWLayer = rwLayer
 	}
-	ctr.RWLayer = rwLayer
 
 	current := idtools.CurrentIdentity()
 	if err := idtools.MkdirAndChown(ctr.Root, 0710, idtools.Identity{UID: current.UID, GID: daemon.IdentityMapping().RootPair().GID}); err != nil {

daemon/daemon.go

@@ -1259,42 +1259,13 @@ func (daemon *Daemon) Shutdown(ctx context.Context) error {
 }
 
 // Mount sets container.BaseFS
-// (is it not set coming in? why is it unset?)
 func (daemon *Daemon) Mount(container *container.Container) error {
-	if container.RWLayer == nil {
-		return errors.New("RWLayer of container " + container.ID + " is unexpectedly nil")
-	}
-	dir, err := container.RWLayer.Mount(container.GetMountLabel())
-	if err != nil {
-		return err
-	}
-	logrus.WithField("container", container.ID).Debugf("container mounted via layerStore: %v", dir)
-
-	if container.BaseFS != "" && container.BaseFS != dir {
-		// The mount path reported by the graph driver should always be trusted on Windows, since the
-		// volume path for a given mounted layer may change over time.  This should only be an error
-		// on non-Windows operating systems.
-		if runtime.GOOS != "windows" {
-			daemon.Unmount(container)
-			return fmt.Errorf("driver %s is returning inconsistent paths for container %s ('%s' then '%s')",
-				container.Driver, container.ID, container.BaseFS, dir)
-		}
-	}
-	container.BaseFS = dir // TODO: combine these fields
-	return nil
+	return daemon.imageService.Mount(context.Background(), container)
 }
 
 // Unmount unsets the container base filesystem
 func (daemon *Daemon) Unmount(container *container.Container) error {
-	if container.RWLayer == nil {
-		return errors.New("RWLayer of container " + container.ID + " is unexpectedly nil")
-	}
-	if err := container.RWLayer.Unmount(); err != nil {
-		logrus.WithField("container", container.ID).WithError(err).Error("error unmounting container")
-		return err
-	}
-
-	return nil
+	return daemon.imageService.Unmount(context.Background(), container)
 }
 
 // Subnets return the IPv4 and IPv6 subnets of networks that are manager by Docker.

daemon/daemon_unix.go

@@ -1394,13 +1394,19 @@ func (daemon *Daemon) registerLinks(container *container.Container, hostConfig *
 // conditionalMountOnStart is a platform specific helper function during the
 // container start to call mount.
 func (daemon *Daemon) conditionalMountOnStart(container *container.Container) error {
-	return daemon.Mount(container)
+	if !daemon.UsesSnapshotter() {
+		return daemon.Mount(container)
+	}
+	return nil
 }
 
 // conditionalUnmountOnCleanup is a platform specific helper function called
 // during the cleanup of a container to unmount.
 func (daemon *Daemon) conditionalUnmountOnCleanup(container *container.Container) error {
-	return daemon.Unmount(container)
+	if !daemon.UsesSnapshotter() {
+		return daemon.Unmount(container)
+	}
+	return nil
 }
 
 func copyBlkioEntry(entries []*statsV1.BlkIOEntry) []types.BlkioStatEntry {

daemon/delete.go

@@ -8,6 +8,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/containerd/containerd/leases"
 	"github.com/docker/docker/api/types"
 	containertypes "github.com/docker/docker/api/types/container"
 	"github.com/docker/docker/container"
@@ -136,6 +137,17 @@ func (daemon *Daemon) cleanupContainer(container *container.Container, config ty
 			return err
 		}
 		container.RWLayer = nil
+	} else {
+		if daemon.UsesSnapshotter() {
+			ls := daemon.containerdCli.LeasesService()
+			lease := leases.Lease{
+				ID: container.ID,
+			}
+			if err := ls.Delete(context.Background(), lease, leases.SynchronousDelete); err != nil {
+				container.SetRemovalError(err)
+				return err
+			}
+		}
 	}
 
 	// Hold the container lock while deleting the container root directory

daemon/image_service.go

@@ -44,6 +44,10 @@ type ImageService interface {
 	CommitImage(ctx context.Context, c backend.CommitConfig) (image.ID, error)
 	SquashImage(id, parent string) (string, error)
 
+	// Containerd related methods
+
+	PrepareSnapshot(ctx context.Context, id string, image string, platform *v1.Platform) error
+
 	// Layers
 
 	GetImageAndReleasableLayer(ctx context.Context, refOrID string, opts backend.GetImageAndLayerOptions) (builder.Image, builder.ROLayer, error)
@@ -54,6 +58,8 @@ type ImageService interface {
 	ReleaseLayer(rwlayer layer.RWLayer) error
 	LayerDiskUsage(ctx context.Context) (int64, error)
 	GetContainerLayerSize(containerID string) (int64, int64)
+	Mount(ctx context.Context, container *container.Container) error
+	Unmount(ctx context.Context, container *container.Container) error
 
 	// Windows specific
 

daemon/images/image.go

@@ -18,6 +18,7 @@ import (
 	"github.com/docker/docker/layer"
 	"github.com/opencontainers/go-digest"
 	specs "github.com/opencontainers/image-spec/specs-go/v1"
+	v1 "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 )
@@ -46,6 +47,11 @@ type manifest struct {
 	Config specs.Descriptor `json:"config"`
 }
 
+func (i *ImageService) PrepareSnapshot(ctx context.Context, id string, image string, platform *v1.Platform) error {
+	// Only makes sense when conatinerd image store is used
+	panic("not implemented")
+}
+
 func (i *ImageService) manifestMatchesPlatform(ctx context.Context, img *image.Image, platform specs.Platform) (bool, error) {
 	logger := logrus.WithField("image", img.ID).WithField("desiredPlatform", platforms.Format(platform))
 

daemon/images/mount.go

@@ -0,0 +1,50 @@
+package images
+
+import (
+	"context"
+	"fmt"
+	"runtime"
+
+	"github.com/docker/docker/container"
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+)
+
+// Mount sets container.BaseFS
+// (is it not set coming in? why is it unset?)
+func (i *ImageService) Mount(ctx context.Context, container *container.Container) error {
+	if container.RWLayer == nil {
+		return errors.New("RWLayer of container " + container.ID + " is unexpectedly nil")
+	}
+	dir, err := container.RWLayer.Mount(container.GetMountLabel())
+	if err != nil {
+		return err
+	}
+	logrus.WithField("container", container.ID).Debugf("container mounted via layerStore: %v", dir)
+
+	if container.BaseFS != "" && container.BaseFS != dir {
+		// The mount path reported by the graph driver should always be trusted on Windows, since the
+		// volume path for a given mounted layer may change over time.  This should only be an error
+		// on non-Windows operating systems.
+		if runtime.GOOS != "windows" {
+			i.Unmount(ctx, container)
+			return fmt.Errorf("Error: driver %s is returning inconsistent paths for container %s ('%s' then '%s')",
+				i.StorageDriver(), container.ID, container.BaseFS, dir)
+		}
+	}
+	container.BaseFS = dir // TODO: combine these fields
+	return nil
+}
+
+// Unmount unsets the container base filesystem
+func (i *ImageService) Unmount(ctx context.Context, container *container.Container) error {
+	if container.RWLayer == nil {
+		return errors.New("RWLayer of container " + container.ID + " is unexpectedly nil")
+	}
+	if err := container.RWLayer.Unmount(); err != nil {
+		logrus.WithField("container", container.ID).WithError(err).Error("error unmounting container")
+		return err
+	}
+
+	return nil
+}

daemon/oci_linux.go

@@ -731,9 +731,9 @@ func WithCommonOptions(daemon *Daemon, c *container.Container) coci.SpecOpts {
 				Path:     c.BaseFS,
 				Readonly: c.HostConfig.ReadonlyRootfs,
 			}
-		}
-		if err := c.SetupWorkingDirectory(daemon.idMapping.RootPair()); err != nil {
-			return err
+			if err := c.SetupWorkingDirectory(daemon.idMapping.RootPair()); err != nil {
+				return err
+			}
 		}
 		cwd := c.Config.WorkingDir
 		if len(cwd) == 0 {
@@ -1017,7 +1017,6 @@ func (daemon *Daemon) createSpec(ctx context.Context, c *container.Container) (r
 		WithResources(c),
 		WithSysctls(c),
 		WithDevices(daemon, c),
-		WithUser(c),
 		WithRlimits(daemon, c),
 		WithNamespaces(daemon, c),
 		WithCapabilities(c),
@@ -1028,6 +1027,20 @@ func (daemon *Daemon) createSpec(ctx context.Context, c *container.Container) (r
 		WithSelinux(c),
 		WithOOMScore(&c.HostConfig.OomScoreAdj),
 	)
+	if daemon.UsesSnapshotter() {
+		s.Root = &specs.Root{
+			Path: "rootfs",
+		}
+		if c.Config.User != "" {
+			opts = append(opts, coci.WithUser(c.Config.User))
+		}
+		if c.Config.WorkingDir != "" {
+			opts = append(opts, coci.WithProcessCwd(c.Config.WorkingDir))
+		}
+	} else {
+		opts = append(opts, WithUser(c))
+	}
+
 	if c.NoNewPrivileges {
 		opts = append(opts, coci.WithNoNewPrivileges)
 	}
@@ -1051,7 +1064,7 @@ func (daemon *Daemon) createSpec(ctx context.Context, c *container.Container) (r
 		snapshotKey = c.ID
 	}
 
-	return &s, coci.ApplyOpts(ctx, nil, &containers.Container{
+	return &s, coci.ApplyOpts(ctx, daemon.containerdCli, &containers.Container{
 		ID:          c.ID,
 		Snapshotter: snapshotter,
 		SnapshotKey: snapshotKey,

daemon/start.go

@@ -5,6 +5,7 @@ import (
 	"runtime"
 	"time"
 
+	"github.com/containerd/containerd"
 	"github.com/docker/docker/api/types"
 	containertypes "github.com/docker/docker/api/types/container"
 	"github.com/docker/docker/container"
@@ -177,7 +178,13 @@ func (daemon *Daemon) containerStart(ctx context.Context, container *container.C
 		return err
 	}
 
-	ctr, err := libcontainerd.ReplaceContainer(ctx, daemon.containerd, container.ID, spec, shim, createOptions)
+	newContainerOpts := []containerd.NewContainerOpts{}
+	if daemon.UsesSnapshotter() {
+		newContainerOpts = append(newContainerOpts, containerd.WithSnapshotter(container.Driver))
+		newContainerOpts = append(newContainerOpts, containerd.WithSnapshot(container.ID))
+	}
+
+	ctr, err := libcontainerd.ReplaceContainer(ctx, daemon.containerd, container.ID, spec, shim, createOptions, newContainerOpts...)
 	if err != nil {
 		return setExitCodeFromError(container.SetExitCode, err)
 	}