[20.10] Backport Restore active mount counts on live-restore

Backporting commit 647c2a6cdd86d79230df1bf690d0b6a2930d6db2 for 20.10

When live-restoring a container the volume driver needs be notified that
there is an active mount for the volume.
Before this change the count is zero until the container stops and the
uint64 overflows pretty much making it so the volume can never be
removed until another daemon restart.

Signed-off-by: Brian Goff <cpuguy83@gmail.com>

daemon/mounts.go

@@ -8,13 +8,24 @@ import (
 	mounttypes "github.com/docker/docker/api/types/mount"
 	"github.com/docker/docker/container"
 	volumesservice "github.com/docker/docker/volume/service"
+	"github.com/sirupsen/logrus"
 )
 
 func (daemon *Daemon) prepareMountPoints(container *container.Container) error {
+	alive := container.IsRunning()
 	for _, config := range container.MountPoints {
 		if err := daemon.lazyInitializeVolume(container.ID, config); err != nil {
 			return err
 		}
+		if alive {
+			logrus.WithFields(logrus.Fields{
+				"container": container.ID,
+				"volume":    config.Volume.Name(),
+			}).Debug("Live-restoring volume for alive container")
+			if err := config.LiveRestore(context.TODO()); err != nil {
+				return err
+			}
+		}
 	}
 	return nil
 }

daemon/volumes.go

@@ -25,7 +25,8 @@ import (
 var (
 	// ErrVolumeReadonly is used to signal an error when trying to copy data into
 	// a volume mount that is not writable.
-	ErrVolumeReadonly = errors.New("mounted volume is marked read-only")
+	ErrVolumeReadonly                     = errors.New("mounted volume is marked read-only")
+	_                 volume.LiveRestorer = (*volumeWrapper)(nil)
 )
 
 type mounts []container.Mount
@@ -387,6 +388,7 @@ func (daemon *Daemon) VolumesService() *service.VolumesService {
 type volumeMounter interface {
 	Mount(ctx context.Context, v *types.Volume, ref string) (string, error)
 	Unmount(ctx context.Context, v *types.Volume, ref string) error
+	LiveRestoreVolume(ctx context.Context, v *types.Volume, ref string) error
 }
 
 type volumeWrapper struct {
@@ -421,3 +423,7 @@ func (v *volumeWrapper) CreatedAt() (time.Time, error) {
 func (v *volumeWrapper) Status() map[string]interface{} {
 	return v.v.Status
 }
+
+func (v *volumeWrapper) LiveRestoreVolume(ctx context.Context, ref string) error {
+	return v.s.LiveRestoreVolume(ctx, v.v, ref)
+}

integration/daemon/daemon_test.go

@@ -66,4 +66,40 @@ func testLiveRestoreVolumeReferences(t *testing.T) {
 		runTest(t, "on-failure")
 		runTest(t, "no")
 	})
+
+	// Make sure that the local volume driver's mount ref count is restored
+	// Addresses https://github.com/moby/moby/issues/44422
+	t.Run("local volume with mount options", func(t *testing.T) {
+		v, err := c.VolumeCreate(ctx, volume.VolumeCreateBody{
+			Driver: "local",
+			Name:   "test-live-restore-volume-references-local",
+			DriverOpts: map[string]string{
+				"type":   "tmpfs",
+				"device": "tmpfs",
+			},
+		})
+		assert.NilError(t, err)
+		m := mount.Mount{
+			Type:   mount.TypeVolume,
+			Source: v.Name,
+			Target: "/foo",
+		}
+		cID := container.Run(ctx, t, c, container.WithMount(m), container.WithCmd("top"))
+		defer c.ContainerRemove(ctx, cID, types.ContainerRemoveOptions{Force: true})
+
+		d.Restart(t, "--live-restore", "--iptables=false")
+
+		// Try to remove the volume
+		// This should fail since its used by a container
+		err = c.VolumeRemove(ctx, v.Name, false)
+		assert.ErrorContains(t, err, "volume is in use")
+
+		// Remove that container which should free the references in the volume
+		err = c.ContainerRemove(ctx, cID, types.ContainerRemoveOptions{Force: true})
+		assert.NilError(t, err)
+
+		// Now we should be able to remove the volume
+		err = c.VolumeRemove(ctx, v.Name, false)
+		assert.NilError(t, err)
+	})
 }

volume/local/local.go

@@ -4,6 +4,7 @@
 package local // import "github.com/docker/docker/volume/local"
 
 import (
+	"context"
 	"encoding/json"
 	"fmt"
 	"os"
@@ -36,6 +37,8 @@ var (
 	// This name is used to create the bind directory, so we need to avoid characters that
 	// would make the path to escape the root directory.
 	volumeNameRegex = names.RestrictedNamePattern
+
+	_ volume.LiveRestorer = (*localVolume)(nil)
 )
 
 type activeMount struct {
@@ -315,14 +318,17 @@ func (v *localVolume) CachedPath() string {
 func (v *localVolume) Mount(id string) (string, error) {
 	v.m.Lock()
 	defer v.m.Unlock()
+	logger := logrus.WithField("volume", v.name)
 	if v.needsMount() {
 		if !v.active.mounted {
+			logger.Debug("Mounting volume")
 			if err := v.mount(); err != nil {
 				return "", errdefs.System(err)
 			}
 			v.active.mounted = true
 		}
 		v.active.count++
+		logger.WithField("active mounts", v.active).Debug("Decremented active mount count")
 	}
 	if err := v.postMount(); err != nil {
 		return "", err
@@ -335,6 +341,7 @@ func (v *localVolume) Mount(id string) (string, error) {
 func (v *localVolume) Unmount(id string) error {
 	v.m.Lock()
 	defer v.m.Unlock()
+	logger := logrus.WithField("volume", v.name)
 
 	// Always decrement the count, even if the unmount fails
 	// Essentially docker doesn't care if this fails, it will send an error, but
@@ -342,12 +349,14 @@ func (v *localVolume) Unmount(id string) error {
 	// this volume can never be removed until a daemon restart occurs.
 	if v.needsMount() {
 		v.active.count--
+		logger.WithField("active mounts", v.active).Debug("Decremented active mount count")
 	}
 
 	if v.active.count > 0 {
 		return nil
 	}
 
+	logger.Debug("Unmounting volume")
 	return v.unmount()
 }
 
@@ -355,6 +364,24 @@ func (v *localVolume) Status() map[string]interface{} {
 	return nil
 }
 
+// LiveRestoreVolume restores reference counts for mounts
+// It is assumed that the volume is already mounted since this is only called for active, live-restored containers.
+func (v *localVolume) LiveRestoreVolume(ctx context.Context, _ string) error {
+	v.m.Lock()
+	defer v.m.Unlock()
+
+	if !v.needsMount() {
+		return nil
+	}
+	v.active.count++
+	v.active.mounted = true
+	logrus.WithFields(logrus.Fields{
+		"volume":        v.name,
+		"active mounts": v.active,
+	}).Debugf("Live restored volume")
+	return nil
+}
+
 // getAddress finds out address/hostname from options
 func getAddress(opts string) string {
 	optsList := strings.Split(opts, ",")

volume/mounts/mounts.go

@@ -1,6 +1,7 @@
 package mounts // import "github.com/docker/docker/volume/mounts"
 
 import (
+	"context"
 	"fmt"
 	"os"
 	"path/filepath"
@@ -12,6 +13,7 @@ import (
 	"github.com/docker/docker/volume"
 	"github.com/opencontainers/selinux/go-selinux/label"
 	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
 )
 
 // MountPoint is the intersection point between a volume and a container. It
@@ -179,3 +181,29 @@ func errInvalidMode(mode string) error {
 func errInvalidSpec(spec string) error {
 	return errors.Errorf("invalid volume specification: '%s'", spec)
 }
+
+func (m *MountPoint) LiveRestore(ctx context.Context) error {
+	if m.Volume == nil {
+		logrus.Debug("No volume to restore")
+		return nil
+	}
+
+	lrv, ok := m.Volume.(volume.LiveRestorer)
+	if !ok {
+		logrus.WithField("volume", m.Volume.Name()).Debugf("Volume does not support live restore: %T", m.Volume)
+		return nil
+	}
+
+	id := m.ID
+	if id == "" {
+		id = stringid.GenerateRandomID()
+	}
+
+	if err := lrv.LiveRestoreVolume(ctx, id); err != nil {
+		return errors.Wrapf(err, "error while restoring volume '%s'", m.Source)
+	}
+
+	m.ID = id
+	m.active++
+	return nil
+}

volume/service/service.go

@@ -265,3 +265,18 @@ func (s *VolumesService) List(ctx context.Context, filter filters.Args) (volumes
 func (s *VolumesService) Shutdown() error {
 	return s.vs.Shutdown()
 }
+
+// LiveRestoreVolume passes through the LiveRestoreVolume call to the volume if it is implemented
+// otherwise it is a no-op.
+func (s *VolumesService) LiveRestoreVolume(ctx context.Context, vol *types.Volume, ref string) error {
+	v, err := s.vs.Get(ctx, vol.Name, opts.WithGetDriver(vol.Driver))
+	if err != nil {
+		return err
+	}
+	rlv, ok := v.(volume.LiveRestorer)
+	if !ok {
+		logrus.WithField("volume", vol.Name).Debugf("volume does not implement LiveRestoreVolume: %T", v)
+		return nil
+	}
+	return rlv.LiveRestoreVolume(ctx, ref)
+}

volume/service/store.go

@@ -25,6 +25,8 @@ const (
 	volumeDataDir = "volumes"
 )
 
+var _ volume.LiveRestorer = (*volumeWrapper)(nil)
+
 type volumeWrapper struct {
 	volume.Volume
 	labels  map[string]string
@@ -68,6 +70,13 @@ func (v volumeWrapper) CachedPath() string {
 	return v.Volume.Path()
 }
 
+func (v volumeWrapper) LiveRestoreVolume(ctx context.Context, ref string) error {
+	if vv, ok := v.Volume.(volume.LiveRestorer); ok {
+		return vv.LiveRestoreVolume(ctx, ref)
+	}
+	return nil
+}
+
 // NewStore creates a new volume store at the given path
 func NewStore(rootPath string, drivers *drivers.Store) (*VolumeStore, error) {
 	vs := &VolumeStore{

volume/volume.go

@@ -1,6 +1,7 @@
 package volume // import "github.com/docker/docker/volume"
 
 import (
+	"context"
 	"time"
 )
 
@@ -67,3 +68,12 @@ type DetailedVolume interface {
 	Scope() string
 	Volume
 }
+
+// LiveRestorer is an optional interface that can be implemented by a volume driver
+// It is used to restore any resources that are necessary for a volume to be used by a live-restored container
+type LiveRestorer interface {
+	// LiveRestoreVolume allows a volume driver which implements this interface to restore any necessary resources (such as reference counting)
+	// This is called only after the daemon is restarted with live-restored containers
+	// It is called once per live-restored container.
+	LiveRestoreVolume(_ context.Context, ref string) error
+}