diff --git a/daemon/mounts.go b/daemon/mounts.go index 383a38e7eb..424e375037 100644 --- a/daemon/mounts.go +++ b/daemon/mounts.go @@ -8,13 +8,24 @@ import ( mounttypes "github.com/docker/docker/api/types/mount" "github.com/docker/docker/container" volumesservice "github.com/docker/docker/volume/service" + "github.com/sirupsen/logrus" ) func (daemon *Daemon) prepareMountPoints(container *container.Container) error { + alive := container.IsRunning() for _, config := range container.MountPoints { if err := daemon.lazyInitializeVolume(container.ID, config); err != nil { return err } + if alive { + logrus.WithFields(logrus.Fields{ + "container": container.ID, + "volume": config.Volume.Name(), + }).Debug("Live-restoring volume for alive container") + if err := config.LiveRestore(context.TODO()); err != nil { + return err + } + } } return nil } diff --git a/daemon/volumes.go b/daemon/volumes.go index c396843831..dfd3352c03 100644 --- a/daemon/volumes.go +++ b/daemon/volumes.go @@ -25,7 +25,8 @@ import ( var ( // ErrVolumeReadonly is used to signal an error when trying to copy data into // a volume mount that is not writable. - ErrVolumeReadonly = errors.New("mounted volume is marked read-only") + ErrVolumeReadonly = errors.New("mounted volume is marked read-only") + _ volume.LiveRestorer = (*volumeWrapper)(nil) ) type mounts []container.Mount @@ -387,6 +388,7 @@ func (daemon *Daemon) VolumesService() *service.VolumesService { type volumeMounter interface { Mount(ctx context.Context, v *types.Volume, ref string) (string, error) Unmount(ctx context.Context, v *types.Volume, ref string) error + LiveRestoreVolume(ctx context.Context, v *types.Volume, ref string) error } type volumeWrapper struct { @@ -421,3 +423,7 @@ func (v *volumeWrapper) CreatedAt() (time.Time, error) { func (v *volumeWrapper) Status() map[string]interface{} { return v.v.Status } + +func (v *volumeWrapper) LiveRestoreVolume(ctx context.Context, ref string) error { + return v.s.LiveRestoreVolume(ctx, v.v, ref) +} diff --git a/integration/daemon/daemon_test.go b/integration/daemon/daemon_test.go index b1ad668dfe..47dac3b762 100644 --- a/integration/daemon/daemon_test.go +++ b/integration/daemon/daemon_test.go @@ -66,4 +66,40 @@ func testLiveRestoreVolumeReferences(t *testing.T) { runTest(t, "on-failure") runTest(t, "no") }) + + // Make sure that the local volume driver's mount ref count is restored + // Addresses https://github.com/moby/moby/issues/44422 + t.Run("local volume with mount options", func(t *testing.T) { + v, err := c.VolumeCreate(ctx, volume.VolumeCreateBody{ + Driver: "local", + Name: "test-live-restore-volume-references-local", + DriverOpts: map[string]string{ + "type": "tmpfs", + "device": "tmpfs", + }, + }) + assert.NilError(t, err) + m := mount.Mount{ + Type: mount.TypeVolume, + Source: v.Name, + Target: "/foo", + } + cID := container.Run(ctx, t, c, container.WithMount(m), container.WithCmd("top")) + defer c.ContainerRemove(ctx, cID, types.ContainerRemoveOptions{Force: true}) + + d.Restart(t, "--live-restore", "--iptables=false") + + // Try to remove the volume + // This should fail since its used by a container + err = c.VolumeRemove(ctx, v.Name, false) + assert.ErrorContains(t, err, "volume is in use") + + // Remove that container which should free the references in the volume + err = c.ContainerRemove(ctx, cID, types.ContainerRemoveOptions{Force: true}) + assert.NilError(t, err) + + // Now we should be able to remove the volume + err = c.VolumeRemove(ctx, v.Name, false) + assert.NilError(t, err) + }) } diff --git a/volume/local/local.go b/volume/local/local.go index 29e3cc9a54..abb885cc24 100644 --- a/volume/local/local.go +++ b/volume/local/local.go @@ -4,6 +4,7 @@ package local // import "github.com/docker/docker/volume/local" import ( + "context" "encoding/json" "fmt" "os" @@ -36,6 +37,8 @@ var ( // This name is used to create the bind directory, so we need to avoid characters that // would make the path to escape the root directory. volumeNameRegex = names.RestrictedNamePattern + + _ volume.LiveRestorer = (*localVolume)(nil) ) type activeMount struct { @@ -315,14 +318,17 @@ func (v *localVolume) CachedPath() string { func (v *localVolume) Mount(id string) (string, error) { v.m.Lock() defer v.m.Unlock() + logger := logrus.WithField("volume", v.name) if v.needsMount() { if !v.active.mounted { + logger.Debug("Mounting volume") if err := v.mount(); err != nil { return "", errdefs.System(err) } v.active.mounted = true } v.active.count++ + logger.WithField("active mounts", v.active).Debug("Decremented active mount count") } if err := v.postMount(); err != nil { return "", err @@ -335,6 +341,7 @@ func (v *localVolume) Mount(id string) (string, error) { func (v *localVolume) Unmount(id string) error { v.m.Lock() defer v.m.Unlock() + logger := logrus.WithField("volume", v.name) // Always decrement the count, even if the unmount fails // Essentially docker doesn't care if this fails, it will send an error, but @@ -342,12 +349,14 @@ func (v *localVolume) Unmount(id string) error { // this volume can never be removed until a daemon restart occurs. if v.needsMount() { v.active.count-- + logger.WithField("active mounts", v.active).Debug("Decremented active mount count") } if v.active.count > 0 { return nil } + logger.Debug("Unmounting volume") return v.unmount() } @@ -355,6 +364,24 @@ func (v *localVolume) Status() map[string]interface{} { return nil } +// LiveRestoreVolume restores reference counts for mounts +// It is assumed that the volume is already mounted since this is only called for active, live-restored containers. +func (v *localVolume) LiveRestoreVolume(ctx context.Context, _ string) error { + v.m.Lock() + defer v.m.Unlock() + + if !v.needsMount() { + return nil + } + v.active.count++ + v.active.mounted = true + logrus.WithFields(logrus.Fields{ + "volume": v.name, + "active mounts": v.active, + }).Debugf("Live restored volume") + return nil +} + // getAddress finds out address/hostname from options func getAddress(opts string) string { optsList := strings.Split(opts, ",") diff --git a/volume/mounts/mounts.go b/volume/mounts/mounts.go index c441e51ed9..d1897d7928 100644 --- a/volume/mounts/mounts.go +++ b/volume/mounts/mounts.go @@ -1,6 +1,7 @@ package mounts // import "github.com/docker/docker/volume/mounts" import ( + "context" "fmt" "os" "path/filepath" @@ -12,6 +13,7 @@ import ( "github.com/docker/docker/volume" "github.com/opencontainers/selinux/go-selinux/label" "github.com/pkg/errors" + "github.com/sirupsen/logrus" ) // MountPoint is the intersection point between a volume and a container. It @@ -179,3 +181,29 @@ func errInvalidMode(mode string) error { func errInvalidSpec(spec string) error { return errors.Errorf("invalid volume specification: '%s'", spec) } + +func (m *MountPoint) LiveRestore(ctx context.Context) error { + if m.Volume == nil { + logrus.Debug("No volume to restore") + return nil + } + + lrv, ok := m.Volume.(volume.LiveRestorer) + if !ok { + logrus.WithField("volume", m.Volume.Name()).Debugf("Volume does not support live restore: %T", m.Volume) + return nil + } + + id := m.ID + if id == "" { + id = stringid.GenerateRandomID() + } + + if err := lrv.LiveRestoreVolume(ctx, id); err != nil { + return errors.Wrapf(err, "error while restoring volume '%s'", m.Source) + } + + m.ID = id + m.active++ + return nil +} diff --git a/volume/service/service.go b/volume/service/service.go index b185471a39..24bff5d807 100644 --- a/volume/service/service.go +++ b/volume/service/service.go @@ -265,3 +265,18 @@ func (s *VolumesService) List(ctx context.Context, filter filters.Args) (volumes func (s *VolumesService) Shutdown() error { return s.vs.Shutdown() } + +// LiveRestoreVolume passes through the LiveRestoreVolume call to the volume if it is implemented +// otherwise it is a no-op. +func (s *VolumesService) LiveRestoreVolume(ctx context.Context, vol *types.Volume, ref string) error { + v, err := s.vs.Get(ctx, vol.Name, opts.WithGetDriver(vol.Driver)) + if err != nil { + return err + } + rlv, ok := v.(volume.LiveRestorer) + if !ok { + logrus.WithField("volume", vol.Name).Debugf("volume does not implement LiveRestoreVolume: %T", v) + return nil + } + return rlv.LiveRestoreVolume(ctx, ref) +} diff --git a/volume/service/store.go b/volume/service/store.go index b06890c2d2..361c201370 100644 --- a/volume/service/store.go +++ b/volume/service/store.go @@ -25,6 +25,8 @@ const ( volumeDataDir = "volumes" ) +var _ volume.LiveRestorer = (*volumeWrapper)(nil) + type volumeWrapper struct { volume.Volume labels map[string]string @@ -68,6 +70,13 @@ func (v volumeWrapper) CachedPath() string { return v.Volume.Path() } +func (v volumeWrapper) LiveRestoreVolume(ctx context.Context, ref string) error { + if vv, ok := v.Volume.(volume.LiveRestorer); ok { + return vv.LiveRestoreVolume(ctx, ref) + } + return nil +} + // NewStore creates a new volume store at the given path func NewStore(rootPath string, drivers *drivers.Store) (*VolumeStore, error) { vs := &VolumeStore{ diff --git a/volume/volume.go b/volume/volume.go index 61c8243979..df641675ae 100644 --- a/volume/volume.go +++ b/volume/volume.go @@ -1,6 +1,7 @@ package volume // import "github.com/docker/docker/volume" import ( + "context" "time" ) @@ -67,3 +68,12 @@ type DetailedVolume interface { Scope() string Volume } + +// LiveRestorer is an optional interface that can be implemented by a volume driver +// It is used to restore any resources that are necessary for a volume to be used by a live-restored container +type LiveRestorer interface { + // LiveRestoreVolume allows a volume driver which implements this interface to restore any necessary resources (such as reference counting) + // This is called only after the daemon is restarted with live-restored containers + // It is called once per live-restored container. + LiveRestoreVolume(_ context.Context, ref string) error +}