devmapper: Provide option to enabled deferred device deletion

Provide a command line option dm.use_deferred_deletion to enable deferred
device deletion feature. By default feature will be turned off.

Not sure if there is much value in deferred deletion being turned on
without deferred removal being turned on. So for now, this feature can
be enabled only if deferred removal is on.

Signed-off-by: Vivek Goyal <vgoyal@redhat.com>

daemon/graphdriver/devmapper/deviceset.go

@@ -40,6 +40,7 @@ var (
 	logLevel                     = devicemapper.LogLevelFatal
 	driverDeferredRemovalSupport = false
 	enableDeferredRemoval        = false
+	enableDeferredDeletion       = false
 )
 
 const deviceSetMetaFile string = "deviceset-metadata"
@@ -106,6 +107,7 @@ type DeviceSet struct {
 	transaction           `json:"-"`
 	overrideUdevSyncCheck bool
 	deferredRemove        bool   // use deferred removal
+	deferredDelete        bool   // use deferred deletion
 	BaseDeviceUUID        string //save UUID of base device
 }
 
@@ -141,6 +143,12 @@ type Status struct {
 	UdevSyncSupported bool
 	// DeferredRemoveEnabled is true then the device is not unmounted.
 	DeferredRemoveEnabled bool
+	// True if deferred deletion is enabled. This is different from
+	// deferred removal. "removal" means that device mapper device is
+	// deactivated. Thin device is still in thin pool and can be activated
+	// again. But "deletion" means that thin device will be deleted from
+	// thin pool and it can't be activated again.
+	DeferredDeleteEnabled bool
 }
 
 // Structure used to export image/container metadata in docker inspect.
@@ -1314,13 +1322,27 @@ func (devices *DeviceSet) initDevmapper(doInit bool) error {
 		return graphdriver.ErrNotSupported
 	}
 
-	// If user asked for deferred removal and both library and driver
-	// supports deferred removal use it.
-	if enableDeferredRemoval && driverDeferredRemovalSupport && devicemapper.LibraryDeferredRemovalSupport == true {
+	// If user asked for deferred removal then check both libdm library
+	// and kernel driver support deferred removal otherwise error out.
+	if enableDeferredRemoval {
+		if !driverDeferredRemovalSupport {
+			return fmt.Errorf("devmapper: Deferred removal can not be enabled as kernel does not support it")
+		}
+		if !devicemapper.LibraryDeferredRemovalSupport {
+			return fmt.Errorf("devmapper: Deferred removal can not be enabled as libdm does not support it")
+		}
 		logrus.Debugf("devmapper: Deferred removal support enabled.")
 		devices.deferredRemove = true
 	}
 
+	if enableDeferredDeletion {
+		if !devices.deferredRemove {
+			return fmt.Errorf("devmapper: Deferred deletion can not be enabled as deferred removal is not enabled. Enable deferred removal using --storage-opt dm.use_deferred_removal=true parameter")
+		}
+		logrus.Debugf("devmapper: Deferred deletion support enabled.")
+		devices.deferredDelete = true
+	}
+
 	// https://github.com/docker/docker/issues/4036
 	if supported := devicemapper.UdevSetSyncSupport(true); !supported {
 		logrus.Warn("Udev sync is not supported. This will lead to unexpected behavior, data loss and errors. For more information, see https://docs.docker.com/reference/commandline/daemon/#daemon-storage-driver-option")
@@ -1996,6 +2018,7 @@ func (devices *DeviceSet) Status() *Status {
 	status.MetadataLoopback = devices.metadataLoopFile
 	status.UdevSyncSupported = devicemapper.UdevSyncSupported()
 	status.DeferredRemoveEnabled = devices.deferredRemove
+	status.DeferredDeleteEnabled = devices.deferredDelete
 
 	totalSizeInSectors, _, dataUsed, dataTotal, metadataUsed, metadataTotal, err := devices.poolStatus()
 	if err == nil {
@@ -2128,6 +2151,12 @@ func NewDeviceSet(root string, doInit bool, options []string) (*DeviceSet, error
 				return nil, err
 			}
 
+		case "dm.use_deferred_deletion":
+			enableDeferredDeletion, err = strconv.ParseBool(val)
+			if err != nil {
+				return nil, err
+			}
+
 		default:
 			return nil, fmt.Errorf("Unknown option %s\n", key)
 		}

daemon/graphdriver/devmapper/driver.go

@@ -84,6 +84,7 @@ func (d *Driver) Status() [][2]string {
 		{"Metadata Space Available", fmt.Sprintf("%s", units.HumanSize(float64(s.Metadata.Available)))},
 		{"Udev Sync Supported", fmt.Sprintf("%v", s.UdevSyncSupported)},
 		{"Deferred Removal Enabled", fmt.Sprintf("%v", s.DeferredRemoveEnabled)},
+		{"Deferred Deletion Enabled", fmt.Sprintf("%v", s.DeferredDeleteEnabled)},
 	}
 	if len(s.DataLoopback) > 0 {
 		status = append(status, [2]string{"Data loop file", s.DataLoopback})

docs/reference/commandline/daemon.md

@@ -368,6 +368,46 @@ options for `zfs` start with `zfs`.
     > Otherwise, set this flag for migrating existing Docker daemons to
     > a daemon with a supported environment.
 
+ *  `dm.use_deferred_removal`
+
+    Enables use of deferred device removal if `libdm` and the kernel driver
+    support the mechanism.
+
+    Deferred device removal means that if device is busy when devices are
+    being removed/deactivated, then a deferred removal is scheduled on
+    device. And devices automatically go away when last user of the device
+    exits.
+
+    For example, when a container exits, its associated thin device is removed.
+    If that device has leaked into some other mount namespace and can't be
+    removed, the container exit still succeeds and this option causes the
+    system to schedule the device for deferred removal. It does not wait in a
+    loop trying to remove a busy device.
+
+    Example use: `docker daemon --storage-opt dm.use_deferred_removal=true`
+
+ *  `dm.use_deferred_deletion`
+
+    Enables use of deferred device deletion for thin pool devices. By default,
+    thin pool device deletion is synchronous. Before a container is deleted,
+    the Docker daemon removes any associated devices. If the storage driver
+    can not remove a device, the container deletion fails and daemon returns.
+
+    `Error deleting container: Error response from daemon: Cannot destroy container`
+
+    To avoid this failure, enable both deferred device deletion and deferred
+    device removal on the daemon.
+
+    `docker daemon --storage-opt dm.use_deferred_deletion=true --storage-opt dm.use_deferred_removal=true`
+
+    With these two options enabled, if a device is busy when the driver is
+    deleting a container, the driver marks the device as deleted. Later, when
+    the device isn't in use, the driver deletes it.
+
+    In general it should be safe to enable this option by default. It will help
+    when unintentional leaking of mount point happens across multiple mount
+    namespaces.
+
 Currently supported options of `zfs`:
 
  * `zfs.fsname`

man/docker-daemon.8.md

@@ -302,6 +302,28 @@ device.
 
 Example use: `docker daemon --storage-opt dm.use_deferred_removal=true`
 
+#### dm.use_deferred_deletion
+
+Enables use of deferred device deletion for thin pool devices. By default,
+thin pool device deletion is synchronous. Before a container is deleted, the
+Docker daemon removes any associated devices. If the storage driver can not
+remove a device, the container deletion fails and daemon returns.
+
+`Error deleting container: Error response from daemon: Cannot destroy container`
+
+To avoid this failure, enable both deferred device deletion and deferred
+device removal on the daemon.
+
+`docker daemon --storage-opt dm.use_deferred_deletion=true --storage-opt dm.use_deferred_removal=true`
+
+With these two options enabled, if a device is busy when the driver is
+deleting a container, the driver marks the device as deleted. Later, when the
+device isn't in use, the driver deletes it.
+
+In general it should be safe to enable this option by default. It will help
+when unintentional leaking of mount point happens across multiple mount
+namespaces.
+
 #### dm.loopdatasize
 
 **Note**: This option configures devicemapper loopback, which should not be used in production.