diff --git a/daemon/graphdriver/devmapper/deviceset.go b/daemon/graphdriver/devmapper/deviceset.go
index 0856758cb7..99bffacd53 100644
--- a/daemon/graphdriver/devmapper/deviceset.go
+++ b/daemon/graphdriver/devmapper/deviceset.go
@@ -1287,12 +1287,20 @@ func (devices *DeviceSet) deactivateDevice(info *DevInfo) error {
 	if err != nil {
 		return err
 	}
-	if devinfo.Exists != 0 {
+
+	if devinfo.Exists == 0 {
+		return nil
+	}
+
+	if devices.deferredRemove {
+		if err := devicemapper.RemoveDeviceDeferred(info.Name()); err != nil {
+			return err
+		}
+	} else {
 		if err := devices.removeDevice(info.Name()); err != nil {
 			return err
 		}
 	}
-
 	return nil
 }