Merge pull request #39270 from kolyshkin/moar-aufs-fixes

aufs: retry umount on ebusy, ignore ENOENT in graphdriver.Mounted

daemon/graphdriver/aufs/aufs.go

@@ -34,7 +34,6 @@ import (
 	"path/filepath"
 	"strings"
 	"sync"
-	"time"
 
 	"github.com/docker/docker/daemon/graphdriver"
 	"github.com/docker/docker/pkg/archive"
@@ -307,36 +306,9 @@ func (a *Driver) Remove(id string) error {
 		mountpoint = a.getMountpoint(id)
 	}
 
-	logger := logger.WithField("layer", id)
-
-	var retries int
-	for {
-		mounted, err := a.mounted(mountpoint)
-		if err != nil {
-			if os.IsNotExist(err) {
-				break
-			}
-			return err
-		}
-		if !mounted {
-			break
-		}
-
-		err = a.unmount(mountpoint)
-		if err == nil {
-			break
-		}
-
-		if errors.Cause(err) != unix.EBUSY {
-			return errors.Wrap(err, "aufs: unmount error")
-		}
-		if retries >= 5 {
-			return errors.Wrap(err, "aufs: unmount error after retries")
-		}
-		// If unmount returns EBUSY, it could be a transient error. Sleep and retry.
-		retries++
-		logger.Warnf("unmount failed due to EBUSY: retry count: %d", retries)
-		time.Sleep(100 * time.Millisecond)
+	if err := a.unmount(mountpoint); err != nil {
+		logger.WithError(err).WithField("method", "Remove()").Warn()
+		return err
 	}
 
 	// Remove the layers file for the id

daemon/graphdriver/aufs/mount.go

@@ -5,17 +5,18 @@ package aufs // import "github.com/docker/docker/daemon/graphdriver/aufs"
 import (
 	"os/exec"
 	"syscall"
+	"time"
 
 	"github.com/docker/docker/pkg/mount"
+	"github.com/pkg/errors"
+	"golang.org/x/sys/unix"
 )
 
 // Unmount the target specified.
 func Unmount(target string) error {
-	const (
-		EINVAL  = 22 // if auplink returns this,
-		retries = 3  // retry a few times
-	)
+	const retries = 5
 
+	// auplink flush
 	for i := 0; ; i++ {
 		out, err := exec.Command("auplink", target, "flush").CombinedOutput()
 		if err == nil {
@@ -27,7 +28,7 @@ func Unmount(target string) error {
 				rc = status.ExitStatus()
 			}
 		}
-		if i >= retries || rc != EINVAL {
+		if i >= retries || rc != int(unix.EINVAL) {
 			logger.WithError(err).WithField("method", "Unmount").Warnf("auplink flush failed: %s", out)
 			break
 		}
@@ -37,5 +38,22 @@ func Unmount(target string) error {
 		logger.Debugf("auplink flush error (retrying %d/%d): %s", i+1, retries, out)
 	}
 
-	return mount.Unmount(target)
+	// unmount
+	var err error
+	for i := 0; i < retries; i++ {
+		err = mount.Unmount(target)
+		switch errors.Cause(err) {
+		case nil:
+			return nil
+		case unix.EBUSY:
+			logger.Debugf("aufs unmount %s failed with EBUSY (retrying %d/%d)", target, i+1, retries)
+			time.Sleep(100 * time.Millisecond)
+			continue // try again
+		default:
+			// any other error is fatal
+			break
+		}
+	}
+
+	return err
 }

daemon/graphdriver/driver_linux.go

@@ -118,6 +118,9 @@ func (c *defaultChecker) IsMounted(path string) bool {
 func Mounted(fsType FsMagic, mountPath string) (bool, error) {
 	var buf unix.Statfs_t
 	if err := unix.Statfs(mountPath, &buf); err != nil {
+		if err == unix.ENOENT { // not exist, thus not mounted
+			err = nil
+		}
 		return false, err
 	}
 	return FsMagic(buf.Type) == fsType, nil