diff --git a/plugin/backend_linux.go b/plugin/backend_linux.go
index 33200d8efa..a5a3f9b5b2 100644
--- a/plugin/backend_linux.go
+++ b/plugin/backend_linux.go
@@ -575,7 +575,7 @@ func (pm *Manager) Remove(name string, config *types.PluginRmConfig) error {
 func getMounts(root string) ([]string, error) {
 	infos, err := mount.GetMounts()
 	if err != nil {
-		return nil, errors.Wrap(err, "failed to read mount table while performing recursive unmount")
+		return nil, errors.Wrap(err, "failed to read mount table")
 	}
 
 	var mounts []string
diff --git a/plugin/manager_linux.go b/plugin/manager_linux.go
index ad66616628..7e734b72d4 100644
--- a/plugin/manager_linux.go
+++ b/plugin/manager_linux.go
@@ -171,9 +171,17 @@ func (pm *Manager) upgradePlugin(p *v2.Plugin, configDigest digest.Digest, blobs
 
 	pdir := filepath.Join(pm.config.Root, p.PluginObj.ID)
 	orig := filepath.Join(pdir, "rootfs")
+
+	// Make sure nothing is mounted
+	// This could happen if the plugin was disabled with `-f` with active mounts.
+	// If there is anything in `orig` is still mounted, this should error out.
+	if err := recursiveUnmount(orig); err != nil {
+		return err
+	}
+
 	backup := orig + "-old"
 	if err := os.Rename(orig, backup); err != nil {
-		return err
+		return errors.Wrap(err, "error backing up plugin data before upgrade")
 	}
 
 	defer func() {