Merge pull request #22168 from cpuguy83/22116_hack_in_layer_refcounts

Add refcounts to graphdrivers that use fsdiff

daemon/graphdriver/counter.go

@@ -0,0 +1,32 @@
+package graphdriver
+
+import "sync"
+
+// RefCounter is a generic counter for use by graphdriver Get/Put calls
+type RefCounter struct {
+	counts map[string]int
+	mu     sync.Mutex
+}
+
+// NewRefCounter returns a new RefCounter
+func NewRefCounter() *RefCounter {
+	return &RefCounter{counts: make(map[string]int)}
+}
+
+// Increment increaes the ref count for the given id and returns the current count
+func (c *RefCounter) Increment(id string) int {
+	c.mu.Lock()
+	c.counts[id]++
+	count := c.counts[id]
+	c.mu.Unlock()
+	return count
+}
+
+// Decrement decreases the ref count for the given id and returns the current count
+func (c *RefCounter) Decrement(id string) int {
+	c.mu.Lock()
+	c.counts[id]--
+	count := c.counts[id]
+	c.mu.Unlock()
+	return count
+}

daemon/graphdriver/devmapper/driver.go

@@ -28,6 +28,7 @@ type Driver struct {
 	home    string
 	uidMaps []idtools.IDMap
 	gidMaps []idtools.IDMap
+	ctr     *graphdriver.RefCounter
 }
 
 // Init creates a driver with the given home and the set of options.
@@ -46,6 +47,7 @@ func Init(home string, options []string, uidMaps, gidMaps []idtools.IDMap) (grap
 		home:      home,
 		uidMaps:   uidMaps,
 		gidMaps:   gidMaps,
+		ctr:       graphdriver.NewRefCounter(),
 	}
 
 	return graphdriver.NewNaiveDiffDriver(d, uidMaps, gidMaps), nil
@@ -157,26 +159,35 @@ func (d *Driver) Remove(id string) error {
 // Get mounts a device with given id into the root filesystem
 func (d *Driver) Get(id, mountLabel string) (string, error) {
 	mp := path.Join(d.home, "mnt", id)
+	if count := d.ctr.Increment(id); count > 1 {
+		return mp, nil
+	}
 
 	uid, gid, err := idtools.GetRootUIDGID(d.uidMaps, d.gidMaps)
 	if err != nil {
+		d.ctr.Decrement(id)
 		return "", err
 	}
+
 	// Create the target directories if they don't exist
 	if err := idtools.MkdirAllAs(path.Join(d.home, "mnt"), 0755, uid, gid); err != nil && !os.IsExist(err) {
+		d.ctr.Decrement(id)
 		return "", err
 	}
 	if err := idtools.MkdirAs(mp, 0755, uid, gid); err != nil && !os.IsExist(err) {
+		d.ctr.Decrement(id)
 		return "", err
 	}
 
 	// Mount the device
 	if err := d.DeviceSet.MountDevice(id, mp, mountLabel); err != nil {
+		d.ctr.Decrement(id)
 		return "", err
 	}
 
 	rootFs := path.Join(mp, "rootfs")
 	if err := idtools.MkdirAllAs(rootFs, 0755, uid, gid); err != nil && !os.IsExist(err) {
+		d.ctr.Decrement(id)
 		d.DeviceSet.UnmountDevice(id, mp)
 		return "", err
 	}
@@ -186,6 +197,7 @@ func (d *Driver) Get(id, mountLabel string) (string, error) {
 		// Create an "id" file with the container/image id in it to help reconstruct this in case
 		// of later problems
 		if err := ioutil.WriteFile(idFile, []byte(id), 0600); err != nil {
+			d.ctr.Decrement(id)
 			d.DeviceSet.UnmountDevice(id, mp)
 			return "", err
 		}
@@ -196,6 +208,9 @@ func (d *Driver) Get(id, mountLabel string) (string, error) {
 
 // Put unmounts a device and removes it.
 func (d *Driver) Put(id string) error {
+	if count := d.ctr.Decrement(id); count > 0 {
+		return nil
+	}
 	mp := path.Join(d.home, "mnt", id)
 	err := d.DeviceSet.UnmountDevice(id, mp)
 	if err != nil {

daemon/graphdriver/overlay/overlay.go

@@ -96,6 +96,7 @@ type Driver struct {
 	pathCache     map[string]string
 	uidMaps       []idtools.IDMap
 	gidMaps       []idtools.IDMap
+	ctr           *graphdriver.RefCounter
 }
 
 var backingFs = "<unknown>"
@@ -155,6 +156,7 @@ func Init(home string, options []string, uidMaps, gidMaps []idtools.IDMap) (grap
 		pathCache: make(map[string]string),
 		uidMaps:   uidMaps,
 		gidMaps:   gidMaps,
+		ctr:       graphdriver.NewRefCounter(),
 	}
 
 	return NaiveDiffDriverWithApply(d, uidMaps, gidMaps), nil
@@ -368,28 +370,39 @@ func (d *Driver) Get(id string, mountLabel string) (string, error) {
 	workDir := path.Join(dir, "work")
 	mergedDir := path.Join(dir, "merged")
 
+	if count := d.ctr.Increment(id); count > 1 {
+		return mergedDir, nil
+	}
+
 	opts := fmt.Sprintf("lowerdir=%s,upperdir=%s,workdir=%s", lowerDir, upperDir, workDir)
 
 	// if it's mounted already, just return
 	mounted, err := d.mounted(mergedDir)
 	if err != nil {
+		d.ctr.Decrement(id)
 		return "", err
 	}
 	if mounted {
+		d.ctr.Decrement(id)
 		return mergedDir, nil
 	}
 
 	if err := syscall.Mount("overlay", mergedDir, "overlay", 0, label.FormatMountLabel(opts, mountLabel)); err != nil {
+		d.ctr.Decrement(id)
 		return "", fmt.Errorf("error creating overlay mount to %s: %v", mergedDir, err)
 	}
 	// chown "workdir/work" to the remapped root UID/GID. Overlay fs inside a
 	// user namespace requires this to move a directory from lower to upper.
 	rootUID, rootGID, err := idtools.GetRootUIDGID(d.uidMaps, d.gidMaps)
 	if err != nil {
+		d.ctr.Decrement(id)
+		syscall.Unmount(mergedDir, 0)
 		return "", err
 	}
 
 	if err := os.Chown(path.Join(workDir, "work"), rootUID, rootGID); err != nil {
+		d.ctr.Decrement(id)
+		syscall.Unmount(mergedDir, 0)
 		return "", err
 	}
 
@@ -406,6 +419,9 @@ func (d *Driver) mounted(dir string) (bool, error) {
 
 // Put unmounts the mount path created for the give id.
 func (d *Driver) Put(id string) error {
+	if count := d.ctr.Decrement(id); count > 0 {
+		return nil
+	}
 	d.pathCacheLock.Lock()
 	mountpoint, exists := d.pathCache[id]
 	d.pathCacheLock.Unlock()

daemon/graphdriver/zfs/zfs.go

@@ -105,6 +105,7 @@ func Init(base string, opt []string, uidMaps, gidMaps []idtools.IDMap) (graphdri
 		filesystemsCache: filesystemsCache,
 		uidMaps:          uidMaps,
 		gidMaps:          gidMaps,
+		ctr:              graphdriver.NewRefCounter(),
 	}
 	return graphdriver.NewNaiveDiffDriver(d, uidMaps, gidMaps), nil
 }
@@ -161,6 +162,7 @@ type Driver struct {
 	filesystemsCache map[string]bool
 	uidMaps          []idtools.IDMap
 	gidMaps          []idtools.IDMap
+	ctr              *graphdriver.RefCounter
 }
 
 func (d *Driver) String() string {
@@ -305,25 +307,35 @@ func (d *Driver) Remove(id string) error {
 // Get returns the mountpoint for the given id after creating the target directories if necessary.
 func (d *Driver) Get(id, mountLabel string) (string, error) {
 	mountpoint := d.mountPath(id)
+	if count := d.ctr.Increment(id); count > 1 {
+		return mountpoint, nil
+	}
+
 	filesystem := d.zfsPath(id)
 	options := label.FormatMountLabel("", mountLabel)
 	logrus.Debugf(`[zfs] mount("%s", "%s", "%s")`, filesystem, mountpoint, options)
 
 	rootUID, rootGID, err := idtools.GetRootUIDGID(d.uidMaps, d.gidMaps)
 	if err != nil {
+		d.ctr.Decrement(id)
 		return "", err
 	}
 	// Create the target directories if they don't exist
 	if err := idtools.MkdirAllAs(mountpoint, 0755, rootUID, rootGID); err != nil {
+		d.ctr.Decrement(id)
 		return "", err
 	}
 
 	if err := mount.Mount(filesystem, mountpoint, "zfs", options); err != nil {
+		d.ctr.Decrement(id)
 		return "", fmt.Errorf("error creating zfs mount of %s to %s: %v", filesystem, mountpoint, err)
 	}
+
 	// this could be our first mount after creation of the filesystem, and the root dir may still have root
 	// permissions instead of the remapped root uid:gid (if user namespaces are enabled):
 	if err := os.Chown(mountpoint, rootUID, rootGID); err != nil {
+		mount.Unmount(mountpoint)
+		d.ctr.Decrement(id)
 		return "", fmt.Errorf("error modifying zfs mountpoint (%s) directory ownership: %v", mountpoint, err)
 	}
 
@@ -332,6 +344,9 @@ func (d *Driver) Get(id, mountLabel string) (string, error) {
 
 // Put removes the existing mountpoint for the given id if it exists.
 func (d *Driver) Put(id string) error {
+	if count := d.ctr.Decrement(id); count > 0 {
+		return nil
+	}
 	mountpoint := d.mountPath(id)
 	mounted, err := graphdriver.Mounted(graphdriver.FsMagicZfs, mountpoint)
 	if err != nil || !mounted {