plugins: container-rootfs-relative paths

Legacy plugins expect host-relative paths (such as for Volume.Mount).
However, a containerized plugin cannot respond with a host-relative
path. Therefore, this commit modifies new volume plugins' paths in Mount
and List to prepend the container's rootfs path.

This introduces a new PropagatedMount field in the Plugin Config.
When it is set for volume plugins, RootfsPropagation is set to rshared
and the path specified by PropagatedMount is bind-mounted with rshared
prior to launching the container. This is so that the daemon code can
access the paths returned by the plugin from the host mount namespace.

Signed-off-by: Tibor Vass <tibor@docker.com>

api/swagger.yaml

@@ -1383,6 +1383,7 @@ definitions:
           - Workdir
           - Network
           - Linux
+          - PropagatedMount
           - Mounts
           - Env
           - Args
@@ -1447,6 +1448,9 @@ definitions:
                 type: "array"
                 items:
                   $ref: "#/definitions/PluginDevice"
+          PropagatedMount:
+            type: "string"
+            x-nullable: false
           Mounts:
             type: "array"
             items:

api/types/plugin.go

@@ -71,6 +71,10 @@ type PluginConfig struct {
 	// Required: true
 	Network PluginConfigNetwork `json:"Network"`
 
+	// propagated mount
+	// Required: true
+	PropagatedMount string `json:"PropagatedMount"`
+
 	// user
 	User PluginConfigUser `json:"User,omitempty"`
 

docs/extend/config.md

@@ -111,6 +111,10 @@ Config provides the base accessible fields for working with V0 plugin format
 
 	  options of the mount.
 
+- **`propagatedMount`** *string*
+
+   path to be mounted as rshared, so that mounts under that path are visible to docker. This is useful for volume plugins.
+
 - **`env`** *PluginEnv array*
 
    env of the plugin, struct consisting of the following fields

docs/extend/plugins_volume.md

@@ -22,6 +22,10 @@ beyond the lifetime of a single Engine host. See the
 
 ## Changelog
 
+### 1.13.0
+
+- If used as part of the v2 plugin architecture, mountpoints that are part of paths returned by plugin have to be mounted under the directory specified by PropagatedMount in the plugin configuration [#26398](https://github.com/docker/docker/pull/26398)
+
 ### 1.12.0
 
 - Add `Status` field to `VolumeDriver.Get` response ([#21006](https://github.com/docker/docker/pull/21006#))

integration-cli/daemon.go

@@ -88,6 +88,11 @@ func NewDaemon(c *check.C) *Daemon {
 	}
 }
 
+// RootDir returns the root directory of the daemon.
+func (d *Daemon) RootDir() string {
+	return d.root
+}
+
 func (d *Daemon) getClientConfig() (*clientConfig, error) {
 	var (
 		transport *http.Transport

integration-cli/docker_cli_daemon_plugins_test.go

@@ -10,6 +10,7 @@ import (
 	"syscall"
 
 	"github.com/docker/docker/pkg/integration/checker"
+	"github.com/docker/docker/pkg/mount"
 	"github.com/go-check/check"
 )
 
@@ -186,7 +187,6 @@ func (s *DockerDaemonSuite) TestVolumePlugin(c *check.C) {
 	testRequires(c, Network, IsAmd64)
 
 	volName := "plugin-volume"
-	volRoot := "/data"
 	destDir := "/tmp/data/"
 	destFile := "foo"
 
@@ -197,13 +197,25 @@ func (s *DockerDaemonSuite) TestVolumePlugin(c *check.C) {
 	if err != nil {
 		c.Fatalf("Could not install plugin: %v %s", err, out)
 	}
+	pluginID, err := s.d.Cmd("plugin", "inspect", "-f", "{{.Id}}", pName)
+	pluginID = strings.TrimSpace(pluginID)
+	if err != nil {
+		c.Fatalf("Could not retrieve plugin ID: %v %s", err, pluginID)
+	}
+	mountpointPrefix := filepath.Join(s.d.RootDir(), "plugins", pluginID, "rootfs")
 	defer func() {
 		if out, err := s.d.Cmd("plugin", "disable", pName); err != nil {
 			c.Fatalf("Could not disable plugin: %v %s", err, out)
 		}
+
 		if out, err := s.d.Cmd("plugin", "remove", pName); err != nil {
 			c.Fatalf("Could not remove plugin: %v %s", err, out)
 		}
+
+		exists, err := existsMountpointWithPrefix(mountpointPrefix)
+		c.Assert(err, checker.IsNil)
+		c.Assert(exists, checker.Equals, false)
+
 	}()
 
 	out, err = s.d.Cmd("volume", "create", "-d", pName, volName)
@@ -231,11 +243,24 @@ func (s *DockerDaemonSuite) TestVolumePlugin(c *check.C) {
 
 	out, err = s.d.Cmd("run", "--rm", "-v", volName+":"+destDir, "busybox", "touch", destDir+destFile)
 	c.Assert(err, checker.IsNil, check.Commentf(out))
-	path := filepath.Join(mountPoint, destFile)
+	path := filepath.Join(s.d.RootDir(), "plugins", pluginID, "rootfs", mountPoint, destFile)
 	_, err = os.Lstat(path)
 	c.Assert(err, checker.IsNil)
 
-	// tiborvass/no-remove is a volume plugin that persists data on disk at /data,
-	// even after the volume is removed. So perform an explicit filesystem cleanup.
-	os.RemoveAll(volRoot)
+	exists, err := existsMountpointWithPrefix(mountpointPrefix)
+	c.Assert(err, checker.IsNil)
+	c.Assert(exists, checker.Equals, true)
+}
+
+func existsMountpointWithPrefix(mountpointPrefix string) (bool, error) {
+	mounts, err := mount.GetMounts()
+	if err != nil {
+		return false, err
+	}
+	for _, mnt := range mounts {
+		if strings.HasPrefix(mnt.Mountpoint, mountpointPrefix) {
+			return true, nil
+		}
+	}
+	return false, nil
 }

integration-cli/docker_cli_plugins_test.go

@@ -11,8 +11,8 @@ import (
 )
 
 var (
-	pluginProcessName = "no-remove"
-	pName             = "tiborvass/no-remove"
+	pluginProcessName = "sample-volume-plugin"
+	pName             = "tiborvass/sample-volume-plugin"
 	pTag              = "latest"
 	pNameWithTag      = pName + ":" + pTag
 )
@@ -33,6 +33,7 @@ func (s *DockerSuite) TestPluginBasicOps(c *check.C) {
 	c.Assert(err, checker.IsNil)
 
 	out, _, err = dockerCmdWithError("plugin", "remove", pNameWithTag)
+	c.Assert(err, checker.NotNil)
 	c.Assert(out, checker.Contains, "is enabled")
 
 	_, _, err = dockerCmdWithError("plugin", "disable", pNameWithTag)

pkg/plugingetter/getter.go

@@ -15,6 +15,7 @@ const (
 type CompatPlugin interface {
 	Client() *plugins.Client
 	Name() string
+	BasePath() string
 	IsV1() bool
 }
 

pkg/plugins/plugins.go

@@ -78,6 +78,12 @@ type Plugin struct {
 	activateWait *sync.Cond
 }
 
+// BasePath returns the path to which all paths returned by the plugin are relative to.
+// For v1 plugins, this always returns the host's root directory.
+func (p *Plugin) BasePath() string {
+	return "/"
+}
+
 // Name returns the name of the plugin.
 func (p *Plugin) Name() string {
 	return p.name

plugin/backend_linux.go

@@ -255,7 +255,7 @@ func (pm *Manager) Push(name string, metaHeader http.Header, authConfig *types.A
 		return err
 	}
 
-	rootfs, err := archive.Tar(filepath.Join(dest, "rootfs"), archive.Gzip)
+	rootfs, err := archive.Tar(p.Rootfs, archive.Gzip)
 	if err != nil {
 		return err
 	}
@@ -293,9 +293,13 @@ func (pm *Manager) Remove(name string, config *types.PluginRmConfig) error {
 		}
 	}
 
+	id := p.GetID()
 	pm.pluginStore.Remove(p)
-	os.RemoveAll(filepath.Join(pm.libRoot, p.GetID()))
-	pm.pluginEventLogger(p.GetID(), name, "remove")
+	pluginDir := filepath.Join(pm.libRoot, id)
+	if err := os.RemoveAll(pluginDir); err != nil {
+		logrus.Warnf("unable to remove %q from plugin remove: %v", pluginDir, err)
+	}
+	pm.pluginEventLogger(id, name, "remove")
 	return nil
 }
 

plugin/distribution/pull.go

@@ -177,7 +177,8 @@ func WritePullData(pd PullData, dest string, extract bool) error {
 	if err := json.Unmarshal(config, &p); err != nil {
 		return err
 	}
-	logrus.Debugf("%#v", p)
+	logrus.Debugf("plugin: %#v", p)
+
 	if err := os.MkdirAll(dest, 0700); err != nil {
 		return err
 	}

plugin/manager.go

@@ -5,10 +5,12 @@ import (
 	"io"
 	"os"
 	"path/filepath"
+	"strings"
 	"sync"
 
 	"github.com/Sirupsen/logrus"
 	"github.com/docker/docker/libcontainerd"
+	"github.com/docker/docker/pkg/mount"
 	"github.com/docker/docker/plugin/store"
 	"github.com/docker/docker/plugin/v2"
 	"github.com/docker/docker/registry"
@@ -63,7 +65,7 @@ func Init(root string, ps *store.Store, remote libcontainerd.Remote, rs registry
 	root = filepath.Join(root, "plugins")
 	manager = &Manager{
 		libRoot:           root,
-		runRoot:           "/run/docker",
+		runRoot:           "/run/docker/plugins",
 		pluginStore:       ps,
 		registryService:   rs,
 		liveRestore:       liveRestore,
@@ -104,6 +106,13 @@ func (pm *Manager) StateChanged(id string, e libcontainerd.StateInfo) error {
 		pm.mu.RUnlock()
 
 		p.RemoveFromDisk()
+
+		if p.PropagatedMount != "" {
+			if err := mount.Unmount(p.PropagatedMount); err != nil {
+				logrus.Warnf("Could not unmount %s: %v", p.PropagatedMount, err)
+			}
+		}
+
 		if restart {
 			pm.enable(p, c, true)
 		}
@@ -141,6 +150,24 @@ func (pm *Manager) reload() error {
 				return
 			}
 
+			if p.Rootfs != "" {
+				p.Rootfs = filepath.Join(pm.libRoot, p.PluginObj.ID, "rootfs")
+			}
+
+			// We should only enable rootfs propagation for certain plugin types that need it.
+			for _, typ := range p.PluginObj.Config.Interface.Types {
+				if typ.Capability == "volumedriver" && typ.Prefix == "docker" && strings.HasPrefix(typ.Version, "1.") {
+					if p.PluginObj.Config.PropagatedMount != "" {
+						// TODO: sanitize PropagatedMount and prevent breakout
+						p.PropagatedMount = filepath.Join(p.Rootfs, p.PluginObj.Config.PropagatedMount)
+						if err := os.MkdirAll(p.PropagatedMount, 0755); err != nil {
+							logrus.Errorf("failed to create PropagatedMount directory at %s: %v", p.PropagatedMount, err)
+							return
+						}
+					}
+				}
+			}
+
 			pm.pluginStore.Update(p)
 			requiresManualRestore := !pm.liveRestore && p.IsEnabled()
 

plugin/manager_linux.go

@@ -11,16 +11,18 @@ import (
 	"github.com/Sirupsen/logrus"
 	"github.com/docker/docker/libcontainerd"
 	"github.com/docker/docker/oci"
+	"github.com/docker/docker/pkg/mount"
 	"github.com/docker/docker/pkg/plugins"
 	"github.com/docker/docker/plugin/v2"
 	specs "github.com/opencontainers/runtime-spec/specs-go"
 )
 
 func (pm *Manager) enable(p *v2.Plugin, c *controller, force bool) error {
+	p.Rootfs = filepath.Join(pm.libRoot, p.PluginObj.ID, "rootfs")
 	if p.IsEnabled() && !force {
 		return fmt.Errorf("plugin %s is already enabled", p.Name())
 	}
-	spec, err := p.InitSpec(oci.DefaultSpec(), pm.libRoot)
+	spec, err := p.InitSpec(oci.DefaultSpec())
 	if err != nil {
 		return err
 	}
@@ -32,6 +34,12 @@ func (pm *Manager) enable(p *v2.Plugin, c *controller, force bool) error {
 	pm.cMap[p] = c
 	pm.mu.Unlock()
 
+	if p.PropagatedMount != "" {
+		if err := mount.MakeRShared(p.PropagatedMount); err != nil {
+			return err
+		}
+	}
+
 	if err := pm.containerdClient.Create(p.GetID(), "", "", specs.Spec(*spec), attachToLog(p.GetID())); err != nil {
 		return err
 	}

plugin/v2/plugin.go

@@ -2,6 +2,7 @@ package v2
 
 import (
 	"encoding/json"
+	"errors"
 	"fmt"
 	"os"
 	"path/filepath"
@@ -22,7 +23,9 @@ type Plugin struct {
 	pClient           *plugins.Client
 	runtimeSourcePath string
 	refCount          int
-	libRoot           string
+	LibRoot           string // TODO: make private
+	PropagatedMount   string // TODO: make private
+	Rootfs            string // TODO: make private
 }
 
 const defaultPluginRuntimeDestination = "/run/docker/plugins"
@@ -45,7 +48,7 @@ func NewPlugin(name, id, runRoot, libRoot, tag string) *Plugin {
 	return &Plugin{
 		PluginObj:         newPluginObj(name, id, tag),
 		runtimeSourcePath: filepath.Join(runRoot, id),
-		libRoot:           libRoot,
+		LibRoot:           libRoot,
 	}
 }
 
@@ -63,6 +66,12 @@ func (p *Plugin) GetRuntimeSourcePath() string {
 	return p.runtimeSourcePath
 }
 
+// BasePath returns the path to which all paths returned by the plugin are relative to.
+// For Plugin objects this returns the host path of the plugin container's rootfs.
+func (p *Plugin) BasePath() string {
+	return p.Rootfs
+}
+
 // Client returns the plugin client.
 func (p *Plugin) Client() *plugins.Client {
 	p.mu.RLock()
@@ -112,7 +121,7 @@ func (p *Plugin) RemoveFromDisk() error {
 
 // InitPlugin populates the plugin object from the plugin config file.
 func (p *Plugin) InitPlugin() error {
-	dt, err := os.Open(filepath.Join(p.libRoot, p.PluginObj.ID, "config.json"))
+	dt, err := os.Open(filepath.Join(p.LibRoot, p.PluginObj.ID, "config.json"))
 	if err != nil {
 		return err
 	}
@@ -123,9 +132,7 @@ func (p *Plugin) InitPlugin() error {
 	}
 
 	p.PluginObj.Settings.Mounts = make([]types.PluginMount, len(p.PluginObj.Config.Mounts))
-	for i, mount := range p.PluginObj.Config.Mounts {
-		p.PluginObj.Settings.Mounts[i] = mount
-	}
+	copy(p.PluginObj.Settings.Mounts, p.PluginObj.Config.Mounts)
 	p.PluginObj.Settings.Env = make([]string, 0, len(p.PluginObj.Config.Env))
 	p.PluginObj.Settings.Devices = make([]types.PluginDevice, 0, len(p.PluginObj.Config.Linux.Devices))
 	copy(p.PluginObj.Settings.Devices, p.PluginObj.Config.Linux.Devices)
@@ -134,13 +141,14 @@ func (p *Plugin) InitPlugin() error {
 			p.PluginObj.Settings.Env = append(p.PluginObj.Settings.Env, fmt.Sprintf("%s=%s", env.Name, *env.Value))
 		}
 	}
+	p.PluginObj.Settings.Args = make([]string, len(p.PluginObj.Config.Args.Value))
 	copy(p.PluginObj.Settings.Args, p.PluginObj.Config.Args.Value)
 
 	return p.writeSettings()
 }
 
 func (p *Plugin) writeSettings() error {
-	f, err := os.Create(filepath.Join(p.libRoot, p.PluginObj.ID, "plugin-settings.json"))
+	f, err := os.Create(filepath.Join(p.LibRoot, p.PluginObj.ID, "plugin-settings.json"))
 	if err != nil {
 		return err
 	}
@@ -287,18 +295,21 @@ func (p *Plugin) SetRefCount(count int) {
 }
 
 // InitSpec creates an OCI spec from the plugin's config.
-func (p *Plugin) InitSpec(s specs.Spec, libRoot string) (*specs.Spec, error) {
-	rootfs := filepath.Join(libRoot, p.PluginObj.ID, "rootfs")
+func (p *Plugin) InitSpec(s specs.Spec) (*specs.Spec, error) {
 	s.Root = specs.Root{
-		Path:     rootfs,
+		Path:     p.Rootfs,
 		Readonly: false, // TODO: all plugins should be readonly? settable in config?
 	}
 
-	userMounts := make(map[string]struct{}, len(p.PluginObj.Config.Mounts))
-	for _, m := range p.PluginObj.Config.Mounts {
+	userMounts := make(map[string]struct{}, len(p.PluginObj.Settings.Mounts))
+	for _, m := range p.PluginObj.Settings.Mounts {
 		userMounts[m.Destination] = struct{}{}
 	}
 
+	if err := os.MkdirAll(p.runtimeSourcePath, 0755); err != nil {
+		return nil, err
+	}
+
 	mounts := append(p.PluginObj.Config.Mounts, types.PluginMount{
 		Source:      &p.runtimeSourcePath,
 		Destination: defaultPluginRuntimeDestination,
@@ -328,27 +339,16 @@ func (p *Plugin) InitSpec(s specs.Spec, libRoot string) (*specs.Spec, error) {
 			})
 	}
 
-	for _, mount := range mounts {
+	for _, mnt := range mounts {
 		m := specs.Mount{
-			Destination: mount.Destination,
-			Type:        mount.Type,
-			Options:     mount.Options,
-		}
-		// TODO: if nil, then it's required and user didn't set it
-		if mount.Source != nil {
-			m.Source = *mount.Source
-		}
-		if m.Source != "" && m.Type == "bind" {
-			fi, err := os.Lstat(filepath.Join(rootfs, m.Destination)) // TODO: followsymlinks
-			if err != nil {
-				return nil, err
-			}
-			if fi.IsDir() {
-				if err := os.MkdirAll(m.Source, 0700); err != nil {
-					return nil, err
-				}
+			Destination: mnt.Destination,
+			Type:        mnt.Type,
+			Options:     mnt.Options,
 		}
+		if mnt.Source == nil {
+			return nil, errors.New("mount source is not specified")
 		}
+		m.Source = *mnt.Source
 		s.Mounts = append(s.Mounts, m)
 	}
 
@@ -360,11 +360,16 @@ func (p *Plugin) InitSpec(s specs.Spec, libRoot string) (*specs.Spec, error) {
 		}
 	}
 
+	if p.PluginObj.Config.PropagatedMount != "" {
+		p.PropagatedMount = filepath.Join(p.Rootfs, p.PluginObj.Config.PropagatedMount)
+		s.Linux.RootfsPropagation = "rshared"
+	}
+
 	if p.PluginObj.Config.Linux.DeviceCreation {
 		rwm := "rwm"
 		s.Linux.Resources.Devices = []specs.DeviceCgroup{{Allow: true, Access: &rwm}}
 	}
-	for _, dev := range p.PluginObj.Config.Linux.Devices {
+	for _, dev := range p.PluginObj.Settings.Devices {
 		path := *dev.Path
 		d, dPermissions, err := oci.DevicesFromPath(path, path, "rwm")
 		if err != nil {

volume/drivers/adapter.go

@@ -2,6 +2,7 @@ package volumedrivers
 
 import (
 	"errors"
+	"path/filepath"
 	"strings"
 
 	"github.com/Sirupsen/logrus"
@@ -14,6 +15,7 @@ var (
 
 type volumeDriverAdapter struct {
 	name         string
+	baseHostPath string
 	capabilities *volume.Capability
 	proxy        *volumeDriverProxy
 }
@@ -30,6 +32,7 @@ func (a *volumeDriverAdapter) Create(name string, opts map[string]string) (volum
 		proxy:        a.proxy,
 		name:         name,
 		driverName:   a.name,
+		baseHostPath: a.baseHostPath,
 	}, nil
 }
 
@@ -37,6 +40,13 @@ func (a *volumeDriverAdapter) Remove(v volume.Volume) error {
 	return a.proxy.Remove(v.Name())
 }
 
+func hostPath(baseHostPath, path string) string {
+	if baseHostPath != "" {
+		path = filepath.Join(baseHostPath, path)
+	}
+	return path
+}
+
 func (a *volumeDriverAdapter) List() ([]volume.Volume, error) {
 	ls, err := a.proxy.List()
 	if err != nil {
@@ -48,8 +58,9 @@ func (a *volumeDriverAdapter) List() ([]volume.Volume, error) {
 		out = append(out, &volumeAdapter{
 			proxy:        a.proxy,
 			name:         vp.Name,
+			baseHostPath: a.baseHostPath,
 			driverName:   a.name,
-			eMount:     vp.Mountpoint,
+			eMount:       hostPath(a.baseHostPath, vp.Mountpoint),
 		})
 	}
 	return out, nil
@@ -72,6 +83,7 @@ func (a *volumeDriverAdapter) Get(name string) (volume.Volume, error) {
 		driverName:   a.Name(),
 		eMount:       v.Mountpoint,
 		status:       v.Status,
+		baseHostPath: a.baseHostPath,
 	}, nil
 }
 
@@ -110,6 +122,7 @@ func (a *volumeDriverAdapter) getCapabilities() volume.Capability {
 type volumeAdapter struct {
 	proxy        *volumeDriverProxy
 	name         string
+	baseHostPath string
 	driverName   string
 	eMount       string // ephemeral host volume path
 	status       map[string]interface{}
@@ -131,7 +144,8 @@ func (a *volumeAdapter) DriverName() string {
 
 func (a *volumeAdapter) Path() string {
 	if len(a.eMount) == 0 {
-		a.eMount, _ = a.proxy.Path(a.name)
+		mountpoint, _ := a.proxy.Path(a.name)
+		a.eMount = hostPath(a.baseHostPath, mountpoint)
 	}
 	return a.eMount
 }
@@ -141,8 +155,8 @@ func (a *volumeAdapter) CachedPath() string {
 }
 
 func (a *volumeAdapter) Mount(id string) (string, error) {
-	var err error
-	a.eMount, err = a.proxy.Mount(a.name, id)
+	mountpoint, err := a.proxy.Mount(a.name, id)
+	a.eMount = hostPath(a.baseHostPath, mountpoint)
 	return a.eMount, err
 }
 

volume/drivers/extpoint.go

@@ -22,9 +22,9 @@ var drivers = &driverExtpoint{
 const extName = "VolumeDriver"
 
 // NewVolumeDriver returns a driver has the given name mapped on the given client.
-func NewVolumeDriver(name string, c client) volume.Driver {
+func NewVolumeDriver(name string, baseHostPath string, c client) volume.Driver {
 	proxy := &volumeDriverProxy{c}
-	return &volumeDriverAdapter{name: name, proxy: proxy}
+	return &volumeDriverAdapter{name: name, baseHostPath: baseHostPath, proxy: proxy}
 }
 
 // volumeDriver defines the available functions that volume plugins must implement.
@@ -117,7 +117,7 @@ func lookup(name string, mode int) (volume.Driver, error) {
 		return nil, fmt.Errorf("Error looking up volume plugin %s: %v", name, err)
 	}
 
-	d := NewVolumeDriver(p.Name(), p.Client())
+	d := NewVolumeDriver(p.Name(), p.BasePath(), p.Client())
 	if err := validateDriver(d); err != nil {
 		return nil, err
 	}
@@ -199,7 +199,7 @@ func GetAllDrivers() ([]volume.Driver, error) {
 			continue
 		}
 
-		ext = NewVolumeDriver(name, p.Client())
+		ext = NewVolumeDriver(name, p.BasePath(), p.Client())
 		if p.IsV1() {
 			drivers.extensions[name] = ext
 		}

volume/drivers/proxy.go

@@ -4,6 +4,7 @@ package volumedrivers
 
 import (
 	"errors"
+
 	"github.com/docker/docker/volume"
 )