Merge pull request #4422 from alexlarsson/internal-mounts

Move all bind-mounts in the container inside the namespace

buildfile.go

@@ -374,7 +374,7 @@ func (b *buildFile) checkPathForAddition(orig string) error {
 func (b *buildFile) addContext(container *runtime.Container, orig, dest string, remote bool) error {
 	var (
 		origPath = path.Join(b.contextPath, orig)
-		destPath = path.Join(container.BasefsPath(), dest)
+		destPath = path.Join(container.RootfsPath(), dest)
 	)
 	// Preserve the trailing '/'
 	if strings.HasSuffix(dest, "/") {

execdriver/driver.go

@@ -97,6 +97,13 @@ type Resources struct {
 	CpuShares  int64 `json:"cpu_shares"`
 }
 
+type Mount struct {
+	Source      string `json:"source"`
+	Destination string `json:"destination"`
+	Writable    bool   `json:"writable"`
+	Private     bool   `json:"private"`
+}
+
 // Process wrapps an os/exec.Cmd to add more metadata
 type Command struct {
 	exec.Cmd `json:"-"`
@@ -114,6 +121,7 @@ type Command struct {
 	Network    *Network   `json:"network"` // if network is nil then networking is disabled
 	Config     []string   `json:"config"`  //  generic values that specific drivers can consume
 	Resources  *Resources `json:"resources"`
+	Mounts     []Mount    `json:"mounts"`
 
 	Terminal     Terminal `json:"-"`             // standard or tty terminal
 	Console      string   `json:"-"`             // dev/console path

execdriver/execdrivers/execdrivers.go

@@ -9,7 +9,7 @@ import (
 	"path"
 )
 
-func NewDriver(name, root string, sysInfo *sysinfo.SysInfo) (execdriver.Driver, error) {
+func NewDriver(name, root, initPath string, sysInfo *sysinfo.SysInfo) (execdriver.Driver, error) {
 	switch name {
 	case "lxc":
 		// we want to five the lxc driver the full docker root because it needs
@@ -17,7 +17,7 @@ func NewDriver(name, root string, sysInfo *sysinfo.SysInfo) (execdriver.Driver,
 		// to be backwards compatible
 		return lxc.NewDriver(root, sysInfo.AppArmor)
 	case "native":
-		return native.NewDriver(path.Join(root, "execdriver", "native"))
+		return native.NewDriver(path.Join(root, "execdriver", "native"), initPath)
 	}
 	return nil, fmt.Errorf("unknown exec driver %s", name)
 }

execdriver/lxc/driver.go

@@ -21,6 +21,10 @@ const DriverName = "lxc"
 
 func init() {
 	execdriver.RegisterInitFunc(DriverName, func(args *execdriver.InitArgs) error {
+		if err := setupEnv(args); err != nil {
+			return err
+		}
+
 		if err := setupHostname(args); err != nil {
 			return err
 		}

execdriver/lxc/init.go

@@ -1,17 +1,47 @@
 package lxc
 
 import (
+	"encoding/json"
 	"fmt"
 	"github.com/dotcloud/docker/execdriver"
 	"github.com/dotcloud/docker/pkg/netlink"
 	"github.com/dotcloud/docker/pkg/user"
 	"github.com/syndtr/gocapability/capability"
+	"io/ioutil"
 	"net"
 	"os"
 	"strings"
 	"syscall"
 )
 
+// Clear environment pollution introduced by lxc-start
+func setupEnv(args *execdriver.InitArgs) error {
+	// Get env
+	var env []string
+	content, err := ioutil.ReadFile(".dockerenv")
+	if err != nil {
+		return fmt.Errorf("Unable to load environment variables: %v", err)
+	}
+	if err := json.Unmarshal(content, &env); err != nil {
+		return fmt.Errorf("Unable to unmarshal environment variables: %v", err)
+	}
+	// Propagate the plugin-specific container env variable
+	env = append(env, "container="+os.Getenv("container"))
+
+	args.Env = env
+
+	os.Clearenv()
+	for _, kv := range args.Env {
+		parts := strings.SplitN(kv, "=", 2)
+		if len(parts) == 1 {
+			parts = append(parts, "")
+		}
+		os.Setenv(parts[0], parts[1])
+	}
+
+	return nil
+}
+
 func setupHostname(args *execdriver.InitArgs) error {
 	hostname := getEnv(args, "HOSTNAME")
 	if hostname == "" {

execdriver/lxc/lxc_template.go

@@ -88,6 +88,14 @@ lxc.mount.entry = {{.Console}} {{escapeFstabSpaces $ROOTFS}}/dev/console none bi
 lxc.mount.entry = devpts {{escapeFstabSpaces $ROOTFS}}/dev/pts devpts newinstance,ptmxmode=0666,nosuid,noexec 0 0
 lxc.mount.entry = shm {{escapeFstabSpaces $ROOTFS}}/dev/shm tmpfs size=65536k,nosuid,nodev,noexec 0 0
 
+{{range $value := .Mounts}}
+{{if $value.Writable}}
+lxc.mount.entry = {{$value.Source}} {{escapeFstabSpaces $ROOTFS}}/{{escapeFstabSpaces $value.Destination}} none bind,rw 0 0
+{{else}}
+lxc.mount.entry = {{$value.Source}} {{escapeFstabSpaces $ROOTFS}}/{{escapeFstabSpaces $value.Destination}} none bind,ro 0 0
+{{end}}
+{{end}}
+
 {{if .Privileged}}
 {{if .AppArmor}}
 lxc.aa_profile = unconfined

execdriver/native/default_template.go

@@ -48,6 +48,10 @@ func createContainer(c *execdriver.Command) *libcontainer.Container {
 	// check to see if we are running in ramdisk to disable pivot root
 	container.NoPivotRoot = os.Getenv("DOCKER_RAMDISK") != ""
 
+	for _, m := range c.Mounts {
+		container.Mounts = append(container.Mounts, libcontainer.Mount{m.Source, m.Destination, m.Writable, m.Private})
+	}
+
 	return container
 }
 

execdriver/native/driver.go

@@ -55,10 +55,11 @@ func init() {
 }
 
 type driver struct {
-	root string
+	root     string
+	initPath string
 }
 
-func NewDriver(root string) (*driver, error) {
+func NewDriver(root, initPath string) (*driver, error) {
 	if err := os.MkdirAll(root, 0700); err != nil {
 		return nil, err
 	}
@@ -66,7 +67,8 @@ func NewDriver(root string) (*driver, error) {
 		return nil, err
 	}
 	return &driver{
-		root: root,
+		root:     root,
+		initPath: initPath,
 	}, nil
 }
 
@@ -210,7 +212,7 @@ func (d *dockerCommandFactory) Create(container *libcontainer.Container, console
 	// we need to join the rootfs because nsinit will setup the rootfs and chroot
 	initPath := filepath.Join(d.c.Rootfs, d.c.InitPath)
 
-	d.c.Path = initPath
+	d.c.Path = d.driver.initPath
 	d.c.Args = append([]string{
 		initPath,
 		"-driver", DriverName,

integration/utils_test.go

@@ -71,7 +71,7 @@ func containerFileExists(eng *engine.Engine, id, dir string, t utils.Fataler) bo
 		t.Fatal(err)
 	}
 	defer c.Unmount()
-	if _, err := os.Stat(path.Join(c.BasefsPath(), dir)); err != nil {
+	if _, err := os.Stat(path.Join(c.RootfsPath(), dir)); err != nil {
 		if os.IsNotExist(err) {
 			return false
 		}

pkg/libcontainer/container.go

@@ -23,6 +23,7 @@ type Container struct {
 	Networks     []*Network      `json:"networks,omitempty"`      // nil for host's network stack
 	Cgroups      *cgroups.Cgroup `json:"cgroups,omitempty"`       // cgroups
 	Context      Context         `json:"context,omitempty"`       // generic context for specific options (apparmor, selinux)
+	Mounts       []Mount         `json:"mounts,omitempty"`
 }
 
 // Network defines configuration for a container's networking stack
@@ -36,3 +37,12 @@ type Network struct {
 	Gateway string  `json:"gateway,omitempty"`
 	Mtu     int     `json:"mtu,omitempty"`
 }
+
+// Bind mounts from the host system to the container
+//
+type Mount struct {
+	Source      string `json:"source"`      // Source path, in the host namespace
+	Destination string `json:"destination"` // Destination path, in the container
+	Writable    bool   `json:"writable"`
+	Private     bool   `json:"private"`
+}

pkg/libcontainer/nsinit/init.go

@@ -51,7 +51,7 @@ func (ns *linuxNs) Init(container *libcontainer.Container, uncleanRootfs, consol
 	if err := system.ParentDeathSignal(); err != nil {
 		return fmt.Errorf("parent death signal %s", err)
 	}
-	if err := setupNewMountNamespace(rootfs, console, container.ReadonlyFs, container.NoPivotRoot); err != nil {
+	if err := setupNewMountNamespace(rootfs, container.Mounts, console, container.ReadonlyFs, container.NoPivotRoot); err != nil {
 		return fmt.Errorf("setup mount namespace %s", err)
 	}
 	if err := setupNetwork(container, context); err != nil {

pkg/libcontainer/nsinit/mount.go

@@ -4,6 +4,7 @@ package nsinit
 
 import (
 	"fmt"
+	"github.com/dotcloud/docker/pkg/libcontainer"
 	"github.com/dotcloud/docker/pkg/system"
 	"io/ioutil"
 	"os"
@@ -19,7 +20,7 @@ const defaultMountFlags = syscall.MS_NOEXEC | syscall.MS_NOSUID | syscall.MS_NOD
 //
 // There is no need to unmount the new mounts because as soon as the mount namespace
 // is no longer in use, the mounts will be removed automatically
-func setupNewMountNamespace(rootfs, console string, readonly, noPivotRoot bool) error {
+func setupNewMountNamespace(rootfs string, bindMounts []libcontainer.Mount, console string, readonly, noPivotRoot bool) error {
 	flag := syscall.MS_PRIVATE
 	if noPivotRoot {
 		flag = syscall.MS_SLAVE
@@ -38,6 +39,23 @@ func setupNewMountNamespace(rootfs, console string, readonly, noPivotRoot bool)
 	if err := mountSystem(rootfs); err != nil {
 		return fmt.Errorf("mount system %s", err)
 	}
+
+	for _, m := range bindMounts {
+		flags := syscall.MS_BIND | syscall.MS_REC
+		if !m.Writable {
+			flags = flags | syscall.MS_RDONLY
+		}
+		dest := filepath.Join(rootfs, m.Destination)
+		if err := system.Mount(m.Source, dest, "bind", uintptr(flags), ""); err != nil {
+			return fmt.Errorf("mounting %s into %s %s", m.Source, dest, err)
+		}
+		if m.Private {
+			if err := system.Mount("", dest, "none", uintptr(syscall.MS_PRIVATE), ""); err != nil {
+				return fmt.Errorf("mounting %s private %s", dest, err)
+			}
+		}
+	}
+
 	if err := copyDevNodes(rootfs); err != nil {
 		return fmt.Errorf("copy dev nodes %s", err)
 	}

runtime/container.go

@@ -529,13 +529,13 @@ func (container *Container) Start() (err error) {
 		return err
 	}
 
-	if err := mountVolumesForContainer(container, envPath); err != nil {
-		return err
-	}
-
 	populateCommand(container)
 	container.command.Env = env
 
+	if err := setupMountsForContainer(container, envPath); err != nil {
+		return err
+	}
+
 	// Setup logging of stdout and stderr to disk
 	if err := container.runtime.LogToDisk(container.stdout, container.logPath("json"), "stdout"); err != nil {
 		return err
@@ -843,8 +843,6 @@ func (container *Container) cleanup() {
 		}
 	}
 
-	unmountVolumesForContainer(container)
-
 	if err := container.Unmount(); err != nil {
 		log.Printf("%v: Failed to umount filesystem: %v", container.ID, err)
 	}
@@ -1039,12 +1037,6 @@ func (container *Container) EnvConfigPath() (string, error) {
 // This method must be exported to be used from the lxc template
 // This directory is only usable when the container is running
 func (container *Container) RootfsPath() string {
-	return path.Join(container.root, "root")
-}
-
-// This is the stand-alone version of the root fs, without any additional mounts.
-// This directory is usable whenever the container is mounted (and not unmounted)
-func (container *Container) BasefsPath() string {
 	return container.basefs
 }
 

runtime/runtime.go

@@ -174,7 +174,6 @@ func (runtime *Runtime) Register(container *Container) error {
 				runtime.execDriver.Kill(command, 9)
 			}
 			// ensure that the filesystem is also unmounted
-			unmountVolumesForContainer(container)
 			if err := container.Unmount(); err != nil {
 				utils.Debugf("ghost unmount error %s", err)
 			}
@@ -185,7 +184,6 @@ func (runtime *Runtime) Register(container *Container) error {
 			utils.Debugf("Container %s was supposed to be running but is not.", container.ID)
 			if runtime.config.AutoRestart {
 				utils.Debugf("Restarting")
-				unmountVolumesForContainer(container)
 				if err := container.Unmount(); err != nil {
 					utils.Debugf("restart unmount error %s", err)
 				}
@@ -733,7 +731,7 @@ func NewRuntimeFromDirectory(config *daemonconfig.Config, eng *engine.Engine) (*
 	}
 
 	sysInfo := sysinfo.New(false)
-	ed, err := execdrivers.NewDriver(config.ExecDriver, config.Root, sysInfo)
+	ed, err := execdrivers.NewDriver(config.ExecDriver, config.Root, sysInitPath, sysInfo)
 	if err != nil {
 		return nil, err
 	}

runtime/volumes.go

@@ -3,10 +3,9 @@ package runtime
 import (
 	"fmt"
 	"github.com/dotcloud/docker/archive"
-	"github.com/dotcloud/docker/pkg/mount"
+	"github.com/dotcloud/docker/execdriver"
 	"github.com/dotcloud/docker/utils"
 	"io/ioutil"
-	"log"
 	"os"
 	"path/filepath"
 	"strings"
@@ -34,92 +33,29 @@ func prepareVolumesForContainer(container *Container) error {
 	return nil
 }
 
-func mountVolumesForContainer(container *Container, envPath string) error {
-	// Setup the root fs as a bind mount of the base fs
-	var (
-		root    = container.RootfsPath()
-		runtime = container.runtime
-	)
-	if err := os.MkdirAll(root, 0755); err != nil && !os.IsExist(err) {
-		return nil
-	}
-
-	// Create a bind mount of the base fs as a place where we can add mounts
-	// without affecting the ability to access the base fs
-	if err := mount.Mount(container.basefs, root, "none", "bind,rw"); err != nil {
-		return err
-	}
-
-	// Make sure the root fs is private so the mounts here don't propagate to basefs
-	if err := mount.ForceMount(root, root, "none", "private"); err != nil {
-		return err
-	}
-
-	// Mount docker specific files into the containers root fs
-	if err := mount.Mount(runtime.sysInitPath, filepath.Join(root, "/.dockerinit"), "none", "bind,ro"); err != nil {
-		return err
-	}
-	if err := mount.Mount(envPath, filepath.Join(root, "/.dockerenv"), "none", "bind,ro"); err != nil {
-		return err
-	}
-	if err := mount.Mount(container.ResolvConfPath, filepath.Join(root, "/etc/resolv.conf"), "none", "bind,ro"); err != nil {
-		return err
+func setupMountsForContainer(container *Container, envPath string) error {
+	mounts := []execdriver.Mount{
+		{container.runtime.sysInitPath, "/.dockerinit", false, true},
+		{envPath, "/.dockerenv", false, true},
+		{container.ResolvConfPath, "/etc/resolv.conf", false, true},
 	}
 
 	if container.HostnamePath != "" && container.HostsPath != "" {
-		if err := mount.Mount(container.HostnamePath, filepath.Join(root, "/etc/hostname"), "none", "bind,ro"); err != nil {
-			return err
-		}
-		if err := mount.Mount(container.HostsPath, filepath.Join(root, "/etc/hosts"), "none", "bind,ro"); err != nil {
-			return err
-		}
+		mounts = append(mounts, execdriver.Mount{container.HostnamePath, "/etc/hostname", false, true})
+		mounts = append(mounts, execdriver.Mount{container.HostsPath, "/etc/hosts", false, true})
 	}
 
 	// Mount user specified volumes
+	// Note, these are not private because you may want propagation of (un)mounts from host
+	// volumes. For instance if you use -v /usr:/usr and the host later mounts /usr/share you
+	// want this new mount in the container
 	for r, v := range container.Volumes {
-		mountAs := "ro"
-		if container.VolumesRW[r] {
-			mountAs = "rw"
-		}
-
-		r = filepath.Join(root, r)
-		if p, err := utils.FollowSymlinkInScope(r, root); err != nil {
-			return err
-		} else {
-			r = p
-		}
-
-		if err := mount.Mount(v, r, "none", fmt.Sprintf("bind,%s", mountAs)); err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
-func unmountVolumesForContainer(container *Container) {
-	var (
-		root   = container.RootfsPath()
-		mounts = []string{
-			root,
-			filepath.Join(root, "/.dockerinit"),
-			filepath.Join(root, "/.dockerenv"),
-			filepath.Join(root, "/etc/resolv.conf"),
-		}
-	)
-
-	if container.HostnamePath != "" && container.HostsPath != "" {
-		mounts = append(mounts, filepath.Join(root, "/etc/hostname"), filepath.Join(root, "/etc/hosts"))
+		mounts = append(mounts, execdriver.Mount{v, r, container.VolumesRW[r], false})
 	}
 
-	for r := range container.Volumes {
-		mounts = append(mounts, filepath.Join(root, r))
-	}
+	container.command.Mounts = mounts
 
-	for i := len(mounts) - 1; i >= 0; i-- {
-		if lastError := mount.Unmount(mounts[i]); lastError != nil {
-			log.Printf("Failed to umount %v: %v", mounts[i], lastError)
-		}
-	}
+	return nil
 }
 
 func applyVolumesFrom(container *Container) error {

sysinit/sysinit.go

@@ -1,33 +1,16 @@
 package sysinit
 
 import (
-	"encoding/json"
 	"flag"
 	"fmt"
 	"github.com/dotcloud/docker/execdriver"
 	_ "github.com/dotcloud/docker/execdriver/lxc"
 	_ "github.com/dotcloud/docker/execdriver/native"
-	"io/ioutil"
 	"log"
 	"os"
-	"strings"
 )
 
-// Clear environment pollution introduced by lxc-start
-func setupEnv(args *execdriver.InitArgs) {
-	os.Clearenv()
-	for _, kv := range args.Env {
-		parts := strings.SplitN(kv, "=", 2)
-		if len(parts) == 1 {
-			parts = append(parts, "")
-		}
-		os.Setenv(parts[0], parts[1])
-	}
-}
-
 func executeProgram(args *execdriver.InitArgs) error {
-	setupEnv(args)
-
 	dockerInitFct, err := execdriver.GetInitFunc(args.Driver)
 	if err != nil {
 		panic(err)
@@ -59,25 +42,12 @@ func SysInit() {
 	)
 	flag.Parse()
 
-	// Get env
-	var env []string
-	content, err := ioutil.ReadFile(".dockerenv")
-	if err != nil {
-		log.Fatalf("Unable to load environment variables: %v", err)
-	}
-	if err := json.Unmarshal(content, &env); err != nil {
-		log.Fatalf("Unable to unmarshal environment variables: %v", err)
-	}
-	// Propagate the plugin-specific container env variable
-	env = append(env, "container="+os.Getenv("container"))
-
 	args := &execdriver.InitArgs{
 		User:       *user,
 		Gateway:    *gateway,
 		Ip:         *ip,
 		WorkDir:    *workDir,
 		Privileged: *privileged,
-		Env:        env,
 		Args:       flag.Args(),
 		Mtu:        *mtu,
 		Driver:     *driver,