From d88fe447df0e87b3a57f9d08b108b141dd72678c Mon Sep 17 00:00:00 2001 From: Mrunal Patel Date: Mon, 3 Aug 2015 15:05:34 -0700 Subject: [PATCH] Add support for sharing /dev/shm/ and /dev/mqueue between containers This changeset creates /dev/shm and /dev/mqueue mounts for each container under /var/lib/containers// and bind mounts them into the container. When --ipc:container is used, then the /dev/shm and /dev/mqueue of the ipc container are used instead of creating new ones for the container. Signed-off-by: Mrunal Patel Docker-DCO-1.1-Signed-off-by: Dan Walsh (github: rhatdan) --- daemon/container.go | 11 ++ daemon/container_unix.go | 100 ++++++++++++++++++ daemon/container_windows.go | 12 +++ daemon/daemon.go | 8 ++ daemon/daemon_linux.go | 44 ++++++++ daemon/daemon_windows.go | 4 + .../native/template/default_template.go | 13 --- 7 files changed, 179 insertions(+), 13 deletions(-) create mode 100644 daemon/daemon_linux.go diff --git a/daemon/container.go b/daemon/container.go index 855ce0296c..96828f4784 100644 --- a/daemon/container.go +++ b/daemon/container.go @@ -292,10 +292,17 @@ func (container *Container) Start() (err error) { return err } + if !(container.hostConfig.IpcMode.IsContainer() || container.hostConfig.IpcMode.IsHost()) { + if err := container.setupIpcDirs(); err != nil { + return err + } + } + mounts, err := container.setupMounts() if err != nil { return err } + mounts = append(mounts, container.ipcMounts()...) container.command.Mounts = mounts return container.waitForStart() @@ -354,6 +361,10 @@ func (container *Container) isNetworkAllocated() bool { func (container *Container) cleanup() { container.ReleaseNetwork() + if err := container.unmountIpcMounts(); err != nil { + logrus.Errorf("%v: Failed to umount ipc filesystems: %v", container.ID, err) + } + if err := container.Unmount(); err != nil { logrus.Errorf("%v: Failed to umount filesystem: %v", container.ID, err) } diff --git a/daemon/container_unix.go b/daemon/container_unix.go index 245a0762cc..5bc89e740b 100644 --- a/daemon/container_unix.go +++ b/daemon/container_unix.go @@ -45,6 +45,8 @@ type Container struct { AppArmorProfile string HostnamePath string HostsPath string + ShmPath string + MqueuePath string MountPoints map[string]*mountPoint ResolvConfPath string UpdateDns bool @@ -184,6 +186,16 @@ func populateCommand(c *Container, env []string) error { } ipc := &execdriver.Ipc{} + var err error + c.ShmPath, err = c.shmPath() + if err != nil { + return err + } + + c.MqueuePath, err = c.mqueuePath() + if err != nil { + return err + } if c.hostConfig.IpcMode.IsContainer() { ic, err := c.getIpcContainer() @@ -191,8 +203,14 @@ func populateCommand(c *Container, env []string) error { return err } ipc.ContainerID = ic.ID + c.ShmPath = ic.ShmPath + c.MqueuePath = ic.MqueuePath } else { ipc.HostIpc = c.hostConfig.IpcMode.IsHost() + if ipc.HostIpc { + c.ShmPath = "/dev/shm" + c.MqueuePath = "/dev/mqueue" + } } pid := &execdriver.Pid{} @@ -1193,3 +1211,85 @@ func (container *Container) removeMountPoints() error { } return nil } + +func (container *Container) shmPath() (string, error) { + return container.GetRootResourcePath("shm") +} +func (container *Container) mqueuePath() (string, error) { + return container.GetRootResourcePath("mqueue") +} + +func (container *Container) setupIpcDirs() error { + shmPath, err := container.shmPath() + if err != nil { + return err + } + + if err := os.MkdirAll(shmPath, 0700); err != nil { + return err + } + + if err := syscall.Mount("shm", shmPath, "tmpfs", uintptr(syscall.MS_NOEXEC|syscall.MS_NOSUID|syscall.MS_NODEV), label.FormatMountLabel("mode=1777,size=65536k", container.GetMountLabel())); err != nil { + return fmt.Errorf("mounting shm tmpfs: %s", err) + } + + mqueuePath, err := container.mqueuePath() + if err != nil { + return err + } + + if err := os.MkdirAll(mqueuePath, 0700); err != nil { + return err + } + + if err := syscall.Mount("mqueue", mqueuePath, "mqueue", uintptr(syscall.MS_NOEXEC|syscall.MS_NOSUID|syscall.MS_NODEV), ""); err != nil { + return fmt.Errorf("mounting mqueue mqueue : %s", err) + } + + return nil +} + +func (container *Container) unmountIpcMounts() error { + if container.hostConfig.IpcMode.IsContainer() || container.hostConfig.IpcMode.IsHost() { + return nil + } + + shmPath, err := container.shmPath() + if err != nil { + return fmt.Errorf("shm path does not exist %v", err) + } + + if err := syscall.Unmount(shmPath, syscall.MNT_DETACH); err != nil { + return fmt.Errorf("failed to umount %s filesystem %v", shmPath, err) + } + + mqueuePath, err := container.mqueuePath() + if err != nil { + return fmt.Errorf("mqueue path does not exist %v", err) + } + + if err := syscall.Unmount(mqueuePath, syscall.MNT_DETACH); err != nil { + return fmt.Errorf("failed to umount %s filesystem %v", mqueuePath, err) + } + + return nil +} + +func (container *Container) ipcMounts() []execdriver.Mount { + var mounts []execdriver.Mount + label.SetFileLabel(container.ShmPath, container.MountLabel) + mounts = append(mounts, execdriver.Mount{ + Source: container.ShmPath, + Destination: "/dev/shm", + Writable: true, + Private: true, + }) + label.SetFileLabel(container.MqueuePath, container.MountLabel) + mounts = append(mounts, execdriver.Mount{ + Source: container.MqueuePath, + Destination: "/dev/mqueue", + Writable: true, + Private: true, + }) + return mounts +} diff --git a/daemon/container_windows.go b/daemon/container_windows.go index cea0268565..bd6dde8d83 100644 --- a/daemon/container_windows.go +++ b/daemon/container_windows.go @@ -172,3 +172,15 @@ func (container *Container) prepareMountPoints() error { func (container *Container) removeMountPoints() error { return nil } + +func (container *Container) setupIpcDirs() error { + return nil +} + +func (container *Container) unmountIpcMounts() error { + return nil +} + +func (container *Container) ipcMounts() []execdriver.Mount { + return nil +} diff --git a/daemon/daemon.go b/daemon/daemon.go index 43e0aabcd5..51f1f26e32 100644 --- a/daemon/daemon.go +++ b/daemon/daemon.go @@ -745,6 +745,10 @@ func NewDaemon(config *Config, registryService *registry.Service) (daemon *Daemo return nil, err } + if err := d.cleanupMounts(); err != nil { + return nil, err + } + return d, nil } @@ -789,6 +793,10 @@ func (daemon *Daemon) Shutdown() error { } } + if err := daemon.cleanupMounts(); err != nil { + return err + } + return nil } diff --git a/daemon/daemon_linux.go b/daemon/daemon_linux.go new file mode 100644 index 0000000000..0c52de3734 --- /dev/null +++ b/daemon/daemon_linux.go @@ -0,0 +1,44 @@ +package daemon + +import ( + "bufio" + "os" + "path/filepath" + "strings" + + "github.com/Sirupsen/logrus" + "github.com/docker/docker/pkg/mount" +) + +// cleanupMounts umounts shm/mqueue mounts for old containers +func (daemon *Daemon) cleanupMounts() error { + logrus.Debugf("Cleaning up old shm/mqueue mounts: start.") + f, err := os.Open("/proc/self/mountinfo") + if err != nil { + return err + } + defer f.Close() + + sc := bufio.NewScanner(f) + for sc.Scan() { + line := sc.Text() + fields := strings.Split(line, " ") + if strings.HasPrefix(fields[4], daemon.repository) { + mnt := fields[4] + mountBase := filepath.Base(mnt) + if mountBase == "mqueue" || mountBase == "shm" { + logrus.Debugf("Unmounting %+v", mnt) + if err := mount.Unmount(mnt); err != nil { + return err + } + } + } + } + + if err := sc.Err(); err != nil { + return err + } + + logrus.Debugf("Cleaning up old shm/mqueue mounts: done.") + return nil +} diff --git a/daemon/daemon_windows.go b/daemon/daemon_windows.go index fdd3b8ff32..0b09537724 100644 --- a/daemon/daemon_windows.go +++ b/daemon/daemon_windows.go @@ -138,3 +138,7 @@ func (daemon *Daemon) newBaseContainer(id string) Container { }, } } + +func (daemon *Daemon) cleanupMounts() error { + return nil +} diff --git a/daemon/execdriver/native/template/default_template.go b/daemon/execdriver/native/template/default_template.go index 7352101dd4..4c5263d1a2 100644 --- a/daemon/execdriver/native/template/default_template.go +++ b/daemon/execdriver/native/template/default_template.go @@ -61,19 +61,6 @@ func New() *configs.Config { Flags: syscall.MS_NOSUID | syscall.MS_NOEXEC, Data: "newinstance,ptmxmode=0666,mode=0620,gid=5", }, - { - Device: "tmpfs", - Source: "shm", - Destination: "/dev/shm", - Data: "mode=1777,size=65536k", - Flags: defaultMountFlags, - }, - { - Source: "mqueue", - Destination: "/dev/mqueue", - Device: "mqueue", - Flags: defaultMountFlags, - }, { Source: "sysfs", Destination: "/sys",