Merge pull request #8479 from vishh/OOM

Provide Out Of Memory information in container status

daemon/daemon.go

@@ -231,7 +231,7 @@ func (daemon *Daemon) register(container *Container, updateSuffixarray bool) err
 		log.Debugf("killing old running container %s", container.ID)
 
 		existingPid := container.Pid
-		container.SetStopped(0)
+		container.SetStopped(&execdriver.ExitStatus{0, false})
 
 		// We only have to handle this for lxc because the other drivers will ensure that
 		// no processes are left when docker dies
@@ -263,7 +263,7 @@ func (daemon *Daemon) register(container *Container, updateSuffixarray bool) err
 
 			log.Debugf("Marking as stopped")
 
-			container.SetStopped(-127)
+			container.SetStopped(&execdriver.ExitStatus{-127, false})
 			if err := container.ToDisk(); err != nil {
 				return err
 			}
@@ -991,7 +991,7 @@ func (daemon *Daemon) Diff(container *Container) (archive.Archive, error) {
 	return daemon.driver.Diff(container.ID, initID)
 }
 
-func (daemon *Daemon) Run(c *Container, pipes *execdriver.Pipes, startCallback execdriver.StartCallback) (int, error) {
+func (daemon *Daemon) Run(c *Container, pipes *execdriver.Pipes, startCallback execdriver.StartCallback) (execdriver.ExitStatus, error) {
 	return daemon.execDriver.Run(c.command, pipes, startCallback)
 }
 

daemon/execdriver/driver.go

@@ -40,9 +40,18 @@ type TtyTerminal interface {
 	Master() *os.File
 }
 
+// ExitStatus provides exit reasons for a container.
+type ExitStatus struct {
+	// The exit code with which the container exited.
+	ExitCode int
+
+	// Whether the container encountered an OOM.
+	OOMKilled bool
+}
+
 type Driver interface {
-	Run(c *Command, pipes *Pipes, startCallback StartCallback) (int, error) // Run executes the process and blocks until the process exits and returns the exit code
-	// Exec executes the process in a running container, blocks until the process exits and returns the exit code
+	Run(c *Command, pipes *Pipes, startCallback StartCallback) (ExitStatus, error) // Run executes the process and blocks until the process exits and returns the exit code
+	// Exec executes the process in an existing container, blocks until the process exits and returns the exit code
 	Exec(c *Command, processConfig *ProcessConfig, pipes *Pipes, startCallback StartCallback) (int, error)
 	Kill(c *Command, sig int) error
 	Pause(c *Command) error

daemon/execdriver/lxc/driver.go

@@ -55,7 +55,7 @@ func (d *driver) Name() string {
 	return fmt.Sprintf("%s-%s", DriverName, version)
 }
 
-func (d *driver) Run(c *execdriver.Command, pipes *execdriver.Pipes, startCallback execdriver.StartCallback) (int, error) {
+func (d *driver) Run(c *execdriver.Command, pipes *execdriver.Pipes, startCallback execdriver.StartCallback) (execdriver.ExitStatus, error) {
 	var (
 		term execdriver.Terminal
 		err  error
@@ -76,11 +76,11 @@ func (d *driver) Run(c *execdriver.Command, pipes *execdriver.Pipes, startCallba
 	})
 
 	if err := d.generateEnvConfig(c); err != nil {
-		return -1, err
+		return execdriver.ExitStatus{-1, false}, err
 	}
 	configPath, err := d.generateLXCConfig(c)
 	if err != nil {
-		return -1, err
+		return execdriver.ExitStatus{-1, false}, err
 	}
 	params := []string{
 		"lxc-start",
@@ -155,11 +155,11 @@ func (d *driver) Run(c *execdriver.Command, pipes *execdriver.Pipes, startCallba
 	c.ProcessConfig.Args = append([]string{name}, arg...)
 
 	if err := nodes.CreateDeviceNodes(c.Rootfs, c.AutoCreatedDevices); err != nil {
-		return -1, err
+		return execdriver.ExitStatus{-1, false}, err
 	}
 
 	if err := c.ProcessConfig.Start(); err != nil {
-		return -1, err
+		return execdriver.ExitStatus{-1, false}, err
 	}
 
 	var (
@@ -183,7 +183,7 @@ func (d *driver) Run(c *execdriver.Command, pipes *execdriver.Pipes, startCallba
 			c.ProcessConfig.Process.Kill()
 			c.ProcessConfig.Wait()
 		}
-		return -1, err
+		return execdriver.ExitStatus{-1, false}, err
 	}
 
 	c.ContainerPid = pid
@@ -194,7 +194,7 @@ func (d *driver) Run(c *execdriver.Command, pipes *execdriver.Pipes, startCallba
 
 	<-waitLock
 
-	return getExitCode(c), waitErr
+	return execdriver.ExitStatus{getExitCode(c), false}, waitErr
 }
 
 /// Return the exit code of the process

daemon/execdriver/native/driver.go

@@ -14,6 +14,7 @@ import (
 	"sync"
 	"syscall"
 
+	log "github.com/Sirupsen/logrus"
 	"github.com/docker/docker/daemon/execdriver"
 	"github.com/docker/docker/pkg/term"
 	"github.com/docker/libcontainer"
@@ -60,11 +61,20 @@ func NewDriver(root, initPath string) (*driver, error) {
 	}, nil
 }
 
-func (d *driver) Run(c *execdriver.Command, pipes *execdriver.Pipes, startCallback execdriver.StartCallback) (int, error) {
+func (d *driver) notifyOnOOM(config *libcontainer.Config) (<-chan struct{}, error) {
+	return fs.NotifyOnOOM(config.Cgroups)
+}
+
+type execOutput struct {
+	exitCode int
+	err      error
+}
+
+func (d *driver) Run(c *execdriver.Command, pipes *execdriver.Pipes, startCallback execdriver.StartCallback) (execdriver.ExitStatus, error) {
 	// take the Command and populate the libcontainer.Config from it
 	container, err := d.createContainer(c)
 	if err != nil {
-		return -1, err
+		return execdriver.ExitStatus{-1, false}, err
 	}
 
 	var term execdriver.Terminal
@@ -75,7 +85,7 @@ func (d *driver) Run(c *execdriver.Command, pipes *execdriver.Pipes, startCallba
 		term, err = execdriver.NewStdConsole(&c.ProcessConfig, pipes)
 	}
 	if err != nil {
-		return -1, err
+		return execdriver.ExitStatus{-1, false}, err
 	}
 	c.ProcessConfig.Terminal = term
 
@@ -92,40 +102,66 @@ func (d *driver) Run(c *execdriver.Command, pipes *execdriver.Pipes, startCallba
 	)
 
 	if err := d.createContainerRoot(c.ID); err != nil {
-		return -1, err
+		return execdriver.ExitStatus{-1, false}, err
 	}
 	defer d.cleanContainer(c.ID)
 
 	if err := d.writeContainerFile(container, c.ID); err != nil {
-		return -1, err
+		return execdriver.ExitStatus{-1, false}, err
 	}
 
-	return namespaces.Exec(container, c.ProcessConfig.Stdin, c.ProcessConfig.Stdout, c.ProcessConfig.Stderr, c.ProcessConfig.Console, dataPath, args, func(container *libcontainer.Config, console, dataPath, init string, child *os.File, args []string) *exec.Cmd {
-		c.ProcessConfig.Path = d.initPath
-		c.ProcessConfig.Args = append([]string{
-			DriverName,
-			"-console", console,
-			"-pipe", "3",
-			"-root", filepath.Join(d.root, c.ID),
-			"--",
-		}, args...)
-
-		// set this to nil so that when we set the clone flags anything else is reset
-		c.ProcessConfig.SysProcAttr = &syscall.SysProcAttr{
-			Cloneflags: uintptr(namespaces.GetNamespaceFlags(container.Namespaces)),
-		}
-		c.ProcessConfig.ExtraFiles = []*os.File{child}
+	execOutputChan := make(chan execOutput, 1)
+	waitForStart := make(chan struct{})
 
-		c.ProcessConfig.Env = container.Env
-		c.ProcessConfig.Dir = container.RootFs
+	go func() {
+		exitCode, err := namespaces.Exec(container, c.ProcessConfig.Stdin, c.ProcessConfig.Stdout, c.ProcessConfig.Stderr, c.ProcessConfig.Console, dataPath, args, func(container *libcontainer.Config, console, dataPath, init string, child *os.File, args []string) *exec.Cmd {
+			c.ProcessConfig.Path = d.initPath
+			c.ProcessConfig.Args = append([]string{
+				DriverName,
+				"-console", console,
+				"-pipe", "3",
+				"-root", filepath.Join(d.root, c.ID),
+				"--",
+			}, args...)
+
+			// set this to nil so that when we set the clone flags anything else is reset
+			c.ProcessConfig.SysProcAttr = &syscall.SysProcAttr{
+				Cloneflags: uintptr(namespaces.GetNamespaceFlags(container.Namespaces)),
+			}
+			c.ProcessConfig.ExtraFiles = []*os.File{child}
 
-		return &c.ProcessConfig.Cmd
-	}, func() {
-		if startCallback != nil {
-			c.ContainerPid = c.ProcessConfig.Process.Pid
-			startCallback(&c.ProcessConfig, c.ContainerPid)
-		}
-	})
+			c.ProcessConfig.Env = container.Env
+			c.ProcessConfig.Dir = container.RootFs
+
+			return &c.ProcessConfig.Cmd
+		}, func() {
+			close(waitForStart)
+			if startCallback != nil {
+				c.ContainerPid = c.ProcessConfig.Process.Pid
+				startCallback(&c.ProcessConfig, c.ContainerPid)
+			}
+		})
+		execOutputChan <- execOutput{exitCode, err}
+	}()
+
+	select {
+	case execOutput := <-execOutputChan:
+		return execdriver.ExitStatus{execOutput.exitCode, false}, execOutput.err
+	case <-waitForStart:
+		break
+	}
+
+	oomKill := false
+	oomKillNotification, err := d.notifyOnOOM(container)
+	if err == nil {
+		_, oomKill = <-oomKillNotification
+	} else {
+		log.Warnf("WARNING: Your kernel does not support OOM notifications: %s", err)
+	}
+	// wait for the container to exit.
+	execOutput := <-execOutputChan
+
+	return execdriver.ExitStatus{execOutput.exitCode, oomKill}, execOutput.err
 }
 
 func (d *driver) Kill(p *execdriver.Command, sig int) error {

daemon/monitor.go

@@ -100,7 +100,7 @@ func (m *containerMonitor) Close() error {
 func (m *containerMonitor) Start() error {
 	var (
 		err        error
-		exitStatus int
+		exitStatus execdriver.ExitStatus
 		// this variable indicates where we in execution flow:
 		// before Run or after
 		afterRun bool
@@ -110,7 +110,7 @@ func (m *containerMonitor) Start() error {
 	defer func() {
 		if afterRun {
 			m.container.Lock()
-			m.container.setStopped(exitStatus)
+			m.container.setStopped(&exitStatus)
 			defer m.container.Unlock()
 		}
 		m.Close()
@@ -138,7 +138,7 @@ func (m *containerMonitor) Start() error {
 			// if we receive an internal error from the initial start of a container then lets
 			// return it instead of entering the restart loop
 			if m.container.RestartCount == 0 {
-				m.container.ExitCode = exitStatus
+				m.container.ExitCode = -1
 				m.resetContainer(false)
 
 				return err
@@ -150,10 +150,10 @@ func (m *containerMonitor) Start() error {
 		// here container.Lock is already lost
 		afterRun = true
 
-		m.resetMonitor(err == nil && exitStatus == 0)
+		m.resetMonitor(err == nil && exitStatus.ExitCode == 0)
 
-		if m.shouldRestart(exitStatus) {
-			m.container.SetRestarting(exitStatus)
+		if m.shouldRestart(exitStatus.ExitCode) {
+			m.container.SetRestarting(&exitStatus)
 			m.container.LogEvent("die")
 			m.resetContainer(true)
 
@@ -164,12 +164,12 @@ func (m *containerMonitor) Start() error {
 			// we need to check this before reentering the loop because the waitForNextRestart could have
 			// been terminated by a request from a user
 			if m.shouldStop {
-				m.container.ExitCode = exitStatus
+				m.container.ExitCode = exitStatus.ExitCode
 				return err
 			}
 			continue
 		}
-		m.container.ExitCode = exitStatus
+		m.container.ExitCode = exitStatus.ExitCode
 		m.container.LogEvent("die")
 		m.resetContainer(true)
 		return err
@@ -209,7 +209,7 @@ func (m *containerMonitor) waitForNextRestart() {
 
 // shouldRestart checks the restart policy and applies the rules to determine if
 // the container's process should be restarted
-func (m *containerMonitor) shouldRestart(exitStatus int) bool {
+func (m *containerMonitor) shouldRestart(exitCode int) bool {
 	m.mux.Lock()
 	defer m.mux.Unlock()
 
@@ -228,7 +228,7 @@ func (m *containerMonitor) shouldRestart(exitStatus int) bool {
 			return false
 		}
 
-		return exitStatus != 0
+		return exitCode != 0
 	}
 
 	return false

daemon/state.go

@@ -5,6 +5,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/docker/docker/daemon/execdriver"
 	"github.com/docker/docker/pkg/units"
 )
 
@@ -13,6 +14,7 @@ type State struct {
 	Running    bool
 	Paused     bool
 	Restarting bool
+	OOMKilled  bool
 	Pid        int
 	ExitCode   int
 	Error      string // contains last known error when starting the container
@@ -149,25 +151,26 @@ func (s *State) setRunning(pid int) {
 	s.waitChan = make(chan struct{})
 }
 
-func (s *State) SetStopped(exitCode int) {
+func (s *State) SetStopped(exitStatus *execdriver.ExitStatus) {
 	s.Lock()
-	s.setStopped(exitCode)
+	s.setStopped(exitStatus)
 	s.Unlock()
 }
 
-func (s *State) setStopped(exitCode int) {
+func (s *State) setStopped(exitStatus *execdriver.ExitStatus) {
 	s.Running = false
 	s.Restarting = false
 	s.Pid = 0
 	s.FinishedAt = time.Now().UTC()
-	s.ExitCode = exitCode
+	s.ExitCode = exitStatus.ExitCode
+	s.OOMKilled = exitStatus.OOMKilled
 	close(s.waitChan) // fire waiters for stop
 	s.waitChan = make(chan struct{})
 }
 
 // SetRestarting is when docker hanldes the auto restart of containers when they are
 // in the middle of a stop and being restarted again
-func (s *State) SetRestarting(exitCode int) {
+func (s *State) SetRestarting(exitStatus *execdriver.ExitStatus) {
 	s.Lock()
 	// we should consider the container running when it is restarting because of
 	// all the checks in docker around rm/stop/etc
@@ -175,7 +178,8 @@ func (s *State) SetRestarting(exitCode int) {
 	s.Restarting = true
 	s.Pid = 0
 	s.FinishedAt = time.Now().UTC()
-	s.ExitCode = exitCode
+	s.ExitCode = exitStatus.ExitCode
+	s.OOMKilled = exitStatus.OOMKilled
 	close(s.waitChan) // fire waiters for stop
 	s.waitChan = make(chan struct{})
 	s.Unlock()

daemon/state_test.go

@@ -4,6 +4,8 @@ import (
 	"sync/atomic"
 	"testing"
 	"time"
+
+	"github.com/docker/docker/daemon/execdriver"
 )
 
 func TestStateRunStop(t *testing.T) {
@@ -47,7 +49,7 @@ func TestStateRunStop(t *testing.T) {
 			atomic.StoreInt64(&exit, int64(exitCode))
 			close(stopped)
 		}()
-		s.SetStopped(i)
+		s.SetStopped(&execdriver.ExitStatus{i, false})
 		if s.IsRunning() {
 			t.Fatal("State is running")
 		}

integration/runtime_test.go

@@ -18,6 +18,7 @@ import (
 
 	log "github.com/Sirupsen/logrus"
 	"github.com/docker/docker/daemon"
+	"github.com/docker/docker/daemon/execdriver"
 	"github.com/docker/docker/engine"
 	"github.com/docker/docker/image"
 	"github.com/docker/docker/nat"
@@ -652,7 +653,7 @@ func TestRestore(t *testing.T) {
 	if err := container3.Run(); err != nil {
 		t.Fatal(err)
 	}
-	container2.SetStopped(0)
+	container2.SetStopped(&execdriver.ExitStatus{0, false})
 }
 
 func TestDefaultContainerName(t *testing.T) {