Merge remote-tracking branch 'origin/257-container_real_running_state-fix'

runtime.go

@@ -7,9 +7,10 @@ import (
 	"io"
 	"io/ioutil"
 	"os"
+	"os/exec"
 	"path"
 	"sort"
-	"sync"
+	"strings"
 	"time"
 )
 
@@ -115,6 +116,7 @@ func (runtime *Runtime) Load(id string) (*Container, error) {
 	if err := container.FromDisk(); err != nil {
 		return nil, err
 	}
+	container.State.initLock()
 	if container.Id != id {
 		return container, fmt.Errorf("Container %s is stored at %s", container.Id, id)
 	}
@@ -132,11 +134,26 @@ func (runtime *Runtime) Register(container *Container) error {
 	if err := validateId(container.Id); err != nil {
 		return err
 	}
+
+	// FIXME: if the container is supposed to be running but is not, auto restart it?
+	// If the container is supposed to be running, make sure of it
+	if container.State.Running {
+		if output, err := exec.Command("lxc-info", "-n", container.Id).CombinedOutput(); err != nil {
+			return err
+		} else {
+			if !strings.Contains(string(output), "RUNNING") {
+				Debugf("Container %s was supposed to be running be is not.", container.Id)
+				container.State.setStopped(-127)
+				if err := container.ToDisk(); err != nil {
+					return err
+				}
+			}
+		}
+	}
+
 	container.runtime = runtime
 	// Setup state lock (formerly in newState()
-	lock := new(sync.Mutex)
-	container.State.stateChangeLock = lock
-	container.State.stateChangeCond = sync.NewCond(lock)
+	container.State.initLock()
 	// Attach to stdout and stderr
 	container.stderr = newWriteBroadcaster()
 	container.stdout = newWriteBroadcaster()
@@ -259,7 +276,6 @@ func NewRuntimeFromDirectory(root string) (*Runtime, error) {
 		// If the auth file does not exist, keep going
 		return nil, err
 	}
-
 	runtime := &Runtime{
 		root:           root,
 		repository:     runtimeRepo,

runtime_test.go

@@ -8,6 +8,7 @@ import (
 	"os/user"
 	"sync"
 	"testing"
+	"time"
 )
 
 // FIXME: this is no longer needed
@@ -289,13 +290,48 @@ func TestRestore(t *testing.T) {
 		t.Fatal(err)
 	}
 	defer runtime1.Destroy(container1)
-	if len(runtime1.List()) != 1 {
-		t.Errorf("Expected 1 container, %v found", len(runtime1.List()))
+
+	// Create a second container meant to be killed
+	container2, err := runtime1.Create(&Config{
+		Image:     GetTestImage(runtime1).Id,
+		Cmd:       []string{"/bin/cat"},
+		OpenStdin: true,
+	},
+	)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer runtime1.Destroy(container2)
+
+	// Start the container non blocking
+	if err := container2.Start(); err != nil {
+		t.Fatal(err)
+	}
+
+	if !container2.State.Running {
+		t.Fatalf("Container %v should appear as running but isn't", container2.Id)
+	}
+
+	// Simulate a crash/manual quit of dockerd: process dies, states stays 'Running'
+	cStdin, _ := container2.StdinPipe()
+	cStdin.Close()
+	if err := container2.WaitTimeout(time.Second); err != nil {
+		t.Fatal(err)
+	}
+	container2.State.Running = true
+	container2.ToDisk()
+
+	if len(runtime1.List()) != 2 {
+		t.Errorf("Expected 2 container, %v found", len(runtime1.List()))
 	}
 	if err := container1.Run(); err != nil {
 		t.Fatal(err)
 	}
 
+	if !container2.State.Running {
+		t.Fatalf("Container %v should appear as running but isn't", container2.Id)
+	}
+
 	// Here are are simulating a docker restart - that is, reloading all containers
 	// from scratch
 	runtime2, err := NewRuntimeFromDirectory(root)
@@ -303,14 +339,24 @@ func TestRestore(t *testing.T) {
 		t.Fatal(err)
 	}
 	defer nuke(runtime2)
-	if len(runtime2.List()) != 1 {
-		t.Errorf("Expected 1 container, %v found", len(runtime2.List()))
+	if len(runtime2.List()) != 2 {
+		t.Errorf("Expected 2 container, %v found", len(runtime2.List()))
+	}
+	runningCount := 0
+	for _, c := range runtime2.List() {
+		if c.State.Running {
+			t.Errorf("Running container found: %v (%v)", c.Id, c.Path)
+			runningCount++
+		}
+	}
+	if runningCount != 0 {
+		t.Fatalf("Expected 0 container alive, %d found", runningCount)
 	}
-	container2 := runtime2.Get(container1.Id)
-	if container2 == nil {
+	container3 := runtime2.Get(container1.Id)
+	if container3 == nil {
 		t.Fatal("Unable to Get container")
 	}
-	if err := container2.Run(); err != nil {
+	if err := container3.Run(); err != nil {
 		t.Fatal(err)
 	}
 }

state.go

@@ -39,6 +39,13 @@ func (s *State) setStopped(exitCode int) {
 	s.broadcast()
 }
 
+func (s *State) initLock() {
+	if s.stateChangeLock == nil {
+		s.stateChangeLock = &sync.Mutex{}
+		s.stateChangeCond = sync.NewCond(s.stateChangeLock)
+	}
+}
+
 func (s *State) broadcast() {
 	s.stateChangeLock.Lock()
 	s.stateChangeCond.Broadcast()