Merge pull request #17594 from mrjana/restart

Fix race with host networking causing `invalid argument` errors

daemon/container_unix.go

@@ -895,17 +895,16 @@ func (container *Container) buildCreateEndpointOptions(n libnetwork.Network) ([]
 }
 
 func (container *Container) allocateNetwork() error {
-	sb := container.getNetworkSandbox()
-	if sb != nil {
-		// Cleanup any stale sandbox left over due to ungraceful daemon shutdown
-		if err := sb.Delete(); err != nil {
-			logrus.Errorf("failed to cleanup up stale network sandbox for container %s", container.ID)
-		}
+	controller := container.daemon.netController
+
+	// Cleanup any stale sandbox left over due to ungraceful daemon shutdown
+	if err := controller.SandboxDestroy(container.ID); err != nil {
+		logrus.Errorf("failed to cleanup up stale network sandbox for container %s", container.ID)
 	}
+
 	updateSettings := false
 	if len(container.NetworkSettings.Networks) == 0 {
 		mode := container.hostConfig.NetworkMode
-		controller := container.daemon.netController
 		if container.Config.NetworkDisabled || mode.IsContainer() {
 			return nil
 		}

hack/vendor.sh

@@ -21,7 +21,7 @@ clone git github.com/vdemeester/shakers 3c10293ce22b900c27acad7b28656196fcc2f73b
 clone git golang.org/x/net 3cffabab72adf04f8e3b01c5baf775361837b5fe https://github.com/golang/net.git
 
 #get libnetwork packages
-clone git github.com/docker/libnetwork 47edb73dd3e64cfcc04234b073872205cd79694a
+clone git github.com/docker/libnetwork e7719596c01a83f9ef24d33e9d609a64acacd7b8
 clone git github.com/armon/go-metrics eb0af217e5e9747e41dd5303755356b62d28e3ec
 clone git github.com/hashicorp/go-msgpack 71c2886f5a673a35f909803f38ece5810165097b
 clone git github.com/hashicorp/memberlist 9a1e242e454d2443df330bdd51a436d5a9058fc4

vendor/src/github.com/docker/libnetwork/controller.go

@@ -100,6 +100,9 @@ type NetworkController interface {
 	// SandboxByID returns the Sandbox which has the passed id. If not found, a types.NotFoundError is returned.
 	SandboxByID(id string) (Sandbox, error)
 
+	// SandboxDestroy destroys a sandbox given a container ID
+	SandboxDestroy(id string) error
+
 	// Stop network controller
 	Stop()
 }
@@ -144,6 +147,8 @@ type controller struct {
 	unWatchCh      chan *endpoint
 	svcDb          map[string]svcMap
 	nmap           map[string]*netWatch
+	defOsSbox      osl.Sandbox
+	sboxOnce       sync.Once
 	sync.Mutex
 }
 
@@ -476,27 +481,37 @@ func (c *controller) NewSandbox(containerID string, options ...SandboxOption) (S
 		return nil, types.BadRequestErrorf("invalid container ID")
 	}
 
-	var existing Sandbox
-	look := SandboxContainerWalker(&existing, containerID)
-	c.WalkSandboxes(look)
-	if existing != nil {
-		return nil, types.BadRequestErrorf("container %s is already present: %v", containerID, existing)
+	var sb *sandbox
+	c.Lock()
+	for _, s := range c.sandboxes {
+		if s.containerID == containerID {
+			// If not a stub, then we already have a complete sandbox.
+			if !s.isStub {
+				c.Unlock()
+				return nil, types.BadRequestErrorf("container %s is already present: %v", containerID, s)
+			}
+
+			// We already have a stub sandbox from the
+			// store. Make use of it so that we don't lose
+			// the endpoints from store but reset the
+			// isStub flag.
+			sb = s
+			sb.isStub = false
+			break
+		}
 	}
+	c.Unlock()
 
 	// Create sandbox and process options first. Key generation depends on an option
-	sb := &sandbox{
-		id:          stringid.GenerateRandomID(),
-		containerID: containerID,
-		endpoints:   epHeap{},
-		epPriority:  map[string]int{},
-		config:      containerConfig{},
-		controller:  c,
-	}
-	// This sandbox may be using an existing osl sandbox, sharing it with another sandbox
-	var peerSb Sandbox
-	c.WalkSandboxes(SandboxKeyWalker(&peerSb, sb.Key()))
-	if peerSb != nil {
-		sb.osSbox = peerSb.(*sandbox).osSbox
+	if sb == nil {
+		sb = &sandbox{
+			id:          stringid.GenerateRandomID(),
+			containerID: containerID,
+			endpoints:   epHeap{},
+			epPriority:  map[string]int{},
+			config:      containerConfig{},
+			controller:  c,
+		}
 	}
 
 	heap.Init(&sb.endpoints)
@@ -507,14 +522,26 @@ func (c *controller) NewSandbox(containerID string, options ...SandboxOption) (S
 		return nil, err
 	}
 
-	c.Lock()
+	if sb.config.useDefaultSandBox {
+		c.sboxOnce.Do(func() {
+			c.defOsSbox, err = osl.NewSandbox(sb.Key(), false)
+		})
+
+		if err != nil {
+			c.sboxOnce = sync.Once{}
+			return nil, fmt.Errorf("failed to create default sandbox: %v", err)
+		}
+
+		sb.osSbox = c.defOsSbox
+	}
+
 	if sb.osSbox == nil && !sb.config.useExternalKey {
 		if sb.osSbox, err = osl.NewSandbox(sb.Key(), !sb.config.useDefaultSandBox); err != nil {
-			c.Unlock()
 			return nil, fmt.Errorf("failed to create new osl sandbox: %v", err)
 		}
 	}
 
+	c.Lock()
 	c.sandboxes[sb.id] = sb
 	c.Unlock()
 	defer func() {
@@ -539,6 +566,11 @@ func (c *controller) Sandboxes() []Sandbox {
 
 	list := make([]Sandbox, 0, len(c.sandboxes))
 	for _, s := range c.sandboxes {
+		// Hide stub sandboxes from libnetwork users
+		if s.isStub {
+			continue
+		}
+
 		list = append(list, s)
 	}
 
@@ -566,6 +598,26 @@ func (c *controller) SandboxByID(id string) (Sandbox, error) {
 	return s, nil
 }
 
+// SandboxDestroy destroys a sandbox given a container ID
+func (c *controller) SandboxDestroy(id string) error {
+	var sb *sandbox
+	c.Lock()
+	for _, s := range c.sandboxes {
+		if s.containerID == id {
+			sb = s
+			break
+		}
+	}
+	c.Unlock()
+
+	// It is not an error if sandbox is not available
+	if sb == nil {
+		return nil
+	}
+
+	return sb.Delete()
+}
+
 // SandboxContainerWalker returns a Sandbox Walker function which looks for an existing Sandbox with the passed containerID
 func SandboxContainerWalker(out *Sandbox, containerID string) SandboxWalker {
 	return func(sb Sandbox) bool {

vendor/src/github.com/docker/libnetwork/sandbox.go

@@ -68,6 +68,7 @@ type sandbox struct {
 	joinLeaveDone chan struct{}
 	dbIndex       uint64
 	dbExists      bool
+	isStub        bool
 	inDelete      bool
 	sync.Mutex
 }
@@ -197,7 +198,7 @@ func (sb *sandbox) Delete() error {
 	// likely not required any more. Drop it.
 	etchosts.Drop(sb.config.hostsPath)
 
-	if sb.osSbox != nil {
+	if sb.osSbox != nil && !sb.config.useDefaultSandBox {
 		sb.osSbox.Destroy()
 	}
 

vendor/src/github.com/docker/libnetwork/sandbox_store.go

@@ -128,6 +128,12 @@ func (sb *sandbox) storeUpdate() error {
 retry:
 	sbs.Eps = nil
 	for _, ep := range sb.getConnectedEndpoints() {
+		// If the endpoint is not persisted then do not add it to
+		// the sandbox checkpoint
+		if ep.Skip() {
+			continue
+		}
+
 		eps := epState{
 			Nid: ep.getNetwork().ID(),
 			Eid: ep.ID(),
@@ -188,6 +194,7 @@ func (c *controller) sandboxCleanup() {
 			endpoints:   epHeap{},
 			epPriority:  map[string]int{},
 			dbIndex:     sbs.dbIndex,
+			isStub:      true,
 			dbExists:    true,
 		}