Merge pull request #19465 from mavenugo/sysd

Vendor libnetwork v0.6.0-rc1

hack/vendor.sh

@@ -27,7 +27,7 @@ clone git github.com/RackSec/srslog 6eb773f331e46fbba8eecb8e794e635e75fc04de
 clone git github.com/imdario/mergo 0.2.1
 
 #get libnetwork packages
-clone git github.com/docker/libnetwork v0.5.6
+clone git github.com/docker/libnetwork v0.6.0-rc1
 clone git github.com/armon/go-metrics eb0af217e5e9747e41dd5303755356b62d28e3ec
 clone git github.com/hashicorp/go-msgpack 71c2886f5a673a35f909803f38ece5810165097b
 clone git github.com/hashicorp/memberlist 9a1e242e454d2443df330bdd51a436d5a9058fc4

integration-cli/docker_cli_network_unix_test.go

@@ -12,6 +12,7 @@ import (
 	"os"
 	"sort"
 	"strings"
+	"time"
 
 	"github.com/docker/docker/pkg/integration/checker"
 	"github.com/docker/docker/runconfig"
@@ -56,6 +57,10 @@ func (s *DockerNetworkSuite) SetUpSuite(c *check.C) {
 	mux := http.NewServeMux()
 	s.server = httptest.NewServer(mux)
 	c.Assert(s.server, check.NotNil, check.Commentf("Failed to start a HTTP Server"))
+	setupRemoteNetworkDrivers(c, mux, s.server.URL, dummyNetworkDriver, dummyIpamDriver)
+}
+
+func setupRemoteNetworkDrivers(c *check.C, mux *http.ServeMux, url, netDrv, ipamDrv string) {
 
 	mux.HandleFunc("/Plugin.Activate", func(w http.ResponseWriter, r *http.Request) {
 		w.Header().Set("Content-Type", "application/vnd.docker.plugins.v1+json")
@@ -199,12 +204,12 @@ func (s *DockerNetworkSuite) SetUpSuite(c *check.C) {
 	err := os.MkdirAll("/etc/docker/plugins", 0755)
 	c.Assert(err, checker.IsNil)
 
-	fileName := fmt.Sprintf("/etc/docker/plugins/%s.spec", dummyNetworkDriver)
-	err = ioutil.WriteFile(fileName, []byte(s.server.URL), 0644)
+	fileName := fmt.Sprintf("/etc/docker/plugins/%s.spec", netDrv)
+	err = ioutil.WriteFile(fileName, []byte(url), 0644)
 	c.Assert(err, checker.IsNil)
 
-	ipamFileName := fmt.Sprintf("/etc/docker/plugins/%s.spec", dummyIpamDriver)
-	err = ioutil.WriteFile(ipamFileName, []byte(s.server.URL), 0644)
+	ipamFileName := fmt.Sprintf("/etc/docker/plugins/%s.spec", ipamDrv)
+	err = ioutil.WriteFile(ipamFileName, []byte(url), 0644)
 	c.Assert(err, checker.IsNil)
 }
 
@@ -830,6 +835,51 @@ func (s *DockerNetworkSuite) TestDockerNetworkOverlayPortMapping(c *check.C) {
 	c.Assert(out, checker.Contains, unpPort2)
 }
 
+func (s *DockerNetworkSuite) TestDockerNetworkDriverUngracefulRestart(c *check.C) {
+	testRequires(c, DaemonIsLinux, NotUserNamespace)
+	dnd := "dnd"
+	did := "did"
+
+	mux := http.NewServeMux()
+	server := httptest.NewServer(mux)
+	setupRemoteNetworkDrivers(c, mux, server.URL, dnd, did)
+
+	s.d.StartWithBusybox()
+	_, err := s.d.Cmd("network", "create", "-d", dnd, "--subnet", "1.1.1.0/24", "net1")
+	c.Assert(err, checker.IsNil)
+
+	_, err = s.d.Cmd("run", "-itd", "--net", "net1", "--name", "foo", "--ip", "1.1.1.10", "busybox", "sh")
+	c.Assert(err, checker.IsNil)
+
+	// Kill daemon and restart
+	if err = s.d.cmd.Process.Kill(); err != nil {
+		c.Fatal(err)
+	}
+
+	server.Close()
+
+	startTime := time.Now().Unix()
+	if err = s.d.Restart(); err != nil {
+		c.Fatal(err)
+	}
+	lapse := time.Now().Unix() - startTime
+	if lapse > 60 {
+		// In normal scenarios, daemon restart takes ~1 second.
+		// Plugin retry mechanism can delay the daemon start. systemd may not like it.
+		// Avoid accessing plugins during daemon bootup
+		c.Logf("daemon restart took too long : %d seconds", lapse)
+	}
+
+	// Restart the custom dummy plugin
+	mux = http.NewServeMux()
+	server = httptest.NewServer(mux)
+	setupRemoteNetworkDrivers(c, mux, server.URL, dnd, did)
+
+	// trying to reuse the same ip must succeed
+	_, err = s.d.Cmd("run", "-itd", "--net", "net1", "--name", "bar", "--ip", "1.1.1.10", "busybox", "sh")
+	c.Assert(err, checker.IsNil)
+}
+
 func (s *DockerNetworkSuite) TestDockerNetworkMacInspect(c *check.C) {
 	// Verify endpoint MAC address is correctly populated in container's network settings
 	nwn := "ov"

vendor/src/github.com/docker/libnetwork/CHANGELOG.md

@@ -1,5 +1,10 @@
 # Changelog
 
+## 0.6.0-rc1 (2016-01-14)
+- Fixes docker/docker#19404
+- Fixes the ungraceful daemon restart issue in systemd with remote network plugin 
+  (https://github.com/docker/libnetwork/issues/813)
+
 ## 0.5.6 (2016-01-14)
 - Setup embedded DNS server correctly on container restart. Fixes docker/docker#19354
 

vendor/src/github.com/docker/libnetwork/controller.go

@@ -387,7 +387,7 @@ func (c *controller) NewNetwork(networkType, name string, options ...NetworkOpti
 
 	// Make sure we have a driver available for this network type
 	// before we allocate anything.
-	if _, err := network.driver(); err != nil {
+	if _, err := network.driver(true); err != nil {
 		return nil, err
 	}
 
@@ -432,7 +432,7 @@ func (c *controller) NewNetwork(networkType, name string, options ...NetworkOpti
 }
 
 func (c *controller) addNetwork(n *network) error {
-	d, err := n.driver()
+	d, err := n.driver(true)
 	if err != nil {
 		return err
 	}

vendor/src/github.com/docker/libnetwork/default_gateway.go

@@ -84,7 +84,7 @@ func (sb *sandbox) clearDefaultGW() error {
 		return nil
 	}
 
-	if err := ep.sbLeave(sb); err != nil {
+	if err := ep.sbLeave(sb, false); err != nil {
 		return fmt.Errorf("container %s: endpoint leaving GW Network failed: %v", sb.containerID, err)
 	}
 	if err := ep.Delete(false); err != nil {

vendor/src/github.com/docker/libnetwork/endpoint.go

@@ -406,7 +406,7 @@ func (ep *endpoint) sbJoin(sbox Sandbox, options ...EndpointOption) error {
 
 	ep.processOptions(options...)
 
-	driver, err := network.driver()
+	driver, err := network.driver(true)
 	if err != nil {
 		return fmt.Errorf("failed to join endpoint: %v", err)
 	}
@@ -533,10 +533,10 @@ func (ep *endpoint) Leave(sbox Sandbox, options ...EndpointOption) error {
 	sb.joinLeaveStart()
 	defer sb.joinLeaveEnd()
 
-	return ep.sbLeave(sbox, options...)
+	return ep.sbLeave(sbox, false, options...)
 }
 
-func (ep *endpoint) sbLeave(sbox Sandbox, options ...EndpointOption) error {
+func (ep *endpoint) sbLeave(sbox Sandbox, force bool, options ...EndpointOption) error {
 	sb, ok := sbox.(*sandbox)
 	if !ok {
 		return types.BadRequestErrorf("not a valid Sandbox interface")
@@ -565,7 +565,7 @@ func (ep *endpoint) sbLeave(sbox Sandbox, options ...EndpointOption) error {
 
 	ep.processOptions(options...)
 
-	d, err := n.driver()
+	d, err := n.driver(!force)
 	if err != nil {
 		return fmt.Errorf("failed to leave endpoint: %v", err)
 	}
@@ -575,9 +575,11 @@ func (ep *endpoint) sbLeave(sbox Sandbox, options ...EndpointOption) error {
 	ep.network = n
 	ep.Unlock()
 
-	if err := d.Leave(n.id, ep.id); err != nil {
-		if _, ok := err.(types.MaskableError); !ok {
-			log.Warnf("driver error disconnecting container %s : %v", ep.name, err)
+	if d != nil {
+		if err := d.Leave(n.id, ep.id); err != nil {
+			if _, ok := err.(types.MaskableError); !ok {
+				log.Warnf("driver error disconnecting container %s : %v", ep.name, err)
+			}
 		}
 	}
 
@@ -649,7 +651,7 @@ func (ep *endpoint) Delete(force bool) error {
 	}
 
 	if sb != nil {
-		if e := ep.sbLeave(sb); e != nil {
+		if e := ep.sbLeave(sb, force); e != nil {
 			log.Warnf("failed to leave sandbox for endpoint %s : %v", name, e)
 		}
 	}
@@ -681,7 +683,7 @@ func (ep *endpoint) Delete(force bool) error {
 	// unwatch for service records
 	n.getController().unWatchSvcRecord(ep)
 
-	if err = ep.deleteEndpoint(); err != nil && !force {
+	if err = ep.deleteEndpoint(force); err != nil && !force {
 		return err
 	}
 
@@ -690,18 +692,22 @@ func (ep *endpoint) Delete(force bool) error {
 	return nil
 }
 
-func (ep *endpoint) deleteEndpoint() error {
+func (ep *endpoint) deleteEndpoint(force bool) error {
 	ep.Lock()
 	n := ep.network
 	name := ep.name
 	epid := ep.id
 	ep.Unlock()
 
-	driver, err := n.driver()
+	driver, err := n.driver(!force)
 	if err != nil {
 		return fmt.Errorf("failed to delete endpoint: %v", err)
 	}
 
+	if driver == nil {
+		return nil
+	}
+
 	if err := driver.DeleteEndpoint(n.id, epid); err != nil {
 		if _, ok := err.(types.ForbiddenError); ok {
 			return err
@@ -913,7 +919,7 @@ func (ep *endpoint) assignAddressVersion(ipVer int, ipam ipamapi.Ipam) error {
 		}
 	}
 	if progAdd != nil {
-		return types.BadRequestErrorf("Invalid preferred address %s: It does not belong to any of this network's subnets")
+		return types.BadRequestErrorf("Invalid preferred address %s: It does not belong to any of this network's subnets", prefAdd)
 	}
 	return fmt.Errorf("no available IPv%d addresses on this network's address pools: %s (%s)", ipVer, n.Name(), n.ID())
 }
@@ -956,7 +962,7 @@ func (c *controller) cleanupLocalEndpoints() {
 		}
 
 		for _, ep := range epl {
-			if err := ep.Delete(false); err != nil {
+			if err := ep.Delete(true); err != nil {
 				log.Warnf("Could not delete local endpoint %s during endpoint cleanup: %v", ep.name, err)
 			}
 		}

vendor/src/github.com/docker/libnetwork/endpoint_info.go

@@ -188,7 +188,7 @@ func (ep *endpoint) DriverInfo() (map[string]interface{}, error) {
 		return nil, fmt.Errorf("could not find network in store for driver info: %v", err)
 	}
 
-	driver, err := n.driver()
+	driver, err := n.driver(true)
 	if err != nil {
 		return nil, fmt.Errorf("failed to get driver info: %v", err)
 	}

vendor/src/github.com/docker/libnetwork/network.go

@@ -149,6 +149,7 @@ type network struct {
 	name         string
 	networkType  string
 	id           string
+	scope        string
 	ipamType     string
 	ipamOptions  map[string]string
 	addrSpace    string
@@ -246,6 +247,7 @@ func (n *network) New() datastore.KVObject {
 	return &network{
 		ctrlr:   n.ctrlr,
 		drvOnce: &sync.Once{},
+		scope:   n.scope,
 	}
 }
 
@@ -295,6 +297,7 @@ func (n *network) CopyTo(o datastore.KVObject) error {
 	dstN.name = n.name
 	dstN.id = n.id
 	dstN.networkType = n.networkType
+	dstN.scope = n.scope
 	dstN.ipamType = n.ipamType
 	dstN.enableIPv6 = n.enableIPv6
 	dstN.persist = n.persist
@@ -337,7 +340,7 @@ func (n *network) CopyTo(o datastore.KVObject) error {
 }
 
 func (n *network) DataScope() string {
-	return n.driverScope()
+	return n.Scope()
 }
 
 func (n *network) getEpCnt() *endpointCnt {
@@ -353,6 +356,7 @@ func (n *network) MarshalJSON() ([]byte, error) {
 	netMap["name"] = n.name
 	netMap["id"] = n.id
 	netMap["networkType"] = n.networkType
+	netMap["scope"] = n.scope
 	netMap["ipamType"] = n.ipamType
 	netMap["addrSpace"] = n.addrSpace
 	netMap["enableIPv6"] = n.enableIPv6
@@ -456,6 +460,9 @@ func (n *network) UnmarshalJSON(b []byte) (err error) {
 	if v, ok := netMap["internal"]; ok {
 		n.internal = v.(bool)
 	}
+	if s, ok := netMap["scope"]; ok {
+		n.scope = s.(string)
+	}
 	return nil
 }
 
@@ -566,7 +573,7 @@ func (n *network) driverScope() string {
 	return dd.capability.DataScope
 }
 
-func (n *network) driver() (driverapi.Driver, error) {
+func (n *network) driver(load bool) (driverapi.Driver, error) {
 	c := n.getController()
 
 	c.Lock()
@@ -574,14 +581,20 @@ func (n *network) driver() (driverapi.Driver, error) {
 	dd, ok := c.drivers[n.networkType]
 	c.Unlock()
 
-	if !ok {
+	if !ok && load {
 		var err error
 		dd, err = c.loadDriver(n.networkType)
 		if err != nil {
 			return nil, err
 		}
+	} else if !ok {
+		// dont fail if driver loading is not required
+		return nil, nil
 	}
 
+	n.Lock()
+	n.scope = dd.capability.DataScope
+	n.Unlock()
 	return dd.driver, nil
 }
 
@@ -631,7 +644,7 @@ func (n *network) Delete() error {
 }
 
 func (n *network) deleteNetwork() error {
-	d, err := n.driver()
+	d, err := n.driver(true)
 	if err != nil {
 		return fmt.Errorf("failed deleting network: %v", err)
 	}
@@ -651,7 +664,7 @@ func (n *network) deleteNetwork() error {
 }
 
 func (n *network) addEndpoint(ep *endpoint) error {
-	d, err := n.driver()
+	d, err := n.driver(true)
 	if err != nil {
 		return fmt.Errorf("failed to add endpoint: %v", err)
 	}
@@ -725,7 +738,7 @@ func (n *network) CreateEndpoint(name string, options ...EndpointOption) (Endpoi
 	}
 	defer func() {
 		if err != nil {
-			if e := ep.deleteEndpoint(); e != nil {
+			if e := ep.deleteEndpoint(false); e != nil {
 				log.Warnf("cleaning up endpoint failed %s : %v", name, e)
 			}
 		}
@@ -1169,7 +1182,9 @@ func (n *network) DriverOptions() map[string]string {
 }
 
 func (n *network) Scope() string {
-	return n.driverScope()
+	n.Lock()
+	defer n.Unlock()
+	return n.scope
 }
 
 func (n *network) IpamConfig() (string, map[string]string, []*IpamConf, []*IpamConf) {

vendor/src/github.com/docker/libnetwork/sandbox.go

@@ -160,6 +160,10 @@ func (sb *sandbox) Statistics() (map[string]*types.InterfaceStatistics, error) {
 }
 
 func (sb *sandbox) Delete() error {
+	return sb.delete(false)
+}
+
+func (sb *sandbox) delete(force bool) error {
 	sb.Lock()
 	if sb.inDelete {
 		sb.Unlock()
@@ -194,11 +198,13 @@ func (sb *sandbox) Delete() error {
 			continue
 		}
 
-		if err := ep.Leave(sb); err != nil {
-			log.Warnf("Failed detaching sandbox %s from endpoint %s: %v\n", sb.ID(), ep.ID(), err)
+		if !force {
+			if err := ep.Leave(sb); err != nil {
+				log.Warnf("Failed detaching sandbox %s from endpoint %s: %v\n", sb.ID(), ep.ID(), err)
+			}
 		}
 
-		if err := ep.Delete(false); err != nil {
+		if err := ep.Delete(force); err != nil {
 			log.Warnf("Failed deleting endpoint %s: %v\n", ep.ID(), err)
 		}
 	}

vendor/src/github.com/docker/libnetwork/sandbox_store.go

@@ -226,7 +226,7 @@ func (c *controller) sandboxCleanup() {
 			heap.Push(&sb.endpoints, ep)
 		}
 
-		if err := sb.Delete(); err != nil {
+		if err := sb.delete(true); err != nil {
 			logrus.Errorf("failed to delete sandbox %s while trying to cleanup: %v", sb.id, err)
 		}
 	}

vendor/src/github.com/docker/libnetwork/store.go

@@ -75,6 +75,7 @@ func (c *controller) getNetworkFromStore(nid string) (*network, error) {
 		}
 
 		n.epCnt = ec
+		n.scope = store.Scope()
 		return n, nil
 	}
 
@@ -107,6 +108,7 @@ func (c *controller) getNetworksForScope(scope string) ([]*network, error) {
 		}
 
 		n.epCnt = ec
+		n.scope = scope
 		nl = append(nl, n)
 	}
 
@@ -140,6 +142,7 @@ func (c *controller) getNetworksFromStore() ([]*network, error) {
 			}
 
 			n.epCnt = ec
+			n.scope = store.Scope()
 			nl = append(nl, n)
 		}
 	}
@@ -148,17 +151,21 @@ func (c *controller) getNetworksFromStore() ([]*network, error) {
 }
 
 func (n *network) getEndpointFromStore(eid string) (*endpoint, error) {
-	store := n.ctrlr.getStore(n.Scope())
-	if store == nil {
-		return nil, fmt.Errorf("could not find endpoint %s: datastore not found for scope %s", eid, n.Scope())
-	}
-
-	ep := &endpoint{id: eid, network: n}
-	err := store.GetObject(datastore.Key(ep.Key()...), ep)
-	if err != nil {
-		return nil, fmt.Errorf("could not find endpoint %s: %v", eid, err)
+	var errors []string
+	for _, store := range n.ctrlr.getStores() {
+		ep := &endpoint{id: eid, network: n}
+		err := store.GetObject(datastore.Key(ep.Key()...), ep)
+		// Continue searching in the next store if the key is not found in this store
+		if err != nil {
+			if err != datastore.ErrKeyNotFound {
+				errors = append(errors, fmt.Sprintf("{%s:%v}, ", store.Scope(), err))
+				log.Debugf("could not find endpoint %s in %s: %v", eid, store.Scope(), err)
+			}
+			continue
+		}
+		return ep, nil
 	}
-	return ep, nil
+	return nil, fmt.Errorf("could not find endpoint %s: %v", eid, errors)
 }
 
 func (n *network) getEndpointsFromStore() ([]*endpoint, error) {