Merge pull request #212 from mrjana/cnm_integ

Add endpoint priority during join and cleanup libnetwork test code

libnetwork/controller.go

@@ -59,7 +59,6 @@ import (
 	"github.com/docker/libnetwork/datastore"
 	"github.com/docker/libnetwork/driverapi"
 	"github.com/docker/libnetwork/hostdiscovery"
-	"github.com/docker/libnetwork/sandbox"
 	"github.com/docker/libnetwork/types"
 	"github.com/docker/swarm/pkg/store"
 )
@@ -91,11 +90,6 @@ type NetworkController interface {
 // When the function returns true, the walk will stop.
 type NetworkWalker func(nw Network) bool
 
-type sandboxData struct {
-	sandbox sandbox.Sandbox
-	refCnt  int
-}
-
 type networkTable map[types.UUID]*network
 type endpointTable map[types.UUID]*endpoint
 type sandboxTable map[string]*sandboxData
@@ -382,51 +376,6 @@ func (c *controller) NetworkByID(id string) (Network, error) {
 	return nil, ErrNoSuchNetwork(id)
 }
 
-func (c *controller) sandboxAdd(key string, create bool) (sandbox.Sandbox, error) {
-	c.Lock()
-	defer c.Unlock()
-
-	sData, ok := c.sandboxes[key]
-	if !ok {
-		sb, err := sandbox.NewSandbox(key, create)
-		if err != nil {
-			return nil, err
-		}
-
-		sData = &sandboxData{sandbox: sb, refCnt: 1}
-		c.sandboxes[key] = sData
-		return sData.sandbox, nil
-	}
-
-	sData.refCnt++
-	return sData.sandbox, nil
-}
-
-func (c *controller) sandboxRm(key string) {
-	c.Lock()
-	defer c.Unlock()
-
-	sData := c.sandboxes[key]
-	sData.refCnt--
-
-	if sData.refCnt == 0 {
-		sData.sandbox.Destroy()
-		delete(c.sandboxes, key)
-	}
-}
-
-func (c *controller) sandboxGet(key string) sandbox.Sandbox {
-	c.Lock()
-	defer c.Unlock()
-
-	sData, ok := c.sandboxes[key]
-	if !ok {
-		return nil
-	}
-
-	return sData.sandbox
-}
-
 func (c *controller) loadDriver(networkType string) (driverapi.Driver, error) {
 	// Plugins pkg performs lazy loading of plugins that acts as remote drivers.
 	// As per the design, this Get call will result in remote driver discovery if there is a corresponding plugin available.

libnetwork/endpoint.go

@@ -8,7 +8,6 @@ import (
 	"path/filepath"
 	"sync"
 
-	"github.com/Sirupsen/logrus"
 	"github.com/docker/docker/pkg/ioutils"
 	"github.com/docker/libnetwork/etchosts"
 	"github.com/docker/libnetwork/netlabel"
@@ -78,6 +77,7 @@ type containerConfig struct {
 	resolvConfPathConfig
 	generic           map[string]interface{}
 	useDefaultSandBox bool
+	prio              int // higher the value, more the priority
 }
 
 type extraHost struct {
@@ -101,7 +101,6 @@ type endpoint struct {
 	name          string
 	id            types.UUID
 	network       *network
-	sandboxInfo   *sandbox.Info
 	iFaces        []*endpointInterface
 	joinInfo      *endpointJoinInfo
 	container     *containerInfo
@@ -233,8 +232,6 @@ func (ep *endpoint) Join(containerID string, options ...EndpointOption) error {
 	container := ep.container
 	network := ep.network
 	epid := ep.id
-	joinInfo := ep.joinInfo
-	ifaces := ep.iFaces
 
 	ep.Unlock()
 	defer func() {
@@ -278,52 +275,32 @@ func (ep *endpoint) Join(containerID string, options ...EndpointOption) error {
 		return err
 	}
 
-	sb, err := ctrlr.sandboxAdd(sboxKey, !container.config.useDefaultSandBox)
+	sb, err := ctrlr.sandboxAdd(sboxKey, !container.config.useDefaultSandBox, ep)
 	if err != nil {
 		return err
 	}
 	defer func() {
 		if err != nil {
-			ctrlr.sandboxRm(sboxKey)
+			ctrlr.sandboxRm(sboxKey, ep)
 		}
 	}()
 
-	for _, i := range ifaces {
-		iface := &sandbox.Interface{
-			SrcName: i.srcName,
-			DstName: i.dstPrefix,
-			Address: &i.addr,
-			Routes:  i.routes,
-		}
-		if i.addrv6.IP.To16() != nil {
-			iface.AddressIPv6 = &i.addrv6
-		}
-		err = sb.AddInterface(iface)
-		if err != nil {
-			return err
-		}
-	}
-	// Set up non-interface routes.
-	for _, r := range ep.joinInfo.StaticRoutes {
-		err = sb.AddStaticRoute(r)
-		if err != nil {
-			return err
-		}
-	}
+	container.data.SandboxKey = sb.Key()
 
-	err = sb.SetGateway(joinInfo.gw)
-	if err != nil {
-		return err
-	}
+	return nil
+}
 
-	err = sb.SetGatewayIPv6(joinInfo.gw6)
-	if err != nil {
-		return err
-	}
+func (ep *endpoint) hasInterface(iName string) bool {
+	ep.Lock()
+	defer ep.Unlock()
 
-	container.data.SandboxKey = sb.Key()
+	for _, iface := range ep.iFaces {
+		if iface.srcName == iName {
+			return true
+		}
+	}
 
-	return nil
+	return false
 }
 
 func (ep *endpoint) Leave(containerID string, options ...EndpointOption) error {
@@ -359,23 +336,7 @@ func (ep *endpoint) Leave(containerID string, options ...EndpointOption) error {
 
 	err = driver.Leave(n.id, ep.id)
 
-	sb := ctrlr.sandboxGet(container.data.SandboxKey)
-	for _, i := range sb.Interfaces() {
-		err = sb.RemoveInterface(i)
-		if err != nil {
-			logrus.Debugf("Remove interface failed: %v", err)
-		}
-	}
-
-	// Remove non-interface routes.
-	for _, r := range ep.joinInfo.StaticRoutes {
-		err = sb.RemoveStaticRoute(r)
-		if err != nil {
-			logrus.Debugf("Remove route failed: %v", err)
-		}
-	}
-
-	ctrlr.sandboxRm(container.data.SandboxKey)
+	ctrlr.sandboxRm(container.data.SandboxKey, ep)
 
 	return err
 }
@@ -632,6 +593,14 @@ func EndpointOptionGeneric(generic map[string]interface{}) EndpointOption {
 	}
 }
 
+// JoinOptionPriority function returns an option setter for priority option to
+// be passed to endpoint Join method.
+func JoinOptionPriority(prio int) EndpointOption {
+	return func(ep *endpoint) {
+		ep.container.config.prio = prio
+	}
+}
+
 // JoinOptionHostname function returns an option setter for hostname option to
 // be passed to endpoint Join method.
 func JoinOptionHostname(name string) EndpointOption {

libnetwork/libnetwork_test.go

@@ -33,35 +33,46 @@ const (
 	bridgeName    = "docker0"
 )
 
+var controller libnetwork.NetworkController
+
 func TestMain(m *testing.M) {
 	if reexec.Init() {
 		return
 	}
-	os.Exit(m.Run())
-}
 
-func createTestController() (libnetwork.NetworkController, error) {
-	controller, err := libnetwork.New("")
+	if err := createController(); err != nil {
+		os.Exit(1)
+	}
+	option := options.Generic{
+		"EnableIPForwarding": true,
+	}
+
+	genericOption := make(map[string]interface{})
+	genericOption[netlabel.GenericData] = option
+
+	err := controller.ConfigureNetworkDriver(bridgeNetType, genericOption)
 	if err != nil {
-		return nil, err
+		//m.Fatal(err)
+		os.Exit(1)
 	}
+
 	libnetwork.SetTestDataStore(controller, datastore.NewCustomDataStore(datastore.NewMockStore()))
-	return controller, nil
+
+	os.Exit(m.Run())
 }
 
-func createTestNetwork(networkType, networkName string, option options.Generic, netOption options.Generic) (libnetwork.Network, error) {
-	controller, err := createTestController()
-	if err != nil {
-		return nil, err
-	}
-	genericOption := make(map[string]interface{})
-	genericOption[netlabel.GenericData] = option
+func createController() error {
+	var err error
 
-	err = controller.ConfigureNetworkDriver(networkType, genericOption)
+	controller, err = libnetwork.New("")
 	if err != nil {
-		return nil, err
+		return err
 	}
 
+	return nil
+}
+
+func createTestNetwork(networkType, networkName string, netOption options.Generic) (libnetwork.Network, error) {
 	network, err := controller.NewNetwork(networkType, networkName,
 		libnetwork.NetworkOptionGeneric(netOption))
 	if err != nil {
@@ -86,8 +97,7 @@ func getPortMapping() []types.PortBinding {
 }
 
 func TestNull(t *testing.T) {
-	network, err := createTestNetwork("null", "testnetwork", options.Generic{},
-		options.Generic{})
+	network, err := createTestNetwork("null", "testnetwork", options.Generic{})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -120,7 +130,7 @@ func TestNull(t *testing.T) {
 }
 
 func TestHost(t *testing.T) {
-	network, err := createTestNetwork("host", "testnetwork", options.Generic{}, options.Generic{})
+	network, err := createTestNetwork("host", "testnetwork", options.Generic{})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -224,9 +234,6 @@ func TestBridge(t *testing.T) {
 	cidrv6.IP = ip
 
 	log.Debug("Adding a bridge")
-	option := options.Generic{
-		"EnableIPForwarding": true,
-	}
 
 	netOption := options.Generic{
 		netlabel.GenericData: options.Generic{
@@ -242,7 +249,7 @@ func TestBridge(t *testing.T) {
 		},
 	}
 
-	network, err := createTestNetwork(bridgeNetType, "testnetwork", option, netOption)
+	network, err := createTestNetwork(bridgeNetType, "testnetwork", netOption)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -282,23 +289,18 @@ func TestUnknownDriver(t *testing.T) {
 		defer netutils.SetupTestNetNS(t)()
 	}
 
-	_, err := createTestNetwork("unknowndriver", "testnetwork", options.Generic{}, options.Generic{})
+	_, err := createTestNetwork("unknowndriver", "testnetwork", options.Generic{})
 	if err == nil {
 		t.Fatal("Expected to fail. But instead succeeded")
 	}
 
-	if _, ok := err.(libnetwork.NetworkTypeError); !ok {
+	if _, ok := err.(types.NotFoundError); !ok {
 		t.Fatalf("Did not fail with expected error. Actual error: %v", err)
 	}
 }
 
 func TestNilRemoteDriver(t *testing.T) {
-	controller, err := createTestController()
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	_, err = controller.NewNetwork("framerelay", "dummy",
+	_, err := controller.NewNetwork("framerelay", "dummy",
 		libnetwork.NetworkOptionGeneric(getEmptyGenericOption()))
 	if err == nil {
 		t.Fatal("Expected to fail. But instead succeeded")
@@ -314,23 +316,15 @@ func TestDuplicateNetwork(t *testing.T) {
 		defer netutils.SetupTestNetNS(t)()
 	}
 
-	controller, err := createTestController()
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	genericOption := make(map[string]interface{})
-	genericOption[netlabel.GenericData] = options.Generic{}
-
-	err = controller.ConfigureNetworkDriver(bridgeNetType, genericOption)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	_, err = controller.NewNetwork(bridgeNetType, "testnetwork", nil)
+	n, err := controller.NewNetwork(bridgeNetType, "testnetwork", nil)
 	if err != nil {
 		t.Fatal(err)
 	}
+	defer func() {
+		if err := n.Delete(); err != nil {
+			t.Fatal(err)
+		}
+	}()
 
 	_, err = controller.NewNetwork(bridgeNetType, "testnetwork")
 	if err == nil {
@@ -347,7 +341,7 @@ func TestNetworkName(t *testing.T) {
 		defer netutils.SetupTestNetNS(t)()
 	}
 
-	_, err := createTestNetwork(bridgeNetType, "", options.Generic{}, options.Generic{})
+	_, err := createTestNetwork(bridgeNetType, "", options.Generic{})
 	if err == nil {
 		t.Fatal("Expected to fail. But instead succeeded")
 	}
@@ -357,10 +351,15 @@ func TestNetworkName(t *testing.T) {
 	}
 
 	networkName := "testnetwork"
-	n, err := createTestNetwork(bridgeNetType, networkName, options.Generic{}, options.Generic{})
+	n, err := createTestNetwork(bridgeNetType, networkName, options.Generic{})
 	if err != nil {
 		t.Fatal(err)
 	}
+	defer func() {
+		if err := n.Delete(); err != nil {
+			t.Fatal(err)
+		}
+	}()
 
 	if n.Name() != networkName {
 		t.Fatalf("Expected network name %s, got %s", networkName, n.Name())
@@ -372,10 +371,15 @@ func TestNetworkType(t *testing.T) {
 		defer netutils.SetupTestNetNS(t)()
 	}
 
-	n, err := createTestNetwork(bridgeNetType, "testnetwork", options.Generic{}, options.Generic{})
+	n, err := createTestNetwork(bridgeNetType, "testnetwork", options.Generic{})
 	if err != nil {
 		t.Fatal(err)
 	}
+	defer func() {
+		if err := n.Delete(); err != nil {
+			t.Fatal(err)
+		}
+	}()
 
 	if n.Type() != bridgeNetType {
 		t.Fatalf("Expected network type %s, got %s", bridgeNetType, n.Type())
@@ -387,10 +391,15 @@ func TestNetworkID(t *testing.T) {
 		defer netutils.SetupTestNetNS(t)()
 	}
 
-	n, err := createTestNetwork(bridgeNetType, "testnetwork", options.Generic{}, options.Generic{})
+	n, err := createTestNetwork(bridgeNetType, "testnetwork", options.Generic{})
 	if err != nil {
 		t.Fatal(err)
 	}
+	defer func() {
+		if err := n.Delete(); err != nil {
+			t.Fatal(err)
+		}
+	}()
 
 	if n.ID() == "" {
 		t.Fatal("Expected non-empty network id")
@@ -402,11 +411,14 @@ func TestDeleteNetworkWithActiveEndpoints(t *testing.T) {
 		defer netutils.SetupTestNetNS(t)()
 	}
 
-	option := options.Generic{
+	netOption := options.Generic{
 		"BridgeName":            bridgeName,
 		"AllowNonDefaultBridge": true}
+	option := options.Generic{
+		netlabel.GenericData: netOption,
+	}
 
-	network, err := createTestNetwork(bridgeNetType, "testnetwork", options.Generic{}, option)
+	network, err := createTestNetwork(bridgeNetType, "testnetwork", option)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -440,11 +452,14 @@ func TestUnknownNetwork(t *testing.T) {
 		defer netutils.SetupTestNetNS(t)()
 	}
 
-	option := options.Generic{
+	netOption := options.Generic{
 		"BridgeName":            bridgeName,
 		"AllowNonDefaultBridge": true}
+	option := options.Generic{
+		netlabel.GenericData: netOption,
+	}
 
-	network, err := createTestNetwork(bridgeNetType, "testnetwork", options.Generic{}, option)
+	network, err := createTestNetwork(bridgeNetType, "testnetwork", option)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -475,12 +490,15 @@ func TestUnknownEndpoint(t *testing.T) {
 	}
 	subnet.IP = ip
 
-	option := options.Generic{
+	netOption := options.Generic{
 		"BridgeName":            bridgeName,
 		"AddressIPv4":           subnet,
 		"AllowNonDefaultBridge": true}
+	option := options.Generic{
+		netlabel.GenericData: netOption,
+	}
 
-	network, err := createTestNetwork(bridgeNetType, "testnetwork", options.Generic{}, option)
+	network, err := createTestNetwork(bridgeNetType, "testnetwork", option)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -523,29 +541,36 @@ func TestNetworkEndpointsWalkers(t *testing.T) {
 		defer netutils.SetupTestNetNS(t)()
 	}
 
-	controller, err := createTestController()
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	err = controller.ConfigureNetworkDriver(bridgeNetType, getEmptyGenericOption())
-	if err != nil {
-		t.Fatal(err)
-	}
-
 	// Create network 1 and add 2 endpoint: ep11, ep12
 	net1, err := controller.NewNetwork(bridgeNetType, "network1")
 	if err != nil {
 		t.Fatal(err)
 	}
+	defer func() {
+		if err := net1.Delete(); err != nil {
+			t.Fatal(err)
+		}
+	}()
+
 	ep11, err := net1.CreateEndpoint("ep11")
 	if err != nil {
 		t.Fatal(err)
 	}
+	defer func() {
+		if err := ep11.Delete(); err != nil {
+			t.Fatal(err)
+		}
+	}()
+
 	ep12, err := net1.CreateEndpoint("ep12")
 	if err != nil {
 		t.Fatal(err)
 	}
+	defer func() {
+		if err := ep12.Delete(); err != nil {
+			t.Fatal(err)
+		}
+	}()
 
 	// Test list methods on net1
 	epList1 := net1.Endpoints()
@@ -607,21 +632,16 @@ func TestControllerQuery(t *testing.T) {
 		defer netutils.SetupTestNetNS(t)()
 	}
 
-	controller, err := createTestController()
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	err = controller.ConfigureNetworkDriver(bridgeNetType, getEmptyGenericOption())
-	if err != nil {
-		t.Fatal(err)
-	}
-
 	// Create network 1
 	net1, err := controller.NewNetwork(bridgeNetType, "network1")
 	if err != nil {
 		t.Fatal(err)
 	}
+	defer func() {
+		if err := net1.Delete(); err != nil {
+			t.Fatal(err)
+		}
+	}()
 
 	_, err = controller.NetworkByName("")
 	if err == nil {
@@ -673,29 +693,36 @@ func TestNetworkQuery(t *testing.T) {
 		defer netutils.SetupTestNetNS(t)()
 	}
 
-	controller, err := createTestController()
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	err = controller.ConfigureNetworkDriver(bridgeNetType, getEmptyGenericOption())
-	if err != nil {
-		t.Fatal(err)
-	}
-
 	// Create network 1 and add 2 endpoint: ep11, ep12
 	net1, err := controller.NewNetwork(bridgeNetType, "network1")
 	if err != nil {
 		t.Fatal(err)
 	}
+	defer func() {
+		if err := net1.Delete(); err != nil {
+			t.Fatal(err)
+		}
+	}()
+
 	ep11, err := net1.CreateEndpoint("ep11")
 	if err != nil {
 		t.Fatal(err)
 	}
+	defer func() {
+		if err := ep11.Delete(); err != nil {
+			t.Fatal(err)
+		}
+	}()
+
 	ep12, err := net1.CreateEndpoint("ep12")
 	if err != nil {
 		t.Fatal(err)
 	}
+	defer func() {
+		if err := ep12.Delete(); err != nil {
+			t.Fatal(err)
+		}
+	}()
 
 	e, err := net1.EndpointByName("ep11")
 	if err != nil {
@@ -770,6 +797,11 @@ func checkSandbox(t *testing.T, info libnetwork.EndpointInfo) {
 	if err != nil {
 		t.Fatalf("Could not find the interface eth0 inside the sandbox: %v", err)
 	}
+
+	_, err = netlink.LinkByName("eth1")
+	if err != nil {
+		t.Fatalf("Could not find the interface eth1 inside the sandbox: %v", err)
+	}
 }
 
 func TestEndpointJoin(t *testing.T) {
@@ -777,18 +809,28 @@ func TestEndpointJoin(t *testing.T) {
 		defer netutils.SetupTestNetNS(t)()
 	}
 
-	n, err := createTestNetwork(bridgeNetType, "testnetwork", options.Generic{}, options.Generic{})
+	n1, err := createTestNetwork(bridgeNetType, "testnetwork1", options.Generic{})
 	if err != nil {
 		t.Fatal(err)
 	}
+	defer func() {
+		if err := n1.Delete(); err != nil {
+			t.Fatal(err)
+		}
+	}()
 
-	ep, err := n.CreateEndpoint("ep1")
+	ep1, err := n1.CreateEndpoint("ep1")
 	if err != nil {
 		t.Fatal(err)
 	}
+	defer func() {
+		if err := ep1.Delete(); err != nil {
+			t.Fatal(err)
+		}
+	}()
 
 	// Validate if ep.Info() only gives me IP address info and not names and gateway during CreateEndpoint()
-	info := ep.Info()
+	info := ep1.Info()
 
 	for _, iface := range info.InterfaceList() {
 		if iface.Address().IP.To4() == nil {
@@ -804,7 +846,7 @@ func TestEndpointJoin(t *testing.T) {
 		t.Fatalf("Expected an empty sandbox key for an empty endpoint. Instead found a non-empty sandbox key: %s", info.SandboxKey())
 	}
 
-	err = ep.Join(containerID,
+	err = ep1.Join(containerID,
 		libnetwork.JoinOptionHostname("test"),
 		libnetwork.JoinOptionDomainname("docker.io"),
 		libnetwork.JoinOptionExtraHost("web", "192.168.0.1"))
@@ -813,14 +855,14 @@ func TestEndpointJoin(t *testing.T) {
 	}
 
 	defer func() {
-		err = ep.Leave(containerID)
+		err = ep1.Leave(containerID)
 		if err != nil {
 			t.Fatal(err)
 		}
 	}()
 
 	// Validate if ep.Info() only gives valid gateway and sandbox key after has container has joined.
-	info = ep.Info()
+	info = ep1.Info()
 	if info.Gateway().To4() == nil {
 		t.Fatalf("Expected a valid gateway for a joined endpoint. Instead found an invalid gateway: %v", info.Gateway())
 	}
@@ -829,6 +871,45 @@ func TestEndpointJoin(t *testing.T) {
 		t.Fatalf("Expected an non-empty sandbox key for a joined endpoint. Instead found a empty sandbox key")
 	}
 
+	// Now test the container joining another network
+	n2, err := createTestNetwork(bridgeNetType, "testnetwork2",
+		options.Generic{
+			netlabel.GenericData: options.Generic{
+				"BridgeName":            "secondary",
+				"AllowNonDefaultBridge": true,
+			},
+		})
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer func() {
+		if err := n2.Delete(); err != nil {
+			t.Fatal(err)
+		}
+	}()
+
+	ep2, err := n2.CreateEndpoint("ep2")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer func() {
+		if err := ep2.Delete(); err != nil {
+			t.Fatal(err)
+		}
+	}()
+
+	err = ep2.Join(containerID)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	defer func() {
+		err = ep2.Leave(containerID)
+		if err != nil {
+			t.Fatal(err)
+		}
+	}()
+
 	checkSandbox(t, info)
 }
 
@@ -837,15 +918,25 @@ func TestEndpointJoinInvalidContainerId(t *testing.T) {
 		defer netutils.SetupTestNetNS(t)()
 	}
 
-	n, err := createTestNetwork(bridgeNetType, "testnetwork", options.Generic{}, options.Generic{})
+	n, err := createTestNetwork(bridgeNetType, "testnetwork", options.Generic{})
 	if err != nil {
 		t.Fatal(err)
 	}
+	defer func() {
+		if err := n.Delete(); err != nil {
+			t.Fatal(err)
+		}
+	}()
 
 	ep, err := n.CreateEndpoint("ep1")
 	if err != nil {
 		t.Fatal(err)
 	}
+	defer func() {
+		if err := ep.Delete(); err != nil {
+			t.Fatal(err)
+		}
+	}()
 
 	err = ep.Join("")
 	if err == nil {
@@ -862,10 +953,15 @@ func TestEndpointDeleteWithActiveContainer(t *testing.T) {
 		defer netutils.SetupTestNetNS(t)()
 	}
 
-	n, err := createTestNetwork(bridgeNetType, "testnetwork", options.Generic{}, options.Generic{})
+	n, err := createTestNetwork(bridgeNetType, "testnetwork", options.Generic{})
 	if err != nil {
 		t.Fatal(err)
 	}
+	defer func() {
+		if err := n.Delete(); err != nil {
+			t.Fatal(err)
+		}
+	}()
 
 	ep, err := n.CreateEndpoint("ep1")
 	if err != nil {
@@ -906,15 +1002,25 @@ func TestEndpointMultipleJoins(t *testing.T) {
 		defer netutils.SetupTestNetNS(t)()
 	}
 
-	n, err := createTestNetwork(bridgeNetType, "testnetwork", options.Generic{}, options.Generic{})
+	n, err := createTestNetwork(bridgeNetType, "testnetwork", options.Generic{})
 	if err != nil {
 		t.Fatal(err)
 	}
+	defer func() {
+		if err := n.Delete(); err != nil {
+			t.Fatal(err)
+		}
+	}()
 
 	ep, err := n.CreateEndpoint("ep1")
 	if err != nil {
 		t.Fatal(err)
 	}
+	defer func() {
+		if err := ep.Delete(); err != nil {
+			t.Fatal(err)
+		}
+	}()
 
 	err = ep.Join(containerID,
 		libnetwork.JoinOptionHostname("test"),
@@ -946,15 +1052,25 @@ func TestEndpointInvalidLeave(t *testing.T) {
 		defer netutils.SetupTestNetNS(t)()
 	}
 
-	n, err := createTestNetwork(bridgeNetType, "testnetwork", options.Generic{}, options.Generic{})
+	n, err := createTestNetwork(bridgeNetType, "testnetwork", options.Generic{})
 	if err != nil {
 		t.Fatal(err)
 	}
+	defer func() {
+		if err := n.Delete(); err != nil {
+			t.Fatal(err)
+		}
+	}()
 
 	ep, err := n.CreateEndpoint("ep1")
 	if err != nil {
 		t.Fatal(err)
 	}
+	defer func() {
+		if err := ep.Delete(); err != nil {
+			t.Fatal(err)
+		}
+	}()
 
 	err = ep.Leave(containerID)
 	if err == nil {
@@ -1007,15 +1123,25 @@ func TestEndpointUpdateParent(t *testing.T) {
 		defer netutils.SetupTestNetNS(t)()
 	}
 
-	n, err := createTestNetwork("bridge", "testnetwork", options.Generic{}, options.Generic{})
+	n, err := createTestNetwork("bridge", "testnetwork", options.Generic{})
 	if err != nil {
 		t.Fatal(err)
 	}
+	defer func() {
+		if err := n.Delete(); err != nil {
+			t.Fatal(err)
+		}
+	}()
 
-	ep1, err := n.CreateEndpoint("ep1", nil)
+	ep1, err := n.CreateEndpoint("ep1")
 	if err != nil {
 		t.Fatal(err)
 	}
+	defer func() {
+		if err := ep1.Delete(); err != nil {
+			t.Fatal(err)
+		}
+	}()
 
 	err = ep1.Join(containerID,
 		libnetwork.JoinOptionHostname("test1"),
@@ -1032,10 +1158,15 @@ func TestEndpointUpdateParent(t *testing.T) {
 		}
 	}()
 
-	ep2, err := n.CreateEndpoint("ep2", nil)
+	ep2, err := n.CreateEndpoint("ep2")
 	if err != nil {
 		t.Fatal(err)
 	}
+	defer func() {
+		if err := ep2.Delete(); err != nil {
+			t.Fatal(err)
+		}
+	}()
 
 	err = ep2.Join("container2",
 		libnetwork.JoinOptionHostname("test2"),
@@ -1087,15 +1218,25 @@ func TestEnableIPv6(t *testing.T) {
 		},
 	}
 
-	n, err := createTestNetwork("bridge", "testnetwork", options.Generic{}, netOption)
+	n, err := createTestNetwork("bridge", "testnetwork", netOption)
 	if err != nil {
 		t.Fatal(err)
 	}
+	defer func() {
+		if err := n.Delete(); err != nil {
+			t.Fatal(err)
+		}
+	}()
 
-	ep1, err := n.CreateEndpoint("ep1", nil)
+	ep1, err := n.CreateEndpoint("ep1")
 	if err != nil {
 		t.Fatal(err)
 	}
+	defer func() {
+		if err := ep1.Delete(); err != nil {
+			t.Fatal(err)
+		}
+	}()
 
 	if err := ioutil.WriteFile("/etc/resolv.conf", tmpResolvConf, 0644); err != nil {
 		t.Fatal(err)
@@ -1154,15 +1295,25 @@ func TestResolvConf(t *testing.T) {
 		}
 	}()
 
-	n, err := createTestNetwork("bridge", "testnetwork", options.Generic{}, options.Generic{})
+	n, err := createTestNetwork("bridge", "testnetwork", options.Generic{})
 	if err != nil {
 		t.Fatal(err)
 	}
+	defer func() {
+		if err := n.Delete(); err != nil {
+			t.Fatal(err)
+		}
+	}()
 
-	ep1, err := n.CreateEndpoint("ep1", nil)
+	ep1, err := n.CreateEndpoint("ep1")
 	if err != nil {
 		t.Fatal(err)
 	}
+	defer func() {
+		if err := ep1.Delete(); err != nil {
+			t.Fatal(err)
+		}
+	}()
 
 	if err := ioutil.WriteFile("/etc/resolv.conf", tmpResolvConf1, 0644); err != nil {
 		t.Fatal(err)
@@ -1339,21 +1490,24 @@ func TestValidRemoteDriver(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	controller, err := libnetwork.New("")
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	_, err = controller.NewNetwork("valid-network-driver", "dummy",
+	n, err := controller.NewNetwork("valid-network-driver", "dummy",
 		libnetwork.NetworkOptionGeneric(getEmptyGenericOption()))
 	if err != nil {
-		t.Fatal(err)
+		// Only fail if we could not find the plugin driver
+		if _, ok := err.(types.NotFoundError); ok {
+			t.Fatal(err)
+		}
+		return
 	}
+	defer func() {
+		if err := n.Delete(); err != nil {
+			t.Fatal(err)
+		}
+	}()
 }
 
 var (
 	once   sync.Once
-	ctrlr  libnetwork.NetworkController
 	start  = make(chan struct{})
 	done   = make(chan chan struct{}, numThreads-1)
 	origns = netns.None()
@@ -1386,17 +1540,7 @@ func createGlobalInstance(t *testing.T) {
 		}
 	}
 
-	ctrlr, err = createTestController()
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	err = ctrlr.ConfigureNetworkDriver(bridgeNetType, getEmptyGenericOption())
-	if err != nil {
-		t.Fatal("configure driver")
-	}
-
-	net, err := ctrlr.NewNetwork(bridgeNetType, "network1")
+	net, err := controller.NewNetwork(bridgeNetType, "network1")
 	if err != nil {
 		t.Fatal("new network")
 	}
@@ -1489,7 +1633,7 @@ func runParallelTests(t *testing.T, thrNumber int) {
 	}
 	defer netns.Set(origns)
 
-	net, err := ctrlr.NetworkByName("network1")
+	net, err := controller.NetworkByName("network1")
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -1524,6 +1668,10 @@ func runParallelTests(t *testing.T, thrNumber int) {
 		if err != nil {
 			t.Fatal(err)
 		}
+
+		if err := net.Delete(); err != nil {
+			t.Fatal(err)
+		}
 	}
 }
 

libnetwork/sandbox/configure_linux.go

@@ -30,7 +30,7 @@ func configureInterface(iface netlink.Link, settings *Interface) error {
 	return nil
 }
 
-func programGateway(path string, gw net.IP) error {
+func programGateway(path string, gw net.IP, isAdd bool) error {
 	runtime.LockOSThread()
 	defer runtime.UnlockOSThread()
 
@@ -57,7 +57,15 @@ func programGateway(path string, gw net.IP) error {
 		return fmt.Errorf("route for the gateway could not be found: %v", err)
 	}
 
-	return netlink.RouteAdd(&netlink.Route{
+	if isAdd {
+		return netlink.RouteAdd(&netlink.Route{
+			Scope:     netlink.SCOPE_UNIVERSE,
+			LinkIndex: gwRoutes[0].LinkIndex,
+			Gw:        gw,
+		})
+	}
+
+	return netlink.RouteDel(&netlink.Route{
 		Scope:     netlink.SCOPE_UNIVERSE,
 		LinkIndex: gwRoutes[0].LinkIndex,
 		Gw:        gw,

libnetwork/sandbox/namespace_linux.go

@@ -308,26 +308,72 @@ func (n *networkNamespace) AddInterface(i *Interface) error {
 }
 
 func (n *networkNamespace) SetGateway(gw net.IP) error {
+	// Silently return if the gateway is empty
 	if len(gw) == 0 {
 		return nil
 	}
 
-	err := programGateway(n.path, gw)
+	err := programGateway(n.path, gw, true)
 	if err == nil {
+		n.Lock()
 		n.sinfo.Gateway = gw
+		n.Unlock()
+	}
+
+	return err
+}
+
+func (n *networkNamespace) UnsetGateway() error {
+	n.Lock()
+	gw := n.sinfo.Gateway
+	n.Unlock()
+
+	// Silently return if the gateway is empty
+	if len(gw) == 0 {
+		return nil
+	}
+
+	err := programGateway(n.path, gw, false)
+	if err == nil {
+		n.Lock()
+		n.sinfo.Gateway = net.IP{}
+		n.Unlock()
 	}
 
 	return err
 }
 
 func (n *networkNamespace) SetGatewayIPv6(gw net.IP) error {
+	// Silently return if the gateway is empty
 	if len(gw) == 0 {
 		return nil
 	}
 
-	err := programGateway(n.path, gw)
+	err := programGateway(n.path, gw, true)
 	if err == nil {
+		n.Lock()
 		n.sinfo.GatewayIPv6 = gw
+		n.Unlock()
+	}
+
+	return err
+}
+
+func (n *networkNamespace) UnsetGatewayIPv6() error {
+	n.Lock()
+	gw := n.sinfo.GatewayIPv6
+	n.Unlock()
+
+	// Silently return if the gateway is empty
+	if len(gw) == 0 {
+		return nil
+	}
+
+	err := programGateway(n.path, gw, false)
+	if err == nil {
+		n.Lock()
+		n.sinfo.GatewayIPv6 = net.IP{}
+		n.Unlock()
 	}
 
 	return err

libnetwork/sandbox/sandbox.go

@@ -35,6 +35,12 @@ type Sandbox interface {
 	// Set default IPv6 gateway for the sandbox
 	SetGatewayIPv6(gw net.IP) error
 
+	// Unset the previously set default IPv4 gateway in the sandbox
+	UnsetGateway() error
+
+	// Unset the previously set default IPv6 gateway in the sandbox
+	UnsetGatewayIPv6() error
+
 	// Add a static route to the sandbox.
 	AddStaticRoute(*types.StaticRoute) error
 

libnetwork/sandboxdata.go

@@ -0,0 +1,246 @@
+package libnetwork
+
+import (
+	"container/heap"
+	"sync"
+
+	"github.com/Sirupsen/logrus"
+	"github.com/docker/libnetwork/sandbox"
+)
+
+type epHeap []*endpoint
+
+type sandboxData struct {
+	sbox      sandbox.Sandbox
+	refCnt    int
+	endpoints epHeap
+	sync.Mutex
+}
+
+func (eh epHeap) Len() int { return len(eh) }
+
+func (eh epHeap) Less(i, j int) bool {
+	eh[i].Lock()
+	eh[j].Lock()
+	defer eh[j].Unlock()
+	defer eh[i].Unlock()
+
+	if eh[i].container.config.prio == eh[j].container.config.prio {
+		return eh[i].network.Name() < eh[j].network.Name()
+	}
+
+	return eh[i].container.config.prio > eh[j].container.config.prio
+}
+
+func (eh epHeap) Swap(i, j int) { eh[i], eh[j] = eh[j], eh[i] }
+
+func (eh *epHeap) Push(x interface{}) {
+	*eh = append(*eh, x.(*endpoint))
+}
+
+func (eh *epHeap) Pop() interface{} {
+	old := *eh
+	n := len(old)
+	x := old[n-1]
+	*eh = old[0 : n-1]
+	return x
+}
+
+func (s *sandboxData) updateGateway(ep *endpoint) error {
+	sb := s.sandbox()
+	if err := sb.UnsetGateway(); err != nil {
+		return err
+	}
+
+	if err := sb.UnsetGatewayIPv6(); err != nil {
+		return err
+	}
+
+	if ep == nil {
+		return nil
+	}
+
+	ep.Lock()
+	joinInfo := ep.joinInfo
+	ep.Unlock()
+
+	if err := sb.SetGateway(joinInfo.gw); err != nil {
+		return err
+	}
+
+	if err := sb.SetGatewayIPv6(joinInfo.gw6); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (s *sandboxData) addEndpoint(ep *endpoint) error {
+	ep.Lock()
+	joinInfo := ep.joinInfo
+	ifaces := ep.iFaces
+	ep.Unlock()
+
+	sb := s.sandbox()
+	for _, i := range ifaces {
+		iface := &sandbox.Interface{
+			SrcName: i.srcName,
+			DstName: i.dstPrefix,
+			Address: &i.addr,
+			Routes:  i.routes,
+		}
+		if i.addrv6.IP.To16() != nil {
+			iface.AddressIPv6 = &i.addrv6
+		}
+
+		if err := sb.AddInterface(iface); err != nil {
+			return err
+		}
+	}
+
+	if joinInfo != nil {
+		// Set up non-interface routes.
+		for _, r := range ep.joinInfo.StaticRoutes {
+			if err := sb.AddStaticRoute(r); err != nil {
+				return err
+			}
+		}
+	}
+
+	s.Lock()
+	heap.Push(&s.endpoints, ep)
+	highEp := s.endpoints[0]
+	s.Unlock()
+
+	if ep == highEp {
+		if err := s.updateGateway(ep); err != nil {
+			return err
+		}
+	}
+
+	s.Lock()
+	s.refCnt++
+	s.Unlock()
+
+	return nil
+}
+
+func (s *sandboxData) rmEndpoint(ep *endpoint) int {
+	ep.Lock()
+	joinInfo := ep.joinInfo
+	ep.Unlock()
+
+	sb := s.sandbox()
+	for _, i := range sb.Interfaces() {
+		// Only remove the interfaces owned by this endpoint from the sandbox.
+		if ep.hasInterface(i.SrcName) {
+			if err := sb.RemoveInterface(i); err != nil {
+				logrus.Debugf("Remove interface failed: %v", err)
+			}
+		}
+	}
+
+	// Remove non-interface routes.
+	for _, r := range joinInfo.StaticRoutes {
+		if err := sb.RemoveStaticRoute(r); err != nil {
+			logrus.Debugf("Remove route failed: %v", err)
+		}
+	}
+
+	// We don't check if s.endpoints is empty here because
+	// it should never be empty during a rmEndpoint call and
+	// if it is we will rightfully panic here
+	s.Lock()
+	highEpBefore := s.endpoints[0]
+	var (
+		i int
+		e *endpoint
+	)
+	for i, e = range s.endpoints {
+		if e == ep {
+			break
+		}
+	}
+	heap.Remove(&s.endpoints, i)
+	var highEpAfter *endpoint
+	if len(s.endpoints) > 0 {
+		highEpAfter = s.endpoints[0]
+	}
+
+	s.Unlock()
+
+	if highEpBefore != highEpAfter {
+		s.updateGateway(highEpAfter)
+	}
+
+	s.Lock()
+	s.refCnt--
+	refCnt := s.refCnt
+	s.Unlock()
+
+	if refCnt == 0 {
+		s.sandbox().Destroy()
+	}
+
+	return refCnt
+}
+
+func (s *sandboxData) sandbox() sandbox.Sandbox {
+	s.Lock()
+	defer s.Unlock()
+
+	return s.sbox
+}
+
+func (c *controller) sandboxAdd(key string, create bool, ep *endpoint) (sandbox.Sandbox, error) {
+	c.Lock()
+	sData, ok := c.sandboxes[key]
+	c.Unlock()
+
+	if !ok {
+		sb, err := sandbox.NewSandbox(key, create)
+		if err != nil {
+			return nil, err
+		}
+
+		sData = &sandboxData{
+			sbox:      sb,
+			endpoints: epHeap{},
+		}
+
+		heap.Init(&sData.endpoints)
+		c.Lock()
+		c.sandboxes[key] = sData
+		c.Unlock()
+	}
+
+	if err := sData.addEndpoint(ep); err != nil {
+		return nil, err
+	}
+
+	return sData.sandbox(), nil
+}
+
+func (c *controller) sandboxRm(key string, ep *endpoint) {
+	c.Lock()
+	sData := c.sandboxes[key]
+	c.Unlock()
+
+	if sData.rmEndpoint(ep) == 0 {
+		c.Lock()
+		delete(c.sandboxes, key)
+		c.Unlock()
+	}
+}
+
+func (c *controller) sandboxGet(key string) sandbox.Sandbox {
+	c.Lock()
+	sData, ok := c.sandboxes[key]
+	c.Unlock()
+
+	if !ok {
+		return nil
+	}
+
+	return sData.sandbox()
+}

libnetwork/sandboxdata_test.go

@@ -0,0 +1,130 @@
+package libnetwork
+
+import "testing"
+
+func createEmptyCtrlr() *controller {
+	return &controller{sandboxes: sandboxTable{}}
+}
+
+func createEmptyEndpoint() *endpoint {
+	return &endpoint{
+		container: &containerInfo{},
+		joinInfo:  &endpointJoinInfo{},
+		iFaces:    []*endpointInterface{},
+	}
+}
+
+func TestSandboxAddEmpty(t *testing.T) {
+	ctrlr := createEmptyCtrlr()
+	ep := createEmptyEndpoint()
+
+	if _, err := ctrlr.sandboxAdd("sandbox1", true, ep); err != nil {
+		t.Fatal(err)
+	}
+
+	if ctrlr.sandboxes["sandbox1"].refCnt != 1 {
+		t.Fatalf("Unexpected sandbox ref count. Expected 1, got %d",
+			ctrlr.sandboxes["sandbox1"].refCnt)
+	}
+
+	ctrlr.sandboxRm("sandbox1", ep)
+	if len(ctrlr.sandboxes) != 0 {
+		t.Fatalf("controller sandboxes is not empty. len = %d", len(ctrlr.sandboxes))
+	}
+}
+
+func TestSandboxAddMultiPrio(t *testing.T) {
+	ctrlr := createEmptyCtrlr()
+	ep1 := createEmptyEndpoint()
+	ep2 := createEmptyEndpoint()
+	ep3 := createEmptyEndpoint()
+
+	ep1.container.config.prio = 1
+	ep2.container.config.prio = 2
+	ep3.container.config.prio = 3
+
+	if _, err := ctrlr.sandboxAdd("sandbox1", true, ep1); err != nil {
+		t.Fatal(err)
+	}
+
+	if _, err := ctrlr.sandboxAdd("sandbox1", true, ep2); err != nil {
+		t.Fatal(err)
+	}
+
+	if _, err := ctrlr.sandboxAdd("sandbox1", true, ep3); err != nil {
+		t.Fatal(err)
+	}
+
+	if ctrlr.sandboxes["sandbox1"].refCnt != 3 {
+		t.Fatalf("Unexpected sandbox ref count. Expected 3, got %d",
+			ctrlr.sandboxes["sandbox1"].refCnt)
+	}
+
+	if ctrlr.sandboxes["sandbox1"].endpoints[0] != ep3 {
+		t.Fatal("Expected ep3 to be at the top of the heap. But did not find ep3 at the top of the heap")
+	}
+
+	ctrlr.sandboxRm("sandbox1", ep3)
+
+	if ctrlr.sandboxes["sandbox1"].endpoints[0] != ep2 {
+		t.Fatal("Expected ep2 to be at the top of the heap after removing ep3. But did not find ep2 at the top of the heap")
+	}
+
+	ctrlr.sandboxRm("sandbox1", ep2)
+
+	if ctrlr.sandboxes["sandbox1"].endpoints[0] != ep1 {
+		t.Fatal("Expected ep1 to be at the top of the heap after removing ep2. But did not find ep1 at the top of the heap")
+	}
+
+	// Re-add ep3 back
+	if _, err := ctrlr.sandboxAdd("sandbox1", true, ep3); err != nil {
+		t.Fatal(err)
+	}
+
+	if ctrlr.sandboxes["sandbox1"].endpoints[0] != ep3 {
+		t.Fatal("Expected ep3 to be at the top of the heap after adding ep3 back. But did not find ep3 at the top of the heap")
+	}
+
+	ctrlr.sandboxRm("sandbox1", ep3)
+	ctrlr.sandboxRm("sandbox1", ep1)
+	if len(ctrlr.sandboxes) != 0 {
+		t.Fatalf("controller sandboxes is not empty. len = %d", len(ctrlr.sandboxes))
+	}
+}
+
+func TestSandboxAddSamePrio(t *testing.T) {
+	ctrlr := createEmptyCtrlr()
+	ep1 := createEmptyEndpoint()
+	ep2 := createEmptyEndpoint()
+
+	ep1.network = &network{name: "aaa"}
+	ep2.network = &network{name: "bbb"}
+
+	if _, err := ctrlr.sandboxAdd("sandbox1", true, ep1); err != nil {
+		t.Fatal(err)
+	}
+
+	if _, err := ctrlr.sandboxAdd("sandbox1", true, ep2); err != nil {
+		t.Fatal(err)
+	}
+
+	if ctrlr.sandboxes["sandbox1"].refCnt != 2 {
+		t.Fatalf("Unexpected sandbox ref count. Expected 2, got %d",
+			ctrlr.sandboxes["sandbox1"].refCnt)
+	}
+
+	if ctrlr.sandboxes["sandbox1"].endpoints[0] != ep1 {
+		t.Fatal("Expected ep1 to be at the top of the heap. But did not find ep1 at the top of the heap")
+	}
+
+	ctrlr.sandboxRm("sandbox1", ep1)
+
+	if ctrlr.sandboxes["sandbox1"].endpoints[0] != ep2 {
+		t.Fatal("Expected ep2 to be at the top of the heap after removing ep3. But did not find ep2 at the top of the heap")
+	}
+
+	ctrlr.sandboxRm("sandbox1", ep2)
+	if len(ctrlr.sandboxes) != 0 {
+		t.Fatalf("controller sandboxes is not empty. len = %d", len(ctrlr.sandboxes))
+	}
+}