From ea8580d1e2d5b2da108261350212818cd91641e2 Mon Sep 17 00:00:00 2001 From: Jana Radhakrishnan Date: Mon, 25 May 2015 06:24:23 +0000 Subject: [PATCH 1/4] Remove only the endpoint owned interfaces Only remove the interfaces owned by the endpoint from the sandbox when the container leaves the endpoint. Signed-off-by: Jana Radhakrishnan --- libnetwork/endpoint.go | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/libnetwork/endpoint.go b/libnetwork/endpoint.go index 5cccf92a5c..2d9dab9f31 100644 --- a/libnetwork/endpoint.go +++ b/libnetwork/endpoint.go @@ -326,6 +326,19 @@ func (ep *endpoint) Join(containerID string, options ...EndpointOption) error { return nil } +func (ep *endpoint) hasInterface(iName string) bool { + ep.Lock() + defer ep.Unlock() + + for _, iface := range ep.iFaces { + if iface.srcName == iName { + return true + } + } + + return false +} + func (ep *endpoint) Leave(containerID string, options ...EndpointOption) error { var err error @@ -361,9 +374,12 @@ func (ep *endpoint) Leave(containerID string, options ...EndpointOption) error { sb := ctrlr.sandboxGet(container.data.SandboxKey) for _, i := range sb.Interfaces() { - err = sb.RemoveInterface(i) - if err != nil { - logrus.Debugf("Remove interface failed: %v", err) + // Only remove the interfaces owned by this endpoint from the sandbox. + if ep.hasInterface(i.SrcName) { + err = sb.RemoveInterface(i) + if err != nil { + logrus.Debugf("Remove interface failed: %v", err) + } } } From c3d02744c68430fb93ee2bec894e230b51b1c68f Mon Sep 17 00:00:00 2001 From: Jana Radhakrishnan Date: Mon, 25 May 2015 04:27:13 +0000 Subject: [PATCH 2/4] Cleanup libnetwork test code - The libnetwork test code had some issues in not properly passing the network options. Fixed it. - Made controller a global value so that every test uses the same controller instance. - Cleaned up endpoint and network objects after every test. - Extended the endpoint join test case to test the same container join two different networks using two different endpoints. Signed-off-by: Jana Radhakrishnan --- libnetwork/libnetwork_test.go | 392 +++++++++++++++++++++++----------- 1 file changed, 270 insertions(+), 122 deletions(-) diff --git a/libnetwork/libnetwork_test.go b/libnetwork/libnetwork_test.go index 4a0083ced5..d33e206126 100644 --- a/libnetwork/libnetwork_test.go +++ b/libnetwork/libnetwork_test.go @@ -33,35 +33,46 @@ const ( bridgeName = "docker0" ) +var controller libnetwork.NetworkController + func TestMain(m *testing.M) { if reexec.Init() { return } - os.Exit(m.Run()) -} -func createTestController() (libnetwork.NetworkController, error) { - controller, err := libnetwork.New("") - if err != nil { - return nil, err + if err := createController(); err != nil { + os.Exit(1) + } + option := options.Generic{ + "EnableIPForwarding": true, } - libnetwork.SetTestDataStore(controller, datastore.NewCustomDataStore(datastore.NewMockStore())) - return controller, nil -} -func createTestNetwork(networkType, networkName string, option options.Generic, netOption options.Generic) (libnetwork.Network, error) { - controller, err := createTestController() - if err != nil { - return nil, err - } genericOption := make(map[string]interface{}) genericOption[netlabel.GenericData] = option - err = controller.ConfigureNetworkDriver(networkType, genericOption) + err := controller.ConfigureNetworkDriver(bridgeNetType, genericOption) if err != nil { - return nil, err + //m.Fatal(err) + os.Exit(1) } + libnetwork.SetTestDataStore(controller, datastore.NewCustomDataStore(datastore.NewMockStore())) + + os.Exit(m.Run()) +} + +func createController() error { + var err error + + controller, err = libnetwork.New("") + if err != nil { + return err + } + + return nil +} + +func createTestNetwork(networkType, networkName string, netOption options.Generic) (libnetwork.Network, error) { network, err := controller.NewNetwork(networkType, networkName, libnetwork.NetworkOptionGeneric(netOption)) if err != nil { @@ -86,8 +97,7 @@ func getPortMapping() []types.PortBinding { } func TestNull(t *testing.T) { - network, err := createTestNetwork("null", "testnetwork", options.Generic{}, - options.Generic{}) + network, err := createTestNetwork("null", "testnetwork", options.Generic{}) if err != nil { t.Fatal(err) } @@ -120,7 +130,7 @@ func TestNull(t *testing.T) { } func TestHost(t *testing.T) { - network, err := createTestNetwork("host", "testnetwork", options.Generic{}, options.Generic{}) + network, err := createTestNetwork("host", "testnetwork", options.Generic{}) if err != nil { t.Fatal(err) } @@ -224,9 +234,6 @@ func TestBridge(t *testing.T) { cidrv6.IP = ip log.Debug("Adding a bridge") - option := options.Generic{ - "EnableIPForwarding": true, - } netOption := options.Generic{ netlabel.GenericData: options.Generic{ @@ -242,7 +249,7 @@ func TestBridge(t *testing.T) { }, } - network, err := createTestNetwork(bridgeNetType, "testnetwork", option, netOption) + network, err := createTestNetwork(bridgeNetType, "testnetwork", netOption) if err != nil { t.Fatal(err) } @@ -282,23 +289,18 @@ func TestUnknownDriver(t *testing.T) { defer netutils.SetupTestNetNS(t)() } - _, err := createTestNetwork("unknowndriver", "testnetwork", options.Generic{}, options.Generic{}) + _, err := createTestNetwork("unknowndriver", "testnetwork", options.Generic{}) if err == nil { t.Fatal("Expected to fail. But instead succeeded") } - if _, ok := err.(libnetwork.NetworkTypeError); !ok { + if _, ok := err.(types.NotFoundError); !ok { t.Fatalf("Did not fail with expected error. Actual error: %v", err) } } func TestNilRemoteDriver(t *testing.T) { - controller, err := createTestController() - if err != nil { - t.Fatal(err) - } - - _, err = controller.NewNetwork("framerelay", "dummy", + _, err := controller.NewNetwork("framerelay", "dummy", libnetwork.NetworkOptionGeneric(getEmptyGenericOption())) if err == nil { t.Fatal("Expected to fail. But instead succeeded") @@ -314,23 +316,15 @@ func TestDuplicateNetwork(t *testing.T) { defer netutils.SetupTestNetNS(t)() } - controller, err := createTestController() - if err != nil { - t.Fatal(err) - } - - genericOption := make(map[string]interface{}) - genericOption[netlabel.GenericData] = options.Generic{} - - err = controller.ConfigureNetworkDriver(bridgeNetType, genericOption) - if err != nil { - t.Fatal(err) - } - - _, err = controller.NewNetwork(bridgeNetType, "testnetwork", nil) + n, err := controller.NewNetwork(bridgeNetType, "testnetwork", nil) if err != nil { t.Fatal(err) } + defer func() { + if err := n.Delete(); err != nil { + t.Fatal(err) + } + }() _, err = controller.NewNetwork(bridgeNetType, "testnetwork") if err == nil { @@ -347,7 +341,7 @@ func TestNetworkName(t *testing.T) { defer netutils.SetupTestNetNS(t)() } - _, err := createTestNetwork(bridgeNetType, "", options.Generic{}, options.Generic{}) + _, err := createTestNetwork(bridgeNetType, "", options.Generic{}) if err == nil { t.Fatal("Expected to fail. But instead succeeded") } @@ -357,10 +351,15 @@ func TestNetworkName(t *testing.T) { } networkName := "testnetwork" - n, err := createTestNetwork(bridgeNetType, networkName, options.Generic{}, options.Generic{}) + n, err := createTestNetwork(bridgeNetType, networkName, options.Generic{}) if err != nil { t.Fatal(err) } + defer func() { + if err := n.Delete(); err != nil { + t.Fatal(err) + } + }() if n.Name() != networkName { t.Fatalf("Expected network name %s, got %s", networkName, n.Name()) @@ -372,10 +371,15 @@ func TestNetworkType(t *testing.T) { defer netutils.SetupTestNetNS(t)() } - n, err := createTestNetwork(bridgeNetType, "testnetwork", options.Generic{}, options.Generic{}) + n, err := createTestNetwork(bridgeNetType, "testnetwork", options.Generic{}) if err != nil { t.Fatal(err) } + defer func() { + if err := n.Delete(); err != nil { + t.Fatal(err) + } + }() if n.Type() != bridgeNetType { t.Fatalf("Expected network type %s, got %s", bridgeNetType, n.Type()) @@ -387,10 +391,15 @@ func TestNetworkID(t *testing.T) { defer netutils.SetupTestNetNS(t)() } - n, err := createTestNetwork(bridgeNetType, "testnetwork", options.Generic{}, options.Generic{}) + n, err := createTestNetwork(bridgeNetType, "testnetwork", options.Generic{}) if err != nil { t.Fatal(err) } + defer func() { + if err := n.Delete(); err != nil { + t.Fatal(err) + } + }() if n.ID() == "" { t.Fatal("Expected non-empty network id") @@ -402,11 +411,14 @@ func TestDeleteNetworkWithActiveEndpoints(t *testing.T) { defer netutils.SetupTestNetNS(t)() } - option := options.Generic{ + netOption := options.Generic{ "BridgeName": bridgeName, "AllowNonDefaultBridge": true} + option := options.Generic{ + netlabel.GenericData: netOption, + } - network, err := createTestNetwork(bridgeNetType, "testnetwork", options.Generic{}, option) + network, err := createTestNetwork(bridgeNetType, "testnetwork", option) if err != nil { t.Fatal(err) } @@ -440,11 +452,14 @@ func TestUnknownNetwork(t *testing.T) { defer netutils.SetupTestNetNS(t)() } - option := options.Generic{ + netOption := options.Generic{ "BridgeName": bridgeName, "AllowNonDefaultBridge": true} + option := options.Generic{ + netlabel.GenericData: netOption, + } - network, err := createTestNetwork(bridgeNetType, "testnetwork", options.Generic{}, option) + network, err := createTestNetwork(bridgeNetType, "testnetwork", option) if err != nil { t.Fatal(err) } @@ -475,12 +490,15 @@ func TestUnknownEndpoint(t *testing.T) { } subnet.IP = ip - option := options.Generic{ + netOption := options.Generic{ "BridgeName": bridgeName, "AddressIPv4": subnet, "AllowNonDefaultBridge": true} + option := options.Generic{ + netlabel.GenericData: netOption, + } - network, err := createTestNetwork(bridgeNetType, "testnetwork", options.Generic{}, option) + network, err := createTestNetwork(bridgeNetType, "testnetwork", option) if err != nil { t.Fatal(err) } @@ -523,29 +541,36 @@ func TestNetworkEndpointsWalkers(t *testing.T) { defer netutils.SetupTestNetNS(t)() } - controller, err := createTestController() - if err != nil { - t.Fatal(err) - } - - err = controller.ConfigureNetworkDriver(bridgeNetType, getEmptyGenericOption()) - if err != nil { - t.Fatal(err) - } - // Create network 1 and add 2 endpoint: ep11, ep12 net1, err := controller.NewNetwork(bridgeNetType, "network1") if err != nil { t.Fatal(err) } + defer func() { + if err := net1.Delete(); err != nil { + t.Fatal(err) + } + }() + ep11, err := net1.CreateEndpoint("ep11") if err != nil { t.Fatal(err) } + defer func() { + if err := ep11.Delete(); err != nil { + t.Fatal(err) + } + }() + ep12, err := net1.CreateEndpoint("ep12") if err != nil { t.Fatal(err) } + defer func() { + if err := ep12.Delete(); err != nil { + t.Fatal(err) + } + }() // Test list methods on net1 epList1 := net1.Endpoints() @@ -607,21 +632,16 @@ func TestControllerQuery(t *testing.T) { defer netutils.SetupTestNetNS(t)() } - controller, err := createTestController() - if err != nil { - t.Fatal(err) - } - - err = controller.ConfigureNetworkDriver(bridgeNetType, getEmptyGenericOption()) - if err != nil { - t.Fatal(err) - } - // Create network 1 net1, err := controller.NewNetwork(bridgeNetType, "network1") if err != nil { t.Fatal(err) } + defer func() { + if err := net1.Delete(); err != nil { + t.Fatal(err) + } + }() _, err = controller.NetworkByName("") if err == nil { @@ -673,29 +693,36 @@ func TestNetworkQuery(t *testing.T) { defer netutils.SetupTestNetNS(t)() } - controller, err := createTestController() - if err != nil { - t.Fatal(err) - } - - err = controller.ConfigureNetworkDriver(bridgeNetType, getEmptyGenericOption()) - if err != nil { - t.Fatal(err) - } - // Create network 1 and add 2 endpoint: ep11, ep12 net1, err := controller.NewNetwork(bridgeNetType, "network1") if err != nil { t.Fatal(err) } + defer func() { + if err := net1.Delete(); err != nil { + t.Fatal(err) + } + }() + ep11, err := net1.CreateEndpoint("ep11") if err != nil { t.Fatal(err) } + defer func() { + if err := ep11.Delete(); err != nil { + t.Fatal(err) + } + }() + ep12, err := net1.CreateEndpoint("ep12") if err != nil { t.Fatal(err) } + defer func() { + if err := ep12.Delete(); err != nil { + t.Fatal(err) + } + }() e, err := net1.EndpointByName("ep11") if err != nil { @@ -770,6 +797,11 @@ func checkSandbox(t *testing.T, info libnetwork.EndpointInfo) { if err != nil { t.Fatalf("Could not find the interface eth0 inside the sandbox: %v", err) } + + _, err = netlink.LinkByName("eth1") + if err != nil { + t.Fatalf("Could not find the interface eth1 inside the sandbox: %v", err) + } } func TestEndpointJoin(t *testing.T) { @@ -777,18 +809,28 @@ func TestEndpointJoin(t *testing.T) { defer netutils.SetupTestNetNS(t)() } - n, err := createTestNetwork(bridgeNetType, "testnetwork", options.Generic{}, options.Generic{}) + n1, err := createTestNetwork(bridgeNetType, "testnetwork1", options.Generic{}) if err != nil { t.Fatal(err) } + defer func() { + if err := n1.Delete(); err != nil { + t.Fatal(err) + } + }() - ep, err := n.CreateEndpoint("ep1") + ep1, err := n1.CreateEndpoint("ep1") if err != nil { t.Fatal(err) } + defer func() { + if err := ep1.Delete(); err != nil { + t.Fatal(err) + } + }() // Validate if ep.Info() only gives me IP address info and not names and gateway during CreateEndpoint() - info := ep.Info() + info := ep1.Info() for _, iface := range info.InterfaceList() { if iface.Address().IP.To4() == nil { @@ -804,7 +846,7 @@ func TestEndpointJoin(t *testing.T) { t.Fatalf("Expected an empty sandbox key for an empty endpoint. Instead found a non-empty sandbox key: %s", info.SandboxKey()) } - err = ep.Join(containerID, + err = ep1.Join(containerID, libnetwork.JoinOptionHostname("test"), libnetwork.JoinOptionDomainname("docker.io"), libnetwork.JoinOptionExtraHost("web", "192.168.0.1")) @@ -813,14 +855,14 @@ func TestEndpointJoin(t *testing.T) { } defer func() { - err = ep.Leave(containerID) + err = ep1.Leave(containerID) if err != nil { t.Fatal(err) } }() // Validate if ep.Info() only gives valid gateway and sandbox key after has container has joined. - info = ep.Info() + info = ep1.Info() if info.Gateway().To4() == nil { t.Fatalf("Expected a valid gateway for a joined endpoint. Instead found an invalid gateway: %v", info.Gateway()) } @@ -829,6 +871,45 @@ func TestEndpointJoin(t *testing.T) { t.Fatalf("Expected an non-empty sandbox key for a joined endpoint. Instead found a empty sandbox key") } + // Now test the container joining another network + n2, err := createTestNetwork(bridgeNetType, "testnetwork2", + options.Generic{ + netlabel.GenericData: options.Generic{ + "BridgeName": "secondary", + "AllowNonDefaultBridge": true, + }, + }) + if err != nil { + t.Fatal(err) + } + defer func() { + if err := n2.Delete(); err != nil { + t.Fatal(err) + } + }() + + ep2, err := n2.CreateEndpoint("ep2") + if err != nil { + t.Fatal(err) + } + defer func() { + if err := ep2.Delete(); err != nil { + t.Fatal(err) + } + }() + + err = ep2.Join(containerID) + if err != nil { + t.Fatal(err) + } + + defer func() { + err = ep2.Leave(containerID) + if err != nil { + t.Fatal(err) + } + }() + checkSandbox(t, info) } @@ -837,15 +918,25 @@ func TestEndpointJoinInvalidContainerId(t *testing.T) { defer netutils.SetupTestNetNS(t)() } - n, err := createTestNetwork(bridgeNetType, "testnetwork", options.Generic{}, options.Generic{}) + n, err := createTestNetwork(bridgeNetType, "testnetwork", options.Generic{}) if err != nil { t.Fatal(err) } + defer func() { + if err := n.Delete(); err != nil { + t.Fatal(err) + } + }() ep, err := n.CreateEndpoint("ep1") if err != nil { t.Fatal(err) } + defer func() { + if err := ep.Delete(); err != nil { + t.Fatal(err) + } + }() err = ep.Join("") if err == nil { @@ -862,10 +953,15 @@ func TestEndpointDeleteWithActiveContainer(t *testing.T) { defer netutils.SetupTestNetNS(t)() } - n, err := createTestNetwork(bridgeNetType, "testnetwork", options.Generic{}, options.Generic{}) + n, err := createTestNetwork(bridgeNetType, "testnetwork", options.Generic{}) if err != nil { t.Fatal(err) } + defer func() { + if err := n.Delete(); err != nil { + t.Fatal(err) + } + }() ep, err := n.CreateEndpoint("ep1") if err != nil { @@ -906,15 +1002,25 @@ func TestEndpointMultipleJoins(t *testing.T) { defer netutils.SetupTestNetNS(t)() } - n, err := createTestNetwork(bridgeNetType, "testnetwork", options.Generic{}, options.Generic{}) + n, err := createTestNetwork(bridgeNetType, "testnetwork", options.Generic{}) if err != nil { t.Fatal(err) } + defer func() { + if err := n.Delete(); err != nil { + t.Fatal(err) + } + }() ep, err := n.CreateEndpoint("ep1") if err != nil { t.Fatal(err) } + defer func() { + if err := ep.Delete(); err != nil { + t.Fatal(err) + } + }() err = ep.Join(containerID, libnetwork.JoinOptionHostname("test"), @@ -946,15 +1052,25 @@ func TestEndpointInvalidLeave(t *testing.T) { defer netutils.SetupTestNetNS(t)() } - n, err := createTestNetwork(bridgeNetType, "testnetwork", options.Generic{}, options.Generic{}) + n, err := createTestNetwork(bridgeNetType, "testnetwork", options.Generic{}) if err != nil { t.Fatal(err) } + defer func() { + if err := n.Delete(); err != nil { + t.Fatal(err) + } + }() ep, err := n.CreateEndpoint("ep1") if err != nil { t.Fatal(err) } + defer func() { + if err := ep.Delete(); err != nil { + t.Fatal(err) + } + }() err = ep.Leave(containerID) if err == nil { @@ -1007,15 +1123,25 @@ func TestEndpointUpdateParent(t *testing.T) { defer netutils.SetupTestNetNS(t)() } - n, err := createTestNetwork("bridge", "testnetwork", options.Generic{}, options.Generic{}) + n, err := createTestNetwork("bridge", "testnetwork", options.Generic{}) if err != nil { t.Fatal(err) } + defer func() { + if err := n.Delete(); err != nil { + t.Fatal(err) + } + }() - ep1, err := n.CreateEndpoint("ep1", nil) + ep1, err := n.CreateEndpoint("ep1") if err != nil { t.Fatal(err) } + defer func() { + if err := ep1.Delete(); err != nil { + t.Fatal(err) + } + }() err = ep1.Join(containerID, libnetwork.JoinOptionHostname("test1"), @@ -1032,10 +1158,15 @@ func TestEndpointUpdateParent(t *testing.T) { } }() - ep2, err := n.CreateEndpoint("ep2", nil) + ep2, err := n.CreateEndpoint("ep2") if err != nil { t.Fatal(err) } + defer func() { + if err := ep2.Delete(); err != nil { + t.Fatal(err) + } + }() err = ep2.Join("container2", libnetwork.JoinOptionHostname("test2"), @@ -1087,15 +1218,25 @@ func TestEnableIPv6(t *testing.T) { }, } - n, err := createTestNetwork("bridge", "testnetwork", options.Generic{}, netOption) + n, err := createTestNetwork("bridge", "testnetwork", netOption) if err != nil { t.Fatal(err) } + defer func() { + if err := n.Delete(); err != nil { + t.Fatal(err) + } + }() - ep1, err := n.CreateEndpoint("ep1", nil) + ep1, err := n.CreateEndpoint("ep1") if err != nil { t.Fatal(err) } + defer func() { + if err := ep1.Delete(); err != nil { + t.Fatal(err) + } + }() if err := ioutil.WriteFile("/etc/resolv.conf", tmpResolvConf, 0644); err != nil { t.Fatal(err) @@ -1154,15 +1295,25 @@ func TestResolvConf(t *testing.T) { } }() - n, err := createTestNetwork("bridge", "testnetwork", options.Generic{}, options.Generic{}) + n, err := createTestNetwork("bridge", "testnetwork", options.Generic{}) if err != nil { t.Fatal(err) } + defer func() { + if err := n.Delete(); err != nil { + t.Fatal(err) + } + }() - ep1, err := n.CreateEndpoint("ep1", nil) + ep1, err := n.CreateEndpoint("ep1") if err != nil { t.Fatal(err) } + defer func() { + if err := ep1.Delete(); err != nil { + t.Fatal(err) + } + }() if err := ioutil.WriteFile("/etc/resolv.conf", tmpResolvConf1, 0644); err != nil { t.Fatal(err) @@ -1339,21 +1490,24 @@ func TestValidRemoteDriver(t *testing.T) { t.Fatal(err) } - controller, err := libnetwork.New("") - if err != nil { - t.Fatal(err) - } - - _, err = controller.NewNetwork("valid-network-driver", "dummy", + n, err := controller.NewNetwork("valid-network-driver", "dummy", libnetwork.NetworkOptionGeneric(getEmptyGenericOption())) if err != nil { - t.Fatal(err) + // Only fail if we could not find the plugin driver + if _, ok := err.(types.NotFoundError); ok { + t.Fatal(err) + } + return } + defer func() { + if err := n.Delete(); err != nil { + t.Fatal(err) + } + }() } var ( once sync.Once - ctrlr libnetwork.NetworkController start = make(chan struct{}) done = make(chan chan struct{}, numThreads-1) origns = netns.None() @@ -1386,17 +1540,7 @@ func createGlobalInstance(t *testing.T) { } } - ctrlr, err = createTestController() - if err != nil { - t.Fatal(err) - } - - err = ctrlr.ConfigureNetworkDriver(bridgeNetType, getEmptyGenericOption()) - if err != nil { - t.Fatal("configure driver") - } - - net, err := ctrlr.NewNetwork(bridgeNetType, "network1") + net, err := controller.NewNetwork(bridgeNetType, "network1") if err != nil { t.Fatal("new network") } @@ -1489,7 +1633,7 @@ func runParallelTests(t *testing.T, thrNumber int) { } defer netns.Set(origns) - net, err := ctrlr.NetworkByName("network1") + net, err := controller.NetworkByName("network1") if err != nil { t.Fatal(err) } @@ -1524,6 +1668,10 @@ func runParallelTests(t *testing.T, thrNumber int) { if err != nil { t.Fatal(err) } + + if err := net.Delete(); err != nil { + t.Fatal(err) + } } } From 36303270c6accbed3aa73e43417262af06ae7749 Mon Sep 17 00:00:00 2001 From: Jana Radhakrishnan Date: Tue, 2 Jun 2015 13:20:15 -0700 Subject: [PATCH 3/4] Introduce UnsetGateway(IPv6) methods Sandbox needs unset gateway methods to cleanup gateway settings to enable smooth transition of the sandbox between endpoints. Signed-off-by: Jana Radhakrishnan --- libnetwork/sandbox/configure_linux.go | 12 +++++-- libnetwork/sandbox/namespace_linux.go | 50 +++++++++++++++++++++++++-- libnetwork/sandbox/sandbox.go | 6 ++++ 3 files changed, 64 insertions(+), 4 deletions(-) diff --git a/libnetwork/sandbox/configure_linux.go b/libnetwork/sandbox/configure_linux.go index 6f106e8647..4022170820 100644 --- a/libnetwork/sandbox/configure_linux.go +++ b/libnetwork/sandbox/configure_linux.go @@ -30,7 +30,7 @@ func configureInterface(iface netlink.Link, settings *Interface) error { return nil } -func programGateway(path string, gw net.IP) error { +func programGateway(path string, gw net.IP, isAdd bool) error { runtime.LockOSThread() defer runtime.UnlockOSThread() @@ -57,7 +57,15 @@ func programGateway(path string, gw net.IP) error { return fmt.Errorf("route for the gateway could not be found: %v", err) } - return netlink.RouteAdd(&netlink.Route{ + if isAdd { + return netlink.RouteAdd(&netlink.Route{ + Scope: netlink.SCOPE_UNIVERSE, + LinkIndex: gwRoutes[0].LinkIndex, + Gw: gw, + }) + } + + return netlink.RouteDel(&netlink.Route{ Scope: netlink.SCOPE_UNIVERSE, LinkIndex: gwRoutes[0].LinkIndex, Gw: gw, diff --git a/libnetwork/sandbox/namespace_linux.go b/libnetwork/sandbox/namespace_linux.go index 3db3a8ee0f..c368d54f41 100644 --- a/libnetwork/sandbox/namespace_linux.go +++ b/libnetwork/sandbox/namespace_linux.go @@ -308,26 +308,72 @@ func (n *networkNamespace) AddInterface(i *Interface) error { } func (n *networkNamespace) SetGateway(gw net.IP) error { + // Silently return if the gateway is empty if len(gw) == 0 { return nil } - err := programGateway(n.path, gw) + err := programGateway(n.path, gw, true) if err == nil { + n.Lock() n.sinfo.Gateway = gw + n.Unlock() + } + + return err +} + +func (n *networkNamespace) UnsetGateway() error { + n.Lock() + gw := n.sinfo.Gateway + n.Unlock() + + // Silently return if the gateway is empty + if len(gw) == 0 { + return nil + } + + err := programGateway(n.path, gw, false) + if err == nil { + n.Lock() + n.sinfo.Gateway = net.IP{} + n.Unlock() } return err } func (n *networkNamespace) SetGatewayIPv6(gw net.IP) error { + // Silently return if the gateway is empty if len(gw) == 0 { return nil } - err := programGateway(n.path, gw) + err := programGateway(n.path, gw, true) if err == nil { + n.Lock() n.sinfo.GatewayIPv6 = gw + n.Unlock() + } + + return err +} + +func (n *networkNamespace) UnsetGatewayIPv6() error { + n.Lock() + gw := n.sinfo.GatewayIPv6 + n.Unlock() + + // Silently return if the gateway is empty + if len(gw) == 0 { + return nil + } + + err := programGateway(n.path, gw, false) + if err == nil { + n.Lock() + n.sinfo.GatewayIPv6 = net.IP{} + n.Unlock() } return err diff --git a/libnetwork/sandbox/sandbox.go b/libnetwork/sandbox/sandbox.go index 0f036034ab..e52fba031c 100644 --- a/libnetwork/sandbox/sandbox.go +++ b/libnetwork/sandbox/sandbox.go @@ -35,6 +35,12 @@ type Sandbox interface { // Set default IPv6 gateway for the sandbox SetGatewayIPv6(gw net.IP) error + // Unset the previously set default IPv4 gateway in the sandbox + UnsetGateway() error + + // Unset the previously set default IPv6 gateway in the sandbox + UnsetGatewayIPv6() error + // Add a static route to the sandbox. AddStaticRoute(*types.StaticRoute) error From a93d08aef58ba8f4e57c9e03304bee256b34c699 Mon Sep 17 00:00:00 2001 From: Jana Radhakrishnan Date: Tue, 2 Jun 2015 13:22:28 -0700 Subject: [PATCH 4/4] Add container join priority option to endpoint When an endpoint is joined by a container it may optionally pass a priority to resolve resource conflicts inside the sandbox when more than one endpoint provides the same kind of resource. If the the priority is the same for two endpoints with conflicting resources then the endpoint network names are used to resolve the conflict. Signed-off-by: Jana Radhakrishnan --- libnetwork/controller.go | 51 ------- libnetwork/endpoint.go | 71 ++-------- libnetwork/sandboxdata.go | 246 +++++++++++++++++++++++++++++++++ libnetwork/sandboxdata_test.go | 130 +++++++++++++++++ 4 files changed, 388 insertions(+), 110 deletions(-) create mode 100644 libnetwork/sandboxdata.go create mode 100644 libnetwork/sandboxdata_test.go diff --git a/libnetwork/controller.go b/libnetwork/controller.go index 8046731cf0..d709d972b0 100644 --- a/libnetwork/controller.go +++ b/libnetwork/controller.go @@ -59,7 +59,6 @@ import ( "github.com/docker/libnetwork/datastore" "github.com/docker/libnetwork/driverapi" "github.com/docker/libnetwork/hostdiscovery" - "github.com/docker/libnetwork/sandbox" "github.com/docker/libnetwork/types" "github.com/docker/swarm/pkg/store" ) @@ -91,11 +90,6 @@ type NetworkController interface { // When the function returns true, the walk will stop. type NetworkWalker func(nw Network) bool -type sandboxData struct { - sandbox sandbox.Sandbox - refCnt int -} - type networkTable map[types.UUID]*network type endpointTable map[types.UUID]*endpoint type sandboxTable map[string]*sandboxData @@ -382,51 +376,6 @@ func (c *controller) NetworkByID(id string) (Network, error) { return nil, ErrNoSuchNetwork(id) } -func (c *controller) sandboxAdd(key string, create bool) (sandbox.Sandbox, error) { - c.Lock() - defer c.Unlock() - - sData, ok := c.sandboxes[key] - if !ok { - sb, err := sandbox.NewSandbox(key, create) - if err != nil { - return nil, err - } - - sData = &sandboxData{sandbox: sb, refCnt: 1} - c.sandboxes[key] = sData - return sData.sandbox, nil - } - - sData.refCnt++ - return sData.sandbox, nil -} - -func (c *controller) sandboxRm(key string) { - c.Lock() - defer c.Unlock() - - sData := c.sandboxes[key] - sData.refCnt-- - - if sData.refCnt == 0 { - sData.sandbox.Destroy() - delete(c.sandboxes, key) - } -} - -func (c *controller) sandboxGet(key string) sandbox.Sandbox { - c.Lock() - defer c.Unlock() - - sData, ok := c.sandboxes[key] - if !ok { - return nil - } - - return sData.sandbox -} - func (c *controller) loadDriver(networkType string) (driverapi.Driver, error) { // Plugins pkg performs lazy loading of plugins that acts as remote drivers. // As per the design, this Get call will result in remote driver discovery if there is a corresponding plugin available. diff --git a/libnetwork/endpoint.go b/libnetwork/endpoint.go index 2d9dab9f31..c357b3298f 100644 --- a/libnetwork/endpoint.go +++ b/libnetwork/endpoint.go @@ -8,7 +8,6 @@ import ( "path/filepath" "sync" - "github.com/Sirupsen/logrus" "github.com/docker/docker/pkg/ioutils" "github.com/docker/libnetwork/etchosts" "github.com/docker/libnetwork/netlabel" @@ -78,6 +77,7 @@ type containerConfig struct { resolvConfPathConfig generic map[string]interface{} useDefaultSandBox bool + prio int // higher the value, more the priority } type extraHost struct { @@ -101,7 +101,6 @@ type endpoint struct { name string id types.UUID network *network - sandboxInfo *sandbox.Info iFaces []*endpointInterface joinInfo *endpointJoinInfo container *containerInfo @@ -233,8 +232,6 @@ func (ep *endpoint) Join(containerID string, options ...EndpointOption) error { container := ep.container network := ep.network epid := ep.id - joinInfo := ep.joinInfo - ifaces := ep.iFaces ep.Unlock() defer func() { @@ -278,49 +275,16 @@ func (ep *endpoint) Join(containerID string, options ...EndpointOption) error { return err } - sb, err := ctrlr.sandboxAdd(sboxKey, !container.config.useDefaultSandBox) + sb, err := ctrlr.sandboxAdd(sboxKey, !container.config.useDefaultSandBox, ep) if err != nil { return err } defer func() { if err != nil { - ctrlr.sandboxRm(sboxKey) + ctrlr.sandboxRm(sboxKey, ep) } }() - for _, i := range ifaces { - iface := &sandbox.Interface{ - SrcName: i.srcName, - DstName: i.dstPrefix, - Address: &i.addr, - Routes: i.routes, - } - if i.addrv6.IP.To16() != nil { - iface.AddressIPv6 = &i.addrv6 - } - err = sb.AddInterface(iface) - if err != nil { - return err - } - } - // Set up non-interface routes. - for _, r := range ep.joinInfo.StaticRoutes { - err = sb.AddStaticRoute(r) - if err != nil { - return err - } - } - - err = sb.SetGateway(joinInfo.gw) - if err != nil { - return err - } - - err = sb.SetGatewayIPv6(joinInfo.gw6) - if err != nil { - return err - } - container.data.SandboxKey = sb.Key() return nil @@ -372,26 +336,7 @@ func (ep *endpoint) Leave(containerID string, options ...EndpointOption) error { err = driver.Leave(n.id, ep.id) - sb := ctrlr.sandboxGet(container.data.SandboxKey) - for _, i := range sb.Interfaces() { - // Only remove the interfaces owned by this endpoint from the sandbox. - if ep.hasInterface(i.SrcName) { - err = sb.RemoveInterface(i) - if err != nil { - logrus.Debugf("Remove interface failed: %v", err) - } - } - } - - // Remove non-interface routes. - for _, r := range ep.joinInfo.StaticRoutes { - err = sb.RemoveStaticRoute(r) - if err != nil { - logrus.Debugf("Remove route failed: %v", err) - } - } - - ctrlr.sandboxRm(container.data.SandboxKey) + ctrlr.sandboxRm(container.data.SandboxKey, ep) return err } @@ -648,6 +593,14 @@ func EndpointOptionGeneric(generic map[string]interface{}) EndpointOption { } } +// JoinOptionPriority function returns an option setter for priority option to +// be passed to endpoint Join method. +func JoinOptionPriority(prio int) EndpointOption { + return func(ep *endpoint) { + ep.container.config.prio = prio + } +} + // JoinOptionHostname function returns an option setter for hostname option to // be passed to endpoint Join method. func JoinOptionHostname(name string) EndpointOption { diff --git a/libnetwork/sandboxdata.go b/libnetwork/sandboxdata.go new file mode 100644 index 0000000000..947fea3863 --- /dev/null +++ b/libnetwork/sandboxdata.go @@ -0,0 +1,246 @@ +package libnetwork + +import ( + "container/heap" + "sync" + + "github.com/Sirupsen/logrus" + "github.com/docker/libnetwork/sandbox" +) + +type epHeap []*endpoint + +type sandboxData struct { + sbox sandbox.Sandbox + refCnt int + endpoints epHeap + sync.Mutex +} + +func (eh epHeap) Len() int { return len(eh) } + +func (eh epHeap) Less(i, j int) bool { + eh[i].Lock() + eh[j].Lock() + defer eh[j].Unlock() + defer eh[i].Unlock() + + if eh[i].container.config.prio == eh[j].container.config.prio { + return eh[i].network.Name() < eh[j].network.Name() + } + + return eh[i].container.config.prio > eh[j].container.config.prio +} + +func (eh epHeap) Swap(i, j int) { eh[i], eh[j] = eh[j], eh[i] } + +func (eh *epHeap) Push(x interface{}) { + *eh = append(*eh, x.(*endpoint)) +} + +func (eh *epHeap) Pop() interface{} { + old := *eh + n := len(old) + x := old[n-1] + *eh = old[0 : n-1] + return x +} + +func (s *sandboxData) updateGateway(ep *endpoint) error { + sb := s.sandbox() + if err := sb.UnsetGateway(); err != nil { + return err + } + + if err := sb.UnsetGatewayIPv6(); err != nil { + return err + } + + if ep == nil { + return nil + } + + ep.Lock() + joinInfo := ep.joinInfo + ep.Unlock() + + if err := sb.SetGateway(joinInfo.gw); err != nil { + return err + } + + if err := sb.SetGatewayIPv6(joinInfo.gw6); err != nil { + return err + } + + return nil +} + +func (s *sandboxData) addEndpoint(ep *endpoint) error { + ep.Lock() + joinInfo := ep.joinInfo + ifaces := ep.iFaces + ep.Unlock() + + sb := s.sandbox() + for _, i := range ifaces { + iface := &sandbox.Interface{ + SrcName: i.srcName, + DstName: i.dstPrefix, + Address: &i.addr, + Routes: i.routes, + } + if i.addrv6.IP.To16() != nil { + iface.AddressIPv6 = &i.addrv6 + } + + if err := sb.AddInterface(iface); err != nil { + return err + } + } + + if joinInfo != nil { + // Set up non-interface routes. + for _, r := range ep.joinInfo.StaticRoutes { + if err := sb.AddStaticRoute(r); err != nil { + return err + } + } + } + + s.Lock() + heap.Push(&s.endpoints, ep) + highEp := s.endpoints[0] + s.Unlock() + + if ep == highEp { + if err := s.updateGateway(ep); err != nil { + return err + } + } + + s.Lock() + s.refCnt++ + s.Unlock() + + return nil +} + +func (s *sandboxData) rmEndpoint(ep *endpoint) int { + ep.Lock() + joinInfo := ep.joinInfo + ep.Unlock() + + sb := s.sandbox() + for _, i := range sb.Interfaces() { + // Only remove the interfaces owned by this endpoint from the sandbox. + if ep.hasInterface(i.SrcName) { + if err := sb.RemoveInterface(i); err != nil { + logrus.Debugf("Remove interface failed: %v", err) + } + } + } + + // Remove non-interface routes. + for _, r := range joinInfo.StaticRoutes { + if err := sb.RemoveStaticRoute(r); err != nil { + logrus.Debugf("Remove route failed: %v", err) + } + } + + // We don't check if s.endpoints is empty here because + // it should never be empty during a rmEndpoint call and + // if it is we will rightfully panic here + s.Lock() + highEpBefore := s.endpoints[0] + var ( + i int + e *endpoint + ) + for i, e = range s.endpoints { + if e == ep { + break + } + } + heap.Remove(&s.endpoints, i) + var highEpAfter *endpoint + if len(s.endpoints) > 0 { + highEpAfter = s.endpoints[0] + } + + s.Unlock() + + if highEpBefore != highEpAfter { + s.updateGateway(highEpAfter) + } + + s.Lock() + s.refCnt-- + refCnt := s.refCnt + s.Unlock() + + if refCnt == 0 { + s.sandbox().Destroy() + } + + return refCnt +} + +func (s *sandboxData) sandbox() sandbox.Sandbox { + s.Lock() + defer s.Unlock() + + return s.sbox +} + +func (c *controller) sandboxAdd(key string, create bool, ep *endpoint) (sandbox.Sandbox, error) { + c.Lock() + sData, ok := c.sandboxes[key] + c.Unlock() + + if !ok { + sb, err := sandbox.NewSandbox(key, create) + if err != nil { + return nil, err + } + + sData = &sandboxData{ + sbox: sb, + endpoints: epHeap{}, + } + + heap.Init(&sData.endpoints) + c.Lock() + c.sandboxes[key] = sData + c.Unlock() + } + + if err := sData.addEndpoint(ep); err != nil { + return nil, err + } + + return sData.sandbox(), nil +} + +func (c *controller) sandboxRm(key string, ep *endpoint) { + c.Lock() + sData := c.sandboxes[key] + c.Unlock() + + if sData.rmEndpoint(ep) == 0 { + c.Lock() + delete(c.sandboxes, key) + c.Unlock() + } +} + +func (c *controller) sandboxGet(key string) sandbox.Sandbox { + c.Lock() + sData, ok := c.sandboxes[key] + c.Unlock() + + if !ok { + return nil + } + + return sData.sandbox() +} diff --git a/libnetwork/sandboxdata_test.go b/libnetwork/sandboxdata_test.go new file mode 100644 index 0000000000..3b76326c3a --- /dev/null +++ b/libnetwork/sandboxdata_test.go @@ -0,0 +1,130 @@ +package libnetwork + +import "testing" + +func createEmptyCtrlr() *controller { + return &controller{sandboxes: sandboxTable{}} +} + +func createEmptyEndpoint() *endpoint { + return &endpoint{ + container: &containerInfo{}, + joinInfo: &endpointJoinInfo{}, + iFaces: []*endpointInterface{}, + } +} + +func TestSandboxAddEmpty(t *testing.T) { + ctrlr := createEmptyCtrlr() + ep := createEmptyEndpoint() + + if _, err := ctrlr.sandboxAdd("sandbox1", true, ep); err != nil { + t.Fatal(err) + } + + if ctrlr.sandboxes["sandbox1"].refCnt != 1 { + t.Fatalf("Unexpected sandbox ref count. Expected 1, got %d", + ctrlr.sandboxes["sandbox1"].refCnt) + } + + ctrlr.sandboxRm("sandbox1", ep) + if len(ctrlr.sandboxes) != 0 { + t.Fatalf("controller sandboxes is not empty. len = %d", len(ctrlr.sandboxes)) + } +} + +func TestSandboxAddMultiPrio(t *testing.T) { + ctrlr := createEmptyCtrlr() + ep1 := createEmptyEndpoint() + ep2 := createEmptyEndpoint() + ep3 := createEmptyEndpoint() + + ep1.container.config.prio = 1 + ep2.container.config.prio = 2 + ep3.container.config.prio = 3 + + if _, err := ctrlr.sandboxAdd("sandbox1", true, ep1); err != nil { + t.Fatal(err) + } + + if _, err := ctrlr.sandboxAdd("sandbox1", true, ep2); err != nil { + t.Fatal(err) + } + + if _, err := ctrlr.sandboxAdd("sandbox1", true, ep3); err != nil { + t.Fatal(err) + } + + if ctrlr.sandboxes["sandbox1"].refCnt != 3 { + t.Fatalf("Unexpected sandbox ref count. Expected 3, got %d", + ctrlr.sandboxes["sandbox1"].refCnt) + } + + if ctrlr.sandboxes["sandbox1"].endpoints[0] != ep3 { + t.Fatal("Expected ep3 to be at the top of the heap. But did not find ep3 at the top of the heap") + } + + ctrlr.sandboxRm("sandbox1", ep3) + + if ctrlr.sandboxes["sandbox1"].endpoints[0] != ep2 { + t.Fatal("Expected ep2 to be at the top of the heap after removing ep3. But did not find ep2 at the top of the heap") + } + + ctrlr.sandboxRm("sandbox1", ep2) + + if ctrlr.sandboxes["sandbox1"].endpoints[0] != ep1 { + t.Fatal("Expected ep1 to be at the top of the heap after removing ep2. But did not find ep1 at the top of the heap") + } + + // Re-add ep3 back + if _, err := ctrlr.sandboxAdd("sandbox1", true, ep3); err != nil { + t.Fatal(err) + } + + if ctrlr.sandboxes["sandbox1"].endpoints[0] != ep3 { + t.Fatal("Expected ep3 to be at the top of the heap after adding ep3 back. But did not find ep3 at the top of the heap") + } + + ctrlr.sandboxRm("sandbox1", ep3) + ctrlr.sandboxRm("sandbox1", ep1) + if len(ctrlr.sandboxes) != 0 { + t.Fatalf("controller sandboxes is not empty. len = %d", len(ctrlr.sandboxes)) + } +} + +func TestSandboxAddSamePrio(t *testing.T) { + ctrlr := createEmptyCtrlr() + ep1 := createEmptyEndpoint() + ep2 := createEmptyEndpoint() + + ep1.network = &network{name: "aaa"} + ep2.network = &network{name: "bbb"} + + if _, err := ctrlr.sandboxAdd("sandbox1", true, ep1); err != nil { + t.Fatal(err) + } + + if _, err := ctrlr.sandboxAdd("sandbox1", true, ep2); err != nil { + t.Fatal(err) + } + + if ctrlr.sandboxes["sandbox1"].refCnt != 2 { + t.Fatalf("Unexpected sandbox ref count. Expected 2, got %d", + ctrlr.sandboxes["sandbox1"].refCnt) + } + + if ctrlr.sandboxes["sandbox1"].endpoints[0] != ep1 { + t.Fatal("Expected ep1 to be at the top of the heap. But did not find ep1 at the top of the heap") + } + + ctrlr.sandboxRm("sandbox1", ep1) + + if ctrlr.sandboxes["sandbox1"].endpoints[0] != ep2 { + t.Fatal("Expected ep2 to be at the top of the heap after removing ep3. But did not find ep2 at the top of the heap") + } + + ctrlr.sandboxRm("sandbox1", ep2) + if len(ctrlr.sandboxes) != 0 { + t.Fatalf("controller sandboxes is not empty. len = %d", len(ctrlr.sandboxes)) + } +}