From a93d08aef58ba8f4e57c9e03304bee256b34c699 Mon Sep 17 00:00:00 2001 From: Jana Radhakrishnan Date: Tue, 2 Jun 2015 13:22:28 -0700 Subject: [PATCH] Add container join priority option to endpoint When an endpoint is joined by a container it may optionally pass a priority to resolve resource conflicts inside the sandbox when more than one endpoint provides the same kind of resource. If the the priority is the same for two endpoints with conflicting resources then the endpoint network names are used to resolve the conflict. Signed-off-by: Jana Radhakrishnan --- libnetwork/controller.go | 51 ------- libnetwork/endpoint.go | 71 ++-------- libnetwork/sandboxdata.go | 246 +++++++++++++++++++++++++++++++++ libnetwork/sandboxdata_test.go | 130 +++++++++++++++++ 4 files changed, 388 insertions(+), 110 deletions(-) create mode 100644 libnetwork/sandboxdata.go create mode 100644 libnetwork/sandboxdata_test.go diff --git a/libnetwork/controller.go b/libnetwork/controller.go index 8046731cf0..d709d972b0 100644 --- a/libnetwork/controller.go +++ b/libnetwork/controller.go @@ -59,7 +59,6 @@ import ( "github.com/docker/libnetwork/datastore" "github.com/docker/libnetwork/driverapi" "github.com/docker/libnetwork/hostdiscovery" - "github.com/docker/libnetwork/sandbox" "github.com/docker/libnetwork/types" "github.com/docker/swarm/pkg/store" ) @@ -91,11 +90,6 @@ type NetworkController interface { // When the function returns true, the walk will stop. type NetworkWalker func(nw Network) bool -type sandboxData struct { - sandbox sandbox.Sandbox - refCnt int -} - type networkTable map[types.UUID]*network type endpointTable map[types.UUID]*endpoint type sandboxTable map[string]*sandboxData @@ -382,51 +376,6 @@ func (c *controller) NetworkByID(id string) (Network, error) { return nil, ErrNoSuchNetwork(id) } -func (c *controller) sandboxAdd(key string, create bool) (sandbox.Sandbox, error) { - c.Lock() - defer c.Unlock() - - sData, ok := c.sandboxes[key] - if !ok { - sb, err := sandbox.NewSandbox(key, create) - if err != nil { - return nil, err - } - - sData = &sandboxData{sandbox: sb, refCnt: 1} - c.sandboxes[key] = sData - return sData.sandbox, nil - } - - sData.refCnt++ - return sData.sandbox, nil -} - -func (c *controller) sandboxRm(key string) { - c.Lock() - defer c.Unlock() - - sData := c.sandboxes[key] - sData.refCnt-- - - if sData.refCnt == 0 { - sData.sandbox.Destroy() - delete(c.sandboxes, key) - } -} - -func (c *controller) sandboxGet(key string) sandbox.Sandbox { - c.Lock() - defer c.Unlock() - - sData, ok := c.sandboxes[key] - if !ok { - return nil - } - - return sData.sandbox -} - func (c *controller) loadDriver(networkType string) (driverapi.Driver, error) { // Plugins pkg performs lazy loading of plugins that acts as remote drivers. // As per the design, this Get call will result in remote driver discovery if there is a corresponding plugin available. diff --git a/libnetwork/endpoint.go b/libnetwork/endpoint.go index 2d9dab9f31..c357b3298f 100644 --- a/libnetwork/endpoint.go +++ b/libnetwork/endpoint.go @@ -8,7 +8,6 @@ import ( "path/filepath" "sync" - "github.com/Sirupsen/logrus" "github.com/docker/docker/pkg/ioutils" "github.com/docker/libnetwork/etchosts" "github.com/docker/libnetwork/netlabel" @@ -78,6 +77,7 @@ type containerConfig struct { resolvConfPathConfig generic map[string]interface{} useDefaultSandBox bool + prio int // higher the value, more the priority } type extraHost struct { @@ -101,7 +101,6 @@ type endpoint struct { name string id types.UUID network *network - sandboxInfo *sandbox.Info iFaces []*endpointInterface joinInfo *endpointJoinInfo container *containerInfo @@ -233,8 +232,6 @@ func (ep *endpoint) Join(containerID string, options ...EndpointOption) error { container := ep.container network := ep.network epid := ep.id - joinInfo := ep.joinInfo - ifaces := ep.iFaces ep.Unlock() defer func() { @@ -278,49 +275,16 @@ func (ep *endpoint) Join(containerID string, options ...EndpointOption) error { return err } - sb, err := ctrlr.sandboxAdd(sboxKey, !container.config.useDefaultSandBox) + sb, err := ctrlr.sandboxAdd(sboxKey, !container.config.useDefaultSandBox, ep) if err != nil { return err } defer func() { if err != nil { - ctrlr.sandboxRm(sboxKey) + ctrlr.sandboxRm(sboxKey, ep) } }() - for _, i := range ifaces { - iface := &sandbox.Interface{ - SrcName: i.srcName, - DstName: i.dstPrefix, - Address: &i.addr, - Routes: i.routes, - } - if i.addrv6.IP.To16() != nil { - iface.AddressIPv6 = &i.addrv6 - } - err = sb.AddInterface(iface) - if err != nil { - return err - } - } - // Set up non-interface routes. - for _, r := range ep.joinInfo.StaticRoutes { - err = sb.AddStaticRoute(r) - if err != nil { - return err - } - } - - err = sb.SetGateway(joinInfo.gw) - if err != nil { - return err - } - - err = sb.SetGatewayIPv6(joinInfo.gw6) - if err != nil { - return err - } - container.data.SandboxKey = sb.Key() return nil @@ -372,26 +336,7 @@ func (ep *endpoint) Leave(containerID string, options ...EndpointOption) error { err = driver.Leave(n.id, ep.id) - sb := ctrlr.sandboxGet(container.data.SandboxKey) - for _, i := range sb.Interfaces() { - // Only remove the interfaces owned by this endpoint from the sandbox. - if ep.hasInterface(i.SrcName) { - err = sb.RemoveInterface(i) - if err != nil { - logrus.Debugf("Remove interface failed: %v", err) - } - } - } - - // Remove non-interface routes. - for _, r := range ep.joinInfo.StaticRoutes { - err = sb.RemoveStaticRoute(r) - if err != nil { - logrus.Debugf("Remove route failed: %v", err) - } - } - - ctrlr.sandboxRm(container.data.SandboxKey) + ctrlr.sandboxRm(container.data.SandboxKey, ep) return err } @@ -648,6 +593,14 @@ func EndpointOptionGeneric(generic map[string]interface{}) EndpointOption { } } +// JoinOptionPriority function returns an option setter for priority option to +// be passed to endpoint Join method. +func JoinOptionPriority(prio int) EndpointOption { + return func(ep *endpoint) { + ep.container.config.prio = prio + } +} + // JoinOptionHostname function returns an option setter for hostname option to // be passed to endpoint Join method. func JoinOptionHostname(name string) EndpointOption { diff --git a/libnetwork/sandboxdata.go b/libnetwork/sandboxdata.go new file mode 100644 index 0000000000..947fea3863 --- /dev/null +++ b/libnetwork/sandboxdata.go @@ -0,0 +1,246 @@ +package libnetwork + +import ( + "container/heap" + "sync" + + "github.com/Sirupsen/logrus" + "github.com/docker/libnetwork/sandbox" +) + +type epHeap []*endpoint + +type sandboxData struct { + sbox sandbox.Sandbox + refCnt int + endpoints epHeap + sync.Mutex +} + +func (eh epHeap) Len() int { return len(eh) } + +func (eh epHeap) Less(i, j int) bool { + eh[i].Lock() + eh[j].Lock() + defer eh[j].Unlock() + defer eh[i].Unlock() + + if eh[i].container.config.prio == eh[j].container.config.prio { + return eh[i].network.Name() < eh[j].network.Name() + } + + return eh[i].container.config.prio > eh[j].container.config.prio +} + +func (eh epHeap) Swap(i, j int) { eh[i], eh[j] = eh[j], eh[i] } + +func (eh *epHeap) Push(x interface{}) { + *eh = append(*eh, x.(*endpoint)) +} + +func (eh *epHeap) Pop() interface{} { + old := *eh + n := len(old) + x := old[n-1] + *eh = old[0 : n-1] + return x +} + +func (s *sandboxData) updateGateway(ep *endpoint) error { + sb := s.sandbox() + if err := sb.UnsetGateway(); err != nil { + return err + } + + if err := sb.UnsetGatewayIPv6(); err != nil { + return err + } + + if ep == nil { + return nil + } + + ep.Lock() + joinInfo := ep.joinInfo + ep.Unlock() + + if err := sb.SetGateway(joinInfo.gw); err != nil { + return err + } + + if err := sb.SetGatewayIPv6(joinInfo.gw6); err != nil { + return err + } + + return nil +} + +func (s *sandboxData) addEndpoint(ep *endpoint) error { + ep.Lock() + joinInfo := ep.joinInfo + ifaces := ep.iFaces + ep.Unlock() + + sb := s.sandbox() + for _, i := range ifaces { + iface := &sandbox.Interface{ + SrcName: i.srcName, + DstName: i.dstPrefix, + Address: &i.addr, + Routes: i.routes, + } + if i.addrv6.IP.To16() != nil { + iface.AddressIPv6 = &i.addrv6 + } + + if err := sb.AddInterface(iface); err != nil { + return err + } + } + + if joinInfo != nil { + // Set up non-interface routes. + for _, r := range ep.joinInfo.StaticRoutes { + if err := sb.AddStaticRoute(r); err != nil { + return err + } + } + } + + s.Lock() + heap.Push(&s.endpoints, ep) + highEp := s.endpoints[0] + s.Unlock() + + if ep == highEp { + if err := s.updateGateway(ep); err != nil { + return err + } + } + + s.Lock() + s.refCnt++ + s.Unlock() + + return nil +} + +func (s *sandboxData) rmEndpoint(ep *endpoint) int { + ep.Lock() + joinInfo := ep.joinInfo + ep.Unlock() + + sb := s.sandbox() + for _, i := range sb.Interfaces() { + // Only remove the interfaces owned by this endpoint from the sandbox. + if ep.hasInterface(i.SrcName) { + if err := sb.RemoveInterface(i); err != nil { + logrus.Debugf("Remove interface failed: %v", err) + } + } + } + + // Remove non-interface routes. + for _, r := range joinInfo.StaticRoutes { + if err := sb.RemoveStaticRoute(r); err != nil { + logrus.Debugf("Remove route failed: %v", err) + } + } + + // We don't check if s.endpoints is empty here because + // it should never be empty during a rmEndpoint call and + // if it is we will rightfully panic here + s.Lock() + highEpBefore := s.endpoints[0] + var ( + i int + e *endpoint + ) + for i, e = range s.endpoints { + if e == ep { + break + } + } + heap.Remove(&s.endpoints, i) + var highEpAfter *endpoint + if len(s.endpoints) > 0 { + highEpAfter = s.endpoints[0] + } + + s.Unlock() + + if highEpBefore != highEpAfter { + s.updateGateway(highEpAfter) + } + + s.Lock() + s.refCnt-- + refCnt := s.refCnt + s.Unlock() + + if refCnt == 0 { + s.sandbox().Destroy() + } + + return refCnt +} + +func (s *sandboxData) sandbox() sandbox.Sandbox { + s.Lock() + defer s.Unlock() + + return s.sbox +} + +func (c *controller) sandboxAdd(key string, create bool, ep *endpoint) (sandbox.Sandbox, error) { + c.Lock() + sData, ok := c.sandboxes[key] + c.Unlock() + + if !ok { + sb, err := sandbox.NewSandbox(key, create) + if err != nil { + return nil, err + } + + sData = &sandboxData{ + sbox: sb, + endpoints: epHeap{}, + } + + heap.Init(&sData.endpoints) + c.Lock() + c.sandboxes[key] = sData + c.Unlock() + } + + if err := sData.addEndpoint(ep); err != nil { + return nil, err + } + + return sData.sandbox(), nil +} + +func (c *controller) sandboxRm(key string, ep *endpoint) { + c.Lock() + sData := c.sandboxes[key] + c.Unlock() + + if sData.rmEndpoint(ep) == 0 { + c.Lock() + delete(c.sandboxes, key) + c.Unlock() + } +} + +func (c *controller) sandboxGet(key string) sandbox.Sandbox { + c.Lock() + sData, ok := c.sandboxes[key] + c.Unlock() + + if !ok { + return nil + } + + return sData.sandbox() +} diff --git a/libnetwork/sandboxdata_test.go b/libnetwork/sandboxdata_test.go new file mode 100644 index 0000000000..3b76326c3a --- /dev/null +++ b/libnetwork/sandboxdata_test.go @@ -0,0 +1,130 @@ +package libnetwork + +import "testing" + +func createEmptyCtrlr() *controller { + return &controller{sandboxes: sandboxTable{}} +} + +func createEmptyEndpoint() *endpoint { + return &endpoint{ + container: &containerInfo{}, + joinInfo: &endpointJoinInfo{}, + iFaces: []*endpointInterface{}, + } +} + +func TestSandboxAddEmpty(t *testing.T) { + ctrlr := createEmptyCtrlr() + ep := createEmptyEndpoint() + + if _, err := ctrlr.sandboxAdd("sandbox1", true, ep); err != nil { + t.Fatal(err) + } + + if ctrlr.sandboxes["sandbox1"].refCnt != 1 { + t.Fatalf("Unexpected sandbox ref count. Expected 1, got %d", + ctrlr.sandboxes["sandbox1"].refCnt) + } + + ctrlr.sandboxRm("sandbox1", ep) + if len(ctrlr.sandboxes) != 0 { + t.Fatalf("controller sandboxes is not empty. len = %d", len(ctrlr.sandboxes)) + } +} + +func TestSandboxAddMultiPrio(t *testing.T) { + ctrlr := createEmptyCtrlr() + ep1 := createEmptyEndpoint() + ep2 := createEmptyEndpoint() + ep3 := createEmptyEndpoint() + + ep1.container.config.prio = 1 + ep2.container.config.prio = 2 + ep3.container.config.prio = 3 + + if _, err := ctrlr.sandboxAdd("sandbox1", true, ep1); err != nil { + t.Fatal(err) + } + + if _, err := ctrlr.sandboxAdd("sandbox1", true, ep2); err != nil { + t.Fatal(err) + } + + if _, err := ctrlr.sandboxAdd("sandbox1", true, ep3); err != nil { + t.Fatal(err) + } + + if ctrlr.sandboxes["sandbox1"].refCnt != 3 { + t.Fatalf("Unexpected sandbox ref count. Expected 3, got %d", + ctrlr.sandboxes["sandbox1"].refCnt) + } + + if ctrlr.sandboxes["sandbox1"].endpoints[0] != ep3 { + t.Fatal("Expected ep3 to be at the top of the heap. But did not find ep3 at the top of the heap") + } + + ctrlr.sandboxRm("sandbox1", ep3) + + if ctrlr.sandboxes["sandbox1"].endpoints[0] != ep2 { + t.Fatal("Expected ep2 to be at the top of the heap after removing ep3. But did not find ep2 at the top of the heap") + } + + ctrlr.sandboxRm("sandbox1", ep2) + + if ctrlr.sandboxes["sandbox1"].endpoints[0] != ep1 { + t.Fatal("Expected ep1 to be at the top of the heap after removing ep2. But did not find ep1 at the top of the heap") + } + + // Re-add ep3 back + if _, err := ctrlr.sandboxAdd("sandbox1", true, ep3); err != nil { + t.Fatal(err) + } + + if ctrlr.sandboxes["sandbox1"].endpoints[0] != ep3 { + t.Fatal("Expected ep3 to be at the top of the heap after adding ep3 back. But did not find ep3 at the top of the heap") + } + + ctrlr.sandboxRm("sandbox1", ep3) + ctrlr.sandboxRm("sandbox1", ep1) + if len(ctrlr.sandboxes) != 0 { + t.Fatalf("controller sandboxes is not empty. len = %d", len(ctrlr.sandboxes)) + } +} + +func TestSandboxAddSamePrio(t *testing.T) { + ctrlr := createEmptyCtrlr() + ep1 := createEmptyEndpoint() + ep2 := createEmptyEndpoint() + + ep1.network = &network{name: "aaa"} + ep2.network = &network{name: "bbb"} + + if _, err := ctrlr.sandboxAdd("sandbox1", true, ep1); err != nil { + t.Fatal(err) + } + + if _, err := ctrlr.sandboxAdd("sandbox1", true, ep2); err != nil { + t.Fatal(err) + } + + if ctrlr.sandboxes["sandbox1"].refCnt != 2 { + t.Fatalf("Unexpected sandbox ref count. Expected 2, got %d", + ctrlr.sandboxes["sandbox1"].refCnt) + } + + if ctrlr.sandboxes["sandbox1"].endpoints[0] != ep1 { + t.Fatal("Expected ep1 to be at the top of the heap. But did not find ep1 at the top of the heap") + } + + ctrlr.sandboxRm("sandbox1", ep1) + + if ctrlr.sandboxes["sandbox1"].endpoints[0] != ep2 { + t.Fatal("Expected ep2 to be at the top of the heap after removing ep3. But did not find ep2 at the top of the heap") + } + + ctrlr.sandboxRm("sandbox1", ep2) + if len(ctrlr.sandboxes) != 0 { + t.Fatalf("controller sandboxes is not empty. len = %d", len(ctrlr.sandboxes)) + } +}