From 52e85b4b9a88ac8e3733f3e831bf8c5f75b33eea Mon Sep 17 00:00:00 2001 From: selansen Date: Tue, 24 Jul 2018 15:46:59 -0400 Subject: [PATCH] Global Default Address Pool support This change brings global default address pool feature into libnetwork. Idea is to reuse same code flow and functions that were implemented for local scope default address pool. Function InitNetworks carries most of the changes. local scope default address pool init should always happen only once. But Global scope default address pool can be initialized multiple times. Signed-off-by: selansen --- libnetwork/drivers/bridge/bridge_test.go | 5 -- libnetwork/ipam/allocator.go | 4 +- libnetwork/ipam/allocator_test.go | 2 - libnetwork/ipams/builtin/builtin_unix.go | 2 +- libnetwork/ipams/builtin/builtin_windows.go | 2 +- libnetwork/ipamutils/utils.go | 75 ++++++++++++++------- libnetwork/ipamutils/utils_test.go | 59 +++++++++++----- libnetwork/netutils/utils_linux.go | 4 +- libnetwork/netutils/utils_test.go | 13 ++-- 9 files changed, 104 insertions(+), 62 deletions(-) diff --git a/libnetwork/drivers/bridge/bridge_test.go b/libnetwork/drivers/bridge/bridge_test.go index e2f073fef6..945e2bd2ec 100644 --- a/libnetwork/drivers/bridge/bridge_test.go +++ b/libnetwork/drivers/bridge/bridge_test.go @@ -10,7 +10,6 @@ import ( "testing" "github.com/docker/libnetwork/driverapi" - "github.com/docker/libnetwork/ipamutils" "github.com/docker/libnetwork/iptables" "github.com/docker/libnetwork/netlabel" "github.com/docker/libnetwork/netutils" @@ -20,10 +19,6 @@ import ( "github.com/vishvananda/netlink" ) -func init() { - ipamutils.InitNetworks(nil) -} - func TestEndpointMarshalling(t *testing.T) { ip1, _ := types.ParseCIDR("172.22.0.9/16") ip2, _ := types.ParseCIDR("2001:db8::9") diff --git a/libnetwork/ipam/allocator.go b/libnetwork/ipam/allocator.go index 44070119dc..aa1a63a112 100644 --- a/libnetwork/ipam/allocator.go +++ b/libnetwork/ipam/allocator.go @@ -46,8 +46,8 @@ func NewAllocator(lcDs, glDs datastore.DataStore) (*Allocator, error) { // Load predefined subnet pools a.predefined = map[string][]*net.IPNet{ - localAddressSpace: ipamutils.PredefinedBroadNetworks, - globalAddressSpace: ipamutils.PredefinedGranularNetworks, + localAddressSpace: ipamutils.PredefinedLocalScopeDefaultNetworks, + globalAddressSpace: ipamutils.PredefinedGlobalScopeDefaultNetworks, } // Initialize asIndices map diff --git a/libnetwork/ipam/allocator_test.go b/libnetwork/ipam/allocator_test.go index 2f5ef3e882..9ce73bc00c 100644 --- a/libnetwork/ipam/allocator_test.go +++ b/libnetwork/ipam/allocator_test.go @@ -17,7 +17,6 @@ import ( "github.com/docker/libnetwork/bitseq" "github.com/docker/libnetwork/datastore" "github.com/docker/libnetwork/ipamapi" - "github.com/docker/libnetwork/ipamutils" _ "github.com/docker/libnetwork/testutils" "github.com/docker/libnetwork/types" "gotest.tools/assert" @@ -57,7 +56,6 @@ func randomLocalStore(needStore bool) (datastore.DataStore, error) { } func getAllocator(store bool) (*Allocator, error) { - ipamutils.InitNetworks(nil) ds, err := randomLocalStore(store) if err != nil { return nil, err diff --git a/libnetwork/ipams/builtin/builtin_unix.go b/libnetwork/ipams/builtin/builtin_unix.go index 0ab861d170..b07fb4b632 100644 --- a/libnetwork/ipams/builtin/builtin_unix.go +++ b/libnetwork/ipams/builtin/builtin_unix.go @@ -35,7 +35,7 @@ func Init(ic ipamapi.Callback, l, g interface{}) error { } } - ipamutils.InitNetworks(GetDefaultIPAddressPool()) + ipamutils.ConfigLocalScopeDefaultNetworks(GetDefaultIPAddressPool()) a, err := ipam.NewAllocator(localDs, globalDs) if err != nil { diff --git a/libnetwork/ipams/builtin/builtin_windows.go b/libnetwork/ipams/builtin/builtin_windows.go index e994595658..7975981eec 100644 --- a/libnetwork/ipams/builtin/builtin_windows.go +++ b/libnetwork/ipams/builtin/builtin_windows.go @@ -37,7 +37,7 @@ func InitDockerDefault(ic ipamapi.Callback, l, g interface{}) error { } } - ipamutils.InitNetworks(nil) + ipamutils.ConfigLocalScopeDefaultNetworks(nil) a, err := ipam.NewAllocator(localDs, globalDs) if err != nil { diff --git a/libnetwork/ipamutils/utils.go b/libnetwork/ipamutils/utils.go index f8eca58e8c..6ecd5204c1 100644 --- a/libnetwork/ipamutils/utils.go +++ b/libnetwork/ipamutils/utils.go @@ -5,23 +5,20 @@ import ( "fmt" "net" "sync" - - "github.com/sirupsen/logrus" ) var ( - // PredefinedBroadNetworks contains a list of 31 IPv4 private networks with host size 16 and 12 - // (172.17-31.x.x/16, 192.168.x.x/20) which do not overlap with the networks in `PredefinedGranularNetworks` - PredefinedBroadNetworks []*net.IPNet - // PredefinedGranularNetworks contains a list of 64K IPv4 private networks with host size 8 - // (10.x.x.x/24) which do not overlap with the networks in `PredefinedBroadNetworks` - PredefinedGranularNetworks []*net.IPNet - initNetworksOnce sync.Once - - defaultBroadNetwork = []*NetworkToSplit{{"172.17.0.0/16", 16}, {"172.18.0.0/16", 16}, {"172.19.0.0/16", 16}, + // PredefinedLocalScopeDefaultNetworks contains a list of 31 IPv4 private networks with host size 16 and 12 + // (172.17-31.x.x/16, 192.168.x.x/20) which do not overlap with the networks in `PredefinedGlobalScopeDefaultNetworks` + PredefinedLocalScopeDefaultNetworks []*net.IPNet + // PredefinedGlobalScopeDefaultNetworks contains a list of 64K IPv4 private networks with host size 8 + // (10.x.x.x/24) which do not overlap with the networks in `PredefinedLocalScopeDefaultNetworks` + PredefinedGlobalScopeDefaultNetworks []*net.IPNet + mutex sync.Mutex + localScopeDefaultNetworks = []*NetworkToSplit{{"172.17.0.0/16", 16}, {"172.18.0.0/16", 16}, {"172.19.0.0/16", 16}, {"172.20.0.0/14", 16}, {"172.24.0.0/14", 16}, {"172.28.0.0/14", 16}, {"192.168.0.0/16", 20}} - defaultGranularNetwork = []*NetworkToSplit{{"10.0.0.0/8", 24}} + globalScopeDefaultNetworks = []*NetworkToSplit{{"10.0.0.0/8", 24}} ) // NetworkToSplit represent a network that has to be split in chunks with mask length Size. @@ -34,19 +31,47 @@ type NetworkToSplit struct { Size int `json:"size"` } -// InitNetworks initializes the broad network pool and the granular network pool -func InitNetworks(defaultAddressPool []*NetworkToSplit) { - initNetworksOnce.Do(func() { - // error ingnored should never fail - PredefinedGranularNetworks, _ = splitNetworks(defaultGranularNetwork) - if defaultAddressPool == nil { - defaultAddressPool = defaultBroadNetwork - } - var err error - if PredefinedBroadNetworks, err = splitNetworks(defaultAddressPool); err != nil { - logrus.WithError(err).Error("InitAddressPools failed to initialize the default address pool") - } - }) +func init() { + var err error + if PredefinedGlobalScopeDefaultNetworks, err = splitNetworks(globalScopeDefaultNetworks); err != nil { + //we are going to panic in case of error as we should never get into this state + panic("InitAddressPools failed to initialize the global scope default address pool") + } + + if PredefinedLocalScopeDefaultNetworks, err = splitNetworks(localScopeDefaultNetworks); err != nil { + //we are going to panic in case of error as we should never get into this state + panic("InitAddressPools failed to initialize the local scope default address pool") + } +} + +// configDefaultNetworks configures local as well global default pool based on input +func configDefaultNetworks(defaultAddressPool []*NetworkToSplit, result *[]*net.IPNet) error { + mutex.Lock() + defer mutex.Unlock() + defaultNetworks, err := splitNetworks(defaultAddressPool) + if err != nil { + return err + } + *result = defaultNetworks + return nil +} + +// ConfigGlobalScopeDefaultNetworks configures global default pool. +// Ideally this will be called from SwarmKit as part of swarm init +func ConfigGlobalScopeDefaultNetworks(defaultAddressPool []*NetworkToSplit) error { + if defaultAddressPool == nil { + defaultAddressPool = globalScopeDefaultNetworks + } + return configDefaultNetworks(defaultAddressPool, &PredefinedGlobalScopeDefaultNetworks) +} + +// ConfigLocalScopeDefaultNetworks configures local default pool. +// Ideally this will be called during libnetwork init +func ConfigLocalScopeDefaultNetworks(defaultAddressPool []*NetworkToSplit) error { + if defaultAddressPool == nil { + return nil + } + return configDefaultNetworks(defaultAddressPool, &PredefinedLocalScopeDefaultNetworks) } // splitNetworks takes a slice of networks, split them accordingly and returns them diff --git a/libnetwork/ipamutils/utils_test.go b/libnetwork/ipamutils/utils_test.go index 7a3f21bfc8..d82cb68366 100644 --- a/libnetwork/ipamutils/utils_test.go +++ b/libnetwork/ipamutils/utils_test.go @@ -2,7 +2,6 @@ package ipamutils import ( "net" - "sync" "testing" _ "github.com/docker/libnetwork/testutils" @@ -34,15 +33,25 @@ func initGranularPredefinedNetworks() []*net.IPNet { return pl } +func initGlobalScopeNetworks() []*net.IPNet { + pl := make([]*net.IPNet, 0, 256*256) + mask := []byte{255, 255, 255, 0} + for i := 0; i < 256; i++ { + for j := 0; j < 256; j++ { + pl = append(pl, &net.IPNet{IP: []byte{30, byte(i), byte(j), 0}, Mask: mask}) + } + } + return pl +} + func TestDefaultNetwork(t *testing.T) { - InitNetworks(nil) - for _, nw := range PredefinedGranularNetworks { + for _, nw := range PredefinedGlobalScopeDefaultNetworks { if ones, bits := nw.Mask.Size(); bits != 32 || ones != 24 { t.Fatalf("Unexpected size for network in granular list: %v", nw) } } - for _, nw := range PredefinedBroadNetworks { + for _, nw := range PredefinedLocalScopeDefaultNetworks { if ones, bits := nw.Mask.Size(); bits != 32 || (ones != 20 && ones != 16) { t.Fatalf("Unexpected size for network in broad list: %v", nw) } @@ -53,7 +62,7 @@ func TestDefaultNetwork(t *testing.T) { for _, v := range originalBroadNets { m[v.String()] = true } - for _, nw := range PredefinedBroadNetworks { + for _, nw := range PredefinedLocalScopeDefaultNetworks { _, ok := m[nw.String()] assert.Check(t, ok) delete(m, nw.String()) @@ -67,7 +76,25 @@ func TestDefaultNetwork(t *testing.T) { for _, v := range originalGranularNets { m[v.String()] = true } - for _, nw := range PredefinedGranularNetworks { + for _, nw := range PredefinedGlobalScopeDefaultNetworks { + _, ok := m[nw.String()] + assert.Check(t, ok) + delete(m, nw.String()) + } + + assert.Check(t, is.Len(m, 0)) +} + +func TestConfigGlobalScopeDefaultNetworks(t *testing.T) { + err := ConfigGlobalScopeDefaultNetworks([]*NetworkToSplit{{"30.0.0.0/8", 24}}) + assert.NilError(t, err) + + originalGlobalScopeNetworks := initGlobalScopeNetworks() + m := make(map[string]bool) + for _, v := range originalGlobalScopeNetworks { + m[v.String()] = true + } + for _, nw := range PredefinedGlobalScopeDefaultNetworks { _, ok := m[nw.String()] assert.Check(t, ok) delete(m, nw.String()) @@ -77,15 +104,15 @@ func TestDefaultNetwork(t *testing.T) { } func TestInitAddressPools(t *testing.T) { - initNetworksOnce = sync.Once{} - InitNetworks([]*NetworkToSplit{{"172.80.0.0/16", 24}, {"172.90.0.0/16", 24}}) + err := ConfigLocalScopeDefaultNetworks([]*NetworkToSplit{{"172.80.0.0/16", 24}, {"172.90.0.0/16", 24}}) + assert.NilError(t, err) - // Check for Random IPAddresses in PredefinedBroadNetworks ex: first , last and middle - assert.Check(t, is.Len(PredefinedBroadNetworks, 512), "Failed to find PredefinedBroadNetworks") - assert.Check(t, is.Equal(PredefinedBroadNetworks[0].String(), "172.80.0.0/24")) - assert.Check(t, is.Equal(PredefinedBroadNetworks[127].String(), "172.80.127.0/24")) - assert.Check(t, is.Equal(PredefinedBroadNetworks[255].String(), "172.80.255.0/24")) - assert.Check(t, is.Equal(PredefinedBroadNetworks[256].String(), "172.90.0.0/24")) - assert.Check(t, is.Equal(PredefinedBroadNetworks[383].String(), "172.90.127.0/24")) - assert.Check(t, is.Equal(PredefinedBroadNetworks[511].String(), "172.90.255.0/24")) + // Check for Random IPAddresses in PredefinedLocalScopeDefaultNetworks ex: first , last and middle + assert.Check(t, is.Len(PredefinedLocalScopeDefaultNetworks, 512), "Failed to find PredefinedLocalScopeDefaultNetworks") + assert.Check(t, is.Equal(PredefinedLocalScopeDefaultNetworks[0].String(), "172.80.0.0/24")) + assert.Check(t, is.Equal(PredefinedLocalScopeDefaultNetworks[127].String(), "172.80.127.0/24")) + assert.Check(t, is.Equal(PredefinedLocalScopeDefaultNetworks[255].String(), "172.80.255.0/24")) + assert.Check(t, is.Equal(PredefinedLocalScopeDefaultNetworks[256].String(), "172.90.0.0/24")) + assert.Check(t, is.Equal(PredefinedLocalScopeDefaultNetworks[383].String(), "172.90.127.0/24")) + assert.Check(t, is.Equal(PredefinedLocalScopeDefaultNetworks[511].String(), "172.90.255.0/24")) } diff --git a/libnetwork/netutils/utils_linux.go b/libnetwork/netutils/utils_linux.go index 870cc12855..10a5e109ee 100644 --- a/libnetwork/netutils/utils_linux.go +++ b/libnetwork/netutils/utils_linux.go @@ -94,8 +94,8 @@ func ElectInterfaceAddresses(name string) ([]*net.IPNet, []*net.IPNet, error) { } if link == nil || len(v4Nets) == 0 { - // Choose from predefined broad networks - v4Net, err := FindAvailableNetwork(ipamutils.PredefinedBroadNetworks) + // Choose from predefined local scope networks + v4Net, err := FindAvailableNetwork(ipamutils.PredefinedLocalScopeDefaultNetworks) if err != nil { return nil, nil, err } diff --git a/libnetwork/netutils/utils_test.go b/libnetwork/netutils/utils_test.go index c24d635cf4..99cdea650f 100644 --- a/libnetwork/netutils/utils_test.go +++ b/libnetwork/netutils/utils_test.go @@ -212,15 +212,14 @@ func TestUtilGenerateRandomMAC(t *testing.T) { func TestNetworkRequest(t *testing.T) { defer testutils.SetupTestOSContext(t)() - ipamutils.InitNetworks(nil) - nw, err := FindAvailableNetwork(ipamutils.PredefinedBroadNetworks) + nw, err := FindAvailableNetwork(ipamutils.PredefinedLocalScopeDefaultNetworks) if err != nil { t.Fatal(err) } var found bool - for _, exp := range ipamutils.PredefinedBroadNetworks { + for _, exp := range ipamutils.PredefinedLocalScopeDefaultNetworks { if types.CompareIPNet(exp, nw) { found = true break @@ -231,13 +230,13 @@ func TestNetworkRequest(t *testing.T) { t.Fatalf("Found unexpected broad network %s", nw) } - nw, err = FindAvailableNetwork(ipamutils.PredefinedGranularNetworks) + nw, err = FindAvailableNetwork(ipamutils.PredefinedGlobalScopeDefaultNetworks) if err != nil { t.Fatal(err) } found = false - for _, exp := range ipamutils.PredefinedGranularNetworks { + for _, exp := range ipamutils.PredefinedGlobalScopeDefaultNetworks { if types.CompareIPNet(exp, nw) { found = true break @@ -255,7 +254,7 @@ func TestNetworkRequest(t *testing.T) { if err != nil { t.Fatal(err) } - nw, err = FindAvailableNetwork(ipamutils.PredefinedBroadNetworks) + nw, err = FindAvailableNetwork(ipamutils.PredefinedLocalScopeDefaultNetworks) if err != nil { t.Fatal(err) } @@ -266,7 +265,6 @@ func TestNetworkRequest(t *testing.T) { func TestElectInterfaceAddressMultipleAddresses(t *testing.T) { defer testutils.SetupTestOSContext(t)() - ipamutils.InitNetworks(nil) nws := []string{"172.101.202.254/16", "172.102.202.254/16"} createInterface(t, "test", nws...) @@ -303,7 +301,6 @@ func TestElectInterfaceAddressMultipleAddresses(t *testing.T) { func TestElectInterfaceAddress(t *testing.T) { defer testutils.SetupTestOSContext(t)() - ipamutils.InitNetworks(nil) nws := "172.101.202.254/16" createInterface(t, "test", nws)