libnw: untangle IPAM allocator from global state

ipam.Allocator is not a singleton, but it references mutable singleton
state. Address that deficiency by refactoring it to instead take the
predefined address spaces as constructor arguments. Unfortunately some
work is needed on the Swarmkit side before the mutable singleton state
can be completely eliminated from the IPAMs.

Signed-off-by: Cory Snider <csnider@mirantis.com>

libnetwork/drivers_ipam.go

@@ -10,7 +10,12 @@ import (
 )
 
 func initIPAMDrivers(r *drvregistry.DrvRegistry, lDs, gDs interface{}, addressPool []*ipamutils.NetworkToSplit) error {
-	builtinIpam.SetDefaultIPAddressPool(addressPool)
+	// TODO: pass address pools as arguments to builtinIpam.Init instead of
+	// indirectly through global mutable state. Swarmkit references that
+	// function so changing its signature breaks the build.
+	if err := builtinIpam.SetDefaultIPAddressPool(addressPool); err != nil {
+		return err
+	}
 	for _, fn := range [](func(ipamapi.Callback, interface{}, interface{}) error){
 		builtinIpam.Init,
 		remoteIpam.Init,

libnetwork/ipam/allocator.go

@@ -10,7 +10,6 @@ import (
 	"github.com/docker/docker/libnetwork/datastore"
 	"github.com/docker/docker/libnetwork/discoverapi"
 	"github.com/docker/docker/libnetwork/ipamapi"
-	"github.com/docker/docker/libnetwork/ipamutils"
 	"github.com/docker/docker/libnetwork/types"
 	"github.com/sirupsen/logrus"
 )
@@ -38,14 +37,14 @@ type Allocator struct {
 }
 
 // NewAllocator returns an instance of libnetwork ipam
-func NewAllocator(lcDs, glDs datastore.DataStore) (*Allocator, error) {
+func NewAllocator(lcDs, glDs datastore.DataStore, lcAs, glAs []*net.IPNet) (*Allocator, error) {
 	a := &Allocator{}
 
 	// Load predefined subnet pools
 
 	a.predefined = map[string][]*net.IPNet{
-		localAddressSpace:  ipamutils.GetLocalScopeDefaultNetworks(),
-		globalAddressSpace: ipamutils.GetGlobalScopeDefaultNetworks(),
+		localAddressSpace:  lcAs,
+		globalAddressSpace: glAs,
 	}
 
 	// Initialize asIndices map

libnetwork/ipam/allocator_test.go

@@ -16,6 +16,7 @@ import (
 	"github.com/docker/docker/libnetwork/bitseq"
 	"github.com/docker/docker/libnetwork/datastore"
 	"github.com/docker/docker/libnetwork/ipamapi"
+	"github.com/docker/docker/libnetwork/ipamutils"
 	"github.com/docker/docker/libnetwork/types"
 	"github.com/docker/libkv/store"
 	"github.com/docker/libkv/store/boltdb"
@@ -61,7 +62,7 @@ func getAllocator(store bool) (*Allocator, error) {
 	if err != nil {
 		return nil, err
 	}
-	return NewAllocator(ds, nil)
+	return NewAllocator(ds, nil, ipamutils.GetLocalScopeDefaultNetworks(), ipamutils.GetGlobalScopeDefaultNetworks())
 }
 
 func TestInt2IP2IntConversion(t *testing.T) {
@@ -1287,7 +1288,7 @@ func testAllocateRandomDeallocate(t *testing.T, pool, subPool string, num int, s
 	ds, err := randomLocalStore(store)
 	assert.NilError(t, err)
 
-	a, err := NewAllocator(ds, nil)
+	a, err := NewAllocator(ds, nil, ipamutils.GetLocalScopeDefaultNetworks(), ipamutils.GetGlobalScopeDefaultNetworks())
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -1355,7 +1356,7 @@ func TestRetrieveFromStore(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	a, err := NewAllocator(ds, nil)
+	a, err := NewAllocator(ds, nil, ipamutils.GetLocalScopeDefaultNetworks(), ipamutils.GetGlobalScopeDefaultNetworks())
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -1370,7 +1371,7 @@ func TestRetrieveFromStore(t *testing.T) {
 	}
 
 	// Restore
-	a1, err := NewAllocator(ds, nil)
+	a1, err := NewAllocator(ds, nil, ipamutils.GetLocalScopeDefaultNetworks(), ipamutils.GetGlobalScopeDefaultNetworks())
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -1392,7 +1393,7 @@ func TestRetrieveFromStore(t *testing.T) {
 	}
 
 	// Restore
-	a2, err := NewAllocator(ds, nil)
+	a2, err := NewAllocator(ds, nil, ipamutils.GetLocalScopeDefaultNetworks(), ipamutils.GetGlobalScopeDefaultNetworks())
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -1409,7 +1410,7 @@ func TestRetrieveFromStore(t *testing.T) {
 	}
 
 	// Restore
-	a3, err := NewAllocator(ds, nil)
+	a3, err := NewAllocator(ds, nil, ipamutils.GetLocalScopeDefaultNetworks(), ipamutils.GetGlobalScopeDefaultNetworks())
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -1426,7 +1427,7 @@ func TestRetrieveFromStore(t *testing.T) {
 	}
 
 	// Restore
-	a4, err := NewAllocator(ds, nil)
+	a4, err := NewAllocator(ds, nil, ipamutils.GetLocalScopeDefaultNetworks(), ipamutils.GetGlobalScopeDefaultNetworks())
 	if err != nil {
 		t.Fatal(err)
 	}

libnetwork/ipams/builtin/builtin.go

@@ -0,0 +1,62 @@
+package builtin
+
+import (
+	"errors"
+	"net"
+
+	"github.com/docker/docker/libnetwork/datastore"
+	"github.com/docker/docker/libnetwork/ipam"
+	"github.com/docker/docker/libnetwork/ipamapi"
+	"github.com/docker/docker/libnetwork/ipamutils"
+)
+
+var (
+	// defaultAddressPool Stores user configured subnet list
+	defaultAddressPool []*net.IPNet
+)
+
+// initBuiltin registers the built-in ipam service with libnetwork
+func initBuiltin(ic ipamapi.Callback, l, g interface{}) error {
+	var (
+		ok                bool
+		localDs, globalDs datastore.DataStore
+	)
+
+	if l != nil {
+		if localDs, ok = l.(datastore.DataStore); !ok {
+			return errors.New("incorrect local datastore passed to built-in ipam init")
+		}
+	}
+
+	if g != nil {
+		if globalDs, ok = g.(datastore.DataStore); !ok {
+			return errors.New("incorrect global datastore passed to built-in ipam init")
+		}
+	}
+
+	var localAddressPool []*net.IPNet
+	if len(defaultAddressPool) > 0 {
+		localAddressPool = append([]*net.IPNet(nil), defaultAddressPool...)
+	} else {
+		localAddressPool = ipamutils.GetLocalScopeDefaultNetworks()
+	}
+
+	a, err := ipam.NewAllocator(localDs, globalDs, localAddressPool, ipamutils.GetGlobalScopeDefaultNetworks())
+	if err != nil {
+		return err
+	}
+
+	cps := &ipamapi.Capability{RequiresRequestReplay: true}
+
+	return ic.RegisterIpamDriverWithCapabilities(ipamapi.DefaultIPAM, a, cps)
+}
+
+// SetDefaultIPAddressPool stores default address pool.
+func SetDefaultIPAddressPool(addressPool []*ipamutils.NetworkToSplit) error {
+	nets, err := ipamutils.SplitNetworks(addressPool)
+	if err != nil {
+		return err
+	}
+	defaultAddressPool = nets
+	return nil
+}

libnetwork/ipams/builtin/builtin_unix.go

@@ -3,60 +3,9 @@
 
 package builtin
 
-import (
-	"errors"
-
-	"github.com/docker/docker/libnetwork/datastore"
-	"github.com/docker/docker/libnetwork/ipam"
-	"github.com/docker/docker/libnetwork/ipamapi"
-	"github.com/docker/docker/libnetwork/ipamutils"
-)
-
-var (
-	// defaultAddressPool Stores user configured subnet list
-	defaultAddressPool []*ipamutils.NetworkToSplit
-)
+import "github.com/docker/docker/libnetwork/ipamapi"
 
 // Init registers the built-in ipam service with libnetwork
 func Init(ic ipamapi.Callback, l, g interface{}) error {
-	var (
-		ok                bool
-		localDs, globalDs datastore.DataStore
-	)
-
-	if l != nil {
-		if localDs, ok = l.(datastore.DataStore); !ok {
-			return errors.New("incorrect local datastore passed to built-in ipam init")
-		}
-	}
-
-	if g != nil {
-		if globalDs, ok = g.(datastore.DataStore); !ok {
-			return errors.New("incorrect global datastore passed to built-in ipam init")
-		}
-	}
-
-	err := ipamutils.ConfigLocalScopeDefaultNetworks(GetDefaultIPAddressPool())
-	if err != nil {
-		return err
-	}
-
-	a, err := ipam.NewAllocator(localDs, globalDs)
-	if err != nil {
-		return err
-	}
-
-	cps := &ipamapi.Capability{RequiresRequestReplay: true}
-
-	return ic.RegisterIpamDriverWithCapabilities(ipamapi.DefaultIPAM, a, cps)
-}
-
-// SetDefaultIPAddressPool stores default address pool.
-func SetDefaultIPAddressPool(addressPool []*ipamutils.NetworkToSplit) {
-	defaultAddressPool = addressPool
-}
-
-// GetDefaultIPAddressPool returns default address pool.
-func GetDefaultIPAddressPool() []*ipamutils.NetworkToSplit {
-	return defaultAddressPool
+	return initBuiltin(ic, l, g)
 }

libnetwork/ipams/builtin/builtin_windows.go

@@ -4,70 +4,18 @@
 package builtin
 
 import (
-	"errors"
-
-	"github.com/docker/docker/libnetwork/datastore"
-	"github.com/docker/docker/libnetwork/ipam"
 	"github.com/docker/docker/libnetwork/ipamapi"
-	"github.com/docker/docker/libnetwork/ipamutils"
-
-	windowsipam "github.com/docker/docker/libnetwork/ipams/windowsipam"
-)
-
-var (
-	// defaultAddressPool Stores user configured subnet list
-	defaultAddressPool []*ipamutils.NetworkToSplit
+	"github.com/docker/docker/libnetwork/ipams/windowsipam"
 )
 
-// InitDockerDefault registers the built-in ipam service with libnetwork
-func InitDockerDefault(ic ipamapi.Callback, l, g interface{}) error {
-	var (
-		ok                bool
-		localDs, globalDs datastore.DataStore
-	)
-
-	if l != nil {
-		if localDs, ok = l.(datastore.DataStore); !ok {
-			return errors.New("incorrect local datastore passed to built-in ipam init")
-		}
-	}
-
-	if g != nil {
-		if globalDs, ok = g.(datastore.DataStore); !ok {
-			return errors.New("incorrect global datastore passed to built-in ipam init")
-		}
-	}
-
-	ipamutils.ConfigLocalScopeDefaultNetworks(nil)
-
-	a, err := ipam.NewAllocator(localDs, globalDs)
-	if err != nil {
-		return err
-	}
-
-	cps := &ipamapi.Capability{RequiresRequestReplay: true}
-
-	return ic.RegisterIpamDriverWithCapabilities(ipamapi.DefaultIPAM, a, cps)
-}
-
 // Init registers the built-in ipam service with libnetwork
 func Init(ic ipamapi.Callback, l, g interface{}) error {
 	initFunc := windowsipam.GetInit(windowsipam.DefaultIPAM)
 
-	err := InitDockerDefault(ic, l, g)
+	err := initBuiltin(ic, l, g)
 	if err != nil {
 		return err
 	}
 
 	return initFunc(ic, l, g)
 }
-
-// SetDefaultIPAddressPool stores default address pool .
-func SetDefaultIPAddressPool(addressPool []*ipamutils.NetworkToSplit) {
-	defaultAddressPool = addressPool
-}
-
-// GetDefaultIPAddressPool returns default address pool .
-func GetDefaultIPAddressPool() []*ipamutils.NetworkToSplit {
-	return defaultAddressPool
-}

libnetwork/ipamutils/utils.go

@@ -33,11 +33,11 @@ type NetworkToSplit struct {
 
 func init() {
 	var err error
-	if PredefinedGlobalScopeDefaultNetworks, err = splitNetworks(globalScopeDefaultNetworks); err != nil {
+	if PredefinedGlobalScopeDefaultNetworks, err = SplitNetworks(globalScopeDefaultNetworks); err != nil {
 		panic("failed to initialize the global scope default address pool: " + err.Error())
 	}
 
-	if PredefinedLocalScopeDefaultNetworks, err = splitNetworks(localScopeDefaultNetworks); err != nil {
+	if PredefinedLocalScopeDefaultNetworks, err = SplitNetworks(localScopeDefaultNetworks); err != nil {
 		panic("failed to initialize the local scope default address pool: " + err.Error())
 	}
 }
@@ -46,7 +46,7 @@ func init() {
 func configDefaultNetworks(defaultAddressPool []*NetworkToSplit, result *[]*net.IPNet) error {
 	mutex.Lock()
 	defer mutex.Unlock()
-	defaultNetworks, err := splitNetworks(defaultAddressPool)
+	defaultNetworks, err := SplitNetworks(defaultAddressPool)
 	if err != nil {
 		return err
 	}
@@ -86,8 +86,8 @@ func ConfigLocalScopeDefaultNetworks(defaultAddressPool []*NetworkToSplit) error
 	return configDefaultNetworks(defaultAddressPool, &PredefinedLocalScopeDefaultNetworks)
 }
 
-// splitNetworks takes a slice of networks, split them accordingly and returns them
-func splitNetworks(list []*NetworkToSplit) ([]*net.IPNet, error) {
+// SplitNetworks takes a slice of networks, split them accordingly and returns them
+func SplitNetworks(list []*NetworkToSplit) ([]*net.IPNet, error) {
 	localPools := make([]*net.IPNet, 0, len(list))
 
 	for _, p := range list {