Merge pull request #585 from aboch/ds2

Default IPAM to handle local ds

libnetwork/ipam/allocator.go

@@ -1,14 +1,11 @@
 package ipam
 
 import (
-	"encoding/json"
 	"fmt"
 	"net"
-	"strings"
 	"sync"
 
 	log "github.com/Sirupsen/logrus"
-
 	"github.com/docker/libkv/store"
 	"github.com/docker/libnetwork/bitseq"
 	"github.com/docker/libnetwork/datastore"
@@ -34,209 +31,86 @@ type Allocator struct {
 	// Predefined pools for default address spaces
 	predefined map[string][]*net.IPNet
 	// Static subnet information
-	subnets map[SubnetKey]*PoolData
+	localSubnets  *PoolsConfig
+	globalSubnets *PoolsConfig
 	// Allocated addresses in each address space's subnet
 	addresses map[SubnetKey]*bitseq.Handle
 	// Datastore
-	store    datastore.DataStore
-	dbIndex  uint64
-	dbExists bool
+	addrSpace2Configs map[string]*PoolsConfig
 	sync.Mutex
 }
 
 // NewAllocator returns an instance of libnetwork ipam
 func NewAllocator(lcDs, glDs datastore.DataStore) (*Allocator, error) {
 	a := &Allocator{}
-	a.subnets = make(map[SubnetKey]*PoolData)
-	a.addresses = make(map[SubnetKey]*bitseq.Handle)
-	a.predefined = make(map[string][]*net.IPNet, 2)
-	a.predefined[localAddressSpace] = initLocalPredefinedPools()
-	a.predefined[globalAddressSpace] = initGlobalPredefinedPools()
-	a.store = glDs
-
-	if a.store == nil {
-		return a, nil
-	}
-
-	// Register for status changes
-	a.watchForChanges()
-
-	// Get the initial subnet configs status from the ds if present.
-	kvPair, err := a.store.KVStore().Get(datastore.Key(a.Key()...))
-	if err != nil {
-		if err != store.ErrKeyNotFound {
-			return nil, fmt.Errorf("failed to retrieve the ipam subnet configs from datastore: %v", err)
-		}
-		return a, nil
-	}
-	a.subnetConfigFromStore(kvPair)
-
-	// Now retrieve the bitmasks for the master pools
-	var inserterList []func() error
-	a.Lock()
-	for k, v := range a.subnets {
-		if v.Range == nil {
-			inserterList = append(inserterList, func() error { return a.insertBitMask(k, v.Pool) })
-		}
-	}
-	a.Unlock()
-
-	// Add the bitmasks, data could come from datastore
-	for _, f := range inserterList {
-		if err := f(); err != nil {
-			return nil, err
-		}
-	}
-
-	return a, nil
-}
-
-func (a *Allocator) subnetConfigFromStore(kvPair *store.KVPair) {
-	a.Lock()
-	if a.dbIndex < kvPair.LastIndex {
-		a.SetValue(kvPair.Value)
-		a.dbIndex = kvPair.LastIndex
-		a.dbExists = true
-	}
-	a.Unlock()
-}
-
-// SubnetKey is the pointer to the configured pools in each address space
-type SubnetKey struct {
-	AddressSpace string
-	Subnet       string
-	ChildSubnet  string
-}
-
-// String returns the string form of the SubnetKey object
-func (s *SubnetKey) String() string {
-	k := fmt.Sprintf("%s/%s", s.AddressSpace, s.Subnet)
-	if s.ChildSubnet != "" {
-		k = fmt.Sprintf("%s/%s", k, s.ChildSubnet)
-	}
-	return k
-}
 
-// FromString populate the SubnetKey object reading it from string
-func (s *SubnetKey) FromString(str string) error {
-	if str == "" || !strings.Contains(str, "/") {
-		return fmt.Errorf("invalid string form for subnetkey: %s", str)
+	a.localSubnets = &PoolsConfig{
+		subnets: map[SubnetKey]*PoolData{},
+		id:      dsConfigKey + "/Pools",
+		scope:   datastore.LocalScope,
+		ds:      lcDs,
+		alloc:   a,
 	}
 
-	p := strings.Split(str, "/")
-	if len(p) != 3 && len(p) != 5 {
-		return fmt.Errorf("invalid string form for subnetkey: %s", str)
-	}
-	s.AddressSpace = p[0]
-	s.Subnet = fmt.Sprintf("%s/%s", p[1], p[2])
-	if len(p) == 5 {
-		s.ChildSubnet = fmt.Sprintf("%s/%s", p[3], p[4])
+	a.globalSubnets = &PoolsConfig{
+		subnets: map[SubnetKey]*PoolData{},
+		id:      dsConfigKey + "/Pools",
+		scope:   datastore.GlobalScope,
+		ds:      glDs,
+		alloc:   a,
 	}
 
-	return nil
-}
-
-// AddressRange specifies first and last ip ordinal which
-// identify a range in a a pool of addresses
-type AddressRange struct {
-	Sub        *net.IPNet
-	Start, End uint32
-}
-
-// String returns the string form of the AddressRange object
-func (r *AddressRange) String() string {
-	return fmt.Sprintf("Sub: %s, range [%d, %d]", r.Sub, r.Start, r.End)
-}
-
-// MarshalJSON returns the JSON encoding of the Range object
-func (r *AddressRange) MarshalJSON() ([]byte, error) {
-	m := map[string]interface{}{
-		"Sub":   r.Sub.String(),
-		"Start": r.Start,
-		"End":   r.End,
+	a.predefined = map[string][]*net.IPNet{
+		localAddressSpace:  initLocalPredefinedPools(),
+		globalAddressSpace: initGlobalPredefinedPools(),
 	}
-	return json.Marshal(m)
-}
 
-// UnmarshalJSON decodes data into the Range object
-func (r *AddressRange) UnmarshalJSON(data []byte) error {
-	m := map[string]interface{}{}
-	err := json.Unmarshal(data, &m)
-	if err != nil {
-		return err
+	a.addrSpace2Configs = map[string]*PoolsConfig{
+		localAddressSpace:  a.localSubnets,
+		globalAddressSpace: a.globalSubnets,
 	}
-	if r.Sub, err = types.ParseCIDR(m["Sub"].(string)); err != nil {
-		return err
-	}
-	r.Start = uint32(m["Start"].(float64))
-	r.End = uint32(m["End"].(float64))
-	return nil
-}
-
-// PoolData contains the configured pool data
-type PoolData struct {
-	ParentKey SubnetKey
-	Pool      *net.IPNet
-	Range     *AddressRange `json:",omitempty"`
-	RefCount  int
-}
 
-// String returns the string form of the PoolData object
-func (p *PoolData) String() string {
-	return fmt.Sprintf("ParentKey: %s, Pool: %s, Range: %s, RefCount: %d",
-		p.ParentKey.String(), p.Pool.String(), p.Range, p.RefCount)
-}
+	a.addresses = make(map[SubnetKey]*bitseq.Handle)
 
-// MarshalJSON returns the JSON encoding of the PoolData object
-func (p *PoolData) MarshalJSON() ([]byte, error) {
-	m := map[string]interface{}{
-		"ParentKey": p.ParentKey,
-		"RefCount":  p.RefCount,
-	}
-	if p.Pool != nil {
-		m["Pool"] = p.Pool.String()
+	cfgs := []struct {
+		cfg *PoolsConfig
+		dsc string
+	}{
+		{a.localSubnets, "local"},
+		{a.globalSubnets, "global"},
 	}
-	if p.Range != nil {
-		m["Range"] = p.Range
-	}
-	return json.Marshal(m)
-}
-
-// UnmarshalJSON decodes data into the PoolData object
-func (p *PoolData) UnmarshalJSON(data []byte) error {
-	var (
-		err error
-		t   struct {
-			ParentKey SubnetKey
-			Pool      string
-			Range     *AddressRange `json:",omitempty"`
-			RefCount  int
+	// Get the initial local/global pools configfrom the datastores
+	var inserterList []func() error
+	for _, e := range cfgs {
+		if e.cfg.ds == nil {
+			continue
 		}
-	)
-
-	if err = json.Unmarshal(data, &t); err != nil {
-		return err
+		if err := e.cfg.watchForChanges(); err != nil {
+			log.Warnf("Error on registering watch for %s datastore: %v", e.dsc, err)
+		}
+		if err := e.cfg.readFromStore(); err != nil && err != store.ErrKeyNotFound {
+			return nil, fmt.Errorf("failed to retrieve the ipam %s pools config from datastore: %v", e.dsc, err)
+		}
+		e.cfg.Lock()
+		for k, v := range e.cfg.subnets {
+			if v.Range == nil {
+				inserterList = append(inserterList, func() error { return a.insertBitMask(e.cfg.ds, k, v.Pool) })
+			}
+		}
+		e.cfg.Unlock()
 	}
-
-	p.ParentKey = t.ParentKey
-	p.Range = t.Range
-	p.RefCount = t.RefCount
-	if t.Pool != "" {
-		if p.Pool, err = types.ParseCIDR(t.Pool); err != nil {
-			return err
+	// Add the bitmasks (data could come from datastore)
+	if inserterList != nil {
+		for _, f := range inserterList {
+			if err := f(); err != nil {
+				return nil, err
+			}
 		}
 	}
 
-	return nil
+	return a, nil
 }
 
-type ipVersion int
-
-const (
-	v4 = 4
-	v6 = 6
-)
-
 // GetDefaultAddressSpaces returns the local and global default address spaces
 func (a *Allocator) GetDefaultAddressSpaces() (string, string, error) {
 	return localAddressSpace, globalAddressSpace, nil
@@ -248,16 +122,22 @@ func (a *Allocator) RequestPool(addressSpace, pool, subPool string, options map[
 	if err != nil {
 		return "", nil, nil, ipamapi.ErrInvalidPool
 	}
+
+	cfg, err := a.getPoolsConfig(addressSpace)
+	if err != nil {
+		return "", nil, nil, err
+	}
+
 retry:
-	insert, err := a.updatePoolDBOnAdd(*k, nw, ipr)
+	insert, err := cfg.updatePoolDBOnAdd(*k, nw, ipr)
 	if err != nil {
 		return "", nil, nil, err
 	}
-	if err := a.writeToStore(); err != nil {
+	if err := cfg.writeToStore(); err != nil {
 		if _, ok := err.(types.RetryError); !ok {
 			return "", nil, nil, types.InternalErrorf("pool configuration failed because of %s", err.Error())
 		}
-		if erru := a.readFromStore(); erru != nil {
+		if erru := cfg.readFromStore(); erru != nil {
 			return "", nil, nil, fmt.Errorf("failed to get updated pool config from datastore (%v) after (%v)", erru, err)
 		}
 		goto retry
@@ -272,16 +152,21 @@ func (a *Allocator) ReleasePool(poolID string) error {
 		return types.BadRequestErrorf("invalid pool id: %s", poolID)
 	}
 
+	cfg, err := a.getPoolsConfig(k.AddressSpace)
+	if err != nil {
+		return err
+	}
+
 retry:
-	remove, err := a.updatePoolDBOnRemoval(k)
+	remove, err := cfg.updatePoolDBOnRemoval(k)
 	if err != nil {
 		return err
 	}
-	if err = a.writeToStore(); err != nil {
+	if err = cfg.writeToStore(); err != nil {
 		if _, ok := err.(types.RetryError); !ok {
 			return types.InternalErrorf("pool (%s) removal failed because of %v", poolID, err)
 		}
-		if erru := a.readFromStore(); erru != nil {
+		if erru := cfg.readFromStore(); erru != nil {
 			return fmt.Errorf("failed to get updated pool config from datastore (%v) after (%v)", erru, err)
 		}
 		goto retry
@@ -290,6 +175,18 @@ retry:
 	return remove()
 }
 
+// Given the address space, returns the local or global PoolConfig based on the
+// address space is local or global. AddressSpace locality is being registered with IPAM out of band.
+func (a *Allocator) getPoolsConfig(addrSpace string) (*PoolsConfig, error) {
+	a.Lock()
+	defer a.Unlock()
+	cfg, ok := a.addrSpace2Configs[addrSpace]
+	if !ok {
+		return nil, types.BadRequestErrorf("cannot find locality of address space: %s", addrSpace)
+	}
+	return cfg, nil
+}
+
 func (a *Allocator) parsePoolRequest(addressSpace, pool, subPool string, v6 bool) (*SubnetKey, *net.IPNet, *net.IPNet, *AddressRange, error) {
 	var (
 		nw, aw *net.IPNet
@@ -327,89 +224,7 @@ func (a *Allocator) parsePoolRequest(addressSpace, pool, subPool string, v6 bool
 	return &SubnetKey{AddressSpace: addressSpace, Subnet: nw.String(), ChildSubnet: subPool}, nw, aw, ipr, nil
 }
 
-func (a *Allocator) updatePoolDBOnAdd(k SubnetKey, nw *net.IPNet, ipr *AddressRange) (func() error, error) {
-	a.Lock()
-	defer a.Unlock()
-
-	// Check if already allocated
-	if p, ok := a.subnets[k]; ok {
-		a.incRefCount(p, 1)
-		return func() error { return nil }, nil
-	}
-
-	// If master pool, check for overlap
-	if ipr == nil {
-		if a.contains(k.AddressSpace, nw) {
-			return nil, ipamapi.ErrPoolOverlap
-		}
-		// This is a new master pool, add it along with corresponding bitmask
-		a.subnets[k] = &PoolData{Pool: nw, RefCount: 1}
-		return func() error { return a.insertBitMask(k, nw) }, nil
-	}
-
-	// This is a new non-master pool
-	p := &PoolData{
-		ParentKey: SubnetKey{AddressSpace: k.AddressSpace, Subnet: k.Subnet},
-		Pool:      nw,
-		Range:     ipr,
-		RefCount:  1,
-	}
-	a.subnets[k] = p
-
-	// Look for parent pool
-	pp, ok := a.subnets[p.ParentKey]
-	if ok {
-		a.incRefCount(pp, 1)
-		return func() error { return nil }, nil
-	}
-
-	// Parent pool does not exist, add it along with corresponding bitmask
-	a.subnets[p.ParentKey] = &PoolData{Pool: nw, RefCount: 1}
-	return func() error { return a.insertBitMask(p.ParentKey, nw) }, nil
-}
-
-func (a *Allocator) updatePoolDBOnRemoval(k SubnetKey) (func() error, error) {
-	a.Lock()
-	defer a.Unlock()
-
-	p, ok := a.subnets[k]
-	if !ok {
-		return nil, ipamapi.ErrBadPool
-	}
-
-	a.incRefCount(p, -1)
-
-	c := p
-	for ok {
-		if c.RefCount == 0 {
-			delete(a.subnets, k)
-			if c.Range == nil {
-				return func() error {
-					bm, err := a.retrieveBitmask(k, c.Pool)
-					if err != nil {
-						return fmt.Errorf("could not find bitmask in datastore for pool %s removal: %v", k.String(), err)
-					}
-					return bm.Destroy()
-				}, nil
-			}
-		}
-		k = c.ParentKey
-		c, ok = a.subnets[k]
-	}
-
-	return func() error { return nil }, nil
-}
-
-func (a *Allocator) incRefCount(p *PoolData, delta int) {
-	c := p
-	ok := true
-	for ok {
-		c.RefCount += delta
-		c, ok = a.subnets[c.ParentKey]
-	}
-}
-
-func (a *Allocator) insertBitMask(key SubnetKey, pool *net.IPNet) error {
+func (a *Allocator) insertBitMask(store datastore.DataStore, key SubnetKey, pool *net.IPNet) error {
 	log.Debugf("Inserting bitmask (%s, %s)", key.String(), pool.String())
 	ipVer := getAddressVersion(pool.IP)
 	ones, bits := pool.Mask.Size()
@@ -421,7 +236,7 @@ func (a *Allocator) insertBitMask(key SubnetKey, pool *net.IPNet) error {
 	}
 
 	// Generate the new address masks. AddressMask content may come from datastore
-	h, err := bitseq.NewHandle(dsDataKey, a.store, key.String(), numAddresses)
+	h, err := bitseq.NewHandle(dsDataKey, store, key.String(), numAddresses)
 	if err != nil {
 		return err
 	}
@@ -434,17 +249,16 @@ func (a *Allocator) insertBitMask(key SubnetKey, pool *net.IPNet) error {
 	a.Lock()
 	a.addresses[key] = h
 	a.Unlock()
-
 	return nil
 }
 
-func (a *Allocator) retrieveBitmask(k SubnetKey, n *net.IPNet) (*bitseq.Handle, error) {
+func (a *Allocator) retrieveBitmask(ds datastore.DataStore, k SubnetKey, n *net.IPNet) (*bitseq.Handle, error) {
 	a.Lock()
 	bm, ok := a.addresses[k]
 	a.Unlock()
 	if !ok {
 		log.Debugf("Retrieving bitmask (%s, %s)", k.String(), n.String())
-		if err := a.insertBitMask(k, n); err != nil {
+		if err := a.insertBitMask(ds, k, n); err != nil {
 			return nil, fmt.Errorf("could not find bitmask in datastore for %s", k.String())
 		}
 		a.Lock()
@@ -475,18 +289,23 @@ func (a *Allocator) getPredefinedPool(as string, ipV6 bool) (*net.IPNet, error)
 		return nil, fmt.Errorf("no default pool availbale for non-default addresss spaces")
 	}
 
+	cfg, err := a.getPoolsConfig(as)
+	if err != nil {
+		return nil, err
+	}
+
 	for _, nw := range a.getPredefineds(as) {
 		if v != getAddressVersion(nw.IP) {
 			continue
 		}
-		a.Lock()
-		_, ok := a.subnets[SubnetKey{AddressSpace: as, Subnet: nw.String()}]
-		a.Unlock()
+		cfg.Lock()
+		_, ok := cfg.subnets[SubnetKey{AddressSpace: as, Subnet: nw.String()}]
+		cfg.Unlock()
 		if ok {
 			continue
 		}
 
-		if !a.contains(as, nw) {
+		if !cfg.contains(as, nw) {
 			if as == localAddressSpace {
 				if err := netutils.CheckRouteOverlaps(nw); err == nil {
 					return nw, nil
@@ -500,38 +319,6 @@ func (a *Allocator) getPredefinedPool(as string, ipV6 bool) (*net.IPNet, error)
 	return nil, types.NotFoundErrorf("could not find an available predefined network")
 }
 
-// Check subnets size. In case configured subnet is v6 and host size is
-// greater than 32 bits, adjust subnet to /96.
-func adjustAndCheckSubnetSize(subnet *net.IPNet) (*net.IPNet, error) {
-	ones, bits := subnet.Mask.Size()
-	if v6 == getAddressVersion(subnet.IP) {
-		if ones < minNetSizeV6 {
-			return nil, ipamapi.ErrInvalidPool
-		}
-		if ones < minNetSizeV6Eff {
-			newMask := net.CIDRMask(minNetSizeV6Eff, bits)
-			return &net.IPNet{IP: subnet.IP, Mask: newMask}, nil
-		}
-	} else {
-		if ones < minNetSize {
-			return nil, ipamapi.ErrInvalidPool
-		}
-	}
-	return subnet, nil
-}
-
-// Checks whether the passed subnet is a superset or subset of any of the subset in the db
-func (a *Allocator) contains(space string, nw *net.IPNet) bool {
-	for k, v := range a.subnets {
-		if space == k.AddressSpace && k.ChildSubnet == "" {
-			if nw.Contains(v.Pool.IP) || v.Pool.Contains(nw.IP) {
-				return true
-			}
-		}
-	}
-	return false
-}
-
 // RequestAddress returns an address from the specified pool ID
 func (a *Allocator) RequestAddress(poolID string, prefAddress net.IP, opts map[string]string) (*net.IPNet, map[string]string, error) {
 	k := SubnetKey{}
@@ -539,26 +326,31 @@ func (a *Allocator) RequestAddress(poolID string, prefAddress net.IP, opts map[s
 		return nil, nil, types.BadRequestErrorf("invalid pool id: %s", poolID)
 	}
 
-	a.Lock()
-	p, ok := a.subnets[k]
+	cfg, err := a.getPoolsConfig(k.AddressSpace)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	cfg.Lock()
+	p, ok := cfg.subnets[k]
 	if !ok {
-		a.Unlock()
+		cfg.Unlock()
 		return nil, nil, types.NotFoundErrorf("cannot find address pool for poolID:%s", poolID)
 	}
 
 	if prefAddress != nil && !p.Pool.Contains(prefAddress) {
-		a.Unlock()
+		cfg.Unlock()
 		return nil, nil, ipamapi.ErrIPOutOfRange
 	}
 
 	c := p
 	for c.Range != nil {
 		k = c.ParentKey
-		c, ok = a.subnets[k]
+		c, ok = cfg.subnets[k]
 	}
-	a.Unlock()
+	cfg.Unlock()
 
-	bm, err := a.retrieveBitmask(k, c.Pool)
+	bm, err := a.retrieveBitmask(cfg.ds, k, c.Pool)
 	if err != nil {
 		return nil, nil, fmt.Errorf("could not find bitmask in datastore for %s on address %v request from pool %s: %v",
 			k.String(), prefAddress, poolID, err)
@@ -578,24 +370,29 @@ func (a *Allocator) ReleaseAddress(poolID string, address net.IP) error {
 		return types.BadRequestErrorf("invalid pool id: %s", poolID)
 	}
 
-	a.Lock()
-	p, ok := a.subnets[k]
+	cfg, err := a.getPoolsConfig(k.AddressSpace)
+	if err != nil {
+		return err
+	}
+
+	cfg.Lock()
+	p, ok := cfg.subnets[k]
 	if !ok {
-		a.Unlock()
+		cfg.Unlock()
 		return ipamapi.ErrBadPool
 	}
 
 	if address == nil || !p.Pool.Contains(address) {
-		a.Unlock()
+		cfg.Unlock()
 		return ipamapi.ErrInvalidRequest
 	}
 
 	c := p
 	for c.Range != nil {
 		k = c.ParentKey
-		c = a.subnets[k]
+		c = cfg.subnets[k]
 	}
-	a.Unlock()
+	cfg.Unlock()
 
 	mask := p.Pool.Mask
 	if p.Range != nil {
@@ -606,7 +403,7 @@ func (a *Allocator) ReleaseAddress(poolID string, address net.IP) error {
 		return fmt.Errorf("failed to release address %s: %v", address.String(), err)
 	}
 
-	bm, err := a.retrieveBitmask(k, c.Pool)
+	bm, err := cfg.alloc.retrieveBitmask(cfg.ds, k, c.Pool)
 	if err != nil {
 		return fmt.Errorf("could not find bitmask in datastore for %s on address %v release from pool %s: %v",
 			k.String(), address, poolID, err)
@@ -652,10 +449,19 @@ func (a *Allocator) DumpDatabase() string {
 	a.Lock()
 	defer a.Unlock()
 
-	s := fmt.Sprintf("\n\nPoolData")
-	for k, config := range a.subnets {
+	s := fmt.Sprintf("\n\nLocal Pool Config")
+	a.localSubnets.Lock()
+	for k, config := range a.localSubnets.subnets {
 		s = fmt.Sprintf("%s%s", s, fmt.Sprintf("\n%v: %v", k, config))
 	}
+	a.localSubnets.Unlock()
+
+	s = fmt.Sprintf("%s\n\nGlobal Pool Config", s)
+	a.globalSubnets.Lock()
+	for k, config := range a.globalSubnets.subnets {
+		s = fmt.Sprintf("%s%s", s, fmt.Sprintf("\n%v: %v", k, config))
+	}
+	a.globalSubnets.Unlock()
 
 	s = fmt.Sprintf("%s\n\nBitmasks", s)
 	for k, bm := range a.addresses {
@@ -663,95 +469,3 @@ func (a *Allocator) DumpDatabase() string {
 	}
 	return s
 }
-
-// It generates the ip address in the passed subnet specified by
-// the passed host address ordinal
-func generateAddress(ordinal uint32, network *net.IPNet) net.IP {
-	var address [16]byte
-
-	// Get network portion of IP
-	if getAddressVersion(network.IP) == v4 {
-		copy(address[:], network.IP.To4())
-	} else {
-		copy(address[:], network.IP)
-	}
-
-	end := len(network.Mask)
-	addIntToIP(address[:end], ordinal)
-
-	return net.IP(address[:end])
-}
-
-func getAddressVersion(ip net.IP) ipVersion {
-	if ip.To4() == nil {
-		return v6
-	}
-	return v4
-}
-
-// Adds the ordinal IP to the current array
-// 192.168.0.0 + 53 => 192.168.53
-func addIntToIP(array []byte, ordinal uint32) {
-	for i := len(array) - 1; i >= 0; i-- {
-		array[i] |= (byte)(ordinal & 0xff)
-		ordinal >>= 8
-	}
-}
-
-// Convert an ordinal to the respective IP address
-func ipToUint32(ip []byte) uint32 {
-	value := uint32(0)
-	for i := 0; i < len(ip); i++ {
-		j := len(ip) - 1 - i
-		value += uint32(ip[i]) << uint(j*8)
-	}
-	return value
-}
-
-func initLocalPredefinedPools() []*net.IPNet {
-	pl := make([]*net.IPNet, 0, 274)
-	mask := []byte{255, 255, 0, 0}
-	for i := 17; i < 32; i++ {
-		pl = append(pl, &net.IPNet{IP: []byte{172, byte(i), 0, 0}, Mask: mask})
-	}
-	for i := 0; i < 256; i++ {
-		pl = append(pl, &net.IPNet{IP: []byte{10, byte(i), 0, 0}, Mask: mask})
-	}
-	mask24 := []byte{255, 255, 255, 0}
-	for i := 42; i < 45; i++ {
-		pl = append(pl, &net.IPNet{IP: []byte{192, 168, byte(i), 0}, Mask: mask24})
-	}
-	return pl
-}
-
-func initGlobalPredefinedPools() []*net.IPNet {
-	pl := make([]*net.IPNet, 0, 256*256)
-	mask := []byte{255, 255, 255, 0}
-	for i := 0; i < 256; i++ {
-		for j := 0; j < 256; j++ {
-			pl = append(pl, &net.IPNet{IP: []byte{10, byte(i), byte(j), 0}, Mask: mask})
-		}
-	}
-	return pl
-}
-
-func getAddressRange(pool string) (*AddressRange, error) {
-	ip, nw, err := net.ParseCIDR(pool)
-	if err != nil {
-		return nil, ipamapi.ErrInvalidSubPool
-	}
-	lIP, e := types.GetHostPartIP(nw.IP, nw.Mask)
-	if e != nil {
-		return nil, fmt.Errorf("failed to compute range's lowest ip address: %v", e)
-	}
-	bIP, e := types.GetBroadcastIP(nw.IP, nw.Mask)
-	if e != nil {
-		return nil, fmt.Errorf("failed to compute range's broadcast ip address: %v", e)
-	}
-	hIP, e := types.GetHostPartIP(bIP, nw.Mask)
-	if e != nil {
-		return nil, fmt.Errorf("failed to compute range's highest ip address: %v", e)
-	}
-	nw.IP = ip
-	return &AddressRange{nw, ipToUint32(types.GetMinimalIP(lIP)), ipToUint32(types.GetMinimalIP(hIP))}, nil
-}

libnetwork/ipam/allocator_test.go

@@ -1,14 +1,15 @@
 package ipam
 
 import (
+	"encoding/json"
 	"fmt"
+	"io/ioutil"
 	"net"
 	"os"
 	"testing"
 	"time"
 
-	"encoding/json"
-
+	"github.com/docker/libkv/store"
 	"github.com/docker/libnetwork/bitseq"
 	"github.com/docker/libnetwork/config"
 	"github.com/docker/libnetwork/datastore"
@@ -18,12 +19,35 @@ import (
 	"github.com/docker/libnetwork/types"
 )
 
-var ds datastore.DataStore
+const (
+	defaultPrefix = "/tmp/libnetwork/test/ipam"
+)
+
+// OptionBoltdbWithRandomDBFile function returns a random dir for local store backend
+func randomLocalStore() (datastore.DataStore, error) {
+	tmp, err := ioutil.TempFile("", "libnetwork-")
+	if err != nil {
+		return nil, fmt.Errorf("Error creating temp file: %v", err)
+	}
+	if err := tmp.Close(); err != nil {
+		return nil, fmt.Errorf("Error closing temp file: %v", err)
+	}
+	return datastore.NewDataStore(&config.DatastoreCfg{
+		Embedded: true,
+		Client: config.DatastoreClientCfg{
+			Provider: "boltdb",
+			Address:  defaultPrefix + tmp.Name(),
+			Config: &store.Config{
+				Bucket:            "libnetwork",
+				ConnectionTimeout: 3 * time.Second,
+			},
+		},
+	})
+}
 
 // enable w/ upper case
-func testMain(m *testing.M) {
+func TestMain(m *testing.M) {
 	var err error
-	ds, err = datastore.NewDataStore(&config.DatastoreCfg{Embedded: false, Client: config.DatastoreClientCfg{Provider: "consul", Address: "127.0.0.1:8500"}})
 	if err != nil {
 		fmt.Println(err)
 	}
@@ -31,16 +55,16 @@ func testMain(m *testing.M) {
 	os.Exit(m.Run())
 }
 
-func getAllocator(t *testing.T, subnet string) (*Allocator, string) {
-	a, err := NewAllocator(nil, ds)
+func getAllocator() (*Allocator, error) {
+	ds, err := randomLocalStore()
 	if err != nil {
-		t.Fatal(err)
+		return nil, err
 	}
-	poolID, _, _, err := a.RequestPool("default", subnet, "", nil, false)
+	a, err := NewAllocator(ds, nil)
 	if err != nil {
-		t.Fatal(err)
+		return nil, err
 	}
-	return a, poolID
+	return a, nil
 }
 
 func TestInt2IP2IntConversion(t *testing.T) {
@@ -70,7 +94,6 @@ func TestGetAddressVersion(t *testing.T) {
 }
 
 func TestKeyString(t *testing.T) {
-
 	k := &SubnetKey{AddressSpace: "default", Subnet: "172.27.0.0/16"}
 	expected := "default/172.27.0.0/16"
 	if expected != k.String() {
@@ -151,8 +174,10 @@ func TestPoolDataMarshal(t *testing.T) {
 }
 
 func TestSubnetsMarshal(t *testing.T) {
-	a, _ := NewAllocator(nil, nil)
-
+	a, err := getAllocator()
+	if err != nil {
+		t.Fatal(err)
+	}
 	pid0, _, _, err := a.RequestPool(localAddressSpace, "192.168.0.0/16", "", nil, false)
 	if err != nil {
 		t.Fatal(err)
@@ -166,9 +191,9 @@ func TestSubnetsMarshal(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	ba := a.Value()
-	a.subnets = make(map[SubnetKey]*PoolData, 0)
-	if err := a.SetValue(ba); err != nil {
+	cfg := a.localSubnets
+	ba := cfg.Value()
+	if err := cfg.SetValue(ba); err != nil {
 		t.Fatal(err)
 	}
 
@@ -192,12 +217,13 @@ func TestSubnetsMarshal(t *testing.T) {
 }
 
 func TestAddSubnets(t *testing.T) {
-	a, err := NewAllocator(nil, nil)
+	a, err := getAllocator()
 	if err != nil {
 		t.Fatal(err)
 	}
+	a.addrSpace2Configs["abc"] = a.addrSpace2Configs[localAddressSpace]
 
-	pid0, _, _, err := a.RequestPool("default", "10.0.0.0/8", "", nil, false)
+	pid0, _, _, err := a.RequestPool(localAddressSpace, "10.0.0.0/8", "", nil, false)
 	if err != nil {
 		t.Fatalf("Unexpected failure in adding subnet")
 	}
@@ -236,22 +262,22 @@ func TestAddSubnets(t *testing.T) {
 		t.Fatalf("returned different pool id for same sub pool requests")
 	}
 
-	pid, _, _, err = a.RequestPool("default", "10.20.2.0/24", "", nil, false)
+	pid, _, _, err = a.RequestPool(localAddressSpace, "10.20.2.0/24", "", nil, false)
 	if err == nil {
 		t.Fatalf("Failed to detect overlapping subnets")
 	}
 
-	_, _, _, err = a.RequestPool("default", "10.128.0.0/9", "", nil, false)
+	_, _, _, err = a.RequestPool(localAddressSpace, "10.128.0.0/9", "", nil, false)
 	if err == nil {
 		t.Fatalf("Failed to detect overlapping subnets")
 	}
 
-	_, _, _, err = a.RequestPool("default", "1003:1:2:3:4:5:6::/112", "", nil, false)
+	_, _, _, err = a.RequestPool(localAddressSpace, "1003:1:2:3:4:5:6::/112", "", nil, false)
 	if err != nil {
 		t.Fatalf("Failed to add v6 subnet: %s", err.Error())
 	}
 
-	_, _, _, err = a.RequestPool("default", "1003:1:2:3::/64", "", nil, false)
+	_, _, _, err = a.RequestPool(localAddressSpace, "1003:1:2:3::/64", "", nil, false)
 	if err == nil {
 		t.Fatalf("Failed to detect overlapping v6 subnet")
 	}
@@ -259,34 +285,35 @@ func TestAddSubnets(t *testing.T) {
 
 func TestAddReleasePoolID(t *testing.T) {
 	var k0, k1, k2 SubnetKey
-	a, err := NewAllocator(nil, nil)
+
+	a, err := getAllocator()
 	if err != nil {
 		t.Fatal(err)
 	}
-
-	pid0, _, _, err := a.RequestPool("default", "10.0.0.0/8", "", nil, false)
+	subnets := a.localSubnets.subnets
+	pid0, _, _, err := a.RequestPool(localAddressSpace, "10.0.0.0/8", "", nil, false)
 	if err != nil {
 		t.Fatalf("Unexpected failure in adding pool")
 	}
 	if err := k0.FromString(pid0); err != nil {
 		t.Fatal(err)
 	}
-	if a.subnets[k0].RefCount != 1 {
-		t.Fatalf("Unexpected ref count for %s: %d", k0, a.subnets[k0].RefCount)
+	if subnets[k0].RefCount != 1 {
+		t.Fatalf("Unexpected ref count for %s: %d", k0, subnets[k0].RefCount)
 	}
 
-	pid1, _, _, err := a.RequestPool("default", "10.0.0.0/8", "10.0.0.0/16", nil, false)
+	pid1, _, _, err := a.RequestPool(localAddressSpace, "10.0.0.0/8", "10.0.0.0/16", nil, false)
 	if err != nil {
 		t.Fatalf("Unexpected failure in adding sub pool")
 	}
 	if err := k1.FromString(pid1); err != nil {
 		t.Fatal(err)
 	}
-	if a.subnets[k1].RefCount != 1 {
-		t.Fatalf("Unexpected ref count for %s: %d", k1, a.subnets[k1].RefCount)
+	if subnets[k1].RefCount != 1 {
+		t.Fatalf("Unexpected ref count for %s: %d", k1, subnets[k1].RefCount)
 	}
 
-	pid2, _, _, err := a.RequestPool("default", "10.0.0.0/8", "10.0.0.0/16", nil, false)
+	pid2, _, _, err := a.RequestPool(localAddressSpace, "10.0.0.0/8", "10.0.0.0/16", nil, false)
 	if err != nil {
 		t.Fatalf("Unexpected failure in adding sub pool")
 	}
@@ -296,63 +323,63 @@ func TestAddReleasePoolID(t *testing.T) {
 	if err := k2.FromString(pid2); err != nil {
 		t.Fatal(err)
 	}
-	if a.subnets[k2].RefCount != 2 {
-		t.Fatalf("Unexpected ref count for %s: %d", k2, a.subnets[k2].RefCount)
+	if subnets[k2].RefCount != 2 {
+		t.Fatalf("Unexpected ref count for %s: %d", k2, subnets[k2].RefCount)
 	}
 
-	if a.subnets[k0].RefCount != 3 {
-		t.Fatalf("Unexpected ref count for %s: %d", k0, a.subnets[k0].RefCount)
+	if subnets[k0].RefCount != 3 {
+		t.Fatalf("Unexpected ref count for %s: %d", k0, subnets[k0].RefCount)
 	}
 
 	if err := a.ReleasePool(pid1); err != nil {
 		t.Fatal(err)
 	}
-	if a.subnets[k0].RefCount != 2 {
-		t.Fatalf("Unexpected ref count for %s: %d", k0, a.subnets[k0].RefCount)
+	if subnets[k0].RefCount != 2 {
+		t.Fatalf("Unexpected ref count for %s: %d", k0, subnets[k0].RefCount)
 	}
 	if err := a.ReleasePool(pid0); err != nil {
 		t.Fatal(err)
 	}
-	if a.subnets[k0].RefCount != 1 {
-		t.Fatalf("Unexpected ref count for %s: %d", k0, a.subnets[k0].RefCount)
+	if subnets[k0].RefCount != 1 {
+		t.Fatalf("Unexpected ref count for %s: %d", k0, subnets[k0].RefCount)
 	}
 
-	pid00, _, _, err := a.RequestPool("default", "10.0.0.0/8", "", nil, false)
+	pid00, _, _, err := a.RequestPool(localAddressSpace, "10.0.0.0/8", "", nil, false)
 	if err != nil {
 		t.Fatalf("Unexpected failure in adding pool")
 	}
 	if pid00 != pid0 {
 		t.Fatalf("main pool should still exist")
 	}
-	if a.subnets[k0].RefCount != 2 {
-		t.Fatalf("Unexpected ref count for %s: %d", k0, a.subnets[k0].RefCount)
+	if subnets[k0].RefCount != 2 {
+		t.Fatalf("Unexpected ref count for %s: %d", k0, subnets[k0].RefCount)
 	}
 
 	if err := a.ReleasePool(pid2); err != nil {
 		t.Fatal(err)
 	}
-	if a.subnets[k0].RefCount != 1 {
-		t.Fatalf("Unexpected ref count for %s: %d", k0, a.subnets[k0].RefCount)
+	if subnets[k0].RefCount != 1 {
+		t.Fatalf("Unexpected ref count for %s: %d", k0, subnets[k0].RefCount)
 	}
 
 	if err := a.ReleasePool(pid00); err != nil {
 		t.Fatal(err)
 	}
-	if bp, ok := a.subnets[k0]; ok {
+	if bp, ok := subnets[k0]; ok {
 		t.Fatalf("Base pool %s is still present: %v", k0, bp)
 	}
 
-	_, _, _, err = a.RequestPool("default", "10.0.0.0/8", "", nil, false)
+	_, _, _, err = a.RequestPool(localAddressSpace, "10.0.0.0/8", "", nil, false)
 	if err != nil {
 		t.Fatalf("Unexpected failure in adding pool")
 	}
-	if a.subnets[k0].RefCount != 1 {
-		t.Fatalf("Unexpected ref count for %s: %d", k0, a.subnets[k0].RefCount)
+	if subnets[k0].RefCount != 1 {
+		t.Fatalf("Unexpected ref count for %s: %d", k0, subnets[k0].RefCount)
 	}
 }
 
 func TestPredefinedPool(t *testing.T) {
-	a, err := NewAllocator(nil, nil)
+	a, err := getAllocator()
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -444,29 +471,31 @@ func TestAdjustAndCheckSubnet(t *testing.T) {
 }
 
 func TestRemoveSubnet(t *testing.T) {
-	a, err := NewAllocator(nil, nil)
+	a, err := getAllocator()
 	if err != nil {
 		t.Fatal(err)
 	}
+	a.addrSpace2Configs["splane"] = a.addrSpace2Configs[localAddressSpace]
 
 	input := []struct {
 		addrSpace string
 		subnet    string
+		v6        bool
 	}{
-		{"default", "192.168.0.0/16"},
-		{"default", "172.17.0.0/16"},
-		{"default", "10.0.0.0/8"},
-		{"default", "2002:1:2:3:4:5:ffff::/112"},
-		{"splane", "172.17.0.0/16"},
-		{"splane", "10.0.0.0/8"},
-		{"splane", "2002:1:2:3:4:5:6::/112"},
-		{"splane", "2002:1:2:3:4:5:ffff::/112"},
+		{localAddressSpace, "192.168.0.0/16", false},
+		{localAddressSpace, "172.17.0.0/16", false},
+		{localAddressSpace, "10.0.0.0/8", false},
+		{localAddressSpace, "2002:1:2:3:4:5:ffff::/112", false},
+		{"splane", "172.17.0.0/16", false},
+		{"splane", "10.0.0.0/8", false},
+		{"splane", "2002:1:2:3:4:5:6::/112", true},
+		{"splane", "2002:1:2:3:4:5:ffff::/112", true},
 	}
 
 	poolIDs := make([]string, len(input))
 
 	for ind, i := range input {
-		if poolIDs[ind], _, _, err = a.RequestPool(i.addrSpace, i.subnet, "", nil, false); err != nil {
+		if poolIDs[ind], _, _, err = a.RequestPool(i.addrSpace, i.subnet, "", nil, i.v6); err != nil {
 			t.Fatalf("Failed to apply input. Can't proceed: %s", err.Error())
 		}
 	}
@@ -479,10 +508,11 @@ func TestRemoveSubnet(t *testing.T) {
 }
 
 func TestGetSameAddress(t *testing.T) {
-	a, err := NewAllocator(nil, nil)
+	a, err := getAllocator()
 	if err != nil {
 		t.Fatal(err)
 	}
+	a.addrSpace2Configs["giallo"] = a.addrSpace2Configs[localAddressSpace]
 
 	pid, _, _, err := a.RequestPool("giallo", "192.168.100.0/24", "", nil, false)
 	if err != nil {
@@ -502,10 +532,11 @@ func TestGetSameAddress(t *testing.T) {
 }
 
 func TestRequestReleaseAddressFromSubPool(t *testing.T) {
-	a, err := NewAllocator(nil, nil)
+	a, err := getAllocator()
 	if err != nil {
 		t.Fatal(err)
 	}
+	a.addrSpace2Configs["rosso"] = a.addrSpace2Configs[localAddressSpace]
 
 	poolID, _, _, err := a.RequestPool("rosso", "172.28.0.0/16", "172.28.30.0/24", nil, false)
 	if err != nil {
@@ -587,11 +618,16 @@ func TestRequestSyntaxCheck(t *testing.T) {
 		pool      = "192.168.0.0/16"
 		subPool   = "192.168.0.0/24"
 		addrSpace = "green"
+		err       error
 	)
 
-	a, _ := NewAllocator(nil, nil)
+	a, err := getAllocator()
+	if err != nil {
+		t.Fatal(err)
+	}
+	a.addrSpace2Configs[addrSpace] = a.addrSpace2Configs[localAddressSpace]
 
-	_, _, _, err := a.RequestPool("", pool, "", nil, false)
+	_, _, _, err = a.RequestPool("", pool, "", nil, false)
 	if err == nil {
 		t.Fatalf("Failed to detect wrong request: empty address space")
 	}
@@ -661,12 +697,14 @@ func TestRequest(t *testing.T) {
 		{"10.0.0.0/8", 256, "10.0.1.0"},
 
 		{"192.168.128.0/18", 4*256 - 1, "192.168.131.255"},
-		{"192.168.240.0/20", 16*256 - 2, "192.168.255.254"},
+		/*
+			{"192.168.240.0/20", 16*256 - 2, "192.168.255.254"},
 
-		{"192.168.0.0/16", 256*256 - 2, "192.168.255.254"},
-		{"10.0.0.0/8", 2 * 256, "10.0.2.0"},
-		{"10.0.0.0/8", 5 * 256, "10.0.5.0"},
-		//{"10.0.0.0/8", 100 * 256 * 254, "10.99.255.254"},
+			{"192.168.0.0/16", 256*256 - 2, "192.168.255.254"},
+			{"10.0.0.0/8", 2 * 256, "10.0.2.0"},
+			{"10.0.0.0/8", 5 * 256, "10.0.5.0"},
+			{"10.0.0.0/8", 100 * 256 * 254, "10.99.255.254"},
+		*/
 	}
 
 	for _, d := range input {
@@ -676,12 +714,19 @@ func TestRequest(t *testing.T) {
 
 func TestRelease(t *testing.T) {
 	var (
-		err    error
-		subnet = "192.168.0.0/16"
+		subnet = "192.168.0.0/23"
 	)
 
-	a, pid := getAllocator(t, subnet)
-	bm := a.addresses[SubnetKey{"default", subnet, ""}]
+	a, err := getAllocator()
+	if err != nil {
+		t.Fatal(err)
+	}
+	pid, _, _, err := a.RequestPool(localAddressSpace, subnet, "", nil, false)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	bm := a.addresses[SubnetKey{localAddressSpace, subnet, ""}]
 
 	// Allocate all addresses
 	for err != ipamapi.ErrNoAvailableIPs {
@@ -711,8 +756,8 @@ func TestRelease(t *testing.T) {
 
 		{"192.168.1.3"},
 
-		{"192.168.255.253"},
-		{"192.168.255.254"},
+		{"192.168.1.253"},
+		{"192.168.1.254"},
 	}
 
 	// One by one, relase the address and request again. We should get the same IP
@@ -773,13 +818,19 @@ func assertGetAddress(t *testing.T, subnet string) {
 
 func assertNRequests(t *testing.T, subnet string, numReq int, lastExpectedIP string) {
 	var (
-		err       error
 		nw        *net.IPNet
 		printTime = false
 	)
 
 	lastIP := net.ParseIP(lastExpectedIP)
-	a, pid := getAllocator(t, subnet)
+	a, err := getAllocator()
+	if err != nil {
+		t.Fatal(err)
+	}
+	pid, _, _, err := a.RequestPool(localAddressSpace, subnet, "", nil, false)
+	if err != nil {
+		t.Fatal(err)
+	}
 
 	i := 0
 	start := time.Now()
@@ -795,31 +846,31 @@ func assertNRequests(t *testing.T, subnet string, numReq int, lastExpectedIP str
 	}
 }
 
-func benchmarkRequest(subnet string) {
-	var err error
-
-	a, _ := NewAllocator(nil, nil)
-	pid, _, _, _ := a.RequestPool("default", subnet, "", nil, false)
-
+func benchmarkRequest(b *testing.B, a *Allocator, subnet string) {
+	pid, _, _, err := a.RequestPool(localAddressSpace, subnet, "", nil, false)
 	for err != ipamapi.ErrNoAvailableIPs {
 		_, _, err = a.RequestAddress(pid, nil, nil)
 	}
 }
 
 func benchMarkRequest(subnet string, b *testing.B) {
+	a, _ := getAllocator()
 	for n := 0; n < b.N; n++ {
-		benchmarkRequest(subnet)
+		benchmarkRequest(b, a, subnet)
 	}
 }
 
 func BenchmarkRequest_24(b *testing.B) {
-	benchmarkRequest("10.0.0.0/24")
+	a, _ := getAllocator()
+	benchmarkRequest(b, a, "10.0.0.0/24")
 }
 
 func BenchmarkRequest_16(b *testing.B) {
-	benchmarkRequest("10.0.0.0/16")
+	a, _ := getAllocator()
+	benchmarkRequest(b, a, "10.0.0.0/16")
 }
 
 func BenchmarkRequest_8(b *testing.B) {
-	benchmarkRequest("10.0.0.0/8")
+	a, _ := getAllocator()
+	benchmarkRequest(b, a, "10.0.0.0/8")
 }

libnetwork/ipam/store.go

@@ -4,95 +4,77 @@ import (
 	"encoding/json"
 
 	log "github.com/Sirupsen/logrus"
+	"github.com/docker/libkv/store"
 	"github.com/docker/libnetwork/datastore"
 	"github.com/docker/libnetwork/types"
 )
 
 // Key provides the Key to be used in KV Store
-func (a *Allocator) Key() []string {
-	a.Lock()
-	defer a.Unlock()
-	return []string{dsConfigKey}
+func (cfg *PoolsConfig) Key() []string {
+	cfg.Lock()
+	defer cfg.Unlock()
+	return []string{cfg.id}
 }
 
 // KeyPrefix returns the immediate parent key that can be used for tree walk
-func (a *Allocator) KeyPrefix() []string {
-	a.Lock()
-	defer a.Unlock()
+func (cfg *PoolsConfig) KeyPrefix() []string {
+	cfg.Lock()
+	defer cfg.Unlock()
 	return []string{dsConfigKey}
 }
 
 // Value marshals the data to be stored in the KV store
-func (a *Allocator) Value() []byte {
-	a.Lock()
-	defer a.Unlock()
-
-	if a.subnets == nil {
-		return []byte{}
-	}
-	m := map[string]interface{}{}
-	for k, v := range a.subnets {
-		m[k.String()] = v
-	}
-
-	b, err := json.Marshal(m)
+func (cfg *PoolsConfig) Value() []byte {
+	b, err := json.Marshal(cfg)
 	if err != nil {
-		log.Warnf("Failed to marshal ipam configured subnets")
+		log.Warnf("Failed to marshal ipam configured pools: %v", err)
 		return nil
 	}
 	return b
 }
 
 // SetValue unmarshalls the data from the KV store.
-func (a *Allocator) SetValue(value []byte) error {
-	var m map[string]*PoolData
-	err := json.Unmarshal(value, &m)
-	if err != nil {
+func (cfg *PoolsConfig) SetValue(value []byte) error {
+	rc := &PoolsConfig{subnets: make(map[SubnetKey]*PoolData)}
+	if err := json.Unmarshal(value, rc); err != nil {
 		return err
 	}
-	for ks, d := range m {
-		k := SubnetKey{}
-		if err := k.FromString(ks); err != nil {
-			return err
-		}
-		a.subnets[k] = d
-	}
+	cfg.subnets = rc.subnets
 	return nil
 }
 
 // Index returns the latest DB Index as seen by this object
-func (a *Allocator) Index() uint64 {
-	a.Lock()
-	defer a.Unlock()
-	return a.dbIndex
+func (cfg *PoolsConfig) Index() uint64 {
+	cfg.Lock()
+	defer cfg.Unlock()
+	return cfg.dbIndex
 }
 
 // SetIndex method allows the datastore to store the latest DB Index into this object
-func (a *Allocator) SetIndex(index uint64) {
-	a.Lock()
-	a.dbIndex = index
-	a.dbExists = true
-	a.Unlock()
+func (cfg *PoolsConfig) SetIndex(index uint64) {
+	cfg.Lock()
+	cfg.dbIndex = index
+	cfg.dbExists = true
+	cfg.Unlock()
 }
 
 // Exists method is true if this object has been stored in the DB.
-func (a *Allocator) Exists() bool {
-	a.Lock()
-	defer a.Unlock()
-	return a.dbExists
+func (cfg *PoolsConfig) Exists() bool {
+	cfg.Lock()
+	defer cfg.Unlock()
+	return cfg.dbExists
 }
 
 // Skip provides a way for a KV Object to avoid persisting it in the KV Store
-func (a *Allocator) Skip() bool {
+func (cfg *PoolsConfig) Skip() bool {
 	return false
 }
 
-func (a *Allocator) watchForChanges() error {
-	if a.store == nil {
+func (cfg *PoolsConfig) watchForChanges() error {
+	if cfg.ds == nil {
 		return nil
 	}
-
-	kvpChan, err := a.store.KVStore().Watch(datastore.Key(a.Key()...), nil)
+	kvpChan, err := cfg.ds.KVStore().Watch(datastore.Key(cfg.Key()...), nil)
 	if err != nil {
 		return err
 	}
@@ -101,7 +83,7 @@ func (a *Allocator) watchForChanges() error {
 			select {
 			case kvPair := <-kvpChan:
 				if kvPair != nil {
-					a.subnetConfigFromStore(kvPair)
+					cfg.readFromKey(kvPair)
 				}
 			}
 		}
@@ -109,50 +91,40 @@ func (a *Allocator) watchForChanges() error {
 	return nil
 }
 
-func (a *Allocator) readFromStore() error {
-	a.Lock()
-	store := a.store
-	a.Unlock()
-
-	if store == nil {
+func (cfg *PoolsConfig) writeToStore() error {
+	if cfg.ds == nil {
 		return nil
 	}
-
-	kvPair, err := a.store.KVStore().Get(datastore.Key(a.Key()...))
-	if err != nil {
-		return err
+	err := cfg.ds.PutObjectAtomic(cfg)
+	if err == datastore.ErrKeyModified {
+		return types.RetryErrorf("failed to perform atomic write (%v). retry might fix the error", err)
 	}
-
-	a.subnetConfigFromStore(kvPair)
-
-	return nil
+	return err
 }
 
-func (a *Allocator) writeToStore() error {
-	a.Lock()
-	store := a.store
-	a.Unlock()
-	if store == nil {
+func (cfg *PoolsConfig) readFromStore() error {
+	if cfg.ds == nil {
 		return nil
 	}
-	err := store.PutObjectAtomic(a)
-	if err == datastore.ErrKeyModified {
-		return types.RetryErrorf("failed to perform atomic write (%v). retry might fix the error", err)
+	return cfg.ds.GetObject(datastore.Key(cfg.Key()...), cfg)
+}
+
+func (cfg *PoolsConfig) readFromKey(kvPair *store.KVPair) {
+	if cfg.dbIndex < kvPair.LastIndex {
+		cfg.SetValue(kvPair.Value)
+		cfg.dbIndex = kvPair.LastIndex
+		cfg.dbExists = true
 	}
-	return err
 }
 
-func (a *Allocator) deleteFromStore() error {
-	a.Lock()
-	store := a.store
-	a.Unlock()
-	if store == nil {
+func (cfg *PoolsConfig) deleteFromStore() error {
+	if cfg.ds == nil {
 		return nil
 	}
-	return store.DeleteObjectAtomic(a)
+	return cfg.ds.DeleteObjectAtomic(cfg)
 }
 
 // DataScope method returns the storage scope of the datastore
-func (a *Allocator) DataScope() datastore.DataScope {
-	return datastore.GlobalScope
+func (cfg *PoolsConfig) DataScope() datastore.DataScope {
+	return cfg.scope
 }

libnetwork/ipam/structures.go

@@ -0,0 +1,302 @@
+package ipam
+
+import (
+	"encoding/json"
+	"fmt"
+	"net"
+	"strings"
+	"sync"
+
+	"github.com/docker/libnetwork/datastore"
+	"github.com/docker/libnetwork/ipamapi"
+	"github.com/docker/libnetwork/types"
+)
+
+// SubnetKey is the pointer to the configured pools in each address space
+type SubnetKey struct {
+	AddressSpace string
+	Subnet       string
+	ChildSubnet  string
+}
+
+// PoolData contains the configured pool data
+type PoolData struct {
+	ParentKey SubnetKey
+	Pool      *net.IPNet
+	Range     *AddressRange `json:",omitempty"`
+	RefCount  int
+}
+
+// PoolsConfig contains the pool configurations
+type PoolsConfig struct {
+	subnets  map[SubnetKey]*PoolData
+	dbIndex  uint64
+	dbExists bool
+	id       string
+	scope    datastore.DataScope
+	ds       datastore.DataStore
+	alloc    *Allocator
+	sync.Mutex
+}
+
+// AddressRange specifies first and last ip ordinal which
+// identify a range in a a pool of addresses
+type AddressRange struct {
+	Sub        *net.IPNet
+	Start, End uint32
+}
+
+// String returns the string form of the AddressRange object
+func (r *AddressRange) String() string {
+	return fmt.Sprintf("Sub: %s, range [%d, %d]", r.Sub, r.Start, r.End)
+}
+
+// MarshalJSON returns the JSON encoding of the Range object
+func (r *AddressRange) MarshalJSON() ([]byte, error) {
+	m := map[string]interface{}{
+		"Sub":   r.Sub.String(),
+		"Start": r.Start,
+		"End":   r.End,
+	}
+	return json.Marshal(m)
+}
+
+// UnmarshalJSON decodes data into the Range object
+func (r *AddressRange) UnmarshalJSON(data []byte) error {
+	m := map[string]interface{}{}
+	err := json.Unmarshal(data, &m)
+	if err != nil {
+		return err
+	}
+	if r.Sub, err = types.ParseCIDR(m["Sub"].(string)); err != nil {
+		return err
+	}
+	r.Start = uint32(m["Start"].(float64))
+	r.End = uint32(m["End"].(float64))
+	return nil
+}
+
+// String returns the string form of the SubnetKey object
+func (s *SubnetKey) String() string {
+	k := fmt.Sprintf("%s/%s", s.AddressSpace, s.Subnet)
+	if s.ChildSubnet != "" {
+		k = fmt.Sprintf("%s/%s", k, s.ChildSubnet)
+	}
+	return k
+}
+
+// FromString populate the SubnetKey object reading it from string
+func (s *SubnetKey) FromString(str string) error {
+	if str == "" || !strings.Contains(str, "/") {
+		return fmt.Errorf("invalid string form for subnetkey: %s", str)
+	}
+
+	p := strings.Split(str, "/")
+	if len(p) != 3 && len(p) != 5 {
+		return fmt.Errorf("invalid string form for subnetkey: %s", str)
+	}
+	s.AddressSpace = p[0]
+	s.Subnet = fmt.Sprintf("%s/%s", p[1], p[2])
+	if len(p) == 5 {
+		s.ChildSubnet = fmt.Sprintf("%s/%s", p[3], p[4])
+	}
+
+	return nil
+}
+
+// String returns the string form of the PoolData object
+func (p *PoolData) String() string {
+	return fmt.Sprintf("ParentKey: %s, Pool: %s, Range: %s, RefCount: %d",
+		p.ParentKey.String(), p.Pool.String(), p.Range, p.RefCount)
+}
+
+// MarshalJSON returns the JSON encoding of the PoolData object
+func (p *PoolData) MarshalJSON() ([]byte, error) {
+	m := map[string]interface{}{
+		"ParentKey": p.ParentKey,
+		"RefCount":  p.RefCount,
+	}
+	if p.Pool != nil {
+		m["Pool"] = p.Pool.String()
+	}
+	if p.Range != nil {
+		m["Range"] = p.Range
+	}
+	return json.Marshal(m)
+}
+
+// UnmarshalJSON decodes data into the PoolData object
+func (p *PoolData) UnmarshalJSON(data []byte) error {
+	var (
+		err error
+		t   struct {
+			ParentKey SubnetKey
+			Pool      string
+			Range     *AddressRange `json:",omitempty"`
+			RefCount  int
+		}
+	)
+
+	if err = json.Unmarshal(data, &t); err != nil {
+		return err
+	}
+
+	p.ParentKey = t.ParentKey
+	p.Range = t.Range
+	p.RefCount = t.RefCount
+	if t.Pool != "" {
+		if p.Pool, err = types.ParseCIDR(t.Pool); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// MarshalJSON returns the JSON encoding of the PoolsConfig object
+func (cfg *PoolsConfig) MarshalJSON() ([]byte, error) {
+	cfg.Lock()
+	defer cfg.Unlock()
+
+	m := map[string]interface{}{
+		"Scope": string(cfg.scope),
+	}
+
+	if cfg.subnets != nil {
+		s := map[string]*PoolData{}
+		for k, v := range cfg.subnets {
+			s[k.String()] = v
+		}
+		m["Subnets"] = s
+	}
+
+	return json.Marshal(m)
+}
+
+// UnmarshalJSON decodes data into the PoolsConfig object
+func (cfg *PoolsConfig) UnmarshalJSON(data []byte) error {
+	cfg.Lock()
+	defer cfg.Unlock()
+
+	m := map[string]interface{}{}
+	err := json.Unmarshal(data, &m)
+	if err != nil {
+		return err
+	}
+
+	cfg.scope = datastore.LocalScope
+	s := m["Scope"].(string)
+	if s == string(datastore.GlobalScope) {
+		cfg.scope = datastore.GlobalScope
+	}
+
+	if v, ok := m["Subnets"]; ok {
+		sb, _ := json.Marshal(v)
+		var s map[string]*PoolData
+		err := json.Unmarshal(sb, &s)
+		if err != nil {
+			return err
+		}
+		for ks, v := range s {
+			k := SubnetKey{}
+			k.FromString(ks)
+			cfg.subnets[k] = v
+		}
+	}
+
+	return nil
+}
+
+func (cfg *PoolsConfig) updatePoolDBOnAdd(k SubnetKey, nw *net.IPNet, ipr *AddressRange) (func() error, error) {
+	cfg.Lock()
+	defer cfg.Unlock()
+
+	// Check if already allocated
+	if p, ok := cfg.subnets[k]; ok {
+		cfg.incRefCount(p, 1)
+		return func() error { return nil }, nil
+	}
+
+	// If master pool, check for overlap
+	if ipr == nil {
+		if cfg.contains(k.AddressSpace, nw) {
+			return nil, ipamapi.ErrPoolOverlap
+		}
+		// This is a new master pool, add it along with corresponding bitmask
+		cfg.subnets[k] = &PoolData{Pool: nw, RefCount: 1}
+		return func() error { return cfg.alloc.insertBitMask(cfg.ds, k, nw) }, nil
+	}
+
+	// This is a new non-master pool
+	p := &PoolData{
+		ParentKey: SubnetKey{AddressSpace: k.AddressSpace, Subnet: k.Subnet},
+		Pool:      nw,
+		Range:     ipr,
+		RefCount:  1,
+	}
+	cfg.subnets[k] = p
+
+	// Look for parent pool
+	pp, ok := cfg.subnets[p.ParentKey]
+	if ok {
+		cfg.incRefCount(pp, 1)
+		return func() error { return nil }, nil
+	}
+
+	// Parent pool does not exist, add it along with corresponding bitmask
+	cfg.subnets[p.ParentKey] = &PoolData{Pool: nw, RefCount: 1}
+	return func() error { return cfg.alloc.insertBitMask(cfg.ds, p.ParentKey, nw) }, nil
+}
+
+func (cfg *PoolsConfig) updatePoolDBOnRemoval(k SubnetKey) (func() error, error) {
+	cfg.Lock()
+	defer cfg.Unlock()
+
+	p, ok := cfg.subnets[k]
+	if !ok {
+		return nil, ipamapi.ErrBadPool
+	}
+
+	cfg.incRefCount(p, -1)
+
+	c := p
+	for ok {
+		if c.RefCount == 0 {
+			delete(cfg.subnets, k)
+			if c.Range == nil {
+				return func() error {
+					bm, err := cfg.alloc.retrieveBitmask(cfg.ds, k, c.Pool)
+					if err != nil {
+						return fmt.Errorf("could not find bitmask in datastore for pool %s removal: %v", k.String(), err)
+					}
+					return bm.Destroy()
+				}, nil
+			}
+		}
+		k = c.ParentKey
+		c, ok = cfg.subnets[k]
+	}
+
+	return func() error { return nil }, nil
+}
+
+func (cfg *PoolsConfig) incRefCount(p *PoolData, delta int) {
+	c := p
+	ok := true
+	for ok {
+		c.RefCount += delta
+		c, ok = cfg.subnets[c.ParentKey]
+	}
+}
+
+// Checks whether the passed subnet is a superset or subset of any of the subset in this config db
+func (cfg *PoolsConfig) contains(space string, nw *net.IPNet) bool {
+	for k, v := range cfg.subnets {
+		if space == k.AddressSpace && k.ChildSubnet == "" {
+			if nw.Contains(v.Pool.IP) || v.Pool.Contains(nw.IP) {
+				return true
+			}
+		}
+	}
+	return false
+}

libnetwork/ipam/utils.go

@@ -0,0 +1,128 @@
+package ipam
+
+import (
+	"fmt"
+	"net"
+
+	"github.com/docker/libnetwork/ipamapi"
+	"github.com/docker/libnetwork/types"
+)
+
+type ipVersion int
+
+const (
+	v4 = 4
+	v6 = 6
+)
+
+func getAddressRange(pool string) (*AddressRange, error) {
+	ip, nw, err := net.ParseCIDR(pool)
+	if err != nil {
+		return nil, ipamapi.ErrInvalidSubPool
+	}
+	lIP, e := types.GetHostPartIP(nw.IP, nw.Mask)
+	if e != nil {
+		return nil, fmt.Errorf("failed to compute range's lowest ip address: %v", e)
+	}
+	bIP, e := types.GetBroadcastIP(nw.IP, nw.Mask)
+	if e != nil {
+		return nil, fmt.Errorf("failed to compute range's broadcast ip address: %v", e)
+	}
+	hIP, e := types.GetHostPartIP(bIP, nw.Mask)
+	if e != nil {
+		return nil, fmt.Errorf("failed to compute range's highest ip address: %v", e)
+	}
+	nw.IP = ip
+	return &AddressRange{nw, ipToUint32(types.GetMinimalIP(lIP)), ipToUint32(types.GetMinimalIP(hIP))}, nil
+}
+
+func initLocalPredefinedPools() []*net.IPNet {
+	pl := make([]*net.IPNet, 0, 274)
+	mask := []byte{255, 255, 0, 0}
+	for i := 17; i < 32; i++ {
+		pl = append(pl, &net.IPNet{IP: []byte{172, byte(i), 0, 0}, Mask: mask})
+	}
+	for i := 0; i < 256; i++ {
+		pl = append(pl, &net.IPNet{IP: []byte{10, byte(i), 0, 0}, Mask: mask})
+	}
+	mask24 := []byte{255, 255, 255, 0}
+	for i := 42; i < 45; i++ {
+		pl = append(pl, &net.IPNet{IP: []byte{192, 168, byte(i), 0}, Mask: mask24})
+	}
+	return pl
+}
+
+func initGlobalPredefinedPools() []*net.IPNet {
+	pl := make([]*net.IPNet, 0, 256*256)
+	mask := []byte{255, 255, 255, 0}
+	for i := 0; i < 256; i++ {
+		for j := 0; j < 256; j++ {
+			pl = append(pl, &net.IPNet{IP: []byte{10, byte(i), byte(j), 0}, Mask: mask})
+		}
+	}
+	return pl
+}
+
+// Check subnets size. In case configured subnet is v6 and host size is
+// greater than 32 bits, adjust subnet to /96.
+func adjustAndCheckSubnetSize(subnet *net.IPNet) (*net.IPNet, error) {
+	ones, bits := subnet.Mask.Size()
+	if v6 == getAddressVersion(subnet.IP) {
+		if ones < minNetSizeV6 {
+			return nil, ipamapi.ErrInvalidPool
+		}
+		if ones < minNetSizeV6Eff {
+			newMask := net.CIDRMask(minNetSizeV6Eff, bits)
+			return &net.IPNet{IP: subnet.IP, Mask: newMask}, nil
+		}
+	} else {
+		if ones < minNetSize {
+			return nil, ipamapi.ErrInvalidPool
+		}
+	}
+	return subnet, nil
+}
+
+// It generates the ip address in the passed subnet specified by
+// the passed host address ordinal
+func generateAddress(ordinal uint32, network *net.IPNet) net.IP {
+	var address [16]byte
+
+	// Get network portion of IP
+	if getAddressVersion(network.IP) == v4 {
+		copy(address[:], network.IP.To4())
+	} else {
+		copy(address[:], network.IP)
+	}
+
+	end := len(network.Mask)
+	addIntToIP(address[:end], ordinal)
+
+	return net.IP(address[:end])
+}
+
+func getAddressVersion(ip net.IP) ipVersion {
+	if ip.To4() == nil {
+		return v6
+	}
+	return v4
+}
+
+// Adds the ordinal IP to the current array
+// 192.168.0.0 + 53 => 192.168.53
+func addIntToIP(array []byte, ordinal uint32) {
+	for i := len(array) - 1; i >= 0; i-- {
+		array[i] |= (byte)(ordinal & 0xff)
+		ordinal >>= 8
+	}
+}
+
+// Convert an ordinal to the respective IP address
+func ipToUint32(ip []byte) uint32 {
+	value := uint32(0)
+	for i := 0; i < len(ip); i++ {
+		j := len(ip) - 1 - i
+		value += uint32(ip[i]) << uint(j*8)
+	}
+	return value
+}