IPAM driver

 - Add IPAM cotract and remote IPAM hooks
 - Add ipam registration in controller
 - Have default IPAM follow ipamapi contract

Signed-off-by: Alessandro Boch <aboch@docker.com>

libnetwork/.gitignore

@@ -29,7 +29,8 @@ cmd/dnet/dnet
 *.tmp
 *.coverprofile
 
-# IDE files
+# IDE files and folders
 .project
+.settings/
 
 libnetwork-build.created

libnetwork/bitseq/sequence.go

@@ -306,8 +306,8 @@ func (h *Handle) validateOrdinal(ordinal uint32) error {
 }
 
 // Destroy removes from the datastore the data belonging to this handle
-func (h *Handle) Destroy() {
-	h.deleteFromStore()
+func (h *Handle) Destroy() error {
+	return h.deleteFromStore()
 }
 
 // ToByteArray converts this handle's data into a byte array

libnetwork/controller.go

@@ -58,6 +58,7 @@ import (
 	"github.com/docker/libnetwork/datastore"
 	"github.com/docker/libnetwork/driverapi"
 	"github.com/docker/libnetwork/hostdiscovery"
+	"github.com/docker/libnetwork/ipamapi"
 	"github.com/docker/libnetwork/osl"
 	"github.com/docker/libnetwork/types"
 )
@@ -116,7 +117,14 @@ type driverData struct {
 	capability driverapi.Capability
 }
 
+type ipamData struct {
+	driver ipamapi.Ipam
+	// default address spaces are provided by ipam driver at registration time
+	defaultLocalAddressSpace, defaultGlobalAddressSpace string
+}
+
 type driverTable map[string]*driverData
+type ipamTable map[string]*ipamData
 type networkTable map[string]*network
 type endpointTable map[string]*endpoint
 type sandboxTable map[string]*sandbox
@@ -125,6 +133,7 @@ type controller struct {
 	id                      string
 	networks                networkTable
 	drivers                 driverTable
+	ipamDrivers             ipamTable
 	sandboxes               sandboxTable
 	cfg                     *config.Config
 	globalStore, localStore datastore.DataStore
@@ -145,11 +154,12 @@ func New(cfgOptions ...config.Option) (NetworkController, error) {
 		cfg.ProcessOptions(cfgOptions...)
 	}
 	c := &controller{
-		id:        stringid.GenerateRandomID(),
-		cfg:       cfg,
-		networks:  networkTable{},
-		sandboxes: sandboxTable{},
-		drivers:   driverTable{}}
+		id:          stringid.GenerateRandomID(),
+		cfg:         cfg,
+		networks:    networkTable{},
+		sandboxes:   sandboxTable{},
+		drivers:     driverTable{},
+		ipamDrivers: ipamTable{}}
 	if err := initDrivers(c); err != nil {
 		return nil, err
 	}
@@ -160,13 +170,26 @@ func New(cfgOptions ...config.Option) (NetworkController, error) {
 			// But it cannot fail creating the Controller
 			log.Debugf("Failed to Initialize Datastore due to %v. Operating in non-clustered mode", err)
 		}
+		if err := c.initLocalStore(); err != nil {
+			log.Debugf("Failed to Initialize LocalDatastore due to %v.", err)
+		}
+	}
+
+	if err := initIpams(c, c.localStore, c.globalStore); err != nil {
+		return nil, err
+	}
+
+	if cfg != nil {
+		if err := c.restoreFromGlobalStore(); err != nil {
+			log.Debugf("Failed to restore from global Datastore due to %v", err)
+		}
 		if err := c.initDiscovery(cfg.Cluster.Watcher); err != nil {
 			// Failing to initalize discovery is a bad situation to be in.
 			// But it cannot fail creating the Controller
 			log.Debugf("Failed to Initialize Discovery : %v", err)
 		}
-		if err := c.initLocalStore(); err != nil {
-			log.Debugf("Failed to Initialize LocalDatastore due to %v.", err)
+		if err := c.restoreFromLocalStore(); err != nil {
+			log.Debugf("Failed to restore from local Datastore due to %v", err)
 		}
 	}
 
@@ -272,6 +295,28 @@ func (c *controller) RegisterDriver(networkType string, driver driverapi.Driver,
 	return nil
 }
 
+func (c *controller) RegisterIpamDriver(name string, driver ipamapi.Ipam) error {
+	if !config.IsValidName(name) {
+		return ErrInvalidName(name)
+	}
+
+	c.Lock()
+	if _, ok := c.ipamDrivers[name]; ok {
+		c.Unlock()
+		return driverapi.ErrActiveRegistration(name)
+	}
+	l, g, err := driver.GetDefaultAddressSpaces()
+	if err != nil {
+		return fmt.Errorf("ipam driver %s failed to return default address spaces: %v", name, err)
+	}
+	c.ipamDrivers[name] = &ipamData{driver: driver, defaultLocalAddressSpace: l, defaultGlobalAddressSpace: g}
+	c.Unlock()
+
+	log.Debugf("Registering ipam provider: %s", name)
+
+	return nil
+}
+
 // NewNetwork creates a new network of the specified network type. The options
 // are network specific and modeled in a generic way.
 func (c *controller) NewNetwork(networkType, name string, options ...NetworkOption) (Network, error) {
@@ -292,6 +337,7 @@ func (c *controller) NewNetwork(networkType, name string, options ...NetworkOpti
 	network := &network{
 		name:        name,
 		networkType: networkType,
+		ipamType:    ipamapi.DefaultIPAM,
 		id:          stringid.GenerateRandomID(),
 		ctrlr:       c,
 		endpoints:   endpointTable{},
@@ -316,7 +362,6 @@ func (c *controller) NewNetwork(networkType, name string, options ...NetworkOpti
 }
 
 func (c *controller) addNetwork(n *network) error {
-
 	c.Lock()
 	// Check if a driver for the specified network type is available
 	dd, ok := c.drivers[n.networkType]
@@ -534,14 +579,39 @@ func (c *controller) loadDriver(networkType string) (*driverData, error) {
 	return dd, nil
 }
 
-func (c *controller) getDriver(networkType string) (*driverData, error) {
+func (c *controller) loadIpamDriver(name string) (*ipamData, error) {
+	if _, err := plugins.Get(name, ipamapi.PluginEndpointType); err != nil {
+		if err == plugins.ErrNotFound {
+			return nil, types.NotFoundErrorf(err.Error())
+		}
+		return nil, err
+	}
 	c.Lock()
-	defer c.Unlock()
-	dd, ok := c.drivers[networkType]
+	id, ok := c.ipamDrivers[name]
+	c.Unlock()
 	if !ok {
-		return nil, types.NotFoundErrorf("driver %s not found", networkType)
+		return nil, ErrInvalidNetworkDriver(name)
 	}
-	return dd, nil
+	return id, nil
+}
+
+func (c *controller) getIPAM(name string) (id *ipamData, err error) {
+	var ok bool
+	c.Lock()
+	id, ok = c.ipamDrivers[name]
+	c.Unlock()
+	if !ok {
+		id, err = c.loadIpamDriver(name)
+	}
+	return id, err
+}
+
+func (c *controller) getIpamDriver(name string) (ipamapi.Ipam, error) {
+	id, err := c.getIPAM(name)
+	if err != nil {
+		return nil, err
+	}
+	return id.driver, nil
 }
 
 func (c *controller) Stop() {

libnetwork/drivers.go

@@ -4,6 +4,9 @@ import (
 	"strings"
 
 	"github.com/docker/libnetwork/driverapi"
+	"github.com/docker/libnetwork/ipamapi"
+	builtinIpam "github.com/docker/libnetwork/ipams/builtin"
+	remoteIpam "github.com/docker/libnetwork/ipams/remote"
 	"github.com/docker/libnetwork/netlabel"
 )
 
@@ -53,3 +56,15 @@ func makeDriverConfig(c *controller, ntype string) map[string]interface{} {
 
 	return config
 }
+
+func initIpams(ic ipamapi.Callback, lDs, gDs interface{}) error {
+	for _, fn := range [](func(ipamapi.Callback, interface{}, interface{}) error){
+		builtinIpam.Init,
+		remoteIpam.Init,
+	} {
+		if err := fn(ic, lDs, gDs); err != nil {
+			return err
+		}
+	}
+	return nil
+}

libnetwork/ipam/allocator.go

@@ -1,57 +1,58 @@
 package ipam
 
 import (
+	"encoding/json"
 	"fmt"
 	"net"
 	"strings"
 	"sync"
 
 	log "github.com/Sirupsen/logrus"
+
 	"github.com/docker/libkv/store"
 	"github.com/docker/libnetwork/bitseq"
 	"github.com/docker/libnetwork/datastore"
+	"github.com/docker/libnetwork/ipamapi"
+	"github.com/docker/libnetwork/netutils"
 	"github.com/docker/libnetwork/types"
 )
 
 const (
+	localAddressSpace  = "LocalDefault"
+	globalAddressSpace = "GlobalDefault"
 	// The biggest configurable host subnets
-	minNetSize   = 8
-	minNetSizeV6 = 64
-	// The effective network size for v6
+	minNetSize      = 8
+	minNetSizeV6    = 64
 	minNetSizeV6Eff = 96
-	// The size of the host subnet used internally, it's the most granular sequence addresses
-	defaultInternalHostSize = 16
 	// datastore keyes for ipam objects
-	dsConfigKey = "ipam-config" // ipam-config/<domain>/<map of subent configs>
-	dsDataKey   = "ipam-data"   // ipam-data/<domain>/<subnet>/<child-sudbnet>/<bitmask>
+	dsConfigKey = "ipam/" + ipamapi.DefaultIPAM + "/config"
+	dsDataKey   = "ipam/" + ipamapi.DefaultIPAM + "/data"
 )
 
 // Allocator provides per address space ipv4/ipv6 book keeping
 type Allocator struct {
-	// The internal subnets host size
-	internalHostSize int
+	// Predefined pools for default address spaces
+	predefined map[string][]*net.IPNet
 	// Static subnet information
-	subnets map[subnetKey]*SubnetInfo
-	// Allocated addresses in each address space's internal subnet
-	addresses map[subnetKey]*bitseq.Handle
+	subnets map[SubnetKey]*PoolData
+	// Allocated addresses in each address space's subnet
+	addresses map[SubnetKey]*bitseq.Handle
 	// Datastore
 	store    datastore.DataStore
-	App      string
-	ID       string
 	dbIndex  uint64
 	dbExists bool
 	sync.Mutex
 }
 
 // NewAllocator returns an instance of libnetwork ipam
-func NewAllocator(ds datastore.DataStore) (*Allocator, error) {
+func NewAllocator(lcDs, glDs datastore.DataStore) (*Allocator, error) {
 	a := &Allocator{}
-	a.subnets = make(map[subnetKey]*SubnetInfo)
-	a.addresses = make(map[subnetKey]*bitseq.Handle)
-	a.internalHostSize = defaultInternalHostSize
-	a.store = ds
-	a.App = "ipam"
-	a.ID = dsConfigKey
+	a.subnets = make(map[SubnetKey]*PoolData)
+	a.addresses = make(map[SubnetKey]*bitseq.Handle)
+	a.predefined = make(map[string][]*net.IPNet, 2)
+	a.predefined[localAddressSpace] = initLocalPredefinedPools()
+	a.predefined[globalAddressSpace] = initGlobalPredefinedPools()
+	a.store = glDs
 
 	if a.store == nil {
 		return a, nil
@@ -70,18 +71,13 @@ func NewAllocator(ds datastore.DataStore) (*Allocator, error) {
 	}
 	a.subnetConfigFromStore(kvPair)
 
-	// Now retrieve the list of small subnets
+	// Now retrieve the bitmasks for the master pools
 	var inserterList []func() error
 	a.Lock()
 	for k, v := range a.subnets {
-		inserterList = append(inserterList,
-			func() error {
-				subnetList, err := getInternalSubnets(v.Subnet, a.internalHostSize)
-				if err != nil {
-					return fmt.Errorf("failed to load address bitmask for configured subnet %s because of %s", v.Subnet.String(), err.Error())
-				}
-				return a.insertAddressMasks(k, subnetList)
-			})
+		if v.Range == nil {
+			inserterList = append(inserterList, func() error { return a.insertBitMask(k, v.Pool) })
+		}
 	}
 	a.Unlock()
 
@@ -98,29 +94,31 @@ func NewAllocator(ds datastore.DataStore) (*Allocator, error) {
 func (a *Allocator) subnetConfigFromStore(kvPair *store.KVPair) {
 	a.Lock()
 	if a.dbIndex < kvPair.LastIndex {
-		a.subnets = byteArrayToSubnets(kvPair.Value)
+		a.SetValue(kvPair.Value)
 		a.dbIndex = kvPair.LastIndex
 		a.dbExists = true
 	}
 	a.Unlock()
 }
 
-// Pointer to the configured subnets in each address space
-type subnetKey struct {
-	addressSpace AddressSpace
-	subnet       string
-	childSubnet  string
+// SubnetKey is the pointer to the configured pools in each address space
+type SubnetKey struct {
+	AddressSpace string
+	Subnet       string
+	ChildSubnet  string
 }
 
-func (s *subnetKey) String() string {
-	k := fmt.Sprintf("%s/%s", s.addressSpace, s.subnet)
-	if s.childSubnet != "" {
-		k = fmt.Sprintf("%s/%s", k, s.childSubnet)
+// String returns the string form of the SubnetKey object
+func (s *SubnetKey) String() string {
+	k := fmt.Sprintf("%s/%s", s.AddressSpace, s.Subnet)
+	if s.ChildSubnet != "" {
+		k = fmt.Sprintf("%s/%s", k, s.ChildSubnet)
 	}
 	return k
 }
 
-func (s *subnetKey) FromString(str string) error {
+// FromString populate the SubnetKey object reading it from string
+func (s *SubnetKey) FromString(str string) error {
 	if str == "" || !strings.Contains(str, "/") {
 		return fmt.Errorf("invalid string form for subnetkey: %s", str)
 	}
@@ -129,26 +127,106 @@ func (s *subnetKey) FromString(str string) error {
 	if len(p) != 3 && len(p) != 5 {
 		return fmt.Errorf("invalid string form for subnetkey: %s", str)
 	}
-	s.addressSpace = AddressSpace(p[0])
-	s.subnet = fmt.Sprintf("%s/%s", p[1], p[2])
+	s.AddressSpace = p[0]
+	s.Subnet = fmt.Sprintf("%s/%s", p[1], p[2])
 	if len(p) == 5 {
-		s.childSubnet = fmt.Sprintf("%s/%s", p[1], p[2])
+		s.ChildSubnet = fmt.Sprintf("%s/%s", p[3], p[4])
 	}
 
 	return nil
 }
 
-func (s *subnetKey) canonicalSubnet() *net.IPNet {
-	if _, sub, err := net.ParseCIDR(s.subnet); err == nil {
-		return sub
+// AddressRange specifies first and last ip ordinal which
+// identify a range in a a pool of addresses
+type AddressRange struct {
+	Sub        *net.IPNet
+	Start, End uint32
+}
+
+// String returns the string form of the AddressRange object
+func (r *AddressRange) String() string {
+	return fmt.Sprintf("Sub: %s, range [%d, %d]", r.Sub, r.Start, r.End)
+}
+
+// MarshalJSON returns the JSON encoding of the Range object
+func (r *AddressRange) MarshalJSON() ([]byte, error) {
+	m := map[string]interface{}{
+		"Sub":   r.Sub.String(),
+		"Start": r.Start,
+		"End":   r.End,
 	}
+	return json.Marshal(m)
+}
+
+// UnmarshalJSON decodes data into the Range object
+func (r *AddressRange) UnmarshalJSON(data []byte) error {
+	m := map[string]interface{}{}
+	err := json.Unmarshal(data, &m)
+	if err != nil {
+		return err
+	}
+	if r.Sub, err = types.ParseCIDR(m["Sub"].(string)); err != nil {
+		return err
+	}
+	r.Start = uint32(m["Start"].(float64))
+	r.End = uint32(m["End"].(float64))
 	return nil
 }
 
-func (s *subnetKey) canonicalChildSubnet() *net.IPNet {
-	if _, sub, err := net.ParseCIDR(s.childSubnet); err == nil {
-		return sub
+// PoolData contains the configured pool data
+type PoolData struct {
+	ParentKey SubnetKey
+	Pool      *net.IPNet
+	Range     *AddressRange `json:",omitempty"`
+	RefCount  int
+}
+
+// String returns the string form of the PoolData object
+func (p *PoolData) String() string {
+	return fmt.Sprintf("ParentKey: %s, Pool: %s, Range: %s, RefCount: %d",
+		p.ParentKey.String(), p.Pool.String(), p.Range, p.RefCount)
+}
+
+// MarshalJSON returns the JSON encoding of the PoolData object
+func (p *PoolData) MarshalJSON() ([]byte, error) {
+	m := map[string]interface{}{
+		"ParentKey": p.ParentKey,
+		"RefCount":  p.RefCount,
+	}
+	if p.Pool != nil {
+		m["Pool"] = p.Pool.String()
+	}
+	if p.Range != nil {
+		m["Range"] = p.Range
+	}
+	return json.Marshal(m)
+}
+
+// UnmarshalJSON decodes data into the PoolData object
+func (p *PoolData) UnmarshalJSON(data []byte) error {
+	var (
+		err error
+		t   struct {
+			ParentKey SubnetKey
+			Pool      string
+			Range     *AddressRange `json:",omitempty"`
+			RefCount  int
+		}
+	)
+
+	if err = json.Unmarshal(data, &t); err != nil {
+		return err
+	}
+
+	p.ParentKey = t.ParentKey
+	p.Range = t.Range
+	p.RefCount = t.RefCount
+	if t.Pool != "" {
+		if p.Pool, err = types.ParseCIDR(t.Pool); err != nil {
+			return err
+		}
 	}
+
 	return nil
 }
 
@@ -159,391 +237,402 @@ const (
 	v6 = 6
 )
 
-/*******************
- * IPAMConf Contract
- ********************/
+// GetDefaultAddressSpaces returns the local and global default address spaces
+func (a *Allocator) GetDefaultAddressSpaces() (string, string, error) {
+	return localAddressSpace, globalAddressSpace, nil
+}
 
-// AddSubnet adds a subnet for the specified address space
-func (a *Allocator) AddSubnet(addrSpace AddressSpace, subnetInfo *SubnetInfo) error {
-	// Sanity check
-	if addrSpace == "" {
-		return ErrInvalidAddressSpace
-	}
-	if subnetInfo == nil || subnetInfo.Subnet == nil {
-		return ErrInvalidSubnet
-	}
-	// Convert to smaller internal subnets (if needed)
-	subnetList, err := getInternalSubnets(subnetInfo.Subnet, a.internalHostSize)
+// RequestPool returns an address pool along with its unique id.
+func (a *Allocator) RequestPool(addressSpace, pool, subPool string, options map[string]string, v6 bool) (string, *net.IPNet, map[string]string, error) {
+	k, nw, aw, ipr, err := a.parsePoolRequest(addressSpace, pool, subPool, v6)
 	if err != nil {
-		return err
+		return "", nil, nil, ipamapi.ErrInvalidPool
 	}
 retry:
-	if a.contains(addrSpace, subnetInfo) {
-		return ErrOverlapSubnet
-	}
-
-	// Store the configured subnet and sync to datatstore
-	key := subnetKey{addrSpace, subnetInfo.Subnet.String(), ""}
-	a.Lock()
-	a.subnets[key] = subnetInfo
-	a.Unlock()
-	err = a.writeToStore()
+	insert, err := a.updatePoolDBOnAdd(*k, nw, ipr)
 	if err != nil {
+		return "", nil, nil, err
+	}
+	if err := a.writeToStore(); err != nil {
 		if _, ok := err.(types.RetryError); !ok {
-			return types.InternalErrorf("subnet configuration failed because of %s", err.Error())
+			return "", nil, nil, types.InternalErrorf("pool configuration failed because of %s", err.Error())
 		}
-		// Update to latest
 		if erru := a.readFromStore(); erru != nil {
-			// Restore and bail out
-			a.Lock()
-			delete(a.addresses, key)
-			a.Unlock()
-			return fmt.Errorf("failed to get updated subnets config from datastore (%v) after (%v)", erru, err)
+			return "", nil, nil, fmt.Errorf("failed to get updated pool config from datastore (%v) after (%v)", erru, err)
 		}
 		goto retry
 	}
-
-	// Insert respective bitmasks for this subnet
-	a.insertAddressMasks(key, subnetList)
-
-	return nil
+	return k.String(), aw, nil, insert()
 }
 
-// Create and insert the internal subnet(s) addresses masks into the address database. Mask data may come from the bitseq datastore.
-func (a *Allocator) insertAddressMasks(parentKey subnetKey, internalSubnetList []*net.IPNet) error {
-	ipVer := getAddressVersion(internalSubnetList[0].IP)
-	num := len(internalSubnetList)
-	ones, bits := internalSubnetList[0].Mask.Size()
-	numAddresses := 1 << uint(bits-ones)
-
-	for i := 0; i < num; i++ {
-		smallKey := subnetKey{parentKey.addressSpace, parentKey.subnet, internalSubnetList[i].String()}
-		limit := uint32(numAddresses)
+// ReleasePool releases the address pool identified by the passed id
+func (a *Allocator) ReleasePool(poolID string) error {
+	k := SubnetKey{}
+	if err := k.FromString(poolID); err != nil {
+		return types.BadRequestErrorf("invalid pool id: %s", poolID)
+	}
 
-		if ipVer == v4 && i == num-1 {
-			// Do not let broadcast address be reserved
-			limit--
+retry:
+	remove, err := a.updatePoolDBOnRemoval(k)
+	if err != nil {
+		return err
+	}
+	if err = a.writeToStore(); err != nil {
+		if _, ok := err.(types.RetryError); !ok {
+			return types.InternalErrorf("pool (%s) removal failed because of %v", poolID, err)
 		}
-
-		// Generate the new address masks. AddressMask content may come from datastore
-		h, err := bitseq.NewHandle(dsDataKey, a.getStore(), smallKey.String(), limit)
-		if err != nil {
-			return err
+		if erru := a.readFromStore(); erru != nil {
+			return fmt.Errorf("failed to get updated pool config from datastore (%v) after (%v)", erru, err)
 		}
+		goto retry
+	}
 
-		if ipVer == v4 && i == 0 {
-			// Do not let network identifier address be reserved
-			h.Set(0)
-		}
+	return remove()
+}
 
-		a.Lock()
-		a.addresses[smallKey] = h
-		a.Unlock()
+func (a *Allocator) parsePoolRequest(addressSpace, pool, subPool string, v6 bool) (*SubnetKey, *net.IPNet, *net.IPNet, *AddressRange, error) {
+	var (
+		nw, aw *net.IPNet
+		ipr    *AddressRange
+		err    error
+	)
+
+	if addressSpace == "" {
+		return nil, nil, nil, nil, ipamapi.ErrInvalidAddressSpace
 	}
-	return nil
-}
 
-// Check subnets size. In case configured subnet is v6 and host size is
-// greater than 32 bits, adjust subnet to /96.
-func adjustAndCheckSubnetSize(subnet *net.IPNet) (*net.IPNet, error) {
-	ones, bits := subnet.Mask.Size()
-	if v6 == getAddressVersion(subnet.IP) {
-		if ones < minNetSizeV6 {
-			return nil, ErrInvalidSubnet
+	if pool == "" && subPool != "" {
+		return nil, nil, nil, nil, ipamapi.ErrInvalidSubPool
+	}
+
+	if pool != "" {
+		if _, nw, err = net.ParseCIDR(pool); err != nil {
+			return nil, nil, nil, nil, ipamapi.ErrInvalidPool
 		}
-		if ones < minNetSizeV6Eff {
-			newMask := net.CIDRMask(minNetSizeV6Eff, bits)
-			return &net.IPNet{IP: subnet.IP, Mask: newMask}, nil
+		if subPool != "" {
+			if ipr, err = getAddressRange(subPool); err != nil {
+				return nil, nil, nil, nil, err
+			}
 		}
 	} else {
-		if ones < minNetSize {
-			return nil, ErrInvalidSubnet
+		if nw, err = a.getPredefinedPool(addressSpace, v6); err != nil {
+			return nil, nil, nil, nil, err
 		}
+
 	}
-	return subnet, nil
+	if aw, err = adjustAndCheckSubnetSize(nw); err != nil {
+		return nil, nil, nil, nil, err
+	}
+
+	return &SubnetKey{AddressSpace: addressSpace, Subnet: nw.String(), ChildSubnet: subPool}, nw, aw, ipr, nil
 }
 
-// Checks whether the passed subnet is a superset or subset of any of the subset in the db
-func (a *Allocator) contains(space AddressSpace, subInfo *SubnetInfo) bool {
+func (a *Allocator) updatePoolDBOnAdd(k SubnetKey, nw *net.IPNet, ipr *AddressRange) (func() error, error) {
 	a.Lock()
 	defer a.Unlock()
-	for k, v := range a.subnets {
-		if space == k.addressSpace {
-			if subInfo.Subnet.Contains(v.Subnet.IP) ||
-				v.Subnet.Contains(subInfo.Subnet.IP) {
-				return true
-			}
+
+	// Check if already allocated
+	if p, ok := a.subnets[k]; ok {
+		a.incRefCount(p, 1)
+		return func() error { return nil }, nil
+	}
+
+	// If master pool, check for overlap
+	if ipr == nil {
+		if a.contains(k.AddressSpace, nw) {
+			return nil, ipamapi.ErrPoolOverlap
 		}
+		// This is a new master pool, add it along with corresponding bitmask
+		a.subnets[k] = &PoolData{Pool: nw, RefCount: 1}
+		return func() error { return a.insertBitMask(k, nw) }, nil
 	}
-	return false
-}
 
-// Splits the passed subnet into N internal subnets with host size equal to internalHostSize.
-// If the subnet's host size is equal to or smaller than internalHostSize, there won't be any
-// split and the return list will contain only the passed subnet.
-func getInternalSubnets(inSubnet *net.IPNet, internalHostSize int) ([]*net.IPNet, error) {
-	var subnetList []*net.IPNet
+	// This is a new non-master pool
+	p := &PoolData{
+		ParentKey: SubnetKey{AddressSpace: k.AddressSpace, Subnet: k.Subnet},
+		Pool:      nw,
+		Range:     ipr,
+		RefCount:  1,
+	}
+	a.subnets[k] = p
 
-	// Sanity check and size adjustment for v6
-	subnet, err := adjustAndCheckSubnetSize(inSubnet)
-	if err != nil {
-		return subnetList, err
+	// Look for parent pool
+	pp, ok := a.subnets[p.ParentKey]
+	if ok {
+		a.incRefCount(pp, 1)
+		return func() error { return nil }, nil
 	}
 
-	// Get network/host subnet information
-	netBits, bits := subnet.Mask.Size()
-	hostBits := bits - netBits
+	// Parent pool does not exist, add it along with corresponding bitmask
+	a.subnets[p.ParentKey] = &PoolData{Pool: nw, RefCount: 1}
+	return func() error { return a.insertBitMask(p.ParentKey, nw) }, nil
+}
 
-	extraBits := hostBits - internalHostSize
-	if extraBits <= 0 {
-		subnetList = make([]*net.IPNet, 1)
-		subnetList[0] = subnet
-	} else {
-		// Split in smaller internal subnets
-		numIntSubs := 1 << uint(extraBits)
-		subnetList = make([]*net.IPNet, numIntSubs)
-
-		// Construct one copy of the internal subnets's mask
-		intNetBits := bits - internalHostSize
-		intMask := net.CIDRMask(intNetBits, bits)
-
-		// Construct the prefix portion for each internal subnet
-		for i := 0; i < numIntSubs; i++ {
-			intIP := make([]byte, len(subnet.IP))
-			copy(intIP, subnet.IP) // IPv6 is too big, just work on the extra portion
-			addIntToIP(intIP, uint32(i<<uint(internalHostSize)))
-			subnetList[i] = &net.IPNet{IP: intIP, Mask: intMask}
+func (a *Allocator) updatePoolDBOnRemoval(k SubnetKey) (func() error, error) {
+	a.Lock()
+	defer a.Unlock()
+
+	p, ok := a.subnets[k]
+	if !ok {
+		return nil, ipamapi.ErrBadPool
+	}
+
+	a.incRefCount(p, -1)
+
+	c := p
+	for ok {
+		if c.RefCount == 0 {
+			delete(a.subnets, k)
+			if c.Range == nil {
+				return func() error {
+					bm, err := a.retrieveBitmask(k, c.Pool)
+					if err != nil {
+						return fmt.Errorf("could not find bitmask in datastore for pool %s removal: %v", k.String(), err)
+					}
+					return bm.Destroy()
+				}, nil
+			}
 		}
+		k = c.ParentKey
+		c, ok = a.subnets[k]
 	}
-	return subnetList, nil
+
+	return func() error { return nil }, nil
 }
 
-// RemoveSubnet removes the subnet from the specified address space
-func (a *Allocator) RemoveSubnet(addrSpace AddressSpace, subnet *net.IPNet) error {
-	if addrSpace == "" {
-		return ErrInvalidAddressSpace
-	}
-	if subnet == nil {
-		return ErrInvalidSubnet
-	}
-retry:
-	// Look for the respective subnet configuration data
-	// Remove it along with the internal subnets
-	subKey := subnetKey{addrSpace, subnet.String(), ""}
-	a.Lock()
-	current, ok := a.subnets[subKey]
-	a.Unlock()
-	if !ok {
-		return ErrSubnetNotFound
+func (a *Allocator) incRefCount(p *PoolData, delta int) {
+	c := p
+	ok := true
+	for ok {
+		c.RefCount += delta
+		c, ok = a.subnets[c.ParentKey]
 	}
+}
 
-	// Remove config and sync to datastore
-	a.Lock()
-	delete(a.subnets, subKey)
-	a.Unlock()
-	err := a.writeToStore()
-	if err != nil {
-		if _, ok := err.(types.RetryError); !ok {
-			return types.InternalErrorf("subnet removal failed because of %s", err.Error())
-		}
-		// Update to latest
-		if erru := a.readFromStore(); erru != nil {
-			// Restore and bail out
-			a.Lock()
-			a.subnets[subKey] = current
-			a.Unlock()
-			return fmt.Errorf("failed to get updated subnets config from datastore (%v) after (%v)", erru, err)
-		}
-		goto retry
+func (a *Allocator) insertBitMask(key SubnetKey, pool *net.IPNet) error {
+	log.Debugf("Inserting bitmask (%s, %s)", key.String(), pool.String())
+	ipVer := getAddressVersion(pool.IP)
+	ones, bits := pool.Mask.Size()
+	numAddresses := uint32(1 << uint(bits-ones))
+
+	if ipVer == v4 {
+		// Do not let broadcast address be reserved
+		numAddresses--
 	}
 
-	// Get the list of smaller internal subnets
-	subnetList, err := getInternalSubnets(subnet, a.internalHostSize)
+	// Generate the new address masks. AddressMask content may come from datastore
+	h, err := bitseq.NewHandle(dsDataKey, a.store, key.String(), numAddresses)
 	if err != nil {
 		return err
 	}
 
-	for _, s := range subnetList {
-		sk := subnetKey{addrSpace, subKey.subnet, s.String()}
-		a.Lock()
-		if bm, ok := a.addresses[sk]; ok {
-			bm.Destroy()
-		}
-		delete(a.addresses, sk)
-		a.Unlock()
+	if ipVer == v4 {
+		// Do not let network identifier address be reserved
+		h.Set(0)
 	}
 
-	return nil
-
-}
+	a.Lock()
+	a.addresses[key] = h
+	a.Unlock()
 
-// AddVendorInfo adds vendor specific data
-func (a *Allocator) AddVendorInfo([]byte) error {
-	// no op for us
 	return nil
 }
 
-/****************
- * IPAM Contract
- ****************/
-
-// Request allows requesting an IPv4 address from the specified address space
-func (a *Allocator) Request(addrSpace AddressSpace, req *AddressRequest) (*AddressResponse, error) {
-	return a.request(addrSpace, req, v4)
-}
-
-// RequestV6 requesting an IPv6 address from the specified address space
-func (a *Allocator) RequestV6(addrSpace AddressSpace, req *AddressRequest) (*AddressResponse, error) {
-	return a.request(addrSpace, req, v6)
+func (a *Allocator) retrieveBitmask(k SubnetKey, n *net.IPNet) (*bitseq.Handle, error) {
+	a.Lock()
+	bm, ok := a.addresses[k]
+	a.Unlock()
+	if !ok {
+		log.Debugf("Retrieving bitmask (%s, %s)", k.String(), n.String())
+		if err := a.insertBitMask(k, n); err != nil {
+			return nil, fmt.Errorf("could not find bitmask in datastore for %s", k.String())
+		}
+		a.Lock()
+		bm = a.addresses[k]
+		a.Unlock()
+	}
+	return bm, nil
 }
 
-func (a *Allocator) request(addrSpace AddressSpace, req *AddressRequest, version ipVersion) (*AddressResponse, error) {
-	// Empty response
-	response := &AddressResponse{}
-
-	// Sanity check
-	if addrSpace == "" {
-		return response, ErrInvalidAddressSpace
+func (a *Allocator) getPredefineds(as string) []*net.IPNet {
+	a.Lock()
+	defer a.Unlock()
+	l := make([]*net.IPNet, 0, len(a.predefined[as]))
+	for _, pool := range a.predefined[as] {
+		l = append(l, pool)
 	}
+	return l
+}
 
-	// Validate request
-	if err := req.Validate(); err != nil {
-		return response, err
+func (a *Allocator) getPredefinedPool(as string, ipV6 bool) (*net.IPNet, error) {
+	var v ipVersion
+	v = v4
+	if ipV6 {
+		v = v6
 	}
 
-	// Check ip version congruence
-	if &req.Subnet != nil && version != getAddressVersion(req.Subnet.IP) {
-		return response, ErrInvalidRequest
+	if as != localAddressSpace && as != globalAddressSpace {
+		return nil, fmt.Errorf("no default pool availbale for non-default addresss spaces")
 	}
 
-	// Look for an address
-	ip, _, err := a.reserveAddress(addrSpace, &req.Subnet, req.Address, version)
-	if err == nil {
-		// Populate response
-		response.Address = ip
+	for _, nw := range a.getPredefineds(as) {
+		if v != getAddressVersion(nw.IP) {
+			continue
+		}
 		a.Lock()
-		response.Subnet = *a.subnets[subnetKey{addrSpace, req.Subnet.String(), ""}]
+		_, ok := a.subnets[SubnetKey{AddressSpace: as, Subnet: nw.String()}]
 		a.Unlock()
+		if ok {
+			continue
+		}
+
+		if !a.contains(as, nw) {
+			if as == localAddressSpace {
+				if err := netutils.CheckRouteOverlaps(nw); err == nil {
+					return nw, nil
+				}
+				continue
+			}
+			return nw, nil
+		}
 	}
 
-	return response, err
+	return nil, types.NotFoundErrorf("could not find an available predefined network")
 }
 
-// Release allows releasing the address from the specified address space
-func (a *Allocator) Release(addrSpace AddressSpace, address net.IP) {
-	var (
-		space *bitseq.Handle
-		sub   *net.IPNet
-	)
+// Check subnets size. In case configured subnet is v6 and host size is
+// greater than 32 bits, adjust subnet to /96.
+func adjustAndCheckSubnetSize(subnet *net.IPNet) (*net.IPNet, error) {
+	ones, bits := subnet.Mask.Size()
+	if v6 == getAddressVersion(subnet.IP) {
+		if ones < minNetSizeV6 {
+			return nil, ipamapi.ErrInvalidPool
+		}
+		if ones < minNetSizeV6Eff {
+			newMask := net.CIDRMask(minNetSizeV6Eff, bits)
+			return &net.IPNet{IP: subnet.IP, Mask: newMask}, nil
+		}
+	} else {
+		if ones < minNetSize {
+			return nil, ipamapi.ErrInvalidPool
+		}
+	}
+	return subnet, nil
+}
+
+// Checks whether the passed subnet is a superset or subset of any of the subset in the db
+func (a *Allocator) contains(space string, nw *net.IPNet) bool {
+	for k, v := range a.subnets {
+		if space == k.AddressSpace && k.ChildSubnet == "" {
+			if nw.Contains(v.Pool.IP) || v.Pool.Contains(nw.IP) {
+				return true
+			}
+		}
+	}
+	return false
+}
 
-	if address == nil {
-		log.Debugf("Requested to remove nil address from address space %s", addrSpace)
-		return
+// RequestAddress returns an address from the specified pool ID
+func (a *Allocator) RequestAddress(poolID string, prefAddress net.IP, opts map[string]string) (*net.IPNet, map[string]string, error) {
+	k := SubnetKey{}
+	if err := k.FromString(poolID); err != nil {
+		return nil, nil, types.BadRequestErrorf("invalid pool id: %s", poolID)
 	}
 
-	ver := getAddressVersion(address)
-	if ver == v4 {
-		address = address.To4()
+	a.Lock()
+	p, ok := a.subnets[k]
+	if !ok {
+		a.Unlock()
+		return nil, nil, types.NotFoundErrorf("cannot find address pool for poolID:%s", poolID)
 	}
 
-	// Find the subnet containing the address
-	for _, subKey := range a.getSubnetList(addrSpace, ver) {
-		sub = subKey.canonicalChildSubnet()
-		if sub.Contains(address) {
-			a.Lock()
-			space = a.addresses[subKey]
-			a.Unlock()
-			break
-		}
+	if prefAddress != nil && !p.Pool.Contains(prefAddress) {
+		a.Unlock()
+		return nil, nil, ipamapi.ErrIPOutOfRange
 	}
-	if space == nil {
-		log.Debugf("Could not find subnet on address space %s containing %s on release", addrSpace, address.String())
-		return
+
+	c := p
+	for c.Range != nil {
+		k = c.ParentKey
+		c, ok = a.subnets[k]
 	}
+	a.Unlock()
 
-	// Retrieve correspondent ordinal in the subnet
-	hostPart, err := types.GetHostPartIP(address, sub.Mask)
+	bm, err := a.retrieveBitmask(k, c.Pool)
 	if err != nil {
-		log.Warnf("Failed to release address %s on address space %s because of internal error: %v", address.String(), addrSpace, err)
-		return
+		return nil, nil, fmt.Errorf("could not find bitmask in datastore for %s on address %v request from pool %s: %v",
+			k.String(), prefAddress, poolID, err)
 	}
-	ordinal := ipToUint32(hostPart)
-
-	// Release it
-	if err := space.Unset(ordinal); err != nil {
-		log.Warnf("Failed to release address %s on address space %s because of internal error: %v", address.String(), addrSpace, err)
+	ip, err := a.getAddress(p.Pool, bm, prefAddress, p.Range)
+	if err != nil {
+		return nil, nil, err
 	}
+
+	return &net.IPNet{IP: ip, Mask: p.Pool.Mask}, nil, nil
 }
 
-func (a *Allocator) reserveAddress(addrSpace AddressSpace, subnet *net.IPNet, prefAddress net.IP, ver ipVersion) (net.IP, *net.IPNet, error) {
-	var keyList []subnetKey
+// ReleaseAddress releases the address from the specified pool ID
+func (a *Allocator) ReleaseAddress(poolID string, address net.IP) error {
+	k := SubnetKey{}
+	if err := k.FromString(poolID); err != nil {
+		return types.BadRequestErrorf("invalid pool id: %s", poolID)
+	}
 
-	// Get the list of pointers to the internal subnets
-	if subnet != nil {
-		// Get the list of smaller internal subnets
-		subnetList, err := getInternalSubnets(subnet, a.internalHostSize)
-		if err != nil {
-			return nil, nil, err
-		}
-		for _, s := range subnetList {
-			keyList = append(keyList, subnetKey{addrSpace, subnet.String(), s.String()})
-		}
-	} else {
-		a.Lock()
-		keyList = a.getSubnetList(addrSpace, ver)
+	a.Lock()
+	p, ok := a.subnets[k]
+	if !ok {
 		a.Unlock()
-	}
-	if len(keyList) == 0 {
-		return nil, nil, ErrNoAvailableSubnet
+		return ipamapi.ErrBadPool
 	}
 
-	for _, key := range keyList {
-		a.Lock()
-		bitmask, ok := a.addresses[key]
+	if address == nil || !p.Pool.Contains(address) {
 		a.Unlock()
-		if !ok {
-			log.Warnf("Did not find a bitmask for subnet key: %s", key.String())
-			continue
-		}
-		address, err := a.getAddress(key.canonicalChildSubnet(), bitmask, prefAddress, ver)
-		if err == nil {
-			return address, subnet, nil
-		}
+		return ipamapi.ErrInvalidRequest
 	}
 
-	return nil, nil, ErrNoAvailableIPs
-}
-
-// Get the list of available internal subnets for the specified address space and the desired ip version
-func (a *Allocator) getSubnetList(addrSpace AddressSpace, ver ipVersion) []subnetKey {
-	var list [1024]subnetKey
-	ind := 0
-	a.Lock()
-	for subKey := range a.addresses {
-		s := subKey.canonicalSubnet()
-		subVer := getAddressVersion(s.IP)
-		if subKey.addressSpace == addrSpace && subVer == ver {
-			list[ind] = subKey
-			ind++
-		}
+	c := p
+	for c.Range != nil {
+		k = c.ParentKey
+		c = a.subnets[k]
 	}
 	a.Unlock()
-	return list[0:ind]
+
+	mask := p.Pool.Mask
+	if p.Range != nil {
+		mask = p.Range.Sub.Mask
+	}
+	h, err := types.GetHostPartIP(address, mask)
+	if err != nil {
+		return fmt.Errorf("failed to release address %s: %v", address.String(), err)
+	}
+
+	bm, err := a.retrieveBitmask(k, c.Pool)
+	if err != nil {
+		return fmt.Errorf("could not find bitmask in datastore for %s on address %v release from pool %s: %v",
+			k.String(), address, poolID, err)
+	}
+	return bm.Unset(ipToUint32(h))
 }
 
-func (a *Allocator) getAddress(subnet *net.IPNet, bitmask *bitseq.Handle, prefAddress net.IP, ver ipVersion) (net.IP, error) {
+func (a *Allocator) getAddress(nw *net.IPNet, bitmask *bitseq.Handle, prefAddress net.IP, ipr *AddressRange) (net.IP, error) {
 	var (
 		ordinal uint32
 		err     error
+		base    *net.IPNet
 	)
 
+	base = types.GetIPNetCopy(nw)
+
 	if bitmask.Unselected() <= 0 {
-		return nil, ErrNoAvailableIPs
+		return nil, ipamapi.ErrNoAvailableIPs
 	}
-	if prefAddress == nil {
+	if ipr == nil && prefAddress == nil {
 		ordinal, err = bitmask.SetAny()
+	} else if ipr != nil {
+		base.IP = ipr.Sub.IP
+		ordinal, err = bitmask.SetAnyInRange(ipr.Start, ipr.End)
 	} else {
-		hostPart, e := types.GetHostPartIP(prefAddress, subnet.Mask)
+		hostPart, e := types.GetHostPartIP(prefAddress, base.Mask)
 		if e != nil {
 			return nil, fmt.Errorf("failed to allocate preferred address %s: %v", prefAddress.String(), e)
 		}
@@ -551,32 +640,28 @@ func (a *Allocator) getAddress(subnet *net.IPNet, bitmask *bitseq.Handle, prefAd
 		err = bitmask.Set(ordinal)
 	}
 	if err != nil {
-		return nil, ErrNoAvailableIPs
+		return nil, ipamapi.ErrNoAvailableIPs
 	}
 
 	// Convert IP ordinal for this subnet into IP address
-	return generateAddress(ordinal, subnet), nil
+	return generateAddress(ordinal, base), nil
 }
 
 // DumpDatabase dumps the internal info
-func (a *Allocator) DumpDatabase() {
+func (a *Allocator) DumpDatabase() string {
 	a.Lock()
 	defer a.Unlock()
+
+	s := fmt.Sprintf("\n\nPoolData")
 	for k, config := range a.subnets {
-		fmt.Printf("\n\n%s:", config.Subnet.String())
-		subnetList, _ := getInternalSubnets(config.Subnet, a.internalHostSize)
-		for _, s := range subnetList {
-			internKey := subnetKey{k.addressSpace, config.Subnet.String(), s.String()}
-			bm := a.addresses[internKey]
-			fmt.Printf("\n\t%s: %s\n\t%d", internKey.childSubnet, bm, bm.Unselected())
-		}
+		s = fmt.Sprintf("%s%s", s, fmt.Sprintf("\n%v: %v", k, config))
 	}
-}
 
-func (a *Allocator) getStore() datastore.DataStore {
-	a.Lock()
-	defer a.Unlock()
-	return a.store
+	s = fmt.Sprintf("%s\n\nBitmasks", s)
+	for k, bm := range a.addresses {
+		s = fmt.Sprintf("%s%s", s, fmt.Sprintf("\n\t%s: %s\n\t%d", k, bm, bm.Unselected()))
+	}
+	return s
 }
 
 // It generates the ip address in the passed subnet specified by
@@ -622,3 +707,51 @@ func ipToUint32(ip []byte) uint32 {
 	}
 	return value
 }
+
+func initLocalPredefinedPools() []*net.IPNet {
+	pl := make([]*net.IPNet, 0, 274)
+	mask := []byte{255, 255, 0, 0}
+	for i := 17; i < 32; i++ {
+		pl = append(pl, &net.IPNet{IP: []byte{172, byte(i), 0, 0}, Mask: mask})
+	}
+	for i := 0; i < 256; i++ {
+		pl = append(pl, &net.IPNet{IP: []byte{10, byte(i), 0, 0}, Mask: mask})
+	}
+	mask24 := []byte{255, 255, 255, 0}
+	for i := 42; i < 45; i++ {
+		pl = append(pl, &net.IPNet{IP: []byte{192, 168, byte(i), 0}, Mask: mask24})
+	}
+	return pl
+}
+
+func initGlobalPredefinedPools() []*net.IPNet {
+	pl := make([]*net.IPNet, 0, 256*256)
+	mask := []byte{255, 255, 255, 0}
+	for i := 0; i < 256; i++ {
+		for j := 0; j < 256; j++ {
+			pl = append(pl, &net.IPNet{IP: []byte{10, byte(i), byte(j), 0}, Mask: mask})
+		}
+	}
+	return pl
+}
+
+func getAddressRange(pool string) (*AddressRange, error) {
+	ip, nw, err := net.ParseCIDR(pool)
+	if err != nil {
+		return nil, ipamapi.ErrInvalidSubPool
+	}
+	lIP, e := types.GetHostPartIP(nw.IP, nw.Mask)
+	if e != nil {
+		return nil, fmt.Errorf("failed to compute range's lowest ip address: %v", e)
+	}
+	bIP, e := types.GetBroadcastIP(nw.IP, nw.Mask)
+	if e != nil {
+		return nil, fmt.Errorf("failed to compute range's broadcast ip address: %v", e)
+	}
+	hIP, e := types.GetHostPartIP(bIP, nw.Mask)
+	if e != nil {
+		return nil, fmt.Errorf("failed to compute range's highest ip address: %v", e)
+	}
+	nw.IP = ip
+	return &AddressRange{nw, ipToUint32(types.GetMinimalIP(lIP)), ipToUint32(types.GetMinimalIP(hIP))}, nil
+}

libnetwork/ipam/allocator_test.go

@@ -7,10 +7,15 @@ import (
 	"testing"
 	"time"
 
+	"encoding/json"
+
 	"github.com/docker/libnetwork/bitseq"
 	"github.com/docker/libnetwork/config"
 	"github.com/docker/libnetwork/datastore"
+	"github.com/docker/libnetwork/ipamapi"
+	"github.com/docker/libnetwork/netutils"
 	_ "github.com/docker/libnetwork/testutils"
+	"github.com/docker/libnetwork/types"
 )
 
 var ds datastore.DataStore
@@ -26,13 +31,16 @@ func testMain(m *testing.M) {
 	os.Exit(m.Run())
 }
 
-func getAllocator(t *testing.T, subnet *net.IPNet) *Allocator {
-	a, err := NewAllocator(ds)
+func getAllocator(t *testing.T, subnet string) (*Allocator, string) {
+	a, err := NewAllocator(nil, ds)
 	if err != nil {
 		t.Fatal(err)
 	}
-	a.AddSubnet("default", &SubnetInfo{Subnet: subnet})
-	return a
+	poolID, _, _, err := a.RequestPool("default", subnet, "", nil, false)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return a, poolID
 }
 
 func TestInt2IP2IntConversion(t *testing.T) {
@@ -63,23 +71,23 @@ func TestGetAddressVersion(t *testing.T) {
 
 func TestKeyString(t *testing.T) {
 
-	k := &subnetKey{addressSpace: "default", subnet: "172.27.0.0/16"}
+	k := &SubnetKey{AddressSpace: "default", Subnet: "172.27.0.0/16"}
 	expected := "default/172.27.0.0/16"
 	if expected != k.String() {
 		t.Fatalf("Unexpected key string: %s", k.String())
 	}
 
-	k2 := &subnetKey{}
+	k2 := &SubnetKey{}
 	err := k2.FromString(expected)
 	if err != nil {
 		t.Fatal(err)
 	}
-	if k2.addressSpace != k.addressSpace || k2.subnet != k.subnet {
-		t.Fatalf("subnetKey.FromString() failed. Expected %v. Got %v", k, k2)
+	if k2.AddressSpace != k.AddressSpace || k2.Subnet != k.Subnet {
+		t.Fatalf("SubnetKey.FromString() failed. Expected %v. Got %v", k, k2)
 	}
 
 	expected = fmt.Sprintf("%s/%s", expected, "172.27.3.0/24")
-	k.childSubnet = "172.27.3.0/24"
+	k.ChildSubnet = "172.27.3.0/24"
 	if expected != k.String() {
 		t.Fatalf("Unexpected key string: %s", k.String())
 	}
@@ -88,64 +96,328 @@ func TestKeyString(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	if k2.addressSpace != k.addressSpace || k2.subnet != k.subnet {
-		t.Fatalf("subnetKey.FromString() failed. Expected %v. Got %v", k, k2)
+	if k2.AddressSpace != k.AddressSpace || k2.Subnet != k.Subnet || k2.ChildSubnet != k.ChildSubnet {
+		t.Fatalf("SubnetKey.FromString() failed. Expected %v. Got %v", k, k2)
+	}
+}
+
+func TestPoolDataMarshal(t *testing.T) {
+	_, nw, err := net.ParseCIDR("172.28.30.1/24")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	p := &PoolData{
+		ParentKey: SubnetKey{AddressSpace: "Blue", Subnet: "172.28.0.0/16"},
+		Pool:      nw,
+		Range:     &AddressRange{Sub: &net.IPNet{IP: net.IP{172, 28, 20, 0}, Mask: net.IPMask{255, 255, 255, 0}}, Start: 0, End: 255},
+		RefCount:  4,
+	}
+
+	ba, err := json.Marshal(p)
+	if err != nil {
+		t.Fatal(err)
+	}
+	var q PoolData
+	err = json.Unmarshal(ba, &q)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if p.ParentKey != q.ParentKey || !types.CompareIPNet(p.Range.Sub, q.Range.Sub) ||
+		p.Range.Start != q.Range.Start || p.Range.End != q.Range.End || p.RefCount != q.RefCount ||
+		!types.CompareIPNet(p.Pool, q.Pool) {
+		t.Fatalf("\n%#v\n%#v", p, &q)
+	}
+
+	p = &PoolData{
+		ParentKey: SubnetKey{AddressSpace: "Blue", Subnet: "172.28.0.0/16"},
+		Pool:      nw,
+		RefCount:  4,
+	}
+
+	ba, err = json.Marshal(p)
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = json.Unmarshal(ba, &q)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if q.Range != nil {
+		t.Fatalf("Unexpected Range")
+	}
+}
+
+func TestSubnetsMarshal(t *testing.T) {
+	a, _ := NewAllocator(nil, nil)
+
+	pid0, _, _, err := a.RequestPool(localAddressSpace, "192.168.0.0/16", "", nil, false)
+	if err != nil {
+		t.Fatal(err)
+	}
+	pid1, _, _, err := a.RequestPool(localAddressSpace, "192.169.0.0/16", "", nil, false)
+	if err != nil {
+		t.Fatal(err)
+	}
+	_, _, err = a.RequestAddress(pid0, nil, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	ba := a.Value()
+	a.subnets = make(map[SubnetKey]*PoolData, 0)
+	if err := a.SetValue(ba); err != nil {
+		t.Fatal(err)
+	}
+
+	expIP := &net.IPNet{IP: net.IP{192, 168, 0, 2}, Mask: net.IPMask{255, 255, 0, 0}}
+	ip, _, err := a.RequestAddress(pid0, nil, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !types.CompareIPNet(expIP, ip) {
+		t.Fatalf("Got unexpected ip after pool config restore: %s", ip)
+	}
+
+	expIP = &net.IPNet{IP: net.IP{192, 169, 0, 1}, Mask: net.IPMask{255, 255, 0, 0}}
+	ip, _, err = a.RequestAddress(pid1, nil, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !types.CompareIPNet(expIP, ip) {
+		t.Fatalf("Got unexpected ip after pool config restore: %s", ip)
 	}
 }
 
 func TestAddSubnets(t *testing.T) {
-	a, err := NewAllocator(nil)
+	a, err := NewAllocator(nil, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	_, sub0, _ := net.ParseCIDR("10.0.0.0/8")
-	err = a.AddSubnet("default", &SubnetInfo{Subnet: sub0})
+	pid0, _, _, err := a.RequestPool("default", "10.0.0.0/8", "", nil, false)
 	if err != nil {
-		t.Fatalf("Unexpected failure in adding subent")
+		t.Fatalf("Unexpected failure in adding subnet")
 	}
 
-	err = a.AddSubnet("abc", &SubnetInfo{Subnet: sub0})
+	pid1, _, _, err := a.RequestPool("abc", "10.0.0.0/8", "", nil, false)
 	if err != nil {
-		t.Fatalf("Unexpected failure in adding overlapping subents to different address spaces")
+		t.Fatalf("Unexpected failure in adding overlapping subnets to different address spaces: %v", err)
 	}
 
-	err = a.AddSubnet("abc", &SubnetInfo{Subnet: sub0})
-	if err == nil {
-		t.Fatalf("Failed to detect overlapping subnets: %s and %s", sub0, sub0)
+	if pid0 == pid1 {
+		t.Fatalf("returned same pool id for same subnets in different namespaces")
 	}
 
-	_, sub1, _ := net.ParseCIDR("10.20.2.0/24")
-	err = a.AddSubnet("default", &SubnetInfo{Subnet: sub1})
-	if err == nil {
-		t.Fatalf("Failed to detect overlapping subnets: %s and %s", sub0, sub1)
+	pid, _, _, err := a.RequestPool("abc", "10.0.0.0/8", "", nil, false)
+	if err != nil {
+		t.Fatalf("Unexpected failure requesting existing subnet: %v", err)
+	}
+	if pid != pid1 {
+		t.Fatalf("returned different pool id for same subnet requests")
 	}
 
-	_, sub2, _ := net.ParseCIDR("10.128.0.0/9")
-	err = a.AddSubnet("default", &SubnetInfo{Subnet: sub2})
+	_, _, _, err = a.RequestPool("abc", "10.128.0.0/9", "", nil, false)
 	if err == nil {
-		t.Fatalf("Failed to detect overlapping subnets: %s and %s", sub1, sub2)
+		t.Fatalf("Expected failure on adding overlapping base subnet")
 	}
 
-	_, sub6, err := net.ParseCIDR("1003:1:2:3:4:5:6::/112")
+	pid2, _, _, err := a.RequestPool("abc", "10.0.0.0/8", "10.128.0.0/9", nil, false)
 	if err != nil {
-		t.Fatalf("Wrong input, Can't proceed: %s", err.Error())
+		t.Fatalf("Unexpected failure on adding sub pool: %v", err)
 	}
-	err = a.AddSubnet("default", &SubnetInfo{Subnet: sub6})
+	pid3, _, _, err := a.RequestPool("abc", "10.0.0.0/8", "10.128.0.0/9", nil, false)
 	if err != nil {
-		t.Fatalf("Failed to add v6 subnet: %s", err.Error())
+		t.Fatalf("Unexpected failure on adding overlapping sub pool: %v", err)
+	}
+	if pid2 != pid3 {
+		t.Fatalf("returned different pool id for same sub pool requests")
+	}
+
+	pid, _, _, err = a.RequestPool("default", "10.20.2.0/24", "", nil, false)
+	if err == nil {
+		t.Fatalf("Failed to detect overlapping subnets")
+	}
+
+	_, _, _, err = a.RequestPool("default", "10.128.0.0/9", "", nil, false)
+	if err == nil {
+		t.Fatalf("Failed to detect overlapping subnets")
 	}
 
-	_, sub6, err = net.ParseCIDR("1003:1:2:3::/64")
+	_, _, _, err = a.RequestPool("default", "1003:1:2:3:4:5:6::/112", "", nil, false)
 	if err != nil {
-		t.Fatalf("Wrong input, Can't proceed: %s", err.Error())
+		t.Fatalf("Failed to add v6 subnet: %s", err.Error())
 	}
-	err = a.AddSubnet("default", &SubnetInfo{Subnet: sub6})
+
+	_, _, _, err = a.RequestPool("default", "1003:1:2:3::/64", "", nil, false)
 	if err == nil {
 		t.Fatalf("Failed to detect overlapping v6 subnet")
 	}
 }
 
+func TestAddReleasePoolID(t *testing.T) {
+	var k0, k1, k2 SubnetKey
+	a, err := NewAllocator(nil, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	pid0, _, _, err := a.RequestPool("default", "10.0.0.0/8", "", nil, false)
+	if err != nil {
+		t.Fatalf("Unexpected failure in adding pool")
+	}
+	if err := k0.FromString(pid0); err != nil {
+		t.Fatal(err)
+	}
+	if a.subnets[k0].RefCount != 1 {
+		t.Fatalf("Unexpected ref count for %s: %d", k0, a.subnets[k0].RefCount)
+	}
+
+	pid1, _, _, err := a.RequestPool("default", "10.0.0.0/8", "10.0.0.0/16", nil, false)
+	if err != nil {
+		t.Fatalf("Unexpected failure in adding sub pool")
+	}
+	if err := k1.FromString(pid1); err != nil {
+		t.Fatal(err)
+	}
+	if a.subnets[k1].RefCount != 1 {
+		t.Fatalf("Unexpected ref count for %s: %d", k1, a.subnets[k1].RefCount)
+	}
+
+	pid2, _, _, err := a.RequestPool("default", "10.0.0.0/8", "10.0.0.0/16", nil, false)
+	if err != nil {
+		t.Fatalf("Unexpected failure in adding sub pool")
+	}
+	if pid0 == pid1 || pid0 == pid2 || pid1 != pid2 {
+		t.Fatalf("Incorrect poolIDs returned %s, %s, %s", pid0, pid1, pid2)
+	}
+	if err := k2.FromString(pid2); err != nil {
+		t.Fatal(err)
+	}
+	if a.subnets[k2].RefCount != 2 {
+		t.Fatalf("Unexpected ref count for %s: %d", k2, a.subnets[k2].RefCount)
+	}
+
+	if a.subnets[k0].RefCount != 3 {
+		t.Fatalf("Unexpected ref count for %s: %d", k0, a.subnets[k0].RefCount)
+	}
+
+	if err := a.ReleasePool(pid1); err != nil {
+		t.Fatal(err)
+	}
+	if a.subnets[k0].RefCount != 2 {
+		t.Fatalf("Unexpected ref count for %s: %d", k0, a.subnets[k0].RefCount)
+	}
+	if err := a.ReleasePool(pid0); err != nil {
+		t.Fatal(err)
+	}
+	if a.subnets[k0].RefCount != 1 {
+		t.Fatalf("Unexpected ref count for %s: %d", k0, a.subnets[k0].RefCount)
+	}
+
+	pid00, _, _, err := a.RequestPool("default", "10.0.0.0/8", "", nil, false)
+	if err != nil {
+		t.Fatalf("Unexpected failure in adding pool")
+	}
+	if pid00 != pid0 {
+		t.Fatalf("main pool should still exist")
+	}
+	if a.subnets[k0].RefCount != 2 {
+		t.Fatalf("Unexpected ref count for %s: %d", k0, a.subnets[k0].RefCount)
+	}
+
+	if err := a.ReleasePool(pid2); err != nil {
+		t.Fatal(err)
+	}
+	if a.subnets[k0].RefCount != 1 {
+		t.Fatalf("Unexpected ref count for %s: %d", k0, a.subnets[k0].RefCount)
+	}
+
+	if err := a.ReleasePool(pid00); err != nil {
+		t.Fatal(err)
+	}
+	if bp, ok := a.subnets[k0]; ok {
+		t.Fatalf("Base pool %s is still present: %v", k0, bp)
+	}
+
+	_, _, _, err = a.RequestPool("default", "10.0.0.0/8", "", nil, false)
+	if err != nil {
+		t.Fatalf("Unexpected failure in adding pool")
+	}
+	if a.subnets[k0].RefCount != 1 {
+		t.Fatalf("Unexpected ref count for %s: %d", k0, a.subnets[k0].RefCount)
+	}
+}
+
+func TestPredefinedPool(t *testing.T) {
+	a, err := NewAllocator(nil, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if _, err := a.getPredefinedPool("blue", false); err == nil {
+		t.Fatalf("Expected failure for non default addr space")
+	}
+
+	i, available, err := getFirstAvailablePool(a, localAddressSpace, 2)
+	if err != nil {
+		t.Skip(err)
+	}
+
+	pid, _, _, err := a.RequestPool(localAddressSpace, available.String(), "", nil, false)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	nw, err := a.getPredefinedPool(localAddressSpace, false)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if nw != a.predefined[localAddressSpace][i+1] {
+		t.Fatalf("Unexpected default network returned: %s", nw)
+	}
+
+	if err := a.ReleasePool(pid); err != nil {
+		t.Fatal(err)
+	}
+
+	nw, err = a.getPredefinedPool(localAddressSpace, false)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if nw != a.predefined[localAddressSpace][i] {
+		t.Fatalf("Unexpected default network returned: %s", nw)
+	}
+
+	i, available, err = getFirstAvailablePool(a, globalAddressSpace, 2)
+	if err != nil {
+		t.Skip(err)
+	}
+	nw, err = a.getPredefinedPool(globalAddressSpace, false)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if nw != available {
+		t.Fatalf("Unexpected default network returned: %s", nw)
+	}
+}
+
+func getFirstAvailablePool(a *Allocator, as string, atLeast int) (int, *net.IPNet, error) {
+	i := 0
+	for i < len(a.predefined[as])-1 {
+		if err := netutils.CheckRouteOverlaps(a.predefined[as][i]); err == nil {
+			break
+		}
+		i++
+	}
+	if i > len(a.predefined[as])-1-atLeast {
+		return 0, nil, fmt.Errorf("Not enough non-overlapping networks to run the test")
+	}
+	return i, a.predefined[as][i], nil
+}
+
 func TestAdjustAndCheckSubnet(t *testing.T) {
 	_, sub6, _ := net.ParseCIDR("1003:1:2:300::/63")
 	_, err := adjustAndCheckSubnetSize(sub6)
@@ -172,13 +444,13 @@ func TestAdjustAndCheckSubnet(t *testing.T) {
 }
 
 func TestRemoveSubnet(t *testing.T) {
-	a, err := NewAllocator(nil)
+	a, err := NewAllocator(nil, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
 
 	input := []struct {
-		addrSpace AddressSpace
+		addrSpace string
 		subnet    string
 	}{
 		{"default", "192.168.0.0/16"},
@@ -191,119 +463,116 @@ func TestRemoveSubnet(t *testing.T) {
 		{"splane", "2002:1:2:3:4:5:ffff::/112"},
 	}
 
-	for _, i := range input {
-		_, sub, err := net.ParseCIDR(i.subnet)
-		if err != nil {
-			t.Fatalf("Wrong input, Can't proceed: %s", err.Error())
-		}
-		err = a.AddSubnet(i.addrSpace, &SubnetInfo{Subnet: sub})
-		if err != nil {
+	poolIDs := make([]string, len(input))
+
+	for ind, i := range input {
+		if poolIDs[ind], _, _, err = a.RequestPool(i.addrSpace, i.subnet, "", nil, false); err != nil {
 			t.Fatalf("Failed to apply input. Can't proceed: %s", err.Error())
 		}
 	}
 
-	_, sub, _ := net.ParseCIDR("172.17.0.0/16")
-	a.RemoveSubnet("default", sub)
-	if len(a.subnets) != 7 {
-		t.Fatalf("Failed to remove subnet info")
-	}
-	list := a.getSubnetList("default", v4)
-	if len(list) != 257 {
-		t.Fatalf("Failed to effectively remove subnet address space")
+	for ind, id := range poolIDs {
+		if err := a.ReleasePool(id); err != nil {
+			t.Fatalf("Failed to release poolID %s (%d)", id, ind)
+		}
 	}
+}
 
-	_, sub, _ = net.ParseCIDR("2002:1:2:3:4:5:ffff::/112")
-	a.RemoveSubnet("default", sub)
-	if len(a.subnets) != 6 {
-		t.Fatalf("Failed to remove subnet info")
-	}
-	list = a.getSubnetList("default", v6)
-	if len(list) != 0 {
-		t.Fatalf("Failed to effectively remove subnet address space")
+func TestGetSameAddress(t *testing.T) {
+	a, err := NewAllocator(nil, nil)
+	if err != nil {
+		t.Fatal(err)
 	}
 
-	_, sub, _ = net.ParseCIDR("2002:1:2:3:4:5:6::/112")
-	a.RemoveSubnet("splane", sub)
-	if len(a.subnets) != 5 {
-		t.Fatalf("Failed to remove subnet info")
-	}
-	list = a.getSubnetList("splane", v6)
-	if len(list) != 1 {
-		t.Fatalf("Failed to effectively remove subnet address space")
+	pid, _, _, err := a.RequestPool("giallo", "192.168.100.0/24", "", nil, false)
+	if err != nil {
+		t.Fatal(err)
 	}
-}
 
-func TestGetInternalSubnets(t *testing.T) {
-	// This function tests the splitting of a parent subnet in small host subnets.
-	// The splitting is controlled by the max host size, which is the first parameter
-	// passed to the function. It basically says if the parent subnet host size is
-	// greater than the max host size, split the parent subnet into N internal small
-	// subnets with host size = max host size to cover the same address space.
-
-	input := []struct {
-		internalHostSize int
-		parentSubnet     string
-		firstIntSubnet   string
-		lastIntSubnet    string
-	}{
-		// Test 8 bits prefix network
-		{24, "10.0.0.0/8", "10.0.0.0/8", "10.0.0.0/8"},
-		{16, "10.0.0.0/8", "10.0.0.0/16", "10.255.0.0/16"},
-		{8, "10.0.0.0/8", "10.0.0.0/24", "10.255.255.0/24"},
-		// Test 16 bits prefix network
-		{16, "192.168.0.0/16", "192.168.0.0/16", "192.168.0.0/16"},
-		{8, "192.168.0.0/16", "192.168.0.0/24", "192.168.255.0/24"},
-		// Test 24 bits prefix network
-		{16, "192.168.57.0/24", "192.168.57.0/24", "192.168.57.0/24"},
-		{8, "192.168.57.0/24", "192.168.57.0/24", "192.168.57.0/24"},
-		// Test non byte multiple host size
-		{24, "10.0.0.0/8", "10.0.0.0/8", "10.0.0.0/8"},
-		{20, "10.0.0.0/12", "10.0.0.0/12", "10.0.0.0/12"},
-		{20, "10.128.0.0/12", "10.128.0.0/12", "10.128.0.0/12"},
-		{12, "10.16.0.0/16", "10.16.0.0/20", "10.16.240.0/20"},
-		{13, "10.0.0.0/8", "10.0.0.0/19", "10.255.224.0/19"},
-		{15, "10.0.0.0/8", "10.0.0.0/17", "10.255.128.0/17"},
-		// Test v6 network
-		{16, "2002:1:2:3:4:5:6000::/110", "2002:1:2:3:4:5:6000:0/112", "2002:1:2:3:4:5:6003:0/112"},
-		{16, "2002:1:2:3:4:5:ff00::/104", "2002:1:2:3:4:5:ff00:0/112", "2002:1:2:3:4:5:ffff:0/112"},
-		{12, "2002:1:2:3:4:5:ffff::/112", "2002:1:2:3:4:5:ffff:0/116", "2002:1:2:3:4:5:ffff:f000/116"},
-		{11, "2002:1:2:3:4:5:ffff::/112", "2002:1:2:3:4:5:ffff:0/117", "2002:1:2:3:4:5:ffff:f800/117"},
+	ip := net.ParseIP("192.168.100.250")
+	_, _, err = a.RequestAddress(pid, ip, nil)
+	if err != nil {
+		t.Fatal(err)
 	}
 
-	for _, d := range input {
-		assertInternalSubnet(t, d.internalHostSize, d.parentSubnet, d.firstIntSubnet, d.lastIntSubnet)
+	_, _, err = a.RequestAddress(pid, ip, nil)
+	if err == nil {
+		t.Fatal(err)
 	}
 }
 
-func TestGetSameAddress(t *testing.T) {
-	a, err := NewAllocator(nil)
+func TestRequestReleaseAddressFromSubPool(t *testing.T) {
+	a, err := NewAllocator(nil, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	addSpace := AddressSpace("giallo")
-	_, subnet, _ := net.ParseCIDR("192.168.100.0/24")
-	if err := a.AddSubnet(addSpace, &SubnetInfo{Subnet: subnet}); err != nil {
+	poolID, _, _, err := a.RequestPool("rosso", "172.28.0.0/16", "172.28.30.0/24", nil, false)
+	if err != nil {
 		t.Fatal(err)
 	}
 
-	ip := net.ParseIP("192.168.100.250")
-	req := &AddressRequest{Subnet: *subnet, Address: ip}
+	var ip *net.IPNet
+	expected := &net.IPNet{IP: net.IP{172, 28, 30, 255}, Mask: net.IPMask{255, 255, 0, 0}}
+	for err == nil {
+		var c *net.IPNet
+		if c, _, err = a.RequestAddress(poolID, nil, nil); err == nil {
+			ip = c
+		}
+	}
+	if err != ipamapi.ErrNoAvailableIPs {
+		t.Fatal(err)
+	}
+	if !types.CompareIPNet(expected, ip) {
+		t.Fatalf("Unexpected last IP from subpool. Expected: %s. Got: %v.", expected, ip)
+	}
+	rp := &net.IPNet{IP: net.IP{172, 28, 30, 97}, Mask: net.IPMask{255, 255, 0, 0}}
+	if err = a.ReleaseAddress(poolID, rp.IP); err != nil {
+		t.Fatal(err)
+	}
+	if ip, _, err = a.RequestAddress(poolID, nil, nil); err != nil {
+		t.Fatal(err)
+	}
+	if !types.CompareIPNet(rp, ip) {
+		t.Fatalf("Unexpected IP from subpool. Expected: %s. Got: %v.", rp, ip)
+	}
 
-	_, err = a.Request(addSpace, req)
+	_, _, _, err = a.RequestPool("rosso", "10.0.0.0/8", "10.0.0.0/16", nil, false)
 	if err != nil {
 		t.Fatal(err)
 	}
-
-	_, err = a.Request(addSpace, req)
-	if err == nil {
+	poolID, _, _, err = a.RequestPool("rosso", "10.0.0.0/16", "10.0.0.0/24", nil, false)
+	if err != nil {
 		t.Fatal(err)
 	}
+	expected = &net.IPNet{IP: net.IP{10, 0, 0, 255}, Mask: net.IPMask{255, 255, 0, 0}}
+	for err == nil {
+		var c *net.IPNet
+		if c, _, err = a.RequestAddress(poolID, nil, nil); err == nil {
+			ip = c
+		}
+	}
+	if err != ipamapi.ErrNoAvailableIPs {
+		t.Fatal(err)
+	}
+	if !types.CompareIPNet(expected, ip) {
+		t.Fatalf("Unexpected last IP from subpool. Expected: %s. Got: %v.", expected, ip)
+	}
+	rp = &net.IPNet{IP: net.IP{10, 0, 0, 79}, Mask: net.IPMask{255, 255, 0, 0}}
+	if err = a.ReleaseAddress(poolID, rp.IP); err != nil {
+		t.Fatal(err)
+	}
+	if ip, _, err = a.RequestAddress(poolID, nil, nil); err != nil {
+		t.Fatal(err)
+	}
+	if !types.CompareIPNet(rp, ip) {
+		t.Fatalf("Unexpected IP from subpool. Expected: %s. Got: %v.", rp, ip)
+	}
 }
 
 func TestGetAddress(t *testing.T) {
 	input := []string{
-		/*"10.0.0.0/8", "10.0.0.0/9", */ "10.0.0.0/10", "10.0.0.0/11", "10.0.0.0/12", "10.0.0.0/13", "10.0.0.0/14",
+		/*"10.0.0.0/8", "10.0.0.0/9", "10.0.0.0/10",*/ "10.0.0.0/11", "10.0.0.0/12", "10.0.0.0/13", "10.0.0.0/14",
 		"10.0.0.0/15", "10.0.0.0/16", "10.0.0.0/17", "10.0.0.0/18", "10.0.0.0/19", "10.0.0.0/20", "10.0.0.0/21",
 		"10.0.0.0/22", "10.0.0.0/23", "10.0.0.0/24", "10.0.0.0/25", "10.0.0.0/26", "10.0.0.0/27", "10.0.0.0/28",
 		"10.0.0.0/29", "10.0.0.0/30", "10.0.0.0/31"}
@@ -313,90 +582,65 @@ func TestGetAddress(t *testing.T) {
 	}
 }
 
-func TestGetSubnetList(t *testing.T) {
-	a, err := NewAllocator(nil)
-	if err != nil {
-		t.Fatal(err)
-	}
-	input := []struct {
-		addrSpace AddressSpace
-		subnet    string
-	}{
-		{"default", "192.168.0.0/16"},
-		{"default", "172.17.0.0/16"},
-		{"default", "10.0.0.0/8"},
-		{"default", "2002:1:2:3:4:5:6::/112"},
-		{"default", "2002:1:2:3:4:5:ffff::/112"},
-		{"splane", "172.17.0.0/16"},
-		{"splane", "10.0.0.0/8"},
-		{"splane", "2002:1:2:3:4:5:ff00::/104"},
-	}
+func TestRequestSyntaxCheck(t *testing.T) {
+	var (
+		pool      = "192.168.0.0/16"
+		subPool   = "192.168.0.0/24"
+		addrSpace = "green"
+	)
 
-	for _, i := range input {
-		_, sub, err := net.ParseCIDR(i.subnet)
-		if err != nil {
-			t.Fatalf("Wrong input, Can't proceed: %s", err.Error())
-		}
-		err = a.AddSubnet(i.addrSpace, &SubnetInfo{Subnet: sub})
-		if err != nil {
-			t.Fatalf("Failed to apply input. Can't proceed: %s", err.Error())
-		}
-	}
+	a, _ := NewAllocator(nil, nil)
 
-	list := a.getSubnetList("default", v4)
-	if len(list) != 258 {
-		t.Fatalf("Incorrect number of internal subnets for ipv4 version. Expected 258. Got %d.", len(list))
+	_, _, _, err := a.RequestPool("", pool, "", nil, false)
+	if err == nil {
+		t.Fatalf("Failed to detect wrong request: empty address space")
 	}
-	list = a.getSubnetList("splane", v4)
-	if len(list) != 257 {
-		t.Fatalf("Incorrect number of internal subnets for ipv4 version. Expected 257. Got %d.", len(list))
+
+	_, _, _, err = a.RequestPool("", pool, subPool, nil, false)
+	if err == nil {
+		t.Fatalf("Failed to detect wrong request: empty address space")
 	}
 
-	list = a.getSubnetList("default", v6)
-	if len(list) != 2 {
-		t.Fatalf("Incorrect number of internal subnets for ipv6 version. Expected 2. Got %d.", len(list))
+	_, _, _, err = a.RequestPool(addrSpace, "", subPool, nil, false)
+	if err == nil {
+		t.Fatalf("Failed to detect wrong request: subPool specified and no pool")
 	}
-	list = a.getSubnetList("splane", v6)
-	if len(list) != 256 {
-		t.Fatalf("Incorrect number of internal subnets for ipv6 version. Expected 256. Got %d.", len(list))
+
+	pid, _, _, err := a.RequestPool(addrSpace, pool, subPool, nil, false)
+	if err != nil {
+		t.Fatalf("Unexpected failure: %v", err)
 	}
 
-}
+	_, _, err = a.RequestAddress("", nil, nil)
+	if err == nil {
+		t.Fatalf("Failed to detect wrong request: no pool id specified")
+	}
 
-func TestRequestSyntaxCheck(t *testing.T) {
-	var (
-		subnet   = "192.168.0.0/16"
-		addSpace = AddressSpace("green")
-	)
+	ip := net.ParseIP("172.17.0.23")
+	_, _, err = a.RequestAddress(pid, ip, nil)
+	if err == nil {
+		t.Fatalf("Failed to detect wrong request: preferred IP from different subnet")
+	}
 
-	a, err := NewAllocator(nil)
+	ip = net.ParseIP("192.168.0.50")
+	_, _, err = a.RequestAddress(pid, ip, nil)
 	if err != nil {
-		t.Fatal(err)
+		t.Fatalf("Unexpected failure: %v", err)
 	}
 
-	// Add subnet and create base request
-	_, sub, _ := net.ParseCIDR(subnet)
-	a.AddSubnet(addSpace, &SubnetInfo{Subnet: sub})
-	req := &AddressRequest{Subnet: *sub}
-
-	// Empty address space request
-	_, err = a.Request("", req)
+	err = a.ReleaseAddress("", ip)
 	if err == nil {
-		t.Fatalf("Failed to detect wrong request: empty address space")
+		t.Fatalf("Failed to detect wrong request: no pool id specified")
 	}
 
-	// Preferred address from different subnet in request
-	req.Address = net.ParseIP("172.17.0.23")
-	_, err = a.Request(addSpace, req)
+	err = a.ReleaseAddress(pid, nil)
 	if err == nil {
-		t.Fatalf("Failed to detect wrong request: preferred IP from different subnet")
+		t.Fatalf("Failed to detect wrong request: no pool id specified")
 	}
 
-	// Preferred address specified and nil subnet
-	req = &AddressRequest{Address: net.ParseIP("172.17.0.23")}
-	_, err = a.Request(addSpace, req)
-	if err == nil {
-		t.Fatalf("Failed to detect wrong request: subnet not specified but preferred address specified")
+	err = a.ReleaseAddress(pid, ip)
+	if err != nil {
+		t.Fatalf("Unexpected failure: %v", err)
 	}
 }
 
@@ -433,18 +677,15 @@ func TestRequest(t *testing.T) {
 func TestRelease(t *testing.T) {
 	var (
 		err    error
-		req    *AddressRequest
 		subnet = "192.168.0.0/16"
 	)
 
-	_, sub, _ := net.ParseCIDR(subnet)
-	a := getAllocator(t, sub)
-	req = &AddressRequest{Subnet: *sub}
-	bm := a.addresses[subnetKey{"default", subnet, subnet}]
+	a, pid := getAllocator(t, subnet)
+	bm := a.addresses[SubnetKey{"default", subnet, ""}]
 
 	// Allocate all addresses
-	for err != ErrNoAvailableIPs {
-		_, err = a.Request("default", req)
+	for err != ipamapi.ErrNoAvailableIPs {
+		_, _, err = a.RequestAddress(pid, nil, nil)
 	}
 
 	toRelease := []struct {
@@ -475,45 +716,24 @@ func TestRelease(t *testing.T) {
 	}
 
 	// One by one, relase the address and request again. We should get the same IP
-	req = &AddressRequest{Subnet: *sub}
 	for i, inp := range toRelease {
-		address := net.ParseIP(inp.address)
-		a.Release("default", address)
+		ip0 := net.ParseIP(inp.address)
+		a.ReleaseAddress(pid, ip0)
 		if bm.Unselected() != 1 {
 			t.Fatalf("Failed to update free address count after release. Expected %d, Found: %d", i+1, bm.Unselected())
 		}
 
-		rsp, err := a.Request("default", req)
+		nw, _, err := a.RequestAddress(pid, nil, nil)
 		if err != nil {
 			t.Fatalf("Failed to obtain the address: %s", err.Error())
 		}
-		if !address.Equal(rsp.Address) {
-			t.Fatalf("Failed to obtain the same address. Expected: %s, Got: %s", address, rsp.Address)
+		ip := nw.IP
+		if !ip0.Equal(ip) {
+			t.Fatalf("Failed to obtain the same address. Expected: %s, Got: %s", ip0, ip)
 		}
 	}
 }
 
-func assertInternalSubnet(t *testing.T, hostSize int, bigSubnet, firstSmall, lastSmall string) {
-	_, subnet, _ := net.ParseCIDR(bigSubnet)
-	list, _ := getInternalSubnets(subnet, hostSize)
-	count := 1
-	ones, bits := subnet.Mask.Size()
-	diff := bits - ones - int(hostSize)
-	if diff > 0 {
-		count <<= uint(diff)
-	}
-
-	if len(list) != count {
-		t.Fatalf("Wrong small subnets number. Expected: %d, Got: %d", count, len(list))
-	}
-	if firstSmall != list[0].String() {
-		t.Fatalf("Wrong first small subent. Expected: %v, Got: %v", firstSmall, list[0])
-	}
-	if lastSmall != list[count-1].String() {
-		t.Fatalf("Wrong last small subent. Expected: %v, Got: %v", lastSmall, list[count-1])
-	}
-}
-
 func assertGetAddress(t *testing.T, subnet string) {
 	var (
 		err       error
@@ -526,15 +746,15 @@ func assertGetAddress(t *testing.T, subnet string) {
 	zeroes := bits - ones
 	numAddresses := 1 << uint(zeroes)
 
-	bm, err := bitseq.NewHandle("ipam_test", nil, "default/192.168.0.0/24", uint32(numAddresses))
+	bm, err := bitseq.NewHandle("ipam_test", nil, "default/"+subnet, uint32(numAddresses))
 	if err != nil {
 		t.Fatal(err)
 	}
 
 	start := time.Now()
 	run := 0
-	for err != ErrNoAvailableIPs {
-		_, err = a.getAddress(sub, bm, nil, v4)
+	for err != ipamapi.ErrNoAvailableIPs {
+		_, err = a.getAddress(sub, bm, nil, nil)
 		run++
 	}
 	if printTime {
@@ -554,59 +774,52 @@ func assertGetAddress(t *testing.T, subnet string) {
 func assertNRequests(t *testing.T, subnet string, numReq int, lastExpectedIP string) {
 	var (
 		err       error
-		req       *AddressRequest
-		rsp       *AddressResponse
+		nw        *net.IPNet
 		printTime = false
 	)
 
-	_, sub, _ := net.ParseCIDR(subnet)
 	lastIP := net.ParseIP(lastExpectedIP)
-
-	a := getAllocator(t, sub)
-	req = &AddressRequest{Subnet: *sub}
+	a, pid := getAllocator(t, subnet)
 
 	i := 0
 	start := time.Now()
 	for ; i < numReq; i++ {
-		rsp, err = a.Request("default", req)
+		nw, _, err = a.RequestAddress(pid, nil, nil)
 	}
 	if printTime {
 		fmt.Printf("\nTaken %v, to allocate %d addresses on %s\n", time.Since(start), numReq, subnet)
 	}
 
-	if !lastIP.Equal(rsp.Address) {
-		t.Fatalf("Wrong last IP. Expected %s. Got: %s (err: %v, ind: %d)", lastExpectedIP, rsp.Address.String(), err, i)
+	if !lastIP.Equal(nw.IP) {
+		t.Fatalf("Wrong last IP. Expected %s. Got: %s (err: %v, ind: %d)", lastExpectedIP, nw.IP.String(), err, i)
 	}
 }
 
-func benchmarkRequest(subnet *net.IPNet) {
+func benchmarkRequest(subnet string) {
 	var err error
 
-	a, _ := NewAllocator(nil)
-	a.internalHostSize = 20
-	a.AddSubnet("default", &SubnetInfo{Subnet: subnet})
-
-	req := &AddressRequest{Subnet: *subnet}
-	for err != ErrNoAvailableIPs {
-		_, err = a.Request("default", req)
+	a, _ := NewAllocator(nil, nil)
+	pid, _, _, _ := a.RequestPool("default", subnet, "", nil, false)
 
+	for err != ipamapi.ErrNoAvailableIPs {
+		_, _, err = a.RequestAddress(pid, nil, nil)
 	}
 }
 
-func benchMarkRequest(subnet *net.IPNet, b *testing.B) {
+func benchMarkRequest(subnet string, b *testing.B) {
 	for n := 0; n < b.N; n++ {
 		benchmarkRequest(subnet)
 	}
 }
 
 func BenchmarkRequest_24(b *testing.B) {
-	benchmarkRequest(&net.IPNet{IP: []byte{10, 0, 0, 0}, Mask: []byte{255, 255, 255, 0}})
+	benchmarkRequest("10.0.0.0/24")
 }
 
 func BenchmarkRequest_16(b *testing.B) {
-	benchmarkRequest(&net.IPNet{IP: []byte{10, 0, 0, 0}, Mask: []byte{255, 255, 0, 0}})
+	benchmarkRequest("10.0.0.0/16")
 }
 
 func BenchmarkRequest_8(b *testing.B) {
-	benchmarkRequest(&net.IPNet{IP: []byte{10, 0, 0, 0}, Mask: []byte{255, 0xfc, 0, 0}})
+	benchmarkRequest("10.0.0.0/8")
 }

libnetwork/ipam/contract.go

@@ -1,100 +0,0 @@
-// Package ipam that specifies the contract the IPAM plugin need to satisfy,
-// decoupling IPAM interface and implementation.
-package ipam
-
-import (
-	"errors"
-	"net"
-)
-
-/**************
- * IPAM Errors
- **************/
-
-// ErrIpamNotAvailable is returned when the plugin prviding the IPAM service is not available
-var (
-	ErrInvalidIpamService       = errors.New("Invalid IPAM Service")
-	ErrInvalidIpamConfigService = errors.New("Invalid IPAM Config Service")
-	ErrIpamNotAvailable         = errors.New("IPAM Service not available")
-	ErrIpamInternalError        = errors.New("IPAM Internal Error")
-	ErrInvalidAddressSpace      = errors.New("Invalid Address Space")
-	ErrInvalidSubnet            = errors.New("Invalid Subnet")
-	ErrInvalidRequest           = errors.New("Invalid Request")
-	ErrSubnetNotFound           = errors.New("Subnet not found")
-	ErrOverlapSubnet            = errors.New("Subnet overlaps with existing subnet on this address space")
-	ErrNoAvailableSubnet        = errors.New("No available subnet")
-	ErrNoAvailableIPs           = errors.New("No available addresses on subnet")
-	ErrIPAlreadyAllocated       = errors.New("Address already in use")
-	ErrIPOutOfRange             = errors.New("Requested address is out of range")
-	ErrSubnetAlreadyRegistered  = errors.New("Subnet already registered on this address space")
-	ErrBadSubnet                = errors.New("Address space does not contain specified subnet")
-)
-
-// AddressSpace identifies a unique pool of network addresses
-type AddressSpace string
-
-/*******************************
- * IPAM Configuration Interface
- *******************************/
-
-// Config represents the interface the IPAM service plugins must implement
-// in order to allow injection/modification of IPAM database.
-// Common key is a addressspace
-type Config interface {
-	// AddSubnet adds a subnet to the specified address space
-	AddSubnet(AddressSpace, *SubnetInfo) error
-	// RemoveSubnet removes a subnet from the specified address space
-	RemoveSubnet(AddressSpace, *net.IPNet) error
-	// AddVendorInfo adds Vendor specific data
-	AddVendorInfo([]byte) error
-}
-
-// SubnetInfo contains the information subnet hosts need in order to communicate
-type SubnetInfo struct {
-	Subnet     *net.IPNet
-	Gateway    net.IP
-	OpaqueData []byte // Vendor specific
-}
-
-/*************************
- * IPAM Service Interface
- *************************/
-
-// IPAM defines the interface that needs to be implemented by IPAM service plugin
-// Common key is a unique address space identifier
-type IPAM interface {
-	// Request address from the specified address space
-	Request(AddressSpace, *AddressRequest) (*AddressResponse, error)
-	// Separate API for IPv6
-	RequestV6(AddressSpace, *AddressRequest) (*AddressResponse, error)
-	// Release the address from the specified address space
-	Release(AddressSpace, net.IP)
-}
-
-// AddressRequest encloses the information a client
-// needs to pass to IPAM when requesting an address
-type AddressRequest struct {
-	Subnet     net.IPNet // Preferred subnet pool (Optional)
-	Address    net.IP    // Preferred address (Optional)
-	Endpoint   string    // For static IP mapping (Optional)
-	OpaqueData []byte    // Vendor specific request data
-}
-
-// Validate runs syntactic validation on this AddressRequest object
-func (req *AddressRequest) Validate() error {
-	var byteArray []byte = req.Address
-
-	// Check preferred address
-	if byteArray != nil && (&req.Subnet == nil || !req.Subnet.Contains(req.Address)) {
-		return ErrInvalidRequest
-	}
-
-	return nil
-}
-
-// AddressResponse represents the IPAM service's
-// response to an address request
-type AddressResponse struct {
-	Address net.IP
-	Subnet  SubnetInfo
-}

libnetwork/ipam/store.go

@@ -2,7 +2,6 @@ package ipam
 
 import (
 	"encoding/json"
-	"net"
 
 	log "github.com/Sirupsen/logrus"
 	"github.com/docker/libnetwork/datastore"
@@ -13,14 +12,14 @@ import (
 func (a *Allocator) Key() []string {
 	a.Lock()
 	defer a.Unlock()
-	return []string{a.App, a.ID}
+	return []string{dsConfigKey}
 }
 
 // KeyPrefix returns the immediate parent key that can be used for tree walk
 func (a *Allocator) KeyPrefix() []string {
 	a.Lock()
 	defer a.Unlock()
-	return []string{a.App}
+	return []string{dsConfigKey}
 }
 
 // Value marshals the data to be stored in the KV store
@@ -31,9 +30,14 @@ func (a *Allocator) Value() []byte {
 	if a.subnets == nil {
 		return []byte{}
 	}
+	m := map[string]interface{}{}
+	for k, v := range a.subnets {
+		m[k.String()] = v
+	}
 
-	b, err := subnetsToByteArray(a.subnets)
+	b, err := json.Marshal(m)
 	if err != nil {
+		log.Warnf("Failed to marshal ipam configured subnets")
 		return nil
 	}
 	return b
@@ -41,52 +45,19 @@ func (a *Allocator) Value() []byte {
 
 // SetValue unmarshalls the data from the KV store.
 func (a *Allocator) SetValue(value []byte) error {
-	a.subnets = byteArrayToSubnets(value)
-	return nil
-}
-
-func subnetsToByteArray(m map[subnetKey]*SubnetInfo) ([]byte, error) {
-	if m == nil {
-		return nil, nil
-	}
-
-	mm := make(map[string]string, len(m))
-	for k, v := range m {
-		mm[k.String()] = v.Subnet.String()
-	}
-
-	return json.Marshal(mm)
-}
-
-func byteArrayToSubnets(ba []byte) map[subnetKey]*SubnetInfo {
-	m := map[subnetKey]*SubnetInfo{}
-
-	if ba == nil || len(ba) == 0 {
-		return m
-	}
-
-	var mm map[string]string
-	err := json.Unmarshal(ba, &mm)
+	var m map[string]*PoolData
+	err := json.Unmarshal(value, &m)
 	if err != nil {
-		log.Warnf("Failed to decode subnets byte array: %v", err)
-		return m
+		return err
 	}
-	for ks, vs := range mm {
-		sk := subnetKey{}
-		if err := sk.FromString(ks); err != nil {
-			log.Warnf("Failed to decode subnets map entry: (%s, %s)", ks, vs)
-			continue
+	for ks, d := range m {
+		k := SubnetKey{}
+		if err := k.FromString(ks); err != nil {
+			return err
 		}
-		si := &SubnetInfo{}
-		_, nw, err := net.ParseCIDR(vs)
-		if err != nil {
-			log.Warnf("Failed to decode subnets map entry value: (%s, %s)", ks, vs)
-			continue
-		}
-		si.Subnet = nw
-		m[sk] = si
+		a.subnets[k] = d
 	}
-	return m
+	return nil
 }
 
 // Index returns the latest DB Index as seen by this object
@@ -130,7 +101,6 @@ func (a *Allocator) watchForChanges() error {
 			select {
 			case kvPair := <-kvpChan:
 				if kvPair != nil {
-					log.Debugf("Got notification for key %v: %v", kvPair.Key, kvPair.Value)
 					a.subnetConfigFromStore(kvPair)
 				}
 			}

libnetwork/ipamapi/contract.go

@@ -0,0 +1,74 @@
+// Package ipamapi specifies the contract the IPAM service (built-in or remote) needs to satisfy.
+package ipamapi
+
+import (
+	"errors"
+	"net"
+)
+
+/********************
+ * IPAM plugin types
+ ********************/
+
+const (
+	// DefaultIPAM is the name of the built-in default ipam driver
+	DefaultIPAM = "default"
+	// PluginEndpointType represents the Endpoint Type used by Plugin system
+	PluginEndpointType = "IPAM"
+	// Gateway is the key for the gateway option
+	Gateway = "gateway"
+)
+
+// Callback provides a Callback interface for registering an IPAM instance into LibNetwork
+type Callback interface {
+	// RegisterDriver provides a way for Remote drivers to dynamically register new NetworkType and associate with a ipam instance
+	RegisterIpamDriver(name string, driver Ipam) error
+}
+
+/**************
+ * IPAM Errors
+ **************/
+
+// Weel-known errors returned by IPAM
+var (
+	ErrInvalidIpamService       = errors.New("Invalid IPAM Service")
+	ErrInvalidIpamConfigService = errors.New("Invalid IPAM Config Service")
+	ErrIpamNotAvailable         = errors.New("IPAM Service not available")
+	ErrIpamInternalError        = errors.New("IPAM Internal Error")
+	ErrInvalidAddressSpace      = errors.New("Invalid Address Space")
+	ErrInvalidPool              = errors.New("Invalid Address Pool")
+	ErrInvalidSubPool           = errors.New("Invalid Address SubPool")
+	ErrInvalidRequest           = errors.New("Invalid Request")
+	ErrPoolNotFound             = errors.New("Address Pool not found")
+	ErrOverlapPool              = errors.New("Address pool overlaps with existing pool on this address space")
+	ErrNoAvailablePool          = errors.New("No available pool")
+	ErrNoAvailableIPs           = errors.New("No available addresses on this pool")
+	ErrIPAlreadyAllocated       = errors.New("Address already in use")
+	ErrIPOutOfRange             = errors.New("Requested address is out of range")
+	ErrPoolOverlap              = errors.New("Pool overlaps with other one on this address space")
+	ErrBadPool                  = errors.New("Address space does not contain specified address pool")
+)
+
+/*******************************
+ * IPAM Service Interface
+ *******************************/
+
+// Ipam represents the interface the IPAM service plugins must implement
+// in order to allow injection/modification of IPAM database.
+type Ipam interface {
+	// GetDefaultAddressSpaces returns the default local and global address spaces for this ipam
+	GetDefaultAddressSpaces() (string, string, error)
+	// RequestPool returns an address pool along with its unique id. Address space is a mandatory field
+	// which denotes a set of non-overlapping pools. pool describes the pool of addrresses in CIDR notation.
+	// subpool indicates a smaller range of addresses from the pool, for now it is specified in CIDR notation.
+	// Both pool and subpool are non mandatory fields. When they are not specified, Ipam driver may choose to
+	// return a self chosen pool for this request. In such case the v6 flag needs to be set appropriately so
+	// that the driver would return the expected ip version pool.
+	RequestPool(addressSpace, pool, subPool string, options map[string]string, v6 bool) (string, *net.IPNet, map[string]string, error)
+	// ReleasePool releases the address pool identified by the passed id
+	ReleasePool(poolID string) error
+	// Request address from the specified pool ID. Input options or preferred IP can be passed.
+	RequestAddress(string, net.IP, map[string]string) (*net.IPNet, map[string]string, error)
+	// Release the address from the specified pool ID
+	ReleaseAddress(string, net.IP) error
+}

libnetwork/ipams/builtin/builtin.go

@@ -0,0 +1,35 @@
+package builtin
+
+import (
+	"fmt"
+
+	"github.com/docker/libnetwork/datastore"
+	"github.com/docker/libnetwork/ipam"
+	"github.com/docker/libnetwork/ipamapi"
+)
+
+// Init registers the built-in ipam service with libnetwork
+func Init(ic ipamapi.Callback, l, g interface{}) error {
+	var (
+		ok                bool
+		localDs, globalDs datastore.DataStore
+	)
+
+	if l != nil {
+		if localDs, ok = l.(datastore.DataStore); !ok {
+			return fmt.Errorf("incorrect local datastore passed to built-in ipam init")
+		}
+	}
+
+	if g != nil {
+		if globalDs, ok = g.(datastore.DataStore); !ok {
+			return fmt.Errorf("incorrect global datastore passed to built-in ipam init")
+		}
+	}
+	a, err := ipam.NewAllocator(localDs, globalDs)
+	if err != nil {
+		return err
+	}
+
+	return ic.RegisterIpamDriver(ipamapi.DefaultIPAM, a)
+}

libnetwork/ipams/remote/api/api.go

@@ -0,0 +1,81 @@
+// Package api defines the data structure to be used in the request/response
+// messages between libnetwork and the remote ipam plugin
+package api
+
+import (
+	"net"
+)
+
+// Response is the basic response structure used in all responses
+type Response struct {
+	Error string
+}
+
+// IsSuccess returns wheter the plugin response is successful
+func (r *Response) IsSuccess() bool {
+	return r.Error == ""
+}
+
+// GetError returns the error from the response, if any.
+func (r *Response) GetError() string {
+	return r.Error
+}
+
+// GetAddressSpacesResponse is the response to the ``get default address spaces`` request message
+type GetAddressSpacesResponse struct {
+	Response
+	LocalDefaultAddressSpace  string
+	GlobalDefaultAddressSpace string
+}
+
+// RequestPoolRequest represents the expected data in a ``request address pool`` request message
+type RequestPoolRequest struct {
+	AddressSpace string
+	Pool         string
+	SubPool      string
+	Options      map[string]string
+	V6           bool
+}
+
+// RequestPoolResponse represents the response message to a ``request address pool`` request
+type RequestPoolResponse struct {
+	Response
+	PoolID string
+	Pool   *net.IPNet
+	Data   map[string]string
+}
+
+// ReleasePoolRequest represents the expected data in a ``release address pool`` request message
+type ReleasePoolRequest struct {
+	PoolID string
+}
+
+// ReleasePoolResponse represents the response message to a ``release address pool`` request
+type ReleasePoolResponse struct {
+	Response
+}
+
+// RequestAddressRequest represents the expected data in a ``request address`` request message
+type RequestAddressRequest struct {
+	PoolID  string
+	Address net.IP
+	Options map[string]string
+}
+
+// RequestAddressResponse represents the expected data in the response message to a ``request address`` request
+type RequestAddressResponse struct {
+	Response
+	Address *net.IPNet
+	Data    map[string]string
+}
+
+// ReleaseAddressRequest represents the expected data in a ``release address`` request message
+type ReleaseAddressRequest struct {
+	PoolID  string
+	Address net.IP
+}
+
+// ReleaseAddressResponse represents the response message to a ``release address`` request
+type ReleaseAddressResponse struct {
+	Response
+}

libnetwork/ipams/remote/remote.go

@@ -0,0 +1,92 @@
+package remote
+
+import (
+	"fmt"
+	"net"
+
+	log "github.com/Sirupsen/logrus"
+	"github.com/docker/docker/pkg/plugins"
+	"github.com/docker/libnetwork/ipamapi"
+	"github.com/docker/libnetwork/ipams/remote/api"
+)
+
+type allocator struct {
+	endpoint *plugins.Client
+	name     string
+}
+
+// PluginResponse is the interface for the plugin request responses
+type PluginResponse interface {
+	IsSuccess() bool
+	GetError() string
+}
+
+func newAllocator(name string, client *plugins.Client) ipamapi.Ipam {
+	a := &allocator{name: name, endpoint: client}
+	return a
+}
+
+// Init registers a remote ipam when its plugin is activated
+func Init(cb ipamapi.Callback, l, g interface{}) error {
+	plugins.Handle(ipamapi.PluginEndpointType, func(name string, client *plugins.Client) {
+		if err := cb.RegisterIpamDriver(name, newAllocator(name, client)); err != nil {
+			log.Errorf("error registering remote ipam %s due to %v", name, err)
+		}
+	})
+	return nil
+}
+
+func (a *allocator) call(methodName string, arg interface{}, retVal PluginResponse) error {
+	method := ipamapi.PluginEndpointType + "." + methodName
+	err := a.endpoint.Call(method, arg, retVal)
+	if err != nil {
+		return err
+	}
+	if !retVal.IsSuccess() {
+		return fmt.Errorf("remote: %s", retVal.GetError())
+	}
+	return nil
+}
+
+// GetDefaultAddressSpaces returns the local and global default address spaces
+func (a *allocator) GetDefaultAddressSpaces() (string, string, error) {
+	res := &api.GetAddressSpacesResponse{}
+	if err := a.call("GetDefaultAddressSpaces", nil, res); err != nil {
+		return "", "", err
+	}
+	return res.LocalDefaultAddressSpace, res.GlobalDefaultAddressSpace, nil
+}
+
+// RequestPool requests an address pool in the specified address space
+func (a *allocator) RequestPool(addressSpace, pool, subPool string, options map[string]string, v6 bool) (string, *net.IPNet, map[string]string, error) {
+	req := &api.RequestPoolRequest{AddressSpace: addressSpace, Pool: pool, SubPool: subPool, Options: options, V6: v6}
+	res := &api.RequestPoolResponse{}
+	if err := a.call("RequestPool", req, res); err != nil {
+		return "", nil, nil, err
+	}
+	return res.PoolID, res.Pool, res.Data, nil
+}
+
+// ReleasePool removes an address pool from the specified address space
+func (a *allocator) ReleasePool(poolID string) error {
+	req := &api.ReleasePoolRequest{PoolID: poolID}
+	res := &api.ReleasePoolResponse{}
+	return a.call("ReleasePool", req, res)
+}
+
+// RequestAddress requests an address from the address pool
+func (a *allocator) RequestAddress(poolID string, address net.IP, options map[string]string) (*net.IPNet, map[string]string, error) {
+	req := &api.RequestAddressRequest{PoolID: poolID, Address: address, Options: options}
+	res := &api.RequestAddressResponse{}
+	if err := a.call("RequestAddress", req, res); err != nil {
+		return nil, nil, err
+	}
+	return res.Address, res.Data, nil
+}
+
+// ReleaseAddress releases the address from the specified address pool
+func (a *allocator) ReleaseAddress(poolID string, address net.IP) error {
+	req := &api.ReleaseAddressRequest{PoolID: poolID, Address: address}
+	res := &api.ReleaseAddressResponse{}
+	return a.call("ReleaseAddress", req, res)
+}

libnetwork/network.go

@@ -60,6 +60,7 @@ type network struct {
 	name        string
 	networkType string
 	id          string
+	ipamType    string
 	driver      driverapi.Driver
 	enableIPv6  bool
 	endpointCnt uint64
@@ -178,6 +179,7 @@ func (n *network) MarshalJSON() ([]byte, error) {
 	netMap["name"] = n.name
 	netMap["id"] = n.id
 	netMap["networkType"] = n.networkType
+	netMap["ipamType"] = n.ipamType
 	netMap["endpointCnt"] = n.endpointCnt
 	netMap["enableIPv6"] = n.enableIPv6
 	netMap["generic"] = n.generic
@@ -193,6 +195,7 @@ func (n *network) UnmarshalJSON(b []byte) (err error) {
 	}
 	n.name = netMap["name"].(string)
 	n.id = netMap["id"].(string)
+	n.ipamType = netMap["ipamType"].(string)
 	n.networkType = netMap["networkType"].(string)
 	n.endpointCnt = uint64(netMap["endpointCnt"].(float64))
 	n.enableIPv6 = netMap["enableIPv6"].(bool)
@@ -239,9 +242,7 @@ func (n *network) processOptions(options ...NetworkOption) {
 func (n *network) Delete() error {
 	var err error
 
-	n.Lock()
-	ctrlr := n.ctrlr
-	n.Unlock()
+	ctrlr := n.getController()
 
 	ctrlr.Lock()
 	_, ok := ctrlr.networks[n.id]
@@ -338,15 +339,12 @@ func (n *network) CreateEndpoint(name string, options ...EndpointOption) (Endpoi
 		return nil, types.ForbiddenErrorf("service endpoint with name %s already exists", name)
 	}
 
-	ep := &endpoint{name: name,
-		generic: make(map[string]interface{})}
+	ep := &endpoint{name: name, generic: make(map[string]interface{})}
 	ep.id = stringid.GenerateRandomID()
 	ep.network = n
 	ep.processOptions(options...)
 
-	n.Lock()
-	ctrlr := n.ctrlr
-	n.Unlock()
+	ctrlr := n.getController()
 
 	n.IncEndpointCnt()
 	if err = ctrlr.updateToStore(n); err != nil {

libnetwork/store.go

@@ -45,14 +45,7 @@ func (c *controller) initGlobalStore() error {
 	c.Lock()
 	c.globalStore = store
 	c.Unlock()
-
-	nws, err := c.getNetworksFromStore(true)
-	if err == nil {
-		c.processNetworkUpdate(nws, nil)
-	} else if err != datastore.ErrKeyNotFound {
-		log.Warnf("failed to read networks from globalstore during init : %v", err)
-	}
-	return c.watchNetworks()
+	return nil
 }
 
 func (c *controller) initLocalStore() error {
@@ -66,14 +59,37 @@ func (c *controller) initLocalStore() error {
 	c.Lock()
 	c.localStore = localStore
 	c.Unlock()
+	return nil
+}
 
-	nws, err := c.getNetworksFromStore(false)
+func (c *controller) restoreFromGlobalStore() error {
+	c.Lock()
+	s := c.globalStore
+	c.Unlock()
+	if s == nil {
+		return nil
+	}
+	c.restore("global")
+	return c.watchNetworks()
+}
+
+func (c *controller) restoreFromLocalStore() error {
+	c.Lock()
+	s := c.localStore
+	c.Unlock()
+	if s != nil {
+		c.restore("local")
+	}
+	return nil
+}
+
+func (c *controller) restore(store string) {
+	nws, err := c.getNetworksFromStore(store == "global")
 	if err == nil {
 		c.processNetworkUpdate(nws, nil)
 	} else if err != datastore.ErrKeyNotFound {
-		log.Warnf("failed to read networks from localstore during init : %v", err)
+		log.Warnf("failed to read networks from %s store during init : %v", store, err)
 	}
-	return nil
 }
 
 func (c *controller) getNetworksFromStore(global bool) ([]*store.KVPair, error) {

libnetwork/store_test.go

@@ -52,7 +52,6 @@ func testLocalBackend(t *testing.T, provider, url string, storeConfig *store.Con
 	genericOption[netlabel.GenericData] = driverOptions
 	cfgOptions = append(cfgOptions, config.OptionDriverConfig("host", genericOption))
 
-	fmt.Printf("URL : %s\n", url)
 	ctrl, err := New(cfgOptions...)
 	if err != nil {
 		t.Fatalf("Error new controller: %v", err)
@@ -139,7 +138,7 @@ func TestLocalStoreLockTimeout(t *testing.T) {
 	}
 	defer ctrl1.Stop()
 	// Use the same boltdb file without closing the previous controller
-	ctrl2, _ := New(cfgOptions...)
+	ctrl2, err := New(cfgOptions...)
 	if err != nil {
 		t.Fatalf("Error new controller: %v", err)
 	}

libnetwork/types/types.go

@@ -222,23 +222,32 @@ func GetMinimalIPNet(nw *net.IPNet) *net.IPNet {
 
 var v4inV6MaskPrefix = []byte{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}
 
-// GetHostPartIP returns the host portion of the ip address identified by the mask.
-// IP address representation is not modified. If address and mask are not compatible
-// an error is returned.
-func GetHostPartIP(ip net.IP, mask net.IPMask) (net.IP, error) {
+// compareIPMask checks if the passed ip and mask are semantically compatible.
+// It returns the byte indexes for the address and mask so that caller can
+// do bitwise operations without modifying address representation.
+func compareIPMask(ip net.IP, mask net.IPMask) (is int, ms int, err error) {
 	// Find the effective starting of address and mask
-	is := 0
-	ms := 0
 	if len(ip) == net.IPv6len && ip.To4() != nil {
 		is = 12
 	}
 	if len(ip[is:]) == net.IPv4len && len(mask) == net.IPv6len && bytes.Equal(mask[:12], v4inV6MaskPrefix) {
 		ms = 12
 	}
-
 	// Check if address and mask are semantically compatible
 	if len(ip[is:]) != len(mask[ms:]) {
-		return nil, fmt.Errorf("cannot compute host portion ip address as ip and mask are not compatible: (%#v, %#v)", ip, mask)
+		err = fmt.Errorf("ip and mask are not compatible: (%#v, %#v)", ip, mask)
+	}
+	return
+}
+
+// GetHostPartIP returns the host portion of the ip address identified by the mask.
+// IP address representation is not modified. If address and mask are not compatible
+// an error is returned.
+func GetHostPartIP(ip net.IP, mask net.IPMask) (net.IP, error) {
+	// Find the effective starting of address and mask
+	is, ms, err := compareIPMask(ip, mask)
+	if err != nil {
+		return nil, fmt.Errorf("cannot compute host portion ip address because %s", err)
 	}
 
 	// Compute host portion
@@ -250,6 +259,34 @@ func GetHostPartIP(ip net.IP, mask net.IPMask) (net.IP, error) {
 	return out, nil
 }
 
+// GetBroadcastIP returns the broadcast ip address for the passed network (ip and mask).
+// IP address representation is not modified. If address and mask are not compatible
+// an error is returned.
+func GetBroadcastIP(ip net.IP, mask net.IPMask) (net.IP, error) {
+	// Find the effective starting of address and mask
+	is, ms, err := compareIPMask(ip, mask)
+	if err != nil {
+		return nil, fmt.Errorf("cannot compute broadcast ip address because %s", err)
+	}
+
+	// Compute broadcast address
+	out := GetIPCopy(ip)
+	for i := 0; i < len(mask[ms:]); i++ {
+		out[is+i] |= ^mask[ms+i]
+	}
+
+	return out, nil
+}
+
+// ParseCIDR returns the *net.IPNet represented by the passed CIDR notation
+func ParseCIDR(cidr string) (n *net.IPNet, e error) {
+	var i net.IP
+	if i, n, e = net.ParseCIDR(cidr); e == nil {
+		n.IP = i
+	}
+	return
+}
+
 const (
 	// NEXTHOP indicates a StaticRoute with an IP next hop.
 	NEXTHOP = iota

libnetwork/types/types_test.go

@@ -111,6 +111,75 @@ func TestErrorConstructors(t *testing.T) {
 	}
 }
 
+func TestCompareIPMask(t *testing.T) {
+	input := []struct {
+		ip    net.IP
+		mask  net.IPMask
+		is    int
+		ms    int
+		isErr bool
+	}{
+		{ // ip in v4Inv6 representation, mask in v4 representation
+			ip:   net.IPv4(172, 28, 30, 1),
+			mask: []byte{0xff, 0xff, 0xff, 0},
+			is:   12,
+			ms:   0,
+		},
+		{ // ip and mask in v4Inv6 representation
+			ip:   net.IPv4(172, 28, 30, 2),
+			mask: []byte{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0},
+			is:   12,
+			ms:   12,
+		},
+		{ // ip in v4 representation, mask in v4Inv6 representation
+			ip:   net.IPv4(172, 28, 30, 3)[12:],
+			mask: []byte{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0},
+			is:   0,
+			ms:   12,
+		},
+		{ // ip and mask in v4 representation
+			ip:   net.IPv4(172, 28, 30, 4)[12:],
+			mask: []byte{0xff, 0xff, 0xff, 0},
+			is:   0,
+			ms:   0,
+		},
+		{ // ip and mask as v6
+			ip:   net.ParseIP("2005:2004:2002:2001:FFFF:ABCD:EEAB:00CD"),
+			mask: []byte{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0, 0, 0},
+			is:   0,
+			ms:   0,
+		},
+		{
+			ip:    net.ParseIP("2005:2004:2002:2001:FFFF:ABCD:EEAB:00CD"),
+			mask:  []byte{0xff, 0xff, 0xff, 0},
+			isErr: true,
+		},
+		{
+			ip:    net.ParseIP("173.32.4.5"),
+			mask:  []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xff, 0xff, 0xff, 0},
+			isErr: true,
+		},
+		{
+			ip:    net.ParseIP("173.32.4.5"),
+			mask:  []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xff, 0xff, 0xff, 0xff, 0xff, 0},
+			isErr: true,
+		},
+	}
+
+	for ind, i := range input {
+		is, ms, err := compareIPMask(i.ip, i.mask)
+		if i.isErr {
+			if err == nil {
+				t.Fatalf("Incorrect error condition for element %d. is: %d, ms: %d, err: %v", ind, is, ms, err)
+			}
+		} else {
+			if i.is != is || i.ms != ms {
+				t.Fatalf("expected is: %d, ms: %d. Got is: %d, ms: %d for element %d", i.is, i.ms, is, ms, ind)
+			}
+		}
+	}
+}
+
 func TestUtilGetHostPortionIP(t *testing.T) {
 	input := []struct {
 		ip   net.IP
@@ -168,3 +237,106 @@ func TestUtilGetHostPortionIP(t *testing.T) {
 		t.Fatalf("Unexpected success")
 	}
 }
+
+func TestUtilGetBroadcastIP(t *testing.T) {
+	input := []struct {
+		ip    net.IP
+		mask  net.IPMask
+		bcast net.IP
+		err   error
+	}{
+		// ip in v4Inv6 representation, mask in v4 representation
+		{
+			ip:    net.IPv4(172, 28, 30, 1),
+			mask:  []byte{0xff, 0xff, 0xff, 0},
+			bcast: net.IPv4(172, 28, 30, 255),
+		},
+		{
+			ip:    net.IPv4(10, 28, 30, 1),
+			mask:  []byte{0xff, 0, 0, 0},
+			bcast: net.IPv4(10, 255, 255, 255),
+		},
+		// ip and mask in v4Inv6 representation
+		{
+			ip:    net.IPv4(172, 28, 30, 2),
+			mask:  []byte{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0},
+			bcast: net.IPv4(172, 28, 30, 255),
+		},
+		{
+			ip:    net.IPv4(172, 28, 30, 2),
+			mask:  []byte{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0, 0},
+			bcast: net.IPv4(172, 28, 255, 255),
+		},
+		// ip in v4 representation, mask in v4Inv6 representation
+		{
+			ip:    net.IPv4(172, 28, 30, 3)[12:],
+			mask:  []byte{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0},
+			bcast: net.IPv4(172, 28, 30, 255)[12:],
+		},
+		{
+			ip:    net.IPv4(172, 28, 30, 3)[12:],
+			mask:  []byte{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0, 0, 0},
+			bcast: net.IPv4(172, 255, 255, 255)[12:],
+		},
+		// ip and mask in v4 representation
+		{
+			ip:    net.IPv4(172, 28, 30, 4)[12:],
+			mask:  []byte{0xff, 0xff, 0xff, 0},
+			bcast: net.IPv4(172, 28, 30, 255)[12:],
+		},
+		{
+			ip:    net.IPv4(172, 28, 30, 4)[12:],
+			mask:  []byte{0xff, 0xff, 0, 0},
+			bcast: net.IPv4(172, 28, 255, 255)[12:],
+		},
+		{ // ip and mask as v6
+			ip:    net.ParseIP("2005:2004:2002:2001:FFFF:ABCD:EEAB:00CD"),
+			mask:  []byte{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0, 0, 0},
+			bcast: net.ParseIP("2005:2004:2002:2001:FFFF:ABCD:EEFF:FFFF"),
+		},
+	}
+
+	for _, i := range input {
+		h, err := GetBroadcastIP(i.ip, i.mask)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if !i.bcast.Equal(h) {
+			t.Fatalf("Failed to return expected host ip. Expected: %s. Got: %s", i.bcast, h)
+		}
+	}
+
+	// ip as v6 and mask as v4 are not compatible
+	if _, err := GetBroadcastIP(net.ParseIP("2005:2004:2002:2001:FFFF:ABCD:EEAB:00CD"), []byte{0xff, 0xff, 0xff, 0}); err == nil {
+		t.Fatalf("Unexpected success")
+	}
+	// ip as v4 and non conventional mask
+	if _, err := GetBroadcastIP(net.ParseIP("173.32.4.5"), []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xff, 0xff, 0xff, 0}); err == nil {
+		t.Fatalf("Unexpected success")
+	}
+	// ip as v4 and non conventional mask
+	if _, err := GetBroadcastIP(net.ParseIP("173.32.4.5"), []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xff, 0xff, 0xff, 0xff, 0xff, 0}); err == nil {
+		t.Fatalf("Unexpected success")
+	}
+}
+
+func TestParseCIDR(t *testing.T) {
+	input := []struct {
+		cidr string
+		ipnw *net.IPNet
+	}{
+		{"192.168.22.44/16", &net.IPNet{IP: net.IP{192, 168, 22, 44}, Mask: net.IPMask{255, 255, 0, 0}}},
+		{"10.10.2.0/24", &net.IPNet{IP: net.IP{10, 10, 2, 0}, Mask: net.IPMask{255, 255, 255, 0}}},
+		{"10.0.0.100/17", &net.IPNet{IP: net.IP{10, 0, 0, 100}, Mask: net.IPMask{255, 255, 128, 0}}},
+	}
+
+	for _, i := range input {
+		nw, err := ParseCIDR(i.cidr)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if !CompareIPNet(nw, i.ipnw) {
+			t.Fatalf("network differ. Expected %v. Got: %v", i.ipnw, nw)
+		}
+	}
+}