moby/libnetwork/cnmallocator/networkallocator.go

package cnmallocator

import (
	"context"
	"fmt"
	"net"
	"strings"

	"github.com/containerd/log"
	"github.com/docker/docker/libnetwork/driverapi"
	"github.com/docker/docker/libnetwork/drivers/remote"
	"github.com/docker/docker/libnetwork/drvregistry"
	"github.com/docker/docker/libnetwork/ipamapi"
	remoteipam "github.com/docker/docker/libnetwork/ipams/remote"
	"github.com/docker/docker/libnetwork/netlabel"
	"github.com/docker/docker/libnetwork/scope"
	"github.com/docker/docker/pkg/plugingetter"
	"github.com/moby/swarmkit/v2/api"
	"github.com/moby/swarmkit/v2/manager/allocator/networkallocator"
	"github.com/pkg/errors"
)

const (
	// DefaultDriver defines the name of the driver to be used by
	// default if a network without any driver name specified is
	// created.
	DefaultDriver = "overlay"
)

// cnmNetworkAllocator acts as the controller for all network related operations
// like managing network and IPAM drivers and also creating and
// deleting networks and the associated resources.
type cnmNetworkAllocator struct {
	// The plugin getter instance used to get network and IPAM driver plugins.
	pg plugingetter.PluginGetter

	// The driver registry for all internal and external IPAM drivers.
	ipamRegistry drvregistry.IPAMs

	// The driver registry for all internal and external network drivers.
	networkRegistry drvregistry.Networks

	// Local network state used by cnmNetworkAllocator to do network management.
	networks map[string]*network

	// Allocator state to indicate if allocation has been
	// successfully completed for this service.
	services map[string]struct{}

	// Allocator state to indicate if allocation has been
	// successfully completed for this task.
	tasks map[string]struct{}

	// Allocator state to indicate if allocation has been
	// successfully completed for this node on this network.
	// outer map key: node id
	// inner map key: network id
	nodes map[string]map[string]struct{}
}

// Local in-memory state related to network that need to be tracked by cnmNetworkAllocator
type network struct {
	// A local cache of the store object.
	nw *api.Network

	// pools is used to save the internal poolIDs needed when
	// releasing the pool.
	pools map[string]string

	// endpoints is a map of endpoint IP to the poolID from which it
	// was allocated.
	endpoints map[string]string

	// isNodeLocal indicates whether the scope of the network's resources
	// is local to the node. If true, it means the resources can only be
	// allocated locally by the node where the network will be deployed.
	// In this the swarm manager will skip the allocations.
	isNodeLocal bool
}

type networkDriver struct {
	driver     driverapi.Driver
	name       string
	capability *driverapi.Capability
}

// NewAllocator returns a new NetworkAllocator handle
func (p *Provider) NewAllocator(netConfig *networkallocator.Config) (networkallocator.NetworkAllocator, error) {
	na := &cnmNetworkAllocator{
		networks: make(map[string]*network),
		services: make(map[string]struct{}),
		tasks:    make(map[string]struct{}),
		nodes:    make(map[string]map[string]struct{}),
		pg:       p.pg,
	}

	for ntype, i := range initializers {
		if err := i(&na.networkRegistry); err != nil {
			return nil, fmt.Errorf("failed to register %q network driver: %w", ntype, err)
		}
	}
	if err := remote.Register(&na.networkRegistry, p.pg); err != nil {
		return nil, fmt.Errorf("failed to initialize network driver plugins: %w", err)
	}

	if err := initIPAMDrivers(&na.ipamRegistry, netConfig); err != nil {
		return nil, err
	}
	if err := remoteipam.Register(&na.ipamRegistry, p.pg); err != nil {
		return nil, fmt.Errorf("failed to initialize IPAM driver plugins: %w", err)
	}

	return na, nil
}

// Allocate allocates all the necessary resources both general
// and driver-specific which may be specified in the NetworkSpec
func (na *cnmNetworkAllocator) Allocate(n *api.Network) error {
	if _, ok := na.networks[n.ID]; ok {
		return fmt.Errorf("network %s already allocated", n.ID)
	}

	d, err := na.resolveDriver(n)
	if err != nil {
		return err
	}

	nw := &network{
		nw:          n,
		endpoints:   make(map[string]string),
		isNodeLocal: d.capability.DataScope == scope.Local,
	}

	// No swarm-level allocation can be provided by the network driver for
	// node-local networks. Only thing needed is populating the driver's name
	// in the driver's state.
	if nw.isNodeLocal {
		n.DriverState = &api.Driver{
			Name: d.name,
		}
		// In order to support backward compatibility with older daemon
		// versions which assumes the network attachment to contains
		// non nil IPAM attribute, passing an empty object
		n.IPAM = &api.IPAMOptions{Driver: &api.Driver{}}
	} else {
		nw.pools, err = na.allocatePools(n)
		if err != nil {
			return errors.Wrapf(err, "failed allocating pools and gateway IP for network %s", n.ID)
		}

		if err := na.allocateDriverState(n); err != nil {
			na.freePools(n, nw.pools)
			return errors.Wrapf(err, "failed while allocating driver state for network %s", n.ID)
		}
	}

	na.networks[n.ID] = nw

	return nil
}

func (na *cnmNetworkAllocator) getNetwork(id string) *network {
	return na.networks[id]
}

// Deallocate frees all the general and driver specific resources
// which were assigned to the passed network.
func (na *cnmNetworkAllocator) Deallocate(n *api.Network) error {
	localNet := na.getNetwork(n.ID)
	if localNet == nil {
		return fmt.Errorf("could not get networker state for network %s", n.ID)
	}

	// No swarm-level resource deallocation needed for node-local networks
	if localNet.isNodeLocal {
		delete(na.networks, n.ID)
		return nil
	}

	if err := na.freeDriverState(n); err != nil {
		return errors.Wrapf(err, "failed to free driver state for network %s", n.ID)
	}

	delete(na.networks, n.ID)

	return na.freePools(n, localNet.pools)
}

// AllocateService allocates all the network resources such as virtual
// IP needed by the service.
func (na *cnmNetworkAllocator) AllocateService(s *api.Service) (err error) {
	defer func() {
		if err != nil {
			na.DeallocateService(s)
		}
	}()

	if s.Endpoint == nil {
		s.Endpoint = &api.Endpoint{}
	}
	s.Endpoint.Spec = s.Spec.Endpoint.Copy()

	// If ResolutionMode is DNSRR do not try allocating VIPs, but
	// free any VIP from previous state.
	if s.Spec.Endpoint != nil && s.Spec.Endpoint.Mode == api.ResolutionModeDNSRoundRobin {
		for _, vip := range s.Endpoint.VirtualIPs {
			if err := na.deallocateVIP(vip); err != nil {
				// don't bail here, deallocate as many as possible.
				log.L.WithError(err).
					WithField("vip.network", vip.NetworkID).
					WithField("vip.addr", vip.Addr).Error("error deallocating vip")
			}
		}

		s.Endpoint.VirtualIPs = nil

		delete(na.services, s.ID)
		return nil
	}

	specNetworks := serviceNetworks(s)

	// Allocate VIPs for all the pre-populated endpoint attachments
	eVIPs := s.Endpoint.VirtualIPs[:0]

vipLoop:
	for _, eAttach := range s.Endpoint.VirtualIPs {
		if na.IsVIPOnIngressNetwork(eAttach) && networkallocator.IsIngressNetworkNeeded(s) {
			if err = na.allocateVIP(eAttach); err != nil {
				return err
			}
			eVIPs = append(eVIPs, eAttach)
			continue vipLoop

		}
		for _, nAttach := range specNetworks {
			if nAttach.Target == eAttach.NetworkID {
				log.L.WithFields(log.Fields{"service_id": s.ID, "vip": eAttach.Addr}).Debug("allocate vip")
				if err = na.allocateVIP(eAttach); err != nil {
					return err
				}
				eVIPs = append(eVIPs, eAttach)
				continue vipLoop
			}
		}
		// If the network of the VIP is not part of the service spec,
		// deallocate the vip
		na.deallocateVIP(eAttach)
	}

networkLoop:
	for _, nAttach := range specNetworks {
		for _, vip := range s.Endpoint.VirtualIPs {
			if vip.NetworkID == nAttach.Target {
				continue networkLoop
			}
		}

		vip := &api.Endpoint_VirtualIP{NetworkID: nAttach.Target}
		if err = na.allocateVIP(vip); err != nil {
			return err
		}

		eVIPs = append(eVIPs, vip)
	}

	if len(eVIPs) > 0 {
		na.services[s.ID] = struct{}{}
	} else {
		delete(na.services, s.ID)
	}

	s.Endpoint.VirtualIPs = eVIPs
	return nil
}

// DeallocateService de-allocates all the network resources such as
// virtual IP associated with the service.
func (na *cnmNetworkAllocator) DeallocateService(s *api.Service) error {
	if s.Endpoint == nil {
		return nil
	}

	for _, vip := range s.Endpoint.VirtualIPs {
		if err := na.deallocateVIP(vip); err != nil {
			// don't bail here, deallocate as many as possible.
			log.L.WithError(err).
				WithField("vip.network", vip.NetworkID).
				WithField("vip.addr", vip.Addr).Error("error deallocating vip")
		}
	}
	s.Endpoint.VirtualIPs = nil

	delete(na.services, s.ID)

	return nil
}

// IsAllocated returns if the passed network has been allocated or not.
func (na *cnmNetworkAllocator) IsAllocated(n *api.Network) bool {
	_, ok := na.networks[n.ID]
	return ok
}

// IsTaskAllocated returns if the passed task has its network resources allocated or not.
func (na *cnmNetworkAllocator) IsTaskAllocated(t *api.Task) bool {
	// If the task is not found in the allocated set, then it is
	// not allocated.
	if _, ok := na.tasks[t.ID]; !ok {
		return false
	}

	// If Networks is empty there is no way this Task is allocated.
	if len(t.Networks) == 0 {
		return false
	}

	// To determine whether the task has its resources allocated,
	// we just need to look at one global scope network (in case of
	// multi-network attachment).  This is because we make sure we
	// allocate for every network or we allocate for none.

	// Find the first global scope network
	for _, nAttach := range t.Networks {
		// If the network is not allocated, the task cannot be allocated.
		localNet, ok := na.networks[nAttach.Network.ID]
		if !ok {
			return false
		}

		// Nothing else to check for local scope network
		if localNet.isNodeLocal {
			continue
		}

		// Addresses empty. Task is not allocated.
		if len(nAttach.Addresses) == 0 {
			return false
		}

		// The allocated IP address not found in local endpoint state. Not allocated.
		if _, ok := localNet.endpoints[nAttach.Addresses[0]]; !ok {
			return false
		}
	}

	return true
}

// IsServiceAllocated returns false if the passed service needs to have network resources allocated/updated.
func (na *cnmNetworkAllocator) IsServiceAllocated(s *api.Service, flags ...func(*networkallocator.ServiceAllocationOpts)) bool {
	specNetworks := serviceNetworks(s)

	// If endpoint mode is VIP and allocator does not have the
	// service in VIP allocated set then it needs to be allocated.
	if len(specNetworks) != 0 &&
		(s.Spec.Endpoint == nil ||
			s.Spec.Endpoint.Mode == api.ResolutionModeVirtualIP) {

		if _, ok := na.services[s.ID]; !ok {
			return false
		}

		if s.Endpoint == nil || len(s.Endpoint.VirtualIPs) == 0 {
			return false
		}

		// If the spec has networks which don't have a corresponding VIP,
		// the service needs to be allocated.
	networkLoop:
		for _, net := range specNetworks {
			for _, vip := range s.Endpoint.VirtualIPs {
				if vip.NetworkID == net.Target {
					continue networkLoop
				}
			}
			return false
		}
	}

	// If the spec no longer has networks attached and has a vip allocated
	// from previous spec the service needs to allocated.
	if s.Endpoint != nil {
	vipLoop:
		for _, vip := range s.Endpoint.VirtualIPs {
			if na.IsVIPOnIngressNetwork(vip) && networkallocator.IsIngressNetworkNeeded(s) {
				// This checks the condition when ingress network is needed
				// but allocation has not been done.
				if _, ok := na.services[s.ID]; !ok {
					return false
				}
				continue vipLoop
			}
			for _, net := range specNetworks {
				if vip.NetworkID == net.Target {
					continue vipLoop
				}
			}
			return false
		}
	}

	// If the endpoint mode is DNSRR and allocator has the service
	// in VIP allocated set then we return to be allocated to make
	// sure the allocator triggers networkallocator to free up the
	// resources if any.
	if s.Spec.Endpoint != nil && s.Spec.Endpoint.Mode == api.ResolutionModeDNSRoundRobin {
		if _, ok := na.services[s.ID]; ok {
			return false
		}
	}

	return true
}

// AllocateTask allocates all the endpoint resources for all the
// networks that a task is attached to.
func (na *cnmNetworkAllocator) AllocateTask(t *api.Task) error {
	for i, nAttach := range t.Networks {
		if localNet := na.getNetwork(nAttach.Network.ID); localNet != nil && localNet.isNodeLocal {
			continue
		}
		if err := na.allocateNetworkIPs(nAttach); err != nil {
			if err := na.releaseEndpoints(t.Networks[:i]); err != nil {
				log.G(context.TODO()).WithError(err).Errorf("failed to release IP addresses while rolling back allocation for task %s network %s", t.ID, nAttach.Network.ID)
			}
			return errors.Wrapf(err, "failed to allocate network IP for task %s network %s", t.ID, nAttach.Network.ID)
		}
	}

	na.tasks[t.ID] = struct{}{}

	return nil
}

// DeallocateTask releases all the endpoint resources for all the
// networks that a task is attached to.
func (na *cnmNetworkAllocator) DeallocateTask(t *api.Task) error {
	delete(na.tasks, t.ID)
	return na.releaseEndpoints(t.Networks)
}

// IsAttachmentAllocated returns if the passed node and network has resources allocated or not.
func (na *cnmNetworkAllocator) IsAttachmentAllocated(node *api.Node, networkAttachment *api.NetworkAttachment) bool {
	if node == nil {
		return false
	}

	if networkAttachment == nil || networkAttachment.Network == nil {
		return false
	}

	// If the node is not found in the allocated set, then it is
	// not allocated.
	if _, ok := na.nodes[node.ID]; !ok {
		return false
	}

	// If the network is not found in the allocated set, then it is
	// not allocated.
	if _, ok := na.nodes[node.ID][networkAttachment.Network.ID]; !ok {
		return false
	}

	// If the network is not allocated, the node cannot be allocated.
	localNet, ok := na.networks[networkAttachment.Network.ID]
	if !ok {
		return false
	}

	// Addresses empty, not allocated.
	if len(networkAttachment.Addresses) == 0 {
		return false
	}

	// The allocated IP address not found in local endpoint state. Not allocated.
	if _, ok := localNet.endpoints[networkAttachment.Addresses[0]]; !ok {
		return false
	}

	return true
}

// AllocateAttachment allocates the IP addresses for a LB in a network
// on a given node
func (na *cnmNetworkAllocator) AllocateAttachment(node *api.Node, networkAttachment *api.NetworkAttachment) error {

	if err := na.allocateNetworkIPs(networkAttachment); err != nil {
		return err
	}

	if na.nodes[node.ID] == nil {
		na.nodes[node.ID] = make(map[string]struct{})
	}
	na.nodes[node.ID][networkAttachment.Network.ID] = struct{}{}

	return nil
}

// DeallocateAttachment deallocates the IP addresses for a LB in a network to
// which the node is attached.
func (na *cnmNetworkAllocator) DeallocateAttachment(node *api.Node, networkAttachment *api.NetworkAttachment) error {

	delete(na.nodes[node.ID], networkAttachment.Network.ID)
	if len(na.nodes[node.ID]) == 0 {
		delete(na.nodes, node.ID)
	}

	return na.releaseEndpoints([]*api.NetworkAttachment{networkAttachment})
}

func (na *cnmNetworkAllocator) releaseEndpoints(networks []*api.NetworkAttachment) error {
	for _, nAttach := range networks {
		localNet := na.getNetwork(nAttach.Network.ID)
		if localNet == nil {
			return fmt.Errorf("could not find network allocator state for network %s", nAttach.Network.ID)
		}

		if localNet.isNodeLocal {
			continue
		}

		ipam, _, _, err := na.resolveIPAM(nAttach.Network)
		if err != nil {
			return errors.Wrap(err, "failed to resolve IPAM while releasing")
		}

		// Do not fail and bail out if we fail to release IP
		// address here. Keep going and try releasing as many
		// addresses as possible.
		for _, addr := range nAttach.Addresses {
			// Retrieve the poolID and immediately nuke
			// out the mapping.
			poolID := localNet.endpoints[addr]
			delete(localNet.endpoints, addr)

			ip, _, err := net.ParseCIDR(addr)
			if err != nil {
				log.G(context.TODO()).Errorf("Could not parse IP address %s while releasing", addr)
				continue
			}

			if err := ipam.ReleaseAddress(poolID, ip); err != nil {
				log.G(context.TODO()).WithError(err).Errorf("IPAM failure while releasing IP address %s", addr)
			}
		}

		// Clear out the address list when we are done with
		// this network.
		nAttach.Addresses = nil
	}

	return nil
}

// allocate virtual IP for a single endpoint attachment of the service.
func (na *cnmNetworkAllocator) allocateVIP(vip *api.Endpoint_VirtualIP) error {
	var opts map[string]string
	localNet := na.getNetwork(vip.NetworkID)
	if localNet == nil {
		return errors.New("networkallocator: could not find local network state")
	}

	if localNet.isNodeLocal {
		return nil
	}

	// If this IP is already allocated in memory we don't need to
	// do anything.
	if _, ok := localNet.endpoints[vip.Addr]; ok {
		return nil
	}

	ipam, _, _, err := na.resolveIPAM(localNet.nw)
	if err != nil {
		return errors.Wrap(err, "failed to resolve IPAM while allocating")
	}

	var addr net.IP
	if vip.Addr != "" {
		var err error

		addr, _, err = net.ParseCIDR(vip.Addr)
		if err != nil {
			return err
		}
	}
	if localNet.nw.IPAM != nil && localNet.nw.IPAM.Driver != nil {
		// set ipam allocation method to serial
		opts = setIPAMSerialAlloc(localNet.nw.IPAM.Driver.Options)
	}

	for _, poolID := range localNet.pools {
		ip, _, err := ipam.RequestAddress(poolID, addr, opts)
		if err != nil && err != ipamapi.ErrNoAvailableIPs && err != ipamapi.ErrIPOutOfRange {
			return errors.Wrap(err, "could not allocate VIP from IPAM")
		}

		// If we got an address then we are done.
		if err == nil {
			ipStr := ip.String()
			localNet.endpoints[ipStr] = poolID
			vip.Addr = ipStr
			return nil
		}
	}

	return errors.New("could not find an available IP while allocating VIP")
}

func (na *cnmNetworkAllocator) deallocateVIP(vip *api.Endpoint_VirtualIP) error {
	localNet := na.getNetwork(vip.NetworkID)
	if localNet == nil {
		return errors.New("networkallocator: could not find local network state")
	}
	if localNet.isNodeLocal {
		return nil
	}
	ipam, _, _, err := na.resolveIPAM(localNet.nw)
	if err != nil {
		return errors.Wrap(err, "failed to resolve IPAM while allocating")
	}

	// Retrieve the poolID and immediately nuke
	// out the mapping.
	poolID := localNet.endpoints[vip.Addr]
	delete(localNet.endpoints, vip.Addr)

	ip, _, err := net.ParseCIDR(vip.Addr)
	if err != nil {
		log.G(context.TODO()).Errorf("Could not parse VIP address %s while releasing", vip.Addr)
		return err
	}

	if err := ipam.ReleaseAddress(poolID, ip); err != nil {
		log.G(context.TODO()).WithError(err).Errorf("IPAM failure while releasing VIP address %s", vip.Addr)
		return err
	}

	return nil
}

// allocate the IP addresses for a single network attachment of the task.
func (na *cnmNetworkAllocator) allocateNetworkIPs(nAttach *api.NetworkAttachment) error {
	var ip *net.IPNet
	var opts map[string]string

	ipam, _, _, err := na.resolveIPAM(nAttach.Network)
	if err != nil {
		return errors.Wrap(err, "failed to resolve IPAM while allocating")
	}

	localNet := na.getNetwork(nAttach.Network.ID)
	if localNet == nil {
		return fmt.Errorf("could not find network allocator state for network %s", nAttach.Network.ID)
	}

	addresses := nAttach.Addresses
	if len(addresses) == 0 {
		addresses = []string{""}
	}

	for i, rawAddr := range addresses {
		var addr net.IP
		if rawAddr != "" {
			var err error
			addr, _, err = net.ParseCIDR(rawAddr)
			if err != nil {
				addr = net.ParseIP(rawAddr)

				if addr == nil {
					return errors.Wrapf(err, "could not parse address string %s", rawAddr)
				}
			}
		}
		// Set the ipam options if the network has an ipam driver.
		if localNet.nw.IPAM != nil && localNet.nw.IPAM.Driver != nil {
			// set ipam allocation method to serial
			opts = setIPAMSerialAlloc(localNet.nw.IPAM.Driver.Options)
		}

		for _, poolID := range localNet.pools {
			var err error

			ip, _, err = ipam.RequestAddress(poolID, addr, opts)
			if err != nil && err != ipamapi.ErrNoAvailableIPs && err != ipamapi.ErrIPOutOfRange {
				return errors.Wrap(err, "could not allocate IP from IPAM")
			}

			// If we got an address then we are done.
			if err == nil {
				ipStr := ip.String()
				localNet.endpoints[ipStr] = poolID
				addresses[i] = ipStr
				nAttach.Addresses = addresses
				return nil
			}
		}
	}

	return errors.New("could not find an available IP")
}

func (na *cnmNetworkAllocator) freeDriverState(n *api.Network) error {
	d, err := na.resolveDriver(n)
	if err != nil {
		return err
	}

	return d.driver.NetworkFree(n.ID)
}

func (na *cnmNetworkAllocator) allocateDriverState(n *api.Network) error {
	d, err := na.resolveDriver(n)
	if err != nil {
		return err
	}

	options := make(map[string]string)
	// reconcile the driver specific options from the network spec
	// and from the operational state retrieved from the store
	if n.Spec.DriverConfig != nil {
		for k, v := range n.Spec.DriverConfig.Options {
			options[k] = v
		}
	}
	if n.DriverState != nil {
		for k, v := range n.DriverState.Options {
			options[k] = v
		}
	}

	// Construct IPAM data for driver consumption.
	ipv4Data := make([]driverapi.IPAMData, 0, len(n.IPAM.Configs))
	for _, ic := range n.IPAM.Configs {
		if ic.Family == api.IPAMConfig_IPV6 {
			continue
		}

		_, subnet, err := net.ParseCIDR(ic.Subnet)
		if err != nil {
			return errors.Wrapf(err, "error parsing subnet %s while allocating driver state", ic.Subnet)
		}

		gwIP := net.ParseIP(ic.Gateway)
		gwNet := &net.IPNet{
			IP:   gwIP,
			Mask: subnet.Mask,
		}

		data := driverapi.IPAMData{
			Pool:    subnet,
			Gateway: gwNet,
		}

		ipv4Data = append(ipv4Data, data)
	}

	ds, err := d.driver.NetworkAllocate(n.ID, options, ipv4Data, nil)
	if err != nil {
		return err
	}

	// Update network object with the obtained driver state.
	n.DriverState = &api.Driver{
		Name:    d.name,
		Options: ds,
	}

	return nil
}

// Resolve network driver
func (na *cnmNetworkAllocator) resolveDriver(n *api.Network) (*networkDriver, error) {
	dName := DefaultDriver
	if n.Spec.DriverConfig != nil && n.Spec.DriverConfig.Name != "" {
		dName = n.Spec.DriverConfig.Name
	}

	d, drvcap := na.networkRegistry.Driver(dName)
	if d == nil {
		err := na.loadDriver(dName)
		if err != nil {
			return nil, err
		}

		d, drvcap = na.networkRegistry.Driver(dName)
		if d == nil {
			return nil, fmt.Errorf("could not resolve network driver %s", dName)
		}
	}

	return &networkDriver{driver: d, capability: &drvcap, name: dName}, nil
}

func (na *cnmNetworkAllocator) loadDriver(name string) error {
	if na.pg == nil {
		return errors.New("plugin store is uninitialized")
	}
	_, err := na.pg.Get(name, driverapi.NetworkPluginEndpointType, plugingetter.Lookup)
	return err
}

// Resolve the IPAM driver
func (na *cnmNetworkAllocator) resolveIPAM(n *api.Network) (ipamapi.Ipam, string, map[string]string, error) {
	dName := ipamapi.DefaultIPAM
	if n.Spec.IPAM != nil && n.Spec.IPAM.Driver != nil && n.Spec.IPAM.Driver.Name != "" {
		dName = n.Spec.IPAM.Driver.Name
	}

	var dOptions map[string]string
	if n.Spec.IPAM != nil && n.Spec.IPAM.Driver != nil && len(n.Spec.IPAM.Driver.Options) != 0 {
		dOptions = n.Spec.IPAM.Driver.Options
	}

	ipam, _ := na.ipamRegistry.IPAM(dName)
	if ipam == nil {
		return nil, "", nil, fmt.Errorf("could not resolve IPAM driver %s", dName)
	}

	return ipam, dName, dOptions, nil
}

func (na *cnmNetworkAllocator) freePools(n *api.Network, pools map[string]string) error {
	ipam, _, _, err := na.resolveIPAM(n)
	if err != nil {
		return errors.Wrapf(err, "failed to resolve IPAM while freeing pools for network %s", n.ID)
	}

	releasePools(ipam, n.IPAM.Configs, pools)
	return nil
}

func releasePools(ipam ipamapi.Ipam, icList []*api.IPAMConfig, pools map[string]string) {
	for _, ic := range icList {
		if err := ipam.ReleaseAddress(pools[ic.Subnet], net.ParseIP(ic.Gateway)); err != nil {
			log.G(context.TODO()).WithError(err).Errorf("Failed to release address %s", ic.Subnet)
		}
	}

	for k, p := range pools {
		if err := ipam.ReleasePool(p); err != nil {
			log.G(context.TODO()).WithError(err).Errorf("Failed to release pool %s", k)
		}
	}
}

func (na *cnmNetworkAllocator) allocatePools(n *api.Network) (map[string]string, error) {
	ipam, dName, dOptions, err := na.resolveIPAM(n)
	if err != nil {
		return nil, err
	}

	// We don't support user defined address spaces yet so just
	// retrieve default address space names for the driver.
	_, asName, err := ipam.GetDefaultAddressSpaces()
	if err != nil {
		return nil, err
	}

	pools := make(map[string]string)

	var ipamConfigs []*api.IPAMConfig

	// If there is non-nil IPAM state always prefer those subnet
	// configs over Spec configs.
	if n.IPAM != nil {
		ipamConfigs = n.IPAM.Configs
	} else if n.Spec.IPAM != nil {
		ipamConfigs = make([]*api.IPAMConfig, len(n.Spec.IPAM.Configs))
		copy(ipamConfigs, n.Spec.IPAM.Configs)
	}

	// Append an empty slot for subnet allocation if there are no
	// IPAM configs from either spec or state.
	if len(ipamConfigs) == 0 {
		ipamConfigs = append(ipamConfigs, &api.IPAMConfig{Family: api.IPAMConfig_IPV4})
	}

	// Update the runtime IPAM configurations with initial state
	n.IPAM = &api.IPAMOptions{
		Driver:  &api.Driver{Name: dName, Options: dOptions},
		Configs: ipamConfigs,
	}

	for i, ic := range ipamConfigs {
		poolID, poolIP, meta, err := ipam.RequestPool(asName, ic.Subnet, ic.Range, dOptions, false)
		if err != nil {
			// Rollback by releasing all the resources allocated so far.
			releasePools(ipam, ipamConfigs[:i], pools)
			return nil, err
		}
		pools[poolIP.String()] = poolID

		// The IPAM contract allows the IPAM driver to autonomously
		// provide a network gateway in response to the pool request.
		// But if the network spec contains a gateway, we will allocate
		// it irrespective of whether the ipam driver returned one already.
		// If none of the above is true, we need to allocate one now, and
		// let the driver know this request is for the network gateway.
		var (
			gwIP *net.IPNet
			ip   net.IP
		)
		if gws, ok := meta[netlabel.Gateway]; ok {
			if ip, gwIP, err = net.ParseCIDR(gws); err != nil {
				return nil, fmt.Errorf("failed to parse gateway address (%v) returned by ipam driver: %v", gws, err)
			}
			gwIP.IP = ip
		}
		if dOptions == nil {
			dOptions = make(map[string]string)
		}
		dOptions[ipamapi.RequestAddressType] = netlabel.Gateway
		// set ipam allocation method to serial
		dOptions = setIPAMSerialAlloc(dOptions)
		defer delete(dOptions, ipamapi.RequestAddressType)

		if ic.Gateway != "" || gwIP == nil {
			gwIP, _, err = ipam.RequestAddress(poolID, net.ParseIP(ic.Gateway), dOptions)
			if err != nil {
				// Rollback by releasing all the resources allocated so far.
				releasePools(ipam, ipamConfigs[:i], pools)
				return nil, err
			}
		}

		if ic.Subnet == "" {
			ic.Subnet = poolIP.String()
		}

		if ic.Gateway == "" {
			ic.Gateway = gwIP.IP.String()
		}

	}

	return pools, nil
}

func serviceNetworks(s *api.Service) []*api.NetworkAttachmentConfig {
	// Always prefer NetworkAttachmentConfig in the TaskSpec
	if len(s.Spec.Task.Networks) == 0 && len(s.Spec.Networks) != 0 {
		return s.Spec.Networks
	}
	return s.Spec.Task.Networks
}

// IsVIPOnIngressNetwork check if the vip is in ingress network
func (na *cnmNetworkAllocator) IsVIPOnIngressNetwork(vip *api.Endpoint_VirtualIP) bool {
	if vip == nil {
		return false
	}

	localNet := na.getNetwork(vip.NetworkID)
	if localNet != nil && localNet.nw != nil {
		return networkallocator.IsIngressNetwork(localNet.nw)
	}
	return false
}

// IsBuiltInDriver returns whether the passed driver is an internal network driver
func IsBuiltInDriver(name string) bool {
	n := strings.ToLower(name)
	_, ok := initializers[n]
	return ok
}

// setIPAMSerialAlloc sets the ipam allocation method to serial
func setIPAMSerialAlloc(opts map[string]string) map[string]string {
	if opts == nil {
		opts = make(map[string]string)
	}
	if _, ok := opts[ipamapi.AllocSerialPrefix]; !ok {
		opts[ipamapi.AllocSerialPrefix] = "true"
	}
	return opts
}