IPvlan and macvlan driver to persist endpoints

Signed-off-by: Alessandro Boch <aboch@docker.com>

libnetwork/drivers/ipvlan/ipvlan.go

@@ -36,11 +36,14 @@ type driver struct {
 }
 
 type endpoint struct {
-	id      string
-	mac     net.HardwareAddr
-	addr    *net.IPNet
-	addrv6  *net.IPNet
-	srcName string
+	id       string
+	nid      string
+	mac      net.HardwareAddr
+	addr     *net.IPNet
+	addrv6   *net.IPNet
+	srcName  string
+	dbIndex  uint64
+	dbExists bool
 }
 
 type network struct {

libnetwork/drivers/ipvlan/ipvlan_endpoint.go

@@ -28,9 +28,9 @@ func (d *driver) CreateEndpoint(nid, eid string, ifInfo driverapi.InterfaceInfo,
 	}
 	ep := &endpoint{
 		id:     eid,
+		nid:    nid,
 		addr:   ifInfo.Address(),
 		addrv6: ifInfo.AddressIPv6(),
-		mac:    ifInfo.MacAddress(),
 	}
 	if ep.addr == nil {
 		return fmt.Errorf("create endpoint was not passed an IP address")
@@ -51,6 +51,11 @@ func (d *driver) CreateEndpoint(nid, eid string, ifInfo driverapi.InterfaceInfo,
 			}
 		}
 	}
+
+	if err := d.storeUpdate(ep); err != nil {
+		return fmt.Errorf("failed to save ipvlan endpoint %s to store: %v", ep.id[0:7], err)
+	}
+
 	n.addEndpoint(ep)
 
 	return nil
@@ -74,5 +79,9 @@ func (d *driver) DeleteEndpoint(nid, eid string) error {
 		ns.NlHandle().LinkDel(link)
 	}
 
+	if err := d.storeDelete(ep); err != nil {
+		logrus.Warnf("Failed to remove ipvlan endpoint %s from store: %v", ep.id[0:7], err)
+	}
+
 	return nil
 }

libnetwork/drivers/ipvlan/ipvlan_joinleave.go

@@ -116,6 +116,9 @@ func (d *driver) Join(nid, eid string, sboxKey string, jinfo driverapi.JoinInfo,
 	if err != nil {
 		return err
 	}
+	if err = d.storeUpdate(ep); err != nil {
+		return fmt.Errorf("failed to save ipvlan endpoint %s to store: %v", ep.id[0:7], err)
+	}
 
 	return nil
 }

libnetwork/drivers/ipvlan/ipvlan_store.go

@@ -3,6 +3,7 @@ package ipvlan
 import (
 	"encoding/json"
 	"fmt"
+	"net"
 
 	"github.com/Sirupsen/logrus"
 	"github.com/docker/libnetwork/datastore"
@@ -11,7 +12,11 @@ import (
 	"github.com/docker/libnetwork/types"
 )
 
-const ipvlanPrefix = "ipvlan" // prefix used for persistent driver storage
+const (
+	ipvlanPrefix         = "ipvlan"
+	ipvlanNetworkPrefix  = ipvlanPrefix + "/network"
+	ipvlanEndpointPrefix = ipvlanPrefix + "/endpoint"
+)
 
 // networkConfiguration for this driver's network specific configuration
 type configuration struct {
@@ -58,7 +63,7 @@ func (d *driver) initStore(option map[string]interface{}) error {
 
 // populateNetworks is invoked at driver init to recreate persistently stored networks
 func (d *driver) populateNetworks() error {
-	kvol, err := d.store.List(datastore.Key(ipvlanPrefix), &configuration{})
+	kvol, err := d.store.List(datastore.Key(ipvlanNetworkPrefix), &configuration{})
 	if err != nil && err != datastore.ErrKeyNotFound {
 		return fmt.Errorf("failed to get ipvlan network configurations from store: %v", err)
 	}
@@ -76,6 +81,34 @@ func (d *driver) populateNetworks() error {
 	return nil
 }
 
+func (d *driver) populateEndpoints() error {
+	kvol, err := d.store.List(datastore.Key(ipvlanEndpointPrefix), &endpoint{})
+	if err != nil && err != datastore.ErrKeyNotFound {
+		return fmt.Errorf("failed to get ipvlan endpoints from store: %v", err)
+	}
+
+	if err == datastore.ErrKeyNotFound {
+		return nil
+	}
+
+	for _, kvo := range kvol {
+		ep := kvo.(*endpoint)
+		n, ok := d.networks[ep.nid]
+		if !ok {
+			logrus.Debugf("Network (%s) not found for restored ipvlan endpoint (%s)", ep.nid[0:7], ep.id[0:7])
+			logrus.Debugf("Deleting stale ipvlan endpoint (%s) from store", ep.nid[0:7])
+			if err := d.storeDelete(ep); err != nil {
+				logrus.Debugf("Failed to delete stale ipvlan endpoint (%s) from store", ep.nid[0:7])
+			}
+			continue
+		}
+		n.endpoints[ep.id] = ep
+		logrus.Debugf("Endpoint (%s) restored to network (%s)", ep.id[0:7], ep.nid[0:7])
+	}
+
+	return nil
+}
+
 // storeUpdate used to update persistent ipvlan network records as they are created
 func (d *driver) storeUpdate(kvObject datastore.KVObject) error {
 	if d.store == nil {
@@ -165,11 +198,11 @@ func (config *configuration) UnmarshalJSON(b []byte) error {
 }
 
 func (config *configuration) Key() []string {
-	return []string{ipvlanPrefix, config.ID}
+	return []string{ipvlanNetworkPrefix, config.ID}
 }
 
 func (config *configuration) KeyPrefix() []string {
-	return []string{ipvlanPrefix}
+	return []string{ipvlanNetworkPrefix}
 }
 
 func (config *configuration) Value() []byte {
@@ -214,3 +247,103 @@ func (config *configuration) CopyTo(o datastore.KVObject) error {
 func (config *configuration) DataScope() string {
 	return datastore.LocalScope
 }
+
+func (ep *endpoint) MarshalJSON() ([]byte, error) {
+	epMap := make(map[string]interface{})
+	epMap["id"] = ep.id
+	epMap["nid"] = ep.nid
+	epMap["SrcName"] = ep.srcName
+	if len(ep.mac) != 0 {
+		epMap["MacAddress"] = ep.mac.String()
+	}
+	if ep.addr != nil {
+		epMap["Addr"] = ep.addr.String()
+	}
+	if ep.addrv6 != nil {
+		epMap["Addrv6"] = ep.addrv6.String()
+	}
+	return json.Marshal(epMap)
+}
+
+func (ep *endpoint) UnmarshalJSON(b []byte) error {
+	var (
+		err   error
+		epMap map[string]interface{}
+	)
+
+	if err = json.Unmarshal(b, &epMap); err != nil {
+		return fmt.Errorf("Failed to unmarshal to ipvlan endpoint: %v", err)
+	}
+
+	if v, ok := epMap["MacAddress"]; ok {
+		if ep.mac, err = net.ParseMAC(v.(string)); err != nil {
+			return types.InternalErrorf("failed to decode ipvlan endpoint MAC address (%s) after json unmarshal: %v", v.(string), err)
+		}
+	}
+	if v, ok := epMap["Addr"]; ok {
+		if ep.addr, err = types.ParseCIDR(v.(string)); err != nil {
+			return types.InternalErrorf("failed to decode ipvlan endpoint IPv4 address (%s) after json unmarshal: %v", v.(string), err)
+		}
+	}
+	if v, ok := epMap["Addrv6"]; ok {
+		if ep.addrv6, err = types.ParseCIDR(v.(string)); err != nil {
+			return types.InternalErrorf("failed to decode ipvlan endpoint IPv6 address (%s) after json unmarshal: %v", v.(string), err)
+		}
+	}
+	ep.id = epMap["id"].(string)
+	ep.nid = epMap["nid"].(string)
+	ep.srcName = epMap["SrcName"].(string)
+
+	return nil
+}
+
+func (ep *endpoint) Key() []string {
+	return []string{ipvlanEndpointPrefix, ep.id}
+}
+
+func (ep *endpoint) KeyPrefix() []string {
+	return []string{ipvlanEndpointPrefix}
+}
+
+func (ep *endpoint) Value() []byte {
+	b, err := json.Marshal(ep)
+	if err != nil {
+		return nil
+	}
+	return b
+}
+
+func (ep *endpoint) SetValue(value []byte) error {
+	return json.Unmarshal(value, ep)
+}
+
+func (ep *endpoint) Index() uint64 {
+	return ep.dbIndex
+}
+
+func (ep *endpoint) SetIndex(index uint64) {
+	ep.dbIndex = index
+	ep.dbExists = true
+}
+
+func (ep *endpoint) Exists() bool {
+	return ep.dbExists
+}
+
+func (ep *endpoint) Skip() bool {
+	return false
+}
+
+func (ep *endpoint) New() datastore.KVObject {
+	return &endpoint{}
+}
+
+func (ep *endpoint) CopyTo(o datastore.KVObject) error {
+	dstEp := o.(*endpoint)
+	*dstEp = *ep
+	return nil
+}
+
+func (ep *endpoint) DataScope() string {
+	return datastore.LocalScope
+}

libnetwork/drivers/macvlan/macvlan.go

@@ -38,11 +38,14 @@ type driver struct {
 }
 
 type endpoint struct {
-	id      string
-	mac     net.HardwareAddr
-	addr    *net.IPNet
-	addrv6  *net.IPNet
-	srcName string
+	id       string
+	nid      string
+	mac      net.HardwareAddr
+	addr     *net.IPNet
+	addrv6   *net.IPNet
+	srcName  string
+	dbIndex  uint64
+	dbExists bool
 }
 
 type network struct {

libnetwork/drivers/macvlan/macvlan_endpoint.go

@@ -26,6 +26,7 @@ func (d *driver) CreateEndpoint(nid, eid string, ifInfo driverapi.InterfaceInfo,
 	}
 	ep := &endpoint{
 		id:     eid,
+		nid:    nid,
 		addr:   ifInfo.Address(),
 		addrv6: ifInfo.AddressIPv6(),
 		mac:    ifInfo.MacAddress(),
@@ -55,6 +56,11 @@ func (d *driver) CreateEndpoint(nid, eid string, ifInfo driverapi.InterfaceInfo,
 			}
 		}
 	}
+
+	if err := d.storeUpdate(ep); err != nil {
+		return fmt.Errorf("failed to save macvlan endpoint %s to store: %v", ep.id[0:7], err)
+	}
+
 	n.addEndpoint(ep)
 
 	return nil
@@ -77,6 +83,8 @@ func (d *driver) DeleteEndpoint(nid, eid string) error {
 	if link, err := ns.NlHandle().LinkByName(ep.srcName); err == nil {
 		ns.NlHandle().LinkDel(link)
 	}
-
+	if err := d.storeDelete(ep); err != nil {
+		logrus.Warnf("Failed to remove macvlan endpoint %s from store: %v", ep.id[0:7], err)
+	}
 	return nil
 }

libnetwork/drivers/macvlan/macvlan_joinleave.go

@@ -77,7 +77,9 @@ func (d *driver) Join(nid, eid string, sboxKey string, jinfo driverapi.JoinInfo,
 	if err != nil {
 		return err
 	}
-
+	if err := d.storeUpdate(ep); err != nil {
+		return fmt.Errorf("failed to save macvlan endpoint %s to store: %v", ep.id[0:7], err)
+	}
 	return nil
 }
 

libnetwork/drivers/macvlan/macvlan_store.go

@@ -3,6 +3,7 @@ package macvlan
 import (
 	"encoding/json"
 	"fmt"
+	"net"
 
 	"github.com/Sirupsen/logrus"
 	"github.com/docker/libnetwork/datastore"
@@ -11,7 +12,11 @@ import (
 	"github.com/docker/libnetwork/types"
 )
 
-const macvlanPrefix = "macvlan" // prefix used for persistent driver storage
+const (
+	macvlanPrefix         = "macvlan"
+	macvlanNetworkPrefix  = macvlanPrefix + "/network"
+	macvlanEndpointPrefix = macvlanPrefix + "/endpoint"
+)
 
 // networkConfiguration for this driver's network specific configuration
 type configuration struct {
@@ -76,6 +81,34 @@ func (d *driver) populateNetworks() error {
 	return nil
 }
 
+func (d *driver) populateEndpoints() error {
+	kvol, err := d.store.List(datastore.Key(macvlanEndpointPrefix), &endpoint{})
+	if err != nil && err != datastore.ErrKeyNotFound {
+		return fmt.Errorf("failed to get macvlan endpoints from store: %v", err)
+	}
+
+	if err == datastore.ErrKeyNotFound {
+		return nil
+	}
+
+	for _, kvo := range kvol {
+		ep := kvo.(*endpoint)
+		n, ok := d.networks[ep.nid]
+		if !ok {
+			logrus.Debugf("Network (%s) not found for restored macvlan endpoint (%s)", ep.nid[0:7], ep.id[0:7])
+			logrus.Debugf("Deleting stale macvlan endpoint (%s) from store", ep.nid[0:7])
+			if err := d.storeDelete(ep); err != nil {
+				logrus.Debugf("Failed to delete stale macvlan endpoint (%s) from store", ep.nid[0:7])
+			}
+			continue
+		}
+		n.endpoints[ep.id] = ep
+		logrus.Debugf("Endpoint (%s) restored to network (%s)", ep.id[0:7], ep.nid[0:7])
+	}
+
+	return nil
+}
+
 // storeUpdate used to update persistent macvlan network records as they are created
 func (d *driver) storeUpdate(kvObject datastore.KVObject) error {
 	if d.store == nil {
@@ -165,11 +198,11 @@ func (config *configuration) UnmarshalJSON(b []byte) error {
 }
 
 func (config *configuration) Key() []string {
-	return []string{macvlanPrefix, config.ID}
+	return []string{macvlanNetworkPrefix, config.ID}
 }
 
 func (config *configuration) KeyPrefix() []string {
-	return []string{macvlanPrefix}
+	return []string{macvlanNetworkPrefix}
 }
 
 func (config *configuration) Value() []byte {
@@ -216,3 +249,103 @@ func (config *configuration) CopyTo(o datastore.KVObject) error {
 func (config *configuration) DataScope() string {
 	return datastore.LocalScope
 }
+
+func (ep *endpoint) MarshalJSON() ([]byte, error) {
+	epMap := make(map[string]interface{})
+	epMap["id"] = ep.id
+	epMap["nid"] = ep.nid
+	epMap["SrcName"] = ep.srcName
+	if len(ep.mac) != 0 {
+		epMap["MacAddress"] = ep.mac.String()
+	}
+	if ep.addr != nil {
+		epMap["Addr"] = ep.addr.String()
+	}
+	if ep.addrv6 != nil {
+		epMap["Addrv6"] = ep.addrv6.String()
+	}
+	return json.Marshal(epMap)
+}
+
+func (ep *endpoint) UnmarshalJSON(b []byte) error {
+	var (
+		err   error
+		epMap map[string]interface{}
+	)
+
+	if err = json.Unmarshal(b, &epMap); err != nil {
+		return fmt.Errorf("Failed to unmarshal to macvlan endpoint: %v", err)
+	}
+
+	if v, ok := epMap["MacAddress"]; ok {
+		if ep.mac, err = net.ParseMAC(v.(string)); err != nil {
+			return types.InternalErrorf("failed to decode macvlan endpoint MAC address (%s) after json unmarshal: %v", v.(string), err)
+		}
+	}
+	if v, ok := epMap["Addr"]; ok {
+		if ep.addr, err = types.ParseCIDR(v.(string)); err != nil {
+			return types.InternalErrorf("failed to decode macvlan endpoint IPv4 address (%s) after json unmarshal: %v", v.(string), err)
+		}
+	}
+	if v, ok := epMap["Addrv6"]; ok {
+		if ep.addrv6, err = types.ParseCIDR(v.(string)); err != nil {
+			return types.InternalErrorf("failed to decode macvlan endpoint IPv6 address (%s) after json unmarshal: %v", v.(string), err)
+		}
+	}
+	ep.id = epMap["id"].(string)
+	ep.nid = epMap["nid"].(string)
+	ep.srcName = epMap["SrcName"].(string)
+
+	return nil
+}
+
+func (ep *endpoint) Key() []string {
+	return []string{macvlanEndpointPrefix, ep.id}
+}
+
+func (ep *endpoint) KeyPrefix() []string {
+	return []string{macvlanEndpointPrefix}
+}
+
+func (ep *endpoint) Value() []byte {
+	b, err := json.Marshal(ep)
+	if err != nil {
+		return nil
+	}
+	return b
+}
+
+func (ep *endpoint) SetValue(value []byte) error {
+	return json.Unmarshal(value, ep)
+}
+
+func (ep *endpoint) Index() uint64 {
+	return ep.dbIndex
+}
+
+func (ep *endpoint) SetIndex(index uint64) {
+	ep.dbIndex = index
+	ep.dbExists = true
+}
+
+func (ep *endpoint) Exists() bool {
+	return ep.dbExists
+}
+
+func (ep *endpoint) Skip() bool {
+	return false
+}
+
+func (ep *endpoint) New() datastore.KVObject {
+	return &endpoint{}
+}
+
+func (ep *endpoint) CopyTo(o datastore.KVObject) error {
+	dstEp := o.(*endpoint)
+	*dstEp = *ep
+	return nil
+}
+
+func (ep *endpoint) DataScope() string {
+	return datastore.LocalScope
+}