Bridge driver to persist endpoints

Signed-off-by: Alessandro Boch <aboch@docker.com>

libnetwork/drivers/bridge/bridge.go

@@ -91,6 +91,7 @@ type connectivityConfiguration struct {
 
 type bridgeEndpoint struct {
 	id              string
+	nid             string
 	srcName         string
 	addr            *net.IPNet
 	addrv6          *net.IPNet
@@ -99,6 +100,8 @@ type bridgeEndpoint struct {
 	containerConfig *containerConfiguration
 	extConnConfig   *connectivityConfiguration
 	portMapping     []types.PortBinding // Operation port bindings
+	dbIndex         uint64
+	dbExists        bool
 }
 
 type bridgeNetwork struct {
@@ -882,7 +885,7 @@ func (d *driver) CreateEndpoint(nid, eid string, ifInfo driverapi.InterfaceInfo,
 
 	// Create and add the endpoint
 	n.Lock()
-	endpoint := &bridgeEndpoint{id: eid, config: epConfig}
+	endpoint := &bridgeEndpoint{id: eid, nid: nid, config: epConfig}
 	n.endpoints[eid] = endpoint
 	n.Unlock()
 
@@ -1009,6 +1012,10 @@ func (d *driver) CreateEndpoint(nid, eid string, ifInfo driverapi.InterfaceInfo,
 		}
 	}
 
+	if err = d.storeUpdate(endpoint); err != nil {
+		return fmt.Errorf("failed to save bridge endpoint %s to store: %v", ep.id[0:7], err)
+	}
+
 	return nil
 }
 
@@ -1069,6 +1076,10 @@ func (d *driver) DeleteEndpoint(nid, eid string) error {
 		d.nlh.LinkDel(link)
 	}
 
+	if err := d.storeDelete(ep); err != nil {
+		logrus.Warnf("Failed to remove bridge endpoint %s from store: %v", ep.id[0:7], err)
+	}
+
 	return nil
 }
 
@@ -1225,6 +1236,11 @@ func (d *driver) ProgramExternalConnectivity(nid, eid string, options map[string
 		return err
 	}
 
+	if err = d.storeUpdate(endpoint); err != nil {
+		endpoint.portMapping = nil
+		return fmt.Errorf("failed to update bridge endpoint %s to store: %v", endpoint.id[0:7], err)
+	}
+
 	if !network.config.EnableICC {
 		return d.link(network, endpoint, true)
 	}

libnetwork/drivers/bridge/bridge_store.go

@@ -12,7 +12,13 @@ import (
 	"github.com/docker/libnetwork/types"
 )
 
-const bridgePrefix = "bridge"
+const (
+	// network config prefix was not specific enough.
+	// To be backward compatible, need custom endpoint
+	// prefix with different root
+	bridgePrefix         = "bridge"
+	bridgeEndpointPrefix = "bridge-endpoint"
+)
 
 func (d *driver) initStore(option map[string]interface{}) error {
 	if data, ok := option[netlabel.LocalKVClient]; ok {
@@ -26,7 +32,15 @@ func (d *driver) initStore(option map[string]interface{}) error {
 			return types.InternalErrorf("bridge driver failed to initialize data store: %v", err)
 		}
 
-		return d.populateNetworks()
+		err = d.populateNetworks()
+		if err != nil {
+			return err
+		}
+
+		err = d.populateEndpoints()
+		if err != nil {
+			return err
+		}
 	}
 
 	return nil
@@ -48,6 +62,36 @@ func (d *driver) populateNetworks() error {
 		if err = d.createNetwork(ncfg); err != nil {
 			logrus.Warnf("could not create bridge network for id %s bridge name %s while booting up from persistent state: %v", ncfg.ID, ncfg.BridgeName, err)
 		}
+		logrus.Debugf("Network (%s) restored", ncfg.ID[0:7])
+	}
+
+	return nil
+}
+
+func (d *driver) populateEndpoints() error {
+	kvol, err := d.store.List(datastore.Key(bridgeEndpointPrefix), &bridgeEndpoint{})
+	if err != nil && err != datastore.ErrKeyNotFound {
+		return fmt.Errorf("failed to get bridge endpoints from store: %v", err)
+	}
+
+	if err == datastore.ErrKeyNotFound {
+		return nil
+	}
+
+	for _, kvo := range kvol {
+		ep := kvo.(*bridgeEndpoint)
+		n, ok := d.networks[ep.nid]
+		if !ok {
+			logrus.Debugf("Network (%s) not found for restored bridge endpoint (%s)", ep.nid[0:7], ep.id[0:7])
+			logrus.Debugf("Deleting stale bridge endpoint (%s) from store", ep.nid[0:7])
+			if err := d.storeDelete(ep); err != nil {
+				logrus.Debugf("Failed to delete stale bridge endpoint (%s) from store", ep.nid[0:7])
+			}
+			continue
+		}
+		n.endpoints[ep.id] = ep
+		n.restorePortAllocations(ep)
+		logrus.Debugf("Endpoint (%s) restored to network (%s)", ep.id[0:7], ep.nid[0:7])
 	}
 
 	return nil
@@ -200,3 +244,135 @@ func (ncfg *networkConfiguration) CopyTo(o datastore.KVObject) error {
 func (ncfg *networkConfiguration) DataScope() string {
 	return datastore.LocalScope
 }
+
+func (ep *bridgeEndpoint) MarshalJSON() ([]byte, error) {
+	epMap := make(map[string]interface{})
+	epMap["id"] = ep.id
+	epMap["nid"] = ep.nid
+	epMap["SrcName"] = ep.srcName
+	epMap["MacAddress"] = ep.macAddress.String()
+	epMap["Addr"] = ep.addr.String()
+	if ep.addrv6 != nil {
+		epMap["Addrv6"] = ep.addrv6.String()
+	}
+	epMap["Config"] = ep.config
+	epMap["ContainerConfig"] = ep.containerConfig
+	epMap["ExternalConnConfig"] = ep.extConnConfig
+	epMap["PortMapping"] = ep.portMapping
+
+	return json.Marshal(epMap)
+}
+
+func (ep *bridgeEndpoint) UnmarshalJSON(b []byte) error {
+	var (
+		err   error
+		epMap map[string]interface{}
+	)
+
+	if err = json.Unmarshal(b, &epMap); err != nil {
+		return fmt.Errorf("Failed to unmarshal to bridge endpoint: %v", err)
+	}
+
+	if v, ok := epMap["MacAddress"]; ok {
+		if ep.macAddress, err = net.ParseMAC(v.(string)); err != nil {
+			return types.InternalErrorf("failed to decode bridge endpoint MAC address (%s) after json unmarshal: %v", v.(string), err)
+		}
+	}
+	if v, ok := epMap["Addr"]; ok {
+		if ep.addr, err = types.ParseCIDR(v.(string)); err != nil {
+			return types.InternalErrorf("failed to decode bridge endpoint IPv4 address (%s) after json unmarshal: %v", v.(string), err)
+		}
+	}
+	if v, ok := epMap["Addrv6"]; ok {
+		if ep.addrv6, err = types.ParseCIDR(v.(string)); err != nil {
+			return types.InternalErrorf("failed to decode bridge endpoint IPv6 address (%s) after json unmarshal: %v", v.(string), err)
+		}
+	}
+	ep.id = epMap["id"].(string)
+	ep.nid = epMap["nid"].(string)
+	ep.srcName = epMap["SrcName"].(string)
+	d, _ := json.Marshal(epMap["Config"])
+	if err := json.Unmarshal(d, &ep.config); err != nil {
+		logrus.Warnf("Failed to decode endpoint config %v", err)
+	}
+	d, _ = json.Marshal(epMap["ContainerConfig"])
+	if err := json.Unmarshal(d, &ep.containerConfig); err != nil {
+		logrus.Warnf("Failed to decode endpoint container config %v", err)
+	}
+	d, _ = json.Marshal(epMap["ExternalConnConfig"])
+	if err := json.Unmarshal(d, &ep.extConnConfig); err != nil {
+		logrus.Warnf("Failed to decode endpoint external connectivity configuration %v", err)
+	}
+	d, _ = json.Marshal(epMap["PortMapping"])
+	if err := json.Unmarshal(d, &ep.portMapping); err != nil {
+		logrus.Warnf("Failed to decode endpoint port mapping %v", err)
+	}
+
+	return nil
+}
+
+func (ep *bridgeEndpoint) Key() []string {
+	return []string{bridgeEndpointPrefix, ep.id}
+}
+
+func (ep *bridgeEndpoint) KeyPrefix() []string {
+	return []string{bridgeEndpointPrefix}
+}
+
+func (ep *bridgeEndpoint) Value() []byte {
+	b, err := json.Marshal(ep)
+	if err != nil {
+		return nil
+	}
+	return b
+}
+
+func (ep *bridgeEndpoint) SetValue(value []byte) error {
+	return json.Unmarshal(value, ep)
+}
+
+func (ep *bridgeEndpoint) Index() uint64 {
+	return ep.dbIndex
+}
+
+func (ep *bridgeEndpoint) SetIndex(index uint64) {
+	ep.dbIndex = index
+	ep.dbExists = true
+}
+
+func (ep *bridgeEndpoint) Exists() bool {
+	return ep.dbExists
+}
+
+func (ep *bridgeEndpoint) Skip() bool {
+	return false
+}
+
+func (ep *bridgeEndpoint) New() datastore.KVObject {
+	return &bridgeEndpoint{}
+}
+
+func (ep *bridgeEndpoint) CopyTo(o datastore.KVObject) error {
+	dstEp := o.(*bridgeEndpoint)
+	*dstEp = *ep
+	return nil
+}
+
+func (ep *bridgeEndpoint) DataScope() string {
+	return datastore.LocalScope
+}
+
+func (n *bridgeNetwork) restorePortAllocations(ep *bridgeEndpoint) {
+	if ep.extConnConfig == nil ||
+		ep.extConnConfig.ExposedPorts == nil ||
+		ep.extConnConfig.PortBindings == nil {
+		return
+	}
+	tmp := ep.extConnConfig.PortBindings
+	ep.extConnConfig.PortBindings = ep.portMapping
+	_, err := n.allocatePorts(ep, n.config.DefaultBindingIP, n.driver.config.EnableUserlandProxy)
+	if err != nil {
+		logrus.Warnf("Failed to reserve existing port mapping for endpoint %s:%v", ep.id[0:7], err)
+	}
+	ep.extConnConfig.PortBindings = tmp
+}

libnetwork/drivers/bridge/bridge_test.go

@@ -1,6 +1,8 @@
 package bridge
 
 import (
+	"bytes"
+	"encoding/json"
 	"fmt"
 	"net"
 	"regexp"
@@ -20,6 +22,150 @@ func init() {
 	ipamutils.InitNetworks()
 }
 
+func TestEndpointMarshalling(t *testing.T) {
+	ip1, _ := types.ParseCIDR("172.22.0.9/16")
+	ip2, _ := types.ParseCIDR("2001:db8::9")
+	mac, _ := net.ParseMAC("ac:bd:24:57:66:77")
+	e := &bridgeEndpoint{
+		id:         "d2c015a1fe5930650cbcd50493efba0500bcebd8ee1f4401a16319f8a567de33",
+		nid:        "ee33fbb43c323f1920b6b35a0101552ac22ede960d0e5245e9738bccc68b2415",
+		addr:       ip1,
+		addrv6:     ip2,
+		macAddress: mac,
+		srcName:    "veth123456",
+		config:     &endpointConfiguration{MacAddress: mac},
+		containerConfig: &containerConfiguration{
+			ParentEndpoints: []string{"one", "due", "three"},
+			ChildEndpoints:  []string{"four", "five", "six"},
+		},
+		extConnConfig: &connectivityConfiguration{
+			ExposedPorts: []types.TransportPort{
+				{
+					Proto: 6,
+					Port:  uint16(18),
+				},
+			},
+			PortBindings: []types.PortBinding{
+				{
+					Proto:       6,
+					IP:          net.ParseIP("17210.33.9.56"),
+					Port:        uint16(18),
+					HostPort:    uint16(3000),
+					HostPortEnd: uint16(14000),
+				},
+			},
+		},
+		portMapping: []types.PortBinding{
+			{
+				Proto:       17,
+				IP:          net.ParseIP("172.33.9.56"),
+				Port:        uint16(99),
+				HostIP:      net.ParseIP("10.10.100.2"),
+				HostPort:    uint16(9900),
+				HostPortEnd: uint16(10000),
+			},
+			{
+				Proto:       6,
+				IP:          net.ParseIP("171.33.9.56"),
+				Port:        uint16(55),
+				HostIP:      net.ParseIP("10.11.100.2"),
+				HostPort:    uint16(5500),
+				HostPortEnd: uint16(55000),
+			},
+		},
+	}
+
+	b, err := json.Marshal(e)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	ee := &bridgeEndpoint{}
+	err = json.Unmarshal(b, ee)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if e.id != ee.id || e.nid != ee.nid || e.srcName != ee.srcName || !bytes.Equal(e.macAddress, ee.macAddress) ||
+		!types.CompareIPNet(e.addr, ee.addr) || !types.CompareIPNet(e.addrv6, ee.addrv6) ||
+		!compareEpConfig(e.config, ee.config) ||
+		!compareContainerConfig(e.containerConfig, ee.containerConfig) ||
+		!compareConnConfig(e.extConnConfig, ee.extConnConfig) ||
+		!compareBindings(e.portMapping, ee.portMapping) {
+		t.Fatalf("JSON marsh/unmarsh failed.\nOriginal:\n%#v\nDecoded:\n%#v", e, ee)
+	}
+}
+
+func compareEpConfig(a, b *endpointConfiguration) bool {
+	if a == b {
+		return true
+	}
+	if a == nil || b == nil {
+		return false
+	}
+	return bytes.Equal(a.MacAddress, b.MacAddress)
+}
+
+func compareContainerConfig(a, b *containerConfiguration) bool {
+	if a == b {
+		return true
+	}
+	if a == nil || b == nil {
+		return false
+	}
+	if len(a.ParentEndpoints) != len(b.ParentEndpoints) ||
+		len(a.ChildEndpoints) != len(b.ChildEndpoints) {
+		return false
+	}
+	for i := 0; i < len(a.ParentEndpoints); i++ {
+		if a.ParentEndpoints[i] != b.ParentEndpoints[i] {
+			return false
+		}
+	}
+	for i := 0; i < len(a.ChildEndpoints); i++ {
+		if a.ChildEndpoints[i] != b.ChildEndpoints[i] {
+			return false
+		}
+	}
+	return true
+}
+
+func compareConnConfig(a, b *connectivityConfiguration) bool {
+	if a == b {
+		return true
+	}
+	if a == nil || b == nil {
+		return false
+	}
+	if len(a.ExposedPorts) != len(b.ExposedPorts) ||
+		len(a.PortBindings) != len(b.PortBindings) {
+		return false
+	}
+	for i := 0; i < len(a.ExposedPorts); i++ {
+		if !a.ExposedPorts[i].Equal(&b.ExposedPorts[i]) {
+			return false
+		}
+	}
+	for i := 0; i < len(a.PortBindings); i++ {
+		if !a.PortBindings[i].Equal(&b.PortBindings[i]) {
+			return false
+		}
+	}
+	return true
+}
+
+func compareBindings(a, b []types.PortBinding) bool {
+	if len(a) != len(b) {
+		return false
+	}
+	for i := 0; i < len(a); i++ {
+		if !a[i].Equal(&b[i]) {
+			return false
+		}
+	}
+	return true
+}
+
 func getIPv4Data(t *testing.T) []driverapi.IPAMData {
 	ipd := driverapi.IPAMData{AddressSpace: "full"}
 	nw, _, err := netutils.ElectInterfaceAddresses("")