Make bridge driver networks persistent

Since libnetwork is going to provide createNetwork
notifications only once when the network is created
bridge network needs to save it's network state in
persistent store so that it becomes available even
after restart.

Signed-off-by: Jana Radhakrishnan <mrjana@docker.com>

libnetwork/datastore/datastore.go

@@ -92,8 +92,7 @@ type KVConstructor interface {
 
 // ScopeCfg represents Datastore configuration.
 type ScopeCfg struct {
-	Embedded bool
-	Client   ScopeClientCfg
+	Client ScopeClientCfg
 }
 
 // ScopeClientCfg represents Datastore Client-only mode configuration
@@ -125,7 +124,6 @@ var (
 func makeDefaultScopes() map[string]*ScopeCfg {
 	def := make(map[string]*ScopeCfg)
 	def[LocalScope] = &ScopeCfg{
-		Embedded: true,
 		Client: ScopeClientCfg{
 			Provider: "boltdb",
 			Address:  defaultPrefix + "/boltdb.db",

libnetwork/datastore/datastore_test.go

@@ -38,7 +38,6 @@ func TestParseKey(t *testing.T) {
 
 func TestInvalidDataStore(t *testing.T) {
 	config := &ScopeCfg{}
-	config.Embedded = false
 	config.Client.Provider = "invalid"
 	config.Client.Address = "localhost:8500"
 	_, err := NewDataStore(GlobalScope, config)

libnetwork/drivers/bridge/bridge.go

@@ -55,6 +55,7 @@ type configuration struct {
 
 // networkConfiguration for network specific configuration
 type networkConfiguration struct {
+	ID                 string
 	BridgeName         string
 	EnableIPv6         bool
 	EnableIPMasquerade bool
@@ -67,6 +68,8 @@ type networkConfiguration struct {
 	AddressIPv6        *net.IPNet
 	DefaultGatewayIPv4 net.IP
 	DefaultGatewayIPv6 net.IP
+	dbIndex            uint64
+	dbExists           bool
 }
 
 // endpointConfiguration represents the user specified configuration for the sandbox endpoint
@@ -109,6 +112,7 @@ type driver struct {
 	natChain    *iptables.ChainInfo
 	filterChain *iptables.ChainInfo
 	networks    map[string]*bridgeNetwork
+	store       datastore.DataStore
 	sync.Mutex
 }
 
@@ -376,6 +380,11 @@ func (d *driver) configure(option map[string]interface{}) error {
 	var config *configuration
 	var err error
 
+	err = d.initStore(option)
+	if err != nil {
+		return err
+	}
+
 	d.Lock()
 	defer d.Unlock()
 
@@ -511,6 +520,8 @@ func parseNetworkOptions(id string, option options.Generic) (*networkConfigurati
 	if config.BridgeName == "" && config.DefaultBridge == false {
 		config.BridgeName = "br-" + id[:12]
 	}
+
+	config.ID = id
 	return config, nil
 }
 
@@ -540,10 +551,6 @@ func (d *driver) getNetworks() []*bridgeNetwork {
 
 // Create a new network using bridge plugin
 func (d *driver) CreateNetwork(id string, option map[string]interface{}, ipV4Data, ipV6Data []driverapi.IPAMData) error {
-	var err error
-
-	defer osl.InitOSContext()()
-
 	// Sanity checks
 	d.Lock()
 	if _, ok := d.networks[id]; ok {
@@ -563,6 +570,18 @@ func (d *driver) CreateNetwork(id string, option map[string]interface{}, ipV4Dat
 		return err
 	}
 
+	if err = d.createNetwork(config); err != nil {
+		return err
+	}
+
+	return d.storeUpdate(config)
+}
+
+func (d *driver) createNetwork(config *networkConfiguration) error {
+	var err error
+
+	defer osl.InitOSContext()()
+
 	networkList := d.getNetworks()
 	for _, nw := range networkList {
 		nw.Lock()
@@ -570,13 +589,13 @@ func (d *driver) CreateNetwork(id string, option map[string]interface{}, ipV4Dat
 		nw.Unlock()
 		if err := nwConfig.Conflicts(config); err != nil {
 			return types.ForbiddenErrorf("cannot create network %s (%s): conflicts with network %s (%s): %s",
-				nwConfig.BridgeName, id, nw.id, nw.config.BridgeName, err.Error())
+				nwConfig.BridgeName, config.ID, nw.id, nw.config.BridgeName, err.Error())
 		}
 	}
 
 	// Create and set network handler in driver
 	network := &bridgeNetwork{
-		id:         id,
+		id:         config.ID,
 		endpoints:  make(map[string]*bridgeEndpoint),
 		config:     config,
 		portMapper: portmapper.New(),
@@ -584,14 +603,14 @@ func (d *driver) CreateNetwork(id string, option map[string]interface{}, ipV4Dat
 	}
 
 	d.Lock()
-	d.networks[id] = network
+	d.networks[config.ID] = network
 	d.Unlock()
 
 	// On failure make sure to reset driver network handler to nil
 	defer func() {
 		if err != nil {
 			d.Lock()
-			delete(d.networks, id)
+			delete(d.networks, config.ID)
 			d.Unlock()
 		}
 	}()
@@ -604,7 +623,7 @@ func (d *driver) CreateNetwork(id string, option map[string]interface{}, ipV4Dat
 	// networks. This step is needed now because driver might have now set the bridge
 	// name on this config struct. And because we need to check for possible address
 	// conflicts, so we need to check against operationa lnetworks.
-	if err = config.conflictsWithNetworks(id, networkList); err != nil {
+	if err = config.conflictsWithNetworks(config.ID, networkList); err != nil {
 		return err
 	}
 
@@ -705,10 +724,6 @@ func (d *driver) DeleteNetwork(nid string) error {
 	config := n.config
 	n.Unlock()
 
-	if config.BridgeName == DefaultBridgeName {
-		return types.ForbiddenErrorf("default network of type \"%s\" cannot be deleted", networkType)
-	}
-
 	d.Lock()
 	delete(d.networks, nid)
 	d.Unlock()
@@ -753,8 +768,12 @@ func (d *driver) DeleteNetwork(nid string) error {
 		return err
 	}
 
-	// Programming
-	return netlink.LinkDel(n.bridge.Link)
+	// We only delete the bridge when it's not the default bridge. This is keep the backward compatible behavior.
+	if !config.DefaultBridge {
+		err = netlink.LinkDel(n.bridge.Link)
+	}
+
+	return d.storeDelete(config)
 }
 
 func addToBridge(ifaceName, bridgeName string) error {

libnetwork/drivers/bridge/bridge_store.go

@@ -0,0 +1,210 @@
+package bridge
+
+import (
+	"encoding/json"
+	"fmt"
+	"net"
+
+	"github.com/Sirupsen/logrus"
+	"github.com/docker/libkv/store"
+	"github.com/docker/libkv/store/boltdb"
+	"github.com/docker/libnetwork/datastore"
+	"github.com/docker/libnetwork/netlabel"
+	"github.com/docker/libnetwork/types"
+)
+
+const bridgePrefix = "bridge"
+
+func (d *driver) initStore(option map[string]interface{}) error {
+	var err error
+
+	provider, provOk := option[netlabel.LocalKVProvider]
+	provURL, urlOk := option[netlabel.LocalKVProviderURL]
+
+	if provOk && urlOk {
+		cfg := &datastore.ScopeCfg{
+			Client: datastore.ScopeClientCfg{
+				Provider: provider.(string),
+				Address:  provURL.(string),
+			},
+		}
+
+		provConfig, confOk := option[netlabel.LocalKVProviderConfig]
+		if confOk {
+			cfg.Client.Config = provConfig.(*store.Config)
+		}
+
+		d.store, err = datastore.NewDataStore(datastore.LocalScope, cfg)
+		if err != nil {
+			return fmt.Errorf("bridge driver failed to initialize data store: %v", err)
+		}
+
+		return d.populateNetworks()
+	}
+
+	return nil
+}
+
+func (d *driver) populateNetworks() error {
+	kvol, err := d.store.List(datastore.Key(bridgePrefix), &networkConfiguration{})
+	if err != nil && err != datastore.ErrKeyNotFound && err != boltdb.ErrBoltBucketNotFound {
+		return fmt.Errorf("failed to get bridge network configurations from store: %v", err)
+	}
+
+	// It's normal for network configuration state to be empty. Just return.
+	if err == datastore.ErrKeyNotFound {
+		return nil
+	}
+
+	for _, kvo := range kvol {
+		ncfg := kvo.(*networkConfiguration)
+		if err = d.createNetwork(ncfg); err != nil {
+			logrus.Warnf("could not create bridge network for id %s bridge name %s while booting up from persistent state", ncfg.ID, ncfg.BridgeName)
+		}
+	}
+
+	return nil
+}
+
+func (d *driver) storeUpdate(kvObject datastore.KVObject) error {
+	if d.store == nil {
+		logrus.Warnf("bridge store not initialized. kv object %s is not added to the store", datastore.Key(kvObject.Key()...))
+		return nil
+	}
+
+	if err := d.store.PutObjectAtomic(kvObject); err != nil {
+		return fmt.Errorf("failed to update bridge store for object type %T: %v", kvObject, err)
+	}
+
+	return nil
+}
+
+func (d *driver) storeDelete(kvObject datastore.KVObject) error {
+	if d.store == nil {
+		logrus.Debugf("bridge store not initialized. kv object %s is not deleted from store", datastore.Key(kvObject.Key()...))
+		return nil
+	}
+
+retry:
+	if err := d.store.DeleteObjectAtomic(kvObject); err != nil {
+		if err == datastore.ErrKeyModified {
+			if err := d.store.GetObject(datastore.Key(kvObject.Key()...), kvObject); err != nil {
+				return fmt.Errorf("could not update the kvobject to latest when trying to delete: %v", err)
+			}
+			goto retry
+		}
+		return err
+	}
+
+	return nil
+}
+
+func (ncfg *networkConfiguration) MarshalJSON() ([]byte, error) {
+	nMap := make(map[string]interface{})
+	nMap["ID"] = ncfg.ID
+	nMap["BridgeName"] = ncfg.BridgeName
+	nMap["EnableIPv6"] = ncfg.EnableIPv6
+	nMap["EnableIPMasquerade"] = ncfg.EnableIPMasquerade
+	nMap["EnableICC"] = ncfg.EnableICC
+	nMap["Mtu"] = ncfg.Mtu
+	nMap["DefaultBridge"] = ncfg.DefaultBridge
+	nMap["DefaultBindingIP"] = ncfg.DefaultBindingIP.String()
+	nMap["DefaultGatewayIPv4"] = ncfg.DefaultGatewayIPv4.String()
+	nMap["DefaultGatewayIPv6"] = ncfg.DefaultGatewayIPv6.String()
+
+	if ncfg.AddressIPv4 != nil {
+		nMap["AddressIPv4"] = ncfg.AddressIPv4.String()
+	}
+
+	if ncfg.AddressIPv6 != nil {
+		nMap["AddressIPv6"] = ncfg.AddressIPv6.String()
+	}
+
+	return json.Marshal(nMap)
+}
+
+func (ncfg *networkConfiguration) UnmarshalJSON(b []byte) error {
+	var (
+		err  error
+		nMap map[string]interface{}
+	)
+
+	if err = json.Unmarshal(b, &nMap); err != nil {
+		return err
+	}
+
+	if v, ok := nMap["AddressIPv4"]; ok {
+		if ncfg.AddressIPv4, err = types.ParseCIDR(v.(string)); err != nil {
+			return types.InternalErrorf("failed to decode bridge network address IPv4 after json unmarshal: %s", v.(string))
+		}
+	}
+
+	if v, ok := nMap["AddressIPv6"]; ok {
+		if ncfg.AddressIPv6, err = types.ParseCIDR(v.(string)); err != nil {
+			return types.InternalErrorf("failed to decode bridge network address IPv6 after json unmarshal: %s", v.(string))
+		}
+	}
+
+	ncfg.DefaultBindingIP = net.ParseIP(nMap["DefaultBindingIP"].(string))
+	ncfg.DefaultGatewayIPv4 = net.ParseIP(nMap["DefaultGatewayIPv4"].(string))
+	ncfg.DefaultGatewayIPv6 = net.ParseIP(nMap["DefaultGatewayIPv6"].(string))
+	ncfg.ID = nMap["ID"].(string)
+	ncfg.BridgeName = nMap["BridgeName"].(string)
+	ncfg.EnableIPv6 = nMap["EnableIPv6"].(bool)
+	ncfg.EnableIPMasquerade = nMap["EnableIPMasquerade"].(bool)
+	ncfg.EnableICC = nMap["EnableICC"].(bool)
+	ncfg.Mtu = int(nMap["Mtu"].(float64))
+
+	return nil
+}
+
+func (ncfg *networkConfiguration) Key() []string {
+	return []string{bridgePrefix, ncfg.ID}
+}
+
+func (ncfg *networkConfiguration) KeyPrefix() []string {
+	return []string{bridgePrefix}
+}
+
+func (ncfg *networkConfiguration) Value() []byte {
+	b, err := json.Marshal(ncfg)
+	if err != nil {
+		return nil
+	}
+	return b
+}
+
+func (ncfg *networkConfiguration) SetValue(value []byte) error {
+	return json.Unmarshal(value, ncfg)
+}
+
+func (ncfg *networkConfiguration) Index() uint64 {
+	return ncfg.dbIndex
+}
+
+func (ncfg *networkConfiguration) SetIndex(index uint64) {
+	ncfg.dbIndex = index
+	ncfg.dbExists = true
+}
+
+func (ncfg *networkConfiguration) Exists() bool {
+	return ncfg.dbExists
+}
+
+func (ncfg *networkConfiguration) Skip() bool {
+	return false
+}
+
+func (ncfg *networkConfiguration) New() datastore.KVObject {
+	return &networkConfiguration{}
+}
+
+func (ncfg *networkConfiguration) CopyTo(o datastore.KVObject) error {
+	dstNcfg := o.(*networkConfiguration)
+	*dstNcfg = *ncfg
+	return nil
+}
+
+func (ncfg *networkConfiguration) DataScope() string {
+	return datastore.LocalScope
+}

libnetwork/drivers/bridge/bridge_test.go

@@ -120,14 +120,6 @@ func TestCreate(t *testing.T) {
 	if _, ok := err.(types.ForbiddenError); !ok {
 		t.Fatalf("Creation of second network with default name failed with unexpected error type")
 	}
-
-	err = d.DeleteNetwork("dummy")
-	if err == nil {
-		t.Fatalf("deletion of network with default name should fail on this driver")
-	}
-	if _, ok := err.(types.ForbiddenError); !ok {
-		t.Fatalf("deletion of network with default name failed with unexpected error type")
-	}
 }
 
 func TestCreateFail(t *testing.T) {

libnetwork/ipam/allocator_test.go

@@ -32,7 +32,6 @@ func randomLocalStore() (datastore.DataStore, error) {
 		return nil, fmt.Errorf("Error closing temp file: %v", err)
 	}
 	return datastore.NewDataStore(datastore.LocalScope, &datastore.ScopeCfg{
-		Embedded: true,
 		Client: datastore.ScopeClientCfg{
 			Provider: "boltdb",
 			Address:  defaultPrefix + tmp.Name(),

libnetwork/network.go

@@ -557,17 +557,6 @@ func (n *network) deleteNetwork() error {
 		return fmt.Errorf("failed deleting network: %v", err)
 	}
 
-	// If it is bridge network type make sure we call the driver about the network
-	// because the network may have been created in some past life of libnetwork.
-	if n.Type() == "bridge" {
-		n.drvOnce.Do(func() {
-			err = n.getController().addNetwork(n)
-		})
-		if err != nil {
-			return err
-		}
-	}
-
 	if err := d.DeleteNetwork(n.ID()); err != nil {
 		// Forbidden Errors should be honored
 		if _, ok := err.(types.ForbiddenError); ok {
@@ -585,17 +574,6 @@ func (n *network) addEndpoint(ep *endpoint) error {
 		return fmt.Errorf("failed to add endpoint: %v", err)
 	}
 
-	// If it is bridge network type make sure we call the driver about the network
-	// because the network may have been created in some past life of libnetwork.
-	if n.Type() == "bridge" {
-		n.drvOnce.Do(func() {
-			err = n.getController().addNetwork(n)
-		})
-		if err != nil {
-			return err
-		}
-	}
-
 	err = d.CreateEndpoint(n.id, ep.id, ep.Interface(), ep.generic)
 	if err != nil {
 		return types.InternalErrorf("failed to create endpoint %s on network %s: %v",