TOML based Configuration support for libnetwork

Signed-off-by: Madhu Venugopal <madhu@docker.com>

libnetwork/README.md

@@ -18,7 +18,7 @@ There are many networking solutions available to suit a broad range of use-cases
 
 ```go
         // Create a new controller instance
-        controller := libnetwork.New()
+        controller := libnetwork.New("/etc/default/libnetwork.toml")
 
         // Select and configure the network driver
         networkType := "bridge"

libnetwork/api/api_test.go

@@ -79,7 +79,7 @@ func i2nL(i interface{}) []*networkResource {
 }
 
 func createTestNetwork(t *testing.T, network string) (libnetwork.NetworkController, libnetwork.Network) {
-	c, err := libnetwork.New()
+	c, err := libnetwork.New("")
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -167,7 +167,7 @@ func TestJson(t *testing.T) {
 func TestCreateDeleteNetwork(t *testing.T) {
 	defer netutils.SetupTestNetNS(t)()
 
-	c, err := libnetwork.New()
+	c, err := libnetwork.New("")
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -245,7 +245,7 @@ func TestCreateDeleteNetwork(t *testing.T) {
 func TestGetNetworksAndEndpoints(t *testing.T) {
 	defer netutils.SetupTestNetNS(t)()
 
-	c, err := libnetwork.New()
+	c, err := libnetwork.New("")
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -509,7 +509,7 @@ func TestGetNetworksAndEndpoints(t *testing.T) {
 }
 
 func TestDetectGetNetworksInvalidQueryComposition(t *testing.T) {
-	c, err := libnetwork.New()
+	c, err := libnetwork.New("")
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -600,7 +600,7 @@ func TestFindNetworkUtil(t *testing.T) {
 func TestCreateDeleteEndpoints(t *testing.T) {
 	defer netutils.SetupTestNetNS(t)()
 
-	c, err := libnetwork.New()
+	c, err := libnetwork.New("")
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -726,7 +726,7 @@ func TestCreateDeleteEndpoints(t *testing.T) {
 func TestJoinLeave(t *testing.T) {
 	defer netutils.SetupTestNetNS(t)()
 
-	c, err := libnetwork.New()
+	c, err := libnetwork.New("")
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -1114,7 +1114,7 @@ func TestwriteJSON(t *testing.T) {
 func TestHttpHandlerUninit(t *testing.T) {
 	defer netutils.SetupTestNetNS(t)()
 
-	c, err := libnetwork.New()
+	c, err := libnetwork.New("")
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -1180,7 +1180,7 @@ func TestHttpHandlerBadBody(t *testing.T) {
 
 	rsp := newWriter()
 
-	c, err := libnetwork.New()
+	c, err := libnetwork.New("")
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -1212,7 +1212,7 @@ func TestEndToEnd(t *testing.T) {
 
 	rsp := newWriter()
 
-	c, err := libnetwork.New()
+	c, err := libnetwork.New("")
 	if err != nil {
 		t.Fatal(err)
 	}

libnetwork/cmd/dnet/dnet.go

@@ -106,7 +106,7 @@ type dnetConnection struct {
 }
 
 func (d *dnetConnection) dnetDaemon() error {
-	controller, err := libnetwork.New()
+	controller, err := libnetwork.New("")
 	if err != nil {
 		fmt.Println("Error starting dnetDaemon :", err)
 		return err

libnetwork/cmd/readme_test/readme.go

@@ -11,7 +11,7 @@ import (
 
 func main() {
 	// Create a new controller instance
-	controller, err := libnetwork.New()
+	controller, err := libnetwork.New("/etc/default/libnetwork.toml")
 	if err != nil {
 		return
 	}

libnetwork/cmd/test/libnetwork.toml

@@ -0,0 +1,12 @@
+title = "LibNetwork Configuration file"
+
+[daemon]
+  debug = false
+[cluster]
+  discovery = "token://swarm-discovery-token"
+  Address = "Cluster-wide reachable Host IP"
+[datastore]
+  embedded = false
+[datastore.client]
+  provider = "consul"
+  Address = "localhost:8500"

libnetwork/cmd/test/main.go

@@ -4,25 +4,32 @@ import (
 	"fmt"
 	"log"
 	"net"
+	"os"
+	"time"
 
 	"github.com/docker/libnetwork"
 	"github.com/docker/libnetwork/options"
 )
 
 func main() {
-	ip, net, _ := net.ParseCIDR("192.168.100.1/24")
-	net.IP = ip
-
-	options := options.Generic{"AddressIPv4": net}
-	controller, err := libnetwork.New()
+	os.Setenv("LIBNETWORK_CFG", "libnetwork.toml")
+	controller, err := libnetwork.New("libnetwork.toml")
 	if err != nil {
 		log.Fatal(err)
 	}
-	netType := "bridge"
+
+	netType := "null"
+	ip, net, _ := net.ParseCIDR("192.168.100.1/24")
+	net.IP = ip
+	options := options.Generic{"AddressIPv4": net}
+
 	err = controller.ConfigureNetworkDriver(netType, options)
-	netw, err := controller.NewNetwork(netType, "dummy")
-	if err != nil {
-		log.Fatal(err)
+	for i := 0; i < 100; i++ {
+		netw, err := controller.NewNetwork(netType, fmt.Sprintf("Gordon-%d", i))
+		if err != nil {
+			log.Fatal(err)
+		}
+		fmt.Println("Network Created Successfully :", netw)
+		time.Sleep(10 * time.Second)
 	}
-	fmt.Printf("Network=%#v\n", netw)
 }

libnetwork/config/config.go

@@ -0,0 +1,43 @@
+package config
+
+import "github.com/BurntSushi/toml"
+
+// Config encapsulates configurations of various Libnetwork components
+type Config struct {
+	Daemon    DaemonCfg
+	Cluster   ClusterCfg
+	Datastore DatastoreCfg
+}
+
+// DaemonCfg represents libnetwork core configuration
+type DaemonCfg struct {
+	Debug bool
+}
+
+// ClusterCfg represents cluster configuration
+type ClusterCfg struct {
+	Discovery string
+	Address   string
+	Heartbeat uint64
+}
+
+// DatastoreCfg represents Datastore configuration.
+type DatastoreCfg struct {
+	Embedded bool
+	Client   DatastoreClientCfg
+}
+
+// DatastoreClientCfg represents Datastore Client-only mode configuration
+type DatastoreClientCfg struct {
+	Provider string
+	Address  string
+}
+
+// ParseConfig parses the libnetwork configuration file
+func ParseConfig(tomlCfgFile string) (*Config, error) {
+	var cfg Config
+	if _, err := toml.DecodeFile(tomlCfgFile, &cfg); err != nil {
+		return nil, err
+	}
+	return &cfg, nil
+}

libnetwork/config/config_test.go

@@ -0,0 +1,19 @@
+package config
+
+import (
+	"testing"
+)
+
+func TestInvalidConfig(t *testing.T) {
+	_, err := ParseConfig("invalid.toml")
+	if err == nil {
+		t.Fatal("Invalid Configuration file must fail")
+	}
+}
+
+func TestConfig(t *testing.T) {
+	cfg, err := ParseConfig("libnetwork.toml")
+	if err != nil {
+		t.Fatal("Error parsing a valid configuration file :", err)
+	}
+}

libnetwork/config/libnetwork.toml

@@ -0,0 +1,12 @@
+title = "LibNetwork Configuration file"
+
+[daemon]
+  debug = false
+[cluster]
+  discovery = "token://swarm-discovery-token"
+  Address = "Cluster-wide reachable Host IP"
+[datastore]
+  embedded = false
+[datastore.client]
+  provider = "consul"
+  Address = "localhost:8500"

libnetwork/controller.go

@@ -3,7 +3,7 @@ Package libnetwork provides the basic functionality and extension points to
 create network namespaces and allocate interfaces for containers to use.
 
         // Create a new controller instance
-        controller, _err := libnetwork.New()
+        controller, _err := libnetwork.New("/etc/default/libnetwork.toml")
 
         // Select and configure the network driver
         networkType := "bridge"
@@ -47,13 +47,14 @@ package libnetwork
 
 import (
 	"encoding/json"
-	"errors"
-	"fmt"
+	"os"
+	"strings"
 	"sync"
 
 	log "github.com/Sirupsen/logrus"
 	"github.com/docker/docker/pkg/plugins"
 	"github.com/docker/docker/pkg/stringid"
+	"github.com/docker/libnetwork/config"
 	"github.com/docker/libnetwork/datastore"
 	"github.com/docker/libnetwork/driverapi"
 	"github.com/docker/libnetwork/sandbox"
@@ -61,9 +62,6 @@ import (
 	"github.com/docker/swarm/pkg/store"
 )
 
-// TODO: Move it to error.go once the error refactoring is done
-var ErrInvalidDatastore = errors.New("Datastore is not initialized")
-
 // NetworkController provides the interface for controller instance which manages
 // networks.
 type NetworkController interface {
@@ -104,12 +102,13 @@ type controller struct {
 	networks  networkTable
 	drivers   driverTable
 	sandboxes sandboxTable
+	cfg       *config.Config
 	store     datastore.DataStore
 	sync.Mutex
 }
 
 // New creates a new instance of network controller.
-func New() (NetworkController, error) {
+func New(configFile string) (NetworkController, error) {
 	c := &controller{
 		networks:  networkTable{},
 		sandboxes: sandboxTable{},
@@ -118,28 +117,53 @@ func New() (NetworkController, error) {
 		return nil, err
 	}
 
-	if err := c.initDataStore(); err != nil {
-		log.Errorf("Failed to Initialize Datastore : %v", err)
-		// TODO : Should we fail if the initDataStore fail here ?
+	if err := c.initConfig(configFile); err == nil {
+		if err := c.initDataStore(); err != nil {
+			// Failing to initalize datastore is a bad situation to be in.
+			// But it cannot fail creating the Controller
+			log.Warnf("Failed to Initialize Datastore due to %v. Operating in non-clustered mode", err)
+		}
+	} else {
+		// Missing Configuration file is not a failure scenario
+		// But without that, datastore cannot be initialized.
+		log.Debugf("Unable to Parse LibNetwork Config file : %v", err)
 	}
 
-	go c.watchNewNetworks()
 	return c, nil
 }
 
-func (c *controller) initDataStore() error {
-	/* TODO : Duh ! make this configurable */
-	config := &datastore.StoreConfiguration{}
-	config.Provider = "consul"
-	config.Addrs = []string{"localhost:8500"}
+const (
+	cfgFileEnv     = "LIBNETWORK_CFG"
+	defaultCfgFile = "/etc/default/libnetwork.toml"
+)
+
+func (c *controller) initConfig(configFile string) error {
+	cfgFile := configFile
+	if strings.Trim(cfgFile, " ") == "" {
+		cfgFile = os.Getenv(cfgFileEnv)
+		if strings.Trim(cfgFile, " ") == "" {
+			cfgFile = defaultCfgFile
+		}
+	}
+	cfg, err := config.ParseConfig(cfgFile)
+	if err != nil {
+		return ErrInvalidConfigFile(cfgFile)
+	}
+	c.Lock()
+	c.cfg = cfg
+	c.Unlock()
+	return nil
+}
 
-	store, err := datastore.NewDataStore(config)
+func (c *controller) initDataStore() error {
+	store, err := datastore.NewDataStore(&c.cfg.Datastore)
 	if err != nil {
 		return err
 	}
 	c.Lock()
 	c.store = store
 	c.Unlock()
+	go c.watchNewNetworks()
 
 	return nil
 }
@@ -233,7 +257,7 @@ func (c *controller) newNetworkFromStore(n *network) {
 }
 
 func (c *controller) addNetworkToStore(n *network) error {
-	if IsReservedNetwork(n.Name()) {
+	if isReservedNetwork(n.Name()) {
 		return nil
 	}
 	if c.store == nil {
@@ -263,7 +287,6 @@ func (c *controller) watchNewNetworks() {
 				// Skip any watch notification for a network that has not changed
 				continue
 			}
-			fmt.Printf("WATCHED : %v = %v\n", kve.Key(), n)
 			c.newNetworkFromStore(&n)
 		}
 	})

libnetwork/datastore/datastore.go

@@ -4,6 +4,7 @@ import (
 	"errors"
 	"strings"
 
+	"github.com/docker/libnetwork/config"
 	"github.com/docker/swarm/pkg/store"
 )
 
@@ -18,14 +19,7 @@ type DataStore interface {
 }
 
 type datastore struct {
-	store  store.Store
-	config *StoreConfiguration
-}
-
-//StoreConfiguration exported
-type StoreConfiguration struct {
-	Addrs    []string
-	Provider string
+	store store.Store
 }
 
 //KV Key Value interface used by objects to be part of the DataStore
@@ -49,8 +43,8 @@ var errInvalidConfiguration = errors.New("Invalid Configuration passed to Datast
 var errInvalidAtomicRequest = errors.New("Invalid Atomic Request")
 
 // newClient used to connect to KV Store
-func newClient(kv string, addrs []string) (DataStore, error) {
-	store, err := store.CreateStore(kv, addrs, store.Config{})
+func newClient(kv string, addrs string) (DataStore, error) {
+	store, err := store.CreateStore(kv, []string{addrs}, store.Config{})
 	if err != nil {
 		return nil, err
 	}
@@ -59,11 +53,17 @@ func newClient(kv string, addrs []string) (DataStore, error) {
 }
 
 // NewDataStore creates a new instance of LibKV data store
-func NewDataStore(config *StoreConfiguration) (DataStore, error) {
-	if config == nil {
+func NewDataStore(cfg *config.DatastoreCfg) (DataStore, error) {
+	if cfg == nil {
 		return nil, errInvalidConfiguration
 	}
-	return newClient(config.Provider, config.Addrs)
+	// TODO : cfg.Embedded case
+	return newClient(cfg.Client.Provider, cfg.Client.Address)
+}
+
+// NewCustomDataStore can be used by clients to plugin cusom datatore that adhers to store.Store
+func NewCustomDataStore(customStore store.Store) DataStore {
+	return &datastore{store: customStore}
 }
 
 func (ds *datastore) KVStore() store.Store {

libnetwork/datastore/datastore_test.go

@@ -4,16 +4,23 @@ import (
 	"encoding/json"
 	"testing"
 
+	"github.com/docker/libnetwork/config"
 	_ "github.com/docker/libnetwork/netutils"
 	"github.com/docker/libnetwork/options"
 )
 
 var dummyKey = "dummy"
 
+// NewCustomDataStore can be used by other Tests in order to use custom datastore
+func NewTestDataStore() DataStore {
+	return &datastore{store: NewMockStore()}
+}
+
 func TestInvalidDataStore(t *testing.T) {
-	config := &StoreConfiguration{}
-	config.Provider = "invalid"
-	config.Addrs = []string{"localhost:8500"}
+	config := &config.DatastoreCfg{}
+	config.Embedded = false
+	config.Client.Provider = "invalid"
+	config.Client.Address = "localhost:8500"
 	_, err := NewDataStore(config)
 	if err == nil {
 		t.Fatal("Invalid Datastore connection configuration must result in a failure")
@@ -21,8 +28,7 @@ func TestInvalidDataStore(t *testing.T) {
 }
 
 func TestKVObjectFlatKey(t *testing.T) {
-	mockStore := newMockStore()
-	store := datastore{store: mockStore}
+	store := NewTestDataStore()
 	expected := dummyKVObject("1000", true)
 	err := store.PutObject(expected)
 	if err != nil {
@@ -41,8 +47,7 @@ func TestKVObjectFlatKey(t *testing.T) {
 }
 
 func TestAtomicKVObjectFlatKey(t *testing.T) {
-	mockStore := newMockStore()
-	store := datastore{store: mockStore}
+	store := NewTestDataStore()
 	expected := dummyKVObject("1111", true)
 	err := store.PutObjectAtomic(expected)
 	if err != nil {

libnetwork/error.go

@@ -79,6 +79,13 @@ func (in ErrInvalidName) Error() string {
 // BadRequest denotes the type of this error
 func (in ErrInvalidName) BadRequest() {}
 
+// ErrInvalidConfigFile type is returned when an invalid LibNetwork config file is detected
+type ErrInvalidConfigFile string
+
+func (cf ErrInvalidConfigFile) Error() string {
+	return fmt.Sprintf("Invalid Config file %q", string(cf))
+}
+
 // NetworkTypeError type is returned when the network type string is not
 // known to libnetwork.
 type NetworkTypeError string

libnetwork/libnetwork_internal_test.go

@@ -3,12 +3,13 @@ package libnetwork
 import (
 	"testing"
 
+	"github.com/docker/libnetwork/datastore"
 	"github.com/docker/libnetwork/driverapi"
 )
 
 func TestDriverRegistration(t *testing.T) {
 	bridgeNetType := "bridge"
-	c, err := New()
+	c, err := New("")
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -24,3 +25,8 @@ func TestDriverRegistration(t *testing.T) {
 		t.Fatalf("Test failed with an error %v", err)
 	}
 }
+
+func SetTestDataStore(c NetworkController, custom datastore.DataStore) {
+	con := c.(*controller)
+	con.store = custom
+}

libnetwork/libnetwork_test.go

@@ -18,6 +18,7 @@ import (
 	"github.com/docker/docker/pkg/plugins"
 	"github.com/docker/docker/pkg/reexec"
 	"github.com/docker/libnetwork"
+	"github.com/docker/libnetwork/datastore"
 	"github.com/docker/libnetwork/driverapi"
 	"github.com/docker/libnetwork/netlabel"
 	"github.com/docker/libnetwork/netutils"
@@ -39,8 +40,17 @@ func TestMain(m *testing.M) {
 	os.Exit(m.Run())
 }
 
+func createTestController() (libnetwork.NetworkController, error) {
+	controller, err := libnetwork.New("")
+	if err != nil {
+		return nil, err
+	}
+	libnetwork.SetTestDataStore(controller, datastore.NewCustomDataStore(datastore.NewMockStore()))
+	return controller, nil
+}
+
 func createTestNetwork(networkType, networkName string, option options.Generic, netOption options.Generic) (libnetwork.Network, error) {
-	controller, err := libnetwork.New()
+	controller, err := createTestController()
 	if err != nil {
 		return nil, err
 	}
@@ -283,7 +293,7 @@ func TestUnknownDriver(t *testing.T) {
 }
 
 func TestNilRemoteDriver(t *testing.T) {
-	controller, err := libnetwork.New()
+	controller, err := createTestController()
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -304,7 +314,7 @@ func TestDuplicateNetwork(t *testing.T) {
 		defer netutils.SetupTestNetNS(t)()
 	}
 
-	controller, err := libnetwork.New()
+	controller, err := createTestController()
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -513,7 +523,7 @@ func TestNetworkEndpointsWalkers(t *testing.T) {
 		defer netutils.SetupTestNetNS(t)()
 	}
 
-	controller, err := libnetwork.New()
+	controller, err := createTestController()
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -597,7 +607,7 @@ func TestControllerQuery(t *testing.T) {
 		defer netutils.SetupTestNetNS(t)()
 	}
 
-	controller, err := libnetwork.New()
+	controller, err := createTestController()
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -663,7 +673,7 @@ func TestNetworkQuery(t *testing.T) {
 		defer netutils.SetupTestNetNS(t)()
 	}
 
-	controller, err := libnetwork.New()
+	controller, err := createTestController()
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -1275,7 +1285,7 @@ func TestInvalidRemoteDriver(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	controller, err := libnetwork.New()
+	controller, err := libnetwork.New("")
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -1329,7 +1339,7 @@ func TestValidRemoteDriver(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	controller, err := libnetwork.New()
+	controller, err := libnetwork.New("")
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -1376,7 +1386,7 @@ func createGlobalInstance(t *testing.T) {
 		}
 	}
 
-	ctrlr, err = libnetwork.New()
+	ctrlr, err = createTestController()
 	if err != nil {
 		t.Fatal(err)
 	}

libnetwork/network.go

@@ -256,7 +256,7 @@ func (n *network) EndpointByID(id string) (Endpoint, error) {
 	return nil, ErrNoSuchEndpoint(id)
 }
 
-func IsReservedNetwork(name string) bool {
+func isReservedNetwork(name string) bool {
 	reserved := []string{"bridge", "none", "host"}
 	for _, r := range reserved {
 		if strings.EqualFold(r, name) {