Merge pull request #14305 from aboch/nwst

Import latest libnetwork

hack/vendor.sh

@@ -18,7 +18,7 @@ clone git golang.org/x/net 3cffabab72adf04f8e3b01c5baf775361837b5fe https://gith
 clone hg code.google.com/p/gosqlite 74691fb6f837
 
 #get libnetwork packages
-clone git github.com/docker/libnetwork 82a1f5634904b57e619fd715ded6903727e00143
+clone git github.com/docker/libnetwork 4c14cd316f40f16bc1c17e420b18a1902dc575a7
 clone git github.com/armon/go-metrics eb0af217e5e9747e41dd5303755356b62d28e3ec
 clone git github.com/hashicorp/go-msgpack 71c2886f5a673a35f909803f38ece5810165097b
 clone git github.com/hashicorp/memberlist 9a1e242e454d2443df330bdd51a436d5a9058fc4

vendor/src/github.com/docker/libnetwork/README.md

@@ -49,7 +49,7 @@ There are many networking solutions available to suit a broad range of use-cases
 
         // A container can join the endpoint by providing the container ID to the join
         // api.
-        // Join acceps Variadic arguments which will be made use of by libnetwork and Drivers
+        // Join accepts Variadic arguments which will be made use of by libnetwork and Drivers
         err = ep.Join("container1",
                 libnetwork.JoinOptionHostname("test"),
                 libnetwork.JoinOptionDomainname("docker.io"))
@@ -57,7 +57,7 @@ There are many networking solutions available to suit a broad range of use-cases
                 return
         }
 
-		// libentwork client can check the endpoint's operational data via the Info() API
+		// libnetwork client can check the endpoint's operational data via the Info() API
 		epInfo, err := ep.DriverInfo()
 		mapData, ok := epInfo[netlabel.PortMap]
 		if ok {

vendor/src/github.com/docker/libnetwork/bitseq/sequence.go

@@ -28,6 +28,7 @@ type Handle struct {
 	app        string
 	id         string
 	dbIndex    uint64
+	dbExists   bool
 	store      datastore.DataStore
 	sync.Mutex
 }
@@ -54,18 +55,10 @@ func NewHandle(app string, ds datastore.DataStore, id string, numElements uint32
 	h.watchForChanges()
 
 	// Get the initial status from the ds if present.
-	// We will be getting an instance without a dbIndex
-	// (GetObject() does not set it): It is ok for now,
-	// it will only cause the first allocation on this
-	// node to go through a retry.
-	var bah []byte
-	if err := h.store.GetObject(datastore.Key(h.Key()...), &bah); err != nil {
-		if err != datastore.ErrKeyNotFound {
-			return nil, err
-		}
-		return h, nil
+	err := h.store.GetObject(datastore.Key(h.Key()...), h)
+	if err != datastore.ErrKeyNotFound {
+		return nil, err
 	}
-	err := h.FromByteArray(bah)
 
 	return h, err
 }
@@ -199,7 +192,14 @@ func (h *Handle) CheckIfAvailable(ordinal int) (int, int, error) {
 func (h *Handle) PushReservation(bytePos, bitPos int, release bool) error {
 	// Create a copy of the current handler
 	h.Lock()
-	nh := &Handle{app: h.app, id: h.id, store: h.store, dbIndex: h.dbIndex, head: h.head.GetCopy()}
+	nh := &Handle{
+		app:      h.app,
+		id:       h.id,
+		store:    h.store,
+		dbIndex:  h.dbIndex,
+		head:     h.head.GetCopy(),
+		dbExists: h.dbExists,
+	}
 	h.Unlock()
 
 	nh.head = PushReservation(bytePos, bitPos, nh.head, release)
@@ -214,7 +214,9 @@ func (h *Handle) PushReservation(bytePos, bitPos int, release bool) error {
 		} else {
 			h.unselected--
 		}
-		h.dbIndex = nh.dbIndex
+		// Can't use SetIndex() since we're locked.
+		h.dbIndex = nh.Index()
+		h.dbExists = true
 		h.Unlock()
 	}
 
@@ -276,12 +278,6 @@ func (h *Handle) Unselected() uint32 {
 	return h.unselected
 }
 
-func (h *Handle) getDBIndex() uint64 {
-	h.Lock()
-	defer h.Unlock()
-	return h.dbIndex
-}
-
 // GetFirstAvailable looks for the first unset bit in passed mask
 func GetFirstAvailable(head *Sequence) (int, int, error) {
 	byteIndex := 0

vendor/src/github.com/docker/libnetwork/bitseq/store.go

@@ -38,6 +38,11 @@ func (h *Handle) Value() []byte {
 	return jv
 }
 
+// SetValue unmarshals the data from the KV store
+func (h *Handle) SetValue(value []byte) error {
+	return h.FromByteArray(value)
+}
+
 // Index returns the latest DB Index as seen by this object
 func (h *Handle) Index() uint64 {
 	h.Lock()
@@ -49,9 +54,17 @@ func (h *Handle) Index() uint64 {
 func (h *Handle) SetIndex(index uint64) {
 	h.Lock()
 	h.dbIndex = index
+	h.dbExists = true
 	h.Unlock()
 }
 
+// Exists method is true if this object has been stored in the DB.
+func (h *Handle) Exists() bool {
+	h.Lock()
+	defer h.Unlock()
+	return h.dbExists
+}
+
 func (h *Handle) watchForChanges() error {
 	h.Lock()
 	store := h.store
@@ -70,14 +83,12 @@ func (h *Handle) watchForChanges() error {
 			select {
 			case kvPair := <-kvpChan:
 				// Only process remote update
-				if kvPair != nil && (kvPair.LastIndex != h.getDBIndex()) {
+				if kvPair != nil && (kvPair.LastIndex != h.Index()) {
 					err := h.fromDsValue(kvPair.Value)
 					if err != nil {
 						log.Warnf("Failed to reconstruct bitseq handle from ds watch: %s", err.Error())
 					} else {
-						h.Lock()
-						h.dbIndex = kvPair.LastIndex
-						h.Unlock()
+						h.SetIndex(kvPair.LastIndex)
 					}
 				}
 			}

vendor/src/github.com/docker/libnetwork/config/config.go

@@ -4,6 +4,7 @@ import (
 	"strings"
 
 	"github.com/BurntSushi/toml"
+	log "github.com/Sirupsen/logrus"
 	"github.com/docker/libnetwork/netlabel"
 )
 
@@ -57,6 +58,7 @@ type Option func(c *Config)
 // OptionDefaultNetwork function returns an option setter for a default network
 func OptionDefaultNetwork(dn string) Option {
 	return func(c *Config) {
+		log.Infof("Option DefaultNetwork: %s", dn)
 		c.Daemon.DefaultNetwork = strings.TrimSpace(dn)
 	}
 }
@@ -64,6 +66,7 @@ func OptionDefaultNetwork(dn string) Option {
 // OptionDefaultDriver function returns an option setter for default driver
 func OptionDefaultDriver(dd string) Option {
 	return func(c *Config) {
+		log.Infof("Option DefaultDriver: %s", dd)
 		c.Daemon.DefaultDriver = strings.TrimSpace(dd)
 	}
 }
@@ -82,6 +85,7 @@ func OptionLabels(labels []string) Option {
 // OptionKVProvider function returns an option setter for kvstore provider
 func OptionKVProvider(provider string) Option {
 	return func(c *Config) {
+		log.Infof("Option OptionKVProvider: %s", provider)
 		c.Datastore.Client.Provider = strings.TrimSpace(provider)
 	}
 }
@@ -89,6 +93,7 @@ func OptionKVProvider(provider string) Option {
 // OptionKVProviderURL function returns an option setter for kvstore url
 func OptionKVProviderURL(url string) Option {
 	return func(c *Config) {
+		log.Infof("Option OptionKVProviderURL: %s", url)
 		c.Datastore.Client.Address = strings.TrimSpace(url)
 	}
 }

vendor/src/github.com/docker/libnetwork/controller.go

@@ -34,7 +34,7 @@ create network namespaces and allocate interfaces for containers to use.
 
 	// A container can join the endpoint by providing the container ID to the join
 	// api.
-	// Join acceps Variadic arguments which will be made use of by libnetwork and Drivers
+	// Join accepts Variadic arguments which will be made use of by libnetwork and Drivers
 	err = ep.Join("container1",
 		libnetwork.JoinOptionHostname("test"),
 		libnetwork.JoinOptionDomainname("docker.io"))
@@ -262,6 +262,7 @@ func (c *controller) NewNetwork(networkType, name string, options ...NetworkOpti
 	}
 
 	if err := c.updateNetworkToStore(network); err != nil {
+		log.Warnf("couldnt create network %s: %v", network.name, err)
 		if e := network.Delete(); e != nil {
 			log.Warnf("couldnt cleanup network %s: %v", network.name, err)
 		}

vendor/src/github.com/docker/libnetwork/datastore/datastore.go

@@ -1,7 +1,6 @@
 package datastore
 
 import (
-	"encoding/json"
 	"reflect"
 	"strings"
 
@@ -14,9 +13,7 @@ import (
 //DataStore exported
 type DataStore interface {
 	// GetObject gets data from datastore and unmarshals to the specified object
-	GetObject(key string, o interface{}) error
-	// GetUpdatedObject gets data from datastore along with its index and unmarshals to the specified object
-	GetUpdatedObject(key string, o interface{}) (uint64, error)
+	GetObject(key string, o KV) error
 	// PutObject adds a new Record based on an object into the datastore
 	PutObject(kvObject KV) error
 	// PutObjectAtomic provides an atomic add and update operation for a Record
@@ -49,10 +46,15 @@ type KV interface {
 	KeyPrefix() []string
 	// Value method lets an object to marshal its content to be stored in the KV store
 	Value() []byte
+	// SetValue is used by the datastore to set the object's value when loaded from the data store.
+	SetValue([]byte) error
 	// Index method returns the latest DB Index as seen by the object
 	Index() uint64
 	// SetIndex method allows the datastore to store the latest DB Index into the object
 	SetIndex(uint64)
+	// True if the object exists in the datastore, false if it hasn't been stored yet.
+	// When SetIndex() is called, the object has been stored.
+	Exists() bool
 }
 
 const (
@@ -121,7 +123,12 @@ func (ds *datastore) PutObjectAtomic(kvObject KV) error {
 		return types.BadRequestErrorf("invalid KV Object with a nil Value for key %s", Key(kvObject.Key()...))
 	}
 
-	previous := &store.KVPair{Key: Key(kvObject.Key()...), LastIndex: kvObject.Index()}
+	var previous *store.KVPair
+	if kvObject.Exists() {
+		previous = &store.KVPair{Key: Key(kvObject.Key()...), LastIndex: kvObject.Index()}
+	} else {
+		previous = nil
+	}
 	_, pair, err := ds.store.AtomicPut(Key(kvObject.Key()...), kvObjValue, previous, nil)
 	if err != nil {
 		return err
@@ -149,24 +156,20 @@ func (ds *datastore) putObjectWithKey(kvObject KV, key ...string) error {
 }
 
 // GetObject returns a record matching the key
-func (ds *datastore) GetObject(key string, o interface{}) error {
+func (ds *datastore) GetObject(key string, o KV) error {
 	kvPair, err := ds.store.Get(key)
 	if err != nil {
 		return err
 	}
-	return json.Unmarshal(kvPair.Value, o)
-}
-
-// GetUpdateObject returns a record matching the key
-func (ds *datastore) GetUpdatedObject(key string, o interface{}) (uint64, error) {
-	kvPair, err := ds.store.Get(key)
+	err = o.SetValue(kvPair.Value)
 	if err != nil {
-		return 0, err
-	}
-	if err := json.Unmarshal(kvPair.Value, o); err != nil {
-		return 0, err
+		return err
 	}
-	return kvPair.LastIndex, nil
+
+	// Make sure the object has a correct view of the DB index in case we need to modify it
+	// and update the DB.
+	o.SetIndex(kvPair.LastIndex)
+	return nil
 }
 
 // DeleteObject unconditionally deletes a record from the store

vendor/src/github.com/docker/libnetwork/datastore/mock_store.go

@@ -93,8 +93,18 @@ func (s *MockStore) NewLock(key string, options *store.LockOptions) (store.Locke
 // modified in the meantime, throws an error if this is the case
 func (s *MockStore) AtomicPut(key string, newValue []byte, previous *store.KVPair, options *store.WriteOptions) (bool, *store.KVPair, error) {
 	mData := s.db[key]
-	if mData != nil && mData.Index != previous.LastIndex {
-		return false, nil, types.BadRequestErrorf("atomic put failed due to mismatched Index")
+
+	if previous == nil {
+		if mData != nil {
+			return false, nil, types.BadRequestErrorf("atomic put failed because key exists")
+		} // Else OK.
+	} else {
+		if mData == nil {
+			return false, nil, types.BadRequestErrorf("atomic put failed because key exists")
+		}
+		if mData != nil && mData.Index != previous.LastIndex {
+			return false, nil, types.BadRequestErrorf("atomic put failed due to mismatched Index")
+		} // Else OK.
 	}
 	err := s.Put(key, newValue, nil)
 	if err != nil {

vendor/src/github.com/docker/libnetwork/drivers/bridge/bridge.go

@@ -108,8 +108,11 @@ func newDriver() driverapi.Driver {
 func Init(dc driverapi.DriverCallback) error {
 	// try to modprobe bridge first
 	// see gh#12177
-	if out, err := exec.Command("modprobe", "-va", "bridge", "nf_nat", "br_netfilter").Output(); err != nil {
-		logrus.Warnf("Running modprobe bridge nf_nat failed with message: %s, error: %v", out, err)
+	if out, err := exec.Command("modprobe", "-va", "bridge", "nf_nat", "br_netfilter").CombinedOutput(); err != nil {
+		logrus.Warnf("Running modprobe bridge nf_nat br_netfilter failed with message: %s, error: %v", out, err)
+	}
+	if err := iptables.FirewalldInit(); err != nil {
+		logrus.Debugf("Fail to initialize firewalld: %v, using raw iptables instead", err)
 	}
 	if err := iptables.RemoveExistingChain(DockerChain, iptables.Nat); err != nil {
 		logrus.Warnf("Failed to remove existing iptables entries in %s : %v", DockerChain, err)
@@ -453,7 +456,7 @@ func (d *driver) getNetwork(id types.UUID) (*bridgeNetwork, error) {
 		return nw, nil
 	}
 
-	return nil, nil
+	return nil, types.NotFoundErrorf("network not found: %s", id)
 }
 
 func parseNetworkGenericOptions(data interface{}) (*networkConfiguration, error) {

vendor/src/github.com/docker/libnetwork/drivers/overlay/ov_network.go

@@ -22,6 +22,7 @@ type network struct {
 	id          types.UUID
 	vni         uint32
 	dbIndex     uint64
+	dbExists    bool
 	sbox        sandbox.Sandbox
 	endpoints   endpointTable
 	ipAllocator *ipallocator.IPAllocator
@@ -260,6 +261,20 @@ func (n *network) Index() uint64 {
 
 func (n *network) SetIndex(index uint64) {
 	n.dbIndex = index
+	n.dbExists = true
+}
+
+func (n *network) Exists() bool {
+	return n.dbExists
+}
+
+func (n *network) SetValue(value []byte) error {
+	var vni uint32
+	err := json.Unmarshal(value, &vni)
+	if err == nil {
+		n.setVxlanID(vni)
+	}
+	return err
 }
 
 func (n *network) writeToStore() error {
@@ -297,8 +312,7 @@ func (n *network) obtainVxlanID() error {
 
 	for {
 		var vxlanID uint32
-		if err := n.driver.store.GetObject(datastore.Key(n.Key()...),
-			&vxlanID); err != nil {
+		if err := n.driver.store.GetObject(datastore.Key(n.Key()...), n); err != nil {
 			if err == datastore.ErrKeyNotFound {
 				vxlanID, err = n.driver.vxlanIdm.GetID()
 				if err != nil {
@@ -318,8 +332,6 @@ func (n *network) obtainVxlanID() error {
 			}
 			return fmt.Errorf("failed to obtain vxlan id from data store: %v", err)
 		}
-
-		n.setVxlanID(vxlanID)
 		return nil
 	}
 }

vendor/src/github.com/docker/libnetwork/endpoint.go

@@ -51,6 +51,9 @@ type Endpoint interface {
 
 	// Delete and detaches this endpoint from the network.
 	Delete() error
+
+	// Retrieve the interfaces' statistics from the sandbox
+	Statistics() (map[string]*sandbox.InterfaceStatistics, error)
 }
 
 // EndpointOption is a option setter function type used to pass varios options to Network
@@ -124,6 +127,7 @@ type endpoint struct {
 	generic       map[string]interface{}
 	joinLeaveDone chan struct{}
 	dbIndex       uint64
+	dbExists      bool
 	sync.Mutex
 }
 
@@ -241,7 +245,7 @@ func (ep *endpoint) KeyPrefix() []string {
 
 func (ep *endpoint) networkIDFromKey(key []string) (types.UUID, error) {
 	// endpoint Key structure : endpoint/network-id/endpoint-id
-	// its an invalid key if the key doesnt have all the 3 key elements above
+	// it's an invalid key if the key doesn't have all the 3 key elements above
 	if key == nil || len(key) < 3 || key[0] != datastore.EndpointKeyPrefix {
 		return types.UUID(""), fmt.Errorf("invalid endpoint key : %v", key)
 	}
@@ -258,6 +262,10 @@ func (ep *endpoint) Value() []byte {
 	return b
 }
 
+func (ep *endpoint) SetValue(value []byte) error {
+	return json.Unmarshal(value, ep)
+}
+
 func (ep *endpoint) Index() uint64 {
 	ep.Lock()
 	defer ep.Unlock()
@@ -268,6 +276,13 @@ func (ep *endpoint) SetIndex(index uint64) {
 	ep.Lock()
 	defer ep.Unlock()
 	ep.dbIndex = index
+	ep.dbExists = true
+}
+
+func (ep *endpoint) Exists() bool {
+	ep.Lock()
+	defer ep.Unlock()
+	return ep.dbExists
 }
 
 func (ep *endpoint) processOptions(options ...EndpointOption) {
@@ -545,6 +560,33 @@ func (ep *endpoint) Delete() error {
 	return nil
 }
 
+func (ep *endpoint) Statistics() (map[string]*sandbox.InterfaceStatistics, error) {
+	m := make(map[string]*sandbox.InterfaceStatistics)
+
+	ep.Lock()
+	n := ep.network
+	skey := ep.container.data.SandboxKey
+	ep.Unlock()
+
+	n.Lock()
+	c := n.ctrlr
+	n.Unlock()
+
+	sbox := c.sandboxGet(skey)
+	if sbox == nil {
+		return m, nil
+	}
+
+	var err error
+	for _, i := range sbox.Info().Interfaces() {
+		if m[i.DstName()], err = i.Statistics(); err != nil {
+			return m, err
+		}
+	}
+
+	return m, nil
+}
+
 func (ep *endpoint) deleteEndpoint() error {
 	ep.Lock()
 	n := ep.network

vendor/src/github.com/docker/libnetwork/network.go

@@ -67,6 +67,7 @@ type network struct {
 	generic     options.Generic
 	dbIndex     uint64
 	svcRecords  svcMap
+	dbExists    bool
 	stopWatchCh chan struct{}
 	sync.Mutex
 }
@@ -116,6 +117,10 @@ func (n *network) Value() []byte {
 	return b
 }
 
+func (n *network) SetValue(value []byte) error {
+	return json.Unmarshal(value, n)
+}
+
 func (n *network) Index() uint64 {
 	n.Lock()
 	defer n.Unlock()
@@ -125,9 +130,16 @@ func (n *network) Index() uint64 {
 func (n *network) SetIndex(index uint64) {
 	n.Lock()
 	n.dbIndex = index
+	n.dbExists = true
 	n.Unlock()
 }
 
+func (n *network) Exists() bool {
+	n.Lock()
+	defer n.Unlock()
+	return n.dbExists
+}
+
 func (n *network) EndpointCnt() uint64 {
 	n.Lock()
 	defer n.Unlock()
@@ -292,7 +304,9 @@ func (n *network) CreateEndpoint(name string, options ...EndpointOption) (Endpoi
 		return nil, types.ForbiddenErrorf("service endpoint with name %s already exists", name)
 	}
 
-	ep := &endpoint{name: name, iFaces: []*endpointInterface{}, generic: make(map[string]interface{})}
+	ep := &endpoint{name: name,
+		iFaces:  []*endpointInterface{},
+		generic: make(map[string]interface{})}
 	ep.id = types.UUID(stringid.GenerateRandomID())
 	ep.network = n
 	ep.processOptions(options...)

vendor/src/github.com/docker/libnetwork/sandbox/interface_linux.go

@@ -3,6 +3,8 @@ package sandbox
 import (
 	"fmt"
 	"net"
+	"os/exec"
+	"regexp"
 	"sync"
 
 	"github.com/docker/libnetwork/types"
@@ -153,6 +155,36 @@ func (i *nwIface) Remove() error {
 	})
 }
 
+// Returns the sandbox's side veth interface statistics
+func (i *nwIface) Statistics() (*InterfaceStatistics, error) {
+	i.Lock()
+	n := i.ns
+	i.Unlock()
+
+	n.Lock()
+	path := n.path
+	n.Unlock()
+
+	s := &InterfaceStatistics{}
+
+	err := nsInvoke(path, func(nsFD int) error { return nil }, func(callerFD int) error {
+		// For some reason ioutil.ReadFile(netStatsFile) reads the file in
+		// the default netns when this code is invoked from docker.
+		// Executing "cat <netStatsFile>" works as expected.
+		data, err := exec.Command("cat", netStatsFile).Output()
+		if err != nil {
+			return fmt.Errorf("failure opening %s: %v", netStatsFile, err)
+		}
+		return scanInterfaceStats(string(data), i.DstName(), s)
+	})
+
+	if err != nil {
+		err = fmt.Errorf("failed to retrieve the statistics for %s in netns %s: %v", i.DstName(), path, err)
+	}
+
+	return s, err
+}
+
 func (n *networkNamespace) findDst(srcName string, isBridge bool) string {
 	n.Lock()
 	defer n.Unlock()
@@ -311,3 +343,28 @@ func setInterfaceRoutes(iface netlink.Link, i *nwIface) error {
 	}
 	return nil
 }
+
+// In older kernels (like the one in Centos 6.6 distro) sysctl does not have netns support. Therefore
+// we cannot gather the statistics from /sys/class/net/<dev>/statistics/<counter> files. Per-netns stats
+// are naturally found in /proc/net/dev in kernels which support netns (ifconfig relyes on that).
+const (
+	netStatsFile = "/proc/net/dev"
+	base         = "[ ]*%s:([ ]+[0-9]+){16}"
+)
+
+func scanInterfaceStats(data, ifName string, i *InterfaceStatistics) error {
+	var (
+		bktStr string
+		bkt    uint64
+	)
+
+	regex := fmt.Sprintf(base, ifName)
+	re := regexp.MustCompile(regex)
+	line := re.FindString(data)
+
+	_, err := fmt.Sscanf(line, "%s %d %d %d %d %d %d %d %d %d %d %d %d %d %d %d %d",
+		&bktStr, &i.RxBytes, &i.RxPackets, &i.RxErrors, &i.RxDropped, &bkt, &bkt, &bkt,
+		&bkt, &i.TxBytes, &i.TxPackets, &i.TxErrors, &i.TxDropped, &bkt, &bkt, &bkt, &bkt)
+
+	return err
+}

vendor/src/github.com/docker/libnetwork/sandbox/sandbox.go

@@ -1,6 +1,7 @@
 package sandbox
 
 import (
+	"fmt"
 	"net"
 
 	"github.com/docker/libnetwork/types"
@@ -146,4 +147,24 @@ type Interface interface {
 	// Remove an interface from the sandbox by renaming to original name
 	// and moving it out of the sandbox.
 	Remove() error
+
+	// Statistics returns the statistics for this interface
+	Statistics() (*InterfaceStatistics, error)
+}
+
+// InterfaceStatistics represents the interface's statistics
+type InterfaceStatistics struct {
+	RxBytes   uint64
+	RxPackets uint64
+	RxErrors  uint64
+	RxDropped uint64
+	TxBytes   uint64
+	TxPackets uint64
+	TxErrors  uint64
+	TxDropped uint64
+}
+
+func (is *InterfaceStatistics) String() string {
+	return fmt.Sprintf("\nRxBytes: %d, RxPackets: %d, RxErrors: %d, RxDropped: %d, TxBytes: %d, TxPackets: %d, TxErrors: %d, TxDropped: %d",
+		is.RxBytes, is.RxPackets, is.RxErrors, is.RxDropped, is.TxBytes, is.TxPackets, is.TxErrors, is.TxDropped)
 }

vendor/src/github.com/docker/libnetwork/store.go

@@ -11,10 +11,7 @@ import (
 )
 
 func (c *controller) validateDatastoreConfig() bool {
-	if c.cfg == nil || c.cfg.Datastore.Client.Provider == "" || c.cfg.Datastore.Client.Address == "" {
-		return false
-	}
-	return true
+	return c.cfg != nil && c.cfg.Datastore.Client.Provider != "" && c.cfg.Datastore.Client.Address != ""
 }
 
 func (c *controller) initDataStore() error {
@@ -197,6 +194,7 @@ func (c *controller) watchNetworks() error {
 					}
 				}
 				c.processNetworkUpdate(nws, &tmpview)
+
 				// Delete processing
 				for k := range tmpview {
 					c.Lock()
@@ -259,7 +257,7 @@ func (n *network) watchEndpoints() error {
 						continue
 					}
 					delete(tmpview, ep.id)
-					ep.dbIndex = epe.LastIndex
+					ep.SetIndex(epe.LastIndex)
 					ep.network = n
 					if n.ctrlr.processEndpointUpdate(&ep) {
 						err = n.ctrlr.newEndpointFromStore(epe.Key, &ep)
@@ -310,15 +308,17 @@ func (c *controller) processNetworkUpdate(nws []*store.KVPair, prune *networkTab
 		if prune != nil {
 			delete(*prune, n.id)
 		}
-		n.dbIndex = kve.LastIndex
+		n.SetIndex(kve.LastIndex)
 		c.Lock()
 		existing, ok := c.networks[n.id]
 		c.Unlock()
 		if ok {
 			existing.Lock()
 			// Skip existing network update
-			if existing.dbIndex != n.dbIndex {
-				existing.dbIndex = n.dbIndex
+			if existing.dbIndex != n.Index() {
+				// Can't use SetIndex() since existing is locked.
+				existing.dbIndex = n.Index()
+				existing.dbExists = true
 				existing.endpointCnt = n.endpointCnt
 			}
 			existing.Unlock()
@@ -353,8 +353,10 @@ func (c *controller) processEndpointUpdate(ep *endpoint) bool {
 
 	ee := existing.(*endpoint)
 	ee.Lock()
-	if ee.dbIndex != ep.dbIndex {
-		ee.dbIndex = ep.dbIndex
+	if ee.dbIndex != ep.Index() {
+		// Can't use SetIndex() because ee is locked.
+		ee.dbIndex = ep.Index()
+		ee.dbExists = true
 		if ee.container != nil && ep.container != nil {
 			// we care only about the container id
 			ee.container.id = ep.container.id