Merge pull request #23524 from aboch/restore

Daemon to support network restore

daemon/cluster/cluster.go

@@ -121,14 +121,6 @@ func New(config Config) (*Cluster, error) {
 	return c, nil
 }
 
-func (c *Cluster) checkCompatibility() error {
-	info, _ := c.config.Backend.SystemInfo()
-	if info != nil && (info.ClusterStore != "" || info.ClusterAdvertise != "") {
-		return fmt.Errorf("swarm mode is incompatible with `--cluster-store` and `--cluster-advertise daemon configuration")
-	}
-	return nil
-}
-
 func (c *Cluster) saveState() error {
 	dt, err := json.Marshal(state{ListenAddr: c.listenAddr})
 	if err != nil {
@@ -173,7 +165,7 @@ func (c *Cluster) reconnectOnFailure(ctx context.Context) {
 }
 
 func (c *Cluster) startNewNode(forceNewCluster bool, listenAddr, joinAddr, secret, cahash string, ismanager bool) (*swarmagent.Node, context.Context, error) {
-	if err := c.checkCompatibility(); err != nil {
+	if err := c.config.Backend.IsSwarmCompatible(); err != nil {
 		return nil, nil, err
 	}
 	c.node = nil

daemon/cluster/executor/backend.go

@@ -32,4 +32,5 @@ type Backend interface {
 	ListContainersForNode(nodeID string) []string
 	SetNetworkBootstrapKeys([]*networktypes.EncryptionKey) error
 	SetClusterProvider(provider cluster.Provider)
+	IsSwarmCompatible() error
 }

daemon/config_solaris.go

@@ -37,3 +37,7 @@ func (config *Config) InstallFlags(cmd *flag.FlagSet, usageFn func(string) strin
 	// Then platform-specific install flags
 	config.attachExperimentalFlags(cmd, usageFn)
 }
+
+func (config *Config) isSwarmCompatible() error {
+	return nil
+}

daemon/config_unix.go

@@ -3,6 +3,7 @@
 package daemon
 
 import (
+	"fmt"
 	"net"
 
 	"github.com/docker/docker/opts"
@@ -120,3 +121,13 @@ func (config *Config) GetAllRuntimes() map[string]types.Runtime {
 	config.reloadLock.Unlock()
 	return rts
 }
+
+func (config *Config) isSwarmCompatible() error {
+	if config.IsValueSet("cluster-store") || config.IsValueSet("cluster-advertise") {
+		return fmt.Errorf("--cluster-store and --cluster-advertise daemon configurations are incompatible with swarm mode")
+	}
+	if config.LiveRestore {
+		return fmt.Errorf("--live-restore daemon configuration is incompatible with swarm mode")
+	}
+	return nil
+}

daemon/config_windows.go

@@ -57,3 +57,7 @@ func (config *Config) GetDefaultRuntimeName() string {
 func (config *Config) GetAllRuntimes() map[string]types.Runtime {
 	return map[string]types.Runtime{}
 }
+
+func (config *Config) isSwarmCompatible() error {
+	return nil
+}

daemon/container_operations.go

@@ -30,7 +30,7 @@ var (
 	getPortMapInfo    = container.GetSandboxPortMapInfo
 )
 
-func (daemon *Daemon) buildSandboxOptions(container *container.Container, n libnetwork.Network) ([]libnetwork.SandboxOption, error) {
+func (daemon *Daemon) buildSandboxOptions(container *container.Container) ([]libnetwork.SandboxOption, error) {
 	var (
 		sboxOptions []libnetwork.SandboxOption
 		err         error
@@ -176,16 +176,19 @@ func (daemon *Daemon) buildSandboxOptions(container *container.Container, n libn
 
 	// Legacy Link feature is supported only for the default bridge network.
 	// return if this call to build join options is not for default bridge network
-	if n.Name() != defaultNetName {
+	// Legacy Link is only supported by docker run --link
+	if _, ok := container.NetworkSettings.Networks[defaultNetName]; !container.HostConfig.NetworkMode.IsDefault() || !ok {
 		return sboxOptions, nil
 	}
 
-	ep, _ := container.GetEndpointInNetwork(n)
-	if ep == nil {
+	if container.NetworkSettings.Networks[defaultNetName].EndpointID == "" {
 		return sboxOptions, nil
 	}
 
-	var childEndpoints, parentEndpoints []string
+	var (
+		childEndpoints, parentEndpoints []string
+		cEndpointID                     string
+	)
 
 	children := daemon.children(container)
 	for linkAlias, child := range children {
@@ -200,9 +203,9 @@ func (daemon *Daemon) buildSandboxOptions(container *container.Container, n libn
 			aliasList = aliasList + " " + child.Name[1:]
 		}
 		sboxOptions = append(sboxOptions, libnetwork.OptionExtraHost(aliasList, child.NetworkSettings.Networks[defaultNetName].IPAddress))
-		cEndpoint, _ := child.GetEndpointInNetwork(n)
-		if cEndpoint != nil && cEndpoint.ID() != "" {
-			childEndpoints = append(childEndpoints, cEndpoint.ID())
+		cEndpointID = child.NetworkSettings.Networks[defaultNetName].EndpointID
+		if cEndpointID != "" {
+			childEndpoints = append(childEndpoints, cEndpointID)
 		}
 	}
 
@@ -219,8 +222,8 @@ func (daemon *Daemon) buildSandboxOptions(container *container.Container, n libn
 			alias,
 			bridgeSettings.IPAddress,
 		))
-		if ep.ID() != "" {
-			parentEndpoints = append(parentEndpoints, ep.ID())
+		if cEndpointID != "" {
+			parentEndpoints = append(parentEndpoints, cEndpointID)
 		}
 	}
 
@@ -312,7 +315,7 @@ func (daemon *Daemon) updateNetwork(container *container.Container) error {
 		return nil
 	}
 
-	options, err := daemon.buildSandboxOptions(container, n)
+	options, err := daemon.buildSandboxOptions(container)
 	if err != nil {
 		return fmt.Errorf("Update network failed: %v", err)
 	}
@@ -570,7 +573,7 @@ func (daemon *Daemon) connectToNetwork(container *container.Container, idOrName
 	}
 
 	if sb == nil {
-		options, err := daemon.buildSandboxOptions(container, n)
+		options, err := daemon.buildSandboxOptions(container)
 		if err != nil {
 			return err
 		}
@@ -709,6 +712,9 @@ func (daemon *Daemon) getNetworkedContainer(containerID, connectedContainerID st
 }
 
 func (daemon *Daemon) releaseNetwork(container *container.Container) {
+	if daemon.netController == nil {
+		return
+	}
 	if container.HostConfig.NetworkMode.IsContainer() || container.Config.NetworkDisabled {
 		return
 	}

daemon/daemon.go

@@ -146,6 +146,7 @@ func (daemon *Daemon) restore() error {
 
 	var migrateLegacyLinks bool
 	restartContainers := make(map[*container.Container]chan struct{})
+	activeSandboxes := make(map[string]interface{})
 	for _, c := range containers {
 		if err := daemon.registerName(c); err != nil {
 			logrus.Errorf("Failed to register container %s: %s", c.ID, err)
@@ -178,6 +179,16 @@ func (daemon *Daemon) restore() error {
 					logrus.Errorf("Failed to restore with containerd: %q", err)
 					return
 				}
+				if !c.HostConfig.NetworkMode.IsContainer() {
+					options, err := daemon.buildSandboxOptions(c)
+					if err != nil {
+						logrus.Warnf("Failed build sandbox option to restore container %s: %v", c.ID, err)
+					}
+					mapLock.Lock()
+					activeSandboxes[c.NetworkSettings.SandboxID] = options
+					mapLock.Unlock()
+				}
+
 			}
 			// fixme: only if not running
 			// get list of containers we need to restart
@@ -209,6 +220,10 @@ func (daemon *Daemon) restore() error {
 		}(c)
 	}
 	wg.Wait()
+	daemon.netController, err = daemon.initNetworkController(daemon.configStore, activeSandboxes)
+	if err != nil {
+		return fmt.Errorf("Error initializing network controller: %v", err)
+	}
 
 	// migrate any legacy links from sqlite
 	linkdbFile := filepath.Join(daemon.root, "linkgraph.db")
@@ -356,6 +371,15 @@ func (daemon *Daemon) SetClusterProvider(clusterProvider cluster.Provider) {
 	daemon.netController.SetClusterProvider(clusterProvider)
 }
 
+// IsSwarmCompatible verifies if the current daemon
+// configuration is compatible with the swarm mode
+func (daemon *Daemon) IsSwarmCompatible() error {
+	if daemon.configStore == nil {
+		return nil
+	}
+	return daemon.configStore.isSwarmCompatible()
+}
+
 // NewDaemon sets up everything for the daemon to be able to service
 // requests from the webserver.
 func NewDaemon(config *Config, registryService registry.Service, containerdRemote libcontainerd.Remote) (daemon *Daemon, err error) {
@@ -530,11 +554,6 @@ func NewDaemon(config *Config, registryService registry.Service, containerdRemot
 		return nil, err
 	}
 
-	d.netController, err = d.initNetworkController(config)
-	if err != nil {
-		return nil, fmt.Errorf("Error initializing network controller: %v", err)
-	}
-
 	sysInfo := sysinfo.New(false)
 	// Check if Devices cgroup is mounted, it is hard requirement for container security,
 	// on Linux.
@@ -912,15 +931,17 @@ func (daemon *Daemon) reloadClusterDiscovery(config *Config) error {
 		}
 	}
 
+	if daemon.clusterProvider != nil {
+		if err := config.isSwarmCompatible(); err != nil {
+			return err
+		}
+	}
+
 	// check discovery modifications
 	if !modifiedDiscoverySettings(daemon.configStore, newAdvertise, newClusterStore, config.ClusterOpts) {
 		return nil
 	}
 
-	if daemon.clusterProvider != nil {
-		return fmt.Errorf("--cluster-store and --cluster-advertise daemon configurations are incompatible with swarm mode")
-	}
-
 	// enable discovery for the first time if it was not previously enabled
 	if daemon.discoveryWatcher == nil {
 		discoveryWatcher, err := initDiscovery(newClusterStore, newAdvertise, config.ClusterOpts)
@@ -947,7 +968,7 @@ func (daemon *Daemon) reloadClusterDiscovery(config *Config) error {
 	if daemon.netController == nil {
 		return nil
 	}
-	netOptions, err := daemon.networkOptions(daemon.configStore)
+	netOptions, err := daemon.networkOptions(daemon.configStore, nil)
 	if err != nil {
 		logrus.Warnf("Failed to reload configuration with network controller: %v", err)
 		return nil
@@ -964,7 +985,7 @@ func isBridgeNetworkDisabled(config *Config) bool {
 	return config.bridgeConfig.Iface == disableNetworkBridge
 }
 
-func (daemon *Daemon) networkOptions(dconfig *Config) ([]nwconfig.Option, error) {
+func (daemon *Daemon) networkOptions(dconfig *Config, activeSandboxes map[string]interface{}) ([]nwconfig.Option, error) {
 	options := []nwconfig.Option{}
 	if dconfig == nil {
 		return options, nil
@@ -999,6 +1020,11 @@ func (daemon *Daemon) networkOptions(dconfig *Config) ([]nwconfig.Option, error)
 
 	options = append(options, nwconfig.OptionLabels(dconfig.Labels))
 	options = append(options, driverOptions(dconfig)...)
+
+	if daemon.configStore != nil && daemon.configStore.LiveRestore && len(activeSandboxes) != 0 {
+		options = append(options, nwconfig.OptionActiveSandboxes(activeSandboxes))
+	}
+
 	return options, nil
 }
 

daemon/daemon_solaris.go

@@ -113,7 +113,7 @@ func configureKernelSecuritySupport(config *Config, driverName string) error {
 	return nil
 }
 
-func (daemon *Daemon) initNetworkController(config *Config) (libnetwork.NetworkController, error) {
+func (daemon *Daemon) initNetworkController(config *Config, activeSandboxes map[string]interface{}) (libnetwork.NetworkController, error) {
 	return nil, nil
 }
 

daemon/daemon_unix.go

@@ -627,8 +627,8 @@ func configureKernelSecuritySupport(config *Config, driverName string) error {
 	return nil
 }
 
-func (daemon *Daemon) initNetworkController(config *Config) (libnetwork.NetworkController, error) {
-	netOptions, err := daemon.networkOptions(config)
+func (daemon *Daemon) initNetworkController(config *Config, activeSandboxes map[string]interface{}) (libnetwork.NetworkController, error) {
+	netOptions, err := daemon.networkOptions(config, activeSandboxes)
 	if err != nil {
 		return nil, err
 	}
@@ -638,16 +638,24 @@ func (daemon *Daemon) initNetworkController(config *Config) (libnetwork.NetworkC
 		return nil, fmt.Errorf("error obtaining controller instance: %v", err)
 	}
 
+	if len(activeSandboxes) > 0 {
+		logrus.Infof("There are old running containers, the network config will not take affect")
+		return controller, nil
+	}
+
 	// Initialize default network on "null"
-	if _, err := controller.NewNetwork("null", "none", "", libnetwork.NetworkOptionPersist(false)); err != nil {
-		return nil, fmt.Errorf("Error creating default \"null\" network: %v", err)
+	if n, _ := controller.NetworkByName("none"); n == nil {
+		if _, err := controller.NewNetwork("null", "none", "", libnetwork.NetworkOptionPersist(true)); err != nil {
+			return nil, fmt.Errorf("Error creating default \"null\" network: %v", err)
+		}
 	}
 
 	// Initialize default network on "host"
-	if _, err := controller.NewNetwork("host", "host", "", libnetwork.NetworkOptionPersist(false)); err != nil {
-		return nil, fmt.Errorf("Error creating default \"host\" network: %v", err)
+	if n, _ := controller.NetworkByName("host"); n == nil {
+		if _, err := controller.NewNetwork("host", "host", "", libnetwork.NetworkOptionPersist(true)); err != nil {
+			return nil, fmt.Errorf("Error creating default \"host\" network: %v", err)
+		}
 	}
-
 	if !config.DisableBridge {
 		// Initialize default driver "bridge"
 		if err := initBridgeDriver(controller, config); err != nil {

daemon/daemon_unix_test.go

@@ -183,7 +183,7 @@ func TestNetworkOptions(t *testing.T) {
 		},
 	}
 
-	if _, err := daemon.networkOptions(dconfigCorrect); err != nil {
+	if _, err := daemon.networkOptions(dconfigCorrect, nil); err != nil {
 		t.Fatalf("Expect networkOptions success, got error: %v", err)
 	}
 
@@ -193,7 +193,7 @@ func TestNetworkOptions(t *testing.T) {
 		},
 	}
 
-	if _, err := daemon.networkOptions(dconfigWrong); err == nil {
+	if _, err := daemon.networkOptions(dconfigWrong, nil); err == nil {
 		t.Fatalf("Expected networkOptions error, got nil")
 	}
 }

daemon/daemon_windows.go

@@ -189,8 +189,8 @@ func configureMaxThreads(config *Config) error {
 	return nil
 }
 
-func (daemon *Daemon) initNetworkController(config *Config) (libnetwork.NetworkController, error) {
-	netOptions, err := daemon.networkOptions(config)
+func (daemon *Daemon) initNetworkController(config *Config, activeSandboxes map[string]interface{}) (libnetwork.NetworkController, error) {
+	netOptions, err := daemon.networkOptions(config, nil)
 	if err != nil {
 		return nil, err
 	}

daemon/network.go

@@ -59,6 +59,9 @@ func (daemon *Daemon) GetNetworkByID(partialID string) (libnetwork.Network, erro
 // GetNetworkByName function returns a network for a given network name.
 func (daemon *Daemon) GetNetworkByName(name string) (libnetwork.Network, error) {
 	c := daemon.netController
+	if c == nil {
+		return nil, libnetwork.ErrNoSuchNetwork(name)
+	}
 	if name == "" {
 		name = c.Config().Daemon.DefaultNetwork
 	}
@@ -68,6 +71,9 @@ func (daemon *Daemon) GetNetworkByName(name string) (libnetwork.Network, error)
 // GetNetworksByID returns a list of networks whose ID partially matches zero or more networks
 func (daemon *Daemon) GetNetworksByID(partialID string) []libnetwork.Network {
 	c := daemon.netController
+	if c == nil {
+		return nil
+	}
 	list := []libnetwork.Network{}
 	l := func(nw libnetwork.Network) bool {
 		if strings.HasPrefix(nw.ID(), partialID) {

hack/vendor.sh

@@ -65,7 +65,7 @@ clone git github.com/RackSec/srslog 259aed10dfa74ea2961eddd1d9847619f6e98837
 clone git github.com/imdario/mergo 0.2.1
 
 #get libnetwork packages
-clone git github.com/docker/libnetwork e8da32ce5693f0ed6823d59c8415baf76c0809ea
+clone git github.com/docker/libnetwork 452dff166e0abd9455b07c835613197f078a34de
 clone git github.com/docker/go-events 39718a26497694185f8fb58a7d6f31947f3dc42d
 clone git github.com/armon/go-radix e39d623f12e8e41c7b5529e9a9dd67a1e2261f80
 clone git github.com/armon/go-metrics eb0af217e5e9747e41dd5303755356b62d28e3ec

integration-cli/docker_cli_network_unix_test.go

@@ -1586,3 +1586,67 @@ func (s *DockerNetworkSuite) TestDockerNetworkCreateDeleteSpecialCharacters(c *c
 	dockerCmd(c, "network", "rm", "kiwl$%^")
 	assertNwNotAvailable(c, "kiwl$%^")
 }
+
+func (s *DockerDaemonSuite) TestDaemonRestartRestoreBridgeNetwork(t *check.C) {
+	testRequires(t, DaemonIsLinux)
+	if err := s.d.StartWithBusybox("--live-restore"); err != nil {
+		t.Fatal(err)
+	}
+	defer s.d.Stop()
+	oldCon := "old"
+
+	_, err := s.d.Cmd("run", "-d", "--name", oldCon, "-p", "80:80", "busybox", "top")
+	if err != nil {
+		t.Fatal(err)
+	}
+	oldContainerIP, err := s.d.Cmd("inspect", "-f", "{{ .NetworkSettings.Networks.bridge.IPAddress }}", oldCon)
+	if err != nil {
+		t.Fatal(err)
+	}
+	// Kill the daemon
+	if err := s.d.Kill(); err != nil {
+		t.Fatal(err)
+	}
+
+	// restart the daemon
+	if err := s.d.Start("--live-restore"); err != nil {
+		t.Fatal(err)
+	}
+
+	// start a new container, the new container's ip should not be the same with
+	// old running container.
+	newCon := "new"
+	_, err = s.d.Cmd("run", "-d", "--name", newCon, "busybox", "top")
+	if err != nil {
+		t.Fatal(err)
+	}
+	newContainerIP, err := s.d.Cmd("inspect", "-f", "{{ .NetworkSettings.Networks.bridge.IPAddress }}", newCon)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if strings.Compare(strings.TrimSpace(oldContainerIP), strings.TrimSpace(newContainerIP)) == 0 {
+		t.Fatalf("new container ip should not equal to old running container  ip")
+	}
+
+	// start a new container, the new container should ping old running container
+	_, err = s.d.Cmd("run", "-t", "busybox", "ping", "-c", "1", oldContainerIP)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// start a new container try to publist port 80:80 will failed
+	out, err := s.d.Cmd("run", "-p", "80:80", "-d", "busybox", "top")
+	if err == nil || !strings.Contains(out, "Bind for 0.0.0.0:80 failed: port is already allocated") {
+		t.Fatalf("80 port is allocated to old running container, it should failed on allocating to new container")
+	}
+
+	// kill old running container and try to allocate again
+	_, err = s.d.Cmd("kill", oldCon)
+	if err != nil {
+		t.Fatal(err)
+	}
+	_, err = s.d.Cmd("run", "-p", "80:80", "-d", "busybox", "top")
+	if err != nil {
+		t.Fatal(err)
+	}
+}

vendor/src/github.com/docker/libnetwork/config/config.go

@@ -15,9 +15,10 @@ import (
 
 // Config encapsulates configurations of various Libnetwork components
 type Config struct {
-	Daemon  DaemonCfg
-	Cluster ClusterCfg
-	Scopes  map[string]*datastore.ScopeCfg
+	Daemon          DaemonCfg
+	Cluster         ClusterCfg
+	Scopes          map[string]*datastore.ScopeCfg
+	ActiveSandboxes map[string]interface{}
 }
 
 // DaemonCfg represents libnetwork core configuration
@@ -245,3 +246,11 @@ func OptionLocalKVProviderConfig(config *store.Config) Option {
 		c.Scopes[datastore.LocalScope].Client.Config = config
 	}
 }
+
+// OptionActiveSandboxes function returns an option setter for passing the sandboxes
+// which were active during previous daemon life
+func OptionActiveSandboxes(sandboxes map[string]interface{}) Option {
+	return func(c *Config) {
+		c.ActiveSandboxes = sandboxes
+	}
+}

vendor/src/github.com/docker/libnetwork/controller.go

@@ -203,15 +203,13 @@ func New(cfgOptions ...config.Option) (NetworkController, error) {
 		}
 	}
 
-	// Reserve pools first before doing cleanup. This is because
-	// if the pools are not populated properly, the cleanups of
-	// endpoint/network and sandbox below will not be able to
-	// release ip subnets and addresses properly into the pool
-	// because the pools won't exist.
+	// Reserve pools first before doing cleanup. Otherwise the
+	// cleanups of endpoint/network and sandbox below will
+	// generate many unnecessary warnings
 	c.reservePools()
 
 	// Cleanup resources
-	c.sandboxCleanup()
+	c.sandboxCleanup(c.cfg.ActiveSandboxes)
 	c.cleanupLocalEndpoints()
 	c.networkCleanup()
 
@@ -671,9 +669,27 @@ func (c *controller) reservePools() {
 				c.Gateway = n.ipamV6Info[i].Gateway.IP.String()
 			}
 		}
+		// Reserve pools
 		if err := n.ipamAllocate(); err != nil {
 			log.Warnf("Failed to allocate ipam pool(s) for network %q (%s): %v", n.Name(), n.ID(), err)
 		}
+		// Reserve existing endpoints' addresses
+		ipam, _, err := n.getController().getIPAMDriver(n.ipamType)
+		if err != nil {
+			log.Warnf("Failed to retrieve ipam driver for network %q (%s) during address reservation", n.Name(), n.ID())
+			continue
+		}
+		epl, err := n.getEndpointsFromStore()
+		if err != nil {
+			log.Warnf("Failed to retrieve list of current endpoints on network %q (%s)", n.Name(), n.ID())
+			continue
+		}
+		for _, ep := range epl {
+			if err := ep.assignAddress(ipam, true, ep.Iface().AddressIPv6() != nil); err != nil {
+				log.Warnf("Failed to reserve current adress for endpoint %q (%s) on network %q (%s)",
+					ep.Name(), ep.ID(), n.Name(), n.ID())
+			}
+		}
 	}
 }
 
@@ -832,7 +848,7 @@ func (c *controller) NewSandbox(containerID string, options ...SandboxOption) (s
 
 	if sb.config.useDefaultSandBox {
 		c.sboxOnce.Do(func() {
-			c.defOsSbox, err = osl.NewSandbox(sb.Key(), false)
+			c.defOsSbox, err = osl.NewSandbox(sb.Key(), false, false)
 		})
 
 		if err != nil {
@@ -844,7 +860,7 @@ func (c *controller) NewSandbox(containerID string, options ...SandboxOption) (s
 	}
 
 	if sb.osSbox == nil && !sb.config.useExternalKey {
-		if sb.osSbox, err = osl.NewSandbox(sb.Key(), !sb.config.useDefaultSandBox); err != nil {
+		if sb.osSbox, err = osl.NewSandbox(sb.Key(), !sb.config.useDefaultSandBox, false); err != nil {
 			return nil, fmt.Errorf("failed to create new osl sandbox: %v", err)
 		}
 	}

vendor/src/github.com/docker/libnetwork/datastore/cache.go

@@ -86,25 +86,52 @@ out:
 	return kmap, nil
 }
 
-func (c *cache) add(kvObject KVObject) error {
+func (c *cache) add(kvObject KVObject, atomic bool) error {
 	kmap, err := c.kmap(kvObject)
 	if err != nil {
 		return err
 	}
 
 	c.Lock()
+	// If atomic is true, cache needs to maintain its own index
+	// for atomicity and the add needs to be atomic.
+	if atomic {
+		if prev, ok := kmap[Key(kvObject.Key()...)]; ok {
+			if prev.Index() != kvObject.Index() {
+				c.Unlock()
+				return ErrKeyModified
+			}
+		}
+
+		// Increment index
+		index := kvObject.Index()
+		index++
+		kvObject.SetIndex(index)
+	}
+
 	kmap[Key(kvObject.Key()...)] = kvObject
 	c.Unlock()
 	return nil
 }
 
-func (c *cache) del(kvObject KVObject) error {
+func (c *cache) del(kvObject KVObject, atomic bool) error {
 	kmap, err := c.kmap(kvObject)
 	if err != nil {
 		return err
 	}
 
 	c.Lock()
+	// If atomic is true, cache needs to maintain its own index
+	// for atomicity and del needs to be atomic.
+	if atomic {
+		if prev, ok := kmap[Key(kvObject.Key()...)]; ok {
+			if prev.Index() != kvObject.Index() {
+				c.Unlock()
+				return ErrKeyModified
+			}
+		}
+	}
+
 	delete(kmap, Key(kvObject.Key()...))
 	c.Unlock()
 	return nil

vendor/src/github.com/docker/libnetwork/datastore/datastore.go

@@ -410,7 +410,9 @@ func (ds *datastore) PutObjectAtomic(kvObject KVObject) error {
 
 add_cache:
 	if ds.cache != nil {
-		return ds.cache.add(kvObject)
+		// If persistent store is skipped, sequencing needs to
+		// happen in cache.
+		return ds.cache.add(kvObject, kvObject.Skip())
 	}
 
 	return nil
@@ -435,7 +437,9 @@ func (ds *datastore) PutObject(kvObject KVObject) error {
 
 add_cache:
 	if ds.cache != nil {
-		return ds.cache.add(kvObject)
+		// If persistent store is skipped, sequencing needs to
+		// happen in cache.
+		return ds.cache.add(kvObject, kvObject.Skip())
 	}
 
 	return nil
@@ -537,7 +541,9 @@ func (ds *datastore) DeleteObject(kvObject KVObject) error {
 
 	// cleaup the cache first
 	if ds.cache != nil {
-		ds.cache.del(kvObject)
+		// If persistent store is skipped, sequencing needs to
+		// happen in cache.
+		ds.cache.del(kvObject, kvObject.Skip())
 	}
 
 	if kvObject.Skip() {
@@ -572,7 +578,9 @@ func (ds *datastore) DeleteObjectAtomic(kvObject KVObject) error {
 del_cache:
 	// cleanup the cache only if AtomicDelete went through successfully
 	if ds.cache != nil {
-		return ds.cache.del(kvObject)
+		// If persistent store is skipped, sequencing needs to
+		// happen in cache.
+		return ds.cache.del(kvObject, kvObject.Skip())
 	}
 
 	return nil
@@ -585,7 +593,9 @@ func (ds *datastore) DeleteTree(kvObject KVObject) error {
 
 	// cleaup the cache first
 	if ds.cache != nil {
-		ds.cache.del(kvObject)
+		// If persistent store is skipped, sequencing needs to
+		// happen in cache.
+		ds.cache.del(kvObject, kvObject.Skip())
 	}
 
 	if kvObject.Skip() {

vendor/src/github.com/docker/libnetwork/drivers/bridge/bridge.go

@@ -91,6 +91,7 @@ type connectivityConfiguration struct {
 
 type bridgeEndpoint struct {
 	id              string
+	nid             string
 	srcName         string
 	addr            *net.IPNet
 	addrv6          *net.IPNet
@@ -99,6 +100,8 @@ type bridgeEndpoint struct {
 	containerConfig *containerConfiguration
 	extConnConfig   *connectivityConfiguration
 	portMapping     []types.PortBinding // Operation port bindings
+	dbIndex         uint64
+	dbExists        bool
 }
 
 type bridgeNetwork struct {
@@ -882,7 +885,7 @@ func (d *driver) CreateEndpoint(nid, eid string, ifInfo driverapi.InterfaceInfo,
 
 	// Create and add the endpoint
 	n.Lock()
-	endpoint := &bridgeEndpoint{id: eid, config: epConfig}
+	endpoint := &bridgeEndpoint{id: eid, nid: nid, config: epConfig}
 	n.endpoints[eid] = endpoint
 	n.Unlock()
 
@@ -1009,6 +1012,10 @@ func (d *driver) CreateEndpoint(nid, eid string, ifInfo driverapi.InterfaceInfo,
 		}
 	}
 
+	if err = d.storeUpdate(endpoint); err != nil {
+		return fmt.Errorf("failed to save bridge endpoint %s to store: %v", ep.id[0:7], err)
+	}
+
 	return nil
 }
 
@@ -1069,6 +1076,10 @@ func (d *driver) DeleteEndpoint(nid, eid string) error {
 		d.nlh.LinkDel(link)
 	}
 
+	if err := d.storeDelete(ep); err != nil {
+		logrus.Warnf("Failed to remove bridge endpoint %s from store: %v", ep.id[0:7], err)
+	}
+
 	return nil
 }
 
@@ -1225,6 +1236,11 @@ func (d *driver) ProgramExternalConnectivity(nid, eid string, options map[string
 		return err
 	}
 
+	if err = d.storeUpdate(endpoint); err != nil {
+		endpoint.portMapping = nil
+		return fmt.Errorf("failed to update bridge endpoint %s to store: %v", endpoint.id[0:7], err)
+	}
+
 	if !network.config.EnableICC {
 		return d.link(network, endpoint, true)
 	}

vendor/src/github.com/docker/libnetwork/drivers/bridge/bridge_store.go

@@ -12,7 +12,13 @@ import (
 	"github.com/docker/libnetwork/types"
 )
 
-const bridgePrefix = "bridge"
+const (
+	// network config prefix was not specific enough.
+	// To be backward compatible, need custom endpoint
+	// prefix with different root
+	bridgePrefix         = "bridge"
+	bridgeEndpointPrefix = "bridge-endpoint"
+)
 
 func (d *driver) initStore(option map[string]interface{}) error {
 	if data, ok := option[netlabel.LocalKVClient]; ok {
@@ -26,7 +32,15 @@ func (d *driver) initStore(option map[string]interface{}) error {
 			return types.InternalErrorf("bridge driver failed to initialize data store: %v", err)
 		}
 
-		return d.populateNetworks()
+		err = d.populateNetworks()
+		if err != nil {
+			return err
+		}
+
+		err = d.populateEndpoints()
+		if err != nil {
+			return err
+		}
 	}
 
 	return nil
@@ -48,6 +62,36 @@ func (d *driver) populateNetworks() error {
 		if err = d.createNetwork(ncfg); err != nil {
 			logrus.Warnf("could not create bridge network for id %s bridge name %s while booting up from persistent state: %v", ncfg.ID, ncfg.BridgeName, err)
 		}
+		logrus.Debugf("Network (%s) restored", ncfg.ID[0:7])
+	}
+
+	return nil
+}
+
+func (d *driver) populateEndpoints() error {
+	kvol, err := d.store.List(datastore.Key(bridgeEndpointPrefix), &bridgeEndpoint{})
+	if err != nil && err != datastore.ErrKeyNotFound {
+		return fmt.Errorf("failed to get bridge endpoints from store: %v", err)
+	}
+
+	if err == datastore.ErrKeyNotFound {
+		return nil
+	}
+
+	for _, kvo := range kvol {
+		ep := kvo.(*bridgeEndpoint)
+		n, ok := d.networks[ep.nid]
+		if !ok {
+			logrus.Debugf("Network (%s) not found for restored bridge endpoint (%s)", ep.nid[0:7], ep.id[0:7])
+			logrus.Debugf("Deleting stale bridge endpoint (%s) from store", ep.nid[0:7])
+			if err := d.storeDelete(ep); err != nil {
+				logrus.Debugf("Failed to delete stale bridge endpoint (%s) from store", ep.nid[0:7])
+			}
+			continue
+		}
+		n.endpoints[ep.id] = ep
+		n.restorePortAllocations(ep)
+		logrus.Debugf("Endpoint (%s) restored to network (%s)", ep.id[0:7], ep.nid[0:7])
 	}
 
 	return nil
@@ -184,7 +228,7 @@ func (ncfg *networkConfiguration) Exists() bool {
 }
 
 func (ncfg *networkConfiguration) Skip() bool {
-	return ncfg.DefaultBridge
+	return false
 }
 
 func (ncfg *networkConfiguration) New() datastore.KVObject {
@@ -200,3 +244,135 @@ func (ncfg *networkConfiguration) CopyTo(o datastore.KVObject) error {
 func (ncfg *networkConfiguration) DataScope() string {
 	return datastore.LocalScope
 }
+
+func (ep *bridgeEndpoint) MarshalJSON() ([]byte, error) {
+	epMap := make(map[string]interface{})
+	epMap["id"] = ep.id
+	epMap["nid"] = ep.nid
+	epMap["SrcName"] = ep.srcName
+	epMap["MacAddress"] = ep.macAddress.String()
+	epMap["Addr"] = ep.addr.String()
+	if ep.addrv6 != nil {
+		epMap["Addrv6"] = ep.addrv6.String()
+	}
+	epMap["Config"] = ep.config
+	epMap["ContainerConfig"] = ep.containerConfig
+	epMap["ExternalConnConfig"] = ep.extConnConfig
+	epMap["PortMapping"] = ep.portMapping
+
+	return json.Marshal(epMap)
+}
+
+func (ep *bridgeEndpoint) UnmarshalJSON(b []byte) error {
+	var (
+		err   error
+		epMap map[string]interface{}
+	)
+
+	if err = json.Unmarshal(b, &epMap); err != nil {
+		return fmt.Errorf("Failed to unmarshal to bridge endpoint: %v", err)
+	}
+
+	if v, ok := epMap["MacAddress"]; ok {
+		if ep.macAddress, err = net.ParseMAC(v.(string)); err != nil {
+			return types.InternalErrorf("failed to decode bridge endpoint MAC address (%s) after json unmarshal: %v", v.(string), err)
+		}
+	}
+	if v, ok := epMap["Addr"]; ok {
+		if ep.addr, err = types.ParseCIDR(v.(string)); err != nil {
+			return types.InternalErrorf("failed to decode bridge endpoint IPv4 address (%s) after json unmarshal: %v", v.(string), err)
+		}
+	}
+	if v, ok := epMap["Addrv6"]; ok {
+		if ep.addrv6, err = types.ParseCIDR(v.(string)); err != nil {
+			return types.InternalErrorf("failed to decode bridge endpoint IPv6 address (%s) after json unmarshal: %v", v.(string), err)
+		}
+	}
+	ep.id = epMap["id"].(string)
+	ep.nid = epMap["nid"].(string)
+	ep.srcName = epMap["SrcName"].(string)
+	d, _ := json.Marshal(epMap["Config"])
+	if err := json.Unmarshal(d, &ep.config); err != nil {
+		logrus.Warnf("Failed to decode endpoint config %v", err)
+	}
+	d, _ = json.Marshal(epMap["ContainerConfig"])
+	if err := json.Unmarshal(d, &ep.containerConfig); err != nil {
+		logrus.Warnf("Failed to decode endpoint container config %v", err)
+	}
+	d, _ = json.Marshal(epMap["ExternalConnConfig"])
+	if err := json.Unmarshal(d, &ep.extConnConfig); err != nil {
+		logrus.Warnf("Failed to decode endpoint external connectivity configuration %v", err)
+	}
+	d, _ = json.Marshal(epMap["PortMapping"])
+	if err := json.Unmarshal(d, &ep.portMapping); err != nil {
+		logrus.Warnf("Failed to decode endpoint port mapping %v", err)
+	}
+
+	return nil
+}
+
+func (ep *bridgeEndpoint) Key() []string {
+	return []string{bridgeEndpointPrefix, ep.id}
+}
+
+func (ep *bridgeEndpoint) KeyPrefix() []string {
+	return []string{bridgeEndpointPrefix}
+}
+
+func (ep *bridgeEndpoint) Value() []byte {
+	b, err := json.Marshal(ep)
+	if err != nil {
+		return nil
+	}
+	return b
+}
+
+func (ep *bridgeEndpoint) SetValue(value []byte) error {
+	return json.Unmarshal(value, ep)
+}
+
+func (ep *bridgeEndpoint) Index() uint64 {
+	return ep.dbIndex
+}
+
+func (ep *bridgeEndpoint) SetIndex(index uint64) {
+	ep.dbIndex = index
+	ep.dbExists = true
+}
+
+func (ep *bridgeEndpoint) Exists() bool {
+	return ep.dbExists
+}
+
+func (ep *bridgeEndpoint) Skip() bool {
+	return false
+}
+
+func (ep *bridgeEndpoint) New() datastore.KVObject {
+	return &bridgeEndpoint{}
+}
+
+func (ep *bridgeEndpoint) CopyTo(o datastore.KVObject) error {
+	dstEp := o.(*bridgeEndpoint)
+	*dstEp = *ep
+	return nil
+}
+
+func (ep *bridgeEndpoint) DataScope() string {
+	return datastore.LocalScope
+}
+
+func (n *bridgeNetwork) restorePortAllocations(ep *bridgeEndpoint) {
+	if ep.extConnConfig == nil ||
+		ep.extConnConfig.ExposedPorts == nil ||
+		ep.extConnConfig.PortBindings == nil {
+		return
+	}
+	tmp := ep.extConnConfig.PortBindings
+	ep.extConnConfig.PortBindings = ep.portMapping
+	_, err := n.allocatePorts(ep, n.config.DefaultBindingIP, n.driver.config.EnableUserlandProxy)
+	if err != nil {
+		logrus.Warnf("Failed to reserve existing port mapping for endpoint %s:%v", ep.id[0:7], err)
+	}
+	ep.extConnConfig.PortBindings = tmp
+}

vendor/src/github.com/docker/libnetwork/drivers/ipvlan/ipvlan.go

@@ -36,11 +36,14 @@ type driver struct {
 }
 
 type endpoint struct {
-	id      string
-	mac     net.HardwareAddr
-	addr    *net.IPNet
-	addrv6  *net.IPNet
-	srcName string
+	id       string
+	nid      string
+	mac      net.HardwareAddr
+	addr     *net.IPNet
+	addrv6   *net.IPNet
+	srcName  string
+	dbIndex  uint64
+	dbExists bool
 }
 
 type network struct {

vendor/src/github.com/docker/libnetwork/drivers/ipvlan/ipvlan_endpoint.go

@@ -28,9 +28,9 @@ func (d *driver) CreateEndpoint(nid, eid string, ifInfo driverapi.InterfaceInfo,
 	}
 	ep := &endpoint{
 		id:     eid,
+		nid:    nid,
 		addr:   ifInfo.Address(),
 		addrv6: ifInfo.AddressIPv6(),
-		mac:    ifInfo.MacAddress(),
 	}
 	if ep.addr == nil {
 		return fmt.Errorf("create endpoint was not passed an IP address")
@@ -51,6 +51,11 @@ func (d *driver) CreateEndpoint(nid, eid string, ifInfo driverapi.InterfaceInfo,
 			}
 		}
 	}
+
+	if err := d.storeUpdate(ep); err != nil {
+		return fmt.Errorf("failed to save ipvlan endpoint %s to store: %v", ep.id[0:7], err)
+	}
+
 	n.addEndpoint(ep)
 
 	return nil
@@ -74,5 +79,9 @@ func (d *driver) DeleteEndpoint(nid, eid string) error {
 		ns.NlHandle().LinkDel(link)
 	}
 
+	if err := d.storeDelete(ep); err != nil {
+		logrus.Warnf("Failed to remove ipvlan endpoint %s from store: %v", ep.id[0:7], err)
+	}
+
 	return nil
 }

vendor/src/github.com/docker/libnetwork/drivers/ipvlan/ipvlan_joinleave.go

@@ -116,6 +116,9 @@ func (d *driver) Join(nid, eid string, sboxKey string, jinfo driverapi.JoinInfo,
 	if err != nil {
 		return err
 	}
+	if err = d.storeUpdate(ep); err != nil {
+		return fmt.Errorf("failed to save ipvlan endpoint %s to store: %v", ep.id[0:7], err)
+	}
 
 	return nil
 }

vendor/src/github.com/docker/libnetwork/drivers/ipvlan/ipvlan_store.go

@@ -3,6 +3,7 @@ package ipvlan
 import (
 	"encoding/json"
 	"fmt"
+	"net"
 
 	"github.com/Sirupsen/logrus"
 	"github.com/docker/libnetwork/datastore"
@@ -11,7 +12,11 @@ import (
 	"github.com/docker/libnetwork/types"
 )
 
-const ipvlanPrefix = "ipvlan" // prefix used for persistent driver storage
+const (
+	ipvlanPrefix         = "ipvlan"
+	ipvlanNetworkPrefix  = ipvlanPrefix + "/network"
+	ipvlanEndpointPrefix = ipvlanPrefix + "/endpoint"
+)
 
 // networkConfiguration for this driver's network specific configuration
 type configuration struct {
@@ -58,7 +63,7 @@ func (d *driver) initStore(option map[string]interface{}) error {
 
 // populateNetworks is invoked at driver init to recreate persistently stored networks
 func (d *driver) populateNetworks() error {
-	kvol, err := d.store.List(datastore.Key(ipvlanPrefix), &configuration{})
+	kvol, err := d.store.List(datastore.Key(ipvlanNetworkPrefix), &configuration{})
 	if err != nil && err != datastore.ErrKeyNotFound {
 		return fmt.Errorf("failed to get ipvlan network configurations from store: %v", err)
 	}
@@ -76,6 +81,34 @@ func (d *driver) populateNetworks() error {
 	return nil
 }
 
+func (d *driver) populateEndpoints() error {
+	kvol, err := d.store.List(datastore.Key(ipvlanEndpointPrefix), &endpoint{})
+	if err != nil && err != datastore.ErrKeyNotFound {
+		return fmt.Errorf("failed to get ipvlan endpoints from store: %v", err)
+	}
+
+	if err == datastore.ErrKeyNotFound {
+		return nil
+	}
+
+	for _, kvo := range kvol {
+		ep := kvo.(*endpoint)
+		n, ok := d.networks[ep.nid]
+		if !ok {
+			logrus.Debugf("Network (%s) not found for restored ipvlan endpoint (%s)", ep.nid[0:7], ep.id[0:7])
+			logrus.Debugf("Deleting stale ipvlan endpoint (%s) from store", ep.nid[0:7])
+			if err := d.storeDelete(ep); err != nil {
+				logrus.Debugf("Failed to delete stale ipvlan endpoint (%s) from store", ep.nid[0:7])
+			}
+			continue
+		}
+		n.endpoints[ep.id] = ep
+		logrus.Debugf("Endpoint (%s) restored to network (%s)", ep.id[0:7], ep.nid[0:7])
+	}
+
+	return nil
+}
+
 // storeUpdate used to update persistent ipvlan network records as they are created
 func (d *driver) storeUpdate(kvObject datastore.KVObject) error {
 	if d.store == nil {
@@ -165,11 +198,11 @@ func (config *configuration) UnmarshalJSON(b []byte) error {
 }
 
 func (config *configuration) Key() []string {
-	return []string{ipvlanPrefix, config.ID}
+	return []string{ipvlanNetworkPrefix, config.ID}
 }
 
 func (config *configuration) KeyPrefix() []string {
-	return []string{ipvlanPrefix}
+	return []string{ipvlanNetworkPrefix}
 }
 
 func (config *configuration) Value() []byte {
@@ -214,3 +247,103 @@ func (config *configuration) CopyTo(o datastore.KVObject) error {
 func (config *configuration) DataScope() string {
 	return datastore.LocalScope
 }
+
+func (ep *endpoint) MarshalJSON() ([]byte, error) {
+	epMap := make(map[string]interface{})
+	epMap["id"] = ep.id
+	epMap["nid"] = ep.nid
+	epMap["SrcName"] = ep.srcName
+	if len(ep.mac) != 0 {
+		epMap["MacAddress"] = ep.mac.String()
+	}
+	if ep.addr != nil {
+		epMap["Addr"] = ep.addr.String()
+	}
+	if ep.addrv6 != nil {
+		epMap["Addrv6"] = ep.addrv6.String()
+	}
+	return json.Marshal(epMap)
+}
+
+func (ep *endpoint) UnmarshalJSON(b []byte) error {
+	var (
+		err   error
+		epMap map[string]interface{}
+	)
+
+	if err = json.Unmarshal(b, &epMap); err != nil {
+		return fmt.Errorf("Failed to unmarshal to ipvlan endpoint: %v", err)
+	}
+
+	if v, ok := epMap["MacAddress"]; ok {
+		if ep.mac, err = net.ParseMAC(v.(string)); err != nil {
+			return types.InternalErrorf("failed to decode ipvlan endpoint MAC address (%s) after json unmarshal: %v", v.(string), err)
+		}
+	}
+	if v, ok := epMap["Addr"]; ok {
+		if ep.addr, err = types.ParseCIDR(v.(string)); err != nil {
+			return types.InternalErrorf("failed to decode ipvlan endpoint IPv4 address (%s) after json unmarshal: %v", v.(string), err)
+		}
+	}
+	if v, ok := epMap["Addrv6"]; ok {
+		if ep.addrv6, err = types.ParseCIDR(v.(string)); err != nil {
+			return types.InternalErrorf("failed to decode ipvlan endpoint IPv6 address (%s) after json unmarshal: %v", v.(string), err)
+		}
+	}
+	ep.id = epMap["id"].(string)
+	ep.nid = epMap["nid"].(string)
+	ep.srcName = epMap["SrcName"].(string)
+
+	return nil
+}
+
+func (ep *endpoint) Key() []string {
+	return []string{ipvlanEndpointPrefix, ep.id}
+}
+
+func (ep *endpoint) KeyPrefix() []string {
+	return []string{ipvlanEndpointPrefix}
+}
+
+func (ep *endpoint) Value() []byte {
+	b, err := json.Marshal(ep)
+	if err != nil {
+		return nil
+	}
+	return b
+}
+
+func (ep *endpoint) SetValue(value []byte) error {
+	return json.Unmarshal(value, ep)
+}
+
+func (ep *endpoint) Index() uint64 {
+	return ep.dbIndex
+}
+
+func (ep *endpoint) SetIndex(index uint64) {
+	ep.dbIndex = index
+	ep.dbExists = true
+}
+
+func (ep *endpoint) Exists() bool {
+	return ep.dbExists
+}
+
+func (ep *endpoint) Skip() bool {
+	return false
+}
+
+func (ep *endpoint) New() datastore.KVObject {
+	return &endpoint{}
+}
+
+func (ep *endpoint) CopyTo(o datastore.KVObject) error {
+	dstEp := o.(*endpoint)
+	*dstEp = *ep
+	return nil
+}
+
+func (ep *endpoint) DataScope() string {
+	return datastore.LocalScope
+}

vendor/src/github.com/docker/libnetwork/drivers/macvlan/macvlan.go

@@ -38,11 +38,14 @@ type driver struct {
 }
 
 type endpoint struct {
-	id      string
-	mac     net.HardwareAddr
-	addr    *net.IPNet
-	addrv6  *net.IPNet
-	srcName string
+	id       string
+	nid      string
+	mac      net.HardwareAddr
+	addr     *net.IPNet
+	addrv6   *net.IPNet
+	srcName  string
+	dbIndex  uint64
+	dbExists bool
 }
 
 type network struct {

vendor/src/github.com/docker/libnetwork/drivers/macvlan/macvlan_endpoint.go

@@ -26,6 +26,7 @@ func (d *driver) CreateEndpoint(nid, eid string, ifInfo driverapi.InterfaceInfo,
 	}
 	ep := &endpoint{
 		id:     eid,
+		nid:    nid,
 		addr:   ifInfo.Address(),
 		addrv6: ifInfo.AddressIPv6(),
 		mac:    ifInfo.MacAddress(),
@@ -55,6 +56,11 @@ func (d *driver) CreateEndpoint(nid, eid string, ifInfo driverapi.InterfaceInfo,
 			}
 		}
 	}
+
+	if err := d.storeUpdate(ep); err != nil {
+		return fmt.Errorf("failed to save macvlan endpoint %s to store: %v", ep.id[0:7], err)
+	}
+
 	n.addEndpoint(ep)
 
 	return nil
@@ -77,6 +83,8 @@ func (d *driver) DeleteEndpoint(nid, eid string) error {
 	if link, err := ns.NlHandle().LinkByName(ep.srcName); err == nil {
 		ns.NlHandle().LinkDel(link)
 	}
-
+	if err := d.storeDelete(ep); err != nil {
+		logrus.Warnf("Failed to remove macvlan endpoint %s from store: %v", ep.id[0:7], err)
+	}
 	return nil
 }

vendor/src/github.com/docker/libnetwork/drivers/macvlan/macvlan_joinleave.go

@@ -77,7 +77,9 @@ func (d *driver) Join(nid, eid string, sboxKey string, jinfo driverapi.JoinInfo,
 	if err != nil {
 		return err
 	}
-
+	if err := d.storeUpdate(ep); err != nil {
+		return fmt.Errorf("failed to save macvlan endpoint %s to store: %v", ep.id[0:7], err)
+	}
 	return nil
 }
 

vendor/src/github.com/docker/libnetwork/drivers/macvlan/macvlan_store.go

@@ -3,6 +3,7 @@ package macvlan
 import (
 	"encoding/json"
 	"fmt"
+	"net"
 
 	"github.com/Sirupsen/logrus"
 	"github.com/docker/libnetwork/datastore"
@@ -11,7 +12,11 @@ import (
 	"github.com/docker/libnetwork/types"
 )
 
-const macvlanPrefix = "macvlan" // prefix used for persistent driver storage
+const (
+	macvlanPrefix         = "macvlan"
+	macvlanNetworkPrefix  = macvlanPrefix + "/network"
+	macvlanEndpointPrefix = macvlanPrefix + "/endpoint"
+)
 
 // networkConfiguration for this driver's network specific configuration
 type configuration struct {
@@ -76,6 +81,34 @@ func (d *driver) populateNetworks() error {
 	return nil
 }
 
+func (d *driver) populateEndpoints() error {
+	kvol, err := d.store.List(datastore.Key(macvlanEndpointPrefix), &endpoint{})
+	if err != nil && err != datastore.ErrKeyNotFound {
+		return fmt.Errorf("failed to get macvlan endpoints from store: %v", err)
+	}
+
+	if err == datastore.ErrKeyNotFound {
+		return nil
+	}
+
+	for _, kvo := range kvol {
+		ep := kvo.(*endpoint)
+		n, ok := d.networks[ep.nid]
+		if !ok {
+			logrus.Debugf("Network (%s) not found for restored macvlan endpoint (%s)", ep.nid[0:7], ep.id[0:7])
+			logrus.Debugf("Deleting stale macvlan endpoint (%s) from store", ep.nid[0:7])
+			if err := d.storeDelete(ep); err != nil {
+				logrus.Debugf("Failed to delete stale macvlan endpoint (%s) from store", ep.nid[0:7])
+			}
+			continue
+		}
+		n.endpoints[ep.id] = ep
+		logrus.Debugf("Endpoint (%s) restored to network (%s)", ep.id[0:7], ep.nid[0:7])
+	}
+
+	return nil
+}
+
 // storeUpdate used to update persistent macvlan network records as they are created
 func (d *driver) storeUpdate(kvObject datastore.KVObject) error {
 	if d.store == nil {
@@ -165,11 +198,11 @@ func (config *configuration) UnmarshalJSON(b []byte) error {
 }
 
 func (config *configuration) Key() []string {
-	return []string{macvlanPrefix, config.ID}
+	return []string{macvlanNetworkPrefix, config.ID}
 }
 
 func (config *configuration) KeyPrefix() []string {
-	return []string{macvlanPrefix}
+	return []string{macvlanNetworkPrefix}
 }
 
 func (config *configuration) Value() []byte {
@@ -216,3 +249,103 @@ func (config *configuration) CopyTo(o datastore.KVObject) error {
 func (config *configuration) DataScope() string {
 	return datastore.LocalScope
 }
+
+func (ep *endpoint) MarshalJSON() ([]byte, error) {
+	epMap := make(map[string]interface{})
+	epMap["id"] = ep.id
+	epMap["nid"] = ep.nid
+	epMap["SrcName"] = ep.srcName
+	if len(ep.mac) != 0 {
+		epMap["MacAddress"] = ep.mac.String()
+	}
+	if ep.addr != nil {
+		epMap["Addr"] = ep.addr.String()
+	}
+	if ep.addrv6 != nil {
+		epMap["Addrv6"] = ep.addrv6.String()
+	}
+	return json.Marshal(epMap)
+}
+
+func (ep *endpoint) UnmarshalJSON(b []byte) error {
+	var (
+		err   error
+		epMap map[string]interface{}
+	)
+
+	if err = json.Unmarshal(b, &epMap); err != nil {
+		return fmt.Errorf("Failed to unmarshal to macvlan endpoint: %v", err)
+	}
+
+	if v, ok := epMap["MacAddress"]; ok {
+		if ep.mac, err = net.ParseMAC(v.(string)); err != nil {
+			return types.InternalErrorf("failed to decode macvlan endpoint MAC address (%s) after json unmarshal: %v", v.(string), err)
+		}
+	}
+	if v, ok := epMap["Addr"]; ok {
+		if ep.addr, err = types.ParseCIDR(v.(string)); err != nil {
+			return types.InternalErrorf("failed to decode macvlan endpoint IPv4 address (%s) after json unmarshal: %v", v.(string), err)
+		}
+	}
+	if v, ok := epMap["Addrv6"]; ok {
+		if ep.addrv6, err = types.ParseCIDR(v.(string)); err != nil {
+			return types.InternalErrorf("failed to decode macvlan endpoint IPv6 address (%s) after json unmarshal: %v", v.(string), err)
+		}
+	}
+	ep.id = epMap["id"].(string)
+	ep.nid = epMap["nid"].(string)
+	ep.srcName = epMap["SrcName"].(string)
+
+	return nil
+}
+
+func (ep *endpoint) Key() []string {
+	return []string{macvlanEndpointPrefix, ep.id}
+}
+
+func (ep *endpoint) KeyPrefix() []string {
+	return []string{macvlanEndpointPrefix}
+}
+
+func (ep *endpoint) Value() []byte {
+	b, err := json.Marshal(ep)
+	if err != nil {
+		return nil
+	}
+	return b
+}
+
+func (ep *endpoint) SetValue(value []byte) error {
+	return json.Unmarshal(value, ep)
+}
+
+func (ep *endpoint) Index() uint64 {
+	return ep.dbIndex
+}
+
+func (ep *endpoint) SetIndex(index uint64) {
+	ep.dbIndex = index
+	ep.dbExists = true
+}
+
+func (ep *endpoint) Exists() bool {
+	return ep.dbExists
+}
+
+func (ep *endpoint) Skip() bool {
+	return false
+}
+
+func (ep *endpoint) New() datastore.KVObject {
+	return &endpoint{}
+}
+
+func (ep *endpoint) CopyTo(o datastore.KVObject) error {
+	dstEp := o.(*endpoint)
+	*dstEp = *ep
+	return nil
+}
+
+func (ep *endpoint) DataScope() string {
+	return datastore.LocalScope
+}

vendor/src/github.com/docker/libnetwork/drivers/overlay/encryption.go

@@ -10,6 +10,7 @@ import (
 
 	log "github.com/Sirupsen/logrus"
 	"github.com/docker/libnetwork/iptables"
+	"github.com/docker/libnetwork/ns"
 	"github.com/docker/libnetwork/types"
 	"github.com/vishvananda/netlink"
 	"strconv"
@@ -214,12 +215,12 @@ func programSA(localIP, remoteIP net.IP, spi *spi, k *key, dir int, add bool) (f
 	var (
 		crypt       *netlink.XfrmStateAlgo
 		action      = "Removing"
-		xfrmProgram = netlink.XfrmStateDel
+		xfrmProgram = ns.NlHandle().XfrmStateDel
 	)
 
 	if add {
 		action = "Adding"
-		xfrmProgram = netlink.XfrmStateAdd
+		xfrmProgram = ns.NlHandle().XfrmStateAdd
 		crypt = &netlink.XfrmStateAlgo{Name: "cbc(aes)", Key: k.value}
 	}
 
@@ -278,10 +279,10 @@ func programSA(localIP, remoteIP net.IP, spi *spi, k *key, dir int, add bool) (f
 
 func programSP(fSA *netlink.XfrmState, rSA *netlink.XfrmState, add bool) error {
 	action := "Removing"
-	xfrmProgram := netlink.XfrmPolicyDel
+	xfrmProgram := ns.NlHandle().XfrmPolicyDel
 	if add {
 		action = "Adding"
-		xfrmProgram = netlink.XfrmPolicyAdd
+		xfrmProgram = ns.NlHandle().XfrmPolicyAdd
 	}
 
 	fullMask := net.CIDRMask(8*len(fSA.Src), 8*len(fSA.Src))
@@ -322,7 +323,7 @@ func programSP(fSA *netlink.XfrmState, rSA *netlink.XfrmState, add bool) error {
 }
 
 func saExists(sa *netlink.XfrmState) (bool, error) {
-	_, err := netlink.XfrmStateGet(sa)
+	_, err := ns.NlHandle().XfrmStateGet(sa)
 	switch err {
 	case nil:
 		return true, nil
@@ -336,7 +337,7 @@ func saExists(sa *netlink.XfrmState) (bool, error) {
 }
 
 func spExists(sp *netlink.XfrmPolicy) (bool, error) {
-	_, err := netlink.XfrmPolicyGet(sp)
+	_, err := ns.NlHandle().XfrmPolicyGet(sp)
 	switch err {
 	case nil:
 		return true, nil
@@ -482,7 +483,7 @@ func updateNodeKey(lIP, rIP net.IP, idxs []*spi, curKeys []*key, newIdx, priIdx,
 			Limits: netlink.XfrmStateLimits{TimeSoft: timeout},
 		}
 		log.Infof("Updating rSA0{%s}", rSA0)
-		if err := netlink.XfrmStateUpdate(rSA0); err != nil {
+		if err := ns.NlHandle().XfrmStateUpdate(rSA0); err != nil {
 			log.Warnf("Failed to update rSA0{%s}: %v", rSA0, err)
 		}
 	}
@@ -518,7 +519,7 @@ func updateNodeKey(lIP, rIP net.IP, idxs []*spi, curKeys []*key, newIdx, priIdx,
 			},
 		}
 		log.Infof("Updating fSP{%s}", fSP1)
-		if err := netlink.XfrmPolicyUpdate(fSP1); err != nil {
+		if err := ns.NlHandle().XfrmPolicyUpdate(fSP1); err != nil {
 			log.Warnf("Failed to update fSP{%s}: %v", fSP1, err)
 		}
 
@@ -533,7 +534,7 @@ func updateNodeKey(lIP, rIP net.IP, idxs []*spi, curKeys []*key, newIdx, priIdx,
 			Limits: netlink.XfrmStateLimits{TimeHard: timeout},
 		}
 		log.Infof("Removing fSA0{%s}", fSA0)
-		if err := netlink.XfrmStateUpdate(fSA0); err != nil {
+		if err := ns.NlHandle().XfrmStateUpdate(fSA0); err != nil {
 			log.Warnf("Failed to remove fSA0{%s}: %v", fSA0, err)
 		}
 	}

vendor/src/github.com/docker/libnetwork/drivers/overlay/joinleave.go

@@ -40,11 +40,11 @@ func (d *driver) Join(nid, eid string, sboxKey string, jinfo driverapi.JoinInfo,
 		return fmt.Errorf("couldn't get vxlan id for %q: %v", s.subnetIP.String(), err)
 	}
 
-	if err := n.joinSandbox(); err != nil {
+	if err := n.joinSandbox(false); err != nil {
 		return fmt.Errorf("network sandbox join failed: %v", err)
 	}
 
-	if err := n.joinSubnetSandbox(s); err != nil {
+	if err := n.joinSubnetSandbox(s, false); err != nil {
 		return fmt.Errorf("subnet sandbox join failed for %q: %v", s.subnetIP.String(), err)
 	}
 
@@ -61,6 +61,10 @@ func (d *driver) Join(nid, eid string, sboxKey string, jinfo driverapi.JoinInfo,
 
 	ep.ifName = containerIfName
 
+	if err := d.writeEndpointToStore(ep); err != nil {
+		return fmt.Errorf("failed to update overlay endpoint %s to local data store: %v", ep.id[0:7], err)
+	}
+
 	nlh := ns.NlHandle()
 
 	// Set the container interface and its peer MTU to 1450 to allow

vendor/src/github.com/docker/libnetwork/drivers/overlay/ov_endpoint.go

@@ -1,22 +1,30 @@
 package overlay
 
 import (
+	"encoding/json"
 	"fmt"
 	"net"
 
 	log "github.com/Sirupsen/logrus"
+	"github.com/docker/libnetwork/datastore"
 	"github.com/docker/libnetwork/driverapi"
 	"github.com/docker/libnetwork/netutils"
 	"github.com/docker/libnetwork/ns"
+	"github.com/docker/libnetwork/types"
 )
 
 type endpointTable map[string]*endpoint
 
+const overlayEndpointPrefix = "overlay/endpoint"
+
 type endpoint struct {
-	id     string
-	ifName string
-	mac    net.HardwareAddr
-	addr   *net.IPNet
+	id       string
+	nid      string
+	ifName   string
+	mac      net.HardwareAddr
+	addr     *net.IPNet
+	dbExists bool
+	dbIndex  uint64
 }
 
 func (n *network) endpoint(eid string) *endpoint {
@@ -60,6 +68,7 @@ func (d *driver) CreateEndpoint(nid, eid string, ifInfo driverapi.InterfaceInfo,
 
 	ep := &endpoint{
 		id:   eid,
+		nid:  n.id,
 		addr: ifInfo.Address(),
 		mac:  ifInfo.MacAddress(),
 	}
@@ -80,6 +89,10 @@ func (d *driver) CreateEndpoint(nid, eid string, ifInfo driverapi.InterfaceInfo,
 
 	n.addEndpoint(ep)
 
+	if err := d.writeEndpointToStore(ep); err != nil {
+		return fmt.Errorf("failed to update overlay endpoint %s to local store: %v", ep.id[0:7], err)
+	}
+
 	return nil
 }
 
@@ -102,6 +115,10 @@ func (d *driver) DeleteEndpoint(nid, eid string) error {
 
 	n.deleteEndpoint(eid)
 
+	if err := d.deleteEndpointFromStore(ep); err != nil {
+		log.Warnf("Failed to delete overlay endpoint %s from local store: %v", ep.id[0:7], err)
+	}
+
 	if ep.ifName == "" {
 		return nil
 	}
@@ -121,3 +138,122 @@ func (d *driver) DeleteEndpoint(nid, eid string) error {
 func (d *driver) EndpointOperInfo(nid, eid string) (map[string]interface{}, error) {
 	return make(map[string]interface{}, 0), nil
 }
+
+func (d *driver) deleteEndpointFromStore(e *endpoint) error {
+	if d.localStore == nil {
+		return fmt.Errorf("overlay local store not initialized, ep not deleted")
+	}
+
+	if err := d.localStore.DeleteObjectAtomic(e); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (d *driver) writeEndpointToStore(e *endpoint) error {
+	if d.localStore == nil {
+		return fmt.Errorf("overlay local store not initialized, ep not added")
+	}
+
+	if err := d.localStore.PutObjectAtomic(e); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (ep *endpoint) DataScope() string {
+	return datastore.LocalScope
+}
+
+func (ep *endpoint) New() datastore.KVObject {
+	return &endpoint{}
+}
+
+func (ep *endpoint) CopyTo(o datastore.KVObject) error {
+	dstep := o.(*endpoint)
+	*dstep = *ep
+	return nil
+}
+
+func (ep *endpoint) Key() []string {
+	return []string{overlayEndpointPrefix, ep.id}
+}
+
+func (ep *endpoint) KeyPrefix() []string {
+	return []string{overlayEndpointPrefix}
+}
+
+func (ep *endpoint) Index() uint64 {
+	return ep.dbIndex
+}
+
+func (ep *endpoint) SetIndex(index uint64) {
+	ep.dbIndex = index
+	ep.dbExists = true
+}
+
+func (ep *endpoint) Exists() bool {
+	return ep.dbExists
+}
+
+func (ep *endpoint) Skip() bool {
+	return false
+}
+
+func (ep *endpoint) Value() []byte {
+	b, err := json.Marshal(ep)
+	if err != nil {
+		return nil
+	}
+	return b
+}
+
+func (ep *endpoint) SetValue(value []byte) error {
+	return json.Unmarshal(value, ep)
+}
+
+func (ep *endpoint) MarshalJSON() ([]byte, error) {
+	epMap := make(map[string]interface{})
+
+	epMap["id"] = ep.id
+	epMap["nid"] = ep.nid
+	if ep.ifName != "" {
+		epMap["ifName"] = ep.ifName
+	}
+	if ep.addr != nil {
+		epMap["addr"] = ep.addr.String()
+	}
+	if len(ep.mac) != 0 {
+		epMap["mac"] = ep.mac.String()
+	}
+
+	return json.Marshal(epMap)
+}
+
+func (ep *endpoint) UnmarshalJSON(value []byte) error {
+	var (
+		err   error
+		epMap map[string]interface{}
+	)
+
+	json.Unmarshal(value, &epMap)
+
+	ep.id = epMap["id"].(string)
+	ep.nid = epMap["nid"].(string)
+	if v, ok := epMap["mac"]; ok {
+		if ep.mac, err = net.ParseMAC(v.(string)); err != nil {
+			return types.InternalErrorf("failed to decode endpoint interface mac address after json unmarshal: %s", v.(string))
+		}
+	}
+	if v, ok := epMap["addr"]; ok {
+		if ep.addr, err = types.ParseCIDR(v.(string)); err != nil {
+			return types.InternalErrorf("failed to decode endpoint interface ipv4 address after json unmarshal: %v", err)
+		}
+	}
+	if v, ok := epMap["ifName"]; ok {
+		ep.ifName = v.(string)
+	}
+
+	return nil
+}

vendor/src/github.com/docker/libnetwork/drivers/overlay/ov_network.go

@@ -195,21 +195,21 @@ func (n *network) incEndpointCount() {
 	n.joinCnt++
 }
 
-func (n *network) joinSandbox() error {
+func (n *network) joinSandbox(restore bool) error {
 	// If there is a race between two go routines here only one will win
 	// the other will wait.
 	n.once.Do(func() {
 		// save the error status of initSandbox in n.initErr so that
 		// all the racing go routines are able to know the status.
-		n.initErr = n.initSandbox()
+		n.initErr = n.initSandbox(restore)
 	})
 
 	return n.initErr
 }
 
-func (n *network) joinSubnetSandbox(s *subnet) error {
+func (n *network) joinSubnetSandbox(s *subnet, restore bool) error {
 	s.once.Do(func() {
-		s.initErr = n.initSubnetSandbox(s)
+		s.initErr = n.initSubnetSandbox(s, restore)
 	})
 	return s.initErr
 }
@@ -386,9 +386,33 @@ func isOverlap(nw *net.IPNet) bool {
 	return false
 }
 
-func (n *network) initSubnetSandbox(s *subnet) error {
-	brName := n.generateBridgeName(s)
-	vxlanName := n.generateVxlanName(s)
+func (n *network) restoreSubnetSandbox(s *subnet, brName, vxlanName string) error {
+	sbox := n.sandbox()
+
+	// restore overlay osl sandbox
+	Ifaces := make(map[string][]osl.IfaceOption)
+	brIfaceOption := make([]osl.IfaceOption, 2)
+	brIfaceOption = append(brIfaceOption, sbox.InterfaceOptions().Address(s.gwIP))
+	brIfaceOption = append(brIfaceOption, sbox.InterfaceOptions().Bridge(true))
+	Ifaces[fmt.Sprintf("%s+%s", brName, "br")] = brIfaceOption
+
+	err := sbox.Restore(Ifaces, nil, nil, nil)
+	if err != nil {
+		return err
+	}
+
+	Ifaces = make(map[string][]osl.IfaceOption)
+	vxlanIfaceOption := make([]osl.IfaceOption, 1)
+	vxlanIfaceOption = append(vxlanIfaceOption, sbox.InterfaceOptions().Master(brName))
+	Ifaces[fmt.Sprintf("%s+%s", vxlanName, "vxlan")] = vxlanIfaceOption
+	err = sbox.Restore(Ifaces, nil, nil, nil)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (n *network) setupSubnetSandbox(s *subnet, brName, vxlanName string) error {
 
 	if hostMode {
 		// Try to delete stale bridge interface if it exists
@@ -451,6 +475,19 @@ func (n *network) initSubnetSandbox(s *subnet) error {
 		}
 	}
 
+	return nil
+}
+
+func (n *network) initSubnetSandbox(s *subnet, restore bool) error {
+	brName := n.generateBridgeName(s)
+	vxlanName := n.generateVxlanName(s)
+
+	if restore {
+		n.restoreSubnetSandbox(s, brName, vxlanName)
+	} else {
+		n.setupSubnetSandbox(s, brName, vxlanName)
+	}
+
 	n.Lock()
 	s.vxlanName = vxlanName
 	s.brName = brName
@@ -494,32 +531,45 @@ func (n *network) cleanupStaleSandboxes() {
 		})
 }
 
-func (n *network) initSandbox() error {
+func (n *network) initSandbox(restore bool) error {
 	n.Lock()
 	n.initEpoch++
 	n.Unlock()
 
 	networkOnce.Do(networkOnceInit)
 
-	if hostMode {
-		if err := addNetworkChain(n.id[:12]); err != nil {
-			return err
+	if !restore {
+		if hostMode {
+			if err := addNetworkChain(n.id[:12]); err != nil {
+				return err
+			}
 		}
+
+		// If there are any stale sandboxes related to this network
+		// from previous daemon life clean it up here
+		n.cleanupStaleSandboxes()
 	}
 
-	// If there are any stale sandboxes related to this network
-	// from previous daemon life clean it up here
-	n.cleanupStaleSandboxes()
+	// In the restore case network sandbox already exist; but we don't know
+	// what epoch number it was created with. It has to be retrieved by
+	// searching the net namespaces.
+	key := ""
+	if restore {
+		key = osl.GenerateKey("-" + n.id)
+	} else {
+		key = osl.GenerateKey(fmt.Sprintf("%d-", n.initEpoch) + n.id)
+	}
 
-	sbox, err := osl.NewSandbox(
-		osl.GenerateKey(fmt.Sprintf("%d-", n.initEpoch)+n.id), !hostMode)
+	sbox, err := osl.NewSandbox(key, !hostMode, restore)
 	if err != nil {
-		return fmt.Errorf("could not create network sandbox: %v", err)
+		return fmt.Errorf("could not get network sandbox (oper %t): %v", restore, err)
 	}
 
 	n.setSandbox(sbox)
 
-	n.driver.peerDbUpdateSandbox(n.id)
+	if !restore {
+		n.driver.peerDbUpdateSandbox(n.id)
+	}
 
 	var nlSock *nl.NetlinkSocket
 	sbox.InvokeFunc(func() {

vendor/src/github.com/docker/libnetwork/drivers/overlay/overlay.go

@@ -13,6 +13,7 @@ import (
 	"github.com/docker/libnetwork/driverapi"
 	"github.com/docker/libnetwork/idm"
 	"github.com/docker/libnetwork/netlabel"
+	"github.com/docker/libnetwork/osl"
 	"github.com/docker/libnetwork/types"
 	"github.com/hashicorp/serf/serf"
 )
@@ -41,6 +42,7 @@ type driver struct {
 	serfInstance *serf.Serf
 	networks     networkTable
 	store        datastore.DataStore
+	localStore   datastore.DataStore
 	vxlanIdm     *idm.Idm
 	once         sync.Once
 	joinOnce     sync.Once
@@ -74,9 +76,75 @@ func Init(dc driverapi.DriverCallback, config map[string]interface{}) error {
 		}
 	}
 
+	if data, ok := config[netlabel.LocalKVClient]; ok {
+		var err error
+		dsc, ok := data.(discoverapi.DatastoreConfigData)
+		if !ok {
+			return types.InternalErrorf("incorrect data in datastore configuration: %v", data)
+		}
+		d.localStore, err = datastore.NewDataStoreFromConfig(dsc)
+		if err != nil {
+			return types.InternalErrorf("failed to initialize local data store: %v", err)
+		}
+	}
+
+	d.restoreEndpoints()
+
 	return dc.RegisterDriver(networkType, d, c)
 }
 
+// Endpoints are stored in the local store. Restore them and reconstruct the overlay sandbox
+func (d *driver) restoreEndpoints() error {
+	if d.localStore == nil {
+		logrus.Warnf("Cannot restore overlay endpoints because local datastore is missing")
+		return nil
+	}
+	kvol, err := d.localStore.List(datastore.Key(overlayEndpointPrefix), &endpoint{})
+	if err != nil && err != datastore.ErrKeyNotFound {
+		return fmt.Errorf("failed to read overlay endpoint from store: %v", err)
+	}
+
+	if err == datastore.ErrKeyNotFound {
+		return nil
+	}
+	for _, kvo := range kvol {
+		ep := kvo.(*endpoint)
+		n := d.network(ep.nid)
+		if n == nil {
+			logrus.Debugf("Network (%s) not found for restored endpoint (%s)", ep.nid, ep.id)
+			continue
+		}
+		n.addEndpoint(ep)
+
+		s := n.getSubnetforIP(ep.addr)
+		if s == nil {
+			return fmt.Errorf("could not find subnet for endpoint %s", ep.id)
+		}
+
+		if err := n.joinSandbox(true); err != nil {
+			return fmt.Errorf("restore network sandbox failed: %v", err)
+		}
+
+		if err := n.joinSubnetSandbox(s, true); err != nil {
+			return fmt.Errorf("restore subnet sandbox failed for %q: %v", s.subnetIP.String(), err)
+		}
+
+		Ifaces := make(map[string][]osl.IfaceOption)
+		vethIfaceOption := make([]osl.IfaceOption, 1)
+		vethIfaceOption = append(vethIfaceOption, n.sbox.InterfaceOptions().Master(s.brName))
+		Ifaces[fmt.Sprintf("%s+%s", "veth", "veth")] = vethIfaceOption
+
+		err := n.sbox.Restore(Ifaces, nil, nil, nil)
+		if err != nil {
+			return fmt.Errorf("failed to restore overlay sandbox: %v", err)
+		}
+
+		n.incEndpointCount()
+		d.peerDbAdd(ep.nid, ep.id, ep.addr.IP, ep.addr.Mask, ep.mac, net.ParseIP(d.bindAddress), true)
+	}
+	return nil
+}
+
 // Fini cleans up the driver resources
 func Fini(drv driverapi.Driver) {
 	d := drv.(*driver)

vendor/src/github.com/docker/libnetwork/drivers/overlay/peerdb.go

@@ -271,7 +271,7 @@ func (d *driver) peerAdd(nid, eid string, peerIP net.IP, peerIPMask net.IPMask,
 		return fmt.Errorf("couldn't get vxlan id for %q: %v", s.subnetIP.String(), err)
 	}
 
-	if err := n.joinSubnetSandbox(s); err != nil {
+	if err := n.joinSubnetSandbox(s, false); err != nil {
 		return fmt.Errorf("subnet sandbox join failed for %q: %v", s.subnetIP.String(), err)
 	}
 

vendor/src/github.com/docker/libnetwork/drivers_experimental_linux.go

@@ -2,14 +2,10 @@
 
 package libnetwork
 
-import (
-	"github.com/docker/libnetwork/drivers/ipvlan"
-	"github.com/docker/libnetwork/drivers/macvlan"
-)
+import "github.com/docker/libnetwork/drivers/ipvlan"
 
 func additionalDrivers() []initializer {
 	return []initializer{
-		{macvlan.Init, "macvlan"},
 		{ipvlan.Init, "ipvlan"},
 	}
 }

vendor/src/github.com/docker/libnetwork/drivers_linux.go

@@ -3,6 +3,7 @@ package libnetwork
 import (
 	"github.com/docker/libnetwork/drivers/bridge"
 	"github.com/docker/libnetwork/drivers/host"
+	"github.com/docker/libnetwork/drivers/macvlan"
 	"github.com/docker/libnetwork/drivers/null"
 	"github.com/docker/libnetwork/drivers/overlay"
 	"github.com/docker/libnetwork/drivers/remote"
@@ -12,6 +13,7 @@ func getInitializers() []initializer {
 	in := []initializer{
 		{bridge.Init, "bridge"},
 		{host.Init, "host"},
+		{macvlan.Init, "macvlan"},
 		{null.Init, "null"},
 		{remote.Init, "remote"},
 		{overlay.Init, "overlay"},

vendor/src/github.com/docker/libnetwork/endpoint.go

@@ -84,6 +84,7 @@ func (ep *endpoint) MarshalJSON() ([]byte, error) {
 	epMap["name"] = ep.name
 	epMap["id"] = ep.id
 	epMap["ep_iface"] = ep.iface
+	epMap["joinInfo"] = ep.joinInfo
 	epMap["exposed_ports"] = ep.exposedPorts
 	if ep.generic != nil {
 		epMap["generic"] = ep.generic
@@ -115,6 +116,9 @@ func (ep *endpoint) UnmarshalJSON(b []byte) (err error) {
 	ib, _ := json.Marshal(epMap["ep_iface"])
 	json.Unmarshal(ib, &ep.iface)
 
+	jb, _ := json.Marshal(epMap["joinInfo"])
+	json.Unmarshal(jb, &ep.joinInfo)
+
 	tb, _ := json.Marshal(epMap["exposed_ports"])
 	var tPorts []types.TransportPort
 	json.Unmarshal(tb, &tPorts)
@@ -235,6 +239,11 @@ func (ep *endpoint) CopyTo(o datastore.KVObject) error {
 		ep.iface.CopyTo(dstEp.iface)
 	}
 
+	if ep.joinInfo != nil {
+		dstEp.joinInfo = &endpointJoinInfo{}
+		ep.joinInfo.CopyTo(dstEp.joinInfo)
+	}
+
 	dstEp.exposedPorts = make([]types.TransportPort, len(ep.exposedPorts))
 	copy(dstEp.exposedPorts, ep.exposedPorts)
 
@@ -1073,6 +1082,13 @@ func (ep *endpoint) releaseAddress() {
 }
 
 func (c *controller) cleanupLocalEndpoints() {
+	// Get used endpoints
+	eps := make(map[string]interface{})
+	for _, sb := range c.sandboxes {
+		for _, ep := range sb.endpoints {
+			eps[ep.id] = true
+		}
+	}
 	nl, err := c.getNetworksForScope(datastore.LocalScope)
 	if err != nil {
 		log.Warnf("Could not get list of networks during endpoint cleanup: %v", err)
@@ -1087,6 +1103,9 @@ func (c *controller) cleanupLocalEndpoints() {
 		}
 
 		for _, ep := range epl {
+			if _, ok := eps[ep.id]; ok {
+				continue
+			}
 			log.Infof("Removing stale endpoint %s (%s)", ep.name, ep.id)
 			if err := ep.Delete(true); err != nil {
 				log.Warnf("Could not delete local endpoint %s during endpoint cleanup: %v", ep.name, err)

vendor/src/github.com/docker/libnetwork/endpoint_info.go

@@ -414,3 +414,56 @@ func (ep *endpoint) DisableGatewayService() {
 
 	ep.joinInfo.disableGatewayService = true
 }
+
+func (epj *endpointJoinInfo) MarshalJSON() ([]byte, error) {
+	epMap := make(map[string]interface{})
+	if epj.gw != nil {
+		epMap["gw"] = epj.gw.String()
+	}
+	if epj.gw6 != nil {
+		epMap["gw6"] = epj.gw6.String()
+	}
+	epMap["disableGatewayService"] = epj.disableGatewayService
+	epMap["StaticRoutes"] = epj.StaticRoutes
+	return json.Marshal(epMap)
+}
+
+func (epj *endpointJoinInfo) UnmarshalJSON(b []byte) error {
+	var (
+		err   error
+		epMap map[string]interface{}
+	)
+	if err = json.Unmarshal(b, &epMap); err != nil {
+		return err
+	}
+	if v, ok := epMap["gw"]; ok {
+		epj.gw6 = net.ParseIP(v.(string))
+	}
+	if v, ok := epMap["gw6"]; ok {
+		epj.gw6 = net.ParseIP(v.(string))
+	}
+	epj.disableGatewayService = epMap["disableGatewayService"].(bool)
+
+	var tStaticRoute []types.StaticRoute
+	if v, ok := epMap["StaticRoutes"]; ok {
+		tb, _ := json.Marshal(v)
+		var tStaticRoute []types.StaticRoute
+		json.Unmarshal(tb, &tStaticRoute)
+	}
+	var StaticRoutes []*types.StaticRoute
+	for _, r := range tStaticRoute {
+		StaticRoutes = append(StaticRoutes, &r)
+	}
+	epj.StaticRoutes = StaticRoutes
+
+	return nil
+}
+
+func (epj *endpointJoinInfo) CopyTo(dstEpj *endpointJoinInfo) error {
+	dstEpj.disableGatewayService = epj.disableGatewayService
+	dstEpj.StaticRoutes = make([]*types.StaticRoute, len(epj.StaticRoutes))
+	copy(dstEpj.StaticRoutes, epj.StaticRoutes)
+	dstEpj.gw = types.GetIPCopy(epj.gw)
+	dstEpj.gw = types.GetIPCopy(epj.gw6)
+	return nil
+}

vendor/src/github.com/docker/libnetwork/networkdb/cluster.go

@@ -311,7 +311,16 @@ func (nDB *NetworkDB) bulkSyncTables() {
 		nid := networks[0]
 		networks = networks[1:]
 
-		completed, err := nDB.bulkSync(nid, false)
+		nDB.RLock()
+		nodes := nDB.networkNodes[nid]
+		nDB.RUnlock()
+
+		// No peer nodes on this network. Move on.
+		if len(nodes) == 0 {
+			continue
+		}
+
+		completed, err := nDB.bulkSync(nid, nodes, false)
 		if err != nil {
 			logrus.Errorf("periodic bulk sync failure for network %s: %v", nid, err)
 			continue
@@ -334,11 +343,7 @@ func (nDB *NetworkDB) bulkSyncTables() {
 	}
 }
 
-func (nDB *NetworkDB) bulkSync(nid string, all bool) ([]string, error) {
-	nDB.RLock()
-	nodes := nDB.networkNodes[nid]
-	nDB.RUnlock()
-
+func (nDB *NetworkDB) bulkSync(nid string, nodes []string, all bool) ([]string, error) {
 	if !all {
 		// If not all, then just pick one.
 		nodes = nDB.mRandomNodes(1, nodes)

vendor/src/github.com/docker/libnetwork/networkdb/delegate.go

@@ -2,6 +2,7 @@ package networkdb
 
 import (
 	"fmt"
+	"net"
 	"time"
 
 	"github.com/Sirupsen/logrus"
@@ -210,8 +211,13 @@ func (nDB *NetworkDB) handleBulkSync(buf []byte) {
 		return
 	}
 
+	var nodeAddr net.IP
+	if node, ok := nDB.nodes[bsm.NodeName]; ok {
+		nodeAddr = node.Addr
+	}
+
 	if err := nDB.bulkSyncNode(bsm.Networks, bsm.NodeName, false); err != nil {
-		logrus.Errorf("Error in responding to bulk sync from node %s: %v", nDB.nodes[bsm.NodeName].Addr, err)
+		logrus.Errorf("Error in responding to bulk sync from node %s: %v", nodeAddr, err)
 	}
 }
 

vendor/src/github.com/docker/libnetwork/networkdb/event_delegate.go

@@ -14,6 +14,7 @@ func (e *eventDelegate) NotifyJoin(n *memberlist.Node) {
 
 func (e *eventDelegate) NotifyLeave(n *memberlist.Node) {
 	e.nDB.deleteNodeTableEntries(n.Name)
+	e.nDB.deleteNetworkNodeEntries(n.Name)
 	e.nDB.Lock()
 	delete(e.nDB.nodes, n.Name)
 	e.nDB.Unlock()

vendor/src/github.com/docker/libnetwork/networkdb/networkdb.go

@@ -286,6 +286,23 @@ func (nDB *NetworkDB) DeleteEntry(tname, nid, key string) error {
 	return nil
 }
 
+func (nDB *NetworkDB) deleteNetworkNodeEntries(deletedNode string) {
+	nDB.Lock()
+	for nid, nodes := range nDB.networkNodes {
+		updatedNodes := make([]string, 0, len(nodes))
+		for _, node := range nodes {
+			if node == deletedNode {
+				continue
+			}
+
+			updatedNodes = append(updatedNodes, node)
+		}
+
+		nDB.networkNodes[nid] = updatedNodes
+	}
+	nDB.Unlock()
+}
+
 func (nDB *NetworkDB) deleteNodeTableEntries(node string) {
 	nDB.Lock()
 	nDB.indexes[byTable].Walk(func(path string, v interface{}) bool {
@@ -359,6 +376,7 @@ func (nDB *NetworkDB) JoinNetwork(nid string) error {
 		RetransmitMult: 4,
 	}
 	nDB.networkNodes[nid] = append(nDB.networkNodes[nid], nDB.config.NodeName)
+	networkNodes := nDB.networkNodes[nid]
 	nDB.Unlock()
 
 	if err := nDB.sendNetworkEvent(nid, NetworkEventTypeJoin, ltime); err != nil {
@@ -366,7 +384,7 @@ func (nDB *NetworkDB) JoinNetwork(nid string) error {
 	}
 
 	logrus.Debugf("%s: joined network %s", nDB.config.NodeName, nid)
-	if _, err := nDB.bulkSync(nid, true); err != nil {
+	if _, err := nDB.bulkSync(nid, networkNodes, true); err != nil {
 		logrus.Errorf("Error bulk syncing while joining network %s: %v", nid, err)
 	}
 

vendor/src/github.com/docker/libnetwork/osl/namespace_linux.go

@@ -2,10 +2,13 @@ package osl
 
 import (
 	"fmt"
+	"io/ioutil"
 	"net"
 	"os"
 	"os/exec"
 	"runtime"
+	"strconv"
+	"strings"
 	"sync"
 	"syscall"
 	"time"
@@ -133,6 +136,39 @@ func GC() {
 // container id.
 func GenerateKey(containerID string) string {
 	maxLen := 12
+	// Read sandbox key from host for overlay
+	if strings.HasPrefix(containerID, "-") {
+		var (
+			index    int
+			indexStr string
+			tmpkey   string
+		)
+		dir, err := ioutil.ReadDir(prefix)
+		if err != nil {
+			return ""
+		}
+
+		for _, v := range dir {
+			id := v.Name()
+			if strings.HasSuffix(id, containerID[:maxLen-1]) {
+				indexStr = strings.TrimSuffix(id, containerID[:maxLen-1])
+				tmpindex, err := strconv.Atoi(indexStr)
+				if err != nil {
+					return ""
+				}
+				if tmpindex > index {
+					index = tmpindex
+					tmpkey = id
+				}
+
+			}
+		}
+		containerID = tmpkey
+		if containerID == "" {
+			return ""
+		}
+	}
+
 	if len(containerID) < maxLen {
 		maxLen = len(containerID)
 	}
@@ -142,10 +178,14 @@ func GenerateKey(containerID string) string {
 
 // NewSandbox provides a new sandbox instance created in an os specific way
 // provided a key which uniquely identifies the sandbox
-func NewSandbox(key string, osCreate bool) (Sandbox, error) {
-	err := createNetworkNamespace(key, osCreate)
-	if err != nil {
-		return nil, err
+func NewSandbox(key string, osCreate, isRestore bool) (Sandbox, error) {
+	if !isRestore {
+		err := createNetworkNamespace(key, osCreate)
+		if err != nil {
+			return nil, err
+		}
+	} else {
+		once.Do(createBasePath)
 	}
 
 	n := &networkNamespace{path: key, isDefault: !osCreate}
@@ -347,3 +387,105 @@ func (n *networkNamespace) Destroy() error {
 	addToGarbagePaths(n.path)
 	return nil
 }
+
+// Restore restore the network namespace
+func (n *networkNamespace) Restore(ifsopt map[string][]IfaceOption, routes []*types.StaticRoute, gw net.IP, gw6 net.IP) error {
+	// restore interfaces
+	for name, opts := range ifsopt {
+		if !strings.Contains(name, "+") {
+			return fmt.Errorf("wrong iface name in restore osl sandbox interface: %s", name)
+		}
+		seps := strings.Split(name, "+")
+		srcName := seps[0]
+		dstPrefix := seps[1]
+		i := &nwIface{srcName: srcName, dstName: dstPrefix, ns: n}
+		i.processInterfaceOptions(opts...)
+		if i.master != "" {
+			i.dstMaster = n.findDst(i.master, true)
+			if i.dstMaster == "" {
+				return fmt.Errorf("could not find an appropriate master %q for %q",
+					i.master, i.srcName)
+			}
+		}
+		if n.isDefault {
+			i.dstName = i.srcName
+		} else {
+			links, err := n.nlHandle.LinkList()
+			if err != nil {
+				return fmt.Errorf("failed to retrieve list of links in network namespace %q during restore", n.path)
+			}
+			// due to the docker network connect/disconnect, so the dstName should
+			// restore from the namespace
+			for _, link := range links {
+				addrs, err := n.nlHandle.AddrList(link, netlink.FAMILY_V4)
+				if err != nil {
+					return err
+				}
+				ifaceName := link.Attrs().Name
+				if strings.HasPrefix(ifaceName, "vxlan") {
+					if i.dstName == "vxlan" {
+						i.dstName = ifaceName
+						break
+					}
+				}
+				// find the interface name by ip
+				if i.address != nil {
+					for _, addr := range addrs {
+						if addr.IPNet.String() == i.address.String() {
+							i.dstName = ifaceName
+							break
+						}
+						continue
+					}
+					if i.dstName == ifaceName {
+						break
+					}
+				}
+				// This is to find the interface name of the pair in overlay sandbox
+				if strings.HasPrefix(ifaceName, "veth") {
+					if i.master != "" && i.dstName == "veth" {
+						i.dstName = ifaceName
+					}
+				}
+			}
+
+			var index int
+			indexStr := strings.TrimPrefix(i.dstName, dstPrefix)
+			if indexStr != "" {
+				index, err = strconv.Atoi(indexStr)
+				if err != nil {
+					return err
+				}
+			}
+			index++
+			n.Lock()
+			if index > n.nextIfIndex {
+				n.nextIfIndex = index
+			}
+			n.iFaces = append(n.iFaces, i)
+			n.Unlock()
+		}
+	}
+
+	// restore routes
+	for _, r := range routes {
+		n.Lock()
+		n.staticRoutes = append(n.staticRoutes, r)
+		n.Unlock()
+	}
+
+	// restore gateway
+	if len(gw) > 0 {
+		n.Lock()
+		n.gw = gw
+		n.Unlock()
+	}
+
+	if len(gw6) > 0 {
+		n.Lock()
+		n.gwv6 = gw6
+		n.Unlock()
+	}
+
+	return nil
+}

vendor/src/github.com/docker/libnetwork/osl/namespace_windows.go

@@ -15,7 +15,7 @@ func GenerateKey(containerID string) string {
 
 // NewSandbox provides a new sandbox instance created in an os specific way
 // provided a key which uniquely identifies the sandbox
-func NewSandbox(key string, osCreate bool) (Sandbox, error) {
+func NewSandbox(key string, osCreate, isRestore bool) (Sandbox, error) {
 	return nil, nil
 }
 

vendor/src/github.com/docker/libnetwork/osl/sandbox.go

@@ -58,6 +58,9 @@ type Sandbox interface {
 
 	// Destroy the sandbox
 	Destroy() error
+
+	// restore sandbox
+	Restore(ifsopt map[string][]IfaceOption, routes []*types.StaticRoute, gw net.IP, gw6 net.IP) error
 }
 
 // NeighborOptionSetter interface defines the option setter methods for interface options

vendor/src/github.com/docker/libnetwork/osl/sandbox_freebsd.go

@@ -15,7 +15,7 @@ func GenerateKey(containerID string) string {
 
 // NewSandbox provides a new sandbox instance created in an os specific way
 // provided a key which uniquely identifies the sandbox
-func NewSandbox(key string, osCreate bool) (Sandbox, error) {
+func NewSandbox(key string, osCreate, isRestore bool) (Sandbox, error) {
 	return nil, nil
 }
 

vendor/src/github.com/docker/libnetwork/osl/sandbox_unsupported.go

@@ -11,7 +11,7 @@ var (
 
 // NewSandbox provides a new sandbox instance created in an os specific way
 // provided a key which uniquely identifies the sandbox
-func NewSandbox(key string, osCreate bool) (Sandbox, error) {
+func NewSandbox(key string, osCreate, isRestore bool) (Sandbox, error) {
 	return nil, ErrNotImplemented
 }
 

vendor/src/github.com/docker/libnetwork/resolver_unix.go

@@ -19,6 +19,13 @@ func init() {
 	reexec.Register("setup-resolver", reexecSetupResolver)
 }
 
+const (
+	// outputChain used for docker embed dns
+	outputChain = "DOCKER_OUTPUT"
+	//postroutingchain used for docker embed dns
+	postroutingchain = "DOCKER_POSTROUTING"
+)
+
 func reexecSetupResolver() {
 	runtime.LockOSThread()
 	defer runtime.UnlockOSThread()
@@ -31,10 +38,10 @@ func reexecSetupResolver() {
 	_, ipPort, _ := net.SplitHostPort(os.Args[2])
 	_, tcpPort, _ := net.SplitHostPort(os.Args[3])
 	rules := [][]string{
-		{"-t", "nat", "-A", "OUTPUT", "-d", resolverIP, "-p", "udp", "--dport", dnsPort, "-j", "DNAT", "--to-destination", os.Args[2]},
-		{"-t", "nat", "-A", "POSTROUTING", "-s", resolverIP, "-p", "udp", "--sport", ipPort, "-j", "SNAT", "--to-source", ":" + dnsPort},
-		{"-t", "nat", "-A", "OUTPUT", "-d", resolverIP, "-p", "tcp", "--dport", dnsPort, "-j", "DNAT", "--to-destination", os.Args[3]},
-		{"-t", "nat", "-A", "POSTROUTING", "-s", resolverIP, "-p", "tcp", "--sport", tcpPort, "-j", "SNAT", "--to-source", ":" + dnsPort},
+		{"-t", "nat", "-I", outputChain, "-d", resolverIP, "-p", "udp", "--dport", dnsPort, "-j", "DNAT", "--to-destination", os.Args[2]},
+		{"-t", "nat", "-I", postroutingchain, "-s", resolverIP, "-p", "udp", "--sport", ipPort, "-j", "SNAT", "--to-source", ":" + dnsPort},
+		{"-t", "nat", "-I", outputChain, "-d", resolverIP, "-p", "tcp", "--dport", dnsPort, "-j", "DNAT", "--to-destination", os.Args[3]},
+		{"-t", "nat", "-I", postroutingchain, "-s", resolverIP, "-p", "tcp", "--sport", tcpPort, "-j", "SNAT", "--to-source", ":" + dnsPort},
 	}
 
 	f, err := os.OpenFile(os.Args[1], os.O_RDONLY, 0)
@@ -50,6 +57,23 @@ func reexecSetupResolver() {
 		os.Exit(3)
 	}
 
+	// insert outputChain and postroutingchain
+	err = iptables.RawCombinedOutputNative("-t", "nat", "-C", "OUTPUT", "-d", resolverIP, "-j", outputChain)
+	if err == nil {
+		iptables.RawCombinedOutputNative("-t", "nat", "-F", outputChain)
+	} else {
+		iptables.RawCombinedOutputNative("-t", "nat", "-N", outputChain)
+		iptables.RawCombinedOutputNative("-t", "nat", "-I", "OUTPUT", "-d", resolverIP, "-j", outputChain)
+	}
+
+	err = iptables.RawCombinedOutputNative("-t", "nat", "-C", "POSTROUTING", "-d", resolverIP, "-j", postroutingchain)
+	if err == nil {
+		iptables.RawCombinedOutputNative("-t", "nat", "-F", postroutingchain)
+	} else {
+		iptables.RawCombinedOutputNative("-t", "nat", "-N", postroutingchain)
+		iptables.RawCombinedOutputNative("-t", "nat", "-I", "POSTROUTING", "-d", resolverIP, "-j", postroutingchain)
+	}
+
 	for _, rule := range rules {
 		if iptables.RawCombinedOutputNative(rule...) != nil {
 			log.Errorf("setting up rule failed, %v", rule)

vendor/src/github.com/docker/libnetwork/sandbox.go

@@ -700,6 +700,52 @@ func (sb *sandbox) releaseOSSbox() {
 	osSbox.Destroy()
 }
 
+func (sb *sandbox) restoreOslSandbox() error {
+	var routes []*types.StaticRoute
+
+	// restore osl sandbox
+	Ifaces := make(map[string][]osl.IfaceOption)
+	for _, ep := range sb.endpoints {
+		var ifaceOptions []osl.IfaceOption
+		ep.Lock()
+		joinInfo := ep.joinInfo
+		i := ep.iface
+		ep.Unlock()
+		ifaceOptions = append(ifaceOptions, sb.osSbox.InterfaceOptions().Address(i.addr), sb.osSbox.InterfaceOptions().Routes(i.routes))
+		if i.addrv6 != nil && i.addrv6.IP.To16() != nil {
+			ifaceOptions = append(ifaceOptions, sb.osSbox.InterfaceOptions().AddressIPv6(i.addrv6))
+		}
+		if i.mac != nil {
+			ifaceOptions = append(ifaceOptions, sb.osSbox.InterfaceOptions().MacAddress(i.mac))
+		}
+		if len(i.llAddrs) != 0 {
+			ifaceOptions = append(ifaceOptions, sb.osSbox.InterfaceOptions().LinkLocalAddresses(i.llAddrs))
+		}
+		Ifaces[fmt.Sprintf("%s+%s", i.srcName, i.dstPrefix)] = ifaceOptions
+		if joinInfo != nil {
+			for _, r := range joinInfo.StaticRoutes {
+				routes = append(routes, r)
+			}
+		}
+		if ep.needResolver() {
+			sb.startResolver()
+		}
+	}
+
+	gwep := sb.getGatewayEndpoint()
+	if gwep == nil {
+		return nil
+	}
+
+	// restore osl sandbox
+	err := sb.osSbox.Restore(Ifaces, routes, gwep.joinInfo.gw, gwep.joinInfo.gw6)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
 func (sb *sandbox) populateNetworkResources(ep *endpoint) error {
 	sb.Lock()
 	if sb.osSbox == nil {

vendor/src/github.com/docker/libnetwork/sandbox_dns_unix.go

@@ -139,6 +139,16 @@ func (sb *sandbox) updateParentHosts() error {
 	return nil
 }
 
+func (sb *sandbox) restorePath() {
+	if sb.config.resolvConfPath == "" {
+		sb.config.resolvConfPath = defaultPrefix + "/" + sb.id + "/resolv.conf"
+	}
+	sb.config.resolvConfHashFile = sb.config.resolvConfPath + ".hash"
+	if sb.config.hostsPath == "" {
+		sb.config.hostsPath = defaultPrefix + "/" + sb.id + "/hosts"
+	}
+}
+
 func (sb *sandbox) setupDNS() error {
 	var newRC *resolvconf.File
 

vendor/src/github.com/docker/libnetwork/sandbox_dns_windows.go

@@ -15,6 +15,9 @@ func (sb *sandbox) setupResolutionFiles() error {
 	return nil
 }
 
+func (sb *sandbox) restorePath() {
+}
+
 func (sb *sandbox) updateHostsFile(ifaceIP string) error {
 	return nil
 }

vendor/src/github.com/docker/libnetwork/sandbox_store.go

@@ -20,12 +20,13 @@ type epState struct {
 }
 
 type sbState struct {
-	ID       string
-	Cid      string
-	c        *controller
-	dbIndex  uint64
-	dbExists bool
-	Eps      []epState
+	ID         string
+	Cid        string
+	c          *controller
+	dbIndex    uint64
+	dbExists   bool
+	Eps        []epState
+	EpPriority map[string]int
 }
 
 func (sbs *sbState) Key() []string {
@@ -106,6 +107,7 @@ func (sbs *sbState) CopyTo(o datastore.KVObject) error {
 	dstSbs.Cid = sbs.Cid
 	dstSbs.dbIndex = sbs.dbIndex
 	dstSbs.dbExists = sbs.dbExists
+	dstSbs.EpPriority = sbs.EpPriority
 
 	for _, eps := range sbs.Eps {
 		dstSbs.Eps = append(dstSbs.Eps, eps)
@@ -120,9 +122,10 @@ func (sbs *sbState) DataScope() string {
 
 func (sb *sandbox) storeUpdate() error {
 	sbs := &sbState{
-		c:   sb.controller,
-		ID:  sb.id,
-		Cid: sb.containerID,
+		c:          sb.controller,
+		ID:         sb.id,
+		Cid:        sb.containerID,
+		EpPriority: sb.epPriority,
 	}
 
 retry:
@@ -166,7 +169,7 @@ func (sb *sandbox) storeDelete() error {
 	return sb.controller.deleteFromStore(sbs)
 }
 
-func (c *controller) sandboxCleanup() {
+func (c *controller) sandboxCleanup(activeSandboxes map[string]interface{}) {
 	store := c.getStore(datastore.LocalScope)
 	if store == nil {
 		logrus.Errorf("Could not find local scope store while trying to cleanup sandboxes")
@@ -192,15 +195,27 @@ func (c *controller) sandboxCleanup() {
 			controller:  sbs.c,
 			containerID: sbs.Cid,
 			endpoints:   epHeap{},
-			epPriority:  map[string]int{},
 			dbIndex:     sbs.dbIndex,
 			isStub:      true,
 			dbExists:    true,
 		}
 
-		sb.osSbox, err = osl.NewSandbox(sb.Key(), true)
+		msg := " for cleanup"
+		create := true
+		isRestore := false
+		if val, ok := activeSandboxes[sb.ID()]; ok {
+			msg = ""
+			sb.isStub = false
+			isRestore = true
+			opts := val.([]SandboxOption)
+			sb.processOptions(opts...)
+			sb.restorePath()
+			create = !sb.config.useDefaultSandBox
+			heap.Init(&sb.endpoints)
+		}
+		sb.osSbox, err = osl.NewSandbox(sb.Key(), create, isRestore)
 		if err != nil {
-			logrus.Errorf("failed to create new osl sandbox while trying to build sandbox for cleanup: %v", err)
+			logrus.Errorf("failed to create osl sandbox while trying to restore sandbox %s%s: %v", sb.ID()[0:7], msg, err)
 			continue
 		}
 
@@ -222,13 +237,34 @@ func (c *controller) sandboxCleanup() {
 					ep = &endpoint{id: eps.Eid, network: n, sandboxID: sbs.ID}
 				}
 			}
-
 			heap.Push(&sb.endpoints, ep)
 		}
 
-		logrus.Infof("Removing stale sandbox %s (%s)", sb.id, sb.containerID)
-		if err := sb.delete(true); err != nil {
-			logrus.Errorf("failed to delete sandbox %s while trying to cleanup: %v", sb.id, err)
+		if _, ok := activeSandboxes[sb.ID()]; !ok {
+			logrus.Infof("Removing stale sandbox %s (%s)", sb.id, sb.containerID)
+			if err := sb.delete(true); err != nil {
+				logrus.Errorf("Failed to delete sandbox %s while trying to cleanup: %v", sb.id, err)
+			}
+			continue
+		}
+
+		// reconstruct osl sandbox field
+		if !sb.config.useDefaultSandBox {
+			if err := sb.restoreOslSandbox(); err != nil {
+				logrus.Errorf("failed to populate fields for osl sandbox %s", sb.ID())
+				continue
+			}
+		} else {
+			c.sboxOnce.Do(func() {
+				c.defOsSbox = sb.osSbox
+			})
+		}
+
+		for _, ep := range sb.endpoints {
+			// Watch for service records
+			if !c.isAgent() {
+				c.watchSvcRecord(ep)
+			}
 		}
 	}
 }