Merge pull request #1244 from aboch/restore

Add network restore to support docker live restore container

libnetwork/config/config.go

@@ -15,9 +15,10 @@ import (
 
 // Config encapsulates configurations of various Libnetwork components
 type Config struct {
-	Daemon  DaemonCfg
-	Cluster ClusterCfg
-	Scopes  map[string]*datastore.ScopeCfg
+	Daemon          DaemonCfg
+	Cluster         ClusterCfg
+	Scopes          map[string]*datastore.ScopeCfg
+	ActiveSandboxes map[string]interface{}
 }
 
 // DaemonCfg represents libnetwork core configuration
@@ -245,3 +246,11 @@ func OptionLocalKVProviderConfig(config *store.Config) Option {
 		c.Scopes[datastore.LocalScope].Client.Config = config
 	}
 }
+
+// OptionActiveSandboxes function returns an option setter for passing the sandboxes
+// which were active during previous daemon life
+func OptionActiveSandboxes(sandboxes map[string]interface{}) Option {
+	return func(c *Config) {
+		c.ActiveSandboxes = sandboxes
+	}
+}

libnetwork/controller.go

@@ -203,15 +203,13 @@ func New(cfgOptions ...config.Option) (NetworkController, error) {
 		}
 	}
 
-	// Reserve pools first before doing cleanup. This is because
-	// if the pools are not populated properly, the cleanups of
-	// endpoint/network and sandbox below will not be able to
-	// release ip subnets and addresses properly into the pool
-	// because the pools won't exist.
+	// Reserve pools first before doing cleanup. Otherwise the
+	// cleanups of endpoint/network and sandbox below will
+	// generate many unnecessary warnings
 	c.reservePools()
 
 	// Cleanup resources
-	c.sandboxCleanup()
+	c.sandboxCleanup(c.cfg.ActiveSandboxes)
 	c.cleanupLocalEndpoints()
 	c.networkCleanup()
 
@@ -671,9 +669,27 @@ func (c *controller) reservePools() {
 				c.Gateway = n.ipamV6Info[i].Gateway.IP.String()
 			}
 		}
+		// Reserve pools
 		if err := n.ipamAllocate(); err != nil {
 			log.Warnf("Failed to allocate ipam pool(s) for network %q (%s): %v", n.Name(), n.ID(), err)
 		}
+		// Reserve existing endpoints' addresses
+		ipam, _, err := n.getController().getIPAMDriver(n.ipamType)
+		if err != nil {
+			log.Warnf("Failed to retrieve ipam driver for network %q (%s) during address reservation", n.Name(), n.ID())
+			continue
+		}
+		epl, err := n.getEndpointsFromStore()
+		if err != nil {
+			log.Warnf("Failed to retrieve list of current endpoints on network %q (%s)", n.Name(), n.ID())
+			continue
+		}
+		for _, ep := range epl {
+			if err := ep.assignAddress(ipam, true, ep.Iface().AddressIPv6() != nil); err != nil {
+				log.Warnf("Failed to reserve current adress for endpoint %q (%s) on network %q (%s)",
+					ep.Name(), ep.ID(), n.Name(), n.ID())
+			}
+		}
 	}
 }
 
@@ -832,7 +848,7 @@ func (c *controller) NewSandbox(containerID string, options ...SandboxOption) (s
 
 	if sb.config.useDefaultSandBox {
 		c.sboxOnce.Do(func() {
-			c.defOsSbox, err = osl.NewSandbox(sb.Key(), false)
+			c.defOsSbox, err = osl.NewSandbox(sb.Key(), false, false)
 		})
 
 		if err != nil {
@@ -844,7 +860,7 @@ func (c *controller) NewSandbox(containerID string, options ...SandboxOption) (s
 	}
 
 	if sb.osSbox == nil && !sb.config.useExternalKey {
-		if sb.osSbox, err = osl.NewSandbox(sb.Key(), !sb.config.useDefaultSandBox); err != nil {
+		if sb.osSbox, err = osl.NewSandbox(sb.Key(), !sb.config.useDefaultSandBox, false); err != nil {
 			return nil, fmt.Errorf("failed to create new osl sandbox: %v", err)
 		}
 	}

libnetwork/drivers/bridge/bridge.go

@@ -91,6 +91,7 @@ type connectivityConfiguration struct {
 
 type bridgeEndpoint struct {
 	id              string
+	nid             string
 	srcName         string
 	addr            *net.IPNet
 	addrv6          *net.IPNet
@@ -99,6 +100,8 @@ type bridgeEndpoint struct {
 	containerConfig *containerConfiguration
 	extConnConfig   *connectivityConfiguration
 	portMapping     []types.PortBinding // Operation port bindings
+	dbIndex         uint64
+	dbExists        bool
 }
 
 type bridgeNetwork struct {
@@ -882,7 +885,7 @@ func (d *driver) CreateEndpoint(nid, eid string, ifInfo driverapi.InterfaceInfo,
 
 	// Create and add the endpoint
 	n.Lock()
-	endpoint := &bridgeEndpoint{id: eid, config: epConfig}
+	endpoint := &bridgeEndpoint{id: eid, nid: nid, config: epConfig}
 	n.endpoints[eid] = endpoint
 	n.Unlock()
 
@@ -1009,6 +1012,10 @@ func (d *driver) CreateEndpoint(nid, eid string, ifInfo driverapi.InterfaceInfo,
 		}
 	}
 
+	if err = d.storeUpdate(endpoint); err != nil {
+		return fmt.Errorf("failed to save bridge endpoint %s to store: %v", ep.id[0:7], err)
+	}
+
 	return nil
 }
 
@@ -1069,6 +1076,10 @@ func (d *driver) DeleteEndpoint(nid, eid string) error {
 		d.nlh.LinkDel(link)
 	}
 
+	if err := d.storeDelete(ep); err != nil {
+		logrus.Warnf("Failed to remove bridge endpoint %s from store: %v", ep.id[0:7], err)
+	}
+
 	return nil
 }
 
@@ -1225,6 +1236,11 @@ func (d *driver) ProgramExternalConnectivity(nid, eid string, options map[string
 		return err
 	}
 
+	if err = d.storeUpdate(endpoint); err != nil {
+		endpoint.portMapping = nil
+		return fmt.Errorf("failed to update bridge endpoint %s to store: %v", endpoint.id[0:7], err)
+	}
+
 	if !network.config.EnableICC {
 		return d.link(network, endpoint, true)
 	}

libnetwork/drivers/bridge/bridge_store.go

@@ -12,7 +12,13 @@ import (
 	"github.com/docker/libnetwork/types"
 )
 
-const bridgePrefix = "bridge"
+const (
+	// network config prefix was not specific enough.
+	// To be backward compatible, need custom endpoint
+	// prefix with different root
+	bridgePrefix         = "bridge"
+	bridgeEndpointPrefix = "bridge-endpoint"
+)
 
 func (d *driver) initStore(option map[string]interface{}) error {
 	if data, ok := option[netlabel.LocalKVClient]; ok {
@@ -26,7 +32,15 @@ func (d *driver) initStore(option map[string]interface{}) error {
 			return types.InternalErrorf("bridge driver failed to initialize data store: %v", err)
 		}
 
-		return d.populateNetworks()
+		err = d.populateNetworks()
+		if err != nil {
+			return err
+		}
+
+		err = d.populateEndpoints()
+		if err != nil {
+			return err
+		}
 	}
 
 	return nil
@@ -48,6 +62,36 @@ func (d *driver) populateNetworks() error {
 		if err = d.createNetwork(ncfg); err != nil {
 			logrus.Warnf("could not create bridge network for id %s bridge name %s while booting up from persistent state: %v", ncfg.ID, ncfg.BridgeName, err)
 		}
+		logrus.Debugf("Network (%s) restored", ncfg.ID[0:7])
+	}
+
+	return nil
+}
+
+func (d *driver) populateEndpoints() error {
+	kvol, err := d.store.List(datastore.Key(bridgeEndpointPrefix), &bridgeEndpoint{})
+	if err != nil && err != datastore.ErrKeyNotFound {
+		return fmt.Errorf("failed to get bridge endpoints from store: %v", err)
+	}
+
+	if err == datastore.ErrKeyNotFound {
+		return nil
+	}
+
+	for _, kvo := range kvol {
+		ep := kvo.(*bridgeEndpoint)
+		n, ok := d.networks[ep.nid]
+		if !ok {
+			logrus.Debugf("Network (%s) not found for restored bridge endpoint (%s)", ep.nid[0:7], ep.id[0:7])
+			logrus.Debugf("Deleting stale bridge endpoint (%s) from store", ep.nid[0:7])
+			if err := d.storeDelete(ep); err != nil {
+				logrus.Debugf("Failed to delete stale bridge endpoint (%s) from store", ep.nid[0:7])
+			}
+			continue
+		}
+		n.endpoints[ep.id] = ep
+		n.restorePortAllocations(ep)
+		logrus.Debugf("Endpoint (%s) restored to network (%s)", ep.id[0:7], ep.nid[0:7])
 	}
 
 	return nil
@@ -184,7 +228,7 @@ func (ncfg *networkConfiguration) Exists() bool {
 }
 
 func (ncfg *networkConfiguration) Skip() bool {
-	return ncfg.DefaultBridge
+	return false
 }
 
 func (ncfg *networkConfiguration) New() datastore.KVObject {
@@ -200,3 +244,135 @@ func (ncfg *networkConfiguration) CopyTo(o datastore.KVObject) error {
 func (ncfg *networkConfiguration) DataScope() string {
 	return datastore.LocalScope
 }
+
+func (ep *bridgeEndpoint) MarshalJSON() ([]byte, error) {
+	epMap := make(map[string]interface{})
+	epMap["id"] = ep.id
+	epMap["nid"] = ep.nid
+	epMap["SrcName"] = ep.srcName
+	epMap["MacAddress"] = ep.macAddress.String()
+	epMap["Addr"] = ep.addr.String()
+	if ep.addrv6 != nil {
+		epMap["Addrv6"] = ep.addrv6.String()
+	}
+	epMap["Config"] = ep.config
+	epMap["ContainerConfig"] = ep.containerConfig
+	epMap["ExternalConnConfig"] = ep.extConnConfig
+	epMap["PortMapping"] = ep.portMapping
+
+	return json.Marshal(epMap)
+}
+
+func (ep *bridgeEndpoint) UnmarshalJSON(b []byte) error {
+	var (
+		err   error
+		epMap map[string]interface{}
+	)
+
+	if err = json.Unmarshal(b, &epMap); err != nil {
+		return fmt.Errorf("Failed to unmarshal to bridge endpoint: %v", err)
+	}
+
+	if v, ok := epMap["MacAddress"]; ok {
+		if ep.macAddress, err = net.ParseMAC(v.(string)); err != nil {
+			return types.InternalErrorf("failed to decode bridge endpoint MAC address (%s) after json unmarshal: %v", v.(string), err)
+		}
+	}
+	if v, ok := epMap["Addr"]; ok {
+		if ep.addr, err = types.ParseCIDR(v.(string)); err != nil {
+			return types.InternalErrorf("failed to decode bridge endpoint IPv4 address (%s) after json unmarshal: %v", v.(string), err)
+		}
+	}
+	if v, ok := epMap["Addrv6"]; ok {
+		if ep.addrv6, err = types.ParseCIDR(v.(string)); err != nil {
+			return types.InternalErrorf("failed to decode bridge endpoint IPv6 address (%s) after json unmarshal: %v", v.(string), err)
+		}
+	}
+	ep.id = epMap["id"].(string)
+	ep.nid = epMap["nid"].(string)
+	ep.srcName = epMap["SrcName"].(string)
+	d, _ := json.Marshal(epMap["Config"])
+	if err := json.Unmarshal(d, &ep.config); err != nil {
+		logrus.Warnf("Failed to decode endpoint config %v", err)
+	}
+	d, _ = json.Marshal(epMap["ContainerConfig"])
+	if err := json.Unmarshal(d, &ep.containerConfig); err != nil {
+		logrus.Warnf("Failed to decode endpoint container config %v", err)
+	}
+	d, _ = json.Marshal(epMap["ExternalConnConfig"])
+	if err := json.Unmarshal(d, &ep.extConnConfig); err != nil {
+		logrus.Warnf("Failed to decode endpoint external connectivity configuration %v", err)
+	}
+	d, _ = json.Marshal(epMap["PortMapping"])
+	if err := json.Unmarshal(d, &ep.portMapping); err != nil {
+		logrus.Warnf("Failed to decode endpoint port mapping %v", err)
+	}
+
+	return nil
+}
+
+func (ep *bridgeEndpoint) Key() []string {
+	return []string{bridgeEndpointPrefix, ep.id}
+}
+
+func (ep *bridgeEndpoint) KeyPrefix() []string {
+	return []string{bridgeEndpointPrefix}
+}
+
+func (ep *bridgeEndpoint) Value() []byte {
+	b, err := json.Marshal(ep)
+	if err != nil {
+		return nil
+	}
+	return b
+}
+
+func (ep *bridgeEndpoint) SetValue(value []byte) error {
+	return json.Unmarshal(value, ep)
+}
+
+func (ep *bridgeEndpoint) Index() uint64 {
+	return ep.dbIndex
+}
+
+func (ep *bridgeEndpoint) SetIndex(index uint64) {
+	ep.dbIndex = index
+	ep.dbExists = true
+}
+
+func (ep *bridgeEndpoint) Exists() bool {
+	return ep.dbExists
+}
+
+func (ep *bridgeEndpoint) Skip() bool {
+	return false
+}
+
+func (ep *bridgeEndpoint) New() datastore.KVObject {
+	return &bridgeEndpoint{}
+}
+
+func (ep *bridgeEndpoint) CopyTo(o datastore.KVObject) error {
+	dstEp := o.(*bridgeEndpoint)
+	*dstEp = *ep
+	return nil
+}
+
+func (ep *bridgeEndpoint) DataScope() string {
+	return datastore.LocalScope
+}
+
+func (n *bridgeNetwork) restorePortAllocations(ep *bridgeEndpoint) {
+	if ep.extConnConfig == nil ||
+		ep.extConnConfig.ExposedPorts == nil ||
+		ep.extConnConfig.PortBindings == nil {
+		return
+	}
+	tmp := ep.extConnConfig.PortBindings
+	ep.extConnConfig.PortBindings = ep.portMapping
+	_, err := n.allocatePorts(ep, n.config.DefaultBindingIP, n.driver.config.EnableUserlandProxy)
+	if err != nil {
+		logrus.Warnf("Failed to reserve existing port mapping for endpoint %s:%v", ep.id[0:7], err)
+	}
+	ep.extConnConfig.PortBindings = tmp
+}

libnetwork/drivers/bridge/bridge_test.go

@@ -1,6 +1,8 @@
 package bridge
 
 import (
+	"bytes"
+	"encoding/json"
 	"fmt"
 	"net"
 	"regexp"
@@ -20,6 +22,150 @@ func init() {
 	ipamutils.InitNetworks()
 }
 
+func TestEndpointMarshalling(t *testing.T) {
+	ip1, _ := types.ParseCIDR("172.22.0.9/16")
+	ip2, _ := types.ParseCIDR("2001:db8::9")
+	mac, _ := net.ParseMAC("ac:bd:24:57:66:77")
+	e := &bridgeEndpoint{
+		id:         "d2c015a1fe5930650cbcd50493efba0500bcebd8ee1f4401a16319f8a567de33",
+		nid:        "ee33fbb43c323f1920b6b35a0101552ac22ede960d0e5245e9738bccc68b2415",
+		addr:       ip1,
+		addrv6:     ip2,
+		macAddress: mac,
+		srcName:    "veth123456",
+		config:     &endpointConfiguration{MacAddress: mac},
+		containerConfig: &containerConfiguration{
+			ParentEndpoints: []string{"one", "due", "three"},
+			ChildEndpoints:  []string{"four", "five", "six"},
+		},
+		extConnConfig: &connectivityConfiguration{
+			ExposedPorts: []types.TransportPort{
+				{
+					Proto: 6,
+					Port:  uint16(18),
+				},
+			},
+			PortBindings: []types.PortBinding{
+				{
+					Proto:       6,
+					IP:          net.ParseIP("17210.33.9.56"),
+					Port:        uint16(18),
+					HostPort:    uint16(3000),
+					HostPortEnd: uint16(14000),
+				},
+			},
+		},
+		portMapping: []types.PortBinding{
+			{
+				Proto:       17,
+				IP:          net.ParseIP("172.33.9.56"),
+				Port:        uint16(99),
+				HostIP:      net.ParseIP("10.10.100.2"),
+				HostPort:    uint16(9900),
+				HostPortEnd: uint16(10000),
+			},
+			{
+				Proto:       6,
+				IP:          net.ParseIP("171.33.9.56"),
+				Port:        uint16(55),
+				HostIP:      net.ParseIP("10.11.100.2"),
+				HostPort:    uint16(5500),
+				HostPortEnd: uint16(55000),
+			},
+		},
+	}
+
+	b, err := json.Marshal(e)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	ee := &bridgeEndpoint{}
+	err = json.Unmarshal(b, ee)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if e.id != ee.id || e.nid != ee.nid || e.srcName != ee.srcName || !bytes.Equal(e.macAddress, ee.macAddress) ||
+		!types.CompareIPNet(e.addr, ee.addr) || !types.CompareIPNet(e.addrv6, ee.addrv6) ||
+		!compareEpConfig(e.config, ee.config) ||
+		!compareContainerConfig(e.containerConfig, ee.containerConfig) ||
+		!compareConnConfig(e.extConnConfig, ee.extConnConfig) ||
+		!compareBindings(e.portMapping, ee.portMapping) {
+		t.Fatalf("JSON marsh/unmarsh failed.\nOriginal:\n%#v\nDecoded:\n%#v", e, ee)
+	}
+}
+
+func compareEpConfig(a, b *endpointConfiguration) bool {
+	if a == b {
+		return true
+	}
+	if a == nil || b == nil {
+		return false
+	}
+	return bytes.Equal(a.MacAddress, b.MacAddress)
+}
+
+func compareContainerConfig(a, b *containerConfiguration) bool {
+	if a == b {
+		return true
+	}
+	if a == nil || b == nil {
+		return false
+	}
+	if len(a.ParentEndpoints) != len(b.ParentEndpoints) ||
+		len(a.ChildEndpoints) != len(b.ChildEndpoints) {
+		return false
+	}
+	for i := 0; i < len(a.ParentEndpoints); i++ {
+		if a.ParentEndpoints[i] != b.ParentEndpoints[i] {
+			return false
+		}
+	}
+	for i := 0; i < len(a.ChildEndpoints); i++ {
+		if a.ChildEndpoints[i] != b.ChildEndpoints[i] {
+			return false
+		}
+	}
+	return true
+}
+
+func compareConnConfig(a, b *connectivityConfiguration) bool {
+	if a == b {
+		return true
+	}
+	if a == nil || b == nil {
+		return false
+	}
+	if len(a.ExposedPorts) != len(b.ExposedPorts) ||
+		len(a.PortBindings) != len(b.PortBindings) {
+		return false
+	}
+	for i := 0; i < len(a.ExposedPorts); i++ {
+		if !a.ExposedPorts[i].Equal(&b.ExposedPorts[i]) {
+			return false
+		}
+	}
+	for i := 0; i < len(a.PortBindings); i++ {
+		if !a.PortBindings[i].Equal(&b.PortBindings[i]) {
+			return false
+		}
+	}
+	return true
+}
+
+func compareBindings(a, b []types.PortBinding) bool {
+	if len(a) != len(b) {
+		return false
+	}
+	for i := 0; i < len(a); i++ {
+		if !a[i].Equal(&b[i]) {
+			return false
+		}
+	}
+	return true
+}
+
 func getIPv4Data(t *testing.T) []driverapi.IPAMData {
 	ipd := driverapi.IPAMData{AddressSpace: "full"}
 	nw, _, err := netutils.ElectInterfaceAddresses("")

libnetwork/drivers/ipvlan/ipvlan.go

@@ -36,11 +36,14 @@ type driver struct {
 }
 
 type endpoint struct {
-	id      string
-	mac     net.HardwareAddr
-	addr    *net.IPNet
-	addrv6  *net.IPNet
-	srcName string
+	id       string
+	nid      string
+	mac      net.HardwareAddr
+	addr     *net.IPNet
+	addrv6   *net.IPNet
+	srcName  string
+	dbIndex  uint64
+	dbExists bool
 }
 
 type network struct {

libnetwork/drivers/ipvlan/ipvlan_endpoint.go

@@ -28,9 +28,9 @@ func (d *driver) CreateEndpoint(nid, eid string, ifInfo driverapi.InterfaceInfo,
 	}
 	ep := &endpoint{
 		id:     eid,
+		nid:    nid,
 		addr:   ifInfo.Address(),
 		addrv6: ifInfo.AddressIPv6(),
-		mac:    ifInfo.MacAddress(),
 	}
 	if ep.addr == nil {
 		return fmt.Errorf("create endpoint was not passed an IP address")
@@ -51,6 +51,11 @@ func (d *driver) CreateEndpoint(nid, eid string, ifInfo driverapi.InterfaceInfo,
 			}
 		}
 	}
+
+	if err := d.storeUpdate(ep); err != nil {
+		return fmt.Errorf("failed to save ipvlan endpoint %s to store: %v", ep.id[0:7], err)
+	}
+
 	n.addEndpoint(ep)
 
 	return nil
@@ -74,5 +79,9 @@ func (d *driver) DeleteEndpoint(nid, eid string) error {
 		ns.NlHandle().LinkDel(link)
 	}
 
+	if err := d.storeDelete(ep); err != nil {
+		logrus.Warnf("Failed to remove ipvlan endpoint %s from store: %v", ep.id[0:7], err)
+	}
+
 	return nil
 }

libnetwork/drivers/ipvlan/ipvlan_joinleave.go

@@ -116,6 +116,9 @@ func (d *driver) Join(nid, eid string, sboxKey string, jinfo driverapi.JoinInfo,
 	if err != nil {
 		return err
 	}
+	if err = d.storeUpdate(ep); err != nil {
+		return fmt.Errorf("failed to save ipvlan endpoint %s to store: %v", ep.id[0:7], err)
+	}
 
 	return nil
 }

libnetwork/drivers/ipvlan/ipvlan_store.go

@@ -3,6 +3,7 @@ package ipvlan
 import (
 	"encoding/json"
 	"fmt"
+	"net"
 
 	"github.com/Sirupsen/logrus"
 	"github.com/docker/libnetwork/datastore"
@@ -11,7 +12,11 @@ import (
 	"github.com/docker/libnetwork/types"
 )
 
-const ipvlanPrefix = "ipvlan" // prefix used for persistent driver storage
+const (
+	ipvlanPrefix         = "ipvlan"
+	ipvlanNetworkPrefix  = ipvlanPrefix + "/network"
+	ipvlanEndpointPrefix = ipvlanPrefix + "/endpoint"
+)
 
 // networkConfiguration for this driver's network specific configuration
 type configuration struct {
@@ -58,7 +63,7 @@ func (d *driver) initStore(option map[string]interface{}) error {
 
 // populateNetworks is invoked at driver init to recreate persistently stored networks
 func (d *driver) populateNetworks() error {
-	kvol, err := d.store.List(datastore.Key(ipvlanPrefix), &configuration{})
+	kvol, err := d.store.List(datastore.Key(ipvlanNetworkPrefix), &configuration{})
 	if err != nil && err != datastore.ErrKeyNotFound {
 		return fmt.Errorf("failed to get ipvlan network configurations from store: %v", err)
 	}
@@ -76,6 +81,34 @@ func (d *driver) populateNetworks() error {
 	return nil
 }
 
+func (d *driver) populateEndpoints() error {
+	kvol, err := d.store.List(datastore.Key(ipvlanEndpointPrefix), &endpoint{})
+	if err != nil && err != datastore.ErrKeyNotFound {
+		return fmt.Errorf("failed to get ipvlan endpoints from store: %v", err)
+	}
+
+	if err == datastore.ErrKeyNotFound {
+		return nil
+	}
+
+	for _, kvo := range kvol {
+		ep := kvo.(*endpoint)
+		n, ok := d.networks[ep.nid]
+		if !ok {
+			logrus.Debugf("Network (%s) not found for restored ipvlan endpoint (%s)", ep.nid[0:7], ep.id[0:7])
+			logrus.Debugf("Deleting stale ipvlan endpoint (%s) from store", ep.nid[0:7])
+			if err := d.storeDelete(ep); err != nil {
+				logrus.Debugf("Failed to delete stale ipvlan endpoint (%s) from store", ep.nid[0:7])
+			}
+			continue
+		}
+		n.endpoints[ep.id] = ep
+		logrus.Debugf("Endpoint (%s) restored to network (%s)", ep.id[0:7], ep.nid[0:7])
+	}
+
+	return nil
+}
+
 // storeUpdate used to update persistent ipvlan network records as they are created
 func (d *driver) storeUpdate(kvObject datastore.KVObject) error {
 	if d.store == nil {
@@ -165,11 +198,11 @@ func (config *configuration) UnmarshalJSON(b []byte) error {
 }
 
 func (config *configuration) Key() []string {
-	return []string{ipvlanPrefix, config.ID}
+	return []string{ipvlanNetworkPrefix, config.ID}
 }
 
 func (config *configuration) KeyPrefix() []string {
-	return []string{ipvlanPrefix}
+	return []string{ipvlanNetworkPrefix}
 }
 
 func (config *configuration) Value() []byte {
@@ -214,3 +247,103 @@ func (config *configuration) CopyTo(o datastore.KVObject) error {
 func (config *configuration) DataScope() string {
 	return datastore.LocalScope
 }
+
+func (ep *endpoint) MarshalJSON() ([]byte, error) {
+	epMap := make(map[string]interface{})
+	epMap["id"] = ep.id
+	epMap["nid"] = ep.nid
+	epMap["SrcName"] = ep.srcName
+	if len(ep.mac) != 0 {
+		epMap["MacAddress"] = ep.mac.String()
+	}
+	if ep.addr != nil {
+		epMap["Addr"] = ep.addr.String()
+	}
+	if ep.addrv6 != nil {
+		epMap["Addrv6"] = ep.addrv6.String()
+	}
+	return json.Marshal(epMap)
+}
+
+func (ep *endpoint) UnmarshalJSON(b []byte) error {
+	var (
+		err   error
+		epMap map[string]interface{}
+	)
+
+	if err = json.Unmarshal(b, &epMap); err != nil {
+		return fmt.Errorf("Failed to unmarshal to ipvlan endpoint: %v", err)
+	}
+
+	if v, ok := epMap["MacAddress"]; ok {
+		if ep.mac, err = net.ParseMAC(v.(string)); err != nil {
+			return types.InternalErrorf("failed to decode ipvlan endpoint MAC address (%s) after json unmarshal: %v", v.(string), err)
+		}
+	}
+	if v, ok := epMap["Addr"]; ok {
+		if ep.addr, err = types.ParseCIDR(v.(string)); err != nil {
+			return types.InternalErrorf("failed to decode ipvlan endpoint IPv4 address (%s) after json unmarshal: %v", v.(string), err)
+		}
+	}
+	if v, ok := epMap["Addrv6"]; ok {
+		if ep.addrv6, err = types.ParseCIDR(v.(string)); err != nil {
+			return types.InternalErrorf("failed to decode ipvlan endpoint IPv6 address (%s) after json unmarshal: %v", v.(string), err)
+		}
+	}
+	ep.id = epMap["id"].(string)
+	ep.nid = epMap["nid"].(string)
+	ep.srcName = epMap["SrcName"].(string)
+
+	return nil
+}
+
+func (ep *endpoint) Key() []string {
+	return []string{ipvlanEndpointPrefix, ep.id}
+}
+
+func (ep *endpoint) KeyPrefix() []string {
+	return []string{ipvlanEndpointPrefix}
+}
+
+func (ep *endpoint) Value() []byte {
+	b, err := json.Marshal(ep)
+	if err != nil {
+		return nil
+	}
+	return b
+}
+
+func (ep *endpoint) SetValue(value []byte) error {
+	return json.Unmarshal(value, ep)
+}
+
+func (ep *endpoint) Index() uint64 {
+	return ep.dbIndex
+}
+
+func (ep *endpoint) SetIndex(index uint64) {
+	ep.dbIndex = index
+	ep.dbExists = true
+}
+
+func (ep *endpoint) Exists() bool {
+	return ep.dbExists
+}
+
+func (ep *endpoint) Skip() bool {
+	return false
+}
+
+func (ep *endpoint) New() datastore.KVObject {
+	return &endpoint{}
+}
+
+func (ep *endpoint) CopyTo(o datastore.KVObject) error {
+	dstEp := o.(*endpoint)
+	*dstEp = *ep
+	return nil
+}
+
+func (ep *endpoint) DataScope() string {
+	return datastore.LocalScope
+}

libnetwork/drivers/macvlan/macvlan.go

@@ -38,11 +38,14 @@ type driver struct {
 }
 
 type endpoint struct {
-	id      string
-	mac     net.HardwareAddr
-	addr    *net.IPNet
-	addrv6  *net.IPNet
-	srcName string
+	id       string
+	nid      string
+	mac      net.HardwareAddr
+	addr     *net.IPNet
+	addrv6   *net.IPNet
+	srcName  string
+	dbIndex  uint64
+	dbExists bool
 }
 
 type network struct {

libnetwork/drivers/macvlan/macvlan_endpoint.go

@@ -26,6 +26,7 @@ func (d *driver) CreateEndpoint(nid, eid string, ifInfo driverapi.InterfaceInfo,
 	}
 	ep := &endpoint{
 		id:     eid,
+		nid:    nid,
 		addr:   ifInfo.Address(),
 		addrv6: ifInfo.AddressIPv6(),
 		mac:    ifInfo.MacAddress(),
@@ -55,6 +56,11 @@ func (d *driver) CreateEndpoint(nid, eid string, ifInfo driverapi.InterfaceInfo,
 			}
 		}
 	}
+
+	if err := d.storeUpdate(ep); err != nil {
+		return fmt.Errorf("failed to save macvlan endpoint %s to store: %v", ep.id[0:7], err)
+	}
+
 	n.addEndpoint(ep)
 
 	return nil
@@ -77,6 +83,8 @@ func (d *driver) DeleteEndpoint(nid, eid string) error {
 	if link, err := ns.NlHandle().LinkByName(ep.srcName); err == nil {
 		ns.NlHandle().LinkDel(link)
 	}
-
+	if err := d.storeDelete(ep); err != nil {
+		logrus.Warnf("Failed to remove macvlan endpoint %s from store: %v", ep.id[0:7], err)
+	}
 	return nil
 }

libnetwork/drivers/macvlan/macvlan_joinleave.go

@@ -77,7 +77,9 @@ func (d *driver) Join(nid, eid string, sboxKey string, jinfo driverapi.JoinInfo,
 	if err != nil {
 		return err
 	}
-
+	if err := d.storeUpdate(ep); err != nil {
+		return fmt.Errorf("failed to save macvlan endpoint %s to store: %v", ep.id[0:7], err)
+	}
 	return nil
 }
 

libnetwork/drivers/macvlan/macvlan_store.go

@@ -3,6 +3,7 @@ package macvlan
 import (
 	"encoding/json"
 	"fmt"
+	"net"
 
 	"github.com/Sirupsen/logrus"
 	"github.com/docker/libnetwork/datastore"
@@ -11,7 +12,11 @@ import (
 	"github.com/docker/libnetwork/types"
 )
 
-const macvlanPrefix = "macvlan" // prefix used for persistent driver storage
+const (
+	macvlanPrefix         = "macvlan"
+	macvlanNetworkPrefix  = macvlanPrefix + "/network"
+	macvlanEndpointPrefix = macvlanPrefix + "/endpoint"
+)
 
 // networkConfiguration for this driver's network specific configuration
 type configuration struct {
@@ -76,6 +81,34 @@ func (d *driver) populateNetworks() error {
 	return nil
 }
 
+func (d *driver) populateEndpoints() error {
+	kvol, err := d.store.List(datastore.Key(macvlanEndpointPrefix), &endpoint{})
+	if err != nil && err != datastore.ErrKeyNotFound {
+		return fmt.Errorf("failed to get macvlan endpoints from store: %v", err)
+	}
+
+	if err == datastore.ErrKeyNotFound {
+		return nil
+	}
+
+	for _, kvo := range kvol {
+		ep := kvo.(*endpoint)
+		n, ok := d.networks[ep.nid]
+		if !ok {
+			logrus.Debugf("Network (%s) not found for restored macvlan endpoint (%s)", ep.nid[0:7], ep.id[0:7])
+			logrus.Debugf("Deleting stale macvlan endpoint (%s) from store", ep.nid[0:7])
+			if err := d.storeDelete(ep); err != nil {
+				logrus.Debugf("Failed to delete stale macvlan endpoint (%s) from store", ep.nid[0:7])
+			}
+			continue
+		}
+		n.endpoints[ep.id] = ep
+		logrus.Debugf("Endpoint (%s) restored to network (%s)", ep.id[0:7], ep.nid[0:7])
+	}
+
+	return nil
+}
+
 // storeUpdate used to update persistent macvlan network records as they are created
 func (d *driver) storeUpdate(kvObject datastore.KVObject) error {
 	if d.store == nil {
@@ -165,11 +198,11 @@ func (config *configuration) UnmarshalJSON(b []byte) error {
 }
 
 func (config *configuration) Key() []string {
-	return []string{macvlanPrefix, config.ID}
+	return []string{macvlanNetworkPrefix, config.ID}
 }
 
 func (config *configuration) KeyPrefix() []string {
-	return []string{macvlanPrefix}
+	return []string{macvlanNetworkPrefix}
 }
 
 func (config *configuration) Value() []byte {
@@ -216,3 +249,103 @@ func (config *configuration) CopyTo(o datastore.KVObject) error {
 func (config *configuration) DataScope() string {
 	return datastore.LocalScope
 }
+
+func (ep *endpoint) MarshalJSON() ([]byte, error) {
+	epMap := make(map[string]interface{})
+	epMap["id"] = ep.id
+	epMap["nid"] = ep.nid
+	epMap["SrcName"] = ep.srcName
+	if len(ep.mac) != 0 {
+		epMap["MacAddress"] = ep.mac.String()
+	}
+	if ep.addr != nil {
+		epMap["Addr"] = ep.addr.String()
+	}
+	if ep.addrv6 != nil {
+		epMap["Addrv6"] = ep.addrv6.String()
+	}
+	return json.Marshal(epMap)
+}
+
+func (ep *endpoint) UnmarshalJSON(b []byte) error {
+	var (
+		err   error
+		epMap map[string]interface{}
+	)
+
+	if err = json.Unmarshal(b, &epMap); err != nil {
+		return fmt.Errorf("Failed to unmarshal to macvlan endpoint: %v", err)
+	}
+
+	if v, ok := epMap["MacAddress"]; ok {
+		if ep.mac, err = net.ParseMAC(v.(string)); err != nil {
+			return types.InternalErrorf("failed to decode macvlan endpoint MAC address (%s) after json unmarshal: %v", v.(string), err)
+		}
+	}
+	if v, ok := epMap["Addr"]; ok {
+		if ep.addr, err = types.ParseCIDR(v.(string)); err != nil {
+			return types.InternalErrorf("failed to decode macvlan endpoint IPv4 address (%s) after json unmarshal: %v", v.(string), err)
+		}
+	}
+	if v, ok := epMap["Addrv6"]; ok {
+		if ep.addrv6, err = types.ParseCIDR(v.(string)); err != nil {
+			return types.InternalErrorf("failed to decode macvlan endpoint IPv6 address (%s) after json unmarshal: %v", v.(string), err)
+		}
+	}
+	ep.id = epMap["id"].(string)
+	ep.nid = epMap["nid"].(string)
+	ep.srcName = epMap["SrcName"].(string)
+
+	return nil
+}
+
+func (ep *endpoint) Key() []string {
+	return []string{macvlanEndpointPrefix, ep.id}
+}
+
+func (ep *endpoint) KeyPrefix() []string {
+	return []string{macvlanEndpointPrefix}
+}
+
+func (ep *endpoint) Value() []byte {
+	b, err := json.Marshal(ep)
+	if err != nil {
+		return nil
+	}
+	return b
+}
+
+func (ep *endpoint) SetValue(value []byte) error {
+	return json.Unmarshal(value, ep)
+}
+
+func (ep *endpoint) Index() uint64 {
+	return ep.dbIndex
+}
+
+func (ep *endpoint) SetIndex(index uint64) {
+	ep.dbIndex = index
+	ep.dbExists = true
+}
+
+func (ep *endpoint) Exists() bool {
+	return ep.dbExists
+}
+
+func (ep *endpoint) Skip() bool {
+	return false
+}
+
+func (ep *endpoint) New() datastore.KVObject {
+	return &endpoint{}
+}
+
+func (ep *endpoint) CopyTo(o datastore.KVObject) error {
+	dstEp := o.(*endpoint)
+	*dstEp = *ep
+	return nil
+}
+
+func (ep *endpoint) DataScope() string {
+	return datastore.LocalScope
+}

libnetwork/drivers/overlay/encryption.go

@@ -10,6 +10,7 @@ import (
 
 	log "github.com/Sirupsen/logrus"
 	"github.com/docker/libnetwork/iptables"
+	"github.com/docker/libnetwork/ns"
 	"github.com/docker/libnetwork/types"
 	"github.com/vishvananda/netlink"
 	"strconv"
@@ -214,12 +215,12 @@ func programSA(localIP, remoteIP net.IP, spi *spi, k *key, dir int, add bool) (f
 	var (
 		crypt       *netlink.XfrmStateAlgo
 		action      = "Removing"
-		xfrmProgram = netlink.XfrmStateDel
+		xfrmProgram = ns.NlHandle().XfrmStateDel
 	)
 
 	if add {
 		action = "Adding"
-		xfrmProgram = netlink.XfrmStateAdd
+		xfrmProgram = ns.NlHandle().XfrmStateAdd
 		crypt = &netlink.XfrmStateAlgo{Name: "cbc(aes)", Key: k.value}
 	}
 
@@ -278,10 +279,10 @@ func programSA(localIP, remoteIP net.IP, spi *spi, k *key, dir int, add bool) (f
 
 func programSP(fSA *netlink.XfrmState, rSA *netlink.XfrmState, add bool) error {
 	action := "Removing"
-	xfrmProgram := netlink.XfrmPolicyDel
+	xfrmProgram := ns.NlHandle().XfrmPolicyDel
 	if add {
 		action = "Adding"
-		xfrmProgram = netlink.XfrmPolicyAdd
+		xfrmProgram = ns.NlHandle().XfrmPolicyAdd
 	}
 
 	fullMask := net.CIDRMask(8*len(fSA.Src), 8*len(fSA.Src))
@@ -322,7 +323,7 @@ func programSP(fSA *netlink.XfrmState, rSA *netlink.XfrmState, add bool) error {
 }
 
 func saExists(sa *netlink.XfrmState) (bool, error) {
-	_, err := netlink.XfrmStateGet(sa)
+	_, err := ns.NlHandle().XfrmStateGet(sa)
 	switch err {
 	case nil:
 		return true, nil
@@ -336,7 +337,7 @@ func saExists(sa *netlink.XfrmState) (bool, error) {
 }
 
 func spExists(sp *netlink.XfrmPolicy) (bool, error) {
-	_, err := netlink.XfrmPolicyGet(sp)
+	_, err := ns.NlHandle().XfrmPolicyGet(sp)
 	switch err {
 	case nil:
 		return true, nil
@@ -482,7 +483,7 @@ func updateNodeKey(lIP, rIP net.IP, idxs []*spi, curKeys []*key, newIdx, priIdx,
 			Limits: netlink.XfrmStateLimits{TimeSoft: timeout},
 		}
 		log.Infof("Updating rSA0{%s}", rSA0)
-		if err := netlink.XfrmStateUpdate(rSA0); err != nil {
+		if err := ns.NlHandle().XfrmStateUpdate(rSA0); err != nil {
 			log.Warnf("Failed to update rSA0{%s}: %v", rSA0, err)
 		}
 	}
@@ -518,7 +519,7 @@ func updateNodeKey(lIP, rIP net.IP, idxs []*spi, curKeys []*key, newIdx, priIdx,
 			},
 		}
 		log.Infof("Updating fSP{%s}", fSP1)
-		if err := netlink.XfrmPolicyUpdate(fSP1); err != nil {
+		if err := ns.NlHandle().XfrmPolicyUpdate(fSP1); err != nil {
 			log.Warnf("Failed to update fSP{%s}: %v", fSP1, err)
 		}
 
@@ -533,7 +534,7 @@ func updateNodeKey(lIP, rIP net.IP, idxs []*spi, curKeys []*key, newIdx, priIdx,
 			Limits: netlink.XfrmStateLimits{TimeHard: timeout},
 		}
 		log.Infof("Removing fSA0{%s}", fSA0)
-		if err := netlink.XfrmStateUpdate(fSA0); err != nil {
+		if err := ns.NlHandle().XfrmStateUpdate(fSA0); err != nil {
 			log.Warnf("Failed to remove fSA0{%s}: %v", fSA0, err)
 		}
 	}

libnetwork/drivers/overlay/joinleave.go

@@ -40,11 +40,11 @@ func (d *driver) Join(nid, eid string, sboxKey string, jinfo driverapi.JoinInfo,
 		return fmt.Errorf("couldn't get vxlan id for %q: %v", s.subnetIP.String(), err)
 	}
 
-	if err := n.joinSandbox(); err != nil {
+	if err := n.joinSandbox(false); err != nil {
 		return fmt.Errorf("network sandbox join failed: %v", err)
 	}
 
-	if err := n.joinSubnetSandbox(s); err != nil {
+	if err := n.joinSubnetSandbox(s, false); err != nil {
 		return fmt.Errorf("subnet sandbox join failed for %q: %v", s.subnetIP.String(), err)
 	}
 
@@ -61,6 +61,10 @@ func (d *driver) Join(nid, eid string, sboxKey string, jinfo driverapi.JoinInfo,
 
 	ep.ifName = containerIfName
 
+	if err := d.writeEndpointToStore(ep); err != nil {
+		return fmt.Errorf("failed to update overlay endpoint %s to local data store: %v", ep.id[0:7], err)
+	}
+
 	nlh := ns.NlHandle()
 
 	// Set the container interface and its peer MTU to 1450 to allow

libnetwork/drivers/overlay/ov_endpoint.go

@@ -1,22 +1,30 @@
 package overlay
 
 import (
+	"encoding/json"
 	"fmt"
 	"net"
 
 	log "github.com/Sirupsen/logrus"
+	"github.com/docker/libnetwork/datastore"
 	"github.com/docker/libnetwork/driverapi"
 	"github.com/docker/libnetwork/netutils"
 	"github.com/docker/libnetwork/ns"
+	"github.com/docker/libnetwork/types"
 )
 
 type endpointTable map[string]*endpoint
 
+const overlayEndpointPrefix = "overlay/endpoint"
+
 type endpoint struct {
-	id     string
-	ifName string
-	mac    net.HardwareAddr
-	addr   *net.IPNet
+	id       string
+	nid      string
+	ifName   string
+	mac      net.HardwareAddr
+	addr     *net.IPNet
+	dbExists bool
+	dbIndex  uint64
 }
 
 func (n *network) endpoint(eid string) *endpoint {
@@ -60,6 +68,7 @@ func (d *driver) CreateEndpoint(nid, eid string, ifInfo driverapi.InterfaceInfo,
 
 	ep := &endpoint{
 		id:   eid,
+		nid:  n.id,
 		addr: ifInfo.Address(),
 		mac:  ifInfo.MacAddress(),
 	}
@@ -80,6 +89,10 @@ func (d *driver) CreateEndpoint(nid, eid string, ifInfo driverapi.InterfaceInfo,
 
 	n.addEndpoint(ep)
 
+	if err := d.writeEndpointToStore(ep); err != nil {
+		return fmt.Errorf("failed to update overlay endpoint %s to local store: %v", ep.id[0:7], err)
+	}
+
 	return nil
 }
 
@@ -102,6 +115,10 @@ func (d *driver) DeleteEndpoint(nid, eid string) error {
 
 	n.deleteEndpoint(eid)
 
+	if err := d.deleteEndpointFromStore(ep); err != nil {
+		log.Warnf("Failed to delete overlay endpoint %s from local store: %v", ep.id[0:7], err)
+	}
+
 	if ep.ifName == "" {
 		return nil
 	}
@@ -121,3 +138,122 @@ func (d *driver) DeleteEndpoint(nid, eid string) error {
 func (d *driver) EndpointOperInfo(nid, eid string) (map[string]interface{}, error) {
 	return make(map[string]interface{}, 0), nil
 }
+
+func (d *driver) deleteEndpointFromStore(e *endpoint) error {
+	if d.localStore == nil {
+		return fmt.Errorf("overlay local store not initialized, ep not deleted")
+	}
+
+	if err := d.localStore.DeleteObjectAtomic(e); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (d *driver) writeEndpointToStore(e *endpoint) error {
+	if d.localStore == nil {
+		return fmt.Errorf("overlay local store not initialized, ep not added")
+	}
+
+	if err := d.localStore.PutObjectAtomic(e); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (ep *endpoint) DataScope() string {
+	return datastore.LocalScope
+}
+
+func (ep *endpoint) New() datastore.KVObject {
+	return &endpoint{}
+}
+
+func (ep *endpoint) CopyTo(o datastore.KVObject) error {
+	dstep := o.(*endpoint)
+	*dstep = *ep
+	return nil
+}
+
+func (ep *endpoint) Key() []string {
+	return []string{overlayEndpointPrefix, ep.id}
+}
+
+func (ep *endpoint) KeyPrefix() []string {
+	return []string{overlayEndpointPrefix}
+}
+
+func (ep *endpoint) Index() uint64 {
+	return ep.dbIndex
+}
+
+func (ep *endpoint) SetIndex(index uint64) {
+	ep.dbIndex = index
+	ep.dbExists = true
+}
+
+func (ep *endpoint) Exists() bool {
+	return ep.dbExists
+}
+
+func (ep *endpoint) Skip() bool {
+	return false
+}
+
+func (ep *endpoint) Value() []byte {
+	b, err := json.Marshal(ep)
+	if err != nil {
+		return nil
+	}
+	return b
+}
+
+func (ep *endpoint) SetValue(value []byte) error {
+	return json.Unmarshal(value, ep)
+}
+
+func (ep *endpoint) MarshalJSON() ([]byte, error) {
+	epMap := make(map[string]interface{})
+
+	epMap["id"] = ep.id
+	epMap["nid"] = ep.nid
+	if ep.ifName != "" {
+		epMap["ifName"] = ep.ifName
+	}
+	if ep.addr != nil {
+		epMap["addr"] = ep.addr.String()
+	}
+	if len(ep.mac) != 0 {
+		epMap["mac"] = ep.mac.String()
+	}
+
+	return json.Marshal(epMap)
+}
+
+func (ep *endpoint) UnmarshalJSON(value []byte) error {
+	var (
+		err   error
+		epMap map[string]interface{}
+	)
+
+	json.Unmarshal(value, &epMap)
+
+	ep.id = epMap["id"].(string)
+	ep.nid = epMap["nid"].(string)
+	if v, ok := epMap["mac"]; ok {
+		if ep.mac, err = net.ParseMAC(v.(string)); err != nil {
+			return types.InternalErrorf("failed to decode endpoint interface mac address after json unmarshal: %s", v.(string))
+		}
+	}
+	if v, ok := epMap["addr"]; ok {
+		if ep.addr, err = types.ParseCIDR(v.(string)); err != nil {
+			return types.InternalErrorf("failed to decode endpoint interface ipv4 address after json unmarshal: %v", err)
+		}
+	}
+	if v, ok := epMap["ifName"]; ok {
+		ep.ifName = v.(string)
+	}
+
+	return nil
+}

libnetwork/drivers/overlay/ov_network.go

@@ -195,21 +195,21 @@ func (n *network) incEndpointCount() {
 	n.joinCnt++
 }
 
-func (n *network) joinSandbox() error {
+func (n *network) joinSandbox(restore bool) error {
 	// If there is a race between two go routines here only one will win
 	// the other will wait.
 	n.once.Do(func() {
 		// save the error status of initSandbox in n.initErr so that
 		// all the racing go routines are able to know the status.
-		n.initErr = n.initSandbox()
+		n.initErr = n.initSandbox(restore)
 	})
 
 	return n.initErr
 }
 
-func (n *network) joinSubnetSandbox(s *subnet) error {
+func (n *network) joinSubnetSandbox(s *subnet, restore bool) error {
 	s.once.Do(func() {
-		s.initErr = n.initSubnetSandbox(s)
+		s.initErr = n.initSubnetSandbox(s, restore)
 	})
 	return s.initErr
 }
@@ -386,9 +386,33 @@ func isOverlap(nw *net.IPNet) bool {
 	return false
 }
 
-func (n *network) initSubnetSandbox(s *subnet) error {
-	brName := n.generateBridgeName(s)
-	vxlanName := n.generateVxlanName(s)
+func (n *network) restoreSubnetSandbox(s *subnet, brName, vxlanName string) error {
+	sbox := n.sandbox()
+
+	// restore overlay osl sandbox
+	Ifaces := make(map[string][]osl.IfaceOption)
+	brIfaceOption := make([]osl.IfaceOption, 2)
+	brIfaceOption = append(brIfaceOption, sbox.InterfaceOptions().Address(s.gwIP))
+	brIfaceOption = append(brIfaceOption, sbox.InterfaceOptions().Bridge(true))
+	Ifaces[fmt.Sprintf("%s+%s", brName, "br")] = brIfaceOption
+
+	err := sbox.Restore(Ifaces, nil, nil, nil)
+	if err != nil {
+		return err
+	}
+
+	Ifaces = make(map[string][]osl.IfaceOption)
+	vxlanIfaceOption := make([]osl.IfaceOption, 1)
+	vxlanIfaceOption = append(vxlanIfaceOption, sbox.InterfaceOptions().Master(brName))
+	Ifaces[fmt.Sprintf("%s+%s", vxlanName, "vxlan")] = vxlanIfaceOption
+	err = sbox.Restore(Ifaces, nil, nil, nil)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (n *network) setupSubnetSandbox(s *subnet, brName, vxlanName string) error {
 
 	if hostMode {
 		// Try to delete stale bridge interface if it exists
@@ -451,6 +475,19 @@ func (n *network) initSubnetSandbox(s *subnet) error {
 		}
 	}
 
+	return nil
+}
+
+func (n *network) initSubnetSandbox(s *subnet, restore bool) error {
+	brName := n.generateBridgeName(s)
+	vxlanName := n.generateVxlanName(s)
+
+	if restore {
+		n.restoreSubnetSandbox(s, brName, vxlanName)
+	} else {
+		n.setupSubnetSandbox(s, brName, vxlanName)
+	}
+
 	n.Lock()
 	s.vxlanName = vxlanName
 	s.brName = brName
@@ -494,32 +531,45 @@ func (n *network) cleanupStaleSandboxes() {
 		})
 }
 
-func (n *network) initSandbox() error {
+func (n *network) initSandbox(restore bool) error {
 	n.Lock()
 	n.initEpoch++
 	n.Unlock()
 
 	networkOnce.Do(networkOnceInit)
 
-	if hostMode {
-		if err := addNetworkChain(n.id[:12]); err != nil {
-			return err
+	if !restore {
+		if hostMode {
+			if err := addNetworkChain(n.id[:12]); err != nil {
+				return err
+			}
 		}
+
+		// If there are any stale sandboxes related to this network
+		// from previous daemon life clean it up here
+		n.cleanupStaleSandboxes()
 	}
 
-	// If there are any stale sandboxes related to this network
-	// from previous daemon life clean it up here
-	n.cleanupStaleSandboxes()
+	// In the restore case network sandbox already exist; but we don't know
+	// what epoch number it was created with. It has to be retrieved by
+	// searching the net namespaces.
+	key := ""
+	if restore {
+		key = osl.GenerateKey("-" + n.id)
+	} else {
+		key = osl.GenerateKey(fmt.Sprintf("%d-", n.initEpoch) + n.id)
+	}
 
-	sbox, err := osl.NewSandbox(
-		osl.GenerateKey(fmt.Sprintf("%d-", n.initEpoch)+n.id), !hostMode)
+	sbox, err := osl.NewSandbox(key, !hostMode, restore)
 	if err != nil {
-		return fmt.Errorf("could not create network sandbox: %v", err)
+		return fmt.Errorf("could not get network sandbox (oper %t): %v", restore, err)
 	}
 
 	n.setSandbox(sbox)
 
-	n.driver.peerDbUpdateSandbox(n.id)
+	if !restore {
+		n.driver.peerDbUpdateSandbox(n.id)
+	}
 
 	var nlSock *nl.NetlinkSocket
 	sbox.InvokeFunc(func() {

libnetwork/drivers/overlay/overlay.go

@@ -13,6 +13,7 @@ import (
 	"github.com/docker/libnetwork/driverapi"
 	"github.com/docker/libnetwork/idm"
 	"github.com/docker/libnetwork/netlabel"
+	"github.com/docker/libnetwork/osl"
 	"github.com/docker/libnetwork/types"
 	"github.com/hashicorp/serf/serf"
 )
@@ -41,6 +42,7 @@ type driver struct {
 	serfInstance *serf.Serf
 	networks     networkTable
 	store        datastore.DataStore
+	localStore   datastore.DataStore
 	vxlanIdm     *idm.Idm
 	once         sync.Once
 	joinOnce     sync.Once
@@ -74,9 +76,75 @@ func Init(dc driverapi.DriverCallback, config map[string]interface{}) error {
 		}
 	}
 
+	if data, ok := config[netlabel.LocalKVClient]; ok {
+		var err error
+		dsc, ok := data.(discoverapi.DatastoreConfigData)
+		if !ok {
+			return types.InternalErrorf("incorrect data in datastore configuration: %v", data)
+		}
+		d.localStore, err = datastore.NewDataStoreFromConfig(dsc)
+		if err != nil {
+			return types.InternalErrorf("failed to initialize local data store: %v", err)
+		}
+	}
+
+	d.restoreEndpoints()
+
 	return dc.RegisterDriver(networkType, d, c)
 }
 
+// Endpoints are stored in the local store. Restore them and reconstruct the overlay sandbox
+func (d *driver) restoreEndpoints() error {
+	if d.localStore == nil {
+		logrus.Warnf("Cannot restore overlay endpoints because local datastore is missing")
+		return nil
+	}
+	kvol, err := d.localStore.List(datastore.Key(overlayEndpointPrefix), &endpoint{})
+	if err != nil && err != datastore.ErrKeyNotFound {
+		return fmt.Errorf("failed to read overlay endpoint from store: %v", err)
+	}
+
+	if err == datastore.ErrKeyNotFound {
+		return nil
+	}
+	for _, kvo := range kvol {
+		ep := kvo.(*endpoint)
+		n := d.network(ep.nid)
+		if n == nil {
+			logrus.Debugf("Network (%s) not found for restored endpoint (%s)", ep.nid, ep.id)
+			continue
+		}
+		n.addEndpoint(ep)
+
+		s := n.getSubnetforIP(ep.addr)
+		if s == nil {
+			return fmt.Errorf("could not find subnet for endpoint %s", ep.id)
+		}
+
+		if err := n.joinSandbox(true); err != nil {
+			return fmt.Errorf("restore network sandbox failed: %v", err)
+		}
+
+		if err := n.joinSubnetSandbox(s, true); err != nil {
+			return fmt.Errorf("restore subnet sandbox failed for %q: %v", s.subnetIP.String(), err)
+		}
+
+		Ifaces := make(map[string][]osl.IfaceOption)
+		vethIfaceOption := make([]osl.IfaceOption, 1)
+		vethIfaceOption = append(vethIfaceOption, n.sbox.InterfaceOptions().Master(s.brName))
+		Ifaces[fmt.Sprintf("%s+%s", "veth", "veth")] = vethIfaceOption
+
+		err := n.sbox.Restore(Ifaces, nil, nil, nil)
+		if err != nil {
+			return fmt.Errorf("failed to restore overlay sandbox: %v", err)
+		}
+
+		n.incEndpointCount()
+		d.peerDbAdd(ep.nid, ep.id, ep.addr.IP, ep.addr.Mask, ep.mac, net.ParseIP(d.bindAddress), true)
+	}
+	return nil
+}
+
 // Fini cleans up the driver resources
 func Fini(drv driverapi.Driver) {
 	d := drv.(*driver)

libnetwork/drivers/overlay/peerdb.go

@@ -271,7 +271,7 @@ func (d *driver) peerAdd(nid, eid string, peerIP net.IP, peerIPMask net.IPMask,
 		return fmt.Errorf("couldn't get vxlan id for %q: %v", s.subnetIP.String(), err)
 	}
 
-	if err := n.joinSubnetSandbox(s); err != nil {
+	if err := n.joinSubnetSandbox(s, false); err != nil {
 		return fmt.Errorf("subnet sandbox join failed for %q: %v", s.subnetIP.String(), err)
 	}
 

libnetwork/endpoint.go

@@ -84,6 +84,7 @@ func (ep *endpoint) MarshalJSON() ([]byte, error) {
 	epMap["name"] = ep.name
 	epMap["id"] = ep.id
 	epMap["ep_iface"] = ep.iface
+	epMap["joinInfo"] = ep.joinInfo
 	epMap["exposed_ports"] = ep.exposedPorts
 	if ep.generic != nil {
 		epMap["generic"] = ep.generic
@@ -115,6 +116,9 @@ func (ep *endpoint) UnmarshalJSON(b []byte) (err error) {
 	ib, _ := json.Marshal(epMap["ep_iface"])
 	json.Unmarshal(ib, &ep.iface)
 
+	jb, _ := json.Marshal(epMap["joinInfo"])
+	json.Unmarshal(jb, &ep.joinInfo)
+
 	tb, _ := json.Marshal(epMap["exposed_ports"])
 	var tPorts []types.TransportPort
 	json.Unmarshal(tb, &tPorts)
@@ -235,6 +239,11 @@ func (ep *endpoint) CopyTo(o datastore.KVObject) error {
 		ep.iface.CopyTo(dstEp.iface)
 	}
 
+	if ep.joinInfo != nil {
+		dstEp.joinInfo = &endpointJoinInfo{}
+		ep.joinInfo.CopyTo(dstEp.joinInfo)
+	}
+
 	dstEp.exposedPorts = make([]types.TransportPort, len(ep.exposedPorts))
 	copy(dstEp.exposedPorts, ep.exposedPorts)
 
@@ -1073,6 +1082,13 @@ func (ep *endpoint) releaseAddress() {
 }
 
 func (c *controller) cleanupLocalEndpoints() {
+	// Get used endpoints
+	eps := make(map[string]interface{})
+	for _, sb := range c.sandboxes {
+		for _, ep := range sb.endpoints {
+			eps[ep.id] = true
+		}
+	}
 	nl, err := c.getNetworksForScope(datastore.LocalScope)
 	if err != nil {
 		log.Warnf("Could not get list of networks during endpoint cleanup: %v", err)
@@ -1087,6 +1103,9 @@ func (c *controller) cleanupLocalEndpoints() {
 		}
 
 		for _, ep := range epl {
+			if _, ok := eps[ep.id]; ok {
+				continue
+			}
 			log.Infof("Removing stale endpoint %s (%s)", ep.name, ep.id)
 			if err := ep.Delete(true); err != nil {
 				log.Warnf("Could not delete local endpoint %s during endpoint cleanup: %v", ep.name, err)

libnetwork/endpoint_info.go

@@ -414,3 +414,56 @@ func (ep *endpoint) DisableGatewayService() {
 
 	ep.joinInfo.disableGatewayService = true
 }
+
+func (epj *endpointJoinInfo) MarshalJSON() ([]byte, error) {
+	epMap := make(map[string]interface{})
+	if epj.gw != nil {
+		epMap["gw"] = epj.gw.String()
+	}
+	if epj.gw6 != nil {
+		epMap["gw6"] = epj.gw6.String()
+	}
+	epMap["disableGatewayService"] = epj.disableGatewayService
+	epMap["StaticRoutes"] = epj.StaticRoutes
+	return json.Marshal(epMap)
+}
+
+func (epj *endpointJoinInfo) UnmarshalJSON(b []byte) error {
+	var (
+		err   error
+		epMap map[string]interface{}
+	)
+	if err = json.Unmarshal(b, &epMap); err != nil {
+		return err
+	}
+	if v, ok := epMap["gw"]; ok {
+		epj.gw6 = net.ParseIP(v.(string))
+	}
+	if v, ok := epMap["gw6"]; ok {
+		epj.gw6 = net.ParseIP(v.(string))
+	}
+	epj.disableGatewayService = epMap["disableGatewayService"].(bool)
+
+	var tStaticRoute []types.StaticRoute
+	if v, ok := epMap["StaticRoutes"]; ok {
+		tb, _ := json.Marshal(v)
+		var tStaticRoute []types.StaticRoute
+		json.Unmarshal(tb, &tStaticRoute)
+	}
+	var StaticRoutes []*types.StaticRoute
+	for _, r := range tStaticRoute {
+		StaticRoutes = append(StaticRoutes, &r)
+	}
+	epj.StaticRoutes = StaticRoutes
+
+	return nil
+}
+
+func (epj *endpointJoinInfo) CopyTo(dstEpj *endpointJoinInfo) error {
+	dstEpj.disableGatewayService = epj.disableGatewayService
+	dstEpj.StaticRoutes = make([]*types.StaticRoute, len(epj.StaticRoutes))
+	copy(dstEpj.StaticRoutes, epj.StaticRoutes)
+	dstEpj.gw = types.GetIPCopy(epj.gw)
+	dstEpj.gw = types.GetIPCopy(epj.gw6)
+	return nil
+}

libnetwork/libnetwork_test.go

@@ -1305,7 +1305,7 @@ func externalKeyTest(t *testing.T, reexec bool) {
 	}
 
 	// Create a new OS sandbox using the osl API before using it in SetKey
-	if extOsBox, err := osl.NewSandbox("ValidKey", true); err != nil {
+	if extOsBox, err := osl.NewSandbox("ValidKey", true, false); err != nil {
 		t.Fatalf("Failed to create new osl sandbox")
 	} else {
 		defer func() {

libnetwork/osl/namespace_linux.go

@@ -2,10 +2,13 @@ package osl
 
 import (
 	"fmt"
+	"io/ioutil"
 	"net"
 	"os"
 	"os/exec"
 	"runtime"
+	"strconv"
+	"strings"
 	"sync"
 	"syscall"
 	"time"
@@ -133,6 +136,39 @@ func GC() {
 // container id.
 func GenerateKey(containerID string) string {
 	maxLen := 12
+	// Read sandbox key from host for overlay
+	if strings.HasPrefix(containerID, "-") {
+		var (
+			index    int
+			indexStr string
+			tmpkey   string
+		)
+		dir, err := ioutil.ReadDir(prefix)
+		if err != nil {
+			return ""
+		}
+
+		for _, v := range dir {
+			id := v.Name()
+			if strings.HasSuffix(id, containerID[:maxLen-1]) {
+				indexStr = strings.TrimSuffix(id, containerID[:maxLen-1])
+				tmpindex, err := strconv.Atoi(indexStr)
+				if err != nil {
+					return ""
+				}
+				if tmpindex > index {
+					index = tmpindex
+					tmpkey = id
+				}
+
+			}
+		}
+		containerID = tmpkey
+		if containerID == "" {
+			return ""
+		}
+	}
+
 	if len(containerID) < maxLen {
 		maxLen = len(containerID)
 	}
@@ -142,10 +178,14 @@ func GenerateKey(containerID string) string {
 
 // NewSandbox provides a new sandbox instance created in an os specific way
 // provided a key which uniquely identifies the sandbox
-func NewSandbox(key string, osCreate bool) (Sandbox, error) {
-	err := createNetworkNamespace(key, osCreate)
-	if err != nil {
-		return nil, err
+func NewSandbox(key string, osCreate, isRestore bool) (Sandbox, error) {
+	if !isRestore {
+		err := createNetworkNamespace(key, osCreate)
+		if err != nil {
+			return nil, err
+		}
+	} else {
+		once.Do(createBasePath)
 	}
 
 	n := &networkNamespace{path: key, isDefault: !osCreate}
@@ -347,3 +387,105 @@ func (n *networkNamespace) Destroy() error {
 	addToGarbagePaths(n.path)
 	return nil
 }
+
+// Restore restore the network namespace
+func (n *networkNamespace) Restore(ifsopt map[string][]IfaceOption, routes []*types.StaticRoute, gw net.IP, gw6 net.IP) error {
+	// restore interfaces
+	for name, opts := range ifsopt {
+		if !strings.Contains(name, "+") {
+			return fmt.Errorf("wrong iface name in restore osl sandbox interface: %s", name)
+		}
+		seps := strings.Split(name, "+")
+		srcName := seps[0]
+		dstPrefix := seps[1]
+		i := &nwIface{srcName: srcName, dstName: dstPrefix, ns: n}
+		i.processInterfaceOptions(opts...)
+		if i.master != "" {
+			i.dstMaster = n.findDst(i.master, true)
+			if i.dstMaster == "" {
+				return fmt.Errorf("could not find an appropriate master %q for %q",
+					i.master, i.srcName)
+			}
+		}
+		if n.isDefault {
+			i.dstName = i.srcName
+		} else {
+			links, err := n.nlHandle.LinkList()
+			if err != nil {
+				return fmt.Errorf("failed to retrieve list of links in network namespace %q during restore", n.path)
+			}
+			// due to the docker network connect/disconnect, so the dstName should
+			// restore from the namespace
+			for _, link := range links {
+				addrs, err := n.nlHandle.AddrList(link, netlink.FAMILY_V4)
+				if err != nil {
+					return err
+				}
+				ifaceName := link.Attrs().Name
+				if strings.HasPrefix(ifaceName, "vxlan") {
+					if i.dstName == "vxlan" {
+						i.dstName = ifaceName
+						break
+					}
+				}
+				// find the interface name by ip
+				if i.address != nil {
+					for _, addr := range addrs {
+						if addr.IPNet.String() == i.address.String() {
+							i.dstName = ifaceName
+							break
+						}
+						continue
+					}
+					if i.dstName == ifaceName {
+						break
+					}
+				}
+				// This is to find the interface name of the pair in overlay sandbox
+				if strings.HasPrefix(ifaceName, "veth") {
+					if i.master != "" && i.dstName == "veth" {
+						i.dstName = ifaceName
+					}
+				}
+			}
+
+			var index int
+			indexStr := strings.TrimPrefix(i.dstName, dstPrefix)
+			if indexStr != "" {
+				index, err = strconv.Atoi(indexStr)
+				if err != nil {
+					return err
+				}
+			}
+			index++
+			n.Lock()
+			if index > n.nextIfIndex {
+				n.nextIfIndex = index
+			}
+			n.iFaces = append(n.iFaces, i)
+			n.Unlock()
+		}
+	}
+
+	// restore routes
+	for _, r := range routes {
+		n.Lock()
+		n.staticRoutes = append(n.staticRoutes, r)
+		n.Unlock()
+	}
+
+	// restore gateway
+	if len(gw) > 0 {
+		n.Lock()
+		n.gw = gw
+		n.Unlock()
+	}
+
+	if len(gw6) > 0 {
+		n.Lock()
+		n.gwv6 = gw6
+		n.Unlock()
+	}
+
+	return nil
+}

libnetwork/osl/namespace_windows.go

@@ -15,7 +15,7 @@ func GenerateKey(containerID string) string {
 
 // NewSandbox provides a new sandbox instance created in an os specific way
 // provided a key which uniquely identifies the sandbox
-func NewSandbox(key string, osCreate bool) (Sandbox, error) {
+func NewSandbox(key string, osCreate, isRestore bool) (Sandbox, error) {
 	return nil, nil
 }
 

libnetwork/osl/sandbox.go

@@ -58,6 +58,9 @@ type Sandbox interface {
 
 	// Destroy the sandbox
 	Destroy() error
+
+	// restore sandbox
+	Restore(ifsopt map[string][]IfaceOption, routes []*types.StaticRoute, gw net.IP, gw6 net.IP) error
 }
 
 // NeighborOptionSetter interface defines the option setter methods for interface options

libnetwork/osl/sandbox_freebsd.go

@@ -15,7 +15,7 @@ func GenerateKey(containerID string) string {
 
 // NewSandbox provides a new sandbox instance created in an os specific way
 // provided a key which uniquely identifies the sandbox
-func NewSandbox(key string, osCreate bool) (Sandbox, error) {
+func NewSandbox(key string, osCreate, isRestore bool) (Sandbox, error) {
 	return nil, nil
 }
 

libnetwork/osl/sandbox_test.go

@@ -25,7 +25,7 @@ func TestSandboxCreate(t *testing.T) {
 		t.Fatalf("Failed to obtain a key: %v", err)
 	}
 
-	s, err := NewSandbox(key, true)
+	s, err := NewSandbox(key, true, false)
 	if err != nil {
 		t.Fatalf("Failed to create a new sandbox: %v", err)
 	}
@@ -77,7 +77,7 @@ func TestSandboxCreateTwice(t *testing.T) {
 		t.Fatalf("Failed to obtain a key: %v", err)
 	}
 
-	_, err = NewSandbox(key, true)
+	_, err = NewSandbox(key, true, false)
 	if err != nil {
 		t.Fatalf("Failed to create a new sandbox: %v", err)
 	}
@@ -85,7 +85,7 @@ func TestSandboxCreateTwice(t *testing.T) {
 
 	// Create another sandbox with the same key to see if we handle it
 	// gracefully.
-	s, err := NewSandbox(key, true)
+	s, err := NewSandbox(key, true, false)
 	if err != nil {
 		t.Fatalf("Failed to create a new sandbox: %v", err)
 	}
@@ -105,7 +105,7 @@ func TestSandboxGC(t *testing.T) {
 		t.Fatalf("Failed to obtain a key: %v", err)
 	}
 
-	s, err := NewSandbox(key, true)
+	s, err := NewSandbox(key, true, false)
 	if err != nil {
 		t.Fatalf("Failed to create a new sandbox: %v", err)
 	}
@@ -127,7 +127,7 @@ func TestAddRemoveInterface(t *testing.T) {
 		t.Fatalf("Failed to obtain a key: %v", err)
 	}
 
-	s, err := NewSandbox(key, true)
+	s, err := NewSandbox(key, true, false)
 	if err != nil {
 		t.Fatalf("Failed to create a new sandbox: %v", err)
 	}

libnetwork/osl/sandbox_unsupported.go

@@ -11,7 +11,7 @@ var (
 
 // NewSandbox provides a new sandbox instance created in an os specific way
 // provided a key which uniquely identifies the sandbox
-func NewSandbox(key string, osCreate bool) (Sandbox, error) {
+func NewSandbox(key string, osCreate, isRestore bool) (Sandbox, error) {
 	return nil, ErrNotImplemented
 }
 

libnetwork/resolver_unix.go

@@ -19,6 +19,13 @@ func init() {
 	reexec.Register("setup-resolver", reexecSetupResolver)
 }
 
+const (
+	// outputChain used for docker embed dns
+	outputChain = "DOCKER_OUTPUT"
+	//postroutingchain used for docker embed dns
+	postroutingchain = "DOCKER_POSTROUTING"
+)
+
 func reexecSetupResolver() {
 	runtime.LockOSThread()
 	defer runtime.UnlockOSThread()
@@ -31,10 +38,10 @@ func reexecSetupResolver() {
 	_, ipPort, _ := net.SplitHostPort(os.Args[2])
 	_, tcpPort, _ := net.SplitHostPort(os.Args[3])
 	rules := [][]string{
-		{"-t", "nat", "-A", "OUTPUT", "-d", resolverIP, "-p", "udp", "--dport", dnsPort, "-j", "DNAT", "--to-destination", os.Args[2]},
-		{"-t", "nat", "-A", "POSTROUTING", "-s", resolverIP, "-p", "udp", "--sport", ipPort, "-j", "SNAT", "--to-source", ":" + dnsPort},
-		{"-t", "nat", "-A", "OUTPUT", "-d", resolverIP, "-p", "tcp", "--dport", dnsPort, "-j", "DNAT", "--to-destination", os.Args[3]},
-		{"-t", "nat", "-A", "POSTROUTING", "-s", resolverIP, "-p", "tcp", "--sport", tcpPort, "-j", "SNAT", "--to-source", ":" + dnsPort},
+		{"-t", "nat", "-I", outputChain, "-d", resolverIP, "-p", "udp", "--dport", dnsPort, "-j", "DNAT", "--to-destination", os.Args[2]},
+		{"-t", "nat", "-I", postroutingchain, "-s", resolverIP, "-p", "udp", "--sport", ipPort, "-j", "SNAT", "--to-source", ":" + dnsPort},
+		{"-t", "nat", "-I", outputChain, "-d", resolverIP, "-p", "tcp", "--dport", dnsPort, "-j", "DNAT", "--to-destination", os.Args[3]},
+		{"-t", "nat", "-I", postroutingchain, "-s", resolverIP, "-p", "tcp", "--sport", tcpPort, "-j", "SNAT", "--to-source", ":" + dnsPort},
 	}
 
 	f, err := os.OpenFile(os.Args[1], os.O_RDONLY, 0)
@@ -50,6 +57,23 @@ func reexecSetupResolver() {
 		os.Exit(3)
 	}
 
+	// insert outputChain and postroutingchain
+	err = iptables.RawCombinedOutputNative("-t", "nat", "-C", "OUTPUT", "-d", resolverIP, "-j", outputChain)
+	if err == nil {
+		iptables.RawCombinedOutputNative("-t", "nat", "-F", outputChain)
+	} else {
+		iptables.RawCombinedOutputNative("-t", "nat", "-N", outputChain)
+		iptables.RawCombinedOutputNative("-t", "nat", "-I", "OUTPUT", "-d", resolverIP, "-j", outputChain)
+	}
+
+	err = iptables.RawCombinedOutputNative("-t", "nat", "-C", "POSTROUTING", "-d", resolverIP, "-j", postroutingchain)
+	if err == nil {
+		iptables.RawCombinedOutputNative("-t", "nat", "-F", postroutingchain)
+	} else {
+		iptables.RawCombinedOutputNative("-t", "nat", "-N", postroutingchain)
+		iptables.RawCombinedOutputNative("-t", "nat", "-I", "POSTROUTING", "-d", resolverIP, "-j", postroutingchain)
+	}
+
 	for _, rule := range rules {
 		if iptables.RawCombinedOutputNative(rule...) != nil {
 			log.Errorf("setting up rule failed, %v", rule)

libnetwork/sandbox.go

@@ -700,6 +700,52 @@ func (sb *sandbox) releaseOSSbox() {
 	osSbox.Destroy()
 }
 
+func (sb *sandbox) restoreOslSandbox() error {
+	var routes []*types.StaticRoute
+
+	// restore osl sandbox
+	Ifaces := make(map[string][]osl.IfaceOption)
+	for _, ep := range sb.endpoints {
+		var ifaceOptions []osl.IfaceOption
+		ep.Lock()
+		joinInfo := ep.joinInfo
+		i := ep.iface
+		ep.Unlock()
+		ifaceOptions = append(ifaceOptions, sb.osSbox.InterfaceOptions().Address(i.addr), sb.osSbox.InterfaceOptions().Routes(i.routes))
+		if i.addrv6 != nil && i.addrv6.IP.To16() != nil {
+			ifaceOptions = append(ifaceOptions, sb.osSbox.InterfaceOptions().AddressIPv6(i.addrv6))
+		}
+		if i.mac != nil {
+			ifaceOptions = append(ifaceOptions, sb.osSbox.InterfaceOptions().MacAddress(i.mac))
+		}
+		if len(i.llAddrs) != 0 {
+			ifaceOptions = append(ifaceOptions, sb.osSbox.InterfaceOptions().LinkLocalAddresses(i.llAddrs))
+		}
+		Ifaces[fmt.Sprintf("%s+%s", i.srcName, i.dstPrefix)] = ifaceOptions
+		if joinInfo != nil {
+			for _, r := range joinInfo.StaticRoutes {
+				routes = append(routes, r)
+			}
+		}
+		if ep.needResolver() {
+			sb.startResolver()
+		}
+	}
+
+	gwep := sb.getGatewayEndpoint()
+	if gwep == nil {
+		return nil
+	}
+
+	// restore osl sandbox
+	err := sb.osSbox.Restore(Ifaces, routes, gwep.joinInfo.gw, gwep.joinInfo.gw6)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
 func (sb *sandbox) populateNetworkResources(ep *endpoint) error {
 	sb.Lock()
 	if sb.osSbox == nil {

libnetwork/sandbox_dns_unix.go

@@ -139,6 +139,16 @@ func (sb *sandbox) updateParentHosts() error {
 	return nil
 }
 
+func (sb *sandbox) restorePath() {
+	if sb.config.resolvConfPath == "" {
+		sb.config.resolvConfPath = defaultPrefix + "/" + sb.id + "/resolv.conf"
+	}
+	sb.config.resolvConfHashFile = sb.config.resolvConfPath + ".hash"
+	if sb.config.hostsPath == "" {
+		sb.config.hostsPath = defaultPrefix + "/" + sb.id + "/hosts"
+	}
+}
+
 func (sb *sandbox) setupDNS() error {
 	var newRC *resolvconf.File
 

libnetwork/sandbox_dns_windows.go

@@ -15,6 +15,9 @@ func (sb *sandbox) setupResolutionFiles() error {
 	return nil
 }
 
+func (sb *sandbox) restorePath() {
+}
+
 func (sb *sandbox) updateHostsFile(ifaceIP string) error {
 	return nil
 }

libnetwork/sandbox_store.go

@@ -20,12 +20,13 @@ type epState struct {
 }
 
 type sbState struct {
-	ID       string
-	Cid      string
-	c        *controller
-	dbIndex  uint64
-	dbExists bool
-	Eps      []epState
+	ID         string
+	Cid        string
+	c          *controller
+	dbIndex    uint64
+	dbExists   bool
+	Eps        []epState
+	EpPriority map[string]int
 }
 
 func (sbs *sbState) Key() []string {
@@ -106,6 +107,7 @@ func (sbs *sbState) CopyTo(o datastore.KVObject) error {
 	dstSbs.Cid = sbs.Cid
 	dstSbs.dbIndex = sbs.dbIndex
 	dstSbs.dbExists = sbs.dbExists
+	dstSbs.EpPriority = sbs.EpPriority
 
 	for _, eps := range sbs.Eps {
 		dstSbs.Eps = append(dstSbs.Eps, eps)
@@ -120,9 +122,10 @@ func (sbs *sbState) DataScope() string {
 
 func (sb *sandbox) storeUpdate() error {
 	sbs := &sbState{
-		c:   sb.controller,
-		ID:  sb.id,
-		Cid: sb.containerID,
+		c:          sb.controller,
+		ID:         sb.id,
+		Cid:        sb.containerID,
+		EpPriority: sb.epPriority,
 	}
 
 retry:
@@ -166,7 +169,7 @@ func (sb *sandbox) storeDelete() error {
 	return sb.controller.deleteFromStore(sbs)
 }
 
-func (c *controller) sandboxCleanup() {
+func (c *controller) sandboxCleanup(activeSandboxes map[string]interface{}) {
 	store := c.getStore(datastore.LocalScope)
 	if store == nil {
 		logrus.Errorf("Could not find local scope store while trying to cleanup sandboxes")
@@ -192,15 +195,27 @@ func (c *controller) sandboxCleanup() {
 			controller:  sbs.c,
 			containerID: sbs.Cid,
 			endpoints:   epHeap{},
-			epPriority:  map[string]int{},
 			dbIndex:     sbs.dbIndex,
 			isStub:      true,
 			dbExists:    true,
 		}
 
-		sb.osSbox, err = osl.NewSandbox(sb.Key(), true)
+		msg := " for cleanup"
+		create := true
+		isRestore := false
+		if val, ok := activeSandboxes[sb.ID()]; ok {
+			msg = ""
+			sb.isStub = false
+			isRestore = true
+			opts := val.([]SandboxOption)
+			sb.processOptions(opts...)
+			sb.restorePath()
+			create = !sb.config.useDefaultSandBox
+			heap.Init(&sb.endpoints)
+		}
+		sb.osSbox, err = osl.NewSandbox(sb.Key(), create, isRestore)
 		if err != nil {
-			logrus.Errorf("failed to create new osl sandbox while trying to build sandbox for cleanup: %v", err)
+			logrus.Errorf("failed to create osl sandbox while trying to restore sandbox %s%s: %v", sb.ID()[0:7], msg, err)
 			continue
 		}
 
@@ -222,13 +237,34 @@ func (c *controller) sandboxCleanup() {
 					ep = &endpoint{id: eps.Eid, network: n, sandboxID: sbs.ID}
 				}
 			}
-
 			heap.Push(&sb.endpoints, ep)
 		}
 
-		logrus.Infof("Removing stale sandbox %s (%s)", sb.id, sb.containerID)
-		if err := sb.delete(true); err != nil {
-			logrus.Errorf("failed to delete sandbox %s while trying to cleanup: %v", sb.id, err)
+		if _, ok := activeSandboxes[sb.ID()]; !ok {
+			logrus.Infof("Removing stale sandbox %s (%s)", sb.id, sb.containerID)
+			if err := sb.delete(true); err != nil {
+				logrus.Errorf("Failed to delete sandbox %s while trying to cleanup: %v", sb.id, err)
+			}
+			continue
+		}
+
+		// reconstruct osl sandbox field
+		if !sb.config.useDefaultSandBox {
+			if err := sb.restoreOslSandbox(); err != nil {
+				logrus.Errorf("failed to populate fields for osl sandbox %s", sb.ID())
+				continue
+			}
+		} else {
+			c.sboxOnce.Do(func() {
+				c.defOsSbox = sb.osSbox
+			})
+		}
+
+		for _, ep := range sb.endpoints {
+			// Watch for service records
+			if !c.isAgent() {
+				c.watchSvcRecord(ep)
+			}
 		}
 	}
 }