Adding synchronization to endpoint addition/deletion to prevent network connectivity issue

Signed-off-by: Pradip Dhara <pradipd@microsoft.com>

libnetwork/drivers/windows/overlay/ov_endpoint_windows.go

@@ -4,6 +4,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"net"
+	"sync"
 
 	"github.com/Microsoft/hcsshim"
 	"github.com/docker/docker/pkg/system"
@@ -29,6 +30,13 @@ type endpoint struct {
 	portMapping    []types.PortBinding // Operation port bindings
 }
 
+var (
+	//Server 2016 (RS1) does not support concurrent add/delete of endpoints.  Therefore, we need
+	//to use this mutex and serialize the add/delete of endpoints on RS1.
+	endpointMu   sync.Mutex
+	windowsBuild = system.GetOSVersion().Build
+)
+
 func validateID(nid, eid string) error {
 	if nid == "" {
 		return fmt.Errorf("invalid network id")
@@ -77,8 +85,7 @@ func (n *network) removeEndpointWithAddress(addr *net.IPNet) {
 
 	if networkEndpoint != nil {
 		logrus.Debugf("Removing stale endpoint from HNS")
-		_, err := hcsshim.HNSEndpointRequest("DELETE", networkEndpoint.profileID, "")
-
+		_, err := endpointRequest("DELETE", networkEndpoint.profileID, "")
 		if err != nil {
 			logrus.Debugf("Failed to delete stale overlay endpoint (%.7s) from hns", networkEndpoint.id)
 		}
@@ -101,8 +108,7 @@ func (d *driver) CreateEndpoint(nid, eid string, ifInfo driverapi.InterfaceInfo,
 	if ep != nil {
 		logrus.Debugf("Deleting stale endpoint %s", eid)
 		n.deleteEndpoint(eid)
-
-		_, err := hcsshim.HNSEndpointRequest("DELETE", ep.profileID, "")
+		_, err := endpointRequest("DELETE", ep.profileID, "")
 		if err != nil {
 			return err
 		}
@@ -179,7 +185,7 @@ func (d *driver) CreateEndpoint(nid, eid string, ifInfo driverapi.InterfaceInfo,
 		return err
 	}
 
-	hnsresponse, err := hcsshim.HNSEndpointRequest("POST", "", string(configurationb))
+	hnsresponse, err := endpointRequest("POST", "", string(configurationb))
 	if err != nil {
 		return err
 	}
@@ -199,7 +205,7 @@ func (d *driver) CreateEndpoint(nid, eid string, ifInfo driverapi.InterfaceInfo,
 
 	ep.portMapping, err = windows.ParsePortBindingPolicies(hnsresponse.Policies)
 	if err != nil {
-		hcsshim.HNSEndpointRequest("DELETE", hnsresponse.Id, "")
+		endpointRequest("DELETE", hnsresponse.Id, "")
 		return err
 	}
 
@@ -225,7 +231,7 @@ func (d *driver) DeleteEndpoint(nid, eid string) error {
 
 	n.deleteEndpoint(eid)
 
-	_, err := hcsshim.HNSEndpointRequest("DELETE", ep.profileID, "")
+	_, err := endpointRequest("DELETE", ep.profileID, "")
 	if err != nil {
 		return err
 	}
@@ -263,3 +269,14 @@ func (d *driver) EndpointOperInfo(nid, eid string) (map[string]interface{}, erro
 
 	return data, nil
 }
+
+func endpointRequest(method, path, request string) (*hcsshim.HNSEndpoint, error) {
+	if windowsBuild == 14393 {
+		endpointMu.Lock()
+	}
+	hnsresponse, err := hcsshim.HNSEndpointRequest(method, path, request)
+	if windowsBuild == 14393 {
+		endpointMu.Unlock()
+	}
+	return hnsresponse, err
+}

libnetwork/drivers/windows/overlay/peerdb_windows.go

@@ -67,8 +67,7 @@ func (d *driver) peerAdd(nid, eid string, peerIP net.IP, peerIPMask net.IPMask,
 		}
 
 		n.removeEndpointWithAddress(addr)
-
-		hnsresponse, err := hcsshim.HNSEndpointRequest("POST", "", string(configurationb))
+		hnsresponse, err := endpointRequest("POST", "", string(configurationb))
 		if err != nil {
 			return err
 		}
@@ -108,7 +107,7 @@ func (d *driver) peerDelete(nid, eid string, peerIP net.IP, peerIPMask net.IPMas
 	}
 
 	if updateDb {
-		_, err := hcsshim.HNSEndpointRequest("DELETE", ep.profileID, "")
+		_, err := endpointRequest("DELETE", ep.profileID, "")
 		if err != nil {
 			return err
 		}