Fix a lb rule race in loadbalancer

When a goroutine which is adding the service and another which is adding
just a destination interleave the destination which is dependent on the
service may not get added and will result in service working at reduced
scale. The fix is to synchronize this with the service mutex.

Signed-off-by: Jana Radhakrishnan <mrjana@docker.com>

libnetwork/service_linux.go

@@ -56,7 +56,13 @@ func (c *controller) addServiceBinding(name, sid, nid, eid string, vip net.IP, i
 	}
 	c.Unlock()
 
+	// Add endpoint IP to special "tasks.svc_name" so that the
+	// applications have access to DNS RR.
+	n.(*network).addSvcRecords("tasks."+name, ip, nil, false)
+
 	s.Lock()
+	defer s.Unlock()
+
 	lb, ok := s.loadBalancers[nid]
 	if !ok {
 		// Create a new load balancer if we are seeing this
@@ -89,11 +95,6 @@ func (c *controller) addServiceBinding(name, sid, nid, eid string, vip net.IP, i
 	}
 
 	lb.backEnds[eid] = ip
-	s.Unlock()
-
-	// Add endpoint IP to special "tasks.svc_name" so that the
-	// applications have access to DNS RR.
-	n.(*network).addSvcRecords("tasks."+name, ip, nil, false)
 
 	// Add loadbalancer service and backend in all sandboxes in
 	// the network only if vip is valid.
@@ -120,17 +121,18 @@ func (c *controller) rmServiceBinding(name, sid, nid, eid string, vip net.IP, in
 	}
 	c.Unlock()
 
+	// Delete the special "tasks.svc_name" backend record.
+	n.(*network).deleteSvcRecords("tasks."+name, ip, nil, false)
+
 	s.Lock()
+	defer s.Unlock()
+
 	lb, ok := s.loadBalancers[nid]
 	if !ok {
-		s.Unlock()
 		return nil
 	}
 
-	// Delete the special "tasks.svc_name" backend record.
-	n.(*network).deleteSvcRecords("tasks."+name, ip, nil, false)
 	delete(lb.backEnds, eid)
-
 	if len(lb.backEnds) == 0 {
 		// All the backends for this service have been
 		// removed. Time to remove the load balancer and also
@@ -153,7 +155,6 @@ func (c *controller) rmServiceBinding(name, sid, nid, eid string, vip net.IP, in
 		// remove the service itself.
 		delete(c.serviceBindings, sid)
 	}
-	s.Unlock()
 
 	// Remove loadbalancer service(if needed) and backend in all
 	// sandboxes in the network only if the vip is valid.
@@ -312,7 +313,7 @@ func (sb *sandbox) addLBBackend(ip, vip net.IP, fwMark uint32, ingressPorts []*P
 	// destination.
 	s.SchedName = ""
 	if err := i.NewDestination(s, d); err != nil && err != syscall.EEXIST {
-		logrus.Errorf("Failed to create real server %s for vip %s fwmark %d: %v", ip, vip, fwMark, err)
+		logrus.Errorf("Failed to create real server %s for vip %s fwmark %d in sb %s: %v", ip, vip, fwMark, sb.containerID, err)
 	}
 }