Always program the kernel state if triggered by l2 or l3 miss

Signed-off-by: Santhosh Manohar <santhosh@docker.com>

libnetwork/drivers/overlay/joinleave.go

@@ -205,7 +205,7 @@ func (d *driver) EventNotify(etype driverapi.EventType, nid, tableName, key stri
 		return
 	}
 
-	d.peerAdd(nid, eid, addr.IP, addr.Mask, mac, vtep, true)
+	d.peerAdd(nid, eid, addr.IP, addr.Mask, mac, vtep, true, false, false)
 }
 
 // Leave method is invoked when a Sandbox detaches from an endpoint.

libnetwork/drivers/overlay/ov_network.go

@@ -646,14 +646,17 @@ func (n *network) watchMiss(nlSock *nl.NetlinkSocket) {
 			}
 
 			var (
-				ip  net.IP
-				mac net.HardwareAddr
+				ip             net.IP
+				mac            net.HardwareAddr
+				l2Miss, l3Miss bool
 			)
 			if neigh.IP.To4() != nil {
 				ip = neigh.IP
+				l3Miss = true
 			} else if neigh.HardwareAddr != nil {
 				mac = []byte(neigh.HardwareAddr)
 				ip = net.IP(mac[2:])
+				l2Miss = true
 			} else {
 				continue
 			}
@@ -679,7 +682,7 @@ func (n *network) watchMiss(nlSock *nl.NetlinkSocket) {
 				continue
 			}
 
-			if err := n.driver.peerAdd(n.id, "dummy", ip, IPmask, mac, vtep, true); err != nil {
+			if err := n.driver.peerAdd(n.id, "dummy", ip, IPmask, mac, vtep, true, l2Miss, l3Miss); err != nil {
 				logrus.Errorf("could not add neighbor entry for missed peer %q: %v", ip, err)
 			}
 		}

libnetwork/drivers/overlay/ov_serf.go

@@ -121,7 +121,7 @@ func (d *driver) processEvent(u serf.UserEvent) {
 	switch action {
 	case "join":
 		if err := d.peerAdd(nid, eid, net.ParseIP(ipStr), net.IPMask(net.ParseIP(maskStr).To4()), mac,
-			net.ParseIP(vtepStr), true); err != nil {
+			net.ParseIP(vtepStr), true, false, false); err != nil {
 			logrus.Errorf("Peer add failed in the driver: %v\n", err)
 		}
 	case "leave":

libnetwork/drivers/overlay/peerdb.go

@@ -236,7 +236,7 @@ func (d *driver) peerDbUpdateSandbox(nid string) {
 		op := func() {
 			if err := d.peerAdd(nid, entry.eid, pKey.peerIP, entry.peerIPMask,
 				pKey.peerMac, entry.vtep,
-				false); err != nil {
+				false, false, false); err != nil {
 				fmt.Printf("peerdbupdate in sandbox failed for ip %s and mac %s: %v",
 					pKey.peerIP, pKey.peerMac, err)
 			}
@@ -254,7 +254,7 @@ func (d *driver) peerDbUpdateSandbox(nid string) {
 }
 
 func (d *driver) peerAdd(nid, eid string, peerIP net.IP, peerIPMask net.IPMask,
-	peerMac net.HardwareAddr, vtep net.IP, updateDb bool) error {
+	peerMac net.HardwareAddr, vtep net.IP, updateDb, l2Miss, l3Miss bool) error {
 
 	if err := validateID(nid, eid); err != nil {
 		return err
@@ -297,12 +297,12 @@ func (d *driver) peerAdd(nid, eid string, peerIP net.IP, peerIPMask net.IPMask,
 	}
 
 	// Add neighbor entry for the peer IP
-	if err := sbox.AddNeighbor(peerIP, peerMac, sbox.NeighborOptions().LinkName(s.vxlanName)); err != nil {
+	if err := sbox.AddNeighbor(peerIP, peerMac, l3Miss, sbox.NeighborOptions().LinkName(s.vxlanName)); err != nil {
 		return fmt.Errorf("could not add neighbor entry into the sandbox: %v", err)
 	}
 
 	// Add fdb entry to the bridge for the peer mac
-	if err := sbox.AddNeighbor(vtep, peerMac, sbox.NeighborOptions().LinkName(s.vxlanName),
+	if err := sbox.AddNeighbor(vtep, peerMac, l2Miss, sbox.NeighborOptions().LinkName(s.vxlanName),
 		sbox.NeighborOptions().Family(syscall.AF_BRIDGE)); err != nil {
 		return fmt.Errorf("could not add fdb entry into the sandbox: %v", err)
 	}

libnetwork/drivers/solaris/overlay/peerdb.go

@@ -279,7 +279,7 @@ func (d *driver) peerAdd(nid, eid string, peerIP net.IP, peerIPMask net.IPMask,
 	}
 
 	// Add neighbor entry for the peer IP
-	if err := sbox.AddNeighbor(peerIP, peerMac, sbox.NeighborOptions().LinkName(s.vxlanName)); err != nil {
+	if err := sbox.AddNeighbor(peerIP, peerMac, false, sbox.NeighborOptions().LinkName(s.vxlanName)); err != nil {
 		return fmt.Errorf("could not add neigbor entry into the sandbox: %v", err)
 	}
 

libnetwork/osl/neigh_linux.go

@@ -72,8 +72,11 @@ func (n *networkNamespace) DeleteNeighbor(dstIP net.IP, dstMac net.HardwareAddr,
 			nlnh.LinkIndex = iface.Attrs().Index
 		}
 
+		// If the kernel deletion fails for the neighbor entry still remote it
+		// from the namespace cache. Otherwise if the neighbor moves back to the
+		// same host again, kernel update can fail.
 		if err := nlh.NeighDel(nlnh); err != nil {
-			return fmt.Errorf("could not delete neighbor entry: %v", err)
+			logrus.Warnf("Deleting neighbor IP %s, mac %s failed, %v", dstIP, dstMac, err)
 		}
 	}
 
@@ -85,21 +88,26 @@ func (n *networkNamespace) DeleteNeighbor(dstIP net.IP, dstMac net.HardwareAddr,
 		}
 	}
 	n.Unlock()
+	logrus.Debugf("Neighbor entry deleted for IP %v, mac %v", dstIP, dstMac)
 
 	return nil
 }
 
-func (n *networkNamespace) AddNeighbor(dstIP net.IP, dstMac net.HardwareAddr, options ...NeighOption) error {
+func (n *networkNamespace) AddNeighbor(dstIP net.IP, dstMac net.HardwareAddr, force bool, options ...NeighOption) error {
 	var (
 		iface netlink.Link
 		err   error
 	)
 
+	// If the namespace already has the neighbor entry but the AddNeighbor is called
+	// because of a miss notification (force flag) program the kernel anyway.
 	nh := n.findNeighbor(dstIP, dstMac)
 	if nh != nil {
-		logrus.Debugf("Neighbor entry already present for IP %v, mac %v", dstIP, dstMac)
-		// If it exists silently return
-		return nil
+		if !force {
+			logrus.Warnf("Neighbor entry already present for IP %v, mac %v", dstIP, dstMac)
+			return nil
+		}
+		logrus.Warnf("Force kernel update, Neighbor entry already present for IP %v, mac %v", dstIP, dstMac)
 	}
 
 	nh = &neigh{
@@ -150,6 +158,7 @@ func (n *networkNamespace) AddNeighbor(dstIP net.IP, dstMac net.HardwareAddr, op
 	n.Lock()
 	n.neighbors = append(n.neighbors, nh)
 	n.Unlock()
+	logrus.Debugf("Neighbor entry added for IP %v, mac %v", dstIP, dstMac)
 
 	return nil
 }

libnetwork/osl/sandbox.go

@@ -39,7 +39,7 @@ type Sandbox interface {
 	RemoveStaticRoute(*types.StaticRoute) error
 
 	// AddNeighbor adds a neighbor entry into the sandbox.
-	AddNeighbor(dstIP net.IP, dstMac net.HardwareAddr, option ...NeighOption) error
+	AddNeighbor(dstIP net.IP, dstMac net.HardwareAddr, force bool, option ...NeighOption) error
 
 	// DeleteNeighbor deletes neighbor entry from the sandbox.
 	DeleteNeighbor(dstIP net.IP, dstMac net.HardwareAddr, osDelete bool) error