Cleanup vxlan interfaces inside namespace

If a new network request is received for a prticular vni, cleanup the
interface with that vni even if it is inside a namespace. This is done
by collecting vni to namespace data during init and later using it to
delete the interface.

Also fixed a long pending issue of the vxlan interface not getting
destroyed even if the sandbox is destroyed. Fixed by first deleting the
vxlan interface first before destroying the sandbox.

Signed-off-by: Jana Radhakrishnan <mrjana@docker.com>

libnetwork/drivers/overlay/ov_network.go

@@ -21,11 +21,14 @@ import (
 	"github.com/docker/libnetwork/types"
 	"github.com/vishvananda/netlink"
 	"github.com/vishvananda/netlink/nl"
+	"github.com/vishvananda/netns"
 )
 
 var (
-	hostMode     bool
-	hostModeOnce sync.Once
+	hostMode    bool
+	networkOnce sync.Once
+	networkMu   sync.Mutex
+	vniTbl      = make(map[uint32]string)
 )
 
 type networkTable map[string]*network
@@ -249,7 +252,48 @@ func (n *network) destroySandbox() {
 	}
 }
 
-func setHostMode() {
+func populateVNITbl() {
+	filepath.Walk(filepath.Dir(osl.GenerateKey("walk")),
+		func(path string, info os.FileInfo, err error) error {
+			_, fname := filepath.Split(path)
+
+			if len(strings.Split(fname, "-")) <= 1 {
+				return nil
+			}
+
+			ns, err := netns.GetFromPath(path)
+			if err != nil {
+				logrus.Errorf("Could not open namespace path %s during vni population: %v", path, err)
+				return nil
+			}
+			defer ns.Close()
+
+			nlh, err := netlink.NewHandleAt(ns)
+			if err != nil {
+				logrus.Errorf("Could not open netlink handle during vni population for ns %s: %v", path, err)
+				return nil
+			}
+			defer nlh.Delete()
+
+			links, err := nlh.LinkList()
+			if err != nil {
+				logrus.Errorf("Failed to list interfaces during vni population for ns %s: %v", path, err)
+				return nil
+			}
+
+			for _, l := range links {
+				if l.Type() == "vxlan" {
+					vniTbl[uint32(l.(*netlink.Vxlan).VxlanId)] = path
+				}
+			}
+
+			return nil
+		})
+}
+
+func networkOnceInit() {
+	populateVNITbl()
+
 	if os.Getenv("_OVERLAY_HOST_MODE") != "" {
 		hostMode = true
 		return
@@ -330,13 +374,36 @@ func (n *network) initSubnetSandbox(s *subnet) error {
 		// Try to delete stale bridge interface if it exists
 		deleteInterface(brName)
 		// Try to delete the vxlan interface by vni if already present
-		deleteVxlanByVNI(n.vxlanID(s))
+		deleteVxlanByVNI("", n.vxlanID(s))
 
 		if isOverlap(s.subnetIP) {
 			return fmt.Errorf("overlay subnet %s has conflicts in the host while running in host mode", s.subnetIP.String())
 		}
 	}
 
+	if !hostMode {
+		// Try to find this subnet's vni is being used in some
+		// other namespace by looking at vniTbl that we just
+		// populated in the once init. If a hit is found then
+		// it must a stale namespace from previous
+		// life. Destroy it completely and reclaim resourced.
+		networkMu.Lock()
+		path, ok := vniTbl[n.vxlanID(s)]
+		networkMu.Unlock()
+
+		if ok {
+			deleteVxlanByVNI(path, n.vxlanID(s))
+			if err := syscall.Unmount(path, syscall.MNT_FORCE); err != nil {
+				logrus.Errorf("unmount of %s failed: %v", path, err)
+			}
+			os.Remove(path)
+
+			networkMu.Lock()
+			delete(vniTbl, n.vxlanID(s))
+			networkMu.Unlock()
+		}
+	}
+
 	// create a bridge and vxlan device for this subnet and move it to the sandbox
 	sbox := n.sandbox()
 
@@ -382,6 +449,8 @@ func (n *network) cleanupStaleSandboxes() {
 
 			pattern := pList[1]
 			if strings.Contains(n.id, pattern) {
+				// Delete all vnis
+				deleteVxlanByVNI(path, 0)
 				syscall.Unmount(path, syscall.MNT_DETACH)
 				os.Remove(path)
 			}
@@ -395,7 +464,7 @@ func (n *network) initSandbox() error {
 	n.initEpoch++
 	n.Unlock()
 
-	hostModeOnce.Do(setHostMode)
+	networkOnce.Do(networkOnceInit)
 
 	if hostMode {
 		if err := addNetworkChain(n.id[:12]); err != nil {

libnetwork/drivers/overlay/ov_utils.go

@@ -6,6 +6,7 @@ import (
 	"github.com/docker/libnetwork/netutils"
 	"github.com/docker/libnetwork/osl"
 	"github.com/vishvananda/netlink"
+	"github.com/vishvananda/netns"
 )
 
 func validateID(nid, eid string) error {
@@ -81,16 +82,42 @@ func deleteInterface(name string) error {
 	return nil
 }
 
-func deleteVxlanByVNI(vni uint32) error {
+func deleteVxlanByVNI(path string, vni uint32) error {
 	defer osl.InitOSContext()()
 
-	links, err := netlink.LinkList()
+	var nlh *netlink.Handle
+	if path == "" {
+		var err error
+		nlh, err = netlink.NewHandle()
+		if err != nil {
+			return fmt.Errorf("failed to get netlink handle for current ns: %v", err)
+		}
+	} else {
+		var (
+			err error
+			ns  netns.NsHandle
+		)
+
+		ns, err = netns.GetFromPath(path)
+		if err != nil {
+			return fmt.Errorf("failed to get ns handle for %s: %v", path, err)
+		}
+		defer ns.Close()
+
+		nlh, err = netlink.NewHandleAt(ns)
+		if err != nil {
+			return fmt.Errorf("failed to get netlink handle for ns %s: %v", path, err)
+		}
+	}
+	defer nlh.Delete()
+
+	links, err := nlh.LinkList()
 	if err != nil {
 		return fmt.Errorf("failed to list interfaces while deleting vxlan interface by vni: %v", err)
 	}
 
 	for _, l := range links {
-		if l.Type() == "vxlan" && l.(*netlink.Vxlan).VxlanId == int(vni) {
+		if l.Type() == "vxlan" && (vni == 0 || l.(*netlink.Vxlan).VxlanId == int(vni)) {
 			err = netlink.LinkDel(l)
 			if err != nil {
 				return fmt.Errorf("error deleting vxlan interface with id %d: %v", vni, err)