From 6e47b07b39d5ccf449516c0494214fbd185ec157 Mon Sep 17 00:00:00 2001 From: Jana Radhakrishnan Date: Sun, 5 Jun 2016 19:28:33 -0700 Subject: [PATCH] Cleanup vxlan interfaces inside namespace If a new network request is received for a prticular vni, cleanup the interface with that vni even if it is inside a namespace. This is done by collecting vni to namespace data during init and later using it to delete the interface. Also fixed a long pending issue of the vxlan interface not getting destroyed even if the sandbox is destroyed. Fixed by first deleting the vxlan interface first before destroying the sandbox. Signed-off-by: Jana Radhakrishnan --- libnetwork/drivers/overlay/ov_network.go | 79 ++++++++++++++++++++++-- libnetwork/drivers/overlay/ov_utils.go | 33 +++++++++- 2 files changed, 104 insertions(+), 8 deletions(-) diff --git a/libnetwork/drivers/overlay/ov_network.go b/libnetwork/drivers/overlay/ov_network.go index 893f8da314..d1cf940491 100644 --- a/libnetwork/drivers/overlay/ov_network.go +++ b/libnetwork/drivers/overlay/ov_network.go @@ -21,11 +21,14 @@ import ( "github.com/docker/libnetwork/types" "github.com/vishvananda/netlink" "github.com/vishvananda/netlink/nl" + "github.com/vishvananda/netns" ) var ( - hostMode bool - hostModeOnce sync.Once + hostMode bool + networkOnce sync.Once + networkMu sync.Mutex + vniTbl = make(map[uint32]string) ) type networkTable map[string]*network @@ -249,7 +252,48 @@ func (n *network) destroySandbox() { } } -func setHostMode() { +func populateVNITbl() { + filepath.Walk(filepath.Dir(osl.GenerateKey("walk")), + func(path string, info os.FileInfo, err error) error { + _, fname := filepath.Split(path) + + if len(strings.Split(fname, "-")) <= 1 { + return nil + } + + ns, err := netns.GetFromPath(path) + if err != nil { + logrus.Errorf("Could not open namespace path %s during vni population: %v", path, err) + return nil + } + defer ns.Close() + + nlh, err := netlink.NewHandleAt(ns) + if err != nil { + logrus.Errorf("Could not open netlink handle during vni population for ns %s: %v", path, err) + return nil + } + defer nlh.Delete() + + links, err := nlh.LinkList() + if err != nil { + logrus.Errorf("Failed to list interfaces during vni population for ns %s: %v", path, err) + return nil + } + + for _, l := range links { + if l.Type() == "vxlan" { + vniTbl[uint32(l.(*netlink.Vxlan).VxlanId)] = path + } + } + + return nil + }) +} + +func networkOnceInit() { + populateVNITbl() + if os.Getenv("_OVERLAY_HOST_MODE") != "" { hostMode = true return @@ -330,13 +374,36 @@ func (n *network) initSubnetSandbox(s *subnet) error { // Try to delete stale bridge interface if it exists deleteInterface(brName) // Try to delete the vxlan interface by vni if already present - deleteVxlanByVNI(n.vxlanID(s)) + deleteVxlanByVNI("", n.vxlanID(s)) if isOverlap(s.subnetIP) { return fmt.Errorf("overlay subnet %s has conflicts in the host while running in host mode", s.subnetIP.String()) } } + if !hostMode { + // Try to find this subnet's vni is being used in some + // other namespace by looking at vniTbl that we just + // populated in the once init. If a hit is found then + // it must a stale namespace from previous + // life. Destroy it completely and reclaim resourced. + networkMu.Lock() + path, ok := vniTbl[n.vxlanID(s)] + networkMu.Unlock() + + if ok { + deleteVxlanByVNI(path, n.vxlanID(s)) + if err := syscall.Unmount(path, syscall.MNT_FORCE); err != nil { + logrus.Errorf("unmount of %s failed: %v", path, err) + } + os.Remove(path) + + networkMu.Lock() + delete(vniTbl, n.vxlanID(s)) + networkMu.Unlock() + } + } + // create a bridge and vxlan device for this subnet and move it to the sandbox sbox := n.sandbox() @@ -382,6 +449,8 @@ func (n *network) cleanupStaleSandboxes() { pattern := pList[1] if strings.Contains(n.id, pattern) { + // Delete all vnis + deleteVxlanByVNI(path, 0) syscall.Unmount(path, syscall.MNT_DETACH) os.Remove(path) } @@ -395,7 +464,7 @@ func (n *network) initSandbox() error { n.initEpoch++ n.Unlock() - hostModeOnce.Do(setHostMode) + networkOnce.Do(networkOnceInit) if hostMode { if err := addNetworkChain(n.id[:12]); err != nil { diff --git a/libnetwork/drivers/overlay/ov_utils.go b/libnetwork/drivers/overlay/ov_utils.go index bcaea768c5..b9266f4188 100644 --- a/libnetwork/drivers/overlay/ov_utils.go +++ b/libnetwork/drivers/overlay/ov_utils.go @@ -6,6 +6,7 @@ import ( "github.com/docker/libnetwork/netutils" "github.com/docker/libnetwork/osl" "github.com/vishvananda/netlink" + "github.com/vishvananda/netns" ) func validateID(nid, eid string) error { @@ -81,16 +82,42 @@ func deleteInterface(name string) error { return nil } -func deleteVxlanByVNI(vni uint32) error { +func deleteVxlanByVNI(path string, vni uint32) error { defer osl.InitOSContext()() - links, err := netlink.LinkList() + var nlh *netlink.Handle + if path == "" { + var err error + nlh, err = netlink.NewHandle() + if err != nil { + return fmt.Errorf("failed to get netlink handle for current ns: %v", err) + } + } else { + var ( + err error + ns netns.NsHandle + ) + + ns, err = netns.GetFromPath(path) + if err != nil { + return fmt.Errorf("failed to get ns handle for %s: %v", path, err) + } + defer ns.Close() + + nlh, err = netlink.NewHandleAt(ns) + if err != nil { + return fmt.Errorf("failed to get netlink handle for ns %s: %v", path, err) + } + } + defer nlh.Delete() + + links, err := nlh.LinkList() if err != nil { return fmt.Errorf("failed to list interfaces while deleting vxlan interface by vni: %v", err) } for _, l := range links { - if l.Type() == "vxlan" && l.(*netlink.Vxlan).VxlanId == int(vni) { + if l.Type() == "vxlan" && (vni == 0 || l.(*netlink.Vxlan).VxlanId == int(vni)) { err = netlink.LinkDel(l) if err != nil { return fmt.Errorf("error deleting vxlan interface with id %d: %v", vni, err)