Merge pull request #46149 from thaJeztah/libnetwork_remove_interface_interface

libnetwork/osl: remove Interface Interface and other funny stories

libnetwork/controller.go

@@ -92,9 +92,7 @@ type Controller struct {
 	svcRecords       map[string]*svcInfo
 	nmap             map[string]*netWatch
 	serviceBindings  map[serviceKey]*service
-	defOsSbox        osl.Sandbox
 	ingressSandbox   *Sandbox
-	sboxOnce         sync.Once
 	agent            *nwAgent
 	networkLocker    *locker.Locker
 	agentInitDone    chan struct{}
@@ -102,6 +100,10 @@ type Controller struct {
 	keys             []*types.EncryptionKey
 	DiagnosticServer *diagnostic.Server
 	mu               sync.Mutex
+
+	// FIXME(thaJeztah): defOsSbox is always nil on non-Linux: move these fields to Linux-only files.
+	defOsSboxOnce sync.Once
+	defOsSbox     osl.Sandbox
 }
 
 // New creates a new instance of network controller.
@@ -937,38 +939,8 @@ func (c *Controller) NewSandbox(containerID string, options ...SandboxOption) (_
 	if err := sb.setupResolutionFiles(); err != nil {
 		return nil, err
 	}
-
-	if sb.config.useDefaultSandBox {
-		var err error
-		c.sboxOnce.Do(func() {
-			c.defOsSbox, err = osl.NewSandbox(sb.Key(), false, false)
-		})
-
-		if err != nil {
-			c.sboxOnce = sync.Once{}
-			return nil, fmt.Errorf("failed to create default sandbox: %v", err)
-		}
-
-		sb.osSbox = c.defOsSbox
-	}
-
-	if sb.osSbox == nil && !sb.config.useExternalKey {
-		var err error
-		if sb.osSbox, err = osl.NewSandbox(sb.Key(), !sb.config.useDefaultSandBox, false); err != nil {
-			return nil, fmt.Errorf("failed to create new osl sandbox: %v", err)
-		}
-	}
-
-	if sb.osSbox != nil {
-		// Apply operating specific knobs on the load balancer sandbox
-		err := sb.osSbox.InvokeFunc(func() {
-			sb.osSbox.ApplyOSTweaks(sb.oslTypes)
-		})
-		if err != nil {
-			log.G(context.TODO()).Errorf("Failed to apply performance tuning sysctls to the sandbox: %v", err)
-		}
-		// Keep this just so performance is not changed
-		sb.osSbox.ApplyOSTweaks(sb.oslTypes)
+	if err := c.setupOSLSandbox(sb); err != nil {
+		return nil, err
 	}
 
 	c.mu.Lock()

libnetwork/controller_linux.go

@@ -1,9 +1,15 @@
 package libnetwork
 
 import (
+	"context"
+	"fmt"
+	"sync"
+
+	"github.com/containerd/containerd/log"
 	"github.com/docker/docker/libnetwork/iptables"
 	"github.com/docker/docker/libnetwork/netlabel"
 	"github.com/docker/docker/libnetwork/options"
+	"github.com/docker/docker/libnetwork/osl"
 )
 
 // enabledIptablesVersions returns the iptables versions that are enabled
@@ -31,3 +37,54 @@ func (c *Controller) enabledIptablesVersions() []iptables.IPVersion {
 	}
 	return versions
 }
+
+// getDefaultOSLSandbox returns the controller's default [osl.Sandbox]. It
+// creates the sandbox if it does not yet exist.
+func (c *Controller) getDefaultOSLSandbox(key string) (osl.Sandbox, error) {
+	var err error
+	c.defOsSboxOnce.Do(func() {
+		c.defOsSbox, err = osl.NewSandbox(key, false, false)
+	})
+
+	if err != nil {
+		c.defOsSboxOnce = sync.Once{}
+		return nil, fmt.Errorf("failed to create default sandbox: %v", err)
+	}
+	return c.defOsSbox, nil
+}
+
+// setupOSLSandbox sets the sandbox [osl.Sandbox], and applies operating-
+// specific configuration.
+//
+// Depending on the Sandbox settings, it may either use the Controller's
+// default sandbox, or configure a new one.
+func (c *Controller) setupOSLSandbox(sb *Sandbox) error {
+	if sb.config.useDefaultSandBox {
+		defSB, err := c.getDefaultOSLSandbox(sb.Key())
+		if err != nil {
+			return err
+		}
+		sb.osSbox = defSB
+	}
+
+	if sb.osSbox == nil && !sb.config.useExternalKey {
+		newSB, err := osl.NewSandbox(sb.Key(), !sb.config.useDefaultSandBox, false)
+		if err != nil {
+			return fmt.Errorf("failed to create new osl sandbox: %v", err)
+		}
+		sb.osSbox = newSB
+	}
+
+	if sb.osSbox != nil {
+		// Apply operating specific knobs on the load balancer sandbox
+		err := sb.osSbox.InvokeFunc(func() {
+			sb.osSbox.ApplyOSTweaks(sb.oslTypes)
+		})
+		if err != nil {
+			log.G(context.TODO()).Errorf("Failed to apply performance tuning sysctls to the sandbox: %v", err)
+		}
+		// Keep this just so performance is not changed
+		sb.osSbox.ApplyOSTweaks(sb.oslTypes)
+	}
+	return nil
+}

libnetwork/controller_others.go

@@ -6,3 +6,7 @@ package libnetwork
 func (c *Controller) enabledIptablesVersions() []any {
 	return nil
 }
+
+func (c *Controller) setupOSLSandbox(_ *Sandbox) error {
+	return nil
+}

libnetwork/drivers/overlay/joinleave.go

@@ -11,6 +11,7 @@ import (
 	"github.com/containerd/containerd/log"
 	"github.com/docker/docker/libnetwork/driverapi"
 	"github.com/docker/docker/libnetwork/ns"
+	"github.com/docker/docker/libnetwork/osl"
 	"github.com/docker/docker/libnetwork/types"
 	"github.com/gogo/protobuf/proto"
 )
@@ -73,8 +74,7 @@ func (d *driver) Join(nid, eid string, sboxKey string, jinfo driverapi.JoinInfo,
 		return err
 	}
 
-	if err = sbox.AddInterface(overlayIfName, "veth",
-		sbox.InterfaceOptions().Master(s.brName)); err != nil {
+	if err = sbox.AddInterface(overlayIfName, "veth", osl.WithMaster(s.brName)); err != nil {
 		return fmt.Errorf("could not add veth pair inside the network sandbox: %v", err)
 	}
 

libnetwork/drivers/overlay/ov_network.go

@@ -426,9 +426,7 @@ func (n *network) setupSubnetSandbox(s *subnet, brName, vxlanName string) error
 	// create a bridge and vxlan device for this subnet and move it to the sandbox
 	sbox := n.sbox
 
-	if err := sbox.AddInterface(brName, "br",
-		sbox.InterfaceOptions().Address(s.gwIP),
-		sbox.InterfaceOptions().Bridge(true)); err != nil {
+	if err := sbox.AddInterface(brName, "br", osl.WithIPv4Address(s.gwIP), osl.WithIsBridge(true)); err != nil {
 		return fmt.Errorf("bridge creation in sandbox failed for subnet %q: %v", s.subnetIP.String(), err)
 	}
 
@@ -437,8 +435,7 @@ func (n *network) setupSubnetSandbox(s *subnet, brName, vxlanName string) error
 		return err
 	}
 
-	if err := sbox.AddInterface(vxlanName, "vxlan",
-		sbox.InterfaceOptions().Master(brName)); err != nil {
+	if err := sbox.AddInterface(vxlanName, "vxlan", osl.WithMaster(brName)); err != nil {
 		// If adding vxlan device to the overlay namespace fails, remove the bridge interface we
 		// already added to the namespace. This allows the caller to try the setup again.
 		for _, iface := range sbox.Interfaces() {

libnetwork/drivers/overlay/peerdb.go

@@ -322,7 +322,7 @@ func (d *driver) peerAddOp(nid, eid string, peerIP net.IP, peerIPMask net.IPMask
 	}
 
 	// Add neighbor entry for the peer IP
-	if err := sbox.AddNeighbor(peerIP, peerMac, l3Miss, sbox.NeighborOptions().LinkName(s.vxlanName)); err != nil {
+	if err := sbox.AddNeighbor(peerIP, peerMac, l3Miss, osl.WithLinkName(s.vxlanName)); err != nil {
 		if _, ok := err.(osl.NeighborSearchError); ok && dbEntries > 1 {
 			// We are in the transient case so only the first configuration is programmed into the kernel
 			// Upon deletion if the active configuration is deleted the next one from the database will be restored
@@ -333,8 +333,7 @@ func (d *driver) peerAddOp(nid, eid string, peerIP net.IP, peerIPMask net.IPMask
 	}
 
 	// Add fdb entry to the bridge for the peer mac
-	if err := sbox.AddNeighbor(vtep, peerMac, l2Miss, sbox.NeighborOptions().LinkName(s.vxlanName),
-		sbox.NeighborOptions().Family(syscall.AF_BRIDGE)); err != nil {
+	if err := sbox.AddNeighbor(vtep, peerMac, l2Miss, osl.WithLinkName(s.vxlanName), osl.WithFamily(syscall.AF_BRIDGE)); err != nil {
 		return fmt.Errorf("could not add fdb entry for nid:%s eid:%s into the sandbox:%v", nid, eid, err)
 	}
 

libnetwork/osl/interface_linux.go

@@ -14,12 +14,12 @@ import (
 	"github.com/vishvananda/netns"
 )
 
-// nwIface represents the settings and identity of a network device.
+// Interface represents the settings and identity of a network device.
 // It is used as a return type for Network.Link, and it is common practice
 // for the caller to use this information when moving interface SrcName from
 // host namespace to DstName in a different net namespace with the appropriate
 // network settings.
-type nwIface struct {
+type Interface struct {
 	srcName     string
 	dstName     string
 	master      string
@@ -34,7 +34,7 @@ type nwIface struct {
 }
 
 // SrcName returns the name of the interface in the origin network namespace.
-func (i *nwIface) SrcName() string {
+func (i *Interface) SrcName() string {
 	return i.srcName
 }
 
@@ -42,46 +42,41 @@ func (i *nwIface) SrcName() string {
 // moved inside a network namespace. When the caller passes in a DstName,
 // it is only expected to pass a prefix. The name will be modified with an
 // auto-generated suffix.
-func (i *nwIface) DstName() string {
+func (i *Interface) DstName() string {
 	return i.dstName
 }
 
-func (i *nwIface) DstMaster() string {
+func (i *Interface) DstMaster() string {
 	return i.dstMaster
 }
 
 // Bridge returns true if the interface is a bridge.
-func (i *nwIface) Bridge() bool {
+func (i *Interface) Bridge() bool {
 	return i.bridge
 }
 
-// Master returns the srcname of the master interface for this interface.
-func (i *nwIface) Master() string {
-	return i.master
-}
-
-func (i *nwIface) MacAddress() net.HardwareAddr {
+func (i *Interface) MacAddress() net.HardwareAddr {
 	return types.GetMacCopy(i.mac)
 }
 
 // Address returns the IPv4 address for the interface.
-func (i *nwIface) Address() *net.IPNet {
+func (i *Interface) Address() *net.IPNet {
 	return types.GetIPNetCopy(i.address)
 }
 
 // AddressIPv6 returns the IPv6 address for the interface.
-func (i *nwIface) AddressIPv6() *net.IPNet {
+func (i *Interface) AddressIPv6() *net.IPNet {
 	return types.GetIPNetCopy(i.addressIPv6)
 }
 
 // LinkLocalAddresses returns the link-local IP addresses assigned to the
 // interface.
-func (i *nwIface) LinkLocalAddresses() []*net.IPNet {
+func (i *Interface) LinkLocalAddresses() []*net.IPNet {
 	return i.llAddrs
 }
 
 // Routes returns IP routes for the interface.
-func (i *nwIface) Routes() []*net.IPNet {
+func (i *Interface) Routes() []*net.IPNet {
 	routes := make([]*net.IPNet, len(i.routes))
 	for index, route := range i.routes {
 		routes[index] = types.GetIPNetCopy(route)
@@ -92,7 +87,7 @@ func (i *nwIface) Routes() []*net.IPNet {
 
 // Remove an interface from the sandbox by renaming to original name
 // and moving it out of the sandbox.
-func (i *nwIface) Remove() error {
+func (i *Interface) Remove() error {
 	i.ns.Lock()
 	isDefault := i.ns.isDefault
 	nlh := i.ns.nlHandle
@@ -143,7 +138,7 @@ func (i *nwIface) Remove() error {
 }
 
 // Statistics returns the sandbox's side veth interface statistics.
-func (i *nwIface) Statistics() (*types.InterfaceStatistics, error) {
+func (i *Interface) Statistics() (*types.InterfaceStatistics, error) {
 	l, err := i.ns.nlHandle.LinkByName(i.DstName())
 	if err != nil {
 		return nil, fmt.Errorf("failed to retrieve the statistics for %s in netns %s: %v", i.DstName(), i.ns.path, err)
@@ -179,13 +174,20 @@ func (n *networkNamespace) findDst(srcName string, isBridge bool) string {
 	return ""
 }
 
+// AddInterface adds an existing Interface to the sandbox. The operation will rename
+// from the Interface SrcName to DstName as it moves, and reconfigure the
+// interface according to the specified settings. The caller is expected
+// to only provide a prefix for DstName. The AddInterface api will auto-generate
+// an appropriate suffix for the DstName to disambiguate.
 func (n *networkNamespace) AddInterface(srcName, dstPrefix string, options ...IfaceOption) error {
-	i := &nwIface{
+	i := &Interface{
 		srcName: srcName,
 		dstName: dstPrefix,
 		ns:      n,
 	}
-	i.processInterfaceOptions(options...)
+	if err := i.processInterfaceOptions(options...); err != nil {
+		return err
+	}
 
 	if i.master != "" {
 		i.dstMaster = n.findDst(i.master, true)
@@ -292,10 +294,10 @@ func (n *networkNamespace) AddInterface(srcName, dstPrefix string, options ...If
 	return nil
 }
 
-func configureInterface(nlh *netlink.Handle, iface netlink.Link, i *nwIface) error {
+func configureInterface(nlh *netlink.Handle, iface netlink.Link, i *Interface) error {
 	ifaceName := iface.Attrs().Name
 	ifaceConfigurators := []struct {
-		Fn         func(*netlink.Handle, netlink.Link, *nwIface) error
+		Fn         func(*netlink.Handle, netlink.Link, *Interface) error
 		ErrMessage string
 	}{
 		{setInterfaceName, fmt.Sprintf("error renaming interface %q to %q", ifaceName, i.DstName())},
@@ -314,7 +316,7 @@ func configureInterface(nlh *netlink.Handle, iface netlink.Link, i *nwIface) err
 	return nil
 }
 
-func setInterfaceMaster(nlh *netlink.Handle, iface netlink.Link, i *nwIface) error {
+func setInterfaceMaster(nlh *netlink.Handle, iface netlink.Link, i *Interface) error {
 	if i.DstMaster() == "" {
 		return nil
 	}
@@ -324,14 +326,14 @@ func setInterfaceMaster(nlh *netlink.Handle, iface netlink.Link, i *nwIface) err
 	})
 }
 
-func setInterfaceMAC(nlh *netlink.Handle, iface netlink.Link, i *nwIface) error {
+func setInterfaceMAC(nlh *netlink.Handle, iface netlink.Link, i *Interface) error {
 	if i.MacAddress() == nil {
 		return nil
 	}
 	return nlh.LinkSetHardwareAddr(iface, i.MacAddress())
 }
 
-func setInterfaceIP(nlh *netlink.Handle, iface netlink.Link, i *nwIface) error {
+func setInterfaceIP(nlh *netlink.Handle, iface netlink.Link, i *Interface) error {
 	if i.Address() == nil {
 		return nil
 	}
@@ -342,7 +344,7 @@ func setInterfaceIP(nlh *netlink.Handle, iface netlink.Link, i *nwIface) error {
 	return nlh.AddrAdd(iface, ipAddr)
 }
 
-func setInterfaceIPv6(nlh *netlink.Handle, iface netlink.Link, i *nwIface) error {
+func setInterfaceIPv6(nlh *netlink.Handle, iface netlink.Link, i *Interface) error {
 	if i.AddressIPv6() == nil {
 		return nil
 	}
@@ -356,7 +358,7 @@ func setInterfaceIPv6(nlh *netlink.Handle, iface netlink.Link, i *nwIface) error
 	return nlh.AddrAdd(iface, ipAddr)
 }
 
-func setInterfaceLinkLocalIPs(nlh *netlink.Handle, iface netlink.Link, i *nwIface) error {
+func setInterfaceLinkLocalIPs(nlh *netlink.Handle, iface netlink.Link, i *Interface) error {
 	for _, llIP := range i.LinkLocalAddresses() {
 		ipAddr := &netlink.Addr{IPNet: llIP}
 		if err := nlh.AddrAdd(iface, ipAddr); err != nil {
@@ -366,11 +368,11 @@ func setInterfaceLinkLocalIPs(nlh *netlink.Handle, iface netlink.Link, i *nwIfac
 	return nil
 }
 
-func setInterfaceName(nlh *netlink.Handle, iface netlink.Link, i *nwIface) error {
+func setInterfaceName(nlh *netlink.Handle, iface netlink.Link, i *Interface) error {
 	return nlh.LinkSetName(iface, i.DstName())
 }
 
-func setInterfaceRoutes(nlh *netlink.Handle, iface netlink.Link, i *nwIface) error {
+func setInterfaceRoutes(nlh *netlink.Handle, iface netlink.Link, i *Interface) error {
 	for _, route := range i.Routes() {
 		err := nlh.RouteAdd(&netlink.Route{
 			Scope:     netlink.SCOPE_LINK,

libnetwork/osl/interface_unsupported.go

@@ -2,4 +2,4 @@
 
 package osl
 
-type nwIface struct{}
+type Interface struct{}

libnetwork/osl/namespace_linux.go

@@ -313,12 +313,13 @@ func createNamespaceFile(path string) (err error) {
 	return err
 }
 
-// The networkNamespace type is the linux implementation of the Sandbox
-// interface. It represents a linux network namespace, and moves an interface
-// into it when called on method AddInterface or sets the gateway etc.
+// networkNamespace represents a network sandbox. It represents a Linux network
+// namespace, and moves an interface into it when called on method AddInterface
+// or sets the gateway etc. It holds a list of Interfaces, routes etc., and more
+// can be added dynamically.
 type networkNamespace struct {
 	path         string
-	iFaces       []*nwIface
+	iFaces       []*Interface
 	gw           net.IP
 	gwv6         net.IP
 	staticRoutes []*types.StaticRoute
@@ -330,22 +331,16 @@ type networkNamespace struct {
 	sync.Mutex
 }
 
-func (n *networkNamespace) Interfaces() []Interface {
-	ifaces := make([]Interface, len(n.iFaces))
-	for i, iface := range n.iFaces {
-		ifaces[i] = iface
-	}
+// Interfaces returns the collection of Interface previously added with the AddInterface
+// method. Note that this doesn't include network interfaces added in any
+// other way (such as the default loopback interface which is automatically
+// created on creation of a sandbox).
+func (n *networkNamespace) Interfaces() []*Interface {
+	ifaces := make([]*Interface, len(n.iFaces))
+	copy(ifaces, n.iFaces)
 	return ifaces
 }
 
-func (n *networkNamespace) InterfaceOptions() IfaceOptionSetter {
-	return n
-}
-
-func (n *networkNamespace) NeighborOptions() NeighborOptionSetter {
-	return n
-}
-
 func (n *networkNamespace) loopbackUp() error {
 	iface, err := n.nlHandle.LinkByName("lo")
 	if err != nil {
@@ -354,10 +349,12 @@ func (n *networkNamespace) loopbackUp() error {
 	return n.nlHandle.LinkSetUp(iface)
 }
 
+// GetLoopbackIfaceName returns the name of the loopback interface
 func (n *networkNamespace) GetLoopbackIfaceName() string {
 	return "lo"
 }
 
+// AddAliasIP adds the passed IP address to the named interface
 func (n *networkNamespace) AddAliasIP(ifName string, ip *net.IPNet) error {
 	iface, err := n.nlHandle.LinkByName(ifName)
 	if err != nil {
@@ -366,6 +363,7 @@ func (n *networkNamespace) AddAliasIP(ifName string, ip *net.IPNet) error {
 	return n.nlHandle.AddrAdd(iface, &netlink.Addr{IPNet: ip})
 }
 
+// RemoveAliasIP removes the passed IP address from the named interface
 func (n *networkNamespace) RemoveAliasIP(ifName string, ip *net.IPNet) error {
 	iface, err := n.nlHandle.LinkByName(ifName)
 	if err != nil {
@@ -374,6 +372,8 @@ func (n *networkNamespace) RemoveAliasIP(ifName string, ip *net.IPNet) error {
 	return n.nlHandle.AddrDel(iface, &netlink.Addr{IPNet: ip})
 }
 
+// DisableARPForVIP disables ARP replies and requests for VIP addresses
+// on a particular interface.
 func (n *networkNamespace) DisableARPForVIP(srcName string) (Err error) {
 	dstName := ""
 	for _, i := range n.Interfaces() {
@@ -404,6 +404,7 @@ func (n *networkNamespace) DisableARPForVIP(srcName string) (Err error) {
 	return
 }
 
+// InvokeFunc invoke a function in the network namespace.
 func (n *networkNamespace) InvokeFunc(f func()) error {
 	path := n.nsPath()
 	newNS, err := netns.GetFromPath(path)
@@ -455,10 +456,12 @@ func (n *networkNamespace) nsPath() string {
 	return n.path
 }
 
+// Key returns the path where the network namespace is mounted.
 func (n *networkNamespace) Key() string {
 	return n.path
 }
 
+// Destroy destroys the sandbox.
 func (n *networkNamespace) Destroy() error {
 	if n.nlHandle != nil {
 		n.nlHandle.Close()
@@ -474,16 +477,18 @@ func (n *networkNamespace) Destroy() error {
 	return nil
 }
 
-// Restore restore the network namespace
+// Restore restores the network namespace.
 func (n *networkNamespace) Restore(ifsopt map[Iface][]IfaceOption, routes []*types.StaticRoute, gw net.IP, gw6 net.IP) error {
 	// restore interfaces
 	for name, opts := range ifsopt {
-		i := &nwIface{
+		i := &Interface{
 			srcName: name.SrcName,
 			dstName: name.DstPrefix,
 			ns:      n,
 		}
-		i.processInterfaceOptions(opts...)
+		if err := i.processInterfaceOptions(opts...); err != nil {
+			return err
+		}
 		if i.master != "" {
 			i.dstMaster = n.findDst(i.master, true)
 			if i.dstMaster == "" {
@@ -602,7 +607,7 @@ func (n *networkNamespace) checkLoV6() {
 	n.loV6Enabled = enable
 }
 
-// ApplyOSTweaks applies linux configs on the sandbox
+// ApplyOSTweaks applies operating system specific knobs on the sandbox.
 func (n *networkNamespace) ApplyOSTweaks(types []SandboxType) {
 	for _, t := range types {
 		switch t {

libnetwork/osl/neigh_linux.go

@@ -42,6 +42,7 @@ func (n *networkNamespace) findNeighbor(dstIP net.IP, dstMac net.HardwareAddr) *
 	return nil
 }
 
+// DeleteNeighbor deletes neighbor entry from the sandbox.
 func (n *networkNamespace) DeleteNeighbor(dstIP net.IP, dstMac net.HardwareAddr, osDelete bool) error {
 	var (
 		iface netlink.Link
@@ -119,6 +120,7 @@ func (n *networkNamespace) DeleteNeighbor(dstIP net.IP, dstMac net.HardwareAddr,
 	return nil
 }
 
+// AddNeighbor adds a neighbor entry into the sandbox.
 func (n *networkNamespace) AddNeighbor(dstIP net.IP, dstMac net.HardwareAddr, force bool, options ...NeighOption) error {
 	var (
 		iface                  netlink.Link

libnetwork/osl/options_linux.go

@@ -10,64 +10,86 @@ func (nh *neigh) processNeighOptions(options ...NeighOption) {
 	}
 }
 
-func (n *networkNamespace) LinkName(name string) NeighOption {
+// WithLinkName sets the srcName of the link to use in the neighbor entry.
+func WithLinkName(name string) NeighOption {
 	return func(nh *neigh) {
 		nh.linkName = name
 	}
 }
 
-func (n *networkNamespace) Family(family int) NeighOption {
+// WithFamily sets the address-family for the neighbor entry. e.g. [syscall.AF_BRIDGE].
+func WithFamily(family int) NeighOption {
 	return func(nh *neigh) {
 		nh.family = family
 	}
 }
 
-func (i *nwIface) processInterfaceOptions(options ...IfaceOption) {
+func (i *Interface) processInterfaceOptions(options ...IfaceOption) error {
 	for _, opt := range options {
 		if opt != nil {
-			opt(i)
+			// TODO(thaJeztah): use multi-error instead of returning early.
+			if err := opt(i); err != nil {
+				return err
+			}
 		}
 	}
+	return nil
 }
 
-func (n *networkNamespace) Bridge(isBridge bool) IfaceOption {
-	return func(i *nwIface) {
+// WithIsBridge sets whether the interface is a bridge.
+func WithIsBridge(isBridge bool) IfaceOption {
+	return func(i *Interface) error {
 		i.bridge = isBridge
+		return nil
 	}
 }
 
-func (n *networkNamespace) Master(name string) IfaceOption {
-	return func(i *nwIface) {
+// WithMaster sets the master interface (if any) for this interface. The
+// master interface name should refer to the srcName of a previously added
+// interface of type bridge.
+func WithMaster(name string) IfaceOption {
+	return func(i *Interface) error {
 		i.master = name
+		return nil
 	}
 }
 
-func (n *networkNamespace) MacAddress(mac net.HardwareAddr) IfaceOption {
-	return func(i *nwIface) {
+// WithMACAddress sets the interface MAC-address.
+func WithMACAddress(mac net.HardwareAddr) IfaceOption {
+	return func(i *Interface) error {
 		i.mac = mac
+		return nil
 	}
 }
 
-func (n *networkNamespace) Address(addr *net.IPNet) IfaceOption {
-	return func(i *nwIface) {
+// WithIPv4Address sets the IPv4 address of the interface.
+func WithIPv4Address(addr *net.IPNet) IfaceOption {
+	return func(i *Interface) error {
 		i.address = addr
+		return nil
 	}
 }
 
-func (n *networkNamespace) AddressIPv6(addr *net.IPNet) IfaceOption {
-	return func(i *nwIface) {
+// WithIPv6Address sets the IPv6 address of the interface.
+func WithIPv6Address(addr *net.IPNet) IfaceOption {
+	return func(i *Interface) error {
 		i.addressIPv6 = addr
+		return nil
 	}
 }
 
-func (n *networkNamespace) LinkLocalAddresses(list []*net.IPNet) IfaceOption {
-	return func(i *nwIface) {
+// WithLinkLocalAddresses set the link-local IP addresses of the interface.
+func WithLinkLocalAddresses(list []*net.IPNet) IfaceOption {
+	return func(i *Interface) error {
 		i.llAddrs = list
+		return nil
 	}
 }
 
-func (n *networkNamespace) Routes(routes []*net.IPNet) IfaceOption {
-	return func(i *nwIface) {
+// WithRoutes sets the interface routes.
+func WithRoutes(routes []*net.IPNet) IfaceOption {
+	return func(i *Interface) error {
 		i.routes = routes
+		return nil
 	}
 }

libnetwork/osl/route_linux.go

@@ -8,6 +8,7 @@ import (
 	"github.com/vishvananda/netlink"
 )
 
+// Gateway returns the IPv4 gateway for the sandbox.
 func (n *networkNamespace) Gateway() net.IP {
 	n.Lock()
 	defer n.Unlock()
@@ -15,6 +16,7 @@ func (n *networkNamespace) Gateway() net.IP {
 	return n.gw
 }
 
+// GatewayIPv6 returns the IPv6 gateway for the sandbox.
 func (n *networkNamespace) GatewayIPv6() net.IP {
 	n.Lock()
 	defer n.Unlock()
@@ -22,6 +24,9 @@ func (n *networkNamespace) GatewayIPv6() net.IP {
 	return n.gwv6
 }
 
+// StaticRoutes returns additional static routes for the sandbox. Note that
+// directly connected routes are stored on the particular interface they
+// refer to.
 func (n *networkNamespace) StaticRoutes() []*types.StaticRoute {
 	n.Lock()
 	defer n.Unlock()
@@ -47,6 +52,7 @@ func (n *networkNamespace) setGatewayIPv6(gwv6 net.IP) {
 	n.Unlock()
 }
 
+// SetGateway sets the default IPv4 gateway for the sandbox.
 func (n *networkNamespace) SetGateway(gw net.IP) error {
 	// Silently return if the gateway is empty
 	if len(gw) == 0 {
@@ -61,6 +67,7 @@ func (n *networkNamespace) SetGateway(gw net.IP) error {
 	return err
 }
 
+// UnsetGateway the previously set default IPv4 gateway in the sandbox.
 func (n *networkNamespace) UnsetGateway() error {
 	gw := n.Gateway()
 
@@ -140,6 +147,7 @@ func (n *networkNamespace) removeRoute(path string, dest *net.IPNet, nh net.IP)
 	})
 }
 
+// SetGatewayIPv6 sets the default IPv6 gateway for the sandbox.
 func (n *networkNamespace) SetGatewayIPv6(gwv6 net.IP) error {
 	// Silently return if the gateway is empty
 	if len(gwv6) == 0 {
@@ -154,6 +162,7 @@ func (n *networkNamespace) SetGatewayIPv6(gwv6 net.IP) error {
 	return err
 }
 
+// UnsetGatewayIPv6 unsets the previously set default IPv6 gateway in the sandbox.
 func (n *networkNamespace) UnsetGatewayIPv6() error {
 	gwv6 := n.GatewayIPv6()
 
@@ -172,6 +181,7 @@ func (n *networkNamespace) UnsetGatewayIPv6() error {
 	return err
 }
 
+// AddStaticRoute adds a static route to the sandbox.
 func (n *networkNamespace) AddStaticRoute(r *types.StaticRoute) error {
 	err := n.programRoute(n.nsPath(), r.Destination, r.NextHop)
 	if err == nil {
@@ -182,6 +192,7 @@ func (n *networkNamespace) AddStaticRoute(r *types.StaticRoute) error {
 	return err
 }
 
+// RemoveStaticRoute removes a static route from the sandbox.
 func (n *networkNamespace) RemoveStaticRoute(r *types.StaticRoute) error {
 	err := n.removeRoute(n.nsPath(), r.Destination, r.NextHop)
 	if err == nil {

libnetwork/osl/sandbox.go

@@ -22,7 +22,7 @@ type Iface struct {
 }
 
 // IfaceOption is a function option type to set interface options.
-type IfaceOption func(i *nwIface)
+type IfaceOption func(i *Interface) error
 
 // NeighOption is a function option type to set neighbor options.
 type NeighOption func(nh *neigh)
@@ -77,12 +77,6 @@ type Sandbox interface {
 	// DeleteNeighbor deletes neighbor entry from the sandbox.
 	DeleteNeighbor(dstIP net.IP, dstMac net.HardwareAddr, osDelete bool) error
 
-	// NeighborOptions returns an interface with methods to set neighbor options.
-	NeighborOptions() NeighborOptionSetter
-
-	// InterfaceOptions an interface with methods to set interface options.
-	InterfaceOptions() IfaceOptionSetter
-
 	// InvokeFunc invoke a function in the network namespace.
 	InvokeFunc(func()) error
 
@@ -98,43 +92,6 @@ type Sandbox interface {
 	Info
 }
 
-// NeighborOptionSetter interface defines the option setter methods for interface options
-type NeighborOptionSetter interface {
-	// LinkName returns an option setter to set the srcName of the link that should
-	// be used in the neighbor entry
-	LinkName(string) NeighOption
-
-	// Family returns an option setter to set the address family for the neighbor
-	// entry. eg. AF_BRIDGE
-	Family(int) NeighOption
-}
-
-// IfaceOptionSetter interface defines the option setter methods for interface options.
-type IfaceOptionSetter interface {
-	// Bridge returns an option setter to set if the interface is a bridge.
-	Bridge(bool) IfaceOption
-
-	// MacAddress returns an option setter to set the MAC address.
-	MacAddress(net.HardwareAddr) IfaceOption
-
-	// Address returns an option setter to set IPv4 address.
-	Address(*net.IPNet) IfaceOption
-
-	// AddressIPv6 returns an option setter to set IPv6 address.
-	AddressIPv6(*net.IPNet) IfaceOption
-
-	// LinkLocalAddresses returns an option setter to set the link-local IP addresses.
-	LinkLocalAddresses([]*net.IPNet) IfaceOption
-
-	// Master returns an option setter to set the master interface if any for this
-	// interface. The master interface name should refer to the srcname of a
-	// previously added interface of type bridge.
-	Master(string) IfaceOption
-
-	// Routes returns an option setter to set interface routes.
-	Routes([]*net.IPNet) IfaceOption
-}
-
 // Info represents all possible information that
 // the driver wants to place in the sandbox which includes
 // interfaces, routes and gateway
@@ -143,7 +100,7 @@ type Info interface {
 	// method. Note that this doesn't include network interfaces added in any
 	// other way (such as the default loopback interface which is automatically
 	// created on creation of a sandbox).
-	Interfaces() []Interface
+	Interfaces() []*Interface
 
 	// Gateway returns the IPv4 gateway for the sandbox.
 	Gateway() net.IP
@@ -156,45 +113,3 @@ type Info interface {
 	// refer to.
 	StaticRoutes() []*types.StaticRoute
 }
-
-// Interface represents the settings and identity of a network device. It is
-// used as a return type for Network.Link, and it is common practice for the
-// caller to use this information when moving interface SrcName from host
-// namespace to DstName in a different net namespace with the appropriate
-// network settings.
-type Interface interface {
-	// SrcName returns the name of the interface in the origin network namespace.
-	SrcName() string
-
-	// DstName returns the name that will be assigned to the interface once
-	// moved inside a network namespace. When the caller passes in a DstName,
-	// it is only expected to pass a prefix. The name will be modified with an
-	// auto-generated suffix.
-	DstName() string
-
-	// Address returns the IPv4 address for the interface.
-	Address() *net.IPNet
-
-	// AddressIPv6 returns the IPv6 address for the interface.
-	AddressIPv6() *net.IPNet
-
-	// LinkLocalAddresses returns the link-local IP addresses assigned to the
-	// interface.
-	LinkLocalAddresses() []*net.IPNet
-
-	// Routes returns IP routes for the interface.
-	Routes() []*net.IPNet
-
-	// Bridge returns true if the interface is a bridge.
-	Bridge() bool
-
-	// Master returns the srcname of the master interface for this interface.
-	Master() string
-
-	// Remove an interface from the sandbox by renaming to original name
-	// and moving it out of the sandbox.
-	Remove() error
-
-	// Statistics returns the statistics for this interface
-	Statistics() (*types.InterfaceStatistics, error)
-}

libnetwork/osl/sandbox_linux_test.go

@@ -85,7 +85,7 @@ func newInfo(t *testing.T, hnd *netlink.Handle) (Sandbox, error) {
 
 	// Store the sandbox side pipe interface
 	// This is needed for cleanup on DeleteEndpoint()
-	intf1 := &nwIface{
+	intf1 := &Interface{
 		srcName:     vethName2,
 		dstName:     sboxIfaceName,
 		address:     addr,
@@ -93,7 +93,7 @@ func newInfo(t *testing.T, hnd *netlink.Handle) (Sandbox, error) {
 		routes:      []*net.IPNet{route},
 	}
 
-	intf2 := &nwIface{
+	intf2 := &Interface{
 		srcName: "testbridge",
 		dstName: sboxIfaceName,
 		bridge:  true,
@@ -107,14 +107,14 @@ func newInfo(t *testing.T, hnd *netlink.Handle) (Sandbox, error) {
 		return nil, err
 	}
 
-	intf3 := &nwIface{
+	intf3 := &Interface{
 		srcName: vethName4,
 		dstName: sboxIfaceName,
 		master:  "testbridge",
 	}
 
 	return &networkNamespace{
-		iFaces: []*nwIface{intf1, intf2, intf3},
+		iFaces: []*Interface{intf1, intf2, intf3},
 		gw:     net.ParseIP("192.168.1.1"),
 		gwv6:   net.ParseIP("fe80::1"),
 	}, nil
@@ -182,7 +182,7 @@ func TestDisableIPv6DAD(t *testing.T) {
 	nlh := n.nlHandle
 
 	ipv6, _ := types.ParseCIDR("2001:db8::44/64")
-	iface := &nwIface{addressIPv6: ipv6, ns: n, dstName: "sideA"}
+	iface := &Interface{addressIPv6: ipv6, ns: n, dstName: "sideA"}
 
 	veth := &netlink.Veth{
 		LinkAttrs: netlink.LinkAttrs{Name: "sideA"},
@@ -242,7 +242,7 @@ func TestSetInterfaceIP(t *testing.T) {
 
 	ipv4, _ := types.ParseCIDR("172.30.0.33/24")
 	ipv6, _ := types.ParseCIDR("2001:db8::44/64")
-	iface := &nwIface{address: ipv4, addressIPv6: ipv6, ns: n, dstName: "sideA"}
+	iface := &Interface{address: ipv4, addressIPv6: ipv6, ns: n, dstName: "sideA"}
 
 	if err := nlh.LinkAdd(&netlink.Veth{
 		LinkAttrs: netlink.LinkAttrs{Name: "sideA"},
@@ -316,7 +316,7 @@ func TestLiveRestore(t *testing.T) {
 
 	ipv4, _ := types.ParseCIDR("172.30.0.33/24")
 	ipv6, _ := types.ParseCIDR("2001:db8::44/64")
-	iface := &nwIface{address: ipv4, addressIPv6: ipv6, ns: n, dstName: "sideA"}
+	iface := &Interface{address: ipv4, addressIPv6: ipv6, ns: n, dstName: "sideA"}
 
 	if err := nlh.LinkAdd(&netlink.Veth{
 		LinkAttrs: netlink.LinkAttrs{Name: "sideA"},
@@ -409,9 +409,9 @@ func TestSandboxCreate(t *testing.T) {
 
 	for _, i := range tbox.Interfaces() {
 		err = s.AddInterface(i.SrcName(), i.DstName(),
-			tbox.InterfaceOptions().Bridge(i.Bridge()),
-			tbox.InterfaceOptions().Address(i.Address()),
-			tbox.InterfaceOptions().AddressIPv6(i.AddressIPv6()))
+			WithIsBridge(i.Bridge()),
+			WithIPv4Address(i.Address()),
+			WithIPv6Address(i.AddressIPv6()))
 		if err != nil {
 			t.Fatalf("Failed to add interfaces to sandbox: %v", err)
 		}
@@ -508,9 +508,10 @@ func TestAddRemoveInterface(t *testing.T) {
 
 	for _, i := range tbox.Interfaces() {
 		err = s.AddInterface(i.SrcName(), i.DstName(),
-			tbox.InterfaceOptions().Bridge(i.Bridge()),
-			tbox.InterfaceOptions().Address(i.Address()),
-			tbox.InterfaceOptions().AddressIPv6(i.AddressIPv6()))
+			WithIsBridge(i.Bridge()),
+			WithIPv4Address(i.Address()),
+			WithIPv6Address(i.AddressIPv6()),
+		)
 		if err != nil {
 			t.Fatalf("Failed to add interfaces to sandbox: %v", err)
 		}
@@ -526,10 +527,12 @@ func TestAddRemoveInterface(t *testing.T) {
 	verifySandbox(t, s, []string{"1", "2"})
 
 	i := tbox.Interfaces()[0]
-	if err := s.AddInterface(i.SrcName(), i.DstName(),
-		tbox.InterfaceOptions().Bridge(i.Bridge()),
-		tbox.InterfaceOptions().Address(i.Address()),
-		tbox.InterfaceOptions().AddressIPv6(i.AddressIPv6())); err != nil {
+	err = s.AddInterface(i.SrcName(), i.DstName(),
+		WithIsBridge(i.Bridge()),
+		WithIPv4Address(i.Address()),
+		WithIPv6Address(i.AddressIPv6()),
+	)
+	if err != nil {
 		t.Fatalf("Failed to add interfaces to sandbox: %v", err)
 	}
 

libnetwork/sandbox.go

@@ -8,7 +8,6 @@ import (
 	"sort"
 	"strings"
 	"sync"
-	"time"
 
 	"github.com/containerd/containerd/log"
 	"github.com/docker/docker/libnetwork/etchosts"
@@ -127,27 +126,6 @@ func (sb *Sandbox) Labels() map[string]interface{} {
 	return opts
 }
 
-// Statistics retrieves the interfaces' statistics for the sandbox.
-func (sb *Sandbox) Statistics() (map[string]*types.InterfaceStatistics, error) {
-	m := make(map[string]*types.InterfaceStatistics)
-
-	sb.mu.Lock()
-	osb := sb.osSbox
-	sb.mu.Unlock()
-	if osb == nil {
-		return m, nil
-	}
-
-	var err error
-	for _, i := range osb.Interfaces() {
-		if m[i.DstName()], err = i.Statistics(); err != nil {
-			return m, err
-		}
-	}
-
-	return m, nil
-}
-
 // Delete destroys this container after detaching it from all connected endpoints.
 func (sb *Sandbox) Delete() error {
 	return sb.delete(false)
@@ -366,35 +344,6 @@ func (sb *Sandbox) getEndpoint(id string) *Endpoint {
 	return nil
 }
 
-func (sb *Sandbox) updateGateway(ep *Endpoint) error {
-	sb.mu.Lock()
-	osSbox := sb.osSbox
-	sb.mu.Unlock()
-	if osSbox == nil {
-		return nil
-	}
-	osSbox.UnsetGateway()     //nolint:errcheck
-	osSbox.UnsetGatewayIPv6() //nolint:errcheck
-
-	if ep == nil {
-		return nil
-	}
-
-	ep.mu.Lock()
-	joinInfo := ep.joinInfo
-	ep.mu.Unlock()
-
-	if err := osSbox.SetGateway(joinInfo.gw); err != nil {
-		return fmt.Errorf("failed to set gateway while updating gateway: %v", err)
-	}
-
-	if err := osSbox.SetGatewayIPv6(joinInfo.gw6); err != nil {
-		return fmt.Errorf("failed to set IPv6 gateway while updating gateway: %v", err)
-	}
-
-	return nil
-}
-
 func (sb *Sandbox) HandleQueryResp(name string, ip net.IP) {
 	for _, ep := range sb.Endpoints() {
 		n := ep.getNetwork()
@@ -417,16 +366,6 @@ func (sb *Sandbox) ResolveIP(ip string) string {
 	return svc
 }
 
-func (sb *Sandbox) ExecFunc(f func()) error {
-	sb.mu.Lock()
-	osSbox := sb.osSbox
-	sb.mu.Unlock()
-	if osSbox != nil {
-		return osSbox.InvokeFunc(f)
-	}
-	return fmt.Errorf("osl sandbox unavailable in ExecFunc for %v", sb.ContainerID())
-}
-
 // ResolveService returns all the backend details about the containers or hosts
 // backing a service. Its purpose is to satisfy an SRV query.
 func (sb *Sandbox) ResolveService(name string) ([]*net.SRV, []net.IP) {
@@ -600,63 +539,6 @@ func (sb *Sandbox) resolveName(req string, networkName string, epList []*Endpoin
 	return nil, ipv6Miss
 }
 
-// SetKey updates the Sandbox Key.
-func (sb *Sandbox) SetKey(basePath string) error {
-	start := time.Now()
-	defer func() {
-		log.G(context.TODO()).Debugf("sandbox set key processing took %s for container %s", time.Since(start), sb.ContainerID())
-	}()
-
-	if basePath == "" {
-		return types.InvalidParameterErrorf("invalid sandbox key")
-	}
-
-	sb.mu.Lock()
-	if sb.inDelete {
-		sb.mu.Unlock()
-		return types.ForbiddenErrorf("failed to SetKey: sandbox %q delete in progress", sb.id)
-	}
-	oldosSbox := sb.osSbox
-	sb.mu.Unlock()
-
-	if oldosSbox != nil {
-		// If we already have an OS sandbox, release the network resources from that
-		// and destroy the OS snab. We are moving into a new home further down. Note that none
-		// of the network resources gets destroyed during the move.
-		sb.releaseOSSbox()
-	}
-
-	osSbox, err := osl.GetSandboxForExternalKey(basePath, sb.Key())
-	if err != nil {
-		return err
-	}
-
-	sb.mu.Lock()
-	sb.osSbox = osSbox
-	sb.mu.Unlock()
-
-	// If the resolver was setup before stop it and set it up in the
-	// new osl sandbox.
-	if oldosSbox != nil && sb.resolver != nil {
-		sb.resolver.Stop()
-
-		if err := sb.osSbox.InvokeFunc(sb.resolver.SetupFunc(0)); err == nil {
-			if err := sb.resolver.Start(); err != nil {
-				log.G(context.TODO()).Errorf("Resolver Start failed for container %s, %q", sb.ContainerID(), err)
-			}
-		} else {
-			log.G(context.TODO()).Errorf("Resolver Setup Function failed for container %s, %q", sb.ContainerID(), err)
-		}
-	}
-
-	for _, ep := range sb.Endpoints() {
-		if err = sb.populateNetworkResources(ep); err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
 // EnableService makes a managed container's service available by adding the
 // endpoint to the service load balancer and service discovery.
 func (sb *Sandbox) EnableService() (err error) {
@@ -703,195 +585,6 @@ func (sb *Sandbox) DisableService() (err error) {
 	return nil
 }
 
-func releaseOSSboxResources(osSbox osl.Sandbox, ep *Endpoint) {
-	for _, i := range osSbox.Interfaces() {
-		// Only remove the interfaces owned by this endpoint from the sandbox.
-		if ep.hasInterface(i.SrcName()) {
-			if err := i.Remove(); err != nil {
-				log.G(context.TODO()).Debugf("Remove interface %s failed: %v", i.SrcName(), err)
-			}
-		}
-	}
-
-	ep.mu.Lock()
-	joinInfo := ep.joinInfo
-	vip := ep.virtualIP
-	lbModeIsDSR := ep.network.loadBalancerMode == loadBalancerModeDSR
-	ep.mu.Unlock()
-
-	if len(vip) > 0 && lbModeIsDSR {
-		ipNet := &net.IPNet{IP: vip, Mask: net.CIDRMask(32, 32)}
-		if err := osSbox.RemoveAliasIP(osSbox.GetLoopbackIfaceName(), ipNet); err != nil {
-			log.G(context.TODO()).WithError(err).Debugf("failed to remove virtual ip %v to loopback", ipNet)
-		}
-	}
-
-	if joinInfo == nil {
-		return
-	}
-
-	// Remove non-interface routes.
-	for _, r := range joinInfo.StaticRoutes {
-		if err := osSbox.RemoveStaticRoute(r); err != nil {
-			log.G(context.TODO()).Debugf("Remove route failed: %v", err)
-		}
-	}
-}
-
-func (sb *Sandbox) releaseOSSbox() {
-	sb.mu.Lock()
-	osSbox := sb.osSbox
-	sb.osSbox = nil
-	sb.mu.Unlock()
-
-	if osSbox == nil {
-		return
-	}
-
-	for _, ep := range sb.Endpoints() {
-		releaseOSSboxResources(osSbox, ep)
-	}
-
-	if err := osSbox.Destroy(); err != nil {
-		log.G(context.TODO()).WithError(err).Error("Error destroying os sandbox")
-	}
-}
-
-func (sb *Sandbox) restoreOslSandbox() error {
-	var routes []*types.StaticRoute
-
-	// restore osl sandbox
-	Ifaces := make(map[osl.Iface][]osl.IfaceOption)
-	for _, ep := range sb.endpoints {
-		ep.mu.Lock()
-		joinInfo := ep.joinInfo
-		i := ep.iface
-		ep.mu.Unlock()
-
-		if i == nil {
-			log.G(context.TODO()).Errorf("error restoring endpoint %s for container %s", ep.Name(), sb.ContainerID())
-			continue
-		}
-
-		ifaceOptions := []osl.IfaceOption{
-			sb.osSbox.InterfaceOptions().Address(i.addr),
-			sb.osSbox.InterfaceOptions().Routes(i.routes),
-		}
-		if i.addrv6 != nil && i.addrv6.IP.To16() != nil {
-			ifaceOptions = append(ifaceOptions, sb.osSbox.InterfaceOptions().AddressIPv6(i.addrv6))
-		}
-		if i.mac != nil {
-			ifaceOptions = append(ifaceOptions, sb.osSbox.InterfaceOptions().MacAddress(i.mac))
-		}
-		if len(i.llAddrs) != 0 {
-			ifaceOptions = append(ifaceOptions, sb.osSbox.InterfaceOptions().LinkLocalAddresses(i.llAddrs))
-		}
-		Ifaces[osl.Iface{SrcName: i.srcName, DstPrefix: i.dstPrefix}] = ifaceOptions
-		if joinInfo != nil {
-			routes = append(routes, joinInfo.StaticRoutes...)
-		}
-		if ep.needResolver() {
-			sb.startResolver(true)
-		}
-	}
-
-	gwep := sb.getGatewayEndpoint()
-	if gwep == nil {
-		return nil
-	}
-
-	// restore osl sandbox
-	return sb.osSbox.Restore(Ifaces, routes, gwep.joinInfo.gw, gwep.joinInfo.gw6)
-}
-
-func (sb *Sandbox) populateNetworkResources(ep *Endpoint) error {
-	sb.mu.Lock()
-	if sb.osSbox == nil {
-		sb.mu.Unlock()
-		return nil
-	}
-	inDelete := sb.inDelete
-	sb.mu.Unlock()
-
-	ep.mu.Lock()
-	joinInfo := ep.joinInfo
-	i := ep.iface
-	lbModeIsDSR := ep.network.loadBalancerMode == loadBalancerModeDSR
-	ep.mu.Unlock()
-
-	if ep.needResolver() {
-		sb.startResolver(false)
-	}
-
-	if i != nil && i.srcName != "" {
-		var ifaceOptions []osl.IfaceOption
-
-		ifaceOptions = append(ifaceOptions, sb.osSbox.InterfaceOptions().Address(i.addr), sb.osSbox.InterfaceOptions().Routes(i.routes))
-		if i.addrv6 != nil && i.addrv6.IP.To16() != nil {
-			ifaceOptions = append(ifaceOptions, sb.osSbox.InterfaceOptions().AddressIPv6(i.addrv6))
-		}
-		if len(i.llAddrs) != 0 {
-			ifaceOptions = append(ifaceOptions, sb.osSbox.InterfaceOptions().LinkLocalAddresses(i.llAddrs))
-		}
-		if i.mac != nil {
-			ifaceOptions = append(ifaceOptions, sb.osSbox.InterfaceOptions().MacAddress(i.mac))
-		}
-
-		if err := sb.osSbox.AddInterface(i.srcName, i.dstPrefix, ifaceOptions...); err != nil {
-			return fmt.Errorf("failed to add interface %s to sandbox: %v", i.srcName, err)
-		}
-
-		if len(ep.virtualIP) > 0 && lbModeIsDSR {
-			if sb.loadBalancerNID == "" {
-				if err := sb.osSbox.DisableARPForVIP(i.srcName); err != nil {
-					return fmt.Errorf("failed disable ARP for VIP: %v", err)
-				}
-			}
-			ipNet := &net.IPNet{IP: ep.virtualIP, Mask: net.CIDRMask(32, 32)}
-			if err := sb.osSbox.AddAliasIP(sb.osSbox.GetLoopbackIfaceName(), ipNet); err != nil {
-				return fmt.Errorf("failed to add virtual ip %v to loopback: %v", ipNet, err)
-			}
-		}
-	}
-
-	if joinInfo != nil {
-		// Set up non-interface routes.
-		for _, r := range joinInfo.StaticRoutes {
-			if err := sb.osSbox.AddStaticRoute(r); err != nil {
-				return fmt.Errorf("failed to add static route %s: %v", r.Destination.String(), err)
-			}
-		}
-	}
-
-	if ep == sb.getGatewayEndpoint() {
-		if err := sb.updateGateway(ep); err != nil {
-			return err
-		}
-	}
-
-	// Make sure to add the endpoint to the populated endpoint set
-	// before populating loadbalancers.
-	sb.mu.Lock()
-	sb.populatedEndpoints[ep.ID()] = struct{}{}
-	sb.mu.Unlock()
-
-	// Populate load balancer only after updating all the other
-	// information including gateway and other routes so that
-	// loadbalancers are populated all the network state is in
-	// place in the sandbox.
-	sb.populateLoadBalancers(ep)
-
-	// Only update the store if we did not come here as part of
-	// sandbox delete. If we came here as part of delete then do
-	// not bother updating the store. The sandbox object will be
-	// deleted anyway
-	if !inDelete {
-		return sb.storeUpdate()
-	}
-
-	return nil
-}
-
 func (sb *Sandbox) clearNetworkResources(origEp *Endpoint) error {
 	ep := sb.getEndpoint(origEp.id)
 	if ep == nil {

libnetwork/sandbox_dns_windows.go

@@ -8,8 +8,6 @@ import (
 
 // Stub implementations for DNS related functions
 
-func (sb *Sandbox) startResolver(bool) {}
-
 func (sb *Sandbox) setupResolutionFiles() error {
 	return nil
 }

libnetwork/sandbox_linux.go

@@ -0,0 +1,318 @@
+package libnetwork
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"time"
+
+	"github.com/containerd/containerd/log"
+	"github.com/docker/docker/libnetwork/osl"
+	"github.com/docker/docker/libnetwork/types"
+)
+
+func releaseOSSboxResources(osSbox osl.Sandbox, ep *Endpoint) {
+	for _, i := range osSbox.Interfaces() {
+		// Only remove the interfaces owned by this endpoint from the sandbox.
+		if ep.hasInterface(i.SrcName()) {
+			if err := i.Remove(); err != nil {
+				log.G(context.TODO()).Debugf("Remove interface %s failed: %v", i.SrcName(), err)
+			}
+		}
+	}
+
+	ep.mu.Lock()
+	joinInfo := ep.joinInfo
+	vip := ep.virtualIP
+	lbModeIsDSR := ep.network.loadBalancerMode == loadBalancerModeDSR
+	ep.mu.Unlock()
+
+	if len(vip) > 0 && lbModeIsDSR {
+		ipNet := &net.IPNet{IP: vip, Mask: net.CIDRMask(32, 32)}
+		if err := osSbox.RemoveAliasIP(osSbox.GetLoopbackIfaceName(), ipNet); err != nil {
+			log.G(context.TODO()).WithError(err).Debugf("failed to remove virtual ip %v to loopback", ipNet)
+		}
+	}
+
+	if joinInfo == nil {
+		return
+	}
+
+	// Remove non-interface routes.
+	for _, r := range joinInfo.StaticRoutes {
+		if err := osSbox.RemoveStaticRoute(r); err != nil {
+			log.G(context.TODO()).Debugf("Remove route failed: %v", err)
+		}
+	}
+}
+
+// Statistics retrieves the interfaces' statistics for the sandbox.
+func (sb *Sandbox) Statistics() (map[string]*types.InterfaceStatistics, error) {
+	m := make(map[string]*types.InterfaceStatistics)
+
+	sb.mu.Lock()
+	osb := sb.osSbox
+	sb.mu.Unlock()
+	if osb == nil {
+		return m, nil
+	}
+
+	var err error
+	for _, i := range osb.Interfaces() {
+		if m[i.DstName()], err = i.Statistics(); err != nil {
+			return m, err
+		}
+	}
+
+	return m, nil
+}
+
+func (sb *Sandbox) updateGateway(ep *Endpoint) error {
+	sb.mu.Lock()
+	osSbox := sb.osSbox
+	sb.mu.Unlock()
+	if osSbox == nil {
+		return nil
+	}
+	osSbox.UnsetGateway()     //nolint:errcheck
+	osSbox.UnsetGatewayIPv6() //nolint:errcheck
+
+	if ep == nil {
+		return nil
+	}
+
+	ep.mu.Lock()
+	joinInfo := ep.joinInfo
+	ep.mu.Unlock()
+
+	if err := osSbox.SetGateway(joinInfo.gw); err != nil {
+		return fmt.Errorf("failed to set gateway while updating gateway: %v", err)
+	}
+
+	if err := osSbox.SetGatewayIPv6(joinInfo.gw6); err != nil {
+		return fmt.Errorf("failed to set IPv6 gateway while updating gateway: %v", err)
+	}
+
+	return nil
+}
+
+func (sb *Sandbox) ExecFunc(f func()) error {
+	sb.mu.Lock()
+	osSbox := sb.osSbox
+	sb.mu.Unlock()
+	if osSbox != nil {
+		return osSbox.InvokeFunc(f)
+	}
+	return fmt.Errorf("osl sandbox unavailable in ExecFunc for %v", sb.ContainerID())
+}
+
+// SetKey updates the Sandbox Key.
+func (sb *Sandbox) SetKey(basePath string) error {
+	start := time.Now()
+	defer func() {
+		log.G(context.TODO()).Debugf("sandbox set key processing took %s for container %s", time.Since(start), sb.ContainerID())
+	}()
+
+	if basePath == "" {
+		return types.InvalidParameterErrorf("invalid sandbox key")
+	}
+
+	sb.mu.Lock()
+	if sb.inDelete {
+		sb.mu.Unlock()
+		return types.ForbiddenErrorf("failed to SetKey: sandbox %q delete in progress", sb.id)
+	}
+	oldosSbox := sb.osSbox
+	sb.mu.Unlock()
+
+	if oldosSbox != nil {
+		// If we already have an OS sandbox, release the network resources from that
+		// and destroy the OS snab. We are moving into a new home further down. Note that none
+		// of the network resources gets destroyed during the move.
+		if err := sb.releaseOSSbox(); err != nil {
+			log.G(context.TODO()).WithError(err).Error("Error destroying os sandbox")
+		}
+	}
+
+	osSbox, err := osl.GetSandboxForExternalKey(basePath, sb.Key())
+	if err != nil {
+		return err
+	}
+
+	sb.mu.Lock()
+	sb.osSbox = osSbox
+	sb.mu.Unlock()
+
+	// If the resolver was setup before stop it and set it up in the
+	// new osl sandbox.
+	if oldosSbox != nil && sb.resolver != nil {
+		sb.resolver.Stop()
+
+		if err := sb.osSbox.InvokeFunc(sb.resolver.SetupFunc(0)); err == nil {
+			if err := sb.resolver.Start(); err != nil {
+				log.G(context.TODO()).Errorf("Resolver Start failed for container %s, %q", sb.ContainerID(), err)
+			}
+		} else {
+			log.G(context.TODO()).Errorf("Resolver Setup Function failed for container %s, %q", sb.ContainerID(), err)
+		}
+	}
+
+	for _, ep := range sb.Endpoints() {
+		if err = sb.populateNetworkResources(ep); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (sb *Sandbox) releaseOSSbox() error {
+	sb.mu.Lock()
+	osSbox := sb.osSbox
+	sb.osSbox = nil
+	sb.mu.Unlock()
+
+	if osSbox == nil {
+		return nil
+	}
+
+	for _, ep := range sb.Endpoints() {
+		releaseOSSboxResources(osSbox, ep)
+	}
+
+	return osSbox.Destroy()
+}
+
+func (sb *Sandbox) restoreOslSandbox() error {
+	var routes []*types.StaticRoute
+
+	// restore osl sandbox
+	interfaces := make(map[osl.Iface][]osl.IfaceOption)
+	for _, ep := range sb.endpoints {
+		ep.mu.Lock()
+		joinInfo := ep.joinInfo
+		i := ep.iface
+		ep.mu.Unlock()
+
+		if i == nil {
+			log.G(context.TODO()).Errorf("error restoring endpoint %s for container %s", ep.Name(), sb.ContainerID())
+			continue
+		}
+
+		ifaceOptions := []osl.IfaceOption{
+			osl.WithIPv4Address(i.addr),
+			osl.WithRoutes(i.routes),
+		}
+		if i.addrv6 != nil && i.addrv6.IP.To16() != nil {
+			ifaceOptions = append(ifaceOptions, osl.WithIPv6Address(i.addrv6))
+		}
+		if i.mac != nil {
+			ifaceOptions = append(ifaceOptions, osl.WithMACAddress(i.mac))
+		}
+		if len(i.llAddrs) != 0 {
+			ifaceOptions = append(ifaceOptions, osl.WithLinkLocalAddresses(i.llAddrs))
+		}
+		interfaces[osl.Iface{SrcName: i.srcName, DstPrefix: i.dstPrefix}] = ifaceOptions
+		if joinInfo != nil {
+			routes = append(routes, joinInfo.StaticRoutes...)
+		}
+		if ep.needResolver() {
+			sb.startResolver(true)
+		}
+	}
+
+	gwep := sb.getGatewayEndpoint()
+	if gwep == nil {
+		return nil
+	}
+
+	// restore osl sandbox
+	return sb.osSbox.Restore(interfaces, routes, gwep.joinInfo.gw, gwep.joinInfo.gw6)
+}
+
+func (sb *Sandbox) populateNetworkResources(ep *Endpoint) error {
+	sb.mu.Lock()
+	if sb.osSbox == nil {
+		sb.mu.Unlock()
+		return nil
+	}
+	inDelete := sb.inDelete
+	sb.mu.Unlock()
+
+	ep.mu.Lock()
+	joinInfo := ep.joinInfo
+	i := ep.iface
+	lbModeIsDSR := ep.network.loadBalancerMode == loadBalancerModeDSR
+	ep.mu.Unlock()
+
+	if ep.needResolver() {
+		sb.startResolver(false)
+	}
+
+	if i != nil && i.srcName != "" {
+		var ifaceOptions []osl.IfaceOption
+
+		ifaceOptions = append(ifaceOptions, osl.WithIPv4Address(i.addr), osl.WithRoutes(i.routes))
+		if i.addrv6 != nil && i.addrv6.IP.To16() != nil {
+			ifaceOptions = append(ifaceOptions, osl.WithIPv6Address(i.addrv6))
+		}
+		if len(i.llAddrs) != 0 {
+			ifaceOptions = append(ifaceOptions, osl.WithLinkLocalAddresses(i.llAddrs))
+		}
+		if i.mac != nil {
+			ifaceOptions = append(ifaceOptions, osl.WithMACAddress(i.mac))
+		}
+
+		if err := sb.osSbox.AddInterface(i.srcName, i.dstPrefix, ifaceOptions...); err != nil {
+			return fmt.Errorf("failed to add interface %s to sandbox: %v", i.srcName, err)
+		}
+
+		if len(ep.virtualIP) > 0 && lbModeIsDSR {
+			if sb.loadBalancerNID == "" {
+				if err := sb.osSbox.DisableARPForVIP(i.srcName); err != nil {
+					return fmt.Errorf("failed disable ARP for VIP: %v", err)
+				}
+			}
+			ipNet := &net.IPNet{IP: ep.virtualIP, Mask: net.CIDRMask(32, 32)}
+			if err := sb.osSbox.AddAliasIP(sb.osSbox.GetLoopbackIfaceName(), ipNet); err != nil {
+				return fmt.Errorf("failed to add virtual ip %v to loopback: %v", ipNet, err)
+			}
+		}
+	}
+
+	if joinInfo != nil {
+		// Set up non-interface routes.
+		for _, r := range joinInfo.StaticRoutes {
+			if err := sb.osSbox.AddStaticRoute(r); err != nil {
+				return fmt.Errorf("failed to add static route %s: %v", r.Destination.String(), err)
+			}
+		}
+	}
+
+	if ep == sb.getGatewayEndpoint() {
+		if err := sb.updateGateway(ep); err != nil {
+			return err
+		}
+	}
+
+	// Make sure to add the endpoint to the populated endpoint set
+	// before populating loadbalancers.
+	sb.mu.Lock()
+	sb.populatedEndpoints[ep.ID()] = struct{}{}
+	sb.mu.Unlock()
+
+	// Populate load balancer only after updating all the other
+	// information including gateway and other routes so that
+	// loadbalancers are populated all the network state is in
+	// place in the sandbox.
+	sb.populateLoadBalancers(ep)
+
+	// Only update the store if we did not come here as part of
+	// sandbox delete. If we came here as part of delete then do
+	// not bother updating the store. The sandbox object will be
+	// deleted anyway
+	if !inDelete {
+		return sb.storeUpdate()
+	}
+
+	return nil
+}

libnetwork/sandbox_store.go

@@ -266,7 +266,8 @@ func (c *Controller) sandboxCleanup(activeSandboxes map[string]interface{}) {
 				continue
 			}
 		} else {
-			c.sboxOnce.Do(func() {
+			// FIXME(thaJeztah): osSbox (and thus defOsSbox) is always nil on non-Linux: move this code to Linux-only files.
+			c.defOsSboxOnce.Do(func() {
 				c.defOsSbox = sb.osSbox
 			})
 		}

libnetwork/sandbox_unsupported.go

@@ -0,0 +1,32 @@
+//go:build !linux
+
+package libnetwork
+
+import "github.com/docker/docker/libnetwork/osl"
+
+func releaseOSSboxResources(osl.Sandbox, *Endpoint) {}
+
+func (sb *Sandbox) updateGateway(*Endpoint) error {
+	// not implemented on Windows (Sandbox.osSbox is always nil)
+	return nil
+}
+
+func (sb *Sandbox) ExecFunc(func()) error {
+	// not implemented on Windows (Sandbox.osSbox is always nil)
+	return nil
+}
+
+func (sb *Sandbox) releaseOSSbox() error {
+	// not implemented on Windows (Sandbox.osSbox is always nil)
+	return nil
+}
+
+func (sb *Sandbox) restoreOslSandbox() error {
+	// not implemented on Windows (Sandbox.osSbox is always nil)
+	return nil
+}
+
+func (sb *Sandbox) populateNetworkResources(*Endpoint) error {
+	// not implemented on Windows (Sandbox.osSbox is always nil)
+	return nil
+}