Merge pull request #44409 from corhere/libnetwork-test-sanity

libnetwork: improve tests & fix some bugs

libnetwork/datastore/datastore.go

@@ -154,13 +154,12 @@ var rootChain = defaultRootChain
 
 // DefaultScopes returns a map of default scopes and its config for clients to use.
 func DefaultScopes(dataDir string) map[string]*ScopeCfg {
+	s := makeDefaultScopes()
 	if dataDir != "" {
-		defaultScopes[LocalScope].Client.Address = dataDir + "/network/files/local-kv.db"
-		return defaultScopes
+		s[LocalScope].Client.Address = dataDir + "/network/files/local-kv.db"
 	}
 
-	defaultScopes[LocalScope].Client.Address = defaultPrefix + "/local-kv.db"
-	return defaultScopes
+	return s
 }
 
 // IsValid checks if the scope config has valid configuration.

libnetwork/drivers/bridge/bridge.go

@@ -22,6 +22,7 @@ import (
 	"github.com/docker/docker/libnetwork/netutils"
 	"github.com/docker/docker/libnetwork/ns"
 	"github.com/docker/docker/libnetwork/options"
+	"github.com/docker/docker/libnetwork/portallocator"
 	"github.com/docker/docker/libnetwork/portmapper"
 	"github.com/docker/docker/libnetwork/types"
 	"github.com/sirupsen/logrus"
@@ -142,7 +143,7 @@ type bridgeNetwork struct {
 }
 
 type driver struct {
-	config            *configuration
+	config            configuration
 	natChain          *iptables.ChainInfo
 	filterChain       *iptables.ChainInfo
 	isolationChain1   *iptables.ChainInfo
@@ -155,12 +156,16 @@ type driver struct {
 	store             datastore.DataStore
 	nlh               *netlink.Handle
 	configNetwork     sync.Mutex
+	portAllocator     *portallocator.PortAllocator // Overridable for tests.
 	sync.Mutex
 }
 
 // New constructs a new bridge driver
 func newDriver() *driver {
-	return &driver{networks: map[string]*bridgeNetwork{}, config: &configuration{}}
+	return &driver{
+		networks:      map[string]*bridgeNetwork{},
+		portAllocator: portallocator.Get(),
+	}
 }
 
 // Init registers a new instance of bridge driver
@@ -348,7 +353,7 @@ func (n *bridgeNetwork) isolateNetwork(enable bool) error {
 
 func (d *driver) configure(option map[string]interface{}) error {
 	var (
-		config            *configuration
+		config            configuration
 		err               error
 		natChain          *iptables.ChainInfo
 		filterChain       *iptables.ChainInfo
@@ -360,20 +365,17 @@ func (d *driver) configure(option map[string]interface{}) error {
 		isolationChain2V6 *iptables.ChainInfo
 	)
 
-	genericData, ok := option[netlabel.GenericData]
-	if !ok || genericData == nil {
-		return nil
-	}
-
-	switch opt := genericData.(type) {
+	switch opt := option[netlabel.GenericData].(type) {
 	case options.Generic:
 		opaqueConfig, err := options.GenerateFromModel(opt, &configuration{})
 		if err != nil {
 			return err
 		}
-		config = opaqueConfig.(*configuration)
+		config = *opaqueConfig.(*configuration)
 	case *configuration:
-		config = opt
+		config = *opt
+	case nil:
+		// No GenericData option set. Use defaults.
 	default:
 		return &ErrInvalidDriverConfig{}
 	}
@@ -688,8 +690,8 @@ func (d *driver) createNetwork(config *networkConfiguration) (err error) {
 		id:           config.ID,
 		endpoints:    make(map[string]*bridgeEndpoint),
 		config:       config,
-		portMapper:   portmapper.New(d.config.UserlandProxyPath),
-		portMapperV6: portmapper.New(d.config.UserlandProxyPath),
+		portMapper:   portmapper.NewWithPortAllocator(d.portAllocator, d.config.UserlandProxyPath),
+		portMapperV6: portmapper.NewWithPortAllocator(d.portAllocator, d.config.UserlandProxyPath),
 		bridge:       bridgeIface,
 		driver:       d,
 	}

libnetwork/drivers/bridge/bridge_test.go

@@ -17,6 +17,7 @@ import (
 	"github.com/docker/docker/libnetwork/netlabel"
 	"github.com/docker/docker/libnetwork/netutils"
 	"github.com/docker/docker/libnetwork/options"
+	"github.com/docker/docker/libnetwork/portallocator"
 	"github.com/docker/docker/libnetwork/testutils"
 	"github.com/docker/docker/libnetwork/types"
 	"github.com/vishvananda/netlink"
@@ -180,9 +181,7 @@ func getIPv4Data(t *testing.T, iface string) []driverapi.IPAMData {
 }
 
 func TestCreateFullOptions(t *testing.T) {
-	if !testutils.IsRunningInContainer() {
-		defer testutils.SetupTestOSContext(t)()
-	}
+	defer testutils.SetupTestOSContext(t)()
 	d := newDriver()
 
 	config := &configuration{
@@ -236,9 +235,7 @@ func TestCreateFullOptions(t *testing.T) {
 }
 
 func TestCreateNoConfig(t *testing.T) {
-	if !testutils.IsRunningInContainer() {
-		defer testutils.SetupTestOSContext(t)()
-	}
+	defer testutils.SetupTestOSContext(t)()
 	d := newDriver()
 
 	netconfig := &networkConfiguration{BridgeName: DefaultBridgeName}
@@ -251,9 +248,7 @@ func TestCreateNoConfig(t *testing.T) {
 }
 
 func TestCreateFullOptionsLabels(t *testing.T) {
-	if !testutils.IsRunningInContainer() {
-		defer testutils.SetupTestOSContext(t)()
-	}
+	defer testutils.SetupTestOSContext(t)()
 	d := newDriver()
 
 	config := &configuration{
@@ -359,9 +354,7 @@ func TestCreateFullOptionsLabels(t *testing.T) {
 }
 
 func TestCreate(t *testing.T) {
-	if !testutils.IsRunningInContainer() {
-		defer testutils.SetupTestOSContext(t)()
-	}
+	defer testutils.SetupTestOSContext(t)()
 
 	d := newDriver()
 
@@ -387,9 +380,7 @@ func TestCreate(t *testing.T) {
 }
 
 func TestCreateFail(t *testing.T) {
-	if !testutils.IsRunningInContainer() {
-		defer testutils.SetupTestOSContext(t)()
-	}
+	defer testutils.SetupTestOSContext(t)()
 
 	d := newDriver()
 
@@ -407,9 +398,7 @@ func TestCreateFail(t *testing.T) {
 }
 
 func TestCreateMultipleNetworks(t *testing.T) {
-	if !testutils.IsRunningInContainer() {
-		defer testutils.SetupTestOSContext(t)()
-	}
+	defer testutils.SetupTestOSContext(t)()
 
 	d := newDriver()
 
@@ -617,10 +606,9 @@ func TestQueryEndpointInfoHairpin(t *testing.T) {
 }
 
 func testQueryEndpointInfo(t *testing.T, ulPxyEnabled bool) {
-	if !testutils.IsRunningInContainer() {
-		defer testutils.SetupTestOSContext(t)()
-	}
+	defer testutils.SetupTestOSContext(t)()
 	d := newDriver()
+	d.portAllocator = portallocator.NewInstance()
 
 	config := &configuration{
 		EnableIPTables:      true,
@@ -720,9 +708,7 @@ func getPortMapping() []types.PortBinding {
 }
 
 func TestLinkContainers(t *testing.T) {
-	if !testutils.IsRunningInContainer() {
-		defer testutils.SetupTestOSContext(t)()
-	}
+	defer testutils.SetupTestOSContext(t)()
 
 	d := newDriver()
 	iptable := iptables.GetIptable(iptables.IPv4)
@@ -876,9 +862,7 @@ func TestLinkContainers(t *testing.T) {
 }
 
 func TestValidateConfig(t *testing.T) {
-	if !testutils.IsRunningInContainer() {
-		defer testutils.SetupTestOSContext(t)()
-	}
+	defer testutils.SetupTestOSContext(t)()
 
 	// Test mtu
 	c := networkConfiguration{Mtu: -2}
@@ -949,9 +933,7 @@ func TestValidateConfig(t *testing.T) {
 }
 
 func TestSetDefaultGw(t *testing.T) {
-	if !testutils.IsRunningInContainer() {
-		defer testutils.SetupTestOSContext(t)()
-	}
+	defer testutils.SetupTestOSContext(t)()
 
 	d := newDriver()
 
@@ -1013,7 +995,7 @@ func TestCleanupIptableRules(t *testing.T) {
 	ipVersions := []iptables.IPVersion{iptables.IPv4, iptables.IPv6}
 
 	for _, version := range ipVersions {
-		if _, _, _, _, err := setupIPChains(&configuration{EnableIPTables: true}, version); err != nil {
+		if _, _, _, _, err := setupIPChains(configuration{EnableIPTables: true}, version); err != nil {
 			t.Fatalf("Error setting up ip chains for %s: %v", version, err)
 		}
 
@@ -1095,32 +1077,35 @@ func TestCreateWithExistingBridge(t *testing.T) {
 }
 
 func TestCreateParallel(t *testing.T) {
-	if !testutils.IsRunningInContainer() {
-		defer testutils.SetupTestOSContext(t)()
-	}
+	c := testutils.SetupTestOSContextEx(t)
+	defer c.Cleanup(t)
 
 	d := newDriver()
+	d.portAllocator = portallocator.NewInstance()
 
 	if err := d.configure(nil); err != nil {
 		t.Fatalf("Failed to setup driver config: %v", err)
 	}
 
+	ipV4Data := getIPv4Data(t, "docker0")
+
 	ch := make(chan error, 100)
 	for i := 0; i < 100; i++ {
-		go func(name string, ch chan<- error) {
+		name := "net" + strconv.Itoa(i)
+		c.Go(t, func() {
 			config := &networkConfiguration{BridgeName: name}
 			genericOption := make(map[string]interface{})
 			genericOption[netlabel.GenericData] = config
-			if err := d.CreateNetwork(name, genericOption, nil, getIPv4Data(t, "docker0"), nil); err != nil {
+			if err := d.CreateNetwork(name, genericOption, nil, ipV4Data, nil); err != nil {
 				ch <- fmt.Errorf("failed to create %s", name)
 				return
 			}
-			if err := d.CreateNetwork(name, genericOption, nil, getIPv4Data(t, "docker0"), nil); err == nil {
+			if err := d.CreateNetwork(name, genericOption, nil, ipV4Data, nil); err == nil {
 				ch <- fmt.Errorf("failed was able to create overlap %s", name)
 				return
 			}
 			ch <- nil
-		}("net"+strconv.Itoa(i), ch)
+		})
 	}
 	// wait for the go routines
 	var success int

libnetwork/drivers/bridge/interface.go

@@ -57,6 +57,10 @@ func (i *bridgeInterface) exists() bool {
 
 // addresses returns all IPv4 addresses and all IPv6 addresses for the bridge interface.
 func (i *bridgeInterface) addresses() ([]netlink.Addr, []netlink.Addr, error) {
+	if !i.exists() {
+		// A nonexistent interface, by definition, cannot have any addresses.
+		return nil, nil, nil
+	}
 	v4addr, err := i.nlh.AddrList(i.Link, netlink.FAMILY_V4)
 	if err != nil {
 		return nil, nil, fmt.Errorf("Failed to retrieve V4 addresses: %v", err)

libnetwork/drivers/bridge/setup_ip_tables.go

@@ -29,7 +29,7 @@ const (
 	IsolationChain2 = "DOCKER-ISOLATION-STAGE-2"
 )
 
-func setupIPChains(config *configuration, version iptables.IPVersion) (*iptables.ChainInfo, *iptables.ChainInfo, *iptables.ChainInfo, *iptables.ChainInfo, error) {
+func setupIPChains(config configuration, version iptables.IPVersion) (*iptables.ChainInfo, *iptables.ChainInfo, *iptables.ChainInfo, *iptables.ChainInfo, error) {
 	// Sanity check.
 	if !config.EnableIPTables {
 		return nil, nil, nil, nil, errors.New("cannot create new chains, EnableIPTable is disabled")

libnetwork/drivers/bridge/setup_ip_tables_test.go

@@ -56,7 +56,7 @@ func TestSetupIPChains(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	driverconfig := &configuration{
+	driverconfig := configuration{
 		EnableIPTables: true,
 	}
 	d := &driver{

libnetwork/endpoint_test.go

@@ -13,9 +13,7 @@ import (
 )
 
 func TestHostsEntries(t *testing.T) {
-	if !testutils.IsRunningInContainer() {
-		defer testutils.SetupTestOSContext(t)()
-	}
+	defer testutils.SetupTestOSContext(t)()
 
 	expectedHostsFile := `127.0.0.1	localhost
 ::1	localhost ip6-localhost ip6-loopback

libnetwork/firewall_linux_test.go

@@ -8,6 +8,7 @@ import (
 	"github.com/docker/docker/libnetwork/iptables"
 	"github.com/docker/docker/libnetwork/netlabel"
 	"github.com/docker/docker/libnetwork/options"
+	"github.com/docker/docker/libnetwork/testutils"
 	"gotest.tools/v3/assert"
 )
 
@@ -19,9 +20,6 @@ const (
 func TestUserChain(t *testing.T) {
 	iptable := iptables.GetIptable(iptables.IPv4)
 
-	nc, err := New()
-	assert.NilError(t, err)
-
 	tests := []struct {
 		iptables  bool
 		insert    bool // insert other rules to FORWARD
@@ -47,10 +45,15 @@ func TestUserChain(t *testing.T) {
 		},
 	}
 
-	resetIptables(t)
 	for _, tc := range tests {
 		tc := tc
 		t.Run(fmt.Sprintf("iptables=%v,insert=%v", tc.iptables, tc.insert), func(t *testing.T) {
+			defer testutils.SetupTestOSContext(t)()
+			defer resetIptables(t)
+
+			nc, err := New()
+			assert.NilError(t, err)
+			defer nc.Stop()
 			c := nc.(*controller)
 			c.cfg.DriverCfg["bridge"] = map[string]interface{}{
 				netlabel.GenericData: options.Generic{
@@ -75,7 +78,6 @@ func TestUserChain(t *testing.T) {
 				assert.Assert(t, err != nil, "chain %v: created unexpectedly", usrChainName)
 			}
 		})
-		resetIptables(t)
 	}
 }
 

libnetwork/libnetwork_internal_test.go

@@ -312,6 +312,8 @@ func compareNwLists(a, b []*net.IPNet) bool {
 }
 
 func TestAuxAddresses(t *testing.T) {
+	defer testutils.SetupTestOSContext(t)()
+
 	c, err := New()
 	if err != nil {
 		t.Fatal(err)
@@ -349,6 +351,8 @@ func TestAuxAddresses(t *testing.T) {
 func TestSRVServiceQuery(t *testing.T) {
 	skip.If(t, runtime.GOOS == "windows", "test only works on linux")
 
+	defer testutils.SetupTestOSContext(t)()
+
 	c, err := New()
 	if err != nil {
 		t.Fatal(err)
@@ -447,6 +451,8 @@ func TestSRVServiceQuery(t *testing.T) {
 func TestServiceVIPReuse(t *testing.T) {
 	skip.If(t, runtime.GOOS == "windows", "test only works on linux")
 
+	defer testutils.SetupTestOSContext(t)()
+
 	c, err := New()
 	if err != nil {
 		t.Fatal(err)
@@ -562,15 +568,9 @@ func TestServiceVIPReuse(t *testing.T) {
 func TestIpamReleaseOnNetDriverFailures(t *testing.T) {
 	skip.If(t, runtime.GOOS == "windows", "test only works on linux")
 
-	if !testutils.IsRunningInContainer() {
-		defer testutils.SetupTestOSContext(t)()
-	}
+	defer testutils.SetupTestOSContext(t)()
 
-	cfgOptions, err := OptionBoltdbWithRandomDBFile()
-	if err != nil {
-		t.Fatal(err)
-	}
-	c, err := New(cfgOptions...)
+	c, err := New(OptionBoltdbWithRandomDBFile(t))
 	if err != nil {
 		t.Fatal(err)
 	}

libnetwork/libnetwork_linux_test.go

@@ -3,13 +3,10 @@ package libnetwork_test
 import (
 	"bytes"
 	"encoding/json"
-	"flag"
 	"fmt"
 	"net"
 	"os"
 	"os/exec"
-	"runtime"
-	"strconv"
 	"strings"
 	"sync"
 	"testing"
@@ -22,93 +19,30 @@ import (
 	"github.com/docker/docker/libnetwork/testutils"
 	"github.com/docker/docker/libnetwork/types"
 	"github.com/docker/docker/pkg/reexec"
+	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	"github.com/vishvananda/netlink"
 	"github.com/vishvananda/netns"
+	"golang.org/x/sync/errgroup"
 )
 
 const (
 	bridgeNetType = "bridge"
 )
 
-var (
-	origins = netns.None()
-	testns  = netns.None()
-)
-
-var createTesthostNetworkOnce sync.Once
-
-func getTesthostNetwork(t *testing.T) libnetwork.Network {
+func makeTesthostNetwork(t *testing.T, c libnetwork.NetworkController) libnetwork.Network {
 	t.Helper()
-	createTesthostNetworkOnce.Do(func() {
-		_, err := createTestNetwork("host", "testhost", options.Generic{}, nil, nil)
-		if err != nil {
-			t.Fatal(err)
-		}
-	})
-	n, err := controller.NetworkByName("testhost")
+	n, err := createTestNetwork(c, "host", "testhost", options.Generic{}, nil, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
 	return n
 }
 
-func createGlobalInstance(t *testing.T) {
-	var err error
-	defer close(start)
-
-	origins, err = netns.Get()
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	if testutils.IsRunningInContainer() {
-		testns = origins
-	} else {
-		testns, err = netns.New()
-		if err != nil {
-			t.Fatal(err)
-		}
-	}
-
-	netOption := options.Generic{
-		netlabel.GenericData: options.Generic{
-			"BridgeName": "network",
-		},
-	}
-
-	net1 := getTesthostNetwork(t)
-	net2, err := createTestNetwork("bridge", "network2", netOption, nil, nil)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	_, err = net1.CreateEndpoint("pep1")
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	_, err = net2.CreateEndpoint("pep2")
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	_, err = net2.CreateEndpoint("pep3")
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	if sboxes[first-1], err = controller.NewSandbox(fmt.Sprintf("%drace", first), libnetwork.OptionUseDefaultSandbox()); err != nil {
-		t.Fatal(err)
-	}
-	for thd := first + 1; thd <= last; thd++ {
-		if sboxes[thd-1], err = controller.NewSandbox(fmt.Sprintf("%drace", thd)); err != nil {
-			t.Fatal(err)
-		}
-	}
-}
-
 func TestHost(t *testing.T) {
+	defer testutils.SetupTestOSContext(t)()
+	controller := newController(t)
+
 	sbx1, err := controller.NewSandbox("host_c1",
 		libnetwork.OptionHostname("test1"),
 		libnetwork.OptionDomainname("docker.io"),
@@ -137,7 +71,7 @@ func TestHost(t *testing.T) {
 		}
 	}()
 
-	network := getTesthostNetwork(t)
+	network := makeTesthostNetwork(t, controller)
 	ep1, err := network.CreateEndpoint("testep1")
 	if err != nil {
 		t.Fatal(err)
@@ -207,9 +141,8 @@ func TestHost(t *testing.T) {
 
 // Testing IPV6 from MAC address
 func TestBridgeIpv6FromMac(t *testing.T) {
-	if !testutils.IsRunningInContainer() {
-		defer testutils.SetupTestOSContext(t)()
-	}
+	defer testutils.SetupTestOSContext(t)()
+	controller := newController(t)
 
 	netOption := options.Generic{
 		netlabel.GenericData: options.Generic{
@@ -283,9 +216,8 @@ func checkSandbox(t *testing.T, info libnetwork.EndpointInfo) {
 }
 
 func TestEndpointJoin(t *testing.T) {
-	if !testutils.IsRunningInContainer() {
-		defer testutils.SetupTestOSContext(t)()
-	}
+	defer testutils.SetupTestOSContext(t)()
+	controller := newController(t)
 
 	// Create network 1 and add 2 endpoint: ep11, ep12
 	netOption := options.Generic{
@@ -411,7 +343,7 @@ func TestEndpointJoin(t *testing.T) {
 	}
 
 	// Now test the container joining another network
-	n2, err := createTestNetwork(bridgeNetType, "testnetwork2",
+	n2, err := createTestNetwork(controller, bridgeNetType, "testnetwork2",
 		options.Generic{
 			netlabel.GenericData: options.Generic{
 				"BridgeName": "testnetwork2",
@@ -459,11 +391,10 @@ func TestExternalKey(t *testing.T) {
 }
 
 func externalKeyTest(t *testing.T, reexec bool) {
-	if !testutils.IsRunningInContainer() {
-		defer testutils.SetupTestOSContext(t)()
-	}
+	defer testutils.SetupTestOSContext(t)()
+	controller := newController(t)
 
-	n, err := createTestNetwork(bridgeNetType, "testnetwork", options.Generic{
+	n, err := createTestNetwork(controller, bridgeNetType, "testnetwork", options.Generic{
 		netlabel.GenericData: options.Generic{
 			"BridgeName": "testnetwork",
 		},
@@ -477,7 +408,7 @@ func externalKeyTest(t *testing.T, reexec bool) {
 		}
 	}()
 
-	n2, err := createTestNetwork(bridgeNetType, "testnetwork2", options.Generic{
+	n2, err := createTestNetwork(controller, bridgeNetType, "testnetwork2", options.Generic{
 		netlabel.GenericData: options.Generic{
 			"BridgeName": "testnetwork2",
 		},
@@ -621,9 +552,8 @@ func reexecSetKey(key string, containerID string, controllerID string) error {
 }
 
 func TestEnableIPv6(t *testing.T) {
-	if !testutils.IsRunningInContainer() {
-		defer testutils.SetupTestOSContext(t)()
-	}
+	defer testutils.SetupTestOSContext(t)()
+	controller := newController(t)
 
 	tmpResolvConf := []byte("search pommesfrites.fr\nnameserver 12.34.56.78\nnameserver 2001:4860:4860::8888\n")
 	expectedResolvConf := []byte("search pommesfrites.fr\nnameserver 127.0.0.11\nnameserver 2001:4860:4860::8888\noptions ndots:0\n")
@@ -647,7 +577,7 @@ func TestEnableIPv6(t *testing.T) {
 	}
 	ipamV6ConfList := []*libnetwork.IpamConf{{PreferredPool: "fe99::/64", Gateway: "fe99::9"}}
 
-	n, err := createTestNetwork("bridge", "testnetwork", netOption, nil, ipamV6ConfList)
+	n, err := createTestNetwork(controller, "bridge", "testnetwork", netOption, nil, ipamV6ConfList)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -699,9 +629,8 @@ func TestEnableIPv6(t *testing.T) {
 }
 
 func TestResolvConfHost(t *testing.T) {
-	if !testutils.IsRunningInContainer() {
-		defer testutils.SetupTestOSContext(t)()
-	}
+	defer testutils.SetupTestOSContext(t)()
+	controller := newController(t)
 
 	tmpResolvConf := []byte("search localhost.net\nnameserver 127.0.0.1\nnameserver 2001:4860:4860::8888\n")
 
@@ -717,7 +646,7 @@ func TestResolvConfHost(t *testing.T) {
 		}
 	}()
 
-	n := getTesthostNetwork(t)
+	n := makeTesthostNetwork(t, controller)
 	ep1, err := n.CreateEndpoint("ep1", libnetwork.CreateOptionDisableResolution())
 	if err != nil {
 		t.Fatal(err)
@@ -775,9 +704,8 @@ func TestResolvConfHost(t *testing.T) {
 }
 
 func TestResolvConf(t *testing.T) {
-	if !testutils.IsRunningInContainer() {
-		defer testutils.SetupTestOSContext(t)()
-	}
+	defer testutils.SetupTestOSContext(t)()
+	controller := newController(t)
 
 	tmpResolvConf1 := []byte("search pommesfrites.fr\nnameserver 12.34.56.78\nnameserver 2001:4860:4860::8888\n")
 	tmpResolvConf2 := []byte("search pommesfrites.fr\nnameserver 112.34.56.78\nnameserver 2001:4860:4860::8888\n")
@@ -801,7 +729,7 @@ func TestResolvConf(t *testing.T) {
 			"BridgeName": "testnetwork",
 		},
 	}
-	n, err := createTestNetwork("bridge", "testnetwork", netOption, nil, nil)
+	n, err := createTestNetwork(controller, "bridge", "testnetwork", netOption, nil, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -915,163 +843,138 @@ func TestResolvConf(t *testing.T) {
 	}
 }
 
-func parallelJoin(t *testing.T, rc libnetwork.Sandbox, ep libnetwork.Endpoint, thrNumber int) {
-	debugf("J%d.", thrNumber)
-	var err error
-
-	sb := sboxes[thrNumber-1]
-	err = ep.Join(sb)
-
-	if err != nil {
-		if _, ok := err.(types.ForbiddenError); !ok {
-			t.Fatalf("thread %d: %v", thrNumber, err)
-		}
-		debugf("JE%d(%v).", thrNumber, err)
-	}
-	debugf("JD%d.", thrNumber)
+type parallelTester struct {
+	osctx      *testutils.OSContext
+	controller libnetwork.NetworkController
+	net1, net2 libnetwork.Network
+	iterCnt    int
 }
 
-func parallelLeave(t *testing.T, rc libnetwork.Sandbox, ep libnetwork.Endpoint, thrNumber int) {
-	debugf("L%d.", thrNumber)
-	var err error
-
-	sb := sboxes[thrNumber-1]
-
-	err = ep.Leave(sb)
-	if err != nil {
-		if _, ok := err.(types.ForbiddenError); !ok {
-			t.Fatalf("thread %d: %v", thrNumber, err)
-		}
-		debugf("LE%d(%v).", thrNumber, err)
-	}
-	debugf("LD%d.", thrNumber)
-}
-
-func runParallelTests(t *testing.T, thrNumber int) {
+func (pt parallelTester) Do(t *testing.T, thrNumber int) error {
 	var (
 		ep  libnetwork.Endpoint
 		sb  libnetwork.Sandbox
 		err error
 	)
 
-	t.Parallel()
+	teardown, err := pt.osctx.Set()
+	if err != nil {
+		return err
+	}
+	defer teardown(t)
 
-	pTest := flag.Lookup("test.parallel")
-	if pTest == nil {
-		t.Skip("Skipped because test.parallel flag not set;")
+	epName := fmt.Sprintf("pep%d", thrNumber)
+
+	if thrNumber == 1 {
+		ep, err = pt.net1.EndpointByName(epName)
+	} else {
+		ep, err = pt.net2.EndpointByName(epName)
 	}
-	numParallel, err := strconv.Atoi(pTest.Value.String())
+
 	if err != nil {
-		t.Fatal(err)
+		return errors.WithStack(err)
 	}
-	if numParallel < numThreads {
-		t.Skip("Skipped because t.parallel was less than ", numThreads)
+	if ep == nil {
+		return errors.New("got nil ep with no error")
 	}
 
-	runtime.LockOSThread()
-	if thrNumber == first {
-		createGlobalInstance(t)
-	} else {
-		<-start
-
-		thrdone := make(chan struct{})
-		done <- thrdone
-		defer close(thrdone)
+	cid := fmt.Sprintf("%drace", thrNumber)
+	pt.controller.WalkSandboxes(libnetwork.SandboxContainerWalker(&sb, cid))
+	if sb == nil {
+		return errors.Errorf("got nil sandbox for container: %s", cid)
+	}
 
-		if thrNumber == last {
-			defer close(done)
+	for i := 0; i < pt.iterCnt; i++ {
+		if err := ep.Join(sb); err != nil {
+			if _, ok := err.(types.ForbiddenError); !ok {
+				return errors.Wrapf(err, "thread %d", thrNumber)
+			}
 		}
-
-		err = netns.Set(testns)
-		if err != nil {
-			runtime.UnlockOSThread()
-			t.Fatal(err)
+		if err := ep.Leave(sb); err != nil {
+			if _, ok := err.(types.ForbiddenError); !ok {
+				return errors.Wrapf(err, "thread %d", thrNumber)
+			}
 		}
 	}
-	defer func() {
-		if err := netns.Set(origins); err != nil {
-			t.Fatalf("Error restoring the current thread's netns: %v", err)
-		} else {
-			runtime.UnlockOSThread()
-		}
-	}()
 
-	net1 := getTesthostNetwork(t)
-	if net1 == nil {
-		t.Fatal("Could not find testhost")
+	if err := errors.WithStack(sb.Delete()); err != nil {
+		return err
 	}
+	return errors.WithStack(ep.Delete(false))
+}
 
-	net2, err := controller.NetworkByName("network2")
-	if err != nil {
-		t.Fatal(err)
-	}
-	if net2 == nil {
-		t.Fatal("Could not find network2")
-	}
+func TestParallel(t *testing.T) {
+	const (
+		first      = 1
+		last       = 3
+		numThreads = last - first + 1
+		iterCnt    = 25
+	)
 
-	epName := fmt.Sprintf("pep%d", thrNumber)
+	osctx := testutils.SetupTestOSContextEx(t)
+	defer osctx.Cleanup(t)
+	controller := newController(t)
 
-	if thrNumber == first {
-		ep, err = net1.EndpointByName(epName)
-	} else {
-		ep, err = net2.EndpointByName(epName)
+	netOption := options.Generic{
+		netlabel.GenericData: options.Generic{
+			"BridgeName": "network",
+		},
 	}
 
+	net1 := makeTesthostNetwork(t, controller)
+	defer net1.Delete()
+	net2, err := createTestNetwork(controller, "bridge", "network2", netOption, nil, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
-	if ep == nil {
-		t.Fatal("Got nil ep with no error")
-	}
+	defer net2.Delete()
 
-	cid := fmt.Sprintf("%drace", thrNumber)
-	controller.WalkSandboxes(libnetwork.SandboxContainerWalker(&sb, cid))
-	if sb == nil {
-		t.Fatalf("Got nil sandbox for container: %s", cid)
+	_, err = net1.CreateEndpoint("pep1")
+	if err != nil {
+		t.Fatal(err)
 	}
 
-	for i := 0; i < iterCnt; i++ {
-		parallelJoin(t, sb, ep, thrNumber)
-		parallelLeave(t, sb, ep, thrNumber)
+	_, err = net2.CreateEndpoint("pep2")
+	if err != nil {
+		t.Fatal(err)
 	}
 
-	debugf("\n")
-
-	err = sb.Delete()
+	_, err = net2.CreateEndpoint("pep3")
 	if err != nil {
 		t.Fatal(err)
 	}
-	if thrNumber == first {
-		for thrdone := range done {
-			<-thrdone
-		}
 
-		if testns != origins {
-			testns.Close()
-		}
-		if err := net2.Delete(); err != nil {
-			t.Fatal(err)
-		}
-	} else {
-		err = ep.Delete(false)
-		if err != nil {
+	sboxes := make([]libnetwork.Sandbox, numThreads)
+	if sboxes[first-1], err = controller.NewSandbox(fmt.Sprintf("%drace", first), libnetwork.OptionUseDefaultSandbox()); err != nil {
+		t.Fatal(err)
+	}
+	for thd := first + 1; thd <= last; thd++ {
+		if sboxes[thd-1], err = controller.NewSandbox(fmt.Sprintf("%drace", thd)); err != nil {
 			t.Fatal(err)
 		}
 	}
-}
 
-func TestParallel1(t *testing.T) {
-	runParallelTests(t, 1)
-}
+	pt := parallelTester{
+		osctx:      osctx,
+		controller: controller,
+		net1:       net1,
+		net2:       net2,
+		iterCnt:    iterCnt,
+	}
 
-func TestParallel2(t *testing.T) {
-	runParallelTests(t, 2)
+	var eg errgroup.Group
+	for i := first; i <= last; i++ {
+		i := i
+		eg.Go(func() error { return pt.Do(t, i) })
+	}
+	if err := eg.Wait(); err != nil {
+		t.Fatalf("%+v", err)
+	}
 }
 
 func TestBridge(t *testing.T) {
-	if !testutils.IsRunningInContainer() {
-		defer testutils.SetupTestOSContext(t)()
-	}
+	defer testutils.SetupTestOSContext(t)()
+	controller := newController(t)
 
 	netOption := options.Generic{
 		netlabel.EnableIPv6: true,
@@ -1084,7 +987,7 @@ func TestBridge(t *testing.T) {
 	ipamV4ConfList := []*libnetwork.IpamConf{{PreferredPool: "192.168.100.0/24", Gateway: "192.168.100.1"}}
 	ipamV6ConfList := []*libnetwork.IpamConf{{PreferredPool: "fe90::/64", Gateway: "fe90::22"}}
 
-	network, err := createTestNetwork(bridgeNetType, "testnetwork", netOption, ipamV4ConfList, ipamV6ConfList)
+	network, err := createTestNetwork(controller, bridgeNetType, "testnetwork", netOption, ipamV4ConfList, ipamV6ConfList)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -1157,11 +1060,10 @@ func isV6Listenable() bool {
 	return v6ListenableCached
 }
 
-func TestParallel3(t *testing.T) {
-	runParallelTests(t, 3)
-}
-
 func TestNullIpam(t *testing.T) {
+	defer testutils.SetupTestOSContext(t)()
+	controller := newController(t)
+
 	_, err := controller.NewNetwork(bridgeNetType, "testnetworkinternal", "", libnetwork.NetworkOptionIpam(ipamapi.NullIPAM, "", nil, nil, nil))
 	if err == nil || err.Error() != "ipv4 pool is empty" {
 		t.Fatal("bridge network should complain empty pool")

libnetwork/libnetwork_test.go

@@ -28,8 +28,6 @@ import (
 	"github.com/sirupsen/logrus"
 )
 
-var controller libnetwork.NetworkController
-
 func TestMain(m *testing.M) {
 	if runtime.GOOS == "windows" {
 		logrus.Info("Test suite does not currently support windows")
@@ -39,39 +37,33 @@ func TestMain(m *testing.M) {
 		return
 	}
 
-	if err := createController(); err != nil {
-		logrus.Errorf("Error creating controller: %v", err)
-		os.Exit(1)
-	}
+	// Cleanup local datastore file
+	_ = os.Remove(datastore.DefaultScopes("")[datastore.LocalScope].Client.Address)
 
-	x := m.Run()
-	controller.Stop()
-	os.Exit(x)
+	os.Exit(m.Run())
 }
 
-func createController() error {
-	var err error
-
-	// Cleanup local datastore file
-	os.Remove(datastore.DefaultScopes("")[datastore.LocalScope].Client.Address)
-
-	option := options.Generic{
-		"EnableIPForwarding": true,
+func newController(t *testing.T) libnetwork.NetworkController {
+	t.Helper()
+	genericOption := map[string]interface{}{
+		netlabel.GenericData: options.Generic{
+			"EnableIPForwarding": true,
+		},
 	}
 
-	genericOption := make(map[string]interface{})
-	genericOption[netlabel.GenericData] = option
-
-	cfgOptions, err := libnetwork.OptionBoltdbWithRandomDBFile()
+	c, err := libnetwork.New(
+		libnetwork.OptionBoltdbWithRandomDBFile(t),
+		config.OptionDriverConfig(bridgeNetType, genericOption),
+	)
 	if err != nil {
-		return err
+		t.Fatal(err)
 	}
-	controller, err = libnetwork.New(append(cfgOptions, config.OptionDriverConfig(bridgeNetType, genericOption))...)
-	return err
+	t.Cleanup(c.Stop)
+	return c
 }
 
-func createTestNetwork(networkType, networkName string, netOption options.Generic, ipamV4Configs, ipamV6Configs []*libnetwork.IpamConf) (libnetwork.Network, error) {
-	return controller.NewNetwork(networkType, networkName, "",
+func createTestNetwork(c libnetwork.NetworkController, networkType, networkName string, netOption options.Generic, ipamV4Configs, ipamV6Configs []*libnetwork.IpamConf) (libnetwork.Network, error) {
+	return c.NewNetwork(networkType, networkName, "",
 		libnetwork.NetworkOptionGeneric(netOption),
 		libnetwork.NetworkOptionIpam(ipamapi.DefaultIPAM, "", ipamV4Configs, ipamV6Configs, nil))
 }
@@ -98,6 +90,9 @@ func isNotFound(err error) bool {
 }
 
 func TestNull(t *testing.T) {
+	defer testutils.SetupTestOSContext(t)()
+	controller := newController(t)
+
 	cnt, err := controller.NewSandbox("null_container",
 		libnetwork.OptionHostname("test"),
 		libnetwork.OptionDomainname("docker.io"),
@@ -106,7 +101,7 @@ func TestNull(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	network, err := createTestNetwork("null", "testnull", options.Generic{}, nil, nil)
+	network, err := createTestNetwork(controller, "null", "testnull", options.Generic{}, nil, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -145,11 +140,10 @@ func TestNull(t *testing.T) {
 }
 
 func TestUnknownDriver(t *testing.T) {
-	if !testutils.IsRunningInContainer() {
-		defer testutils.SetupTestOSContext(t)()
-	}
+	defer testutils.SetupTestOSContext(t)()
+	controller := newController(t)
 
-	_, err := createTestNetwork("unknowndriver", "testnetwork", options.Generic{}, nil, nil)
+	_, err := createTestNetwork(controller, "unknowndriver", "testnetwork", options.Generic{}, nil, nil)
 	if err == nil {
 		t.Fatal("Expected to fail. But instead succeeded")
 	}
@@ -160,6 +154,9 @@ func TestUnknownDriver(t *testing.T) {
 }
 
 func TestNilRemoteDriver(t *testing.T) {
+	defer testutils.SetupTestOSContext(t)()
+	controller := newController(t)
+
 	_, err := controller.NewNetwork("framerelay", "dummy", "",
 		libnetwork.NetworkOptionGeneric(getEmptyGenericOption()))
 	if err == nil {
@@ -172,9 +169,8 @@ func TestNilRemoteDriver(t *testing.T) {
 }
 
 func TestNetworkName(t *testing.T) {
-	if !testutils.IsRunningInContainer() {
-		defer testutils.SetupTestOSContext(t)()
-	}
+	defer testutils.SetupTestOSContext(t)()
+	controller := newController(t)
 
 	netOption := options.Generic{
 		netlabel.GenericData: options.Generic{
@@ -182,7 +178,7 @@ func TestNetworkName(t *testing.T) {
 		},
 	}
 
-	_, err := createTestNetwork(bridgeNetType, "", netOption, nil, nil)
+	_, err := createTestNetwork(controller, bridgeNetType, "", netOption, nil, nil)
 	if err == nil {
 		t.Fatal("Expected to fail. But instead succeeded")
 	}
@@ -192,7 +188,7 @@ func TestNetworkName(t *testing.T) {
 	}
 
 	networkName := "testnetwork"
-	n, err := createTestNetwork(bridgeNetType, networkName, netOption, nil, nil)
+	n, err := createTestNetwork(controller, bridgeNetType, networkName, netOption, nil, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -208,9 +204,8 @@ func TestNetworkName(t *testing.T) {
 }
 
 func TestNetworkType(t *testing.T) {
-	if !testutils.IsRunningInContainer() {
-		defer testutils.SetupTestOSContext(t)()
-	}
+	defer testutils.SetupTestOSContext(t)()
+	controller := newController(t)
 
 	netOption := options.Generic{
 		netlabel.GenericData: options.Generic{
@@ -218,7 +213,7 @@ func TestNetworkType(t *testing.T) {
 		},
 	}
 
-	n, err := createTestNetwork(bridgeNetType, "testnetwork", netOption, nil, nil)
+	n, err := createTestNetwork(controller, bridgeNetType, "testnetwork", netOption, nil, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -234,9 +229,8 @@ func TestNetworkType(t *testing.T) {
 }
 
 func TestNetworkID(t *testing.T) {
-	if !testutils.IsRunningInContainer() {
-		defer testutils.SetupTestOSContext(t)()
-	}
+	defer testutils.SetupTestOSContext(t)()
+	controller := newController(t)
 
 	netOption := options.Generic{
 		netlabel.GenericData: options.Generic{
@@ -244,7 +238,7 @@ func TestNetworkID(t *testing.T) {
 		},
 	}
 
-	n, err := createTestNetwork(bridgeNetType, "testnetwork", netOption, nil, nil)
+	n, err := createTestNetwork(controller, bridgeNetType, "testnetwork", netOption, nil, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -260,9 +254,8 @@ func TestNetworkID(t *testing.T) {
 }
 
 func TestDeleteNetworkWithActiveEndpoints(t *testing.T) {
-	if !testutils.IsRunningInContainer() {
-		defer testutils.SetupTestOSContext(t)()
-	}
+	defer testutils.SetupTestOSContext(t)()
+	controller := newController(t)
 
 	netOption := options.Generic{
 		"BridgeName": "testnetwork",
@@ -271,7 +264,7 @@ func TestDeleteNetworkWithActiveEndpoints(t *testing.T) {
 		netlabel.GenericData: netOption,
 	}
 
-	network, err := createTestNetwork(bridgeNetType, "testnetwork", option, nil, nil)
+	network, err := createTestNetwork(controller, bridgeNetType, "testnetwork", option, nil, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -301,9 +294,8 @@ func TestDeleteNetworkWithActiveEndpoints(t *testing.T) {
 }
 
 func TestNetworkConfig(t *testing.T) {
-	if !testutils.IsRunningInContainer() {
-		defer testutils.SetupTestOSContext(t)()
-	}
+	defer testutils.SetupTestOSContext(t)()
+	controller := newController(t)
 
 	// Verify config network cannot inherit another config network
 	_, err := controller.NewNetwork("bridge", "config_network0", "",
@@ -403,9 +395,8 @@ func TestNetworkConfig(t *testing.T) {
 }
 
 func TestUnknownNetwork(t *testing.T) {
-	if !testutils.IsRunningInContainer() {
-		defer testutils.SetupTestOSContext(t)()
-	}
+	defer testutils.SetupTestOSContext(t)()
+	controller := newController(t)
 
 	netOption := options.Generic{
 		"BridgeName": "testnetwork",
@@ -414,7 +405,7 @@ func TestUnknownNetwork(t *testing.T) {
 		netlabel.GenericData: netOption,
 	}
 
-	network, err := createTestNetwork(bridgeNetType, "testnetwork", option, nil, nil)
+	network, err := createTestNetwork(controller, bridgeNetType, "testnetwork", option, nil, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -435,9 +426,8 @@ func TestUnknownNetwork(t *testing.T) {
 }
 
 func TestUnknownEndpoint(t *testing.T) {
-	if !testutils.IsRunningInContainer() {
-		defer testutils.SetupTestOSContext(t)()
-	}
+	defer testutils.SetupTestOSContext(t)()
+	controller := newController(t)
 
 	netOption := options.Generic{
 		"BridgeName": "testnetwork",
@@ -447,7 +437,7 @@ func TestUnknownEndpoint(t *testing.T) {
 	}
 	ipamV4ConfList := []*libnetwork.IpamConf{{PreferredPool: "192.168.100.0/24"}}
 
-	network, err := createTestNetwork(bridgeNetType, "testnetwork", option, ipamV4ConfList, nil)
+	network, err := createTestNetwork(controller, bridgeNetType, "testnetwork", option, ipamV4ConfList, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -477,9 +467,8 @@ func TestUnknownEndpoint(t *testing.T) {
 }
 
 func TestNetworkEndpointsWalkers(t *testing.T) {
-	if !testutils.IsRunningInContainer() {
-		defer testutils.SetupTestOSContext(t)()
-	}
+	defer testutils.SetupTestOSContext(t)()
+	controller := newController(t)
 
 	// Create network 1 and add 2 endpoint: ep11, ep12
 	netOption := options.Generic{
@@ -488,7 +477,7 @@ func TestNetworkEndpointsWalkers(t *testing.T) {
 		},
 	}
 
-	net1, err := createTestNetwork(bridgeNetType, "network1", netOption, nil, nil)
+	net1, err := createTestNetwork(controller, bridgeNetType, "network1", netOption, nil, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -560,7 +549,7 @@ func TestNetworkEndpointsWalkers(t *testing.T) {
 		},
 	}
 
-	net2, err := createTestNetwork(bridgeNetType, "network2", netOption, nil, nil)
+	net2, err := createTestNetwork(controller, bridgeNetType, "network2", netOption, nil, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -607,16 +596,15 @@ func TestNetworkEndpointsWalkers(t *testing.T) {
 }
 
 func TestDuplicateEndpoint(t *testing.T) {
-	if !testutils.IsRunningInContainer() {
-		defer testutils.SetupTestOSContext(t)()
-	}
+	defer testutils.SetupTestOSContext(t)()
+	controller := newController(t)
 
 	netOption := options.Generic{
 		netlabel.GenericData: options.Generic{
 			"BridgeName": "testnetwork",
 		},
 	}
-	n, err := createTestNetwork(bridgeNetType, "testnetwork", netOption, nil, nil)
+	n, err := createTestNetwork(controller, bridgeNetType, "testnetwork", netOption, nil, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -656,9 +644,8 @@ func TestDuplicateEndpoint(t *testing.T) {
 }
 
 func TestControllerQuery(t *testing.T) {
-	if !testutils.IsRunningInContainer() {
-		defer testutils.SetupTestOSContext(t)()
-	}
+	defer testutils.SetupTestOSContext(t)()
+	controller := newController(t)
 
 	// Create network 1
 	netOption := options.Generic{
@@ -666,7 +653,7 @@ func TestControllerQuery(t *testing.T) {
 			"BridgeName": "network1",
 		},
 	}
-	net1, err := createTestNetwork(bridgeNetType, "network1", netOption, nil, nil)
+	net1, err := createTestNetwork(controller, bridgeNetType, "network1", netOption, nil, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -682,7 +669,7 @@ func TestControllerQuery(t *testing.T) {
 			"BridgeName": "network2",
 		},
 	}
-	net2, err := createTestNetwork(bridgeNetType, "network2", netOption, nil, nil)
+	net2, err := createTestNetwork(controller, bridgeNetType, "network2", netOption, nil, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -758,9 +745,8 @@ func TestControllerQuery(t *testing.T) {
 }
 
 func TestNetworkQuery(t *testing.T) {
-	if !testutils.IsRunningInContainer() {
-		defer testutils.SetupTestOSContext(t)()
-	}
+	defer testutils.SetupTestOSContext(t)()
+	controller := newController(t)
 
 	// Create network 1 and add 2 endpoint: ep11, ep12
 	netOption := options.Generic{
@@ -768,7 +754,7 @@ func TestNetworkQuery(t *testing.T) {
 			"BridgeName": "network1",
 		},
 	}
-	net1, err := createTestNetwork(bridgeNetType, "network1", netOption, nil, nil)
+	net1, err := createTestNetwork(controller, bridgeNetType, "network1", netOption, nil, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -907,11 +893,10 @@ func (f *fakeSandbox) DisableService() error {
 }
 
 func TestEndpointDeleteWithActiveContainer(t *testing.T) {
-	if !testutils.IsRunningInContainer() {
-		defer testutils.SetupTestOSContext(t)()
-	}
+	defer testutils.SetupTestOSContext(t)()
+	controller := newController(t)
 
-	n, err := createTestNetwork(bridgeNetType, "testnetwork", options.Generic{
+	n, err := createTestNetwork(controller, bridgeNetType, "testnetwork", options.Generic{
 		netlabel.GenericData: options.Generic{
 			"BridgeName": "testnetwork",
 		},
@@ -925,7 +910,7 @@ func TestEndpointDeleteWithActiveContainer(t *testing.T) {
 		}
 	}()
 
-	n2, err := createTestNetwork(bridgeNetType, "testnetwork2", options.Generic{
+	n2, err := createTestNetwork(controller, bridgeNetType, "testnetwork2", options.Generic{
 		netlabel.GenericData: options.Generic{
 			"BridgeName": "testnetwork2",
 		},
@@ -982,11 +967,10 @@ func TestEndpointDeleteWithActiveContainer(t *testing.T) {
 }
 
 func TestEndpointMultipleJoins(t *testing.T) {
-	if !testutils.IsRunningInContainer() {
-		defer testutils.SetupTestOSContext(t)()
-	}
+	defer testutils.SetupTestOSContext(t)()
+	controller := newController(t)
 
-	n, err := createTestNetwork(bridgeNetType, "testmultiple", options.Generic{
+	n, err := createTestNetwork(controller, bridgeNetType, "testmultiple", options.Generic{
 		netlabel.GenericData: options.Generic{
 			"BridgeName": "testmultiple",
 		},
@@ -1056,11 +1040,10 @@ func TestEndpointMultipleJoins(t *testing.T) {
 }
 
 func TestLeaveAll(t *testing.T) {
-	if !testutils.IsRunningInContainer() {
-		defer testutils.SetupTestOSContext(t)()
-	}
+	defer testutils.SetupTestOSContext(t)()
+	controller := newController(t)
 
-	n, err := createTestNetwork(bridgeNetType, "testnetwork", options.Generic{
+	n, err := createTestNetwork(controller, bridgeNetType, "testnetwork", options.Generic{
 		netlabel.GenericData: options.Generic{
 			"BridgeName": "testnetwork",
 		},
@@ -1075,7 +1058,7 @@ func TestLeaveAll(t *testing.T) {
 		}
 	}()
 
-	n2, err := createTestNetwork(bridgeNetType, "testnetwork2", options.Generic{
+	n2, err := createTestNetwork(controller, bridgeNetType, "testnetwork2", options.Generic{
 		netlabel.GenericData: options.Generic{
 			"BridgeName": "testnetwork2",
 		},
@@ -1121,11 +1104,10 @@ func TestLeaveAll(t *testing.T) {
 }
 
 func TestContainerInvalidLeave(t *testing.T) {
-	if !testutils.IsRunningInContainer() {
-		defer testutils.SetupTestOSContext(t)()
-	}
+	defer testutils.SetupTestOSContext(t)()
+	controller := newController(t)
 
-	n, err := createTestNetwork(bridgeNetType, "testnetwork", options.Generic{
+	n, err := createTestNetwork(controller, bridgeNetType, "testnetwork", options.Generic{
 		netlabel.GenericData: options.Generic{
 			"BridgeName": "testnetwork",
 		},
@@ -1187,11 +1169,10 @@ func TestContainerInvalidLeave(t *testing.T) {
 }
 
 func TestEndpointUpdateParent(t *testing.T) {
-	if !testutils.IsRunningInContainer() {
-		defer testutils.SetupTestOSContext(t)()
-	}
+	defer testutils.SetupTestOSContext(t)()
+	controller := newController(t)
 
-	n, err := createTestNetwork(bridgeNetType, "testnetwork", options.Generic{
+	n, err := createTestNetwork(controller, bridgeNetType, "testnetwork", options.Generic{
 		netlabel.GenericData: options.Generic{
 			"BridgeName": "testnetwork",
 		},
@@ -1334,6 +1315,7 @@ func TestValidRemoteDriver(t *testing.T) {
 		t.Fatal(err)
 	}
 
+	controller := newController(t)
 	n, err := controller.NewNetwork("valid-network-driver", "dummy", "",
 		libnetwork.NetworkOptionGeneric(getEmptyGenericOption()))
 	if err != nil {
@@ -1349,23 +1331,3 @@ func TestValidRemoteDriver(t *testing.T) {
 		}
 	}()
 }
-
-var (
-	start  = make(chan struct{})
-	done   = make(chan chan struct{}, numThreads-1)
-	sboxes = make([]libnetwork.Sandbox, numThreads)
-)
-
-const (
-	iterCnt    = 25
-	numThreads = 3
-	first      = 1
-	last       = numThreads
-	debug      = false
-)
-
-func debugf(format string, a ...interface{}) {
-	if debug {
-		fmt.Printf(format, a...)
-	}
-}

libnetwork/netutils/utils_linux.go

@@ -24,6 +24,7 @@ var (
 
 // CheckRouteOverlaps checks whether the passed network overlaps with any existing routes
 func CheckRouteOverlaps(toCheck *net.IPNet) error {
+	networkGetRoutesFct := networkGetRoutesFct
 	if networkGetRoutesFct == nil {
 		networkGetRoutesFct = ns.NlHandle().RouteList
 	}

libnetwork/osl/namespace_linux.go

@@ -420,12 +420,6 @@ func (n *networkNamespace) DisableARPForVIP(srcName string) (Err error) {
 }
 
 func (n *networkNamespace) InvokeFunc(f func()) error {
-	origNS, err := netns.Get()
-	if err != nil {
-		return fmt.Errorf("failed to get original network namespace: %w", err)
-	}
-	defer origNS.Close()
-
 	path := n.nsPath()
 	newNS, err := netns.GetFromPath(path)
 	if err != nil {
@@ -436,6 +430,18 @@ func (n *networkNamespace) InvokeFunc(f func()) error {
 	done := make(chan error, 1)
 	go func() {
 		runtime.LockOSThread()
+		// InvokeFunc() could have been called from a goroutine with
+		// tampered thread state, e.g. from another InvokeFunc()
+		// callback. The outer goroutine's thread state cannot be
+		// trusted.
+		origNS, err := netns.Get()
+		if err != nil {
+			runtime.UnlockOSThread()
+			done <- fmt.Errorf("failed to get original network namespace: %w", err)
+			return
+		}
+		defer origNS.Close()
+
 		if err := netns.Set(newNS); err != nil {
 			runtime.UnlockOSThread()
 			done <- err

libnetwork/portallocator/portallocator.go

@@ -83,12 +83,13 @@ func Get() *PortAllocator {
 	// When this happens singleton behavior will be removed. Clients do not
 	// need to worry about this, they will not see a change in behavior.
 	once.Do(func() {
-		instance = newInstance()
+		instance = NewInstance()
 	})
 	return instance
 }
 
-func newInstance() *PortAllocator {
+// NewInstance is meant for use by libnetwork tests. It is not meant to be called directly.
+func NewInstance() *PortAllocator {
 	start, end, err := getDynamicPortRange()
 	if err != nil {
 		logrus.WithError(err).Infof("falling back to default port range %d-%d", defaultPortRangeStart, defaultPortRangeEnd)

libnetwork/portallocator/portallocator_test.go

@@ -6,7 +6,7 @@ import (
 )
 
 func resetPortAllocator() {
-	instance = newInstance()
+	instance = NewInstance()
 }
 
 func TestRequestNewPort(t *testing.T) {

libnetwork/resolver_test.go

@@ -7,6 +7,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/docker/docker/libnetwork/testutils"
 	"github.com/miekg/dns"
 	"github.com/sirupsen/logrus"
 	"gotest.tools/v3/skip"
@@ -76,6 +77,7 @@ func checkDNSRRType(t *testing.T, actual, expected uint16) {
 func TestDNSIPQuery(t *testing.T) {
 	skip.If(t, runtime.GOOS == "windows", "test only works on linux")
 
+	defer testutils.SetupTestOSContext(t)()
 	c, err := New()
 	if err != nil {
 		t.Fatal(err)
@@ -214,6 +216,9 @@ func waitForLocalDNSServer(t *testing.T) {
 func TestDNSProxyServFail(t *testing.T) {
 	skip.If(t, runtime.GOOS == "windows", "test only works on linux")
 
+	osctx := testutils.SetupTestOSContextEx(t)
+	defer osctx.Cleanup(t)
+
 	c, err := New()
 	if err != nil {
 		t.Fatal(err)
@@ -247,9 +252,9 @@ func TestDNSProxyServFail(t *testing.T) {
 	// use TCP for predictable results. Connection tests (to figure out DNS server initialization) don't work with UDP
 	server := &dns.Server{Addr: "127.0.0.1:53", Net: "tcp"}
 	srvErrCh := make(chan error, 1)
-	go func() {
+	osctx.Go(t, func() {
 		srvErrCh <- server.ListenAndServe()
-	}()
+	})
 	defer func() {
 		server.Shutdown() //nolint:errcheck
 		if err := <-srvErrCh; err != nil {

libnetwork/sandbox_test.go

@@ -25,14 +25,14 @@ func getTestEnv(t *testing.T, opts ...[]NetworkOption) (NetworkController, []Net
 	genericOption := make(map[string]interface{})
 	genericOption[netlabel.GenericData] = option
 
-	cfgOptions, err := OptionBoltdbWithRandomDBFile()
-	if err != nil {
-		t.Fatal(err)
-	}
-	c, err := New(append(cfgOptions, config.OptionDriverConfig(netType, genericOption))...)
+	c, err := New(
+		OptionBoltdbWithRandomDBFile(t),
+		config.OptionDriverConfig(netType, genericOption),
+	)
 	if err != nil {
 		t.Fatal(err)
 	}
+	t.Cleanup(c.Stop)
 
 	if len(opts) == 0 {
 		return c, nil
@@ -82,9 +82,7 @@ func TestSandboxAddEmpty(t *testing.T) {
 
 // // If different priorities are specified, internal option and ipv6 addresses mustn't influence endpoint order
 func TestSandboxAddMultiPrio(t *testing.T) {
-	if !testutils.IsRunningInContainer() {
-		defer testutils.SetupTestOSContext(t)()
-	}
+	defer testutils.SetupTestOSContext(t)()
 
 	opts := [][]NetworkOption{
 		{NetworkOptionEnableIPv6(true), NetworkOptionIpam(ipamapi.DefaultIPAM, "", nil, []*IpamConf{{PreferredPool: "fe90::/64"}}, nil)},
@@ -169,9 +167,7 @@ func TestSandboxAddMultiPrio(t *testing.T) {
 }
 
 func TestSandboxAddSamePrio(t *testing.T) {
-	if !testutils.IsRunningInContainer() {
-		defer testutils.SetupTestOSContext(t)()
-	}
+	defer testutils.SetupTestOSContext(t)()
 
 	opts := [][]NetworkOption{
 		{},

libnetwork/service_common_test.go

@@ -6,6 +6,7 @@ import (
 	"testing"
 
 	"github.com/docker/docker/libnetwork/resolvconf"
+	"github.com/docker/docker/libnetwork/testutils"
 	"gotest.tools/v3/assert"
 	is "gotest.tools/v3/assert/cmp"
 	"gotest.tools/v3/skip"
@@ -14,6 +15,7 @@ import (
 func TestCleanupServiceDiscovery(t *testing.T) {
 	skip.If(t, runtime.GOOS == "windows", "test only works on linux")
 
+	defer testutils.SetupTestOSContext(t)()
 	c, err := New()
 	assert.NilError(t, err)
 	defer c.Stop()

libnetwork/store_linux_test.go

@@ -17,14 +17,11 @@ func TestBoltdbBackend(t *testing.T) {
 }
 
 func TestNoPersist(t *testing.T) {
-	cfgOptions, err := OptionBoltdbWithRandomDBFile()
-	if err != nil {
-		t.Fatalf("Error creating random boltdb file : %v", err)
-	}
-	ctrl, err := New(cfgOptions...)
+	ctrl, err := New(OptionBoltdbWithRandomDBFile(t))
 	if err != nil {
 		t.Fatalf("Error new controller: %v", err)
 	}
+	defer ctrl.Stop()
 	nw, err := ctrl.NewNetwork("host", "host", "", NetworkOptionPersist(false))
 	if err != nil {
 		t.Fatalf("Error creating default \"host\" network: %v", err)

libnetwork/store_test.go

@@ -1,8 +1,8 @@
 package libnetwork
 
 import (
-	"fmt"
 	"os"
+	"path/filepath"
 	"testing"
 
 	"github.com/docker/docker/libnetwork/config"
@@ -27,6 +27,7 @@ func testLocalBackend(t *testing.T, provider, url string, storeConfig *store.Con
 	if err != nil {
 		t.Fatalf("Error new controller: %v", err)
 	}
+	defer ctrl.Stop()
 	nw, err := ctrl.NewNetwork("host", "host", "")
 	if err != nil {
 		t.Fatalf("Error creating default \"host\" network: %v", err)
@@ -49,41 +50,38 @@ func testLocalBackend(t *testing.T, provider, url string, storeConfig *store.Con
 	if err != nil {
 		t.Fatalf("Error creating controller: %v", err)
 	}
+	defer ctrl.Stop()
 	if _, err = ctrl.NetworkByID(nw.ID()); err != nil {
 		t.Fatalf("Error getting network %v", err)
 	}
 }
 
 // OptionBoltdbWithRandomDBFile function returns a random dir for local store backend
-func OptionBoltdbWithRandomDBFile() ([]config.Option, error) {
-	tmp, err := os.CreateTemp("", "libnetwork-")
-	if err != nil {
-		return nil, fmt.Errorf("Error creating temp file: %v", err)
+func OptionBoltdbWithRandomDBFile(t *testing.T) config.Option {
+	t.Helper()
+	tmp := filepath.Join(t.TempDir(), "bolt.db")
+	if err := os.WriteFile(tmp, nil, 0o600); err != nil {
+		t.Fatal(err)
 	}
-	if err := tmp.Close(); err != nil {
-		return nil, fmt.Errorf("Error closing temp file: %v", err)
+
+	return func(c *config.Config) {
+		config.OptionLocalKVProvider("boltdb")(c)
+		config.OptionLocalKVProviderURL(tmp)(c)
+		config.OptionLocalKVProviderConfig(&store.Config{Bucket: "testBackend"})(c)
 	}
-	cfgOptions := []config.Option{}
-	cfgOptions = append(cfgOptions, config.OptionLocalKVProvider("boltdb"))
-	cfgOptions = append(cfgOptions, config.OptionLocalKVProviderURL(tmp.Name()))
-	sCfg := &store.Config{Bucket: "testBackend"}
-	cfgOptions = append(cfgOptions, config.OptionLocalKVProviderConfig(sCfg))
-	return cfgOptions, nil
 }
 
 func TestMultipleControllersWithSameStore(t *testing.T) {
-	cfgOptions, err := OptionBoltdbWithRandomDBFile()
-	if err != nil {
-		t.Fatalf("Error getting random boltdb configs %v", err)
-	}
-	ctrl1, err := New(cfgOptions...)
+	cfgOptions := OptionBoltdbWithRandomDBFile(t)
+	ctrl1, err := New(cfgOptions)
 	if err != nil {
 		t.Fatalf("Error new controller: %v", err)
 	}
 	defer ctrl1.Stop()
 	// Use the same boltdb file without closing the previous controller
-	_, err = New(cfgOptions...)
+	ctrl2, err := New(cfgOptions)
 	if err != nil {
 		t.Fatalf("Local store must support concurrent controllers")
 	}
+	ctrl2.Stop()
 }

libnetwork/testutils/context.go

@@ -0,0 +1,41 @@
+package testutils
+
+import "testing"
+
+// Logger is used to log non-fatal messages during tests.
+type Logger interface {
+	Logf(format string, args ...any)
+}
+
+var _ Logger = (*testing.T)(nil)
+
+// SetupTestOSContext joins the current goroutine to a new network namespace,
+// and returns its associated teardown function.
+//
+// Example usage:
+//
+//	defer SetupTestOSContext(t)()
+func SetupTestOSContext(t *testing.T) func() {
+	c := SetupTestOSContextEx(t)
+	return func() { c.Cleanup(t) }
+}
+
+// Go starts running fn in a new goroutine inside the test OS context.
+func (c *OSContext) Go(t *testing.T, fn func()) {
+	t.Helper()
+	errCh := make(chan error, 1)
+	go func() {
+		teardown, err := c.Set()
+		if err != nil {
+			errCh <- err
+			return
+		}
+		defer teardown(t)
+		close(errCh)
+		fn()
+	}()
+
+	if err := <-errCh; err != nil {
+		t.Fatalf("%+v", err)
+	}
+}

libnetwork/testutils/context_unix.go

@@ -4,42 +4,52 @@
 package testutils
 
 import (
+	"fmt"
 	"runtime"
+	"strconv"
 	"testing"
 
 	"github.com/docker/docker/libnetwork/ns"
+	"github.com/pkg/errors"
 	"github.com/vishvananda/netns"
+	"golang.org/x/sys/unix"
 )
 
-// SetupTestOSContext joins a new network namespace, and returns its associated
-// teardown function.
+// OSContext is a handle to a test OS context.
+type OSContext struct {
+	origNS, newNS netns.NsHandle
+
+	tid    int
+	caller string // The file:line where SetupTestOSContextEx was called, for interpolating into error messages.
+}
+
+// SetupTestOSContextEx joins the current goroutine to a new network namespace.
+//
+// Compared to [SetupTestOSContext], this function allows goroutines to be
+// spawned which are associated with the same OS context via the returned
+// OSContext value.
 //
 // Example usage:
 //
-//	defer SetupTestOSContext(t)()
-func SetupTestOSContext(t *testing.T) func() {
+//	c := SetupTestOSContext(t)
+//	defer c.Cleanup(t)
+func SetupTestOSContextEx(t *testing.T) *OSContext {
+	runtime.LockOSThread()
 	origNS, err := netns.Get()
 	if err != nil {
+		runtime.UnlockOSThread()
 		t.Fatalf("Failed to open initial netns: %v", err)
 	}
-	restore := func() {
-		if err := netns.Set(origNS); err != nil {
-			t.Logf("Warning: failed to restore thread netns (%v)", err)
-		} else {
-			runtime.UnlockOSThread()
-		}
 
-		if err := origNS.Close(); err != nil {
-			t.Logf("Warning: netns closing failed (%v)", err)
-		}
+	c := OSContext{
+		tid:    unix.Gettid(),
+		origNS: origNS,
 	}
-
-	runtime.LockOSThread()
-	newNS, err := netns.New()
+	c.newNS, err = netns.New()
 	if err != nil {
 		// netns.New() is not atomic: it could have encountered an error
 		// after unsharing the current thread's network namespace.
-		restore()
+		c.restore(t)
 		t.Fatalf("Failed to enter netns: %v", err)
 	}
 
@@ -47,11 +57,113 @@ func SetupTestOSContext(t *testing.T) func() {
 	// sure to re-initialize initNs context
 	ns.Init()
 
-	return func() {
-		if err := newNS.Close(); err != nil {
-			t.Logf("Warning: netns closing failed (%v)", err)
-		}
-		restore()
-		ns.Init()
+	nl := ns.NlHandle()
+	lo, err := nl.LinkByName("lo")
+	if err != nil {
+		c.restore(t)
+		t.Fatalf("Failed to get handle to loopback interface 'lo' in new netns: %v", err)
+	}
+	if err := nl.LinkSetUp(lo); err != nil {
+		c.restore(t)
+		t.Fatalf("Failed to enable loopback interface in new netns: %v", err)
+	}
+
+	_, file, line, ok := runtime.Caller(0)
+	if ok {
+		c.caller = file + ":" + strconv.Itoa(line)
+	}
+
+	return &c
+}
+
+// Cleanup tears down the OS context. It must be called from the same goroutine
+// as the [SetupTestOSContextEx] call which returned c.
+//
+// Explicit cleanup is required as (*testing.T).Cleanup() makes no guarantees
+// about which goroutine the cleanup functions are invoked on.
+func (c *OSContext) Cleanup(t *testing.T) {
+	t.Helper()
+	if unix.Gettid() != c.tid {
+		t.Fatalf("c.Cleanup() must be called from the same goroutine as SetupTestOSContextEx() (%s)", c.caller)
 	}
+	if err := c.newNS.Close(); err != nil {
+		t.Logf("Warning: netns closing failed (%v)", err)
+	}
+	c.restore(t)
+	ns.Init()
+}
+
+func (c *OSContext) restore(t *testing.T) {
+	t.Helper()
+	if err := netns.Set(c.origNS); err != nil {
+		t.Logf("Warning: failed to restore thread netns (%v)", err)
+	} else {
+		runtime.UnlockOSThread()
+	}
+
+	if err := c.origNS.Close(); err != nil {
+		t.Logf("Warning: netns closing failed (%v)", err)
+	}
+}
+
+// Set sets the OS context of the calling goroutine to c and returns a teardown
+// function to restore the calling goroutine's OS context and release resources.
+// The teardown function accepts an optional Logger argument.
+//
+// This is a lower-level interface which is less ergonomic than c.Go() but more
+// composable with other goroutine-spawning utilities such as [sync.WaitGroup]
+// or [golang.org/x/sync/errgroup.Group].
+//
+// Example usage:
+//
+//	func TestFoo(t *testing.T) {
+//		osctx := testutils.SetupTestOSContextEx(t)
+//		defer osctx.Cleanup(t)
+//		var eg errgroup.Group
+//		eg.Go(func() error {
+//			teardown, err := osctx.Set()
+//			if err != nil {
+//				return err
+//			}
+//			defer teardown(t)
+//			// ...
+//		})
+//		if err := eg.Wait(); err != nil {
+//			t.Fatalf("%+v", err)
+//		}
+//	}
+func (c *OSContext) Set() (func(Logger), error) {
+	runtime.LockOSThread()
+	orig, err := netns.Get()
+	if err != nil {
+		runtime.UnlockOSThread()
+		return nil, errors.Wrap(err, "failed to open initial netns for goroutine")
+	}
+	if err := errors.WithStack(netns.Set(c.newNS)); err != nil {
+		runtime.UnlockOSThread()
+		return nil, errors.Wrap(err, "failed to set goroutine network namespace")
+	}
+
+	tid := unix.Gettid()
+	_, file, line, callerOK := runtime.Caller(0)
+
+	return func(log Logger) {
+		if unix.Gettid() != tid {
+			msg := "teardown function must be called from the same goroutine as c.Set()"
+			if callerOK {
+				msg += fmt.Sprintf(" (%s:%d)", file, line)
+			}
+			panic(msg)
+		}
+
+		if err := netns.Set(orig); err != nil && log != nil {
+			log.Logf("Warning: failed to restore goroutine thread netns (%v)", err)
+		} else {
+			runtime.UnlockOSThread()
+		}
+
+		if err := orig.Close(); err != nil && log != nil {
+			log.Logf("Warning: netns closing failed (%v)", err)
+		}
+	}, nil
 }

libnetwork/testutils/context_windows.go

@@ -2,12 +2,14 @@ package testutils
 
 import "testing"
 
-// SetupTestOSContext joins a new network namespace, and returns its associated
-// teardown function.
-//
-// Example usage:
-//
-//	defer SetupTestOSContext(t)()
-func SetupTestOSContext(t *testing.T) func() {
-	return func() {}
+type OSContext struct{}
+
+func SetupTestOSContextEx(*testing.T) *OSContext {
+	return nil
+}
+
+func (*OSContext) Cleanup(t *testing.T) {}
+
+func (*OSContext) Set() (func(Logger), error) {
+	return func(Logger) {}, nil
 }

libnetwork/testutils/net.go

@@ -1,11 +0,0 @@
-package testutils
-
-import (
-	"os"
-)
-
-// IsRunningInContainer returns whether the test is running inside a container.
-func IsRunningInContainer() bool {
-	_, err := os.Stat("/.dockerenv")
-	return err == nil
-}