integration-cli: move each test suite to its own TestX testing function

Signed-off-by: Tibor Vass <tibor@docker.com>
(cherry picked from commit f1c1cd436ae1252778417121e77d084172f628a1)
Signed-off-by: Sebastiaan van Stijn <github@gone.nl>

Jenkinsfile

@@ -307,10 +307,10 @@ pipeline {
                                 TEST_INTEGRATION_DEST=1 CONTAINER_NAME=${CONTAINER_NAME}-1 TEST_SKIP_INTEGRATION_CLI=1 run_tests test-integration-flaky &
 
                                 # integration-cli first set
-                                TEST_INTEGRATION_DEST=2 CONTAINER_NAME=${CONTAINER_NAME}-2 TEST_SKIP_INTEGRATION=1 TESTFLAGS="-test.run /(DockerSuite|DockerNetworkSuite|DockerHubPullSuite|DockerRegistrySuite|DockerSchema1RegistrySuite|DockerRegistryAuthTokenSuite|DockerRegistryAuthHtpasswdSuite)/" run_tests &
+                                TEST_INTEGRATION_DEST=2 CONTAINER_NAME=${CONTAINER_NAME}-2 TEST_SKIP_INTEGRATION=1 TESTFLAGS="-test.run Test(DockerSuite|DockerNetworkSuite|DockerHubPullSuite|DockerRegistrySuite|DockerSchema1RegistrySuite|DockerRegistryAuthTokenSuite|DockerRegistryAuthHtpasswdSuite)/" run_tests &
 
                                 # integration-cli second set
-                                TEST_INTEGRATION_DEST=3 CONTAINER_NAME=${CONTAINER_NAME}-3 TEST_SKIP_INTEGRATION=1 TESTFLAGS="-test.run /(DockerSwarmSuite|DockerDaemonSuite|DockerExternalVolumeSuite)/" run_tests &
+                                TEST_INTEGRATION_DEST=3 CONTAINER_NAME=${CONTAINER_NAME}-3 TEST_SKIP_INTEGRATION=1 TESTFLAGS="-test.run Test(DockerSwarmSuite|DockerDaemonSuite|DockerExternalVolumeSuite)/" run_tests &
 
                                 set +x
                                 c=0

docs/contributing/test.md

@@ -174,13 +174,13 @@ flag's value is passed as arguments to the `go test` command. For example, from
 your local host you can run the `TestBuild` test with this command:
 
 ```bash
-$ TESTFLAGS='-test.run /DockerSuite/TestBuild*' make test-integration
+$ TESTFLAGS='-test.run TestDockerSuite/TestBuild*' make test-integration
 ```
 
 To run the same test inside your Docker development container, you do this:
 
 ```bash
-# TESTFLAGS='-test.run /DockerSuite/TestBuild*' hack/make.sh binary test-integration
+# TESTFLAGS='-test.run TestDockerSuite/TestBuild*' hack/make.sh binary test-integration
 ```
 
 ## Test the Windows binary against a Linux daemon
@@ -228,11 +228,11 @@ run a Bash terminal on Windows.
     ```
 
     Should you wish to run a single test such as one with the name
-    'TestExample', you can pass in `TESTFLAGS='-test.run //TestExample'`. For
+    'TestExample', you can pass in `TESTFLAGS='-test.run /TestExample'`. For
     example
 
     ```bash
-    $ TESTFLAGS='-test.run //TestExample' hack/make.sh binary test-integration
+    $ TESTFLAGS='-test.run /TestExample' hack/make.sh binary test-integration
     ```
 
 You can now choose to make changes to the Moby source or the tests. If you

hack/make/.integration-test-helpers

@@ -3,7 +3,7 @@
 # For integration-cli test, we use [gocheck](https://labix.org/gocheck), if you want
 # to run certain tests on your local host, you should run with command:
 #
-#     TESTFLAGS='-test.run /DockerSuite/TestBuild*' ./hack/make.sh binary test-integration
+#     TESTFLAGS='-test.run TestDockerSuite/TestBuild*' ./hack/make.sh binary test-integration
 #
 
 if [ -z "${MAKEDIR}" ]; then

integration-cli/check_test.go

@@ -9,7 +9,6 @@ import (
 	"os"
 	"path"
 	"path/filepath"
-	"runtime"
 	"strconv"
 	"sync"
 	"syscall"
@@ -45,6 +44,8 @@ var (
 
 	// the docker client binary to use
 	dockerBinary = ""
+
+	testEnvOnce sync.Once
 )
 
 func init() {
@@ -74,25 +75,72 @@ func TestMain(m *testing.M) {
 	os.Exit(m.Run())
 }
 
-func Test(t *testing.T) {
-	cli.SetTestEnvironment(testEnv)
-	fakestorage.SetTestEnvironment(&testEnv.Execution)
-	ienv.ProtectAll(t, &testEnv.Execution)
+func ensureTestEnvSetup(t *testing.T) {
+	testEnvOnce.Do(func() {
+		cli.SetTestEnvironment(testEnv)
+		fakestorage.SetTestEnvironment(&testEnv.Execution)
+		ienv.ProtectAll(t, &testEnv.Execution)
+	})
+}
+
+func TestDockerSuite(t *testing.T) {
+	ensureTestEnvSetup(t)
 	suite.Run(t, &DockerSuite{})
+}
+
+func TestDockerRegistrySuite(t *testing.T) {
+	ensureTestEnvSetup(t)
 	suite.Run(t, &DockerRegistrySuite{ds: &DockerSuite{}})
+}
+
+func TestDockerSchema1RegistrySuite(t *testing.T) {
+	ensureTestEnvSetup(t)
 	suite.Run(t, &DockerSchema1RegistrySuite{ds: &DockerSuite{}})
+}
+
+func TestDockerRegistryAuthHtpasswdSuite(t *testing.T) {
+	ensureTestEnvSetup(t)
 	suite.Run(t, &DockerRegistryAuthHtpasswdSuite{ds: &DockerSuite{}})
+}
+
+func TestDockerRegistryAuthTokenSuite(t *testing.T) {
+	ensureTestEnvSetup(t)
 	suite.Run(t, &DockerRegistryAuthTokenSuite{ds: &DockerSuite{}})
+}
+
+func TestDockerDaemonSuite(t *testing.T) {
+	ensureTestEnvSetup(t)
 	suite.Run(t, &DockerDaemonSuite{ds: &DockerSuite{}})
+}
+
+func TestDockerSwarmSuite(t *testing.T) {
+	ensureTestEnvSetup(t)
 	suite.Run(t, &DockerSwarmSuite{ds: &DockerSuite{}})
+}
+
+func TestDockerPluginSuite(t *testing.T) {
+	ensureTestEnvSetup(t)
 	suite.Run(t, &DockerPluginSuite{ds: &DockerSuite{}})
-	if runtime.GOOS != "windows" {
-		suite.Run(t, &DockerExternalVolumeSuite{ds: &DockerSuite{}})
-		suite.Run(t, &DockerNetworkSuite{ds: &DockerSuite{}})
-		// FIXME. Temporarily turning this off for Windows as GH16039 was breaking
-		// Windows to Linux CI @icecrime
-		suite.Run(t, newDockerHubPullSuite())
-	}
+}
+
+func TestDockerExternalVolumeSuite(t *testing.T) {
+	ensureTestEnvSetup(t)
+	testRequires(t, DaemonIsLinux)
+	suite.Run(t, &DockerExternalVolumeSuite{ds: &DockerSuite{}})
+}
+
+func TestDockerNetworkSuite(t *testing.T) {
+	ensureTestEnvSetup(t)
+	testRequires(t, DaemonIsLinux)
+	suite.Run(t, &DockerExternalVolumeSuite{ds: &DockerSuite{}})
+}
+
+func TestDockerHubPullSuite(t *testing.T) {
+	ensureTestEnvSetup(t)
+	// FIXME. Temporarily turning this off for Windows as GH16039 was breaking
+	// Windows to Linux CI @icecrime
+	testRequires(t, DaemonIsLinux)
+	suite.Run(t, newDockerHubPullSuite())
 }
 
 type DockerSuite struct {

internal/test/suite/suite.go

@@ -30,13 +30,12 @@ func Run(t *testing.T, suite interface{}) {
 	}()
 
 	methodFinder := reflect.TypeOf(suite)
-	suiteName := methodFinder.Elem().Name()
 	for index := 0; index < methodFinder.NumMethod(); index++ {
 		method := methodFinder.Method(index)
 		if !methodFilter(method.Name, method.Type) {
 			continue
 		}
-		t.Run(suiteName+"/"+method.Name, func(t *testing.T) {
+		t.Run(method.Name, func(t *testing.T) {
 			defer failOnPanic(t)
 
 			if !suiteSetupDone {