Jenkinsfile: reduce time of integration tests by dividing tests into 3 parallel runs

Signed-off-by: Tibor Vass <tibor@docker.com>

Jenkinsfile

@@ -80,7 +80,7 @@ pipeline {
                                 '''
                             }
                         }
-			stage("Static") {
+                        stage("Static") {
                             steps {
                                 sh '''
                                 docker run --rm -t --privileged \
@@ -203,20 +203,61 @@ pipeline {
                         }
                         stage("Run tests") {
                             steps {
-                                sh '''
+                                sh '''#!/bin/bash
+                                # bash is needed so 'jobs -p' works properly
+                                # it also accepts setting inline envvars for functions without explicitly exporting
+ 
+                                run_tests() {
+                                        [ -n "$TESTDEBUG" ] && rm= || rm=--rm;
+                                        docker run $rm -t --privileged \
+                                          -v "$WORKSPACE/bundles:/go/src/github.com/docker/docker/bundles" \
+                                          -v "$WORKSPACE/.git:/go/src/github.com/docker/docker/.git" \
+                                          --name "$CONTAINER_NAME" \
+                                          -e KEEPBUNDLE=1 \
+                                          -e TESTDEBUG \
+                                          -e TESTFLAGS \
+                                          -e TEST_INTEGRATION_DEST \
+                                          -e TEST_SKIP_INTEGRATION \
+                                          -e TEST_SKIP_INTEGRATION_CLI \
+                                          -e DOCKER_GITCOMMIT=${GIT_COMMIT} \
+                                          -e DOCKER_GRAPHDRIVER \
+                                          docker:${GIT_COMMIT} \
+                                          hack/make.sh \
+                                            "$1" \
+                                            test-integration
+                                }
+
+                                trap "exit" INT TERM
+                                trap 'pids=$(jobs -p); echo "Remaining pids to kill: [$pids]"; [ -z "$pids" ] || kill $pids' EXIT
+
+                                CONTAINER_NAME=docker-pr$BUILD_NUMBER
+
                                 docker run --rm -t --privileged \
                                   -v "$WORKSPACE/bundles:/go/src/github.com/docker/docker/bundles" \
                                   -v "$WORKSPACE/.git:/go/src/github.com/docker/docker/.git" \
-                                  --name docker-pr$BUILD_NUMBER \
+                                  --name ${CONTAINER_NAME}-build \
                                   -e DOCKER_EXPERIMENTAL \
                                   -e DOCKER_GITCOMMIT=${GIT_COMMIT} \
                                   -e DOCKER_GRAPHDRIVER \
                                   docker:${GIT_COMMIT} \
                                   hack/make.sh \
-                                    binary-daemon \
-                                    dynbinary-daemon \
-                                    test-integration-flaky \
-                                    test-integration \
+                                    dynbinary-daemon
+
+                                # flaky + integration
+                                TEST_INTEGRATION_DEST=1 CONTAINER_NAME=${CONTAINER_NAME}-1 TEST_SKIP_INTEGRATION_CLI=1 run_tests test-integration-flaky &
+
+                                # integration-cli first set
+                                TEST_INTEGRATION_DEST=2 CONTAINER_NAME=${CONTAINER_NAME}-2 TEST_SKIP_INTEGRATION=1 TESTFLAGS="-check.f ^(DockerSuite|DockerNetworkSuite|DockerHubPullSuite|DockerRegistrySuite|DockerSchema1RegistrySuite|DockerRegistryAuthTokenSuite|DockerRegistryAuthHtpasswdSuite)" run_tests &
+
+                                # integration-cli second set
+                                TEST_INTEGRATION_DEST=3 CONTAINER_NAME=${CONTAINER_NAME}-3 TEST_SKIP_INTEGRATION=1 TESTFLAGS="-check.f ^(DockerSwarmSuite|DockerDaemonSuite|DockerExternalVolumeSuite)" run_tests &
+
+                                set +x
+                                c=0
+                                for job in $(jobs -p); do
+                                        wait ${job} || c=$?
+                                done
+                                exit $c
                                 '''
                             }
                         }

Makefile

@@ -53,6 +53,7 @@ DOCKER_ENVS := \
 	-e DOCKER_TEST_HOST \
 	-e DOCKER_USERLANDPROXY \
 	-e DOCKERD_ARGS \
+	-e TEST_INTEGRATION_DEST \
 	-e TEST_INTEGRATION_DIR \
 	-e TEST_SKIP_INTEGRATION \
 	-e TEST_SKIP_INTEGRATION_CLI \

hack/make/test-integration

@@ -1,15 +1,21 @@
 #!/usr/bin/env bash
 set -e -o pipefail
 
+if [ -n "$TEST_INTEGRATION_DEST" ]; then
+	export DEST="$ABS_DEST/$TEST_INTEGRATION_DEST"
+	export DOCKER_INTEGRATION_DAEMON_DEST="$DEST"
+	mkdir -p "$DEST"
+fi
+
 source hack/make/.integration-test-helpers
 
 if [ ! -z "${TEST_SKIP_INTEGRATION}" ] && [ ! -z "${TEST_SKIP_INTEGRATION_CLI}" ]; then
-	echo integration and integraiton-cli skipped according to env vars
+	echo integration and integration-cli skipped according to env vars
 	exit 0
 fi
 
-
 (
+	env
 	build_test_suite_binaries
 	bundle .integration-daemon-start
 	bundle .integration-daemon-setup

internal/test/daemon/daemon.go

@@ -98,7 +98,6 @@ func New(t testingT, ops ...func(*Daemon)) *Daemon {
 	if ht, ok := t.(test.HelperT); ok {
 		ht.Helper()
 	}
-	t.Log("Creating a new daemon")
 	dest := os.Getenv("DOCKER_INTEGRATION_DAEMON_DEST")
 	if dest == "" {
 		dest = os.Getenv("DEST")
@@ -109,6 +108,7 @@ func New(t testingT, ops ...func(*Daemon)) *Daemon {
 	case testNamer:
 		dest = filepath.Join(dest, v.TestName())
 	}
+	t.Logf("Creating a new daemon at: %s", dest)
 	assert.Check(t, dest != "", "Please set the DOCKER_INTEGRATION_DAEMON_DEST or the DEST environment variable")
 
 	storageDriver := os.Getenv("DOCKER_GRAPHDRIVER")