Merge pull request #24948 from dnephin/support-iterative-integration-testing

Support running integration tests without restarting a daemon

Dockerfile

@@ -201,6 +201,8 @@ ENV DOCKER_BUILDTAGS apparmor pkcs11 seccomp selinux
 
 # Let us use a .bashrc file
 RUN ln -sfv $PWD/.bashrc ~/.bashrc
+# Add integration helps to bashrc
+RUN echo "source $PWD/hack/make/.integration-test-helpers" >> /etc/bash.bashrc
 
 # Register Docker's bash completion.
 RUN ln -sv $PWD/contrib/completion/bash/docker /etc/bash_completion.d/docker

hack/make.sh

@@ -28,8 +28,6 @@ export SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 export MAKEDIR="$SCRIPTDIR/make"
 export PKG_CONFIG=${PKG_CONFIG:-pkg-config}
 
-: ${TEST_REPEAT:=0}
-
 # We're a nice, sexy, little shell script, and people might try to run us;
 # but really, they shouldn't. We want to be in a container!
 inContainer="AssumeSoInitially"
@@ -224,65 +222,6 @@ if \
 	HAVE_GO_TEST_COVER=1
 fi
 
-# If $TESTFLAGS is set in the environment, it is passed as extra arguments to 'go test'.
-# You can use this to select certain tests to run, eg.
-#
-#     TESTFLAGS='-test.run ^TestBuild$' ./hack/make.sh test-unit
-#
-# For integration-cli test, we use [gocheck](https://labix.org/gocheck), if you want
-# to run certain tests on your local host, you should run with command:
-#
-#     TESTFLAGS='-check.f DockerSuite.TestBuild*' ./hack/make.sh binary test-integration-cli
-#
-go_test_dir() {
-	dir=$1
-	coverpkg=$2
-	testcover=()
-	testcoverprofile=()
-	testbinary="$DEST/test.main"
-	if [ "$HAVE_GO_TEST_COVER" ]; then
-		# if our current go install has -cover, we want to use it :)
-		mkdir -p "$DEST/coverprofiles"
-		coverprofile="docker${dir#.}"
-		coverprofile="$ABS_DEST/coverprofiles/${coverprofile//\//-}"
-		testcover=( -test.cover )
-		testcoverprofile=( -test.coverprofile "$coverprofile" $coverpkg )
-	fi
-	(
-		echo '+ go test' $TESTFLAGS "${DOCKER_PKG}${dir#.}"
-		cd "$dir"
-		export DEST="$ABS_DEST" # we're in a subshell, so this is safe -- our integration-cli tests need DEST, and "cd" screws it up
-		go test -c -o "$testbinary" ${testcover[@]} -ldflags "$LDFLAGS" "${BUILDFLAGS[@]}"
-		i=0
-		while ((++i)); do
-			test_env "$testbinary" ${testcoverprofile[@]} $TESTFLAGS
-			if [ $i -gt "$TEST_REPEAT" ]; then
-				break
-			fi
-			echo "Repeating test ($i)"
-		done
-	)
-}
-test_env() {
-	# use "env -i" to tightly control the environment variables that bleed into the tests
-	env -i \
-		DEST="$DEST" \
-		DOCKER_TLS_VERIFY="$DOCKER_TEST_TLS_VERIFY" \
-		DOCKER_CERT_PATH="$DOCKER_TEST_CERT_PATH" \
-		DOCKER_ENGINE_GOARCH="$DOCKER_ENGINE_GOARCH" \
-		DOCKER_GRAPHDRIVER="$DOCKER_GRAPHDRIVER" \
-		DOCKER_USERLANDPROXY="$DOCKER_USERLANDPROXY" \
-		DOCKER_HOST="$DOCKER_HOST" \
-		DOCKER_REMAP_ROOT="$DOCKER_REMAP_ROOT" \
-		DOCKER_REMOTE_DAEMON="$DOCKER_REMOTE_DAEMON" \
-		GOPATH="$GOPATH" \
-		GOTRACEBACK=all \
-		HOME="$ABS_DEST/fake-HOME" \
-		PATH="$PATH" \
-		TEMP="$TEMP" \
-		"$@"
-}
-
 # a helper to provide ".exe" when it's appropriate
 binary_extension() {
 	if [ "$(go env GOOS)" = 'windows' ]; then

hack/make/.integration-test-helpers

@@ -0,0 +1,68 @@
+#!/bin/bash
+
+: ${TEST_REPEAT:=0}
+
+bundle_test_integration_cli() {
+	TESTFLAGS="$TESTFLAGS -check.v -check.timeout=${TIMEOUT} -test.timeout=360m"
+	go_test_dir ./integration-cli
+}
+
+# If $TESTFLAGS is set in the environment, it is passed as extra arguments to 'go test'.
+# You can use this to select certain tests to run, eg.
+#
+#     TESTFLAGS='-test.run ^TestBuild$' ./hack/make.sh test-unit
+#
+# For integration-cli test, we use [gocheck](https://labix.org/gocheck), if you want
+# to run certain tests on your local host, you should run with command:
+#
+#     TESTFLAGS='-check.f DockerSuite.TestBuild*' ./hack/make.sh binary test-integration-cli
+#
+go_test_dir() {
+	dir=$1
+	coverpkg=$2
+	testcover=()
+	testcoverprofile=()
+	testbinary="$DEST/test.main"
+	if [ "$HAVE_GO_TEST_COVER" ]; then
+		# if our current go install has -cover, we want to use it :)
+		mkdir -p "$DEST/coverprofiles"
+		coverprofile="docker${dir#.}"
+		coverprofile="$ABS_DEST/coverprofiles/${coverprofile//\//-}"
+		testcover=( -test.cover )
+		testcoverprofile=( -test.coverprofile "$coverprofile" $coverpkg )
+	fi
+	(
+		echo '+ go test' $TESTFLAGS "${DOCKER_PKG}${dir#.}"
+		cd "$dir"
+		export DEST="$ABS_DEST" # in a subshell this is safe -- our integration-cli tests need DEST, and "cd" screws it up
+		go test -c -o "$testbinary" ${testcover[@]} -ldflags "$LDFLAGS" "${BUILDFLAGS[@]}"
+		i=0
+		while ((++i)); do
+			test_env "$testbinary" ${testcoverprofile[@]} $TESTFLAGS
+			if [ $i -gt "$TEST_REPEAT" ]; then
+				break
+			fi
+			echo "Repeating test ($i)"
+		done
+	)
+}
+
+test_env() {
+	# use "env -i" to tightly control the environment variables that bleed into the tests
+	env -i \
+		DEST="$DEST" \
+		DOCKER_TLS_VERIFY="$DOCKER_TEST_TLS_VERIFY" \
+		DOCKER_CERT_PATH="$DOCKER_TEST_CERT_PATH" \
+		DOCKER_ENGINE_GOARCH="$DOCKER_ENGINE_GOARCH" \
+		DOCKER_GRAPHDRIVER="$DOCKER_GRAPHDRIVER" \
+		DOCKER_USERLANDPROXY="$DOCKER_USERLANDPROXY" \
+		DOCKER_HOST="$DOCKER_HOST" \
+		DOCKER_REMAP_ROOT="$DOCKER_REMAP_ROOT" \
+		DOCKER_REMOTE_DAEMON="$DOCKER_REMOTE_DAEMON" \
+		GOPATH="$GOPATH" \
+		GOTRACEBACK=all \
+		HOME="$ABS_DEST/fake-HOME" \
+		PATH="$PATH" \
+		TEMP="$TEMP" \
+		"$@"
+}

hack/make/test-docker-py

@@ -1,6 +1,8 @@
 #!/bin/bash
 set -e
 
+source hack/make/.integration-test-helpers
+
 # subshell so that we can export PATH without breaking other things
 (
 	bundle .integration-daemon-start

hack/make/test-integration-cli

@@ -1,10 +1,7 @@
 #!/bin/bash
 set -e
 
-bundle_test_integration_cli() {
-	TESTFLAGS="$TESTFLAGS -check.v -check.timeout=${TIMEOUT} -test.timeout=360m"
-	go_test_dir ./integration-cli
-}
+source hack/make/.integration-test-helpers
 
 # subshell so that we can export PATH without breaking other things
 (

hack/make/test-integration-shell

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+bundle .integration-daemon-start
+bundle .integration-daemon-setup
+
+export ABS_DEST
+bash +e