Merge pull request #39628 from cpuguy83/test_setup_improvements

Improvements to the test runners

Makefile

@@ -54,9 +54,13 @@ DOCKER_ENVS := \
 	-e DOCKER_USERLANDPROXY \
 	-e DOCKERD_ARGS \
 	-e TEST_INTEGRATION_DIR \
+	-e TEST_SKIP_INTEGRATION \
+	-e TEST_SKIP_INTEGRATION_CLI \
 	-e TESTDEBUG \
 	-e TESTDIRS \
 	-e TESTFLAGS \
+	-e TESTFLAGS_INTEGRATION \
+	-e TESTFLAGS_INTEGRATION_CLI \
 	-e TIMEOUT \
 	-e VALIDATE_REPO \
 	-e VALIDATE_BRANCH \
@@ -182,8 +186,13 @@ test-docker-py: build ## run the docker-py tests
 
 test-integration-cli: test-integration ## (DEPRECATED) use test-integration
 
+ifneq ($(and $(TEST_SKIP_INTEGRATION),$(TEST_SKIP_INTEGRATION_CLI)),)
+test-integration:
+	@echo Both integrations suites skipped per environment variables
+else
 test-integration: build ## run the integration tests
 	$(DOCKER_RUN_DOCKER) hack/make.sh dynbinary test-integration
+endif
 
 test-integration-flaky: build ## run the stress test for all new integration tests
 	$(DOCKER_RUN_DOCKER) hack/make.sh dynbinary test-integration-flaky

hack/make.ps1

@@ -327,19 +327,16 @@ Function Run-UnitTests() {
 # Run the integration tests
 Function Run-IntegrationTests() {
     $env:DOCKER_INTEGRATION_DAEMON_DEST = $root + "\bundles\tmp"
-    $dirs =  Get-ChildItem -Path integration -Directory -Recurse
+    $dirs = go list -test -f '{{- if ne .ForTest `"`" -}}{{- .Dir -}}{{- end -}}' .\integration\...
     $integration_api_dirs = @()
     ForEach($dir in $dirs) {
-        $RelativePath = "." + $dir.FullName -replace "$($PWD.Path -replace "\\","\\")",""
-        If ($RelativePath -notmatch '(^.\\integration($|\\internal)|\\testdata)') {
-            $integration_api_dirs += $dir
-            Write-Host "Building test suite binary $RelativePath"
-            go test -c -o "$RelativePath\test.exe" $RelativePath
-        }
+        $integration_api_dirs += $dir
+        Write-Host "Building test suite binary $dir"
+        go test -c -o "$dir\test.exe" $dir
     }
 
     ForEach($dir in $integration_api_dirs) {
-        Set-Location $dir.FullName
+        Set-Location $dir
         Write-Host "Running $($PWD.Path)"
         $pinfo = New-Object System.Diagnostics.ProcessStartInfo
         $pinfo.FileName = "$($PWD.Path)\test.exe"

hack/make/.integration-test-helpers

@@ -5,6 +5,18 @@
 #
 #     TESTFLAGS='-check.f DockerSuite.TestBuild*' ./hack/make.sh binary test-integration
 #
+
+if [[ "${TESTFLAGS}" = *-check.f* ]]; then
+	echo Skipping integration tests since TESTFLAGS includes integration-cli only flags
+	TEST_SKIP_INTEGRATION=1
+fi
+
+if [[ "${TESTFLAGS}" = *-test.run* ]]; then
+	echo Skipping integration-cli tests since TESTFLAGS includes integration only flags
+	TEST_SKIP_INTEGRATION_CLI=1
+fi
+
+
 if [ -z ${MAKEDIR} ]; then
 	export MAKEDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 fi
@@ -15,26 +27,24 @@ source "$MAKEDIR/.go-autogen"
 : ${TESTFLAGS:=}
 : ${TESTDEBUG:=}
 
-integration_api_dirs=${TEST_INTEGRATION_DIR:-"$(
-	find ./integration -type d |
-	grep -vE '(^./integration($|/internal)|/testdata)')"}
+integration_api_dirs=${TEST_INTEGRATION_DIR:-"$(go list  -test -f '{{- if ne .ForTest "" -}}{{- .Dir -}}{{- end -}}'  ./integration/...)"}
 
 run_test_integration() {
 	set_platform_timeout
-	if [[ "$TESTFLAGS" != *-check.f* ]]; then
+	if [ -z "${TEST_SKIP_INTEGRATION}" ]; then
 		run_test_integration_suites
 	fi
-	if [[ "$TESTFLAGS" != *-test.run* ]]; then
+	if [ -z "${TEST_SKIP_INTEGRATION_CLI}" ]; then
 		run_test_integration_legacy_suites
 	fi
 }
 
 run_test_integration_suites() {
-	local flags="-test.v -test.timeout=${TIMEOUT} $TESTFLAGS"
+	local flags="-test.v -test.timeout=${TIMEOUT} $TESTFLAGS ${TESTFLAGS_INTEGRATION}"
 	for dir in ${integration_api_dirs}; do
 		if ! (
 			cd "$dir"
-			echo "Running $PWD"
+			echo "Running $PWD flags=${flags}"
 			test_env ./test.main ${flags}
 		); then exit 1; fi
 	done
@@ -42,9 +52,9 @@ run_test_integration_suites() {
 
 run_test_integration_legacy_suites() {
 	(
-		flags="-check.v -check.timeout=${TIMEOUT} -test.timeout=360m $TESTFLAGS"
+		flags="-check.v -check.timeout=${TIMEOUT} -test.timeout=360m $TESTFLAGS ${TESTFLAGS_INTEGRATION_CLI}"
 		cd integration-cli
-		echo "Running $PWD"
+		echo "Running $PWD flags=${flags}"
 		test_env ./test.main $flags
 	)
 }
@@ -54,10 +64,14 @@ build_test_suite_binaries() {
 		echo "Skipping building test binaries; as DOCKER_INTEGRATION_TESTS_VERIFIED is set"
 		return
 	fi
-	build_test_suite_binary ./integration-cli "test.main"
-	for dir in ${integration_api_dirs}; do
-		build_test_suite_binary "$dir" "test.main"
-	done
+	if [ -z "${TEST_SKIP_INTEGRATION_CLI}" ]; then
+		build_test_suite_binary ./integration-cli "test.main"
+	fi
+	if [ -z "${TEST_SKIP_INTEGRATION}" ]; then
+		for dir in ${integration_api_dirs}; do
+			build_test_suite_binary "$dir" "test.main"
+		done
+	fi
 }
 
 # Build a binary for a test suite package

hack/make/test-integration

@@ -3,6 +3,12 @@ set -e -o pipefail
 
 source hack/make/.integration-test-helpers
 
+if [ ! -z "${TEST_SKIP_INTEGRATION}" ] && [ ! -z "${TEST_SKIP_INTEGRATION_CLI}" ]; then
+	echo integration and integraiton-cli skipped according to env vars
+	exit 0
+fi
+
+
 (
 	build_test_suite_binaries
 	bundle .integration-daemon-start

integration/plugin/logging/cmd/close_on_start/main_test.go

@@ -1 +0,0 @@
-package main

integration/plugin/logging/cmd/cmd_test.go

@@ -1 +0,0 @@
-package cmd

integration/plugin/logging/cmd/dummy/main_test.go

@@ -1 +0,0 @@
-package main

integration/plugin/volumes/cmd/cmd_test.go

@@ -1 +0,0 @@
-package cmd

integration/plugin/volumes/cmd/dummy/main_test.go

@@ -1 +0,0 @@
-package main