Merge pull request #34000 from dnephin/test-integration-api

Introduce `test-integration` target

.gitignore

@@ -4,7 +4,7 @@
 *.exe
 *.exe~
 *.orig
-*.test
+test.main
 .*.swp
 .DS_Store
 # a .bashrc may be added to customize the build environment

CONTRIBUTING.md

@@ -160,6 +160,10 @@ it! Take a look at existing tests for inspiration. [Run the full test
 suite](https://docs.docker.com/opensource/project/test-and-docs/) on your branch before
 submitting a pull request.
 
+If your changes need integration tests, write them against the API. The `cli`
+integration tests are slowly either migrated to API tests or moved away as unit
+tests in `docker/cli` and end-to-end tests for docker.
+
 Update the documentation when creating or modifying features. Test your
 documentation changes for clarity, concision, and correctness, as well as a
 clean documentation build. See our contributors guide for [our style

Dockerfile

@@ -9,7 +9,7 @@
 # docker run -v `pwd`:/go/src/github.com/docker/docker --privileged -i -t docker bash
 #
 # # Run the test suite:
-# docker run -e DOCKER_GITCOMMIT=foo --privileged docker hack/make.sh test-unit test-integration-cli test-docker-py
+# docker run -e DOCKER_GITCOMMIT=foo --privileged docker hack/make.sh test-unit test-integration test-docker-py
 #
 # # Publish a release:
 # docker run --privileged \

Dockerfile.aarch64

@@ -9,7 +9,7 @@
 # docker run -v `pwd`:/go/src/github.com/docker/docker --privileged -i -t docker bash
 #
 # # Run the test suite:
-# docker run --privileged docker hack/make.sh test-unit test-integration-cli test-docker-py
+# docker run --privileged docker hack/make.sh test-unit test-integration test-docker-py
 #
 # Note: AppArmor used to mess with privileged mode, but this is no longer
 # the case. Therefore, you don't have to disable it anymore.

Dockerfile.armhf

@@ -9,7 +9,7 @@
 # docker run -v `pwd`:/go/src/github.com/docker/docker --privileged -i -t docker bash
 #
 # # Run the test suite:
-# docker run --privileged docker hack/make.sh test-unit test-integration-cli test-docker-py
+# docker run --privileged docker hack/make.sh test-unit test-integration test-docker-py
 #
 # Note: AppArmor used to mess with privileged mode, but this is no longer
 # the case. Therefore, you don't have to disable it anymore.

Dockerfile.ppc64le

@@ -9,7 +9,7 @@
 # docker run -v `pwd`:/go/src/github.com/docker/docker --privileged -i -t docker bash
 #
 # # Run the test suite:
-# docker run --privileged docker hack/make.sh test-unit test-integration-cli test-docker-py
+# docker run --privileged docker hack/make.sh test-unit test-integration test-docker-py
 #
 # Note: AppArmor used to mess with privileged mode, but this is no longer
 # the case. Therefore, you don't have to disable it anymore.

Dockerfile.s390x

@@ -9,7 +9,7 @@
 # docker run -v `pwd`:/go/src/github.com/docker/docker --privileged -i -t docker bash
 #
 # # Run the test suite:
-# docker run --privileged docker hack/make.sh test-unit test-integration-cli test-docker-py
+# docker run --privileged docker hack/make.sh test-unit test-integration test-docker-py
 #
 # Note: AppArmor used to mess with privileged mode, but this is no longer
 # the case. Therefore, you don't have to disable it anymore.

Dockerfile.simple

@@ -1,7 +1,7 @@
 # docker build -t docker:simple -f Dockerfile.simple .
 # docker run --rm docker:simple hack/make.sh dynbinary
 # docker run --rm --privileged docker:simple hack/dind hack/make.sh test-unit
-# docker run --rm --privileged -v /var/lib/docker docker:simple hack/dind hack/make.sh dynbinary test-integration-cli
+# docker run --rm --privileged -v /var/lib/docker docker:simple hack/dind hack/make.sh dynbinary test-integration
 
 # This represents the bare minimum required to build and test Docker.
 

Makefile

@@ -1,4 +1,4 @@
-.PHONY: all binary dynbinary build cross deb help init-go-pkg-cache install manpages rpm run shell test test-docker-py test-integration-cli test-unit tgz validate win
+.PHONY: all binary dynbinary build cross deb help init-go-pkg-cache install manpages rpm run shell test test-docker-py test-integration test-unit tgz validate win
 
 # set the graph driver as the current graphdriver if not set
 DOCKER_GRAPHDRIVER := $(if $(DOCKER_GRAPHDRIVER),$(DOCKER_GRAPHDRIVER),$(shell docker info 2>&1 | grep "Storage Driver" | sed 's/.*: //'))
@@ -149,13 +149,15 @@ shell: build ## start a shell inside the build env
 	$(DOCKER_RUN_DOCKER) bash
 
 test: build ## run the unit, integration and docker-py tests
-	$(DOCKER_RUN_DOCKER) hack/make.sh dynbinary cross test-unit test-integration-cli test-docker-py
+	$(DOCKER_RUN_DOCKER) hack/make.sh dynbinary cross test-unit test-integration test-docker-py
 
 test-docker-py: build ## run the docker-py tests
 	$(DOCKER_RUN_DOCKER) hack/make.sh dynbinary test-docker-py
 
-test-integration-cli: build ## run the integration tests
-	$(DOCKER_RUN_DOCKER) hack/make.sh build-integration-test-binary dynbinary test-integration-cli
+test-integration-cli: test-integration ## (DEPRECATED) use test-integration
+
+test-integration: build ## run the integration tests
+	$(DOCKER_RUN_DOCKER) hack/make.sh dynbinary test-integration
 
 test-unit: build ## run the unit tests
 	$(DOCKER_RUN_DOCKER) hack/make.sh test-unit

hack/README.md

@@ -37,14 +37,14 @@ More information is found within `make.ps1` by the author, @jhowardmsft
 - Referenced via `make test` when running tests on a local machine,
 or directly referenced when running tests inside a Docker development container.  
 - When running on a local machine, `make test` to run all tests found in
-`test`, `test-unit`, `test-integration-cli`, and `test-docker-py` on
+`test`, `test-unit`, `test-integration`, and `test-docker-py` on
 your local machine. The default timeout is set in `make.sh` to 60 minutes
 (`${TIMEOUT:=60m}`), since it currently takes up to an hour to run
 all of the tests.
 - When running inside a Docker development container, `hack/make.sh` does
 not have a single target that runs all the tests. You need to provide a
 single command line with multiple targets that performs the same thing.
-An example referenced from [Run targets inside a development container](https://docs.docker.com/opensource/project/test-and-docs/#run-targets-inside-a-development-container): `root@5f8630b873fe:/go/src/github.com/moby/moby# hack/make.sh dynbinary binary cross test-unit test-integration-cli test-docker-py`
+An example referenced from [Run targets inside a development container](https://docs.docker.com/opensource/project/test-and-docs/#run-targets-inside-a-development-container): `root@5f8630b873fe:/go/src/github.com/moby/moby# hack/make.sh dynbinary binary cross test-unit test-integration test-docker-py`
 - For more information related to testing outside the scope of this README,
 refer to
 [Run tests and test documentation](https://docs.docker.com/opensource/project/test-and-docs/)

hack/make.ps1

@@ -318,7 +318,7 @@ Function Run-UnitTests() {
     $pkgList = $pkgList | Select-String -Pattern "github.com/docker/docker"
     $pkgList = $pkgList | Select-String -NotMatch "github.com/docker/docker/vendor"
     $pkgList = $pkgList | Select-String -NotMatch "github.com/docker/docker/man"
-    $pkgList = $pkgList | Select-String -NotMatch "github.com/docker/docker/integration-cli"
+    $pkgList = $pkgList | Select-String -NotMatch "github.com/docker/docker/integration"
     $pkgList = $pkgList -replace "`r`n", " "
     $goTestCommand = "go test" + $raceParm + " -cover -ldflags -w -tags """ + "autogen daemon" + """ -a """ + "-test.timeout=10m" + """ $pkgList"
     Invoke-Expression $goTestCommand

hack/make.sh

@@ -60,7 +60,7 @@ DEFAULT_BUNDLES=(
 	dynbinary
 
 	test-unit
-	test-integration-cli
+	test-integration
 	test-docker-py
 
 	cross

hack/make/.ensure-emptyfs

@@ -1,23 +1,23 @@
 #!/usr/bin/env bash
 set -e
 
-if ! docker inspect -t image emptyfs &> /dev/null; then
-	# let's build a "docker save" tarball for "emptyfs"
+if ! docker image inspect emptyfs > /dev/null; then
+	# build a "docker save" tarball for "emptyfs"
 	# see https://github.com/docker/docker/pull/5262
 	# and also https://github.com/docker/docker/issues/4242
 	dir="$DEST/emptyfs"
-	mkdir -p "$dir"
+	uuid=511136ea3c5a64f264b78b5433614aec563103b4d4702f3ba7d4d2698e22c158
+	mkdir -p "$dir/$uuid"
 	(
-		cd "$dir"
-		echo '{"emptyfs":{"latest":"511136ea3c5a64f264b78b5433614aec563103b4d4702f3ba7d4d2698e22c158"}}' > repositories
-		mkdir -p 511136ea3c5a64f264b78b5433614aec563103b4d4702f3ba7d4d2698e22c158
-		(
-			cd 511136ea3c5a64f264b78b5433614aec563103b4d4702f3ba7d4d2698e22c158
-			echo '{"id":"511136ea3c5a64f264b78b5433614aec563103b4d4702f3ba7d4d2698e22c158","comment":"Imported from -","created":"2013-06-13T14:03:50.821769-07:00","container_config":{"Hostname":"","Domainname":"","User":"","Memory":0,"MemorySwap":0,"CpuShares":0,"AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"PortSpecs":null,"ExposedPorts":null,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":null,"Cmd":null,"Image":"","Volumes":null,"WorkingDir":"","Entrypoint":null,"NetworkDisabled":false,"OnBuild":null},"docker_version":"0.4.0","architecture":"x86_64","Size":0}' > json
-			echo '1.0' > VERSION
-			tar -cf layer.tar --files-from /dev/null
-		)
+		echo '{"emptyfs":{"latest":"511136ea3c5a64f264b78b5433614aec563103b4d4702f3ba7d4d2698e22c158"}}' > "$dir/repositories"
+		cd "$dir/$uuid"
+		echo '{"id":"511136ea3c5a64f264b78b5433614aec563103b4d4702f3ba7d4d2698e22c158","comment":"Imported from -","created":"2013-06-13T14:03:50.821769-07:00","container_config":{"Hostname":"","Domainname":"","User":"","Memory":0,"MemorySwap":0,"CpuShares":0,"AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"PortSpecs":null,"ExposedPorts":null,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":null,"Cmd":null,"Image":"","Volumes":null,"WorkingDir":"","Entrypoint":null,"NetworkDisabled":false,"OnBuild":null},"docker_version":"0.4.0","architecture":"x86_64","Size":0}' > json
+		echo '1.0' > VERSION
+		tar -cf layer.tar --files-from /dev/null
+	)
+	(
+		[ -n "$TESTDEBUG" ] && set -x
+		tar -cC "$dir" . | docker load
 	)
-	( set -x; tar -cC "$dir" . | docker load )
 	rm -rf "$dir"
 fi

hack/make/.integration-daemon-setup

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 set -e
 
-bundle .detect-daemon-osarch
+source "$MAKEDIR/.detect-daemon-osarch"
 if [ "$DOCKER_ENGINE_GOOS" != "windows" ]; then
 	bundle .ensure-emptyfs
 fi

hack/make/.integration-daemon-start

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-# see test-integration-cli for example usage of this script
+# see test-integration for example usage of this script
 
 base="$ABS_DEST/.."
 export PATH="$base/binary-daemon:$base/dynbinary-daemon:$PATH"
@@ -76,24 +76,26 @@ if [ -z "$DOCKER_TEST_HOST" ]; then
 		# see https://github.com/docker/libcontainer/blob/master/apparmor/apparmor.go#L16
 		export container=""
 		(
-			set -x
+			[ -n "$TESTDEBUG" ] && set -x
 			/etc/init.d/apparmor start
 		)
 	fi
 
-	export DOCKER_HOST="unix://$(cd "$DEST" && pwd)/docker.sock" # "pwd" tricks to make sure $DEST is an absolute path, not a relative one
-	( set -x; exec \
-		dockerd --debug \
-		--host "$DOCKER_HOST" \
-		--storage-driver "$DOCKER_GRAPHDRIVER" \
-		--pidfile "$DEST/docker.pid" \
-		--userland-proxy="$DOCKER_USERLANDPROXY" \
-		$storage_params \
-		$extra_params \
-			&> "$DEST/docker.log"
+	# "pwd" tricks to make sure $DEST is an absolute path, not a relative one
+	export DOCKER_HOST="unix://$(cd "$DEST" && pwd)/docker.sock"
+	(
+		echo "Starting dockerd"
+		[ -n "$TESTDEBUG" ] && set -x
+		exec \
+			dockerd --debug \
+			--host "$DOCKER_HOST" \
+			--storage-driver "$DOCKER_GRAPHDRIVER" \
+			--pidfile "$DEST/docker.pid" \
+			--userland-proxy="$DOCKER_USERLANDPROXY" \
+			$storage_params \
+			$extra_params \
+				&> "$DEST/docker.log"
 	) &
-	# make sure that if the script exits unexpectedly, we stop this daemon we just started
-	trap 'bundle .integration-daemon-stop' EXIT
 else
 	export DOCKER_HOST="$DOCKER_TEST_HOST"
 fi

hack/make/.integration-daemon-stop

@@ -1,11 +1,12 @@
 #!/usr/bin/env bash
 
 if [ ! "$(go env GOOS)" = 'windows' ]; then
-	trap - EXIT # reset EXIT trap applied in .integration-daemon-start
-
 	for pidFile in $(find "$DEST" -name docker.pid); do
-		pid=$(set -x; cat "$pidFile")
-		( set -x; kill "$pid" )
+		pid=$([ -n "$TESTDEBUG" ] && set -x; cat "$pidFile")
+		(
+			[ -n "$TESTDEBUG" ] && set -x
+			kill "$pid"
+		)
 		if ! wait "$pid"; then
 			echo >&2 "warning: PID $pid from $pidFile had a nonzero exit code"
 		fi
@@ -15,7 +16,7 @@ if [ ! "$(go env GOOS)" = 'windows' ]; then
 		# Stop apparmor if it is enabled
 		if [ -e "/sys/module/apparmor/parameters/enabled" ] && [ "$(cat /sys/module/apparmor/parameters/enabled)" == "Y" ]; then
 			(
-				set -x
+				[ -n "$TESTDEBUG" ] && set -x
 				/etc/init.d/apparmor stop
 			)
 		fi

hack/make/.integration-test-helpers

@@ -1,65 +1,72 @@
 #!/usr/bin/env bash
-
-: ${TEST_REPEAT:=0}
-
-bundle_test_integration_cli() {
-	TESTFLAGS="$TESTFLAGS -check.v -check.timeout=${TIMEOUT} -test.timeout=360m"
-	go_test_dir integration-cli $DOCKER_INTEGRATION_TESTS_VERIFIED
-}
-
-# If $TESTFLAGS is set in the environment, it is passed as extra arguments to 'go test'.
-# You can use this to select certain tests to run, e.g.
-#
-#     TESTFLAGS='-test.run ^TestBuild$' ./hack/make.sh test-unit
 #
 # For integration-cli test, we use [gocheck](https://labix.org/gocheck), if you want
 # to run certain tests on your local host, you should run with command:
 #
-#     TESTFLAGS='-check.f DockerSuite.TestBuild*' ./hack/make.sh binary test-integration-cli
+#     TESTFLAGS='-check.f DockerSuite.TestBuild*' ./hack/make.sh binary test-integration
 #
-go_test_dir() {
-	dir=$1
-	precompiled=$2
-	testbinary="$ABS_DEST/test.main"
-	testcover=()
-	testcoverprofile=()
+
+source "$SCRIPTDIR/make/.go-autogen"
+
+: ${TEST_REPEAT:=1}
+
+integration_api_dirs=("$(
+	find ./integration -type d |
+	grep -vE '^(./integration$|./integration/util)')")
+
+run_test_integration() {
+	local flags="-test.v -test.timeout=${TIMEOUT} $TESTFLAGS"
+	for dir in $integration_api_dirs; do
+		(
+			cd $dir
+			echo "Running $PWD"
+			test_env ./test.main $flags
+		)
+	done
+
 	(
-		set -e
-		mkdir -p "$DEST/coverprofiles"
-		export DEST="$ABS_DEST" # in a subshell this is safe -- our integration-cli tests need DEST, and "cd" screws it up
-		if [ -z $precompiled ]; then
-			ensure_test_dir $1 $testbinary
-		fi
-		cd "$dir"
-		i=0
-		while ((++i)); do
-			test_env "$testbinary" $TESTFLAGS
-			if [ $i -gt "$TEST_REPEAT" ]; then
-				break
-			fi
-			echo "Repeating test ($i)"
-		done
+		flags="-check.v -check.timeout=${TIMEOUT} -test.timeout=360m $TESTFLAGS"
+		cd integration-cli
+		echo "Running $PWD"
+		test_env ./test.main $flags
 	)
 }
 
-ensure_test_dir() {
-	(
-		# make sure a test dir will compile
-		dir="$1"
-		out="$2"
-		echo Building test dir: "$dir"
-		set -xe
-		cd "$dir"
-		go test -c -o "$out" -ldflags "$LDFLAGS" "${BUILDFLAGS[@]}"
-	)
+build_test_suite_binaries() {
+	build_test_suite_binary ./integration-cli "test.main"
+	for dir in $integration_api_dirs; do
+		build_test_suite_binary "$dir" "test.main"
+	done
+}
+
+# Build a binary for a test suite package
+build_test_suite_binary() {
+	local dir="$1"
+	local out="$2"
+	echo Building test suite binary "$dir/$out"
+	go test -c -o "$dir/$out" -ldflags "$LDFLAGS" "${BUILDFLAGS[@]}" "$dir"
 }
 
+cleanup_test_suite_binaries() {
+	[ -n "$TESTDEBUG" ] && return
+	echo "Removing test suite binaries"
+	find integration* -name test.main | xargs -r rm
+}
+
+repeat() {
+	for i in $(seq 1 $TEST_REPEAT); do
+		echo "Running integration-test (iteration $i)"
+		$@
+	done
+}
+
+# use "env -i" to tightly control the environment variables that bleed into the tests
 test_env() {
 	(
-		set -xe
-		# use "env -i" to tightly control the environment variables that bleed into the tests
+		set -e
+		[ -n "$TESTDEBUG" ] && set -x
 		env -i \
-			DEST="$DEST" \
+			DEST="$ABS_DEST" \
 			DOCKER_CLI_VERSION="$DOCKER_CLI_VERSION" \
 			DOCKER_API_VERSION="$DOCKER_API_VERSION" \
 			DOCKER_INTEGRATION_DAEMON_DEST="$DOCKER_INTEGRATION_DAEMON_DEST" \
@@ -82,3 +89,19 @@ test_env() {
 			"$@"
 	)
 }
+   
+
+error_on_leaked_containerd_shims() {
+	if [ "$(go env GOOS)" == 'windows' ]; then
+		return
+	fi
+
+	leftovers=$(ps -ax -o pid,cmd |
+	            awk '$2 == "docker-containerd-shim" && $4 ~ /.*\/bundles\/.*\/test-integration/ { print $1 }')
+	if [ -n "$leftovers" ]; then
+		ps aux
+		kill -9 $leftovers 2> /dev/null
+		echo "!!!! WARNING you have left over shim(s), Cleanup your test !!!!"
+		exit 1
+	fi
+}

hack/make/build-integration-test-binary

@@ -1,13 +0,0 @@
-#!/usr/bin/env bash
-set -e
-
-rm -rf "$DEST"
-DEST="$ABS_DEST/../test-integration-cli"
-
-source "$SCRIPTDIR/make/.go-autogen"
-
-if [ -z $DOCKER_INTEGRATION_TESTS_VERIFIED ]; then
-	source ${MAKEDIR}/.integration-test-helpers
-	ensure_test_dir integration-cli "$DEST/test.main"
-	export DOCKER_INTEGRATION_TESTS_VERIFIED=1
-fi

hack/make/test-integration

@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+set -e
+
+source "${MAKEDIR}/.go-autogen"
+source hack/make/.integration-test-helpers
+
+(
+	build_test_suite_binaries
+	bundle .integration-daemon-start
+	bundle .integration-daemon-setup
+
+	local testexit=0
+	( repeat run_test_integration ) || testexit=$?
+
+	# Always run cleanup, even if the subshell fails
+	bundle .integration-daemon-stop
+	cleanup_test_suite_binaries
+	error_on_leaked_containerd_shims
+
+	exit $testexit
+
+) 2>&1 | tee -a "$DEST/test.log"

hack/make/test-integration-cli

@@ -1,29 +1,6 @@
 #!/usr/bin/env bash
 set -e
+echo "WARNING: test-integration-cli is DEREPCATED. Use test-integration." >&2
 
-source "${MAKEDIR}/.go-autogen"
-source hack/make/.integration-test-helpers
-
-# subshell so that we can export PATH without breaking other things
-(
-	bundle .integration-daemon-start
-
-	bundle .integration-daemon-setup
-
-	bundle_test_integration_cli
-
-	bundle .integration-daemon-stop
-
-	if [ "$(go env GOOS)" != 'windows' ]
-	then
-		leftovers=$(ps -ax -o pid,cmd | awk '$2 == "docker-containerd-shim" && $4 ~ /.*\/bundles\/.*\/test-integration-cli/ { print $1 }')
-		if [ -n "$leftovers" ]
-		then
-			ps aux
-			kill -9 $leftovers 2> /dev/null
-			echo "!!!! WARNING you have left over shim(s), Cleanup your test !!!!"
-			exit 1
-		fi
-	fi
-
-) 2>&1 | tee -a "$DEST/test.log"
+# TODO: remove this and exit 1 once CI has changed to use test-integration
+bundle test-integration

hack/make/test-integration-shell

@@ -5,3 +5,5 @@ bundle .integration-daemon-setup
 
 export ABS_DEST
 bash +e
+
+bundle .integration-daemon-stop

hack/release.sh

@@ -57,7 +57,7 @@ if [ "$1" != '--release-regardless-of-test-failure' ]; then
 	RELEASE_BUNDLES=(
 		test-unit
 		"${RELEASE_BUNDLES[@]}"
-		test-integration-cli
+		test-integration
 	)
 fi
 

hack/test/unit

@@ -16,7 +16,7 @@ TESTFLAGS+=" -test.timeout=${TIMEOUT:-5m}"
 BUILDFLAGS=( -tags "netgo seccomp libdm_no_deferred_remove" )
 TESTDIRS="${TESTDIRS:-"./..."}"
 
-exclude_paths="/vendor/|/integration-cli"
+exclude_paths="/vendor/|/integration"
 if [ "$(go env GOHOSTOS)" = 'solaris' ]; then
 	exclude_paths="$exclude_paths|/daemon/graphdriver"
 fi

integration-cli/check_test.go

@@ -4,10 +4,8 @@ import (
 	"fmt"
 	"net/http/httptest"
 	"os"
-	"os/exec"
 	"path"
 	"path/filepath"
-	"strings"
 	"sync"
 	"syscall"
 	"testing"
@@ -72,17 +70,7 @@ func TestMain(m *testing.M) {
 func Test(t *testing.T) {
 	cli.EnsureTestEnvIsLoaded(t)
 	fakestorage.EnsureTestEnvIsLoaded(t)
-	cmd := exec.Command(dockerBinary, "images", "-f", "dangling=false", "--format", "{{.Repository}}:{{.Tag}}")
-	cmd.Env = appendBaseEnv(true)
-	out, err := cmd.CombinedOutput()
-	if err != nil {
-		panic(fmt.Errorf("err=%v\nout=%s\n", err, out))
-	}
-	images := strings.Split(strings.TrimSpace(string(out)), "\n")
-	testEnv.ProtectImage(t, images...)
-	if testEnv.DaemonPlatform() == "linux" {
-		ensureFrozenImagesLinux(t)
-	}
+	environment.ProtectImages(t, testEnv)
 	check.TestingT(t)
 }
 

integration-cli/docker_api_create_test.go

@@ -11,82 +11,6 @@ import (
 	"github.com/go-check/check"
 )
 
-func (s *DockerSuite) TestAPICreateWithNotExistImage(c *check.C) {
-	name := "test"
-	config := map[string]interface{}{
-		"Image":   "test456:v1",
-		"Volumes": map[string]struct{}{"/tmp": {}},
-	}
-
-	status, body, err := request.SockRequest("POST", "/containers/create?name="+name, config, daemonHost())
-	c.Assert(err, check.IsNil)
-	c.Assert(status, check.Equals, http.StatusNotFound)
-	expected := "No such image: test456:v1"
-	c.Assert(getErrorMessage(c, body), checker.Contains, expected)
-
-	config2 := map[string]interface{}{
-		"Image":   "test456",
-		"Volumes": map[string]struct{}{"/tmp": {}},
-	}
-
-	status, body, err = request.SockRequest("POST", "/containers/create?name="+name, config2, daemonHost())
-	c.Assert(err, check.IsNil)
-	c.Assert(status, check.Equals, http.StatusNotFound)
-	expected = "No such image: test456:latest"
-	c.Assert(getErrorMessage(c, body), checker.Equals, expected)
-
-	config3 := map[string]interface{}{
-		"Image": "sha256:0cb40641836c461bc97c793971d84d758371ed682042457523e4ae701efeaaaa",
-	}
-
-	status, body, err = request.SockRequest("POST", "/containers/create?name="+name, config3, daemonHost())
-	c.Assert(err, check.IsNil)
-	c.Assert(status, check.Equals, http.StatusNotFound)
-	expected = "No such image: sha256:0cb40641836c461bc97c793971d84d758371ed682042457523e4ae701efeaaaa"
-	c.Assert(getErrorMessage(c, body), checker.Equals, expected)
-
-}
-
-// Test for #25099
-func (s *DockerSuite) TestAPICreateEmptyEnv(c *check.C) {
-	name := "test1"
-	config := map[string]interface{}{
-		"Image": "busybox",
-		"Env":   []string{"", "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"},
-		"Cmd":   []string{"true"},
-	}
-
-	status, body, err := request.SockRequest("POST", "/containers/create?name="+name, config, daemonHost())
-	c.Assert(err, check.IsNil)
-	c.Assert(status, check.Equals, http.StatusInternalServerError)
-	expected := "invalid environment variable:"
-	c.Assert(getErrorMessage(c, body), checker.Contains, expected)
-
-	name = "test2"
-	config = map[string]interface{}{
-		"Image": "busybox",
-		"Env":   []string{"=", "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"},
-		"Cmd":   []string{"true"},
-	}
-	status, body, err = request.SockRequest("POST", "/containers/create?name="+name, config, daemonHost())
-	c.Assert(err, check.IsNil)
-	c.Assert(status, check.Equals, http.StatusInternalServerError)
-	expected = "invalid environment variable: ="
-	c.Assert(getErrorMessage(c, body), checker.Contains, expected)
-
-	name = "test3"
-	config = map[string]interface{}{
-		"Image": "busybox",
-		"Env":   []string{"=foo", "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"},
-		"Cmd":   []string{"true"},
-	}
-	status, body, err = request.SockRequest("POST", "/containers/create?name="+name, config, daemonHost())
-	c.Assert(err, check.IsNil)
-	c.Assert(status, check.Equals, http.StatusInternalServerError)
-	expected = "invalid environment variable: =foo"
-	c.Assert(getErrorMessage(c, body), checker.Contains, expected)
-}
-
 func (s *DockerSuite) TestAPICreateWithInvalidHealthcheckParams(c *check.C) {
 	// test invalid Interval in Healthcheck: less than 0s
 	name := "test1"

integration-cli/environment/protect.go

@@ -1,5 +1,16 @@
 package environment
 
+import (
+	"strings"
+
+	"github.com/docker/docker/integration-cli/fixtures/load"
+	icmd "github.com/docker/docker/pkg/testutil/cmd"
+)
+
+type protectedElements struct {
+	images map[string]struct{}
+}
+
 // ProtectImage adds the specified image(s) to be protected in case of clean
 func (e *Execution) ProtectImage(t testingT, images ...string) {
 	for _, image := range images {
@@ -7,6 +18,31 @@ func (e *Execution) ProtectImage(t testingT, images ...string) {
 	}
 }
 
-type protectedElements struct {
-	images map[string]struct{}
+// ProtectImages protects existing images and on linux frozen images from being
+// cleaned up at the end of test runs
+func ProtectImages(t testingT, testEnv *Execution) {
+	images := getExistingImages(t, testEnv)
+
+	if testEnv.DaemonPlatform() == "linux" {
+		images = append(images, ensureFrozenImagesLinux(t, testEnv)...)
+	}
+	testEnv.ProtectImage(t, images...)
+}
+
+func getExistingImages(t testingT, testEnv *Execution) []string {
+	// TODO: use API instead of cli
+	result := icmd.RunCommand(testEnv.dockerBinary, "images", "-f", "dangling=false", "--format", "{{.Repository}}:{{.Tag}}")
+	result.Assert(t, icmd.Success)
+	return strings.Split(strings.TrimSpace(result.Stdout()), "\n")
+}
+
+func ensureFrozenImagesLinux(t testingT, testEnv *Execution) []string {
+	images := []string{"busybox:latest", "hello-world:frozen", "debian:jessie"}
+	err := load.FrozenImagesLinux(testEnv.DockerBinary(), images...)
+	if err != nil {
+		result := icmd.RunCommand(testEnv.DockerBinary(), "image", "ls")
+		t.Logf(result.String())
+		t.Fatalf("%+v", err)
+	}
+	return images
 }

integration-cli/fixtures_linux_daemon_test.go

@@ -24,16 +24,6 @@ type logT interface {
 	Logf(string, ...interface{})
 }
 
-func ensureFrozenImagesLinux(t testingT) {
-	images := []string{"busybox:latest", "hello-world:frozen", "debian:jessie"}
-	err := load.FrozenImagesLinux(dockerBinary, images...)
-	if err != nil {
-		t.Logf(dockerCmdWithError("images"))
-		t.Fatalf("%+v", err)
-	}
-	defer testEnv.ProtectImage(t, images...)
-}
-
 var ensureSyscallTestOnce sync.Once
 
 func ensureSyscallTest(c *check.C) {

integration/container/create_test.go

@@ -0,0 +1,93 @@
+package container
+
+import (
+	"context"
+	"strconv"
+	"testing"
+
+	"github.com/docker/docker/api/types/container"
+	"github.com/docker/docker/api/types/network"
+	"github.com/docker/docker/integration/util/request"
+	"github.com/docker/docker/pkg/testutil"
+)
+
+func TestCreateFailsWhenIdentifierDoesNotExist(t *testing.T) {
+	defer setupTest(t)()
+	client := request.NewAPIClient(t)
+
+	testCases := []struct {
+		doc           string
+		image         string
+		expectedError string
+	}{
+		{
+			doc:           "image and tag",
+			image:         "test456:v1",
+			expectedError: "No such image: test456:v1",
+		},
+		{
+			doc:           "image no tag",
+			image:         "test456",
+			expectedError: "No such image: test456",
+		},
+		{
+			doc:           "digest",
+			image:         "sha256:0cb40641836c461bc97c793971d84d758371ed682042457523e4ae701efeaaaa",
+			expectedError: "No such image: sha256:0cb40641836c461bc97c793971d84d758371ed682042457523e4ae701efeaaaa",
+		},
+	}
+
+	for _, tc := range testCases {
+		tc := tc
+		t.Run(tc.doc, func(t *testing.T) {
+			t.Parallel()
+			_, err := client.ContainerCreate(context.Background(),
+				&container.Config{Image: tc.image},
+				&container.HostConfig{},
+				&network.NetworkingConfig{},
+				"foo",
+			)
+			testutil.ErrorContains(t, err, tc.expectedError)
+		})
+	}
+}
+
+func TestCreateWithInvalidEnv(t *testing.T) {
+	defer setupTest(t)()
+	client := request.NewAPIClient(t)
+
+	testCases := []struct {
+		env           string
+		expectedError string
+	}{
+		{
+			env:           "",
+			expectedError: "invalid environment variable:",
+		},
+		{
+			env:           "=",
+			expectedError: "invalid environment variable: =",
+		},
+		{
+			env:           "=foo",
+			expectedError: "invalid environment variable: =foo",
+		},
+	}
+
+	for index, tc := range testCases {
+		tc := tc
+		t.Run(strconv.Itoa(index), func(t *testing.T) {
+			t.Parallel()
+			_, err := client.ContainerCreate(context.Background(),
+				&container.Config{
+					Image: "busybox",
+					Env:   []string{tc.env},
+				},
+				&container.HostConfig{},
+				&network.NetworkingConfig{},
+				"foo",
+			)
+			testutil.ErrorContains(t, err, tc.expectedError)
+		})
+	}
+}

integration/container/main_test.go

@@ -0,0 +1,37 @@
+package container
+
+import (
+	"fmt"
+	"os"
+	"testing"
+
+	"github.com/docker/docker/integration-cli/environment"
+)
+
+var (
+	testEnv *environment.Execution
+)
+
+func TestMain(m *testing.M) {
+	var err error
+	testEnv, err = environment.New()
+	if err != nil {
+		fmt.Println(err)
+		os.Exit(1)
+	}
+
+	// TODO: replace this with `testEnv.Print()` to print the full env
+	if testEnv.LocalDaemon() {
+		fmt.Println("INFO: Testing against a local daemon")
+	} else {
+		fmt.Println("INFO: Testing against a remote daemon")
+	}
+
+	res := m.Run()
+	os.Exit(res)
+}
+
+func setupTest(t *testing.T) func() {
+	environment.ProtectImages(t, testEnv)
+	return func() { testEnv.Clean(t, testEnv.DockerBinary()) }
+}

integration/doc.go

@@ -0,0 +1,3 @@
+// Package integration provides integrations tests for Moby (API).
+// These tests require a daemon (dockerd for now) to run.
+package integration

integration/util/request/client.go

@@ -0,0 +1,15 @@
+package request
+
+import (
+	"testing"
+
+	"github.com/docker/docker/client"
+	"github.com/stretchr/testify/require"
+)
+
+// NewAPIClient returns a docker API client configured from environment variables
+func NewAPIClient(t *testing.T) client.APIClient {
+	clt, err := client.NewEnvClient()
+	require.NoError(t, err)
+	return clt
+}

project/ARM.md

@@ -17,7 +17,7 @@ From the root of the Docker/Docker repo one can use make to execute the followin
 - make default
 - make shell
 - make test-unit
-- make test-integration-cli
+- make test-integration
 - make
 
 The Makefile does include logic to determine on which OS and architecture the Docker Development Image is built.