Introduce test-integration target (and deprecate/freeze test-integration-cli)

This adds a new package `integration` where `engine` integration tests
should live. Those integration tests should not depends on any `cli`
components (except from the `dockerd` daemon for now — to actually
start a daemon).

Signed-off-by: Vincent Demeester <vincent@sbr.pm>

CONTRIBUTING.md

@@ -160,6 +160,10 @@ it! Take a look at existing tests for inspiration. [Run the full test
 suite](https://docs.docker.com/opensource/project/test-and-docs/) on your branch before
 submitting a pull request.
 
+If your changes need integration tests, write them against the API. The `cli`
+integration tests are slowly either migrated to API tests or moved away as unit
+tests in `docker/cli` and end-to-end tests for docker.
+
 Update the documentation when creating or modifying features. Test your
 documentation changes for clarity, concision, and correctness, as well as a
 clean documentation build. See our contributors guide for [our style

Makefile

@@ -154,8 +154,11 @@ test: build ## run the unit, integration and docker-py tests
 test-docker-py: build ## run the docker-py tests
 	$(DOCKER_RUN_DOCKER) hack/make.sh dynbinary test-docker-py
 
-test-integration-cli: build ## run the integration tests
+test-integration-cli: build ## (DEPRECATED) use test-integration
-	$(DOCKER_RUN_DOCKER) hack/make.sh build-integration-test-binary dynbinary test-integration-cli
+	$(DOCKER_RUN_DOCKER) hack/make.sh build-integration-test-cli-binary dynbinary test-integration
+
+test-integration: build ## run the integration tests
+	$(DOCKER_RUN_DOCKER) hack/make.sh dynbinary test-integration
 
 test-unit: build ## run the unit tests
 	$(DOCKER_RUN_DOCKER) hack/make.sh test-unit

hack/make.sh

@@ -60,6 +60,7 @@ DEFAULT_BUNDLES=(
 	dynbinary
 
 	test-unit
+	test-integration
 	test-integration-cli
 	test-docker-py
 

hack/make/.integration-test-helpers

@@ -7,6 +7,25 @@ bundle_test_integration_cli() {
 	go_test_dir integration-cli $DOCKER_INTEGRATION_TESTS_VERIFIED
 }
 
+bundle_test_integration() {
+	TESTFLAGS="$TESTFLAGS -v -test.timeout=60m"
+	(
+		set -e
+		cd integration
+		INCBUILD="-i"
+		count=0
+		for flag in "${BUILDFLAGS[@]}"; do
+			if [ "${flag}" == ${INCBUILD} ]; then
+				unset BUILDFLAGS[${count}]
+				break
+			fi
+			count=$[ ${count} + 1 ]
+		done
+		echo go test -ldflags "$LDFLAGS" "${BUILDFLAGS[@]}" $TESTFLAGS ./...
+		go test -ldflags "$LDFLAGS" "${BUILDFLAGS[@]}" $TESTFLAGS ./...
+	)
+}
+
 # If $TESTFLAGS is set in the environment, it is passed as extra arguments to 'go test'.
 # You can use this to select certain tests to run, e.g.
 #

hack/make/test-integration

@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+set -e
+
+source hack/make/.integration-test-helpers
+
+# subshell so that we can export PATH without breaking other things
+(
+	bundle .integration-daemon-start
+
+	bundle .integration-daemon-setup
+
+	bundle_test_integration
+
+	bundle .integration-daemon-stop
+
+	if [ "$(go env GOOS)" != 'windows' ]
+	then
+		leftovers=$(ps -ax -o pid,cmd | awk '$2 == "docker-containerd-shim" && $4 ~ /.*\/bundles\/.*\/test-integration-cli/ { print $1 }')
+		if [ -n "$leftovers" ]
+		then
+			ps aux
+			kill -9 $leftovers 2> /dev/null
+			echo "!!!! WARNING you have left over shim(s), Cleanup your test !!!!"
+			exit 1
+		fi
+	fi
+
+) 2>&1 | tee -a "$DEST/test.log"

integration-cli/docker_api_create_test.go

@@ -11,82 +11,6 @@ import (
 	"github.com/go-check/check"
 )
 
-func (s *DockerSuite) TestAPICreateWithNotExistImage(c *check.C) {
-	name := "test"
-	config := map[string]interface{}{
-		"Image":   "test456:v1",
-		"Volumes": map[string]struct{}{"/tmp": {}},
-	}
-
-	status, body, err := request.SockRequest("POST", "/containers/create?name="+name, config, daemonHost())
-	c.Assert(err, check.IsNil)
-	c.Assert(status, check.Equals, http.StatusNotFound)
-	expected := "No such image: test456:v1"
-	c.Assert(getErrorMessage(c, body), checker.Contains, expected)
-
-	config2 := map[string]interface{}{
-		"Image":   "test456",
-		"Volumes": map[string]struct{}{"/tmp": {}},
-	}
-
-	status, body, err = request.SockRequest("POST", "/containers/create?name="+name, config2, daemonHost())
-	c.Assert(err, check.IsNil)
-	c.Assert(status, check.Equals, http.StatusNotFound)
-	expected = "No such image: test456:latest"
-	c.Assert(getErrorMessage(c, body), checker.Equals, expected)
-
-	config3 := map[string]interface{}{
-		"Image": "sha256:0cb40641836c461bc97c793971d84d758371ed682042457523e4ae701efeaaaa",
-	}
-
-	status, body, err = request.SockRequest("POST", "/containers/create?name="+name, config3, daemonHost())
-	c.Assert(err, check.IsNil)
-	c.Assert(status, check.Equals, http.StatusNotFound)
-	expected = "No such image: sha256:0cb40641836c461bc97c793971d84d758371ed682042457523e4ae701efeaaaa"
-	c.Assert(getErrorMessage(c, body), checker.Equals, expected)
-
-}
-
-// Test for #25099
-func (s *DockerSuite) TestAPICreateEmptyEnv(c *check.C) {
-	name := "test1"
-	config := map[string]interface{}{
-		"Image": "busybox",
-		"Env":   []string{"", "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"},
-		"Cmd":   []string{"true"},
-	}
-
-	status, body, err := request.SockRequest("POST", "/containers/create?name="+name, config, daemonHost())
-	c.Assert(err, check.IsNil)
-	c.Assert(status, check.Equals, http.StatusInternalServerError)
-	expected := "invalid environment variable:"
-	c.Assert(getErrorMessage(c, body), checker.Contains, expected)
-
-	name = "test2"
-	config = map[string]interface{}{
-		"Image": "busybox",
-		"Env":   []string{"=", "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"},
-		"Cmd":   []string{"true"},
-	}
-	status, body, err = request.SockRequest("POST", "/containers/create?name="+name, config, daemonHost())
-	c.Assert(err, check.IsNil)
-	c.Assert(status, check.Equals, http.StatusInternalServerError)
-	expected = "invalid environment variable: ="
-	c.Assert(getErrorMessage(c, body), checker.Contains, expected)
-
-	name = "test3"
-	config = map[string]interface{}{
-		"Image": "busybox",
-		"Env":   []string{"=foo", "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"},
-		"Cmd":   []string{"true"},
-	}
-	status, body, err = request.SockRequest("POST", "/containers/create?name="+name, config, daemonHost())
-	c.Assert(err, check.IsNil)
-	c.Assert(status, check.Equals, http.StatusInternalServerError)
-	expected = "invalid environment variable: =foo"
-	c.Assert(getErrorMessage(c, body), checker.Contains, expected)
-}
-
 func (s *DockerSuite) TestAPICreateWithInvalidHealthcheckParams(c *check.C) {
 	// test invalid Interval in Healthcheck: less than 0s
 	name := "test1"

integration/container/create_test.go

@@ -0,0 +1,141 @@
+package container
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"testing"
+
+	"strconv"
+
+	"github.com/docker/docker/api/types/container"
+	"github.com/docker/docker/api/types/network"
+	"github.com/docker/docker/client"
+	"github.com/docker/docker/integration-cli/environment"
+	"github.com/docker/docker/integration-cli/fixtures/load"
+	"github.com/stretchr/testify/require"
+)
+
+var (
+	testEnv *environment.Execution
+)
+
+func TestMain(m *testing.M) {
+	var err error
+	testEnv, err = environment.New()
+	if err != nil {
+		fmt.Println(err)
+		os.Exit(1)
+	}
+
+	if testEnv.LocalDaemon() {
+		fmt.Println("INFO: Testing against a local daemon")
+	} else {
+		fmt.Println("INFO: Testing against a remote daemon")
+	}
+
+	// TODO: ensure and protect images
+	res := m.Run()
+	os.Exit(res)
+}
+
+func TestAPICreateWithNotExistImage(t *testing.T) {
+	defer setupTest(t)()
+	clt := createClient(t)
+
+	testCases := []struct {
+		image         string
+		expectedError string
+	}{
+		{
+			image:         "test456:v1",
+			expectedError: "No such image: test456:v1",
+		},
+		{
+			image:         "test456",
+			expectedError: "No such image: test456",
+		},
+		{
+			image:         "sha256:0cb40641836c461bc97c793971d84d758371ed682042457523e4ae701efeaaaa",
+			expectedError: "No such image: sha256:0cb40641836c461bc97c793971d84d758371ed682042457523e4ae701efeaaaa",
+		},
+	}
+
+	for index, tc := range testCases {
+		tc := tc
+		t.Run(strconv.Itoa(index), func(t *testing.T) {
+			t.Parallel()
+			_, err := clt.ContainerCreate(context.Background(),
+				&container.Config{
+					Image: tc.image,
+				},
+				&container.HostConfig{},
+				&network.NetworkingConfig{},
+				"foo",
+			)
+			require.Error(t, err)
+			require.Contains(t, err.Error(), tc.expectedError)
+		})
+	}
+}
+
+func TestAPICreateEmptyEnv(t *testing.T) {
+	defer setupTest(t)()
+	clt := createClient(t)
+
+	testCases := []struct {
+		env           string
+		expectedError string
+	}{
+		{
+			env:           "",
+			expectedError: "invalid environment variable:",
+		},
+		{
+			env:           "=",
+			expectedError: "invalid environment variable: =",
+		},
+		{
+			env:           "=foo",
+			expectedError: "invalid environment variable: =foo",
+		},
+	}
+
+	for index, tc := range testCases {
+		tc := tc
+		t.Run(strconv.Itoa(index), func(t *testing.T) {
+			t.Parallel()
+			_, err := clt.ContainerCreate(context.Background(),
+				&container.Config{
+					Image: "busybox",
+					Env:   []string{tc.env},
+				},
+				&container.HostConfig{},
+				&network.NetworkingConfig{},
+				"foo",
+			)
+			require.Error(t, err)
+			require.Contains(t, err.Error(), tc.expectedError)
+		})
+	}
+}
+
+func createClient(t *testing.T) client.APIClient {
+	clt, err := client.NewEnvClient()
+	require.NoError(t, err)
+	return clt
+}
+
+func setupTest(t *testing.T) func() {
+	if testEnv.DaemonPlatform() == "linux" {
+		images := []string{"busybox:latest", "hello-world:frozen", "debian:jessie"}
+		err := load.FrozenImagesLinux(testEnv.DockerBinary(), images...)
+		if err != nil {
+			t.Fatalf("%+v", err)
+		}
+		defer testEnv.ProtectImage(t, images...)
+	}
+	return func() {
+		testEnv.Clean(t, testEnv.DockerBinary())
+	}
+}

integration/doc.go

@@ -0,0 +1,3 @@
+// Package integration provides integrations tests for Moby (API).
+// These tests require a daemon (dockerd for now) to run.
+package integration