Bump docker-py to 4.0.2, and run tests from upstream repository

This removes all the installation steps for docker-py from the
Dockerfile, and instead builds the upstream Dockerfile, and runs
docker-py tests in a container.

To test;

```
make test-docker-py

...

Removing bundles/

---> Making bundle: dynbinary (in bundles/dynbinary)
Building: bundles/dynbinary-daemon/dockerd-dev
Created binary: bundles/dynbinary-daemon/dockerd-dev

---> Making bundle: test-docker-py (in bundles/test-docker-py)
---> Making bundle: .integration-daemon-start (in bundles/test-docker-py)
Using test binary docker
Starting dockerd
INFO: Waiting for daemon to start...
.
INFO: Building docker-sdk-python3:3.7.0...
sha256:686428ae28479e9b5c8fdad1cadc9b7a39b462e66bd13a7e35bd79c6a152a402
INFO: Starting docker-py tests...
============================= test session starts ==============================
platform linux -- Python 3.6.8, pytest-4.1.0, py-1.8.0, pluggy-0.9.0
rootdir: /src, inifile: pytest.ini
plugins: timeout-1.3.3, cov-2.6.1
collected 359 items

tests/integration/api_build_test.py .......s....
....
```

Signed-off-by: Sebastiaan van Stijn <github@gone.nl>
(cherry picked from commit 7bfe48cc00318f9d4cf388237855012aafac56b0)
Signed-off-by: Sebastiaan van Stijn <github@gone.nl>

Dockerfile

@@ -73,17 +73,6 @@ RUN set -x \
 	   esac \
 	&& rm -rf "$GOPATH"
 
-
-
-FROM base AS docker-py
-# Get the "docker-py" source so we can run their integration tests
-ENV DOCKER_PY_COMMIT ac922192959870774ad8428344d9faa0555f7ba6
-RUN git clone https://github.com/docker/docker-py.git /build \
-	&& cd /build \
-	&& git checkout -q $DOCKER_PY_COMMIT
-
-
-
 FROM base AS swagger
 # Install go-swagger for validating swagger.yaml
 ENV GO_SWAGGER_COMMIT c28258affb0b6251755d92489ef685af8d4ff3eb
@@ -94,7 +83,6 @@ RUN set -x \
 	&& go build -o /build/swagger github.com/go-swagger/go-swagger/cmd/swagger \
 	&& rm -rf "$GOPATH"
 
-
 FROM base AS frozen-images
 RUN apt-get update && apt-get install -y jq ca-certificates --no-install-recommends
 # Get useful and necessary Hub images so we can "docker load" locally instead of pulling
@@ -228,25 +216,12 @@ RUN apt-get update && apt-get install -y \
 	jq \
 	libcap2-bin \
 	libdevmapper-dev \
-# libffi-dev and libssl-dev appear to be required for compiling paramiko on s390x/ppc64le
-	libffi-dev \
-	libssl-dev \
 	libudev-dev \
 	libsystemd-dev \
 	binutils-mingw-w64 \
 	g++-mingw-w64-x86-64 \
 	net-tools \
 	pigz \
-	python-backports.ssl-match-hostname \
-	python-dev \
-# python-cffi appears to be required for compiling paramiko on s390x/ppc64le
-	python-cffi \
-	python-mock \
-	python-pip \
-	python-requests \
-	python-setuptools \
-	python-websocket \
-	python-wheel \
 	thin-provisioning-tools \
 	vim \
 	vim-common \
@@ -270,16 +245,6 @@ COPY --from=proxy /build/ /usr/local/bin/
 COPY --from=dockercli /build/ /usr/local/cli
 COPY --from=registry /build/registry* /usr/local/bin/
 COPY --from=criu /build/ /usr/local/
-COPY --from=docker-py /build/ /docker-py
-# TODO: This is for the docker-py tests, which shouldn't really be needed for
-# this image, but currently CI is expecting to run this image. This should be
-# split out into a separate image, including all the `python-*` deps installed
-# above.
-RUN cd /docker-py \
-	&& pip install docker-pycreds==0.4.0 \
-	&& pip install paramiko==2.4.2 \
-	&& pip install yamllint==1.5.0 \
-	&& pip install -r test-requirements.txt
 COPY --from=rootlesskit /build/ /usr/local/bin/
 COPY --from=djs55/vpnkit@sha256:e508a17cfacc8fd39261d5b4e397df2b953690da577e2c987a47630cd0c42f8e /vpnkit /usr/local/bin/vpnkit.x86_64
 

hack/make/test-docker-py

@@ -3,18 +3,47 @@ set -e
 
 source hack/make/.integration-test-helpers
 
-# subshell so that we can export PATH without breaking other things
+# The commit or tag to use for testing
+# TODO docker 17.06 cli client used in CI fails to build using a sha;
+# unable to prepare context: unable to 'git clone' to temporary context directory: error fetching: error: no such remote ref ead0bb9e08c13dd3d1712759491eee06bf5a5602
+#: exit status 128
+: ${DOCKER_PY_COMMIT:=4.0.2}
 (
 	bundle .integration-daemon-start
 
-	dockerPy='/docker-py'
-	[ -d "$dockerPy" ] || {
-		dockerPy="$DEST/docker-py"
-		git clone https://github.com/docker/docker-py.git "$dockerPy"
-	}
+	docker_host_scheme=`echo "${DOCKER_HOST}" | cut -d: -f1 -`
 
-	# exporting PYTHONPATH to import "docker" from our local docker-py
-	test_env PYTHONPATH="$dockerPy" py.test --junitxml="$DEST/results.xml" "$dockerPy/tests/integration"
+	case "${docker_host_scheme}" in
+		unix)
+			# trim the tcp:// scheme, and bind-mount the docker socket into the container
+			run_opts="-v ${DOCKER_HOST#unix://}:/var/run/docker.sock"
+			;;
 
+		tcp)
+			# run container in host-mode networking so that it can connect to the
+			# daemon from the current networking namespace (e.g., to connect to localhost)
+			run_opts="--network=host -e DOCKER_HOST=${DOCKER_HOST}"
+			;;
+
+		*)
+			echo "WARN: Skipping test-docker-py: connecting to docker daemon using ${docker_host_scheme} (${DOCKER_HOST}) not supported"
+			bundle .integration-daemon-stop
+			return 0
+	esac
+
+	docker_py_image="docker-sdk-python3:${DOCKER_PY_COMMIT}"
+	if ! docker image inspect "dockerPyImage" &> /dev/null; then
+		echo INFO: Building ${docker_py_image}...
+		(
+			[ -n "${TESTDEBUG}" ] && set -x
+			exec docker build --quiet -t ${docker_py_image} -f tests/Dockerfile "https://github.com/docker/docker-py.git#${DOCKER_PY_COMMIT}"
+		)
+	fi
+
+	echo INFO: Starting docker-py tests...
+	(
+		[ -n "${TESTDEBUG}" ] && set -x
+		exec docker run -t --rm ${run_opts} ${docker_py_image} py.test tests/integration
+	)
 	bundle .integration-daemon-stop
 ) 2>&1 | tee -a "$DEST/test.log"