From 0f190f798f7b9f90bd008fe8fb0fc087ec965642 Mon Sep 17 00:00:00 2001 From: Kir Kolyshkin Date: Tue, 18 Sep 2018 13:28:49 -0700 Subject: [PATCH 1/7] go {build,test}: rm -i option, add go cache volume Looks like -i (together with DOCKER_INCREMENTAL_BINARY etc) were used to get faster incremental builds. Nowdays (since Go 1.10) this is no longer the case, as go build cache is used [1]. Here's a quote: > You do not have to use "go test -i" or "go build -i" or > "go install" just to get fast incremental builds. We will > not have to teach new users those workarounds anymore. > Everything will just be fast. To enable go cache between builds, add a volume for /root/.cache. [1] https://groups.google.com/forum/#!msg/golang-dev/qfa3mHN4ZPA/X2UzjNV1BAAJ Signed-off-by: Kir Kolyshkin (cherry picked from commit bdcd81d3301a053eefc320de16ac842ec47ed459) Signed-off-by: Sebastiaan van Stijn --- Makefile | 31 ++++++++----------------- docs/contributing/set-up-dev-env.md | 2 +- hack/integration-cli-on-swarm/README.md | 1 - hack/make.sh | 8 ------- hack/test/unit | 4 ---- 5 files changed, 11 insertions(+), 35 deletions(-) diff --git a/Makefile b/Makefile index 7767409a6f..2c64b809d7 100644 --- a/Makefile +++ b/Makefile @@ -1,10 +1,8 @@ -.PHONY: all binary dynbinary build cross help init-go-pkg-cache install manpages run shell test test-docker-py test-integration test-unit validate win +.PHONY: all binary dynbinary build cross help install manpages run shell test test-docker-py test-integration test-unit validate win # set the graph driver as the current graphdriver if not set DOCKER_GRAPHDRIVER := $(if $(DOCKER_GRAPHDRIVER),$(DOCKER_GRAPHDRIVER),$(shell docker info 2>&1 | grep "Storage Driver" | sed 's/.*: //')) export DOCKER_GRAPHDRIVER -DOCKER_INCREMENTAL_BINARY := $(if $(DOCKER_INCREMENTAL_BINARY),$(DOCKER_INCREMENTAL_BINARY),1) -export DOCKER_INCREMENTAL_BINARY # get OS/Arch of docker engine DOCKER_OSARCH := $(shell bash -c 'source hack/make/.detect-daemon-osarch && echo $${DOCKER_ENGINE_OSARCH}') @@ -44,7 +42,6 @@ DOCKER_ENVS := \ -e DOCKER_EXPERIMENTAL \ -e DOCKER_GITCOMMIT \ -e DOCKER_GRAPHDRIVER \ - -e DOCKER_INCREMENTAL_BINARY \ -e DOCKER_LDFLAGS \ -e DOCKER_PORT \ -e DOCKER_REMAP_ROOT \ @@ -84,14 +81,10 @@ DOCKER_MOUNT := $(if $(DOCKER_MOUNT),$(DOCKER_MOUNT),-v /go/src/github.com/docke # This allows to set the docker-dev container name DOCKER_CONTAINER_NAME := $(if $(CONTAINER_NAME),--name $(CONTAINER_NAME),) -# enable package cache if DOCKER_INCREMENTAL_BINARY and DOCKER_MOUNT (i.e.DOCKER_HOST) are set -PKGCACHE_MAP := gopath:/go/pkg goroot-linux_amd64:/usr/local/go/pkg/linux_amd64 goroot-linux_amd64_netgo:/usr/local/go/pkg/linux_amd64_netgo -PKGCACHE_VOLROOT := dockerdev-go-pkg-cache -PKGCACHE_VOL := $(if $(PKGCACHE_DIR),$(CURDIR)/$(PKGCACHE_DIR)/,$(PKGCACHE_VOLROOT)-) -DOCKER_MOUNT_PKGCACHE := $(if $(DOCKER_INCREMENTAL_BINARY),$(shell echo $(PKGCACHE_MAP) | sed -E 's@([^ ]*)@-v "$(PKGCACHE_VOL)\1"@g'),) +DOCKER_MOUNT_CACHE := -v docker-dev-cache:/root/.cache DOCKER_MOUNT_CLI := $(if $(DOCKER_CLI_PATH),-v $(shell dirname $(DOCKER_CLI_PATH)):/usr/local/cli,) DOCKER_MOUNT_BASH_COMPLETION := $(if $(DOCKER_BASH_COMPLETION_PATH),-v $(shell dirname $(DOCKER_BASH_COMPLETION_PATH)):/usr/local/completion/bash,) -DOCKER_MOUNT := $(DOCKER_MOUNT) $(DOCKER_MOUNT_PKGCACHE) $(DOCKER_MOUNT_CLI) $(DOCKER_MOUNT_BASH_COMPLETION) +DOCKER_MOUNT := $(DOCKER_MOUNT) $(DOCKER_MOUNT_CACHE) $(DOCKER_MOUNT_CLI) $(DOCKER_MOUNT_BASH_COMPLETION) GIT_BRANCH := $(shell git rev-parse --abbrev-ref HEAD 2>/dev/null) GIT_BRANCH_CLEAN := $(shell echo $(GIT_BRANCH) | sed -e "s/[^[:alnum:]]/-/g") @@ -133,19 +126,19 @@ binary: build ## build the linux binaries dynbinary: build ## build the linux dynbinaries $(DOCKER_RUN_DOCKER) hack/make.sh dynbinary -build: bundles init-go-pkg-cache +build: bundles $(warning The docker client CLI has moved to github.com/docker/cli. For a dev-test cycle involving the CLI, run:${\n} DOCKER_CLI_PATH=/host/path/to/cli/binary make shell ${\n} then change the cli and compile into a binary at the same location.${\n}) docker build ${BUILD_APT_MIRROR} ${DOCKER_BUILD_ARGS} -t "$(DOCKER_IMAGE)" -f "$(DOCKERFILE)" . bundles: mkdir bundles -clean: clean-pkg-cache-vol ## clean up cached resources +.PHONY: clean +clean: clean-cache -clean-pkg-cache-vol: - @- $(foreach mapping,$(PKGCACHE_MAP), \ - $(shell docker volume rm $(PKGCACHE_VOLROOT)-$(shell echo $(mapping) | awk -F':/' '{ print $$1 }') > /dev/null 2>&1) \ - ) +.PHONY: clean-cache +clean-cache: + docker volume rm -f docker-dev-cache cross: build ## cross build the binaries for darwin, freebsd and\nwindows $(DOCKER_RUN_DOCKER) hack/make.sh dynbinary binary cross @@ -153,9 +146,6 @@ cross: build ## cross build the binaries for darwin, freebsd and\nwindows help: ## this help @awk 'BEGIN {FS = ":.*?## "} /^[a-zA-Z_-]+:.*?## / {sub("\\\\n",sprintf("\n%22c"," "), $$2);printf "\033[36m%-20s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST) -init-go-pkg-cache: - $(if $(PKGCACHE_DIR), mkdir -p $(shell echo $(PKGCACHE_MAP) | sed -E 's@([^: ]*):[^ ]*@$(PKGCACHE_DIR)/\1@g')) - install: ## install the linux binaries KEEPBUNDLE=1 hack/make.sh install-binary @@ -206,12 +196,11 @@ build-integration-cli-on-swarm: build ## build images and binary for running int go build -buildmode=pie -o ./hack/integration-cli-on-swarm/integration-cli-on-swarm ./hack/integration-cli-on-swarm/host @echo "Building $(INTEGRATION_CLI_MASTER_IMAGE)" docker build -t $(INTEGRATION_CLI_MASTER_IMAGE) hack/integration-cli-on-swarm/agent -# For worker, we don't use `docker build` so as to enable DOCKER_INCREMENTAL_BINARY and so on @echo "Building $(INTEGRATION_CLI_WORKER_IMAGE) from $(DOCKER_IMAGE)" $(eval tmp := integration-cli-worker-tmp) # We mount pkgcache, but not bundle (bundle needs to be baked into the image) # For avoiding bakings DOCKER_GRAPHDRIVER and so on to image, we cannot use $(DOCKER_ENVS) here - docker run -t -d --name $(tmp) -e DOCKER_GITCOMMIT -e BUILDFLAGS -e DOCKER_INCREMENTAL_BINARY --privileged $(DOCKER_MOUNT_PKGCACHE) $(DOCKER_IMAGE) top + docker run -t -d --name $(tmp) -e DOCKER_GITCOMMIT -e BUILDFLAGS --privileged $(DOCKER_IMAGE) top docker exec $(tmp) hack/make.sh build-integration-test-binary dynbinary docker exec $(tmp) go build -buildmode=pie -o /worker github.com/docker/docker/hack/integration-cli-on-swarm/agent/worker docker commit -c 'ENTRYPOINT ["/worker"]' $(tmp) $(INTEGRATION_CLI_WORKER_IMAGE) diff --git a/docs/contributing/set-up-dev-env.md b/docs/contributing/set-up-dev-env.md index 3d56c0b8c7..f0619cb464 100644 --- a/docs/contributing/set-up-dev-env.md +++ b/docs/contributing/set-up-dev-env.md @@ -130,7 +130,7 @@ can take over 15 minutes to complete. ```none Successfully built 3d872560918e Successfully tagged docker-dev:dry-run-test - docker run --rm -i --privileged -e BUILDFLAGS -e KEEPBUNDLE -e DOCKER_BUILD_GOGC -e DOCKER_BUILD_PKGS -e DOCKER_CLIENTONLY -e DOCKER_DEBUG -e DOCKER_EXPERIMENTAL -e DOCKER_GITCOMMIT -e DOCKER_GRAPHDRIVER=devicemapper -e DOCKER_INCREMENTAL_BINARY -e DOCKER_REMAP_ROOT -e DOCKER_STORAGE_OPTS -e DOCKER_USERLANDPROXY -e TESTDIRS -e TESTFLAGS -e TIMEOUT -v "home/ubuntu/repos/docker/bundles:/go/src/github.com/docker/docker/bundles" -t "docker-dev:dry-run-test" bash + docker run --rm -i --privileged -e BUILDFLAGS -e KEEPBUNDLE -e DOCKER_BUILD_GOGC -e DOCKER_BUILD_PKGS -e DOCKER_CLIENTONLY -e DOCKER_DEBUG -e DOCKER_EXPERIMENTAL -e DOCKER_GITCOMMIT -e DOCKER_GRAPHDRIVER=devicemapper -e DOCKER_REMAP_ROOT -e DOCKER_STORAGE_OPTS -e DOCKER_USERLANDPROXY -e TESTDIRS -e TESTFLAGS -e TIMEOUT -v "home/ubuntu/repos/docker/bundles:/go/src/github.com/docker/docker/bundles" -t "docker-dev:dry-run-test" bash # ``` diff --git a/hack/integration-cli-on-swarm/README.md b/hack/integration-cli-on-swarm/README.md index 4f4f67d4f4..852b36cea1 100644 --- a/hack/integration-cli-on-swarm/README.md +++ b/hack/integration-cli-on-swarm/README.md @@ -36,7 +36,6 @@ while the client is supposed to be running on a laptop, e.g. Docker for Mac/Wind Following environment variables are known to work in this step: - `BUILDFLAGS` - - `DOCKER_INCREMENTAL_BINARY` Note: during the transition into Moby Project, you might need to create a symbolic link `$GOPATH/src/github.com/docker/docker` to `$GOPATH/src/github.com/moby/moby`. diff --git a/hack/make.sh b/hack/make.sh index 2f4ece3cdb..62c72a09e2 100755 --- a/hack/make.sh +++ b/hack/make.sh @@ -148,14 +148,6 @@ EXTLDFLAGS_STATIC='-static' ORIG_BUILDFLAGS=( -tags "autogen netgo osusergo static_build $DOCKER_BUILDTAGS" -installsuffix netgo ) # see https://github.com/golang/go/issues/9369#issuecomment-69864440 for why -installsuffix is necessary here -# When $DOCKER_INCREMENTAL_BINARY is set in the environment, enable incremental -# builds by installing dependent packages to the GOPATH. -REBUILD_FLAG="-a" -if [ "$DOCKER_INCREMENTAL_BINARY" == "1" ] || [ "$DOCKER_INCREMENTAL_BINARY" == "true" ]; then - REBUILD_FLAG="-i" -fi -ORIG_BUILDFLAGS+=( $REBUILD_FLAG ) - BUILDFLAGS=( $BUILDFLAGS "${ORIG_BUILDFLAGS[@]}" ) # Test timeout. diff --git a/hack/test/unit b/hack/test/unit index d0e85f1ad1..ac27f68c30 100755 --- a/hack/test/unit +++ b/hack/test/unit @@ -19,10 +19,6 @@ TESTDIRS="${TESTDIRS:-"./..."}" exclude_paths="/vendor/|/integration" pkg_list=$(go list $TESTDIRS | grep -vE "($exclude_paths)") -# install test dependencies once before running tests for each package. This -# significantly reduces the runtime. -go test -i "${BUILDFLAGS[@]}" $pkg_list - for pkg in $pkg_list; do go test "${BUILDFLAGS[@]}" \ -cover \ From ecc423df7fde7f0edc963f03d70da133f03bb63c Mon Sep 17 00:00:00 2001 From: Jean Rouge Date: Sat, 6 Oct 2018 11:06:45 -0700 Subject: [PATCH 2/7] Allow to override the Makefile's `DOCKER_MOUNT` variable Through the env variable of the same name. The idea here is pretty simple: I/O perf on native mounted disks on non-Linux (notably Mac OS) is just terrible, thus making it a real pain to develop: one has to choose between re-building the image after every single change (eg to run a test) or just work directly inside the same container (eg with vim, but even then one would have to re-configure their dev container every time it gets destroyed - containers, after all, are not supposed to be long-lived). Allowing to override `DOCKER_MOUNT` makes it easy for everyone to decide what their volume/syncing strategy is; for example one can choose to use [docker-sync](https://github.com/EugenMayer/docker-sync) This patch won't change anything for anyone who doesn't set the `DOCKER_MOUNT` env variable in their environment. Signed-off-by: Jean Rouge (cherry picked from commit aea6fdf3d340835a1b0af208839ce42ace3a5b89) Signed-off-by: Sebastiaan van Stijn --- Makefile | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 2c64b809d7..741b05c4ab 100644 --- a/Makefile +++ b/Makefile @@ -71,6 +71,9 @@ DOCKER_ENVS := \ # (default to no bind mount if DOCKER_HOST is set) # note: BINDDIR is supported for backwards-compatibility here BIND_DIR := $(if $(BINDDIR),$(BINDDIR),$(if $(DOCKER_HOST),,bundles)) + +# DOCKER_MOUNT can be overriden, but use at your own risk! +ifndef DOCKER_MOUNT DOCKER_MOUNT := $(if $(BIND_DIR),-v "$(CURDIR)/$(BIND_DIR):/go/src/github.com/docker/docker/$(BIND_DIR)") # This allows the test suite to be able to run without worrying about the underlying fs used by the container running the daemon (e.g. aufs-on-aufs), so long as the host running the container is running a supported fs. @@ -78,13 +81,14 @@ DOCKER_MOUNT := $(if $(BIND_DIR),-v "$(CURDIR)/$(BIND_DIR):/go/src/github.com/do # Note that `BIND_DIR` will already be set to `bundles` if `DOCKER_HOST` is not set (see above BIND_DIR line), in such case this will do nothing since `DOCKER_MOUNT` will already be set. DOCKER_MOUNT := $(if $(DOCKER_MOUNT),$(DOCKER_MOUNT),-v /go/src/github.com/docker/docker/bundles) -v "$(CURDIR)/.git:/go/src/github.com/docker/docker/.git" -# This allows to set the docker-dev container name -DOCKER_CONTAINER_NAME := $(if $(CONTAINER_NAME),--name $(CONTAINER_NAME),) - DOCKER_MOUNT_CACHE := -v docker-dev-cache:/root/.cache DOCKER_MOUNT_CLI := $(if $(DOCKER_CLI_PATH),-v $(shell dirname $(DOCKER_CLI_PATH)):/usr/local/cli,) DOCKER_MOUNT_BASH_COMPLETION := $(if $(DOCKER_BASH_COMPLETION_PATH),-v $(shell dirname $(DOCKER_BASH_COMPLETION_PATH)):/usr/local/completion/bash,) DOCKER_MOUNT := $(DOCKER_MOUNT) $(DOCKER_MOUNT_CACHE) $(DOCKER_MOUNT_CLI) $(DOCKER_MOUNT_BASH_COMPLETION) +endif # ifndef DOCKER_MOUNT + +# This allows to set the docker-dev container name +DOCKER_CONTAINER_NAME := $(if $(CONTAINER_NAME),--name $(CONTAINER_NAME),) GIT_BRANCH := $(shell git rev-parse --abbrev-ref HEAD 2>/dev/null) GIT_BRANCH_CLEAN := $(shell echo $(GIT_BRANCH) | sed -e "s/[^[:alnum:]]/-/g") From e5a039169f389120be3fbc2a9db45ef8bc99eae5 Mon Sep 17 00:00:00 2001 From: Mohammad Nasirifar Date: Thu, 13 Dec 2018 20:26:10 -0500 Subject: [PATCH 3/7] Use BuildKit to skip source code COPY if BIND_DIR set build the final stage of the Dockerfile (including COPY ...) if no BIND_DIR is used. if BIND_DIR is used, build the dev stage, thus skipping the COPY. Original author: @thaJeztah Signed-off-by: Mohammad Nasirifar (cherry picked from commit e6d7df2e5d313800414b955e10a26d6687e7a1bf) Signed-off-by: Sebastiaan van Stijn --- Dockerfile | 2 ++ Makefile | 6 +++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 15ccf0f829..a2dcbb8624 100644 --- a/Dockerfile +++ b/Dockerfile @@ -245,5 +245,7 @@ WORKDIR /go/src/github.com/docker/docker VOLUME /var/lib/docker # Wrap all commands in the "docker-in-docker" script to allow nested containers ENTRYPOINT ["hack/dind"] + +FROM dev AS final # Upload docker source COPY . /go/src/github.com/docker/docker diff --git a/Makefile b/Makefile index 741b05c4ab..8c2c66030e 100644 --- a/Makefile +++ b/Makefile @@ -34,6 +34,7 @@ DOCKER_ENVS := \ -e KEEPBUNDLE \ -e DOCKER_BUILD_ARGS \ -e DOCKER_BUILD_GOGC \ + -e DOCKER_BUILD_OPTS \ -e DOCKER_BUILD_PKGS \ -e DOCKER_BUILDKIT \ -e DOCKER_BASH_COMPLETION_PATH \ @@ -116,6 +117,9 @@ INTERACTIVE := $(shell [ -t 0 ] && echo 1 || echo 0) ifeq ($(INTERACTIVE), 1) DOCKER_FLAGS += -t endif +ifeq ($(BIND_DIR), .) + DOCKER_BUILD_OPTS += --target=dev +endif DOCKER_RUN_DOCKER := $(DOCKER_FLAGS) "$(DOCKER_IMAGE)" @@ -132,7 +136,7 @@ dynbinary: build ## build the linux dynbinaries build: bundles $(warning The docker client CLI has moved to github.com/docker/cli. For a dev-test cycle involving the CLI, run:${\n} DOCKER_CLI_PATH=/host/path/to/cli/binary make shell ${\n} then change the cli and compile into a binary at the same location.${\n}) - docker build ${BUILD_APT_MIRROR} ${DOCKER_BUILD_ARGS} -t "$(DOCKER_IMAGE)" -f "$(DOCKERFILE)" . + docker build ${BUILD_APT_MIRROR} ${DOCKER_BUILD_ARGS} ${DOCKER_BUILD_OPTS} -t "$(DOCKER_IMAGE)" -f "$(DOCKERFILE)" . bundles: mkdir bundles From 733b86683f347f922affc714d305d9f03fe9b47e Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Thu, 10 Jan 2019 12:56:26 +0100 Subject: [PATCH 4/7] Makefile: make help: fix newline wrapping, and missing targets This patch; - adds support for multiple newlines - removes the 1-space indentation of wrapped lines - allows numerical characters in targets (0-9) Given these targets: ```Makefile .PHONY: foobar foobar: ## runs the foobar lorum ipsum.\nand so pn\nand so on echo foobar .PHONY: e2e-tests e2e-tests: ## runs the end-to-end tests echo e2e-tests ``` Before this change, the output of `make help` was ``` foobar runs the foobar lorum ipsum. and so pn\nand so on ``` After this change, the output is: ``` foobar runs the foobar lorum ipsum. and so pn and so on e2e-tests runs the end-to-end tests ``` Signed-off-by: Sebastiaan van Stijn (cherry picked from commit 202c9d8c98614e7cce2017f5c99d3d783fe8b509) Signed-off-by: Sebastiaan van Stijn --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 8c2c66030e..686b62ffcf 100644 --- a/Makefile +++ b/Makefile @@ -152,7 +152,7 @@ cross: build ## cross build the binaries for darwin, freebsd and\nwindows $(DOCKER_RUN_DOCKER) hack/make.sh dynbinary binary cross help: ## this help - @awk 'BEGIN {FS = ":.*?## "} /^[a-zA-Z_-]+:.*?## / {sub("\\\\n",sprintf("\n%22c"," "), $$2);printf "\033[36m%-20s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST) + @awk 'BEGIN {FS = ":.*?## "} /^[a-zA-Z0-9_-]+:.*?## / {gsub("\\\\n",sprintf("\n%22c",""), $$2);printf "\033[36m%-20s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST) install: ## install the linux binaries KEEPBUNDLE=1 hack/make.sh install-binary From e5c0923b277055c977994ebfdc689346c3104a16 Mon Sep 17 00:00:00 2001 From: Olli Janatuinen Date: Mon, 7 Jan 2019 19:05:54 +0200 Subject: [PATCH 5/7] CI: Introduce flaky test finder comparing PR commit(s) to HEAD of moby/moby master branch and if founds new (or renamed) integration tests will run stress tests for them. Signed-off-by: Olli Janatuinen (cherry picked from commit 8a8fd37f6fb53716cb4b3a7e93e1e3cf385927e2) Signed-off-by: Sebastiaan van Stijn --- Makefile | 3 +++ hack/ci/janky | 1 + hack/make/test-integration-flaky | 26 ++++++++++++++++++++++++++ 3 files changed, 30 insertions(+) create mode 100644 hack/make/test-integration-flaky diff --git a/Makefile b/Makefile index 686b62ffcf..67f6c3dab1 100644 --- a/Makefile +++ b/Makefile @@ -174,6 +174,9 @@ test-integration-cli: test-integration ## (DEPRECATED) use test-integration test-integration: build ## run the integration tests $(DOCKER_RUN_DOCKER) hack/make.sh dynbinary test-integration +test-integration-flaky: build ## run the stress test for all new integration tests + $(DOCKER_RUN_DOCKER) hack/make.sh dynbinary test-integration-flaky + test-unit: build ## run the unit tests $(DOCKER_RUN_DOCKER) hack/test/unit diff --git a/hack/ci/janky b/hack/ci/janky index f2bdfbf326..88cb9d9c61 100755 --- a/hack/ci/janky +++ b/hack/ci/janky @@ -13,5 +13,6 @@ hack/make.sh \ binary-daemon \ dynbinary \ test-docker-py \ + test-integration-flaky \ test-integration \ cross diff --git a/hack/make/test-integration-flaky b/hack/make/test-integration-flaky new file mode 100644 index 0000000000..00723a75bb --- /dev/null +++ b/hack/make/test-integration-flaky @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +set -e -o pipefail + +source hack/validate/.validate +new_tests=$( + validate_diff --diff-filter=ACMR --unified=0 -- 'integration/*_test.go' | + grep -E '^(\+func )(.*)(\*testing)' || true +) + +if [ -z "$new_tests" ]; then + echo 'No new tests added to integration.' + return +fi + +echo +echo "Found new integrations tests:" +echo "$new_tests" +echo "Running stress test for them." + +( + TESTARRAY=$(echo "$new_tests" | sed 's/+func //' | awk -F'\\(' '{print $1}' | tr '\n' '|') + export TESTFLAGS="-test.count 5 -test.run ${TESTARRAY%?}" + export TEST_REPEAT=5 + echo "Using test flags: $TESTFLAGS" + source hack/make/test-integration +) From a9c1bfc1b1cd94968b399ffdddef4b908a722bb3 Mon Sep 17 00:00:00 2001 From: Tibor Vass Date: Fri, 8 Feb 2019 00:37:41 +0000 Subject: [PATCH 6/7] hack: Have TIMEOUT take -test.count into account when testing for flakiness Signed-off-by: Tibor Vass (cherry picked from commit 42dcfc894a64e9b4c1751b21db1537b3b68a36d8) Signed-off-by: Sebastiaan van Stijn --- hack/make/test-integration-flaky | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/hack/make/test-integration-flaky b/hack/make/test-integration-flaky index 00723a75bb..14fb034b75 100644 --- a/hack/make/test-integration-flaky +++ b/hack/make/test-integration-flaky @@ -19,8 +19,19 @@ echo "Running stress test for them." ( TESTARRAY=$(echo "$new_tests" | sed 's/+func //' | awk -F'\\(' '{print $1}' | tr '\n' '|') - export TESTFLAGS="-test.count 5 -test.run ${TESTARRAY%?}" + # Note: TEST_REPEAT will make the test suite run 5 times, restarting the daemon + # whereas testcount will make each test run 5 times in a row under the same daemon. + # This will make a total of 25 runs for each test in TESTARRAY. export TEST_REPEAT=5 + local testcount=5 + # However, TIMEOUT needs to take testcount into account, or a premature time out may happen. + # The following ugliness will: + # - remove last character (usually 'm' from '10m') + # - multiply by testcount + # - add last character back + export TIMEOUT=$((${TIMEOUT::-1} * $testcount))${TIMEOUT:$((${#TIMEOUT}-1)):1} + + export TESTFLAGS="-test.count $testcount -test.run ${TESTARRAY%?}" echo "Using test flags: $TESTFLAGS" source hack/make/test-integration ) From e64cd6abed308a26d9048d4d0e4c52207c2ad5df Mon Sep 17 00:00:00 2001 From: Brian Goff Date: Sat, 9 Mar 2019 18:27:26 -0800 Subject: [PATCH 7/7] Enable buildkit for Makefile build target This is set only if it is not already set. This should give a little speedup to CI builds. Signed-off-by: Brian Goff (cherry picked from commit 1275a001a68722494d090d5beca6749a83710cc2) Signed-off-by: Sebastiaan van Stijn --- Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 67f6c3dab1..e2539683e1 100644 --- a/Makefile +++ b/Makefile @@ -134,9 +134,10 @@ binary: build ## build the linux binaries dynbinary: build ## build the linux dynbinaries $(DOCKER_RUN_DOCKER) hack/make.sh dynbinary +build: DOCKER_BUILDKIT ?= 1 build: bundles $(warning The docker client CLI has moved to github.com/docker/cli. For a dev-test cycle involving the CLI, run:${\n} DOCKER_CLI_PATH=/host/path/to/cli/binary make shell ${\n} then change the cli and compile into a binary at the same location.${\n}) - docker build ${BUILD_APT_MIRROR} ${DOCKER_BUILD_ARGS} ${DOCKER_BUILD_OPTS} -t "$(DOCKER_IMAGE)" -f "$(DOCKERFILE)" . + DOCKER_BUILDKIT="${DOCKER_BUILDKIT}" docker build ${BUILD_APT_MIRROR} ${DOCKER_BUILD_ARGS} ${DOCKER_BUILD_OPTS} -t "$(DOCKER_IMAGE)" -f "$(DOCKERFILE)" . bundles: mkdir bundles