Merge pull request #44034 from crazy-max/gha-test

ci: gha test workflow for integration and unit test

.github/actions/setup-runner/action.yml

@@ -0,0 +1,27 @@
+name: 'Setup Runner'
+description: 'Composite action to set up the GitHub Runner for jobs in the test.yml workflow'
+
+runs:
+  using: composite
+  steps:
+    - run: |
+        sudo modprobe ip_vs
+        sudo modprobe ipv6
+        sudo modprobe ip6table_filter
+        sudo modprobe -r overlay
+        sudo modprobe overlay redirect_dir=off
+      shell: bash
+    - run: |
+        if [ ! -e /etc/docker/daemon.json ]; then
+         echo '{}' | tee /etc/docker/daemon.json >/dev/null
+        fi
+        DOCKERD_CONFIG=$(jq '.+{"experimental":true,"live-restore":true,"ipv6":true,"fixed-cidr-v6":"2001:db8:1::/64"}' /etc/docker/daemon.json)
+        sudo tee /etc/docker/daemon.json <<<"$DOCKERD_CONFIG" >/dev/null
+        sudo service docker restart
+      shell: bash
+    - run: |
+        ./contrib/check-config.sh || true
+      shell: bash
+    - run: |
+        docker info
+      shell: bash

.github/workflows/.windows.yml

@@ -1,5 +1,5 @@
 # reusable workflow
-name: windows
+name: .windows
 
 on:
   workflow_call:

.github/workflows/test.yml

@@ -0,0 +1,361 @@
+name: test
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - 'master'
+      - '[0-9]+.[0-9]{2}'
+    tags:
+      - 'v*'
+  pull_request:
+
+env:
+  GO_VERSION: 1.19.1
+  GOTESTLIST_VERSION: v0.2.0
+  ITG_CLI_MATRIX_SIZE: 6
+  BUILDX: docker buildx
+  USE_BUILDX: 1
+  DOCKER_EXPERIMENTAL: 1
+  DOCKER_GRAPHDRIVER: overlay2
+
+jobs:
+  build-dev:
+    runs-on: ubuntu-20.04
+    strategy:
+      fail-fast: false
+      matrix:
+        mode:
+          - ""
+          - systemd
+    steps:
+      -
+        name: Prepare
+        run: |
+          if [ "${{ matrix.mode }}" = "systemd" ]; then
+            echo "SYSTEMD=true" >> $GITHUB_ENV
+          fi
+      -
+        name: Checkout
+        uses: actions/checkout@v3
+      -
+        name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+      -
+        name: Build dev image
+        uses: docker/bake-action@v2
+        with:
+          targets: dev
+          set: |
+            *.cache-from=type=gha,scope=dev${{ matrix.mode }}
+            *.cache-to=type=gha,scope=dev${{ matrix.mode }},mode=max
+            *.output=type=cacheonly
+
+  unit:
+    runs-on: ubuntu-20.04
+    needs:
+      - build-dev
+    steps:
+      -
+        name: Checkout
+        uses: actions/checkout@v3
+      -
+        name: Set up runner
+        uses: ./.github/actions/setup-runner
+      -
+        name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+      -
+        name: Build dev image
+        uses: docker/bake-action@v2
+        with:
+          targets: dev
+          set: |
+            dev.cache-from=type=gha,scope=dev
+      -
+        name: Test
+        run: |
+          make -o build test-unit
+      -
+        name: Prepare reports
+        if: always()
+        run: |
+          mkdir -p bundles /tmp/reports
+          find bundles -path '*/root/*overlay2' -prune -o -type f \( -name '*-report.json' -o -name '*.log' -o -name '*.out' -o -name '*.prof' -o -name '*-report.xml' \) -print | xargs sudo tar -czf /tmp/reports.tar.gz
+          tar -xzf /tmp/reports.tar.gz -C /tmp/reports
+          sudo chown -R $(id -u):$(id -g) /tmp/reports
+          tree -nh /tmp/reports
+      -
+        name: Send to Codecov
+        uses: codecov/codecov-action@v3
+        with:
+          directory: ./bundles
+          env_vars: RUNNER_OS
+          flags: unit
+      -
+        name: Upload reports
+        if: always()
+        uses: actions/upload-artifact@v3
+        with:
+          name: unit-reports
+          path: /tmp/reports/*
+
+  docker-py:
+    runs-on: ubuntu-20.04
+    needs:
+      - build-dev
+    steps:
+      -
+        name: Checkout
+        uses: actions/checkout@v3
+      -
+        name: Set up runner
+        uses: ./.github/actions/setup-runner
+      -
+        name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+      -
+        name: Build dev image
+        uses: docker/bake-action@v2
+        with:
+          targets: dev
+          set: |
+            dev.cache-from=type=gha,scope=dev
+      -
+        name: Test
+        run: |
+          make -o build test-docker-py
+      -
+        name: Prepare reports
+        if: always()
+        run: |
+          mkdir -p bundles /tmp/reports
+          find bundles -path '*/root/*overlay2' -prune -o -type f \( -name '*-report.json' -o -name '*.log' -o -name '*.out' -o -name '*.prof' -o -name '*-report.xml' \) -print | xargs sudo tar -czf /tmp/reports.tar.gz
+          tar -xzf /tmp/reports.tar.gz -C /tmp/reports
+          sudo chown -R $(id -u):$(id -g) /tmp/reports
+          tree -nh /tmp/reports
+      -
+        name: Test daemon logs
+        if: always()
+        run: |
+          cat bundles/test-docker-py/docker.log
+      -
+        name: Upload reports
+        if: always()
+        uses: actions/upload-artifact@v3
+        with:
+          name: docker-py-reports
+          path: /tmp/reports/*
+
+  integration-flaky:
+    runs-on: ubuntu-20.04
+    needs:
+      - build-dev
+    steps:
+      -
+        name: Checkout
+        uses: actions/checkout@v3
+      -
+        name: Set up runner
+        uses: ./.github/actions/setup-runner
+      -
+        name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+      -
+        name: Build dev image
+        uses: docker/bake-action@v2
+        with:
+          targets: dev
+          set: |
+            dev.cache-from=type=gha,scope=dev
+      -
+        name: Test
+        run: |
+          make -o build test-integration-flaky
+        env:
+          TEST_SKIP_INTEGRATION_CLI: 1
+
+  integration:
+    runs-on: ${{ matrix.os }}
+    needs:
+      - build-dev
+    strategy:
+      fail-fast: false
+      matrix:
+        os:
+          - ubuntu-20.04
+          - ubuntu-22.04
+        mode:
+          - ""
+          - rootless
+          - systemd
+          #- rootless-systemd FIXME: https://github.com/moby/moby/issues/44084
+    steps:
+      -
+        name: Checkout
+        uses: actions/checkout@v3
+      -
+        name: Set up runner
+        uses: ./.github/actions/setup-runner
+      -
+        name: Prepare
+        run: |
+          CACHE_DEV_SCOPE=dev
+          if [[ "${{ matrix.mode }}" == *"rootless"* ]]; then
+            echo "DOCKER_ROOTLESS=1" >> $GITHUB_ENV
+          fi
+          if [[ "${{ matrix.mode }}" == *"systemd"* ]]; then
+            echo "SYSTEMD=true" >> $GITHUB_ENV
+            CACHE_DEV_SCOPE="${CACHE_DEV_SCOPE}systemd"
+          fi
+          echo "CACHE_DEV_SCOPE=${CACHE_DEV_SCOPE}" >> $GITHUB_ENV
+      -
+        name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+      -
+        name: Build dev image
+        uses: docker/bake-action@v2
+        with:
+          targets: dev
+          set: |
+            dev.cache-from=type=gha,scope=${{ env.CACHE_DEV_SCOPE }}
+      -
+        name: Test
+        run: |
+          make -o build test-integration
+        env:
+          TEST_SKIP_INTEGRATION_CLI: 1
+          TESTCOVERAGE: 1
+      -
+        name: Prepare reports
+        if: always()
+        run: |
+          reportsPath="/tmp/reports/${{ matrix.os }}"
+          if [ -n "${{ matrix.mode }}" ]; then
+            reportsPath="$reportsPath-${{ matrix.mode }}"
+          fi
+          mkdir -p bundles $reportsPath
+          find bundles -path '*/root/*overlay2' -prune -o -type f \( -name '*-report.json' -o -name '*.log' -o -name '*.out' -o -name '*.prof' -o -name '*-report.xml' \) -print | xargs sudo tar -czf /tmp/reports.tar.gz
+          tar -xzf /tmp/reports.tar.gz -C $reportsPath
+          sudo chown -R $(id -u):$(id -g) $reportsPath
+          tree -nh $reportsPath
+      -
+        name: Send to Codecov
+        uses: codecov/codecov-action@v3
+        with:
+          directory: ./bundles/test-integration
+          env_vars: RUNNER_OS
+          flags: integration,${{ matrix.mode }}
+      -
+        name: Test daemon logs
+        if: always()
+        run: |
+          cat bundles/test-integration/docker.log
+      -
+        name: Upload reports
+        if: always()
+        uses: actions/upload-artifact@v3
+        with:
+          name: integration-reports
+          path: /tmp/reports/*
+
+  integration-cli-prepare:
+    runs-on: ubuntu-20.04
+    outputs:
+      matrix: ${{ steps.tests.outputs.matrix }}
+    steps:
+      -
+        name: Checkout
+        uses: actions/checkout@v3
+      -
+        name: Set up Go
+        uses: actions/setup-go@v3
+        with:
+          go-version: ${{ env.GO_VERSION }}
+      -
+        name: Install gotestlist
+        run:
+          go install github.com/crazy-max/gotestlist/cmd/gotestlist@${{ env.GOTESTLIST_VERSION }}
+      -
+        name: Create matrix
+        id: tests
+        working-directory: ./integration-cli
+        run: |
+          # Distribute integration-cli tests for the matrix in integration-test job.
+          # Also prepend ./... to the matrix. This is a special case to run "Test integration" step exclusively.
+          matrix="$(gotestlist -d ${{ env.ITG_CLI_MATRIX_SIZE }} ./...)"
+          matrix="$(echo "$matrix" | jq -c '. |= ["./..."] + .')"
+          echo "::set-output name=matrix::$matrix"
+      -
+        name: Show matrix
+        run: |
+          echo ${{ steps.tests.outputs.matrix }}
+
+  integration-cli:
+    runs-on: ubuntu-20.04
+    needs:
+      - build-dev
+      - integration-cli-prepare
+    strategy:
+      fail-fast: false
+      matrix:
+        test: ${{ fromJson(needs.integration-cli-prepare.outputs.matrix) }}
+    steps:
+      -
+        name: Checkout
+        uses: actions/checkout@v3
+      -
+        name: Set up runner
+        uses: ./.github/actions/setup-runner
+      -
+        name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+      -
+        name: Build dev image
+        uses: docker/bake-action@v2
+        with:
+          targets: dev
+          set: |
+            dev.cache-from=type=gha,scope=dev
+      -
+        name: Test
+        run: |
+          make -o build test-integration
+        env:
+          TEST_SKIP_INTEGRATION: 1
+          TESTCOVERAGE: 1
+          TESTFLAGS: "-test.run (${{ matrix.test }})/"
+      -
+        name: Prepare reports
+        if: always()
+        run: |
+          reportsPath=/tmp/reports/$(echo -n "${{ matrix.test }}" | sha256sum | cut -d " " -f 1)
+          mkdir -p bundles $reportsPath
+          echo "${{ matrix.test }}" | tr -s '|' '\n' | tee -a "$reportsPath/tests.txt"
+          find bundles -path '*/root/*overlay2' -prune -o -type f \( -name '*-report.json' -o -name '*.log' -o -name '*.out' -o -name '*.prof' -o -name '*-report.xml' \) -print | xargs sudo tar -czf /tmp/reports.tar.gz
+          tar -xzf /tmp/reports.tar.gz -C $reportsPath
+          sudo chown -R $(id -u):$(id -g) $reportsPath
+          tree -nh $reportsPath
+      -
+        name: Send to Codecov
+        uses: codecov/codecov-action@v3
+        with:
+          directory: ./bundles/test-integration
+          env_vars: RUNNER_OS
+          flags: integration-cli
+      -
+        name: Test daemon logs
+        if: always()
+        run: |
+          cat bundles/test-integration/docker.log
+      -
+        name: Upload reports
+        if: always()
+        uses: actions/upload-artifact@v3
+        with:
+          name: integration-cli-reports
+          path: /tmp/reports/*

Jenkinsfile

@@ -234,7 +234,7 @@ pipeline {
                                 sh '''
                                 bundleName=unit
                                 echo "Creating ${bundleName}-bundles.tar.gz"
-                                tar -czvf ${bundleName}-bundles.tar.gz bundles/junit-report*.xml bundles/go-test-report*.json bundles/profile*.out
+                                tar -czvf ${bundleName}-bundles.tar.gz bundles/junit-report*.xml bundles/go-test-report*.json bundles/coverage*.out
                                 '''
 
                                 archiveArtifacts artifacts: '*-bundles.tar.gz', allowEmptyArchive: true

Makefile

@@ -69,10 +69,12 @@ DOCKER_ENVS := \
 	-e DOCKER_USERLANDPROXY \
 	-e DOCKERD_ARGS \
 	-e DELVE_PORT \
+	-e GITHUB_ACTIONS \
 	-e TEST_FORCE_VALIDATE \
 	-e TEST_INTEGRATION_DIR \
 	-e TEST_SKIP_INTEGRATION \
 	-e TEST_SKIP_INTEGRATION_CLI \
+	-e TESTCOVERAGE \
 	-e TESTDEBUG \
 	-e TESTDIRS \
 	-e TESTFLAGS \
@@ -118,7 +120,7 @@ DOCKER_IMAGE := docker-dev
 DOCKER_PORT_FORWARD := $(if $(DOCKER_PORT),-p "$(DOCKER_PORT)",)
 DELVE_PORT_FORWARD := $(if $(DELVE_PORT),-p "$(DELVE_PORT)",)
 
-DOCKER_FLAGS := $(DOCKER) run --rm -i --privileged $(DOCKER_CONTAINER_NAME) $(DOCKER_ENVS) $(DOCKER_MOUNT) $(DOCKER_PORT_FORWARD) $(DELVE_PORT_FORWARD)
+DOCKER_FLAGS := $(DOCKER) run --rm --privileged $(DOCKER_CONTAINER_NAME) $(DOCKER_ENVS) $(DOCKER_MOUNT) $(DOCKER_PORT_FORWARD) $(DELVE_PORT_FORWARD)
 BUILD_APT_MIRROR := $(if $(DOCKER_BUILD_APT_MIRROR),--build-arg APT_MIRROR=$(DOCKER_BUILD_APT_MIRROR))
 export BUILD_APT_MIRROR
 
@@ -137,6 +139,14 @@ ifeq ($(INTERACTIVE), 1)
 	DOCKER_FLAGS += -t
 endif
 
+# on GitHub Runners input device is not a TTY but we allocate a pseudo-one,
+# otherwise keep STDIN open even if not attached if not a GitHub Runner.
+ifeq ($(GITHUB_ACTIONS),true)
+	DOCKER_FLAGS += -t
+else
+	DOCKER_FLAGS += -i
+endif
+
 DOCKER_RUN_DOCKER := $(DOCKER_FLAGS) "$(DOCKER_IMAGE)"
 
 DOCKER_BUILD_ARGS += --build-arg=GO_VERSION

docker-bake.hcl

@@ -35,3 +35,24 @@ target "cross" {
   }
   target = "cross"
 }
+
+#
+# dev
+#
+
+variable "DEV_IMAGE" {
+  default = "docker-dev"
+}
+variable "SYSTEMD" {
+  default = "false"
+}
+
+target "dev" {
+  inherits = ["_common"]
+  target = "final"
+  args = {
+    SYSTEMD = SYSTEMD
+  }
+  tags = [DEV_IMAGE]
+  output = ["type=docker"]
+}

hack/make/.integration-test-helpers

@@ -16,6 +16,7 @@ source "${MAKEDIR}/.go-autogen"
 : "${TEST_REPEAT:=1}"
 : "${TESTFLAGS:=}"
 : "${TESTDEBUG:=}"
+: "${TESTCOVERAGE:=}"
 : "${GOCACHE:=$(go env GOCACHE)}"
 
 setup_integration_test_filter() {
@@ -80,7 +81,13 @@ run_test_integration_suites() {
 			# Finally, we use periods as separator (instead of slashes) to be more
 			# in line with Java package names (which is what junit.xml was designed for)
 			pkgname="$(go env GOARCH).${pkgname//\//.}"
-			echo "Running $PWD (${pkgname}) flags=${flags}"
+
+			pkgtestflags=$flags
+			if [ -n "${TESTCOVERAGE}" ]; then
+				pkgtestflags="$pkgtestflags -test.coverprofile=${ABS_DEST}/${pkgname//./-}-coverage.out"
+			fi
+
+			echo "Running $PWD (${pkgname}) flags=${pkgtestflags}"
 			[ -n "$TESTDEBUG" ] && set -x
 			# shellcheck disable=SC2086
 			test_env gotestsum \
@@ -88,7 +95,7 @@ run_test_integration_suites() {
 				--jsonfile="${ABS_DEST}/${pkgname//./-}-go-test-report.json" \
 				--junitfile="${ABS_DEST}/${pkgname//./-}-junit-report.xml" \
 				--raw-command \
-				-- go tool test2json -p "${pkgname}" -t ./test.main ${flags}
+				-- go tool test2json -p "${pkgname}" -t ./test.main ${pkgtestflags}
 		); then exit 1; fi
 	done
 }
@@ -112,8 +119,12 @@ build_test_suite_binaries() {
 build_test_suite_binary() {
 	local dir="$1"
 	local out="$2"
+	local testflags
 	echo Building test suite binary "$dir/$out"
-	go test -c -o "$dir/$out" -ldflags "$LDFLAGS" "${BUILDFLAGS[@]}" "$dir"
+	if [ -n "${TESTCOVERAGE}" ]; then
+		testflags="-cover -covermode=atomic"
+	fi
+	go test ${testflags} -c -o "$dir/$out" -ldflags "$LDFLAGS" "${BUILDFLAGS[@]}" "$dir"
 }
 
 cleanup_test_suite_binaries() {
@@ -150,6 +161,7 @@ test_env() {
 			DOCKER_REMOTE_DAEMON="$DOCKER_REMOTE_DAEMON" \
 			DOCKER_ROOTLESS="$DOCKER_ROOTLESS" \
 			DOCKERFILE="$DOCKERFILE" \
+			GITHUB_ACTIONS="$GITHUB_ACTIONS" \
 			GOCACHE="$GOCACHE" \
 			GOPATH="$GOPATH" \
 			GOTRACEBACK=all \

hack/test/unit

@@ -32,7 +32,7 @@ if [ -n "${base_pkg_list}" ]; then
 	gotestsum --format=standard-quiet --jsonfile=bundles/go-test-report.json --junitfile=bundles/junit-report.xml -- \
 		"${BUILDFLAGS[@]}" \
 		-cover \
-		-coverprofile=bundles/profile.out \
+		-coverprofile=bundles/coverage.out \
 		-covermode=atomic \
 		${TESTFLAGS} \
 		${base_pkg_list}
@@ -44,7 +44,7 @@ if [ -n "${libnetwork_pkg_list}" ]; then
 	gotestsum --format=standard-quiet --jsonfile=bundles/go-test-report-libnetwork.json --junitfile=bundles/junit-report-libnetwork.xml -- \
 		"${BUILDFLAGS[@]}" \
 		-cover \
-		-coverprofile=bundles/profile-libnetwork.out \
+		-coverprofile=bundles/coverage-libnetwork.out \
 		-covermode=atomic \
 		-p=1 \
 		${TESTFLAGS} \

integration-cli/docker_cli_events_unix_test.go

@@ -19,6 +19,7 @@ import (
 	"golang.org/x/sys/unix"
 	"gotest.tools/v3/assert"
 	is "gotest.tools/v3/assert/cmp"
+	"gotest.tools/v3/skip"
 )
 
 // #5979
@@ -50,6 +51,7 @@ func (s *DockerCLIEventSuite) TestEventsRedirectStdout(c *testing.T) {
 
 func (s *DockerCLIEventSuite) TestEventsOOMDisableFalse(c *testing.T) {
 	testRequires(c, DaemonIsLinux, oomControl, memoryLimitSupport, swapMemorySupport, NotPpc64le)
+	skip.If(c, GitHubActions, "FIXME: https://github.com/moby/moby/pull/36541")
 
 	errChan := make(chan error, 1)
 	go func() {
@@ -80,6 +82,7 @@ func (s *DockerCLIEventSuite) TestEventsOOMDisableFalse(c *testing.T) {
 
 func (s *DockerCLIEventSuite) TestEventsOOMDisableTrue(c *testing.T) {
 	testRequires(c, DaemonIsLinux, oomControl, memoryLimitSupport, NotArm, swapMemorySupport, NotPpc64le)
+	skip.If(c, GitHubActions, "FIXME: https://github.com/moby/moby/pull/36541")
 
 	errChan := make(chan error, 1)
 	observer, err := newEventObserver(c)

integration-cli/docker_cli_plugins_test.go

@@ -17,6 +17,7 @@ import (
 	"github.com/docker/docker/integration-cli/daemon"
 	"github.com/docker/docker/testutil/fixtures/plugin"
 	"gotest.tools/v3/assert"
+	"gotest.tools/v3/skip"
 )
 
 var (
@@ -220,6 +221,7 @@ func (ps *DockerPluginSuite) TestPluginInstallArgs(c *testing.T) {
 
 func (ps *DockerPluginSuite) TestPluginInstallImage(c *testing.T) {
 	testRequires(c, IsAmd64)
+	skip.If(c, GitHubActions, "FIXME: https://github.com/moby/moby/issues/43996")
 
 	repoName := fmt.Sprintf("%v/dockercli/busybox", privateRegistryURL)
 	// tag the image to upload it to the private registry

integration-cli/requirements_test.go

@@ -69,6 +69,10 @@ func UnixCli() bool {
 	return isUnixCli
 }
 
+func GitHubActions() bool {
+	return os.Getenv("GITHUB_ACTIONS") != ""
+}
+
 func Network() bool {
 	// Set a timeout on the GET at 15s
 	const timeout = 15 * time.Second

integration/container/nat_test.go

@@ -56,6 +56,7 @@ func TestNetworkLocalhostTCPNat(t *testing.T) {
 }
 
 func TestNetworkLoopbackNat(t *testing.T) {
+	skip.If(t, testEnv.GitHubActions, "FIXME: https://github.com/moby/moby/issues/41561")
 	skip.If(t, testEnv.OSType == "windows", "FIXME")
 	skip.If(t, testEnv.IsRemoteDaemon)
 

testutil/environment/environment.go

@@ -224,5 +224,5 @@ func EnsureFrozenImagesLinux(testEnv *Execution) error {
 
 // GitHubActions is true if test is executed on a GitHub Runner.
 func (e *Execution) GitHubActions() bool {
-	return os.Getenv("GITHUB_ACTIONS") == "true"
+	return os.Getenv("GITHUB_ACTIONS") != ""
 }