Merge pull request #44546 from crazy-max/dockerfile-cross

Dockerfile: use TARGETPLATFORM to build Docker

.github/workflows/buildkit.yml

@@ -13,7 +13,7 @@ on:
   pull_request:
 
 env:
-  BUNDLES_OUTPUT: ./bundles
+  DESTDIR: ./build
 
 jobs:
   validate-dco:
@@ -40,7 +40,7 @@ jobs:
         uses: actions/upload-artifact@v3
         with:
           name: binary
-          path: ${{ env.BUNDLES_OUTPUT }}
+          path: ${{ env.DESTDIR }}
           if-no-files-found: error
           retention-days: 1
 
@@ -106,7 +106,7 @@ jobs:
         env:
           CONTEXT: "."
           TEST_DOCKERD: "1"
-          TEST_DOCKERD_BINARY: "./build/moby/binary-daemon/dockerd"
+          TEST_DOCKERD_BINARY: "./build/moby/dockerd"
           TESTPKGS: "./${{ matrix.pkg }}"
           TESTFLAGS: "-v --parallel=1 --timeout=30m --run=//worker=dockerd$"
         working-directory: buildkit

.github/workflows/ci.yml

@@ -15,7 +15,7 @@ on:
   pull_request:
 
 env:
-  BUNDLES_OUTPUT: ./bundles
+  DESTDIR: ./build
 
 jobs:
   validate-dco:
@@ -45,32 +45,53 @@ jobs:
         uses: docker/bake-action@v2
         with:
           targets: ${{ matrix.target }}
+      -
+        name: List artifacts
+        run: |
+          tree -nh ${{ env.DESTDIR }}
+      -
+        name: Check artifacts
+        run: |
+          find ${{ env.DESTDIR }} -type f -exec file -e ascii -- {} +
       -
         name: Upload artifacts
         uses: actions/upload-artifact@v3
         with:
           name: ${{ matrix.target }}
-          path: ${{ env.BUNDLES_OUTPUT }}
+          path: ${{ env.DESTDIR }}
           if-no-files-found: error
           retention-days: 7
 
+  prepare-cross:
+    runs-on: ubuntu-latest
+    needs:
+      - validate-dco
+    outputs:
+      matrix: ${{ steps.platforms.outputs.matrix }}
+    steps:
+      -
+        name: Checkout
+        uses: actions/checkout@v3
+      -
+        name: Create matrix
+        id: platforms
+        run: |
+          matrix="$(docker buildx bake binary-cross --print | jq -cr '.target."binary-cross".platforms')"
+          echo "matrix=$matrix" >> $GITHUB_OUTPUT
+      -
+        name: Show matrix
+        run: |
+          echo ${{ steps.platforms.outputs.matrix }}
+
   cross:
     runs-on: ubuntu-20.04
     needs:
       - validate-dco
+      - prepare-cross
     strategy:
       fail-fast: false
       matrix:
-        platform:
-          - linux/amd64
-          - linux/arm/v5
-          - linux/arm/v6
-          - linux/arm/v7
-          - linux/arm64
-          - linux/ppc64le
-          - linux/s390x
-          - windows/amd64
-          - windows/arm64
+        platform: ${{ fromJson(needs.prepare-cross.outputs.matrix) }}
     steps:
       -
         name: Checkout
@@ -89,14 +110,22 @@ jobs:
         name: Build
         uses: docker/bake-action@v2
         with:
-          targets: cross
-        env:
-          DOCKER_CROSSPLATFORMS: ${{ matrix.platform }}
+          targets: all
+          set: |
+            *.platform=${{ matrix.platform }}
+      -
+        name: List artifacts
+        run: |
+          tree -nh ${{ env.DESTDIR }}
+      -
+        name: Check artifacts
+        run: |
+          find ${{ env.DESTDIR }} -type f -exec file -e ascii -- {} +
       -
         name: Upload artifacts
         uses: actions/upload-artifact@v3
         with:
           name: cross-${{ env.PLATFORM_PAIR }}
-          path: ${{ env.BUNDLES_OUTPUT }}
+          path: ${{ env.DESTDIR }}
           if-no-files-found: error
           retention-days: 7

Dockerfile

@@ -8,7 +8,6 @@ ARG XX_VERSION=1.1.2
 ARG VPNKIT_VERSION=0.5.0
 ARG DOCKERCLI_VERSION=v17.06.2-ce
 
-ARG CROSS="false"
 ARG SYSTEMD="false"
 ARG DEBIAN_FRONTEND=noninteractive
 ARG DOCKER_STATIC=1
@@ -128,60 +127,6 @@ RUN /download-frozen-image-v2.sh /build \
         hello-world:latest@sha256:d58e752213a51785838f9eed2b7a498ffa1cb3aa7f946dda11af39286c3db9a9 \
         arm32v7/hello-world:latest@sha256:50b8560ad574c779908da71f7ce370c0a2471c098d44d1c8f6b513c5a55eeeb1
 
-FROM base AS cross-false
-
-FROM --platform=linux/amd64 base AS cross-true
-ARG DEBIAN_FRONTEND
-RUN dpkg --add-architecture arm64
-RUN dpkg --add-architecture armel
-RUN dpkg --add-architecture armhf
-RUN dpkg --add-architecture ppc64el
-RUN dpkg --add-architecture s390x
-RUN --mount=type=cache,sharing=locked,id=moby-cross-true-aptlib,target=/var/lib/apt \
-    --mount=type=cache,sharing=locked,id=moby-cross-true-aptcache,target=/var/cache/apt \
-        apt-get update && apt-get install -y --no-install-recommends \
-            crossbuild-essential-arm64 \
-            crossbuild-essential-armel \
-            crossbuild-essential-armhf \
-            crossbuild-essential-ppc64el \
-            crossbuild-essential-s390x
-
-FROM cross-${CROSS} AS dev-base
-
-FROM dev-base AS runtime-dev-cross-false
-ARG DEBIAN_FRONTEND
-RUN --mount=type=cache,sharing=locked,id=moby-cross-false-aptlib,target=/var/lib/apt \
-    --mount=type=cache,sharing=locked,id=moby-cross-false-aptcache,target=/var/cache/apt \
-        apt-get update && apt-get install -y --no-install-recommends \
-            binutils-mingw-w64 \
-            g++-mingw-w64-x86-64 \
-            libapparmor-dev \
-            libbtrfs-dev \
-            libdevmapper-dev \
-            libseccomp-dev \
-            libsystemd-dev \
-            libudev-dev
-
-FROM --platform=linux/amd64 runtime-dev-cross-false AS runtime-dev-cross-true
-ARG DEBIAN_FRONTEND
-# These crossbuild packages rely on gcc-<arch>, but this doesn't want to install
-# on non-amd64 systems, so other architectures cannot crossbuild amd64.
-RUN --mount=type=cache,sharing=locked,id=moby-cross-true-aptlib,target=/var/lib/apt \
-    --mount=type=cache,sharing=locked,id=moby-cross-true-aptcache,target=/var/cache/apt \
-        apt-get update && apt-get install -y --no-install-recommends \
-            libapparmor-dev:arm64 \
-            libapparmor-dev:armel \
-            libapparmor-dev:armhf \
-            libapparmor-dev:ppc64el \
-            libapparmor-dev:s390x \
-            libseccomp-dev:arm64 \
-            libseccomp-dev:armel \
-            libseccomp-dev:armhf \
-            libseccomp-dev:ppc64el \
-            libseccomp-dev:s390x
-
-FROM runtime-dev-cross-${CROSS} AS runtime-dev
-
 # delve
 FROM base AS delve-src
 WORKDIR /usr/src/delve
@@ -494,8 +439,44 @@ FROM containerutil-build AS containerutil-windows-amd64
 FROM containerutil-windows-${TARGETARCH} AS containerutil-windows
 FROM containerutil-${TARGETOS} AS containerutil
 
-# TODO: Some of this is only really needed for testing, it would be nice to split this up
-FROM runtime-dev AS dev-systemd-false
+FROM base AS dev-systemd-false
+COPY --from=dockercli     /build/ /usr/local/cli
+COPY --from=frozen-images /build/ /docker-frozen-images
+COPY --from=swagger       /build/ /usr/local/bin/
+COPY --from=delve         /build/ /usr/local/bin/
+COPY --from=tomll         /build/ /usr/local/bin/
+COPY --from=gowinres      /build/ /usr/local/bin/
+COPY --from=tini          /build/ /usr/local/bin/
+COPY --from=registry      /build/ /usr/local/bin/
+COPY --from=criu          /build/ /usr/local/bin/
+COPY --from=gotestsum     /build/ /usr/local/bin/
+COPY --from=golangci_lint /build/ /usr/local/bin/
+COPY --from=shfmt         /build/ /usr/local/bin/
+COPY --from=runc          /build/ /usr/local/bin/
+COPY --from=containerd    /build/ /usr/local/bin/
+COPY --from=rootlesskit   /build/ /usr/local/bin/
+COPY --from=vpnkit        /       /usr/local/bin/
+COPY --from=containerutil /build/ /usr/local/bin/
+COPY --from=crun          /build/ /usr/local/bin/
+COPY hack/dockerfile/etc/docker/  /etc/docker/
+ENV PATH=/usr/local/cli:$PATH
+WORKDIR /go/src/github.com/docker/docker
+VOLUME /var/lib/docker
+VOLUME /home/unprivilegeduser/.local/share/docker
+# Wrap all commands in the "docker-in-docker" script to allow nested containers
+ENTRYPOINT ["hack/dind"]
+
+FROM dev-systemd-false AS dev-systemd-true
+RUN --mount=type=cache,sharing=locked,id=moby-dev-aptlib,target=/var/lib/apt \
+    --mount=type=cache,sharing=locked,id=moby-dev-aptcache,target=/var/cache/apt \
+        apt-get update && apt-get install -y --no-install-recommends \
+            dbus \
+            dbus-user-session \
+            systemd \
+            systemd-sysv
+ENTRYPOINT ["hack/dind-systemd"]
+
+FROM dev-systemd-${SYSTEMD} AS dev-base
 ARG DEBIAN_FRONTEND
 RUN groupadd -r docker
 RUN useradd --create-home --gid docker unprivilegeduser \
@@ -540,115 +521,95 @@ RUN --mount=type=cache,sharing=locked,id=moby-dev-aptlib,target=/var/lib/apt \
             xz-utils \
             zip \
             zstd
-
-
 # Switch to use iptables instead of nftables (to match the CI hosts)
 # TODO use some kind of runtime auto-detection instead if/when nftables is supported (https://github.com/moby/moby/issues/26824)
 RUN update-alternatives --set iptables  /usr/sbin/iptables-legacy  || true \
  && update-alternatives --set ip6tables /usr/sbin/ip6tables-legacy || true \
  && update-alternatives --set arptables /usr/sbin/arptables-legacy || true
-
 ARG YAMLLINT_VERSION=1.27.1
 RUN pip3 install yamllint==${YAMLLINT_VERSION}
-
-COPY --from=dockercli     /build/ /usr/local/cli
-COPY --from=frozen-images /build/ /docker-frozen-images
-COPY --from=swagger       /build/ /usr/local/bin/
-COPY --from=delve         /build/ /usr/local/bin/
-COPY --from=tomll         /build/ /usr/local/bin/
-COPY --from=gowinres      /build/ /usr/local/bin/
-COPY --from=tini          /build/ /usr/local/bin/
-COPY --from=registry      /build/ /usr/local/bin/
-COPY --from=criu          /build/ /usr/local/bin/
-COPY --from=gotestsum     /build/ /usr/local/bin/
-COPY --from=golangci_lint /build/ /usr/local/bin/
-COPY --from=shfmt         /build/ /usr/local/bin/
-COPY --from=runc          /build/ /usr/local/bin/
-COPY --from=containerd    /build/ /usr/local/bin/
-COPY --from=rootlesskit   /build/ /usr/local/bin/
-COPY --from=vpnkit        /       /usr/local/bin/
-COPY --from=containerutil /build/ /usr/local/bin/
-COPY --from=crun          /build/ /usr/local/bin/
-COPY hack/dockerfile/etc/docker/  /etc/docker/
-ENV PATH=/usr/local/cli:$PATH
-ARG DOCKER_BUILDTAGS
-ENV DOCKER_BUILDTAGS="${DOCKER_BUILDTAGS}"
-WORKDIR /go/src/github.com/docker/docker
-VOLUME /var/lib/docker
-VOLUME /home/unprivilegeduser/.local/share/docker
-# Wrap all commands in the "docker-in-docker" script to allow nested containers
-ENTRYPOINT ["hack/dind"]
-
-FROM dev-systemd-false AS dev-systemd-true
 RUN --mount=type=cache,sharing=locked,id=moby-dev-aptlib,target=/var/lib/apt \
     --mount=type=cache,sharing=locked,id=moby-dev-aptcache,target=/var/cache/apt \
-        apt-get update && apt-get install -y --no-install-recommends \
-            dbus \
-            dbus-user-session \
-            systemd \
-            systemd-sysv
-ENTRYPOINT ["hack/dind-systemd"]
-
-FROM dev-systemd-${SYSTEMD} AS dev
+        apt-get update && apt-get install --no-install-recommends -y \
+            gcc \
+            pkg-config \
+            dpkg-dev \
+            libapparmor-dev \
+            libbtrfs-dev \
+            libdevmapper-dev \
+            libseccomp-dev \
+            libsecret-1-dev \
+            libsystemd-dev \
+            libudev-dev
 
-FROM runtime-dev AS binary-base
+FROM base AS build
+COPY --from=gowinres /build/ /usr/local/bin/
+WORKDIR /go/src/github.com/docker/docker
+ENV GO111MODULE=off
+ENV CGO_ENABLED=1
+ARG DEBIAN_FRONTEND
+ARG TARGETPLATFORM
+RUN --mount=type=cache,sharing=locked,id=moby-build-aptlib,target=/var/lib/apt \
+    --mount=type=cache,sharing=locked,id=moby-build-aptcache,target=/var/cache/apt \
+        xx-apt-get install --no-install-recommends -y \
+            gcc \
+            libapparmor-dev \
+            libbtrfs-dev \
+            libc6-dev \
+            libdevmapper-dev \
+            libseccomp-dev \
+            libsecret-1-dev \
+            libsystemd-dev \
+            libudev-dev
+ARG DOCKER_BUILDTAGS
+ARG DOCKER_DEBUG
 ARG DOCKER_GITCOMMIT=HEAD
-ENV DOCKER_GITCOMMIT=${DOCKER_GITCOMMIT}
+ARG DOCKER_LDFLAGS
+ARG DOCKER_STATIC
 ARG VERSION
-ENV VERSION=${VERSION}
 ARG PLATFORM
-ENV PLATFORM=${PLATFORM}
 ARG PRODUCT
-ENV PRODUCT=${PRODUCT}
 ARG DEFAULT_PRODUCT_LICENSE
-ENV DEFAULT_PRODUCT_LICENSE=${DEFAULT_PRODUCT_LICENSE}
 ARG PACKAGER_NAME
-ENV PACKAGER_NAME=${PACKAGER_NAME}
-ARG DOCKER_BUILDTAGS
-ENV DOCKER_BUILDTAGS="${DOCKER_BUILDTAGS}"
-ENV PREFIX=/build
-# TODO: This is here because hack/make.sh binary copies these extras binaries
-# from $PATH into the bundles dir.
-# It would be nice to handle this in a different way.
-COPY --from=tini          /build/ /usr/local/bin/
-COPY --from=runc          /build/ /usr/local/bin/
-COPY --from=containerd    /build/ /usr/local/bin/
-COPY --from=rootlesskit   /build/ /usr/local/bin/
-COPY --from=vpnkit        /       /usr/local/bin/
-COPY --from=containerutil /build/ /usr/local/bin/
-COPY --from=gowinres      /build/ /usr/local/bin/
-WORKDIR /go/src/github.com/docker/docker
-
-FROM binary-base AS build-binary
-RUN --mount=type=cache,target=/root/.cache \
-    --mount=type=bind,target=.,ro \
-    --mount=type=tmpfs,target=cli/winresources/dockerd \
-    --mount=type=tmpfs,target=cli/winresources/docker-proxy \
-        hack/make.sh binary
-
-FROM binary-base AS build-dynbinary
-RUN --mount=type=cache,target=/root/.cache \
-    --mount=type=bind,target=.,ro \
-    --mount=type=tmpfs,target=cli/winresources/dockerd \
-    --mount=type=tmpfs,target=cli/winresources/docker-proxy \
-        hack/make.sh dynbinary
-
-FROM binary-base AS build-cross
-ARG DOCKER_CROSSPLATFORMS
-RUN --mount=type=cache,target=/root/.cache \
-    --mount=type=bind,target=.,ro \
+# PREFIX overrides DEST dir in make.sh script otherwise it fails because of
+# read only mount in current work dir
+ENV PREFIX=/tmp
+RUN --mount=type=bind,target=. \
     --mount=type=tmpfs,target=cli/winresources/dockerd \
     --mount=type=tmpfs,target=cli/winresources/docker-proxy \
-        hack/make.sh cross
+    --mount=type=cache,target=/root/.cache/go-build,id=moby-build-$TARGETPLATFORM <<EOT
+  set -e
+  target=$([ "$DOCKER_STATIC" = "1" ] && echo "binary" || echo "dynbinary")
+  xx-go --wrap
+  ./hack/make.sh $target
+  xx-verify $([ "$DOCKER_STATIC" = "1" ] && echo "--static") /tmp/bundles/${target}-daemon/dockerd$([ "$(xx-info os)" = "windows" ] && echo ".exe")
+  xx-verify $([ "$DOCKER_STATIC" = "1" ] && echo "--static") /tmp/bundles/${target}-daemon/docker-proxy$([ "$(xx-info os)" = "windows" ] && echo ".exe")
+  mkdir /build
+  mv /tmp/bundles/${target}-daemon/* /build/
+EOT
 
+# usage:
+# > docker buildx bake binary
+# > DOCKER_STATIC=0 docker buildx bake binary
+# or
+# > make binary
+# > make dynbinary
 FROM scratch AS binary
-COPY --from=build-binary /build/bundles/ /
-
-FROM scratch AS dynbinary
-COPY --from=build-dynbinary /build/bundles/ /
-
-FROM scratch AS cross
-COPY --from=build-cross /build/bundles/ /
-
-FROM dev AS final
-COPY . /go/src/github.com/docker/docker
+COPY --from=build /build/ /
+
+# usage:
+# > docker buildx bake all
+FROM scratch AS all
+COPY --from=tini          /build/ /
+COPY --from=runc          /build/ /
+COPY --from=containerd    /build/ /
+COPY --from=rootlesskit   /build/ /
+COPY --from=containerutil /build/ /
+COPY --from=vpnkit        /       /
+COPY --from=build         /build  /
+
+# usage:
+# > make shell
+# > SYSTEMD=true make shell
+FROM dev-base AS dev
+COPY . .

Makefile

@@ -31,7 +31,6 @@ export VALIDATE_ORIGIN_BRANCH
 # make DOCKER_LDFLAGS="-X github.com/docker/docker/daemon/graphdriver.priority=overlay2,devicemapper" dynbinary
 #
 DOCKER_ENVS := \
-	-e DOCKER_CROSSPLATFORMS \
 	-e BUILD_APT_MIRROR \
 	-e BUILDFLAGS \
 	-e KEEPBUNDLE \
@@ -143,17 +142,7 @@ endif
 
 BUILD_OPTS := ${BUILD_APT_MIRROR} ${DOCKER_BUILD_ARGS} ${DOCKER_BUILD_OPTS} -f "$(DOCKERFILE)"
 BUILD_CMD := $(BUILDX) build
-
-# This is used for the legacy "build" target and anything still depending on it
-BUILD_CROSS =
-ifdef DOCKER_CROSS
-BUILD_CROSS = --build-arg CROSS=$(DOCKER_CROSS)
-endif
-ifdef DOCKER_CROSSPLATFORMS
-BUILD_CROSS = --build-arg CROSS=true
-endif
-
-VERSION_AUTOGEN_ARGS = --build-arg VERSION --build-arg DOCKER_GITCOMMIT --build-arg PRODUCT --build-arg PLATFORM --build-arg DEFAULT_PRODUCT_LICENSE --build-arg PACKAGER_NAME
+BAKE_CMD := $(BUILDX) bake
 
 default: binary
 
@@ -161,14 +150,13 @@ all: build ## validate all checks, build linux binaries, run all tests,\ncross b
 	$(DOCKER_RUN_DOCKER) bash -c 'hack/validate/default && hack/make.sh'
 
 binary: bundles ## build statically linked linux binaries
-	$(BUILD_CMD) $(BUILD_OPTS) --output=bundles/ --target=$@ $(VERSION_AUTOGEN_ARGS) .
+	$(BAKE_CMD) binary
 
 dynbinary: bundles ## build dynamically linked linux binaries
-	$(BUILD_CMD) $(BUILD_OPTS) --output=bundles/ --target=$@ $(VERSION_AUTOGEN_ARGS) .
+	$(BAKE_CMD) dynbinary
 
-cross: BUILD_OPTS += --build-arg CROSS=true --build-arg DOCKER_CROSSPLATFORMS
-cross: bundles ## cross build the binaries for darwin, freebsd and\nwindows
-	$(BUILD_CMD) $(BUILD_OPTS) --output=bundles/ --target=$@ $(VERSION_AUTOGEN_ARGS) .
+cross: bundles ## cross build the binaries
+	$(BAKE_CMD) binary-cross
 
 bundles:
 	mkdir bundles
@@ -191,18 +179,18 @@ run: build ## run the docker daemon in a container
  
 .PHONY: build
 ifeq ($(BIND_DIR), .)
-build: shell_target := --target=dev
+build: shell_target := --target=dev-base
 else
-build: shell_target := --target=final
+build: shell_target := --target=dev
 endif
 build: bundles
-	$(BUILD_CMD) $(BUILD_OPTS) $(shell_target) --load $(BUILD_CROSS) -t "$(DOCKER_IMAGE)" .
+	$(BUILD_CMD) $(BUILD_OPTS) $(shell_target) --load -t "$(DOCKER_IMAGE)" .
 
 shell: build  ## start a shell inside the build env
 	$(DOCKER_RUN_DOCKER) bash
 
 test: build test-unit ## run the unit, integration and docker-py tests
-	$(DOCKER_RUN_DOCKER) hack/make.sh dynbinary cross test-integration test-docker-py
+	$(DOCKER_RUN_DOCKER) hack/make.sh dynbinary test-integration test-docker-py
 
 test-docker-py: build ## run the docker-py tests
 	$(DOCKER_RUN_DOCKER) hack/make.sh dynbinary test-docker-py
@@ -229,8 +217,8 @@ validate: build ## validate DCO, Seccomp profile generation, gofmt,\n./pkg/ isol
 validate-%: build ## validate specific check
 	$(DOCKER_RUN_DOCKER) hack/validate/$*
 
-win: build ## cross build the binary for windows
-	$(DOCKER_RUN_DOCKER) DOCKER_CROSSPLATFORMS=windows/amd64 hack/make.sh cross
+win: bundles ## cross build the binary for windows
+	$(BAKE_CMD) --set *.platform=windows/amd64 binary
 
 .PHONY: swagger-gen
 swagger-gen:

docker-bake.hcl

@@ -1,18 +1,87 @@
-variable "BUNDLES_OUTPUT" {
-  default = "./bundles"
+variable "APT_MIRROR" {
+  default = "cdn-fastly.deb.debian.org"
+}
+variable "DOCKER_DEBUG" {
+  default = ""
 }
 variable "DOCKER_STATIC" {
   default = "1"
 }
-variable "DOCKER_CROSSPLATFORMS" {
+variable "DOCKER_LDFLAGS" {
+  default = ""
+}
+variable "DOCKER_BUILDTAGS" {
   default = ""
 }
+variable "DOCKER_GITCOMMIT" {
+  default = "HEAD"
+}
+
+# Docker version such as 23.0.0-dev. Automatically generated through Git ref.
+variable "VERSION" {
+  default = ""
+}
+
+# The platform name, such as "Docker Engine - Community".
+variable "PLATFORM" {
+  default = ""
+}
+
+# The product name, used to set version.ProductName, which is used to set
+# BuildKit's ExportedProduct variable in order to show useful error messages
+# to users when a certain version of the product doesn't support a BuildKit feature.
+variable "PRODUCT" {
+  default = ""
+}
+
+# Sets the version.DefaultProductLicense string, such as "Community Engine".
+# This field can contain a summary of the product license of the daemon if a
+# commercial license has been applied to the daemon.
+variable "DEFAULT_PRODUCT_LICENSE" {
+  default = ""
+}
+
+# The name of the packager (e.g. "Docker, Inc."). This used to set CompanyName
+# in the manifest.
+variable "PACKAGER_NAME" {
+  default = ""
+}
+
+# GITHUB_REF is the actual ref that triggers the workflow and used as version
+# when tag is pushed: https://docs.github.com/en/actions/learn-github-actions/environment-variables#default-environment-variables
+variable "GITHUB_REF" {
+  default = ""
+}
+
+# GITHUB_SHA is the commit SHA that triggered the workflow and used as commit.
+# https://docs.github.com/en/actions/learn-github-actions/environment-variables#default-environment-variables
+variable "GITHUB_SHA" {
+  default = ""
+}
+
+# Defines the output folder
+variable "DESTDIR" {
+  default = ""
+}
+function "bindir" {
+  params = [defaultdir]
+  result = DESTDIR != "" ? DESTDIR : "./bundles/${defaultdir}"
+}
 
 target "_common" {
   args = {
     BUILDKIT_CONTEXT_KEEP_GIT_DIR = 1
-    APT_MIRROR = "cdn-fastly.deb.debian.org"
+    APT_MIRROR = APT_MIRROR
+    DOCKER_DEBUG = DOCKER_DEBUG
     DOCKER_STATIC = DOCKER_STATIC
+    DOCKER_LDFLAGS = DOCKER_LDFLAGS
+    DOCKER_BUILDTAGS = DOCKER_BUILDTAGS
+    DOCKER_GITCOMMIT = DOCKER_GITCOMMIT != "" ? DOCKER_GITCOMMIT : GITHUB_SHA
+    VERSION = VERSION != "" ? VERSION : GITHUB_REF
+    PLATFORM = PLATFORM
+    PRODUCT = PRODUCT
+    DEFAULT_PRODUCT_LICENSE = DEFAULT_PRODUCT_LICENSE
+    PACKAGER_NAME = PACKAGER_NAME
   }
 }
 
@@ -20,43 +89,69 @@ group "default" {
   targets = ["binary"]
 }
 
+target "_platforms" {
+  platforms = [
+    "linux/amd64",
+    "linux/arm/v5",
+    "linux/arm/v6",
+    "linux/arm/v7",
+    "linux/arm64",
+    "linux/ppc64le",
+    "linux/s390x",
+    "windows/amd64"
+  ]
+}
+
+#
+# build dockerd and docker-proxy
+#
+
 target "binary" {
   inherits = ["_common"]
   target = "binary"
-  output = [BUNDLES_OUTPUT]
+  output = [bindir(DOCKER_STATIC == "1" ? "binary" : "dynbinary")]
 }
 
 target "dynbinary" {
   inherits = ["binary"]
-  target = "dynbinary"
-}
-
-target "cross" {
-  inherits = ["binary"]
+  output = [bindir("dynbinary")]
   args = {
-    CROSS = "true"
-    DOCKER_CROSSPLATFORMS = DOCKER_CROSSPLATFORMS
+    DOCKER_STATIC = "0"
   }
-  target = "cross"
+}
+
+target "binary-cross" {
+  inherits = ["binary", "_platforms"]
 }
 
 #
-# dev
+# same as binary but with extra tools as well (containerd, runc, ...)
 #
 
-variable "DEV_IMAGE" {
-  default = "docker-dev"
+target "all" {
+  inherits = ["_common"]
+  target = "all"
+  output = [bindir(DOCKER_STATIC == "1" ? "binary" : "dynbinary")]
+}
+
+target "all-cross" {
+  inherits = ["all", "_platforms"]
 }
+
+#
+# dev
+#
+
 variable "SYSTEMD" {
   default = "false"
 }
 
 target "dev" {
   inherits = ["_common"]
-  target = "final"
+  target = "dev"
   args = {
     SYSTEMD = SYSTEMD
   }
-  tags = [DEV_IMAGE]
+  tags = ["docker-dev"]
   output = ["type=docker"]
 }

dockerversion/version_lib.go

@@ -6,7 +6,6 @@ var (
 	GitCommit             = "library-import"
 	Version               = "library-import"
 	BuildTime             = "library-import"
-	IAmStatic             = "library-import"
 	PlatformName          = ""
 	ProductName           = ""
 	DefaultProductLicense = ""

docs/contributing/README.md

@@ -6,5 +6,6 @@
  * (Optional) [Configure project for IDE](set-up-ide.md)
  * [Configure Git for contributing](set-up-git.md)
  * [Work with a development container](set-up-dev-env.md)
+ * [Containerized build and cross compilation](ctn-build.md)
  * [Run tests and test documentation](test.md)
  * [Debugging the daemon](debug.md)

docs/contributing/ctn-build.md

@@ -0,0 +1,28 @@
+The `Dockerfile` supports building and cross compiling docker daemon and extra
+tools using [Docker Buildx](https://github.com/docker/buildx) and [BuildKit](https://github.com/moby/buildkit).
+A [bake definition](https://docs.docker.com/build/bake/file-definition/) named
+`docker-bake.hcl` is in place to ease the build process:
+
+```shell
+# build binaries for the current host platform
+# output to ./bundles/binary-daemon by default
+docker buildx bake
+# or
+docker buildx bake binary
+
+# build binaries for the current host platform
+# output to ./bin
+DESTDIR=./bin docker buildx bake
+
+# build dynamically linked binaries
+# output to ./bundles/dynbinary-daemon by default
+DOCKER_STATIC=0 docker buildx bake
+# or
+docker buildx bake dynbinary
+
+# build binaries for all supported platforms
+docker buildx bake binary-cross
+
+# build binaries for a specific platform
+docker buildx bake --set *.platform=linux/arm64
+```

docs/contributing/set-up-dev-env.md

@@ -150,10 +150,10 @@ can take over 15 minutes to complete.
    Removing bundles/
 
    ---> Making bundle: binary (in bundles/binary)
-   Building: bundles/binary-daemon/dockerd-17.06.0-dev
-   Created binary: bundles/binary-daemon/dockerd-17.06.0-dev
-   Copying nested executables into bundles/binary-daemon
-
+   Building bundles/binary-daemon/dockerd (linux/amd64)...
+   Created binary: bundles/binary-daemon/dockerd
+   Building bundles/binary-daemon/docker-proxy (linux/amd64)...
+   Created binary:bundles/binary-daemon/docker-proxy
    ```
 
 7. Run `make install`, which copies the binary to the container's

docs/contributing/test.md

@@ -123,7 +123,7 @@ Try this now.
 4.  Run the tests using the `hack/make.sh` script.
 
     ```bash
-    # hack/make.sh dynbinary binary cross test-integration test-docker-py
+    # hack/make.sh dynbinary binary test-integration test-docker-py
     ```
 
     The tests run just as they did within your local host.
@@ -132,11 +132,11 @@ Try this now.
     just the integration tests:
 
     ```bash
-    # hack/make.sh dynbinary binary cross test-integration
+    # hack/make.sh dynbinary binary test-integration
     ```
 
     Most test targets require that you build these precursor targets first:
-    `dynbinary binary cross`
+    `dynbinary binary`
 
 
 ## Run unit tests

hack/README.md

@@ -44,7 +44,7 @@ all of the tests.
 - When running inside a Docker development container, `hack/make.sh` does
 not have a single target that runs all the tests. You need to provide a
 single command line with multiple targets that performs the same thing.
-An example referenced from [Run targets inside a development container](https://docs.docker.com/opensource/project/test-and-docs/#run-targets-inside-a-development-container): `root@5f8630b873fe:/go/src/github.com/moby/moby# hack/make.sh dynbinary binary cross test-unit test-integration test-docker-py`
+An example referenced from [Run targets inside a development container](https://docs.docker.com/opensource/project/test-and-docs/#run-targets-inside-a-development-container): `root@5f8630b873fe:/go/src/github.com/moby/moby# hack/make.sh dynbinary binary test-unit test-integration test-docker-py`
 - For more information related to testing outside the scope of this README,
 refer to
 [Run tests and test documentation](https://docs.docker.com/opensource/project/test-and-docs/)

hack/make.sh

@@ -36,10 +36,17 @@ DEFAULT_BUNDLES=(
 	dynbinary
 	test-integration
 	test-docker-py
-	cross
 )
 
 VERSION=${VERSION:-dev}
+if [[ $VERSION == refs/tags/* ]]; then
+	VERSION=${VERSION#refs/tags/}
+elif [[ $VERSION == refs/heads/* ]]; then
+	VERSION=$(sed <<< "${VERSION#refs/heads/}" -r 's#/+#-#g')
+elif [[ $VERSION == refs/pull/* ]]; then
+	VERSION=pr-$(grep <<< "$VERSION" -o '[0-9]\+')
+fi
+
 ! BUILDTIME=$(date -u -d "@${SOURCE_DATE_EPOCH:-$(date +%s)}" --rfc-3339 ns 2> /dev/null | sed -e 's/ /T/')
 if [ "$DOCKER_GITCOMMIT" ]; then
 	GITCOMMIT="$DOCKER_GITCOMMIT"
@@ -91,17 +98,12 @@ fi
 # functionality. We favour libdm_dlsym_deferred_remove over
 # libdm_no_deferred_remove in dynamic cases because the binary could be shipped
 # with a newer libdevmapper than the one it was built with.
-if
-	command -v gcc &> /dev/null \
-		&& ! (echo -e '#include <libdevmapper.h>\nint main() { dm_task_deferred_remove(NULL); }' | gcc -xc - -o /dev/null $(pkg-config --libs devmapper) &> /dev/null) \
-		;
-then
+if command -v gcc &> /dev/null && ! (echo -e '#include <libdevmapper.h>\nint main() { dm_task_deferred_remove(NULL); }' | gcc -xc - -o /dev/null $(${PKG_CONFIG} --libs devmapper 2> /dev/null) &> /dev/null); then
 	add_buildtag libdm dlsym_deferred_remove
 fi
 
 # Use these flags when compiling the tests and final binary
 
-IAMSTATIC='true'
 if [ -z "$DOCKER_DEBUG" ]; then
 	LDFLAGS='-w'
 fi

hack/make/.binary

@@ -16,72 +16,84 @@ source "${MAKEDIR}/.go-autogen"
 (
 	export GOGC=${DOCKER_BUILD_GOGC:-1000}
 
-	if [ "$(go env GOOS)/$(go env GOARCH)" != "$(go env GOHOSTOS)/$(go env GOHOSTARCH)" ]; then
-		# must be cross-compiling!
+	# for non-sandboxed invocation
+	if ! command -v xx-go > /dev/null 2>&1; then
+		if [ "$(go env GOOS)/$(go env GOARCH)" != "$(go env GOHOSTOS)/$(go env GOHOSTARCH)" ]; then
+			# must be cross-compiling!
+			case "$(go env GOOS)/$(go env GOARCH)" in
+				windows/amd64)
+					export CC="${CC:-x86_64-w64-mingw32-gcc}"
+					export CGO_ENABLED=1
+					;;
+				linux/arm)
+					case "${GOARM}" in
+						5)
+							export CC="${CC:-arm-linux-gnueabi-gcc}"
+							export CGO_ENABLED=1
+							export CGO_CFLAGS="-march=armv5t"
+							export CGO_CXXFLAGS="-march=armv5t"
+							;;
+						6)
+							export CC="${CC:-arm-linux-gnueabi-gcc}"
+							export CGO_ENABLED=1
+							export CGO_CFLAGS="-march=armv6"
+							export CGO_CXXFLAGS="-march=armv6"
+							;;
+						7)
+							export CC="${CC:-arm-linux-gnueabihf-gcc}"
+							export CGO_ENABLED=1
+							export CGO_CFLAGS="-march=armv7-a"
+							export CGO_CXXFLAGS="-march=armv7-a"
+							;;
+						*)
+							export CC="${CC:-arm-linux-gnueabihf-gcc}"
+							export CGO_ENABLED=1
+							;;
+					esac
+					;;
+				linux/arm64)
+					export CC="${CC:-aarch64-linux-gnu-gcc}"
+					export CGO_ENABLED=1
+					;;
+				linux/amd64)
+					export CC="${CC:-x86_64-linux-gnu-gcc}"
+					export CGO_ENABLED=1
+					;;
+				linux/ppc64le)
+					export CC="${CC:-powerpc64le-linux-gnu-gcc}"
+					export CGO_ENABLED=1
+					;;
+				linux/s390x)
+					export CC="${CC:-s390x-linux-gnu-gcc}"
+					export CGO_ENABLED=1
+					;;
+			esac
+		fi
+	fi
+
+	# -buildmode=pie is not supported on Windows arm64 and Linux mips*, ppc64be
+	# https://github.com/golang/go/blob/go1.19.4/src/cmd/internal/sys/supported.go#L125-L132
+	if ! [ "$DOCKER_STATIC" = "1" ]; then
 		case "$(go env GOOS)/$(go env GOARCH)" in
-			windows/amd64)
-				export CC="${CC:-x86_64-w64-mingw32-gcc}"
-				export CGO_ENABLED=1
-				;;
-			linux/arm)
-				case "${GOARM}" in
-					5)
-						export CC="${CC:-arm-linux-gnueabi-gcc}"
-						export CGO_ENABLED=1
-						export CGO_CFLAGS="-march=armv5t"
-						export CGO_CXXFLAGS="-march=armv5t"
-						;;
-					6)
-						export CC="${CC:-arm-linux-gnueabi-gcc}"
-						export CGO_ENABLED=1
-						export CGO_CFLAGS="-march=armv6"
-						export CGO_CXXFLAGS="-march=armv6"
-						;;
-					7)
-						export CC="${CC:-arm-linux-gnueabihf-gcc}"
-						export CGO_ENABLED=1
-						export CGO_CFLAGS="-march=armv7-a"
-						export CGO_CXXFLAGS="-march=armv7-a"
-						;;
-					*)
-						export CC="${CC:-arm-linux-gnueabihf-gcc}"
-						export CGO_ENABLED=1
-						;;
-				esac
-				;;
-			linux/arm64)
-				export CC="${CC:-aarch64-linux-gnu-gcc}"
-				export CGO_ENABLED=1
-				;;
-			linux/amd64)
-				export CC="${CC:-x86_64-linux-gnu-gcc}"
-				export CGO_ENABLED=1
-				;;
-			linux/ppc64le)
-				export CC="${CC:-powerpc64le-linux-gnu-gcc}"
-				export CGO_ENABLED=1
-				;;
-			linux/s390x)
-				export CC="${CC:-s390x-linux-gnu-gcc}"
-				export CGO_ENABLED=1
+			windows/arm64 | linux/mips* | linux/ppc64) ;;
+			*)
+				BUILDFLAGS+=("-buildmode=pie")
 				;;
 		esac
 	fi
 
-	# -buildmode=pie is not supported on Windows and Linux on mips, riscv64 and ppc64be.
-	# https://github.com/golang/go/blob/77aa209b386a184e7f4b44938f2a05a1b5c5a3cf/src/cmd/internal/sys/supported.go#L89-L99
-	case "$(go env GOOS)/$(go env GOARCH)" in
-		windows/* | linux/mips* | linux/riscv* | linux/ppc64) ;;
-		# TODO remove windows in Go 1.15+: https://github.com/golang/go/commit/95f382139043059a2a0780ba577b53893408f7e4
-		# TODO remove riscv64 in Go 1.16+: https://github.com/golang/go/commit/8eb846fd37eb7bded8a1cf6932be2c59069863e5
-
-		*)
-			BUILDFLAGS+=("-buildmode=pie")
-			;;
-	esac
+	# only necessary for non-sandboxed invocation where TARGETPLATFORM is empty
+	PLATFORM_NAME=$TARGETPLATFORM
+	if [ -z "$PLATFORM_NAME" ]; then
+		PLATFORM_NAME="$(go env GOOS)/$(go env GOARCH)"
+		if [ -n "$(go env GOARM)" ]; then
+			PLATFORM_NAME+="/$(go env GOARM)"
+		elif [ -n "$(go env GOAMD64)" ] && [ "$(go env GOAMD64)" != "v1" ]; then
+			PLATFORM_NAME+="/$(go env GOAMD64)"
+		fi
+	fi
 
-	echo "Building: $DEST/$BINARY_FULLNAME"
-	echo "GOOS=\"${GOOS}\" GOARCH=\"${GOARCH}\" GOARM=\"${GOARM}\""
+	echo "Building $([ "$DOCKER_STATIC" = "1" ] && echo "static" || echo "dynamic") $DEST/$BINARY_FULLNAME ($PLATFORM_NAME)..."
 	go build \
 		-o "$DEST/$BINARY_FULLNAME" \
 		"${BUILDFLAGS[@]}" \

hack/make/.go-autogen

@@ -4,7 +4,6 @@ LDFLAGS="${LDFLAGS} \
 	-X \"github.com/docker/docker/dockerversion.Version=${VERSION}\" \
 	-X \"github.com/docker/docker/dockerversion.GitCommit=${GITCOMMIT}\" \
 	-X \"github.com/docker/docker/dockerversion.BuildTime=${BUILDTIME}\" \
-	-X \"github.com/docker/docker/dockerversion.IAmStatic=${IAMSTATIC:-true}\" \
 	-X \"github.com/docker/docker/dockerversion.PlatformName=${PLATFORM}\" \
 	-X \"github.com/docker/docker/dockerversion.ProductName=${PRODUCT}\" \
 	-X \"github.com/docker/docker/dockerversion.DefaultProductLicense=${DEFAULT_PRODUCT_LICENSE}\" \

hack/make/binary-daemon

@@ -1,35 +1,11 @@
 #!/usr/bin/env bash
 set -e
 
-copy_binaries() {
-	local dir="$1"
-
-	# Add nested executables to bundle dir so we have complete set of
-	# them available, but only if the native OS/ARCH is the same as the
-	# OS/ARCH of the build target
-	if [ "$(go env GOOS)/$(go env GOARCH)" != "$(go env GOHOSTOS)/$(go env GOHOSTARCH)" ]; then
-		return
-	fi
-	if [ ! -x /usr/local/bin/runc ]; then
-		return
-	fi
-	echo "Copying nested executables into $dir"
-	for file in containerd containerd-shim-runc-v2 ctr runc docker-init rootlesskit rootlesskit-docker-proxy dockerd-rootless.sh dockerd-rootless-setuptool.sh; do
-		cp -f "$(command -v "$file")" "$dir/"
-	done
-	# vpnkit might not be available for the target platform, see vpnkit stage in
-	# the Dockerfile for more information.
-	if command -v vpnkit > /dev/null 2>&1; then
-		cp -f "$(command -v vpnkit)" "$dir/"
-	fi
-}
-
 [ -z "$KEEPDEST" ] && rm -rf "$DEST"
 
 (
+	DOCKER_STATIC=1
 	GO_PACKAGE='github.com/docker/docker/cmd/dockerd'
 	BINARY_NAME='dockerd'
-
 	source "${MAKEDIR}/.binary"
-	copy_binaries "$DEST"
 )

hack/make/binary-proxy

@@ -5,8 +5,8 @@ set -e
 (
 	export CGO_ENABLED=0
 
+	DOCKER_STATIC=1
 	GO_PACKAGE='github.com/docker/docker/cmd/docker-proxy'
 	BINARY_NAME='docker-proxy'
-
 	source "${MAKEDIR}/.binary"
 )

hack/make/containerutility

@@ -1,20 +0,0 @@
-#!/usr/bin/env bash
-set -e
-
-: "${CONTAINER_UTILITY_COMMIT:=aa1ba87e99b68e0113bd27ec26c60b88f9d4ccd9}"
-
-(
-	git clone https://github.com/docker/windows-container-utility.git "$GOPATH/src/github.com/docker/windows-container-utility"
-	cd "$GOPATH/src/github.com/docker/windows-container-utility"
-	git checkout -q "$CONTAINER_UTILITY_COMMIT"
-
-	echo Building: ${DEST}/containerutility.exe
-
-	(
-		make
-	)
-
-	mkdir -p ${ABS_DEST}
-
-	cp containerutility.exe ${ABS_DEST}/containerutility.exe
-)

hack/make/cross

@@ -1,37 +0,0 @@
-#!/usr/bin/env bash
-set -e
-
-# if we have our linux/amd64 version compiled, let's symlink it in
-if [ -x "${DEST}/../binary-daemon/dockerd" ]; then
-	arch=$(go env GOHOSTARCH)
-	mkdir -p "$DEST/linux/${arch}"
-	(
-		cd "${DEST}/linux/${arch}"
-		ln -sf ../../../binary-daemon/* ./
-	)
-	echo "Created symlinks:" "${DEST}/linux/${arch}/"*
-fi
-
-DOCKER_CROSSPLATFORMS=${DOCKER_CROSSPLATFORMS:-"linux/amd64 windows/amd64 linux/ppc64le linux/s390x"}
-
-for platform in ${DOCKER_CROSSPLATFORMS}; do
-	(
-		export KEEPDEST=1
-		export DEST="${DEST}/${platform}" # bundles/VERSION/cross/GOOS/GOARCH/docker-VERSION
-		export GOOS=${platform%%/*}
-		export GOARCH=${platform#*/}
-
-		if [[ "${GOARCH}" = "arm/"* ]]; then
-			GOARM=${GOARCH##*/v}
-			GOARCH=${GOARCH%/v*}
-			export GOARM
-		fi
-
-		echo "Cross building: ${DEST}"
-		mkdir -p "${DEST}"
-		ABS_DEST="$(cd "${DEST}" && pwd -P)"
-		source "${MAKEDIR}/binary"
-
-		source "${MAKEDIR}/cross-platform-dependent"
-	)
-done

hack/make/cross-platform-dependent

@@ -1,6 +0,0 @@
-#!/usr/bin/env bash
-set -e
-
-if [ ${platform} == "windows/amd64" ]; then
-	source "${MAKEDIR}/containerutility"
-fi

hack/make/dynbinary-daemon

@@ -4,7 +4,6 @@ set -e
 [ -z "$KEEPDEST" ] && rm -rf "$DEST"
 
 (
-	export IAMSTATIC='false'
 	export LDFLAGS_STATIC_DOCKER=''
 	export BUILDFLAGS=("${BUILDFLAGS[@]/netgo /}")        # disable netgo, since we don't need it for a dynamic binary
 	export BUILDFLAGS=("${BUILDFLAGS[@]/osusergo /}")     # ditto for osusergo

hack/make/dynbinary-proxy

@@ -3,7 +3,6 @@
 set -e
 
 (
-	export IAMSTATIC='false'
 	export LDFLAGS_STATIC_DOCKER=''
 	export BUILDFLAGS=("${BUILDFLAGS[@]/netgo /}")        # disable netgo, since we don't need it for a dynamic binary
 	export BUILDFLAGS=("${BUILDFLAGS[@]/osusergo /}")     # ditto for osusergo