From a85bdbe1caa018d0ce1baeaec8cd418655c31ee5 Mon Sep 17 00:00:00 2001
From: CrazyMax <crazy-max@users.noreply.github.com>
Date: Sun, 1 Jan 2023 17:47:17 +0100
Subject: [PATCH 01/13] Dockerfile: use TARGETPLATFORM to build Docker

Better support for cross compilation so we can fully rely
on `--platform` flag of buildx for a seamless integration.

This removes unnecessary extra cross logic in the Dockerfile,
DOCKER_CROSSPLATFORMS and CROSS vars and some hack scripts as well.

Non-sandboxed build invocation is still supported and dev stages
in the Dockerfile have been updated accordingly.

Bake definition and GitHub Actions workflows have been updated
accordingly as well.

Signed-off-by: CrazyMax <crazy-max@users.noreply.github.com>
(cherry picked from commit 8086f4012330d1c1058e07fc4e5e4522dd432c20)
---
 .github/workflows/ci.yml            |  39 +++--
 Dockerfile                          | 260 +++++++++++-----------------
 Makefile                            |  30 +---
 docker-bake.hcl                     |  33 ++--
 docs/contributing/set-up-dev-env.md |   8 +-
 docs/contributing/test.md           |   6 +-
 hack/README.md                      |   2 +-
 hack/make.sh                        |   1 -
 hack/make/.binary                   | 117 +++++++------
 hack/make/binary-daemon             |  25 ---
 hack/make/cross                     |  37 ----
 hack/make/cross-platform-dependent  |   6 -
 12 files changed, 235 insertions(+), 329 deletions(-)
 delete mode 100644 hack/make/cross
 delete mode 100644 hack/make/cross-platform-dependent

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 7f2c7821ff..a34abc2425 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -54,23 +54,36 @@ jobs:
           if-no-files-found: error
           retention-days: 7
 
+  prepare-cross:
+    runs-on: ubuntu-latest
+    needs:
+      - validate-dco
+    outputs:
+      matrix: ${{ steps.platforms.outputs.matrix }}
+    steps:
+      -
+        name: Checkout
+        uses: actions/checkout@v3
+      -
+        name: Create matrix
+        id: platforms
+        run: |
+          matrix="$(docker buildx bake binary-cross --print | jq -cr '.target."binary-cross".platforms')"
+          echo "matrix=$matrix" >> $GITHUB_OUTPUT
+      -
+        name: Show matrix
+        run: |
+          echo ${{ steps.platforms.outputs.matrix }}
+
   cross:
     runs-on: ubuntu-20.04
     needs:
       - validate-dco
+      - prepare-cross
     strategy:
       fail-fast: false
       matrix:
-        platform:
-          - linux/amd64
-          - linux/arm/v5
-          - linux/arm/v6
-          - linux/arm/v7
-          - linux/arm64
-          - linux/ppc64le
-          - linux/s390x
-          - windows/amd64
-          - windows/arm64
+        platform: ${{ fromJson(needs.prepare-cross.outputs.matrix) }}
     steps:
       -
         name: Checkout
@@ -89,9 +102,9 @@ jobs:
         name: Build
         uses: docker/bake-action@v2
         with:
-          targets: cross
-        env:
-          DOCKER_CROSSPLATFORMS: ${{ matrix.platform }}
+          targets: binary
+          set: |
+            *.platform=${{ matrix.platform }}
       -
         name: Upload artifacts
         uses: actions/upload-artifact@v3
diff --git a/Dockerfile b/Dockerfile
index 0777e02661..e830658774 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -8,7 +8,6 @@ ARG XX_VERSION=1.1.2
 ARG VPNKIT_VERSION=0.5.0
 ARG DOCKERCLI_VERSION=v17.06.2-ce
 
-ARG CROSS="false"
 ARG SYSTEMD="false"
 ARG DEBIAN_FRONTEND=noninteractive
 ARG DOCKER_STATIC=1
@@ -128,60 +127,6 @@ RUN /download-frozen-image-v2.sh /build \
         hello-world:latest@sha256:d58e752213a51785838f9eed2b7a498ffa1cb3aa7f946dda11af39286c3db9a9 \
         arm32v7/hello-world:latest@sha256:50b8560ad574c779908da71f7ce370c0a2471c098d44d1c8f6b513c5a55eeeb1
 
-FROM base AS cross-false
-
-FROM --platform=linux/amd64 base AS cross-true
-ARG DEBIAN_FRONTEND
-RUN dpkg --add-architecture arm64
-RUN dpkg --add-architecture armel
-RUN dpkg --add-architecture armhf
-RUN dpkg --add-architecture ppc64el
-RUN dpkg --add-architecture s390x
-RUN --mount=type=cache,sharing=locked,id=moby-cross-true-aptlib,target=/var/lib/apt \
-    --mount=type=cache,sharing=locked,id=moby-cross-true-aptcache,target=/var/cache/apt \
-        apt-get update && apt-get install -y --no-install-recommends \
-            crossbuild-essential-arm64 \
-            crossbuild-essential-armel \
-            crossbuild-essential-armhf \
-            crossbuild-essential-ppc64el \
-            crossbuild-essential-s390x
-
-FROM cross-${CROSS} AS dev-base
-
-FROM dev-base AS runtime-dev-cross-false
-ARG DEBIAN_FRONTEND
-RUN --mount=type=cache,sharing=locked,id=moby-cross-false-aptlib,target=/var/lib/apt \
-    --mount=type=cache,sharing=locked,id=moby-cross-false-aptcache,target=/var/cache/apt \
-        apt-get update && apt-get install -y --no-install-recommends \
-            binutils-mingw-w64 \
-            g++-mingw-w64-x86-64 \
-            libapparmor-dev \
-            libbtrfs-dev \
-            libdevmapper-dev \
-            libseccomp-dev \
-            libsystemd-dev \
-            libudev-dev
-
-FROM --platform=linux/amd64 runtime-dev-cross-false AS runtime-dev-cross-true
-ARG DEBIAN_FRONTEND
-# These crossbuild packages rely on gcc-<arch>, but this doesn't want to install
-# on non-amd64 systems, so other architectures cannot crossbuild amd64.
-RUN --mount=type=cache,sharing=locked,id=moby-cross-true-aptlib,target=/var/lib/apt \
-    --mount=type=cache,sharing=locked,id=moby-cross-true-aptcache,target=/var/cache/apt \
-        apt-get update && apt-get install -y --no-install-recommends \
-            libapparmor-dev:arm64 \
-            libapparmor-dev:armel \
-            libapparmor-dev:armhf \
-            libapparmor-dev:ppc64el \
-            libapparmor-dev:s390x \
-            libseccomp-dev:arm64 \
-            libseccomp-dev:armel \
-            libseccomp-dev:armhf \
-            libseccomp-dev:ppc64el \
-            libseccomp-dev:s390x
-
-FROM runtime-dev-cross-${CROSS} AS runtime-dev
-
 # delve
 FROM base AS delve-src
 WORKDIR /usr/src/delve
@@ -494,8 +439,47 @@ FROM containerutil-build AS containerutil-windows-amd64
 FROM containerutil-windows-${TARGETARCH} AS containerutil-windows
 FROM containerutil-${TARGETOS} AS containerutil
 
-# TODO: Some of this is only really needed for testing, it would be nice to split this up
-FROM runtime-dev AS dev-systemd-false
+FROM base AS dev-systemd-false
+COPY --from=dockercli     /build/ /usr/local/cli
+COPY --from=frozen-images /build/ /docker-frozen-images
+COPY --from=swagger       /build/ /usr/local/bin/
+COPY --from=delve         /build/ /usr/local/bin/
+COPY --from=tomll         /build/ /usr/local/bin/
+COPY --from=gowinres      /build/ /usr/local/bin/
+COPY --from=tini          /build/ /usr/local/bin/
+COPY --from=registry      /build/ /usr/local/bin/
+COPY --from=criu          /build/ /usr/local/bin/
+COPY --from=gotestsum     /build/ /usr/local/bin/
+COPY --from=golangci_lint /build/ /usr/local/bin/
+COPY --from=shfmt         /build/ /usr/local/bin/
+COPY --from=runc          /build/ /usr/local/bin/
+COPY --from=containerd    /build/ /usr/local/bin/
+COPY --from=rootlesskit   /build/ /usr/local/bin/
+COPY --from=vpnkit        /       /usr/local/bin/
+COPY --from=containerutil /build/ /usr/local/bin/
+COPY --from=crun          /build/ /usr/local/bin/
+COPY hack/dockerfile/etc/docker/  /etc/docker/
+ENV PATH=/usr/local/cli:$PATH
+WORKDIR /go/src/github.com/docker/docker
+VOLUME /var/lib/docker
+VOLUME /home/unprivilegeduser/.local/share/docker
+# Wrap all commands in the "docker-in-docker" script to allow nested containers
+ENTRYPOINT ["hack/dind"]
+
+FROM dev-systemd-false AS dev-systemd-true
+RUN --mount=type=cache,sharing=locked,id=moby-dev-aptlib,target=/var/lib/apt \
+    --mount=type=cache,sharing=locked,id=moby-dev-aptcache,target=/var/cache/apt \
+        apt-get update && apt-get install -y --no-install-recommends \
+            dbus \
+            dbus-user-session \
+            systemd \
+            systemd-sysv
+RUN mkdir -p hack \
+  && curl -o hack/dind-systemd https://raw.githubusercontent.com/AkihiroSuda/containerized-systemd/b70bac0daeea120456764248164c21684ade7d0d/docker-entrypoint.sh \
+  && chmod +x hack/dind-systemd
+ENTRYPOINT ["hack/dind-systemd"]
+
+FROM dev-systemd-${SYSTEMD} AS dev
 ARG DEBIAN_FRONTEND
 RUN groupadd -r docker
 RUN useradd --create-home --gid docker unprivilegeduser \
@@ -539,118 +523,84 @@ RUN --mount=type=cache,sharing=locked,id=moby-dev-aptlib,target=/var/lib/apt \
             xz-utils \
             zip \
             zstd
-
-
 # Switch to use iptables instead of nftables (to match the CI hosts)
 # TODO use some kind of runtime auto-detection instead if/when nftables is supported (https://github.com/moby/moby/issues/26824)
 RUN update-alternatives --set iptables  /usr/sbin/iptables-legacy  || true \
  && update-alternatives --set ip6tables /usr/sbin/ip6tables-legacy || true \
  && update-alternatives --set arptables /usr/sbin/arptables-legacy || true
-
 ARG YAMLLINT_VERSION=1.27.1
 RUN pip3 install yamllint==${YAMLLINT_VERSION}
-
-COPY --from=dockercli     /build/ /usr/local/cli
-COPY --from=frozen-images /build/ /docker-frozen-images
-COPY --from=swagger       /build/ /usr/local/bin/
-COPY --from=delve         /build/ /usr/local/bin/
-COPY --from=tomll         /build/ /usr/local/bin/
-COPY --from=gowinres      /build/ /usr/local/bin/
-COPY --from=tini          /build/ /usr/local/bin/
-COPY --from=registry      /build/ /usr/local/bin/
-COPY --from=criu          /build/ /usr/local/bin/
-COPY --from=gotestsum     /build/ /usr/local/bin/
-COPY --from=golangci_lint /build/ /usr/local/bin/
-COPY --from=shfmt         /build/ /usr/local/bin/
-COPY --from=runc          /build/ /usr/local/bin/
-COPY --from=containerd    /build/ /usr/local/bin/
-COPY --from=rootlesskit   /build/ /usr/local/bin/
-COPY --from=vpnkit        /       /usr/local/bin/
-COPY --from=containerutil /build/ /usr/local/bin/
-COPY --from=crun          /build/ /usr/local/bin/
-COPY hack/dockerfile/etc/docker/  /etc/docker/
-ENV PATH=/usr/local/cli:$PATH
-ARG DOCKER_BUILDTAGS
-ENV DOCKER_BUILDTAGS="${DOCKER_BUILDTAGS}"
-WORKDIR /go/src/github.com/docker/docker
-VOLUME /var/lib/docker
-VOLUME /home/unprivilegeduser/.local/share/docker
-# Wrap all commands in the "docker-in-docker" script to allow nested containers
-ENTRYPOINT ["hack/dind"]
-
-FROM dev-systemd-false AS dev-systemd-true
 RUN --mount=type=cache,sharing=locked,id=moby-dev-aptlib,target=/var/lib/apt \
     --mount=type=cache,sharing=locked,id=moby-dev-aptcache,target=/var/cache/apt \
-        apt-get update && apt-get install -y --no-install-recommends \
-            dbus \
-            dbus-user-session \
-            systemd \
-            systemd-sysv
-RUN mkdir -p hack \
-  && curl -o hack/dind-systemd https://raw.githubusercontent.com/AkihiroSuda/containerized-systemd/b70bac0daeea120456764248164c21684ade7d0d/docker-entrypoint.sh \
-  && chmod +x hack/dind-systemd
-ENTRYPOINT ["hack/dind-systemd"]
+        apt-get update && apt-get install --no-install-recommends -y \
+            gcc \
+            pkg-config \
+            dpkg-dev \
+            libapparmor-dev \
+            libbtrfs-dev \
+            libdevmapper-dev \
+            libseccomp-dev \
+            libsecret-1-dev \
+            libsystemd-dev \
+            libudev-dev
 
-FROM dev-systemd-${SYSTEMD} AS dev
-
-FROM runtime-dev AS binary-base
-ARG DOCKER_GITCOMMIT=HEAD
-ENV DOCKER_GITCOMMIT=${DOCKER_GITCOMMIT}
-ARG VERSION
-ENV VERSION=${VERSION}
-ARG PLATFORM
-ENV PLATFORM=${PLATFORM}
-ARG PRODUCT
-ENV PRODUCT=${PRODUCT}
-ARG DEFAULT_PRODUCT_LICENSE
-ENV DEFAULT_PRODUCT_LICENSE=${DEFAULT_PRODUCT_LICENSE}
-ARG PACKAGER_NAME
-ENV PACKAGER_NAME=${PACKAGER_NAME}
-ARG DOCKER_BUILDTAGS
-ENV DOCKER_BUILDTAGS="${DOCKER_BUILDTAGS}"
-ENV PREFIX=/build
-# TODO: This is here because hack/make.sh binary copies these extras binaries
-# from $PATH into the bundles dir.
-# It would be nice to handle this in a different way.
-COPY --from=tini          /build/ /usr/local/bin/
-COPY --from=runc          /build/ /usr/local/bin/
-COPY --from=containerd    /build/ /usr/local/bin/
-COPY --from=rootlesskit   /build/ /usr/local/bin/
-COPY --from=vpnkit        /       /usr/local/bin/
-COPY --from=containerutil /build/ /usr/local/bin/
-COPY --from=gowinres      /build/ /usr/local/bin/
+FROM base AS build
+COPY --from=gowinres /build/ /usr/local/bin/
 WORKDIR /go/src/github.com/docker/docker
-
-FROM binary-base AS build-binary
-RUN --mount=type=cache,target=/root/.cache \
-    --mount=type=bind,target=.,ro \
+ENV GO111MODULE=off
+ENV CGO_ENABLED=1
+ARG DEBIAN_FRONTEND
+ARG TARGETPLATFORM
+RUN --mount=type=cache,sharing=locked,id=moby-build-aptlib,target=/var/lib/apt \
+    --mount=type=cache,sharing=locked,id=moby-build-aptcache,target=/var/cache/apt \
+        xx-apt-get install --no-install-recommends -y \
+            gcc \
+            libapparmor-dev \
+            libbtrfs-dev \
+            libc6-dev \
+            libdevmapper-dev \
+            libseccomp-dev \
+            libsecret-1-dev \
+            libsystemd-dev \
+            libudev-dev
+ARG DOCKER_BUILDTAGS
+ARG DOCKER_DEBUG
+ARG DOCKER_GITCOMMIT=HEAD
+ARG DOCKER_LDFLAGS
+ARG DOCKER_STATIC
+ARG VERSION
+ARG PLATFORM
+ARG PRODUCT
+ARG DEFAULT_PRODUCT_LICENSE
+ARG PACKAGER_NAME
+# PREFIX overrides DEST dir in make.sh script otherwise it fails because of
+# read only mount in current work dir
+ENV PREFIX=/tmp
+RUN --mount=type=bind,target=. \
     --mount=type=tmpfs,target=cli/winresources/dockerd \
     --mount=type=tmpfs,target=cli/winresources/docker-proxy \
-        hack/make.sh binary
-
-FROM binary-base AS build-dynbinary
-RUN --mount=type=cache,target=/root/.cache \
-    --mount=type=bind,target=.,ro \
-    --mount=type=tmpfs,target=cli/winresources/dockerd \
-    --mount=type=tmpfs,target=cli/winresources/docker-proxy \
-        hack/make.sh dynbinary
-
-FROM binary-base AS build-cross
-ARG DOCKER_CROSSPLATFORMS
-RUN --mount=type=cache,target=/root/.cache \
-    --mount=type=bind,target=.,ro \
-    --mount=type=tmpfs,target=cli/winresources/dockerd \
-    --mount=type=tmpfs,target=cli/winresources/docker-proxy \
-        hack/make.sh cross
+    --mount=type=cache,target=/root/.cache/go-build,id=moby-build-$TARGETPLATFORM <<EOT
+  set -e
+  target=$([ "$DOCKER_STATIC" = "1" ] && echo "binary" || echo "dynbinary")
+  xx-go --wrap
+  ./hack/make.sh $target
+  xx-verify /tmp/bundles/${target}-daemon/dockerd$([ "$(xx-info os)" = "windows" ] && echo ".exe")
+  xx-verify /tmp/bundles/${target}-daemon/docker-proxy$([ "$(xx-info os)" = "windows" ] && echo ".exe")
+  mkdir /build
+  mv /tmp/bundles/${target}-daemon/* /build/
+EOT
 
+# usage:
+# > docker buildx bake binary
+# > DOCKER_STATIC=0 docker buildx bake binary
+# or
+# > make binary
+# > make dynbinary
 FROM scratch AS binary
-COPY --from=build-binary /build/bundles/ /
-
-FROM scratch AS dynbinary
-COPY --from=build-dynbinary /build/bundles/ /
-
-FROM scratch AS cross
-COPY --from=build-cross /build/bundles/ /
+COPY --from=build /build/ /
 
+# usage:
+# > make shell
+# > SYSTEMD=true make shell
 FROM dev AS final
-COPY . /go/src/github.com/docker/docker
+COPY . .
diff --git a/Makefile b/Makefile
index 3de756456a..92f1d41b9e 100644
--- a/Makefile
+++ b/Makefile
@@ -31,7 +31,6 @@ export VALIDATE_ORIGIN_BRANCH
 # make DOCKER_LDFLAGS="-X github.com/docker/docker/daemon/graphdriver.priority=overlay2,devicemapper" dynbinary
 #
 DOCKER_ENVS := \
-	-e DOCKER_CROSSPLATFORMS \
 	-e BUILD_APT_MIRROR \
 	-e BUILDFLAGS \
 	-e KEEPBUNDLE \
@@ -142,17 +141,7 @@ endif
 
 BUILD_OPTS := ${BUILD_APT_MIRROR} ${DOCKER_BUILD_ARGS} ${DOCKER_BUILD_OPTS} -f "$(DOCKERFILE)"
 BUILD_CMD := $(BUILDX) build
-
-# This is used for the legacy "build" target and anything still depending on it
-BUILD_CROSS =
-ifdef DOCKER_CROSS
-BUILD_CROSS = --build-arg CROSS=$(DOCKER_CROSS)
-endif
-ifdef DOCKER_CROSSPLATFORMS
-BUILD_CROSS = --build-arg CROSS=true
-endif
-
-VERSION_AUTOGEN_ARGS = --build-arg VERSION --build-arg DOCKER_GITCOMMIT --build-arg PRODUCT --build-arg PLATFORM --build-arg DEFAULT_PRODUCT_LICENSE --build-arg PACKAGER_NAME
+BAKE_CMD := $(BUILDX) bake
 
 default: binary
 
@@ -160,14 +149,13 @@ all: build ## validate all checks, build linux binaries, run all tests,\ncross b
 	$(DOCKER_RUN_DOCKER) bash -c 'hack/validate/default && hack/make.sh'
 
 binary: bundles ## build statically linked linux binaries
-	$(BUILD_CMD) $(BUILD_OPTS) --output=bundles/ --target=$@ $(VERSION_AUTOGEN_ARGS) .
+	$(BAKE_CMD) binary
 
 dynbinary: bundles ## build dynamically linked linux binaries
-	$(BUILD_CMD) $(BUILD_OPTS) --output=bundles/ --target=$@ $(VERSION_AUTOGEN_ARGS) .
+	$(BAKE_CMD) dynbinary
 
-cross: BUILD_OPTS += --build-arg CROSS=true --build-arg DOCKER_CROSSPLATFORMS
-cross: bundles ## cross build the binaries for darwin, freebsd and\nwindows
-	$(BUILD_CMD) $(BUILD_OPTS) --output=bundles/ --target=$@ $(VERSION_AUTOGEN_ARGS) .
+cross: bundles ## cross build the binaries
+	$(BAKE_CMD) binary-cross
 
 bundles:
 	mkdir bundles
@@ -195,13 +183,13 @@ else
 build: shell_target := --target=final
 endif
 build: bundles
-	$(BUILD_CMD) $(BUILD_OPTS) $(shell_target) --load $(BUILD_CROSS) -t "$(DOCKER_IMAGE)" .
+	$(BUILD_CMD) $(BUILD_OPTS) $(shell_target) --load -t "$(DOCKER_IMAGE)" .
 
 shell: build  ## start a shell inside the build env
 	$(DOCKER_RUN_DOCKER) bash
 
 test: build test-unit ## run the unit, integration and docker-py tests
-	$(DOCKER_RUN_DOCKER) hack/make.sh dynbinary cross test-integration test-docker-py
+	$(DOCKER_RUN_DOCKER) hack/make.sh dynbinary test-integration test-docker-py
 
 test-docker-py: build ## run the docker-py tests
 	$(DOCKER_RUN_DOCKER) hack/make.sh dynbinary test-docker-py
@@ -228,8 +216,8 @@ validate: build ## validate DCO, Seccomp profile generation, gofmt,\n./pkg/ isol
 validate-%: build ## validate specific check
 	$(DOCKER_RUN_DOCKER) hack/validate/$*
 
-win: build ## cross build the binary for windows
-	$(DOCKER_RUN_DOCKER) DOCKER_CROSSPLATFORMS=windows/amd64 hack/make.sh cross
+win: bundles ## cross build the binary for windows
+	$(BAKE_CMD) --set *.platform=windows/amd64 binary
 
 .PHONY: swagger-gen
 swagger-gen:
diff --git a/docker-bake.hcl b/docker-bake.hcl
index fc9d0b7630..e46a64a9a6 100644
--- a/docker-bake.hcl
+++ b/docker-bake.hcl
@@ -4,9 +4,6 @@ variable "BUNDLES_OUTPUT" {
 variable "DOCKER_STATIC" {
   default = "1"
 }
-variable "DOCKER_CROSSPLATFORMS" {
-  default = ""
-}
 
 target "_common" {
   args = {
@@ -20,6 +17,23 @@ group "default" {
   targets = ["binary"]
 }
 
+target "_platforms" {
+  platforms = [
+    "linux/amd64",
+    "linux/arm/v5",
+    "linux/arm/v6",
+    "linux/arm/v7",
+    "linux/arm64",
+    "linux/ppc64le",
+    "linux/s390x",
+    "windows/amd64"
+  ]
+}
+
+#
+# build dockerd and docker-proxy
+#
+
 target "binary" {
   inherits = ["_common"]
   target = "binary"
@@ -28,16 +42,13 @@ target "binary" {
 
 target "dynbinary" {
   inherits = ["binary"]
-  target = "dynbinary"
+  args = {
+    DOCKER_STATIC = "0"
+  }
 }
 
-target "cross" {
-  inherits = ["binary"]
-  args = {
-    CROSS = "true"
-    DOCKER_CROSSPLATFORMS = DOCKER_CROSSPLATFORMS
-  }
-  target = "cross"
+target "binary-cross" {
+  inherits = ["binary", "_platforms"]
 }
 
 #
diff --git a/docs/contributing/set-up-dev-env.md b/docs/contributing/set-up-dev-env.md
index d3efcd029f..e8ef9b23aa 100644
--- a/docs/contributing/set-up-dev-env.md
+++ b/docs/contributing/set-up-dev-env.md
@@ -150,10 +150,10 @@ can take over 15 minutes to complete.
    Removing bundles/
 
    ---> Making bundle: binary (in bundles/binary)
-   Building: bundles/binary-daemon/dockerd-17.06.0-dev
-   Created binary: bundles/binary-daemon/dockerd-17.06.0-dev
-   Copying nested executables into bundles/binary-daemon
-
+   Building bundles/binary-daemon/dockerd (linux/amd64)...
+   Created binary: bundles/binary-daemon/dockerd
+   Building bundles/binary-daemon/docker-proxy (linux/amd64)...
+   Created binary:bundles/binary-daemon/docker-proxy
    ```
 
 7. Run `make install`, which copies the binary to the container's
diff --git a/docs/contributing/test.md b/docs/contributing/test.md
index 099b92c7d8..fe162a9d42 100644
--- a/docs/contributing/test.md
+++ b/docs/contributing/test.md
@@ -123,7 +123,7 @@ Try this now.
 4.  Run the tests using the `hack/make.sh` script.
 
     ```bash
-    # hack/make.sh dynbinary binary cross test-integration test-docker-py
+    # hack/make.sh dynbinary binary test-integration test-docker-py
     ```
 
     The tests run just as they did within your local host.
@@ -132,11 +132,11 @@ Try this now.
     just the integration tests:
 
     ```bash
-    # hack/make.sh dynbinary binary cross test-integration
+    # hack/make.sh dynbinary binary test-integration
     ```
 
     Most test targets require that you build these precursor targets first:
-    `dynbinary binary cross`
+    `dynbinary binary`
 
 
 ## Run unit tests
diff --git a/hack/README.md b/hack/README.md
index 8310b45831..5d1c47d239 100644
--- a/hack/README.md
+++ b/hack/README.md
@@ -44,7 +44,7 @@ all of the tests.
 - When running inside a Docker development container, `hack/make.sh` does
 not have a single target that runs all the tests. You need to provide a
 single command line with multiple targets that performs the same thing.
-An example referenced from [Run targets inside a development container](https://docs.docker.com/opensource/project/test-and-docs/#run-targets-inside-a-development-container): `root@5f8630b873fe:/go/src/github.com/moby/moby# hack/make.sh dynbinary binary cross test-unit test-integration test-docker-py`
+An example referenced from [Run targets inside a development container](https://docs.docker.com/opensource/project/test-and-docs/#run-targets-inside-a-development-container): `root@5f8630b873fe:/go/src/github.com/moby/moby# hack/make.sh dynbinary binary test-unit test-integration test-docker-py`
 - For more information related to testing outside the scope of this README,
 refer to
 [Run tests and test documentation](https://docs.docker.com/opensource/project/test-and-docs/)
diff --git a/hack/make.sh b/hack/make.sh
index 427def3aca..85bf430dff 100755
--- a/hack/make.sh
+++ b/hack/make.sh
@@ -36,7 +36,6 @@ DEFAULT_BUNDLES=(
 	dynbinary
 	test-integration
 	test-docker-py
-	cross
 )
 
 VERSION=${VERSION:-dev}
diff --git a/hack/make/.binary b/hack/make/.binary
index 20bedb9a77..8b412b5a72 100644
--- a/hack/make/.binary
+++ b/hack/make/.binary
@@ -16,56 +16,59 @@ source "${MAKEDIR}/.go-autogen"
 (
 	export GOGC=${DOCKER_BUILD_GOGC:-1000}
 
-	if [ "$(go env GOOS)/$(go env GOARCH)" != "$(go env GOHOSTOS)/$(go env GOHOSTARCH)" ]; then
-		# must be cross-compiling!
-		case "$(go env GOOS)/$(go env GOARCH)" in
-			windows/amd64)
-				export CC="${CC:-x86_64-w64-mingw32-gcc}"
-				export CGO_ENABLED=1
-				;;
-			linux/arm)
-				case "${GOARM}" in
-					5)
-						export CC="${CC:-arm-linux-gnueabi-gcc}"
-						export CGO_ENABLED=1
-						export CGO_CFLAGS="-march=armv5t"
-						export CGO_CXXFLAGS="-march=armv5t"
-						;;
-					6)
-						export CC="${CC:-arm-linux-gnueabi-gcc}"
-						export CGO_ENABLED=1
-						export CGO_CFLAGS="-march=armv6"
-						export CGO_CXXFLAGS="-march=armv6"
-						;;
-					7)
-						export CC="${CC:-arm-linux-gnueabihf-gcc}"
-						export CGO_ENABLED=1
-						export CGO_CFLAGS="-march=armv7-a"
-						export CGO_CXXFLAGS="-march=armv7-a"
-						;;
-					*)
-						export CC="${CC:-arm-linux-gnueabihf-gcc}"
-						export CGO_ENABLED=1
-						;;
-				esac
-				;;
-			linux/arm64)
-				export CC="${CC:-aarch64-linux-gnu-gcc}"
-				export CGO_ENABLED=1
-				;;
-			linux/amd64)
-				export CC="${CC:-x86_64-linux-gnu-gcc}"
-				export CGO_ENABLED=1
-				;;
-			linux/ppc64le)
-				export CC="${CC:-powerpc64le-linux-gnu-gcc}"
-				export CGO_ENABLED=1
-				;;
-			linux/s390x)
-				export CC="${CC:-s390x-linux-gnu-gcc}"
-				export CGO_ENABLED=1
-				;;
-		esac
+	# for non-sandboxed invocation
+	if ! command -v xx-go > /dev/null 2>&1; then
+		if [ "$(go env GOOS)/$(go env GOARCH)" != "$(go env GOHOSTOS)/$(go env GOHOSTARCH)" ]; then
+			# must be cross-compiling!
+			case "$(go env GOOS)/$(go env GOARCH)" in
+				windows/amd64)
+					export CC="${CC:-x86_64-w64-mingw32-gcc}"
+					export CGO_ENABLED=1
+					;;
+				linux/arm)
+					case "${GOARM}" in
+						5)
+							export CC="${CC:-arm-linux-gnueabi-gcc}"
+							export CGO_ENABLED=1
+							export CGO_CFLAGS="-march=armv5t"
+							export CGO_CXXFLAGS="-march=armv5t"
+							;;
+						6)
+							export CC="${CC:-arm-linux-gnueabi-gcc}"
+							export CGO_ENABLED=1
+							export CGO_CFLAGS="-march=armv6"
+							export CGO_CXXFLAGS="-march=armv6"
+							;;
+						7)
+							export CC="${CC:-arm-linux-gnueabihf-gcc}"
+							export CGO_ENABLED=1
+							export CGO_CFLAGS="-march=armv7-a"
+							export CGO_CXXFLAGS="-march=armv7-a"
+							;;
+						*)
+							export CC="${CC:-arm-linux-gnueabihf-gcc}"
+							export CGO_ENABLED=1
+							;;
+					esac
+					;;
+				linux/arm64)
+					export CC="${CC:-aarch64-linux-gnu-gcc}"
+					export CGO_ENABLED=1
+					;;
+				linux/amd64)
+					export CC="${CC:-x86_64-linux-gnu-gcc}"
+					export CGO_ENABLED=1
+					;;
+				linux/ppc64le)
+					export CC="${CC:-powerpc64le-linux-gnu-gcc}"
+					export CGO_ENABLED=1
+					;;
+				linux/s390x)
+					export CC="${CC:-s390x-linux-gnu-gcc}"
+					export CGO_ENABLED=1
+					;;
+			esac
+		fi
 	fi
 
 	# -buildmode=pie is not supported on Windows and Linux on mips, riscv64 and ppc64be.
@@ -80,8 +83,18 @@ source "${MAKEDIR}/.go-autogen"
 			;;
 	esac
 
-	echo "Building: $DEST/$BINARY_FULLNAME"
-	echo "GOOS=\"${GOOS}\" GOARCH=\"${GOARCH}\" GOARM=\"${GOARM}\""
+	# only necessary for non-sandboxed invocation where TARGETPLATFORM is empty
+	PLATFORM_NAME=$TARGETPLATFORM
+	if [ -z "$PLATFORM_NAME" ]; then
+		PLATFORM_NAME="$(go env GOOS)/$(go env GOARCH)"
+		if [ -n "$(go env GOARM)" ]; then
+			PLATFORM_NAME+="/$(go env GOARM)"
+		elif [ -n "$(go env GOAMD64)" ] && [ "$(go env GOAMD64)" != "v1" ]; then
+			PLATFORM_NAME+="/$(go env GOAMD64)"
+		fi
+	fi
+
+	echo "Building $DEST/$BINARY_FULLNAME ($PLATFORM_NAME)..."
 	go build \
 		-o "$DEST/$BINARY_FULLNAME" \
 		"${BUILDFLAGS[@]}" \
diff --git a/hack/make/binary-daemon b/hack/make/binary-daemon
index 07be976710..02ec3ce56b 100644
--- a/hack/make/binary-daemon
+++ b/hack/make/binary-daemon
@@ -1,35 +1,10 @@
 #!/usr/bin/env bash
 set -e
 
-copy_binaries() {
-	local dir="$1"
-
-	# Add nested executables to bundle dir so we have complete set of
-	# them available, but only if the native OS/ARCH is the same as the
-	# OS/ARCH of the build target
-	if [ "$(go env GOOS)/$(go env GOARCH)" != "$(go env GOHOSTOS)/$(go env GOHOSTARCH)" ]; then
-		return
-	fi
-	if [ ! -x /usr/local/bin/runc ]; then
-		return
-	fi
-	echo "Copying nested executables into $dir"
-	for file in containerd containerd-shim-runc-v2 ctr runc docker-init rootlesskit rootlesskit-docker-proxy dockerd-rootless.sh dockerd-rootless-setuptool.sh; do
-		cp -f "$(command -v "$file")" "$dir/"
-	done
-	# vpnkit might not be available for the target platform, see vpnkit stage in
-	# the Dockerfile for more information.
-	if command -v vpnkit > /dev/null 2>&1; then
-		cp -f "$(command -v vpnkit)" "$dir/"
-	fi
-}
-
 [ -z "$KEEPDEST" ] && rm -rf "$DEST"
 
 (
 	GO_PACKAGE='github.com/docker/docker/cmd/dockerd'
 	BINARY_NAME='dockerd'
-
 	source "${MAKEDIR}/.binary"
-	copy_binaries "$DEST"
 )
diff --git a/hack/make/cross b/hack/make/cross
deleted file mode 100644
index 1e2d5d628d..0000000000
--- a/hack/make/cross
+++ /dev/null
@@ -1,37 +0,0 @@
-#!/usr/bin/env bash
-set -e
-
-# if we have our linux/amd64 version compiled, let's symlink it in
-if [ -x "${DEST}/../binary-daemon/dockerd" ]; then
-	arch=$(go env GOHOSTARCH)
-	mkdir -p "$DEST/linux/${arch}"
-	(
-		cd "${DEST}/linux/${arch}"
-		ln -sf ../../../binary-daemon/* ./
-	)
-	echo "Created symlinks:" "${DEST}/linux/${arch}/"*
-fi
-
-DOCKER_CROSSPLATFORMS=${DOCKER_CROSSPLATFORMS:-"linux/amd64 windows/amd64 linux/ppc64le linux/s390x"}
-
-for platform in ${DOCKER_CROSSPLATFORMS}; do
-	(
-		export KEEPDEST=1
-		export DEST="${DEST}/${platform}" # bundles/VERSION/cross/GOOS/GOARCH/docker-VERSION
-		export GOOS=${platform%%/*}
-		export GOARCH=${platform#*/}
-
-		if [[ "${GOARCH}" = "arm/"* ]]; then
-			GOARM=${GOARCH##*/v}
-			GOARCH=${GOARCH%/v*}
-			export GOARM
-		fi
-
-		echo "Cross building: ${DEST}"
-		mkdir -p "${DEST}"
-		ABS_DEST="$(cd "${DEST}" && pwd -P)"
-		source "${MAKEDIR}/binary"
-
-		source "${MAKEDIR}/cross-platform-dependent"
-	)
-done
diff --git a/hack/make/cross-platform-dependent b/hack/make/cross-platform-dependent
deleted file mode 100644
index 21824ed7c9..0000000000
--- a/hack/make/cross-platform-dependent
+++ /dev/null
@@ -1,6 +0,0 @@
-#!/usr/bin/env bash
-set -e
-
-if [ ${platform} == "windows/amd64" ]; then
-	source "${MAKEDIR}/containerutility"
-fi

From 5d4ddce5ed54250794c25e4fca5c227c5fe8ec2f Mon Sep 17 00:00:00 2001
From: CrazyMax <crazy-max@users.noreply.github.com>
Date: Sun, 1 Jan 2023 17:49:23 +0100
Subject: [PATCH 02/13] hack: remove containerutility script

this script is not used anymore. containerutility is
built in the Dockerfile.

Signed-off-by: CrazyMax <crazy-max@users.noreply.github.com>
(cherry picked from commit 04c90b8cf5c624072c8e74f66899e440875ef422)
---
 hack/make/containerutility | 20 --------------------
 1 file changed, 20 deletions(-)
 delete mode 100644 hack/make/containerutility

diff --git a/hack/make/containerutility b/hack/make/containerutility
deleted file mode 100644
index 8525d971f6..0000000000
--- a/hack/make/containerutility
+++ /dev/null
@@ -1,20 +0,0 @@
-#!/usr/bin/env bash
-set -e
-
-: "${CONTAINER_UTILITY_COMMIT:=aa1ba87e99b68e0113bd27ec26c60b88f9d4ccd9}"
-
-(
-	git clone https://github.com/docker/windows-container-utility.git "$GOPATH/src/github.com/docker/windows-container-utility"
-	cd "$GOPATH/src/github.com/docker/windows-container-utility"
-	git checkout -q "$CONTAINER_UTILITY_COMMIT"
-
-	echo Building: ${DEST}/containerutility.exe
-
-	(
-		make
-	)
-
-	mkdir -p ${ABS_DEST}
-
-	cp containerutility.exe ${ABS_DEST}/containerutility.exe
-)

From 29758a067ff5b12c12c76ec7335aa3e40aa49078 Mon Sep 17 00:00:00 2001
From: CrazyMax <crazy-max@users.noreply.github.com>
Date: Tue, 27 Dec 2022 01:36:10 +0100
Subject: [PATCH 03/13] IAmStatic not used anymore

Signed-off-by: CrazyMax <crazy-max@users.noreply.github.com>
(cherry picked from commit 84ea9ee0f90d54e8a9bafbc35c86b5c4be31cd35)
---
 dockerversion/version_lib.go | 1 -
 hack/make.sh                 | 1 -
 hack/make/.go-autogen        | 1 -
 hack/make/dynbinary-daemon   | 1 -
 hack/make/dynbinary-proxy    | 1 -
 5 files changed, 5 deletions(-)

diff --git a/dockerversion/version_lib.go b/dockerversion/version_lib.go
index 0004619f02..c510e269e0 100644
--- a/dockerversion/version_lib.go
+++ b/dockerversion/version_lib.go
@@ -6,7 +6,6 @@ var (
 	GitCommit             = "library-import"
 	Version               = "library-import"
 	BuildTime             = "library-import"
-	IAmStatic             = "library-import"
 	PlatformName          = ""
 	ProductName           = ""
 	DefaultProductLicense = ""
diff --git a/hack/make.sh b/hack/make.sh
index 85bf430dff..5205742990 100755
--- a/hack/make.sh
+++ b/hack/make.sh
@@ -102,7 +102,6 @@ fi
 
 # Use these flags when compiling the tests and final binary
 
-IAMSTATIC='true'
 if [ -z "$DOCKER_DEBUG" ]; then
 	LDFLAGS='-w'
 fi
diff --git a/hack/make/.go-autogen b/hack/make/.go-autogen
index 07205f5f4b..de1708dce7 100644
--- a/hack/make/.go-autogen
+++ b/hack/make/.go-autogen
@@ -4,7 +4,6 @@ LDFLAGS="${LDFLAGS} \
 	-X \"github.com/docker/docker/dockerversion.Version=${VERSION}\" \
 	-X \"github.com/docker/docker/dockerversion.GitCommit=${GITCOMMIT}\" \
 	-X \"github.com/docker/docker/dockerversion.BuildTime=${BUILDTIME}\" \
-	-X \"github.com/docker/docker/dockerversion.IAmStatic=${IAMSTATIC:-true}\" \
 	-X \"github.com/docker/docker/dockerversion.PlatformName=${PLATFORM}\" \
 	-X \"github.com/docker/docker/dockerversion.ProductName=${PRODUCT}\" \
 	-X \"github.com/docker/docker/dockerversion.DefaultProductLicense=${DEFAULT_PRODUCT_LICENSE}\" \
diff --git a/hack/make/dynbinary-daemon b/hack/make/dynbinary-daemon
index 7d659695d4..efb69df754 100644
--- a/hack/make/dynbinary-daemon
+++ b/hack/make/dynbinary-daemon
@@ -4,7 +4,6 @@ set -e
 [ -z "$KEEPDEST" ] && rm -rf "$DEST"
 
 (
-	export IAMSTATIC='false'
 	export LDFLAGS_STATIC_DOCKER=''
 	export BUILDFLAGS=("${BUILDFLAGS[@]/netgo /}")        # disable netgo, since we don't need it for a dynamic binary
 	export BUILDFLAGS=("${BUILDFLAGS[@]/osusergo /}")     # ditto for osusergo
diff --git a/hack/make/dynbinary-proxy b/hack/make/dynbinary-proxy
index ff408b299e..048bd9d2b2 100644
--- a/hack/make/dynbinary-proxy
+++ b/hack/make/dynbinary-proxy
@@ -3,7 +3,6 @@
 set -e
 
 (
-	export IAMSTATIC='false'
 	export LDFLAGS_STATIC_DOCKER=''
 	export BUILDFLAGS=("${BUILDFLAGS[@]/netgo /}")        # disable netgo, since we don't need it for a dynamic binary
 	export BUILDFLAGS=("${BUILDFLAGS[@]/osusergo /}")     # ditto for osusergo

From 75596bee284096d82543728fc34eacb5c70eb441 Mon Sep 17 00:00:00 2001
From: CrazyMax <crazy-max@users.noreply.github.com>
Date: Tue, 27 Dec 2022 01:51:08 +0100
Subject: [PATCH 04/13] hack: use PKG_CONFIG var when checking libdevmapper

Signed-off-by: CrazyMax <crazy-max@users.noreply.github.com>
(cherry picked from commit e37985f59009186dee26c42e14f03d7a11390f2c)
---
 hack/make.sh | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/hack/make.sh b/hack/make.sh
index 5205742990..4a000a65d5 100755
--- a/hack/make.sh
+++ b/hack/make.sh
@@ -92,11 +92,7 @@ fi
 # functionality. We favour libdm_dlsym_deferred_remove over
 # libdm_no_deferred_remove in dynamic cases because the binary could be shipped
 # with a newer libdevmapper than the one it was built with.
-if
-	command -v gcc &> /dev/null \
-		&& ! (echo -e '#include <libdevmapper.h>\nint main() { dm_task_deferred_remove(NULL); }' | gcc -xc - -o /dev/null $(pkg-config --libs devmapper) &> /dev/null) \
-		;
-then
+if command -v gcc &> /dev/null && ! (echo -e '#include <libdevmapper.h>\nint main() { dm_task_deferred_remove(NULL); }' | gcc -xc - -o /dev/null $(${PKG_CONFIG} --libs devmapper 2> /dev/null) &> /dev/null); then
 	add_buildtag libdm dlsym_deferred_remove
 fi
 

From 7d8c689923c5e56c4f475e0306399616a972f20f Mon Sep 17 00:00:00 2001
From: CrazyMax <crazy-max@users.noreply.github.com>
Date: Tue, 27 Dec 2022 02:45:43 +0100
Subject: [PATCH 05/13] hack: do not set buildmode pie for statically linked
 binary

Signed-off-by: CrazyMax <crazy-max@users.noreply.github.com>
(cherry picked from commit 347a5f68da9d262762e3cf1b8191067ff786a017)
---
 Dockerfile              |  4 ++--
 hack/make/.binary       | 22 ++++++++++++----------
 hack/make/binary-daemon |  1 +
 hack/make/binary-proxy  |  2 +-
 4 files changed, 16 insertions(+), 13 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index e830658774..0465ca9733 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -584,8 +584,8 @@ RUN --mount=type=bind,target=. \
   target=$([ "$DOCKER_STATIC" = "1" ] && echo "binary" || echo "dynbinary")
   xx-go --wrap
   ./hack/make.sh $target
-  xx-verify /tmp/bundles/${target}-daemon/dockerd$([ "$(xx-info os)" = "windows" ] && echo ".exe")
-  xx-verify /tmp/bundles/${target}-daemon/docker-proxy$([ "$(xx-info os)" = "windows" ] && echo ".exe")
+  xx-verify $([ "$DOCKER_STATIC" = "1" ] && echo "--static") /tmp/bundles/${target}-daemon/dockerd$([ "$(xx-info os)" = "windows" ] && echo ".exe")
+  xx-verify $([ "$DOCKER_STATIC" = "1" ] && echo "--static") /tmp/bundles/${target}-daemon/docker-proxy$([ "$(xx-info os)" = "windows" ] && echo ".exe")
   mkdir /build
   mv /tmp/bundles/${target}-daemon/* /build/
 EOT
diff --git a/hack/make/.binary b/hack/make/.binary
index 8b412b5a72..8c1769ec1e 100644
--- a/hack/make/.binary
+++ b/hack/make/.binary
@@ -73,15 +73,17 @@ source "${MAKEDIR}/.go-autogen"
 
 	# -buildmode=pie is not supported on Windows and Linux on mips, riscv64 and ppc64be.
 	# https://github.com/golang/go/blob/77aa209b386a184e7f4b44938f2a05a1b5c5a3cf/src/cmd/internal/sys/supported.go#L89-L99
-	case "$(go env GOOS)/$(go env GOARCH)" in
-		windows/* | linux/mips* | linux/riscv* | linux/ppc64) ;;
-		# TODO remove windows in Go 1.15+: https://github.com/golang/go/commit/95f382139043059a2a0780ba577b53893408f7e4
-		# TODO remove riscv64 in Go 1.16+: https://github.com/golang/go/commit/8eb846fd37eb7bded8a1cf6932be2c59069863e5
-
-		*)
-			BUILDFLAGS+=("-buildmode=pie")
-			;;
-	esac
+	if ! [ "$DOCKER_STATIC" = "1" ]; then
+		case "$(go env GOOS)/$(go env GOARCH)" in
+			windows/* | linux/mips* | linux/riscv* | linux/ppc64) ;;
+			# TODO remove windows in Go 1.15+: https://github.com/golang/go/commit/95f382139043059a2a0780ba577b53893408f7e4
+			# TODO remove riscv64 in Go 1.16+: https://github.com/golang/go/commit/8eb846fd37eb7bded8a1cf6932be2c59069863e5
+	
+			*)
+				BUILDFLAGS+=("-buildmode=pie")
+				;;
+		esac
+	fi
 
 	# only necessary for non-sandboxed invocation where TARGETPLATFORM is empty
 	PLATFORM_NAME=$TARGETPLATFORM
@@ -94,7 +96,7 @@ source "${MAKEDIR}/.go-autogen"
 		fi
 	fi
 
-	echo "Building $DEST/$BINARY_FULLNAME ($PLATFORM_NAME)..."
+	echo "Building $([ "$DOCKER_STATIC" = "1" ] && echo "static" || echo "dynamic") $DEST/$BINARY_FULLNAME ($PLATFORM_NAME)..."
 	go build \
 		-o "$DEST/$BINARY_FULLNAME" \
 		"${BUILDFLAGS[@]}" \
diff --git a/hack/make/binary-daemon b/hack/make/binary-daemon
index 02ec3ce56b..0d8877bc02 100644
--- a/hack/make/binary-daemon
+++ b/hack/make/binary-daemon
@@ -4,6 +4,7 @@ set -e
 [ -z "$KEEPDEST" ] && rm -rf "$DEST"
 
 (
+	DOCKER_STATIC=1
 	GO_PACKAGE='github.com/docker/docker/cmd/dockerd'
 	BINARY_NAME='dockerd'
 	source "${MAKEDIR}/.binary"
diff --git a/hack/make/binary-proxy b/hack/make/binary-proxy
index 011cf9d500..9fa51f76cd 100644
--- a/hack/make/binary-proxy
+++ b/hack/make/binary-proxy
@@ -5,8 +5,8 @@ set -e
 (
 	export CGO_ENABLED=0
 
+	DOCKER_STATIC=1
 	GO_PACKAGE='github.com/docker/docker/cmd/docker-proxy'
 	BINARY_NAME='docker-proxy'
-
 	source "${MAKEDIR}/.binary"
 )

From 171471b613f19532d355c0154cecbf29d1506594 Mon Sep 17 00:00:00 2001
From: CrazyMax <crazy-max@users.noreply.github.com>
Date: Tue, 27 Dec 2022 04:20:34 +0100
Subject: [PATCH 06/13] hack: update buildmode pie condition

Signed-off-by: CrazyMax <crazy-max@users.noreply.github.com>
(cherry picked from commit 491467b0b926b38888ad2239bbad96b71b35fa91)
---
 hack/make/.binary | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/hack/make/.binary b/hack/make/.binary
index 8c1769ec1e..1b66ff7fbf 100644
--- a/hack/make/.binary
+++ b/hack/make/.binary
@@ -71,14 +71,11 @@ source "${MAKEDIR}/.go-autogen"
 		fi
 	fi
 
-	# -buildmode=pie is not supported on Windows and Linux on mips, riscv64 and ppc64be.
-	# https://github.com/golang/go/blob/77aa209b386a184e7f4b44938f2a05a1b5c5a3cf/src/cmd/internal/sys/supported.go#L89-L99
+	# -buildmode=pie is not supported on Windows arm64 and Linux mips*, ppc64be
+	# https://github.com/golang/go/blob/go1.19.4/src/cmd/internal/sys/supported.go#L125-L132
 	if ! [ "$DOCKER_STATIC" = "1" ]; then
 		case "$(go env GOOS)/$(go env GOARCH)" in
-			windows/* | linux/mips* | linux/riscv* | linux/ppc64) ;;
-			# TODO remove windows in Go 1.15+: https://github.com/golang/go/commit/95f382139043059a2a0780ba577b53893408f7e4
-			# TODO remove riscv64 in Go 1.16+: https://github.com/golang/go/commit/8eb846fd37eb7bded8a1cf6932be2c59069863e5
-	
+			windows/arm64 | linux/mips* | linux/ppc64) ;;
 			*)
 				BUILDFLAGS+=("-buildmode=pie")
 				;;

From 3b90bb6ad36f0f1358b609dd65810565a18c4cfc Mon Sep 17 00:00:00 2001
From: CrazyMax <crazy-max@users.noreply.github.com>
Date: Mon, 26 Dec 2022 17:45:50 +0100
Subject: [PATCH 07/13] bake: define DESTDIR

Keep the same output dir format in the bake definition
as the one used in make scripts.

Signed-off-by: CrazyMax <crazy-max@users.noreply.github.com>
(cherry picked from commit 9bcf5bed05a47359258a731062a1756abbe1e914)
---
 .github/workflows/buildkit.yml |  6 +++---
 .github/workflows/ci.yml       |  6 +++---
 docker-bake.hcl                | 15 +++++++++++----
 3 files changed, 17 insertions(+), 10 deletions(-)

diff --git a/.github/workflows/buildkit.yml b/.github/workflows/buildkit.yml
index baa27a7757..d131d9c9ba 100644
--- a/.github/workflows/buildkit.yml
+++ b/.github/workflows/buildkit.yml
@@ -13,7 +13,7 @@ on:
   pull_request:
 
 env:
-  BUNDLES_OUTPUT: ./bundles
+  DESTDIR: ./build
 
 jobs:
   validate-dco:
@@ -40,7 +40,7 @@ jobs:
         uses: actions/upload-artifact@v3
         with:
           name: binary
-          path: ${{ env.BUNDLES_OUTPUT }}
+          path: ${{ env.DESTDIR }}
           if-no-files-found: error
           retention-days: 1
 
@@ -106,7 +106,7 @@ jobs:
         env:
           CONTEXT: "."
           TEST_DOCKERD: "1"
-          TEST_DOCKERD_BINARY: "./build/moby/binary-daemon/dockerd"
+          TEST_DOCKERD_BINARY: "./build/moby/dockerd"
           TESTPKGS: "./${{ matrix.pkg }}"
           TESTFLAGS: "-v --parallel=1 --timeout=30m --run=//worker=dockerd$"
         working-directory: buildkit
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index a34abc2425..97ee4e73b3 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -15,7 +15,7 @@ on:
   pull_request:
 
 env:
-  BUNDLES_OUTPUT: ./bundles
+  DESTDIR: ./build
 
 jobs:
   validate-dco:
@@ -50,7 +50,7 @@ jobs:
         uses: actions/upload-artifact@v3
         with:
           name: ${{ matrix.target }}
-          path: ${{ env.BUNDLES_OUTPUT }}
+          path: ${{ env.DESTDIR }}
           if-no-files-found: error
           retention-days: 7
 
@@ -110,6 +110,6 @@ jobs:
         uses: actions/upload-artifact@v3
         with:
           name: cross-${{ env.PLATFORM_PAIR }}
-          path: ${{ env.BUNDLES_OUTPUT }}
+          path: ${{ env.DESTDIR }}
           if-no-files-found: error
           retention-days: 7
diff --git a/docker-bake.hcl b/docker-bake.hcl
index e46a64a9a6..4cfc01eced 100644
--- a/docker-bake.hcl
+++ b/docker-bake.hcl
@@ -1,10 +1,16 @@
-variable "BUNDLES_OUTPUT" {
-  default = "./bundles"
-}
 variable "DOCKER_STATIC" {
   default = "1"
 }
 
+# Defines the output folder
+variable "DESTDIR" {
+  default = ""
+}
+function "bindir" {
+  params = [defaultdir]
+  result = DESTDIR != "" ? DESTDIR : "./bundles/${defaultdir}"
+}
+
 target "_common" {
   args = {
     BUILDKIT_CONTEXT_KEEP_GIT_DIR = 1
@@ -37,11 +43,12 @@ target "_platforms" {
 target "binary" {
   inherits = ["_common"]
   target = "binary"
-  output = [BUNDLES_OUTPUT]
+  output = [bindir(DOCKER_STATIC == "1" ? "binary" : "dynbinary")]
 }
 
 target "dynbinary" {
   inherits = ["binary"]
+  output = [bindir("dynbinary")]
   args = {
     DOCKER_STATIC = "0"
   }

From f29fe2daec9b9f0c790257110d3bad51ae39b068 Mon Sep 17 00:00:00 2001
From: CrazyMax <crazy-max@users.noreply.github.com>
Date: Sun, 1 Jan 2023 17:53:01 +0100
Subject: [PATCH 08/13] bake: define additional vars

Signed-off-by: CrazyMax <crazy-max@users.noreply.github.com>
(cherry picked from commit 0b3af098e12d6960407d6f667055d1ef88a77afb)
---
 docker-bake.hcl | 56 ++++++++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 55 insertions(+), 1 deletion(-)

diff --git a/docker-bake.hcl b/docker-bake.hcl
index 4cfc01eced..d0c7cf29d4 100644
--- a/docker-bake.hcl
+++ b/docker-bake.hcl
@@ -1,6 +1,51 @@
+variable "APT_MIRROR" {
+  default = "cdn-fastly.deb.debian.org"
+}
+variable "DOCKER_DEBUG" {
+  default = ""
+}
 variable "DOCKER_STATIC" {
   default = "1"
 }
+variable "DOCKER_LDFLAGS" {
+  default = ""
+}
+variable "DOCKER_BUILDTAGS" {
+  default = ""
+}
+variable "DOCKER_GITCOMMIT" {
+  default = "HEAD"
+}
+
+# Docker version such as 23.0.0-dev. Automatically generated through Git ref.
+variable "VERSION" {
+  default = ""
+}
+
+# The platform name, such as "Docker Engine - Community".
+variable "PLATFORM" {
+  default = ""
+}
+
+# The product name, used to set version.ProductName, which is used to set
+# BuildKit's ExportedProduct variable in order to show useful error messages
+# to users when a certain version of the product doesn't support a BuildKit feature.
+variable "PRODUCT" {
+  default = ""
+}
+
+# Sets the version.DefaultProductLicense string, such as "Community Engine".
+# This field can contain a summary of the product license of the daemon if a
+# commercial license has been applied to the daemon.
+variable "DEFAULT_PRODUCT_LICENSE" {
+  default = ""
+}
+
+# The name of the packager (e.g. "Docker, Inc."). This used to set CompanyName
+# in the manifest.
+variable "PACKAGER_NAME" {
+  default = ""
+}
 
 # Defines the output folder
 variable "DESTDIR" {
@@ -14,8 +59,17 @@ function "bindir" {
 target "_common" {
   args = {
     BUILDKIT_CONTEXT_KEEP_GIT_DIR = 1
-    APT_MIRROR = "cdn-fastly.deb.debian.org"
+    APT_MIRROR = APT_MIRROR
+    DOCKER_DEBUG = DOCKER_DEBUG
     DOCKER_STATIC = DOCKER_STATIC
+    DOCKER_LDFLAGS = DOCKER_LDFLAGS
+    DOCKER_BUILDTAGS = DOCKER_BUILDTAGS
+    DOCKER_GITCOMMIT = DOCKER_GITCOMMIT
+    VERSION = VERSION
+    PLATFORM = PLATFORM
+    PRODUCT = PRODUCT
+    DEFAULT_PRODUCT_LICENSE = DEFAULT_PRODUCT_LICENSE
+    PACKAGER_NAME = PACKAGER_NAME
   }
 }
 

From 7e286efcd3deb9ea3dec428abfc5f74591096d39 Mon Sep 17 00:00:00 2001
From: CrazyMax <crazy-max@users.noreply.github.com>
Date: Mon, 26 Dec 2022 18:46:26 +0100
Subject: [PATCH 09/13] ci: use GITHUB_REF and GITHUB_SHA to set version and
 commit

Signed-off-by: CrazyMax <crazy-max@users.noreply.github.com>
(cherry picked from commit 27ba1766e79c0ac7ae3f43ba7dc8d622ca6cdf42)
---
 docker-bake.hcl | 16 ++++++++++++++--
 hack/make.sh    |  8 ++++++++
 2 files changed, 22 insertions(+), 2 deletions(-)

diff --git a/docker-bake.hcl b/docker-bake.hcl
index d0c7cf29d4..46b110e45f 100644
--- a/docker-bake.hcl
+++ b/docker-bake.hcl
@@ -47,6 +47,18 @@ variable "PACKAGER_NAME" {
   default = ""
 }
 
+# GITHUB_REF is the actual ref that triggers the workflow and used as version
+# when tag is pushed: https://docs.github.com/en/actions/learn-github-actions/environment-variables#default-environment-variables
+variable "GITHUB_REF" {
+  default = ""
+}
+
+# GITHUB_SHA is the commit SHA that triggered the workflow and used as commit.
+# https://docs.github.com/en/actions/learn-github-actions/environment-variables#default-environment-variables
+variable "GITHUB_SHA" {
+  default = ""
+}
+
 # Defines the output folder
 variable "DESTDIR" {
   default = ""
@@ -64,8 +76,8 @@ target "_common" {
     DOCKER_STATIC = DOCKER_STATIC
     DOCKER_LDFLAGS = DOCKER_LDFLAGS
     DOCKER_BUILDTAGS = DOCKER_BUILDTAGS
-    DOCKER_GITCOMMIT = DOCKER_GITCOMMIT
-    VERSION = VERSION
+    DOCKER_GITCOMMIT = DOCKER_GITCOMMIT != "" ? DOCKER_GITCOMMIT : GITHUB_SHA
+    VERSION = VERSION != "" ? VERSION : GITHUB_REF
     PLATFORM = PLATFORM
     PRODUCT = PRODUCT
     DEFAULT_PRODUCT_LICENSE = DEFAULT_PRODUCT_LICENSE
diff --git a/hack/make.sh b/hack/make.sh
index 4a000a65d5..b30f3d6e44 100755
--- a/hack/make.sh
+++ b/hack/make.sh
@@ -39,6 +39,14 @@ DEFAULT_BUNDLES=(
 )
 
 VERSION=${VERSION:-dev}
+if [[ $VERSION == refs/tags/* ]]; then
+	VERSION=${VERSION#refs/tags/}
+elif [[ $VERSION == refs/heads/* ]]; then
+	VERSION=$(sed <<< "${VERSION#refs/heads/}" -r 's#/+#-#g')
+elif [[ $VERSION == refs/pull/* ]]; then
+	VERSION=pr-$(grep <<< "$VERSION" -o '[0-9]\+')
+fi
+
 ! BUILDTIME=$(date -u -d "@${SOURCE_DATE_EPOCH:-$(date +%s)}" --rfc-3339 ns 2> /dev/null | sed -e 's/ /T/')
 if [ "$DOCKER_GITCOMMIT" ]; then
 	GITCOMMIT="$DOCKER_GITCOMMIT"

From 8fbfc2676f6b9621d9118ec89e12ca2c39a7ccb5 Mon Sep 17 00:00:00 2001
From: CrazyMax <crazy-max@users.noreply.github.com>
Date: Mon, 26 Dec 2022 17:55:57 +0100
Subject: [PATCH 10/13] Dockerfile: rename dev stages

Also remove DEV_IMAGE bake var that is not used

Signed-off-by: CrazyMax <crazy-max@users.noreply.github.com>
(cherry picked from commit b565b496ba9c173555737e152f75a151f5619ca7)
---
 Dockerfile      | 4 ++--
 Makefile        | 4 ++--
 docker-bake.hcl | 7 ++-----
 3 files changed, 6 insertions(+), 9 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 0465ca9733..2011cee82b 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -479,7 +479,7 @@ RUN mkdir -p hack \
   && chmod +x hack/dind-systemd
 ENTRYPOINT ["hack/dind-systemd"]
 
-FROM dev-systemd-${SYSTEMD} AS dev
+FROM dev-systemd-${SYSTEMD} AS dev-base
 ARG DEBIAN_FRONTEND
 RUN groupadd -r docker
 RUN useradd --create-home --gid docker unprivilegeduser \
@@ -602,5 +602,5 @@ COPY --from=build /build/ /
 # usage:
 # > make shell
 # > SYSTEMD=true make shell
-FROM dev AS final
+FROM dev-base AS dev
 COPY . .
diff --git a/Makefile b/Makefile
index 92f1d41b9e..1993c52a91 100644
--- a/Makefile
+++ b/Makefile
@@ -178,9 +178,9 @@ run: build ## run the docker daemon in a container
  
 .PHONY: build
 ifeq ($(BIND_DIR), .)
-build: shell_target := --target=dev
+build: shell_target := --target=dev-base
 else
-build: shell_target := --target=final
+build: shell_target := --target=dev
 endif
 build: bundles
 	$(BUILD_CMD) $(BUILD_OPTS) $(shell_target) --load -t "$(DOCKER_IMAGE)" .
diff --git a/docker-bake.hcl b/docker-bake.hcl
index 46b110e45f..98e02cca98 100644
--- a/docker-bake.hcl
+++ b/docker-bake.hcl
@@ -128,19 +128,16 @@ target "binary-cross" {
 # dev
 #
 
-variable "DEV_IMAGE" {
-  default = "docker-dev"
-}
 variable "SYSTEMD" {
   default = "false"
 }
 
 target "dev" {
   inherits = ["_common"]
-  target = "final"
+  target = "dev"
   args = {
     SYSTEMD = SYSTEMD
   }
-  tags = [DEV_IMAGE]
+  tags = ["docker-dev"]
   output = ["type=docker"]
 }

From 3ef71f908504bf280571badb1ed1121fd6c3cb91 Mon Sep 17 00:00:00 2001
From: CrazyMax <crazy-max@users.noreply.github.com>
Date: Mon, 26 Dec 2022 18:37:06 +0100
Subject: [PATCH 11/13] ci: add extra steps to check artifacts

Signed-off-by: CrazyMax <crazy-max@users.noreply.github.com>
(cherry picked from commit 113e95ec0228596fecbe1f47130c06044fce1a04)
---
 .github/workflows/ci.yml | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 97ee4e73b3..88952bb973 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -45,6 +45,14 @@ jobs:
         uses: docker/bake-action@v2
         with:
           targets: ${{ matrix.target }}
+      -
+        name: List artifacts
+        run: |
+          tree -nh ${{ env.DESTDIR }}
+      -
+        name: Check artifacts
+        run: |
+          find ${{ env.DESTDIR }} -type f -exec file -e ascii -- {} +
       -
         name: Upload artifacts
         uses: actions/upload-artifact@v3
@@ -105,6 +113,14 @@ jobs:
           targets: binary
           set: |
             *.platform=${{ matrix.platform }}
+      -
+        name: List artifacts
+        run: |
+          tree -nh ${{ env.DESTDIR }}
+      -
+        name: Check artifacts
+        run: |
+          find ${{ env.DESTDIR }} -type f -exec file -e ascii -- {} +
       -
         name: Upload artifacts
         uses: actions/upload-artifact@v3

From 2dd577955c416f5e27add490057352edaaff7bfd Mon Sep 17 00:00:00 2001
From: CrazyMax <crazy-max@users.noreply.github.com>
Date: Thu, 29 Dec 2022 15:35:55 +0100
Subject: [PATCH 12/13] Dockerfile: add "all" stage to build binaries and extra
 tools

We still need a stage that build binaries and extra tools as well for
docker-ce-packaging repo: https://github.com/docker/docker-ce-packaging/blob/ff110508ffce04a9246ef52af4e7a458422e941a/static/Makefile#L41-L57

This could be removed if we create a package for each project
like it's done in docker-packaging repo: https://github.com/docker/packaging/tree/main/pkg

Signed-off-by: CrazyMax <crazy-max@users.noreply.github.com>
(cherry picked from commit e8a82ed24de073ccf3191b0d5b3df3d2b1f69903)
---
 .github/workflows/ci.yml |  2 +-
 Dockerfile               | 11 +++++++++++
 docker-bake.hcl          | 14 ++++++++++++++
 3 files changed, 26 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 88952bb973..8ff824328b 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -110,7 +110,7 @@ jobs:
         name: Build
         uses: docker/bake-action@v2
         with:
-          targets: binary
+          targets: all
           set: |
             *.platform=${{ matrix.platform }}
       -
diff --git a/Dockerfile b/Dockerfile
index 2011cee82b..11da703015 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -599,6 +599,17 @@ EOT
 FROM scratch AS binary
 COPY --from=build /build/ /
 
+# usage:
+# > docker buildx bake all
+FROM scratch AS all
+COPY --from=tini          /build/ /
+COPY --from=runc          /build/ /
+COPY --from=containerd    /build/ /
+COPY --from=rootlesskit   /build/ /
+COPY --from=containerutil /build/ /
+COPY --from=vpnkit        /       /
+COPY --from=build         /build  /
+
 # usage:
 # > make shell
 # > SYSTEMD=true make shell
diff --git a/docker-bake.hcl b/docker-bake.hcl
index 98e02cca98..95c1c6bcff 100644
--- a/docker-bake.hcl
+++ b/docker-bake.hcl
@@ -124,6 +124,20 @@ target "binary-cross" {
   inherits = ["binary", "_platforms"]
 }
 
+#
+# same as binary but with extra tools as well (containerd, runc, ...)
+#
+
+target "all" {
+  inherits = ["_common"]
+  target = "all"
+  output = [bindir(DOCKER_STATIC == "1" ? "binary" : "dynbinary")]
+}
+
+target "all-cross" {
+  inherits = ["all", "_platforms"]
+}
+
 #
 # dev
 #

From d778bbab845c5679b80a0491f82fdba3998ced66 Mon Sep 17 00:00:00 2001
From: CrazyMax <crazy-max@users.noreply.github.com>
Date: Sat, 20 Aug 2022 13:58:04 +0200
Subject: [PATCH 13/13] docs: contributing docs update for cross comp

Signed-off-by: CrazyMax <crazy-max@users.noreply.github.com>
(cherry picked from commit 83280d44d0e62a1144ea9a97762f2555ce9f5e30)
---
 docs/contributing/README.md    |  1 +
 docs/contributing/ctn-build.md | 28 ++++++++++++++++++++++++++++
 2 files changed, 29 insertions(+)
 create mode 100644 docs/contributing/ctn-build.md

diff --git a/docs/contributing/README.md b/docs/contributing/README.md
index d419e52c14..fb2e798061 100644
--- a/docs/contributing/README.md
+++ b/docs/contributing/README.md
@@ -6,5 +6,6 @@
  * (Optional) [Configure project for IDE](set-up-ide.md)
  * [Configure Git for contributing](set-up-git.md)
  * [Work with a development container](set-up-dev-env.md)
+ * [Containerized build and cross compilation](ctn-build.md)
  * [Run tests and test documentation](test.md)
  * [Debugging the daemon](debug.md)
diff --git a/docs/contributing/ctn-build.md b/docs/contributing/ctn-build.md
new file mode 100644
index 0000000000..de924e6d36
--- /dev/null
+++ b/docs/contributing/ctn-build.md
@@ -0,0 +1,28 @@
+The `Dockerfile` supports building and cross compiling docker daemon and extra
+tools using [Docker Buildx](https://github.com/docker/buildx) and [BuildKit](https://github.com/moby/buildkit).
+A [bake definition](https://docs.docker.com/build/bake/file-definition/) named
+`docker-bake.hcl` is in place to ease the build process:
+
+```shell
+# build binaries for the current host platform
+# output to ./bundles/binary-daemon by default
+docker buildx bake
+# or
+docker buildx bake binary
+
+# build binaries for the current host platform
+# output to ./bin
+DESTDIR=./bin docker buildx bake
+
+# build dynamically linked binaries
+# output to ./bundles/dynbinary-daemon by default
+DOCKER_STATIC=0 docker buildx bake
+# or
+docker buildx bake dynbinary
+
+# build binaries for all supported platforms
+docker buildx bake binary-cross
+
+# build binaries for a specific platform
+docker buildx bake --set *.platform=linux/arm64
+```