Merge pull request #43275 from kponichtera/43274-delve-debugger

Added Delve debugger to the development container

Dockerfile

@@ -148,6 +148,18 @@ RUN --mount=type=cache,sharing=locked,id=moby-cross-true-aptlib,target=/var/lib/
 
 FROM runtime-dev-cross-${CROSS} AS runtime-dev
 
+FROM base AS delve
+# DELVE_VERSION specifies the version of the Delve debugger binary
+# from the https://github.com/go-delve/delve repository.
+# It can be used to run Docker with a possibility of
+# attaching debugger to it.
+#
+ARG DELVE_VERSION=v1.8.1
+RUN --mount=type=cache,target=/root/.cache/go-build \
+    --mount=type=cache,target=/go/pkg/mod \
+        GOBIN=/build/ GO111MODULE=on go install "github.com/go-delve/delve/cmd/dlv@${DELVE_VERSION}" \
+     && /build/dlv --help
+
 FROM base AS tomll
 # GOTOML_VERSION specifies the version of the tomll binary to build and install
 # from the https://github.com/pelletier/go-toml repository. This binary is used
@@ -308,6 +320,7 @@ RUN pip3 install yamllint==1.26.1
 COPY --from=dockercli     /build/ /usr/local/cli
 COPY --from=frozen-images /build/ /docker-frozen-images
 COPY --from=swagger       /build/ /usr/local/bin/
+COPY --from=delve         /build/ /usr/local/bin/
 COPY --from=tomll         /build/ /usr/local/bin/
 COPY --from=gowinres      /build/ /usr/local/bin/
 COPY --from=tini          /build/ /usr/local/bin/

Makefile

@@ -68,6 +68,7 @@ DOCKER_ENVS := \
 	-e DOCKER_TEST_HOST \
 	-e DOCKER_USERLANDPROXY \
 	-e DOCKERD_ARGS \
+	-e DELVE_PORT \
 	-e TEST_FORCE_VALIDATE \
 	-e TEST_INTEGRATION_DIR \
 	-e TEST_SKIP_INTEGRATION \
@@ -115,8 +116,9 @@ DOCKER_CONTAINER_NAME := $(if $(CONTAINER_NAME),--name $(CONTAINER_NAME),)
 
 DOCKER_IMAGE := docker-dev
 DOCKER_PORT_FORWARD := $(if $(DOCKER_PORT),-p "$(DOCKER_PORT)",)
+DELVE_PORT_FORWARD := $(if $(DELVE_PORT),-p "$(DELVE_PORT)",)
 
-DOCKER_FLAGS := $(DOCKER) run --rm -i --privileged $(DOCKER_CONTAINER_NAME) $(DOCKER_ENVS) $(DOCKER_MOUNT) $(DOCKER_PORT_FORWARD)
+DOCKER_FLAGS := $(DOCKER) run --rm -i --privileged $(DOCKER_CONTAINER_NAME) $(DOCKER_ENVS) $(DOCKER_MOUNT) $(DOCKER_PORT_FORWARD) $(DELVE_PORT_FORWARD)
 BUILD_APT_MIRROR := $(if $(DOCKER_BUILD_APT_MIRROR),--build-arg APT_MIRROR=$(DOCKER_BUILD_APT_MIRROR))
 export BUILD_APT_MIRROR
 

contrib/dockerd-rootless.sh

@@ -77,6 +77,8 @@ if [ -z "$mtu" ]; then
 	mtu=1500
 fi
 
+dockerd="${DOCKERD:-dockerd}"
+
 if [ -z "$_DOCKERD_ROOTLESS_CHILD" ]; then
 	_DOCKERD_ROOTLESS_CHILD=1
 	export _DOCKERD_ROOTLESS_CHILD
@@ -128,5 +130,6 @@ else
 		mount --rbind ${realpath_etc_ssl} /etc/ssl
 	fi
 
-	exec dockerd $@
+	# shellcheck disable=SC2086
+	exec $dockerd "$@"
 fi

docs/contributing/README.md

@@ -7,3 +7,4 @@
  * [Configure Git for contributing](set-up-git.md)
  * [Work with a development container](set-up-dev-env.md)
  * [Run tests and test documentation](test.md)
+ * [Debugging the daemon](debug.md)

docs/contributing/debug.md

@@ -0,0 +1,66 @@
+### Debugging the daemon
+
+The Docker daemon inside the development container can be debugged with [Delve](https://github.com/go-delve/delve).
+
+Delve debugger listens on a port, which has to be exposed outside the development container.
+Also, in order to be able to debug the daemon, it has to be compiled with the debugging symbols.
+This can be done by launching the development container with the following command:
+
+```bash
+$ make BIND_DIR=. DOCKER_DEBUG=1 DELVE_PORT=127.0.0.1:2345:2345 shell
+```
+
+The `DOCKER_DEBUG` variable disables build optimizations, allowing to debug the binary,
+while `DELVE_PORT` publishes the specified port for use with the debugger.
+
+The `DELVE_PORT` variable accepts the port in the same format as Docker CLI's `--publish` (`-p`) option.
+This means that the port can be published in multiple ways:
+
+1. `DELVE_PORT=127.0.0.1:2345:2345` - exposes debugger on port `2345` for local development only (recommended)
+2. `DELVE_PORT=2345:2345` - exposes debugger on port `2345` without binding to specific IP
+3. `DELVE_PORT=2345` - same as above
+
+**IMPORTANT:** Publishing the port without binding it to localhost (127.0.0.1) might expose the debugger
+outside the developer's machine and is not recommended.
+
+## Running Docker daemon with debugger attached
+
+1. Run development container with build optimizations disabled and Delve enabled:
+   ```bash
+   $ make BIND_DIR=. DOCKER_DEBUG=1 DELVE_PORT=127.0.0.1:2345:2345 shell
+   ```
+2. Inside the development container:
+   1. Build the Docker daemon:
+      ```bash
+      $ ./hack/make.sh binary
+      ```
+   2. Install the newly-built daemon:
+      ```bash
+      $ make install
+      ```
+   3. Run the daemon through the `make.sh` script:
+      ```bash
+      $ ./hack/make.sh run
+      ```
+      The execution will stop and wait for the IDE or Delve CLI to attach
+      to the port, specified with the `DELVE_PORT` variable.
+      Once the IDE or Delve CLI is attached, the execution will continue.
+   
+## Debugging from IDE (on example of GoLand 2021.3)
+
+1. Open the project in GoLand
+2. Click *Add Configuration* on the taskbar
+   ![GoLand - adding configuration](images/goland_add_config.png)
+3. Create the *Go Remote* configuration. 
+   No changes are necessary, unless a different port is to be used.
+   ![GoLand - adding remote configuration](images/goland_debug_config.png)
+4. Run the Docker binary in the development container, as described in the previous section.
+   Make sure that the port in the `DELVE_PORT` variable corresponds to one, used in the *Go Remote* configuration.
+5. Run the *Go Remote* configuration.
+   The Docker daemon will continue execution inside the container and debugger will stop it on the breakpoints.
+   ![GoLand - run Go Remote configuration](images/goland_run_debug_config.png)
+
+## Where to go next
+
+Congratulations, you have experienced how to use Delve to debug the Docker daemon
+and how to configure an IDE to make use of it.

docs/contributing/test.md

@@ -250,3 +250,4 @@ jobs can be triggered and re-ran by the Moby maintainers
 
 Congratulations, you have successfully completed the basics you need to
 understand the Moby test framework.
+In the next section you'll [learn how to debug Docker daemon, running inside the development container](debug.md).

hack/make/run

@@ -8,6 +8,8 @@ if ! command -v dockerd &> /dev/null; then
 	false
 fi
 
+DOCKER_COMMAND="$(command -v dockerd)"
+
 DOCKER_GRAPHDRIVER=${DOCKER_GRAPHDRIVER:-vfs}
 DOCKER_USERLANDPROXY=${DOCKER_USERLANDPROXY:-true}
 
@@ -23,8 +25,11 @@ fi
 
 listen_port=2375
 if [ -n "$DOCKER_PORT" ]; then
-	IFS=':' read -r -a ports <<< "$DOCKER_PORT"
-	listen_port="${ports[-1]}"
+	listen_port="${DOCKER_PORT##*:}"
+fi
+
+if [ -n "$DELVE_PORT" ]; then
+	delve_listen_port="${DELVE_PORT##*:}"
 fi
 
 extra_params="$DOCKERD_ARGS"
@@ -36,7 +41,6 @@ if [ -n "$DOCKER_EXPERIMENTAL" ]; then
 	extra_params="$extra_params --experimental"
 fi
 
-dockerd="dockerd"
 socket=/var/run/docker.sock
 if [ -n "$DOCKER_ROOTLESS" ]; then
 	user="unprivilegeduser"
@@ -44,17 +48,48 @@ if [ -n "$DOCKER_ROOTLESS" ]; then
 	# shellcheck disable=SC2174
 	mkdir -p -m 700 "/tmp/docker-${uid}"
 	chown $user "/tmp/docker-${uid}"
-	dockerd="sudo -u $user -E XDG_RUNTIME_DIR=/tmp/docker-${uid} -E HOME=/home/${user} -- dockerd-rootless.sh"
 	socket=/tmp/docker-${uid}/docker.sock
 fi
 
-args="--debug \
-	--host "tcp://0.0.0.0:${listen_port}" --host "unix://${socket}" \
-	--storage-driver "${DOCKER_GRAPHDRIVER}" \
-	--userland-proxy="${DOCKER_USERLANDPROXY}" \
-	$storage_params \
-	$extra_params"
+# shellcheck disable=SC2206
+args=(
+	--debug
+	--host="tcp://0.0.0.0:${listen_port}"
+	--host="unix://${socket}"
+	--storage-driver="${DOCKER_GRAPHDRIVER}"
+	--userland-proxy="${DOCKER_USERLANDPROXY}"
+	$storage_params
+	$extra_params
+)
+
+dockerd=("$DOCKER_COMMAND")
+
+if [ -n "$DELVE_PORT" ]; then
+	dockerd=(
+		dlv
+		--listen="0.0.0.0:$delve_listen_port"
+		--headless=true
+		--log
+		--api-version=2
+		--only-same-user=false
+		--check-go-version=false
+		--accept-multiclient
+		exec "${dockerd[@]}" --
+	)
+fi
+
+if [ -n "$DOCKER_ROOTLESS" ]; then
+	dockerd=(
+		sudo -u "$user"
+		-E DOCKERD="${dockerd[*]}"
+		-E XDG_RUNTIME_DIR="/tmp/docker-${uid}"
+		-E XDG_CONFIG_HOME="/home/${user}/.config"
+		-E HOME="/home/${user}"
+		--
+		dockerd-rootless.sh
+	)
+fi
 
-echo "${dockerd} ${args}"
+set -x
 # shellcheck disable=SC2086
-exec "${dockerd}" ${args}
+exec "${dockerd[@]}" "${args[@]}"