Added Delve debugger to the development container (including instructions in the contribution guide).

Signed-off-by: Konrad Ponichtera <konpon96@gmail.com>

Dockerfile

@@ -149,6 +149,18 @@ RUN --mount=type=cache,sharing=locked,id=moby-cross-true-aptlib,target=/var/lib/
 
 FROM runtime-dev-cross-${CROSS} AS runtime-dev
 
+FROM base AS delve
+# DELVE_VERSION specifies the version of the Delve debugger binary
+# from the https://github.com/go-delve/delve repository.
+# It can be used to run Docker with a possibility of
+# attaching debugger to it.
+#
+ARG DELVE_VERSION=v1.8.1
+RUN --mount=type=cache,target=/root/.cache/go-build \
+    --mount=type=cache,target=/go/pkg/mod \
+        GOBIN=/build/ GO111MODULE=on go install "github.com/go-delve/delve/cmd/dlv@${DELVE_VERSION}" \
+     && /build/dlv --help
+
 FROM base AS tomll
 # GOTOML_VERSION specifies the version of the tomll binary to build and install
 # from the https://github.com/pelletier/go-toml repository. This binary is used
@@ -301,6 +313,7 @@ RUN pip3 install yamllint==1.26.1
 COPY --from=dockercli     /build/ /usr/local/cli
 COPY --from=frozen-images /build/ /docker-frozen-images
 COPY --from=swagger       /build/ /usr/local/bin/
+COPY --from=delve         /build/ /usr/local/bin/
 COPY --from=tomll         /build/ /usr/local/bin/
 COPY --from=tini          /build/ /usr/local/bin/
 COPY --from=registry      /build/ /usr/local/bin/

Makefile

@@ -68,6 +68,7 @@ DOCKER_ENVS := \
 	-e DOCKER_TEST_HOST \
 	-e DOCKER_USERLANDPROXY \
 	-e DOCKERD_ARGS \
+	-e DELVE_PORT \
 	-e TEST_FORCE_VALIDATE \
 	-e TEST_INTEGRATION_DIR \
 	-e TEST_SKIP_INTEGRATION \
@@ -114,8 +115,9 @@ DOCKER_CONTAINER_NAME := $(if $(CONTAINER_NAME),--name $(CONTAINER_NAME),)
 
 DOCKER_IMAGE := docker-dev
 DOCKER_PORT_FORWARD := $(if $(DOCKER_PORT),-p "$(DOCKER_PORT)",)
+DELVE_PORT_FORWARD := $(if $(DELVE_PORT),-p "$(DELVE_PORT)",)
 
-DOCKER_FLAGS := $(DOCKER) run --rm -i --privileged $(DOCKER_CONTAINER_NAME) $(DOCKER_ENVS) $(DOCKER_MOUNT) $(DOCKER_PORT_FORWARD)
+DOCKER_FLAGS := $(DOCKER) run --rm -i --privileged $(DOCKER_CONTAINER_NAME) $(DOCKER_ENVS) $(DOCKER_MOUNT) $(DOCKER_PORT_FORWARD) $(DELVE_PORT_FORWARD)
 BUILD_APT_MIRROR := $(if $(DOCKER_BUILD_APT_MIRROR),--build-arg APT_MIRROR=$(DOCKER_BUILD_APT_MIRROR))
 export BUILD_APT_MIRROR
 

contrib/dockerd-rootless.sh

@@ -77,6 +77,8 @@ if [ -z "$mtu" ]; then
 	mtu=1500
 fi
 
+dockerd="${DOCKERD:-dockerd}"
+
 if [ -z "$_DOCKERD_ROOTLESS_CHILD" ]; then
 	_DOCKERD_ROOTLESS_CHILD=1
 	export _DOCKERD_ROOTLESS_CHILD
@@ -128,5 +130,7 @@ else
 		mount --rbind ${realpath_etc_ssl} /etc/ssl
 	fi
 
-	exec dockerd $@
+	# shellcheck disable=SC2068
+	# shellcheck disable=SC2086
+	exec $dockerd $@
 fi

docs/contributing/README.md

@@ -6,3 +6,4 @@
  * [Configure Git for contributing](set-up-git.md)
  * [Work with a development container](set-up-dev-env.md)
  * [Run tests and test documentation](test.md)
+ * [Debugging the daemon](debug.md)

docs/contributing/debug.md

@@ -0,0 +1,66 @@
+### Debugging the daemon
+
+The Docker daemon inside the development container can be debugged with [Delve](https://github.com/go-delve/delve).
+
+Delve debugger listens on a port, which has to be exposed outside the development container.
+Also, in order to be able to debug the daemon, it has to be compiled with the debugging symbols.
+This can be done by launching the development container with the following command:
+
+```bash
+$ make BIND_DIR=. DOCKER_DEBUG=1 DELVE_PORT=127.0.0.1:2345:2345 shell
+```
+
+The `DOCKER_DEBUG` variable disables build optimizations, allowing to debug the binary,
+while `DELVE_PORT` publishes the specified port for use with the debugger.
+
+The `DELVE_PORT` variable accepts the port in the same format as Docker CLI's `--publish` (`-p`) option.
+This means that the port can be published in multiple ways:
+
+1. `DELVE_PORT=127.0.0.1:2345:2345` - exposes debugger on port `2345` for local development only (recommended)
+2. `DELVE_PORT=2345:2345` - exposes debugger on port `2345` without binding to specific IP
+3. `DELVE_PORT=2345` - same as above
+
+**IMPORTANT:** Publishing the port without binding it to localhost (127.0.0.1) might expose the debugger
+outside the developer's machine and is not recommended.
+
+## Running Docker daemon with debugger attached
+
+1. Run development container with build optimizations disabled and Delve enabled:
+   ```bash
+   $ make BIND_DIR=. DOCKER_DEBUG=1 DELVE_PORT=127.0.0.1:2345:2345 shell
+   ```
+2. Inside the development container:
+   1. Build the Docker daemon:
+      ```bash
+      $ ./hack/make.sh binary
+      ```
+   2. Install the newly-built daemon:
+      ```bash
+      $ make install
+      ```
+   3. Run the daemon through the `make.sh` script:
+      ```bash
+      $ ./hack/make.sh run
+      ```
+      The execution will stop and wait for the IDE or Delve CLI to attach
+      to the port, specified with the `DELVE_PORT` variable.
+      Once the IDE or Delve CLI is attached, the execution will continue.
+   
+## Debugging from IDE (on example of GoLand 2021.3)
+
+1. Open the project in GoLand
+2. Click *Add Configuration* on the taskbar
+   ![GoLand - adding configuration](images/goland_add_config.png)
+3. Create the *Go Remote* configuration. 
+   No changes are necessary, unless a different port is to be used.
+   ![GoLand - adding remote configuration](images/goland_debug_config.png)
+4. Run the Docker binary in the development container, as described in the previous section.
+   Make sure that the port in the `DELVE_PORT` variable corresponds to one, used in the *Go Remote* configuration.
+5. Run the *Go Remote* configuration.
+   The Docker daemon will continue execution inside the container and debugger will stop it on the breakpoints.
+   ![GoLand - run Go Remote configuration](images/goland_run_debug_config.png)
+
+## Where to go next
+
+Congratulations, you have experienced how to use Delve to debug the Docker daemon
+and how to configure an IDE to make use of it.

docs/contributing/test.md

@@ -250,3 +250,4 @@ jobs can be triggered and re-ran by the Moby maintainers
 
 Congratulations, you have successfully completed the basics you need to
 understand the Moby test framework.
+In the next section you'll [learn how to debug Docker daemon, running inside the development container](debug.md).

hack/make/run

@@ -27,6 +27,11 @@ if [ -n "$DOCKER_PORT" ]; then
 	listen_port="${ports[-1]}"
 fi
 
+if [ -n "$DELVE_PORT" ]; then
+	IFS=':' read -r -a ports <<< "$DELVE_PORT"
+	delve_listen_port="${ports[-1]}"
+fi
+
 extra_params="$DOCKERD_ARGS"
 if [ "$DOCKER_REMAP_ROOT" ]; then
 	extra_params="$extra_params --userns-remap $DOCKER_REMAP_ROOT"
@@ -36,7 +41,7 @@ if [ -n "$DOCKER_EXPERIMENTAL" ]; then
 	extra_params="$extra_params --experimental"
 fi
 
-dockerd="dockerd"
+dockerd="$(command -v dockerd)"
 socket=/var/run/docker.sock
 if [ -n "$DOCKER_ROOTLESS" ]; then
 	user="unprivilegeduser"
@@ -44,7 +49,6 @@ if [ -n "$DOCKER_ROOTLESS" ]; then
 	# shellcheck disable=SC2174
 	mkdir -p -m 700 "/tmp/docker-${uid}"
 	chown $user "/tmp/docker-${uid}"
-	dockerd="sudo -u $user -E XDG_RUNTIME_DIR=/tmp/docker-${uid} -E HOME=/home/${user} -- dockerd-rootless.sh"
 	socket=/tmp/docker-${uid}/docker.sock
 fi
 
@@ -55,6 +59,29 @@ args="--debug \
 	$storage_params \
 	$extra_params"
 
+if [ -n "$DELVE_PORT" ]; then
+	dockerd="dlv --listen=0.0.0.0:$delve_listen_port \
+		--headless=true \
+		--log \
+		--api-version=2 \
+		--only-same-user=false \
+		--check-go-version=false \
+		--accept-multiclient \
+		exec ${dockerd} --"
+fi
+
 echo "${dockerd} ${args}"
+
+if [ -n "$DOCKER_ROOTLESS" ]; then
+	# shellcheck disable=SC2068
 	# shellcheck disable=SC2086
-exec "${dockerd}" ${args}
+	exec sudo -u $user \
+		-E DOCKERD="$dockerd" \
+		-E XDG_RUNTIME_DIR=/tmp/docker-${uid} \
+		-E XDG_CONFIG_HOME=/home/${user}/.config \
+		-E HOME=/home/${user} \
+		-- /go/src/github.com/docker/docker/contrib/dockerd-rootless.sh ${args}
+else
+	# shellcheck disable=SC2086
+	exec ${dockerd} ${args}
+fi