hack: support $DOCKER_ROOTLESS for testing rootless

```
$ DOCKER_EXPERIMENTAL=1 DOCKER_ROOTLESS=1 TEST_SKIP_INTEGRATION_CLI=1 \
 make test-integration
```

test-integration-cli is unsupported currently.
Also, tests that spawn custom daemon (testutil/daemon) are skipped.

Signed-off-by: Akihiro Suda <akihiro.suda.cz@hco.ntt.co.jp>

Dockerfile

@@ -261,7 +261,9 @@ FROM djs55/vpnkit@sha256:${VPNKIT_DIGEST} AS vpnkit
 FROM runtime-dev AS dev-systemd-false
 ARG DEBIAN_FRONTEND
 RUN groupadd -r docker
-RUN useradd --create-home --gid docker unprivilegeduser
+RUN useradd --create-home --gid docker unprivilegeduser \
+ && mkdir -p /home/unprivilegeduser/.local/share/docker \
+ && chown -R unprivilegeduser /home/unprivilegeduser
 # Let us use a .bashrc file
 RUN ln -sfv /go/src/github.com/docker/docker/.bashrc ~/.bashrc
 # Activate bash completion and include Docker's completion if mounted with DOCKER_BASH_COMPLETION_PATH
@@ -288,7 +290,9 @@ RUN --mount=type=cache,sharing=locked,id=moby-dev-aptlib,target=/var/lib/apt \
             python3-pip \
             python3-setuptools \
             python3-wheel \
+            sudo \
             thin-provisioning-tools \
+            uidmap \
             vim \
             vim-common \
             xfsprogs \
@@ -325,6 +329,7 @@ ARG DOCKER_BUILDTAGS
 ENV DOCKER_BUILDTAGS="${DOCKER_BUILDTAGS}"
 WORKDIR /go/src/github.com/docker/docker
 VOLUME /var/lib/docker
+VOLUME /home/unprivilegeduser/.local/share/docker
 # Wrap all commands in the "docker-in-docker" script to allow nested containers
 ENTRYPOINT ["hack/dind"]
 

Jenkinsfile

@@ -11,6 +11,7 @@ pipeline {
         booleanParam(name: 'unit_validate', defaultValue: true, description: 'amd64 (x86_64) unit tests and vendor check')
         booleanParam(name: 'validate_force', defaultValue: false, description: 'force validation steps to be run, even if no changes were detected')
         booleanParam(name: 'amd64', defaultValue: true, description: 'amd64 (x86_64) Build/Test')
+        booleanParam(name: 'rootless', defaultValue: true, description: 'amd64 (x86_64) Build/Test (Rootless mode)')
         booleanParam(name: 'arm64', defaultValue: true, description: 'ARM (arm64) Build/Test')
         booleanParam(name: 's390x', defaultValue: true, description: 'IBM Z (s390x) Build/Test')
         booleanParam(name: 'ppc64le', defaultValue: true, description: 'PowerPC (ppc64le) Build/Test')
@@ -380,6 +381,94 @@ pipeline {
                         }
                     }
                 }
+                stage('rootless') {
+                    when {
+                        beforeAgent true
+                        expression { params.rootless }
+                    }
+                    agent { label 'amd64 && ubuntu-1804 && overlay2' }
+                    stages {
+                        stage("Print info") {
+                            steps {
+                                sh 'docker version'
+                                sh 'docker info'
+                                sh '''
+                                echo "check-config.sh version: ${CHECK_CONFIG_COMMIT}"
+                                curl -fsSL -o ${WORKSPACE}/check-config.sh "https://raw.githubusercontent.com/moby/moby/${CHECK_CONFIG_COMMIT}/contrib/check-config.sh" \
+                                && bash ${WORKSPACE}/check-config.sh || true
+                                '''
+                            }
+                        }
+                        stage("Build dev image") {
+                            steps {
+                                sh '''
+                                docker build --force-rm --build-arg APT_MIRROR -t docker:${GIT_COMMIT} .
+                                '''
+                            }
+                        }
+                        stage("Integration tests") {
+                            environment {
+                                DOCKER_EXPERIMENTAL = '1'
+                                DOCKER_ROOTLESS = '1'
+                                TEST_SKIP_INTEGRATION_CLI = '1'
+                            }
+                            steps {
+                                sh '''
+                                docker run --rm -t --privileged \
+                                  -v "$WORKSPACE/bundles:/go/src/github.com/docker/docker/bundles" \
+                                  --name docker-pr$BUILD_NUMBER \
+                                  -e DOCKER_GITCOMMIT=${GIT_COMMIT} \
+                                  -e DOCKER_GRAPHDRIVER \
+                                  -e DOCKER_EXPERIMENTAL \
+                                  -e DOCKER_ROOTLESS \
+                                  -e TEST_SKIP_INTEGRATION_CLI \
+                                  -e TIMEOUT \
+                                  -e VALIDATE_REPO=${GIT_URL} \
+                                  -e VALIDATE_BRANCH=${CHANGE_TARGET} \
+                                  docker:${GIT_COMMIT} \
+                                  hack/make.sh \
+                                    dynbinary \
+                                    test-integration
+                                '''
+                            }
+                            post {
+                                always {
+                                    junit testResults: 'bundles/**/*-report.xml', allowEmptyResults: true
+                                }
+                            }
+                        }
+                    }
+
+                    post {
+                        always {
+                            sh '''
+                            echo "Ensuring container killed."
+                            docker rm -vf docker-pr$BUILD_NUMBER || true
+                            '''
+
+                            sh '''
+                            echo "Chowning /workspace to jenkins user"
+                            docker run --rm -v "$WORKSPACE:/workspace" busybox chown -R "$(id -u):$(id -g)" /workspace
+                            '''
+
+                            catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE', message: 'Failed to create bundles.tar.gz') {
+                                sh '''
+                                bundleName=amd64-rootless
+                                echo "Creating ${bundleName}-bundles.tar.gz"
+                                # exclude overlay2 directories
+                                find bundles -path '*/root/*overlay2' -prune -o -type f \\( -name '*-report.json' -o -name '*.log' -o -name '*.prof' -o -name '*-report.xml' \\) -print | xargs tar -czf ${bundleName}-bundles.tar.gz
+                                '''
+
+                                archiveArtifacts artifacts: '*-bundles.tar.gz', allowEmptyArchive: true
+                            }
+                        }
+                        cleanup {
+                            sh 'make clean'
+                            deleteDir()
+                        }
+                    }
+                }
+
                 stage('s390x') {
                     when {
                         beforeAgent true

Makefile

@@ -61,6 +61,7 @@ DOCKER_ENVS := \
 	-e DOCKER_LDFLAGS \
 	-e DOCKER_PORT \
 	-e DOCKER_REMAP_ROOT \
+	-e DOCKER_ROOTLESS \
 	-e DOCKER_STORAGE_OPTS \
 	-e DOCKER_TEST_HOST \
 	-e DOCKER_USERLANDPROXY \

hack/make/.integration-daemon-start

@@ -63,6 +63,26 @@ if [ "$DOCKER_EXPERIMENTAL" ]; then
 	extra_params="$extra_params --experimental"
 fi
 
+dockerd="dockerd"
+if [ -n "$DOCKER_ROOTLESS" ]; then
+	if [ -z "$DOCKER_EXPERIMENTAL" ]; then
+		echo >&2 '# DOCKER_ROOTLESS requires DOCKER_EXPERIMENTAL to be set'
+		exit 1
+	fi
+	if [ -z "$TEST_SKIP_INTEGRATION_CLI" ]; then
+		echo >&2 '# DOCKER_ROOTLESS requires TEST_SKIP_INTEGRATION_CLI to be set'
+		exit 1
+	fi
+	ln -sf "$(command -v vpnkit."$(uname -m)")" /usr/local/bin/vpnkit
+	user="unprivilegeduser"
+	uid=$(id -u $user)
+	# shellcheck disable=SC2174
+	mkdir -p -m 700 "/tmp/docker-${uid}"
+	chown "$user" "/tmp/docker-${uid}"
+	chmod -R o+w "$DEST"
+	dockerd="sudo -u $user -E -E XDG_RUNTIME_DIR=/tmp/docker-${uid} -E HOME=/home/${user} -E PATH=$PATH -- dockerd-rootless.sh"
+fi
+
 if [ -z "$DOCKER_TEST_HOST" ]; then
 	# Start apparmor if it is enabled
 	if [ -e "/sys/module/apparmor/parameters/enabled" ] && [ "$(cat /sys/module/apparmor/parameters/enabled)" == "Y" ]; then
@@ -81,7 +101,7 @@ if [ -z "$DOCKER_TEST_HOST" ]; then
 		echo "Starting dockerd"
 		[ -n "$TESTDEBUG" ] && set -x
 		exec \
-			dockerd --debug \
+			${dockerd} --debug \
 			--host "$DOCKER_HOST" \
 			--storage-driver "$DOCKER_GRAPHDRIVER" \
 			--pidfile "$DEST/docker.pid" \

hack/make/.integration-test-helpers

@@ -147,6 +147,7 @@ test_env() {
 			DOCKER_HOST="$DOCKER_HOST" \
 			DOCKER_REMAP_ROOT="$DOCKER_REMAP_ROOT" \
 			DOCKER_REMOTE_DAEMON="$DOCKER_REMOTE_DAEMON" \
+			DOCKER_ROOTLESS="$DOCKER_ROOTLESS" \
 			DOCKERFILE="$DOCKERFILE" \
 			GOPATH="$GOPATH" \
 			GOTRACEBACK=all \

hack/make/run

@@ -32,12 +32,33 @@ if [ "$DOCKER_REMAP_ROOT" ]; then
 	extra_params="$extra_params --userns-remap $DOCKER_REMAP_ROOT"
 fi
 
+if [ -n "$DOCKER_EXPERIMENTAL" ]; then
+	extra_params="$extra_params --experimental"
+fi
+
+dockerd="dockerd"
+socket=/var/run/docker.sock
+if [ -n "$DOCKER_ROOTLESS" ]; then
+	if [ -z "$DOCKER_EXPERIMENTAL" ]; then
+		echo >&2 '# DOCKER_ROOTLESS requires DOCKER_EXPERIMENTAL to be set'
+		exit 1
+	fi
+	user="unprivilegeduser"
+	uid=$(id -u $user)
+	# shellcheck disable=SC2174
+	mkdir -p -m 700 "/tmp/docker-${uid}"
+	chown $user "/tmp/docker-${uid}"
+	dockerd="sudo -u $user -E XDG_RUNTIME_DIR=/tmp/docker-${uid} -E HOME=/home/${user} -- dockerd-rootless.sh"
+	socket=/tmp/docker-${uid}/docker.sock
+fi
+
 args="--debug \
-	--host tcp://0.0.0.0:${listen_port} --host unix:///var/run/docker.sock \
+	--host "tcp://0.0.0.0:${listen_port}" --host "unix://${socket}" \
 	--storage-driver "${DOCKER_GRAPHDRIVER}" \
 	--userland-proxy="${DOCKER_USERLANDPROXY}" \
 	$storage_params \
 	$extra_params"
 
-echo dockerd ${args}
+echo "${dockerd} ${args}"
-exec dockerd ${args}
+# shellcheck disable=SC2086
+exec "${dockerd}" ${args}

integration/container/ipcmode_linux_test.go

@@ -116,6 +116,7 @@ func TestIpcModePrivate(t *testing.T) {
 // also exists on the host.
 func TestIpcModeShareable(t *testing.T) {
 	skip.If(t, testEnv.IsRemoteDaemon)
+	skip.If(t, testEnv.IsRootless, "cannot test /dev/shm in rootless")
 
 	testIpcNonePrivateShareable(t, "shareable", true, true)
 }
@@ -191,6 +192,7 @@ func TestAPIIpcModeShareableAndContainer(t *testing.T) {
 func TestAPIIpcModeHost(t *testing.T) {
 	skip.If(t, testEnv.IsRemoteDaemon)
 	skip.If(t, testEnv.IsUserNamespace)
+	skip.If(t, testEnv.IsRootless, "cannot test /dev/shm in rootless")
 
 	cfg := containertypes.Config{
 		Image: "busybox",
@@ -262,6 +264,7 @@ func testDaemonIpcPrivateShareable(t *testing.T, mustBeShared bool, arg ...strin
 // TestDaemonIpcModeShareable checks that --default-ipc-mode shareable works as intended.
 func TestDaemonIpcModeShareable(t *testing.T) {
 	skip.If(t, testEnv.IsRemoteDaemon)
+	skip.If(t, testEnv.IsRootless, "cannot test /dev/shm in rootless")
 
 	testDaemonIpcPrivateShareable(t, true, "--default-ipc-mode", "shareable")
 }

integration/container/kill_test.go

@@ -148,7 +148,9 @@ func TestKillDifferentUserContainer(t *testing.T) {
 }
 
 func TestInspectOomKilledTrue(t *testing.T) {
-	skip.If(t, testEnv.DaemonInfo.OSType == "windows" || !testEnv.DaemonInfo.MemoryLimit || !testEnv.DaemonInfo.SwapLimit)
+	skip.If(t, testEnv.DaemonInfo.OSType == "windows")
+	skip.If(t, testEnv.DaemonInfo.CgroupDriver == "none")
+	skip.If(t, !testEnv.DaemonInfo.MemoryLimit || !testEnv.DaemonInfo.SwapLimit)
 
 	defer setupTest(t)()
 	ctx := context.Background()

integration/container/links_linux_test.go

@@ -16,6 +16,7 @@ import (
 
 func TestLinksEtcHostsContentMatch(t *testing.T) {
 	skip.If(t, testEnv.IsRemoteDaemon)
+	skip.If(t, testEnv.IsRootless, "rootless mode has different view of /etc/hosts")
 
 	hosts, err := ioutil.ReadFile("/etc/hosts")
 	skip.If(t, os.IsNotExist(err))

integration/container/mounts_linux_test.go

@@ -214,6 +214,7 @@ func TestMountDaemonRoot(t *testing.T) {
 func TestContainerBindMountNonRecursive(t *testing.T) {
 	skip.If(t, testEnv.IsRemoteDaemon)
 	skip.If(t, versions.LessThan(testEnv.DaemonAPIVersion(), "1.40"), "BindOptions.NonRecursive requires API v1.40")
+	skip.If(t, testEnv.IsRootless, "cannot be tested because RootlessKit executes the daemon in private mount namespace (https://github.com/rootless-containers/rootlesskit/issues/97)")
 
 	defer setupTest(t)()
 

integration/container/pause_test.go

@@ -20,6 +20,7 @@ import (
 
 func TestPause(t *testing.T) {
 	skip.If(t, testEnv.DaemonInfo.OSType == "windows" && testEnv.DaemonInfo.Isolation == "process")
+	skip.If(t, testEnv.DaemonInfo.CgroupDriver == "none")
 
 	defer setupTest(t)()
 	client := testEnv.APIClient()
@@ -67,6 +68,7 @@ func TestPauseFailsOnWindowsServerContainers(t *testing.T) {
 func TestPauseStopPausedContainer(t *testing.T) {
 	skip.If(t, testEnv.DaemonInfo.OSType == "windows")
 	skip.If(t, versions.LessThan(testEnv.DaemonAPIVersion(), "1.31"), "broken in earlier versions")
+	skip.If(t, testEnv.DaemonInfo.CgroupDriver == "none")
 	defer setupTest(t)()
 	client := testEnv.APIClient()
 	ctx := context.Background()

integration/container/run_linux_test.go

@@ -20,6 +20,7 @@ import (
 func TestKernelTCPMemory(t *testing.T) {
 	skip.If(t, testEnv.DaemonInfo.OSType != "linux")
 	skip.If(t, versions.LessThan(testEnv.DaemonAPIVersion(), "1.40"), "skip test from new feature")
+	skip.If(t, testEnv.DaemonInfo.CgroupDriver == "none")
 	skip.If(t, !testEnv.DaemonInfo.KernelMemoryTCP)
 
 	defer setupTest(t)()
@@ -57,6 +58,11 @@ func TestNISDomainname(t *testing.T) {
 	skip.If(t, versions.LessThan(testEnv.DaemonAPIVersion(), "1.40"), "skip test from new feature")
 	skip.If(t, testEnv.DaemonInfo.OSType != "linux")
 
+	// Rootless supports custom Hostname but doesn't support custom Domainname
+	//  OCI runtime create failed: container_linux.go:349: starting container process caused "process_linux.go:449: container init caused \
+	//  "write sysctl key kernel.domainname: open /proc/sys/kernel/domainname: permission denied\"": unknown.
+	skip.If(t, testEnv.IsRootless, "rootless mode doesn't support setting Domainname (TODO: https://github.com/moby/moby/issues/40632)")
+
 	defer setupTest(t)()
 	client := testEnv.APIClient()
 	ctx := context.Background()

integration/container/stats_test.go

@@ -16,6 +16,7 @@ import (
 )
 
 func TestStats(t *testing.T) {
+	skip.If(t, testEnv.DaemonInfo.CgroupDriver == "none")
 	skip.If(t, !testEnv.DaemonInfo.MemoryLimit)
 
 	defer setupTest(t)()

integration/container/update_linux_test.go

@@ -19,6 +19,7 @@ import (
 
 func TestUpdateMemory(t *testing.T) {
 	skip.If(t, testEnv.DaemonInfo.OSType == "windows")
+	skip.If(t, testEnv.DaemonInfo.CgroupDriver == "none")
 	skip.If(t, !testEnv.DaemonInfo.MemoryLimit)
 	skip.If(t, !testEnv.DaemonInfo.SwapLimit)
 
@@ -68,6 +69,7 @@ func TestUpdateMemory(t *testing.T) {
 }
 
 func TestUpdateCPUQuota(t *testing.T) {
+	skip.If(t, testEnv.DaemonInfo.CgroupDriver == "none")
 	defer setupTest(t)()
 	client := testEnv.APIClient()
 	ctx := context.Background()
@@ -106,6 +108,7 @@ func TestUpdateCPUQuota(t *testing.T) {
 
 func TestUpdatePidsLimit(t *testing.T) {
 	skip.If(t, testEnv.DaemonInfo.OSType == "windows")
+	skip.If(t, testEnv.DaemonInfo.CgroupDriver == "none")
 	skip.If(t, !testEnv.DaemonInfo.PidsLimit)
 
 	defer setupTest(t)()

integration/network/service_test.go

@@ -29,6 +29,7 @@ func TestDaemonRestartWithLiveRestore(t *testing.T) {
 	skip.If(t, testEnv.OSType == "windows")
 	skip.If(t, testEnv.IsRemoteDaemon)
 	skip.If(t, versions.LessThan(testEnv.DaemonAPIVersion(), "1.38"), "skip test from new feature")
+	skip.If(t, testEnv.IsRootless, "rootless mode has different view of network")
 	d := daemon.New(t)
 	defer d.Stop(t)
 	d.Start(t)
@@ -52,6 +53,7 @@ func TestDaemonDefaultNetworkPools(t *testing.T) {
 	// Remove docker0 bridge and the start daemon defining the predefined address pools
 	skip.If(t, testEnv.IsRemoteDaemon)
 	skip.If(t, versions.LessThan(testEnv.DaemonAPIVersion(), "1.38"), "skip test from new feature")
+	skip.If(t, testEnv.IsRootless, "rootless mode has different view of network")
 	defaultNetworkBridge := "docker0"
 	delInterface(t, defaultNetworkBridge)
 	d := daemon.New(t)
@@ -94,6 +96,7 @@ func TestDaemonRestartWithExistingNetwork(t *testing.T) {
 	skip.If(t, testEnv.OSType == "windows")
 	skip.If(t, testEnv.IsRemoteDaemon)
 	skip.If(t, versions.LessThan(testEnv.DaemonAPIVersion(), "1.38"), "skip test from new feature")
+	skip.If(t, testEnv.IsRootless, "rootless mode has different view of network")
 	defaultNetworkBridge := "docker0"
 	d := daemon.New(t)
 	d.Start(t)
@@ -127,6 +130,7 @@ func TestDaemonRestartWithExistingNetworkWithDefaultPoolRange(t *testing.T) {
 	skip.If(t, testEnv.OSType == "windows")
 	skip.If(t, testEnv.IsRemoteDaemon)
 	skip.If(t, versions.LessThan(testEnv.DaemonAPIVersion(), "1.38"), "skip test from new feature")
+	skip.If(t, testEnv.IsRootless, "rootless mode has different view of network")
 	defaultNetworkBridge := "docker0"
 	d := daemon.New(t)
 	d.Start(t)
@@ -177,6 +181,7 @@ func TestDaemonWithBipAndDefaultNetworkPool(t *testing.T) {
 	skip.If(t, testEnv.OSType == "windows")
 	skip.If(t, testEnv.IsRemoteDaemon)
 	skip.If(t, versions.LessThan(testEnv.DaemonAPIVersion(), "1.38"), "skip test from new feature")
+	skip.If(t, testEnv.IsRootless, "rootless mode has different view of network")
 	defaultNetworkBridge := "docker0"
 	d := daemon.New(t)
 	defer d.Stop(t)
@@ -199,6 +204,7 @@ func TestDaemonWithBipAndDefaultNetworkPool(t *testing.T) {
 
 func TestServiceWithPredefinedNetwork(t *testing.T) {
 	skip.If(t, testEnv.OSType == "windows")
+	skip.If(t, testEnv.IsRootless, "rootless mode doesn't support Swarm-mode")
 	defer setupTest(t)()
 	d := swarm.NewSwarm(t, testEnv)
 	defer d.Stop(t)
@@ -228,6 +234,7 @@ const ingressNet = "ingress"
 
 func TestServiceRemoveKeepsIngressNetwork(t *testing.T) {
 	t.Skip("FLAKY_TEST")
+	skip.If(t, testEnv.IsRootless, "rootless mode doesn't support Swarm-mode")
 
 	skip.If(t, testEnv.OSType == "windows")
 	defer setupTest(t)()
@@ -318,6 +325,7 @@ func noServices(ctx context.Context, client client.ServiceAPIClient) func(log po
 func TestServiceWithDataPathPortInit(t *testing.T) {
 	skip.If(t, testEnv.OSType == "windows")
 	skip.If(t, versions.LessThan(testEnv.DaemonAPIVersion(), "1.40"), "DataPathPort was added in API v1.40")
+	skip.If(t, testEnv.IsRootless, "rootless mode doesn't support Swarm-mode")
 	defer setupTest(t)()
 	var datapathPort uint32 = 7777
 	d := swarm.NewSwarm(t, testEnv, daemon.WithSwarmDataPathPort(datapathPort))
@@ -384,6 +392,7 @@ func TestServiceWithDataPathPortInit(t *testing.T) {
 
 func TestServiceWithDefaultAddressPoolInit(t *testing.T) {
 	skip.If(t, testEnv.OSType == "windows")
+	skip.If(t, testEnv.IsRootless, "rootless mode doesn't support Swarm-mode")
 	defer setupTest(t)()
 	d := swarm.NewSwarm(t, testEnv,
 		daemon.WithSwarmDefaultAddrPool([]string{"20.20.0.0/16"}),

testutil/daemon/daemon.go

@@ -151,6 +151,10 @@ func New(t testing.TB, ops ...Option) *Daemon {
 
 	assert.Check(t, dest != "", "Please set the DOCKER_INTEGRATION_DAEMON_DEST or the DEST environment variable")
 
+	if os.Getenv("DOCKER_ROOTLESS") != "" {
+		t.Skip("github.com/docker/docker/testutil/daemon.Daemon doesn't support DOCKER_ROOTLESS")
+	}
+
 	d, err := NewDaemon(dest, ops...)
 	assert.NilError(t, err, "could not create daemon at %q", dest)
 
@@ -227,6 +231,9 @@ func (d *Daemon) Cleanup(t testing.TB) {
 // Start starts the daemon and return once it is ready to receive requests.
 func (d *Daemon) Start(t testing.TB, args ...string) {
 	t.Helper()
+	if os.Getenv("DOCKER_ROOTLESS") != "" {
+		t.Skip("github.com/docker/docker/testutil/daemon.Daemon doesn't support DOCKER_ROOTLESS")
+	}
 	if err := d.StartWithError(args...); err != nil {
 		t.Fatalf("[%s] failed to start daemon with arguments %v : %v", d.id, d.args, err)
 	}

testutil/environment/environment.go

@@ -162,6 +162,11 @@ func (e *Execution) IsUserNamespace() bool {
 	return root != ""
 }
 
+// IsRootless returns whether the rootless mode is enabled
+func (e *Execution) IsRootless() bool {
+	return os.Getenv("DOCKER_ROOTLESS") != ""
+}
+
 // HasExistingImage checks whether there is an image with the given reference.
 // Note that this is done by filtering and then checking whether there were any
 // results -- so ambiguous references might result in false-positives.