hack: support $DOCKER_ROOTLESS for testing rootless

```
$ DOCKER_EXPERIMENTAL=1 DOCKER_ROOTLESS=1 TEST_SKIP_INTEGRATION_CLI=1 \
 make test-integration
```

test-integration-cli is unsupported currently.
Also, tests that spawn custom daemon (testutil/daemon) are skipped.

Signed-off-by: Akihiro Suda <akihiro.suda.cz@hco.ntt.co.jp>
This commit is contained in:
Akihiro Suda 2020-02-18 18:43:56 +09:00
parent e67f42e21d
commit 5bf33adba2
17 changed files with 183 additions and 6 deletions

View file

@ -261,7 +261,9 @@ FROM djs55/vpnkit@sha256:${VPNKIT_DIGEST} AS vpnkit
FROM runtime-dev AS dev-systemd-false
ARG DEBIAN_FRONTEND
RUN groupadd -r docker
RUN useradd --create-home --gid docker unprivilegeduser
RUN useradd --create-home --gid docker unprivilegeduser \
&& mkdir -p /home/unprivilegeduser/.local/share/docker \
&& chown -R unprivilegeduser /home/unprivilegeduser
# Let us use a .bashrc file
RUN ln -sfv /go/src/github.com/docker/docker/.bashrc ~/.bashrc
# Activate bash completion and include Docker's completion if mounted with DOCKER_BASH_COMPLETION_PATH
@ -288,7 +290,9 @@ RUN --mount=type=cache,sharing=locked,id=moby-dev-aptlib,target=/var/lib/apt \
python3-pip \
python3-setuptools \
python3-wheel \
sudo \
thin-provisioning-tools \
uidmap \
vim \
vim-common \
xfsprogs \
@ -325,6 +329,7 @@ ARG DOCKER_BUILDTAGS
ENV DOCKER_BUILDTAGS="${DOCKER_BUILDTAGS}"
WORKDIR /go/src/github.com/docker/docker
VOLUME /var/lib/docker
VOLUME /home/unprivilegeduser/.local/share/docker
# Wrap all commands in the "docker-in-docker" script to allow nested containers
ENTRYPOINT ["hack/dind"]

89
Jenkinsfile vendored
View file

@ -11,6 +11,7 @@ pipeline {
booleanParam(name: 'unit_validate', defaultValue: true, description: 'amd64 (x86_64) unit tests and vendor check')
booleanParam(name: 'validate_force', defaultValue: false, description: 'force validation steps to be run, even if no changes were detected')
booleanParam(name: 'amd64', defaultValue: true, description: 'amd64 (x86_64) Build/Test')
booleanParam(name: 'rootless', defaultValue: true, description: 'amd64 (x86_64) Build/Test (Rootless mode)')
booleanParam(name: 'arm64', defaultValue: true, description: 'ARM (arm64) Build/Test')
booleanParam(name: 's390x', defaultValue: true, description: 'IBM Z (s390x) Build/Test')
booleanParam(name: 'ppc64le', defaultValue: true, description: 'PowerPC (ppc64le) Build/Test')
@ -380,6 +381,94 @@ pipeline {
}
}
}
stage('rootless') {
when {
beforeAgent true
expression { params.rootless }
}
agent { label 'amd64 && ubuntu-1804 && overlay2' }
stages {
stage("Print info") {
steps {
sh 'docker version'
sh 'docker info'
sh '''
echo "check-config.sh version: ${CHECK_CONFIG_COMMIT}"
curl -fsSL -o ${WORKSPACE}/check-config.sh "https://raw.githubusercontent.com/moby/moby/${CHECK_CONFIG_COMMIT}/contrib/check-config.sh" \
&& bash ${WORKSPACE}/check-config.sh || true
'''
}
}
stage("Build dev image") {
steps {
sh '''
docker build --force-rm --build-arg APT_MIRROR -t docker:${GIT_COMMIT} .
'''
}
}
stage("Integration tests") {
environment {
DOCKER_EXPERIMENTAL = '1'
DOCKER_ROOTLESS = '1'
TEST_SKIP_INTEGRATION_CLI = '1'
}
steps {
sh '''
docker run --rm -t --privileged \
-v "$WORKSPACE/bundles:/go/src/github.com/docker/docker/bundles" \
--name docker-pr$BUILD_NUMBER \
-e DOCKER_GITCOMMIT=${GIT_COMMIT} \
-e DOCKER_GRAPHDRIVER \
-e DOCKER_EXPERIMENTAL \
-e DOCKER_ROOTLESS \
-e TEST_SKIP_INTEGRATION_CLI \
-e TIMEOUT \
-e VALIDATE_REPO=${GIT_URL} \
-e VALIDATE_BRANCH=${CHANGE_TARGET} \
docker:${GIT_COMMIT} \
hack/make.sh \
dynbinary \
test-integration
'''
}
post {
always {
junit testResults: 'bundles/**/*-report.xml', allowEmptyResults: true
}
}
}
}
post {
always {
sh '''
echo "Ensuring container killed."
docker rm -vf docker-pr$BUILD_NUMBER || true
'''
sh '''
echo "Chowning /workspace to jenkins user"
docker run --rm -v "$WORKSPACE:/workspace" busybox chown -R "$(id -u):$(id -g)" /workspace
'''
catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE', message: 'Failed to create bundles.tar.gz') {
sh '''
bundleName=amd64-rootless
echo "Creating ${bundleName}-bundles.tar.gz"
# exclude overlay2 directories
find bundles -path '*/root/*overlay2' -prune -o -type f \\( -name '*-report.json' -o -name '*.log' -o -name '*.prof' -o -name '*-report.xml' \\) -print | xargs tar -czf ${bundleName}-bundles.tar.gz
'''
archiveArtifacts artifacts: '*-bundles.tar.gz', allowEmptyArchive: true
}
}
cleanup {
sh 'make clean'
deleteDir()
}
}
}
stage('s390x') {
when {
beforeAgent true

View file

@ -61,6 +61,7 @@ DOCKER_ENVS := \
-e DOCKER_LDFLAGS \
-e DOCKER_PORT \
-e DOCKER_REMAP_ROOT \
-e DOCKER_ROOTLESS \
-e DOCKER_STORAGE_OPTS \
-e DOCKER_TEST_HOST \
-e DOCKER_USERLANDPROXY \

View file

@ -63,6 +63,26 @@ if [ "$DOCKER_EXPERIMENTAL" ]; then
extra_params="$extra_params --experimental"
fi
dockerd="dockerd"
if [ -n "$DOCKER_ROOTLESS" ]; then
if [ -z "$DOCKER_EXPERIMENTAL" ]; then
echo >&2 '# DOCKER_ROOTLESS requires DOCKER_EXPERIMENTAL to be set'
exit 1
fi
if [ -z "$TEST_SKIP_INTEGRATION_CLI" ]; then
echo >&2 '# DOCKER_ROOTLESS requires TEST_SKIP_INTEGRATION_CLI to be set'
exit 1
fi
ln -sf "$(command -v vpnkit."$(uname -m)")" /usr/local/bin/vpnkit
user="unprivilegeduser"
uid=$(id -u $user)
# shellcheck disable=SC2174
mkdir -p -m 700 "/tmp/docker-${uid}"
chown "$user" "/tmp/docker-${uid}"
chmod -R o+w "$DEST"
dockerd="sudo -u $user -E -E XDG_RUNTIME_DIR=/tmp/docker-${uid} -E HOME=/home/${user} -E PATH=$PATH -- dockerd-rootless.sh"
fi
if [ -z "$DOCKER_TEST_HOST" ]; then
# Start apparmor if it is enabled
if [ -e "/sys/module/apparmor/parameters/enabled" ] && [ "$(cat /sys/module/apparmor/parameters/enabled)" == "Y" ]; then
@ -81,7 +101,7 @@ if [ -z "$DOCKER_TEST_HOST" ]; then
echo "Starting dockerd"
[ -n "$TESTDEBUG" ] && set -x
exec \
dockerd --debug \
${dockerd} --debug \
--host "$DOCKER_HOST" \
--storage-driver "$DOCKER_GRAPHDRIVER" \
--pidfile "$DEST/docker.pid" \

View file

@ -147,6 +147,7 @@ test_env() {
DOCKER_HOST="$DOCKER_HOST" \
DOCKER_REMAP_ROOT="$DOCKER_REMAP_ROOT" \
DOCKER_REMOTE_DAEMON="$DOCKER_REMOTE_DAEMON" \
DOCKER_ROOTLESS="$DOCKER_ROOTLESS" \
DOCKERFILE="$DOCKERFILE" \
GOPATH="$GOPATH" \
GOTRACEBACK=all \

View file

@ -32,12 +32,33 @@ if [ "$DOCKER_REMAP_ROOT" ]; then
extra_params="$extra_params --userns-remap $DOCKER_REMAP_ROOT"
fi
if [ -n "$DOCKER_EXPERIMENTAL" ]; then
extra_params="$extra_params --experimental"
fi
dockerd="dockerd"
socket=/var/run/docker.sock
if [ -n "$DOCKER_ROOTLESS" ]; then
if [ -z "$DOCKER_EXPERIMENTAL" ]; then
echo >&2 '# DOCKER_ROOTLESS requires DOCKER_EXPERIMENTAL to be set'
exit 1
fi
user="unprivilegeduser"
uid=$(id -u $user)
# shellcheck disable=SC2174
mkdir -p -m 700 "/tmp/docker-${uid}"
chown $user "/tmp/docker-${uid}"
dockerd="sudo -u $user -E XDG_RUNTIME_DIR=/tmp/docker-${uid} -E HOME=/home/${user} -- dockerd-rootless.sh"
socket=/tmp/docker-${uid}/docker.sock
fi
args="--debug \
--host tcp://0.0.0.0:${listen_port} --host unix:///var/run/docker.sock \
--host "tcp://0.0.0.0:${listen_port}" --host "unix://${socket}" \
--storage-driver "${DOCKER_GRAPHDRIVER}" \
--userland-proxy="${DOCKER_USERLANDPROXY}" \
$storage_params \
$extra_params"
echo dockerd ${args}
exec dockerd ${args}
echo "${dockerd} ${args}"
# shellcheck disable=SC2086
exec "${dockerd}" ${args}

View file

@ -116,6 +116,7 @@ func TestIpcModePrivate(t *testing.T) {
// also exists on the host.
func TestIpcModeShareable(t *testing.T) {
skip.If(t, testEnv.IsRemoteDaemon)
skip.If(t, testEnv.IsRootless, "cannot test /dev/shm in rootless")
testIpcNonePrivateShareable(t, "shareable", true, true)
}
@ -191,6 +192,7 @@ func TestAPIIpcModeShareableAndContainer(t *testing.T) {
func TestAPIIpcModeHost(t *testing.T) {
skip.If(t, testEnv.IsRemoteDaemon)
skip.If(t, testEnv.IsUserNamespace)
skip.If(t, testEnv.IsRootless, "cannot test /dev/shm in rootless")
cfg := containertypes.Config{
Image: "busybox",
@ -262,6 +264,7 @@ func testDaemonIpcPrivateShareable(t *testing.T, mustBeShared bool, arg ...strin
// TestDaemonIpcModeShareable checks that --default-ipc-mode shareable works as intended.
func TestDaemonIpcModeShareable(t *testing.T) {
skip.If(t, testEnv.IsRemoteDaemon)
skip.If(t, testEnv.IsRootless, "cannot test /dev/shm in rootless")
testDaemonIpcPrivateShareable(t, true, "--default-ipc-mode", "shareable")
}

View file

@ -148,7 +148,9 @@ func TestKillDifferentUserContainer(t *testing.T) {
}
func TestInspectOomKilledTrue(t *testing.T) {
skip.If(t, testEnv.DaemonInfo.OSType == "windows" || !testEnv.DaemonInfo.MemoryLimit || !testEnv.DaemonInfo.SwapLimit)
skip.If(t, testEnv.DaemonInfo.OSType == "windows")
skip.If(t, testEnv.DaemonInfo.CgroupDriver == "none")
skip.If(t, !testEnv.DaemonInfo.MemoryLimit || !testEnv.DaemonInfo.SwapLimit)
defer setupTest(t)()
ctx := context.Background()

View file

@ -16,6 +16,7 @@ import (
func TestLinksEtcHostsContentMatch(t *testing.T) {
skip.If(t, testEnv.IsRemoteDaemon)
skip.If(t, testEnv.IsRootless, "rootless mode has different view of /etc/hosts")
hosts, err := ioutil.ReadFile("/etc/hosts")
skip.If(t, os.IsNotExist(err))

View file

@ -214,6 +214,7 @@ func TestMountDaemonRoot(t *testing.T) {
func TestContainerBindMountNonRecursive(t *testing.T) {
skip.If(t, testEnv.IsRemoteDaemon)
skip.If(t, versions.LessThan(testEnv.DaemonAPIVersion(), "1.40"), "BindOptions.NonRecursive requires API v1.40")
skip.If(t, testEnv.IsRootless, "cannot be tested because RootlessKit executes the daemon in private mount namespace (https://github.com/rootless-containers/rootlesskit/issues/97)")
defer setupTest(t)()

View file

@ -20,6 +20,7 @@ import (
func TestPause(t *testing.T) {
skip.If(t, testEnv.DaemonInfo.OSType == "windows" && testEnv.DaemonInfo.Isolation == "process")
skip.If(t, testEnv.DaemonInfo.CgroupDriver == "none")
defer setupTest(t)()
client := testEnv.APIClient()
@ -67,6 +68,7 @@ func TestPauseFailsOnWindowsServerContainers(t *testing.T) {
func TestPauseStopPausedContainer(t *testing.T) {
skip.If(t, testEnv.DaemonInfo.OSType == "windows")
skip.If(t, versions.LessThan(testEnv.DaemonAPIVersion(), "1.31"), "broken in earlier versions")
skip.If(t, testEnv.DaemonInfo.CgroupDriver == "none")
defer setupTest(t)()
client := testEnv.APIClient()
ctx := context.Background()

View file

@ -20,6 +20,7 @@ import (
func TestKernelTCPMemory(t *testing.T) {
skip.If(t, testEnv.DaemonInfo.OSType != "linux")
skip.If(t, versions.LessThan(testEnv.DaemonAPIVersion(), "1.40"), "skip test from new feature")
skip.If(t, testEnv.DaemonInfo.CgroupDriver == "none")
skip.If(t, !testEnv.DaemonInfo.KernelMemoryTCP)
defer setupTest(t)()
@ -57,6 +58,11 @@ func TestNISDomainname(t *testing.T) {
skip.If(t, versions.LessThan(testEnv.DaemonAPIVersion(), "1.40"), "skip test from new feature")
skip.If(t, testEnv.DaemonInfo.OSType != "linux")
// Rootless supports custom Hostname but doesn't support custom Domainname
// OCI runtime create failed: container_linux.go:349: starting container process caused "process_linux.go:449: container init caused \
// "write sysctl key kernel.domainname: open /proc/sys/kernel/domainname: permission denied\"": unknown.
skip.If(t, testEnv.IsRootless, "rootless mode doesn't support setting Domainname (TODO: https://github.com/moby/moby/issues/40632)")
defer setupTest(t)()
client := testEnv.APIClient()
ctx := context.Background()

View file

@ -16,6 +16,7 @@ import (
)
func TestStats(t *testing.T) {
skip.If(t, testEnv.DaemonInfo.CgroupDriver == "none")
skip.If(t, !testEnv.DaemonInfo.MemoryLimit)
defer setupTest(t)()

View file

@ -19,6 +19,7 @@ import (
func TestUpdateMemory(t *testing.T) {
skip.If(t, testEnv.DaemonInfo.OSType == "windows")
skip.If(t, testEnv.DaemonInfo.CgroupDriver == "none")
skip.If(t, !testEnv.DaemonInfo.MemoryLimit)
skip.If(t, !testEnv.DaemonInfo.SwapLimit)
@ -68,6 +69,7 @@ func TestUpdateMemory(t *testing.T) {
}
func TestUpdateCPUQuota(t *testing.T) {
skip.If(t, testEnv.DaemonInfo.CgroupDriver == "none")
defer setupTest(t)()
client := testEnv.APIClient()
ctx := context.Background()
@ -106,6 +108,7 @@ func TestUpdateCPUQuota(t *testing.T) {
func TestUpdatePidsLimit(t *testing.T) {
skip.If(t, testEnv.DaemonInfo.OSType == "windows")
skip.If(t, testEnv.DaemonInfo.CgroupDriver == "none")
skip.If(t, !testEnv.DaemonInfo.PidsLimit)
defer setupTest(t)()

View file

@ -29,6 +29,7 @@ func TestDaemonRestartWithLiveRestore(t *testing.T) {
skip.If(t, testEnv.OSType == "windows")
skip.If(t, testEnv.IsRemoteDaemon)
skip.If(t, versions.LessThan(testEnv.DaemonAPIVersion(), "1.38"), "skip test from new feature")
skip.If(t, testEnv.IsRootless, "rootless mode has different view of network")
d := daemon.New(t)
defer d.Stop(t)
d.Start(t)
@ -52,6 +53,7 @@ func TestDaemonDefaultNetworkPools(t *testing.T) {
// Remove docker0 bridge and the start daemon defining the predefined address pools
skip.If(t, testEnv.IsRemoteDaemon)
skip.If(t, versions.LessThan(testEnv.DaemonAPIVersion(), "1.38"), "skip test from new feature")
skip.If(t, testEnv.IsRootless, "rootless mode has different view of network")
defaultNetworkBridge := "docker0"
delInterface(t, defaultNetworkBridge)
d := daemon.New(t)
@ -94,6 +96,7 @@ func TestDaemonRestartWithExistingNetwork(t *testing.T) {
skip.If(t, testEnv.OSType == "windows")
skip.If(t, testEnv.IsRemoteDaemon)
skip.If(t, versions.LessThan(testEnv.DaemonAPIVersion(), "1.38"), "skip test from new feature")
skip.If(t, testEnv.IsRootless, "rootless mode has different view of network")
defaultNetworkBridge := "docker0"
d := daemon.New(t)
d.Start(t)
@ -127,6 +130,7 @@ func TestDaemonRestartWithExistingNetworkWithDefaultPoolRange(t *testing.T) {
skip.If(t, testEnv.OSType == "windows")
skip.If(t, testEnv.IsRemoteDaemon)
skip.If(t, versions.LessThan(testEnv.DaemonAPIVersion(), "1.38"), "skip test from new feature")
skip.If(t, testEnv.IsRootless, "rootless mode has different view of network")
defaultNetworkBridge := "docker0"
d := daemon.New(t)
d.Start(t)
@ -177,6 +181,7 @@ func TestDaemonWithBipAndDefaultNetworkPool(t *testing.T) {
skip.If(t, testEnv.OSType == "windows")
skip.If(t, testEnv.IsRemoteDaemon)
skip.If(t, versions.LessThan(testEnv.DaemonAPIVersion(), "1.38"), "skip test from new feature")
skip.If(t, testEnv.IsRootless, "rootless mode has different view of network")
defaultNetworkBridge := "docker0"
d := daemon.New(t)
defer d.Stop(t)
@ -199,6 +204,7 @@ func TestDaemonWithBipAndDefaultNetworkPool(t *testing.T) {
func TestServiceWithPredefinedNetwork(t *testing.T) {
skip.If(t, testEnv.OSType == "windows")
skip.If(t, testEnv.IsRootless, "rootless mode doesn't support Swarm-mode")
defer setupTest(t)()
d := swarm.NewSwarm(t, testEnv)
defer d.Stop(t)
@ -228,6 +234,7 @@ const ingressNet = "ingress"
func TestServiceRemoveKeepsIngressNetwork(t *testing.T) {
t.Skip("FLAKY_TEST")
skip.If(t, testEnv.IsRootless, "rootless mode doesn't support Swarm-mode")
skip.If(t, testEnv.OSType == "windows")
defer setupTest(t)()
@ -318,6 +325,7 @@ func noServices(ctx context.Context, client client.ServiceAPIClient) func(log po
func TestServiceWithDataPathPortInit(t *testing.T) {
skip.If(t, testEnv.OSType == "windows")
skip.If(t, versions.LessThan(testEnv.DaemonAPIVersion(), "1.40"), "DataPathPort was added in API v1.40")
skip.If(t, testEnv.IsRootless, "rootless mode doesn't support Swarm-mode")
defer setupTest(t)()
var datapathPort uint32 = 7777
d := swarm.NewSwarm(t, testEnv, daemon.WithSwarmDataPathPort(datapathPort))
@ -384,6 +392,7 @@ func TestServiceWithDataPathPortInit(t *testing.T) {
func TestServiceWithDefaultAddressPoolInit(t *testing.T) {
skip.If(t, testEnv.OSType == "windows")
skip.If(t, testEnv.IsRootless, "rootless mode doesn't support Swarm-mode")
defer setupTest(t)()
d := swarm.NewSwarm(t, testEnv,
daemon.WithSwarmDefaultAddrPool([]string{"20.20.0.0/16"}),

View file

@ -151,6 +151,10 @@ func New(t testing.TB, ops ...Option) *Daemon {
assert.Check(t, dest != "", "Please set the DOCKER_INTEGRATION_DAEMON_DEST or the DEST environment variable")
if os.Getenv("DOCKER_ROOTLESS") != "" {
t.Skip("github.com/docker/docker/testutil/daemon.Daemon doesn't support DOCKER_ROOTLESS")
}
d, err := NewDaemon(dest, ops...)
assert.NilError(t, err, "could not create daemon at %q", dest)
@ -227,6 +231,9 @@ func (d *Daemon) Cleanup(t testing.TB) {
// Start starts the daemon and return once it is ready to receive requests.
func (d *Daemon) Start(t testing.TB, args ...string) {
t.Helper()
if os.Getenv("DOCKER_ROOTLESS") != "" {
t.Skip("github.com/docker/docker/testutil/daemon.Daemon doesn't support DOCKER_ROOTLESS")
}
if err := d.StartWithError(args...); err != nil {
t.Fatalf("[%s] failed to start daemon with arguments %v : %v", d.id, d.args, err)
}

View file

@ -162,6 +162,11 @@ func (e *Execution) IsUserNamespace() bool {
return root != ""
}
// IsRootless returns whether the rootless mode is enabled
func (e *Execution) IsRootless() bool {
return os.Getenv("DOCKER_ROOTLESS") != ""
}
// HasExistingImage checks whether there is an image with the given reference.
// Note that this is done by filtering and then checking whether there were any
// results -- so ambiguous references might result in false-positives.