Merge pull request #41948 from AkihiroSuda/cherrypick-41892-1903

[19.03 backport] pkg/archive: allow mknodding FIFO inside userns

.dockerignore

@@ -3,5 +3,4 @@ bundles
 vendor/pkg
 .go-pkg-cache
 .git
-hack/integration-cli-on-swarm/integration-cli-on-swarm
 

.github/CODEOWNERS

@@ -12,6 +12,5 @@ daemon/graphdriver/overlay2/**          @dmcgowan
 daemon/graphdriver/windows/**           @johnstep @jhowardmsft
 daemon/logger/awslogs/**                @samuelkarp  
 hack/**                                 @tianon
-hack/integration-cli-on-swarm/**        @AkihiroSuda
 plugin/**                               @cpuguy83
 project/**                              @thaJeztah

.gitignore

@@ -3,6 +3,7 @@
 #  please consider a global .gitignore https://help.github.com/articles/ignoring-files
 *.exe
 *.exe~
+*.gz
 *.orig
 test.main
 .*.swp
@@ -19,6 +20,6 @@ contrib/builder/rpm/*/changelog
 dockerversion/version_autogen.go
 dockerversion/version_autogen_unix.go
 vendor/pkg/
-hack/integration-cli-on-swarm/integration-cli-on-swarm
-coverage.txt
+go-test-report.json
 profile.out
+junit-report.xml

Dockerfile

@@ -25,132 +25,156 @@
 #
 
 ARG CROSS="false"
+# IMPORTANT: When updating this please note that stdlib archive/tar pkg is vendored
+ARG GO_VERSION=1.13.15
+ARG DEBIAN_FRONTEND=noninteractive
+ARG VPNKIT_DIGEST=e508a17cfacc8fd39261d5b4e397df2b953690da577e2c987a47630cd0c42f8e
 
-FROM golang:1.12.4 AS base
-# allow replacing httpredir or deb mirror
-ARG APT_MIRROR=deb.debian.org
-RUN sed -ri "s/(httpredir|deb).debian.org/$APT_MIRROR/g" /etc/apt/sources.list
+FROM golang:${GO_VERSION}-buster AS base
+ARG APT_MIRROR
+RUN sed -ri "s/(httpredir|deb).debian.org/${APT_MIRROR:-deb.debian.org}/g" /etc/apt/sources.list \
+ && sed -ri "s/(security).debian.org/${APT_MIRROR:-security.debian.org}/g" /etc/apt/sources.list
+ENV GO111MODULE=off
 
 FROM base AS criu
-# Install CRIU for checkpoint/restore support
-ENV CRIU_VERSION 3.11
+ARG DEBIAN_FRONTEND
 # Install dependency packages specific to criu
-RUN apt-get update && apt-get install -y \
-	libnet-dev \
-	libprotobuf-c0-dev \
-	libprotobuf-dev \
-	libnl-3-dev \
-	libcap-dev \
-	protobuf-compiler \
-	protobuf-c-compiler \
-	python-protobuf \
-	&& mkdir -p /usr/src/criu \
-	&& curl -sSL https://github.com/checkpoint-restore/criu/archive/v${CRIU_VERSION}.tar.gz | tar -C /usr/src/criu/ -xz --strip-components=1 \
-	&& cd /usr/src/criu \
-	&& make \
-	&& make PREFIX=/build/ install-criu
+RUN apt-get update && apt-get install -y --no-install-recommends \
+        libcap-dev \
+        libnet-dev \
+        libnl-3-dev \
+        libprotobuf-c-dev \
+        libprotobuf-dev \
+        protobuf-c-compiler \
+        protobuf-compiler \
+        python-protobuf \
+    && rm -rf /var/lib/apt/lists/*
+
+# Install CRIU for checkpoint/restore support
+ARG CRIU_VERSION=3.14
+RUN mkdir -p /usr/src/criu \
+    && curl -sSL https://github.com/checkpoint-restore/criu/archive/v${CRIU_VERSION}.tar.gz | tar -C /usr/src/criu/ -xz --strip-components=1 \
+    && cd /usr/src/criu \
+    && make \
+    && make PREFIX=/build/ install-criu
 
 FROM base AS registry
+# Install two versions of the registry. The first is an older version that
+# only supports schema1 manifests. The second is a newer version that supports
+# both. This allows integration-cli tests to cover push/pull with both schema1
+# and schema2 manifests.
+ENV REGISTRY_COMMIT_SCHEMA1 ec87e9b6971d831f0eff752ddb54fb64693e51cd
 ENV REGISTRY_COMMIT 47a064d4195a9b56133891bbb13620c3ac83a827
 RUN set -x \
-	&& export GOPATH="$(mktemp -d)" \
-	&& git clone https://github.com/docker/distribution.git "$GOPATH/src/github.com/docker/distribution" \
-	&& (cd "$GOPATH/src/github.com/docker/distribution" && git checkout -q "$REGISTRY_COMMIT") \
-	&& GOPATH="$GOPATH/src/github.com/docker/distribution/Godeps/_workspace:$GOPATH" \
-		go build -buildmode=pie -o /build/registry-v2 github.com/docker/distribution/cmd/registry \
-	&& rm -rf "$GOPATH"
-
-
-
-FROM base AS docker-py
-# Get the "docker-py" source so we can run their integration tests
-ENV DOCKER_PY_COMMIT ac922192959870774ad8428344d9faa0555f7ba6
-RUN git clone https://github.com/docker/docker-py.git /build \
-	&& cd /build \
-	&& git checkout -q $DOCKER_PY_COMMIT
-
-
+    && export GOPATH="$(mktemp -d)" \
+    && git clone https://github.com/docker/distribution.git "$GOPATH/src/github.com/docker/distribution" \
+    && (cd "$GOPATH/src/github.com/docker/distribution" && git checkout -q "$REGISTRY_COMMIT") \
+    && GOPATH="$GOPATH/src/github.com/docker/distribution/Godeps/_workspace:$GOPATH" \
+        go build -buildmode=pie -o /build/registry-v2 github.com/docker/distribution/cmd/registry \
+    && case $(dpkg --print-architecture) in \
+        amd64|ppc64*|s390x) \
+        (cd "$GOPATH/src/github.com/docker/distribution" && git checkout -q "$REGISTRY_COMMIT_SCHEMA1"); \
+        GOPATH="$GOPATH/src/github.com/docker/distribution/Godeps/_workspace:$GOPATH"; \
+            go build -buildmode=pie -o /build/registry-v2-schema1 github.com/docker/distribution/cmd/registry; \
+        ;; \
+       esac \
+    && rm -rf "$GOPATH"
 
 FROM base AS swagger
 # Install go-swagger for validating swagger.yaml
-ENV GO_SWAGGER_COMMIT c28258affb0b6251755d92489ef685af8d4ff3eb
+# This is https://github.com/kolyshkin/go-swagger/tree/golang-1.13-fix
+# TODO: move to under moby/ or fix upstream go-swagger to work for us.
+ENV GO_SWAGGER_COMMIT 5793aa66d4b4112c2602c716516e24710e4adbb5
 RUN set -x \
-	&& export GOPATH="$(mktemp -d)" \
-	&& git clone https://github.com/go-swagger/go-swagger.git "$GOPATH/src/github.com/go-swagger/go-swagger" \
-	&& (cd "$GOPATH/src/github.com/go-swagger/go-swagger" && git checkout -q "$GO_SWAGGER_COMMIT") \
-	&& go build -o /build/swagger github.com/go-swagger/go-swagger/cmd/swagger \
-	&& rm -rf "$GOPATH"
-
+    && export GOPATH="$(mktemp -d)" \
+    && git clone https://github.com/kolyshkin/go-swagger.git "$GOPATH/src/github.com/go-swagger/go-swagger" \
+    && (cd "$GOPATH/src/github.com/go-swagger/go-swagger" && git checkout -q "$GO_SWAGGER_COMMIT") \
+    && go build -o /build/swagger github.com/go-swagger/go-swagger/cmd/swagger \
+    && rm -rf "$GOPATH"
 
 FROM base AS frozen-images
-RUN apt-get update && apt-get install -y jq ca-certificates --no-install-recommends
+ARG DEBIAN_FRONTEND
+RUN apt-get update && apt-get install -y --no-install-recommends \
+        ca-certificates \
+        jq \
+    && rm -rf /var/lib/apt/lists/*
 # Get useful and necessary Hub images so we can "docker load" locally instead of pulling
 COPY contrib/download-frozen-image-v2.sh /
 RUN /download-frozen-image-v2.sh /build \
-	buildpack-deps:jessie@sha256:dd86dced7c9cd2a724e779730f0a53f93b7ef42228d4344b25ce9a42a1486251 \
-	busybox:latest@sha256:bbc3a03235220b170ba48a157dd097dd1379299370e1ed99ce976df0355d24f0 \
-	busybox:glibc@sha256:0b55a30394294ab23b9afd58fab94e61a923f5834fba7ddbae7f8e0c11ba85e6 \
-	debian:jessie@sha256:287a20c5f73087ab406e6b364833e3fb7b3ae63ca0eb3486555dc27ed32c6e60 \
-	hello-world:latest@sha256:be0cd392e45be79ffeffa6b05338b98ebb16c87b255f48e297ec7f98e123905c
+        buildpack-deps:jessie@sha256:dd86dced7c9cd2a724e779730f0a53f93b7ef42228d4344b25ce9a42a1486251 \
+        busybox:latest@sha256:bbc3a03235220b170ba48a157dd097dd1379299370e1ed99ce976df0355d24f0 \
+        busybox:glibc@sha256:0b55a30394294ab23b9afd58fab94e61a923f5834fba7ddbae7f8e0c11ba85e6 \
+        debian:jessie@sha256:287a20c5f73087ab406e6b364833e3fb7b3ae63ca0eb3486555dc27ed32c6e60 \
+        hello-world:latest@sha256:be0cd392e45be79ffeffa6b05338b98ebb16c87b255f48e297ec7f98e123905c
 # See also ensureFrozenImagesLinux() in "integration-cli/fixtures_linux_daemon_test.go" (which needs to be updated when adding images to this list)
 
 FROM base AS cross-false
 
 FROM base AS cross-true
-RUN dpkg --add-architecture armhf
+ARG DEBIAN_FRONTEND
 RUN dpkg --add-architecture arm64
 RUN dpkg --add-architecture armel
+RUN dpkg --add-architecture armhf
 RUN if [ "$(go env GOHOSTARCH)" = "amd64" ]; then \
-	apt-get update \
-	&& apt-get install -y --no-install-recommends \
-		crossbuild-essential-armhf \
-		crossbuild-essential-arm64 \
-		crossbuild-essential-armel; \
-	fi
+        apt-get update && apt-get install -y --no-install-recommends \
+        crossbuild-essential-arm64 \
+        crossbuild-essential-armel \
+        crossbuild-essential-armhf \
+        && rm -rf /var/lib/apt/lists/*; \
+    fi
 
 FROM cross-${CROSS} as dev-base
 
 FROM dev-base AS runtime-dev-cross-false
-RUN apt-get update && apt-get install -y \
-	libapparmor-dev \
-	libseccomp-dev
+ARG DEBIAN_FRONTEND
+RUN apt-get update && apt-get install -y --no-install-recommends \
+        libapparmor-dev \
+        libseccomp-dev \
+    && rm -rf /var/lib/apt/lists/*
 
 FROM cross-true AS runtime-dev-cross-true
+ARG DEBIAN_FRONTEND
 # These crossbuild packages rely on gcc-<arch>, but this doesn't want to install
 # on non-amd64 systems.
 # Additionally, the crossbuild-amd64 is currently only on debian:buster, so
 # other architectures cannnot crossbuild amd64.
 RUN if [ "$(go env GOHOSTARCH)" = "amd64" ]; then \
-	apt-get update \
-	&& apt-get install -y \
-		libseccomp-dev:armhf \
-		libseccomp-dev:arm64 \
-		libseccomp-dev:armel \
-		libapparmor-dev:armhf \
-		libapparmor-dev:arm64 \
-		libapparmor-dev:armel \
-		# install this arches seccomp here due to compat issues with the v0 builder
-		# This is as opposed to inheriting from runtime-dev-cross-false
-		libapparmor-dev \
-		libseccomp-dev; \
-	fi
+        apt-get update && apt-get install -y --no-install-recommends \
+            libapparmor-dev:arm64 \
+            libapparmor-dev:armel \
+            libapparmor-dev:armhf \
+            libseccomp-dev:arm64 \
+            libseccomp-dev:armel \
+            libseccomp-dev:armhf \
+            # install this arches seccomp here due to compat issues with the v0 builder
+            # This is as opposed to inheriting from runtime-dev-cross-false
+            libapparmor-dev \
+            libseccomp-dev \
+        && rm -rf /var/lib/apt/lists/*; \
+    fi
 
 FROM runtime-dev-cross-${CROSS} AS runtime-dev
 
 FROM base AS tomlv
 ENV INSTALL_BINARY_NAME=tomlv
+ARG TOMLV_COMMIT
 COPY hack/dockerfile/install/install.sh ./install.sh
 COPY hack/dockerfile/install/$INSTALL_BINARY_NAME.installer ./
 RUN PREFIX=/build ./install.sh $INSTALL_BINARY_NAME
 
 FROM base AS vndr
 ENV INSTALL_BINARY_NAME=vndr
+ARG VNDR_COMMIT
 COPY hack/dockerfile/install/install.sh ./install.sh
 COPY hack/dockerfile/install/$INSTALL_BINARY_NAME.installer ./
 RUN PREFIX=/build ./install.sh $INSTALL_BINARY_NAME
 
 FROM dev-base AS containerd
-RUN apt-get update && apt-get install -y btrfs-tools
+ARG DEBIAN_FRONTEND
+ARG CONTAINERD_COMMIT
+RUN apt-get update && apt-get install -y --no-install-recommends \
+        libbtrfs-dev \
+    && rm -rf /var/lib/apt/lists/*
 ENV INSTALL_BINARY_NAME=containerd
 COPY hack/dockerfile/install/install.sh ./install.sh
 COPY hack/dockerfile/install/$INSTALL_BINARY_NAME.installer ./
@@ -158,6 +182,7 @@ RUN PREFIX=/build ./install.sh $INSTALL_BINARY_NAME
 
 FROM dev-base AS proxy
 ENV INSTALL_BINARY_NAME=proxy
+ARG LIBNETWORK_COMMIT
 COPY hack/dockerfile/install/install.sh ./install.sh
 COPY hack/dockerfile/install/$INSTALL_BINARY_NAME.installer ./
 RUN PREFIX=/build ./install.sh $INSTALL_BINARY_NAME
@@ -168,20 +193,36 @@ COPY hack/dockerfile/install/install.sh ./install.sh
 COPY hack/dockerfile/install/$INSTALL_BINARY_NAME.installer ./
 RUN PREFIX=/build ./install.sh $INSTALL_BINARY_NAME
 
+FROM base AS gotestsum
+ENV INSTALL_BINARY_NAME=gotestsum
+ARG GOTESTSUM_COMMIT
+COPY hack/dockerfile/install/install.sh ./install.sh
+COPY hack/dockerfile/install/$INSTALL_BINARY_NAME.installer ./
+RUN PREFIX=/build ./install.sh $INSTALL_BINARY_NAME
+
 FROM dev-base AS dockercli
 ENV INSTALL_BINARY_NAME=dockercli
+ARG DOCKERCLI_CHANNEL
+ARG DOCKERCLI_VERSION
 COPY hack/dockerfile/install/install.sh ./install.sh
 COPY hack/dockerfile/install/$INSTALL_BINARY_NAME.installer ./
 RUN PREFIX=/build ./install.sh $INSTALL_BINARY_NAME
 
 FROM runtime-dev AS runc
 ENV INSTALL_BINARY_NAME=runc
+ARG RUNC_COMMIT
+ARG RUNC_BUILDTAGS
 COPY hack/dockerfile/install/install.sh ./install.sh
 COPY hack/dockerfile/install/$INSTALL_BINARY_NAME.installer ./
 RUN PREFIX=/build ./install.sh $INSTALL_BINARY_NAME
 
 FROM dev-base AS tini
-RUN apt-get update && apt-get install -y cmake vim-common
+ARG DEBIAN_FRONTEND
+ARG TINI_COMMIT
+RUN apt-get update && apt-get install -y --no-install-recommends \
+        cmake \
+        vim-common \
+    && rm -rf /var/lib/apt/lists/*
 COPY hack/dockerfile/install/install.sh ./install.sh
 ENV INSTALL_BINARY_NAME=tini
 COPY hack/dockerfile/install/$INSTALL_BINARY_NAME.installer ./
@@ -189,13 +230,17 @@ RUN PREFIX=/build ./install.sh $INSTALL_BINARY_NAME
 
 FROM dev-base AS rootlesskit
 ENV INSTALL_BINARY_NAME=rootlesskit
+ARG ROOTLESSKIT_COMMIT
 COPY hack/dockerfile/install/install.sh ./install.sh
 COPY hack/dockerfile/install/$INSTALL_BINARY_NAME.installer ./
 RUN PREFIX=/build/ ./install.sh $INSTALL_BINARY_NAME
 COPY ./contrib/dockerd-rootless.sh /build
 
+FROM djs55/vpnkit@sha256:${VPNKIT_DIGEST} AS vpnkit
+
 # TODO: Some of this is only really needed for testing, it would be nice to split this up
 FROM runtime-dev AS dev
+ARG DEBIAN_FRONTEND
 RUN groupadd -r docker
 RUN useradd --create-home --gid docker unprivilegeduser
 # Let us use a .bashrc file
@@ -206,74 +251,61 @@ RUN ln -s /usr/local/completion/bash/docker /etc/bash_completion.d/docker
 RUN ldconfig
 # This should only install packages that are specifically needed for the dev environment and nothing else
 # Do you really need to add another package here? Can it be done in a different build stage?
-RUN apt-get update && apt-get install -y \
-	apparmor \
-	aufs-tools \
-	bash-completion \
-	btrfs-tools \
-	iptables \
-	jq \
-	libcap2-bin \
-	libdevmapper-dev \
-# libffi-dev and libssl-dev appear to be required for compiling paramiko on s390x/ppc64le
-	libffi-dev \
-	libssl-dev \
-	libudev-dev \
-	libsystemd-dev \
-	binutils-mingw-w64 \
-	g++-mingw-w64-x86-64 \
-	net-tools \
-	pigz \
-	python-backports.ssl-match-hostname \
-	python-dev \
-# python-cffi appears to be required for compiling paramiko on s390x/ppc64le
-	python-cffi \
-	python-mock \
-	python-pip \
-	python-requests \
-	python-setuptools \
-	python-websocket \
-	python-wheel \
-	thin-provisioning-tools \
-	vim \
-	vim-common \
-	xfsprogs \
-	zip \
-	bzip2 \
-	xz-utils \
-	libprotobuf-c1 \
-	libnet1 \
-	libnl-3-200 \
-	--no-install-recommends
-COPY --from=swagger /build/swagger* /usr/local/bin/
+RUN apt-get update && apt-get install -y --no-install-recommends \
+        apparmor \
+        aufs-tools \
+        bash-completion \
+        binutils-mingw-w64 \
+        libbtrfs-dev \
+        bzip2 \
+        g++-mingw-w64-x86-64 \
+        iptables \
+        jq \
+        libcap2-bin \
+        libdevmapper-dev \
+        libnet1 \
+        libnl-3-200 \
+        libprotobuf-c1 \
+        libsystemd-dev \
+        libudev-dev \
+        net-tools \
+        pigz \
+        python3-pip \
+        python3-setuptools \
+        python3-wheel \
+        thin-provisioning-tools \
+        vim \
+        vim-common \
+        xfsprogs \
+        xz-utils \
+        zip \
+    && rm -rf /var/lib/apt/lists/*
+
+# Switch to use iptables instead of nftables (to match the host machine)
+RUN update-alternatives --set iptables  /usr/sbin/iptables-legacy  || true \
+ && update-alternatives --set ip6tables /usr/sbin/ip6tables-legacy || true \
+ && update-alternatives --set arptables /usr/sbin/arptables-legacy || true
+
+RUN pip3 install yamllint==1.16.0
+
+COPY --from=dockercli     /build/ /usr/local/cli
 COPY --from=frozen-images /build/ /docker-frozen-images
-COPY --from=gometalinter /build/ /usr/local/bin/
-COPY --from=tomlv /build/ /usr/local/bin/
-COPY --from=vndr /build/ /usr/local/bin/
-COPY --from=tini /build/ /usr/local/bin/
-COPY --from=runc /build/ /usr/local/bin/
-COPY --from=containerd /build/ /usr/local/bin/
-COPY --from=proxy /build/ /usr/local/bin/
-COPY --from=dockercli /build/ /usr/local/cli
-COPY --from=registry /build/registry* /usr/local/bin/
-COPY --from=criu /build/ /usr/local/
-COPY --from=docker-py /build/ /docker-py
-# TODO: This is for the docker-py tests, which shouldn't really be needed for
-# this image, but currently CI is expecting to run this image. This should be
-# split out into a separate image, including all the `python-*` deps installed
-# above.
-RUN cd /docker-py \
-	&& pip install docker-pycreds==0.4.0 \
-	&& pip install paramiko==2.4.2 \
-	&& pip install yamllint==1.5.0 \
-	&& pip install -r test-requirements.txt
-COPY --from=rootlesskit /build/ /usr/local/bin/
-COPY --from=djs55/vpnkit@sha256:e508a17cfacc8fd39261d5b4e397df2b953690da577e2c987a47630cd0c42f8e /vpnkit /usr/local/bin/vpnkit.x86_64
+COPY --from=swagger       /build/ /usr/local/bin/
+COPY --from=tomlv         /build/ /usr/local/bin/
+COPY --from=tini          /build/ /usr/local/bin/
+COPY --from=registry      /build/ /usr/local/bin/
+COPY --from=criu          /build/ /usr/local/
+COPY --from=vndr          /build/ /usr/local/bin/
+COPY --from=gotestsum     /build/ /usr/local/bin/
+COPY --from=gometalinter  /build/ /usr/local/bin/
+COPY --from=runc          /build/ /usr/local/bin/
+COPY --from=containerd    /build/ /usr/local/bin/
+COPY --from=rootlesskit   /build/ /usr/local/bin/
+COPY --from=vpnkit        /vpnkit /usr/local/bin/vpnkit.x86_64
+COPY --from=proxy         /build/ /usr/local/bin/
 
 ENV PATH=/usr/local/cli:$PATH
 ENV DOCKER_BUILDTAGS apparmor seccomp selinux
-# Options for hack/validate/gometalinter
-ENV GOMETALINTER_OPTS="--deadline=2m"
 WORKDIR /go/src/github.com/docker/docker
 VOLUME /var/lib/docker
 # Wrap all commands in the "docker-in-docker" script to allow nested containers

Dockerfile.e2e

@@ -1,6 +1,7 @@
-## Step 1: Build tests
-FROM golang:1.12.4-alpine as builder
+ARG GO_VERSION=1.13.15
 
+FROM golang:${GO_VERSION}-alpine AS base
+ENV GO111MODULE=off
 RUN apk --no-cache add \
     bash \
     btrfs-progs-dev \
@@ -9,37 +10,55 @@ RUN apk --no-cache add \
     lvm2-dev \
     jq
 
+RUN mkdir -p /build/
 RUN mkdir -p /go/src/github.com/docker/docker/
 WORKDIR /go/src/github.com/docker/docker/
 
-# Generate frozen images
-COPY contrib/download-frozen-image-v2.sh contrib/download-frozen-image-v2.sh
-RUN contrib/download-frozen-image-v2.sh /output/docker-frozen-images \
+FROM base AS frozen-images
+# Get useful and necessary Hub images so we can "docker load" locally instead of pulling
+COPY contrib/download-frozen-image-v2.sh /
+RUN /download-frozen-image-v2.sh /build \
 	buildpack-deps:jessie@sha256:dd86dced7c9cd2a724e779730f0a53f93b7ef42228d4344b25ce9a42a1486251 \
 	busybox:latest@sha256:bbc3a03235220b170ba48a157dd097dd1379299370e1ed99ce976df0355d24f0 \
 	busybox:glibc@sha256:0b55a30394294ab23b9afd58fab94e61a923f5834fba7ddbae7f8e0c11ba85e6 \
 	debian:jessie@sha256:287a20c5f73087ab406e6b364833e3fb7b3ae63ca0eb3486555dc27ed32c6e60 \
 	hello-world:latest@sha256:be0cd392e45be79ffeffa6b05338b98ebb16c87b255f48e297ec7f98e123905c
+# See also ensureFrozenImagesLinux() in "integration-cli/fixtures_linux_daemon_test.go" (which needs to be updated when adding images to this list)
 
-# Install dockercli
-# Please edit hack/dockerfile/install/<name>.installer to update them.
-COPY hack/dockerfile/install hack/dockerfile/install
-RUN ./hack/dockerfile/install/install.sh dockercli
-
-# Set tag and add sources
-ARG DOCKER_GITCOMMIT
-ENV DOCKER_GITCOMMIT=${DOCKER_GITCOMMIT:-undefined}
-ADD . .
+FROM base AS dockercli
+ENV INSTALL_BINARY_NAME=dockercli
+COPY hack/dockerfile/install/install.sh ./install.sh
+COPY hack/dockerfile/install/$INSTALL_BINARY_NAME.installer ./
+RUN PREFIX=/build ./install.sh $INSTALL_BINARY_NAME
 
 # Build DockerSuite.TestBuild* dependency
-RUN CGO_ENABLED=0 go build -buildmode=pie -o /output/httpserver github.com/docker/docker/contrib/httpserver
+FROM base AS contrib
+COPY contrib/syscall-test           /build/syscall-test
+COPY contrib/httpserver/Dockerfile  /build/httpserver/Dockerfile
+COPY contrib/httpserver             contrib/httpserver
+RUN CGO_ENABLED=0 go build -buildmode=pie -o /build/httpserver/httpserver github.com/docker/docker/contrib/httpserver
 
-# Build the integration tests and copy the resulting binaries to /output/tests
+# Build the integration tests and copy the resulting binaries to /build/tests
+FROM base AS builder
+
+# Set tag and add sources
+COPY . .
+# Copy test sources tests that use assert can print errors
+RUN mkdir -p /build${PWD} && find integration integration-cli -name \*_test.go -exec cp --parents '{}' /build${PWD} \;
+# Build and install test binaries
+ARG DOCKER_GITCOMMIT=undefined
 RUN hack/make.sh build-integration-test-binary
-RUN mkdir -p /output/tests && find . -name test.main -exec cp --parents '{}' /output/tests \;
+RUN mkdir -p /build/tests && find . -name test.main -exec cp --parents '{}' /build/tests \;
 
-## Step 2: Generate testing image
-FROM alpine:3.8 as runner
+## Generate testing image
+FROM alpine:3.10 as runner
+
+ENV DOCKER_REMOTE_DAEMON=1
+ENV DOCKER_INTEGRATION_DAEMON_DEST=/
+ENTRYPOINT ["/scripts/run.sh"]
+
+# Add an unprivileged user to be used for tests which need it
+RUN addgroup docker && adduser -D -G docker unprivilegeduser -s /bin/ash
 
 # GNU tar is used for generating the emptyfs image
 RUN apk --no-cache add \
@@ -52,21 +71,14 @@ RUN apk --no-cache add \
     tar \
     xz
 
-# Add an unprivileged user to be used for tests which need it
-RUN addgroup docker && adduser -D -G docker unprivilegeduser -s /bin/ash
+COPY hack/test/e2e-run.sh       /scripts/run.sh
+COPY hack/make/.ensure-emptyfs  /scripts/ensure-emptyfs.sh
 
-COPY contrib/httpserver/Dockerfile /tests/contrib/httpserver/Dockerfile
-COPY contrib/syscall-test /tests/contrib/syscall-test
-COPY integration-cli/fixtures /tests/integration-cli/fixtures
+COPY integration/testdata       /tests/integration/testdata
+COPY integration/build/testdata /tests/integration/build/testdata
+COPY integration-cli/fixtures   /tests/integration-cli/fixtures
 
-COPY hack/test/e2e-run.sh /scripts/run.sh
-COPY hack/make/.ensure-emptyfs /scripts/ensure-emptyfs.sh
-
-COPY --from=builder /output/docker-frozen-images /docker-frozen-images
-COPY --from=builder /output/httpserver /tests/contrib/httpserver/httpserver
-COPY --from=builder /output/tests /tests
-COPY --from=builder /usr/local/bin/docker /usr/bin/docker
-
-ENV DOCKER_REMOTE_DAEMON=1 DOCKER_INTEGRATION_DAEMON_DEST=/
-
-ENTRYPOINT ["/scripts/run.sh"]
+COPY --from=frozen-images /build/ /docker-frozen-images
+COPY --from=dockercli     /build/ /usr/bin/
+COPY --from=contrib       /build/ /tests/contrib/
+COPY --from=builder       /build/ /

Dockerfile.simple

@@ -5,7 +5,10 @@
 
 # This represents the bare minimum required to build and test Docker.
 
-FROM golang:1.12.4
+ARG GO_VERSION=1.13.15
+
+FROM golang:${GO_VERSION}-stretch
+ENV GO111MODULE=off
 
 # allow replacing httpredir or deb mirror
 ARG APT_MIRROR=deb.debian.org

Dockerfile.windows

@@ -45,8 +45,8 @@
 #
 # 1. Clone the sources from github.com:
 #
-#    >>   git clone https://github.com/docker/docker.git C:\go\src\github.com\docker\docker
-#    >>   Cloning into 'C:\go\src\github.com\docker\docker'...
+#    >>   git clone https://github.com/docker/docker.git C:\gopath\src\github.com\docker\docker
+#    >>   Cloning into 'C:\gopath\src\github.com\docker\docker'...
 #    >>   remote: Counting objects: 186216, done.
 #    >>   remote: Compressing objects: 100% (21/21), done.
 #    >>   remote: Total 186216 (delta 5), reused 0 (delta 0), pack-reused 186195
@@ -59,7 +59,7 @@
 #
 # 2. Change directory to the cloned docker sources:
 #
-#    >>   cd C:\go\src\github.com\docker\docker 
+#    >>   cd C:\gopath\src\github.com\docker\docker 
 #
 #
 # 3. Build a docker image with the components required to build the docker binaries from source
@@ -79,8 +79,8 @@
 # 5. Copy the binaries out of the container, replacing HostPath with an appropriate destination 
 #    folder on the host system where you want the binaries to be located.
 #
-#    >>   docker cp binaries:C:\go\src\github.com\docker\docker\bundles\docker.exe C:\HostPath\docker.exe
-#    >>   docker cp binaries:C:\go\src\github.com\docker\docker\bundles\dockerd.exe C:\HostPath\dockerd.exe
+#    >>   docker cp binaries:C:\gopath\src\github.com\docker\docker\bundles\docker.exe C:\HostPath\docker.exe
+#    >>   docker cp binaries:C:\gopath\src\github.com\docker\docker\bundles\dockerd.exe C:\HostPath\dockerd.exe
 #
 #
 # 6. (Optional) Remove the interim container holding the built executable binaries:
@@ -147,7 +147,7 @@
 # The docker integration tests do not currently run in a container on Windows, predominantly
 # due to Windows not supporting privileged mode, so anything using a volume would fail.
 # They (along with the rest of the docker CI suite) can be run using 
-# https://github.com/jhowardmsft/docker-w2wCIScripts/blob/master/runCI/Invoke-DockerCI.ps1.
+# https://github.com/kevpar/docker-w2wCIScripts/blob/master/runCI/Invoke-DockerCI.ps1.
 #
 # -----------------------------------------------------------------------------------------
 
@@ -165,12 +165,15 @@ FROM microsoft/windowsservercore
 # Use PowerShell as the default shell
 SHELL ["powershell", "-Command", "$ErrorActionPreference = 'Stop'; $ProgressPreference = 'SilentlyContinue';"]
 
+ARG GO_VERSION=1.13.15
+
 # Environment variable notes:
 #  - GO_VERSION must be consistent with 'Dockerfile' used by Linux.
 #  - FROM_DOCKERFILE is used for detection of building within a container.
-ENV GO_VERSION=1.12.4 `
+ENV GO_VERSION=${GO_VERSION} `
     GIT_VERSION=2.11.1 `
-    GOPATH=C:\go `
+    GOPATH=C:\gopath `
+    GO111MODULE=off `
     FROM_DOCKERFILE=1
 
 RUN `
@@ -212,16 +215,17 @@ RUN `
   Download-File $location C:\gitsetup.zip; `
   `
   Write-Host INFO: Downloading go...; `
-  Download-File $('https://golang.org/dl/go'+$Env:GO_VERSION+'.windows-amd64.zip') C:\go.zip; `
+  $dlGoVersion=$Env:GO_VERSION -replace '\.0$',''; `
+  Download-File "https://golang.org/dl/go${dlGoVersion}.windows-amd64.zip" C:\go.zip; `
   `
   Write-Host INFO: Downloading compiler 1 of 3...; `
-  Download-File https://raw.githubusercontent.com/jhowardmsft/docker-tdmgcc/master/gcc.zip C:\gcc.zip; `
+  Download-File https://raw.githubusercontent.com/moby/docker-tdmgcc/master/gcc.zip C:\gcc.zip; `
   `
   Write-Host INFO: Downloading compiler 2 of 3...; `
-  Download-File https://raw.githubusercontent.com/jhowardmsft/docker-tdmgcc/master/runtime.zip C:\runtime.zip; `
+  Download-File https://raw.githubusercontent.com/moby/docker-tdmgcc/master/runtime.zip C:\runtime.zip; `
   `
   Write-Host INFO: Downloading compiler 3 of 3...; `
-  Download-File https://raw.githubusercontent.com/jhowardmsft/docker-tdmgcc/master/binutils.zip C:\binutils.zip; `
+  Download-File https://raw.githubusercontent.com/moby/docker-tdmgcc/master/binutils.zip C:\binutils.zip; `
   `
   Write-Host INFO: Extracting git...; `
   Expand-Archive C:\gitsetup.zip C:\git-tmp; `
@@ -246,7 +250,7 @@ RUN `
   Remove-Item C:\gitsetup.zip; `
   `
   Write-Host INFO: Creating source directory...; `
-  New-Item -ItemType Directory -Path C:\go\src\github.com\docker\docker | Out-Null; `
+  New-Item -ItemType Directory -Path ${GOPATH}\src\github.com\docker\docker | Out-Null; `
   `
   Write-Host INFO: Configuring git core.autocrlf...; `
   C:\git\cmd\git config --global core.autocrlf true; `
@@ -257,7 +261,7 @@ RUN `
 ENTRYPOINT ["powershell.exe"]
 
 # Set the working directory to the location of the sources
-WORKDIR C:\go\src\github.com\docker\docker
+WORKDIR ${GOPATH}\src\github.com\docker\docker
 
 # Copy the sources into the container
 COPY . .

MAINTAINERS

@@ -111,7 +111,7 @@
 			# still stumble into him in our issue tracker, or on IRC.
 			"erikh",
 
-			# Evan Hazlett is the creator of of the Shipyard and Interlock open source projects,
+			# Evan Hazlett is the creator of the Shipyard and Interlock open source projects,
 			# and the author of "Orca", which became the foundation of Docker Universal Control
 			# Plane (UCP). As a maintainer, Evan helped integrating SwarmKit (secrets, tasks)
 			# into the Docker engine.

Makefile

@@ -50,11 +50,18 @@ DOCKER_ENVS := \
 	-e DOCKER_PORT \
 	-e DOCKER_REMAP_ROOT \
 	-e DOCKER_STORAGE_OPTS \
+	-e DOCKER_TEST_HOST \
 	-e DOCKER_USERLANDPROXY \
 	-e DOCKERD_ARGS \
 	-e TEST_INTEGRATION_DIR \
+	-e TEST_SKIP_INTEGRATION \
+	-e TEST_SKIP_INTEGRATION_CLI \
+	-e TESTDEBUG \
 	-e TESTDIRS \
 	-e TESTFLAGS \
+	-e TESTFLAGS_INTEGRATION \
+	-e TESTFLAGS_INTEGRATION_CLI \
+	-e TEST_FILTER \
 	-e TIMEOUT \
 	-e VALIDATE_REPO \
 	-e VALIDATE_BRANCH \
@@ -79,6 +86,7 @@ BIND_DIR := $(if $(BINDDIR),$(BINDDIR),$(if $(DOCKER_HOST),,bundles))
 # DOCKER_MOUNT can be overriden, but use at your own risk!
 ifndef DOCKER_MOUNT
 DOCKER_MOUNT := $(if $(BIND_DIR),-v "$(CURDIR)/$(BIND_DIR):/go/src/github.com/docker/docker/$(BIND_DIR)")
+DOCKER_MOUNT := $(if $(DOCKER_BINDDIR_MOUNT_OPTS),$(DOCKER_MOUNT):$(DOCKER_BINDDIR_MOUNT_OPTS),$(DOCKER_MOUNT))
 
 # This allows the test suite to be able to run without worrying about the underlying fs used by the container running the daemon (e.g. aufs-on-aufs), so long as the host running the container is running a supported fs.
 # The volume will be cleaned up when the container is removed due to `--rm`.
@@ -105,9 +113,6 @@ export BUILD_APT_MIRROR
 
 SWAGGER_DOCS_PORT ?= 9000
 
-INTEGRATION_CLI_MASTER_IMAGE := $(if $(INTEGRATION_CLI_MASTER_IMAGE), $(INTEGRATION_CLI_MASTER_IMAGE), integration-cli-master)
-INTEGRATION_CLI_WORKER_IMAGE := $(if $(INTEGRATION_CLI_WORKER_IMAGE), $(INTEGRATION_CLI_WORKER_IMAGE), integration-cli-worker)
-
 define \n
 
 
@@ -150,7 +155,7 @@ build: DOCKER_BUILD_ARGS += --build-arg=CROSS=$(DOCKER_CROSS)
 build: DOCKER_BUILDKIT ?= 1
 build: bundles
 	$(warning The docker client CLI has moved to github.com/docker/cli. For a dev-test cycle involving the CLI, run:${\n} DOCKER_CLI_PATH=/host/path/to/cli/binary make shell ${\n} then change the cli and compile into a binary at the same location.${\n})
-	DOCKER_BUILDKIT="${DOCKER_BUILDKIT}" docker build ${BUILD_APT_MIRROR} ${DOCKER_BUILD_ARGS} ${DOCKER_BUILD_OPTS} -t "$(DOCKER_IMAGE)" -f "$(DOCKERFILE)" .
+	DOCKER_BUILDKIT="${DOCKER_BUILDKIT}" docker build --build-arg=GO_VERSION ${BUILD_APT_MIRROR} ${DOCKER_BUILD_ARGS} ${DOCKER_BUILD_OPTS} -t "$(DOCKER_IMAGE)" -f "$(DOCKERFILE)" .
 
 bundles:
 	mkdir bundles
@@ -182,8 +187,13 @@ test-docker-py: build ## run the docker-py tests
 
 test-integration-cli: test-integration ## (DEPRECATED) use test-integration
 
+ifneq ($(and $(TEST_SKIP_INTEGRATION),$(TEST_SKIP_INTEGRATION_CLI)),)
+test-integration:
+	@echo Both integrations suites skipped per environment variables
+else
 test-integration: build ## run the integration tests
 	$(DOCKER_RUN_DOCKER) hack/make.sh dynbinary test-integration
+endif
 
 test-integration-flaky: build ## run the stress test for all new integration tests
 	$(DOCKER_RUN_DOCKER) hack/make.sh dynbinary test-integration-flaky
@@ -212,18 +222,3 @@ swagger-docs: ## preview the API documentation
 		-e 'REDOC_OPTIONS=hide-hostname="true" lazy-rendering' \
 		-p $(SWAGGER_DOCS_PORT):80 \
 		bfirsh/redoc:1.6.2
-
-build-integration-cli-on-swarm: build ## build images and binary for running integration-cli on Swarm in parallel
-	@echo "Building hack/integration-cli-on-swarm (if build fails, please refer to hack/integration-cli-on-swarm/README.md)"
-	go build -buildmode=pie -o ./hack/integration-cli-on-swarm/integration-cli-on-swarm ./hack/integration-cli-on-swarm/host
-	@echo "Building $(INTEGRATION_CLI_MASTER_IMAGE)"
-	docker build -t $(INTEGRATION_CLI_MASTER_IMAGE) hack/integration-cli-on-swarm/agent
-	@echo "Building $(INTEGRATION_CLI_WORKER_IMAGE) from $(DOCKER_IMAGE)"
-	$(eval tmp := integration-cli-worker-tmp)
-# We mount pkgcache, but not bundle (bundle needs to be baked into the image)
-# For avoiding bakings DOCKER_GRAPHDRIVER and so on to image, we cannot use $(DOCKER_ENVS) here
-	docker run -t -d --name $(tmp) -e DOCKER_GITCOMMIT -e BUILDFLAGS --privileged $(DOCKER_IMAGE) top
-	docker exec $(tmp) hack/make.sh build-integration-test-binary dynbinary
-	docker exec $(tmp) go build -buildmode=pie -o /worker github.com/docker/docker/hack/integration-cli-on-swarm/agent/worker
-	docker commit -c 'ENTRYPOINT ["/worker"]' $(tmp) $(INTEGRATION_CLI_WORKER_IMAGE)
-	docker rm -f $(tmp)

NOTICE

@@ -3,7 +3,7 @@ Copyright 2012-2017 Docker, Inc.
 
 This product includes software developed at Docker, Inc. (https://www.docker.com).
 
-This product contains software (https://github.com/kr/pty) developed
+This product contains software (https://github.com/creack/pty) developed
 by Keith Rarick, licensed under the MIT License.
 
 The following is courtesy of our legal counsel:

TESTING.md

@@ -67,6 +67,8 @@ If a remote daemon is detected, the test will be skipped.
 
 ## Running tests
 
+### Unit Tests
+
 To run the unit test suite:
 
 ```
@@ -82,8 +84,36 @@ The following environment variables may be used to run a subset of tests:
 * `TESTFLAGS` - flags passed to `go test`, to run tests which match a pattern
   use `TESTFLAGS="-test.run TestNameOrPrefix"`
 
+### Integration Tests
+
 To run the integration test suite:
 
 ```
 make test-integration
 ```
+
+This make target runs both the "integration" suite and the "integration-cli"
+suite.
+
+You can specify which integration test dirs to build and run by specifying
+the list of dirs in the TEST_INTEGRATION_DIR environment variable.
+
+You can also explicitly skip either suite by setting (any value) in
+TEST_SKIP_INTEGRATION and/or TEST_SKIP_INTEGRATION_CLI environment variables.
+
+Flags specific to each suite can be set in the TESTFLAGS_INTEGRATION and
+TESTFLAGS_INTEGRATION_CLI environment variables.
+
+If all you want is to specity a test filter to run, you can set the
+`TEST_FILTER` environment variable. This ends up getting passed directly to `go
+test -run` (or `go test -check-f`, dpenending on the test suite). It will also
+automatically set the other above mentioned environment variables accordingly.
+
+### Go Version
+
+You can change a version of golang used for building stuff that is being tested
+by setting `GO_VERSION` variable, for example:
+
+```
+make GO_VERSION=1.12.8 test
+```

api/server/backend/build/backend.go

@@ -91,7 +91,9 @@ func (b *Backend) Build(ctx context.Context, config backend.BuildConfig) (string
 		stdout := config.ProgressWriter.StdoutFormatter
 		fmt.Fprintf(stdout, "Successfully built %s\n", stringid.TruncateID(imageID))
 	}
-	err = tagger.TagImages(image.ID(imageID))
+	if imageID != "" {
+		err = tagger.TagImages(image.ID(imageID))
+	}
 	return imageID, err
 }
 

api/server/middleware/debug.go

@@ -41,7 +41,7 @@ func DebugRequestMiddleware(handler func(ctx context.Context, w http.ResponseWri
 
 		var postForm map[string]interface{}
 		if err := json.Unmarshal(b, &postForm); err == nil {
-			maskSecretKeys(postForm, r.RequestURI)
+			maskSecretKeys(postForm)
 			formStr, errMarshal := json.Marshal(postForm)
 			if errMarshal == nil {
 				logrus.Debugf("form data: %s", string(formStr))
@@ -54,41 +54,37 @@ func DebugRequestMiddleware(handler func(ctx context.Context, w http.ResponseWri
 	}
 }
 
-func maskSecretKeys(inp interface{}, path string) {
-	// Remove any query string from the path
-	idx := strings.Index(path, "?")
-	if idx != -1 {
-		path = path[:idx]
-	}
-	// Remove trailing / characters
-	path = strings.TrimRight(path, "/")
-
+func maskSecretKeys(inp interface{}) {
 	if arr, ok := inp.([]interface{}); ok {
 		for _, f := range arr {
-			maskSecretKeys(f, path)
+			maskSecretKeys(f)
 		}
 		return
 	}
 
 	if form, ok := inp.(map[string]interface{}); ok {
+		scrub := []string{
+			// Note: The Data field contains the base64-encoded secret in 'secret'
+			// and 'config' create and update requests. Currently, no other POST
+			// API endpoints use a data field, so we scrub this field unconditionally.
+			// Change this handling to be conditional if a new endpoint is added
+			// in future where this field should not be scrubbed.
+			"data",
+			"jointoken",
+			"password",
+			"secret",
+			"signingcakey",
+			"unlockkey",
+		}
 	loop0:
 		for k, v := range form {
-			for _, m := range []string{"password", "secret", "jointoken", "unlockkey", "signingcakey"} {
+			for _, m := range scrub {
 				if strings.EqualFold(m, k) {
 					form[k] = "*****"
 					continue loop0
 				}
 			}
-			maskSecretKeys(v, path)
-		}
-
-		// Route-specific redactions
-		if strings.HasSuffix(path, "/secrets/create") {
-			for k := range form {
-				if k == "Data" {
-					form[k] = "*****"
-				}
-			}
+			maskSecretKeys(v)
 		}
 	}
 }

api/server/middleware/debug_test.go

@@ -9,31 +9,25 @@ import (
 
 func TestMaskSecretKeys(t *testing.T) {
 	tests := []struct {
-		path     string
+		doc      string
 		input    map[string]interface{}
 		expected map[string]interface{}
 	}{
 		{
-			path:     "/v1.30/secrets/create",
+			doc:      "secret/config create and update requests",
 			input:    map[string]interface{}{"Data": "foo", "Name": "name", "Labels": map[string]interface{}{}},
 			expected: map[string]interface{}{"Data": "*****", "Name": "name", "Labels": map[string]interface{}{}},
 		},
 		{
-			path:     "/v1.30/secrets/create//",
-			input:    map[string]interface{}{"Data": "foo", "Name": "name", "Labels": map[string]interface{}{}},
-			expected: map[string]interface{}{"Data": "*****", "Name": "name", "Labels": map[string]interface{}{}},
-		},
-
-		{
-			path:     "/secrets/create?key=val",
-			input:    map[string]interface{}{"Data": "foo", "Name": "name", "Labels": map[string]interface{}{}},
-			expected: map[string]interface{}{"Data": "*****", "Name": "name", "Labels": map[string]interface{}{}},
-		},
-		{
-			path: "/v1.30/some/other/path",
+			doc: "masking other fields (recursively)",
 			input: map[string]interface{}{
-				"password": "pass",
+				"password":     "pass",
+				"secret":       "secret",
+				"jointoken":    "jointoken",
+				"unlockkey":    "unlockkey",
+				"signingcakey": "signingcakey",
 				"other": map[string]interface{}{
+					"password":     "pass",
 					"secret":       "secret",
 					"jointoken":    "jointoken",
 					"unlockkey":    "unlockkey",
@@ -41,8 +35,13 @@ func TestMaskSecretKeys(t *testing.T) {
 				},
 			},
 			expected: map[string]interface{}{
-				"password": "*****",
+				"password":     "*****",
+				"secret":       "*****",
+				"jointoken":    "*****",
+				"unlockkey":    "*****",
+				"signingcakey": "*****",
 				"other": map[string]interface{}{
+					"password":     "*****",
 					"secret":       "*****",
 					"jointoken":    "*****",
 					"unlockkey":    "*****",
@@ -50,10 +49,27 @@ func TestMaskSecretKeys(t *testing.T) {
 				},
 			},
 		},
+		{
+			doc: "case insensitive field matching",
+			input: map[string]interface{}{
+				"PASSWORD": "pass",
+				"other": map[string]interface{}{
+					"PASSWORD": "pass",
+				},
+			},
+			expected: map[string]interface{}{
+				"PASSWORD": "*****",
+				"other": map[string]interface{}{
+					"PASSWORD": "*****",
+				},
+			},
+		},
 	}
 
 	for _, testcase := range tests {
-		maskSecretKeys(testcase.input, testcase.path)
-		assert.Check(t, is.DeepEqual(testcase.expected, testcase.input))
+		t.Run(testcase.doc, func(t *testing.T) {
+			maskSecretKeys(testcase.input)
+			assert.Check(t, is.DeepEqual(testcase.expected, testcase.input))
+		})
 	}
 }

api/templates/server/operation.gotmpl

@@ -1,4 +1,4 @@
-package {{ .Package }}
+package {{ .Package }} // import "github.com/docker/docker/api/types/{{ .Package }}"
 
 // ----------------------------------------------------------------------------
 // DO NOT EDIT THIS FILE

api/types/client.go

@@ -50,7 +50,7 @@ type ContainerCommitOptions struct {
 
 // ContainerExecInspect holds information returned by exec inspect.
 type ContainerExecInspect struct {
-	ExecID      string
+	ExecID      string `json:"ID"`
 	ContainerID string
 	Running     bool
 	ExitCode    int

api/types/container/container_changes.go

@@ -1,4 +1,4 @@
-package container
+package container // import "github.com/docker/docker/api/types/container"
 
 // ----------------------------------------------------------------------------
 // DO NOT EDIT THIS FILE

api/types/container/container_create.go

@@ -1,4 +1,4 @@
-package container
+package container // import "github.com/docker/docker/api/types/container"
 
 // ----------------------------------------------------------------------------
 // DO NOT EDIT THIS FILE

api/types/container/container_top.go

@@ -1,4 +1,4 @@
-package container
+package container // import "github.com/docker/docker/api/types/container"
 
 // ----------------------------------------------------------------------------
 // DO NOT EDIT THIS FILE
@@ -11,7 +11,9 @@ package container
 // swagger:model ContainerTopOKBody
 type ContainerTopOKBody struct {
 
-	// Each process running in the container, where each is process is an array of values corresponding to the titles
+	// Each process running in the container, where each is process
+	// is an array of values corresponding to the titles.
+	//
 	// Required: true
 	Processes [][]string `json:"Processes"`
 

api/types/container/container_update.go

@@ -1,4 +1,4 @@
-package container
+package container // import "github.com/docker/docker/api/types/container"
 
 // ----------------------------------------------------------------------------
 // DO NOT EDIT THIS FILE

api/types/container/container_wait.go

@@ -1,4 +1,4 @@
-package container
+package container // import "github.com/docker/docker/api/types/container"
 
 // ----------------------------------------------------------------------------
 // DO NOT EDIT THIS FILE

api/types/container/host_config.go

@@ -338,7 +338,6 @@ type Resources struct {
 	Devices              []DeviceMapping // List of devices to map inside the container
 	DeviceCgroupRules    []string        // List of rule to be added to the device cgroup
 	DeviceRequests       []DeviceRequest // List of device requests for device drivers
-	DiskQuota            int64           // Disk limit (in bytes)
 	KernelMemory         int64           // Kernel memory limit (in bytes)
 	KernelMemoryTCP      int64           // Hard limit for kernel TCP buffer memory (in bytes)
 	MemoryReservation    int64           // Memory soft limit (in bytes)

api/types/filters/parse.go

@@ -36,6 +36,15 @@ func NewArgs(initialArgs ...KeyValuePair) Args {
 	return args
 }
 
+// Keys returns all the keys in list of Args
+func (args Args) Keys() []string {
+	keys := make([]string, 0, len(args.fields))
+	for k := range args.fields {
+		keys = append(keys, k)
+	}
+	return keys
+}
+
 // MarshalJSON returns a JSON byte representation of the Args
 func (args Args) MarshalJSON() ([]byte, error) {
 	if len(args.fields) == 0 {

api/types/image/image_history.go

@@ -1,4 +1,4 @@
-package image
+package image // import "github.com/docker/docker/api/types/image"
 
 // ----------------------------------------------------------------------------
 // DO NOT EDIT THIS FILE

api/types/volume.go

@@ -27,10 +27,13 @@ type Volume struct {
 	Name string `json:"Name"`
 
 	// The driver specific options used when creating the volume.
+	//
 	// Required: true
 	Options map[string]string `json:"Options"`
 
-	// The level at which the volume exists. Either `global` for cluster-wide, or `local` for machine level.
+	// The level at which the volume exists. Either `global` for cluster-wide,
+	// or `local` for machine level.
+	//
 	// Required: true
 	Scope string `json:"Scope"`
 

api/types/volume/volume_create.go

@@ -1,4 +1,4 @@
-package volume
+package volume // import "github.com/docker/docker/api/types/volume"
 
 // ----------------------------------------------------------------------------
 // DO NOT EDIT THIS FILE
@@ -15,7 +15,9 @@ type VolumeCreateBody struct {
 	// Required: true
 	Driver string `json:"Driver"`
 
-	// A mapping of driver options and values. These options are passed directly to the driver and are driver specific.
+	// A mapping of driver options and values. These options are
+	// passed directly to the driver and are driver specific.
+	//
 	// Required: true
 	DriverOpts map[string]string `json:"DriverOpts"`
 
@@ -24,6 +26,7 @@ type VolumeCreateBody struct {
 	Labels map[string]string `json:"Labels"`
 
 	// The new volume's name. If not specified, Docker generates a name.
+	//
 	// Required: true
 	Name string `json:"Name"`
 }

api/types/volume/volume_list.go

@@ -1,4 +1,4 @@
-package volume
+package volume // import "github.com/docker/docker/api/types/volume"
 
 // ----------------------------------------------------------------------------
 // DO NOT EDIT THIS FILE
@@ -17,7 +17,8 @@ type VolumeListOKBody struct {
 	// Required: true
 	Volumes []*types.Volume `json:"Volumes"`
 
-	// Warnings that occurred when fetching the list of volumes
+	// Warnings that occurred when fetching the list of volumes.
+	//
 	// Required: true
 	Warnings []string `json:"Warnings"`
 }

builder/builder-next/adapters/containerimage/pull.go

@@ -135,7 +135,7 @@ func (is *imageSource) resolveRemote(ctx context.Context, ref string, platform *
 		dt   []byte
 	}
 	res, err := is.g.Do(ctx, ref, func(ctx context.Context) (interface{}, error) {
-		dgst, dt, err := imageutil.Config(ctx, ref, is.getResolver(ctx, is.ResolverOpt, ref, sm), is.ContentStore, platform)
+		dgst, dt, err := imageutil.Config(ctx, ref, is.getResolver(ctx, is.ResolverOpt, ref, sm), is.ContentStore, nil, platform)
 		if err != nil {
 			return nil, err
 		}
@@ -526,6 +526,9 @@ func (p *puller) Snapshot(ctx context.Context) (cache.ImmutableRef, error) {
 	layers := make([]xfer.DownloadDescriptor, 0, len(mfst.Layers))
 
 	for i, desc := range mfst.Layers {
+		if err := desc.Digest.Validate(); err != nil {
+			return nil, errors.Wrap(err, "layer digest could not be validated")
+		}
 		ongoing.add(desc)
 		layers = append(layers, &layerDescriptor{
 			desc:    desc,
@@ -545,10 +548,10 @@ func (p *puller) Snapshot(ctx context.Context) (cache.ImmutableRef, error) {
 
 	r := image.NewRootFS()
 	rootFS, release, err := p.is.DownloadManager.Download(ctx, *r, runtime.GOOS, layers, pkgprogress.ChanOutput(pchan))
+	stopProgress()
 	if err != nil {
 		return nil, err
 	}
-	stopProgress()
 
 	ref, err := p.is.CacheAccessor.GetFromSnapshotter(ctx, string(rootFS.ChainID()), cache.WithDescription(fmt.Sprintf("pulled from %s", p.ref)))
 	release()
@@ -828,9 +831,9 @@ type resolverCache struct {
 }
 
 type cachedResolver struct {
+	counter int64 // needs to be 64bit aligned for 32bit systems
 	timeout time.Time
 	remotes.Resolver
-	counter int64
 }
 
 func (cr *cachedResolver) Resolve(ctx context.Context, ref string) (name string, desc ocispec.Descriptor, err error) {
@@ -842,7 +845,7 @@ func (r *resolverCache) Add(ctx context.Context, ref string, resolver remotes.Re
 	r.mu.Lock()
 	defer r.mu.Unlock()
 
-	ref = r.domain(ref) + "-" + session.FromContext(ctx)
+	ref = r.repo(ref) + "-" + session.FromContext(ctx)
 
 	cr, ok := r.m[ref]
 	cr.timeout = time.Now().Add(time.Minute)
@@ -855,19 +858,19 @@ func (r *resolverCache) Add(ctx context.Context, ref string, resolver remotes.Re
 	return &cr
 }
 
-func (r *resolverCache) domain(refStr string) string {
+func (r *resolverCache) repo(refStr string) string {
 	ref, err := distreference.ParseNormalizedNamed(refStr)
 	if err != nil {
 		return refStr
 	}
-	return distreference.Domain(ref)
+	return ref.Name()
 }
 
 func (r *resolverCache) Get(ctx context.Context, ref string) remotes.Resolver {
 	r.mu.Lock()
 	defer r.mu.Unlock()
 
-	ref = r.domain(ref) + "-" + session.FromContext(ctx)
+	ref = r.repo(ref) + "-" + session.FromContext(ctx)
 
 	cr, ok := r.m[ref]
 	if !ok {

builder/builder-next/adapters/snapshot/snapshot.go

@@ -26,9 +26,10 @@ var keySize = []byte("size")
 
 // Opt defines options for creating the snapshotter
 type Opt struct {
-	GraphDriver graphdriver.Driver
-	LayerStore  layer.Store
-	Root        string
+	GraphDriver     graphdriver.Driver
+	LayerStore      layer.Store
+	Root            string
+	IdentityMapping *idtools.IdentityMapping
 }
 
 type graphIDRegistrar interface {
@@ -74,8 +75,12 @@ func NewSnapshotter(opt Opt) (snapshot.SnapshotterBase, error) {
 	return s, nil
 }
 
+func (s *snapshotter) Name() string {
+	return "default"
+}
+
 func (s *snapshotter) IdentityMapping() *idtools.IdentityMapping {
-	return nil
+	return s.opt.IdentityMapping
 }
 
 func (s *snapshotter) Prepare(ctx context.Context, key, parent string, opts ...snapshots.Opt) error {
@@ -249,24 +254,24 @@ func (s *snapshotter) Mounts(ctx context.Context, key string) (snapshot.Mountabl
 		id := identity.NewID()
 		var rwlayer layer.RWLayer
 		return &mountable{
-			acquire: func() ([]mount.Mount, error) {
+			idmap: s.opt.IdentityMapping,
+			acquire: func() ([]mount.Mount, func() error, error) {
 				rwlayer, err = s.opt.LayerStore.CreateRWLayer(id, l.ChainID(), nil)
 				if err != nil {
-					return nil, err
+					return nil, nil, err
 				}
 				rootfs, err := rwlayer.Mount("")
 				if err != nil {
-					return nil, err
+					return nil, nil, err
 				}
 				return []mount.Mount{{
-					Source:  rootfs.Path(),
-					Type:    "bind",
-					Options: []string{"rbind"},
-				}}, nil
-			},
-			release: func() error {
-				_, err := s.opt.LayerStore.ReleaseRWLayer(rwlayer)
-				return err
+						Source:  rootfs.Path(),
+						Type:    "bind",
+						Options: []string{"rbind"},
+					}}, func() error {
+						_, err := s.opt.LayerStore.ReleaseRWLayer(rwlayer)
+						return err
+					}, nil
 			},
 		}, nil
 	}
@@ -274,19 +279,19 @@ func (s *snapshotter) Mounts(ctx context.Context, key string) (snapshot.Mountabl
 	id, _ := s.getGraphDriverID(key)
 
 	return &mountable{
-		acquire: func() ([]mount.Mount, error) {
+		idmap: s.opt.IdentityMapping,
+		acquire: func() ([]mount.Mount, func() error, error) {
 			rootfs, err := s.opt.GraphDriver.Get(id, "")
 			if err != nil {
-				return nil, err
+				return nil, nil, err
 			}
 			return []mount.Mount{{
-				Source:  rootfs.Path(),
-				Type:    "bind",
-				Options: []string{"rbind"},
-			}}, nil
-		},
-		release: func() error {
-			return s.opt.GraphDriver.Put(id)
+					Source:  rootfs.Path(),
+					Type:    "bind",
+					Options: []string{"rbind"},
+				}}, func() error {
+					return s.opt.GraphDriver.Put(id)
+				}, nil
 		},
 	}, nil
 }
@@ -433,31 +438,33 @@ func (s *snapshotter) Close() error {
 type mountable struct {
 	mu       sync.Mutex
 	mounts   []mount.Mount
-	acquire  func() ([]mount.Mount, error)
+	acquire  func() ([]mount.Mount, func() error, error)
 	release  func() error
 	refCount int
+	idmap    *idtools.IdentityMapping
 }
 
-func (m *mountable) Mount() ([]mount.Mount, error) {
+func (m *mountable) Mount() ([]mount.Mount, func() error, error) {
 	m.mu.Lock()
 	defer m.mu.Unlock()
 
 	if m.mounts != nil {
 		m.refCount++
-		return m.mounts, nil
+		return m.mounts, m.releaseMount, nil
 	}
 
-	mounts, err := m.acquire()
+	mounts, release, err := m.acquire()
 	if err != nil {
-		return nil, err
+		return nil, nil, err
 	}
 	m.mounts = mounts
+	m.release = release
 	m.refCount = 1
 
-	return m.mounts, nil
+	return m.mounts, m.releaseMount, nil
 }
 
-func (m *mountable) Release() error {
+func (m *mountable) releaseMount() error {
 	m.mu.Lock()
 	defer m.mu.Unlock()
 
@@ -472,9 +479,12 @@ func (m *mountable) Release() error {
 	}
 
 	m.mounts = nil
+	defer func() {
+		m.release = nil
+	}()
 	return m.release()
 }
 
 func (m *mountable) IdentityMapping() *idtools.IdentityMapping {
-	return nil
+	return m.idmap
 }

builder/builder-next/builder.go

@@ -17,6 +17,7 @@ import (
 	"github.com/docker/docker/builder"
 	"github.com/docker/docker/daemon/config"
 	"github.com/docker/docker/daemon/images"
+	"github.com/docker/docker/pkg/idtools"
 	"github.com/docker/docker/pkg/streamformatter"
 	"github.com/docker/docker/pkg/system"
 	"github.com/docker/libnetwork"
@@ -73,6 +74,9 @@ type Opt struct {
 	ResolverOpt         resolver.ResolveOptionsFunc
 	BuilderConfig       config.BuilderConfig
 	Rootless            bool
+	IdentityMapping     *idtools.IdentityMapping
+	DNSConfig           config.DNSConfig
+	ApparmorProfile     string
 }
 
 // Builder can build using BuildKit backend
@@ -88,6 +92,10 @@ type Builder struct {
 func New(opt Opt) (*Builder, error) {
 	reqHandler := newReqBodyHandler(tracing.DefaultTransport)
 
+	if opt.IdentityMapping != nil && opt.IdentityMapping.Empty() {
+		opt.IdentityMapping = nil
+	}
+
 	c, err := newController(reqHandler, opt)
 	if err != nil {
 		return nil, err
@@ -234,7 +242,9 @@ func (b *Builder) Build(ctx context.Context, opt backend.BuildConfig) (*builder.
 		}
 
 		defer func() {
+			b.mu.Lock()
 			delete(b.jobs, buildID)
+			b.mu.Unlock()
 		}()
 	}
 

builder/builder-next/controller.go

@@ -8,6 +8,7 @@ import (
 	"github.com/containerd/containerd/content/local"
 	"github.com/containerd/containerd/platforms"
 	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/filters"
 	"github.com/docker/docker/builder/builder-next/adapters/containerimage"
 	"github.com/docker/docker/builder/builder-next/adapters/localinlinecache"
 	"github.com/docker/docker/builder/builder-next/adapters/snapshot"
@@ -38,7 +39,7 @@ import (
 )
 
 func newController(rt http.RoundTripper, opt Opt) (*control.Controller, error) {
-	if err := os.MkdirAll(opt.Root, 0700); err != nil {
+	if err := os.MkdirAll(opt.Root, 0711); err != nil {
 		return nil, err
 	}
 
@@ -55,9 +56,10 @@ func newController(rt http.RoundTripper, opt Opt) (*control.Controller, error) {
 	}
 
 	sbase, err := snapshot.NewSnapshotter(snapshot.Opt{
-		GraphDriver: driver,
-		LayerStore:  dist.LayerStore,
-		Root:        root,
+		GraphDriver:     driver,
+		LayerStore:      dist.LayerStore,
+		Root:            root,
+		IdentityMapping: opt.IdentityMapping,
 	})
 	if err != nil {
 		return nil, err
@@ -112,7 +114,9 @@ func newController(rt http.RoundTripper, opt Opt) (*control.Controller, error) {
 		return nil, err
 	}
 
-	exec, err := newExecutor(root, opt.DefaultCgroupParent, opt.NetworkController, opt.Rootless)
+	dns := getDNSConfig(opt.DNSConfig)
+
+	exec, err := newExecutor(root, opt.DefaultCgroupParent, opt.NetworkController, dns, opt.Rootless, opt.IdentityMapping, opt.ApparmorProfile)
 	if err != nil {
 		return nil, err
 	}
@@ -192,10 +196,7 @@ func newController(rt http.RoundTripper, opt Opt) (*control.Controller, error) {
 		ResolveCacheExporterFuncs: map[string]remotecache.ResolveCacheExporterFunc{
 			"inline": inlineremotecache.ResolveCacheExporterFunc(),
 		},
-		Entitlements: []string{
-			string(entitlements.EntitlementNetworkHost),
-			// string(entitlements.EntitlementSecurityInsecure),
-		},
+		Entitlements: getEntitlements(opt.BuilderConfig),
 	})
 }
 
@@ -229,7 +230,7 @@ func getGCPolicy(conf config.BuilderConfig, root string) ([]client.PruneInfo, er
 				gcPolicy[i], err = toBuildkitPruneInfo(types.BuildCachePruneOptions{
 					All:         p.All,
 					KeepStorage: b,
-					Filters:     p.Filter,
+					Filters:     filters.Args(p.Filter),
 				})
 				if err != nil {
 					return nil, err
@@ -251,3 +252,15 @@ func parsePlatforms(platformsStr []string) ([]specs.Platform, error) {
 	}
 	return out, nil
 }
+
+func getEntitlements(conf config.BuilderConfig) []string {
+	var ents []string
+	// Incase of no config settings, NetworkHost should be enabled & SecurityInsecure must be disabled.
+	if conf.Entitlements.NetworkHost == nil || *conf.Entitlements.NetworkHost {
+		ents = append(ents, string(entitlements.EntitlementNetworkHost))
+	}
+	if conf.Entitlements.SecurityInsecure != nil && *conf.Entitlements.SecurityInsecure {
+		ents = append(ents, string(entitlements.EntitlementSecurityInsecure))
+	}
+	return ents
+}

builder/builder-next/executor_unix.go

@@ -8,8 +8,12 @@ import (
 	"strconv"
 	"sync"
 
+	"github.com/docker/docker/daemon/config"
+	"github.com/docker/docker/pkg/idtools"
+	"github.com/docker/docker/pkg/stringid"
 	"github.com/docker/libnetwork"
 	"github.com/moby/buildkit/executor"
+	"github.com/moby/buildkit/executor/oci"
 	"github.com/moby/buildkit/executor/runcexecutor"
 	"github.com/moby/buildkit/identity"
 	"github.com/moby/buildkit/solver/pb"
@@ -20,7 +24,7 @@ import (
 
 const networkName = "bridge"
 
-func newExecutor(root, cgroupParent string, net libnetwork.NetworkController, rootless bool) (executor.Executor, error) {
+func newExecutor(root, cgroupParent string, net libnetwork.NetworkController, dnsConfig *oci.DNSConfig, rootless bool, idmap *idtools.IdentityMapping, apparmorProfile string) (executor.Executor, error) {
 	networkProviders := map[pb.NetMode]network.Provider{
 		pb.NetMode_UNSET: &bridgeProvider{NetworkController: net, Root: filepath.Join(root, "net")},
 		pb.NetMode_HOST:  network.NewHostProvider(),
@@ -31,6 +35,10 @@ func newExecutor(root, cgroupParent string, net libnetwork.NetworkController, ro
 		CommandCandidates:   []string{"runc"},
 		DefaultCgroupParent: cgroupParent,
 		Rootless:            rootless,
+		NoPivot:             os.Getenv("DOCKER_RAMDISK") != "",
+		IdentityMapping:     idmap,
+		DNS:                 dnsConfig,
+		ApparmorProfile:     apparmorProfile,
 	}, networkProviders)
 }
 
@@ -94,11 +102,12 @@ func (iface *lnInterface) Set(s *specs.Spec) {
 		logrus.WithError(iface.err).Error("failed to set networking spec")
 		return
 	}
+	shortNetCtlrID := stringid.TruncateID(iface.provider.NetworkController.ID())
 	// attach netns to bridge within the container namespace, using reexec in a prestart hook
 	s.Hooks = &specs.Hooks{
 		Prestart: []specs.Hook{{
 			Path: filepath.Join("/proc", strconv.Itoa(os.Getpid()), "exe"),
-			Args: []string{"libnetwork-setkey", "-exec-root=" + iface.provider.Config().Daemon.ExecRoot, iface.sbx.ContainerID(), iface.provider.NetworkController.ID()},
+			Args: []string{"libnetwork-setkey", "-exec-root=" + iface.provider.Config().Daemon.ExecRoot, iface.sbx.ContainerID(), shortNetCtlrID},
 		}},
 	}
 }
@@ -114,3 +123,14 @@ func (iface *lnInterface) Close() error {
 	}
 	return iface.err
 }
+
+func getDNSConfig(cfg config.DNSConfig) *oci.DNSConfig {
+	if cfg.DNS != nil || cfg.DNSSearch != nil || cfg.DNSOptions != nil {
+		return &oci.DNSConfig{
+			Nameservers:   cfg.DNS,
+			SearchDomains: cfg.DNSSearch,
+			Options:       cfg.DNSOptions,
+		}
+	}
+	return nil
+}

builder/builder-next/executor_windows.go

@@ -5,12 +5,15 @@ import (
 	"errors"
 	"io"
 
+	"github.com/docker/docker/daemon/config"
+	"github.com/docker/docker/pkg/idtools"
 	"github.com/docker/libnetwork"
 	"github.com/moby/buildkit/cache"
 	"github.com/moby/buildkit/executor"
+	"github.com/moby/buildkit/executor/oci"
 )
 
-func newExecutor(_, _ string, _ libnetwork.NetworkController, _ bool) (executor.Executor, error) {
+func newExecutor(_, _ string, _ libnetwork.NetworkController, _ *oci.DNSConfig, _ bool, _ *idtools.IdentityMapping, _ string) (executor.Executor, error) {
 	return &winExecutor{}, nil
 }
 
@@ -20,3 +23,7 @@ type winExecutor struct {
 func (e *winExecutor) Exec(ctx context.Context, meta executor.Meta, rootfs cache.Mountable, mounts []executor.Mount, stdin io.ReadCloser, stdout, stderr io.WriteCloser) error {
 	return errors.New("buildkit executor not implemented for windows")
 }
+
+func getDNSConfig(config.DNSConfig) *oci.DNSConfig {
+	return nil
+}

builder/builder-next/exporter/export.go

@@ -117,12 +117,12 @@ func (e *imageExporterInstance) Export(ctx context.Context, inp exporter.Source)
 		layersDone := oneOffProgress(ctx, "exporting layers")
 
 		if err := ref.Finalize(ctx, true); err != nil {
-			return nil, err
+			return nil, layersDone(err)
 		}
 
 		diffIDs, err := e.opt.Differ.EnsureLayer(ctx, ref.ID())
 		if err != nil {
-			return nil, err
+			return nil, layersDone(err)
 		}
 
 		diffs = make([]digest.Digest, len(diffIDs))

builder/builder-next/exporter/writer.go

@@ -137,6 +137,37 @@ func normalizeLayersAndHistory(diffs []digest.Digest, history []ocispec.History,
 		history[i] = h
 	}
 
+	// Find the first new layer time. Otherwise, the history item for a first
+	// metadata command would be the creation time of a base image layer.
+	// If there is no such then the last layer with timestamp.
+	var created *time.Time
+	var noCreatedTime bool
+	for _, h := range history {
+		if h.Created != nil {
+			created = h.Created
+			if noCreatedTime {
+				break
+			}
+		} else {
+			noCreatedTime = true
+		}
+	}
+
+	// Fill in created times for all history items to be either the first new
+	// layer time or the previous layer.
+	noCreatedTime = false
+	for i, h := range history {
+		if h.Created != nil {
+			if noCreatedTime {
+				created = h.Created
+			}
+		} else {
+			noCreatedTime = true
+			h.Created = created
+		}
+		history[i] = h
+	}
+
 	return diffs, history
 }
 

builder/builder-next/reqbodyhandler.go

@@ -35,6 +35,7 @@ func (h *reqBodyHandler) newRequest(rc io.ReadCloser) (string, func()) {
 		h.mu.Lock()
 		delete(h.requests, id)
 		h.mu.Unlock()
+		rc.Close()
 	}
 }
 

builder/builder-next/worker/worker.go

@@ -7,6 +7,7 @@ import (
 	"io/ioutil"
 	nethttp "net/http"
 	"runtime"
+	"strings"
 	"time"
 
 	"github.com/containerd/containerd/content"
@@ -43,6 +44,7 @@ import (
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
+	bolt "go.etcd.io/bbolt"
 )
 
 const labelCreatedAt = "buildkit/createdat"
@@ -257,6 +259,47 @@ func (w *Worker) GetRemote(ctx context.Context, ref cache.ImmutableRef, createIf
 	}, nil
 }
 
+// PruneCacheMounts removes the current cache snapshots for specified IDs
+func (w *Worker) PruneCacheMounts(ctx context.Context, ids []string) error {
+	mu := ops.CacheMountsLocker()
+	mu.Lock()
+	defer mu.Unlock()
+
+	for _, id := range ids {
+		id = "cache-dir:" + id
+		sis, err := w.MetadataStore.Search(id)
+		if err != nil {
+			return err
+		}
+		for _, si := range sis {
+			for _, k := range si.Indexes() {
+				if k == id || strings.HasPrefix(k, id+":") {
+					if siCached := w.CacheManager.Metadata(si.ID()); siCached != nil {
+						si = siCached
+					}
+					if err := cache.CachePolicyDefault(si); err != nil {
+						return err
+					}
+					si.Queue(func(b *bolt.Bucket) error {
+						return si.SetValue(b, k, nil)
+					})
+					if err := si.Commit(); err != nil {
+						return err
+					}
+					// if ref is unused try to clean it up right away by releasing it
+					if mref, err := w.CacheManager.GetMutable(ctx, si.ID()); err == nil {
+						go mref.Release(context.TODO())
+					}
+					break
+				}
+			}
+		}
+	}
+
+	ops.ClearActiveCacheMounts()
+	return nil
+}
+
 // FromRemote converts a remote snapshot reference to a local one
 func (w *Worker) FromRemote(ctx context.Context, remote *solver.Remote) (cache.ImmutableRef, error) {
 	rootfs, err := getLayers(ctx, remote.Descriptors)

builder/dockerfile/copy.go

@@ -556,13 +556,15 @@ func copyFile(archiver Archiver, source, dest *copyEndpoint, identity *idtools.I
 			return errors.Wrapf(err, "failed to create new directory")
 		}
 	} else {
+		// Normal containers
 		if identity == nil {
-			if err := os.MkdirAll(filepath.Dir(dest.path), 0755); err != nil {
+			// Use system.MkdirAll here, which is a custom version of os.MkdirAll
+			// modified for use on Windows to handle volume GUID paths (\\?\{dae8d3ac-b9a1-11e9-88eb-e8554b2ba1db}\path\)
+			if err := system.MkdirAll(filepath.Dir(dest.path), 0755, ""); err != nil {
 				return err
 			}
 		} else {
 			if err := idtools.MkdirAllAndChownNew(filepath.Dir(dest.path), 0755, *identity); err != nil {
-				// Normal containers
 				return errors.Wrapf(err, "failed to create new directory")
 			}
 		}

builder/dockerfile/evaluator_test.go

@@ -24,8 +24,11 @@ func init() {
 	reexec.Init()
 }
 
-func initDispatchTestCases() []dispatchTestCase {
-	dispatchTestCases := []dispatchTestCase{
+func TestDispatch(t *testing.T) {
+	if runtime.GOOS != "windows" {
+		skip.If(t, os.Getuid() != 0, "skipping test that requires root")
+	}
+	testCases := []dispatchTestCase{
 		{
 			name: "ADD multiple files to file",
 			cmd: &instructions.AddCommand{SourcesAndDest: instructions.SourcesAndDest{
@@ -92,56 +95,46 @@ func initDispatchTestCases() []dispatchTestCase {
 			}},
 			expectedError: "source can't be a URL for COPY",
 			files:         nil,
-		}}
-
-	return dispatchTestCases
-}
-
-func TestDispatch(t *testing.T) {
-	if runtime.GOOS != "windows" {
-		skip.If(t, os.Getuid() != 0, "skipping test that requires root")
+		},
 	}
-	testCases := initDispatchTestCases()
 
-	for _, testCase := range testCases {
-		executeTestCase(t, testCase)
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			contextDir, cleanup := createTestTempDir(t, "", "builder-dockerfile-test")
+			defer cleanup()
+
+			for filename, content := range tc.files {
+				createTestTempFile(t, contextDir, filename, content, 0777)
+			}
+
+			tarStream, err := archive.Tar(contextDir, archive.Uncompressed)
+
+			if err != nil {
+				t.Fatalf("Error when creating tar stream: %s", err)
+			}
+
+			defer func() {
+				if err = tarStream.Close(); err != nil {
+					t.Fatalf("Error when closing tar stream: %s", err)
+				}
+			}()
+
+			context, err := remotecontext.FromArchive(tarStream)
+
+			if err != nil {
+				t.Fatalf("Error when creating tar context: %s", err)
+			}
+
+			defer func() {
+				if err = context.Close(); err != nil {
+					t.Fatalf("Error when closing tar context: %s", err)
+				}
+			}()
+
+			b := newBuilderWithMockBackend()
+			sb := newDispatchRequest(b, '`', context, NewBuildArgs(make(map[string]*string)), newStagesBuildResults())
+			err = dispatch(sb, tc.cmd)
+			assert.Check(t, is.ErrorContains(err, tc.expectedError))
+		})
 	}
 }
-
-func executeTestCase(t *testing.T, testCase dispatchTestCase) {
-	contextDir, cleanup := createTestTempDir(t, "", "builder-dockerfile-test")
-	defer cleanup()
-
-	for filename, content := range testCase.files {
-		createTestTempFile(t, contextDir, filename, content, 0777)
-	}
-
-	tarStream, err := archive.Tar(contextDir, archive.Uncompressed)
-
-	if err != nil {
-		t.Fatalf("Error when creating tar stream: %s", err)
-	}
-
-	defer func() {
-		if err = tarStream.Close(); err != nil {
-			t.Fatalf("Error when closing tar stream: %s", err)
-		}
-	}()
-
-	context, err := remotecontext.FromArchive(tarStream)
-
-	if err != nil {
-		t.Fatalf("Error when creating tar context: %s", err)
-	}
-
-	defer func() {
-		if err = context.Close(); err != nil {
-			t.Fatalf("Error when closing tar context: %s", err)
-		}
-	}()
-
-	b := newBuilderWithMockBackend()
-	sb := newDispatchRequest(b, '`', context, NewBuildArgs(make(map[string]*string)), newStagesBuildResults())
-	err = dispatch(sb, testCase.cmd)
-	assert.Check(t, is.ErrorContains(err, testCase.expectedError))
-}

client/client_unix.go

@@ -1,4 +1,4 @@
-// +build linux freebsd openbsd darwin
+// +build linux freebsd openbsd netbsd darwin dragonfly
 
 package client // import "github.com/docker/docker/client"
 

client/ping.go

@@ -31,6 +31,8 @@ func (cli *Client) Ping(ctx context.Context) (types.Ping, error) {
 			// Server handled the request, so parse the response
 			return parsePingResponse(cli, serverResp)
 		}
+	} else if IsErrConnectionFailed(err) {
+		return ping, err
 	}
 
 	req, err = cli.buildRequest("GET", path.Join(cli.basePath, "/_ping"), nil, nil)

client/request.go

@@ -143,8 +143,7 @@ func (cli *Client) doRequest(ctx context.Context, req *http.Request) (serverResp
 
 		// Don't decorate context sentinel errors; users may be comparing to
 		// them directly.
-		switch err {
-		case context.Canceled, context.DeadlineExceeded:
+		if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
 			return serverResp, err
 		}
 
@@ -178,7 +177,13 @@ func (cli *Client) doRequest(ctx context.Context, req *http.Request) (serverResp
 		// this is localised - for example in French the error would be
 		// `open //./pipe/docker_engine: Le fichier spécifié est introuvable.`
 		if strings.Contains(err.Error(), `open //./pipe/docker_engine`) {
-			err = errors.New(err.Error() + " In the default daemon configuration on Windows, the docker client must be run elevated to connect. This error may also indicate that the docker daemon is not running.")
+			// Checks if client is running with elevated privileges
+			if f, elevatedErr := os.Open("\\\\.\\PHYSICALDRIVE0"); elevatedErr == nil {
+				err = errors.Wrap(err, "In the default daemon configuration on Windows, the docker client must be run with elevated privileges to connect.")
+			} else {
+				f.Close()
+				err = errors.Wrap(err, "This error may indicate that the docker daemon is not running.")
+			}
 		}
 
 		return serverResp, errors.Wrap(err, "error during connect")

cmd/dockerd/config.go

@@ -3,8 +3,10 @@ package main
 import (
 	"runtime"
 
+	"github.com/docker/docker/daemon"
 	"github.com/docker/docker/daemon/config"
 	"github.com/docker/docker/opts"
+	"github.com/docker/docker/plugin/executor/containerd"
 	"github.com/docker/docker/registry"
 	"github.com/spf13/pflag"
 )
@@ -12,6 +14,8 @@ import (
 const (
 	// defaultShutdownTimeout is the default shutdown timeout for the daemon
 	defaultShutdownTimeout = 15
+	// defaultTrustKeyFile is the default filename for the trust key
+	defaultTrustKeyFile = "key.json"
 )
 
 // installCommonConfigFlags adds flags to the pflag.FlagSet to configure the daemon
@@ -83,7 +87,13 @@ func installCommonConfigFlags(conf *config.Config, flags *pflag.FlagSet) error {
 
 	conf.MaxConcurrentDownloads = &maxConcurrentDownloads
 	conf.MaxConcurrentUploads = &maxConcurrentUploads
-	return nil
+
+	flags.StringVar(&conf.ContainerdNamespace, "containerd-namespace", daemon.ContainersNamespace, "Containerd namespace to use")
+	if err := flags.MarkHidden("containerd-namespace"); err != nil {
+		return err
+	}
+	flags.StringVar(&conf.ContainerdPluginNamespace, "containerd-plugins-namespace", containerd.PluginNamespace, "Containerd namespace to use for plugins")
+	return flags.MarkHidden("containerd-plugins-namespace")
 }
 
 func installRegistryServiceFlags(options *registry.ServiceOptions, flags *pflag.FlagSet) {

cmd/dockerd/config_common_unix.go

@@ -9,12 +9,11 @@ import (
 	"github.com/docker/docker/daemon/config"
 	"github.com/docker/docker/opts"
 	"github.com/docker/docker/pkg/homedir"
-	"github.com/docker/docker/rootless"
 	"github.com/spf13/pflag"
 )
 
 func getDefaultPidFile() (string, error) {
-	if !rootless.RunningWithNonRootUsername() {
+	if !honorXDG {
 		return "/var/run/docker.pid", nil
 	}
 	runtimeDir, err := homedir.GetRuntimeDir()
@@ -25,7 +24,7 @@ func getDefaultPidFile() (string, error) {
 }
 
 func getDefaultDataRoot() (string, error) {
-	if !rootless.RunningWithNonRootUsername() {
+	if !honorXDG {
 		return "/var/lib/docker", nil
 	}
 	dataHome, err := homedir.GetDataHome()
@@ -36,7 +35,7 @@ func getDefaultDataRoot() (string, error) {
 }
 
 func getDefaultExecRoot() (string, error) {
-	if !rootless.RunningWithNonRootUsername() {
+	if !honorXDG {
 		return "/var/run/docker", nil
 	}
 	runtimeDir, err := homedir.GetRuntimeDir()

cmd/dockerd/config_unix.go

@@ -3,10 +3,13 @@
 package main
 
 import (
+	"os/exec"
+
 	"github.com/docker/docker/daemon/config"
 	"github.com/docker/docker/opts"
 	"github.com/docker/docker/rootless"
 	"github.com/docker/go-units"
+	"github.com/pkg/errors"
 	"github.com/spf13/pflag"
 )
 
@@ -35,7 +38,16 @@ func installConfigFlags(conf *config.Config, flags *pflag.FlagSet) error {
 	flags.BoolVar(&conf.BridgeConfig.EnableIPv6, "ipv6", false, "Enable IPv6 networking")
 	flags.StringVar(&conf.BridgeConfig.FixedCIDRv6, "fixed-cidr-v6", "", "IPv6 subnet for fixed IPs")
 	flags.BoolVar(&conf.BridgeConfig.EnableUserlandProxy, "userland-proxy", true, "Use userland proxy for loopback traffic")
-	flags.StringVar(&conf.BridgeConfig.UserlandProxyPath, "userland-proxy-path", "", "Path to the userland proxy binary")
+	defaultUserlandProxyPath := ""
+	if rootless.RunningWithRootlessKit() {
+		var err error
+		// use rootlesskit-docker-proxy for exposing the ports in RootlessKit netns to the initial namespace.
+		defaultUserlandProxyPath, err = exec.LookPath(rootless.RootlessKitDockerProxyBinary)
+		if err != nil {
+			return errors.Wrapf(err, "running with RootlessKit, but %s not installed", rootless.RootlessKitDockerProxyBinary)
+		}
+	}
+	flags.StringVar(&conf.BridgeConfig.UserlandProxyPath, "userland-proxy-path", defaultUserlandProxyPath, "Path to the userland proxy binary")
 	flags.StringVar(&conf.CgroupParent, "cgroup-parent", "", "Set parent cgroup for all containers")
 	flags.StringVar(&conf.RemappedRoot, "userns-remap", "", "User/Group setting for user namespaces")
 	flags.BoolVar(&conf.LiveRestoreEnabled, "live-restore", false, "Enable live restore of docker when containers are still running")
@@ -49,7 +61,8 @@ func installConfigFlags(conf *config.Config, flags *pflag.FlagSet) error {
 	flags.BoolVar(&conf.NoNewPrivileges, "no-new-privileges", false, "Set no-new-privileges by default for new containers")
 	flags.StringVar(&conf.IpcMode, "default-ipc-mode", config.DefaultIpcMode, `Default mode for containers ipc ("shareable" | "private")`)
 	flags.Var(&conf.NetworkConfig.DefaultAddressPools, "default-address-pool", "Default address pools for node specific local networks")
-	// Mostly users don't need to set this flag explicitly.
-	flags.BoolVar(&conf.Rootless, "rootless", rootless.RunningWithNonRootUsername(), "Enable rootless mode (experimental)")
+	// rootless needs to be explicitly specified for running "rootful" dockerd in rootless dockerd (#38702)
+	// Note that defaultUserlandProxyPath and honorXDG are configured according to the value of rootless.RunningWithRootlessKit, not the value of --rootless.
+	flags.BoolVar(&conf.Rootless, "rootless", rootless.RunningWithRootlessKit(), "Enable rootless mode; typically used with RootlessKit (experimental)")
 	return nil
 }

cmd/dockerd/daemon.go

@@ -91,6 +91,8 @@ func (cli *DaemonCli) start(opts *daemonOptions) (err error) {
 		return err
 	}
 
+	logrus.Info("Starting up")
+
 	cli.configFile = &opts.configFile
 	cli.flags = opts.flags
 
@@ -103,6 +105,12 @@ func (cli *DaemonCli) start(opts *daemonOptions) (err error) {
 		if cli.Config.IsRootless() {
 			logrus.Warn("Running in rootless mode. Cgroups, AppArmor, and CRIU are disabled.")
 		}
+		if rootless.RunningWithRootlessKit() {
+			logrus.Info("Running with RootlessKit integration")
+			if !cli.Config.IsRootless() {
+				return fmt.Errorf("rootless mode needs to be enabled for running with RootlessKit")
+			}
+		}
 	} else {
 		if cli.Config.IsRootless() {
 			return fmt.Errorf("rootless mode is supported only when running in experimental mode")
@@ -200,14 +208,10 @@ func (cli *DaemonCli) start(opts *daemonOptions) (err error) {
 		return errors.Wrap(err, "failed to validate authorization plugin")
 	}
 
-	// TODO: move into startMetricsServer()
-	if cli.Config.MetricsAddress != "" {
-		if !d.HasExperimental() {
-			return errors.Wrap(err, "metrics-addr is only supported when experimental is enabled")
-		}
-		if err := startMetricsServer(cli.Config.MetricsAddress); err != nil {
-			return err
-		}
+	cli.d = d
+
+	if err := cli.startMetricsServer(cli.Config.MetricsAddress); err != nil {
+		return err
 	}
 
 	c, err := createAndStartCluster(cli, d)
@@ -222,8 +226,6 @@ func (cli *DaemonCli) start(opts *daemonOptions) (err error) {
 
 	logrus.Info("Daemon has completed initialization")
 
-	cli.d = d
-
 	routerOptions, err := newRouterOptions(cli.Config, d)
 	if err != nil {
 		return err
@@ -260,6 +262,7 @@ func (cli *DaemonCli) start(opts *daemonOptions) (err error) {
 		return errors.Wrap(errAPI, "shutting down due to ServeAPI error")
 	}
 
+	logrus.Info("Daemon shutdown complete")
 	return nil
 }
 
@@ -309,6 +312,9 @@ func newRouterOptions(config *config.Config, d *daemon.Daemon) (routerOptions, e
 		ResolverOpt:         d.NewResolveOptionsFunc(),
 		BuilderConfig:       config.Builder,
 		Rootless:            d.Rootless(),
+		IdentityMapping:     d.IdentityMapping(),
+		DNSConfig:           config.DNSConfig,
+		ApparmorProfile:     daemon.DefaultApparmorProfile(),
 	})
 	if err != nil {
 		return opts, err
@@ -415,6 +421,14 @@ func loadDaemonCliConfig(opts *daemonOptions) (*config.Config, error) {
 		conf.CommonTLSOptions.KeyFile = opts.TLSOptions.KeyFile
 	}
 
+	if conf.TrustKeyPath == "" {
+		daemonConfDir, err := getDaemonConfDir(conf.Root)
+		if err != nil {
+			return nil, err
+		}
+		conf.TrustKeyPath = filepath.Join(daemonConfDir, defaultTrustKeyFile)
+	}
+
 	if flags.Changed("graph") && flags.Changed("data-root") {
 		return nil, errors.New(`cannot specify both "--graph" and "--data-root" option`)
 	}
@@ -589,11 +603,17 @@ func newAPIServerConfig(cli *DaemonCli) (*apiserver.Config, error) {
 
 func loadListeners(cli *DaemonCli, serverConfig *apiserver.Config) ([]string, error) {
 	var hosts []string
+	seen := make(map[string]struct{}, len(cli.Config.Hosts))
+
 	for i := 0; i < len(cli.Config.Hosts); i++ {
 		var err error
-		if cli.Config.Hosts[i], err = dopts.ParseHost(cli.Config.TLS, rootless.RunningWithNonRootUsername(), cli.Config.Hosts[i]); err != nil {
+		if cli.Config.Hosts[i], err = dopts.ParseHost(cli.Config.TLS, honorXDG, cli.Config.Hosts[i]); err != nil {
 			return nil, errors.Wrapf(err, "error parsing -H %s", cli.Config.Hosts[i])
 		}
+		if _, ok := seen[cli.Config.Hosts[i]]; ok {
+			continue
+		}
+		seen[cli.Config.Hosts[i]] = struct{}{}
 
 		protoAddr := cli.Config.Hosts[i]
 		protoAddrParts := strings.SplitN(protoAddr, "://", 2)
@@ -668,9 +688,9 @@ func validateAuthzPlugins(requestedPlugins []string, pg plugingetter.PluginGette
 	return nil
 }
 
-func systemContainerdRunning(isRootless bool) (string, bool, error) {
+func systemContainerdRunning(honorXDG bool) (string, bool, error) {
 	addr := containerddefaults.DefaultAddress
-	if isRootless {
+	if honorXDG {
 		runtimeDir, err := homedir.GetRuntimeDir()
 		if err != nil {
 			return "", false, err

cmd/dockerd/daemon_unix.go

@@ -18,14 +18,14 @@ import (
 	"github.com/docker/docker/daemon/config"
 	"github.com/docker/docker/libcontainerd/supervisor"
 	"github.com/docker/docker/pkg/homedir"
-	"github.com/docker/docker/rootless"
 	"github.com/docker/libnetwork/portallocator"
 	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
 	"golang.org/x/sys/unix"
 )
 
 func getDefaultDaemonConfigDir() (string, error) {
-	if !rootless.RunningWithNonRootUsername() {
+	if !honorXDG {
 		return "/etc/docker", nil
 	}
 	// NOTE: CLI uses ~/.docker while the daemon uses ~/.config/docker, because
@@ -58,6 +58,10 @@ func setDefaultUmask() error {
 	return nil
 }
 
+func getDaemonConfDir(_ string) (string, error) {
+	return getDefaultDaemonConfigDir()
+}
+
 func (cli *DaemonCli) getPlatformContainerdDaemonOpts() ([]supervisor.DaemonOpt, error) {
 	opts := []supervisor.DaemonOpt{
 		supervisor.WithOOMScore(cli.Config.OOMScoreAdjust),
@@ -148,11 +152,12 @@ func newCgroupParent(config *config.Config) string {
 func (cli *DaemonCli) initContainerD(ctx context.Context) (func(time.Duration) error, error) {
 	var waitForShutdown func(time.Duration) error
 	if cli.Config.ContainerdAddr == "" {
-		systemContainerdAddr, ok, err := systemContainerdRunning(cli.Config.IsRootless())
+		systemContainerdAddr, ok, err := systemContainerdRunning(honorXDG)
 		if err != nil {
 			return nil, errors.Wrap(err, "could not determine whether the system containerd is running")
 		}
 		if !ok {
+			logrus.Debug("Containerd not running, starting daemon managed containerd")
 			opts, err := cli.getContainerdDaemonOpts()
 			if err != nil {
 				return nil, errors.Wrap(err, "failed to generate containerd options")
@@ -162,6 +167,7 @@ func (cli *DaemonCli) initContainerD(ctx context.Context) (func(time.Duration) e
 			if err != nil {
 				return nil, errors.Wrap(err, "failed to start containerd")
 			}
+			logrus.Debug("Started daemon managed containerd")
 			cli.Config.ContainerdAddr = r.Address()
 
 			// Try to wait for containerd to shutdown

cmd/dockerd/daemon_windows.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"net"
 	"os"
+	"path/filepath"
 	"time"
 
 	"github.com/docker/docker/daemon/config"
@@ -23,6 +24,10 @@ func setDefaultUmask() error {
 	return nil
 }
 
+func getDaemonConfDir(root string) (string, error) {
+	return filepath.Join(root, `\config`), nil
+}
+
 // preNotifySystem sends a message to the host when the API is active, but before the daemon is
 func preNotifySystem() {
 	// start the service now to prevent timeouts waiting for daemon to start

cmd/dockerd/docker.go

@@ -10,11 +10,16 @@ import (
 	"github.com/docker/docker/pkg/jsonmessage"
 	"github.com/docker/docker/pkg/reexec"
 	"github.com/docker/docker/pkg/term"
+	"github.com/docker/docker/rootless"
 	"github.com/moby/buildkit/util/apicaps"
 	"github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 )
 
+var (
+	honorXDG bool
+)
+
 func newDaemonCommand() (*cobra.Command, error) {
 	opts := newDaemonOptions(config.New())
 
@@ -53,6 +58,14 @@ func init() {
 	if dockerversion.ProductName != "" {
 		apicaps.ExportedProduct = dockerversion.ProductName
 	}
+	// When running with RootlessKit, $XDG_RUNTIME_DIR, $XDG_DATA_HOME, and $XDG_CONFIG_HOME needs to be
+	// honored as the default dirs, because we are unlikely to have permissions to access the system-wide
+	// directories.
+	//
+	// Note that even running with --rootless, when not running with RootlessKit, honorXDG needs to be kept false,
+	// because the system-wide directories in the current mount namespace are expected to be accessible.
+	// ("rootful" dockerd in rootless dockerd, #38702)
+	honorXDG = rootless.RunningWithRootlessKit()
 }
 
 func main() {

cmd/dockerd/metrics.go

@@ -5,10 +5,19 @@ import (
 	"net/http"
 
 	"github.com/docker/go-metrics"
+	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 )
 
-func startMetricsServer(addr string) error {
+func (cli *DaemonCli) startMetricsServer(addr string) error {
+	if addr == "" {
+		return nil
+	}
+
+	if !cli.d.HasExperimental() {
+		return errors.New("metrics-addr is only supported when experimental is enabled")
+	}
+
 	if err := allocateDaemonPort(addr); err != nil {
 		return err
 	}

container/container.go

@@ -730,7 +730,7 @@ func (i *rio) Close() error {
 }
 
 func (i *rio) Wait() {
-	i.sc.Wait()
+	i.sc.Wait(context.Background())
 
 	i.IO.Wait()
 }

container/container_unix.go

@@ -20,7 +20,6 @@ import (
 	"github.com/opencontainers/selinux/go-selinux/label"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
-	"golang.org/x/sys/unix"
 )
 
 const (
@@ -136,7 +135,7 @@ func (container *Container) CopyImagePathContent(v volume.Volume, destination st
 		return err
 	}
 
-	id := stringid.GenerateNonCryptoID()
+	id := stringid.GenerateRandomID()
 	path, err := v.Mount(id)
 	if err != nil {
 		return err
@@ -147,7 +146,7 @@ func (container *Container) CopyImagePathContent(v volume.Volume, destination st
 			logrus.Warnf("error while unmounting volume %s: %v", v.Name(), err)
 		}
 	}()
-	if err := label.Relabel(path, container.MountLabel, true); err != nil && err != unix.ENOTSUP {
+	if err := label.Relabel(path, container.MountLabel, true); err != nil && !errors.Is(err, syscall.ENOTSUP) {
 		return err
 	}
 	return copyExistingContents(rootfs, path)
@@ -190,7 +189,7 @@ func (container *Container) UnmountIpcMount() error {
 	if shmPath == "" {
 		return nil
 	}
-	if err = mount.Unmount(shmPath); err != nil && !os.IsNotExist(err) {
+	if err = mount.Unmount(shmPath); err != nil && !os.IsNotExist(errors.Cause(err)) {
 		return err
 	}
 	return nil

container/container_windows.go

@@ -153,7 +153,6 @@ func (container *Container) UpdateContainer(hostConfig *containertypes.HostConfi
 		resources.CpusetMems != "" ||
 		len(resources.Devices) != 0 ||
 		len(resources.DeviceCgroupRules) != 0 ||
-		resources.DiskQuota != 0 ||
 		resources.KernelMemory != 0 ||
 		resources.MemoryReservation != 0 ||
 		resources.MemorySwap != 0 ||

container/health.go

@@ -22,7 +22,7 @@ func (s *Health) String() string {
 	case types.Starting:
 		return "health: starting"
 	default: // Healthy and Unhealthy are clear on their own
-		return s.Health.Status
+		return status
 	}
 }
 

container/state.go

@@ -17,7 +17,7 @@ import (
 type State struct {
 	sync.Mutex
 	// Note that `Running` and `Paused` are not mutually exclusive:
-	// When pausing a container (on Linux), the cgroups freezer is used to suspend
+	// When pausing a container (on Linux), the freezer cgroup is used to suspend
 	// all processes in the container. Freezing the process requires the process to
 	// be running. As a result, paused containers are both `Running` _and_ `Paused`.
 	Running           bool

container/stream/streams.go

@@ -1,6 +1,7 @@
 package stream // import "github.com/docker/docker/container/stream"
 
 import (
+	"context"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -24,11 +25,12 @@ import (
 // copied and delivered to all StdoutPipe and StderrPipe consumers, using
 // a kind of "broadcaster".
 type Config struct {
-	sync.WaitGroup
+	wg        sync.WaitGroup
 	stdout    *broadcaster.Unbuffered
 	stderr    *broadcaster.Unbuffered
 	stdin     io.ReadCloser
 	stdinPipe io.WriteCloser
+	dio       *cio.DirectIO
 }
 
 // NewConfig creates a stream config and initializes
@@ -115,14 +117,15 @@ func (c *Config) CloseStreams() error {
 
 // CopyToPipe connects streamconfig with a libcontainerd.IOPipe
 func (c *Config) CopyToPipe(iop *cio.DirectIO) {
+	c.dio = iop
 	copyFunc := func(w io.Writer, r io.ReadCloser) {
-		c.Add(1)
+		c.wg.Add(1)
 		go func() {
 			if _, err := pools.Copy(w, r); err != nil {
 				logrus.Errorf("stream copy error: %v", err)
 			}
 			r.Close()
-			c.Done()
+			c.wg.Done()
 		}()
 	}
 
@@ -144,3 +147,23 @@ func (c *Config) CopyToPipe(iop *cio.DirectIO) {
 		}
 	}
 }
+
+// Wait for the stream to close
+// Wait supports timeouts via the context to unblock and forcefully
+// close the io streams
+func (c *Config) Wait(ctx context.Context) {
+	done := make(chan struct{}, 1)
+	go func() {
+		c.wg.Wait()
+		close(done)
+	}()
+	select {
+	case <-done:
+	case <-ctx.Done():
+		if c.dio != nil {
+			c.dio.Cancel()
+			c.dio.Wait()
+			c.dio.Close()
+		}
+	}
+}

contrib/apparmor/template.go

@@ -29,10 +29,11 @@ profile /usr/bin/docker (attach_disconnected, complain) {
   capability,
   owner /** rw,
   @{DOCKER_GRAPH_PATH}/** rwl,
-  @{DOCKER_GRAPH_PATH}/linkgraph.db k,
   @{DOCKER_GRAPH_PATH}/network/files/boltdb.db k,
   @{DOCKER_GRAPH_PATH}/network/files/local-kv.db k,
-  @{DOCKER_GRAPH_PATH}/[0-9]*.[0-9]*/linkgraph.db k,
+  # For user namespaces:
+  @{DOCKER_GRAPH_PATH}/[0-9]*.[0-9]*/network/files/boltdb.db k,
+  @{DOCKER_GRAPH_PATH}/[0-9]*.[0-9]*/network/files/local-kv.db k,
 
   # For non-root client use:
   /dev/urandom r,

contrib/dockerd-rootless.sh

@@ -3,13 +3,13 @@
 #
 # Usage: dockerd-rootless.sh --experimental [DOCKERD_OPTIONS]
 # Currently, specifying --experimental is mandatory.
-# Also, to expose ports, you need to specify
-# --userland-proxy-path=/path/to/rootlesskit-docker-proxy
 #
 # External dependencies:
 # * newuidmap and newgidmap needs to be installed.
 # * /etc/subuid and /etc/subgid needs to be configured for the current user.
-# * Either slirp4netns (v0.3+) or VPNKit needs to be installed.
+# * Either one of slirp4netns (>= v0.4.0), VPNKit, lxc-user-nic needs to be installed.
+#   slirp4netns is used by default if installed. Otherwise fallsback to VPNKit.
+#   The default value can be overridden with $DOCKERD_ROOTLESS_ROOTLESSKIT_NET=(slirp4netns|vpnkit|lxc-user-nic)
 #
 # See the documentation for the further information.
 
@@ -35,24 +35,35 @@ if [ -z $rootlesskit ]; then
 	exit 1
 fi
 
-net=""
-mtu=""
-if which slirp4netns >/dev/null 2>&1; then
-	if slirp4netns --help | grep -- --disable-host-loopback; then
-		net=slirp4netns
-		mtu=65520
-	else
-		echo "slirp4netns does not support --disable-host-loopback. Falling back to VPNKit."
+: "${DOCKERD_ROOTLESS_ROOTLESSKIT_NET:=}"
+: "${DOCKERD_ROOTLESS_ROOTLESSKIT_MTU:=}"
+: "${DOCKERD_ROOTLESS_ROOTLESSKIT_SLIRP4NETNS_SANDBOX:=auto}"
+: "${DOCKERD_ROOTLESS_ROOTLESSKIT_SLIRP4NETNS_SECCOMP:=auto}"
+net=$DOCKERD_ROOTLESS_ROOTLESSKIT_NET
+mtu=$DOCKERD_ROOTLESS_ROOTLESSKIT_MTU
+if [ -z $net ]; then
+	if which slirp4netns >/dev/null 2>&1; then
+		# If --netns-type is present in --help, slirp4netns is >= v0.4.0.
+		if slirp4netns --help | grep -qw -- --netns-type; then
+			net=slirp4netns
+			if [ -z $mtu ]; then
+				mtu=65520
+			fi
+		else
+			echo "slirp4netns found but seems older than v0.4.0. Falling back to VPNKit."
+		fi
+	fi
+	if [ -z $net ]; then
+		if which vpnkit >/dev/null 2>&1; then
+			net=vpnkit
+		else
+			echo "Either slirp4netns (>= v0.4.0) or vpnkit needs to be installed"
+			exit 1
+		fi
 	fi
 fi
-if [ -z $net ]; then
-	if which vpnkit >/dev/null 2>&1; then
-		net=vpnkit
-		mtu=1500
-	else
-		echo "Either slirp4netns (v0.3+) or vpnkit needs to be installed"
-		exit 1
-	fi
+if [ -z $mtu ]; then
+	mtu=1500
 fi
 
 if [ -z $_DOCKERD_ROOTLESS_CHILD ]; then
@@ -65,8 +76,11 @@ if [ -z $_DOCKERD_ROOTLESS_CHILD ]; then
 	#         namespace from being unexpectedly unmounted when `/etc/resolv.conf` is recreated on the host
 	#         (by either systemd-networkd or NetworkManager)
 	# * /run: copy-up is required so that we can create /run/docker (hardcoded for plugins) in our namespace
-	$rootlesskit \
-		--net=$net --mtu=$mtu --disable-host-loopback --port-driver=builtin \
+	exec $rootlesskit \
+		--net=$net --mtu=$mtu \
+		--slirp4netns-sandbox=$DOCKERD_ROOTLESS_ROOTLESSKIT_SLIRP4NETNS_SANDBOX \
+		--slirp4netns-seccomp=$DOCKERD_ROOTLESS_ROOTLESSKIT_SLIRP4NETNS_SECCOMP \
+		--disable-host-loopback --port-driver=builtin \
 		--copy-up=/etc --copy-up=/run \
 		$DOCKERD_ROOTLESS_ROOTLESSKIT_FLAGS \
 		$0 $@
@@ -74,6 +88,6 @@ else
 	[ $_DOCKERD_ROOTLESS_CHILD = 1 ]
 	# remove the symlinks for the existing files in the parent namespace if any,
 	# so that we can create our own files in our mount namespace.
-	rm -f /run/docker /run/xtables.lock
-	dockerd $@
+	rm -f /run/docker /run/containerd /run/xtables.lock
+	exec dockerd $@
 fi

daemon/apparmor_default.go

@@ -14,6 +14,14 @@ const (
 	defaultApparmorProfile = "docker-default"
 )
 
+// DefaultApparmorProfile returns the name of the default apparmor profile
+func DefaultApparmorProfile() string {
+	if apparmor.IsEnabled() {
+		return defaultApparmorProfile
+	}
+	return ""
+}
+
 func ensureDefaultAppArmorProfile() error {
 	if apparmor.IsEnabled() {
 		loaded, err := aaprofile.IsLoaded(defaultApparmorProfile)

daemon/apparmor_default_unsupported.go

@@ -5,3 +5,8 @@ package daemon // import "github.com/docker/docker/daemon"
 func ensureDefaultAppArmorProfile() error {
 	return nil
 }
+
+// DefaultApparmorProfile returns an empty string.
+func DefaultApparmorProfile() string {
+	return ""
+}

daemon/archive.go

@@ -31,18 +31,19 @@ type archiver interface {
 }
 
 // helper functions to extract or archive
-func extractArchive(i interface{}, src io.Reader, dst string, opts *archive.TarOptions) error {
+func extractArchive(i interface{}, src io.Reader, dst string, opts *archive.TarOptions, root string) error {
 	if ea, ok := i.(extractor); ok {
 		return ea.ExtractArchive(src, dst, opts)
 	}
-	return chrootarchive.Untar(src, dst, opts)
+
+	return chrootarchive.UntarWithRoot(src, dst, opts, root)
 }
 
-func archivePath(i interface{}, src string, opts *archive.TarOptions) (io.ReadCloser, error) {
+func archivePath(i interface{}, src string, opts *archive.TarOptions, root string) (io.ReadCloser, error) {
 	if ap, ok := i.(archiver); ok {
 		return ap.ArchivePath(src, opts)
 	}
-	return archive.TarWithOptions(src, opts)
+	return chrootarchive.Tar(src, opts, root)
 }
 
 // ContainerCopy performs a deprecated operation of archiving the resource at
@@ -235,10 +236,16 @@ func (daemon *Daemon) containerArchivePath(container *container.Container, path
 	if driver.Base(resolvedPath) == "." {
 		resolvedPath += string(driver.Separator()) + "."
 	}
-	sourceDir, sourceBase := driver.Dir(resolvedPath), driver.Base(resolvedPath)
+
+	sourceDir := resolvedPath
+	sourceBase := "."
+
+	if stat.Mode&os.ModeDir == 0 { // not dir
+		sourceDir, sourceBase = driver.Split(resolvedPath)
+	}
 	opts := archive.TarResourceRebaseOpts(sourceBase, driver.Base(absPath))
 
-	data, err := archivePath(driver, sourceDir, opts)
+	data, err := archivePath(driver, sourceDir, opts, container.BaseFS.Path())
 	if err != nil {
 		return nil, nil, err
 	}
@@ -367,7 +374,7 @@ func (daemon *Daemon) containerExtractToDir(container *container.Container, path
 		}
 	}
 
-	if err := extractArchive(driver, content, resolvedPath, options); err != nil {
+	if err := extractArchive(driver, content, resolvedPath, options, container.BaseFS.Path()); err != nil {
 		return err
 	}
 
@@ -425,14 +432,11 @@ func (daemon *Daemon) containerCopy(container *container.Container, resource str
 		d, f := driver.Split(basePath)
 		basePath = d
 		filter = []string{f}
-	} else {
-		filter = []string{driver.Base(basePath)}
-		basePath = driver.Dir(basePath)
 	}
 	archive, err := archivePath(driver, basePath, &archive.TarOptions{
 		Compression:  archive.Uncompressed,
 		IncludeFiles: filter,
-	})
+	}, container.BaseFS.Path())
 	if err != nil {
 		return nil, err
 	}

daemon/bindmount_unix.go

@@ -1,5 +0,0 @@
-// +build linux freebsd
-
-package daemon // import "github.com/docker/docker/daemon"
-
-const bindMountType = "bind"

daemon/cluster/cluster.go

@@ -58,6 +58,7 @@ import (
 	swarmnode "github.com/docker/swarmkit/node"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
+	"google.golang.org/grpc"
 )
 
 const swarmDirName = "swarm"
@@ -399,7 +400,10 @@ func (c *Cluster) Cleanup() {
 func managerStats(client swarmapi.ControlClient, currentNodeID string) (current bool, reachable int, unreachable int, err error) {
 	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
 	defer cancel()
-	nodes, err := client.ListNodes(ctx, &swarmapi.ListNodesRequest{})
+	nodes, err := client.ListNodes(
+		ctx, &swarmapi.ListNodesRequest{},
+		grpc.MaxCallRecvMsgSize(defaultRecvSizeForListResponse),
+	)
 	if err != nil {
 		return false, 0, 0, err
 	}

daemon/cluster/executor/container/container.go

@@ -6,7 +6,6 @@ import (
 	"net"
 	"strconv"
 	"strings"
-	"time"
 
 	"github.com/sirupsen/logrus"
 
@@ -31,10 +30,6 @@ import (
 )
 
 const (
-	// Explicitly use the kernel's default setting for CPU quota of 100ms.
-	// https://www.kernel.org/doc/Documentation/scheduler/sched-bwc.txt
-	cpuQuotaPeriod = 100 * time.Millisecond
-
 	// systemLabelPrefix represents the reserved namespace for system labels.
 	systemLabelPrefix = "com.docker.swarm"
 )
@@ -451,9 +446,7 @@ func (c *containerConfig) resources() enginecontainer.Resources {
 	}
 
 	if r.Limits.NanoCPUs > 0 {
-		// CPU Period must be set in microseconds.
-		resources.CPUPeriod = int64(cpuQuotaPeriod / time.Microsecond)
-		resources.CPUQuota = r.Limits.NanoCPUs * resources.CPUPeriod / 1e9
+		resources.NanoCPUs = r.Limits.NanoCPUs
 	}
 
 	return resources

daemon/cluster/executor/container/controller.go

@@ -369,11 +369,17 @@ func (r *controller) Shutdown(ctx context.Context) error {
 	}
 
 	if err := r.adapter.shutdown(ctx); err != nil {
-		if isUnknownContainer(err) || isStoppedContainer(err) {
-			return nil
+		if !(isUnknownContainer(err) || isStoppedContainer(err)) {
+			return err
 		}
+	}
 
-		return err
+	// Try removing networks referenced in this task in case this
+	// task is the last one referencing it
+	if err := r.adapter.removeNetworks(ctx); err != nil {
+		if !isUnknownContainer(err) {
+			return err
+		}
 	}
 
 	return nil
@@ -419,15 +425,6 @@ func (r *controller) Remove(ctx context.Context) error {
 		log.G(ctx).WithError(err).Debug("shutdown failed on removal")
 	}
 
-	// Try removing networks referenced in this task in case this
-	// task is the last one referencing it
-	if err := r.adapter.removeNetworks(ctx); err != nil {
-		if isUnknownContainer(err) {
-			return nil
-		}
-		return err
-	}
-
 	if err := r.adapter.remove(ctx); err != nil {
 		if isUnknownContainer(err) {
 			return nil
@@ -511,7 +508,9 @@ func (r *controller) Logs(ctx context.Context, publisher exec.LogPublisher, opti
 	var (
 		// use a rate limiter to keep things under control but also provides some
 		// ability coalesce messages.
-		limiter = rate.NewLimiter(rate.Every(time.Second), 10<<20) // 10 MB/s
+		// this will implement a "token bucket" of size 10 MB, initially full and refilled
+		// at rate 10 MB tokens per second.
+		limiter = rate.NewLimiter(10<<20, 10<<20) // 10 MB/s
 		msgctx  = api.LogContext{
 			NodeID:    r.task.NodeID,
 			ServiceID: r.task.ServiceID,

daemon/cluster/nodes.go

@@ -8,6 +8,7 @@ import (
 	"github.com/docker/docker/daemon/cluster/convert"
 	"github.com/docker/docker/errdefs"
 	swarmapi "github.com/docker/swarmkit/api"
+	"google.golang.org/grpc"
 )
 
 // GetNodes returns a list of all nodes known to a cluster.
@@ -30,7 +31,9 @@ func (c *Cluster) GetNodes(options apitypes.NodeListOptions) ([]types.Node, erro
 
 	r, err := state.controlClient.ListNodes(
 		ctx,
-		&swarmapi.ListNodesRequest{Filters: filters})
+		&swarmapi.ListNodesRequest{Filters: filters},
+		grpc.MaxCallRecvMsgSize(defaultRecvSizeForListResponse),
+	)
 	if err != nil {
 		return nil, err
 	}

daemon/cluster/secrets.go

@@ -7,6 +7,7 @@ import (
 	types "github.com/docker/docker/api/types/swarm"
 	"github.com/docker/docker/daemon/cluster/convert"
 	swarmapi "github.com/docker/swarmkit/api"
+	"google.golang.org/grpc"
 )
 
 // GetSecret returns a secret from a managed swarm cluster
@@ -44,7 +45,9 @@ func (c *Cluster) GetSecrets(options apitypes.SecretListOptions) ([]types.Secret
 	defer cancel()
 
 	r, err := state.controlClient.ListSecrets(ctx,
-		&swarmapi.ListSecretsRequest{Filters: filters})
+		&swarmapi.ListSecretsRequest{Filters: filters},
+		grpc.MaxCallRecvMsgSize(defaultRecvSizeForListResponse),
+	)
 	if err != nil {
 		return nil, err
 	}

daemon/cluster/swarm.go

@@ -19,6 +19,7 @@ import (
 	swarmnode "github.com/docker/swarmkit/node"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
+	"google.golang.org/grpc"
 )
 
 // Init initializes new cluster from user provided request.
@@ -32,6 +33,7 @@ func (c *Cluster) Init(req types.InitRequest) (string, error) {
 			// API handlers to finish before shutting down the node.
 			c.mu.Lock()
 			if !c.nr.nodeState.IsManager() {
+				c.mu.Unlock()
 				return "", errSwarmNotManager
 			}
 			c.mu.Unlock()
@@ -449,7 +451,10 @@ func (c *Cluster) Info() types.Info {
 
 		info.Cluster = &swarm.ClusterInfo
 
-		if r, err := state.controlClient.ListNodes(ctx, &swarmapi.ListNodesRequest{}); err != nil {
+		if r, err := state.controlClient.ListNodes(
+			ctx, &swarmapi.ListNodesRequest{},
+			grpc.MaxCallRecvMsgSize(defaultRecvSizeForListResponse),
+		); err != nil {
 			info.Error = err.Error()
 		} else {
 			info.Nodes = len(r.Nodes)

daemon/config/builder.go

@@ -1,12 +1,57 @@
 package config
 
-import "github.com/docker/docker/api/types/filters"
+import (
+	"encoding/json"
+	"fmt"
+	"sort"
+	"strings"
+
+	"github.com/docker/docker/api/types/filters"
+)
 
 // BuilderGCRule represents a GC rule for buildkit cache
 type BuilderGCRule struct {
-	All         bool         `json:",omitempty"`
-	Filter      filters.Args `json:",omitempty"`
-	KeepStorage string       `json:",omitempty"`
+	All         bool            `json:",omitempty"`
+	Filter      BuilderGCFilter `json:",omitempty"`
+	KeepStorage string          `json:",omitempty"`
+}
+
+// BuilderGCFilter contains garbage-collection filter rules for a BuildKit builder
+type BuilderGCFilter filters.Args
+
+// MarshalJSON returns a JSON byte representation of the BuilderGCFilter
+func (x *BuilderGCFilter) MarshalJSON() ([]byte, error) {
+	f := filters.Args(*x)
+	keys := f.Keys()
+	sort.Strings(keys)
+	arr := make([]string, 0, len(keys))
+	for _, k := range keys {
+		values := f.Get(k)
+		for _, v := range values {
+			arr = append(arr, fmt.Sprintf("%s=%s", k, v))
+		}
+	}
+	return json.Marshal(arr)
+}
+
+// UnmarshalJSON fills the BuilderGCFilter values structure from JSON input
+func (x *BuilderGCFilter) UnmarshalJSON(data []byte) error {
+	var arr []string
+	f := filters.NewArgs()
+	if err := json.Unmarshal(data, &arr); err != nil {
+		// backwards compat for deprecated buggy form
+		err := json.Unmarshal(data, &f)
+		*x = BuilderGCFilter(f)
+		return err
+	}
+	for _, s := range arr {
+		fields := strings.SplitN(s, "=", 2)
+		name := strings.ToLower(strings.TrimSpace(fields[0]))
+		value := strings.TrimSpace(fields[1])
+		f.Add(name, value)
+	}
+	*x = BuilderGCFilter(f)
+	return nil
 }
 
 // BuilderGCConfig contains GC config for a buildkit builder
@@ -16,7 +61,14 @@ type BuilderGCConfig struct {
 	DefaultKeepStorage string          `json:",omitempty"`
 }
 
+// BuilderEntitlements contains settings to enable/disable entitlements
+type BuilderEntitlements struct {
+	NetworkHost      *bool `json:"network-host,omitempty"`
+	SecurityInsecure *bool `json:"security-insecure,omitempty"`
+}
+
 // BuilderConfig contains config for the builder
 type BuilderConfig struct {
-	GC BuilderGCConfig `json:",omitempty"`
+	GC           BuilderGCConfig     `json:",omitempty"`
+	Entitlements BuilderEntitlements `json:",omitempty"`
 }

daemon/config/builder_test.go

@@ -0,0 +1,44 @@
+package config
+
+import (
+	"testing"
+
+	"github.com/docker/docker/api/types/filters"
+	"github.com/google/go-cmp/cmp"
+	"gotest.tools/assert"
+	"gotest.tools/fs"
+)
+
+func TestBuilderGC(t *testing.T) {
+	tempFile := fs.NewFile(t, "config", fs.WithContent(`{
+  "builder": {
+    "gc": {
+      "enabled": true,
+      "policy": [
+        {"keepStorage": "10GB", "filter": ["unused-for=2200h"]},
+        {"keepStorage": "50GB", "filter": {"unused-for": {"3300h": true}}},
+        {"keepStorage": "100GB", "all": true}
+      ]
+    }
+  }
+}`))
+	defer tempFile.Remove()
+	configFile := tempFile.Path()
+
+	cfg, err := MergeDaemonConfigurations(&Config{}, nil, configFile)
+	assert.NilError(t, err)
+	assert.Assert(t, cfg.Builder.GC.Enabled)
+	f1 := filters.NewArgs()
+	f1.Add("unused-for", "2200h")
+	f2 := filters.NewArgs()
+	f2.Add("unused-for", "3300h")
+	expectedPolicy := []BuilderGCRule{
+		{KeepStorage: "10GB", Filter: BuilderGCFilter(f1)},
+		{KeepStorage: "50GB", Filter: BuilderGCFilter(f2)}, /* parsed from deprecated form */
+		{KeepStorage: "100GB", All: true},
+	}
+	assert.DeepEqual(t, cfg.Builder.GC.Policy, expectedPolicy, cmp.AllowUnexported(BuilderGCFilter{}))
+	// double check to please the skeptics
+	assert.Assert(t, filters.Args(cfg.Builder.GC.Policy[0].Filter).UniqueExactMatch("unused-for", "2200h"))
+	assert.Assert(t, filters.Args(cfg.Builder.GC.Policy[1].Filter).UniqueExactMatch("unused-for", "3300h"))
+}

daemon/config/config.go

@@ -63,6 +63,8 @@ var flatOptions = map[string]bool{
 var skipValidateOptions = map[string]bool{
 	"features": true,
 	"builder":  true,
+	// Corresponding flag has been removed because it was already unusable
+	"deprecated-key-path": true,
 }
 
 // skipDuplicates contains configuration keys that
@@ -107,6 +109,13 @@ type CommonTLSOptions struct {
 	KeyFile  string `json:"tlskey,omitempty"`
 }
 
+// DNSConfig defines the DNS configurations.
+type DNSConfig struct {
+	DNS        []string `json:"dns,omitempty"`
+	DNSOptions []string `json:"dns-opts,omitempty"`
+	DNSSearch  []string `json:"dns-search,omitempty"`
+}
+
 // CommonConfig defines the configuration of a docker daemon which is
 // common across platforms.
 // It includes json tags to deserialize configuration from a file
@@ -117,9 +126,6 @@ type CommonConfig struct {
 	AutoRestart           bool                      `json:"-"`
 	Context               map[string][]string       `json:"-"`
 	DisableBridge         bool                      `json:"-"`
-	DNS                   []string                  `json:"dns,omitempty"`
-	DNSOptions            []string                  `json:"dns-opts,omitempty"`
-	DNSSearch             []string                  `json:"dns-search,omitempty"`
 	ExecOptions           []string                  `json:"exec-opts,omitempty"`
 	GraphDriver           string                    `json:"storage-driver,omitempty"`
 	GraphOptions          []string                  `json:"storage-opts,omitempty"`
@@ -134,6 +140,12 @@ type CommonConfig struct {
 	SocketGroup           string                    `json:"group,omitempty"`
 	CorsHeaders           string                    `json:"api-cors-header,omitempty"`
 
+	// TrustKeyPath is used to generate the daemon ID and for signing schema 1 manifests
+	// when pushing to a registry which does not support schema 2. This field is marked as
+	// deprecated because schema 1 manifests are deprecated in favor of schema 2 and the
+	// daemon ID will use a dedicated identifier not shared with exported signatures.
+	TrustKeyPath string `json:"deprecated-key-path,omitempty"`
+
 	// LiveRestoreEnabled determines whether we should keep containers
 	// alive upon daemon shutdown/start
 	LiveRestoreEnabled bool `json:"live-restore,omitempty"`
@@ -192,6 +204,7 @@ type CommonConfig struct {
 
 	MetricsAddress string `json:"metrics-addr"`
 
+	DNSConfig
 	LogConfig
 	BridgeConfig // bridgeConfig holds bridge network specific configuration.
 	NetworkConfig
@@ -222,6 +235,9 @@ type CommonConfig struct {
 	Features map[string]bool `json:"features,omitempty"`
 
 	Builder BuilderConfig `json:"builder,omitempty"`
+
+	ContainerdNamespace       string `json:"containerd-namespace,omitempty"`
+	ContainerdPluginNamespace string `json:"containerd-plugin-namespace,omitempty"`
 }
 
 // IsValueSet returns true if a configuration value
@@ -239,7 +255,6 @@ func New() *Config {
 	config := Config{}
 	config.LogConfig.Config = make(map[string]string)
 	config.ClusterOpts = make(map[string]string)
-
 	return &config
 }
 

daemon/config/config_test.go

@@ -244,28 +244,36 @@ func TestValidateConfigurationErrors(t *testing.T) {
 		{
 			config: &Config{
 				CommonConfig: CommonConfig{
-					DNS: []string{"1.1.1.1o"},
+					DNSConfig: DNSConfig{
+						DNS: []string{"1.1.1.1o"},
+					},
 				},
 			},
 		},
 		{
 			config: &Config{
 				CommonConfig: CommonConfig{
-					DNS: []string{"2.2.2.2", "1.1.1.1o"},
+					DNSConfig: DNSConfig{
+						DNS: []string{"2.2.2.2", "1.1.1.1o"},
+					},
 				},
 			},
 		},
 		{
 			config: &Config{
 				CommonConfig: CommonConfig{
-					DNSSearch: []string{"123456"},
+					DNSConfig: DNSConfig{
+						DNSSearch: []string{"123456"},
+					},
 				},
 			},
 		},
 		{
 			config: &Config{
 				CommonConfig: CommonConfig{
-					DNSSearch: []string{"a.b.c", "123456"},
+					DNSConfig: DNSConfig{
+						DNSSearch: []string{"a.b.c", "123456"},
+					},
 				},
 			},
 		},
@@ -329,14 +337,18 @@ func TestValidateConfiguration(t *testing.T) {
 		{
 			config: &Config{
 				CommonConfig: CommonConfig{
-					DNS: []string{"1.1.1.1"},
+					DNSConfig: DNSConfig{
+						DNS: []string{"1.1.1.1"},
+					},
 				},
 			},
 		},
 		{
 			config: &Config{
 				CommonConfig: CommonConfig{
-					DNSSearch: []string{"a.b.c"},
+					DNSConfig: DNSConfig{
+						DNSSearch: []string{"a.b.c"},
+					},
 				},
 			},
 		},

daemon/container_operations.go

@@ -237,7 +237,10 @@ func (daemon *Daemon) buildSandboxOptions(container *container.Container) ([]lib
 
 func (daemon *Daemon) updateNetworkSettings(container *container.Container, n libnetwork.Network, endpointConfig *networktypes.EndpointSettings) error {
 	if container.NetworkSettings == nil {
-		container.NetworkSettings = &network.Settings{Networks: make(map[string]*network.EndpointSettings)}
+		container.NetworkSettings = &network.Settings{}
+	}
+	if container.NetworkSettings.Networks == nil {
+		container.NetworkSettings.Networks = make(map[string]*network.EndpointSettings)
 	}
 
 	if !container.HostConfig.NetworkMode.IsHost() && containertypes.NetworkMode(n.Type()).IsHost() {
@@ -354,6 +357,15 @@ func (daemon *Daemon) findAndAttachNetwork(container *container.Container, idOrN
 		if container.Managed || !n.Info().Dynamic() {
 			return n, nil, nil
 		}
+		// Throw an error if the container is already attached to the network
+		if container.NetworkSettings.Networks != nil {
+			networkName := n.Name()
+			containerName := strings.TrimPrefix(container.Name, "/")
+			if network, ok := container.NetworkSettings.Networks[networkName]; ok && network.EndpointID != "" {
+				err := fmt.Errorf("%s is already attached to network %s", containerName, networkName)
+				return n, nil, errdefs.Conflict(err)
+			}
+		}
 	}
 
 	var addresses []string
@@ -497,7 +509,7 @@ func (daemon *Daemon) updateContainerNetworkSettings(container *container.Contai
 	}
 }
 
-func (daemon *Daemon) allocateNetwork(container *container.Container) error {
+func (daemon *Daemon) allocateNetwork(container *container.Container) (retErr error) {
 	start := time.Now()
 	controller := daemon.netController
 
@@ -565,7 +577,7 @@ func (daemon *Daemon) allocateNetwork(container *container.Container) error {
 			}
 			updateSandboxNetworkSettings(container, sb)
 			defer func() {
-				if err != nil {
+				if retErr != nil {
 					sb.Delete()
 				}
 			}()

daemon/container_operations_unix.go

@@ -411,5 +411,5 @@ func (daemon *Daemon) setupContainerMountsRoot(c *container.Container) error {
 	if err != nil {
 		return err
 	}
-	return idtools.MkdirAllAndChown(p, 0700, daemon.idMapping.RootPair())
+	return idtools.MkdirAllAndChown(p, 0701, idtools.CurrentIdentity())
 }

daemon/create.go

@@ -194,12 +194,10 @@ func (daemon *Daemon) create(opts createOpts) (retC *container.Container, retErr
 	}
 	container.RWLayer = rwLayer
 
-	rootIDs := daemon.idMapping.RootPair()
-
-	if err := idtools.MkdirAndChown(container.Root, 0700, rootIDs); err != nil {
+	if err := idtools.MkdirAndChown(container.Root, 0701, idtools.CurrentIdentity()); err != nil {
 		return nil, err
 	}
-	if err := idtools.MkdirAndChown(container.CheckpointDir(), 0700, rootIDs); err != nil {
+	if err := idtools.MkdirAndChown(container.CheckpointDir(), 0700, idtools.CurrentIdentity()); err != nil {
 		return nil, err
 	}
 

daemon/create_unix.go

@@ -41,12 +41,14 @@ func (daemon *Daemon) createContainerOSSpecificSettings(container *container.Con
 	}
 
 	for spec := range config.Volumes {
-		name := stringid.GenerateNonCryptoID()
+		name := stringid.GenerateRandomID()
 		destination := filepath.Clean(spec)
 
 		// Skip volumes for which we already have something mounted on that
 		// destination because of a --volume-from.
-		if container.IsDestinationMounted(destination) {
+		if container.HasMountFor(destination) {
+			logrus.WithField("container", container.ID).WithField("destination", spec).Debug("mountpoint already exists, skipping anonymous volume")
+			// Not an error, this could easily have come from the image config.
 			continue
 		}
 		path, err := container.GetResourcePath(destination)

daemon/create_windows.go

@@ -38,7 +38,7 @@ func (daemon *Daemon) createContainerOSSpecificSettings(container *container.Con
 
 		// If the mountpoint doesn't have a name, generate one.
 		if len(mp.Name) == 0 {
-			mp.Name = stringid.GenerateNonCryptoID()
+			mp.Name = stringid.GenerateRandomID()
 		}
 
 		// Skip volumes for which we already have something mounted on that

daemon/daemon.go

@@ -335,7 +335,7 @@ func (daemon *Daemon) restore() error {
 			}
 			if !alive && process != nil {
 				ec, exitedAt, err = process.Delete(context.Background())
-				if err != nil {
+				if err != nil && !errdefs.IsNotFound(err) {
 					logrus.WithError(err).Errorf("Failed to delete container %s from containerd", c.ID)
 					return
 				}
@@ -748,7 +748,7 @@ func NewDaemon(ctx context.Context, config *config.Config, pluginStore *plugin.S
 	}
 
 	// set up the tmpDir to use a canonical path
-	tmp, err := prepareTempDir(config.Root, rootIDs)
+	tmp, err := prepareTempDir(config.Root)
 	if err != nil {
 		return nil, fmt.Errorf("Unable to get the TempDir under %s: %s", config.Root, err)
 	}
@@ -813,7 +813,7 @@ func NewDaemon(ctx context.Context, config *config.Config, pluginStore *plugin.S
 	}
 
 	daemonRepo := filepath.Join(config.Root, "containers")
-	if err := idtools.MkdirAllAndChown(daemonRepo, 0700, rootIDs); err != nil {
+	if err := idtools.MkdirAllAndChown(daemonRepo, 0701, idtools.CurrentIdentity()); err != nil {
 		return nil, err
 	}
 
@@ -866,6 +866,24 @@ func NewDaemon(ctx context.Context, config *config.Config, pluginStore *plugin.S
 	registerMetricsPluginCallback(d.PluginStore, metricsSockPath)
 
 	gopts := []grpc.DialOption{
+		// WithBlock makes sure that the following containerd request
+		// is reliable.
+		//
+		// NOTE: In one edge case with high load pressure, kernel kills
+		// dockerd, containerd and containerd-shims caused by OOM.
+		// When both dockerd and containerd restart, but containerd
+		// will take time to recover all the existing containers. Before
+		// containerd serving, dockerd will failed with gRPC error.
+		// That bad thing is that restore action will still ignore the
+		// any non-NotFound errors and returns running state for
+		// already stopped container. It is unexpected behavior. And
+		// we need to restart dockerd to make sure that anything is OK.
+		//
+		// It is painful. Add WithBlock can prevent the edge case. And
+		// n common case, the containerd will be serving in shortly.
+		// It is not harm to add WithBlock for containerd connection.
+		grpc.WithBlock(),
+
 		grpc.WithInsecure(),
 		grpc.WithBackoffMaxDelay(3 * time.Second),
 		grpc.WithDialer(dialer.Dialer),
@@ -875,7 +893,7 @@ func NewDaemon(ctx context.Context, config *config.Config, pluginStore *plugin.S
 		grpc.WithDefaultCallOptions(grpc.MaxCallSendMsgSize(defaults.DefaultMaxSendMsgSize)),
 	}
 	if config.ContainerdAddr != "" {
-		d.containerdCli, err = containerd.New(config.ContainerdAddr, containerd.WithDefaultNamespace(ContainersNamespace), containerd.WithDialOpts(gopts), containerd.WithTimeout(60*time.Second))
+		d.containerdCli, err = containerd.New(config.ContainerdAddr, containerd.WithDefaultNamespace(config.ContainerdNamespace), containerd.WithDialOpts(gopts), containerd.WithTimeout(60*time.Second))
 		if err != nil {
 			return nil, errors.Wrapf(err, "failed to dial %q", config.ContainerdAddr)
 		}
@@ -887,13 +905,13 @@ func NewDaemon(ctx context.Context, config *config.Config, pluginStore *plugin.S
 		// Windows is not currently using containerd, keep the
 		// client as nil
 		if config.ContainerdAddr != "" {
-			pluginCli, err = containerd.New(config.ContainerdAddr, containerd.WithDefaultNamespace(pluginexec.PluginNamespace), containerd.WithDialOpts(gopts), containerd.WithTimeout(60*time.Second))
+			pluginCli, err = containerd.New(config.ContainerdAddr, containerd.WithDefaultNamespace(config.ContainerdPluginNamespace), containerd.WithDialOpts(gopts), containerd.WithTimeout(60*time.Second))
 			if err != nil {
 				return nil, errors.Wrapf(err, "failed to dial %q", config.ContainerdAddr)
 			}
 		}
 
-		return pluginexec.New(ctx, getPluginExecRoot(config.Root), pluginCli, m)
+		return pluginexec.New(ctx, getPluginExecRoot(config.Root), pluginCli, config.ContainerdPluginNamespace, m)
 	}
 
 	// Plugin system initialization should happen before restore. Do not change order.
@@ -960,7 +978,7 @@ func NewDaemon(ctx context.Context, config *config.Config, pluginStore *plugin.S
 		return nil, err
 	}
 
-	uuid, err := loadOrCreateUUID(filepath.Join(config.Root, "engine_uuid"))
+	trustKey, err := loadOrCreateTrustKey(config.TrustKeyPath)
 	if err != nil {
 		return nil, err
 	}
@@ -1005,7 +1023,7 @@ func NewDaemon(ctx context.Context, config *config.Config, pluginStore *plugin.S
 		return nil, errors.New("Devices cgroup isn't mounted")
 	}
 
-	d.ID = uuid
+	d.ID = trustKey.PublicKey().KeyID()
 	d.repository = daemonRepo
 	d.containers = container.NewMemoryStore()
 	if d.containersReplica, err = container.NewViewDB(); err != nil {
@@ -1036,11 +1054,12 @@ func NewDaemon(ctx context.Context, config *config.Config, pluginStore *plugin.S
 		MaxConcurrentUploads:      *config.MaxConcurrentUploads,
 		ReferenceStore:            rs,
 		RegistryService:           registryService,
+		TrustKey:                  trustKey,
 	})
 
 	go d.execCommandGC()
 
-	d.containerd, err = libcontainerd.NewClient(ctx, d.containerdCli, filepath.Join(config.ExecRoot, "containerd"), ContainersNamespace, d)
+	d.containerd, err = libcontainerd.NewClient(ctx, d.containerdCli, filepath.Join(config.ExecRoot, "containerd"), config.ContainerdNamespace, d)
 	if err != nil {
 		return nil, err
 	}
@@ -1270,7 +1289,7 @@ func (daemon *Daemon) Subnets() ([]net.IPNet, []net.IPNet) {
 // prepareTempDir prepares and returns the default directory to use
 // for temporary files.
 // If it doesn't exist, it is created. If it exists, its content is removed.
-func prepareTempDir(rootDir string, rootIdentity idtools.Identity) (string, error) {
+func prepareTempDir(rootDir string) (string, error) {
 	var tmpDir string
 	if tmpDir = os.Getenv("DOCKER_TMPDIR"); tmpDir == "" {
 		tmpDir = filepath.Join(rootDir, "tmp")
@@ -1288,9 +1307,7 @@ func prepareTempDir(rootDir string, rootIdentity idtools.Identity) (string, erro
 			}
 		}
 	}
-	// We don't remove the content of tmpdir if it's not the default,
-	// it may hold things that do not belong to us.
-	return tmpDir, idtools.MkdirAllAndChown(tmpDir, 0700, rootIdentity)
+	return tmpDir, idtools.MkdirAllAndChown(tmpDir, 0700, idtools.CurrentIdentity())
 }
 
 func (daemon *Daemon) setGenericResources(conf *config.Config) error {

daemon/daemon_linux.go

@@ -9,18 +9,13 @@ import (
 	"strings"
 
 	"github.com/docker/docker/daemon/config"
-	"github.com/docker/docker/internal/procfs"
 	"github.com/docker/docker/pkg/fileutils"
 	"github.com/docker/docker/pkg/mount"
+	"github.com/docker/libnetwork/resolvconf"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 )
 
-const (
-	defaultResolvConf   = "/etc/resolv.conf"
-	alternateResolvConf = "/run/systemd/resolve/resolv.conf"
-)
-
 // On Linux, plugins use a static path for storing execution state,
 // instead of deriving path from daemon's exec-root. This is because
 // plugin socket files are created here and they cannot exceed max
@@ -148,20 +143,5 @@ func setupResolvConf(config *config.Config) {
 	if config.ResolvConf != "" {
 		return
 	}
-
-	config.ResolvConf = defaultResolvConf
-	pids, err := procfs.PidOf("systemd-resolved")
-	if err != nil {
-		logrus.Errorf("unable to check systemd-resolved status: %s", err)
-		return
-	}
-	if len(pids) > 0 && pids[0] > 0 {
-		_, err := os.Stat(alternateResolvConf)
-		if err == nil {
-			logrus.Infof("systemd-resolved is running, so using resolvconf: %s", alternateResolvConf)
-			config.ResolvConf = alternateResolvConf
-			return
-		}
-		logrus.Infof("systemd-resolved is running, but %s is not present, fallback to %s", alternateResolvConf, defaultResolvConf)
-	}
+	config.ResolvConf = resolvconf.Path()
 }

daemon/daemon_unix.go

@@ -16,7 +16,7 @@ import (
 	"strings"
 	"time"
 
-	containerd_cgroups "github.com/containerd/cgroups"
+	statsV1 "github.com/containerd/cgroups/stats/v1"
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/blkiodev"
 	pblkiodev "github.com/docker/docker/api/types/blkiodev"
@@ -24,6 +24,7 @@ import (
 	"github.com/docker/docker/container"
 	"github.com/docker/docker/daemon/config"
 	"github.com/docker/docker/daemon/initlayer"
+	"github.com/docker/docker/errdefs"
 	"github.com/docker/docker/opts"
 	"github.com/docker/docker/pkg/containerfs"
 	"github.com/docker/docker/pkg/idtools"
@@ -73,6 +74,7 @@ const (
 	// constant for cgroup drivers
 	cgroupFsDriver      = "cgroupfs"
 	cgroupSystemdDriver = "systemd"
+	cgroupNoneDriver    = "none"
 
 	// DefaultRuntimeName is the default runtime to be used by
 	// containerd if none is specified
@@ -191,8 +193,9 @@ func getBlkioWeightDevices(config containertypes.Resources) ([]specs.LinuxWeight
 		}
 		weight := weightDevice.Weight
 		d := specs.LinuxWeightDevice{Weight: &weight}
-		d.Major = int64(stat.Rdev / 256)
-		d.Minor = int64(stat.Rdev % 256)
+		// The type is 32bit on mips.
+		d.Major = int64(unix.Major(uint64(stat.Rdev))) // nolint: unconvert
+		d.Minor = int64(unix.Minor(uint64(stat.Rdev))) // nolint: unconvert
 		blkioWeightDevices = append(blkioWeightDevices, d)
 	}
 
@@ -262,8 +265,9 @@ func getBlkioThrottleDevices(devs []*blkiodev.ThrottleDevice) ([]specs.LinuxThro
 			return nil, err
 		}
 		d := specs.LinuxThrottleDevice{Rate: d.Rate}
-		d.Major = int64(stat.Rdev / 256)
-		d.Minor = int64(stat.Rdev % 256)
+		// the type is 32bit on mips
+		d.Major = int64(unix.Major(uint64(stat.Rdev))) // nolint: unconvert
+		d.Minor = int64(unix.Minor(uint64(stat.Rdev))) // nolint: unconvert
 		throttleDevices = append(throttleDevices, d)
 	}
 
@@ -575,6 +579,9 @@ func verifyPlatformContainerResources(resources *containertypes.Resources, sysIn
 }
 
 func (daemon *Daemon) getCgroupDriver() string {
+	if daemon.Rootless() {
+		return cgroupNoneDriver
+	}
 	cgroupDriver := cgroupFsDriver
 
 	if UsingSystemd(daemon.configStore) {
@@ -601,6 +608,9 @@ func VerifyCgroupDriver(config *config.Config) error {
 	if cd == "" || cd == cgroupFsDriver || cd == cgroupSystemdDriver {
 		return nil
 	}
+	if cd == cgroupNoneDriver {
+		return fmt.Errorf("native.cgroupdriver option %s is internally used and cannot be specified manually", cd)
+	}
 	return fmt.Errorf("native.cgroupdriver option %s not supported", cd)
 }
 
@@ -728,6 +738,9 @@ func (daemon *Daemon) initRuntimes(runtimes map[string]types.Runtime) (err error
 
 // verifyDaemonSettings performs validation of daemon config struct
 func verifyDaemonSettings(conf *config.Config) error {
+	if conf.ContainerdNamespace == conf.ContainerdPluginNamespace {
+		return errors.New("containers namespace and plugins namespace cannot be the same")
+	}
 	// Check for mutually incompatible config options
 	if conf.BridgeConfig.Iface != "" && conf.BridgeConfig.IP != "" {
 		return fmt.Errorf("You specified -b & --bip, mutually exclusive options. Please specify only one")
@@ -954,11 +967,11 @@ func initBridgeDriver(controller libnetwork.NetworkController, config *config.Co
 	}
 
 	if config.BridgeConfig.IP != "" {
-		ipamV4Conf.PreferredPool = config.BridgeConfig.IP
-		ip, _, err := net.ParseCIDR(config.BridgeConfig.IP)
+		ip, ipNet, err := net.ParseCIDR(config.BridgeConfig.IP)
 		if err != nil {
 			return err
 		}
+		ipamV4Conf.PreferredPool = ipNet.String()
 		ipamV4Conf.Gateway = ip.String()
 	} else if bridgeName == bridge.DefaultBridgeName && ipamV4Conf.PreferredPool != "" {
 		logrus.Infof("Default bridge (%s) is assigned with an IP address %s. Daemon option --bip can be used to set a preferred IP address", bridgeName, ipamV4Conf.PreferredPool)
@@ -1163,16 +1176,37 @@ func setupRemappedRoot(config *config.Config) (*idtools.IdentityMapping, error)
 		// update remapped root setting now that we have resolved them to actual names
 		config.RemappedRoot = fmt.Sprintf("%s:%s", username, groupname)
 
+		// try with username:groupname, uid:groupname, username:gid, uid:gid,
+		// but keep the original error message (err)
 		mappings, err := idtools.NewIdentityMapping(username, groupname)
-		if err != nil {
+		if err == nil {
+			return mappings, nil
+		}
+		user, lookupErr := idtools.LookupUser(username)
+		if lookupErr != nil {
 			return nil, errors.Wrap(err, "Can't create ID mappings")
 		}
-		return mappings, nil
+		logrus.Infof("Can't create ID mappings with username:groupname %s:%s, try uid:groupname %d:%s", username, groupname, user.Uid, groupname)
+		mappings, lookupErr = idtools.NewIdentityMapping(fmt.Sprintf("%d", user.Uid), groupname)
+		if lookupErr == nil {
+			return mappings, nil
+		}
+		logrus.Infof("Can't create ID mappings with uid:groupname %d:%s, try username:gid %s:%d", user.Uid, groupname, username, user.Gid)
+		mappings, lookupErr = idtools.NewIdentityMapping(username, fmt.Sprintf("%d", user.Gid))
+		if lookupErr == nil {
+			return mappings, nil
+		}
+		logrus.Infof("Can't create ID mappings with username:gid %s:%d, try uid:gid %d:%d", username, user.Gid, user.Uid, user.Gid)
+		mappings, lookupErr = idtools.NewIdentityMapping(fmt.Sprintf("%d", user.Uid), fmt.Sprintf("%d", user.Gid))
+		if lookupErr == nil {
+			return mappings, nil
+		}
+		return nil, errors.Wrap(err, "Can't create ID mappings")
 	}
 	return &idtools.IdentityMapping{}, nil
 }
 
-func setupDaemonRoot(config *config.Config, rootDir string, rootIdentity idtools.Identity) error {
+func setupDaemonRoot(config *config.Config, rootDir string, remappedRoot idtools.Identity) error {
 	config.Root = rootDir
 	// the docker root metadata directory needs to have execute permissions for all users (g+x,o+x)
 	// so that syscalls executing as non-root, operating on subdirectories of the graph root
@@ -1197,10 +1231,16 @@ func setupDaemonRoot(config *config.Config, rootDir string, rootIdentity idtools
 	// a new subdirectory with ownership set to the remapped uid/gid (so as to allow
 	// `chdir()` to work for containers namespaced to that uid/gid)
 	if config.RemappedRoot != "" {
-		config.Root = filepath.Join(rootDir, fmt.Sprintf("%d.%d", rootIdentity.UID, rootIdentity.GID))
+		id := idtools.CurrentIdentity()
+		// First make sure the current root dir has the correct perms.
+		if err := idtools.MkdirAllAndChown(config.Root, 0701, id); err != nil {
+			return errors.Wrapf(err, "could not create or set daemon root permissions: %s", config.Root)
+		}
+
+		config.Root = filepath.Join(rootDir, fmt.Sprintf("%d.%d", remappedRoot.UID, remappedRoot.GID))
 		logrus.Debugf("Creating user namespaced daemon root: %s", config.Root)
 		// Create the root directory if it doesn't exist
-		if err := idtools.MkdirAllAndChown(config.Root, 0700, rootIdentity); err != nil {
+		if err := idtools.MkdirAllAndChown(config.Root, 0701, id); err != nil {
 			return fmt.Errorf("Cannot create daemon root: %s: %v", config.Root, err)
 		}
 		// we also need to verify that any pre-existing directories in the path to
@@ -1213,7 +1253,7 @@ func setupDaemonRoot(config *config.Config, rootDir string, rootIdentity idtools
 			if dirPath == "/" {
 				break
 			}
-			if !idtools.CanAccess(dirPath, rootIdentity) {
+			if !idtools.CanAccess(dirPath, remappedRoot) {
 				return fmt.Errorf("a subdirectory in your graphroot path (%s) restricts access to the remapped root uid/gid; please fix by allowing 'o+x' permissions on existing directories", config.Root)
 			}
 		}
@@ -1258,6 +1298,10 @@ func setupDaemonRootPropagation(cfg *config.Config) error {
 		return nil
 	}
 
+	if err := os.MkdirAll(filepath.Dir(cleanupFile), 0700); err != nil {
+		return errors.Wrap(err, "error creating dir to store mount cleanup file")
+	}
+
 	if err := ioutil.WriteFile(cleanupFile, nil, 0600); err != nil {
 		return errors.Wrap(err, "error writing file to signal mount cleanup on shutdown")
 	}
@@ -1283,12 +1327,26 @@ func (daemon *Daemon) registerLinks(container *container.Container, hostConfig *
 		}
 		child, err := daemon.GetContainer(name)
 		if err != nil {
+			if errdefs.IsNotFound(err) {
+				// Trying to link to a non-existing container is not valid, and
+				// should return an "invalid parameter" error. Returning a "not
+				// found" error here would make the client report the container's
+				// image could not be found (see moby/moby#39823)
+				err = errdefs.InvalidParameter(err)
+			}
 			return errors.Wrapf(err, "could not get container for %s", name)
 		}
 		for child.HostConfig.NetworkMode.IsContainer() {
 			parts := strings.SplitN(string(child.HostConfig.NetworkMode), ":", 2)
 			child, err = daemon.GetContainer(parts[1])
 			if err != nil {
+				if errdefs.IsNotFound(err) {
+					// Trying to link to a non-existing container is not valid, and
+					// should return an "invalid parameter" error. Returning a "not
+					// found" error here would make the client report the container's
+					// image could not be found (see moby/moby#39823)
+					err = errdefs.InvalidParameter(err)
+				}
 				return errors.Wrapf(err, "Could not get container for %s", parts[1])
 			}
 		}
@@ -1318,7 +1376,7 @@ func (daemon *Daemon) conditionalUnmountOnCleanup(container *container.Container
 	return daemon.Unmount(container)
 }
 
-func copyBlkioEntry(entries []*containerd_cgroups.BlkIOEntry) []types.BlkioStatEntry {
+func copyBlkioEntry(entries []*statsV1.BlkIOEntry) []types.BlkioStatEntry {
 	out := make([]types.BlkioStatEntry, len(entries))
 	for i, re := range entries {
 		out[i] = types.BlkioStatEntry{

daemon/daemon_unix_test.go

@@ -6,12 +6,15 @@ import (
 	"errors"
 	"io/ioutil"
 	"os"
+	"path/filepath"
 	"testing"
 
+	"github.com/docker/docker/api/types/blkiodev"
 	containertypes "github.com/docker/docker/api/types/container"
 	"github.com/docker/docker/container"
 	"github.com/docker/docker/daemon/config"
 	"github.com/docker/docker/pkg/sysinfo"
+	"golang.org/x/sys/unix"
 	"gotest.tools/assert"
 	is "gotest.tools/assert/cmp"
 )
@@ -376,3 +379,61 @@ func sysInfo(t *testing.T, opts ...func(*sysinfo.SysInfo)) sysinfo.SysInfo {
 	}
 	return si
 }
+
+const (
+	// prepare major 0x1FD(509 in decimal) and minor 0x130(304)
+	DEVNO  = 0x11FD30
+	MAJOR  = 509
+	MINOR  = 304
+	WEIGHT = 1024
+)
+
+func deviceTypeMock(t *testing.T, testAndCheck func(string)) {
+	if os.Getuid() != 0 {
+		t.Skip("root required") // for mknod
+	}
+
+	t.Parallel()
+
+	tempDir, err := ioutil.TempDir("", "tempDevDir"+t.Name())
+	assert.NilError(t, err, "create temp file")
+	tempFile := filepath.Join(tempDir, "dev")
+
+	defer os.RemoveAll(tempDir)
+
+	if err = unix.Mknod(tempFile, unix.S_IFCHR, DEVNO); err != nil {
+		t.Fatalf("mknod error %s(%x): %v", tempFile, DEVNO, err)
+	}
+
+	testAndCheck(tempFile)
+}
+
+func TestGetBlkioWeightDevices(t *testing.T) {
+	deviceTypeMock(t, func(tempFile string) {
+		mockResource := containertypes.Resources{
+			BlkioWeightDevice: []*blkiodev.WeightDevice{{Path: tempFile, Weight: WEIGHT}},
+		}
+
+		weightDevs, err := getBlkioWeightDevices(mockResource)
+
+		assert.NilError(t, err, "getBlkioWeightDevices")
+		assert.Check(t, is.Len(weightDevs, 1), "getBlkioWeightDevices")
+		assert.Check(t, weightDevs[0].Major == MAJOR, "get major device type")
+		assert.Check(t, weightDevs[0].Minor == MINOR, "get minor device type")
+		assert.Check(t, *weightDevs[0].Weight == WEIGHT, "get device weight")
+	})
+}
+
+func TestGetBlkioThrottleDevices(t *testing.T) {
+	deviceTypeMock(t, func(tempFile string) {
+		mockDevs := []*blkiodev.ThrottleDevice{{Path: tempFile, Rate: WEIGHT}}
+
+		retDevs, err := getBlkioThrottleDevices(mockDevs)
+
+		assert.NilError(t, err, "getBlkioThrottleDevices")
+		assert.Check(t, is.Len(retDevs, 1), "getBlkioThrottleDevices")
+		assert.Check(t, retDevs[0].Major == MAJOR, "get major device type")
+		assert.Check(t, retDevs[0].Minor == MINOR, "get minor device type")
+		assert.Check(t, retDevs[0].Rate == WEIGHT, "get device rate")
+	})
+}

daemon/daemon_windows.go

@@ -3,7 +3,9 @@ package daemon // import "github.com/docker/docker/daemon"
 import (
 	"context"
 	"fmt"
+	"math"
 	"path/filepath"
+	"runtime"
 	"strings"
 
 	"github.com/Microsoft/hcsshim"
@@ -40,9 +42,10 @@ const (
 	windowsMaxCPUPercent = 100
 )
 
-// Windows doesn't really have rlimits.
+// Windows containers are much larger than Linux containers and each of them
+// have > 20 system processes which why we use much smaller parallelism value.
 func adjustParallelLimit(n int, limit int) int {
-	return limit
+	return int(math.Max(1, math.Floor(float64(runtime.NumCPU())*.8)))
 }
 
 // Windows has no concept of an execution state directory. So use config.Root here.

daemon/exec/exec.go

@@ -1,6 +1,7 @@
 package exec // import "github.com/docker/docker/daemon/exec"
 
 import (
+	"context"
 	"runtime"
 	"sync"
 
@@ -39,7 +40,7 @@ type Config struct {
 // NewConfig initializes the a new exec configuration
 func NewConfig() *Config {
 	return &Config{
-		ID:           stringid.GenerateNonCryptoID(),
+		ID:           stringid.GenerateRandomID(),
 		StreamConfig: stream.NewConfig(),
 		Started:      make(chan struct{}),
 	}
@@ -58,7 +59,7 @@ func (i *rio) Close() error {
 }
 
 func (i *rio) Wait() {
-	i.sc.Wait()
+	i.sc.Wait(context.Background())
 
 	i.IO.Wait()
 }

daemon/export.go

@@ -70,7 +70,7 @@ func (daemon *Daemon) containerExport(container *container.Container) (arch io.R
 		Compression: archive.Uncompressed,
 		UIDMaps:     daemon.idMapping.UIDs(),
 		GIDMaps:     daemon.idMapping.GIDs(),
-	})
+	}, basefs.Path())
 	if err != nil {
 		rwlayer.Unmount()
 		return nil, err

daemon/graphdriver/aufs/aufs.go

@@ -72,7 +72,6 @@ func init() {
 
 // Driver contains information about the filesystem mounted.
 type Driver struct {
-	sync.Mutex
 	root          string
 	uidMaps       []idtools.IDMap
 	gidMaps       []idtools.IDMap
@@ -81,6 +80,7 @@ type Driver struct {
 	pathCache     map[string]string
 	naiveDiff     graphdriver.DiffDriver
 	locker        *locker.Locker
+	mntL          sync.Mutex
 }
 
 // Init returns a new AUFS driver.
@@ -130,18 +130,15 @@ func Init(root string, options []string, uidMaps, gidMaps []idtools.IDMap) (grap
 		locker:    locker.New(),
 	}
 
-	rootUID, rootGID, err := idtools.GetRootUIDGID(uidMaps, gidMaps)
-	if err != nil {
-		return nil, err
-	}
+	currentID := idtools.CurrentIdentity()
 	// Create the root aufs driver dir
-	if err := idtools.MkdirAllAndChown(root, 0700, idtools.Identity{UID: rootUID, GID: rootGID}); err != nil {
+	if err := idtools.MkdirAllAndChown(root, 0701, currentID); err != nil {
 		return nil, err
 	}
 
 	// Populate the dir structure
 	for _, p := range paths {
-		if err := idtools.MkdirAllAndChown(path.Join(root, p), 0700, idtools.Identity{UID: rootUID, GID: rootGID}); err != nil {
+		if err := idtools.MkdirAllAndChown(path.Join(root, p), 0701, currentID); err != nil {
 			return nil, err
 		}
 	}
@@ -327,11 +324,11 @@ func (a *Driver) Remove(id string) error {
 			break
 		}
 
-		if err != unix.EBUSY {
-			return errors.Wrapf(err, "aufs: unmount error: %s", mountpoint)
+		if errors.Cause(err) != unix.EBUSY {
+			return errors.Wrap(err, "aufs: unmount error")
 		}
 		if retries >= 5 {
-			return errors.Wrapf(err, "aufs: unmount error after retries: %s", mountpoint)
+			return errors.Wrap(err, "aufs: unmount error after retries")
 		}
 		// If unmount returns EBUSY, it could be a transient error. Sleep and retry.
 		retries++
@@ -437,7 +434,7 @@ func (a *Driver) Put(id string) error {
 
 	err := a.unmount(m)
 	if err != nil {
-		logger.Debugf("Failed to unmount %s aufs: %v", id, err)
+		logger.WithError(err).WithField("method", "Put()").Warn()
 	}
 	return err
 }
@@ -547,9 +544,6 @@ func (a *Driver) getParentLayerPaths(id string) ([]string, error) {
 }
 
 func (a *Driver) mount(id string, target string, mountLabel string, layers []string) error {
-	a.Lock()
-	defer a.Unlock()
-
 	// If the id is mounted or we get an error return
 	if mounted, err := a.mounted(target); err != nil || mounted {
 		return err
@@ -564,9 +558,6 @@ func (a *Driver) mount(id string, target string, mountLabel string, layers []str
 }
 
 func (a *Driver) unmount(mountPath string) error {
-	a.Lock()
-	defer a.Unlock()
-
 	if mounted, err := a.mounted(mountPath); err != nil || !mounted {
 		return err
 	}
@@ -579,23 +570,20 @@ func (a *Driver) mounted(mountpoint string) (bool, error) {
 
 // Cleanup aufs and unmount all mountpoints
 func (a *Driver) Cleanup() error {
-	var dirs []string
-	if err := filepath.Walk(a.mntPath(), func(path string, info os.FileInfo, err error) error {
-		if err != nil {
-			return err
-		}
-		if !info.IsDir() {
-			return nil
-		}
-		dirs = append(dirs, path)
-		return nil
-	}); err != nil {
-		return err
+	dir := a.mntPath()
+	files, err := ioutil.ReadDir(dir)
+	if err != nil {
+		return errors.Wrap(err, "aufs readdir error")
 	}
+	for _, f := range files {
+		if !f.IsDir() {
+			continue
+		}
+
+		m := path.Join(dir, f.Name())
 
-	for _, m := range dirs {
 		if err := a.unmount(m); err != nil {
-			logger.Debugf("error unmounting %s: %s", m, err)
+			logger.WithError(err).WithField("method", "Cleanup()").Warn()
 		}
 	}
 	return mount.RecursiveUnmount(a.root)
@@ -604,7 +592,7 @@ func (a *Driver) Cleanup() error {
 func (a *Driver) aufsMount(ro []string, rw, target, mountLabel string) (err error) {
 	defer func() {
 		if err != nil {
-			Unmount(target)
+			mount.Unmount(target)
 		}
 	}()
 
@@ -632,14 +620,29 @@ func (a *Driver) aufsMount(ro []string, rw, target, mountLabel string) (err erro
 		opts += ",dirperm1"
 	}
 	data := label.FormatMountLabel(fmt.Sprintf("%s,%s", string(b[:bp]), opts), mountLabel)
-	if err = unix.Mount("none", target, "aufs", 0, data); err != nil {
+	a.mntL.Lock()
+	err = unix.Mount("none", target, "aufs", 0, data)
+	a.mntL.Unlock()
+	if err != nil {
+		err = errors.Wrap(err, "mount target="+target+" data="+data)
 		return
 	}
 
-	for ; index < len(ro); index++ {
-		layer := fmt.Sprintf(":%s=ro+wh", ro[index])
-		data := label.FormatMountLabel(fmt.Sprintf("append%s", layer), mountLabel)
-		if err = unix.Mount("none", target, "aufs", unix.MS_REMOUNT, data); err != nil {
+	for index < len(ro) {
+		bp = 0
+		for ; index < len(ro); index++ {
+			layer := fmt.Sprintf("append:%s=ro+wh,", ro[index])
+			if bp+len(layer) > len(b) {
+				break
+			}
+			bp += copy(b[bp:], layer)
+		}
+		data := label.FormatMountLabel(string(b[:bp]), mountLabel)
+		a.mntL.Lock()
+		err = unix.Mount("none", target, "aufs", unix.MS_REMOUNT, data)
+		a.mntL.Unlock()
+		if err != nil {
+			err = errors.Wrap(err, "mount target="+target+" flags=MS_REMOUNT data="+data)
 			return
 		}
 	}

daemon/graphdriver/aufs/aufs_test.go

@@ -731,7 +731,7 @@ func BenchmarkConcurrentAccess(b *testing.B) {
 	// create a bunch of ids
 	var ids []string
 	for i := 0; i < numConcurrent; i++ {
-		ids = append(ids, stringid.GenerateNonCryptoID())
+		ids = append(ids, stringid.GenerateRandomID())
 	}
 
 	if err := d.Create(ids[0], "", nil); err != nil {

daemon/graphdriver/aufs/mount.go

@@ -4,14 +4,38 @@ package aufs // import "github.com/docker/docker/daemon/graphdriver/aufs"
 
 import (
 	"os/exec"
+	"syscall"
 
-	"golang.org/x/sys/unix"
+	"github.com/docker/docker/pkg/mount"
 )
 
 // Unmount the target specified.
 func Unmount(target string) error {
-	if err := exec.Command("auplink", target, "flush").Run(); err != nil {
-		logger.WithError(err).Warnf("Couldn't run auplink before unmount %s", target)
+	const (
+		EINVAL  = 22 // if auplink returns this,
+		retries = 3  // retry a few times
+	)
+
+	for i := 0; ; i++ {
+		out, err := exec.Command("auplink", target, "flush").CombinedOutput()
+		if err == nil {
+			break
+		}
+		rc := 0
+		if exiterr, ok := err.(*exec.ExitError); ok {
+			if status, ok := exiterr.Sys().(syscall.WaitStatus); ok {
+				rc = status.ExitStatus()
+			}
+		}
+		if i >= retries || rc != EINVAL {
+			logger.WithError(err).WithField("method", "Unmount").Warnf("auplink flush failed: %s", out)
+			break
+		}
+		// auplink failed to find target in /proc/self/mounts because
+		// kernel can't guarantee continuity while reading from it
+		// while mounts table is being changed
+		logger.Debugf("auplink flush error (retrying %d/%d): %s", i+1, retries, out)
 	}
-	return unix.Unmount(target, 0)
+
+	return mount.Unmount(target)
 }

daemon/graphdriver/btrfs/btrfs.go

@@ -70,11 +70,7 @@ func Init(home string, options []string, uidMaps, gidMaps []idtools.IDMap) (grap
 		return nil, graphdriver.ErrPrerequisites
 	}
 
-	rootUID, rootGID, err := idtools.GetRootUIDGID(uidMaps, gidMaps)
-	if err != nil {
-		return nil, err
-	}
-	if err := idtools.MkdirAllAndChown(home, 0700, idtools.Identity{UID: rootUID, GID: rootGID}); err != nil {
+	if err := idtools.MkdirAllAndChown(home, 0701, idtools.CurrentIdentity()); err != nil {
 		return nil, err
 	}
 
@@ -525,7 +521,7 @@ func (d *Driver) Create(id, parent string, opts *graphdriver.CreateOpts) error {
 	if err != nil {
 		return err
 	}
-	if err := idtools.MkdirAllAndChown(subvolumes, 0700, idtools.Identity{UID: rootUID, GID: rootGID}); err != nil {
+	if err := idtools.MkdirAllAndChown(subvolumes, 0701, idtools.CurrentIdentity()); err != nil {
 		return err
 	}
 	if parent == "" {
@@ -560,7 +556,7 @@ func (d *Driver) Create(id, parent string, opts *graphdriver.CreateOpts) error {
 		if err := d.setStorageSize(path.Join(subvolumes, id), driver); err != nil {
 			return err
 		}
-		if err := idtools.MkdirAllAndChown(quotas, 0700, idtools.Identity{UID: rootUID, GID: rootGID}); err != nil {
+		if err := idtools.MkdirAllAndChown(quotas, 0700, idtools.CurrentIdentity()); err != nil {
 			return err
 		}
 		if err := ioutil.WriteFile(path.Join(quotas, id), []byte(fmt.Sprint(driver.options.size)), 0644); err != nil {

daemon/graphdriver/btrfs/version.go

@@ -1,4 +1,4 @@
-// +build linux,!btrfs_noversion
+// +build linux
 
 package btrfs // import "github.com/docker/docker/daemon/graphdriver/btrfs"
 

daemon/graphdriver/btrfs/version_none.go

@@ -1,14 +0,0 @@
-// +build linux,btrfs_noversion
-
-package btrfs // import "github.com/docker/docker/daemon/graphdriver/btrfs"
-
-// TODO(vbatts) remove this work-around once supported linux distros are on
-// btrfs utilities of >= 3.16.1
-
-func btrfsBuildVersion() string {
-	return "-"
-}
-
-func btrfsLibVersion() int {
-	return -1
-}

daemon/graphdriver/btrfs/version_test.go

@@ -1,4 +1,4 @@
-// +build linux,!btrfs_noversion
+// +build linux
 
 package btrfs // import "github.com/docker/docker/daemon/graphdriver/btrfs"
 

daemon/graphdriver/copy/copy.go

@@ -2,14 +2,6 @@
 
 package copy // import "github.com/docker/docker/daemon/graphdriver/copy"
 
-/*
-#include <linux/fs.h>
-
-#ifndef FICLONE
-#define FICLONE		_IOW(0x94, 9, int)
-#endif
-*/
-import "C"
 import (
 	"container/list"
 	"fmt"
@@ -50,7 +42,7 @@ func copyRegular(srcPath, dstPath string, fileinfo os.FileInfo, copyWithFileRang
 	defer dstFile.Close()
 
 	if *copyWithFileClone {
-		_, _, err = unix.Syscall(unix.SYS_IOCTL, dstFile.Fd(), C.FICLONE, srcFile.Fd())
+		err = fiClone(srcFile, dstFile)
 		if err == nil {
 			return nil
 		}
@@ -154,7 +146,8 @@ func DirCopy(srcDir, dstDir string, copyMode Mode, copyXattrs bool) error {
 
 		switch mode := f.Mode(); {
 		case mode.IsRegular():
-			id := fileID{dev: stat.Dev, ino: stat.Ino}
+			//the type is 32bit on mips
+			id := fileID{dev: uint64(stat.Dev), ino: stat.Ino} // nolint: unconvert
 			if copyMode == Hardlink {
 				isHardlink = true
 				if err2 := os.Link(srcPath, dstPath); err2 != nil {

daemon/graphdriver/copy/copy_cgo.go

@@ -0,0 +1,22 @@
+// +build linux,cgo
+
+package copy // import "github.com/docker/docker/daemon/graphdriver/copy"
+
+/*
+#include <linux/fs.h>
+
+#ifndef FICLONE
+#define FICLONE		_IOW(0x94, 9, int)
+#endif
+*/
+import "C"
+import (
+	"os"
+
+	"golang.org/x/sys/unix"
+)
+
+func fiClone(srcFile, dstFile *os.File) error {
+	_, _, err := unix.Syscall(unix.SYS_IOCTL, dstFile.Fd(), C.FICLONE, srcFile.Fd())
+	return err
+}

daemon/graphdriver/copy/copy_nocgo.go

@@ -0,0 +1,13 @@
+// +build linux,!cgo
+
+package copy // import "github.com/docker/docker/daemon/graphdriver/copy"
+
+import (
+	"os"
+
+	"golang.org/x/sys/unix"
+)
+
+func fiClone(srcFile, dstFile *os.File) error {
+	return unix.ENOSYS
+}

daemon/graphdriver/devmapper/deviceset.go

@@ -540,8 +540,14 @@ func xfsSupported() error {
 		return err // error text is descriptive enough
 	}
 
-	// Check if kernel supports xfs filesystem or not.
-	exec.Command("modprobe", "xfs").Run()
+	mountTarget, err := ioutil.TempDir("", "supportsXFS")
+	if err != nil {
+		return errors.Wrapf(err, "error checking for xfs support")
+	}
+
+	/* The mounting will fail--after the module has been loaded.*/
+	defer os.RemoveAll(mountTarget)
+	unix.Mount("none", mountTarget, "xfs", 0, "")
 
 	f, err := os.Open("/proc/filesystems")
 	if err != nil {
@@ -1527,7 +1533,8 @@ func getDeviceMajorMinor(file *os.File) (uint64, uint64, error) {
 		return 0, 0, err
 	}
 
-	dev := stat.Rdev
+	// the type is 32bit on mips
+	dev := uint64(stat.Rdev) // nolint: unconvert
 	majorNum := major(dev)
 	minorNum := minor(dev)
 
@@ -1738,7 +1745,8 @@ func (devices *DeviceSet) initDevmapper(doInit bool) (retErr error) {
 	//	- Managed by docker
 	//	- The target of this device is at major <maj> and minor <min>
 	//	- If <inode> is defined, use that file inside the device as a loopback image. Otherwise use the device itself.
-	devices.devicePrefix = fmt.Sprintf("docker-%d:%d-%d", major(st.Dev), minor(st.Dev), st.Ino)
+	// The type Dev in Stat_t is 32bit on mips.
+	devices.devicePrefix = fmt.Sprintf("docker-%d:%d-%d", major(uint64(st.Dev)), minor(uint64(st.Dev)), st.Ino) // nolint: unconvert
 	logger.Debugf("Generated prefix: %s", devices.devicePrefix)
 
 	// Check for the existence of the thin-pool device

daemon/graphdriver/overlay/overlay.go

@@ -3,12 +3,10 @@
 package overlay // import "github.com/docker/docker/daemon/graphdriver/overlay"
 
 import (
-	"bufio"
 	"fmt"
 	"io"
 	"io/ioutil"
 	"os"
-	"os/exec"
 	"path"
 	"path/filepath"
 	"strconv"
@@ -124,10 +122,6 @@ func Init(home string, options []string, uidMaps, gidMaps []idtools.IDMap) (grap
 		return nil, err
 	}
 
-	if err := supportsOverlay(); err != nil {
-		return nil, graphdriver.ErrNotSupported
-	}
-
 	// Perform feature detection on /var/lib/docker/overlay if it's an existing directory.
 	// This covers situations where /var/lib/docker/overlay is a mount, and on a different
 	// filesystem than /var/lib/docker.
@@ -137,6 +131,11 @@ func Init(home string, options []string, uidMaps, gidMaps []idtools.IDMap) (grap
 		testdir = filepath.Dir(testdir)
 	}
 
+	if err := overlayutils.SupportsOverlay(testdir, false); err != nil {
+		logrus.WithField("storage-driver", "overlay").Error(err)
+		return nil, graphdriver.ErrNotSupported
+	}
+
 	fsMagic, err := graphdriver.GetFSMagic(testdir)
 	if err != nil {
 		return nil, err
@@ -145,12 +144,6 @@ func Init(home string, options []string, uidMaps, gidMaps []idtools.IDMap) (grap
 		backingFs = fsName
 	}
 
-	switch fsMagic {
-	case graphdriver.FsMagicAufs, graphdriver.FsMagicBtrfs, graphdriver.FsMagicEcryptfs, graphdriver.FsMagicNfsFs, graphdriver.FsMagicOverlay, graphdriver.FsMagicZfs:
-		logrus.WithField("storage-driver", "overlay").Errorf("'overlay' is not supported over %s", backingFs)
-		return nil, graphdriver.ErrIncompatibleFS
-	}
-
 	supportsDType, err := fsutils.SupportsDType(testdir)
 	if err != nil {
 		return nil, err
@@ -163,12 +156,8 @@ func Init(home string, options []string, uidMaps, gidMaps []idtools.IDMap) (grap
 		logrus.WithField("storage-driver", "overlay").Warn(overlayutils.ErrDTypeNotSupported("overlay", backingFs))
 	}
 
-	rootUID, rootGID, err := idtools.GetRootUIDGID(uidMaps, gidMaps)
-	if err != nil {
-		return nil, err
-	}
 	// Create the driver home dir
-	if err := idtools.MkdirAllAndChown(home, 0700, idtools.Identity{UID: rootUID, GID: rootGID}); err != nil {
+	if err := idtools.MkdirAllAndChown(home, 0701, idtools.CurrentIdentity()); err != nil {
 		return nil, err
 	}
 
@@ -200,27 +189,6 @@ func parseOptions(options []string) (*overlayOptions, error) {
 	return o, nil
 }
 
-func supportsOverlay() error {
-	// We can try to modprobe overlay first before looking at
-	// proc/filesystems for when overlay is supported
-	exec.Command("modprobe", "overlay").Run()
-
-	f, err := os.Open("/proc/filesystems")
-	if err != nil {
-		return err
-	}
-	defer f.Close()
-
-	s := bufio.NewScanner(f)
-	for s.Scan() {
-		if s.Text() == "nodev\toverlay" {
-			return nil
-		}
-	}
-	logrus.WithField("storage-driver", "overlay").Error("'overlay' not found as a supported filesystem on this host. Please ensure kernel is new enough and has overlay support loaded.")
-	return graphdriver.ErrNotSupported
-}
-
 func (d *Driver) String() string {
 	return "overlay"
 }
@@ -293,10 +261,11 @@ func (d *Driver) Create(id, parent string, opts *graphdriver.CreateOpts) (retErr
 	}
 	root := idtools.Identity{UID: rootUID, GID: rootGID}
 
-	if err := idtools.MkdirAllAndChown(path.Dir(dir), 0700, root); err != nil {
+	currentID := idtools.CurrentIdentity()
+	if err := idtools.MkdirAllAndChown(path.Dir(dir), 0701, currentID); err != nil {
 		return err
 	}
-	if err := idtools.MkdirAndChown(dir, 0700, root); err != nil {
+	if err := idtools.MkdirAndChown(dir, 0701, currentID); err != nil {
 		return err
 	}
 
@@ -309,6 +278,7 @@ func (d *Driver) Create(id, parent string, opts *graphdriver.CreateOpts) (retErr
 
 	// Toplevel images are just a "root" dir
 	if parent == "" {
+		// This must be 0755 otherwise unprivileged users will in the container will not be able to read / in the container
 		return idtools.MkdirAndChown(path.Join(dir, "root"), 0755, root)
 	}
 
@@ -329,7 +299,7 @@ func (d *Driver) Create(id, parent string, opts *graphdriver.CreateOpts) (retErr
 		if err := idtools.MkdirAndChown(path.Join(dir, "work"), 0700, root); err != nil {
 			return err
 		}
-		return ioutil.WriteFile(path.Join(dir, "lower-id"), []byte(parent), 0666)
+		return ioutil.WriteFile(path.Join(dir, "lower-id"), []byte(parent), 0600)
 	}
 
 	// Otherwise, copy the upper and the lower-id from the parent
@@ -339,7 +309,7 @@ func (d *Driver) Create(id, parent string, opts *graphdriver.CreateOpts) (retErr
 		return err
 	}
 
-	if err := ioutil.WriteFile(path.Join(dir, "lower-id"), lowerID, 0666); err != nil {
+	if err := ioutil.WriteFile(path.Join(dir, "lower-id"), lowerID, 0600); err != nil {
 		return err
 	}
 

daemon/graphdriver/overlay2/check.go

@@ -99,35 +99,3 @@ func doesSupportNativeDiff(d string) error {
 
 	return nil
 }
-
-// supportsMultipleLowerDir checks if the system supports multiple lowerdirs,
-// which is required for the overlay2 driver. On 4.x kernels, multiple lowerdirs
-// are always available (so this check isn't needed), and backported to RHEL and
-// CentOS 3.x kernels (3.10.0-693.el7.x86_64 and up). This function is to detect
-// support on those kernels, without doing a kernel version compare.
-func supportsMultipleLowerDir(d string) error {
-	td, err := ioutil.TempDir(d, "multiple-lowerdir-check")
-	if err != nil {
-		return err
-	}
-	defer func() {
-		if err := os.RemoveAll(td); err != nil {
-			logger.Warnf("Failed to remove check directory %v: %v", td, err)
-		}
-	}()
-
-	for _, dir := range []string{"lower1", "lower2", "upper", workDirName, mergedDirName} {
-		if err := os.Mkdir(filepath.Join(td, dir), 0755); err != nil {
-			return err
-		}
-	}
-
-	opts := fmt.Sprintf("lowerdir=%s:%s,upperdir=%s,workdir=%s", path.Join(td, "lower2"), path.Join(td, "lower1"), path.Join(td, "upper"), path.Join(td, workDirName))
-	if err := unix.Mount("overlay", filepath.Join(td, mergedDirName), "overlay", 0, opts); err != nil {
-		return errors.Wrap(err, "failed to mount overlay")
-	}
-	if err := unix.Unmount(filepath.Join(td, mergedDirName), 0); err != nil {
-		logger.Warnf("Failed to unmount check directory %v: %v", filepath.Join(td, mergedDirName), err)
-	}
-	return nil
-}

daemon/graphdriver/overlay2/overlay.go

@@ -3,14 +3,12 @@
 package overlay2 // import "github.com/docker/docker/daemon/graphdriver/overlay2"
 
 import (
-	"bufio"
 	"context"
 	"errors"
 	"fmt"
 	"io"
 	"io/ioutil"
 	"os"
-	"os/exec"
 	"path"
 	"path/filepath"
 	"strconv"
@@ -29,7 +27,6 @@ import (
 	"github.com/docker/docker/pkg/locker"
 	"github.com/docker/docker/pkg/mount"
 	"github.com/docker/docker/pkg/parsers"
-	"github.com/docker/docker/pkg/parsers/kernel"
 	"github.com/docker/docker/pkg/system"
 	"github.com/docker/go-units"
 	rsystem "github.com/opencontainers/runc/libcontainer/system"
@@ -134,16 +131,6 @@ func Init(home string, options []string, uidMaps, gidMaps []idtools.IDMap) (grap
 		return nil, err
 	}
 
-	if err := supportsOverlay(); err != nil {
-		return nil, graphdriver.ErrNotSupported
-	}
-
-	// require kernel 4.0.0 to ensure multiple lower dirs are supported
-	v, err := kernel.GetKernelVersion()
-	if err != nil {
-		return nil, err
-	}
-
 	// Perform feature detection on /var/lib/docker/overlay2 if it's an existing directory.
 	// This covers situations where /var/lib/docker/overlay2 is a mount, and on a different
 	// filesystem than /var/lib/docker.
@@ -153,6 +140,11 @@ func Init(home string, options []string, uidMaps, gidMaps []idtools.IDMap) (grap
 		testdir = filepath.Dir(testdir)
 	}
 
+	if err := overlayutils.SupportsOverlay(testdir, true); err != nil {
+		logger.Error(err)
+		return nil, graphdriver.ErrNotSupported
+	}
+
 	fsMagic, err := graphdriver.GetFSMagic(testdir)
 	if err != nil {
 		return nil, err
@@ -161,32 +153,6 @@ func Init(home string, options []string, uidMaps, gidMaps []idtools.IDMap) (grap
 		backingFs = fsName
 	}
 
-	switch fsMagic {
-	case graphdriver.FsMagicAufs, graphdriver.FsMagicEcryptfs, graphdriver.FsMagicNfsFs, graphdriver.FsMagicOverlay, graphdriver.FsMagicZfs:
-		logger.Errorf("'overlay2' is not supported over %s", backingFs)
-		return nil, graphdriver.ErrIncompatibleFS
-	case graphdriver.FsMagicBtrfs:
-		// Support for OverlayFS on BTRFS was added in kernel 4.7
-		// See https://btrfs.wiki.kernel.org/index.php/Changelog
-		if kernel.CompareKernelVersion(*v, kernel.VersionInfo{Kernel: 4, Major: 7, Minor: 0}) < 0 {
-			if !opts.overrideKernelCheck {
-				logger.Errorf("'overlay2' requires kernel 4.7 to use on %s", backingFs)
-				return nil, graphdriver.ErrIncompatibleFS
-			}
-			logger.Warn("Using pre-4.7.0 kernel for overlay2 on btrfs, may require kernel update")
-		}
-	}
-
-	if kernel.CompareKernelVersion(*v, kernel.VersionInfo{Kernel: 4, Major: 0, Minor: 0}) < 0 {
-		if opts.overrideKernelCheck {
-			logger.Warn("Using pre-4.0.0 kernel for overlay2, mount failures may require kernel update")
-		} else {
-			if err := supportsMultipleLowerDir(testdir); err != nil {
-				logger.Debugf("Multiple lower dirs not supported: %v", err)
-				return nil, graphdriver.ErrNotSupported
-			}
-		}
-	}
 	supportsDType, err := fsutils.SupportsDType(testdir)
 	if err != nil {
 		return nil, err
@@ -199,12 +165,7 @@ func Init(home string, options []string, uidMaps, gidMaps []idtools.IDMap) (grap
 		logger.Warn(overlayutils.ErrDTypeNotSupported("overlay2", backingFs))
 	}
 
-	rootUID, rootGID, err := idtools.GetRootUIDGID(uidMaps, gidMaps)
-	if err != nil {
-		return nil, err
-	}
-	// Create the driver home dir
-	if err := idtools.MkdirAllAndChown(path.Join(home, linkDir), 0700, idtools.Identity{UID: rootUID, GID: rootGID}); err != nil {
+	if err := idtools.MkdirAllAndChown(path.Join(home, linkDir), 0701, idtools.CurrentIdentity()); err != nil {
 		return nil, err
 	}
 
@@ -275,27 +236,6 @@ func parseOptions(options []string) (*overlayOptions, error) {
 	return o, nil
 }
 
-func supportsOverlay() error {
-	// We can try to modprobe overlay first before looking at
-	// proc/filesystems for when overlay is supported
-	exec.Command("modprobe", "overlay").Run()
-
-	f, err := os.Open("/proc/filesystems")
-	if err != nil {
-		return err
-	}
-	defer f.Close()
-
-	s := bufio.NewScanner(f)
-	for s.Scan() {
-		if s.Text() == "nodev\toverlay" {
-			return nil
-		}
-	}
-	logger.Error("'overlay' not found as a supported filesystem on this host. Please ensure kernel is new enough and has overlay support loaded.")
-	return graphdriver.ErrNotSupported
-}
-
 func useNaiveDiff(home string) bool {
 	useNaiveDiffLock.Do(func() {
 		if err := doesSupportNativeDiff(home); err != nil {
@@ -394,11 +334,12 @@ func (d *Driver) create(id, parent string, opts *graphdriver.CreateOpts) (retErr
 		return err
 	}
 	root := idtools.Identity{UID: rootUID, GID: rootGID}
+	current := idtools.CurrentIdentity()
 
-	if err := idtools.MkdirAllAndChown(path.Dir(dir), 0700, root); err != nil {
+	if err := idtools.MkdirAllAndChown(path.Dir(dir), 0701, current); err != nil {
 		return err
 	}
-	if err := idtools.MkdirAndChown(dir, 0700, root); err != nil {
+	if err := idtools.MkdirAndChown(dir, 0701, current); err != nil {
 		return err
 	}
 
@@ -446,6 +387,10 @@ func (d *Driver) create(id, parent string, opts *graphdriver.CreateOpts) (retErr
 		return err
 	}
 
+	if err := ioutil.WriteFile(path.Join(d.dir(parent), "committed"), []byte{}, 0600); err != nil {
+		return err
+	}
+
 	lower, err := d.getLower(parent)
 	if err != nil {
 		return err
@@ -592,7 +537,20 @@ func (d *Driver) Get(id, mountLabel string) (_ containerfs.ContainerFS, retErr e
 	for i, s := range splitLowers {
 		absLowers[i] = path.Join(d.home, s)
 	}
-	opts := indexOff + "lowerdir=" + strings.Join(absLowers, ":") + ",upperdir=" + diffDir + ",workdir=" + workDir
+	var readonly bool
+	if _, err := os.Stat(path.Join(dir, "committed")); err == nil {
+		readonly = true
+	} else if !os.IsNotExist(err) {
+		return nil, err
+	}
+
+	var opts string
+	if readonly {
+		opts = indexOff + "lowerdir=" + diffDir + ":" + strings.Join(absLowers, ":")
+	} else {
+		opts = indexOff + "lowerdir=" + strings.Join(absLowers, ":") + ",upperdir=" + diffDir + ",workdir=" + workDir
+	}
+
 	mountData := label.FormatMountLabel(opts, mountLabel)
 	mount := unix.Mount
 	mountTarget := mergedDir
@@ -612,7 +570,11 @@ func (d *Driver) Get(id, mountLabel string) (_ containerfs.ContainerFS, retErr e
 	// fit within a page and relative links make the mount data much
 	// smaller at the expense of requiring a fork exec to chroot.
 	if len(mountData) > pageSize {
-		opts = indexOff + "lowerdir=" + string(lowers) + ",upperdir=" + path.Join(id, diffDirName) + ",workdir=" + path.Join(id, workDirName)
+		if readonly {
+			opts = indexOff + "lowerdir=" + path.Join(id, diffDirName) + ":" + string(lowers)
+		} else {
+			opts = indexOff + "lowerdir=" + string(lowers) + ",upperdir=" + path.Join(id, diffDirName) + ",workdir=" + path.Join(id, workDirName)
+		}
 		mountData = label.FormatMountLabel(opts, mountLabel)
 		if len(mountData) > pageSize {
 			return nil, fmt.Errorf("cannot mount layer, mount label too large %d", len(mountData))
@@ -628,10 +590,12 @@ func (d *Driver) Get(id, mountLabel string) (_ containerfs.ContainerFS, retErr e
 		return nil, fmt.Errorf("error creating overlay mount to %s: %v", mergedDir, err)
 	}
 
-	// chown "workdir/work" to the remapped root UID/GID. Overlay fs inside a
-	// user namespace requires this to move a directory from lower to upper.
-	if err := os.Chown(path.Join(workDir, workDirName), rootUID, rootGID); err != nil {
-		return nil, err
+	if !readonly {
+		// chown "workdir/work" to the remapped root UID/GID. Overlay fs inside a
+		// user namespace requires this to move a directory from lower to upper.
+		if err := os.Chown(path.Join(workDir, workDirName), rootUID, rootGID); err != nil {
+			return nil, err
+		}
 	}
 
 	return containerfs.NewLocalContainerFS(mergedDir), nil