archive: do not use overlayWhiteoutConverter for UserNS

overlay2 no longer sets `archive.OverlayWhiteoutFormat` when
running in UserNS, so we can remove the complicated logic in the
archive package.

Signed-off-by: Akihiro Suda <akihiro.suda.cz@hco.ntt.co.jp>
(cherry picked from commit 6322dfc217)
Signed-off-by: Akihiro Suda <akihiro.suda.cz@hco.ntt.co.jp>

daemon/graphdriver/overlay2/overlay.go

@@ -15,7 +15,6 @@ import (
 	"strings"
 	"sync"
 
-	"github.com/containerd/containerd/sys"
 	"github.com/docker/docker/daemon/graphdriver"
 	"github.com/docker/docker/daemon/graphdriver/overlayutils"
 	"github.com/docker/docker/pkg/archive"
@@ -682,6 +681,7 @@ func (d *Driver) ApplyDiff(id string, parent string, diff io.Reader) (size int64
 		return d.naiveDiff.ApplyDiff(id, parent, diff)
 	}
 
+	// never reach here if we are running in UserNS
 	applyDir := d.getDiffPath(id)
 
 	logger.Debugf("Applying tar in %s", applyDir)
@@ -690,7 +690,6 @@ func (d *Driver) ApplyDiff(id string, parent string, diff io.Reader) (size int64
 		UIDMaps:        d.uidMaps,
 		GIDMaps:        d.gidMaps,
 		WhiteoutFormat: archive.OverlayWhiteoutFormat,
-		InUserNS:       sys.RunningInUserNS(),
 	}); err != nil {
 		return 0, err
 	}
@@ -721,6 +720,7 @@ func (d *Driver) Diff(id, parent string) (io.ReadCloser, error) {
 		return d.naiveDiff.Diff(id, parent)
 	}
 
+	// never reach here if we are running in UserNS
 	diffPath := d.getDiffPath(id)
 	logger.Debugf("Tar with options on %s", diffPath)
 	return archive.TarWithOptions(diffPath, &archive.TarOptions{

pkg/archive/archive.go

@@ -739,13 +739,18 @@ func TarWithOptions(srcPath string, options *TarOptions) (io.ReadCloser, error)
 		return nil, err
 	}
 
+	whiteoutConverter, err := getWhiteoutConverter(options.WhiteoutFormat, options.InUserNS)
+	if err != nil {
+		return nil, err
+	}
+
 	go func() {
 		ta := newTarAppender(
 			idtools.NewIDMappingsFromMaps(options.UIDMaps, options.GIDMaps),
 			compressWriter,
 			options.ChownOpts,
 		)
-		ta.WhiteoutConverter = getWhiteoutConverter(options.WhiteoutFormat, options.InUserNS)
+		ta.WhiteoutConverter = whiteoutConverter
 
 		defer func() {
 			// Make sure to check the error on Close.
@@ -903,7 +908,10 @@ func Unpack(decompressedArchive io.Reader, dest string, options *TarOptions) err
 	var dirs []*tar.Header
 	idMapping := idtools.NewIDMappingsFromMaps(options.UIDMaps, options.GIDMaps)
 	rootIDs := idMapping.RootPair()
-	whiteoutConverter := getWhiteoutConverter(options.WhiteoutFormat, options.InUserNS)
+	whiteoutConverter, err := getWhiteoutConverter(options.WhiteoutFormat, options.InUserNS)
+	if err != nil {
+		return err
+	}
 
 	// Iterate through the files in the archive.
 loop:

pkg/archive/archive_linux.go

@@ -2,29 +2,26 @@ package archive // import "github.com/docker/docker/pkg/archive"
 
 import (
 	"archive/tar"
-	"fmt"
-	"io/ioutil"
 	"os"
 	"path/filepath"
 	"strings"
-	"syscall"
 
-	"github.com/containerd/continuity/fs"
 	"github.com/docker/docker/pkg/system"
-	"github.com/moby/sys/mount"
 	"github.com/pkg/errors"
 	"golang.org/x/sys/unix"
 )
 
-func getWhiteoutConverter(format WhiteoutFormat, inUserNS bool) tarWhiteoutConverter {
+func getWhiteoutConverter(format WhiteoutFormat, inUserNS bool) (tarWhiteoutConverter, error) {
 	if format == OverlayWhiteoutFormat {
-		return overlayWhiteoutConverter{inUserNS: inUserNS}
+		if inUserNS {
+			return nil, errors.New("specifying OverlayWhiteoutFormat is not allowed in userns")
+		}
+		return overlayWhiteoutConverter{}, nil
 	}
-	return nil
+	return nil, nil
 }
 
 type overlayWhiteoutConverter struct {
-	inUserNS bool
 }
 
 func (overlayWhiteoutConverter) ConvertWrite(hdr *tar.Header, path string, fi os.FileInfo) (wo *tar.Header, err error) {
@@ -77,13 +74,7 @@ func (c overlayWhiteoutConverter) ConvertRead(hdr *tar.Header, path string) (boo
 	if base == WhiteoutOpaqueDir {
 		err := unix.Setxattr(dir, "trusted.overlay.opaque", []byte{'y'}, 0)
 		if err != nil {
-			if c.inUserNS {
-				if err = replaceDirWithOverlayOpaque(dir); err != nil {
-					return false, errors.Wrapf(err, "replaceDirWithOverlayOpaque(%q) failed", dir)
-				}
-			} else {
-				return false, errors.Wrapf(err, "setxattr(%q, trusted.overlay.opaque=y)", dir)
-			}
+			return false, errors.Wrapf(err, "setxattr(%q, trusted.overlay.opaque=y)", dir)
 		}
 		// don't write the file itself
 		return false, err
@@ -95,19 +86,7 @@ func (c overlayWhiteoutConverter) ConvertRead(hdr *tar.Header, path string) (boo
 		originalPath := filepath.Join(dir, originalBase)
 
 		if err := unix.Mknod(originalPath, unix.S_IFCHR, 0); err != nil {
-			if c.inUserNS {
-				// Ubuntu and a few distros support overlayfs in userns.
-				//
-				// Although we can't call mknod directly in userns (at least on bionic kernel 4.15),
-				// we can still create 0,0 char device using mknodChar0Overlay().
-				//
-				// NOTE: we don't need this hack for the containerd snapshotter+unpack model.
-				if err := mknodChar0Overlay(originalPath); err != nil {
-					return false, errors.Wrapf(err, "failed to mknodChar0UserNS(%q)", originalPath)
-				}
-			} else {
-				return false, errors.Wrapf(err, "failed to mknod(%q, S_IFCHR, 0)", originalPath)
-			}
+			return false, errors.Wrapf(err, "failed to mknod(%q, S_IFCHR, 0)", originalPath)
 		}
 		if err := os.Chown(originalPath, hdr.Uid, hdr.Gid); err != nil {
 			return false, err
@@ -119,146 +98,3 @@ func (c overlayWhiteoutConverter) ConvertRead(hdr *tar.Header, path string) (boo
 
 	return true, nil
 }
-
-// mknodChar0Overlay creates 0,0 char device by mounting overlayfs and unlinking.
-// This function can be used for creating 0,0 char device in userns on Ubuntu.
-//
-// Steps:
-// * Mkdir lower,upper,merged,work
-// * Create lower/dummy
-// * Mount overlayfs
-// * Unlink merged/dummy
-// * Unmount overlayfs
-// * Make sure a 0,0 char device is created as upper/dummy
-// * Rename upper/dummy to cleansedOriginalPath
-func mknodChar0Overlay(cleansedOriginalPath string) error {
-	dir := filepath.Dir(cleansedOriginalPath)
-	tmp, err := ioutil.TempDir(dir, "mc0o")
-	if err != nil {
-		return errors.Wrapf(err, "failed to create a tmp directory under %s", dir)
-	}
-	defer os.RemoveAll(tmp)
-	lower := filepath.Join(tmp, "l")
-	upper := filepath.Join(tmp, "u")
-	work := filepath.Join(tmp, "w")
-	merged := filepath.Join(tmp, "m")
-	for _, s := range []string{lower, upper, work, merged} {
-		if err := os.MkdirAll(s, 0700); err != nil {
-			return errors.Wrapf(err, "failed to mkdir %s", s)
-		}
-	}
-	dummyBase := "d"
-	lowerDummy := filepath.Join(lower, dummyBase)
-	if err := ioutil.WriteFile(lowerDummy, []byte{}, 0600); err != nil {
-		return errors.Wrapf(err, "failed to create a dummy lower file %s", lowerDummy)
-	}
-	// lowerdir needs ":" to be escaped: https://github.com/moby/moby/issues/40939#issuecomment-627098286
-	lowerEscaped := strings.ReplaceAll(lower, ":", "\\:")
-	mOpts := fmt.Sprintf("lowerdir=%s,upperdir=%s,workdir=%s", lowerEscaped, upper, work)
-	if err := mount.Mount("overlay", merged, "overlay", mOpts); err != nil {
-		return err
-	}
-	mergedDummy := filepath.Join(merged, dummyBase)
-	if err := os.Remove(mergedDummy); err != nil {
-		syscall.Unmount(merged, 0)
-		return errors.Wrapf(err, "failed to unlink %s", mergedDummy)
-	}
-	if err := syscall.Unmount(merged, 0); err != nil {
-		return errors.Wrapf(err, "failed to unmount %s", merged)
-	}
-	upperDummy := filepath.Join(upper, dummyBase)
-	if err := isChar0(upperDummy); err != nil {
-		return err
-	}
-	if err := os.Rename(upperDummy, cleansedOriginalPath); err != nil {
-		return errors.Wrapf(err, "failed to rename %s to %s", upperDummy, cleansedOriginalPath)
-	}
-	return nil
-}
-
-func isChar0(path string) error {
-	osStat, err := os.Stat(path)
-	if err != nil {
-		return errors.Wrapf(err, "failed to stat %s", path)
-	}
-	st, ok := osStat.Sys().(*syscall.Stat_t)
-	if !ok {
-		return errors.Errorf("got unsupported stat for %s", path)
-	}
-	if os.FileMode(st.Mode)&syscall.S_IFMT != syscall.S_IFCHR {
-		return errors.Errorf("%s is not a character device, got mode=%d", path, st.Mode)
-	}
-	if st.Rdev != 0 {
-		return errors.Errorf("%s is not a 0,0 character device, got Rdev=%d", path, st.Rdev)
-	}
-	return nil
-}
-
-// replaceDirWithOverlayOpaque replaces path with a new directory with trusted.overlay.opaque
-// xattr. The contents of the directory are preserved.
-func replaceDirWithOverlayOpaque(path string) error {
-	if path == "/" {
-		return errors.New("replaceDirWithOverlayOpaque: path must not be \"/\"")
-	}
-	dir := filepath.Dir(path)
-	tmp, err := ioutil.TempDir(dir, "rdwoo")
-	if err != nil {
-		return errors.Wrapf(err, "failed to create a tmp directory under %s", dir)
-	}
-	defer os.RemoveAll(tmp)
-	// newPath is a new empty directory crafted with trusted.overlay.opaque xattr.
-	// we copy the content of path into newPath, remove path, and rename newPath to path.
-	newPath, err := createDirWithOverlayOpaque(tmp)
-	if err != nil {
-		return errors.Wrapf(err, "createDirWithOverlayOpaque(%q) failed", tmp)
-	}
-	if err := fs.CopyDir(newPath, path); err != nil {
-		return errors.Wrapf(err, "CopyDir(%q, %q) failed", newPath, path)
-	}
-	if err := os.RemoveAll(path); err != nil {
-		return err
-	}
-	return os.Rename(newPath, path)
-}
-
-// createDirWithOverlayOpaque creates a directory with trusted.overlay.opaque xattr,
-// without calling setxattr, so as to allow creating opaque dir in userns on Ubuntu.
-func createDirWithOverlayOpaque(tmp string) (string, error) {
-	lower := filepath.Join(tmp, "l")
-	upper := filepath.Join(tmp, "u")
-	work := filepath.Join(tmp, "w")
-	merged := filepath.Join(tmp, "m")
-	for _, s := range []string{lower, upper, work, merged} {
-		if err := os.MkdirAll(s, 0700); err != nil {
-			return "", errors.Wrapf(err, "failed to mkdir %s", s)
-		}
-	}
-	dummyBase := "d"
-	lowerDummy := filepath.Join(lower, dummyBase)
-	if err := os.MkdirAll(lowerDummy, 0700); err != nil {
-		return "", errors.Wrapf(err, "failed to create a dummy lower directory %s", lowerDummy)
-	}
-	// lowerdir needs ":" to be escaped: https://github.com/moby/moby/issues/40939#issuecomment-627098286
-	lowerEscaped := strings.ReplaceAll(lower, ":", "\\:")
-	mOpts := fmt.Sprintf("lowerdir=%s,upperdir=%s,workdir=%s", lowerEscaped, upper, work)
-	if err := mount.Mount("overlay", merged, "overlay", mOpts); err != nil {
-		return "", err
-	}
-	mergedDummy := filepath.Join(merged, dummyBase)
-	if err := os.Remove(mergedDummy); err != nil {
-		syscall.Unmount(merged, 0)
-		return "", errors.Wrapf(err, "failed to rmdir %s", mergedDummy)
-	}
-	// upperDummy becomes a 0,0-char device file here
-	if err := os.Mkdir(mergedDummy, 0700); err != nil {
-		syscall.Unmount(merged, 0)
-		return "", errors.Wrapf(err, "failed to mkdir %s", mergedDummy)
-	}
-	// upperDummy becomes a directory with trusted.overlay.opaque xattr
-	// (but can't be verified in userns)
-	if err := syscall.Unmount(merged, 0); err != nil {
-		return "", errors.Wrapf(err, "failed to unmount %s", merged)
-	}
-	upperDummy := filepath.Join(upper, dummyBase)
-	return upperDummy, nil
-}

pkg/archive/archive_linux_test.go

@@ -1,19 +1,14 @@
 package archive // import "github.com/docker/docker/pkg/archive"
 
 import (
-	"fmt"
 	"io/ioutil"
 	"os"
-	"os/exec"
 	"path/filepath"
 	"syscall"
 	"testing"
 
 	"github.com/containerd/containerd/sys"
-	"github.com/docker/docker/pkg/reexec"
 	"github.com/docker/docker/pkg/system"
-	"github.com/moby/sys/mount"
-	"github.com/pkg/errors"
 	"golang.org/x/sys/unix"
 	"gotest.tools/v3/assert"
 	"gotest.tools/v3/skip"
@@ -167,129 +162,3 @@ func TestOverlayTarAUFSUntar(t *testing.T) {
 	checkFileMode(t, filepath.Join(dst, "d2", "f1"), 0660)
 	checkFileMode(t, filepath.Join(dst, "d3", WhiteoutPrefix+"f1"), 0600)
 }
-
-func unshareCmd(cmd *exec.Cmd) {
-	cmd.SysProcAttr = &syscall.SysProcAttr{
-		Cloneflags: syscall.CLONE_NEWUSER | syscall.CLONE_NEWNS,
-		UidMappings: []syscall.SysProcIDMap{
-			{
-				ContainerID: 0,
-				HostID:      os.Geteuid(),
-				Size:        1,
-			},
-		},
-		GidMappings: []syscall.SysProcIDMap{
-			{
-				ContainerID: 0,
-				HostID:      os.Getegid(),
-				Size:        1,
-			},
-		},
-	}
-}
-
-const (
-	reexecSupportsUserNSOverlay = "docker-test-supports-userns-overlay"
-	reexecMknodChar0            = "docker-test-userns-mknod-char0"
-	reexecSetOpaque             = "docker-test-userns-set-opaque"
-)
-
-func supportsOverlay(dir string) error {
-	lower := filepath.Join(dir, "l")
-	upper := filepath.Join(dir, "u")
-	work := filepath.Join(dir, "w")
-	merged := filepath.Join(dir, "m")
-	for _, s := range []string{lower, upper, work, merged} {
-		if err := os.MkdirAll(s, 0700); err != nil {
-			return err
-		}
-	}
-	mOpts := fmt.Sprintf("lowerdir=%s,upperdir=%s,workdir=%s", lower, upper, work)
-	if err := mount.Mount("overlay", merged, "overlay", mOpts); err != nil {
-		return err
-	}
-	if err := mount.Unmount(merged); err != nil {
-		return err
-	}
-	return nil
-}
-
-// supportsUserNSOverlay returns nil error if overlay is supported in userns.
-// Only Ubuntu and a few distros support overlay in userns (by patching the kernel).
-// https://lists.ubuntu.com/archives/kernel-team/2014-February/038091.html
-// As of kernel 4.19, the patch is not merged to the upstream.
-func supportsUserNSOverlay() error {
-	tmp, err := ioutil.TempDir("", "docker-test-supports-userns-overlay")
-	if err != nil {
-		return err
-	}
-	defer os.RemoveAll(tmp)
-	cmd := reexec.Command(reexecSupportsUserNSOverlay, tmp)
-	unshareCmd(cmd)
-	out, err := cmd.CombinedOutput()
-	if err != nil {
-		return errors.Wrapf(err, "output: %q", string(out))
-	}
-	return nil
-}
-
-// isOpaque returns nil error if the dir has trusted.overlay.opaque=y.
-// isOpaque needs to be called in the initial userns.
-func isOpaque(dir string) error {
-	xattrOpaque, err := system.Lgetxattr(dir, "trusted.overlay.opaque")
-	if err != nil {
-		return errors.Wrapf(err, "failed to read opaque flag of %s", dir)
-	}
-	if string(xattrOpaque) != "y" {
-		return errors.Errorf("expected \"y\", got %q", string(xattrOpaque))
-	}
-	return nil
-}
-
-func TestReexecUserNSOverlayWhiteoutConverter(t *testing.T) {
-	skip.If(t, os.Getuid() != 0, "skipping test that requires root")
-	skip.If(t, sys.RunningInUserNS(), "skipping test that requires initial userns")
-	if err := supportsUserNSOverlay(); err != nil {
-		t.Skipf("skipping test that requires kernel support for overlay-in-userns: %v", err)
-	}
-	tmp, err := ioutil.TempDir("", "docker-test-userns-overlay")
-	assert.NilError(t, err)
-	defer os.RemoveAll(tmp)
-
-	char0 := filepath.Join(tmp, "char0")
-	cmd := reexec.Command(reexecMknodChar0, char0)
-	unshareCmd(cmd)
-	out, err := cmd.CombinedOutput()
-	assert.NilError(t, err, string(out))
-	assert.NilError(t, isChar0(char0))
-
-	opaqueDir := filepath.Join(tmp, "opaquedir")
-	err = os.MkdirAll(opaqueDir, 0755)
-	assert.NilError(t, err, string(out))
-	cmd = reexec.Command(reexecSetOpaque, opaqueDir)
-	unshareCmd(cmd)
-	out, err = cmd.CombinedOutput()
-	assert.NilError(t, err, string(out))
-	assert.NilError(t, isOpaque(opaqueDir))
-}
-
-func init() {
-	reexec.Register(reexecSupportsUserNSOverlay, func() {
-		if err := supportsOverlay(os.Args[1]); err != nil {
-			panic(err)
-		}
-	})
-	reexec.Register(reexecMknodChar0, func() {
-		if err := mknodChar0Overlay(os.Args[1]); err != nil {
-			panic(err)
-		}
-	})
-	reexec.Register(reexecSetOpaque, func() {
-		if err := replaceDirWithOverlayOpaque(os.Args[1]); err != nil {
-			panic(err)
-		}
-	})
-	if reexec.Init() {
-		os.Exit(0)
-	}
-}

pkg/archive/archive_other.go

@@ -2,6 +2,6 @@
 
 package archive // import "github.com/docker/docker/pkg/archive"
 
-func getWhiteoutConverter(format WhiteoutFormat, inUserNS bool) tarWhiteoutConverter {
-	return nil
+func getWhiteoutConverter(format WhiteoutFormat, inUserNS bool) (tarWhiteoutConverter, error) {
+	return nil, nil
 }