Use pivot_root instead of chroot for chrootarchive

This fixes one issue with Docker running under a grsec kernel, which
denies chmod and mknod under chroot.

Note, if pivot_root fails it will still fallback to chroot.

Signed-off-by: Brian Goff <cpuguy83@gmail.com>

pkg/chrootarchive/archive_unix.go

@@ -11,19 +11,11 @@ import (
 	"io/ioutil"
 	"os"
 	"runtime"
-	"syscall"
 
 	"github.com/docker/docker/pkg/archive"
 	"github.com/docker/docker/pkg/reexec"
 )
 
-func chroot(path string) error {
-	if err := syscall.Chroot(path); err != nil {
-		return err
-	}
-	return syscall.Chdir("/")
-}
-
 // untar is the entry-point for docker-untar on re-exec. This is not used on
 // Windows as it does not support chroot, hence no point sandboxing through
 // chroot and rexec.

pkg/chrootarchive/chroot_linux.go

@@ -0,0 +1,90 @@
+package chrootarchive
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"syscall"
+)
+
+// chroot on linux uses pivot_root instead of chroot
+// pivot_root takes a new root and an old root.
+// Old root must be a sub-dir of new root, it is where the current rootfs will reside after the call to pivot_root.
+// New root is where the new rootfs is set to.
+// Old root is removed after the call to pivot_root so it is no longer available under the new root.
+// This is similar to how libcontainer sets up a container's rootfs
+func chroot(path string) (err error) {
+	// Create new mount namespace so mounts don't leak
+	if err := syscall.Unshare(syscall.CLONE_NEWNS); err != nil {
+		return fmt.Errorf("Error creating mount namespace before pivot: %v", err)
+	}
+	// path must be a different fs for pivot_root, so bind-mount to itself to ensure this
+	if err := syscall.Mount(path, path, "", syscall.MS_BIND, ""); err != nil {
+		return fmt.Errorf("Error mounting pivot dir before pivot: %v", err)
+	}
+
+	// setup oldRoot for pivot_root
+	pivotDir, err := ioutil.TempDir(path, ".pivot_root")
+	if err != nil {
+		return fmt.Errorf("Error setting up pivot dir: %v", err)
+	}
+
+	var mounted bool
+	defer func() {
+		if mounted {
+			// make sure pivotDir is not mounted before we try to remove it
+			if errCleanup := syscall.Unmount(pivotDir, syscall.MNT_DETACH); errCleanup != nil {
+				if err == nil {
+					err = errCleanup
+				}
+				return
+			}
+		}
+
+		errCleanup := os.Remove(pivotDir)
+		if errCleanup != nil {
+			errCleanup = fmt.Errorf("Error cleaning up after pivot: %v", errCleanup)
+			if err == nil {
+				err = errCleanup
+			}
+		}
+	}()
+
+	if err := syscall.PivotRoot(path, pivotDir); err != nil {
+		// If pivot fails, fall back to the normal chroot
+		return realChroot(path)
+	}
+	mounted = true
+
+	// This is the new path for where the old root (prior to the pivot) has been moved to
+	// This dir contains the rootfs of the caller, which we need to remove so it is not visible during extraction
+	pivotDir = filepath.Join("/", filepath.Base(pivotDir))
+
+	if err := syscall.Chdir("/"); err != nil {
+		return fmt.Errorf("Error changing to new root: %v", err)
+	}
+
+	// Make the pivotDir (where the old root lives) private so it can be unmounted without propagating to the host
+	if err := syscall.Mount("", pivotDir, "", syscall.MS_PRIVATE|syscall.MS_REC, ""); err != nil {
+		return fmt.Errorf("Error making old root private after pivot: %v", err)
+	}
+
+	// Now unmount the old root so it's no longer visible from the new root
+	if err := syscall.Unmount(pivotDir, syscall.MNT_DETACH); err != nil {
+		return fmt.Errorf("Error while unmounting old root after pivot: %v", err)
+	}
+	mounted = false
+
+	return nil
+}
+
+func realChroot(path string) error {
+	if err := syscall.Chroot(path); err != nil {
+		return fmt.Errorf("Error after fallback to chroot: %v", err)
+	}
+	if err := syscall.Chdir("/"); err != nil {
+		return fmt.Errorf("Error chaning to new root after chroot: %v", err)
+	}
+	return nil
+}

pkg/chrootarchive/chroot_unix.go

@@ -0,0 +1,12 @@
+// +build !windows,!linux
+
+package chrootarchive
+
+import "syscall"
+
+func chroot(path string) error {
+	if err := syscall.Chroot(path); err != nil {
+		return err
+	}
+	return syscall.Chdir("/")
+}