Merge pull request #7490 from crosbymichael/reexec

Use argv0 as reexec implementation for dockerinit

daemon/execdriver/driver.go

@@ -20,49 +20,7 @@ var (
 	ErrDriverNotFound          = errors.New("The requested docker init has not been found")
 )
 
-var dockerInitFcts map[string]InitFunc
-
-type (
-	StartCallback func(*Command)
-	InitFunc      func(i *InitArgs) error
-)
-
-func RegisterInitFunc(name string, fct InitFunc) error {
-	if dockerInitFcts == nil {
-		dockerInitFcts = make(map[string]InitFunc)
-	}
-	if _, ok := dockerInitFcts[name]; ok {
-		return ErrDriverAlreadyRegistered
-	}
-	dockerInitFcts[name] = fct
-	return nil
-}
-
-func GetInitFunc(name string) (InitFunc, error) {
-	fct, ok := dockerInitFcts[name]
-	if !ok {
-		return nil, ErrDriverNotFound
-	}
-	return fct, nil
-}
-
-// Args provided to the init function for a driver
-type InitArgs struct {
-	User       string
-	Gateway    string
-	Ip         string
-	WorkDir    string
-	Privileged bool
-	Env        []string
-	Args       []string
-	Mtu        int
-	Driver     string
-	Console    string
-	Pipe       int
-	Root       string
-	CapAdd     string
-	CapDrop    string
-}
+type StartCallback func(*Command)
 
 // Driver specific information based on
 // processes registered with the driver

daemon/execdriver/lxc/driver.go

@@ -5,12 +5,10 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
-	"log"
 	"os"
 	"os/exec"
 	"path"
 	"path/filepath"
-	"runtime"
 	"strconv"
 	"strings"
 	"syscall"
@@ -27,34 +25,6 @@ import (
 
 const DriverName = "lxc"
 
-func init() {
-	execdriver.RegisterInitFunc(DriverName, func(args *execdriver.InitArgs) error {
-		runtime.LockOSThread()
-		if err := setupEnv(args); err != nil {
-			return err
-		}
-		if err := setupHostname(args); err != nil {
-			return err
-		}
-		if err := setupNetworking(args); err != nil {
-			return err
-		}
-		if err := finalizeNamespace(args); err != nil {
-			return err
-		}
-
-		path, err := exec.LookPath(args.Args[0])
-		if err != nil {
-			log.Printf("Unable to locate %v", args.Args[0])
-			os.Exit(127)
-		}
-		if err := syscall.Exec(path, args.Args, os.Environ()); err != nil {
-			return fmt.Errorf("dockerinit unable to execute %s - %s", path, err)
-		}
-		panic("Unreachable")
-	})
-}
-
 type driver struct {
 	root       string // root path for the driver to use
 	initPath   string
@@ -67,6 +37,7 @@ func NewDriver(root, initPath string, apparmor bool) (*driver, error) {
 	if err := linkLxcStart(root); err != nil {
 		return nil, err
 	}
+
 	return &driver{
 		apparmor:   apparmor,
 		root:       root,
@@ -108,8 +79,6 @@ func (d *driver) Run(c *execdriver.Command, pipes *execdriver.Pipes, startCallba
 		"-f", configPath,
 		"--",
 		c.InitPath,
-		"-driver",
-		DriverName,
 	}
 
 	if c.Network.Interface != nil {

daemon/execdriver/lxc/init.go

@@ -2,19 +2,116 @@ package lxc
 
 import (
 	"encoding/json"
+	"flag"
 	"fmt"
 	"io/ioutil"
+	"log"
 	"net"
 	"os"
+	"os/exec"
+	"runtime"
 	"strings"
 	"syscall"
 
-	"github.com/docker/docker/daemon/execdriver"
+	"github.com/docker/docker/reexec"
 	"github.com/docker/libcontainer/netlink"
 )
 
+// Args provided to the init function for a driver
+type InitArgs struct {
+	User       string
+	Gateway    string
+	Ip         string
+	WorkDir    string
+	Privileged bool
+	Env        []string
+	Args       []string
+	Mtu        int
+	Console    string
+	Pipe       int
+	Root       string
+	CapAdd     string
+	CapDrop    string
+}
+
+func init() {
+	// like always lxc requires a hack to get this to work
+	reexec.Register("/.dockerinit", dockerInititalizer)
+}
+
+func dockerInititalizer() {
+	initializer()
+}
+
+// initializer is the lxc driver's init function that is run inside the namespace to setup
+// additional configurations
+func initializer() {
+	runtime.LockOSThread()
+
+	args := getArgs()
+
+	if err := setupNamespace(args); err != nil {
+		log.Fatal(err)
+	}
+}
+
+func setupNamespace(args *InitArgs) error {
+	if err := setupEnv(args); err != nil {
+		return err
+	}
+	if err := setupHostname(args); err != nil {
+		return err
+	}
+	if err := setupNetworking(args); err != nil {
+		return err
+	}
+	if err := finalizeNamespace(args); err != nil {
+		return err
+	}
+
+	path, err := exec.LookPath(args.Args[0])
+	if err != nil {
+		log.Printf("Unable to locate %v", args.Args[0])
+		os.Exit(127)
+	}
+
+	if err := syscall.Exec(path, args.Args, os.Environ()); err != nil {
+		return fmt.Errorf("dockerinit unable to execute %s - %s", path, err)
+	}
+
+	return nil
+}
+
+func getArgs() *InitArgs {
+	var (
+		// Get cmdline arguments
+		user       = flag.String("u", "", "username or uid")
+		gateway    = flag.String("g", "", "gateway address")
+		ip         = flag.String("i", "", "ip address")
+		workDir    = flag.String("w", "", "workdir")
+		privileged = flag.Bool("privileged", false, "privileged mode")
+		mtu        = flag.Int("mtu", 1500, "interface mtu")
+		capAdd     = flag.String("cap-add", "", "capabilities to add")
+		capDrop    = flag.String("cap-drop", "", "capabilities to drop")
+	)
+
+	flag.Parse()
+
+	return &InitArgs{
+		User:       *user,
+		Gateway:    *gateway,
+		Ip:         *ip,
+		WorkDir:    *workDir,
+		Privileged: *privileged,
+		Args:       flag.Args(),
+		Mtu:        *mtu,
+		CapAdd:     *capAdd,
+		CapDrop:    *capDrop,
+	}
+}
+
 // Clear environment pollution introduced by lxc-start
-func setupEnv(args *execdriver.InitArgs) error {
+func setupEnv(args *InitArgs) error {
 	// Get env
 	var env []string
 	content, err := ioutil.ReadFile(".dockerenv")
@@ -41,7 +138,7 @@ func setupEnv(args *execdriver.InitArgs) error {
 	return nil
 }
 
-func setupHostname(args *execdriver.InitArgs) error {
+func setupHostname(args *InitArgs) error {
 	hostname := getEnv(args, "HOSTNAME")
 	if hostname == "" {
 		return nil
@@ -50,7 +147,7 @@ func setupHostname(args *execdriver.InitArgs) error {
 }
 
 // Setup networking
-func setupNetworking(args *execdriver.InitArgs) error {
+func setupNetworking(args *InitArgs) error {
 	if args.Ip != "" {
 		// eth0
 		iface, err := net.InterfaceByName("eth0")
@@ -95,7 +192,7 @@ func setupNetworking(args *execdriver.InitArgs) error {
 }
 
 // Setup working directory
-func setupWorkingDirectory(args *execdriver.InitArgs) error {
+func setupWorkingDirectory(args *InitArgs) error {
 	if args.WorkDir == "" {
 		return nil
 	}
@@ -105,7 +202,7 @@ func setupWorkingDirectory(args *execdriver.InitArgs) error {
 	return nil
 }
 
-func getEnv(args *execdriver.InitArgs, key string) string {
+func getEnv(args *InitArgs, key string) string {
 	for _, kv := range args.Env {
 		parts := strings.SplitN(kv, "=", 2)
 		if parts[0] == key && len(parts) == 2 {

daemon/execdriver/lxc/lxc_init_linux.go

@@ -17,7 +17,7 @@ func setHostname(hostname string) error {
 	return syscall.Sethostname([]byte(hostname))
 }
 
-func finalizeNamespace(args *execdriver.InitArgs) error {
+func finalizeNamespace(args *InitArgs) error {
 	if err := utils.CloseExecFrom(3); err != nil {
 		return err
 	}

daemon/execdriver/native/driver.go

@@ -22,7 +22,6 @@ import (
 	"github.com/docker/libcontainer/cgroups/systemd"
 	consolepkg "github.com/docker/libcontainer/console"
 	"github.com/docker/libcontainer/namespaces"
-	"github.com/docker/libcontainer/syncpipe"
 	"github.com/docker/libcontainer/system"
 )
 
@@ -31,38 +30,6 @@ const (
 	Version    = "0.2"
 )
 
-func init() {
-	execdriver.RegisterInitFunc(DriverName, func(args *execdriver.InitArgs) error {
-		var container *libcontainer.Config
-		f, err := os.Open(filepath.Join(args.Root, "container.json"))
-		if err != nil {
-			return err
-		}
-
-		if err := json.NewDecoder(f).Decode(&container); err != nil {
-			f.Close()
-			return err
-		}
-		f.Close()
-
-		rootfs, err := os.Getwd()
-		if err != nil {
-			return err
-		}
-
-		syncPipe, err := syncpipe.NewSyncPipeFromFd(0, uintptr(args.Pipe))
-		if err != nil {
-			return err
-		}
-
-		if err := namespaces.Init(container, rootfs, args.Console, syncPipe, args.Args); err != nil {
-			return err
-		}
-
-		return nil
-	})
-}
-
 type activeContainer struct {
 	container *libcontainer.Config
 	cmd       *exec.Cmd
@@ -133,13 +100,9 @@ func (d *driver) Run(c *execdriver.Command, pipes *execdriver.Pipes, startCallba
 	}
 
 	return namespaces.Exec(container, c.Stdin, c.Stdout, c.Stderr, c.Console, c.Rootfs, dataPath, args, func(container *libcontainer.Config, console, rootfs, dataPath, init string, child *os.File, args []string) *exec.Cmd {
-		// we need to join the rootfs because namespaces will setup the rootfs and chroot
-		initPath := filepath.Join(c.Rootfs, c.InitPath)
-
 		c.Path = d.initPath
 		c.Args = append([]string{
-			initPath,
-			"-driver", DriverName,
+			DriverName,
 			"-console", console,
 			"-pipe", "3",
 			"-root", filepath.Join(d.root, c.ID),

daemon/execdriver/native/init.go

@@ -0,0 +1,65 @@
+// +build linux
+
+package native
+
+import (
+	"encoding/json"
+	"flag"
+	"fmt"
+	"os"
+	"path/filepath"
+	"runtime"
+
+	"github.com/docker/docker/reexec"
+	"github.com/docker/libcontainer"
+	"github.com/docker/libcontainer/namespaces"
+	"github.com/docker/libcontainer/syncpipe"
+)
+
+func init() {
+	reexec.Register(DriverName, initializer)
+}
+
+func initializer() {
+	runtime.LockOSThread()
+
+	var (
+		pipe    = flag.Int("pipe", 0, "sync pipe fd")
+		console = flag.String("console", "", "console (pty slave) path")
+		root    = flag.String("root", ".", "root path for configuration files")
+	)
+
+	flag.Parse()
+
+	var container *libcontainer.Config
+	f, err := os.Open(filepath.Join(*root, "container.json"))
+	if err != nil {
+		writeError(err)
+	}
+
+	if err := json.NewDecoder(f).Decode(&container); err != nil {
+		f.Close()
+		writeError(err)
+	}
+	f.Close()
+
+	rootfs, err := os.Getwd()
+	if err != nil {
+		writeError(err)
+	}
+
+	syncPipe, err := syncpipe.NewSyncPipeFromFd(0, uintptr(*pipe))
+	if err != nil {
+		writeError(err)
+	}
+
+	if err := namespaces.Init(container, rootfs, *console, syncPipe, flag.Args()); err != nil {
+		writeError(err)
+	}
+
+	panic("Unreachable")
+}
+
+func writeError(err error) {
+	fmt.Sprint(os.Stderr, err)
+}

docker/client.go

@@ -8,10 +8,6 @@ import (
 
 const CanDaemon = false
 
-func mainSysinit() {
-	log.Fatal("This is a client-only binary - running it as 'dockerinit' is not supported.")
-}
-
 func mainDaemon() {
 	log.Fatal("This is a client-only binary - running the Docker daemon is not supported.")
 }

docker/daemon.go

@@ -7,20 +7,16 @@ import (
 	"net"
 
 	"github.com/docker/docker/builtins"
+	_ "github.com/docker/docker/daemon/execdriver/lxc"
+	_ "github.com/docker/docker/daemon/execdriver/native"
 	"github.com/docker/docker/dockerversion"
 	"github.com/docker/docker/engine"
 	flag "github.com/docker/docker/pkg/mflag"
 	"github.com/docker/docker/pkg/signal"
-	"github.com/docker/docker/sysinit"
 )
 
 const CanDaemon = true
 
-func mainSysinit() {
-	// Running in init mode
-	sysinit.SysInit()
-}
-
 func mainDaemon() {
 	if flag.NArg() != 0 {
 		flag.Usage()

docker/docker.go

@@ -13,6 +13,7 @@ import (
 	"github.com/docker/docker/api/client"
 	"github.com/docker/docker/dockerversion"
 	flag "github.com/docker/docker/pkg/mflag"
+	"github.com/docker/docker/reexec"
 	"github.com/docker/docker/utils"
 )
 
@@ -23,8 +24,7 @@ const (
 )
 
 func main() {
-	if selfPath := utils.SelfPath(); strings.Contains(selfPath, ".dockerinit") {
-		mainSysinit()
+	if reexec.Init() {
 		return
 	}
 

dockerinit/dockerinit.go

@@ -1,11 +1,12 @@
 package main
 
 import (
-	"github.com/docker/docker/sysinit"
+	_ "github.com/docker/docker/daemon/execdriver/lxc"
+	_ "github.com/docker/docker/daemon/execdriver/native"
+	"github.com/docker/docker/reexec"
 )
 
 func main() {
 	// Running in init mode
-	sysinit.SysInit()
-	return
+	reexec.Init()
 }

integration/runtime_test.go

@@ -20,8 +20,8 @@ import (
 	"github.com/docker/docker/engine"
 	"github.com/docker/docker/image"
 	"github.com/docker/docker/nat"
+	"github.com/docker/docker/reexec"
 	"github.com/docker/docker/runconfig"
-	"github.com/docker/docker/sysinit"
 	"github.com/docker/docker/utils"
 )
 
@@ -94,8 +94,7 @@ func init() {
 	os.Setenv("DOCKER_TMPDIR", unitTestDockerTmpdir)
 
 	// Hack to run sys init during unit testing
-	if selfPath := utils.SelfPath(); strings.Contains(selfPath, ".dockerinit") {
-		sysinit.SysInit()
+	if reexec.Init() {
 		return
 	}
 

reexec/README.md

@@ -0,0 +1,5 @@
+## reexec
+
+The `reexec` package facilitates the busybox style reexec of the docker binary that we require because 
+of the forking limitations of using Go.  Handlers can be registered with a name and the argv 0 of 
+the exec of the binary will be used to find and execute custom init paths.

reexec/reexec.go

@@ -0,0 +1,30 @@
+package reexec
+
+import (
+	"fmt"
+	"os"
+)
+
+var registeredInitializers = make(map[string]func())
+
+// Register adds an initialization func under the specified name
+func Register(name string, initializer func()) {
+	if _, exists := registeredInitializers[name]; exists {
+		panic(fmt.Sprintf("reexec func already registred under name %q", name))
+	}
+
+	registeredInitializers[name] = initializer
+}
+
+// Init is called as the first part of the exec process and returns true if an
+// initialization function was called.
+func Init() bool {
+	initializer, exists := registeredInitializers[os.Args[0]]
+	if exists {
+		initializer()
+
+		return true
+	}
+
+	return false
+}

sysinit/README.md

@@ -1,4 +0,0 @@
-Sys Init code
-
-This code is run INSIDE the container and is responsible for setting
-up the environment before running the actual process

sysinit/sysinit.go

@@ -1,72 +0,0 @@
-package sysinit
-
-import (
-	"flag"
-	"fmt"
-	"log"
-	"os"
-	"runtime"
-
-	"github.com/docker/docker/daemon/execdriver"
-	_ "github.com/docker/docker/daemon/execdriver/lxc"
-	_ "github.com/docker/docker/daemon/execdriver/native"
-)
-
-func executeProgram(args *execdriver.InitArgs) error {
-	dockerInitFct, err := execdriver.GetInitFunc(args.Driver)
-	if err != nil {
-		panic(err)
-	}
-	return dockerInitFct(args)
-}
-
-// Sys Init code
-// This code is run INSIDE the container and is responsible for setting
-// up the environment before running the actual process
-func SysInit() {
-	// The very first thing that we should do is lock the thread so that other
-	// system level options will work and not have issues, i.e. setns
-	runtime.LockOSThread()
-
-	if len(os.Args) <= 1 {
-		fmt.Println("You should not invoke dockerinit manually")
-		os.Exit(1)
-	}
-
-	var (
-		// Get cmdline arguments
-		user       = flag.String("u", "", "username or uid")
-		gateway    = flag.String("g", "", "gateway address")
-		ip         = flag.String("i", "", "ip address")
-		workDir    = flag.String("w", "", "workdir")
-		privileged = flag.Bool("privileged", false, "privileged mode")
-		mtu        = flag.Int("mtu", 1500, "interface mtu")
-		driver     = flag.String("driver", "", "exec driver")
-		pipe       = flag.Int("pipe", 0, "sync pipe fd")
-		console    = flag.String("console", "", "console (pty slave) path")
-		root       = flag.String("root", ".", "root path for configuration files")
-		capAdd     = flag.String("cap-add", "", "capabilities to add")
-		capDrop    = flag.String("cap-drop", "", "capabilities to drop")
-	)
-	flag.Parse()
-
-	args := &execdriver.InitArgs{
-		User:       *user,
-		Gateway:    *gateway,
-		Ip:         *ip,
-		WorkDir:    *workDir,
-		Privileged: *privileged,
-		Args:       flag.Args(),
-		Mtu:        *mtu,
-		Driver:     *driver,
-		Console:    *console,
-		Pipe:       *pipe,
-		Root:       *root,
-		CapAdd:     *capAdd,
-		CapDrop:    *capDrop,
-	}
-
-	if err := executeProgram(args); err != nil {
-		log.Fatal(err)
-	}
-}