Reexec external key handling

Signed-off-by: Madhu Venugopal <madhu@docker.com>

libnetwork/controller.go

@@ -65,6 +65,9 @@ import (
 // NetworkController provides the interface for controller instance which manages
 // networks.
 type NetworkController interface {
+	// ID provides an unique identity for the controller
+	ID() string
+
 	// ConfigureNetworkDriver applies the passed options to the driver instance for the specified network type
 	ConfigureNetworkDriver(networkType string, options map[string]interface{}) error
 
@@ -99,8 +102,8 @@ type NetworkController interface {
 	// SandboxByID returns the Sandbox which has the passed id. If not found, a types.NotFoundError is returned.
 	SandboxByID(id string) (Sandbox, error)
 
-	// GC triggers immediate garbage collection of resources which are garbage collected.
-	GC()
+	// Stop network controller
+	Stop()
 }
 
 // NetworkWalker is a client provided function which will be used to walk the Networks.
@@ -122,11 +125,13 @@ type endpointTable map[string]*endpoint
 type sandboxTable map[string]*sandbox
 
 type controller struct {
-	networks  networkTable
-	drivers   driverTable
-	sandboxes sandboxTable
-	cfg       *config.Config
-	store     datastore.DataStore
+	id             string
+	networks       networkTable
+	drivers        driverTable
+	sandboxes      sandboxTable
+	cfg            *config.Config
+	store          datastore.DataStore
+	extKeyListener net.Listener
 	sync.Mutex
 }
 
@@ -138,6 +143,7 @@ func New(cfgOptions ...config.Option) (NetworkController, error) {
 		cfg.ProcessOptions(cfgOptions...)
 	}
 	c := &controller{
+		id:        stringid.GenerateRandomID(),
 		cfg:       cfg,
 		networks:  networkTable{},
 		sandboxes: sandboxTable{},
@@ -159,9 +165,17 @@ func New(cfgOptions ...config.Option) (NetworkController, error) {
 		}
 	}
 
+	if err := c.startExternalKeyListener(); err != nil {
+		return nil, err
+	}
+
 	return c, nil
 }
 
+func (c *controller) ID() string {
+	return c.id
+}
+
 func (c *controller) validateHostDiscoveryConfig() bool {
 	if c.cfg == nil || c.cfg.Cluster.Discovery == "" || c.cfg.Cluster.Address == "" {
 		return false
@@ -514,6 +528,7 @@ func (c *controller) isDriverGlobalScoped(networkType string) (bool, error) {
 	return false, nil
 }
 
-func (c *controller) GC() {
+func (c *controller) Stop() {
+	c.stopExternalKeyListener()
 	osl.GC()
 }

libnetwork/libnetwork_test.go

@@ -2,6 +2,7 @@ package libnetwork_test
 
 import (
 	"bytes"
+	"encoding/json"
 	"flag"
 	"fmt"
 	"io/ioutil"
@@ -9,8 +10,10 @@ import (
 	"net/http"
 	"net/http/httptest"
 	"os"
+	"os/exec"
 	"runtime"
 	"strconv"
+	"strings"
 	"sync"
 	"testing"
 
@@ -25,6 +28,8 @@ import (
 	"github.com/docker/libnetwork/osl"
 	"github.com/docker/libnetwork/testutils"
 	"github.com/docker/libnetwork/types"
+	"github.com/opencontainers/runc/libcontainer"
+	"github.com/opencontainers/runc/libcontainer/configs"
 	"github.com/vishvananda/netlink"
 	"github.com/vishvananda/netns"
 )
@@ -1193,6 +1198,14 @@ func (f *fakeSandbox) SetKey(key string) error {
 }
 
 func TestExternalKey(t *testing.T) {
+	externalKeyTest(t, false)
+}
+
+func TestExternalKeyWithReexec(t *testing.T) {
+	externalKeyTest(t, true)
+}
+
+func externalKeyTest(t *testing.T, reexec bool) {
 	if !testutils.IsRunningInContainer() {
 		defer testutils.SetupTestOSContext(t)()
 	}
@@ -1264,9 +1277,16 @@ func TestExternalKey(t *testing.T) {
 		t.Fatalf("Expected to have a valid Sandbox")
 	}
 
-	// Setting an non-existing key (namespace) must fail
-	if err := sbox.SetKey("this-must-fail"); err == nil {
-		t.Fatalf("Setkey must fail if the corresponding namespace is not created")
+	if reexec {
+		err := reexecSetKey("this-must-fail", containerID, controller.ID())
+		if err == nil {
+			t.Fatalf("SetExternalKey must fail if the corresponding namespace is not created")
+		}
+	} else {
+		// Setting an non-existing key (namespace) must fail
+		if err := sbox.SetKey("this-must-fail"); err == nil {
+			t.Fatalf("Setkey must fail if the corresponding namespace is not created")
+		}
 	}
 
 	// Create a new OS sandbox using the osl API before using it in SetKey
@@ -1274,8 +1294,15 @@ func TestExternalKey(t *testing.T) {
 		t.Fatalf("Failed to create new osl sandbox")
 	}
 
-	if err := sbox.SetKey("ValidKey"); err != nil {
-		t.Fatalf("Setkey failed with %v", err)
+	if reexec {
+		err := reexecSetKey("ValidKey", containerID, controller.ID())
+		if err != nil {
+			t.Fatalf("SetExternalKey failed with %v", err)
+		}
+	} else {
+		if err := sbox.SetKey("ValidKey"); err != nil {
+			t.Fatalf("Setkey failed with %v", err)
+		}
 	}
 
 	// Join endpoint to sandbox after SetKey
@@ -1299,6 +1326,28 @@ func TestExternalKey(t *testing.T) {
 	checkSandbox(t, ep.Info())
 }
 
+func reexecSetKey(key string, containerID string, controllerID string) error {
+	var (
+		state libcontainer.State
+		b     []byte
+		err   error
+	)
+
+	state.NamespacePaths = make(map[configs.NamespaceType]string)
+	state.NamespacePaths[configs.NamespaceType("NEWNET")] = key
+	if b, err = json.Marshal(state); err != nil {
+		return err
+	}
+	cmd := &exec.Cmd{
+		Path:   reexec.Self(),
+		Args:   append([]string{"libnetwork-setkey"}, containerID, controllerID),
+		Stdin:  strings.NewReader(string(b)),
+		Stdout: os.Stdout,
+		Stderr: os.Stderr,
+	}
+	return cmd.Run()
+}
+
 func TestEndpointDeleteWithActiveContainer(t *testing.T) {
 	if !testutils.IsRunningInContainer() {
 		defer testutils.SetupTestOSContext(t)()

libnetwork/sandbox_externalkey.go

@@ -0,0 +1,185 @@
+package libnetwork
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net"
+	"os"
+
+	"github.com/Sirupsen/logrus"
+	"github.com/docker/docker/pkg/reexec"
+	"github.com/docker/libnetwork/types"
+	"github.com/opencontainers/runc/libcontainer"
+	"github.com/opencontainers/runc/libcontainer/configs"
+)
+
+type setKeyData struct {
+	ContainerID string
+	Key         string
+}
+
+func init() {
+	reexec.Register("libnetwork-setkey", processSetKeyReexec)
+}
+
+const udsBase = "/var/lib/docker/network/files/"
+const success = "success"
+
+// processSetKeyReexec is a private function that must be called only on an reexec path
+// It expects 3 args { [0] = "libnetwork-setkey", [1] = <container-id>, [2] = <controller-id> }
+// It also expects libcontainer.State as a json string in <stdin>
+// Refer to https://github.com/opencontainers/runc/pull/160/ for more information
+func processSetKeyReexec() {
+	var err error
+
+	// Return a failure to the calling process via ExitCode
+	defer func() {
+		if err != nil {
+			logrus.Fatalf("%v", err)
+		}
+	}()
+
+	// expecting 3 args {[0]="libnetwork-setkey", [1]=<container-id>, [2]=<controller-id> }
+	if len(os.Args) < 3 {
+		err = fmt.Errorf("Re-exec expects 3 args, received : %d", len(os.Args))
+		return
+	}
+	containerID := os.Args[1]
+
+	// We expect libcontainer.State as a json string in <stdin>
+	stateBuf, err := ioutil.ReadAll(os.Stdin)
+	if err != nil {
+		return
+	}
+	var state libcontainer.State
+	if err = json.Unmarshal(stateBuf, &state); err != nil {
+		return
+	}
+
+	controllerID := os.Args[2]
+	key := state.NamespacePaths[configs.NamespaceType("NEWNET")]
+
+	err = SetExternalKey(controllerID, containerID, key)
+	return
+}
+
+// SetExternalKey provides a convenient way to set an External key to a sandbox
+func SetExternalKey(controllerID string, containerID string, key string) error {
+	keyData := setKeyData{
+		ContainerID: containerID,
+		Key:         key}
+
+	c, err := net.Dial("unix", udsBase+controllerID+".sock")
+	if err != nil {
+		return err
+	}
+	defer c.Close()
+
+	if err = sendKey(c, keyData); err != nil {
+		return fmt.Errorf("sendKey failed with : %v", err)
+	}
+	return processReturn(c)
+}
+
+func sendKey(c net.Conn, data setKeyData) error {
+	var err error
+	defer func() {
+		if err != nil {
+			c.Close()
+		}
+	}()
+
+	var b []byte
+	if b, err = json.Marshal(data); err != nil {
+		return err
+	}
+
+	_, err = c.Write(b)
+	return err
+}
+
+func processReturn(r io.Reader) error {
+	buf := make([]byte, 1024)
+	n, err := r.Read(buf[:])
+	if err != nil {
+		return fmt.Errorf("failed to read buf in processReturn : %v", err)
+	}
+	if string(buf[0:n]) != success {
+		return fmt.Errorf(string(buf[0:n]))
+	}
+	return nil
+}
+
+func (c *controller) startExternalKeyListener() error {
+	if err := os.MkdirAll(udsBase, 0600); err != nil {
+		return err
+	}
+	uds := udsBase + c.id + ".sock"
+	l, err := net.Listen("unix", uds)
+	if err != nil {
+		return err
+	}
+	if err := os.Chmod(uds, 0600); err != nil {
+		l.Close()
+		return err
+	}
+	c.Lock()
+	c.extKeyListener = l
+	c.Unlock()
+
+	go c.acceptClientConnections(uds, l)
+	return nil
+}
+
+func (c *controller) acceptClientConnections(sock string, l net.Listener) {
+	for {
+		conn, err := l.Accept()
+		if err != nil {
+			if _, err1 := os.Stat(sock); os.IsNotExist(err1) {
+				logrus.Warnf("Unix socket %s doesnt exist. cannot accept client connections", sock)
+				return
+			}
+			logrus.Errorf("Error accepting connection %v", err)
+			continue
+		}
+		go func() {
+			err := c.processExternalKey(conn)
+			ret := success
+			if err != nil {
+				ret = err.Error()
+			}
+
+			_, err = conn.Write([]byte(ret))
+			if err != nil {
+				logrus.Errorf("Error returning to the client %v", err)
+			}
+		}()
+	}
+}
+
+func (c *controller) processExternalKey(conn net.Conn) error {
+	buf := make([]byte, 1280)
+	nr, err := conn.Read(buf)
+	if err != nil {
+		return err
+	}
+	var s setKeyData
+	if err = json.Unmarshal(buf[0:nr], &s); err != nil {
+		return err
+	}
+
+	var sandbox Sandbox
+	search := SandboxContainerWalker(&sandbox, s.ContainerID)
+	c.WalkSandboxes(search)
+	if sandbox == nil {
+		return types.BadRequestErrorf("no sandbox present for %s", s.ContainerID)
+	}
+
+	return sandbox.SetKey(s.Key)
+}
+
+func (c *controller) stopExternalKeyListener() {
+	c.extKeyListener.Close()
+}