Merge pull request #16644 from dhiltgen/discovery_tls

Add TLS support for discovery backend

daemon/config.go

@@ -39,6 +39,10 @@ type CommonConfig struct {
 	// mechanism.
 	ClusterStore string
 
+	// ClusterOpts is used to pass options to the discovery package for tuning libkv settings, such
+	// as TLS configuration settings.
+	ClusterOpts map[string]string
+
 	// ClusterAdvertise is the network endpoint that the Engine advertises for the purpose of node
 	// discovery. This should be a 'host:port' combination on which that daemon instance is
 	// reachable by other hosts.
@@ -68,4 +72,5 @@ func (config *Config) InstallCommonFlags(cmd *flag.FlagSet, usageFn func(string)
 	cmd.Var(opts.NewMapOpts(config.LogConfig.Config, nil), []string{"-log-opt"}, usageFn("Set log driver options"))
 	cmd.StringVar(&config.ClusterAdvertise, []string{"-cluster-advertise"}, "", usageFn("Address of the daemon instance to advertise"))
 	cmd.StringVar(&config.ClusterStore, []string{"-cluster-store"}, "", usageFn("Set the cluster store"))
+	cmd.Var(opts.NewMapOpts(config.ClusterOpts, nil), []string{"-cluster-store-opt"}, usageFn("Set cluster store options"))
 }

daemon/daemon.go

@@ -757,7 +757,7 @@ func NewDaemon(config *Config, registryService *registry.Service) (daemon *Daemo
 	// DiscoveryWatcher version.
 	if config.ClusterStore != "" && config.ClusterAdvertise != "" {
 		var err error
-		if d.discoveryWatcher, err = initDiscovery(config.ClusterStore, config.ClusterAdvertise); err != nil {
+		if d.discoveryWatcher, err = initDiscovery(config.ClusterStore, config.ClusterAdvertise, config.ClusterOpts); err != nil {
 			return nil, fmt.Errorf("discovery initialization failed (%v)", err)
 		}
 	}

daemon/discovery.go

@@ -20,12 +20,12 @@ const (
 
 // initDiscovery initialized the nodes discovery subsystem by connecting to the specified backend
 // and start a registration loop to advertise the current node under the specified address.
-func initDiscovery(backend, address string) (discovery.Backend, error) {
+func initDiscovery(backend, address string, clusterOpts map[string]string) (discovery.Backend, error) {
 	var (
 		discoveryBackend discovery.Backend
 		err              error
 	)
-	if discoveryBackend, err = discovery.New(backend, defaultDiscoveryHeartbeat, defaultDiscoveryTTL); err != nil {
+	if discoveryBackend, err = discovery.New(backend, defaultDiscoveryHeartbeat, defaultDiscoveryTTL, clusterOpts); err != nil {
 		return nil, err
 	}
 

docker/daemon.go

@@ -71,6 +71,7 @@ func NewDaemonCli() *DaemonCli {
 	// TODO(tiborvass): remove InstallFlags?
 	daemonConfig := new(daemon.Config)
 	daemonConfig.LogConfig.Config = make(map[string]string)
+	daemonConfig.ClusterOpts = make(map[string]string)
 	daemonConfig.InstallFlags(daemonFlags, presentInHelp)
 	daemonConfig.InstallFlags(flag.CommandLine, absentFromHelp)
 	registryOptions := new(registry.Options)

docs/reference/commandline/daemon.md

@@ -24,6 +24,7 @@ weight = -1
       --default-gateway-v6=""                Container default gateway IPv6 address
       --cluster-store=""                     URL of the distributed storage backend
       --cluster-advertise=""                 Address of the daemon instance to advertise
+      --cluster-store-opt=map[]              Set cluster options
       --dns=[]                               DNS server to use
       --dns-opt=[]                           DNS options to use
       --dns-search=[]                        DNS search domains to use
@@ -537,6 +538,20 @@ please check the [run](run.md) reference.
 daemon instance should use when advertising itself to the cluster. The daemon
 should be reachable by remote hosts on this 'host:port' combination.
 
+The daemon uses [libkv](https://github.com/docker/libkv/) to advertise
+the node within the cluster.  Some Key/Value backends support mutual
+TLS, and the client TLS settings used by the daemon can be configured
+using the `--cluster-store-opt` flag, specifying the paths to PEM encoded
+files. For example:
+
+```bash
+    --cluster-advertise 192.168.1.2:2376 \
+    --cluster-store etcd://192.168.1.2:2379 \
+    --cluster-store-opt kv.cacertfile=/path/to/ca.pem \
+    --cluster-store-opt kv.certfile=/path/to/cert.pem \
+    --cluster-store-opt kv.keyfile=/path/to/key.pem
+```
+
 ## Miscellaneous options
 
 IP masquerading uses address translation to allow containers without a public

pkg/discovery/backends.go

@@ -41,11 +41,11 @@ func parse(rawurl string) (string, string) {
 
 // New returns a new Discovery given a URL, heartbeat and ttl settings.
 // Returns an error if the URL scheme is not supported.
-func New(rawurl string, heartbeat time.Duration, ttl time.Duration) (Backend, error) {
+func New(rawurl string, heartbeat time.Duration, ttl time.Duration, clusterOpts map[string]string) (Backend, error) {
 	scheme, uri := parse(rawurl)
 	if backend, exists := backends[scheme]; exists {
 		log.WithFields(log.Fields{"name": scheme, "uri": uri}).Debug("Initializing discovery service")
-		err := backend.Initialize(uri, heartbeat, ttl)
+		err := backend.Initialize(uri, heartbeat, ttl, clusterOpts)
 		return backend, err
 	}
 

pkg/discovery/discovery.go

@@ -27,8 +27,8 @@ type Backend interface {
 	// Watcher must be provided by every backend.
 	Watcher
 
-	// Initialize the discovery with URIs, a heartbeat and a ttl.
-	Initialize(string, time.Duration, time.Duration) error
+	// Initialize the discovery with URIs, a heartbeat, a ttl and optional settings.
+	Initialize(string, time.Duration, time.Duration, map[string]string) error
 
 	// Register to the discovery.
 	Register(string) error

pkg/discovery/file/file.go

@@ -25,7 +25,7 @@ func Init() {
 }
 
 // Initialize is exported
-func (s *Discovery) Initialize(path string, heartbeat time.Duration, ttl time.Duration) error {
+func (s *Discovery) Initialize(path string, heartbeat time.Duration, ttl time.Duration, _ map[string]string) error {
 	s.path = path
 	s.heartbeat = heartbeat
 	return nil

pkg/discovery/file/file_test.go

@@ -19,12 +19,12 @@ var _ = check.Suite(&DiscoverySuite{})
 
 func (s *DiscoverySuite) TestInitialize(c *check.C) {
 	d := &Discovery{}
-	d.Initialize("/path/to/file", 1000, 0)
+	d.Initialize("/path/to/file", 1000, 0, nil)
 	c.Assert(d.path, check.Equals, "/path/to/file")
 }
 
 func (s *DiscoverySuite) TestNew(c *check.C) {
-	d, err := discovery.New("file:///path/to/file", 0, 0)
+	d, err := discovery.New("file:///path/to/file", 0, 0, nil)
 	c.Assert(err, check.IsNil)
 	c.Assert(d.(*Discovery).path, check.Equals, "/path/to/file")
 }
@@ -81,7 +81,7 @@ func (s *DiscoverySuite) TestWatch(c *check.C) {
 
 	// Set up file discovery.
 	d := &Discovery{}
-	d.Initialize(tmp.Name(), 1000, 0)
+	d.Initialize(tmp.Name(), 1000, 0, nil)
 	stopCh := make(chan struct{})
 	ch, errCh := d.Watch(stopCh)
 

pkg/discovery/kv/kv.go

@@ -8,6 +8,7 @@ import (
 
 	log "github.com/Sirupsen/logrus"
 	"github.com/docker/docker/pkg/discovery"
+	"github.com/docker/docker/pkg/tlsconfig"
 	"github.com/docker/libkv"
 	"github.com/docker/libkv/store"
 	"github.com/docker/libkv/store/consul"
@@ -47,7 +48,7 @@ func Init() {
 }
 
 // Initialize is exported
-func (s *Discovery) Initialize(uris string, heartbeat time.Duration, ttl time.Duration) error {
+func (s *Discovery) Initialize(uris string, heartbeat time.Duration, ttl time.Duration, clusterOpts map[string]string) error {
 	var (
 		parts = strings.SplitN(uris, "/", 2)
 		addrs = strings.Split(parts[0], ",")
@@ -63,9 +64,34 @@ func (s *Discovery) Initialize(uris string, heartbeat time.Duration, ttl time.Du
 	s.ttl = ttl
 	s.path = path.Join(s.prefix, discoveryPath)
 
+	var config *store.Config
+	if clusterOpts["kv.cacertfile"] != "" && clusterOpts["kv.certfile"] != "" && clusterOpts["kv.keyfile"] != "" {
+		log.Info("Initializing discovery with TLS")
+		tlsConfig, err := tlsconfig.Client(tlsconfig.Options{
+			CAFile:   clusterOpts["kv.cacertfile"],
+			CertFile: clusterOpts["kv.certfile"],
+			KeyFile:  clusterOpts["kv.keyfile"],
+		})
+		if err != nil {
+			return err
+		}
+		config = &store.Config{
+			// Set ClientTLS to trigger https (bug in libkv/etcd)
+			ClientTLS: &store.ClientTLSConfig{
+				CACertFile: clusterOpts["kv.cacertfile"],
+				CertFile:   clusterOpts["kv.certfile"],
+				KeyFile:    clusterOpts["kv.keyfile"],
+			},
+			// The actual TLS config that will be used
+			TLS: tlsConfig,
+		}
+	} else {
+		log.Info("Initializing discovery without TLS")
+	}
+
 	// Creates a new store, will ignore options given
 	// if not supported by the chosen store
-	s.store, err = libkv.NewStore(s.backend, addrs, nil)
+	s.store, err = libkv.NewStore(s.backend, addrs, config)
 	return err
 }
 

pkg/discovery/kv/kv_test.go

@@ -2,11 +2,14 @@ package kv
 
 import (
 	"errors"
+	"io/ioutil"
+	"os"
 	"path"
 	"testing"
 	"time"
 
 	"github.com/docker/docker/pkg/discovery"
+	"github.com/docker/libkv"
 	"github.com/docker/libkv/store"
 
 	"github.com/go-check/check"
@@ -24,7 +27,7 @@ func (ds *DiscoverySuite) TestInitialize(c *check.C) {
 		Endpoints: []string{"127.0.0.1"},
 	}
 	d := &Discovery{backend: store.CONSUL}
-	d.Initialize("127.0.0.1", 0, 0)
+	d.Initialize("127.0.0.1", 0, 0, nil)
 	d.store = storeMock
 
 	s := d.store.(*FakeStore)
@@ -36,7 +39,7 @@ func (ds *DiscoverySuite) TestInitialize(c *check.C) {
 		Endpoints: []string{"127.0.0.1:1234"},
 	}
 	d = &Discovery{backend: store.CONSUL}
-	d.Initialize("127.0.0.1:1234/path", 0, 0)
+	d.Initialize("127.0.0.1:1234/path", 0, 0, nil)
 	d.store = storeMock
 
 	s = d.store.(*FakeStore)
@@ -48,7 +51,7 @@ func (ds *DiscoverySuite) TestInitialize(c *check.C) {
 		Endpoints: []string{"127.0.0.1:1234", "127.0.0.2:1234", "127.0.0.3:1234"},
 	}
 	d = &Discovery{backend: store.CONSUL}
-	d.Initialize("127.0.0.1:1234,127.0.0.2:1234,127.0.0.3:1234/path", 0, 0)
+	d.Initialize("127.0.0.1:1234,127.0.0.2:1234,127.0.0.3:1234/path", 0, 0, nil)
 	d.store = storeMock
 
 	s = d.store.(*FakeStore)
@@ -60,6 +63,150 @@ func (ds *DiscoverySuite) TestInitialize(c *check.C) {
 	c.Assert(d.path, check.Equals, "path/"+discoveryPath)
 }
 
+// Extremely limited mock store so we can test initialization
+type Mock struct {
+	// Endpoints passed to InitializeMock
+	Endpoints []string
+
+	// Options passed to InitializeMock
+	Options *store.Config
+}
+
+func NewMock(endpoints []string, options *store.Config) (store.Store, error) {
+	s := &Mock{}
+	s.Endpoints = endpoints
+	s.Options = options
+	return s, nil
+}
+func (s *Mock) Put(key string, value []byte, opts *store.WriteOptions) error {
+	return errors.New("Put not supported")
+}
+func (s *Mock) Get(key string) (*store.KVPair, error) {
+	return nil, errors.New("Get not supported")
+}
+func (s *Mock) Delete(key string) error {
+	return errors.New("Delete not supported")
+}
+
+// Exists mock
+func (s *Mock) Exists(key string) (bool, error) {
+	return false, errors.New("Exists not supported")
+}
+
+// Watch mock
+func (s *Mock) Watch(key string, stopCh <-chan struct{}) (<-chan *store.KVPair, error) {
+	return nil, errors.New("Watch not supported")
+}
+
+// WatchTree mock
+func (s *Mock) WatchTree(prefix string, stopCh <-chan struct{}) (<-chan []*store.KVPair, error) {
+	return nil, errors.New("WatchTree not supported")
+}
+
+// NewLock mock
+func (s *Mock) NewLock(key string, options *store.LockOptions) (store.Locker, error) {
+	return nil, errors.New("NewLock not supported")
+}
+
+// List mock
+func (s *Mock) List(prefix string) ([]*store.KVPair, error) {
+	return nil, errors.New("List not supported")
+}
+
+// DeleteTree mock
+func (s *Mock) DeleteTree(prefix string) error {
+	return errors.New("DeleteTree not supported")
+}
+
+// AtomicPut mock
+func (s *Mock) AtomicPut(key string, value []byte, previous *store.KVPair, opts *store.WriteOptions) (bool, *store.KVPair, error) {
+	return false, nil, errors.New("AtomicPut not supported")
+}
+
+// AtomicDelete mock
+func (s *Mock) AtomicDelete(key string, previous *store.KVPair) (bool, error) {
+	return false, errors.New("AtomicDelete not supported")
+}
+
+// Close mock
+func (s *Mock) Close() {
+	return
+}
+
+func (ds *DiscoverySuite) TestInitializeWithCerts(c *check.C) {
+	cert := `-----BEGIN CERTIFICATE-----
+MIIDCDCCAfKgAwIBAgIICifG7YeiQOEwCwYJKoZIhvcNAQELMBIxEDAOBgNVBAMT
+B1Rlc3QgQ0EwHhcNMTUxMDAxMjMwMDAwWhcNMjAwOTI5MjMwMDAwWjASMRAwDgYD
+VQQDEwdUZXN0IENBMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA1wRC
+O+flnLTK5ImjTurNRHwSejuqGbc4CAvpB0hS+z0QlSs4+zE9h80aC4hz+6caRpds
++J908Q+RvAittMHbpc7VjbZP72G6fiXk7yPPl6C10HhRSoSi3nY+B7F2E8cuz14q
+V2e+ejhWhSrBb/keyXpcyjoW1BOAAJ2TIclRRkICSCZrpXUyXxAvzXfpFXo1RhSb
+UywN11pfiCQzDUN7sPww9UzFHuAHZHoyfTr27XnJYVUerVYrCPq8vqfn//01qz55
+Xs0hvzGdlTFXhuabFtQnKFH5SNwo/fcznhB7rePOwHojxOpXTBepUCIJLbtNnWFT
+V44t9gh5IqIWtoBReQIDAQABo2YwZDAOBgNVHQ8BAf8EBAMCAAYwEgYDVR0TAQH/
+BAgwBgEB/wIBAjAdBgNVHQ4EFgQUZKUI8IIjIww7X/6hvwggQK4bD24wHwYDVR0j
+BBgwFoAUZKUI8IIjIww7X/6hvwggQK4bD24wCwYJKoZIhvcNAQELA4IBAQDES2cz
+7sCQfDCxCIWH7X8kpi/JWExzUyQEJ0rBzN1m3/x8ySRxtXyGekimBqQwQdFqlwMI
+xzAQKkh3ue8tNSzRbwqMSyH14N1KrSxYS9e9szJHfUasoTpQGPmDmGIoRJuq1h6M
+ej5x1SCJ7GWCR6xEXKUIE9OftXm9TdFzWa7Ja3OHz/mXteii8VXDuZ5ACq6EE5bY
+8sP4gcICfJ5fTrpTlk9FIqEWWQrCGa5wk95PGEj+GJpNogjXQ97wVoo/Y3p1brEn
+t5zjN9PAq4H1fuCMdNNA+p1DHNwd+ELTxcMAnb2ajwHvV6lKPXutrTFc4umJToBX
+FpTxDmJHEV4bzUzh
+-----END CERTIFICATE-----
+`
+	key := `-----BEGIN RSA PRIVATE KEY-----
+MIIEpQIBAAKCAQEA1wRCO+flnLTK5ImjTurNRHwSejuqGbc4CAvpB0hS+z0QlSs4
++zE9h80aC4hz+6caRpds+J908Q+RvAittMHbpc7VjbZP72G6fiXk7yPPl6C10HhR
+SoSi3nY+B7F2E8cuz14qV2e+ejhWhSrBb/keyXpcyjoW1BOAAJ2TIclRRkICSCZr
+pXUyXxAvzXfpFXo1RhSbUywN11pfiCQzDUN7sPww9UzFHuAHZHoyfTr27XnJYVUe
+rVYrCPq8vqfn//01qz55Xs0hvzGdlTFXhuabFtQnKFH5SNwo/fcznhB7rePOwHoj
+xOpXTBepUCIJLbtNnWFTV44t9gh5IqIWtoBReQIDAQABAoIBAHSWipORGp/uKFXj
+i/mut776x8ofsAxhnLBARQr93ID+i49W8H7EJGkOfaDjTICYC1dbpGrri61qk8sx
+qX7p3v/5NzKwOIfEpirgwVIqSNYe/ncbxnhxkx6tXtUtFKmEx40JskvSpSYAhmmO
+1XSx0E/PWaEN/nLgX/f1eWJIlxlQkk3QeqL+FGbCXI48DEtlJ9+MzMu4pAwZTpj5
+5qtXo5JJ0jRGfJVPAOznRsYqv864AhMdMIWguzk6EGnbaCWwPcfcn+h9a5LMdony
+MDHfBS7bb5tkF3+AfnVY3IBMVx7YlsD9eAyajlgiKu4zLbwTRHjXgShy+4Oussz0
+ugNGnkECgYEA/hi+McrZC8C4gg6XqK8+9joD8tnyDZDz88BQB7CZqABUSwvjDqlP
+L8hcwo/lzvjBNYGkqaFPUICGWKjeCtd8pPS2DCVXxDQX4aHF1vUur0uYNncJiV3N
+XQz4Iemsa6wnKf6M67b5vMXICw7dw0HZCdIHD1hnhdtDz0uVpeevLZ8CgYEA2KCT
+Y43lorjrbCgMqtlefkr3GJA9dey+hTzCiWEOOqn9RqGoEGUday0sKhiLofOgmN2B
+LEukpKIey8s+Q/cb6lReajDVPDsMweX8i7hz3Wa4Ugp4Xa5BpHqu8qIAE2JUZ7bU
+t88aQAYE58pUF+/Lq1QzAQdrjjzQBx6SrBxieecCgYEAvukoPZEC8mmiN1VvbTX+
+QFHmlZha3QaDxChB+QUe7bMRojEUL/fVnzkTOLuVFqSfxevaI/km9n0ac5KtAchV
+xjp2bTnBb5EUQFqjopYktWA+xO07JRJtMfSEmjZPbbay1kKC7rdTfBm961EIHaRj
+xZUf6M+rOE8964oGrdgdLlECgYEA046GQmx6fh7/82FtdZDRQp9tj3SWQUtSiQZc
+qhO59Lq8mjUXz+MgBuJXxkiwXRpzlbaFB0Bca1fUoYw8o915SrDYf/Zu2OKGQ/qa
+V81sgiVmDuEgycR7YOlbX6OsVUHrUlpwhY3hgfMe6UtkMvhBvHF/WhroBEIJm1pV
+PXZ/CbMCgYEApNWVktFBjOaYfY6SNn4iSts1jgsQbbpglg3kT7PLKjCAhI6lNsbk
+dyT7ut01PL6RaW4SeQWtrJIVQaM6vF3pprMKqlc5XihOGAmVqH7rQx9rtQB5TicL
+BFrwkQE4HQtQBV60hYQUzzlSk44VFDz+jxIEtacRHaomDRh2FtOTz+I=
+-----END RSA PRIVATE KEY-----
+`
+	certFile, err := ioutil.TempFile("", "cert")
+	c.Assert(err, check.IsNil)
+	defer os.Remove(certFile.Name())
+	certFile.Write([]byte(cert))
+	certFile.Close()
+	keyFile, err := ioutil.TempFile("", "key")
+	c.Assert(err, check.IsNil)
+	defer os.Remove(keyFile.Name())
+	keyFile.Write([]byte(key))
+	keyFile.Close()
+
+	libkv.AddStore("mock", NewMock)
+	d := &Discovery{backend: "mock"}
+	err = d.Initialize("127.0.0.3:1234", 0, 0, map[string]string{
+		"kv.cacertfile": certFile.Name(),
+		"kv.certfile":   certFile.Name(),
+		"kv.keyfile":    keyFile.Name(),
+	})
+	c.Assert(err, check.IsNil)
+	s := d.store.(*Mock)
+	c.Assert(s.Options.TLS, check.NotNil)
+	c.Assert(s.Options.TLS.RootCAs, check.NotNil)
+	c.Assert(s.Options.TLS.Certificates, check.HasLen, 1)
+}
+
 func (ds *DiscoverySuite) TestWatch(c *check.C) {
 	mockCh := make(chan []*store.KVPair)
 
@@ -69,7 +216,7 @@ func (ds *DiscoverySuite) TestWatch(c *check.C) {
 	}
 
 	d := &Discovery{backend: store.CONSUL}
-	d.Initialize("127.0.0.1:1234/path", 0, 0)
+	d.Initialize("127.0.0.1:1234/path", 0, 0, nil)
 	d.store = storeMock
 
 	expected := discovery.Entries{

pkg/discovery/nodes/nodes.go

@@ -23,7 +23,7 @@ func Init() {
 }
 
 // Initialize is exported
-func (s *Discovery) Initialize(uris string, _ time.Duration, _ time.Duration) error {
+func (s *Discovery) Initialize(uris string, _ time.Duration, _ time.Duration, _ map[string]string) error {
 	for _, input := range strings.Split(uris, ",") {
 		for _, ip := range discovery.Generate(input) {
 			entry, err := discovery.NewEntry(ip)

pkg/discovery/nodes/nodes_test.go

@@ -17,7 +17,7 @@ var _ = check.Suite(&DiscoverySuite{})
 
 func (s *DiscoverySuite) TestInitialize(c *check.C) {
 	d := &Discovery{}
-	d.Initialize("1.1.1.1:1111,2.2.2.2:2222", 0, 0)
+	d.Initialize("1.1.1.1:1111,2.2.2.2:2222", 0, 0, nil)
 	c.Assert(len(d.entries), check.Equals, 2)
 	c.Assert(d.entries[0].String(), check.Equals, "1.1.1.1:1111")
 	c.Assert(d.entries[1].String(), check.Equals, "2.2.2.2:2222")
@@ -25,7 +25,7 @@ func (s *DiscoverySuite) TestInitialize(c *check.C) {
 
 func (s *DiscoverySuite) TestInitializeWithPattern(c *check.C) {
 	d := &Discovery{}
-	d.Initialize("1.1.1.[1:2]:1111,2.2.2.[2:4]:2222", 0, 0)
+	d.Initialize("1.1.1.[1:2]:1111,2.2.2.[2:4]:2222", 0, 0, nil)
 	c.Assert(len(d.entries), check.Equals, 5)
 	c.Assert(d.entries[0].String(), check.Equals, "1.1.1.1:1111")
 	c.Assert(d.entries[1].String(), check.Equals, "1.1.1.2:1111")
@@ -36,7 +36,7 @@ func (s *DiscoverySuite) TestInitializeWithPattern(c *check.C) {
 
 func (s *DiscoverySuite) TestWatch(c *check.C) {
 	d := &Discovery{}
-	d.Initialize("1.1.1.1:1111,2.2.2.2:2222", 0, 0)
+	d.Initialize("1.1.1.1:1111,2.2.2.2:2222", 0, 0, nil)
 	expected := discovery.Entries{
 		&discovery.Entry{Host: "1.1.1.1", Port: "1111"},
 		&discovery.Entry{Host: "2.2.2.2", Port: "2222"},