Inroduce SWARM --data-path-addr flag

This new flag will allow the configuration of an interface that
can be used for data path traffic to be isolated from control
plane traffic. This flag is simply percolated down to libnetwork
and will be used by all the global scope drivers (today overlay)

Negative test added for invalid flag arguments

Signed-off-by: Flavio Crisciani <flavio.crisciani@docker.com>

api/types/swarm/swarm.go

@@ -132,6 +132,7 @@ type ExternalCA struct {
 type InitRequest struct {
 	ListenAddr       string
 	AdvertiseAddr    string
+	DataPathAddr     string
 	ForceNewCluster  bool
 	Spec             Spec
 	AutoLockManagers bool
@@ -142,6 +143,7 @@ type InitRequest struct {
 type JoinRequest struct {
 	ListenAddr    string
 	AdvertiseAddr string
+	DataPathAddr  string
 	RemoteAddrs   []string
 	JoinToken     string // accept by secret
 	Availability  NodeAvailability

cli/command/swarm/init.go

@@ -19,6 +19,7 @@ type initOptions struct {
 	listenAddr NodeAddrOption
 	// Not a NodeAddrOption because it has no default port.
 	advertiseAddr   string
+	dataPathAddr    string
 	forceNewCluster bool
 	availability    string
 }
@@ -40,6 +41,7 @@ func newInitCommand(dockerCli command.Cli) *cobra.Command {
 	flags := cmd.Flags()
 	flags.Var(&opts.listenAddr, flagListenAddr, "Listen address (format: <ip|interface>[:port])")
 	flags.StringVar(&opts.advertiseAddr, flagAdvertiseAddr, "", "Advertised address (format: <ip|interface>[:port])")
+	flags.StringVar(&opts.dataPathAddr, flagDataPathAddr, "", "Address or interface to use for data path traffic (format: <ip|interface>)")
 	flags.BoolVar(&opts.forceNewCluster, "force-new-cluster", false, "Force create a new cluster from current state")
 	flags.BoolVar(&opts.autolock, flagAutolock, false, "Enable manager autolocking (requiring an unlock key to start a stopped manager)")
 	flags.StringVar(&opts.availability, flagAvailability, "active", `Availability of the node ("active"|"pause"|"drain")`)
@@ -54,6 +56,7 @@ func runInit(dockerCli command.Cli, flags *pflag.FlagSet, opts initOptions) erro
 	req := swarm.InitRequest{
 		ListenAddr:       opts.listenAddr.String(),
 		AdvertiseAddr:    opts.advertiseAddr,
+		DataPathAddr:     opts.dataPathAddr,
 		ForceNewCluster:  opts.forceNewCluster,
 		Spec:             opts.swarmOptions.ToSpec(flags),
 		AutoLockManagers: opts.swarmOptions.autolock,

cli/command/swarm/join.go

@@ -19,6 +19,7 @@ type joinOptions struct {
 	listenAddr NodeAddrOption
 	// Not a NodeAddrOption because it has no default port.
 	advertiseAddr string
+	dataPathAddr  string
 	token         string
 	availability  string
 }
@@ -41,6 +42,7 @@ func newJoinCommand(dockerCli command.Cli) *cobra.Command {
 	flags := cmd.Flags()
 	flags.Var(&opts.listenAddr, flagListenAddr, "Listen address (format: <ip|interface>[:port])")
 	flags.StringVar(&opts.advertiseAddr, flagAdvertiseAddr, "", "Advertised address (format: <ip|interface>[:port])")
+	flags.StringVar(&opts.dataPathAddr, flagDataPathAddr, "", "Address or interface to use for data path traffic (format: <ip|interface>)")
 	flags.StringVar(&opts.token, flagToken, "", "Token for entry into the swarm")
 	flags.StringVar(&opts.availability, flagAvailability, "active", `Availability of the node ("active"|"pause"|"drain")`)
 	return cmd
@@ -54,6 +56,7 @@ func runJoin(dockerCli command.Cli, flags *pflag.FlagSet, opts joinOptions) erro
 		JoinToken:     opts.token,
 		ListenAddr:    opts.listenAddr.String(),
 		AdvertiseAddr: opts.advertiseAddr,
+		DataPathAddr:  opts.dataPathAddr,
 		RemoteAddrs:   []string{opts.remote},
 	}
 	if flags.Changed(flagAvailability) {

cli/command/swarm/opts.go

@@ -19,6 +19,7 @@ const (
 	flagDispatcherHeartbeat = "dispatcher-heartbeat"
 	flagListenAddr          = "listen-addr"
 	flagAdvertiseAddr       = "advertise-addr"
+	flagDataPathAddr        = "data-path-addr"
 	flagQuiet               = "quiet"
 	flagRotate              = "rotate"
 	flagToken               = "token"

contrib/completion/bash/docker

@@ -3301,7 +3301,7 @@ _docker_swarm_init() {
 
 	case "$cur" in
 		-*)
-			COMPREPLY=( $( compgen -W "--advertise-addr --autolock --availability --cert-expiry --dispatcher-heartbeat --external-ca --force-new-cluster --help --listen-addr --max-snapshots --snapshot-interval --task-history-limit" -- "$cur" ) )
+			COMPREPLY=( $( compgen -W "--advertise-addr --data-path-addr --autolock --availability --cert-expiry --dispatcher-heartbeat --external-ca --force-new-cluster --help --listen-addr --max-snapshots --snapshot-interval --task-history-limit" -- "$cur" ) )
 			;;
 	esac
 }
@@ -3337,7 +3337,7 @@ _docker_swarm_join() {
 
 	case "$cur" in
 		-*)
-			COMPREPLY=( $( compgen -W "--advertise-addr --availability --help --listen-addr --token" -- "$cur" ) )
+			COMPREPLY=( $( compgen -W "--advertise-addr --data-path-addr --availability --help --listen-addr --token" -- "$cur" ) )
 			;;
 		*:)
 			COMPREPLY=( $( compgen -W "2377" -- "${cur##*:}" ) )

contrib/completion/zsh/_docker

@@ -2267,6 +2267,7 @@ __docker_swarm_subcommand() {
             _arguments $(__docker_arguments) \
                 $opts_help \
                 "($help)--advertise-addr=[Advertised address]:ip\:port: " \
+                "($help)--data-path-addr=[Data path IP or interface]:ip " \
                 "($help)--autolock[Enable manager autolocking]" \
                 "($help)--availability=[Availability of the node]:availability:(active drain pause)" \
                 "($help)--cert-expiry=[Validity period for node certificates]:duration: " \
@@ -2282,6 +2283,7 @@ __docker_swarm_subcommand() {
             _arguments $(__docker_arguments) -A '-*' \
                 $opts_help \
                 "($help)--advertise-addr=[Advertised address]:ip\:port: " \
+                "($help)--data-path-addr=[Data path IP or interface]:ip " \
                 "($help)--availability=[Availability of the node]:availability:(active drain pause)" \
                 "($help)--listen-addr=[Listen address]:ip\:port: " \
                 "($help)--token=[Token for entry into the swarm]:secret: " \

daemon/cluster/cluster.go

@@ -270,6 +270,16 @@ func (c *Cluster) GetAdvertiseAddress() string {
 	return c.currentNodeState().actualLocalAddr
 }
 
+// GetDataPathAddress returns the address to be used for the data path traffic, if specified.
+func (c *Cluster) GetDataPathAddress() string {
+	c.mu.RLock()
+	defer c.mu.RUnlock()
+	if c.nr != nil {
+		return c.nr.config.DataPathAddr
+	}
+	return ""
+}
+
 // GetRemoteAddress returns a known advertise address of a remote manager if
 // available.
 // todo: change to array/connect with info

daemon/cluster/listen_addr.go

@@ -10,8 +10,10 @@ var (
 	errNoSuchInterface         = errors.New("no such interface")
 	errNoIP                    = errors.New("could not find the system's IP address")
 	errMustSpecifyListenAddr   = errors.New("must specify a listening address because the address to advertise is not recognized as a system address, and a system's IP address to use could not be uniquely identified")
+	errBadNetworkIdentifier    = errors.New("must specify a valid IP address or interface name")
 	errBadListenAddr           = errors.New("listen address must be an IP address or network interface (with optional port number)")
 	errBadAdvertiseAddr        = errors.New("advertise address must be a non-zero IP address or network interface (with optional port number)")
+	errBadDataPathAddr         = errors.New("data path address must be a non-zero IP address or network interface (without a port number)")
 	errBadDefaultAdvertiseAddr = errors.New("default advertise address must be a non-zero IP address or network interface (without a port number)")
 )
 
@@ -20,23 +22,17 @@ func resolveListenAddr(specifiedAddr string) (string, string, error) {
 	if err != nil {
 		return "", "", fmt.Errorf("could not parse listen address %s", specifiedAddr)
 	}
-
 	// Does the host component match any of the interface names on the
 	// system? If so, use the address from that interface.
-	interfaceAddr, err := resolveInterfaceAddr(specifiedHost)
-	if err == nil {
-		return interfaceAddr.String(), specifiedPort, nil
+	specifiedIP, err := resolveInputIPAddr(specifiedHost, true)
+	if err != nil {
+		if err == errBadNetworkIdentifier {
+			err = errBadListenAddr
 		}
-	if err != errNoSuchInterface {
 		return "", "", err
 	}
 
-	// If it's not an interface, it must be an IP (for now)
-	if net.ParseIP(specifiedHost) == nil {
-		return "", "", errBadListenAddr
-	}
-
-	return specifiedHost, specifiedPort, nil
+	return specifiedIP.String(), specifiedPort, nil
 }
 
 func (c *Cluster) resolveAdvertiseAddr(advertiseAddr, listenAddrPort string) (string, string, error) {
@@ -57,43 +53,32 @@ func (c *Cluster) resolveAdvertiseAddr(advertiseAddr, listenAddrPort string) (st
 			advertiseHost = advertiseAddr
 			advertisePort = listenAddrPort
 		}
-
 		// Does the host component match any of the interface names on the
 		// system? If so, use the address from that interface.
-		interfaceAddr, err := resolveInterfaceAddr(advertiseHost)
-		if err == nil {
-			return interfaceAddr.String(), advertisePort, nil
+		advertiseIP, err := resolveInputIPAddr(advertiseHost, false)
+		if err != nil {
+			if err == errBadNetworkIdentifier {
+				err = errBadAdvertiseAddr
 			}
-		if err != errNoSuchInterface {
 			return "", "", err
 		}
 
-		// If it's not an interface, it must be an IP (for now)
-		if ip := net.ParseIP(advertiseHost); ip == nil || ip.IsUnspecified() {
-			return "", "", errBadAdvertiseAddr
-		}
-
-		return advertiseHost, advertisePort, nil
+		return advertiseIP.String(), advertisePort, nil
 	}
 
 	if c.config.DefaultAdvertiseAddr != "" {
 		// Does the default advertise address component match any of the
 		// interface names on the system? If so, use the address from
 		// that interface.
-		interfaceAddr, err := resolveInterfaceAddr(c.config.DefaultAdvertiseAddr)
-		if err == nil {
-			return interfaceAddr.String(), listenAddrPort, nil
+		defaultAdvertiseIP, err := resolveInputIPAddr(c.config.DefaultAdvertiseAddr, false)
+		if err != nil {
+			if err == errBadNetworkIdentifier {
+				err = errBadDefaultAdvertiseAddr
 			}
-		if err != errNoSuchInterface {
 			return "", "", err
 		}
 
-		// If it's not an interface, it must be an IP (for now)
-		if ip := net.ParseIP(c.config.DefaultAdvertiseAddr); ip == nil || ip.IsUnspecified() {
-			return "", "", errBadDefaultAdvertiseAddr
-		}
-
-		return c.config.DefaultAdvertiseAddr, listenAddrPort, nil
+		return defaultAdvertiseIP.String(), listenAddrPort, nil
 	}
 
 	systemAddr, err := c.resolveSystemAddr()
@@ -103,6 +88,22 @@ func (c *Cluster) resolveAdvertiseAddr(advertiseAddr, listenAddrPort string) (st
 	return systemAddr.String(), listenAddrPort, nil
 }
 
+func resolveDataPathAddr(dataPathAddr string) (string, error) {
+	if dataPathAddr == "" {
+		// dataPathAddr is not defined
+		return "", nil
+	}
+	// If a data path flag is specified try to resolve the IP address.
+	dataPathIP, err := resolveInputIPAddr(dataPathAddr, false)
+	if err != nil {
+		if err == errBadNetworkIdentifier {
+			err = errBadDataPathAddr
+		}
+		return "", err
+	}
+	return dataPathIP.String(), nil
+}
+
 func resolveInterfaceAddr(specifiedInterface string) (net.IP, error) {
 	// Use a specific interface's IP address.
 	intf, err := net.InterfaceByName(specifiedInterface)
@@ -149,6 +150,30 @@ func resolveInterfaceAddr(specifiedInterface string) (net.IP, error) {
 	return interfaceAddr6, nil
 }
 
+// resolveInputIPAddr tries to resolve the IP address from the string passed as input
+// - tries to match the string as an interface name, if so returns the IP address associated with it
+// - on failure of previous step tries to parse the string as an IP address itself
+//	 if succeeds returns the IP address
+func resolveInputIPAddr(input string, isUnspecifiedValid bool) (net.IP, error) {
+	// Try to see if it is an interface name
+	interfaceAddr, err := resolveInterfaceAddr(input)
+	if err == nil {
+		return interfaceAddr, nil
+	}
+	// String matched interface but there is a potential ambiguity to be resolved
+	if err != errNoSuchInterface {
+		return nil, err
+	}
+
+	// String is not an interface check if it is a valid IP
+	if ip := net.ParseIP(input); ip != nil && (isUnspecifiedValid || !ip.IsUnspecified()) {
+		return ip, nil
+	}
+
+	// Not valid IP found
+	return nil, errBadNetworkIdentifier
+}
+
 func (c *Cluster) resolveSystemAddrViaSubnetCheck() (net.IP, error) {
 	// Use the system's only IP address, or fail if there are
 	// multiple addresses to choose from. Skip interfaces which

daemon/cluster/noderunner.go

@@ -47,6 +47,9 @@ type nodeStartConfig struct {
 	// AdvertiseAddr is the address other nodes should connect to,
 	// including a port.
 	AdvertiseAddr string
+	// DataPathAddr is the address that has to be used for the data path
+	DataPathAddr string
+
 	joinAddr        string
 	forceNewCluster bool
 	joinToken       string

daemon/cluster/swarm.go

@@ -54,6 +54,11 @@ func (c *Cluster) Init(req types.InitRequest) (string, error) {
 		return "", err
 	}
 
+	dataPathAddr, err := resolveDataPathAddr(req.DataPathAddr)
+	if err != nil {
+		return "", err
+	}
+
 	localAddr := listenHost
 
 	// If the local address is undetermined, the advertise address
@@ -93,6 +98,7 @@ func (c *Cluster) Init(req types.InitRequest) (string, error) {
 		LocalAddr:       localAddr,
 		ListenAddr:      net.JoinHostPort(listenHost, listenPort),
 		AdvertiseAddr:   net.JoinHostPort(advertiseHost, advertisePort),
+		DataPathAddr:    dataPathAddr,
 		availability:    req.Availability,
 	})
 	if err != nil {
@@ -155,12 +161,18 @@ func (c *Cluster) Join(req types.JoinRequest) error {
 		}
 	}
 
+	dataPathAddr, err := resolveDataPathAddr(req.DataPathAddr)
+	if err != nil {
+		return err
+	}
+
 	clearPersistentState(c.root)
 
 	nr, err := c.newNodeRunner(nodeStartConfig{
 		RemoteAddr:    req.RemoteAddrs[0],
 		ListenAddr:    net.JoinHostPort(listenHost, listenPort),
 		AdvertiseAddr: advertiseAddr,
+		DataPathAddr:  dataPathAddr,
 		joinAddr:      req.RemoteAddrs[0],
 		joinToken:     req.JoinToken,
 		availability:  req.Availability,

docs/reference/commandline/swarm_init.md

@@ -25,6 +25,7 @@ Options:
       --autolock                        Enable manager autolocking (requiring an unlock key to start a stopped manager)
       --availability string             Availability of the node ("active"|"pause"|"drain") (default "active")
       --cert-expiry duration            Validity period for node certificates (ns|us|ms|s|m|h) (default 2160h0m0s)
+      --data-path-addr string           Address or interface to use for data path traffic (format: <ip|interface>)
       --dispatcher-heartbeat duration   Dispatcher heartbeat period (ns|us|ms|s|m|h) (default 5s)
       --external-ca external-ca         Specifications of one or more certificate signing endpoints
       --force-new-cluster               Force create a new cluster from current state
@@ -118,6 +119,15 @@ for example `--advertise-addr eth0:2377`.
 Specifying a port is optional. If the value is a bare IP address or interface
 name, the default port 2377 will be used.
 
+### `--data-path-addr`
+
+This flag specifies the address that global scope network drivers will publish towards
+other nodes in order to reach the containers running on this node.
+Using this parameter it is then possible to separate the container's data traffic from the
+management traffic of the cluster.
+If unspecified, Docker will use the same IP address or interface that is used for the
+advertise address.
+
 ### `--task-history-limit`
 
 This flag sets up task history retention limit.

docs/reference/commandline/swarm_join.md

@@ -23,6 +23,7 @@ Join a swarm as a node and/or manager
 Options:
       --advertise-addr string   Advertised address (format: <ip|interface>[:port])
       --availability string     Availability of the node ("active"|"pause"|"drain") (default "active")
+      --data-path-addr string   Address or interface to use for data path traffic (format: <ip|interface>)
       --help                    Print usage
       --listen-addr node-addr   Listen address (format: <ip|interface>[:port]) (default 0.0.0.0:2377)
       --token string            Token for entry into the swarm
@@ -95,6 +96,15 @@ name, the default port 2377 will be used.
 
 This flag is generally not necessary when joining an existing swarm.
 
+### `--data-path-addr`
+
+This flag specifies the address that global scope network drivers will publish towards
+other nodes in order to reach the containers running on this node.
+Using this parameter it is then possible to separate the container's data traffic from the
+management traffic of the cluster.
+If unspecified, Docker will use the same IP address or interface that is used for the
+advertise address.
+
 ### `--token string`
 
 Secret value required for nodes to join the swarm

integration-cli/docker_cli_swarm_test.go

@@ -1932,3 +1932,15 @@ func (s *DockerSwarmSuite) TestSwarmServiceLsFilterMode(c *check.C) {
 	c.Assert(out, checker.Contains, "top1")
 	c.Assert(out, checker.Not(checker.Contains), "top2")
 }
+
+func (s *DockerSwarmSuite) TestSwarmInitUnspecifiedDataPathAddr(c *check.C) {
+	d := s.AddDaemon(c, false, false)
+
+	out, err := d.Cmd("swarm", "init", "--data-path-addr", "0.0.0.0")
+	c.Assert(err, checker.NotNil)
+	c.Assert(out, checker.Contains, "data path address must be a non-zero IP")
+
+	out, err = d.Cmd("swarm", "init", "--data-path-addr", "0.0.0.0:2000")
+	c.Assert(err, checker.NotNil)
+	c.Assert(out, checker.Contains, "data path address must be a non-zero IP")
+}