VXLAN UDP Port configuration support

This commit contains changes to configure DataPathPort
option. By default we use 4789 port number. But this commit
will allow user to configure port number during swarm init.
DataPathPort can't be modified after swarm init.
Signed-off-by: selansen <elango.siva@docker.com>

api/server/router/swarm/cluster_routes.go

@@ -28,11 +28,16 @@ func (sr *swarmRouter) initCluster(ctx context.Context, w http.ResponseWriter, r
 		return errdefs.InvalidParameter(err)
 	}
 	version := httputils.VersionFromContext(ctx)
+
 	// DefaultAddrPool and SubnetSize were added in API 1.39. Ignore on older API versions.
 	if versions.LessThan(version, "1.39") {
 		req.DefaultAddrPool = nil
 		req.SubnetSize = 0
 	}
+	// DataPathPort was added in API 1.40. Ignore this option on older API versions.
+	if versions.LessThan(version, "1.40") {
+		req.DataPathPort = 0
+	}
 	nodeID, err := sr.backend.Init(req)
 	if err != nil {
 		logrus.Errorf("Error initializing swarm: %v", err)

api/swagger.yaml

@@ -2465,6 +2465,15 @@ definitions:
         description: "Whether there is currently a root CA rotation in progress for the swarm"
         type: "boolean"
         example: false
+      DataPathPort:
+        description: |
+          DataPathPort specifies the data path port number for data traffic.
+          Acceptable port range is 1024 to 49151.
+          If no port is set or is set to 0, the default port (4789) is used.
+        type: "integer"
+        format: "uint32"
+        default: 4789
+        example: 4789
       DefaultAddrPool:
         description: |
           Default Address Pool specifies default subnet pools for global scope networks.
@@ -8877,6 +8886,13 @@ paths:
                   nodes in order to reach the containers running on this node. Using this parameter it is possible to
                   separate the container data traffic from the management traffic of the cluster.
                 type: "string"
+              DataPathPort:
+                description: |
+                  DataPathPort specifies the data path port number for data traffic.
+                  Acceptable port range is 1024 to 49151.
+                  if no port is set or is set to 0, default port 4789 will be used.
+                type: "integer"
+                format: "uint32"
               DefaultAddrPool:
                 description: |
                   Default Address Pool specifies default subnet pools for global scope networks.
@@ -8897,6 +8913,7 @@ paths:
             example:
               ListenAddr: "0.0.0.0:2377"
               AdvertiseAddr: "192.168.1.1:2377"
+              DataPathPort: 4789
               DefaultAddrPool: ["10.10.0.0/8", "20.20.0.0/8"]
               SubnetSize: 24
               ForceNewCluster: false

api/types/swarm/swarm.go

@@ -14,6 +14,7 @@ type ClusterInfo struct {
 	RootRotationInProgress bool
 	DefaultAddrPool        []string
 	SubnetSize             uint32
+	DataPathPort           uint32
 }
 
 // Swarm represents a swarm.
@@ -153,6 +154,7 @@ type InitRequest struct {
 	ListenAddr       string
 	AdvertiseAddr    string
 	DataPathAddr     string
+	DataPathPort     uint32
 	ForceNewCluster  bool
 	Spec             Spec
 	AutoLockManagers bool

daemon/cluster/convert/swarm.go

@@ -42,6 +42,7 @@ func SwarmFromGRPC(c swarmapi.Cluster) types.Swarm {
 			RootRotationInProgress: c.RootCA.RootRotation != nil,
 			DefaultAddrPool:        c.DefaultAddressPool,
 			SubnetSize:             c.SubnetSize,
+			DataPathPort:           c.VXLANUDPPort,
 		},
 		JoinTokens: types.JoinTokens{
 			Worker:  c.RootCA.JoinTokens.Worker,

daemon/cluster/listen_addr.go

@@ -123,6 +123,25 @@ func validateDefaultAddrPool(defaultAddrPool []string, size uint32) error {
 	return nil
 }
 
+// getDataPathPort validates vxlan udp port (data path port) number.
+// if no port is set, the default (4789) is returned
+// valid port numbers are between 1024 and 49151
+func getDataPathPort(portNum uint32) (uint32, error) {
+	// if the value comes as 0 by any reason we set it to default value 4789
+	if portNum == 0 {
+		portNum = 4789
+		return portNum, nil
+	}
+	// IANA procedures for each range in detail
+	// The Well Known Ports, aka the System Ports, from 0-1023
+	// The Registered Ports, aka the User Ports, from 1024-49151
+	// The Dynamic Ports, aka the Private Ports, from 49152-65535
+	// So we can allow range between 1024 to 49151
+	if portNum < 1024 || portNum > 49151 {
+		return 0, fmt.Errorf("Datapath port number is not in valid range (1024-49151) : %d", portNum)
+	}
+	return portNum, nil
+}
 func resolveDataPathAddr(dataPathAddr string) (string, error) {
 	if dataPathAddr == "" {
 		// dataPathAddr is not defined

daemon/cluster/noderunner.go

@@ -57,6 +57,8 @@ type nodeStartConfig struct {
 	DefaultAddressPool []string
 	// SubnetSize contains subnet size of DefaultAddressPool
 	SubnetSize uint32
+	// DataPathPort contains Data path port (VXLAN UDP port) number that is used for data traffic.
+	DataPathPort uint32
 	// JoinInProgress is set to true if a join operation has started, but
 	// not completed yet.
 	JoinInProgress bool
@@ -125,6 +127,7 @@ func (n *nodeRunner) start(conf nodeStartConfig) error {
 		NetworkConfig: &swarmallocator.NetworkConfig{
 			DefaultAddrPool: conf.DefaultAddressPool,
 			SubnetSize:      conf.SubnetSize,
+			VXLANUDPPort:    conf.DataPathPort,
 		},
 		JoinAddr:  joinAddr,
 		StateDir:  n.cluster.root,

daemon/cluster/swarm.go

@@ -96,6 +96,12 @@ func (c *Cluster) Init(req types.InitRequest) (string, error) {
 	if err := validateDefaultAddrPool(req.DefaultAddrPool, req.SubnetSize); err != nil {
 		return "", err
 	}
+
+	port, err := getDataPathPort(req.DataPathPort)
+	if err != nil {
+		return "", err
+	}
+
 	nr, err := c.newNodeRunner(nodeStartConfig{
 		forceNewCluster:    req.ForceNewCluster,
 		autolock:           req.AutoLockManagers,
@@ -106,6 +112,7 @@ func (c *Cluster) Init(req types.InitRequest) (string, error) {
 		DefaultAddressPool: req.DefaultAddrPool,
 		SubnetSize:         req.SubnetSize,
 		availability:       req.Availability,
+		DataPathPort:       port,
 	})
 	if err != nil {
 		return "", err

docs/api/version-history.md

@@ -29,6 +29,9 @@ keywords: "API, Docker, rcli, REST, documentation"
   to return those without the specified labels.
 * `POST /containers/create`, `GET /containers/{id}/json`, and `GET /containers/json` now supports
   `BindOptions.NonRecursive`.
+* `POST /swarm/init` now accepts a `DataPathPort` property to set data path port number.
+* `GET /info` now returns information about `DataPathPort` that is currently used in swarm
+* `GET /swarm` endpoint now returns DataPathPort info
 
 ## V1.39 API changes
 

integration/network/service_test.go

@@ -321,6 +321,65 @@ func noServices(client client.ServiceAPIClient) func(log poll.LogT) poll.Result
 	}
 }
 
+func TestServiceWithDataPathPortInit(t *testing.T) {
+	skip.If(t, testEnv.OSType == "windows")
+	skip.If(t, versions.LessThan(testEnv.DaemonAPIVersion(), "1.40"), "DataPathPort was added in API v1.40")
+	defer setupTest(t)()
+	var ops = []func(*daemon.Daemon){}
+	var datapathPort uint32 = 7777
+	ops = append(ops, daemon.WithSwarmDataPathPort(datapathPort))
+	d := swarm.NewSwarm(t, testEnv, ops...)
+
+	cli := d.NewClientT(t)
+	defer cli.Close()
+
+	// Create a overlay network
+	name := "saanvisthira" + t.Name()
+	network.CreateNoError(t, context.Background(), cli, name,
+		network.WithDriver("overlay"))
+
+	var instances uint64 = 1
+	serviceID := swarm.CreateService(t, d,
+		swarm.ServiceWithReplicas(instances),
+		swarm.ServiceWithNetwork(name),
+	)
+
+	poll.WaitOn(t, serviceRunningCount(cli, serviceID, instances), swarm.ServicePoll)
+
+	info := d.Info(t)
+	assert.Equal(t, info.Swarm.Cluster.DataPathPort, datapathPort)
+	err := cli.ServiceRemove(context.Background(), serviceID)
+	assert.NilError(t, err)
+	d.SwarmLeave(true)
+	d.Stop(t)
+
+	// Clean up , set it back to original one to make sure other tests don't fail
+	// call without datapath port option.
+	ops = []func(*daemon.Daemon){}
+	d = swarm.NewSwarm(t, testEnv, ops...)
+	cli = d.NewClientT(t)
+
+	// Create a overlay network
+	name = "saanvisthira" + t.Name()
+	network.CreateNoError(t, context.Background(), cli, name,
+		network.WithDriver("overlay"))
+
+	serviceID = swarm.CreateService(t, d,
+		swarm.ServiceWithReplicas(instances),
+		swarm.ServiceWithNetwork(name),
+	)
+
+	poll.WaitOn(t, serviceRunningCount(cli, serviceID, instances), swarm.ServicePoll)
+
+	info = d.Info(t)
+	var defaultDataPathPort uint32 = 4789
+	assert.Equal(t, info.Swarm.Cluster.DataPathPort, defaultDataPathPort)
+	err = cli.ServiceRemove(context.Background(), serviceID)
+	assert.NilError(t, err)
+	d.SwarmLeave(true)
+	defer d.Stop(t)
+}
+
 func TestServiceWithDefaultAddressPoolInit(t *testing.T) {
 	skip.If(t, testEnv.OSType == "windows")
 	defer setupTest(t)()

internal/test/daemon/daemon.go

@@ -76,6 +76,7 @@ type Daemon struct {
 	SwarmPort       int // FIXME(vdemeester) should probably not be exported
 	DefaultAddrPool []string
 	SubnetSize      uint32
+	DataPathPort    uint32
 	// cached information
 	CachedInfo types.Info
 }

internal/test/daemon/ops.go

@@ -48,6 +48,13 @@ func WithSwarmDefaultAddrPoolSubnetSize(subnetSize uint32) func(*Daemon) {
 	}
 }
 
+// WithSwarmDataPathPort sets the  swarm datapath port to use for swarm mode
+func WithSwarmDataPathPort(datapathPort uint32) func(*Daemon) {
+	return func(d *Daemon) {
+		d.DataPathPort = datapathPort
+	}
+}
+
 // WithEnvironment sets options from internal/test/environment.Execution struct
 func WithEnvironment(e environment.Execution) func(*Daemon) {
 	return func(d *Daemon) {

internal/test/daemon/swarm.go

@@ -85,6 +85,9 @@ func (d *Daemon) SwarmInit(t assert.TestingT, req swarm.InitRequest) {
 		req.DefaultAddrPool = d.DefaultAddrPool
 		req.SubnetSize = d.SubnetSize
 	}
+	if d.DataPathPort > 0 {
+		req.DataPathPort = d.DataPathPort
+	}
 	cli := d.NewClientT(t)
 	defer cli.Close()
 	_, err := cli.SwarmInit(context.Background(), req)