Merge pull request #31714 from aboch/cingr

Allow user to replace ingress network

api/server/router/network/network_routes.go

@@ -294,6 +294,7 @@ func (n *networkRouter) buildNetworkResource(nw libnetwork.Network) *types.Netwo
 	r.EnableIPv6 = info.IPv6Enabled()
 	r.Internal = info.Internal()
 	r.Attachable = info.Attachable()
+	r.Ingress = info.Ingress()
 	r.Options = info.DriverOptions()
 	r.Containers = make(map[string]types.EndpointResource)
 	buildIpamResources(r, info)

api/swagger.yaml

@@ -1117,6 +1117,8 @@ definitions:
         type: "boolean"
       Attachable:
         type: "boolean"
+      Ingress:
+        type: "boolean"
       Containers:
         type: "object"
         additionalProperties:
@@ -1145,6 +1147,7 @@ definitions:
           foo: "bar"
       Internal: false
       Attachable: false
+      Ingress: false
       Containers:
         19a4d5d687db25203351ed79d478946f861258f018fe384f229f2efa4b23513c:
           Name: "test"
@@ -6211,6 +6214,7 @@ paths:
                 EnableIPv6: false
                 Internal: false
                 Attachable: false
+                Ingress: false
                 IPAM:
                   Driver: "default"
                   Config:
@@ -6237,6 +6241,7 @@ paths:
                 EnableIPv6: false
                 Internal: false
                 Attachable: false
+                Ingress: false
                 IPAM:
                   Driver: "default"
                   Config: []
@@ -6250,6 +6255,7 @@ paths:
                 EnableIPv6: false
                 Internal: false
                 Attachable: false
+                Ingress: false
                 IPAM:
                   Driver: "default"
                   Config: []
@@ -6383,6 +6389,9 @@ paths:
               Attachable:
                 description: "Globally scoped network is manually attachable by regular containers from workers in swarm mode."
                 type: "boolean"
+              Ingress:
+                description: "Ingress network is the network which provides the routing-mesh in swarm mode."
+                type: "boolean"
               IPAM:
                 description: "Optional custom IP scheme for the network."
                 $ref: "#/definitions/IPAM"
@@ -6416,6 +6425,7 @@ paths:
                   foo: "bar"
               Internal: true
               Attachable: false
+              Ingress: false
               Options:
                 com.docker.network.bridge.default_bridge: "true"
                 com.docker.network.bridge.enable_icc: "true"

api/types/swarm/network.go

@@ -82,6 +82,7 @@ type NetworkSpec struct {
 	IPv6Enabled         bool         `json:",omitempty"`
 	Internal            bool         `json:",omitempty"`
 	Attachable          bool         `json:",omitempty"`
+	Ingress             bool         `json:",omitempty"`
 	IPAMOptions         *IPAMOptions `json:",omitempty"`
 }
 

api/types/types.go

@@ -400,6 +400,7 @@ type NetworkResource struct {
 	IPAM       network.IPAM                   // IPAM is the network's IP Address Management
 	Internal   bool                           // Internal represents if the network is used internal only
 	Attachable bool                           // Attachable represents if the global scope is manually attachable by regular containers from workers in swarm mode.
+	Ingress    bool                           // Ingress indicates the network is providing the routing-mesh for the swarm cluster.
 	Containers map[string]EndpointResource    // Containers contains endpoints belonging to the network
 	Options    map[string]string              // Options holds the network specific options to use for when creating the network
 	Labels     map[string]string              // Labels holds metadata specific to the network being created
@@ -431,6 +432,7 @@ type NetworkCreate struct {
 	IPAM           *network.IPAM
 	Internal       bool
 	Attachable     bool
+	Ingress        bool
 	Options        map[string]string
 	Labels         map[string]string
 }

cli/command/network/create.go

@@ -24,6 +24,7 @@ type createOptions struct {
 	internal   bool
 	ipv6       bool
 	attachable bool
+	ingress    bool
 
 	ipamDriver  string
 	ipamSubnet  []string
@@ -59,6 +60,8 @@ func newCreateCommand(dockerCli *command.DockerCli) *cobra.Command {
 	flags.BoolVar(&opts.ipv6, "ipv6", false, "Enable IPv6 networking")
 	flags.BoolVar(&opts.attachable, "attachable", false, "Enable manual container attachment")
 	flags.SetAnnotation("attachable", "version", []string{"1.25"})
+	flags.BoolVar(&opts.ingress, "ingress", false, "Create swarm routing-mesh network")
+	flags.SetAnnotation("ingress", "version", []string{"1.29"})
 
 	flags.StringVar(&opts.ipamDriver, "ipam-driver", "default", "IP Address Management Driver")
 	flags.StringSliceVar(&opts.ipamSubnet, "subnet", []string{}, "Subnet in CIDR format that represents a network segment")
@@ -92,6 +95,7 @@ func runCreate(dockerCli *command.DockerCli, opts createOptions) error {
 		Internal:       opts.internal,
 		EnableIPv6:     opts.ipv6,
 		Attachable:     opts.attachable,
+		Ingress:        opts.ingress,
 		Labels:         runconfigopts.ConvertKVStringsToMap(opts.labels.GetAll()),
 	}
 

cli/command/network/remove.go

@@ -22,12 +22,22 @@ func newRemoveCommand(dockerCli *command.DockerCli) *cobra.Command {
 	}
 }
 
+const ingressWarning = "WARNING! Before removing the routing-mesh network, " +
+	"make sure all the nodes in your swarm run the same docker engine version. " +
+	"Otherwise, removal may not be effective and functionality of newly create " +
+	"ingress networks will be impaired.\nAre you sure you want to continue?"
+
 func runRemove(dockerCli *command.DockerCli, networks []string) error {
 	client := dockerCli.Client()
 	ctx := context.Background()
 	status := 0
 
 	for _, name := range networks {
+		if nw, _, err := client.NetworkInspectWithRaw(ctx, name, false); err == nil &&
+			nw.Ingress &&
+			!command.PromptForConfirmation(dockerCli.In(), dockerCli.Out(), ingressWarning) {
+			continue
+		}
 		if err := client.NetworkRemove(ctx, name); err != nil {
 			fmt.Fprintf(dockerCli.Err(), "%s\n", err)
 			status = 1

daemon/cluster/convert/network.go

@@ -28,6 +28,7 @@ func networkFromGRPC(n *swarmapi.Network) types.Network {
 				IPv6Enabled: n.Spec.Ipv6Enabled,
 				Internal:    n.Spec.Internal,
 				Attachable:  n.Spec.Attachable,
+				Ingress:     n.Spec.Ingress,
 				IPAMOptions: ipamFromGRPC(n.Spec.IPAM),
 			},
 			IPAMOptions: ipamFromGRPC(n.IPAM),
@@ -156,6 +157,7 @@ func BasicNetworkFromGRPC(n swarmapi.Network) basictypes.NetworkResource {
 		IPAM:       ipam,
 		Internal:   spec.Internal,
 		Attachable: spec.Attachable,
+		Ingress:    spec.Ingress,
 		Labels:     n.Spec.Annotations.Labels,
 	}
 
@@ -181,6 +183,7 @@ func BasicNetworkCreateToGRPC(create basictypes.NetworkCreateRequest) swarmapi.N
 		Ipv6Enabled: create.EnableIPv6,
 		Internal:    create.Internal,
 		Attachable:  create.Attachable,
+		Ingress:     create.Ingress,
 	}
 	if create.IPAM != nil {
 		driver := create.IPAM.Driver

daemon/cluster/executor/backend.go

@@ -28,6 +28,7 @@ type Backend interface {
 	DeleteManagedNetwork(name string) error
 	FindNetwork(idName string) (libnetwork.Network, error)
 	SetupIngress(req clustertypes.NetworkCreateRequest, nodeIP string) error
+	ReleaseIngress() error
 	PullImage(ctx context.Context, image, tag string, metaHeaders map[string][]string, authConfig *types.AuthConfig, outStream io.Writer) error
 	CreateManagedContainer(config types.ContainerCreateConfig) (container.ContainerCreateCreatedBody, error)
 	ContainerStart(name string, hostConfig *container.HostConfig, checkpoint string, checkpointDir string) error

daemon/cluster/executor/container/container.go

@@ -575,6 +575,7 @@ func (c *containerConfig) networkCreateRequest(name string) (clustertypes.Networ
 		Labels:         na.Network.Spec.Annotations.Labels,
 		Internal:       na.Network.Spec.Internal,
 		Attachable:     na.Network.Spec.Attachable,
+		Ingress:        na.Network.Spec.Ingress,
 		EnableIPv6:     na.Network.Spec.Ipv6Enabled,
 		CheckDuplicate: true,
 	}

daemon/cluster/executor/container/executor.go

@@ -116,6 +116,7 @@ func (e *executor) Describe(ctx context.Context) (*api.NodeDescription, error) {
 func (e *executor) Configure(ctx context.Context, node *api.Node) error {
 	na := node.Attachment
 	if na == nil {
+		e.backend.ReleaseIngress()
 		return nil
 	}
 
@@ -125,6 +126,7 @@ func (e *executor) Configure(ctx context.Context, node *api.Node) error {
 			Driver: na.Network.IPAM.Driver.Name,
 		},
 		Options:        na.Network.DriverState.Options,
+		Ingress:        true,
 		CheckDuplicate: true,
 	}
 

daemon/network.go

@@ -6,6 +6,7 @@ import (
 	"runtime"
 	"sort"
 	"strings"
+	"sync"
 
 	"github.com/Sirupsen/logrus"
 	apierrors "github.com/docker/docker/api/errors"
@@ -99,15 +100,40 @@ func (daemon *Daemon) getAllNetworks() []libnetwork.Network {
 	return daemon.netController.Networks()
 }
 
-func isIngressNetwork(name string) bool {
-	return name == "ingress"
+type ingressJob struct {
+	create *clustertypes.NetworkCreateRequest
+	ip     net.IP
 }
 
-var ingressChan = make(chan struct{}, 1)
+var (
+	ingressWorkerOnce  sync.Once
+	ingressJobsChannel chan *ingressJob
+	ingressID          string
+)
+
+func (daemon *Daemon) startIngressWorker() {
+	ingressJobsChannel = make(chan *ingressJob, 100)
+	go func() {
+		for {
+			select {
+			case r := <-ingressJobsChannel:
+				if r.create != nil {
+					daemon.setupIngress(r.create, r.ip, ingressID)
+					ingressID = r.create.ID
+				} else {
+					daemon.releaseIngress(ingressID)
+					ingressID = ""
+				}
+			}
+		}
+	}()
+}
 
-func ingressWait() func() {
-	ingressChan <- struct{}{}
-	return func() { <-ingressChan }
+// enqueueIngressJob adds a ingress add/rm request to the worker queue.
+// It guarantees the worker is started.
+func (daemon *Daemon) enqueueIngressJob(job *ingressJob) {
+	ingressWorkerOnce.Do(daemon.startIngressWorker)
+	ingressJobsChannel <- job
 }
 
 // SetupIngress setups ingress networking.
@@ -116,72 +142,93 @@ func (daemon *Daemon) SetupIngress(create clustertypes.NetworkCreateRequest, nod
 	if err != nil {
 		return err
 	}
+	daemon.enqueueIngressJob(&ingressJob{&create, ip})
+	return nil
+}
 
-	go func() {
-		controller := daemon.netController
-		controller.AgentInitWait()
+// ReleaseIngress releases the ingress networking.
+func (daemon *Daemon) ReleaseIngress() error {
+	daemon.enqueueIngressJob(&ingressJob{nil, nil})
+	return nil
+}
 
-		if n, err := daemon.GetNetworkByName(create.Name); err == nil && n != nil && n.ID() != create.ID {
-			if err := controller.SandboxDestroy("ingress-sbox"); err != nil {
-				logrus.Errorf("Failed to delete stale ingress sandbox: %v", err)
-				return
-			}
+func (daemon *Daemon) setupIngress(create *clustertypes.NetworkCreateRequest, ip net.IP, staleID string) {
+	controller := daemon.netController
+	controller.AgentInitWait()
 
-			// Cleanup any stale endpoints that might be left over during previous iterations
-			epList := n.Endpoints()
-			for _, ep := range epList {
-				if err := ep.Delete(true); err != nil {
-					logrus.Errorf("Failed to delete endpoint %s (%s): %v", ep.Name(), ep.ID(), err)
-				}
-			}
+	if staleID != "" && staleID != create.ID {
+		daemon.releaseIngress(staleID)
+	}
 
-			if err := n.Delete(); err != nil {
-				logrus.Errorf("Failed to delete stale ingress network %s: %v", n.ID(), err)
-				return
-			}
+	if _, err := daemon.createNetwork(create.NetworkCreateRequest, create.ID, true); err != nil {
+		// If it is any other error other than already
+		// exists error log error and return.
+		if _, ok := err.(libnetwork.NetworkNameError); !ok {
+			logrus.Errorf("Failed creating ingress network: %v", err)
+			return
 		}
+		// Otherwise continue down the call to create or recreate sandbox.
+	}
 
-		if _, err := daemon.createNetwork(create.NetworkCreateRequest, create.ID, true); err != nil {
-			// If it is any other error other than already
-			// exists error log error and return.
-			if _, ok := err.(libnetwork.NetworkNameError); !ok {
-				logrus.Errorf("Failed creating ingress network: %v", err)
-				return
-			}
+	n, err := daemon.GetNetworkByID(create.ID)
+	if err != nil {
+		logrus.Errorf("Failed getting ingress network by id after creating: %v", err)
+	}
 
-			// Otherwise continue down the call to create or recreate sandbox.
+	sb, err := controller.NewSandbox("ingress-sbox", libnetwork.OptionIngress())
+	if err != nil {
+		if _, ok := err.(networktypes.ForbiddenError); !ok {
+			logrus.Errorf("Failed creating ingress sandbox: %v", err)
 		}
+		return
+	}
 
-		n, err := daemon.GetNetworkByID(create.ID)
-		if err != nil {
-			logrus.Errorf("Failed getting ingress network by id after creating: %v", err)
-			return
-		}
+	ep, err := n.CreateEndpoint("ingress-endpoint", libnetwork.CreateOptionIpam(ip, nil, nil, nil))
+	if err != nil {
+		logrus.Errorf("Failed creating ingress endpoint: %v", err)
+		return
+	}
 
-		sb, err := controller.NewSandbox("ingress-sbox", libnetwork.OptionIngress())
-		if err != nil {
-			if _, ok := err.(networktypes.ForbiddenError); !ok {
-				logrus.Errorf("Failed creating ingress sandbox: %v", err)
-			}
-			return
-		}
+	if err := ep.Join(sb, nil); err != nil {
+		logrus.Errorf("Failed joining ingress sandbox to ingress endpoint: %v", err)
+		return
+	}
 
-		ep, err := n.CreateEndpoint("ingress-endpoint", libnetwork.CreateOptionIpam(ip, nil, nil, nil))
-		if err != nil {
-			logrus.Errorf("Failed creating ingress endpoint: %v", err)
-			return
-		}
+	if err := sb.EnableService(); err != nil {
+		logrus.Errorf("Failed enabling service for ingress sandbox")
+	}
+}
 
-		if err := ep.Join(sb, nil); err != nil {
-			logrus.Errorf("Failed joining ingress sandbox to ingress endpoint: %v", err)
-		}
+func (daemon *Daemon) releaseIngress(id string) {
+	controller := daemon.netController
 
-		if err := sb.EnableService(); err != nil {
-			logrus.WithError(err).Error("Failed enabling service for ingress sandbox")
+	if err := controller.SandboxDestroy("ingress-sbox"); err != nil {
+		logrus.Errorf("Failed to delete ingress sandbox: %v", err)
+	}
+
+	if id == "" {
+		return
+	}
+
+	n, err := controller.NetworkByID(id)
+	if err != nil {
+		logrus.Errorf("failed to retrieve ingress network %s: %v", id, err)
+		return
+	}
+
+	for _, ep := range n.Endpoints() {
+		if err := ep.Delete(true); err != nil {
+			logrus.Errorf("Failed to delete endpoint %s (%s): %v", ep.Name(), ep.ID(), err)
+			return
 		}
-	}()
+	}
 
-	return nil
+	if err := n.Delete(); err != nil {
+		logrus.Errorf("Failed to delete ingress network %s: %v", n.ID(), err)
+		return
+	}
+
+	return
 }
 
 // SetNetworkBootstrapKeys sets the bootstrap keys.
@@ -228,13 +275,6 @@ func (daemon *Daemon) CreateNetwork(create types.NetworkCreateRequest) (*types.N
 }
 
 func (daemon *Daemon) createNetwork(create types.NetworkCreateRequest, id string, agent bool) (*types.NetworkCreateResponse, error) {
-	// If there is a pending ingress network creation wait here
-	// since ingress network creation can happen via node download
-	// from manager or task download.
-	if isIngressNetwork(create.Name) {
-		defer ingressWait()()
-	}
-
 	if runconfig.IsPreDefinedNetwork(create.Name) && !agent {
 		err := fmt.Errorf("%s is a pre-defined network and cannot be created", create.Name)
 		return nil, apierrors.NewRequestForbiddenError(err)
@@ -267,6 +307,7 @@ func (daemon *Daemon) createNetwork(create types.NetworkCreateRequest, id string
 		libnetwork.NetworkOptionDriverOpts(create.Options),
 		libnetwork.NetworkOptionLabels(create.Labels),
 		libnetwork.NetworkOptionAttachable(create.Attachable),
+		libnetwork.NetworkOptionIngress(create.Ingress),
 	}
 
 	if create.IPAM != nil {
@@ -286,10 +327,6 @@ func (daemon *Daemon) createNetwork(create types.NetworkCreateRequest, id string
 		nwOptions = append(nwOptions, libnetwork.NetworkOptionPersist(false))
 	}
 
-	if isIngressNetwork(create.Name) {
-		nwOptions = append(nwOptions, libnetwork.NetworkOptionIngress())
-	}
-
 	n, err := c.NewNetwork(driver, create.Name, id, nwOptions...)
 	if err != nil {
 		return nil, err

daemon/prune.go

@@ -231,7 +231,8 @@ func (daemon *Daemon) clusterNetworksPrune(pruneFilters filters.Args) (*types.Ne
 	}
 	networkIsInUse := regexp.MustCompile(`network ([[:alnum:]]+) is in use`)
 	for _, nw := range networks {
-		if nw.Name == "ingress" {
+		if nw.Ingress {
+			// Routing-mesh network removal has to be explicitly invoked by user
 			continue
 		}
 		if !until.IsZero() && nw.Created.After(until) {

docs/api/version-history.md

@@ -17,6 +17,10 @@ keywords: "API, Docker, rcli, REST, documentation"
 
 [Docker Engine API v1.29](https://docs.docker.com/engine/api/v1.29/) documentation
 
+
+* `DELETE /networks/(name)` now allows to remove the ingress network, the one used to provide the routing-mesh.
+* `POST /networks/create` now supports creating the ingress network, by specifying an `Ingress` boolean field. As of now this is supported only when using the overlay network driver.
+* `GET /networks/(name)` now returns an `Ingress` field showing whether the network is the ingress one.
 * `GET /networks/` now supports a `scope` filter to filter networks based on the network mode (`swarm`, `global`, or `local`).
 
 ## v1.28 API changes

docs/reference/commandline/network_create.md

@@ -22,6 +22,7 @@ Create a network
 
 Options:
       --attachable           Enable manual container attachment
+      --ingress              Specify the network provides the routing-mesh
       --aux-address value    Auxiliary IPv4 or IPv6 addresses used by Network
                              driver (default map[])
   -d, --driver string        Driver to manage the Network (default "bridge")
@@ -195,6 +196,23 @@ connects a bridge network to it to provide external connectivity. If you want
 to create an externally isolated `overlay` network, you can specify the
 `--internal` option.
 
+### Network ingress mode
+
+You can create the network which will be used to provide the routing-mesh in the
+swarm cluster. You do so by specifying `--ingress` when creating the network. Only
+one ingress network can be created at the time. The network can be removed only
+if no services depend on it. Any option available when creating a overlay network
+is also available when creating the ingress network, besides the `--attachable` option.
+
+```bash
+$ docker network create -d overlay \
+  --subnet=10.11.0.0/16 \
+  --ingress \
+  --opt com.docker.network.mtu=9216 \
+  --opt encrypted=true \
+  my-ingress-network
+```
+
 ## Related commands
 
 * [network inspect](network_inspect.md)

integration-cli/docker_cli_swarm_test.go

@@ -10,6 +10,7 @@ import (
 	"net/http"
 	"net/http/httptest"
 	"os"
+	"os/exec"
 	"path/filepath"
 	"strings"
 	"time"
@@ -19,6 +20,7 @@ import (
 	"github.com/docker/docker/integration-cli/checker"
 	"github.com/docker/docker/integration-cli/cli"
 	"github.com/docker/docker/integration-cli/daemon"
+	"github.com/docker/docker/pkg/testutil"
 	icmd "github.com/docker/docker/pkg/testutil/cmd"
 	"github.com/docker/libnetwork/driverapi"
 	"github.com/docker/libnetwork/ipamapi"
@@ -413,14 +415,57 @@ func (s *DockerSwarmSuite) TestOverlayAttachableReleaseResourcesOnFailure(c *che
 	c.Assert(err, checker.IsNil, check.Commentf(out))
 }
 
-func (s *DockerSwarmSuite) TestSwarmRemoveInternalNetwork(c *check.C) {
+func (s *DockerSwarmSuite) TestSwarmIngressNetwork(c *check.C) {
 	d := s.AddDaemon(c, true, true)
 
-	name := "ingress"
-	out, err := d.Cmd("network", "rm", name)
+	// Ingress network can be removed
+	out, _, err := testutil.RunCommandPipelineWithOutput(
+		exec.Command("echo", "Y"),
+		exec.Command("docker", "-H", d.Sock(), "network", "rm", "ingress"),
+	)
+	c.Assert(err, checker.IsNil, check.Commentf(out))
+
+	// And recreated
+	out, err = d.Cmd("network", "create", "-d", "overlay", "--ingress", "new-ingress")
+	c.Assert(err, checker.IsNil, check.Commentf(out))
+
+	// But only one is allowed
+	out, err = d.Cmd("network", "create", "-d", "overlay", "--ingress", "another-ingress")
 	c.Assert(err, checker.NotNil)
-	c.Assert(strings.TrimSpace(out), checker.Contains, name)
-	c.Assert(strings.TrimSpace(out), checker.Contains, "is a pre-defined network and cannot be removed")
+	c.Assert(strings.TrimSpace(out), checker.Contains, "is already present")
+
+	// It cannot be removed if it is being used
+	out, err = d.Cmd("service", "create", "--name", "srv1", "-p", "9000:8000", "busybox", "top")
+	c.Assert(err, checker.IsNil, check.Commentf(out))
+	out, _, err = testutil.RunCommandPipelineWithOutput(
+		exec.Command("echo", "Y"),
+		exec.Command("docker", "-H", d.Sock(), "network", "rm", "new-ingress"),
+	)
+	c.Assert(err, checker.NotNil)
+	c.Assert(strings.TrimSpace(out), checker.Contains, "ingress network cannot be removed because service")
+
+	// But it can be removed once no more services depend on it
+	out, err = d.Cmd("service", "update", "--publish-rm", "9000:8000", "srv1")
+	c.Assert(err, checker.IsNil, check.Commentf(out))
+	out, _, err = testutil.RunCommandPipelineWithOutput(
+		exec.Command("echo", "Y"),
+		exec.Command("docker", "-H", d.Sock(), "network", "rm", "new-ingress"),
+	)
+	c.Assert(err, checker.IsNil, check.Commentf(out))
+
+	// A service which needs the ingress network cannot be created if no ingress is present
+	out, err = d.Cmd("service", "create", "--name", "srv2", "-p", "500:500", "busybox", "top")
+	c.Assert(err, checker.NotNil)
+	c.Assert(strings.TrimSpace(out), checker.Contains, "no ingress network is present")
+
+	// An existing service cannot be updated to use the ingress nw if the nw is not present
+	out, err = d.Cmd("service", "update", "--publish-add", "9000:8000", "srv1")
+	c.Assert(err, checker.NotNil)
+	c.Assert(strings.TrimSpace(out), checker.Contains, "no ingress network is present")
+
+	// But services which do not need routing mesh can be created regardless
+	out, err = d.Cmd("service", "create", "--name", "srv3", "--endpoint-mode", "dnsrr", "busybox", "top")
+	c.Assert(err, checker.IsNil, check.Commentf(out))
 }
 
 // Test case for #24108, also the case from:

man/src/network/create.md

@@ -117,3 +117,20 @@ By default, when you connect a container to an `overlay` network, Docker also
 connects a bridge network to it to provide external connectivity. If you want
 to create an externally isolated `overlay` network, you can specify the
 `--internal` option.
+
+### Network ingress mode
+
+You can create the network which will be used to provide the routing-mesh in the
+swarm cluster. You do so by specifying `--ingress` when creating the network. Only
+one ingress network can be created at the time. The network can be removed only
+if no services depend on it. Any option available when creating a overlay network
+is also available when creating the ingress network, besides the `--attachable` option.
+
+```bash
+$ docker network create -d overlay \
+  --subnet=10.11.0.0/16 \
+  --ingress \
+  --opt com.docker.network.mtu=9216 \
+  --opt encrypted=true \
+  my-ingress-network
+```

man/src/network/inspect.md

@@ -32,6 +32,7 @@ $ sudo docker network inspect bridge
             ]
         },
         "Internal": false,
+        "Ingress": false,
         "Containers": {
             "bda12f8922785d1f160be70736f26c1e331ab8aaf8ed8d56728508f2e2fd4727": {
                 "Name": "container2",
@@ -116,6 +117,7 @@ $ docker network inspect --verbose ov1
         },
         "Internal": false,
         "Attachable": false,
+        "Ingress": false,
         "Containers": {
             "020403bd88a15f60747fd25d1ad5fa1272eb740e8a97fc547d8ad07b2f721c5e": {
                 "Name": "s1.1.pjn2ik0sfgkfzed3h0s00gs9o",

runconfig/hostconfig_unix.go

@@ -19,7 +19,7 @@ func DefaultDaemonNetworkMode() container.NetworkMode {
 // IsPreDefinedNetwork indicates if a network is predefined by the daemon
 func IsPreDefinedNetwork(network string) bool {
 	n := container.NetworkMode(network)
-	return n.IsBridge() || n.IsHost() || n.IsNone() || n.IsDefault() || network == "ingress"
+	return n.IsBridge() || n.IsHost() || n.IsNone() || n.IsDefault()
 }
 
 // validateNetMode ensures that the various combinations of requested

vendor.conf

@@ -24,7 +24,7 @@ github.com/RackSec/srslog 456df3a81436d29ba874f3590eeeee25d666f8a5
 github.com/imdario/mergo 0.2.1
 
 #get libnetwork packages
-github.com/docker/libnetwork 4610dd67c7b9828bb4719d8aa2ac53a7f1f739d2
+github.com/docker/libnetwork b6cb1eee1e7fc27ee05f0eb830d3e60e67a88565
 github.com/docker/go-events 18b43f1bc85d9cdd42c05a6cd2d444c7a200a894
 github.com/armon/go-radix e39d623f12e8e41c7b5529e9a9dd67a1e2261f80
 github.com/armon/go-metrics eb0af217e5e9747e41dd5303755356b62d28e3ec
@@ -105,7 +105,7 @@ github.com/docker/containerd 422e31ce907fd9c3833a38d7b8fdd023e5a76e73
 github.com/tonistiigi/fifo 1405643975692217d6720f8b54aeee1bf2cd5cf4
 
 # cluster
-github.com/docker/swarmkit 0e2d9ebcea9d5bbd4a06b3b964fb96356801f880
+github.com/docker/swarmkit 9fdea50c14492b6e1f472813849794d36bfef217
 github.com/golang/mock bd3c8e81be01eef76d4b503f5e687d2d1354d2d9
 github.com/gogo/protobuf 8d70fb3182befc465c4a1eac8ad4d38ff49778e2
 github.com/cloudflare/cfssl 7fb22c8cba7ecaf98e4082d22d65800cf45e042a

vendor/github.com/docker/libnetwork/controller.go

@@ -682,6 +682,10 @@ func (c *controller) NewNetwork(networkType, name string, id string, options ...
 		return nil, err
 	}
 
+	if network.ingress && cap.DataScope != datastore.GlobalScope {
+		return nil, types.ForbiddenErrorf("Ingress network can only be global scope network")
+	}
+
 	if cap.DataScope == datastore.GlobalScope && !c.isDistributedControl() && !network.dynamic {
 		if c.isManager() {
 			// For non-distributed controlled environment, globalscoped non-dynamic networks are redirected to Manager
@@ -1161,15 +1165,29 @@ func (c *controller) clearIngress(clusterLeave bool) {
 	c.ingressSandbox = nil
 	c.Unlock()
 
+	var n *network
 	if ingressSandbox != nil {
+		for _, ep := range ingressSandbox.getConnectedEndpoints() {
+			if nw := ep.getNetwork(); nw.ingress {
+				n = nw
+				break
+			}
+		}
 		if err := ingressSandbox.Delete(); err != nil {
 			logrus.Warnf("Could not delete ingress sandbox while leaving: %v", err)
 		}
 	}
 
-	n, err := c.NetworkByName("ingress")
-	if err != nil && clusterLeave {
-		logrus.Warnf("Could not find ingress network while leaving: %v", err)
+	if n == nil {
+		for _, nw := range c.Networks() {
+			if nw.Info().Ingress() {
+				n = nw.(*network)
+				break
+			}
+		}
+	}
+	if n == nil && clusterLeave {
+		logrus.Warnf("Could not find ingress network while leaving")
 	}
 
 	if n != nil {

vendor/github.com/docker/libnetwork/drivers/bridge/bridge.go

@@ -28,11 +28,11 @@ import (
 )
 
 const (
-	networkType             = "bridge"
-	vethPrefix              = "veth"
-	vethLen                 = 7
-	containerVethPrefix     = "eth"
-	maxAllocatePortAttempts = 10
+	networkType                = "bridge"
+	vethPrefix                 = "veth"
+	vethLen                    = 7
+	defaultContainerVethPrefix = "eth"
+	maxAllocatePortAttempts    = 10
 )
 
 const (
@@ -55,14 +55,15 @@ type configuration struct {
 
 // networkConfiguration for network specific configuration
 type networkConfiguration struct {
-	ID                 string
-	BridgeName         string
-	EnableIPv6         bool
-	EnableIPMasquerade bool
-	EnableICC          bool
-	Mtu                int
-	DefaultBindingIP   net.IP
-	DefaultBridge      bool
+	ID                   string
+	BridgeName           string
+	EnableIPv6           bool
+	EnableIPMasquerade   bool
+	EnableICC            bool
+	Mtu                  int
+	DefaultBindingIP     net.IP
+	DefaultBridge        bool
+	ContainerIfacePrefix string
 	// Internal fields set after ipam data parsing
 	AddressIPv4        *net.IPNet
 	AddressIPv6        *net.IPNet
@@ -239,6 +240,8 @@ func (c *networkConfiguration) fromLabels(labels map[string]string) error {
 			if c.DefaultBindingIP = net.ParseIP(value); c.DefaultBindingIP == nil {
 				return parseErr(label, value, "nil ip")
 			}
+		case netlabel.ContainerIfacePrefix:
+			c.ContainerIfacePrefix = value
 		}
 	}
 
@@ -1221,6 +1224,10 @@ func (d *driver) Join(nid, eid string, sboxKey string, jinfo driverapi.JoinInfo,
 	}
 
 	iNames := jinfo.InterfaceName()
+	containerVethPrefix := defaultContainerVethPrefix
+	if network.config.ContainerIfacePrefix != "" {
+		containerVethPrefix = network.config.ContainerIfacePrefix
+	}
 	err = iNames.SetNames(endpoint.srcName, containerVethPrefix)
 	if err != nil {
 		return err

vendor/github.com/docker/libnetwork/drivers/bridge/bridge_store.go

@@ -143,6 +143,7 @@ func (ncfg *networkConfiguration) MarshalJSON() ([]byte, error) {
 	nMap["DefaultBindingIP"] = ncfg.DefaultBindingIP.String()
 	nMap["DefaultGatewayIPv4"] = ncfg.DefaultGatewayIPv4.String()
 	nMap["DefaultGatewayIPv6"] = ncfg.DefaultGatewayIPv6.String()
+	nMap["ContainerIfacePrefix"] = ncfg.ContainerIfacePrefix
 	nMap["BridgeIfaceCreator"] = ncfg.BridgeIfaceCreator
 
 	if ncfg.AddressIPv4 != nil {
@@ -178,6 +179,10 @@ func (ncfg *networkConfiguration) UnmarshalJSON(b []byte) error {
 		}
 	}
 
+	if v, ok := nMap["ContainerIfacePrefix"]; ok {
+		ncfg.ContainerIfacePrefix = v.(string)
+	}
+
 	ncfg.DefaultBridge = nMap["DefaultBridge"].(bool)
 	ncfg.DefaultBindingIP = net.ParseIP(nMap["DefaultBindingIP"].(string))
 	ncfg.DefaultGatewayIPv4 = net.ParseIP(nMap["DefaultGatewayIPv4"].(string))

vendor/github.com/docker/libnetwork/netlabel/labels.go

@@ -50,6 +50,9 @@ const (
 
 	// Internal constant represents that the network is internal which disables default gateway service
 	Internal = Prefix + ".internal"
+
+	// ContainerIfacePrefix can be used to override the interface prefix used inside the container
+	ContainerIfacePrefix = Prefix + ".container_iface_prefix"
 )
 
 var (

vendor/github.com/docker/libnetwork/network.go

@@ -66,6 +66,7 @@ type NetworkInfo interface {
 	IPv6Enabled() bool
 	Internal() bool
 	Attachable() bool
+	Ingress() bool
 	Labels() map[string]string
 	Dynamic() bool
 	Created() time.Time
@@ -615,9 +616,9 @@ func NetworkOptionGeneric(generic map[string]interface{}) NetworkOption {
 
 // NetworkOptionIngress returns an option setter to indicate if a network is
 // an ingress network.
-func NetworkOptionIngress() NetworkOption {
+func NetworkOptionIngress(ingress bool) NetworkOption {
 	return func(n *network) {
-		n.ingress = true
+		n.ingress = ingress
 	}
 }
 
@@ -1589,6 +1590,13 @@ func (n *network) Attachable() bool {
 	return n.attachable
 }
 
+func (n *network) Ingress() bool {
+	n.Lock()
+	defer n.Unlock()
+
+	return n.ingress
+}
+
 func (n *network) Dynamic() bool {
 	n.Lock()
 	defer n.Unlock()

vendor/github.com/docker/libnetwork/osl/interface_linux.go

@@ -241,8 +241,8 @@ func (n *networkNamespace) AddInterface(srcName, dstPrefix string, options ...If
 	if n.isDefault {
 		i.dstName = i.srcName
 	} else {
-		i.dstName = fmt.Sprintf("%s%d", i.dstName, n.nextIfIndex)
-		n.nextIfIndex++
+		i.dstName = fmt.Sprintf("%s%d", dstPrefix, n.nextIfIndex[dstPrefix])
+		n.nextIfIndex[dstPrefix]++
 	}
 
 	path := n.path

vendor/github.com/docker/libnetwork/osl/namespace_linux.go

@@ -48,7 +48,7 @@ type networkNamespace struct {
 	gwv6         net.IP
 	staticRoutes []*types.StaticRoute
 	neighbors    []*neigh
-	nextIfIndex  int
+	nextIfIndex  map[string]int
 	isDefault    bool
 	nlHandle     *netlink.Handle
 	loV6Enabled  bool
@@ -203,7 +203,7 @@ func NewSandbox(key string, osCreate, isRestore bool) (Sandbox, error) {
 		once.Do(createBasePath)
 	}
 
-	n := &networkNamespace{path: key, isDefault: !osCreate}
+	n := &networkNamespace{path: key, isDefault: !osCreate, nextIfIndex: make(map[string]int)}
 
 	sboxNs, err := netns.GetFromPath(n.path)
 	if err != nil {
@@ -256,7 +256,7 @@ func GetSandboxForExternalKey(basePath string, key string) (Sandbox, error) {
 	if err := mountNetworkNamespace(basePath, key); err != nil {
 		return nil, err
 	}
-	n := &networkNamespace{path: key}
+	n := &networkNamespace{path: key, nextIfIndex: make(map[string]int)}
 
 	sboxNs, err := netns.GetFromPath(n.path)
 	if err != nil {
@@ -495,8 +495,8 @@ func (n *networkNamespace) Restore(ifsopt map[string][]IfaceOption, routes []*ty
 			}
 			index++
 			n.Lock()
-			if index > n.nextIfIndex {
-				n.nextIfIndex = index
+			if index > n.nextIfIndex[dstPrefix] {
+				n.nextIfIndex[dstPrefix] = index
 			}
 			n.iFaces = append(n.iFaces, i)
 			n.Unlock()

vendor/github.com/docker/swarmkit/api/specs.pb.go

@@ -591,6 +591,11 @@ type NetworkSpec struct {
 	// enabled(default case) no manual attachment to this network
 	// can happen.
 	Attachable bool `protobuf:"varint,6,opt,name=attachable,proto3" json:"attachable,omitempty"`
+	// Ingress indicates this network will provide the routing-mesh.
+	// In older versions, the network providing the routing mesh was
+	// swarm internally created only and it was identified by the name
+	// "ingress" and the label "com.docker.swarm.internal": "true".
+	Ingress bool `protobuf:"varint,7,opt,name=ingress,proto3" json:"ingress,omitempty"`
 }
 
 func (m *NetworkSpec) Reset()                    { *m = NetworkSpec{} }
@@ -1795,6 +1800,16 @@ func (m *NetworkSpec) MarshalTo(dAtA []byte) (int, error) {
 		}
 		i++
 	}
+	if m.Ingress {
+		dAtA[i] = 0x38
+		i++
+		if m.Ingress {
+			dAtA[i] = 1
+		} else {
+			dAtA[i] = 0
+		}
+		i++
+	}
 	return i, nil
 }
 
@@ -2255,6 +2270,9 @@ func (m *NetworkSpec) Size() (n int) {
 	if m.Attachable {
 		n += 2
 	}
+	if m.Ingress {
+		n += 2
+	}
 	return n
 }
 
@@ -2502,6 +2520,7 @@ func (this *NetworkSpec) String() string {
 		`Internal:` + fmt.Sprintf("%v", this.Internal) + `,`,
 		`IPAM:` + strings.Replace(fmt.Sprintf("%v", this.IPAM), "IPAMOptions", "IPAMOptions", 1) + `,`,
 		`Attachable:` + fmt.Sprintf("%v", this.Attachable) + `,`,
+		`Ingress:` + fmt.Sprintf("%v", this.Ingress) + `,`,
 		`}`,
 	}, "")
 	return s
@@ -4688,6 +4707,26 @@ func (m *NetworkSpec) Unmarshal(dAtA []byte) error {
 				}
 			}
 			m.Attachable = bool(v != 0)
+		case 7:
+			if wireType != 0 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Ingress", wireType)
+			}
+			var v int
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowSpecs
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				v |= (int(b) & 0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			m.Ingress = bool(v != 0)
 		default:
 			iNdEx = preIndex
 			skippy, err := skipSpecs(dAtA[iNdEx:])
@@ -5218,112 +5257,113 @@ var (
 func init() { proto.RegisterFile("specs.proto", fileDescriptorSpecs) }
 
 var fileDescriptorSpecs = []byte{
-	// 1707 bytes of a gzipped FileDescriptorProto
+	// 1717 bytes of a gzipped FileDescriptorProto
 	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xac, 0x57, 0x41, 0x73, 0x1b, 0xb7,
-	0x15, 0x26, 0x25, 0x8a, 0x5a, 0xbe, 0xa5, 0x6c, 0x1a, 0x75, 0xd2, 0x35, 0xdd, 0x90, 0x34, 0xe3,
+	0x15, 0x16, 0x25, 0x8a, 0x5a, 0xbe, 0xa5, 0x6c, 0x1a, 0x75, 0xd2, 0x35, 0xdd, 0x50, 0x34, 0xe3,
 	0xa6, 0x4a, 0x33, 0xa5, 0xa6, 0x6a, 0x27, 0x75, 0xea, 0x66, 0x5a, 0x52, 0x64, 0x65, 0x55, 0x95,
 	0xcc, 0x01, 0x15, 0x77, 0x7c, 0xe2, 0x80, 0xbb, 0x10, 0xb9, 0xa3, 0xe5, 0x62, 0x0b, 0x60, 0x99,
-	0xe1, 0xad, 0xc7, 0x8c, 0x0f, 0x3d, 0xf5, 0xaa, 0xe9, 0xa1, 0xbf, 0xa1, 0xff, 0xc1, 0xc7, 0x1e,
-	0x7b, 0xd2, 0x34, 0xfc, 0x0b, 0xfd, 0x01, 0xed, 0x00, 0x0b, 0x92, 0xcb, 0x64, 0x15, 0x7b, 0x26,
-	0xbe, 0xe1, 0xbd, 0xfd, 0xbe, 0x07, 0xe0, 0xe1, 0xc3, 0xc3, 0x5b, 0xb0, 0x45, 0x44, 0x5d, 0xd1,
-	0x8a, 0x38, 0x93, 0x0c, 0x21, 0x8f, 0xb9, 0x57, 0x94, 0xb7, 0xc4, 0x97, 0x84, 0x4f, 0xaf, 0x7c,
-	0xd9, 0x9a, 0xfd, 0xbc, 0x6a, 0xcb, 0x79, 0x44, 0x0d, 0xa0, 0x7a, 0x7f, 0xcc, 0xc6, 0x4c, 0x0f,
-	0x0f, 0xd4, 0xc8, 0x78, 0x6b, 0x63, 0xc6, 0xc6, 0x01, 0x3d, 0xd0, 0xd6, 0x28, 0xbe, 0x3c, 0xf0,
-	0x62, 0x4e, 0xa4, 0xcf, 0xc2, 0xe4, 0x7b, 0xf3, 0xba, 0x00, 0xd6, 0x39, 0xf3, 0xe8, 0x20, 0xa2,
-	0x2e, 0x3a, 0x06, 0x9b, 0x84, 0x21, 0x93, 0x1a, 0x20, 0x9c, 0x7c, 0x23, 0xbf, 0x6f, 0x1f, 0xd6,
-	0x5b, 0xdf, 0x9e, 0xb9, 0xd5, 0x5e, 0xc3, 0x3a, 0x85, 0xd7, 0x37, 0xf5, 0x1c, 0x4e, 0x33, 0xd1,
-	0x6f, 0xa1, 0xec, 0x51, 0xe1, 0x73, 0xea, 0x0d, 0x39, 0x0b, 0xa8, 0xb3, 0xd5, 0xc8, 0xef, 0xdf,
-	0x39, 0xfc, 0x51, 0x56, 0x24, 0x35, 0x39, 0x66, 0x01, 0xc5, 0xb6, 0x61, 0x28, 0x03, 0x1d, 0x03,
-	0x4c, 0xe9, 0x74, 0x44, 0xb9, 0x98, 0xf8, 0x91, 0xb3, 0xad, 0xe9, 0x3f, 0xb9, 0x8d, 0xae, 0xd6,
-	0xde, 0x3a, 0x5b, 0xc1, 0x71, 0x8a, 0x8a, 0xce, 0xa0, 0x4c, 0x66, 0xc4, 0x0f, 0xc8, 0xc8, 0x0f,
-	0x7c, 0x39, 0x77, 0x0a, 0x3a, 0xd4, 0xc7, 0xdf, 0x19, 0xaa, 0x9d, 0x22, 0xe0, 0x0d, 0x7a, 0xd3,
-	0x03, 0x58, 0x4f, 0x84, 0x3e, 0x82, 0xdd, 0x7e, 0xef, 0xbc, 0x7b, 0x72, 0x7e, 0x5c, 0xc9, 0x55,
-	0x1f, 0xbc, 0xba, 0x6e, 0xbc, 0xa7, 0x62, 0xac, 0x01, 0x7d, 0x1a, 0x7a, 0x7e, 0x38, 0x46, 0xfb,
-	0x60, 0xb5, 0x8f, 0x8e, 0x7a, 0xfd, 0x8b, 0x5e, 0xb7, 0x92, 0xaf, 0x56, 0x5f, 0x5d, 0x37, 0xde,
-	0xdf, 0x04, 0xb6, 0x5d, 0x97, 0x46, 0x92, 0x7a, 0xd5, 0xc2, 0x57, 0xff, 0xa8, 0xe5, 0x9a, 0x5f,
-	0xe5, 0xa1, 0x9c, 0x5e, 0x04, 0xfa, 0x08, 0x8a, 0xed, 0xa3, 0x8b, 0x93, 0x17, 0xbd, 0x4a, 0x6e,
-	0x4d, 0x4f, 0x23, 0xda, 0xae, 0xf4, 0x67, 0x14, 0x3d, 0x86, 0x9d, 0x7e, 0xfb, 0x8b, 0x41, 0xaf,
-	0x92, 0x5f, 0x2f, 0x27, 0x0d, 0xeb, 0x93, 0x58, 0x68, 0x54, 0x17, 0xb7, 0x4f, 0xce, 0x2b, 0x5b,
-	0xd9, 0xa8, 0x2e, 0x27, 0x7e, 0x68, 0x96, 0xf2, 0xf7, 0x02, 0xd8, 0x03, 0xca, 0x67, 0xbe, 0xfb,
-	0x8e, 0x25, 0xf2, 0x29, 0x14, 0x24, 0x11, 0x57, 0x5a, 0x1a, 0x76, 0xb6, 0x34, 0x2e, 0x88, 0xb8,
-	0x52, 0x93, 0x1a, 0xba, 0xc6, 0x2b, 0x65, 0x70, 0x1a, 0x05, 0xbe, 0x4b, 0x24, 0xf5, 0xb4, 0x32,
-	0xec, 0xc3, 0x1f, 0x67, 0xb1, 0xf1, 0x0a, 0x65, 0xd6, 0xff, 0x2c, 0x87, 0x53, 0x54, 0xf4, 0x14,
-	0x8a, 0xe3, 0x80, 0x8d, 0x48, 0xa0, 0x35, 0x61, 0x1f, 0x3e, 0xca, 0x0a, 0x72, 0xac, 0x11, 0xeb,
-	0x00, 0x86, 0x82, 0x9e, 0x40, 0x31, 0x8e, 0x3c, 0x22, 0xa9, 0x53, 0xd4, 0xe4, 0x46, 0x16, 0xf9,
-	0x0b, 0x8d, 0x38, 0x62, 0xe1, 0xa5, 0x3f, 0xc6, 0x06, 0x8f, 0x4e, 0xc1, 0x0a, 0xa9, 0xfc, 0x92,
-	0xf1, 0x2b, 0xe1, 0xec, 0x36, 0xb6, 0xf7, 0xed, 0xc3, 0x4f, 0x32, 0xc5, 0x98, 0x60, 0xda, 0x52,
-	0x12, 0x77, 0x32, 0xa5, 0xa1, 0x4c, 0xc2, 0x74, 0xb6, 0x9c, 0x3c, 0x5e, 0x05, 0x40, 0xbf, 0x01,
-	0x8b, 0x86, 0x5e, 0xc4, 0xfc, 0x50, 0x3a, 0xd6, 0xed, 0x0b, 0xe9, 0x19, 0x8c, 0x4a, 0x26, 0x5e,
-	0x31, 0x14, 0x9b, 0xb3, 0x20, 0x18, 0x11, 0xf7, 0xca, 0x29, 0xbd, 0xe5, 0x36, 0x56, 0x8c, 0x4e,
-	0x11, 0x0a, 0x53, 0xe6, 0xd1, 0xe6, 0x01, 0xdc, 0xfb, 0x56, 0xaa, 0x51, 0x15, 0x2c, 0x93, 0xea,
-	0x44, 0x23, 0x05, 0xbc, 0xb2, 0x9b, 0x77, 0x61, 0x6f, 0x23, 0xad, 0xcd, 0xbf, 0x16, 0xc0, 0x5a,
-	0x9e, 0x35, 0x6a, 0x43, 0xc9, 0x65, 0xa1, 0x24, 0x7e, 0x48, 0xb9, 0x91, 0x57, 0xe6, 0xc9, 0x1c,
-	0x2d, 0x41, 0x8a, 0xf5, 0x2c, 0x87, 0xd7, 0x2c, 0xf4, 0x7b, 0x28, 0x71, 0x2a, 0x58, 0xcc, 0x5d,
-	0x2a, 0x8c, 0xbe, 0xf6, 0xb3, 0x15, 0x92, 0x80, 0x30, 0xfd, 0x73, 0xec, 0x73, 0xaa, 0xb2, 0x2c,
-	0xf0, 0x9a, 0x8a, 0x9e, 0xc2, 0x2e, 0xa7, 0x42, 0x12, 0x2e, 0xbf, 0x4b, 0x22, 0x38, 0x81, 0xf4,
-	0x59, 0xe0, 0xbb, 0x73, 0xbc, 0x64, 0xa0, 0xa7, 0x50, 0x8a, 0x02, 0xe2, 0xea, 0xa8, 0xce, 0x8e,
-	0xa6, 0x7f, 0x90, 0x45, 0xef, 0x2f, 0x41, 0x78, 0x8d, 0x47, 0x9f, 0x01, 0x04, 0x6c, 0x3c, 0xf4,
-	0xb8, 0x3f, 0xa3, 0xdc, 0x48, 0xac, 0x9a, 0xc5, 0xee, 0x6a, 0x04, 0x2e, 0x05, 0x6c, 0x9c, 0x0c,
-	0xd1, 0xf1, 0xf7, 0xd2, 0x57, 0x4a, 0x5b, 0xa7, 0x00, 0x64, 0xf5, 0xd5, 0xa8, 0xeb, 0xe3, 0xb7,
-	0x0a, 0x65, 0x4e, 0x24, 0x45, 0x47, 0x8f, 0xa0, 0x7c, 0xc9, 0xb8, 0x4b, 0x87, 0xe6, 0xd6, 0x94,
-	0xb4, 0x26, 0x6c, 0xed, 0x4b, 0xf4, 0xd5, 0x29, 0xc1, 0x2e, 0x8f, 0x43, 0xe9, 0x4f, 0x69, 0xf3,
-	0x14, 0xde, 0xcb, 0x0c, 0x8a, 0x0e, 0xa1, 0xbc, 0x3a, 0xe6, 0xa1, 0xef, 0x69, 0x7d, 0x94, 0x3a,
-	0x77, 0x17, 0x37, 0x75, 0x7b, 0xa5, 0x87, 0x93, 0x2e, 0xb6, 0x57, 0xa0, 0x13, 0xaf, 0xf9, 0x37,
-	0x0b, 0xf6, 0x36, 0xc4, 0x82, 0xee, 0xc3, 0x8e, 0x3f, 0x25, 0x63, 0x9a, 0xd0, 0x71, 0x62, 0xa0,
-	0x1e, 0x14, 0x03, 0x32, 0xa2, 0x81, 0x92, 0x8c, 0x4a, 0xdb, 0xcf, 0xde, 0xa8, 0xba, 0xd6, 0x1f,
-	0x35, 0xbe, 0x17, 0x4a, 0x3e, 0xc7, 0x86, 0x8c, 0x1c, 0xd8, 0x75, 0xd9, 0x74, 0x4a, 0x42, 0x55,
-	0x9c, 0xb6, 0xf7, 0x4b, 0x78, 0x69, 0x22, 0x04, 0x05, 0xc2, 0xc7, 0xc2, 0x29, 0x68, 0xb7, 0x1e,
-	0xa3, 0x0a, 0x6c, 0xd3, 0x70, 0xe6, 0xec, 0x68, 0x97, 0x1a, 0x2a, 0x8f, 0xe7, 0x27, 0x67, 0x5e,
-	0xc2, 0x6a, 0xa8, 0x78, 0xb1, 0xa0, 0xdc, 0xd9, 0xd5, 0x2e, 0x3d, 0x46, 0xbf, 0x82, 0xe2, 0x94,
-	0xc5, 0xa1, 0x14, 0x8e, 0xa5, 0x17, 0xfb, 0x20, 0x6b, 0xb1, 0x67, 0x0a, 0x61, 0x8a, 0xa7, 0x81,
-	0xa3, 0x1e, 0xdc, 0x13, 0x92, 0x45, 0xc3, 0x31, 0x27, 0x2e, 0x1d, 0x46, 0x94, 0xfb, 0xcc, 0x33,
-	0x97, 0xff, 0x41, 0x2b, 0xe9, 0x15, 0x5a, 0xcb, 0x5e, 0xa1, 0xd5, 0x35, 0xbd, 0x02, 0xbe, 0xab,
-	0x38, 0xc7, 0x8a, 0xd2, 0xd7, 0x0c, 0xd4, 0x87, 0x72, 0x14, 0x07, 0xc1, 0x90, 0x45, 0xc9, 0x3b,
-	0x00, 0x3a, 0xc2, 0x5b, 0xa4, 0xac, 0x1f, 0x07, 0xc1, 0xf3, 0x84, 0x84, 0xed, 0x68, 0x6d, 0xa0,
-	0xf7, 0xa1, 0x38, 0xe6, 0x2c, 0x8e, 0x84, 0x63, 0xeb, 0x64, 0x18, 0x0b, 0x7d, 0x0e, 0xbb, 0x82,
-	0xba, 0x9c, 0x4a, 0xe1, 0x94, 0xf5, 0x56, 0x3f, 0xcc, 0x9a, 0x64, 0xa0, 0x21, 0x98, 0x5e, 0x52,
-	0x4e, 0x43, 0x97, 0xe2, 0x25, 0x07, 0x3d, 0x80, 0x6d, 0x29, 0xe7, 0xce, 0x5e, 0x23, 0xbf, 0x6f,
-	0x75, 0x76, 0x17, 0x37, 0xf5, 0xed, 0x8b, 0x8b, 0x97, 0x58, 0xf9, 0x54, 0x8d, 0x9a, 0x30, 0x21,
-	0x43, 0x32, 0xa5, 0xce, 0x1d, 0x9d, 0xdb, 0x95, 0x8d, 0x5e, 0x02, 0x78, 0xa1, 0x18, 0xba, 0xfa,
-	0x52, 0x38, 0x77, 0xf5, 0xee, 0x3e, 0x79, 0xf3, 0xee, 0xba, 0xe7, 0x03, 0x53, 0xa7, 0xf7, 0x16,
-	0x37, 0xf5, 0xd2, 0xca, 0xc4, 0x25, 0x2f, 0x14, 0xc9, 0x10, 0x75, 0xc0, 0x9e, 0x50, 0x12, 0xc8,
-	0x89, 0x3b, 0xa1, 0xee, 0x95, 0x53, 0xb9, 0xbd, 0xf0, 0x3e, 0xd3, 0x30, 0x13, 0x21, 0x4d, 0x52,
-	0x0a, 0x56, 0x4b, 0x15, 0xce, 0x3d, 0x9d, 0xab, 0xc4, 0x40, 0x1f, 0x00, 0xb0, 0x88, 0x86, 0x43,
-	0x21, 0x3d, 0x3f, 0x74, 0x90, 0xda, 0x32, 0x2e, 0x29, 0xcf, 0x40, 0x39, 0xd0, 0x43, 0x55, 0x16,
-	0x89, 0x37, 0x64, 0x61, 0x30, 0x77, 0x7e, 0xa0, 0xbf, 0x5a, 0xca, 0xf1, 0x3c, 0x0c, 0xe6, 0xa8,
-	0x0e, 0xb6, 0xd6, 0x85, 0xf0, 0xc7, 0x21, 0x09, 0x9c, 0xfb, 0x3a, 0x1f, 0xa0, 0x5c, 0x03, 0xed,
-	0xa9, 0x7e, 0x06, 0x76, 0x4a, 0xee, 0x4a, 0xa6, 0x57, 0x74, 0x6e, 0x6e, 0x90, 0x1a, 0xaa, 0x35,
-	0xcd, 0x48, 0x10, 0x27, 0xcd, 0x5e, 0x09, 0x27, 0xc6, 0xaf, 0xb7, 0x9e, 0xe4, 0xab, 0x87, 0x60,
-	0xa7, 0x8e, 0x1d, 0x7d, 0x08, 0x7b, 0x9c, 0x8e, 0x7d, 0x21, 0xf9, 0x7c, 0x48, 0x62, 0x39, 0x71,
-	0x7e, 0xa7, 0x09, 0xe5, 0xa5, 0xb3, 0x1d, 0xcb, 0x49, 0x75, 0x08, 0xeb, 0xec, 0xa1, 0x06, 0xd8,
-	0xea, 0x54, 0x04, 0xe5, 0x33, 0xca, 0xd5, 0x83, 0xa2, 0x36, 0x9d, 0x76, 0x29, 0xf5, 0x08, 0x4a,
-	0xb8, 0x3b, 0xd1, 0x97, 0xb7, 0x84, 0x8d, 0xa5, 0x6e, 0xe3, 0x52, 0xa2, 0xe6, 0x36, 0x1a, 0xb3,
-	0xf9, 0xdf, 0x3c, 0x94, 0xd3, 0xef, 0x22, 0x3a, 0x4a, 0xde, 0x33, 0xbd, 0xa5, 0x3b, 0x87, 0x07,
-	0x6f, 0x7a, 0x47, 0xf5, 0xeb, 0x11, 0xc4, 0x2a, 0xd8, 0x99, 0x6a, 0x61, 0x35, 0x19, 0xfd, 0x12,
-	0x76, 0x22, 0xc6, 0xe5, 0xb2, 0x86, 0xd4, 0x32, 0x2b, 0x3e, 0xe3, 0xcb, 0x6a, 0x9b, 0x80, 0x9b,
-	0x13, 0xb8, 0xb3, 0x19, 0x0d, 0x3d, 0x86, 0xed, 0x17, 0x27, 0xfd, 0x4a, 0xae, 0xfa, 0xf0, 0xd5,
-	0x75, 0xe3, 0x87, 0x9b, 0x1f, 0x5f, 0xf8, 0x5c, 0xc6, 0x24, 0x38, 0xe9, 0xa3, 0x9f, 0xc2, 0x4e,
-	0xf7, 0x7c, 0x80, 0x71, 0x25, 0x5f, 0xad, 0xbf, 0xba, 0x6e, 0x3c, 0xdc, 0xc4, 0xa9, 0x4f, 0x2c,
-	0x0e, 0x3d, 0xcc, 0x46, 0xab, 0x76, 0xee, 0x9f, 0x5b, 0x60, 0x9b, 0xd2, 0xfa, 0xae, 0x3b, 0xfe,
-	0xbd, 0xe4, 0xb5, 0x5a, 0xde, 0x99, 0xad, 0x37, 0x3e, 0x5a, 0xe5, 0x84, 0x60, 0xce, 0xf8, 0x11,
-	0x94, 0xfd, 0x68, 0xf6, 0xe9, 0x90, 0x86, 0x64, 0x14, 0x98, 0xce, 0xce, 0xc2, 0xb6, 0xf2, 0xf5,
-	0x12, 0x97, 0xba, 0xb0, 0x7e, 0x28, 0x29, 0x0f, 0x4d, 0xcf, 0x66, 0xe1, 0x95, 0x8d, 0x3e, 0x87,
-	0x82, 0x1f, 0x91, 0xa9, 0x79, 0x69, 0x33, 0x77, 0x70, 0xd2, 0x6f, 0x9f, 0x19, 0x0d, 0x76, 0xac,
-	0xc5, 0x4d, 0xbd, 0xa0, 0x1c, 0x58, 0xd3, 0x50, 0x6d, 0xf9, 0xd8, 0xa9, 0x99, 0x74, 0xf1, 0xb5,
-	0x70, 0xca, 0xd3, 0xfc, 0x5f, 0x01, 0xec, 0xa3, 0x20, 0x16, 0xd2, 0x3c, 0x21, 0xef, 0x2c, 0x6f,
-	0x2f, 0xe1, 0x1e, 0xd1, 0xcd, 0x3f, 0x09, 0x55, 0x3d, 0xd6, 0x4d, 0x84, 0xc9, 0xdd, 0xe3, 0xcc,
-	0x70, 0x2b, 0x70, 0xd2, 0x70, 0x74, 0x8a, 0x2a, 0xa6, 0x93, 0xc7, 0x15, 0xf2, 0x8d, 0x2f, 0x68,
-	0x00, 0x7b, 0x8c, 0xbb, 0x13, 0x2a, 0x64, 0x52, 0xc5, 0x4d, 0xb3, 0x9c, 0xf9, 0x1b, 0xf5, 0x3c,
-	0x0d, 0x34, 0x25, 0x2c, 0x59, 0xed, 0x66, 0x0c, 0xf4, 0x04, 0x0a, 0x9c, 0x5c, 0x2e, 0x1b, 0xa2,
-	0x4c, 0x7d, 0x63, 0x72, 0x29, 0x37, 0x42, 0x68, 0x06, 0xfa, 0x03, 0x80, 0xe7, 0x8b, 0x88, 0x48,
-	0x77, 0x42, 0xb9, 0x39, 0xa7, 0xcc, 0x2d, 0x76, 0x57, 0xa8, 0x8d, 0x28, 0x29, 0x36, 0x3a, 0x85,
-	0x92, 0x4b, 0x96, 0x4a, 0x2b, 0xde, 0xfe, 0x07, 0x71, 0xd4, 0x36, 0x21, 0x2a, 0x2a, 0xc4, 0xe2,
-	0xa6, 0x6e, 0x2d, 0x3d, 0xd8, 0x72, 0x89, 0x51, 0xde, 0x29, 0xec, 0xa9, 0x3f, 0x8b, 0xa1, 0x47,
-	0x2f, 0x49, 0x1c, 0x48, 0xa1, 0x1f, 0xda, 0x5b, 0x4a, 0xb2, 0x6a, 0x53, 0xbb, 0x06, 0x67, 0xd6,
-	0x55, 0x96, 0x29, 0x1f, 0xfa, 0x13, 0xdc, 0xa3, 0xa1, 0xcb, 0xe7, 0x5a, 0x67, 0xcb, 0x15, 0x5a,
-	0xb7, 0x6f, 0xb6, 0xb7, 0x02, 0x6f, 0x6c, 0xb6, 0x42, 0xbf, 0xe1, 0x6f, 0xfa, 0x00, 0xc9, 0x23,
-	0xf7, 0x6e, 0xf5, 0x87, 0xa0, 0xe0, 0x11, 0x49, 0xb4, 0xe4, 0xca, 0x58, 0x8f, 0x3b, 0xce, 0xeb,
-	0xaf, 0x6b, 0xb9, 0x7f, 0x7f, 0x5d, 0xcb, 0xfd, 0x65, 0x51, 0xcb, 0xbf, 0x5e, 0xd4, 0xf2, 0xff,
-	0x5a, 0xd4, 0xf2, 0xff, 0x59, 0xd4, 0xf2, 0xa3, 0xa2, 0x6e, 0x0d, 0x7e, 0xf1, 0xff, 0x00, 0x00,
-	0x00, 0xff, 0xff, 0xed, 0xbe, 0x26, 0xe6, 0x9a, 0x10, 0x00, 0x00,
+	0xe1, 0xad, 0xc7, 0x8c, 0x0f, 0x3d, 0xf5, 0xaa, 0xe9, 0xa1, 0x7f, 0xc6, 0xb7, 0xf6, 0xd8, 0x93,
+	0xa6, 0xe1, 0x5f, 0xe8, 0x0f, 0x68, 0x07, 0x58, 0x2c, 0xb9, 0x4c, 0x56, 0xb1, 0x67, 0xe2, 0x1b,
+	0xde, 0xdb, 0xef, 0x7b, 0x00, 0x1e, 0x3e, 0x3c, 0xbc, 0x05, 0x5b, 0x44, 0xd4, 0x15, 0xad, 0x88,
+	0x33, 0xc9, 0x10, 0xf2, 0x98, 0x7b, 0x45, 0x79, 0x4b, 0x7c, 0x49, 0xf8, 0xf4, 0xca, 0x97, 0xad,
+	0xd9, 0xcf, 0x6b, 0xb6, 0x9c, 0x47, 0xd4, 0x00, 0x6a, 0xf7, 0xc7, 0x6c, 0xcc, 0xf4, 0xf0, 0x40,
+	0x8d, 0x8c, 0xb7, 0x3e, 0x66, 0x6c, 0x1c, 0xd0, 0x03, 0x6d, 0x8d, 0xe2, 0xcb, 0x03, 0x2f, 0xe6,
+	0x44, 0xfa, 0x2c, 0x4c, 0xbe, 0x37, 0xaf, 0x8b, 0x60, 0x9d, 0x33, 0x8f, 0x0e, 0x22, 0xea, 0xa2,
+	0x63, 0xb0, 0x49, 0x18, 0x32, 0xa9, 0x01, 0xc2, 0x29, 0x34, 0x0a, 0xfb, 0xf6, 0xe1, 0x5e, 0xeb,
+	0xdb, 0x33, 0xb7, 0xda, 0x2b, 0x58, 0xa7, 0xf8, 0xfa, 0x66, 0x6f, 0x03, 0x67, 0x99, 0xe8, 0xb7,
+	0x50, 0xf1, 0xa8, 0xf0, 0x39, 0xf5, 0x86, 0x9c, 0x05, 0xd4, 0xd9, 0x6c, 0x14, 0xf6, 0xef, 0x1c,
+	0xfe, 0x28, 0x2f, 0x92, 0x9a, 0x1c, 0xb3, 0x80, 0x62, 0xdb, 0x30, 0x94, 0x81, 0x8e, 0x01, 0xa6,
+	0x74, 0x3a, 0xa2, 0x5c, 0x4c, 0xfc, 0xc8, 0xd9, 0xd2, 0xf4, 0x9f, 0xdc, 0x46, 0x57, 0x6b, 0x6f,
+	0x9d, 0x2d, 0xe1, 0x38, 0x43, 0x45, 0x67, 0x50, 0x21, 0x33, 0xe2, 0x07, 0x64, 0xe4, 0x07, 0xbe,
+	0x9c, 0x3b, 0x45, 0x1d, 0xea, 0xe3, 0xef, 0x0c, 0xd5, 0xce, 0x10, 0xf0, 0x1a, 0xbd, 0xe9, 0x01,
+	0xac, 0x26, 0x42, 0x1f, 0xc1, 0x4e, 0xbf, 0x77, 0xde, 0x3d, 0x39, 0x3f, 0xae, 0x6e, 0xd4, 0x1e,
+	0xbc, 0xba, 0x6e, 0xbc, 0xa7, 0x62, 0xac, 0x00, 0x7d, 0x1a, 0x7a, 0x7e, 0x38, 0x46, 0xfb, 0x60,
+	0xb5, 0x8f, 0x8e, 0x7a, 0xfd, 0x8b, 0x5e, 0xb7, 0x5a, 0xa8, 0xd5, 0x5e, 0x5d, 0x37, 0xde, 0x5f,
+	0x07, 0xb6, 0x5d, 0x97, 0x46, 0x92, 0x7a, 0xb5, 0xe2, 0x57, 0xff, 0xa8, 0x6f, 0x34, 0xbf, 0x2a,
+	0x40, 0x25, 0xbb, 0x08, 0xf4, 0x11, 0x94, 0xda, 0x47, 0x17, 0x27, 0x2f, 0x7a, 0xd5, 0x8d, 0x15,
+	0x3d, 0x8b, 0x68, 0xbb, 0xd2, 0x9f, 0x51, 0xf4, 0x18, 0xb6, 0xfb, 0xed, 0x2f, 0x06, 0xbd, 0x6a,
+	0x61, 0xb5, 0x9c, 0x2c, 0xac, 0x4f, 0x62, 0xa1, 0x51, 0x5d, 0xdc, 0x3e, 0x39, 0xaf, 0x6e, 0xe6,
+	0xa3, 0xba, 0x9c, 0xf8, 0xa1, 0x59, 0xca, 0xdf, 0x8b, 0x60, 0x0f, 0x28, 0x9f, 0xf9, 0xee, 0x3b,
+	0x96, 0xc8, 0xa7, 0x50, 0x94, 0x44, 0x5c, 0x69, 0x69, 0xd8, 0xf9, 0xd2, 0xb8, 0x20, 0xe2, 0x4a,
+	0x4d, 0x6a, 0xe8, 0x1a, 0xaf, 0x94, 0xc1, 0x69, 0x14, 0xf8, 0x2e, 0x91, 0xd4, 0xd3, 0xca, 0xb0,
+	0x0f, 0x7f, 0x9c, 0xc7, 0xc6, 0x4b, 0x94, 0x59, 0xff, 0xb3, 0x0d, 0x9c, 0xa1, 0xa2, 0xa7, 0x50,
+	0x1a, 0x07, 0x6c, 0x44, 0x02, 0xad, 0x09, 0xfb, 0xf0, 0x51, 0x5e, 0x90, 0x63, 0x8d, 0x58, 0x05,
+	0x30, 0x14, 0xf4, 0x04, 0x4a, 0x71, 0xe4, 0x11, 0x49, 0x9d, 0x92, 0x26, 0x37, 0xf2, 0xc8, 0x5f,
+	0x68, 0xc4, 0x11, 0x0b, 0x2f, 0xfd, 0x31, 0x36, 0x78, 0x74, 0x0a, 0x56, 0x48, 0xe5, 0x97, 0x8c,
+	0x5f, 0x09, 0x67, 0xa7, 0xb1, 0xb5, 0x6f, 0x1f, 0x7e, 0x92, 0x2b, 0xc6, 0x04, 0xd3, 0x96, 0x92,
+	0xb8, 0x93, 0x29, 0x0d, 0x65, 0x12, 0xa6, 0xb3, 0xe9, 0x14, 0xf0, 0x32, 0x00, 0xfa, 0x0d, 0x58,
+	0x34, 0xf4, 0x22, 0xe6, 0x87, 0xd2, 0xb1, 0x6e, 0x5f, 0x48, 0xcf, 0x60, 0x54, 0x32, 0xf1, 0x92,
+	0xa1, 0xd8, 0x9c, 0x05, 0xc1, 0x88, 0xb8, 0x57, 0x4e, 0xf9, 0x2d, 0xb7, 0xb1, 0x64, 0x74, 0x4a,
+	0x50, 0x9c, 0x32, 0x8f, 0x36, 0x0f, 0xe0, 0xde, 0xb7, 0x52, 0x8d, 0x6a, 0x60, 0x99, 0x54, 0x27,
+	0x1a, 0x29, 0xe2, 0xa5, 0xdd, 0xbc, 0x0b, 0xbb, 0x6b, 0x69, 0x6d, 0xfe, 0xb5, 0x08, 0x56, 0x7a,
+	0xd6, 0xa8, 0x0d, 0x65, 0x97, 0x85, 0x92, 0xf8, 0x21, 0xe5, 0x46, 0x5e, 0xb9, 0x27, 0x73, 0x94,
+	0x82, 0x14, 0xeb, 0xd9, 0x06, 0x5e, 0xb1, 0xd0, 0xef, 0xa1, 0xcc, 0xa9, 0x60, 0x31, 0x77, 0xa9,
+	0x30, 0xfa, 0xda, 0xcf, 0x57, 0x48, 0x02, 0xc2, 0xf4, 0xcf, 0xb1, 0xcf, 0xa9, 0xca, 0xb2, 0xc0,
+	0x2b, 0x2a, 0x7a, 0x0a, 0x3b, 0x9c, 0x0a, 0x49, 0xb8, 0xfc, 0x2e, 0x89, 0xe0, 0x04, 0xd2, 0x67,
+	0x81, 0xef, 0xce, 0x71, 0xca, 0x40, 0x4f, 0xa1, 0x1c, 0x05, 0xc4, 0xd5, 0x51, 0x9d, 0x6d, 0x4d,
+	0xff, 0x20, 0x8f, 0xde, 0x4f, 0x41, 0x78, 0x85, 0x47, 0x9f, 0x01, 0x04, 0x6c, 0x3c, 0xf4, 0xb8,
+	0x3f, 0xa3, 0xdc, 0x48, 0xac, 0x96, 0xc7, 0xee, 0x6a, 0x04, 0x2e, 0x07, 0x6c, 0x9c, 0x0c, 0xd1,
+	0xf1, 0xf7, 0xd2, 0x57, 0x46, 0x5b, 0xa7, 0x00, 0x64, 0xf9, 0xd5, 0xa8, 0xeb, 0xe3, 0xb7, 0x0a,
+	0x65, 0x4e, 0x24, 0x43, 0x47, 0x8f, 0xa0, 0x72, 0xc9, 0xb8, 0x4b, 0x87, 0xe6, 0xd6, 0x94, 0xb5,
+	0x26, 0x6c, 0xed, 0x4b, 0xf4, 0xd5, 0x29, 0xc3, 0x0e, 0x8f, 0x43, 0xe9, 0x4f, 0x69, 0xf3, 0x14,
+	0xde, 0xcb, 0x0d, 0x8a, 0x0e, 0xa1, 0xb2, 0x3c, 0xe6, 0xa1, 0xef, 0x69, 0x7d, 0x94, 0x3b, 0x77,
+	0x17, 0x37, 0x7b, 0xf6, 0x52, 0x0f, 0x27, 0x5d, 0x6c, 0x2f, 0x41, 0x27, 0x5e, 0xf3, 0x6f, 0x16,
+	0xec, 0xae, 0x89, 0x05, 0xdd, 0x87, 0x6d, 0x7f, 0x4a, 0xc6, 0x34, 0xa1, 0xe3, 0xc4, 0x40, 0x3d,
+	0x28, 0x05, 0x64, 0x44, 0x03, 0x25, 0x19, 0x95, 0xb6, 0x9f, 0xbd, 0x51, 0x75, 0xad, 0x3f, 0x6a,
+	0x7c, 0x2f, 0x94, 0x7c, 0x8e, 0x0d, 0x19, 0x39, 0xb0, 0xe3, 0xb2, 0xe9, 0x94, 0x84, 0xaa, 0x38,
+	0x6d, 0xed, 0x97, 0x71, 0x6a, 0x22, 0x04, 0x45, 0xc2, 0xc7, 0xc2, 0x29, 0x6a, 0xb7, 0x1e, 0xa3,
+	0x2a, 0x6c, 0xd1, 0x70, 0xe6, 0x6c, 0x6b, 0x97, 0x1a, 0x2a, 0x8f, 0xe7, 0x27, 0x67, 0x5e, 0xc6,
+	0x6a, 0xa8, 0x78, 0xb1, 0xa0, 0xdc, 0xd9, 0xd1, 0x2e, 0x3d, 0x46, 0xbf, 0x82, 0xd2, 0x94, 0xc5,
+	0xa1, 0x14, 0x8e, 0xa5, 0x17, 0xfb, 0x20, 0x6f, 0xb1, 0x67, 0x0a, 0x61, 0x8a, 0xa7, 0x81, 0xa3,
+	0x1e, 0xdc, 0x13, 0x92, 0x45, 0xc3, 0x31, 0x27, 0x2e, 0x1d, 0x46, 0x94, 0xfb, 0xcc, 0x33, 0x97,
+	0xff, 0x41, 0x2b, 0xe9, 0x15, 0x5a, 0x69, 0xaf, 0xd0, 0xea, 0x9a, 0x5e, 0x01, 0xdf, 0x55, 0x9c,
+	0x63, 0x45, 0xe9, 0x6b, 0x06, 0xea, 0x43, 0x25, 0x8a, 0x83, 0x60, 0xc8, 0xa2, 0xe4, 0x1d, 0x00,
+	0x1d, 0xe1, 0x2d, 0x52, 0xd6, 0x8f, 0x83, 0xe0, 0x79, 0x42, 0xc2, 0x76, 0xb4, 0x32, 0xd0, 0xfb,
+	0x50, 0x1a, 0x73, 0x16, 0x47, 0xc2, 0xb1, 0x75, 0x32, 0x8c, 0x85, 0x3e, 0x87, 0x1d, 0x41, 0x5d,
+	0x4e, 0xa5, 0x70, 0x2a, 0x7a, 0xab, 0x1f, 0xe6, 0x4d, 0x32, 0xd0, 0x10, 0x4c, 0x2f, 0x29, 0xa7,
+	0xa1, 0x4b, 0x71, 0xca, 0x41, 0x0f, 0x60, 0x4b, 0xca, 0xb9, 0xb3, 0xdb, 0x28, 0xec, 0x5b, 0x9d,
+	0x9d, 0xc5, 0xcd, 0xde, 0xd6, 0xc5, 0xc5, 0x4b, 0xac, 0x7c, 0xaa, 0x46, 0x4d, 0x98, 0x90, 0x21,
+	0x99, 0x52, 0xe7, 0x8e, 0xce, 0xed, 0xd2, 0x46, 0x2f, 0x01, 0xbc, 0x50, 0x0c, 0x5d, 0x7d, 0x29,
+	0x9c, 0xbb, 0x7a, 0x77, 0x9f, 0xbc, 0x79, 0x77, 0xdd, 0xf3, 0x81, 0xa9, 0xd3, 0xbb, 0x8b, 0x9b,
+	0xbd, 0xf2, 0xd2, 0xc4, 0x65, 0x2f, 0x14, 0xc9, 0x10, 0x75, 0xc0, 0x9e, 0x50, 0x12, 0xc8, 0x89,
+	0x3b, 0xa1, 0xee, 0x95, 0x53, 0xbd, 0xbd, 0xf0, 0x3e, 0xd3, 0x30, 0x13, 0x21, 0x4b, 0x52, 0x0a,
+	0x56, 0x4b, 0x15, 0xce, 0x3d, 0x9d, 0xab, 0xc4, 0x40, 0x1f, 0x00, 0xb0, 0x88, 0x86, 0x43, 0x21,
+	0x3d, 0x3f, 0x74, 0x90, 0xda, 0x32, 0x2e, 0x2b, 0xcf, 0x40, 0x39, 0xd0, 0x43, 0x55, 0x16, 0x89,
+	0x37, 0x64, 0x61, 0x30, 0x77, 0x7e, 0xa0, 0xbf, 0x5a, 0xca, 0xf1, 0x3c, 0x0c, 0xe6, 0x68, 0x0f,
+	0x6c, 0xad, 0x0b, 0xe1, 0x8f, 0x43, 0x12, 0x38, 0xf7, 0x75, 0x3e, 0x40, 0xb9, 0x06, 0xda, 0x53,
+	0xfb, 0x0c, 0xec, 0x8c, 0xdc, 0x95, 0x4c, 0xaf, 0xe8, 0xdc, 0xdc, 0x20, 0x35, 0x54, 0x6b, 0x9a,
+	0x91, 0x20, 0x4e, 0x9a, 0xbd, 0x32, 0x4e, 0x8c, 0x5f, 0x6f, 0x3e, 0x29, 0xd4, 0x0e, 0xc1, 0xce,
+	0x1c, 0x3b, 0xfa, 0x10, 0x76, 0x39, 0x1d, 0xfb, 0x42, 0xf2, 0xf9, 0x90, 0xc4, 0x72, 0xe2, 0xfc,
+	0x4e, 0x13, 0x2a, 0xa9, 0xb3, 0x1d, 0xcb, 0x49, 0x6d, 0x08, 0xab, 0xec, 0xa1, 0x06, 0xd8, 0xea,
+	0x54, 0x04, 0xe5, 0x33, 0xca, 0xd5, 0x83, 0xa2, 0x36, 0x9d, 0x75, 0x29, 0xf5, 0x08, 0x4a, 0xb8,
+	0x3b, 0xd1, 0x97, 0xb7, 0x8c, 0x8d, 0xa5, 0x6e, 0x63, 0x2a, 0x51, 0x73, 0x1b, 0x8d, 0xd9, 0xfc,
+	0x6f, 0x01, 0x2a, 0xd9, 0x77, 0x11, 0x1d, 0x25, 0xef, 0x99, 0xde, 0xd2, 0x9d, 0xc3, 0x83, 0x37,
+	0xbd, 0xa3, 0xfa, 0xf5, 0x08, 0x62, 0x15, 0xec, 0x4c, 0xb5, 0xb0, 0x9a, 0x8c, 0x7e, 0x09, 0xdb,
+	0x11, 0xe3, 0x32, 0xad, 0x21, 0xf5, 0xdc, 0x8a, 0xcf, 0x78, 0x5a, 0x6d, 0x13, 0x70, 0x73, 0x02,
+	0x77, 0xd6, 0xa3, 0xa1, 0xc7, 0xb0, 0xf5, 0xe2, 0xa4, 0x5f, 0xdd, 0xa8, 0x3d, 0x7c, 0x75, 0xdd,
+	0xf8, 0xe1, 0xfa, 0xc7, 0x17, 0x3e, 0x97, 0x31, 0x09, 0x4e, 0xfa, 0xe8, 0xa7, 0xb0, 0xdd, 0x3d,
+	0x1f, 0x60, 0x5c, 0x2d, 0xd4, 0xf6, 0x5e, 0x5d, 0x37, 0x1e, 0xae, 0xe3, 0xd4, 0x27, 0x16, 0x87,
+	0x1e, 0x66, 0xa3, 0x65, 0x3b, 0xf7, 0xcf, 0x4d, 0xb0, 0x4d, 0x69, 0x7d, 0xd7, 0x1d, 0xff, 0x6e,
+	0xf2, 0x5a, 0xa5, 0x77, 0x66, 0xf3, 0x8d, 0x8f, 0x56, 0x25, 0x21, 0x98, 0x33, 0x7e, 0x04, 0x15,
+	0x3f, 0x9a, 0x7d, 0x3a, 0xa4, 0x21, 0x19, 0x05, 0xa6, 0xb3, 0xb3, 0xb0, 0xad, 0x7c, 0xbd, 0xc4,
+	0xa5, 0x2e, 0xac, 0x1f, 0x4a, 0xca, 0x43, 0xd3, 0xb3, 0x59, 0x78, 0x69, 0xa3, 0xcf, 0xa1, 0xe8,
+	0x47, 0x64, 0x6a, 0x5e, 0xda, 0xdc, 0x1d, 0x9c, 0xf4, 0xdb, 0x67, 0x46, 0x83, 0x1d, 0x6b, 0x71,
+	0xb3, 0x57, 0x54, 0x0e, 0xac, 0x69, 0xa8, 0x9e, 0x3e, 0x76, 0x6a, 0x26, 0x5d, 0x7c, 0x2d, 0x9c,
+	0xf1, 0x28, 0x1d, 0xf9, 0xe1, 0x98, 0x53, 0x21, 0x74, 0x19, 0xb6, 0x70, 0x6a, 0x36, 0xff, 0x57,
+	0x04, 0xfb, 0x28, 0x88, 0x85, 0x34, 0x8f, 0xcb, 0x3b, 0xcb, 0xe8, 0x4b, 0xb8, 0x47, 0xf4, 0x6f,
+	0x01, 0x09, 0x55, 0xa5, 0xd6, 0xed, 0x85, 0xc9, 0xea, 0xe3, 0xdc, 0x70, 0x4b, 0x70, 0xd2, 0x8a,
+	0x74, 0x4a, 0x2a, 0xa6, 0x53, 0xc0, 0x55, 0xf2, 0x8d, 0x2f, 0x68, 0x00, 0xbb, 0x8c, 0xbb, 0x13,
+	0x2a, 0x64, 0x52, 0xdf, 0x4d, 0x1b, 0x9d, 0xfb, 0x83, 0xf5, 0x3c, 0x0b, 0x34, 0xc5, 0x2d, 0x59,
+	0xed, 0x7a, 0x0c, 0xf4, 0x04, 0x8a, 0x9c, 0x5c, 0xa6, 0xad, 0x52, 0xae, 0xf2, 0x31, 0xb9, 0x94,
+	0x6b, 0x21, 0x34, 0x03, 0xfd, 0x01, 0xc0, 0xf3, 0x45, 0x44, 0xa4, 0x3b, 0xa1, 0xdc, 0x9c, 0x60,
+	0xee, 0x16, 0xbb, 0x4b, 0xd4, 0x5a, 0x94, 0x0c, 0x1b, 0x9d, 0x42, 0xd9, 0x25, 0xa9, 0x06, 0x4b,
+	0xb7, 0xff, 0x5b, 0x1c, 0xb5, 0x4d, 0x88, 0xaa, 0x0a, 0xb1, 0xb8, 0xd9, 0xb3, 0x52, 0x0f, 0xb6,
+	0x5c, 0x62, 0x34, 0x79, 0x0a, 0xbb, 0xea, 0x9f, 0x63, 0xe8, 0xd1, 0x4b, 0x12, 0x07, 0x32, 0x39,
+	0xfb, 0x5b, 0x8a, 0xb5, 0x6a, 0x60, 0xbb, 0x06, 0x67, 0xd6, 0x55, 0x91, 0x19, 0x1f, 0xfa, 0x13,
+	0xdc, 0xa3, 0xa1, 0xcb, 0xe7, 0x5a, 0x81, 0xe9, 0x0a, 0xad, 0xdb, 0x37, 0xdb, 0x5b, 0x82, 0xd7,
+	0x36, 0x5b, 0xa5, 0xdf, 0xf0, 0x37, 0x7d, 0x80, 0xe4, 0xf9, 0x7b, 0xb7, 0xfa, 0x43, 0x50, 0xf4,
+	0x88, 0x24, 0x5a, 0x72, 0x15, 0xac, 0xc7, 0x1d, 0xe7, 0xf5, 0xd7, 0xf5, 0x8d, 0x7f, 0x7f, 0x5d,
+	0xdf, 0xf8, 0xcb, 0xa2, 0x5e, 0x78, 0xbd, 0xa8, 0x17, 0xfe, 0xb5, 0xa8, 0x17, 0xfe, 0xb3, 0xa8,
+	0x17, 0x46, 0x25, 0xdd, 0x34, 0xfc, 0xe2, 0xff, 0x01, 0x00, 0x00, 0xff, 0xff, 0x64, 0x44, 0x1e,
+	0x4f, 0xb4, 0x10, 0x00, 0x00,
 }

vendor/github.com/docker/swarmkit/api/specs.proto

@@ -321,6 +321,12 @@ message NetworkSpec {
 	// enabled(default case) no manual attachment to this network
 	// can happen.
 	bool attachable = 6;
+	
+	// Ingress indicates this network will provide the routing-mesh.
+	// In older versions, the network providing the routing mesh was
+	// swarm internally created only and it was identified by the name
+	// "ingress" and the label "com.docker.swarm.internal": "true".
+	bool ingress = 7;
 }
 
 // ClusterSpec specifies global cluster settings.

vendor/github.com/docker/swarmkit/ca/certificates.go

@@ -113,12 +113,39 @@ type LocalSigner struct {
 	Key []byte
 }
 
-// RootCA is the representation of everything we need to sign certificates
+// RootCA is the representation of everything we need to sign certificates and/or to verify certificates
+//
+// RootCA.Cert:          [signing CA cert][CA cert1][CA cert2]
+// RootCA.Intermediates: [intermediate CA1][intermediate CA2][intermediate CA3]
+// RootCA.Signer.Key:    [signing CA key]
+//
+// Requirements:
+//
+// - [signing CA key] must be the private key for [signing CA cert]
+// - [signing CA cert] must be the first cert in RootCA.Cert
+//
+// - [intermediate CA1] must have the same public key and subject as [signing CA cert], because otherwise when
+//   appended to a leaf certificate, the intermediates will not form a chain (because [intermediate CA1] won't because
+//   the signer of the leaf certificate)
+// - [intermediate CA1] must be signed by [intermediate CA2], which must be signed by [intermediate CA3]
+//
+// - When we issue a certificate, the intermediates will be appended so that the certificate looks like:
+//   [leaf signed by signing CA cert][intermediate CA1][intermediate CA2][intermediate CA3]
+// - [leaf signed by signing CA cert][intermediate CA1][intermediate CA2][intermediate CA3] is guaranteed to form a
+//   valid chain from [leaf signed by signing CA cert] to one of the root certs ([signing CA cert], [CA cert1], [CA cert2])
+//   using zero or more of the intermediate certs ([intermediate CA1][intermediate CA2][intermediate CA3]) as intermediates
+//
 type RootCA struct {
 	// Cert contains a bundle of PEM encoded Certificate for the Root CA, the first one of which
 	// must correspond to the key in the local signer, if provided
 	Cert []byte
 
+	// Intermediates contains a bundle of PEM encoded intermediate CA certificates to append to any
+	// issued TLS (leaf) certificates. The first one must have the same public key and subject as the
+	// signing root certificate, and the rest must form a chain, each one certifying the one above it,
+	// as per RFC5246 section 7.4.2.
+	Intermediates []byte
+
 	// Pool is the root pool used to validate TLS certificates
 	Pool *x509.CertPool
 
@@ -306,7 +333,7 @@ func (rca *RootCA) ParseValidateAndSignCSR(csrBytes []byte, cn, ou, org string)
 		return nil, errors.Wrap(err, "failed to sign node certificate")
 	}
 
-	return cert, nil
+	return append(cert, rca.Intermediates...), nil
 }
 
 // CrossSignCACertificate takes a CA root certificate and generates an intermediate CA from it signed with the current root signer
@@ -348,7 +375,7 @@ func (rca *RootCA) CrossSignCACertificate(otherCAPEM []byte) ([]byte, error) {
 // NewRootCA creates a new RootCA object from unparsed PEM cert bundle and key byte
 // slices. key may be nil, and in this case NewRootCA will return a RootCA
 // without a signer.
-func NewRootCA(certBytes, keyBytes []byte, certExpiry time.Duration) (RootCA, error) {
+func NewRootCA(certBytes, keyBytes []byte, certExpiry time.Duration, intermediates []byte) (RootCA, error) {
 	// Parse all the certificates in the cert bundle
 	parsedCerts, err := helpers.ParseCertificatesPEM(certBytes)
 	if err != nil {
@@ -368,7 +395,6 @@ func NewRootCA(certBytes, keyBytes []byte, certExpiry time.Duration) (RootCA, er
 		default:
 			return RootCA{}, fmt.Errorf("unsupported signature algorithm: %s", cert.SignatureAlgorithm.String())
 		}
-
 		// Check to see if all of the certificates are valid, self-signed root CA certs
 		selfpool := x509.NewCertPool()
 		selfpool.AddCert(cert)
@@ -381,9 +407,28 @@ func NewRootCA(certBytes, keyBytes []byte, certExpiry time.Duration) (RootCA, er
 	// Calculate the digest for our Root CA bundle
 	digest := digest.FromBytes(certBytes)
 
+	// We do not yet support arbitrary chains of intermediates (e.g. the case of an offline root, and the swarm CA is an
+	// intermediate CA). We currently only intermediates for which the first intermediate is cross-signed version of the
+	// CA signing cert (the first cert of the root certs) for the purposes of root rotation.  If we wanted to support
+	// offline roots, we'd have to separate the CA signing cert from the self-signed root certs, but this intermediate
+	// validation logic should remain the same.  Either the first intermediate would BE the intermediate CA we sign with
+	// (in which case it'd have the same subject and public key), or it would be a cross-signed intermediate with the
+	// same subject and public key as our signing cert (which could be either an intermediate cert or a self-signed root
+	// cert).
+	if len(intermediates) > 0 {
+		parsedIntermediates, err := ValidateCertChain(pool, intermediates, false)
+		if err != nil {
+			return RootCA{}, errors.Wrap(err, "invalid intermediate chain")
+		}
+		if !bytes.Equal(parsedIntermediates[0].RawSubject, parsedCerts[0].RawSubject) ||
+			!bytes.Equal(parsedIntermediates[0].RawSubjectPublicKeyInfo, parsedCerts[0].RawSubjectPublicKeyInfo) {
+			return RootCA{}, errors.New("invalid intermediate chain - the first intermediate must have the same subject and public key as the root")
+		}
+	}
+
 	if len(keyBytes) == 0 {
-		// This RootCA does not have a valid signer.
-		return RootCA{Cert: certBytes, Digest: digest, Pool: pool}, nil
+		// This RootCA does not have a valid signer
+		return RootCA{Cert: certBytes, Intermediates: intermediates, Digest: digest, Pool: pool}, nil
 	}
 
 	var (
@@ -434,7 +479,7 @@ func NewRootCA(certBytes, keyBytes []byte, certExpiry time.Duration) (RootCA, er
 		}
 	}
 
-	return RootCA{Signer: &LocalSigner{Signer: signer, Key: keyBytes}, Digest: digest, Cert: certBytes, Pool: pool}, nil
+	return RootCA{Signer: &LocalSigner{Signer: signer, Key: keyBytes}, Intermediates: intermediates, Digest: digest, Cert: certBytes, Pool: pool}, nil
 }
 
 // ValidateCertChain checks checks that the certificates provided chain up to the root pool provided.  In addition
@@ -586,7 +631,7 @@ func GetLocalRootCA(paths CertPaths) (RootCA, error) {
 		key = nil
 	}
 
-	return NewRootCA(cert, key, DefaultNodeCertExpiration)
+	return NewRootCA(cert, key, DefaultNodeCertExpiration, nil)
 }
 
 func getGRPCConnection(creds credentials.TransportCredentials, connBroker *connectionbroker.Broker, forceRemote bool) (*connectionbroker.Conn, error) {
@@ -641,7 +686,7 @@ func GetRemoteCA(ctx context.Context, d digest.Digest, connBroker *connectionbro
 
 	// NewRootCA will validate that the certificates are otherwise valid and create a RootCA object.
 	// Since there is no key, the certificate expiry does not matter and will not be used.
-	return NewRootCA(response.Certificate, nil, DefaultNodeCertExpiration)
+	return NewRootCA(response.Certificate, nil, DefaultNodeCertExpiration, nil)
 }
 
 // CreateRootCA creates a Certificate authority for a new Swarm Cluster, potentially
@@ -660,7 +705,7 @@ func CreateRootCA(rootCN string, paths CertPaths) (RootCA, error) {
 		return RootCA{}, err
 	}
 
-	rootCA, err := NewRootCA(cert, key, DefaultNodeCertExpiration)
+	rootCA, err := NewRootCA(cert, key, DefaultNodeCertExpiration, nil)
 	if err != nil {
 		return RootCA{}, err
 	}

vendor/github.com/docker/swarmkit/ca/config.go

@@ -132,7 +132,7 @@ func (s *SecurityConfig) UpdateRootCA(cert, key []byte, certExpiry time.Duration
 	s.mu.Lock()
 	defer s.mu.Unlock()
 
-	rootCA, err := NewRootCA(cert, key, certExpiry)
+	rootCA, err := NewRootCA(cert, key, certExpiry, nil)
 	if err != nil {
 		return err
 	}

vendor/github.com/docker/swarmkit/ca/external.go

@@ -96,7 +96,7 @@ func (eca *ExternalCA) Sign(ctx context.Context, req signer.SignRequest) (cert [
 	for _, url := range urls {
 		cert, err = makeExternalSignRequest(ctx, client, url, csrJSON)
 		if err == nil {
-			return cert, err
+			return append(cert, eca.rootCA.Intermediates...), err
 		}
 		logrus.Debugf("unable to proxy certificate signing request to %s: %s", url, err)
 	}

vendor/github.com/docker/swarmkit/manager/allocator/network.go

@@ -6,7 +6,6 @@ import (
 
 	"github.com/docker/go-events"
 	"github.com/docker/swarmkit/api"
-	"github.com/docker/swarmkit/identity"
 	"github.com/docker/swarmkit/log"
 	"github.com/docker/swarmkit/manager/allocator/networkallocator"
 	"github.com/docker/swarmkit/manager/state"
@@ -18,42 +17,18 @@ import (
 
 const (
 	// Network allocator Voter ID for task allocation vote.
-	networkVoter = "network"
-
-	ingressNetworkName = "ingress"
-	ingressSubnet      = "10.255.0.0/16"
-
+	networkVoter           = "network"
 	allocatedStatusMessage = "pending task scheduling"
 )
 
 var (
+	// ErrNoIngress is returned when no ingress network is found in store
+	ErrNoIngress = errors.New("no ingress network found")
 	errNoChanges = errors.New("task unchanged")
 
 	retryInterval = 5 * time.Minute
 )
 
-func newIngressNetwork() *api.Network {
-	return &api.Network{
-		Spec: api.NetworkSpec{
-			Annotations: api.Annotations{
-				Name: ingressNetworkName,
-				Labels: map[string]string{
-					"com.docker.swarm.internal": "true",
-				},
-			},
-			DriverConfig: &api.Driver{},
-			IPAM: &api.IPAMOptions{
-				Driver: &api.Driver{},
-				Configs: []*api.IPAMConfig{
-					{
-						Subnet: ingressSubnet,
-					},
-				},
-			},
-		},
-	}
-}
-
 // Network context information which is used throughout the network allocation code.
 type networkContext struct {
 	ingressNetwork *api.Network
@@ -97,7 +72,6 @@ func (a *Allocator) doNetworkInit(ctx context.Context) (err error) {
 		unallocatedTasks:    make(map[string]*api.Task),
 		unallocatedServices: make(map[string]*api.Service),
 		unallocatedNetworks: make(map[string]*api.Network),
-		ingressNetwork:      newIngressNetwork(),
 		lastRetry:           time.Now(),
 	}
 	a.netCtx = nc
@@ -108,63 +82,38 @@ func (a *Allocator) doNetworkInit(ctx context.Context) (err error) {
 		}
 	}()
 
-	// Check if we have the ingress network. If not found create
-	// it before reading all network objects for allocation.
-	var networks []*api.Network
-	a.store.View(func(tx store.ReadTx) {
-		networks, err = store.FindNetworks(tx, store.ByName(ingressNetworkName))
-		if len(networks) > 0 {
-			nc.ingressNetwork = networks[0]
-		}
-	})
-	if err != nil {
-		return errors.Wrap(err, "failed to find ingress network during init")
-	}
-
-	// If ingress network is not found, create one right away
-	// using the predefined template.
-	if len(networks) == 0 {
-		if err := a.store.Update(func(tx store.Tx) error {
-			nc.ingressNetwork.ID = identity.NewID()
-			if err := store.CreateNetwork(tx, nc.ingressNetwork); err != nil {
-				return err
-			}
-
-			return nil
-		}); err != nil {
-			return errors.Wrap(err, "failed to create ingress network")
-		}
-
-		a.store.View(func(tx store.ReadTx) {
-			networks, err = store.FindNetworks(tx, store.ByName(ingressNetworkName))
-			if len(networks) > 0 {
-				nc.ingressNetwork = networks[0]
-			}
-		})
-		if err != nil {
-			return errors.Wrap(err, "failed to find ingress network after creating it")
-		}
-
-	}
-
-	// Try to complete ingress network allocation before anything else so
-	// that the we can get the preferred subnet for ingress
-	// network.
-	if !na.IsAllocated(nc.ingressNetwork) {
-		if err := a.allocateNetwork(ctx, nc.ingressNetwork); err != nil {
-			log.G(ctx).WithError(err).Error("failed allocating ingress network during init")
-		} else if _, err := a.store.Batch(func(batch *store.Batch) error {
-			if err := a.commitAllocatedNetwork(ctx, batch, nc.ingressNetwork); err != nil {
+	// Ingress network is now created at cluster's first time creation.
+	// Check if we have the ingress network. If found, make sure it is
+	// allocated, before reading all network objects for allocation.
+	// If not found, it means it was removed by user, nothing to do here.
+	ingressNetwork, err := GetIngressNetwork(a.store)
+	switch err {
+	case nil:
+		// Try to complete ingress network allocation before anything else so
+		// that the we can get the preferred subnet for ingress network.
+		nc.ingressNetwork = ingressNetwork
+		if !na.IsAllocated(nc.ingressNetwork) {
+			if err := a.allocateNetwork(ctx, nc.ingressNetwork); err != nil {
+				log.G(ctx).WithError(err).Error("failed allocating ingress network during init")
+			} else if _, err := a.store.Batch(func(batch *store.Batch) error {
+				if err := a.commitAllocatedNetwork(ctx, batch, nc.ingressNetwork); err != nil {
+					log.G(ctx).WithError(err).Error("failed committing allocation of ingress network during init")
+				}
+				return nil
+			}); err != nil {
 				log.G(ctx).WithError(err).Error("failed committing allocation of ingress network during init")
 			}
-			return nil
-		}); err != nil {
-			log.G(ctx).WithError(err).Error("failed committing allocation of ingress network during init")
 		}
+	case ErrNoIngress:
+		// Ingress network is not present in store, It means user removed it
+		// and did not create a new one.
+	default:
+		return errors.Wrap(err, "failure while looking for ingress network during init")
 	}
 
 	// Allocate networks in the store so far before we started
 	// watching.
+	var networks []*api.Network
 	a.store.View(func(tx store.ReadTx) {
 		networks, err = store.FindNetworks(tx, store.All)
 	})
@@ -196,43 +145,12 @@ func (a *Allocator) doNetworkInit(ctx context.Context) (err error) {
 		log.G(ctx).WithError(err).Error("failed committing allocation of networks during init")
 	}
 
-	// Allocate nodes in the store so far before we process watched events.
-	var nodes []*api.Node
-	a.store.View(func(tx store.ReadTx) {
-		nodes, err = store.FindNodes(tx, store.All)
-	})
-	if err != nil {
-		return errors.Wrap(err, "error listing all nodes in store while trying to allocate during init")
-	}
-
-	var allocatedNodes []*api.Node
-	for _, node := range nodes {
-		if na.IsNodeAllocated(node) {
-			continue
-		}
-
-		if node.Attachment == nil {
-			node.Attachment = &api.NetworkAttachment{}
-		}
-
-		node.Attachment.Network = nc.ingressNetwork.Copy()
-		if err := a.allocateNode(ctx, node); err != nil {
-			log.G(ctx).WithError(err).Errorf("Failed to allocate network resources for node %s during init", node.ID)
-			continue
-		}
-
-		allocatedNodes = append(allocatedNodes, node)
-	}
-
-	if _, err := a.store.Batch(func(batch *store.Batch) error {
-		for _, node := range allocatedNodes {
-			if err := a.commitAllocatedNode(ctx, batch, node); err != nil {
-				log.G(ctx).WithError(err).Errorf("Failed to commit allocation of network resources for node %s during init", node.ID)
-			}
+	// Allocate nodes in the store so far before we process watched events,
+	// if the ingress network is present.
+	if nc.ingressNetwork != nil {
+		if err := a.allocateNodes(ctx); err != nil {
+			return err
 		}
-		return nil
-	}); err != nil {
-		log.G(ctx).WithError(err).Error("Failed to commit allocation of network resources for nodes during init")
 	}
 
 	// Allocate services in the store so far before we process watched events.
@@ -346,6 +264,12 @@ func (a *Allocator) doNetworkAlloc(ctx context.Context, ev events.Event) {
 			break
 		}
 
+		if IsIngressNetwork(n) && nc.ingressNetwork != nil {
+			log.G(ctx).Errorf("Cannot allocate ingress network %s (%s) because another ingress network is already present: %s (%s)",
+				n.ID, n.Spec.Annotations.Name, nc.ingressNetwork.ID, nc.ingressNetwork.Spec.Annotations)
+			break
+		}
+
 		if err := a.allocateNetwork(ctx, n); err != nil {
 			log.G(ctx).WithError(err).Errorf("Failed allocation for network %s", n.ID)
 			break
@@ -356,9 +280,24 @@ func (a *Allocator) doNetworkAlloc(ctx context.Context, ev events.Event) {
 		}); err != nil {
 			log.G(ctx).WithError(err).Errorf("Failed to commit allocation for network %s", n.ID)
 		}
+
+		if IsIngressNetwork(n) {
+			nc.ingressNetwork = n
+			err := a.allocateNodes(ctx)
+			if err != nil {
+				log.G(ctx).WithError(err).Error(err)
+			}
+		}
 	case state.EventDeleteNetwork:
 		n := v.Network.Copy()
 
+		if IsIngressNetwork(n) && nc.ingressNetwork.ID == n.ID {
+			nc.ingressNetwork = nil
+			if err := a.deallocateNodes(ctx); err != nil {
+				log.G(ctx).WithError(err).Error(err)
+			}
+		}
+
 		// The assumption here is that all dependent objects
 		// have been cleaned up when we are here so the only
 		// thing that needs to happen is free the network
@@ -467,7 +406,7 @@ func (a *Allocator) doNodeAlloc(ctx context.Context, ev events.Event) {
 		return
 	}
 
-	if !nc.nwkAllocator.IsNodeAllocated(node) {
+	if !nc.nwkAllocator.IsNodeAllocated(node) && nc.ingressNetwork != nil {
 		if node.Attachment == nil {
 			node.Attachment = &api.NetworkAttachment{}
 		}
@@ -486,6 +425,85 @@ func (a *Allocator) doNodeAlloc(ctx context.Context, ev events.Event) {
 	}
 }
 
+func (a *Allocator) allocateNodes(ctx context.Context) error {
+	// Allocate nodes in the store so far before we process watched events.
+	var (
+		allocatedNodes []*api.Node
+		nodes          []*api.Node
+		err            error
+		nc             = a.netCtx
+	)
+
+	a.store.View(func(tx store.ReadTx) {
+		nodes, err = store.FindNodes(tx, store.All)
+	})
+	if err != nil {
+		return errors.Wrap(err, "error listing all nodes in store while trying to allocate network resources")
+	}
+
+	for _, node := range nodes {
+		if nc.nwkAllocator.IsNodeAllocated(node) {
+			continue
+		}
+
+		if node.Attachment == nil {
+			node.Attachment = &api.NetworkAttachment{}
+		}
+
+		node.Attachment.Network = nc.ingressNetwork.Copy()
+		if err := a.allocateNode(ctx, node); err != nil {
+			log.G(ctx).WithError(err).Errorf("Failed to allocate network resources for node %s", node.ID)
+			continue
+		}
+
+		allocatedNodes = append(allocatedNodes, node)
+	}
+
+	if _, err := a.store.Batch(func(batch *store.Batch) error {
+		for _, node := range allocatedNodes {
+			if err := a.commitAllocatedNode(ctx, batch, node); err != nil {
+				log.G(ctx).WithError(err).Errorf("Failed to commit allocation of network resources for node %s", node.ID)
+			}
+		}
+		return nil
+	}); err != nil {
+		log.G(ctx).WithError(err).Error("Failed to commit allocation of network resources for nodes")
+	}
+
+	return nil
+}
+
+func (a *Allocator) deallocateNodes(ctx context.Context) error {
+	var (
+		nodes []*api.Node
+		nc    = a.netCtx
+		err   error
+	)
+
+	a.store.View(func(tx store.ReadTx) {
+		nodes, err = store.FindNodes(tx, store.All)
+	})
+	if err != nil {
+		return fmt.Errorf("error listing all nodes in store while trying to free network resources")
+	}
+
+	for _, node := range nodes {
+		if nc.nwkAllocator.IsNodeAllocated(node) {
+			if err := nc.nwkAllocator.DeallocateNode(node); err != nil {
+				log.G(ctx).WithError(err).Errorf("Failed freeing network resources for node %s", node.ID)
+			}
+			node.Attachment = nil
+			if _, err := a.store.Batch(func(batch *store.Batch) error {
+				return a.commitAllocatedNode(ctx, batch, node)
+			}); err != nil {
+				log.G(ctx).WithError(err).Errorf("Failed to commit deallocation of network resources for node %s", node.ID)
+			}
+		}
+	}
+
+	return nil
+}
+
 // taskReadyForNetworkVote checks if the task is ready for a network
 // vote to move it to PENDING state.
 func taskReadyForNetworkVote(t *api.Task, s *api.Service, nc *networkContext) bool {
@@ -711,6 +729,9 @@ func (a *Allocator) allocateService(ctx context.Context, s *api.Service) error {
 		// world. Automatically attach the service to the ingress
 		// network only if it is not already done.
 		if isIngressNetworkNeeded(s) {
+			if nc.ingressNetwork == nil {
+				return fmt.Errorf("ingress network is missing")
+			}
 			var found bool
 			for _, vip := range s.Endpoint.VirtualIPs {
 				if vip.NetworkID == nc.ingressNetwork.ID {
@@ -1022,3 +1043,36 @@ func updateTaskStatus(t *api.Task, newStatus api.TaskState, message string) {
 	t.Status.Message = message
 	t.Status.Timestamp = ptypes.MustTimestampProto(time.Now())
 }
+
+// IsIngressNetwork returns whether the passed network is an ingress network.
+func IsIngressNetwork(nw *api.Network) bool {
+	if nw.Spec.Ingress {
+		return true
+	}
+	// Check if legacy defined ingress network
+	_, ok := nw.Spec.Annotations.Labels["com.docker.swarm.internal"]
+	return ok && nw.Spec.Annotations.Name == "ingress"
+}
+
+// GetIngressNetwork fetches the ingress network from store.
+// ErrNoIngress will be returned if the ingress network is not present,
+// nil otherwise. In case of any other failure in accessing the store,
+// the respective error will be reported as is.
+func GetIngressNetwork(s *store.MemoryStore) (*api.Network, error) {
+	var (
+		networks []*api.Network
+		err      error
+	)
+	s.View(func(tx store.ReadTx) {
+		networks, err = store.FindNetworks(tx, store.All)
+	})
+	if err != nil {
+		return nil, err
+	}
+	for _, n := range networks {
+		if IsIngressNetwork(n) {
+			return n, nil
+		}
+	}
+	return nil, ErrNoIngress
+}

vendor/github.com/docker/swarmkit/manager/controlapi/network.go

@@ -1,7 +1,6 @@
 package controlapi
 
 import (
-	"fmt"
 	"net"
 
 	"github.com/docker/docker/pkg/plugingetter"
@@ -9,6 +8,7 @@ import (
 	"github.com/docker/libnetwork/ipamapi"
 	"github.com/docker/swarmkit/api"
 	"github.com/docker/swarmkit/identity"
+	"github.com/docker/swarmkit/manager/allocator"
 	"github.com/docker/swarmkit/manager/state/store"
 	"golang.org/x/net/context"
 	"google.golang.org/grpc"
@@ -75,6 +75,14 @@ func validateNetworkSpec(spec *api.NetworkSpec, pg plugingetter.PluginGetter) er
 		return grpc.Errorf(codes.InvalidArgument, errInvalidArgument.Error())
 	}
 
+	if spec.Ingress && spec.DriverConfig != nil && spec.DriverConfig.Name != "overlay" {
+		return grpc.Errorf(codes.Unimplemented, "only overlay driver is currently supported for ingress network")
+	}
+
+	if spec.Attachable && spec.Ingress {
+		return grpc.Errorf(codes.InvalidArgument, "ingress network cannot be attachable")
+	}
+
 	if err := validateAnnotations(spec.Annotations); err != nil {
 		return err
 	}
@@ -94,16 +102,10 @@ func validateNetworkSpec(spec *api.NetworkSpec, pg plugingetter.PluginGetter) er
 // - Returns `InvalidArgument` if the NetworkSpec is malformed.
 // - Returns an error if the creation fails.
 func (s *Server) CreateNetwork(ctx context.Context, request *api.CreateNetworkRequest) (*api.CreateNetworkResponse, error) {
-	// if you change this function, you have to change createInternalNetwork in
-	// the tests to match it (except the part where we check the label).
 	if err := validateNetworkSpec(request.Spec, s.pg); err != nil {
 		return nil, err
 	}
 
-	if _, ok := request.Spec.Annotations.Labels["com.docker.swarm.internal"]; ok {
-		return nil, grpc.Errorf(codes.PermissionDenied, "label com.docker.swarm.internal is for predefined internal networks and cannot be applied by users")
-	}
-
 	// TODO(mrjana): Consider using `Name` as a primary key to handle
 	// duplicate creations. See #65
 	n := &api.Network{
@@ -112,6 +114,13 @@ func (s *Server) CreateNetwork(ctx context.Context, request *api.CreateNetworkRe
 	}
 
 	err := s.store.Update(func(tx store.Tx) error {
+		if request.Spec.Ingress {
+			if n, err := allocator.GetIngressNetwork(s.store); err == nil {
+				return grpc.Errorf(codes.AlreadyExists, "ingress network (%s) is already present", n.ID)
+			} else if err != allocator.ErrNoIngress {
+				return grpc.Errorf(codes.Internal, "failed ingress network presence check: %v", err)
+			}
+		}
 		return store.CreateNetwork(tx, n)
 	})
 	if err != nil {
@@ -152,44 +161,70 @@ func (s *Server) RemoveNetwork(ctx context.Context, request *api.RemoveNetworkRe
 		return nil, grpc.Errorf(codes.InvalidArgument, errInvalidArgument.Error())
 	}
 
-	err := s.store.Update(func(tx store.Tx) error {
-		services, err := store.FindServices(tx, store.ByReferencedNetworkID(request.NetworkID))
+	var (
+		n  *api.Network
+		rm = s.removeNetwork
+	)
+
+	s.store.View(func(tx store.ReadTx) {
+		n = store.GetNetwork(tx, request.NetworkID)
+	})
+	if n == nil {
+		return nil, grpc.Errorf(codes.NotFound, "network %s not found", request.NetworkID)
+	}
+
+	if allocator.IsIngressNetwork(n) {
+		rm = s.removeIngressNetwork
+	}
+
+	if err := rm(n.ID); err != nil {
+		if err == store.ErrNotExist {
+			return nil, grpc.Errorf(codes.NotFound, "network %s not found", request.NetworkID)
+		}
+		return nil, err
+	}
+	return &api.RemoveNetworkResponse{}, nil
+}
+
+func (s *Server) removeNetwork(id string) error {
+	return s.store.Update(func(tx store.Tx) error {
+		services, err := store.FindServices(tx, store.ByReferencedNetworkID(id))
 		if err != nil {
-			return grpc.Errorf(codes.Internal, "could not find services using network %s: %v", request.NetworkID, err)
+			return grpc.Errorf(codes.Internal, "could not find services using network %s: %v", id, err)
 		}
 
 		if len(services) != 0 {
-			return grpc.Errorf(codes.FailedPrecondition, "network %s is in use by service %s", request.NetworkID, services[0].ID)
+			return grpc.Errorf(codes.FailedPrecondition, "network %s is in use by service %s", id, services[0].ID)
 		}
 
-		tasks, err := store.FindTasks(tx, store.ByReferencedNetworkID(request.NetworkID))
+		tasks, err := store.FindTasks(tx, store.ByReferencedNetworkID(id))
 		if err != nil {
-			return grpc.Errorf(codes.Internal, "could not find tasks using network %s: %v", request.NetworkID, err)
+			return grpc.Errorf(codes.Internal, "could not find tasks using network %s: %v", id, err)
 		}
 
 		for _, t := range tasks {
 			if t.DesiredState <= api.TaskStateRunning && t.Status.State <= api.TaskStateRunning {
-				return grpc.Errorf(codes.FailedPrecondition, "network %s is in use by task %s", request.NetworkID, t.ID)
+				return grpc.Errorf(codes.FailedPrecondition, "network %s is in use by task %s", id, t.ID)
 			}
 		}
 
-		nw := store.GetNetwork(tx, request.NetworkID)
-		if _, ok := nw.Spec.Annotations.Labels["com.docker.swarm.internal"]; ok {
-			networkDescription := nw.ID
-			if nw.Spec.Annotations.Name != "" {
-				networkDescription = fmt.Sprintf("%s (%s)", nw.Spec.Annotations.Name, nw.ID)
+		return store.DeleteNetwork(tx, id)
+	})
+}
+
+func (s *Server) removeIngressNetwork(id string) error {
+	return s.store.Update(func(tx store.Tx) error {
+		services, err := store.FindServices(tx, store.All)
+		if err != nil {
+			return grpc.Errorf(codes.Internal, "could not find services using network %s: %v", id, err)
+		}
+		for _, srv := range services {
+			if doesServiceNeedIngress(srv) {
+				return grpc.Errorf(codes.FailedPrecondition, "ingress network cannot be removed because service %s depends on it", srv.ID)
 			}
-			return grpc.Errorf(codes.PermissionDenied, "%s is a pre-defined network and cannot be removed", networkDescription)
 		}
-		return store.DeleteNetwork(tx, request.NetworkID)
+		return store.DeleteNetwork(tx, id)
 	})
-	if err != nil {
-		if err == store.ErrNotExist {
-			return nil, grpc.Errorf(codes.NotFound, "network %s not found", request.NetworkID)
-		}
-		return nil, err
-	}
-	return &api.RemoveNetworkResponse{}, nil
 }
 
 func filterNetworks(candidates []*api.Network, filters ...func(*api.Network) bool) []*api.Network {

vendor/github.com/docker/swarmkit/manager/controlapi/service.go

@@ -11,6 +11,7 @@ import (
 	"github.com/docker/distribution/reference"
 	"github.com/docker/swarmkit/api"
 	"github.com/docker/swarmkit/identity"
+	"github.com/docker/swarmkit/manager/allocator"
 	"github.com/docker/swarmkit/manager/constraint"
 	"github.com/docker/swarmkit/manager/state/store"
 	"github.com/docker/swarmkit/protobuf/ptypes"
@@ -288,7 +289,7 @@ func (s *Server) validateNetworks(networks []*api.NetworkAttachmentConfig) error
 		if network == nil {
 			continue
 		}
-		if _, ok := network.Spec.Annotations.Labels["com.docker.swarm.internal"]; ok {
+		if network.Spec.Internal {
 			return grpc.Errorf(codes.InvalidArgument,
 				"Service cannot be explicitly attached to %q network which is a swarm internal network",
 				network.Spec.Annotations.Name)
@@ -424,6 +425,36 @@ func (s *Server) checkSecretExistence(tx store.Tx, spec *api.ServiceSpec) error
 	return nil
 }
 
+func doesServiceNeedIngress(srv *api.Service) bool {
+	// Only VIP mode with target ports needs routing mesh.
+	// If no endpoint is specified, it defaults to VIP mode but no target ports
+	// are specified, so the service does not need the routing mesh.
+	if srv.Spec.Endpoint == nil || srv.Spec.Endpoint.Mode != api.ResolutionModeVirtualIP {
+		return false
+	}
+	// Go through the ports' config
+	for _, p := range srv.Spec.Endpoint.Ports {
+		if p.PublishMode != api.PublishModeIngress {
+			continue
+		}
+		if p.PublishedPort != 0 {
+			return true
+		}
+	}
+	// Go through the ports' state
+	if srv.Endpoint != nil {
+		for _, p := range srv.Endpoint.Ports {
+			if p.PublishMode != api.PublishModeIngress {
+				continue
+			}
+			if p.PublishedPort != 0 {
+				return true
+			}
+		}
+	}
+	return false
+}
+
 // CreateService creates and returns a Service based on the provided ServiceSpec.
 // - Returns `InvalidArgument` if the ServiceSpec is malformed.
 // - Returns `Unimplemented` if the ServiceSpec references unimplemented features.
@@ -449,6 +480,12 @@ func (s *Server) CreateService(ctx context.Context, request *api.CreateServiceRe
 		Spec: *request.Spec,
 	}
 
+	if doesServiceNeedIngress(service) {
+		if _, err := allocator.GetIngressNetwork(s.store); err == allocator.ErrNoIngress {
+			return nil, grpc.Errorf(codes.FailedPrecondition, "service needs ingress network, but no ingress network is present")
+		}
+	}
+
 	err := s.store.Update(func(tx store.Tx) error {
 		// Check to see if all the secrets being added exist as objects
 		// in our datastore
@@ -578,6 +615,12 @@ func (s *Server) UpdateService(ctx context.Context, request *api.UpdateServiceRe
 			service.UpdateStatus = nil
 		}
 
+		if doesServiceNeedIngress(service) {
+			if _, err := allocator.GetIngressNetwork(s.store); err == allocator.ErrNoIngress {
+				return grpc.Errorf(codes.FailedPrecondition, "service needs ingress network, but no ingress network is present")
+			}
+		}
+
 		return store.UpdateService(tx, service)
 	})
 	if err != nil {

vendor/github.com/docker/swarmkit/manager/dirty.go

@@ -47,7 +47,7 @@ func (m *Manager) IsStateDirty() (bool, error) {
 		if structField.Type.Kind() != reflect.Slice {
 			panic("unexpected field type in StoreSnapshot")
 		}
-		if structField.Name != "Nodes" && structField.Name != "Clusters" && field.Len() != 0 {
+		if structField.Name != "Nodes" && structField.Name != "Clusters" && structField.Name != "Networks" && field.Len() != 0 {
 			// One of the other data types has an entry
 			return true, nil
 		}

vendor/github.com/docker/swarmkit/manager/manager.go

@@ -19,6 +19,7 @@ import (
 	"github.com/docker/swarmkit/api"
 	"github.com/docker/swarmkit/ca"
 	"github.com/docker/swarmkit/connectionbroker"
+	"github.com/docker/swarmkit/identity"
 	"github.com/docker/swarmkit/log"
 	"github.com/docker/swarmkit/manager/allocator"
 	"github.com/docker/swarmkit/manager/controlapi"
@@ -892,7 +893,18 @@ func (m *Manager) becomeLeader(ctx context.Context) {
 			rootCA))
 		// Add Node entry for ourself, if one
 		// doesn't exist already.
-		store.CreateNode(tx, managerNode(nodeID, m.config.Availability))
+		freshCluster := nil == store.CreateNode(tx, managerNode(nodeID, m.config.Availability))
+
+		if freshCluster {
+			// This is a fresh swarm cluster. Add to store now any initial
+			// cluster resource, like the default ingress network which
+			// provides the routing mesh for this cluster.
+			log.G(ctx).Info("Creating default ingress network")
+			if err := store.CreateNetwork(tx, newIngressNetwork()); err != nil {
+				log.G(ctx).WithError(err).Error("failed to create default ingress network")
+			}
+		}
+
 		return nil
 	})
 
@@ -1084,3 +1096,28 @@ func managerNode(nodeID string, availability api.NodeSpec_Availability) *api.Nod
 		},
 	}
 }
+
+// newIngressNetwork returns the network object for the default ingress
+// network, the network which provides the routing mesh. Caller will save to
+// store this object once, at fresh cluster creation. It is expected to
+// call this function inside a store update transaction.
+func newIngressNetwork() *api.Network {
+	return &api.Network{
+		ID: identity.NewID(),
+		Spec: api.NetworkSpec{
+			Ingress: true,
+			Annotations: api.Annotations{
+				Name: "ingress",
+			},
+			DriverConfig: &api.Driver{},
+			IPAM: &api.IPAMOptions{
+				Driver: &api.Driver{},
+				Configs: []*api.IPAMConfig{
+					{
+						Subnet: "10.255.0.0/16",
+					},
+				},
+			},
+		},
+	}
+}

vendor/github.com/docker/swarmkit/manager/scheduler/nodeset.go

@@ -111,6 +111,10 @@ func (ns *nodeSet) tree(serviceID string, preferences []*api.PlacementPreference
 			tree = next
 		}
 
+		if node.ActiveTasksCountByService != nil {
+			tree.tasks += node.ActiveTasksCountByService[serviceID]
+		}
+
 		if tree.nodeHeap.lessFunc == nil {
 			tree.nodeHeap.lessFunc = nodeLess
 		}

vendor/github.com/docker/swarmkit/manager/scheduler/scheduler.go

@@ -1,7 +1,6 @@
 package scheduler
 
 import (
-	"container/list"
 	"time"
 
 	"github.com/docker/swarmkit/api"
@@ -30,7 +29,7 @@ type schedulingDecision struct {
 // Scheduler assigns tasks to nodes.
 type Scheduler struct {
 	store           *store.MemoryStore
-	unassignedTasks *list.List
+	unassignedTasks map[string]*api.Task
 	// preassignedTasks already have NodeID, need resource validation
 	preassignedTasks map[string]*api.Task
 	nodeSet          nodeSet
@@ -47,7 +46,7 @@ type Scheduler struct {
 func New(store *store.MemoryStore) *Scheduler {
 	return &Scheduler{
 		store:            store,
-		unassignedTasks:  list.New(),
+		unassignedTasks:  make(map[string]*api.Task),
 		preassignedTasks: make(map[string]*api.Task),
 		allTasks:         make(map[string]*api.Task),
 		stopChan:         make(chan struct{}),
@@ -191,7 +190,7 @@ func (s *Scheduler) Stop() {
 
 // enqueue queues a task for scheduling.
 func (s *Scheduler) enqueue(t *api.Task) {
-	s.unassignedTasks.PushBack(t)
+	s.unassignedTasks[t.ID] = t
 }
 
 func (s *Scheduler) createTask(ctx context.Context, t *api.Task) int {
@@ -333,15 +332,12 @@ func (s *Scheduler) processPreassignedTasks(ctx context.Context) {
 // tick attempts to schedule the queue.
 func (s *Scheduler) tick(ctx context.Context) {
 	tasksByCommonSpec := make(map[string]map[string]*api.Task)
-	schedulingDecisions := make(map[string]schedulingDecision, s.unassignedTasks.Len())
+	schedulingDecisions := make(map[string]schedulingDecision, len(s.unassignedTasks))
 
-	var next *list.Element
-	for e := s.unassignedTasks.Front(); e != nil; e = next {
-		next = e.Next()
-		t := s.allTasks[e.Value.(*api.Task).ID]
+	for taskID, t := range s.unassignedTasks {
 		if t == nil || t.NodeID != "" {
 			// task deleted or already assigned
-			s.unassignedTasks.Remove(e)
+			delete(s.unassignedTasks, taskID)
 			continue
 		}
 
@@ -362,8 +358,8 @@ func (s *Scheduler) tick(ctx context.Context) {
 		if tasksByCommonSpec[taskGroupKey] == nil {
 			tasksByCommonSpec[taskGroupKey] = make(map[string]*api.Task)
 		}
-		tasksByCommonSpec[taskGroupKey][t.ID] = t
-		s.unassignedTasks.Remove(e)
+		tasksByCommonSpec[taskGroupKey][taskID] = t
+		delete(s.unassignedTasks, taskID)
 	}
 
 	for _, taskGroup := range tasksByCommonSpec {
@@ -602,6 +598,12 @@ func (s *Scheduler) scheduleNTasksOnNodes(ctx context.Context, n int, taskGroup
 	nodeIter := 0
 	nodeCount := len(nodes)
 	for taskID, t := range taskGroup {
+		// Skip tasks which were already scheduled because they ended
+		// up in two groups at once.
+		if _, exists := schedulingDecisions[taskID]; exists {
+			continue
+		}
+
 		node := &nodes[nodeIter%nodeCount]
 
 		log.G(ctx).WithField("task.id", t.ID).Debugf("assigning to node %s", node.ID)