0ct0pu5/moby
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Merge pull request #31714 from aboch/cingr

Browse source 
Allow user to replace ingress network
This commit is contained in:
Madhu Venugopal 2017-03-27 07:30:42 -07:00 • committed by GitHub
commit 04295d26df
38 changed files with 867 additions and 401 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
api/server/router/network/network_routes.go
View file

@ -294,6 +294,7 @@ func (n *networkRouter) buildNetworkResource(nw libnetwork.Network) *types.Netwo
r.EnableIPv6 = info.IPv6Enabled()
r.Internal = info.Internal()
r.Attachable = info.Attachable()
r.Ingress = info.Ingress()
r.Options = info.DriverOptions()
r.Containers = make(map[string]types.EndpointResource)
buildIpamResources(r, info)

10
api/swagger.yaml
View file

@ -1117,6 +1117,8 @@ definitions:
type: "boolean"
Attachable:
type: "boolean"
Ingress:
type: "boolean"
Containers:
type: "object"
additionalProperties:
@ -1145,6 +1147,7 @@ definitions:
foo: "bar"
Internal: false
Attachable: false
Ingress: false
Containers:
19a4d5d687db25203351ed79d478946f861258f018fe384f229f2efa4b23513c:
Name: "test"
@ -6211,6 +6214,7 @@ paths:
EnableIPv6: false
Internal: false
Attachable: false
Ingress: false
IPAM:
Driver: "default"
Config:
@ -6237,6 +6241,7 @@ paths:
EnableIPv6: false
Internal: false
Attachable: false
Ingress: false
IPAM:
Driver: "default"
Config: []
@ -6250,6 +6255,7 @@ paths:
EnableIPv6: false
Internal: false
Attachable: false
Ingress: false
IPAM:
Driver: "default"
Config: []
@ -6383,6 +6389,9 @@ paths:
Attachable:
description: "Globally scoped network is manually attachable by regular containers from workers in swarm mode."
type: "boolean"
Ingress:
description: "Ingress network is the network which provides the routing-mesh in swarm mode."
type: "boolean"
IPAM:
description: "Optional custom IP scheme for the network."
$ref: "#/definitions/IPAM"
@ -6416,6 +6425,7 @@ paths:
foo: "bar"
Internal: true
Attachable: false
Ingress: false
Options:
com.docker.network.bridge.default_bridge: "true"
com.docker.network.bridge.enable_icc: "true"

1
api/types/swarm/network.go
View file

@ -82,6 +82,7 @@ type NetworkSpec struct {
IPv6Enabled bool `json:",omitempty"`
Internal bool `json:",omitempty"`
Attachable bool `json:",omitempty"`
Ingress bool `json:",omitempty"`
IPAMOptions *IPAMOptions `json:",omitempty"`
}

2
api/types/types.go
View file

@ -400,6 +400,7 @@ type NetworkResource struct {
IPAM network.IPAM // IPAM is the network's IP Address Management
Internal bool // Internal represents if the network is used internal only
Attachable bool // Attachable represents if the global scope is manually attachable by regular containers from workers in swarm mode.
Ingress bool // Ingress indicates the network is providing the routing-mesh for the swarm cluster.
Containers map[string]EndpointResource // Containers contains endpoints belonging to the network
Options map[string]string // Options holds the network specific options to use for when creating the network
Labels map[string]string // Labels holds metadata specific to the network being created
@ -431,6 +432,7 @@ type NetworkCreate struct {
IPAM *network.IPAM
Internal bool
Attachable bool
Ingress bool
Options map[string]string
Labels map[string]string
}

4
cli/command/network/create.go
View file

@ -24,6 +24,7 @@ type createOptions struct {
internal bool
ipv6 bool
attachable bool
ingress bool
ipamDriver string
ipamSubnet []string
@ -59,6 +60,8 @@ func newCreateCommand(dockerCli *command.DockerCli) *cobra.Command {
flags.BoolVar(&opts.ipv6, "ipv6", false, "Enable IPv6 networking")
flags.BoolVar(&opts.attachable, "attachable", false, "Enable manual container attachment")
flags.SetAnnotation("attachable", "version", []string{"1.25"})
flags.BoolVar(&opts.ingress, "ingress", false, "Create swarm routing-mesh network")
flags.SetAnnotation("ingress", "version", []string{"1.29"})
flags.StringVar(&opts.ipamDriver, "ipam-driver", "default", "IP Address Management Driver")
flags.StringSliceVar(&opts.ipamSubnet, "subnet", []string{}, "Subnet in CIDR format that represents a network segment")
@ -92,6 +95,7 @@ func runCreate(dockerCli *command.DockerCli, opts createOptions) error {
Internal: opts.internal,
EnableIPv6: opts.ipv6,
Attachable: opts.attachable,
Ingress: opts.ingress,
Labels: runconfigopts.ConvertKVStringsToMap(opts.labels.GetAll()),
}

10
cli/command/network/remove.go
View file

@ -22,12 +22,22 @@ func newRemoveCommand(dockerCli *command.DockerCli) *cobra.Command {
}
}
const ingressWarning = "WARNING! Before removing the routing-mesh network, " +
"make sure all the nodes in your swarm run the same docker engine version. " +
"Otherwise, removal may not be effective and functionality of newly create " +
"ingress networks will be impaired.\nAre you sure you want to continue?"
func runRemove(dockerCli *command.DockerCli, networks []string) error {
client := dockerCli.Client()
ctx := context.Background()
status := 0
for _, name := range networks {
if nw, _, err := client.NetworkInspectWithRaw(ctx, name, false); err == nil &&
nw.Ingress &&
!command.PromptForConfirmation(dockerCli.In(), dockerCli.Out(), ingressWarning) {
continue
}
if err := client.NetworkRemove(ctx, name); err != nil {
fmt.Fprintf(dockerCli.Err(), "%s\n", err)
status = 1

3
daemon/cluster/convert/network.go
View file

@ -28,6 +28,7 @@ func networkFromGRPC(n *swarmapi.Network) types.Network {
IPv6Enabled: n.Spec.Ipv6Enabled,
Internal: n.Spec.Internal,
Attachable: n.Spec.Attachable,
Ingress: n.Spec.Ingress,
IPAMOptions: ipamFromGRPC(n.Spec.IPAM),
},
IPAMOptions: ipamFromGRPC(n.IPAM),
@ -156,6 +157,7 @@ func BasicNetworkFromGRPC(n swarmapi.Network) basictypes.NetworkResource {
IPAM: ipam,
Internal: spec.Internal,
Attachable: spec.Attachable,
Ingress: spec.Ingress,
Labels: n.Spec.Annotations.Labels,
}
@ -181,6 +183,7 @@ func BasicNetworkCreateToGRPC(create basictypes.NetworkCreateRequest) swarmapi.N
Ipv6Enabled: create.EnableIPv6,
Internal: create.Internal,
Attachable: create.Attachable,
Ingress: create.Ingress,
}
if create.IPAM != nil {
driver := create.IPAM.Driver

1
daemon/cluster/executor/backend.go
View file

@ -28,6 +28,7 @@ type Backend interface {
DeleteManagedNetwork(name string) error
FindNetwork(idName string) (libnetwork.Network, error)
SetupIngress(req clustertypes.NetworkCreateRequest, nodeIP string) error
ReleaseIngress() error
PullImage(ctx context.Context, image, tag string, metaHeaders map[string][]string, authConfig *types.AuthConfig, outStream io.Writer) error
CreateManagedContainer(config types.ContainerCreateConfig) (container.ContainerCreateCreatedBody, error)
ContainerStart(name string, hostConfig *container.HostConfig, checkpoint string, checkpointDir string) error

1
daemon/cluster/executor/container/container.go
View file

@ -575,6 +575,7 @@ func (c *containerConfig) networkCreateRequest(name string) (clustertypes.Networ
Labels: na.Network.Spec.Annotations.Labels,
Internal: na.Network.Spec.Internal,
Attachable: na.Network.Spec.Attachable,
Ingress: na.Network.Spec.Ingress,
EnableIPv6: na.Network.Spec.Ipv6Enabled,
CheckDuplicate: true,
}

2
daemon/cluster/executor/container/executor.go
View file

@ -116,6 +116,7 @@ func (e *executor) Describe(ctx context.Context) (*api.NodeDescription, error) {
func (e *executor) Configure(ctx context.Context, node *api.Node) error {
na := node.Attachment
if na == nil {
e.backend.ReleaseIngress()
return nil
}
@ -125,6 +126,7 @@ func (e *executor) Configure(ctx context.Context, node *api.Node) error {
Driver: na.Network.IPAM.Driver.Name,
},
Options: na.Network.DriverState.Options,
Ingress: true,
CheckDuplicate: true,
}

201
daemon/network.go
View file

@ -6,6 +6,7 @@ import (
"runtime"
"sort"
"strings"
"sync"
"github.com/Sirupsen/logrus"
apierrors "github.com/docker/docker/api/errors"
@ -99,15 +100,40 @@ func (daemon *Daemon) getAllNetworks() []libnetwork.Network {
return daemon.netController.Networks()
}
func isIngressNetwork(name string) bool {
return name == "ingress"
type ingressJob struct {
create *clustertypes.NetworkCreateRequest
ip net.IP
}
var ingressChan = make(chan struct{}, 1)
var (
ingressWorkerOnce sync.Once
ingressJobsChannel chan *ingressJob
ingressID string
)
func ingressWait() func() {
ingressChan <- struct{}{}
return func() { <-ingressChan }
func (daemon *Daemon) startIngressWorker() {
ingressJobsChannel = make(chan *ingressJob, 100)
go func() {
for {
select {
case r := <-ingressJobsChannel:
if r.create != nil {
daemon.setupIngress(r.create, r.ip, ingressID)
ingressID = r.create.ID
} else {
daemon.releaseIngress(ingressID)
ingressID = ""
}
}
}
}()
}
// enqueueIngressJob adds a ingress add/rm request to the worker queue.
// It guarantees the worker is started.
func (daemon *Daemon) enqueueIngressJob(job *ingressJob) {
ingressWorkerOnce.Do(daemon.startIngressWorker)
ingressJobsChannel <- job
}
// SetupIngress setups ingress networking.
@ -116,74 +142,95 @@ func (daemon *Daemon) SetupIngress(create clustertypes.NetworkCreateRequest, nod
if err != nil {
return err
}
go func() {
controller := daemon.netController
controller.AgentInitWait()
if n, err := daemon.GetNetworkByName(create.Name); err == nil && n != nil && n.ID() != create.ID {
if err := controller.SandboxDestroy("ingress-sbox"); err != nil {
logrus.Errorf("Failed to delete stale ingress sandbox: %v", err)
return
}
// Cleanup any stale endpoints that might be left over during previous iterations
epList := n.Endpoints()
for _, ep := range epList {
if err := ep.Delete(true); err != nil {
logrus.Errorf("Failed to delete endpoint %s (%s): %v", ep.Name(), ep.ID(), err)
}
}
if err := n.Delete(); err != nil {
logrus.Errorf("Failed to delete stale ingress network %s: %v", n.ID(), err)
return
}
}
if _, err := daemon.createNetwork(create.NetworkCreateRequest, create.ID, true); err != nil {
// If it is any other error other than already
// exists error log error and return.
if _, ok := err.(libnetwork.NetworkNameError); !ok {
logrus.Errorf("Failed creating ingress network: %v", err)
return
}
// Otherwise continue down the call to create or recreate sandbox.
}
n, err := daemon.GetNetworkByID(create.ID)
if err != nil {
logrus.Errorf("Failed getting ingress network by id after creating: %v", err)
return
}
sb, err := controller.NewSandbox("ingress-sbox", libnetwork.OptionIngress())
if err != nil {
if _, ok := err.(networktypes.ForbiddenError); !ok {
logrus.Errorf("Failed creating ingress sandbox: %v", err)
}
return
}
ep, err := n.CreateEndpoint("ingress-endpoint", libnetwork.CreateOptionIpam(ip, nil, nil, nil))
if err != nil {
logrus.Errorf("Failed creating ingress endpoint: %v", err)
return
}
if err := ep.Join(sb, nil); err != nil {
logrus.Errorf("Failed joining ingress sandbox to ingress endpoint: %v", err)
}
if err := sb.EnableService(); err != nil {
logrus.WithError(err).Error("Failed enabling service for ingress sandbox")
}
}()
daemon.enqueueIngressJob(&ingressJob{&create, ip})
return nil
}
// ReleaseIngress releases the ingress networking.
func (daemon *Daemon) ReleaseIngress() error {
daemon.enqueueIngressJob(&ingressJob{nil, nil})
return nil
}
func (daemon *Daemon) setupIngress(create *clustertypes.NetworkCreateRequest, ip net.IP, staleID string) {
controller := daemon.netController
controller.AgentInitWait()
if staleID != "" && staleID != create.ID {
daemon.releaseIngress(staleID)
}
if _, err := daemon.createNetwork(create.NetworkCreateRequest, create.ID, true); err != nil {
// If it is any other error other than already
// exists error log error and return.
if _, ok := err.(libnetwork.NetworkNameError); !ok {
logrus.Errorf("Failed creating ingress network: %v", err)
return
}
// Otherwise continue down the call to create or recreate sandbox.
}
n, err := daemon.GetNetworkByID(create.ID)
if err != nil {
logrus.Errorf("Failed getting ingress network by id after creating: %v", err)
}
sb, err := controller.NewSandbox("ingress-sbox", libnetwork.OptionIngress())
if err != nil {
if _, ok := err.(networktypes.ForbiddenError); !ok {
logrus.Errorf("Failed creating ingress sandbox: %v", err)
}
return
}
ep, err := n.CreateEndpoint("ingress-endpoint", libnetwork.CreateOptionIpam(ip, nil, nil, nil))
if err != nil {
logrus.Errorf("Failed creating ingress endpoint: %v", err)
return
}
if err := ep.Join(sb, nil); err != nil {
logrus.Errorf("Failed joining ingress sandbox to ingress endpoint: %v", err)
return
}
if err := sb.EnableService(); err != nil {
logrus.Errorf("Failed enabling service for ingress sandbox")
}
}
func (daemon *Daemon) releaseIngress(id string) {
controller := daemon.netController
if err := controller.SandboxDestroy("ingress-sbox"); err != nil {
logrus.Errorf("Failed to delete ingress sandbox: %v", err)
}
if id == "" {
return
}
n, err := controller.NetworkByID(id)
if err != nil {
logrus.Errorf("failed to retrieve ingress network %s: %v", id, err)
return
}
for _, ep := range n.Endpoints() {
if err := ep.Delete(true); err != nil {
logrus.Errorf("Failed to delete endpoint %s (%s): %v", ep.Name(), ep.ID(), err)
return
}
}
if err := n.Delete(); err != nil {
logrus.Errorf("Failed to delete ingress network %s: %v", n.ID(), err)
return
}
return
}
// SetNetworkBootstrapKeys sets the bootstrap keys.
func (daemon *Daemon) SetNetworkBootstrapKeys(keys []*networktypes.EncryptionKey) error {
return daemon.netController.SetKeys(keys)
@ -228,13 +275,6 @@ func (daemon *Daemon) CreateNetwork(create types.NetworkCreateRequest) (*types.N
}
func (daemon *Daemon) createNetwork(create types.NetworkCreateRequest, id string, agent bool) (*types.NetworkCreateResponse, error) {
// If there is a pending ingress network creation wait here
// since ingress network creation can happen via node download
// from manager or task download.
if isIngressNetwork(create.Name) {
defer ingressWait()()
}
if runconfig.IsPreDefinedNetwork(create.Name) && !agent {
err := fmt.Errorf("%s is a pre-defined network and cannot be created", create.Name)
return nil, apierrors.NewRequestForbiddenError(err)
@ -267,6 +307,7 @@ func (daemon *Daemon) createNetwork(create types.NetworkCreateRequest, id string
libnetwork.NetworkOptionDriverOpts(create.Options),
libnetwork.NetworkOptionLabels(create.Labels),
libnetwork.NetworkOptionAttachable(create.Attachable),
libnetwork.NetworkOptionIngress(create.Ingress),
}
if create.IPAM != nil {
@ -286,10 +327,6 @@ func (daemon *Daemon) createNetwork(create types.NetworkCreateRequest, id string
nwOptions = append(nwOptions, libnetwork.NetworkOptionPersist(false))
}
if isIngressNetwork(create.Name) {
nwOptions = append(nwOptions, libnetwork.NetworkOptionIngress())
}
n, err := c.NewNetwork(driver, create.Name, id, nwOptions...)
if err != nil {
return nil, err

3
daemon/prune.go
View file

@ -231,7 +231,8 @@ func (daemon *Daemon) clusterNetworksPrune(pruneFilters filters.Args) (*types.Ne
}
networkIsInUse := regexp.MustCompile(`network ([[:alnum:]]+) is in use`)
for _, nw := range networks {
if nw.Name == "ingress" {
if nw.Ingress {
// Routing-mesh network removal has to be explicitly invoked by user
continue
}
if !until.IsZero() && nw.Created.After(until) {

4
docs/api/version-history.md
View file

@ -17,6 +17,10 @@ keywords: "API, Docker, rcli, REST, documentation"
[Docker Engine API v1.29](https://docs.docker.com/engine/api/v1.29/) documentation
* `DELETE /networks/(name)` now allows to remove the ingress network, the one used to provide the routing-mesh.
* `POST /networks/create` now supports creating the ingress network, by specifying an `Ingress` boolean field. As of now this is supported only when using the overlay network driver.
* `GET /networks/(name)` now returns an `Ingress` field showing whether the network is the ingress one.
* `GET /networks/` now supports a `scope` filter to filter networks based on the network mode (`swarm`, `global`, or `local`).
## v1.28 API changes

18
docs/reference/commandline/network_create.md
View file

@ -22,6 +22,7 @@ Create a network
Options:
--attachable Enable manual container attachment
--ingress Specify the network provides the routing-mesh
--aux-address value Auxiliary IPv4 or IPv6 addresses used by Network
driver (default map[])
-d, --driver string Driver to manage the Network (default "bridge")
@ -195,6 +196,23 @@ connects a bridge network to it to provide external connectivity. If you want
to create an externally isolated `overlay` network, you can specify the
`--internal` option.
### Network ingress mode
You can create the network which will be used to provide the routing-mesh in the
swarm cluster. You do so by specifying `--ingress` when creating the network. Only
one ingress network can be created at the time. The network can be removed only
if no services depend on it. Any option available when creating a overlay network
is also available when creating the ingress network, besides the `--attachable` option.
```bash
$ docker network create -d overlay \
--subnet=10.11.0.0/16 \
--ingress \
--opt com.docker.network.mtu=9216 \
--opt encrypted=true \
my-ingress-network
```
## Related commands
* [network inspect](network_inspect.md)

55
integration-cli/docker_cli_swarm_test.go
View file

@ -10,6 +10,7 @@ import (
"net/http"
"net/http/httptest"
"os"
"os/exec"
"path/filepath"
"strings"
"time"
@ -19,6 +20,7 @@ import (
"github.com/docker/docker/integration-cli/checker"
"github.com/docker/docker/integration-cli/cli"
"github.com/docker/docker/integration-cli/daemon"
"github.com/docker/docker/pkg/testutil"
icmd "github.com/docker/docker/pkg/testutil/cmd"
"github.com/docker/libnetwork/driverapi"
"github.com/docker/libnetwork/ipamapi"
@ -413,14 +415,57 @@ func (s *DockerSwarmSuite) TestOverlayAttachableReleaseResourcesOnFailure(c *che
c.Assert(err, checker.IsNil, check.Commentf(out))
}
func (s *DockerSwarmSuite) TestSwarmRemoveInternalNetwork(c *check.C) {
func (s *DockerSwarmSuite) TestSwarmIngressNetwork(c *check.C) {
d := s.AddDaemon(c, true, true)
name := "ingress"
out, err := d.Cmd("network", "rm", name)
// Ingress network can be removed
out, _, err := testutil.RunCommandPipelineWithOutput(
exec.Command("echo", "Y"),
exec.Command("docker", "-H", d.Sock(), "network", "rm", "ingress"),
)
c.Assert(err, checker.IsNil, check.Commentf(out))
// And recreated
out, err = d.Cmd("network", "create", "-d", "overlay", "--ingress", "new-ingress")
c.Assert(err, checker.IsNil, check.Commentf(out))
// But only one is allowed
out, err = d.Cmd("network", "create", "-d", "overlay", "--ingress", "another-ingress")
c.Assert(err, checker.NotNil)
c.Assert(strings.TrimSpace(out), checker.Contains, name)
c.Assert(strings.TrimSpace(out), checker.Contains, "is a pre-defined network and cannot be removed")
c.Assert(strings.TrimSpace(out), checker.Contains, "is already present")
// It cannot be removed if it is being used
out, err = d.Cmd("service", "create", "--name", "srv1", "-p", "9000:8000", "busybox", "top")
c.Assert(err, checker.IsNil, check.Commentf(out))
out, _, err = testutil.RunCommandPipelineWithOutput(
exec.Command("echo", "Y"),
exec.Command("docker", "-H", d.Sock(), "network", "rm", "new-ingress"),
)
c.Assert(err, checker.NotNil)
c.Assert(strings.TrimSpace(out), checker.Contains, "ingress network cannot be removed because service")
// But it can be removed once no more services depend on it
out, err = d.Cmd("service", "update", "--publish-rm", "9000:8000", "srv1")
c.Assert(err, checker.IsNil, check.Commentf(out))
out, _, err = testutil.RunCommandPipelineWithOutput(
exec.Command("echo", "Y"),
exec.Command("docker", "-H", d.Sock(), "network", "rm", "new-ingress"),
)
c.Assert(err, checker.IsNil, check.Commentf(out))
// A service which needs the ingress network cannot be created if no ingress is present
out, err = d.Cmd("service", "create", "--name", "srv2", "-p", "500:500", "busybox", "top")
c.Assert(err, checker.NotNil)
c.Assert(strings.TrimSpace(out), checker.Contains, "no ingress network is present")
// An existing service cannot be updated to use the ingress nw if the nw is not present
out, err = d.Cmd("service", "update", "--publish-add", "9000:8000", "srv1")
c.Assert(err, checker.NotNil)
c.Assert(strings.TrimSpace(out), checker.Contains, "no ingress network is present")
// But services which do not need routing mesh can be created regardless
out, err = d.Cmd("service", "create", "--name", "srv3", "--endpoint-mode", "dnsrr", "busybox", "top")
c.Assert(err, checker.IsNil, check.Commentf(out))
}
// Test case for #24108, also the case from:

17
man/src/network/create.md
View file

@ -117,3 +117,20 @@ By default, when you connect a container to an `overlay` network, Docker also
connects a bridge network to it to provide external connectivity. If you want
to create an externally isolated `overlay` network, you can specify the
`--internal` option.
### Network ingress mode
You can create the network which will be used to provide the routing-mesh in the
swarm cluster. You do so by specifying `--ingress` when creating the network. Only
one ingress network can be created at the time. The network can be removed only
if no services depend on it. Any option available when creating a overlay network
is also available when creating the ingress network, besides the `--attachable` option.
```bash
$ docker network create -d overlay \
--subnet=10.11.0.0/16 \
--ingress \
--opt com.docker.network.mtu=9216 \
--opt encrypted=true \
my-ingress-network
```

2
man/src/network/inspect.md
View file

@ -32,6 +32,7 @@ $ sudo docker network inspect bridge
]
},
"Internal": false,
"Ingress": false,
"Containers": {
"bda12f8922785d1f160be70736f26c1e331ab8aaf8ed8d56728508f2e2fd4727": {
"Name": "container2",
@ -116,6 +117,7 @@ $ docker network inspect --verbose ov1
},
"Internal": false,
"Attachable": false,
"Ingress": false,
"Containers": {
"020403bd88a15f60747fd25d1ad5fa1272eb740e8a97fc547d8ad07b2f721c5e": {
"Name": "s1.1.pjn2ik0sfgkfzed3h0s00gs9o",

2
runconfig/hostconfig_unix.go
View file

@ -19,7 +19,7 @@ func DefaultDaemonNetworkMode() container.NetworkMode {
// IsPreDefinedNetwork indicates if a network is predefined by the daemon
func IsPreDefinedNetwork(network string) bool {
n := container.NetworkMode(network)
return n.IsBridge() || n.IsHost() || n.IsNone() || n.IsDefault() || network == "ingress"
return n.IsBridge() || n.IsHost() || n.IsNone() || n.IsDefault()
}
// validateNetMode ensures that the various combinations of requested

4
vendor.conf
View file

@ -24,7 +24,7 @@ github.com/RackSec/srslog 456df3a81436d29ba874f3590eeeee25d666f8a5
github.com/imdario/mergo 0.2.1
#get libnetwork packages
github.com/docker/libnetwork 4610dd67c7b9828bb4719d8aa2ac53a7f1f739d2
github.com/docker/libnetwork b6cb1eee1e7fc27ee05f0eb830d3e60e67a88565
github.com/docker/go-events 18b43f1bc85d9cdd42c05a6cd2d444c7a200a894
github.com/armon/go-radix e39d623f12e8e41c7b5529e9a9dd67a1e2261f80
github.com/armon/go-metrics eb0af217e5e9747e41dd5303755356b62d28e3ec
@ -105,7 +105,7 @@ github.com/docker/containerd 422e31ce907fd9c3833a38d7b8fdd023e5a76e73
github.com/tonistiigi/fifo 1405643975692217d6720f8b54aeee1bf2cd5cf4
# cluster
github.com/docker/swarmkit 0e2d9ebcea9d5bbd4a06b3b964fb96356801f880
github.com/docker/swarmkit 9fdea50c14492b6e1f472813849794d36bfef217
github.com/golang/mock bd3c8e81be01eef76d4b503f5e687d2d1354d2d9
github.com/gogo/protobuf 8d70fb3182befc465c4a1eac8ad4d38ff49778e2
github.com/cloudflare/cfssl 7fb22c8cba7ecaf98e4082d22d65800cf45e042a

24
vendor/github.com/docker/libnetwork/controller.go generated vendored
View file

@ -682,6 +682,10 @@ func (c *controller) NewNetwork(networkType, name string, id string, options ...
return nil, err
}
if network.ingress && cap.DataScope != datastore.GlobalScope {
return nil, types.ForbiddenErrorf("Ingress network can only be global scope network")
}
if cap.DataScope == datastore.GlobalScope && !c.isDistributedControl() && !network.dynamic {
if c.isManager() {
// For non-distributed controlled environment, globalscoped non-dynamic networks are redirected to Manager
@ -1161,15 +1165,29 @@ func (c *controller) clearIngress(clusterLeave bool) {
c.ingressSandbox = nil
c.Unlock()
var n *network
if ingressSandbox != nil {
for _, ep := range ingressSandbox.getConnectedEndpoints() {
if nw := ep.getNetwork(); nw.ingress {
n = nw
break
}
}
if err := ingressSandbox.Delete(); err != nil {
logrus.Warnf("Could not delete ingress sandbox while leaving: %v", err)
}
}
n, err := c.NetworkByName("ingress")
if err != nil && clusterLeave {
logrus.Warnf("Could not find ingress network while leaving: %v", err)
if n == nil {
for _, nw := range c.Networks() {
if nw.Info().Ingress() {
n = nw.(*network)
break
}
}
}
if n == nil && clusterLeave {
logrus.Warnf("Could not find ingress network while leaving")
}
if n != nil {

33
vendor/github.com/docker/libnetwork/drivers/bridge/bridge.go generated vendored
View file

@ -28,11 +28,11 @@ import (
)
const (
networkType = "bridge"
vethPrefix = "veth"
vethLen = 7
containerVethPrefix = "eth"
maxAllocatePortAttempts = 10
networkType = "bridge"
vethPrefix = "veth"
vethLen = 7
defaultContainerVethPrefix = "eth"
maxAllocatePortAttempts = 10
)
const (
@ -55,14 +55,15 @@ type configuration struct {
// networkConfiguration for network specific configuration
type networkConfiguration struct {
ID string
BridgeName string
EnableIPv6 bool
EnableIPMasquerade bool
EnableICC bool
Mtu int
DefaultBindingIP net.IP
DefaultBridge bool
ID string
BridgeName string
EnableIPv6 bool
EnableIPMasquerade bool
EnableICC bool
Mtu int
DefaultBindingIP net.IP
DefaultBridge bool
ContainerIfacePrefix string
// Internal fields set after ipam data parsing
AddressIPv4 *net.IPNet
AddressIPv6 *net.IPNet
@ -239,6 +240,8 @@ func (c *networkConfiguration) fromLabels(labels map[string]string) error {
if c.DefaultBindingIP = net.ParseIP(value); c.DefaultBindingIP == nil {
return parseErr(label, value, "nil ip")
}
case netlabel.ContainerIfacePrefix:
c.ContainerIfacePrefix = value
}
}
@ -1221,6 +1224,10 @@ func (d *driver) Join(nid, eid string, sboxKey string, jinfo driverapi.JoinInfo,
}
iNames := jinfo.InterfaceName()
containerVethPrefix := defaultContainerVethPrefix
if network.config.ContainerIfacePrefix != "" {
containerVethPrefix = network.config.ContainerIfacePrefix
}
err = iNames.SetNames(endpoint.srcName, containerVethPrefix)
if err != nil {
return err

5
vendor/github.com/docker/libnetwork/drivers/bridge/bridge_store.go generated vendored
View file

@ -143,6 +143,7 @@ func (ncfg *networkConfiguration) MarshalJSON() ([]byte, error) {
nMap["DefaultBindingIP"] = ncfg.DefaultBindingIP.String()
nMap["DefaultGatewayIPv4"] = ncfg.DefaultGatewayIPv4.String()
nMap["DefaultGatewayIPv6"] = ncfg.DefaultGatewayIPv6.String()
nMap["ContainerIfacePrefix"] = ncfg.ContainerIfacePrefix
nMap["BridgeIfaceCreator"] = ncfg.BridgeIfaceCreator
if ncfg.AddressIPv4 != nil {
@ -178,6 +179,10 @@ func (ncfg *networkConfiguration) UnmarshalJSON(b []byte) error {
}
}
if v, ok := nMap["ContainerIfacePrefix"]; ok {
ncfg.ContainerIfacePrefix = v.(string)
}
ncfg.DefaultBridge = nMap["DefaultBridge"].(bool)
ncfg.DefaultBindingIP = net.ParseIP(nMap["DefaultBindingIP"].(string))
ncfg.DefaultGatewayIPv4 = net.ParseIP(nMap["DefaultGatewayIPv4"].(string))

3
vendor/github.com/docker/libnetwork/netlabel/labels.go generated vendored
View file

@ -50,6 +50,9 @@ const (
// Internal constant represents that the network is internal which disables default gateway service
Internal = Prefix + ".internal"
// ContainerIfacePrefix can be used to override the interface prefix used inside the container
ContainerIfacePrefix = Prefix + ".container_iface_prefix"
)
var (

12
vendor/github.com/docker/libnetwork/network.go generated vendored
View file

@ -66,6 +66,7 @@ type NetworkInfo interface {
IPv6Enabled() bool
Internal() bool
Attachable() bool
Ingress() bool
Labels() map[string]string
Dynamic() bool
Created() time.Time
@ -615,9 +616,9 @@ func NetworkOptionGeneric(generic map[string]interface{}) NetworkOption {
// NetworkOptionIngress returns an option setter to indicate if a network is
// an ingress network.
func NetworkOptionIngress() NetworkOption {
func NetworkOptionIngress(ingress bool) NetworkOption {
return func(n *network) {
n.ingress = true
n.ingress = ingress
}
}
@ -1589,6 +1590,13 @@ func (n *network) Attachable() bool {
return n.attachable
}
func (n *network) Ingress() bool {
n.Lock()
defer n.Unlock()
return n.ingress
}
func (n *network) Dynamic() bool {
n.Lock()
defer n.Unlock()

4
vendor/github.com/docker/libnetwork/osl/interface_linux.go generated vendored
View file

@ -241,8 +241,8 @@ func (n *networkNamespace) AddInterface(srcName, dstPrefix string, options ...If
if n.isDefault {
i.dstName = i.srcName
} else {
i.dstName = fmt.Sprintf("%s%d", i.dstName, n.nextIfIndex)
n.nextIfIndex++
i.dstName = fmt.Sprintf("%s%d", dstPrefix, n.nextIfIndex[dstPrefix])
n.nextIfIndex[dstPrefix]++
}
path := n.path

10
vendor/github.com/docker/libnetwork/osl/namespace_linux.go generated vendored
View file

@ -48,7 +48,7 @@ type networkNamespace struct {
gwv6 net.IP
staticRoutes []*types.StaticRoute
neighbors []*neigh
nextIfIndex int
nextIfIndex map[string]int
isDefault bool
nlHandle *netlink.Handle
loV6Enabled bool
@ -203,7 +203,7 @@ func NewSandbox(key string, osCreate, isRestore bool) (Sandbox, error) {
once.Do(createBasePath)
}
n := &networkNamespace{path: key, isDefault: !osCreate}
n := &networkNamespace{path: key, isDefault: !osCreate, nextIfIndex: make(map[string]int)}
sboxNs, err := netns.GetFromPath(n.path)
if err != nil {
@ -256,7 +256,7 @@ func GetSandboxForExternalKey(basePath string, key string) (Sandbox, error) {
if err := mountNetworkNamespace(basePath, key); err != nil {
return nil, err
}
n := &networkNamespace{path: key}
n := &networkNamespace{path: key, nextIfIndex: make(map[string]int)}
sboxNs, err := netns.GetFromPath(n.path)
if err != nil {
@ -495,8 +495,8 @@ func (n *networkNamespace) Restore(ifsopt map[string][]IfaceOption, routes []*ty
}
index++
n.Lock()
if index > n.nextIfIndex {
n.nextIfIndex = index
if index > n.nextIfIndex[dstPrefix] {
n.nextIfIndex[dstPrefix] = index
}
n.iFaces = append(n.iFaces, i)
n.Unlock()

250
vendor/github.com/docker/swarmkit/api/specs.pb.go generated vendored
View file

@ -591,6 +591,11 @@ type NetworkSpec struct {
// enabled(default case) no manual attachment to this network
// can happen.
Attachable bool `protobuf:"varint,6,opt,name=attachable,proto3" json:"attachable,omitempty"`
// Ingress indicates this network will provide the routing-mesh.
// In older versions, the network providing the routing mesh was
// swarm internally created only and it was identified by the name
// "ingress" and the label "com.docker.swarm.internal": "true".
Ingress bool `protobuf:"varint,7,opt,name=ingress,proto3" json:"ingress,omitempty"`
}
func (m *NetworkSpec) Reset() { *m = NetworkSpec{} }
@ -1795,6 +1800,16 @@ func (m *NetworkSpec) MarshalTo(dAtA []byte) (int, error) {
}
i++
}
if m.Ingress {
dAtA[i] = 0x38
i++
if m.Ingress {
dAtA[i] = 1
} else {
dAtA[i] = 0
}
i++
}
return i, nil
}
@ -2255,6 +2270,9 @@ func (m *NetworkSpec) Size() (n int) {
if m.Attachable {
n += 2
}
if m.Ingress {
n += 2
}
return n
}
@ -2502,6 +2520,7 @@ func (this *NetworkSpec) String() string {
`Internal:` + fmt.Sprintf("%v", this.Internal) + `,`,
`IPAM:` + strings.Replace(fmt.Sprintf("%v", this.IPAM), "IPAMOptions", "IPAMOptions", 1) + `,`,
`Attachable:` + fmt.Sprintf("%v", this.Attachable) + `,`,
`Ingress:` + fmt.Sprintf("%v", this.Ingress) + `,`,
`}`,
}, "")
return s
@ -4688,6 +4707,26 @@ func (m *NetworkSpec) Unmarshal(dAtA []byte) error {
}
}
m.Attachable = bool(v != 0)
case 7:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field Ingress", wireType)
}
var v int
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowSpecs
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
v |= (int(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
m.Ingress = bool(v != 0)
default:
iNdEx = preIndex
skippy, err := skipSpecs(dAtA[iNdEx:])
@ -5218,112 +5257,113 @@ var (
func init() { proto.RegisterFile("specs.proto", fileDescriptorSpecs) }
var fileDescriptorSpecs = []byte{
// 1707 bytes of a gzipped FileDescriptorProto
// 1717 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xac, 0x57, 0x41, 0x73, 0x1b, 0xb7,
0x15, 0x26, 0x25, 0x8a, 0x5a, 0xbe, 0xa5, 0x6c, 0x1a, 0x75, 0xd2, 0x35, 0xdd, 0x90, 0x34, 0xe3,
0x15, 0x16, 0x25, 0x8a, 0x5a, 0xbe, 0xa5, 0x6c, 0x1a, 0x75, 0xd2, 0x35, 0xdd, 0x50, 0x34, 0xe3,
0xa6, 0x4a, 0x33, 0xa5, 0xa6, 0x6a, 0x27, 0x75, 0xea, 0x66, 0x5a, 0x52, 0x64, 0x65, 0x55, 0x95,
0xcc, 0x01, 0x15, 0x77, 0x7c, 0xe2, 0x80, 0xbb, 0x10, 0xb9, 0xa3, 0xe5, 0x62, 0x0b, 0x60, 0x99,
0xe1, 0xad, 0xc7, 0x8c, 0x0f, 0x3d, 0xf5, 0xaa, 0xe9, 0xa1, 0xbf, 0xa1, 0xff, 0xc1, 0xc7, 0x1e,
0x7b, 0xd2, 0x34, 0xfc, 0x0b, 0xfd, 0x01, 0xed, 0x00, 0x0b, 0x92, 0xcb, 0x64, 0x15, 0x7b, 0x26,
0xbe, 0xe1, 0xbd, 0xfd, 0xbe, 0x07, 0xe0, 0xe1, 0xc3, 0xc3, 0x5b, 0xb0, 0x45, 0x44, 0x5d, 0xd1,
0x8a, 0x38, 0x93, 0x0c, 0x21, 0x8f, 0xb9, 0x57, 0x94, 0xb7, 0xc4, 0x97, 0x84, 0x4f, 0xaf, 0x7c,
0xd9, 0x9a, 0xfd, 0xbc, 0x6a, 0xcb, 0x79, 0x44, 0x0d, 0xa0, 0x7a, 0x7f, 0xcc, 0xc6, 0x4c, 0x0f,
0x0f, 0xd4, 0xc8, 0x78, 0x6b, 0x63, 0xc6, 0xc6, 0x01, 0x3d, 0xd0, 0xd6, 0x28, 0xbe, 0x3c, 0xf0,
0x62, 0x4e, 0xa4, 0xcf, 0xc2, 0xe4, 0x7b, 0xf3, 0xba, 0x00, 0xd6, 0x39, 0xf3, 0xe8, 0x20, 0xa2,
0x2e, 0x3a, 0x06, 0x9b, 0x84, 0x21, 0x93, 0x1a, 0x20, 0x9c, 0x7c, 0x23, 0xbf, 0x6f, 0x1f, 0xd6,
0x5b, 0xdf, 0x9e, 0xb9, 0xd5, 0x5e, 0xc3, 0x3a, 0x85, 0xd7, 0x37, 0xf5, 0x1c, 0x4e, 0x33, 0xd1,
0x6f, 0xa1, 0xec, 0x51, 0xe1, 0x73, 0xea, 0x0d, 0x39, 0x0b, 0xa8, 0xb3, 0xd5, 0xc8, 0xef, 0xdf,
0x39, 0xfc, 0x51, 0x56, 0x24, 0x35, 0x39, 0x66, 0x01, 0xc5, 0xb6, 0x61, 0x28, 0x03, 0x1d, 0x03,
0x4c, 0xe9, 0x74, 0x44, 0xb9, 0x98, 0xf8, 0x91, 0xb3, 0xad, 0xe9, 0x3f, 0xb9, 0x8d, 0xae, 0xd6,
0xde, 0x3a, 0x5b, 0xc1, 0x71, 0x8a, 0x8a, 0xce, 0xa0, 0x4c, 0x66, 0xc4, 0x0f, 0xc8, 0xc8, 0x0f,
0x7c, 0x39, 0x77, 0x0a, 0x3a, 0xd4, 0xc7, 0xdf, 0x19, 0xaa, 0x9d, 0x22, 0xe0, 0x0d, 0x7a, 0xd3,
0x03, 0x58, 0x4f, 0x84, 0x3e, 0x82, 0xdd, 0x7e, 0xef, 0xbc, 0x7b, 0x72, 0x7e, 0x5c, 0xc9, 0x55,
0x1f, 0xbc, 0xba, 0x6e, 0xbc, 0xa7, 0x62, 0xac, 0x01, 0x7d, 0x1a, 0x7a, 0x7e, 0x38, 0x46, 0xfb,
0x60, 0xb5, 0x8f, 0x8e, 0x7a, 0xfd, 0x8b, 0x5e, 0xb7, 0x92, 0xaf, 0x56, 0x5f, 0x5d, 0x37, 0xde,
0xdf, 0x04, 0xb6, 0x5d, 0x97, 0x46, 0x92, 0x7a, 0xd5, 0xc2, 0x57, 0xff, 0xa8, 0xe5, 0x9a, 0x5f,
0xe5, 0xa1, 0x9c, 0x5e, 0x04, 0xfa, 0x08, 0x8a, 0xed, 0xa3, 0x8b, 0x93, 0x17, 0xbd, 0x4a, 0x6e,
0x4d, 0x4f, 0x23, 0xda, 0xae, 0xf4, 0x67, 0x14, 0x3d, 0x86, 0x9d, 0x7e, 0xfb, 0x8b, 0x41, 0xaf,
0x92, 0x5f, 0x2f, 0x27, 0x0d, 0xeb, 0x93, 0x58, 0x68, 0x54, 0x17, 0xb7, 0x4f, 0xce, 0x2b, 0x5b,
0xd9, 0xa8, 0x2e, 0x27, 0x7e, 0x68, 0x96, 0xf2, 0xf7, 0x02, 0xd8, 0x03, 0xca, 0x67, 0xbe, 0xfb,
0x8e, 0x25, 0xf2, 0x29, 0x14, 0x24, 0x11, 0x57, 0x5a, 0x1a, 0x76, 0xb6, 0x34, 0x2e, 0x88, 0xb8,
0x52, 0x93, 0x1a, 0xba, 0xc6, 0x2b, 0x65, 0x70, 0x1a, 0x05, 0xbe, 0x4b, 0x24, 0xf5, 0xb4, 0x32,
0xec, 0xc3, 0x1f, 0x67, 0xb1, 0xf1, 0x0a, 0x65, 0xd6, 0xff, 0x2c, 0x87, 0x53, 0x54, 0xf4, 0x14,
0x8a, 0xe3, 0x80, 0x8d, 0x48, 0xa0, 0x35, 0x61, 0x1f, 0x3e, 0xca, 0x0a, 0x72, 0xac, 0x11, 0xeb,
0x00, 0x86, 0x82, 0x9e, 0x40, 0x31, 0x8e, 0x3c, 0x22, 0xa9, 0x53, 0xd4, 0xe4, 0x46, 0x16, 0xf9,
0x0b, 0x8d, 0x38, 0x62, 0xe1, 0xa5, 0x3f, 0xc6, 0x06, 0x8f, 0x4e, 0xc1, 0x0a, 0xa9, 0xfc, 0x92,
0xf1, 0x2b, 0xe1, 0xec, 0x36, 0xb6, 0xf7, 0xed, 0xc3, 0x4f, 0x32, 0xc5, 0x98, 0x60, 0xda, 0x52,
0x12, 0x77, 0x32, 0xa5, 0xa1, 0x4c, 0xc2, 0x74, 0xb6, 0x9c, 0x3c, 0x5e, 0x05, 0x40, 0xbf, 0x01,
0x8b, 0x86, 0x5e, 0xc4, 0xfc, 0x50, 0x3a, 0xd6, 0xed, 0x0b, 0xe9, 0x19, 0x8c, 0x4a, 0x26, 0x5e,
0x31, 0x14, 0x9b, 0xb3, 0x20, 0x18, 0x11, 0xf7, 0xca, 0x29, 0xbd, 0xe5, 0x36, 0x56, 0x8c, 0x4e,
0x11, 0x0a, 0x53, 0xe6, 0xd1, 0xe6, 0x01, 0xdc, 0xfb, 0x56, 0xaa, 0x51, 0x15, 0x2c, 0x93, 0xea,
0x44, 0x23, 0x05, 0xbc, 0xb2, 0x9b, 0x77, 0x61, 0x6f, 0x23, 0xad, 0xcd, 0xbf, 0x16, 0xc0, 0x5a,
0x9e, 0x35, 0x6a, 0x43, 0xc9, 0x65, 0xa1, 0x24, 0x7e, 0x48, 0xb9, 0x91, 0x57, 0xe6, 0xc9, 0x1c,
0x2d, 0x41, 0x8a, 0xf5, 0x2c, 0x87, 0xd7, 0x2c, 0xf4, 0x7b, 0x28, 0x71, 0x2a, 0x58, 0xcc, 0x5d,
0x2a, 0x8c, 0xbe, 0xf6, 0xb3, 0x15, 0x92, 0x80, 0x30, 0xfd, 0x73, 0xec, 0x73, 0xaa, 0xb2, 0x2c,
0xf0, 0x9a, 0x8a, 0x9e, 0xc2, 0x2e, 0xa7, 0x42, 0x12, 0x2e, 0xbf, 0x4b, 0x22, 0x38, 0x81, 0xf4,
0x59, 0xe0, 0xbb, 0x73, 0xbc, 0x64, 0xa0, 0xa7, 0x50, 0x8a, 0x02, 0xe2, 0xea, 0xa8, 0xce, 0x8e,
0xa6, 0x7f, 0x90, 0x45, 0xef, 0x2f, 0x41, 0x78, 0x8d, 0x47, 0x9f, 0x01, 0x04, 0x6c, 0x3c, 0xf4,
0xb8, 0x3f, 0xa3, 0xdc, 0x48, 0xac, 0x9a, 0xc5, 0xee, 0x6a, 0x04, 0x2e, 0x05, 0x6c, 0x9c, 0x0c,
0xd1, 0xf1, 0xf7, 0xd2, 0x57, 0x4a, 0x5b, 0xa7, 0x00, 0x64, 0xf5, 0xd5, 0xa8, 0xeb, 0xe3, 0xb7,
0x0a, 0x65, 0x4e, 0x24, 0x45, 0x47, 0x8f, 0xa0, 0x7c, 0xc9, 0xb8, 0x4b, 0x87, 0xe6, 0xd6, 0x94,
0xb4, 0x26, 0x6c, 0xed, 0x4b, 0xf4, 0xd5, 0x29, 0xc1, 0x2e, 0x8f, 0x43, 0xe9, 0x4f, 0x69, 0xf3,
0x14, 0xde, 0xcb, 0x0c, 0x8a, 0x0e, 0xa1, 0xbc, 0x3a, 0xe6, 0xa1, 0xef, 0x69, 0x7d, 0x94, 0x3a,
0x77, 0x17, 0x37, 0x75, 0x7b, 0xa5, 0x87, 0x93, 0x2e, 0xb6, 0x57, 0xa0, 0x13, 0xaf, 0xf9, 0x37,
0x0b, 0xf6, 0x36, 0xc4, 0x82, 0xee, 0xc3, 0x8e, 0x3f, 0x25, 0x63, 0x9a, 0xd0, 0x71, 0x62, 0xa0,
0x1e, 0x14, 0x03, 0x32, 0xa2, 0x81, 0x92, 0x8c, 0x4a, 0xdb, 0xcf, 0xde, 0xa8, 0xba, 0xd6, 0x1f,
0x35, 0xbe, 0x17, 0x4a, 0x3e, 0xc7, 0x86, 0x8c, 0x1c, 0xd8, 0x75, 0xd9, 0x74, 0x4a, 0x42, 0x55,
0x9c, 0xb6, 0xf7, 0x4b, 0x78, 0x69, 0x22, 0x04, 0x05, 0xc2, 0xc7, 0xc2, 0x29, 0x68, 0xb7, 0x1e,
0xa3, 0x0a, 0x6c, 0xd3, 0x70, 0xe6, 0xec, 0x68, 0x97, 0x1a, 0x2a, 0x8f, 0xe7, 0x27, 0x67, 0x5e,
0xc2, 0x6a, 0xa8, 0x78, 0xb1, 0xa0, 0xdc, 0xd9, 0xd5, 0x2e, 0x3d, 0x46, 0xbf, 0x82, 0xe2, 0x94,
0xc5, 0xa1, 0x14, 0x8e, 0xa5, 0x17, 0xfb, 0x20, 0x6b, 0xb1, 0x67, 0x0a, 0x61, 0x8a, 0xa7, 0x81,
0xa3, 0x1e, 0xdc, 0x13, 0x92, 0x45, 0xc3, 0x31, 0x27, 0x2e, 0x1d, 0x46, 0x94, 0xfb, 0xcc, 0x33,
0x97, 0xff, 0x41, 0x2b, 0xe9, 0x15, 0x5a, 0xcb, 0x5e, 0xa1, 0xd5, 0x35, 0xbd, 0x02, 0xbe, 0xab,
0x38, 0xc7, 0x8a, 0xd2, 0xd7, 0x0c, 0xd4, 0x87, 0x72, 0x14, 0x07, 0xc1, 0x90, 0x45, 0xc9, 0x3b,
0x00, 0x3a, 0xc2, 0x5b, 0xa4, 0xac, 0x1f, 0x07, 0xc1, 0xf3, 0x84, 0x84, 0xed, 0x68, 0x6d, 0xa0,
0xf7, 0xa1, 0x38, 0xe6, 0x2c, 0x8e, 0x84, 0x63, 0xeb, 0x64, 0x18, 0x0b, 0x7d, 0x0e, 0xbb, 0x82,
0xba, 0x9c, 0x4a, 0xe1, 0x94, 0xf5, 0x56, 0x3f, 0xcc, 0x9a, 0x64, 0xa0, 0x21, 0x98, 0x5e, 0x52,
0x4e, 0x43, 0x97, 0xe2, 0x25, 0x07, 0x3d, 0x80, 0x6d, 0x29, 0xe7, 0xce, 0x5e, 0x23, 0xbf, 0x6f,
0x75, 0x76, 0x17, 0x37, 0xf5, 0xed, 0x8b, 0x8b, 0x97, 0x58, 0xf9, 0x54, 0x8d, 0x9a, 0x30, 0x21,
0x43, 0x32, 0xa5, 0xce, 0x1d, 0x9d, 0xdb, 0x95, 0x8d, 0x5e, 0x02, 0x78, 0xa1, 0x18, 0xba, 0xfa,
0x52, 0x38, 0x77, 0xf5, 0xee, 0x3e, 0x79, 0xf3, 0xee, 0xba, 0xe7, 0x03, 0x53, 0xa7, 0xf7, 0x16,
0x37, 0xf5, 0xd2, 0xca, 0xc4, 0x25, 0x2f, 0x14, 0xc9, 0x10, 0x75, 0xc0, 0x9e, 0x50, 0x12, 0xc8,
0x89, 0x3b, 0xa1, 0xee, 0x95, 0x53, 0xb9, 0xbd, 0xf0, 0x3e, 0xd3, 0x30, 0x13, 0x21, 0x4d, 0x52,
0x0a, 0x56, 0x4b, 0x15, 0xce, 0x3d, 0x9d, 0xab, 0xc4, 0x40, 0x1f, 0x00, 0xb0, 0x88, 0x86, 0x43,
0x21, 0x3d, 0x3f, 0x74, 0x90, 0xda, 0x32, 0x2e, 0x29, 0xcf, 0x40, 0x39, 0xd0, 0x43, 0x55, 0x16,
0x89, 0x37, 0x64, 0x61, 0x30, 0x77, 0x7e, 0xa0, 0xbf, 0x5a, 0xca, 0xf1, 0x3c, 0x0c, 0xe6, 0xa8,
0x0e, 0xb6, 0xd6, 0x85, 0xf0, 0xc7, 0x21, 0x09, 0x9c, 0xfb, 0x3a, 0x1f, 0xa0, 0x5c, 0x03, 0xed,
0xa9, 0x7e, 0x06, 0x76, 0x4a, 0xee, 0x4a, 0xa6, 0x57, 0x74, 0x6e, 0x6e, 0x90, 0x1a, 0xaa, 0x35,
0xcd, 0x48, 0x10, 0x27, 0xcd, 0x5e, 0x09, 0x27, 0xc6, 0xaf, 0xb7, 0x9e, 0xe4, 0xab, 0x87, 0x60,
0xa7, 0x8e, 0x1d, 0x7d, 0x08, 0x7b, 0x9c, 0x8e, 0x7d, 0x21, 0xf9, 0x7c, 0x48, 0x62, 0x39, 0x71,
0x7e, 0xa7, 0x09, 0xe5, 0xa5, 0xb3, 0x1d, 0xcb, 0x49, 0x75, 0x08, 0xeb, 0xec, 0xa1, 0x06, 0xd8,
0xea, 0x54, 0x04, 0xe5, 0x33, 0xca, 0xd5, 0x83, 0xa2, 0x36, 0x9d, 0x76, 0x29, 0xf5, 0x08, 0x4a,
0xb8, 0x3b, 0xd1, 0x97, 0xb7, 0x84, 0x8d, 0xa5, 0x6e, 0xe3, 0x52, 0xa2, 0xe6, 0x36, 0x1a, 0xb3,
0xf9, 0xdf, 0x3c, 0x94, 0xd3, 0xef, 0x22, 0x3a, 0x4a, 0xde, 0x33, 0xbd, 0xa5, 0x3b, 0x87, 0x07,
0x6f, 0x7a, 0x47, 0xf5, 0xeb, 0x11, 0xc4, 0x2a, 0xd8, 0x99, 0x6a, 0x61, 0x35, 0x19, 0xfd, 0x12,
0x76, 0x22, 0xc6, 0xe5, 0xb2, 0x86, 0xd4, 0x32, 0x2b, 0x3e, 0xe3, 0xcb, 0x6a, 0x9b, 0x80, 0x9b,
0x13, 0xb8, 0xb3, 0x19, 0x0d, 0x3d, 0x86, 0xed, 0x17, 0x27, 0xfd, 0x4a, 0xae, 0xfa, 0xf0, 0xd5,
0x75, 0xe3, 0x87, 0x9b, 0x1f, 0x5f, 0xf8, 0x5c, 0xc6, 0x24, 0x38, 0xe9, 0xa3, 0x9f, 0xc2, 0x4e,
0xf7, 0x7c, 0x80, 0x71, 0x25, 0x5f, 0xad, 0xbf, 0xba, 0x6e, 0x3c, 0xdc, 0xc4, 0xa9, 0x4f, 0x2c,
0x0e, 0x3d, 0xcc, 0x46, 0xab, 0x76, 0xee, 0x9f, 0x5b, 0x60, 0x9b, 0xd2, 0xfa, 0xae, 0x3b, 0xfe,
0xbd, 0xe4, 0xb5, 0x5a, 0xde, 0x99, 0xad, 0x37, 0x3e, 0x5a, 0xe5, 0x84, 0x60, 0xce, 0xf8, 0x11,
0x94, 0xfd, 0x68, 0xf6, 0xe9, 0x90, 0x86, 0x64, 0x14, 0x98, 0xce, 0xce, 0xc2, 0xb6, 0xf2, 0xf5,
0x12, 0x97, 0xba, 0xb0, 0x7e, 0x28, 0x29, 0x0f, 0x4d, 0xcf, 0x66, 0xe1, 0x95, 0x8d, 0x3e, 0x87,
0x82, 0x1f, 0x91, 0xa9, 0x79, 0x69, 0x33, 0x77, 0x70, 0xd2, 0x6f, 0x9f, 0x19, 0x0d, 0x76, 0xac,
0xc5, 0x4d, 0xbd, 0xa0, 0x1c, 0x58, 0xd3, 0x50, 0x6d, 0xf9, 0xd8, 0xa9, 0x99, 0x74, 0xf1, 0xb5,
0x70, 0xca, 0xd3, 0xfc, 0x5f, 0x01, 0xec, 0xa3, 0x20, 0x16, 0xd2, 0x3c, 0x21, 0xef, 0x2c, 0x6f,
0x2f, 0xe1, 0x1e, 0xd1, 0xcd, 0x3f, 0x09, 0x55, 0x3d, 0xd6, 0x4d, 0x84, 0xc9, 0xdd, 0xe3, 0xcc,
0x70, 0x2b, 0x70, 0xd2, 0x70, 0x74, 0x8a, 0x2a, 0xa6, 0x93, 0xc7, 0x15, 0xf2, 0x8d, 0x2f, 0x68,
0x00, 0x7b, 0x8c, 0xbb, 0x13, 0x2a, 0x64, 0x52, 0xc5, 0x4d, 0xb3, 0x9c, 0xf9, 0x1b, 0xf5, 0x3c,
0x0d, 0x34, 0x25, 0x2c, 0x59, 0xed, 0x66, 0x0c, 0xf4, 0x04, 0x0a, 0x9c, 0x5c, 0x2e, 0x1b, 0xa2,
0x4c, 0x7d, 0x63, 0x72, 0x29, 0x37, 0x42, 0x68, 0x06, 0xfa, 0x03, 0x80, 0xe7, 0x8b, 0x88, 0x48,
0x77, 0x42, 0xb9, 0x39, 0xa7, 0xcc, 0x2d, 0x76, 0x57, 0xa8, 0x8d, 0x28, 0x29, 0x36, 0x3a, 0x85,
0x92, 0x4b, 0x96, 0x4a, 0x2b, 0xde, 0xfe, 0x07, 0x71, 0xd4, 0x36, 0x21, 0x2a, 0x2a, 0xc4, 0xe2,
0xa6, 0x6e, 0x2d, 0x3d, 0xd8, 0x72, 0x89, 0x51, 0xde, 0x29, 0xec, 0xa9, 0x3f, 0x8b, 0xa1, 0x47,
0x2f, 0x49, 0x1c, 0x48, 0xa1, 0x1f, 0xda, 0x5b, 0x4a, 0xb2, 0x6a, 0x53, 0xbb, 0x06, 0x67, 0xd6,
0x55, 0x96, 0x29, 0x1f, 0xfa, 0x13, 0xdc, 0xa3, 0xa1, 0xcb, 0xe7, 0x5a, 0x67, 0xcb, 0x15, 0x5a,
0xb7, 0x6f, 0xb6, 0xb7, 0x02, 0x6f, 0x6c, 0xb6, 0x42, 0xbf, 0xe1, 0x6f, 0xfa, 0x00, 0xc9, 0x23,
0xf7, 0x6e, 0xf5, 0x87, 0xa0, 0xe0, 0x11, 0x49, 0xb4, 0xe4, 0xca, 0x58, 0x8f, 0x3b, 0xce, 0xeb,
0xaf, 0x6b, 0xb9, 0x7f, 0x7f, 0x5d, 0xcb, 0xfd, 0x65, 0x51, 0xcb, 0xbf, 0x5e, 0xd4, 0xf2, 0xff,
0x5a, 0xd4, 0xf2, 0xff, 0x59, 0xd4, 0xf2, 0xa3, 0xa2, 0x6e, 0x0d, 0x7e, 0xf1, 0xff, 0x00, 0x00,
0x00, 0xff, 0xff, 0xed, 0xbe, 0x26, 0xe6, 0x9a, 0x10, 0x00, 0x00,
0xe1, 0xad, 0xc7, 0x8c, 0x0f, 0x3d, 0xf5, 0xaa, 0xe9, 0xa1, 0x7f, 0xc6, 0xb7, 0xf6, 0xd8, 0x93,
0xa6, 0xe1, 0x5f, 0xe8, 0x0f, 0x68, 0x07, 0x58, 0x2c, 0xb9, 0x4c, 0x56, 0xb1, 0x67, 0xe2, 0x1b,
0xde, 0xdb, 0xef, 0x7b, 0x00, 0x1e, 0x3e, 0x3c, 0xbc, 0x05, 0x5b, 0x44, 0xd4, 0x15, 0xad, 0x88,
0x33, 0xc9, 0x10, 0xf2, 0x98, 0x7b, 0x45, 0x79, 0x4b, 0x7c, 0x49, 0xf8, 0xf4, 0xca, 0x97, 0xad,
0xd9, 0xcf, 0x6b, 0xb6, 0x9c, 0x47, 0xd4, 0x00, 0x6a, 0xf7, 0xc7, 0x6c, 0xcc, 0xf4, 0xf0, 0x40,
0x8d, 0x8c, 0xb7, 0x3e, 0x66, 0x6c, 0x1c, 0xd0, 0x03, 0x6d, 0x8d, 0xe2, 0xcb, 0x03, 0x2f, 0xe6,
0x44, 0xfa, 0x2c, 0x4c, 0xbe, 0x37, 0xaf, 0x8b, 0x60, 0x9d, 0x33, 0x8f, 0x0e, 0x22, 0xea, 0xa2,
0x63, 0xb0, 0x49, 0x18, 0x32, 0xa9, 0x01, 0xc2, 0x29, 0x34, 0x0a, 0xfb, 0xf6, 0xe1, 0x5e, 0xeb,
0xdb, 0x33, 0xb7, 0xda, 0x2b, 0x58, 0xa7, 0xf8, 0xfa, 0x66, 0x6f, 0x03, 0x67, 0x99, 0xe8, 0xb7,
0x50, 0xf1, 0xa8, 0xf0, 0x39, 0xf5, 0x86, 0x9c, 0x05, 0xd4, 0xd9, 0x6c, 0x14, 0xf6, 0xef, 0x1c,
0xfe, 0x28, 0x2f, 0x92, 0x9a, 0x1c, 0xb3, 0x80, 0x62, 0xdb, 0x30, 0x94, 0x81, 0x8e, 0x01, 0xa6,
0x74, 0x3a, 0xa2, 0x5c, 0x4c, 0xfc, 0xc8, 0xd9, 0xd2, 0xf4, 0x9f, 0xdc, 0x46, 0x57, 0x6b, 0x6f,
0x9d, 0x2d, 0xe1, 0x38, 0x43, 0x45, 0x67, 0x50, 0x21, 0x33, 0xe2, 0x07, 0x64, 0xe4, 0x07, 0xbe,
0x9c, 0x3b, 0x45, 0x1d, 0xea, 0xe3, 0xef, 0x0c, 0xd5, 0xce, 0x10, 0xf0, 0x1a, 0xbd, 0xe9, 0x01,
0xac, 0x26, 0x42, 0x1f, 0xc1, 0x4e, 0xbf, 0x77, 0xde, 0x3d, 0x39, 0x3f, 0xae, 0x6e, 0xd4, 0x1e,
0xbc, 0xba, 0x6e, 0xbc, 0xa7, 0x62, 0xac, 0x00, 0x7d, 0x1a, 0x7a, 0x7e, 0x38, 0x46, 0xfb, 0x60,
0xb5, 0x8f, 0x8e, 0x7a, 0xfd, 0x8b, 0x5e, 0xb7, 0x5a, 0xa8, 0xd5, 0x5e, 0x5d, 0x37, 0xde, 0x5f,
0x07, 0xb6, 0x5d, 0x97, 0x46, 0x92, 0x7a, 0xb5, 0xe2, 0x57, 0xff, 0xa8, 0x6f, 0x34, 0xbf, 0x2a,
0x40, 0x25, 0xbb, 0x08, 0xf4, 0x11, 0x94, 0xda, 0x47, 0x17, 0x27, 0x2f, 0x7a, 0xd5, 0x8d, 0x15,
0x3d, 0x8b, 0x68, 0xbb, 0xd2, 0x9f, 0x51, 0xf4, 0x18, 0xb6, 0xfb, 0xed, 0x2f, 0x06, 0xbd, 0x6a,
0x61, 0xb5, 0x9c, 0x2c, 0xac, 0x4f, 0x62, 0xa1, 0x51, 0x5d, 0xdc, 0x3e, 0x39, 0xaf, 0x6e, 0xe6,
0xa3, 0xba, 0x9c, 0xf8, 0xa1, 0x59, 0xca, 0xdf, 0x8b, 0x60, 0x0f, 0x28, 0x9f, 0xf9, 0xee, 0x3b,
0x96, 0xc8, 0xa7, 0x50, 0x94, 0x44, 0x5c, 0x69, 0x69, 0xd8, 0xf9, 0xd2, 0xb8, 0x20, 0xe2, 0x4a,
0x4d, 0x6a, 0xe8, 0x1a, 0xaf, 0x94, 0xc1, 0x69, 0x14, 0xf8, 0x2e, 0x91, 0xd4, 0xd3, 0xca, 0xb0,
0x0f, 0x7f, 0x9c, 0xc7, 0xc6, 0x4b, 0x94, 0x59, 0xff, 0xb3, 0x0d, 0x9c, 0xa1, 0xa2, 0xa7, 0x50,
0x1a, 0x07, 0x6c, 0x44, 0x02, 0xad, 0x09, 0xfb, 0xf0, 0x51, 0x5e, 0x90, 0x63, 0x8d, 0x58, 0x05,
0x30, 0x14, 0xf4, 0x04, 0x4a, 0x71, 0xe4, 0x11, 0x49, 0x9d, 0x92, 0x26, 0x37, 0xf2, 0xc8, 0x5f,
0x68, 0xc4, 0x11, 0x0b, 0x2f, 0xfd, 0x31, 0x36, 0x78, 0x74, 0x0a, 0x56, 0x48, 0xe5, 0x97, 0x8c,
0x5f, 0x09, 0x67, 0xa7, 0xb1, 0xb5, 0x6f, 0x1f, 0x7e, 0x92, 0x2b, 0xc6, 0x04, 0xd3, 0x96, 0x92,
0xb8, 0x93, 0x29, 0x0d, 0x65, 0x12, 0xa6, 0xb3, 0xe9, 0x14, 0xf0, 0x32, 0x00, 0xfa, 0x0d, 0x58,
0x34, 0xf4, 0x22, 0xe6, 0x87, 0xd2, 0xb1, 0x6e, 0x5f, 0x48, 0xcf, 0x60, 0x54, 0x32, 0xf1, 0x92,
0xa1, 0xd8, 0x9c, 0x05, 0xc1, 0x88, 0xb8, 0x57, 0x4e, 0xf9, 0x2d, 0xb7, 0xb1, 0x64, 0x74, 0x4a,
0x50, 0x9c, 0x32, 0x8f, 0x36, 0x0f, 0xe0, 0xde, 0xb7, 0x52, 0x8d, 0x6a, 0x60, 0x99, 0x54, 0x27,
0x1a, 0x29, 0xe2, 0xa5, 0xdd, 0xbc, 0x0b, 0xbb, 0x6b, 0x69, 0x6d, 0xfe, 0xb5, 0x08, 0x56, 0x7a,
0xd6, 0xa8, 0x0d, 0x65, 0x97, 0x85, 0x92, 0xf8, 0x21, 0xe5, 0x46, 0x5e, 0xb9, 0x27, 0x73, 0x94,
0x82, 0x14, 0xeb, 0xd9, 0x06, 0x5e, 0xb1, 0xd0, 0xef, 0xa1, 0xcc, 0xa9, 0x60, 0x31, 0x77, 0xa9,
0x30, 0xfa, 0xda, 0xcf, 0x57, 0x48, 0x02, 0xc2, 0xf4, 0xcf, 0xb1, 0xcf, 0xa9, 0xca, 0xb2, 0xc0,
0x2b, 0x2a, 0x7a, 0x0a, 0x3b, 0x9c, 0x0a, 0x49, 0xb8, 0xfc, 0x2e, 0x89, 0xe0, 0x04, 0xd2, 0x67,
0x81, 0xef, 0xce, 0x71, 0xca, 0x40, 0x4f, 0xa1, 0x1c, 0x05, 0xc4, 0xd5, 0x51, 0x9d, 0x6d, 0x4d,
0xff, 0x20, 0x8f, 0xde, 0x4f, 0x41, 0x78, 0x85, 0x47, 0x9f, 0x01, 0x04, 0x6c, 0x3c, 0xf4, 0xb8,
0x3f, 0xa3, 0xdc, 0x48, 0xac, 0x96, 0xc7, 0xee, 0x6a, 0x04, 0x2e, 0x07, 0x6c, 0x9c, 0x0c, 0xd1,
0xf1, 0xf7, 0xd2, 0x57, 0x46, 0x5b, 0xa7, 0x00, 0x64, 0xf9, 0xd5, 0xa8, 0xeb, 0xe3, 0xb7, 0x0a,
0x65, 0x4e, 0x24, 0x43, 0x47, 0x8f, 0xa0, 0x72, 0xc9, 0xb8, 0x4b, 0x87, 0xe6, 0xd6, 0x94, 0xb5,
0x26, 0x6c, 0xed, 0x4b, 0xf4, 0xd5, 0x29, 0xc3, 0x0e, 0x8f, 0x43, 0xe9, 0x4f, 0x69, 0xf3, 0x14,
0xde, 0xcb, 0x0d, 0x8a, 0x0e, 0xa1, 0xb2, 0x3c, 0xe6, 0xa1, 0xef, 0x69, 0x7d, 0x94, 0x3b, 0x77,
0x17, 0x37, 0x7b, 0xf6, 0x52, 0x0f, 0x27, 0x5d, 0x6c, 0x2f, 0x41, 0x27, 0x5e, 0xf3, 0x6f, 0x16,
0xec, 0xae, 0x89, 0x05, 0xdd, 0x87, 0x6d, 0x7f, 0x4a, 0xc6, 0x34, 0xa1, 0xe3, 0xc4, 0x40, 0x3d,
0x28, 0x05, 0x64, 0x44, 0x03, 0x25, 0x19, 0x95, 0xb6, 0x9f, 0xbd, 0x51, 0x75, 0xad, 0x3f, 0x6a,
0x7c, 0x2f, 0x94, 0x7c, 0x8e, 0x0d, 0x19, 0x39, 0xb0, 0xe3, 0xb2, 0xe9, 0x94, 0x84, 0xaa, 0x38,
0x6d, 0xed, 0x97, 0x71, 0x6a, 0x22, 0x04, 0x45, 0xc2, 0xc7, 0xc2, 0x29, 0x6a, 0xb7, 0x1e, 0xa3,
0x2a, 0x6c, 0xd1, 0x70, 0xe6, 0x6c, 0x6b, 0x97, 0x1a, 0x2a, 0x8f, 0xe7, 0x27, 0x67, 0x5e, 0xc6,
0x6a, 0xa8, 0x78, 0xb1, 0xa0, 0xdc, 0xd9, 0xd1, 0x2e, 0x3d, 0x46, 0xbf, 0x82, 0xd2, 0x94, 0xc5,
0xa1, 0x14, 0x8e, 0xa5, 0x17, 0xfb, 0x20, 0x6f, 0xb1, 0x67, 0x0a, 0x61, 0x8a, 0xa7, 0x81, 0xa3,
0x1e, 0xdc, 0x13, 0x92, 0x45, 0xc3, 0x31, 0x27, 0x2e, 0x1d, 0x46, 0x94, 0xfb, 0xcc, 0x33, 0x97,
0xff, 0x41, 0x2b, 0xe9, 0x15, 0x5a, 0x69, 0xaf, 0xd0, 0xea, 0x9a, 0x5e, 0x01, 0xdf, 0x55, 0x9c,
0x63, 0x45, 0xe9, 0x6b, 0x06, 0xea, 0x43, 0x25, 0x8a, 0x83, 0x60, 0xc8, 0xa2, 0xe4, 0x1d, 0x00,
0x1d, 0xe1, 0x2d, 0x52, 0xd6, 0x8f, 0x83, 0xe0, 0x79, 0x42, 0xc2, 0x76, 0xb4, 0x32, 0xd0, 0xfb,
0x50, 0x1a, 0x73, 0x16, 0x47, 0xc2, 0xb1, 0x75, 0x32, 0x8c, 0x85, 0x3e, 0x87, 0x1d, 0x41, 0x5d,
0x4e, 0xa5, 0x70, 0x2a, 0x7a, 0xab, 0x1f, 0xe6, 0x4d, 0x32, 0xd0, 0x10, 0x4c, 0x2f, 0x29, 0xa7,
0xa1, 0x4b, 0x71, 0xca, 0x41, 0x0f, 0x60, 0x4b, 0xca, 0xb9, 0xb3, 0xdb, 0x28, 0xec, 0x5b, 0x9d,
0x9d, 0xc5, 0xcd, 0xde, 0xd6, 0xc5, 0xc5, 0x4b, 0xac, 0x7c, 0xaa, 0x46, 0x4d, 0x98, 0x90, 0x21,
0x99, 0x52, 0xe7, 0x8e, 0xce, 0xed, 0xd2, 0x46, 0x2f, 0x01, 0xbc, 0x50, 0x0c, 0x5d, 0x7d, 0x29,
0x9c, 0xbb, 0x7a, 0x77, 0x9f, 0xbc, 0x79, 0x77, 0xdd, 0xf3, 0x81, 0xa9, 0xd3, 0xbb, 0x8b, 0x9b,
0xbd, 0xf2, 0xd2, 0xc4, 0x65, 0x2f, 0x14, 0xc9, 0x10, 0x75, 0xc0, 0x9e, 0x50, 0x12, 0xc8, 0x89,
0x3b, 0xa1, 0xee, 0x95, 0x53, 0xbd, 0xbd, 0xf0, 0x3e, 0xd3, 0x30, 0x13, 0x21, 0x4b, 0x52, 0x0a,
0x56, 0x4b, 0x15, 0xce, 0x3d, 0x9d, 0xab, 0xc4, 0x40, 0x1f, 0x00, 0xb0, 0x88, 0x86, 0x43, 0x21,
0x3d, 0x3f, 0x74, 0x90, 0xda, 0x32, 0x2e, 0x2b, 0xcf, 0x40, 0x39, 0xd0, 0x43, 0x55, 0x16, 0x89,
0x37, 0x64, 0x61, 0x30, 0x77, 0x7e, 0xa0, 0xbf, 0x5a, 0xca, 0xf1, 0x3c, 0x0c, 0xe6, 0x68, 0x0f,
0x6c, 0xad, 0x0b, 0xe1, 0x8f, 0x43, 0x12, 0x38, 0xf7, 0x75, 0x3e, 0x40, 0xb9, 0x06, 0xda, 0x53,
0xfb, 0x0c, 0xec, 0x8c, 0xdc, 0x95, 0x4c, 0xaf, 0xe8, 0xdc, 0xdc, 0x20, 0x35, 0x54, 0x6b, 0x9a,
0x91, 0x20, 0x4e, 0x9a, 0xbd, 0x32, 0x4e, 0x8c, 0x5f, 0x6f, 0x3e, 0x29, 0xd4, 0x0e, 0xc1, 0xce,
0x1c, 0x3b, 0xfa, 0x10, 0x76, 0x39, 0x1d, 0xfb, 0x42, 0xf2, 0xf9, 0x90, 0xc4, 0x72, 0xe2, 0xfc,
0x4e, 0x13, 0x2a, 0xa9, 0xb3, 0x1d, 0xcb, 0x49, 0x6d, 0x08, 0xab, 0xec, 0xa1, 0x06, 0xd8, 0xea,
0x54, 0x04, 0xe5, 0x33, 0xca, 0xd5, 0x83, 0xa2, 0x36, 0x9d, 0x75, 0x29, 0xf5, 0x08, 0x4a, 0xb8,
0x3b, 0xd1, 0x97, 0xb7, 0x8c, 0x8d, 0xa5, 0x6e, 0x63, 0x2a, 0x51, 0x73, 0x1b, 0x8d, 0xd9, 0xfc,
0x6f, 0x01, 0x2a, 0xd9, 0x77, 0x11, 0x1d, 0x25, 0xef, 0x99, 0xde, 0xd2, 0x9d, 0xc3, 0x83, 0x37,
0xbd, 0xa3, 0xfa, 0xf5, 0x08, 0x62, 0x15, 0xec, 0x4c, 0xb5, 0xb0, 0x9a, 0x8c, 0x7e, 0x09, 0xdb,
0x11, 0xe3, 0x32, 0xad, 0x21, 0xf5, 0xdc, 0x8a, 0xcf, 0x78, 0x5a, 0x6d, 0x13, 0x70, 0x73, 0x02,
0x77, 0xd6, 0xa3, 0xa1, 0xc7, 0xb0, 0xf5, 0xe2, 0xa4, 0x5f, 0xdd, 0xa8, 0x3d, 0x7c, 0x75, 0xdd,
0xf8, 0xe1, 0xfa, 0xc7, 0x17, 0x3e, 0x97, 0x31, 0x09, 0x4e, 0xfa, 0xe8, 0xa7, 0xb0, 0xdd, 0x3d,
0x1f, 0x60, 0x5c, 0x2d, 0xd4, 0xf6, 0x5e, 0x5d, 0x37, 0x1e, 0xae, 0xe3, 0xd4, 0x27, 0x16, 0x87,
0x1e, 0x66, 0xa3, 0x65, 0x3b, 0xf7, 0xcf, 0x4d, 0xb0, 0x4d, 0x69, 0x7d, 0xd7, 0x1d, 0xff, 0x6e,
0xf2, 0x5a, 0xa5, 0x77, 0x66, 0xf3, 0x8d, 0x8f, 0x56, 0x25, 0x21, 0x98, 0x33, 0x7e, 0x04, 0x15,
0x3f, 0x9a, 0x7d, 0x3a, 0xa4, 0x21, 0x19, 0x05, 0xa6, 0xb3, 0xb3, 0xb0, 0xad, 0x7c, 0xbd, 0xc4,
0xa5, 0x2e, 0xac, 0x1f, 0x4a, 0xca, 0x43, 0xd3, 0xb3, 0x59, 0x78, 0x69, 0xa3, 0xcf, 0xa1, 0xe8,
0x47, 0x64, 0x6a, 0x5e, 0xda, 0xdc, 0x1d, 0x9c, 0xf4, 0xdb, 0x67, 0x46, 0x83, 0x1d, 0x6b, 0x71,
0xb3, 0x57, 0x54, 0x0e, 0xac, 0x69, 0xa8, 0x9e, 0x3e, 0x76, 0x6a, 0x26, 0x5d, 0x7c, 0x2d, 0x9c,
0xf1, 0x28, 0x1d, 0xf9, 0xe1, 0x98, 0x53, 0x21, 0x74, 0x19, 0xb6, 0x70, 0x6a, 0x36, 0xff, 0x57,
0x04, 0xfb, 0x28, 0x88, 0x85, 0x34, 0x8f, 0xcb, 0x3b, 0xcb, 0xe8, 0x4b, 0xb8, 0x47, 0xf4, 0x6f,
0x01, 0x09, 0x55, 0xa5, 0xd6, 0xed, 0x85, 0xc9, 0xea, 0xe3, 0xdc, 0x70, 0x4b, 0x70, 0xd2, 0x8a,
0x74, 0x4a, 0x2a, 0xa6, 0x53, 0xc0, 0x55, 0xf2, 0x8d, 0x2f, 0x68, 0x00, 0xbb, 0x8c, 0xbb, 0x13,
0x2a, 0x64, 0x52, 0xdf, 0x4d, 0x1b, 0x9d, 0xfb, 0x83, 0xf5, 0x3c, 0x0b, 0x34, 0xc5, 0x2d, 0x59,
0xed, 0x7a, 0x0c, 0xf4, 0x04, 0x8a, 0x9c, 0x5c, 0xa6, 0xad, 0x52, 0xae, 0xf2, 0x31, 0xb9, 0x94,
0x6b, 0x21, 0x34, 0x03, 0xfd, 0x01, 0xc0, 0xf3, 0x45, 0x44, 0xa4, 0x3b, 0xa1, 0xdc, 0x9c, 0x60,
0xee, 0x16, 0xbb, 0x4b, 0xd4, 0x5a, 0x94, 0x0c, 0x1b, 0x9d, 0x42, 0xd9, 0x25, 0xa9, 0x06, 0x4b,
0xb7, 0xff, 0x5b, 0x1c, 0xb5, 0x4d, 0x88, 0xaa, 0x0a, 0xb1, 0xb8, 0xd9, 0xb3, 0x52, 0x0f, 0xb6,
0x5c, 0x62, 0x34, 0x79, 0x0a, 0xbb, 0xea, 0x9f, 0x63, 0xe8, 0xd1, 0x4b, 0x12, 0x07, 0x32, 0x39,
0xfb, 0x5b, 0x8a, 0xb5, 0x6a, 0x60, 0xbb, 0x06, 0x67, 0xd6, 0x55, 0x91, 0x19, 0x1f, 0xfa, 0x13,
0xdc, 0xa3, 0xa1, 0xcb, 0xe7, 0x5a, 0x81, 0xe9, 0x0a, 0xad, 0xdb, 0x37, 0xdb, 0x5b, 0x82, 0xd7,
0x36, 0x5b, 0xa5, 0xdf, 0xf0, 0x37, 0x7d, 0x80, 0xe4, 0xf9, 0x7b, 0xb7, 0xfa, 0x43, 0x50, 0xf4,
0x88, 0x24, 0x5a, 0x72, 0x15, 0xac, 0xc7, 0x1d, 0xe7, 0xf5, 0xd7, 0xf5, 0x8d, 0x7f, 0x7f, 0x5d,
0xdf, 0xf8, 0xcb, 0xa2, 0x5e, 0x78, 0xbd, 0xa8, 0x17, 0xfe, 0xb5, 0xa8, 0x17, 0xfe, 0xb3, 0xa8,
0x17, 0x46, 0x25, 0xdd, 0x34, 0xfc, 0xe2, 0xff, 0x01, 0x00, 0x00, 0xff, 0xff, 0x64, 0x44, 0x1e,
0x4f, 0xb4, 0x10, 0x00, 0x00,
}

6
vendor/github.com/docker/swarmkit/api/specs.proto generated vendored
View file

@ -321,6 +321,12 @@ message NetworkSpec {
// enabled(default case) no manual attachment to this network
// can happen.
bool attachable = 6;
// Ingress indicates this network will provide the routing-mesh.
// In older versions, the network providing the routing mesh was
// swarm internally created only and it was identified by the name
// "ingress" and the label "com.docker.swarm.internal": "true".
bool ingress = 7;
}
// ClusterSpec specifies global cluster settings.

65
vendor/github.com/docker/swarmkit/ca/certificates.go generated vendored
View file

@ -113,12 +113,39 @@ type LocalSigner struct {
Key []byte
}
// RootCA is the representation of everything we need to sign certificates
// RootCA is the representation of everything we need to sign certificates and/or to verify certificates
//
// RootCA.Cert: [signing CA cert][CA cert1][CA cert2]
// RootCA.Intermediates: [intermediate CA1][intermediate CA2][intermediate CA3]
// RootCA.Signer.Key: [signing CA key]
//
// Requirements:
//
// - [signing CA key] must be the private key for [signing CA cert]
// - [signing CA cert] must be the first cert in RootCA.Cert
//
// - [intermediate CA1] must have the same public key and subject as [signing CA cert], because otherwise when
// appended to a leaf certificate, the intermediates will not form a chain (because [intermediate CA1] won't because
// the signer of the leaf certificate)
// - [intermediate CA1] must be signed by [intermediate CA2], which must be signed by [intermediate CA3]
//
// - When we issue a certificate, the intermediates will be appended so that the certificate looks like:
// [leaf signed by signing CA cert][intermediate CA1][intermediate CA2][intermediate CA3]
// - [leaf signed by signing CA cert][intermediate CA1][intermediate CA2][intermediate CA3] is guaranteed to form a
// valid chain from [leaf signed by signing CA cert] to one of the root certs ([signing CA cert], [CA cert1], [CA cert2])
// using zero or more of the intermediate certs ([intermediate CA1][intermediate CA2][intermediate CA3]) as intermediates
//
type RootCA struct {
// Cert contains a bundle of PEM encoded Certificate for the Root CA, the first one of which
// must correspond to the key in the local signer, if provided
Cert []byte
// Intermediates contains a bundle of PEM encoded intermediate CA certificates to append to any
// issued TLS (leaf) certificates. The first one must have the same public key and subject as the
// signing root certificate, and the rest must form a chain, each one certifying the one above it,
// as per RFC5246 section 7.4.2.
Intermediates []byte
// Pool is the root pool used to validate TLS certificates
Pool *x509.CertPool
@ -306,7 +333,7 @@ func (rca *RootCA) ParseValidateAndSignCSR(csrBytes []byte, cn, ou, org string)
return nil, errors.Wrap(err, "failed to sign node certificate")
}
return cert, nil
return append(cert, rca.Intermediates...), nil
}
// CrossSignCACertificate takes a CA root certificate and generates an intermediate CA from it signed with the current root signer
@ -348,7 +375,7 @@ func (rca *RootCA) CrossSignCACertificate(otherCAPEM []byte) ([]byte, error) {
// NewRootCA creates a new RootCA object from unparsed PEM cert bundle and key byte
// slices. key may be nil, and in this case NewRootCA will return a RootCA
// without a signer.
func NewRootCA(certBytes, keyBytes []byte, certExpiry time.Duration) (RootCA, error) {
func NewRootCA(certBytes, keyBytes []byte, certExpiry time.Duration, intermediates []byte) (RootCA, error) {
// Parse all the certificates in the cert bundle
parsedCerts, err := helpers.ParseCertificatesPEM(certBytes)
if err != nil {
@ -368,7 +395,6 @@ func NewRootCA(certBytes, keyBytes []byte, certExpiry time.Duration) (RootCA, er
default:
return RootCA{}, fmt.Errorf("unsupported signature algorithm: %s", cert.SignatureAlgorithm.String())
}
// Check to see if all of the certificates are valid, self-signed root CA certs
selfpool := x509.NewCertPool()
selfpool.AddCert(cert)
@ -381,9 +407,28 @@ func NewRootCA(certBytes, keyBytes []byte, certExpiry time.Duration) (RootCA, er
// Calculate the digest for our Root CA bundle
digest := digest.FromBytes(certBytes)
// We do not yet support arbitrary chains of intermediates (e.g. the case of an offline root, and the swarm CA is an
// intermediate CA). We currently only intermediates for which the first intermediate is cross-signed version of the
// CA signing cert (the first cert of the root certs) for the purposes of root rotation. If we wanted to support
// offline roots, we'd have to separate the CA signing cert from the self-signed root certs, but this intermediate
// validation logic should remain the same. Either the first intermediate would BE the intermediate CA we sign with
// (in which case it'd have the same subject and public key), or it would be a cross-signed intermediate with the
// same subject and public key as our signing cert (which could be either an intermediate cert or a self-signed root
// cert).
if len(intermediates) > 0 {
parsedIntermediates, err := ValidateCertChain(pool, intermediates, false)
if err != nil {
return RootCA{}, errors.Wrap(err, "invalid intermediate chain")
}
if !bytes.Equal(parsedIntermediates[0].RawSubject, parsedCerts[0].RawSubject) ||
!bytes.Equal(parsedIntermediates[0].RawSubjectPublicKeyInfo, parsedCerts[0].RawSubjectPublicKeyInfo) {
return RootCA{}, errors.New("invalid intermediate chain - the first intermediate must have the same subject and public key as the root")
}
}
if len(keyBytes) == 0 {
// This RootCA does not have a valid signer.
return RootCA{Cert: certBytes, Digest: digest, Pool: pool}, nil
// This RootCA does not have a valid signer
return RootCA{Cert: certBytes, Intermediates: intermediates, Digest: digest, Pool: pool}, nil
}
var (
@ -434,7 +479,7 @@ func NewRootCA(certBytes, keyBytes []byte, certExpiry time.Duration) (RootCA, er
}
}
return RootCA{Signer: &LocalSigner{Signer: signer, Key: keyBytes}, Digest: digest, Cert: certBytes, Pool: pool}, nil
return RootCA{Signer: &LocalSigner{Signer: signer, Key: keyBytes}, Intermediates: intermediates, Digest: digest, Cert: certBytes, Pool: pool}, nil
}
// ValidateCertChain checks checks that the certificates provided chain up to the root pool provided. In addition
@ -586,7 +631,7 @@ func GetLocalRootCA(paths CertPaths) (RootCA, error) {
key = nil
}
return NewRootCA(cert, key, DefaultNodeCertExpiration)
return NewRootCA(cert, key, DefaultNodeCertExpiration, nil)
}
func getGRPCConnection(creds credentials.TransportCredentials, connBroker *connectionbroker.Broker, forceRemote bool) (*connectionbroker.Conn, error) {
@ -641,7 +686,7 @@ func GetRemoteCA(ctx context.Context, d digest.Digest, connBroker *connectionbro
// NewRootCA will validate that the certificates are otherwise valid and create a RootCA object.
// Since there is no key, the certificate expiry does not matter and will not be used.
return NewRootCA(response.Certificate, nil, DefaultNodeCertExpiration)
return NewRootCA(response.Certificate, nil, DefaultNodeCertExpiration, nil)
}
// CreateRootCA creates a Certificate authority for a new Swarm Cluster, potentially
@ -660,7 +705,7 @@ func CreateRootCA(rootCN string, paths CertPaths) (RootCA, error) {
return RootCA{}, err
}
rootCA, err := NewRootCA(cert, key, DefaultNodeCertExpiration)
rootCA, err := NewRootCA(cert, key, DefaultNodeCertExpiration, nil)
if err != nil {
return RootCA{}, err
}

2
vendor/github.com/docker/swarmkit/ca/config.go generated vendored
View file

@ -132,7 +132,7 @@ func (s *SecurityConfig) UpdateRootCA(cert, key []byte, certExpiry time.Duration
s.mu.Lock()
defer s.mu.Unlock()
rootCA, err := NewRootCA(cert, key, certExpiry)
rootCA, err := NewRootCA(cert, key, certExpiry, nil)
if err != nil {
return err
}

2
vendor/github.com/docker/swarmkit/ca/external.go generated vendored
View file

@ -96,7 +96,7 @@ func (eca *ExternalCA) Sign(ctx context.Context, req signer.SignRequest) (cert [
for _, url := range urls {
cert, err = makeExternalSignRequest(ctx, client, url, csrJSON)
if err == nil {
return cert, err
return append(cert, eca.rootCA.Intermediates...), err
}
logrus.Debugf("unable to proxy certificate signing request to %s: %s", url, err)
}

286
vendor/github.com/docker/swarmkit/manager/allocator/network.go generated vendored
View file

@ -6,7 +6,6 @@ import (
"github.com/docker/go-events"
"github.com/docker/swarmkit/api"
"github.com/docker/swarmkit/identity"
"github.com/docker/swarmkit/log"
"github.com/docker/swarmkit/manager/allocator/networkallocator"
"github.com/docker/swarmkit/manager/state"
@ -18,42 +17,18 @@ import (
const (
// Network allocator Voter ID for task allocation vote.
networkVoter = "network"
ingressNetworkName = "ingress"
ingressSubnet = "10.255.0.0/16"
networkVoter = "network"
allocatedStatusMessage = "pending task scheduling"
)
var (
// ErrNoIngress is returned when no ingress network is found in store
ErrNoIngress = errors.New("no ingress network found")
errNoChanges = errors.New("task unchanged")
retryInterval = 5 * time.Minute
)
func newIngressNetwork() *api.Network {
return &api.Network{
Spec: api.NetworkSpec{
Annotations: api.Annotations{
Name: ingressNetworkName,
Labels: map[string]string{
"com.docker.swarm.internal": "true",
},
},
DriverConfig: &api.Driver{},
IPAM: &api.IPAMOptions{
Driver: &api.Driver{},
Configs: []*api.IPAMConfig{
{
Subnet: ingressSubnet,
},
},
},
},
}
}
// Network context information which is used throughout the network allocation code.
type networkContext struct {
ingressNetwork *api.Network
@ -97,7 +72,6 @@ func (a *Allocator) doNetworkInit(ctx context.Context) (err error) {
unallocatedTasks: make(map[string]*api.Task),
unallocatedServices: make(map[string]*api.Service),
unallocatedNetworks: make(map[string]*api.Network),
ingressNetwork: newIngressNetwork(),
lastRetry: time.Now(),
}
a.netCtx = nc
@ -108,63 +82,38 @@ func (a *Allocator) doNetworkInit(ctx context.Context) (err error) {
}
}()
// Check if we have the ingress network. If not found create
// it before reading all network objects for allocation.
var networks []*api.Network
a.store.View(func(tx store.ReadTx) {
networks, err = store.FindNetworks(tx, store.ByName(ingressNetworkName))
if len(networks) > 0 {
nc.ingressNetwork = networks[0]
}
})
if err != nil {
return errors.Wrap(err, "failed to find ingress network during init")
}
// If ingress network is not found, create one right away
// using the predefined template.
if len(networks) == 0 {
if err := a.store.Update(func(tx store.Tx) error {
nc.ingressNetwork.ID = identity.NewID()
if err := store.CreateNetwork(tx, nc.ingressNetwork); err != nil {
return err
}
return nil
}); err != nil {
return errors.Wrap(err, "failed to create ingress network")
}
a.store.View(func(tx store.ReadTx) {
networks, err = store.FindNetworks(tx, store.ByName(ingressNetworkName))
if len(networks) > 0 {
nc.ingressNetwork = networks[0]
}
})
if err != nil {
return errors.Wrap(err, "failed to find ingress network after creating it")
}
}
// Try to complete ingress network allocation before anything else so
// that the we can get the preferred subnet for ingress
// network.
if !na.IsAllocated(nc.ingressNetwork) {
if err := a.allocateNetwork(ctx, nc.ingressNetwork); err != nil {
log.G(ctx).WithError(err).Error("failed allocating ingress network during init")
} else if _, err := a.store.Batch(func(batch *store.Batch) error {
if err := a.commitAllocatedNetwork(ctx, batch, nc.ingressNetwork); err != nil {
// Ingress network is now created at cluster's first time creation.
// Check if we have the ingress network. If found, make sure it is
// allocated, before reading all network objects for allocation.
// If not found, it means it was removed by user, nothing to do here.
ingressNetwork, err := GetIngressNetwork(a.store)
switch err {
case nil:
// Try to complete ingress network allocation before anything else so
// that the we can get the preferred subnet for ingress network.
nc.ingressNetwork = ingressNetwork
if !na.IsAllocated(nc.ingressNetwork) {
if err := a.allocateNetwork(ctx, nc.ingressNetwork); err != nil {
log.G(ctx).WithError(err).Error("failed allocating ingress network during init")
} else if _, err := a.store.Batch(func(batch *store.Batch) error {
if err := a.commitAllocatedNetwork(ctx, batch, nc.ingressNetwork); err != nil {
log.G(ctx).WithError(err).Error("failed committing allocation of ingress network during init")
}
return nil
}); err != nil {
log.G(ctx).WithError(err).Error("failed committing allocation of ingress network during init")
}
return nil
}); err != nil {
log.G(ctx).WithError(err).Error("failed committing allocation of ingress network during init")
}
case ErrNoIngress:
// Ingress network is not present in store, It means user removed it
// and did not create a new one.
default:
return errors.Wrap(err, "failure while looking for ingress network during init")
}
// Allocate networks in the store so far before we started
// watching.
var networks []*api.Network
a.store.View(func(tx store.ReadTx) {
networks, err = store.FindNetworks(tx, store.All)
})
@ -196,43 +145,12 @@ func (a *Allocator) doNetworkInit(ctx context.Context) (err error) {
log.G(ctx).WithError(err).Error("failed committing allocation of networks during init")
}
// Allocate nodes in the store so far before we process watched events.
var nodes []*api.Node
a.store.View(func(tx store.ReadTx) {
nodes, err = store.FindNodes(tx, store.All)
})
if err != nil {
return errors.Wrap(err, "error listing all nodes in store while trying to allocate during init")
}
var allocatedNodes []*api.Node
for _, node := range nodes {
if na.IsNodeAllocated(node) {
continue
// Allocate nodes in the store so far before we process watched events,
// if the ingress network is present.
if nc.ingressNetwork != nil {
if err := a.allocateNodes(ctx); err != nil {
return err
}
if node.Attachment == nil {
node.Attachment = &api.NetworkAttachment{}
}
node.Attachment.Network = nc.ingressNetwork.Copy()
if err := a.allocateNode(ctx, node); err != nil {
log.G(ctx).WithError(err).Errorf("Failed to allocate network resources for node %s during init", node.ID)
continue
}
allocatedNodes = append(allocatedNodes, node)
}
if _, err := a.store.Batch(func(batch *store.Batch) error {
for _, node := range allocatedNodes {
if err := a.commitAllocatedNode(ctx, batch, node); err != nil {
log.G(ctx).WithError(err).Errorf("Failed to commit allocation of network resources for node %s during init", node.ID)
}
}
return nil
}); err != nil {
log.G(ctx).WithError(err).Error("Failed to commit allocation of network resources for nodes during init")
}
// Allocate services in the store so far before we process watched events.
@ -346,6 +264,12 @@ func (a *Allocator) doNetworkAlloc(ctx context.Context, ev events.Event) {
break
}
if IsIngressNetwork(n) && nc.ingressNetwork != nil {
log.G(ctx).Errorf("Cannot allocate ingress network %s (%s) because another ingress network is already present: %s (%s)",
n.ID, n.Spec.Annotations.Name, nc.ingressNetwork.ID, nc.ingressNetwork.Spec.Annotations)
break
}
if err := a.allocateNetwork(ctx, n); err != nil {
log.G(ctx).WithError(err).Errorf("Failed allocation for network %s", n.ID)
break
@ -356,9 +280,24 @@ func (a *Allocator) doNetworkAlloc(ctx context.Context, ev events.Event) {
}); err != nil {
log.G(ctx).WithError(err).Errorf("Failed to commit allocation for network %s", n.ID)
}
if IsIngressNetwork(n) {
nc.ingressNetwork = n
err := a.allocateNodes(ctx)
if err != nil {
log.G(ctx).WithError(err).Error(err)
}
}
case state.EventDeleteNetwork:
n := v.Network.Copy()
if IsIngressNetwork(n) && nc.ingressNetwork.ID == n.ID {
nc.ingressNetwork = nil
if err := a.deallocateNodes(ctx); err != nil {
log.G(ctx).WithError(err).Error(err)
}
}
// The assumption here is that all dependent objects
// have been cleaned up when we are here so the only
// thing that needs to happen is free the network
@ -467,7 +406,7 @@ func (a *Allocator) doNodeAlloc(ctx context.Context, ev events.Event) {
return
}
if !nc.nwkAllocator.IsNodeAllocated(node) {
if !nc.nwkAllocator.IsNodeAllocated(node) && nc.ingressNetwork != nil {
if node.Attachment == nil {
node.Attachment = &api.NetworkAttachment{}
}
@ -486,6 +425,85 @@ func (a *Allocator) doNodeAlloc(ctx context.Context, ev events.Event) {
}
}
func (a *Allocator) allocateNodes(ctx context.Context) error {
// Allocate nodes in the store so far before we process watched events.
var (
allocatedNodes []*api.Node
nodes []*api.Node
err error
nc = a.netCtx
)
a.store.View(func(tx store.ReadTx) {
nodes, err = store.FindNodes(tx, store.All)
})
if err != nil {
return errors.Wrap(err, "error listing all nodes in store while trying to allocate network resources")
}
for _, node := range nodes {
if nc.nwkAllocator.IsNodeAllocated(node) {
continue
}
if node.Attachment == nil {
node.Attachment = &api.NetworkAttachment{}
}
node.Attachment.Network = nc.ingressNetwork.Copy()
if err := a.allocateNode(ctx, node); err != nil {
log.G(ctx).WithError(err).Errorf("Failed to allocate network resources for node %s", node.ID)
continue
}
allocatedNodes = append(allocatedNodes, node)
}
if _, err := a.store.Batch(func(batch *store.Batch) error {
for _, node := range allocatedNodes {
if err := a.commitAllocatedNode(ctx, batch, node); err != nil {
log.G(ctx).WithError(err).Errorf("Failed to commit allocation of network resources for node %s", node.ID)
}
}
return nil
}); err != nil {
log.G(ctx).WithError(err).Error("Failed to commit allocation of network resources for nodes")
}
return nil
}
func (a *Allocator) deallocateNodes(ctx context.Context) error {
var (
nodes []*api.Node
nc = a.netCtx
err error
)
a.store.View(func(tx store.ReadTx) {
nodes, err = store.FindNodes(tx, store.All)
})
if err != nil {
return fmt.Errorf("error listing all nodes in store while trying to free network resources")
}
for _, node := range nodes {
if nc.nwkAllocator.IsNodeAllocated(node) {
if err := nc.nwkAllocator.DeallocateNode(node); err != nil {
log.G(ctx).WithError(err).Errorf("Failed freeing network resources for node %s", node.ID)
}
node.Attachment = nil
if _, err := a.store.Batch(func(batch *store.Batch) error {
return a.commitAllocatedNode(ctx, batch, node)
}); err != nil {
log.G(ctx).WithError(err).Errorf("Failed to commit deallocation of network resources for node %s", node.ID)
}
}
}
return nil
}
// taskReadyForNetworkVote checks if the task is ready for a network
// vote to move it to PENDING state.
func taskReadyForNetworkVote(t *api.Task, s *api.Service, nc *networkContext) bool {
@ -711,6 +729,9 @@ func (a *Allocator) allocateService(ctx context.Context, s *api.Service) error {
// world. Automatically attach the service to the ingress
// network only if it is not already done.
if isIngressNetworkNeeded(s) {
if nc.ingressNetwork == nil {
return fmt.Errorf("ingress network is missing")
}
var found bool
for _, vip := range s.Endpoint.VirtualIPs {
if vip.NetworkID == nc.ingressNetwork.ID {
@ -1022,3 +1043,36 @@ func updateTaskStatus(t *api.Task, newStatus api.TaskState, message string) {
t.Status.Message = message
t.Status.Timestamp = ptypes.MustTimestampProto(time.Now())
}
// IsIngressNetwork returns whether the passed network is an ingress network.
func IsIngressNetwork(nw *api.Network) bool {
if nw.Spec.Ingress {
return true
}
// Check if legacy defined ingress network
_, ok := nw.Spec.Annotations.Labels["com.docker.swarm.internal"]
return ok && nw.Spec.Annotations.Name == "ingress"
}
// GetIngressNetwork fetches the ingress network from store.
// ErrNoIngress will be returned if the ingress network is not present,
// nil otherwise. In case of any other failure in accessing the store,
// the respective error will be reported as is.
func GetIngressNetwork(s *store.MemoryStore) (*api.Network, error) {
var (
networks []*api.Network
err error
)
s.View(func(tx store.ReadTx) {
networks, err = store.FindNetworks(tx, store.All)
})
if err != nil {
return nil, err
}
for _, n := range networks {
if IsIngressNetwork(n) {
return n, nil
}
}
return nil, ErrNoIngress
}

109
vendor/github.com/docker/swarmkit/manager/controlapi/network.go generated vendored
View file

@ -1,7 +1,6 @@
package controlapi
import (
"fmt"
"net"
"github.com/docker/docker/pkg/plugingetter"
@ -9,6 +8,7 @@ import (
"github.com/docker/libnetwork/ipamapi"
"github.com/docker/swarmkit/api"
"github.com/docker/swarmkit/identity"
"github.com/docker/swarmkit/manager/allocator"
"github.com/docker/swarmkit/manager/state/store"
"golang.org/x/net/context"
"google.golang.org/grpc"
@ -75,6 +75,14 @@ func validateNetworkSpec(spec *api.NetworkSpec, pg plugingetter.PluginGetter) er
return grpc.Errorf(codes.InvalidArgument, errInvalidArgument.Error())
}
if spec.Ingress && spec.DriverConfig != nil && spec.DriverConfig.Name != "overlay" {
return grpc.Errorf(codes.Unimplemented, "only overlay driver is currently supported for ingress network")
}
if spec.Attachable && spec.Ingress {
return grpc.Errorf(codes.InvalidArgument, "ingress network cannot be attachable")
}
if err := validateAnnotations(spec.Annotations); err != nil {
return err
}
@ -94,16 +102,10 @@ func validateNetworkSpec(spec *api.NetworkSpec, pg plugingetter.PluginGetter) er
// - Returns `InvalidArgument` if the NetworkSpec is malformed.
// - Returns an error if the creation fails.
func (s *Server) CreateNetwork(ctx context.Context, request *api.CreateNetworkRequest) (*api.CreateNetworkResponse, error) {
// if you change this function, you have to change createInternalNetwork in
// the tests to match it (except the part where we check the label).
if err := validateNetworkSpec(request.Spec, s.pg); err != nil {
return nil, err
}
if _, ok := request.Spec.Annotations.Labels["com.docker.swarm.internal"]; ok {
return nil, grpc.Errorf(codes.PermissionDenied, "label com.docker.swarm.internal is for predefined internal networks and cannot be applied by users")
}
// TODO(mrjana): Consider using `Name` as a primary key to handle
// duplicate creations. See #65
n := &api.Network{
@ -112,6 +114,13 @@ func (s *Server) CreateNetwork(ctx context.Context, request *api.CreateNetworkRe
}
err := s.store.Update(func(tx store.Tx) error {
if request.Spec.Ingress {
if n, err := allocator.GetIngressNetwork(s.store); err == nil {
return grpc.Errorf(codes.AlreadyExists, "ingress network (%s) is already present", n.ID)
} else if err != allocator.ErrNoIngress {
return grpc.Errorf(codes.Internal, "failed ingress network presence check: %v", err)
}
}
return store.CreateNetwork(tx, n)
})
if err != nil {
@ -152,38 +161,23 @@ func (s *Server) RemoveNetwork(ctx context.Context, request *api.RemoveNetworkRe
return nil, grpc.Errorf(codes.InvalidArgument, errInvalidArgument.Error())
}
err := s.store.Update(func(tx store.Tx) error {
services, err := store.FindServices(tx, store.ByReferencedNetworkID(request.NetworkID))
if err != nil {
return grpc.Errorf(codes.Internal, "could not find services using network %s: %v", request.NetworkID, err)
}
var (
n *api.Network
rm = s.removeNetwork
)
if len(services) != 0 {
return grpc.Errorf(codes.FailedPrecondition, "network %s is in use by service %s", request.NetworkID, services[0].ID)
}
tasks, err := store.FindTasks(tx, store.ByReferencedNetworkID(request.NetworkID))
if err != nil {
return grpc.Errorf(codes.Internal, "could not find tasks using network %s: %v", request.NetworkID, err)
}
for _, t := range tasks {
if t.DesiredState <= api.TaskStateRunning && t.Status.State <= api.TaskStateRunning {
return grpc.Errorf(codes.FailedPrecondition, "network %s is in use by task %s", request.NetworkID, t.ID)
}
}
nw := store.GetNetwork(tx, request.NetworkID)
if _, ok := nw.Spec.Annotations.Labels["com.docker.swarm.internal"]; ok {
networkDescription := nw.ID
if nw.Spec.Annotations.Name != "" {
networkDescription = fmt.Sprintf("%s (%s)", nw.Spec.Annotations.Name, nw.ID)
}
return grpc.Errorf(codes.PermissionDenied, "%s is a pre-defined network and cannot be removed", networkDescription)
}
return store.DeleteNetwork(tx, request.NetworkID)
s.store.View(func(tx store.ReadTx) {
n = store.GetNetwork(tx, request.NetworkID)
})
if err != nil {
if n == nil {
return nil, grpc.Errorf(codes.NotFound, "network %s not found", request.NetworkID)
}
if allocator.IsIngressNetwork(n) {
rm = s.removeIngressNetwork
}
if err := rm(n.ID); err != nil {
if err == store.ErrNotExist {
return nil, grpc.Errorf(codes.NotFound, "network %s not found", request.NetworkID)
}
@ -192,6 +186,47 @@ func (s *Server) RemoveNetwork(ctx context.Context, request *api.RemoveNetworkRe
return &api.RemoveNetworkResponse{}, nil
}
func (s *Server) removeNetwork(id string) error {
return s.store.Update(func(tx store.Tx) error {
services, err := store.FindServices(tx, store.ByReferencedNetworkID(id))
if err != nil {
return grpc.Errorf(codes.Internal, "could not find services using network %s: %v", id, err)
}
if len(services) != 0 {
return grpc.Errorf(codes.FailedPrecondition, "network %s is in use by service %s", id, services[0].ID)
}
tasks, err := store.FindTasks(tx, store.ByReferencedNetworkID(id))
if err != nil {
return grpc.Errorf(codes.Internal, "could not find tasks using network %s: %v", id, err)
}
for _, t := range tasks {
if t.DesiredState <= api.TaskStateRunning && t.Status.State <= api.TaskStateRunning {
return grpc.Errorf(codes.FailedPrecondition, "network %s is in use by task %s", id, t.ID)
}
}
return store.DeleteNetwork(tx, id)
})
}
func (s *Server) removeIngressNetwork(id string) error {
return s.store.Update(func(tx store.Tx) error {
services, err := store.FindServices(tx, store.All)
if err != nil {
return grpc.Errorf(codes.Internal, "could not find services using network %s: %v", id, err)
}
for _, srv := range services {
if doesServiceNeedIngress(srv) {
return grpc.Errorf(codes.FailedPrecondition, "ingress network cannot be removed because service %s depends on it", srv.ID)
}
}
return store.DeleteNetwork(tx, id)
})
}
func filterNetworks(candidates []*api.Network, filters ...func(*api.Network) bool) []*api.Network {
result := []*api.Network{}

45
vendor/github.com/docker/swarmkit/manager/controlapi/service.go generated vendored
View file

@ -11,6 +11,7 @@ import (
"github.com/docker/distribution/reference"
"github.com/docker/swarmkit/api"
"github.com/docker/swarmkit/identity"
"github.com/docker/swarmkit/manager/allocator"
"github.com/docker/swarmkit/manager/constraint"
"github.com/docker/swarmkit/manager/state/store"
"github.com/docker/swarmkit/protobuf/ptypes"
@ -288,7 +289,7 @@ func (s *Server) validateNetworks(networks []*api.NetworkAttachmentConfig) error
if network == nil {
continue
}
if _, ok := network.Spec.Annotations.Labels["com.docker.swarm.internal"]; ok {
if network.Spec.Internal {
return grpc.Errorf(codes.InvalidArgument,
"Service cannot be explicitly attached to %q network which is a swarm internal network",
network.Spec.Annotations.Name)
@ -424,6 +425,36 @@ func (s *Server) checkSecretExistence(tx store.Tx, spec *api.ServiceSpec) error
return nil
}
func doesServiceNeedIngress(srv *api.Service) bool {
// Only VIP mode with target ports needs routing mesh.
// If no endpoint is specified, it defaults to VIP mode but no target ports
// are specified, so the service does not need the routing mesh.
if srv.Spec.Endpoint == nil || srv.Spec.Endpoint.Mode != api.ResolutionModeVirtualIP {
return false
}
// Go through the ports' config
for _, p := range srv.Spec.Endpoint.Ports {
if p.PublishMode != api.PublishModeIngress {
continue
}
if p.PublishedPort != 0 {
return true
}
}
// Go through the ports' state
if srv.Endpoint != nil {
for _, p := range srv.Endpoint.Ports {
if p.PublishMode != api.PublishModeIngress {
continue
}
if p.PublishedPort != 0 {
return true
}
}
}
return false
}
// CreateService creates and returns a Service based on the provided ServiceSpec.
// - Returns `InvalidArgument` if the ServiceSpec is malformed.
// - Returns `Unimplemented` if the ServiceSpec references unimplemented features.
@ -449,6 +480,12 @@ func (s *Server) CreateService(ctx context.Context, request *api.CreateServiceRe
Spec: *request.Spec,
}
if doesServiceNeedIngress(service) {
if _, err := allocator.GetIngressNetwork(s.store); err == allocator.ErrNoIngress {
return nil, grpc.Errorf(codes.FailedPrecondition, "service needs ingress network, but no ingress network is present")
}
}
err := s.store.Update(func(tx store.Tx) error {
// Check to see if all the secrets being added exist as objects
// in our datastore
@ -578,6 +615,12 @@ func (s *Server) UpdateService(ctx context.Context, request *api.UpdateServiceRe
service.UpdateStatus = nil
}
if doesServiceNeedIngress(service) {
if _, err := allocator.GetIngressNetwork(s.store); err == allocator.ErrNoIngress {
return grpc.Errorf(codes.FailedPrecondition, "service needs ingress network, but no ingress network is present")
}
}
return store.UpdateService(tx, service)
})
if err != nil {

2
vendor/github.com/docker/swarmkit/manager/dirty.go generated vendored
View file

@ -47,7 +47,7 @@ func (m *Manager) IsStateDirty() (bool, error) {
if structField.Type.Kind() != reflect.Slice {
panic("unexpected field type in StoreSnapshot")
}
if structField.Name != "Nodes" && structField.Name != "Clusters" && field.Len() != 0 {
if structField.Name != "Nodes" && structField.Name != "Clusters" && structField.Name != "Networks" && field.Len() != 0 {
// One of the other data types has an entry
return true, nil
}

39
vendor/github.com/docker/swarmkit/manager/manager.go generated vendored
View file

@ -19,6 +19,7 @@ import (
"github.com/docker/swarmkit/api"
"github.com/docker/swarmkit/ca"
"github.com/docker/swarmkit/connectionbroker"
"github.com/docker/swarmkit/identity"
"github.com/docker/swarmkit/log"
"github.com/docker/swarmkit/manager/allocator"
"github.com/docker/swarmkit/manager/controlapi"
@ -892,7 +893,18 @@ func (m *Manager) becomeLeader(ctx context.Context) {
rootCA))
// Add Node entry for ourself, if one
// doesn't exist already.
store.CreateNode(tx, managerNode(nodeID, m.config.Availability))
freshCluster := nil == store.CreateNode(tx, managerNode(nodeID, m.config.Availability))
if freshCluster {
// This is a fresh swarm cluster. Add to store now any initial
// cluster resource, like the default ingress network which
// provides the routing mesh for this cluster.
log.G(ctx).Info("Creating default ingress network")
if err := store.CreateNetwork(tx, newIngressNetwork()); err != nil {
log.G(ctx).WithError(err).Error("failed to create default ingress network")
}
}
return nil
})
@ -1084,3 +1096,28 @@ func managerNode(nodeID string, availability api.NodeSpec_Availability) *api.Nod
},
}
}
// newIngressNetwork returns the network object for the default ingress
// network, the network which provides the routing mesh. Caller will save to
// store this object once, at fresh cluster creation. It is expected to
// call this function inside a store update transaction.
func newIngressNetwork() *api.Network {
return &api.Network{
ID: identity.NewID(),
Spec: api.NetworkSpec{
Ingress: true,
Annotations: api.Annotations{
Name: "ingress",
},
DriverConfig: &api.Driver{},
IPAM: &api.IPAMOptions{
Driver: &api.Driver{},
Configs: []*api.IPAMConfig{
{
Subnet: "10.255.0.0/16",
},
},
},
},
}
}

4
vendor/github.com/docker/swarmkit/manager/scheduler/nodeset.go generated vendored
View file

@ -111,6 +111,10 @@ func (ns *nodeSet) tree(serviceID string, preferences []*api.PlacementPreference
tree = next
}
if node.ActiveTasksCountByService != nil {
tree.tasks += node.ActiveTasksCountByService[serviceID]
}
if tree.nodeHeap.lessFunc == nil {
tree.nodeHeap.lessFunc = nodeLess
}

26
vendor/github.com/docker/swarmkit/manager/scheduler/scheduler.go generated vendored
View file

@ -1,7 +1,6 @@
package scheduler
import (
"container/list"
"time"
"github.com/docker/swarmkit/api"
@ -30,7 +29,7 @@ type schedulingDecision struct {
// Scheduler assigns tasks to nodes.
type Scheduler struct {
store *store.MemoryStore
unassignedTasks *list.List
unassignedTasks map[string]*api.Task
// preassignedTasks already have NodeID, need resource validation
preassignedTasks map[string]*api.Task
nodeSet nodeSet
@ -47,7 +46,7 @@ type Scheduler struct {
func New(store *store.MemoryStore) *Scheduler {
return &Scheduler{
store: store,
unassignedTasks: list.New(),
unassignedTasks: make(map[string]*api.Task),
preassignedTasks: make(map[string]*api.Task),
allTasks: make(map[string]*api.Task),
stopChan: make(chan struct{}),
@ -191,7 +190,7 @@ func (s *Scheduler) Stop() {
// enqueue queues a task for scheduling.
func (s *Scheduler) enqueue(t *api.Task) {
s.unassignedTasks.PushBack(t)
s.unassignedTasks[t.ID] = t
}
func (s *Scheduler) createTask(ctx context.Context, t *api.Task) int {
@ -333,15 +332,12 @@ func (s *Scheduler) processPreassignedTasks(ctx context.Context) {
// tick attempts to schedule the queue.
func (s *Scheduler) tick(ctx context.Context) {
tasksByCommonSpec := make(map[string]map[string]*api.Task)
schedulingDecisions := make(map[string]schedulingDecision, s.unassignedTasks.Len())
schedulingDecisions := make(map[string]schedulingDecision, len(s.unassignedTasks))
var next *list.Element
for e := s.unassignedTasks.Front(); e != nil; e = next {
next = e.Next()
t := s.allTasks[e.Value.(*api.Task).ID]
for taskID, t := range s.unassignedTasks {
if t == nil || t.NodeID != "" {
// task deleted or already assigned
s.unassignedTasks.Remove(e)
delete(s.unassignedTasks, taskID)
continue
}
@ -362,8 +358,8 @@ func (s *Scheduler) tick(ctx context.Context) {
if tasksByCommonSpec[taskGroupKey] == nil {
tasksByCommonSpec[taskGroupKey] = make(map[string]*api.Task)
}
tasksByCommonSpec[taskGroupKey][t.ID] = t
s.unassignedTasks.Remove(e)
tasksByCommonSpec[taskGroupKey][taskID] = t
delete(s.unassignedTasks, taskID)
}
for _, taskGroup := range tasksByCommonSpec {
@ -602,6 +598,12 @@ func (s *Scheduler) scheduleNTasksOnNodes(ctx context.Context, n int, taskGroup
nodeIter := 0
nodeCount := len(nodes)
for taskID, t := range taskGroup {
// Skip tasks which were already scheduled because they ended
// up in two groups at once.
if _, exists := schedulingDecisions[taskID]; exists {
continue
}
node := &nodes[nodeIter%nodeCount]
log.G(ctx).WithField("task.id", t.ID).Debugf("assigning to node %s", node.ID)