From a327c231b5c68c13b7dcde2fdc83b8e4cec59c43 Mon Sep 17 00:00:00 2001 From: Diogo Monica Date: Wed, 27 Jul 2016 21:17:00 -0700 Subject: [PATCH] Add --force to node removal Signed-off-by: Diogo Monica --- api/client/node/remove.go | 20 +- api/server/router/swarm/backend.go | 2 +- api/server/router/swarm/cluster_routes.go | 8 +- daemon/cluster/cluster.go | 4 +- docs/reference/commandline/node_rm.md | 21 +- hack/vendor.sh | 5 +- integration-cli/daemon_swarm.go | 11 + integration-cli/docker_api_swarm_test.go | 31 +++ .../docker/engine-api/client/client_darwin.go | 4 - .../docker/engine-api/client/client_unix.go | 2 +- .../docker/engine-api/client/interface.go | 2 +- .../docker/engine-api/client/node_remove.go | 17 +- .../docker/engine-api/types/client.go | 7 +- .../github.com/docker/swarmkit/agent/agent.go | 4 +- .../github.com/docker/swarmkit/agent/node.go | 4 +- .../github.com/docker/swarmkit/agent/task.go | 2 +- .../docker/swarmkit/api/control.pb.go | 222 ++++++++++-------- .../docker/swarmkit/api/control.proto | 1 + .../github.com/docker/swarmkit/ca/server.go | 10 +- .../swarmkit/manager/allocator/network.go | 2 +- .../swarmkit/manager/controlapi/node.go | 2 +- .../swarmkit/manager/dispatcher/dispatcher.go | 43 ++-- .../swarmkit/manager/dispatcher/nodes.go | 29 ++- .../docker/swarmkit/manager/manager.go | 19 +- .../swarmkit/manager/raftpicker/raftpicker.go | 92 +++++--- .../swarmkit/manager/state/raft/raft.go | 18 +- .../docker/swarmkit/picker/picker.go | 26 +- 27 files changed, 403 insertions(+), 205 deletions(-) delete mode 100644 vendor/src/github.com/docker/engine-api/client/client_darwin.go diff --git a/api/client/node/remove.go b/api/client/node/remove.go index 4966f26675..bb54a831e8 100644 --- a/api/client/node/remove.go +++ b/api/client/node/remove.go @@ -7,26 +7,36 @@ import ( "github.com/docker/docker/api/client" "github.com/docker/docker/cli" + "github.com/docker/engine-api/types" "github.com/spf13/cobra" ) +type removeOptions struct { + force bool +} + func newRemoveCommand(dockerCli *client.DockerCli) *cobra.Command { - return &cobra.Command{ - Use: "rm NODE [NODE...]", + opts := removeOptions{} + + cmd := &cobra.Command{ + Use: "rm [OPTIONS] NODE [NODE...]", Aliases: []string{"remove"}, Short: "Remove one or more nodes from the swarm", Args: cli.RequiresMinArgs(1), RunE: func(cmd *cobra.Command, args []string) error { - return runRemove(dockerCli, args) + return runRemove(dockerCli, args, opts) }, } + flags := cmd.Flags() + flags.BoolVar(&opts.force, "force", false, "Force remove an active node") + return cmd } -func runRemove(dockerCli *client.DockerCli, args []string) error { +func runRemove(dockerCli *client.DockerCli, args []string, opts removeOptions) error { client := dockerCli.Client() ctx := context.Background() for _, nodeID := range args { - err := client.NodeRemove(ctx, nodeID) + err := client.NodeRemove(ctx, nodeID, types.NodeRemoveOptions{Force: opts.force}) if err != nil { return err } diff --git a/api/server/router/swarm/backend.go b/api/server/router/swarm/backend.go index 36ede04b84..e51ce7ba6b 100644 --- a/api/server/router/swarm/backend.go +++ b/api/server/router/swarm/backend.go @@ -20,7 +20,7 @@ type Backend interface { GetNodes(basictypes.NodeListOptions) ([]types.Node, error) GetNode(string) (types.Node, error) UpdateNode(string, uint64, types.NodeSpec) error - RemoveNode(string) error + RemoveNode(string, bool) error GetTasks(basictypes.TaskListOptions) ([]types.Task, error) GetTask(string) (types.Task, error) } diff --git a/api/server/router/swarm/cluster_routes.go b/api/server/router/swarm/cluster_routes.go index 911284b40d..53b49b7d59 100644 --- a/api/server/router/swarm/cluster_routes.go +++ b/api/server/router/swarm/cluster_routes.go @@ -219,7 +219,13 @@ func (sr *swarmRouter) updateNode(ctx context.Context, w http.ResponseWriter, r } func (sr *swarmRouter) removeNode(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error { - if err := sr.backend.RemoveNode(vars["id"]); err != nil { + if err := httputils.ParseForm(r); err != nil { + return err + } + + force := httputils.BoolValue(r, "force") + + if err := sr.backend.RemoveNode(vars["id"], force); err != nil { logrus.Errorf("Error removing node %s: %v", vars["id"], err) return err } diff --git a/daemon/cluster/cluster.go b/daemon/cluster/cluster.go index 3dfbeaa89a..a0816316b1 100644 --- a/daemon/cluster/cluster.go +++ b/daemon/cluster/cluster.go @@ -1023,7 +1023,7 @@ func (c *Cluster) UpdateNode(nodeID string, version uint64, spec types.NodeSpec) } // RemoveNode removes a node from a cluster -func (c *Cluster) RemoveNode(input string) error { +func (c *Cluster) RemoveNode(input string, force bool) error { c.RLock() defer c.RUnlock() @@ -1039,7 +1039,7 @@ func (c *Cluster) RemoveNode(input string) error { return err } - if _, err := c.client.RemoveNode(ctx, &swarmapi.RemoveNodeRequest{NodeID: node.ID}); err != nil { + if _, err := c.client.RemoveNode(ctx, &swarmapi.RemoveNodeRequest{NodeID: node.ID, Force: force}); err != nil { return err } return nil diff --git a/docs/reference/commandline/node_rm.md b/docs/reference/commandline/node_rm.md index e2f913e1a8..f5d5daaf1c 100644 --- a/docs/reference/commandline/node_rm.md +++ b/docs/reference/commandline/node_rm.md @@ -11,7 +11,7 @@ parent = "smn_cli" # node rm ```markdown -Usage: docker node rm NODE [NODE...] +Usage: docker node rm [OPTIONS] NODE [NODE...] Remove one or more nodes from the swarm @@ -19,6 +19,7 @@ Aliases: rm, remove Options: + --force Force remove an active node --help Print usage ``` @@ -30,6 +31,24 @@ Example output: $ docker node rm swarm-node-02 Node swarm-node-02 removed from swarm +Removes nodes from the swarm that are in the down state. Attempting to remove +an active node will result in an error: + +```bash +$ docker node rm swarm-node-03 +Error response from daemon: rpc error: code = 9 desc = node swarm-node-03 is not down and can't be removed +``` + +If a worker node becomes compromised, exhibits unexpected or unwanted behavior, or if you lose access to it so +that a clean shutdown is impossible, you can use the force option. + +```bash +$ docker node rm --force swarm-node-03 +Node swarm-node-03 removed from swarm +``` + +Note that manager nodes have to be demoted to worker nodes before they can be removed +from the cluster. ## Related information diff --git a/hack/vendor.sh b/hack/vendor.sh index 6396ffa9d3..87738fced2 100755 --- a/hack/vendor.sh +++ b/hack/vendor.sh @@ -60,7 +60,8 @@ clone git golang.org/x/net 2beffdc2e92c8a3027590f898fe88f69af48a3f8 https://gith clone git golang.org/x/sys eb2c74142fd19a79b3f237334c7384d5167b1b46 https://github.com/golang/sys.git clone git github.com/docker/go-units 651fc226e7441360384da338d0fd37f2440ffbe3 clone git github.com/docker/go-connections fa2850ff103453a9ad190da0df0af134f0314b3d -clone git github.com/docker/engine-api 3d1601b9d2436a70b0dfc045a23f6503d19195df + +clone git github.com/docker/engine-api 228c7390a733320d48697cb41ae8cde4942cd3e5 clone git github.com/RackSec/srslog 259aed10dfa74ea2961eddd1d9847619f6e98837 clone git github.com/imdario/mergo 0.2.1 @@ -139,7 +140,7 @@ clone git github.com/docker/docker-credential-helpers v0.3.0 clone git github.com/docker/containerd 0ac3cd1be170d180b2baed755e8f0da547ceb267 # cluster -clone git github.com/docker/swarmkit 9d4c2f73124e70f8fa85f9076635b827d17b109f +clone git github.com/docker/swarmkit e1c0d64515d839b76e2ef33d396c74933753ffaf clone git github.com/golang/mock bd3c8e81be01eef76d4b503f5e687d2d1354d2d9 clone git github.com/gogo/protobuf 43a2e0b1c32252bfbbdf81f7faa7a88fb3fa4028 clone git github.com/cloudflare/cfssl b895b0549c0ff676f92cf09ba971ae02bb41367b diff --git a/integration-cli/daemon_swarm.go b/integration-cli/daemon_swarm.go index d132c5ede1..6425793042 100644 --- a/integration-cli/daemon_swarm.go +++ b/integration-cli/daemon_swarm.go @@ -208,6 +208,17 @@ func (d *SwarmDaemon) getNode(c *check.C, id string) *swarm.Node { return &node } +func (d *SwarmDaemon) removeNode(c *check.C, id string, force bool) { + url := "/nodes/" + id + if force { + url += "?force=1" + } + + status, out, err := d.SockRequest("DELETE", url, nil) + c.Assert(status, checker.Equals, http.StatusOK, check.Commentf("output: %q", string(out))) + c.Assert(err, checker.IsNil) +} + func (d *SwarmDaemon) updateNode(c *check.C, id string, f ...nodeConstructor) { for i := 0; ; i++ { node := d.getNode(c, id) diff --git a/integration-cli/docker_api_swarm_test.go b/integration-cli/docker_api_swarm_test.go index b872ae6b08..acdba81f2c 100644 --- a/integration-cli/docker_api_swarm_test.go +++ b/integration-cli/docker_api_swarm_test.go @@ -544,6 +544,37 @@ func (s *DockerSwarmSuite) TestApiSwarmNodeUpdate(c *check.C) { c.Assert(n.Spec.Availability, checker.Equals, swarm.NodeAvailabilityPause) } +func (s *DockerSwarmSuite) TestApiSwarmNodeRemove(c *check.C) { + testRequires(c, Network) + d1 := s.AddDaemon(c, true, true) + d2 := s.AddDaemon(c, true, false) + _ = s.AddDaemon(c, true, false) + + nodes := d1.listNodes(c) + c.Assert(len(nodes), checker.Equals, 3, check.Commentf("nodes: %#v", nodes)) + + // Getting the info so we can take the NodeID + d2Info, err := d2.info() + c.Assert(err, checker.IsNil) + + // forceful removal of d2 should work + d1.removeNode(c, d2Info.NodeID, true) + + nodes = d1.listNodes(c) + c.Assert(len(nodes), checker.Equals, 2, check.Commentf("nodes: %#v", nodes)) + + // Restart the node that was removed + err = d2.Restart() + c.Assert(err, checker.IsNil) + + // Give some time for the node to rejoin + time.Sleep(1 * time.Second) + + // Make sure the node didn't rejoin + nodes = d1.listNodes(c) + c.Assert(len(nodes), checker.Equals, 2, check.Commentf("nodes: %#v", nodes)) +} + func (s *DockerSwarmSuite) TestApiSwarmNodeDrainPause(c *check.C) { d1 := s.AddDaemon(c, true, true) d2 := s.AddDaemon(c, true, false) diff --git a/vendor/src/github.com/docker/engine-api/client/client_darwin.go b/vendor/src/github.com/docker/engine-api/client/client_darwin.go deleted file mode 100644 index 4b47a178c4..0000000000 --- a/vendor/src/github.com/docker/engine-api/client/client_darwin.go +++ /dev/null @@ -1,4 +0,0 @@ -package client - -// DefaultDockerHost defines os specific default if DOCKER_HOST is unset -const DefaultDockerHost = "tcp://127.0.0.1:2375" diff --git a/vendor/src/github.com/docker/engine-api/client/client_unix.go b/vendor/src/github.com/docker/engine-api/client/client_unix.go index 572c5f87a7..89de892c85 100644 --- a/vendor/src/github.com/docker/engine-api/client/client_unix.go +++ b/vendor/src/github.com/docker/engine-api/client/client_unix.go @@ -1,4 +1,4 @@ -// +build linux freebsd solaris openbsd +// +build linux freebsd solaris openbsd darwin package client diff --git a/vendor/src/github.com/docker/engine-api/client/interface.go b/vendor/src/github.com/docker/engine-api/client/interface.go index 2e0491ab70..1b4fa4216f 100644 --- a/vendor/src/github.com/docker/engine-api/client/interface.go +++ b/vendor/src/github.com/docker/engine-api/client/interface.go @@ -94,7 +94,7 @@ type NetworkAPIClient interface { type NodeAPIClient interface { NodeInspectWithRaw(ctx context.Context, nodeID string) (swarm.Node, []byte, error) NodeList(ctx context.Context, options types.NodeListOptions) ([]swarm.Node, error) - NodeRemove(ctx context.Context, nodeID string) error + NodeRemove(ctx context.Context, nodeID string, options types.NodeRemoveOptions) error NodeUpdate(ctx context.Context, nodeID string, version swarm.Version, node swarm.NodeSpec) error } diff --git a/vendor/src/github.com/docker/engine-api/client/node_remove.go b/vendor/src/github.com/docker/engine-api/client/node_remove.go index a22ee93f4b..a9cf8ba857 100644 --- a/vendor/src/github.com/docker/engine-api/client/node_remove.go +++ b/vendor/src/github.com/docker/engine-api/client/node_remove.go @@ -1,10 +1,21 @@ package client -import "golang.org/x/net/context" +import ( + "net/url" + + "github.com/docker/engine-api/types" + + "golang.org/x/net/context" +) // NodeRemove removes a Node. -func (cli *Client) NodeRemove(ctx context.Context, nodeID string) error { - resp, err := cli.delete(ctx, "/nodes/"+nodeID, nil, nil) +func (cli *Client) NodeRemove(ctx context.Context, nodeID string, options types.NodeRemoveOptions) error { + query := url.Values{} + if options.Force { + query.Set("force", "1") + } + + resp, err := cli.delete(ctx, "/nodes/"+nodeID, query, nil) ensureReaderClosed(resp) return err } diff --git a/vendor/src/github.com/docker/engine-api/types/client.go b/vendor/src/github.com/docker/engine-api/types/client.go index def3f06197..c6d244d316 100644 --- a/vendor/src/github.com/docker/engine-api/types/client.go +++ b/vendor/src/github.com/docker/engine-api/types/client.go @@ -241,11 +241,16 @@ func (v VersionResponse) ServerOK() bool { return v.Server != nil } -// NodeListOptions holds parameters to list nodes with. +// NodeListOptions holds parameters to list nodes with. type NodeListOptions struct { Filter filters.Args } +// NodeRemoveOptions holds parameters to remove nodes with. +type NodeRemoveOptions struct { + Force bool +} + // ServiceCreateOptions contains the options to use when creating a service. type ServiceCreateOptions struct { // EncodedRegistryAuth is the encoded registry authorization credentials to diff --git a/vendor/src/github.com/docker/swarmkit/agent/agent.go b/vendor/src/github.com/docker/swarmkit/agent/agent.go index 2933a268d7..6f2f276521 100644 --- a/vendor/src/github.com/docker/swarmkit/agent/agent.go +++ b/vendor/src/github.com/docker/swarmkit/agent/agent.go @@ -12,12 +12,12 @@ import ( ) const ( - initialSessionFailureBackoff = time.Second + initialSessionFailureBackoff = 100 * time.Millisecond maxSessionFailureBackoff = 8 * time.Second ) // Agent implements the primary node functionality for a member of a swarm -// cluster. The primary functionality id to run and report on the status of +// cluster. The primary functionality is to run and report on the status of // tasks assigned to the node. type Agent struct { config *Config diff --git a/vendor/src/github.com/docker/swarmkit/agent/node.go b/vendor/src/github.com/docker/swarmkit/agent/node.go index 61f229b264..6de6e57d26 100644 --- a/vendor/src/github.com/docker/swarmkit/agent/node.go +++ b/vendor/src/github.com/docker/swarmkit/agent/node.go @@ -187,7 +187,7 @@ func (n *Node) run(ctx context.Context) (err error) { if n.config.JoinAddr != "" || n.config.ForceNewCluster { n.remotes = newPersistentRemotes(filepath.Join(n.config.StateDir, stateFilename)) if n.config.JoinAddr != "" { - n.remotes.Observe(api.Peer{Addr: n.config.JoinAddr}, 1) + n.remotes.Observe(api.Peer{Addr: n.config.JoinAddr}, picker.DefaultObservationWeight) } } @@ -647,7 +647,7 @@ func (n *Node) runManager(ctx context.Context, securityConfig *ca.SecurityConfig go func(ready chan struct{}) { select { case <-ready: - n.remotes.Observe(api.Peer{NodeID: n.nodeID, Addr: n.config.ListenRemoteAPI}, 5) + n.remotes.Observe(api.Peer{NodeID: n.nodeID, Addr: n.config.ListenRemoteAPI}, picker.DefaultObservationWeight) case <-connCtx.Done(): } }(ready) diff --git a/vendor/src/github.com/docker/swarmkit/agent/task.go b/vendor/src/github.com/docker/swarmkit/agent/task.go index 005ffdf973..2e6acb2a57 100644 --- a/vendor/src/github.com/docker/swarmkit/agent/task.go +++ b/vendor/src/github.com/docker/swarmkit/agent/task.go @@ -200,7 +200,7 @@ func (tm *taskManager) run(ctx context.Context) { cancel() // cancel outstanding if necessary. } else { // If this channel op fails, it means there is already a - // message un the run queue. + // message on the run queue. select { case run <- struct{}{}: default: diff --git a/vendor/src/github.com/docker/swarmkit/api/control.pb.go b/vendor/src/github.com/docker/swarmkit/api/control.pb.go index 1cdc86741f..3c4fe26bab 100644 --- a/vendor/src/github.com/docker/swarmkit/api/control.pb.go +++ b/vendor/src/github.com/docker/swarmkit/api/control.pb.go @@ -106,6 +106,7 @@ func (*UpdateNodeResponse) Descriptor() ([]byte, []int) { return fileDescriptorC // RemoveNodeRequest requests to delete the specified node from store. type RemoveNodeRequest struct { NodeID string `protobuf:"bytes,1,opt,name=node_id,json=nodeId,proto3" json:"node_id,omitempty"` + Force bool `protobuf:"varint,2,opt,name=force,proto3" json:"force,omitempty"` } func (m *RemoveNodeRequest) Reset() { *m = RemoveNodeRequest{} } @@ -786,6 +787,7 @@ func (m *RemoveNodeRequest) Copy() *RemoveNodeRequest { o := &RemoveNodeRequest{ NodeID: m.NodeID, + Force: m.Force, } return o @@ -1473,9 +1475,10 @@ func (this *RemoveNodeRequest) GoString() string { if this == nil { return "nil" } - s := make([]string, 0, 5) + s := make([]string, 0, 6) s = append(s, "&api.RemoveNodeRequest{") s = append(s, "NodeID: "+fmt.Sprintf("%#v", this.NodeID)+",\n") + s = append(s, "Force: "+fmt.Sprintf("%#v", this.Force)+",\n") s = append(s, "}") return strings.Join(s, "") } @@ -2938,6 +2941,16 @@ func (m *RemoveNodeRequest) MarshalTo(data []byte) (int, error) { i = encodeVarintControl(data, i, uint64(len(m.NodeID))) i += copy(data[i:], m.NodeID) } + if m.Force { + data[i] = 0x10 + i++ + if m.Force { + data[i] = 1 + } else { + data[i] = 0 + } + i++ + } return i, nil } @@ -4692,6 +4705,9 @@ func (m *RemoveNodeRequest) Size() (n int) { if l > 0 { n += 1 + l + sovControl(uint64(l)) } + if m.Force { + n += 2 + } return n } @@ -5286,6 +5302,7 @@ func (this *RemoveNodeRequest) String() string { } s := strings.Join([]string{`&RemoveNodeRequest{`, `NodeID:` + fmt.Sprintf("%v", this.NodeID) + `,`, + `Force:` + fmt.Sprintf("%v", this.Force) + `,`, `}`, }, "") return s @@ -6617,6 +6634,26 @@ func (m *RemoveNodeRequest) Unmarshal(data []byte) error { } m.NodeID = string(data[iNdEx:postIndex]) iNdEx = postIndex + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Force", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowControl + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := data[iNdEx] + iNdEx++ + v |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.Force = bool(v != 0) default: iNdEx = preIndex skippy, err := skipControl(data[iNdEx:]) @@ -10521,99 +10558,100 @@ var ( ) var fileDescriptorControl = []byte{ - // 1498 bytes of a gzipped FileDescriptorProto + // 1512 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xcc, 0x59, 0xcf, 0x6f, 0x1b, 0xc5, - 0x17, 0xaf, 0x9d, 0x34, 0x8e, 0x9f, 0x6b, 0xb7, 0x9e, 0xba, 0xfa, 0x46, 0x6e, 0xbf, 0x09, 0xda, + 0x17, 0xaf, 0x9d, 0x34, 0x8e, 0x9f, 0x6b, 0xb7, 0x9e, 0xba, 0xfa, 0x46, 0x6e, 0xbf, 0x0d, 0xda, 0xd2, 0x34, 0x91, 0x82, 0x03, 0x8e, 0x2a, 0x02, 0x48, 0x20, 0x9c, 0xd0, 0xca, 0xd0, 0x86, 0x6a, - 0xd3, 0x02, 0xb7, 0xc8, 0xb1, 0xa7, 0x61, 0xf1, 0x8f, 0x35, 0xbb, 0x9b, 0xb4, 0x11, 0x17, 0x38, + 0xd3, 0x02, 0xb7, 0xc8, 0xb1, 0x27, 0x61, 0xf1, 0x8f, 0x35, 0xbb, 0x9b, 0xb4, 0x11, 0x17, 0x38, 0x20, 0xf1, 0x27, 0x70, 0xe5, 0xca, 0x81, 0x7f, 0x81, 0x6b, 0xc4, 0x89, 0x0b, 0x12, 0xa7, 0x88, 0xf6, 0xc4, 0x09, 0xf1, 0x17, 0x20, 0xe6, 0xc7, 0x9b, 0xdd, 0xf5, 0x7a, 0x76, 0x6d, 0x27, 0x41, - 0xe9, 0xc1, 0xca, 0xee, 0xcc, 0xe7, 0xfd, 0x98, 0x79, 0x9f, 0xf7, 0xf6, 0xcd, 0x04, 0xf2, 0x4d, - 0xbb, 0xe7, 0x39, 0x76, 0xa7, 0xd2, 0x77, 0x6c, 0xcf, 0x26, 0xa4, 0x65, 0x37, 0xdb, 0xd4, 0xa9, - 0xb8, 0x4f, 0x1b, 0x4e, 0xb7, 0x6d, 0x79, 0x95, 0x83, 0x37, 0xca, 0x39, 0xb7, 0x4f, 0x9b, 0xae, - 0x04, 0x94, 0xf3, 0xf6, 0xee, 0x17, 0xb4, 0xe9, 0xa9, 0xd7, 0x9c, 0x77, 0xd8, 0xa7, 0xea, 0xa5, - 0xb4, 0x67, 0xef, 0xd9, 0xe2, 0x71, 0x95, 0x3f, 0xe1, 0xe8, 0xd5, 0x7e, 0x67, 0x7f, 0xcf, 0xea, - 0xad, 0xca, 0x3f, 0x72, 0xd0, 0xb8, 0x03, 0x85, 0x7b, 0xd4, 0xdb, 0xb2, 0x5b, 0xd4, 0xa4, 0x5f, - 0xee, 0x53, 0xd7, 0x23, 0x37, 0x21, 0xd3, 0x63, 0xaf, 0x3b, 0x56, 0x6b, 0x2e, 0xf5, 0x4a, 0x6a, - 0x29, 0x5b, 0x83, 0x17, 0xc7, 0x0b, 0x33, 0x1c, 0x51, 0xdf, 0x34, 0x67, 0xf8, 0x54, 0xbd, 0x65, - 0xbc, 0x07, 0x97, 0x7d, 0x31, 0xb7, 0x6f, 0xf7, 0x5c, 0x4a, 0x56, 0x60, 0x9a, 0x4f, 0x0a, 0xa1, - 0x5c, 0x75, 0xae, 0x32, 0xbc, 0x80, 0x8a, 0xc0, 0x0b, 0x94, 0x71, 0x3c, 0x05, 0x57, 0xee, 0x5b, - 0xae, 0x50, 0xe1, 0x2a, 0xd3, 0x77, 0x21, 0xf3, 0xc4, 0xea, 0x78, 0xd4, 0x71, 0x51, 0xcb, 0x8a, - 0x4e, 0x4b, 0x54, 0xac, 0x72, 0x57, 0xca, 0x98, 0x4a, 0xb8, 0xfc, 0xcd, 0x14, 0x64, 0x70, 0x90, - 0x94, 0xe0, 0x62, 0xaf, 0xd1, 0xa5, 0x5c, 0xe3, 0xd4, 0x52, 0xd6, 0x94, 0x2f, 0x64, 0x15, 0x72, - 0x56, 0x6b, 0xa7, 0xef, 0xd0, 0x27, 0xd6, 0x33, 0x36, 0x97, 0xe6, 0x73, 0xb5, 0x02, 0x5b, 0x28, - 0xd4, 0x37, 0x1f, 0xe2, 0xa8, 0x09, 0x56, 0x4b, 0x3d, 0x93, 0x87, 0x30, 0xd3, 0x69, 0xec, 0xd2, - 0x8e, 0x3b, 0x37, 0xc5, 0xb0, 0xb9, 0xea, 0xfa, 0x24, 0x9e, 0x55, 0xee, 0x0b, 0xd1, 0x0f, 0x58, - 0x80, 0x0f, 0x4d, 0xd4, 0x43, 0xea, 0x90, 0xeb, 0xd2, 0xee, 0x2e, 0x9b, 0xfe, 0xdc, 0xea, 0xbb, - 0x73, 0xd3, 0x4c, 0x6d, 0xa1, 0x7a, 0x3b, 0x6e, 0xdb, 0xb6, 0x59, 0xe8, 0x2b, 0x0f, 0x7c, 0xbc, - 0x19, 0x96, 0x25, 0x55, 0xb8, 0xc8, 0x98, 0xc3, 0xd6, 0x71, 0x51, 0x28, 0xb9, 0x11, 0xbb, 0xf7, - 0x0c, 0x64, 0x4a, 0x28, 0x0b, 0x73, 0x9e, 0x6f, 0x45, 0xb0, 0x07, 0x33, 0x62, 0x7f, 0x2e, 0xf1, - 0x41, 0xb5, 0xea, 0xf2, 0x5b, 0x90, 0x0b, 0xb9, 0x4e, 0xae, 0xc0, 0x54, 0x9b, 0x1e, 0x4a, 0x5a, - 0x98, 0xfc, 0x91, 0xef, 0xee, 0x41, 0xa3, 0xb3, 0x4f, 0xd9, 0x0e, 0xf2, 0x31, 0xf9, 0xf2, 0x76, - 0x7a, 0x3d, 0x65, 0x6c, 0x40, 0x31, 0xb4, 0x1d, 0xc8, 0x91, 0x0a, 0x0b, 0x06, 0x1f, 0x10, 0xc1, - 0x48, 0x22, 0x89, 0x84, 0x19, 0x3f, 0xa6, 0xa0, 0xf8, 0xb8, 0xdf, 0x6a, 0x78, 0x74, 0x52, 0x86, - 0x92, 0x77, 0xe1, 0x92, 0x00, 0x1d, 0xb0, 0x4d, 0xb2, 0xec, 0x9e, 0x70, 0x30, 0x57, 0xbd, 0xae, - 0xb3, 0xf8, 0x89, 0x84, 0x98, 0x39, 0x2e, 0x80, 0x2f, 0xe4, 0x75, 0x98, 0xe6, 0xe9, 0xc6, 0xc2, - 0xcd, 0xe5, 0x6e, 0x24, 0xc5, 0xc5, 0x14, 0x48, 0xa3, 0x06, 0x24, 0xec, 0xeb, 0x89, 0xd2, 0x62, - 0x1d, 0x8a, 0x26, 0xed, 0xda, 0x07, 0x13, 0xaf, 0xd7, 0x28, 0x01, 0x09, 0x4b, 0x4a, 0xeb, 0x98, - 0xde, 0x8f, 0x1a, 0x6e, 0x3b, 0xa4, 0xcc, 0x63, 0xaf, 0x11, 0x65, 0x1c, 0xc1, 0x95, 0xf1, 0x29, - 0x3f, 0xbd, 0xa5, 0x58, 0xb0, 0x0e, 0x3e, 0x99, 0xb4, 0x0e, 0x81, 0x17, 0xa8, 0x60, 0x1d, 0x13, - 0x9b, 0xf6, 0xd7, 0x11, 0xb6, 0x6e, 0xfc, 0x83, 0xe5, 0x82, 0x0f, 0x9e, 0xa0, 0x5c, 0x84, 0xc5, - 0x86, 0xcb, 0xc5, 0x0f, 0xe7, 0x58, 0x2e, 0x74, 0x9e, 0x69, 0xcb, 0x05, 0x73, 0xc1, 0xa5, 0xce, - 0x81, 0xd5, 0xe4, 0x3c, 0x90, 0xe5, 0x02, 0x5d, 0xd8, 0x96, 0xc3, 0xf5, 0x4d, 0xe6, 0x02, 0x42, - 0xea, 0x2d, 0x97, 0x2c, 0xc2, 0x2c, 0xb2, 0x46, 0xd6, 0x85, 0x6c, 0x2d, 0xc7, 0xd0, 0x19, 0x49, - 0x1b, 0xb6, 0x7a, 0xc9, 0x1b, 0x97, 0x6c, 0x42, 0x81, 0xa5, 0x9a, 0xe5, 0xd0, 0xd6, 0x8e, 0xeb, - 0x31, 0xf6, 0xca, 0x4a, 0x50, 0xa8, 0xfe, 0x3f, 0x2e, 0xc4, 0xdb, 0x1c, 0x65, 0xe6, 0x51, 0x48, - 0xbc, 0x69, 0xca, 0x49, 0xe6, 0x3f, 0x29, 0x27, 0xb8, 0x5d, 0x41, 0x39, 0xe1, 0xac, 0x49, 0x2c, - 0x27, 0x82, 0x46, 0x12, 0x66, 0x7c, 0x04, 0xa5, 0x0d, 0x87, 0x32, 0x7f, 0x71, 0xcb, 0x14, 0x91, - 0xd6, 0x30, 0xd7, 0x25, 0x8b, 0x16, 0x74, 0x6a, 0x50, 0x22, 0x94, 0xee, 0x5b, 0x70, 0x2d, 0xa2, - 0x0c, 0xbd, 0xba, 0x03, 0x19, 0x0c, 0x03, 0x2a, 0xbc, 0x9e, 0xa0, 0xd0, 0x54, 0x58, 0xe3, 0x7d, - 0x28, 0xb2, 0x9c, 0x8b, 0x78, 0xb6, 0x02, 0x10, 0x44, 0x1d, 0xb3, 0x26, 0xcf, 0xc2, 0x98, 0xf5, - 0x83, 0x6e, 0x66, 0xfd, 0x98, 0xb3, 0xf5, 0x91, 0xb0, 0x8a, 0xd3, 0xf9, 0xf3, 0x73, 0x0a, 0x4a, - 0xb2, 0x9e, 0x9d, 0xc6, 0x27, 0x46, 0xaf, 0xcb, 0x0a, 0x3d, 0x41, 0x29, 0x2e, 0xa0, 0x8c, 0xaa, - 0xc6, 0x6b, 0x03, 0xd5, 0x78, 0xfc, 0x08, 0x45, 0x16, 0x70, 0xba, 0x1d, 0xd9, 0x84, 0x92, 0x2c, - 0x4d, 0xa7, 0x0a, 0xd2, 0xff, 0xe0, 0x5a, 0x44, 0x0b, 0xd6, 0xb8, 0x3f, 0xd3, 0x70, 0x95, 0x73, - 0x1c, 0xc7, 0xfd, 0x32, 0x57, 0x8f, 0x96, 0xb9, 0xd5, 0xb8, 0x62, 0x12, 0x91, 0x1c, 0xae, 0x74, - 0xdf, 0xa6, 0xcf, 0xbc, 0xd2, 0x6d, 0x47, 0x2a, 0xdd, 0x3b, 0x13, 0x3a, 0xa7, 0x2d, 0x76, 0x43, - 0xd5, 0x64, 0xfa, 0x6c, 0xab, 0xc9, 0xc7, 0x50, 0x1a, 0x74, 0x09, 0x89, 0xf1, 0x26, 0xcc, 0x62, - 0xa0, 0x54, 0x4d, 0x49, 0x64, 0x86, 0x0f, 0x0e, 0x2a, 0xcb, 0x16, 0xf5, 0x9e, 0xda, 0x4e, 0x7b, - 0x82, 0xca, 0x82, 0x12, 0xba, 0xca, 0xe2, 0x2b, 0x0b, 0x78, 0xdb, 0x93, 0x43, 0x49, 0xbc, 0x55, - 0x52, 0x0a, 0x6b, 0x3c, 0x16, 0x95, 0x25, 0xe2, 0x19, 0x61, 0x7d, 0x09, 0xdb, 0x4d, 0xdc, 0x2f, - 0xf1, 0xcc, 0x89, 0x8c, 0x32, 0x9c, 0xc8, 0xe9, 0x80, 0xc8, 0x28, 0xcb, 0x89, 0x8c, 0x00, 0xbf, - 0xda, 0x9c, 0x91, 0x8f, 0x9f, 0xa9, 0xdc, 0x3a, 0x73, 0x37, 0xfd, 0x7c, 0x8b, 0x78, 0xea, 0xe7, - 0x1b, 0x8e, 0x9f, 0x20, 0xdf, 0x22, 0x92, 0x2f, 0x57, 0xbe, 0xc5, 0x38, 0x77, 0x9e, 0xf9, 0x16, - 0xb8, 0x14, 0xe4, 0x1b, 0x06, 0x2a, 0x31, 0xdf, 0x54, 0xe4, 0x7c, 0x30, 0x7e, 0x2c, 0x37, 0x3a, - 0xfb, 0x2e, 0x5b, 0x53, 0xa8, 0x0e, 0x37, 0xe5, 0x48, 0xa4, 0x0e, 0x23, 0x8e, 0xf3, 0x02, 0x01, - 0x3e, 0x7d, 0x7d, 0x15, 0x01, 0x7d, 0x11, 0x92, 0x44, 0x5f, 0x25, 0xa5, 0xb0, 0x3e, 0x97, 0x70, - 0xe2, 0x04, 0x5c, 0x8a, 0x48, 0xbe, 0x5c, 0x5c, 0x8a, 0x71, 0xee, 0x3c, 0xb9, 0x14, 0xb8, 0x14, - 0x70, 0x09, 0xa3, 0x91, 0xc8, 0x25, 0x15, 0x3a, 0x1f, 0x6c, 0xec, 0x43, 0xf1, 0x43, 0xdb, 0xea, - 0x3d, 0xb2, 0xdb, 0xb4, 0x67, 0xda, 0xac, 0x9d, 0xe5, 0x0d, 0x47, 0x05, 0xae, 0x3a, 0xfc, 0x99, - 0xee, 0x70, 0xc2, 0x31, 0x46, 0x79, 0x7c, 0x5a, 0x78, 0x38, 0x6b, 0x16, 0xe5, 0xd4, 0xa7, 0x62, - 0x46, 0xc8, 0xb1, 0xe3, 0x62, 0x09, 0xf1, 0xdd, 0x46, 0xaf, 0xb1, 0xe7, 0x0b, 0xa4, 0x85, 0x00, - 0x91, 0x73, 0x0f, 0xe4, 0x94, 0x90, 0x30, 0xbe, 0x4b, 0xab, 0xfe, 0xea, 0x34, 0x34, 0xe6, 0xfd, - 0x95, 0x42, 0x4f, 0xd2, 0x5f, 0xa1, 0xcc, 0x04, 0xfd, 0x15, 0x5a, 0x0f, 0xbe, 0x53, 0xe4, 0x1e, - 0xcc, 0x3a, 0xb8, 0x5f, 0x2c, 0xc8, 0x5c, 0xf0, 0x96, 0x4e, 0x70, 0x68, 0x73, 0x6b, 0xd3, 0x47, - 0xc7, 0x0b, 0x17, 0x4c, 0x5f, 0x38, 0x68, 0xd4, 0xce, 0x26, 0x1b, 0xab, 0xbf, 0x15, 0x21, 0xb3, - 0x21, 0xaf, 0xd3, 0x88, 0x05, 0x19, 0xbc, 0xa9, 0x22, 0x86, 0x4e, 0x78, 0xf0, 0xf6, 0xab, 0x7c, - 0x33, 0x11, 0x83, 0x5f, 0x8e, 0x6b, 0xbf, 0xfc, 0xf4, 0xd7, 0xf7, 0xe9, 0xcb, 0x90, 0x17, 0xa0, - 0xd7, 0x30, 0xe2, 0xc4, 0x86, 0xac, 0x7f, 0xe5, 0x41, 0x5e, 0x1d, 0xe7, 0x82, 0xa8, 0x7c, 0x6b, - 0x04, 0x2a, 0xd9, 0xa0, 0x03, 0x10, 0xdc, 0x38, 0x10, 0xad, 0xae, 0xa1, 0xdb, 0x93, 0xf2, 0xe2, - 0x28, 0xd8, 0x48, 0x9b, 0xc1, 0x3d, 0x83, 0xde, 0xe6, 0xd0, 0x0d, 0x86, 0xde, 0xa6, 0xe6, 0xba, - 0x22, 0xc6, 0xa6, 0x8c, 0x21, 0x3f, 0xc9, 0xc5, 0xc6, 0x30, 0x74, 0xcf, 0x10, 0x1b, 0xc3, 0x81, - 0x1b, 0x85, 0xe4, 0x18, 0x8a, 0x73, 0x66, 0x7c, 0x0c, 0xc3, 0xa7, 0xf6, 0xf8, 0x18, 0x0e, 0x1c, - 0x56, 0x47, 0xee, 0xa7, 0x58, 0x5e, 0xc2, 0x7e, 0x86, 0x57, 0xb8, 0x38, 0x0a, 0x36, 0xd2, 0x66, - 0x70, 0x4e, 0xd4, 0xdb, 0x1c, 0x3a, 0x8a, 0xea, 0x6d, 0x0e, 0x1f, 0x37, 0xe3, 0x6c, 0x3e, 0x83, - 0x4b, 0xe1, 0x96, 0x9b, 0xdc, 0x1e, 0xf3, 0x9c, 0x50, 0x5e, 0x1a, 0x0d, 0x4c, 0xb6, 0xfc, 0x15, - 0xe4, 0x07, 0x0e, 0xea, 0x44, 0xab, 0x51, 0x77, 0x31, 0x50, 0x5e, 0x1e, 0x03, 0x39, 0xd2, 0xf8, - 0xc0, 0x19, 0x54, 0x6f, 0x5c, 0x77, 0xce, 0xd6, 0x1b, 0xd7, 0x1e, 0x68, 0x13, 0x8c, 0x0f, 0x1c, - 0x35, 0xf5, 0xc6, 0x75, 0x67, 0x5a, 0xbd, 0x71, 0xfd, 0xb9, 0x35, 0x91, 0x64, 0xd8, 0xba, 0xc5, - 0x92, 0x6c, 0xb0, 0xdd, 0x8f, 0x25, 0x59, 0xb4, 0x77, 0x4f, 0x26, 0x99, 0xea, 0x33, 0xe3, 0x49, - 0x16, 0x69, 0x8e, 0xe3, 0x49, 0x16, 0x6d, 0x59, 0x47, 0x92, 0x4c, 0x2d, 0x38, 0x81, 0x64, 0x91, - 0x35, 0x2f, 0x8f, 0x81, 0x1c, 0x33, 0xce, 0x89, 0xc6, 0x75, 0xe7, 0xab, 0xa4, 0x38, 0x8f, 0x69, - 0x5c, 0xc6, 0x19, 0xbf, 0xc1, 0xb1, 0x71, 0x1e, 0xec, 0x71, 0x62, 0xe3, 0x1c, 0x69, 0x00, 0x46, - 0xc4, 0x59, 0xf5, 0x80, 0xf1, 0x71, 0x8e, 0x34, 0xae, 0xf1, 0x71, 0x8e, 0xb6, 0x93, 0x23, 0xf3, - 0x59, 0x2d, 0x38, 0x21, 0x9f, 0x23, 0x6b, 0x5e, 0x1e, 0x03, 0x99, 0x68, 0xbc, 0x76, 0xe3, 0xe8, - 0xf9, 0xfc, 0x85, 0xdf, 0xd9, 0xef, 0xef, 0xe7, 0xf3, 0xa9, 0xaf, 0x5f, 0xcc, 0xa7, 0x8e, 0xd8, - 0xef, 0x57, 0xf6, 0xfb, 0x83, 0xfd, 0x76, 0x67, 0xc4, 0x7f, 0xf4, 0xd6, 0xfe, 0x0d, 0x00, 0x00, - 0xff, 0xff, 0xf3, 0xcc, 0x22, 0xcd, 0x4a, 0x1c, 0x00, 0x00, + 0xe9, 0xc1, 0xca, 0xee, 0xcc, 0xe7, 0xcd, 0x7b, 0x33, 0x9f, 0xcf, 0xbc, 0x7d, 0x33, 0x81, 0x7c, + 0xd3, 0xee, 0x79, 0x8e, 0xdd, 0xa9, 0xf4, 0x1d, 0xdb, 0xb3, 0x09, 0x69, 0xd9, 0xcd, 0x36, 0x75, + 0x2a, 0xee, 0xd3, 0x86, 0xd3, 0x6d, 0x5b, 0x5e, 0xe5, 0xe0, 0x8d, 0x72, 0xce, 0xed, 0xd3, 0xa6, + 0x2b, 0x01, 0xe5, 0xbc, 0xbd, 0xf3, 0x05, 0x6d, 0x7a, 0xea, 0x35, 0xe7, 0x1d, 0xf6, 0xa9, 0x7a, + 0x29, 0xed, 0xd9, 0x7b, 0xb6, 0x78, 0x5c, 0xe1, 0x4f, 0xd8, 0x7a, 0xb5, 0xdf, 0xd9, 0xdf, 0xb3, + 0x7a, 0x2b, 0xf2, 0x8f, 0x6c, 0x34, 0xee, 0x42, 0xe1, 0x3e, 0xf5, 0x36, 0xed, 0x16, 0x35, 0xe9, + 0x97, 0xfb, 0xd4, 0xf5, 0xc8, 0x2d, 0xc8, 0xf4, 0xd8, 0xeb, 0xb6, 0xd5, 0x9a, 0x4b, 0xbd, 0x92, + 0x5a, 0xcc, 0xd6, 0xe0, 0xc5, 0xf1, 0xfc, 0x0c, 0x47, 0xd4, 0x37, 0xcc, 0x19, 0xde, 0x55, 0x6f, + 0x19, 0xef, 0xc1, 0x65, 0xdf, 0xcc, 0xed, 0xdb, 0x3d, 0x97, 0x92, 0x65, 0x98, 0xe6, 0x9d, 0xc2, + 0x28, 0x57, 0x9d, 0xab, 0x0c, 0x4f, 0xa0, 0x22, 0xf0, 0x02, 0x65, 0x1c, 0x4f, 0xc1, 0x95, 0x07, + 0x96, 0x2b, 0x86, 0x70, 0x95, 0xeb, 0x7b, 0x90, 0xd9, 0xb5, 0x3a, 0x1e, 0x75, 0x5c, 0x1c, 0x65, + 0x59, 0x37, 0x4a, 0xd4, 0xac, 0x72, 0x4f, 0xda, 0x98, 0xca, 0xb8, 0xfc, 0xcd, 0x14, 0x64, 0xb0, + 0x91, 0x94, 0xe0, 0x62, 0xaf, 0xd1, 0xa5, 0x7c, 0xc4, 0xa9, 0xc5, 0xac, 0x29, 0x5f, 0xc8, 0x0a, + 0xe4, 0xac, 0xd6, 0x76, 0xdf, 0xa1, 0xbb, 0xd6, 0x33, 0xd6, 0x97, 0xe6, 0x7d, 0xb5, 0x02, 0x9b, + 0x28, 0xd4, 0x37, 0x1e, 0x61, 0xab, 0x09, 0x56, 0x4b, 0x3d, 0x93, 0x47, 0x30, 0xd3, 0x69, 0xec, + 0xd0, 0x8e, 0x3b, 0x37, 0xc5, 0xb0, 0xb9, 0xea, 0xda, 0x24, 0x91, 0x55, 0x1e, 0x08, 0xd3, 0x0f, + 0x18, 0xc1, 0x87, 0x26, 0x8e, 0x43, 0xea, 0x90, 0xeb, 0xd2, 0xee, 0x0e, 0xeb, 0xfe, 0xdc, 0xea, + 0xbb, 0x73, 0xd3, 0x6c, 0xd8, 0x42, 0xf5, 0x4e, 0xdc, 0xb2, 0x6d, 0x31, 0xea, 0x2b, 0x0f, 0x7d, + 0xbc, 0x19, 0xb6, 0x25, 0x55, 0xb8, 0xc8, 0x94, 0xc3, 0xe6, 0x71, 0x51, 0x0c, 0x72, 0x23, 0x76, + 0xed, 0x19, 0xc8, 0x94, 0x50, 0x46, 0x73, 0x9e, 0x2f, 0x45, 0xb0, 0x06, 0x33, 0x62, 0x7d, 0x2e, + 0xf1, 0x46, 0x35, 0xeb, 0xf2, 0x5b, 0x90, 0x0b, 0x85, 0x4e, 0xae, 0xc0, 0x54, 0x9b, 0x1e, 0x4a, + 0x59, 0x98, 0xfc, 0x91, 0xaf, 0xee, 0x41, 0xa3, 0xb3, 0x4f, 0xd9, 0x0a, 0xf2, 0x36, 0xf9, 0xf2, + 0x76, 0x7a, 0x2d, 0x65, 0xac, 0x43, 0x31, 0xb4, 0x1c, 0xa8, 0x91, 0x0a, 0x23, 0x83, 0x37, 0x08, + 0x32, 0x92, 0x44, 0x22, 0x61, 0xc6, 0x8f, 0x29, 0x28, 0x3e, 0xe9, 0xb7, 0x1a, 0x1e, 0x9d, 0x54, + 0xa1, 0xe4, 0x5d, 0xb8, 0x24, 0x40, 0x07, 0x6c, 0x91, 0x2c, 0xbb, 0x27, 0x02, 0xcc, 0x55, 0xaf, + 0xeb, 0x3c, 0x7e, 0x22, 0x21, 0x66, 0x8e, 0x1b, 0xe0, 0x0b, 0x79, 0x1d, 0xa6, 0xf9, 0x76, 0x63, + 0x74, 0x73, 0xbb, 0x1b, 0x49, 0xbc, 0x98, 0x02, 0x69, 0xd4, 0x80, 0x84, 0x63, 0x3d, 0xd1, 0xb6, + 0xd8, 0x84, 0xa2, 0x49, 0xbb, 0xf6, 0xc1, 0xe4, 0xf3, 0x65, 0x4c, 0xec, 0xda, 0x4e, 0x53, 0x32, + 0x31, 0x6b, 0xca, 0x17, 0xa3, 0x04, 0x24, 0x3c, 0x9e, 0x8c, 0x09, 0x37, 0xfd, 0xe3, 0x86, 0xdb, + 0x0e, 0xb9, 0xf0, 0xd8, 0x6b, 0xc4, 0x05, 0x47, 0x70, 0x17, 0xbc, 0xcb, 0xdf, 0xf4, 0xd2, 0x2c, + 0x98, 0x1d, 0xef, 0x4c, 0x9a, 0x9d, 0xc0, 0x0b, 0x94, 0xb1, 0xa6, 0x66, 0x37, 0xb1, 0x6b, 0x7f, + 0x1e, 0x61, 0xef, 0xc6, 0x3f, 0x98, 0x44, 0x78, 0xe3, 0x09, 0x92, 0x48, 0xd8, 0x6c, 0x38, 0x89, + 0xfc, 0x70, 0x8e, 0x49, 0x44, 0x17, 0x99, 0x36, 0x89, 0xb0, 0x10, 0x5c, 0xea, 0x1c, 0x58, 0x4d, + 0xae, 0x0e, 0x99, 0x44, 0x30, 0x84, 0x2d, 0xd9, 0x5c, 0xdf, 0x60, 0x21, 0x20, 0xa4, 0xde, 0x72, + 0xc9, 0x02, 0xcc, 0xa2, 0x96, 0x64, 0xb6, 0xc8, 0xd6, 0x72, 0x0c, 0x9d, 0x91, 0x62, 0x62, 0xb3, + 0x97, 0x6a, 0x72, 0xc9, 0x06, 0x14, 0xd8, 0x06, 0xb4, 0x1c, 0xda, 0xda, 0x76, 0x3d, 0xa6, 0x69, + 0x99, 0x1f, 0x0a, 0xd5, 0xff, 0xc7, 0x51, 0xbc, 0xc5, 0x51, 0x66, 0x1e, 0x8d, 0xc4, 0x9b, 0x26, + 0xc9, 0x64, 0xfe, 0x93, 0x24, 0x83, 0xcb, 0x15, 0x24, 0x19, 0xae, 0x9a, 0xc4, 0x24, 0x23, 0x64, + 0x24, 0x61, 0xc6, 0x47, 0x50, 0x5a, 0x77, 0x28, 0x8b, 0x17, 0x97, 0x4c, 0x09, 0x69, 0x15, 0x33, + 0x80, 0x54, 0xd1, 0xbc, 0x6e, 0x18, 0xb4, 0x08, 0x25, 0x81, 0x4d, 0xb8, 0x16, 0x19, 0x0c, 0xa3, + 0xba, 0x0b, 0x19, 0xa4, 0x01, 0x07, 0xbc, 0x9e, 0x30, 0xa0, 0xa9, 0xb0, 0xc6, 0xfb, 0x50, 0x64, + 0x7b, 0x2e, 0x12, 0xd9, 0x32, 0x40, 0xc0, 0x3a, 0xee, 0x9a, 0x3c, 0xa3, 0x31, 0xeb, 0x93, 0x6e, + 0x66, 0x7d, 0xce, 0xd9, 0xfc, 0x48, 0x78, 0x88, 0xd3, 0xc5, 0xf3, 0x73, 0x0a, 0x4a, 0x32, 0xcb, + 0x9d, 0x26, 0x26, 0x26, 0xaf, 0xcb, 0x0a, 0x3d, 0x41, 0x82, 0x2e, 0xa0, 0x8d, 0xca, 0xd1, 0xab, + 0x03, 0x39, 0x7a, 0x7c, 0x86, 0x22, 0x13, 0x38, 0xdd, 0x8a, 0x6c, 0x40, 0x49, 0xa6, 0xa6, 0x53, + 0x91, 0xf4, 0x3f, 0xb8, 0x16, 0x19, 0x05, 0x73, 0xdc, 0x9f, 0x69, 0xb8, 0xca, 0x35, 0x8e, 0xed, + 0x7e, 0x9a, 0xab, 0x47, 0xd3, 0xdc, 0x4a, 0x5c, 0x32, 0x89, 0x58, 0x0e, 0x67, 0xba, 0x6f, 0xd3, + 0x67, 0x9e, 0xe9, 0xb6, 0x22, 0x99, 0xee, 0x9d, 0x09, 0x83, 0xd3, 0x26, 0xbb, 0xa1, 0x6c, 0x32, + 0x7d, 0xb6, 0xd9, 0xe4, 0x63, 0x28, 0x0d, 0x86, 0x84, 0xc2, 0x78, 0x13, 0x66, 0x91, 0x28, 0x95, + 0x53, 0x12, 0x95, 0xe1, 0x83, 0x83, 0xcc, 0xb2, 0x49, 0xbd, 0xa7, 0xb6, 0xd3, 0x9e, 0x20, 0xb3, + 0xa0, 0x85, 0x2e, 0xb3, 0xf8, 0x83, 0x05, 0xba, 0xed, 0xc9, 0xa6, 0x24, 0xdd, 0x2a, 0x2b, 0x85, + 0x35, 0x9e, 0x88, 0xcc, 0x12, 0x89, 0x8c, 0xb0, 0x6a, 0x85, 0xad, 0x26, 0xae, 0x97, 0x78, 0xe6, + 0x42, 0x46, 0x1b, 0x2e, 0xe4, 0x74, 0x20, 0x64, 0xb4, 0xe5, 0x42, 0x46, 0x80, 0x9f, 0x6d, 0xce, + 0x28, 0xc6, 0xcf, 0xd4, 0xde, 0x3a, 0xf3, 0x30, 0xfd, 0xfd, 0x16, 0x89, 0xd4, 0xdf, 0x6f, 0xd8, + 0x7e, 0x82, 0xfd, 0x16, 0xb1, 0x7c, 0xb9, 0xf6, 0x5b, 0x4c, 0x70, 0xe7, 0xb9, 0xdf, 0x82, 0x90, + 0x82, 0xfd, 0x86, 0x44, 0x25, 0xee, 0x37, 0xc5, 0x9c, 0x0f, 0xc6, 0x8f, 0xe5, 0x7a, 0x67, 0xdf, + 0x65, 0x73, 0x0a, 0xe5, 0xe1, 0xa6, 0x6c, 0x89, 0xe4, 0x61, 0xc4, 0x71, 0x5d, 0x20, 0xc0, 0x97, + 0xaf, 0x3f, 0x44, 0x20, 0x5f, 0x84, 0x24, 0xc9, 0x57, 0x59, 0x29, 0xac, 0xaf, 0x25, 0xec, 0x38, + 0x81, 0x96, 0x22, 0x96, 0x2f, 0x97, 0x96, 0x62, 0x82, 0x3b, 0x4f, 0x2d, 0x05, 0x21, 0x05, 0x5a, + 0x42, 0x36, 0x12, 0xb5, 0xa4, 0xa8, 0xf3, 0xc1, 0xc6, 0x3e, 0x14, 0x3f, 0xb4, 0xad, 0xde, 0x63, + 0xbb, 0x4d, 0x7b, 0xa6, 0xcd, 0xca, 0x59, 0x5e, 0x70, 0x54, 0xe0, 0xaa, 0xc3, 0x9f, 0xe9, 0x36, + 0x17, 0x1c, 0x53, 0x94, 0xc7, 0xbb, 0x45, 0x84, 0xb3, 0x66, 0x51, 0x76, 0x7d, 0x2a, 0x7a, 0x84, + 0x1d, 0x3b, 0x44, 0x96, 0x10, 0xdf, 0x6d, 0xf4, 0x1a, 0x7b, 0xbe, 0x81, 0x3c, 0xa3, 0x11, 0xd9, + 0xf7, 0x50, 0x76, 0x09, 0x0b, 0xe3, 0xbb, 0xb4, 0xaa, 0xaf, 0x4e, 0x23, 0x63, 0x5e, 0x5f, 0x29, + 0xf4, 0x24, 0xf5, 0x15, 0xda, 0x4c, 0x50, 0x5f, 0xa1, 0xf7, 0xe0, 0x3b, 0x45, 0xee, 0xc3, 0xac, + 0x83, 0xeb, 0xc5, 0x48, 0xe6, 0x86, 0xb7, 0x75, 0x86, 0x43, 0x8b, 0x5b, 0x9b, 0x3e, 0x3a, 0x9e, + 0xbf, 0x60, 0xfa, 0xc6, 0x41, 0xa1, 0x76, 0x36, 0xbb, 0xb1, 0xfa, 0x5b, 0x11, 0x32, 0xeb, 0xf2, + 0x92, 0x8d, 0x58, 0x90, 0xc1, 0xfb, 0x2b, 0x62, 0xe8, 0x8c, 0x07, 0xef, 0xc4, 0xca, 0xb7, 0x12, + 0x31, 0xf8, 0xe5, 0xb8, 0xf6, 0xcb, 0x4f, 0x7f, 0x7d, 0x9f, 0xbe, 0x0c, 0x79, 0x01, 0x7a, 0x0d, + 0x19, 0x27, 0x36, 0x64, 0xfd, 0x8b, 0x10, 0xf2, 0xea, 0x38, 0xd7, 0x46, 0xe5, 0xdb, 0x23, 0x50, + 0xc9, 0x0e, 0x1d, 0x80, 0xe0, 0x1e, 0x82, 0x68, 0xc7, 0x1a, 0xba, 0x53, 0x29, 0x2f, 0x8c, 0x82, + 0x8d, 0xf4, 0x19, 0xdc, 0x33, 0xe8, 0x7d, 0x0e, 0xdd, 0x6b, 0xe8, 0x7d, 0x6a, 0xae, 0x2b, 0x62, + 0x7c, 0x4a, 0x0e, 0xf9, 0x49, 0x2e, 0x96, 0xc3, 0xd0, 0x3d, 0x43, 0x2c, 0x87, 0x03, 0x37, 0x0a, + 0xc9, 0x1c, 0x8a, 0x73, 0x66, 0x3c, 0x87, 0xe1, 0x53, 0x7b, 0x3c, 0x87, 0x03, 0x87, 0xd5, 0x91, + 0xeb, 0x29, 0xa6, 0x97, 0xb0, 0x9e, 0xe1, 0x19, 0x2e, 0x8c, 0x82, 0x8d, 0xf4, 0x19, 0x9c, 0x13, + 0xf5, 0x3e, 0x87, 0x8e, 0xa2, 0x7a, 0x9f, 0xc3, 0xc7, 0xcd, 0x38, 0x9f, 0xcf, 0xe0, 0x52, 0xb8, + 0xe4, 0x26, 0x77, 0xc6, 0x3c, 0x27, 0x94, 0x17, 0x47, 0x03, 0x93, 0x3d, 0x7f, 0x05, 0xf9, 0x81, + 0x83, 0x3a, 0xd1, 0x8e, 0xa8, 0xbb, 0x18, 0x28, 0x2f, 0x8d, 0x81, 0x1c, 0xe9, 0x7c, 0xe0, 0x0c, + 0xaa, 0x77, 0xae, 0x3b, 0x67, 0xeb, 0x9d, 0x6b, 0x0f, 0xb4, 0x09, 0xce, 0x07, 0x8e, 0x9a, 0x7a, + 0xe7, 0xba, 0x33, 0xad, 0xde, 0xb9, 0xfe, 0xdc, 0x9a, 0x28, 0x32, 0x2c, 0xdd, 0x62, 0x45, 0x36, + 0x58, 0xee, 0xc7, 0x8a, 0x2c, 0x5a, 0xbb, 0x27, 0x8b, 0x4c, 0xd5, 0x99, 0xf1, 0x22, 0x8b, 0x14, + 0xc7, 0xf1, 0x22, 0x8b, 0x96, 0xac, 0x23, 0x45, 0xa6, 0x26, 0x9c, 0x20, 0xb2, 0xc8, 0x9c, 0x97, + 0xc6, 0x40, 0x8e, 0xc9, 0x73, 0xa2, 0x73, 0xdd, 0xf9, 0x2a, 0x89, 0xe7, 0x31, 0x9d, 0x4b, 0x9e, + 0xf1, 0x1b, 0x1c, 0xcb, 0xf3, 0x60, 0x8d, 0x13, 0xcb, 0x73, 0xa4, 0x00, 0x18, 0xc1, 0xb3, 0xaa, + 0x01, 0xe3, 0x79, 0x8e, 0x14, 0xae, 0xf1, 0x3c, 0x47, 0xcb, 0xc9, 0x91, 0xfb, 0x59, 0x4d, 0x38, + 0x61, 0x3f, 0x47, 0xe6, 0xbc, 0x34, 0x06, 0x32, 0xd1, 0x79, 0xed, 0xc6, 0xd1, 0xf3, 0x9b, 0x17, + 0x7e, 0x67, 0xbf, 0xbf, 0x9f, 0xdf, 0x4c, 0x7d, 0xfd, 0xe2, 0x66, 0xea, 0x88, 0xfd, 0x7e, 0x65, + 0xbf, 0x3f, 0xd8, 0x6f, 0x67, 0x46, 0xfc, 0x9f, 0x6f, 0xf5, 0xdf, 0x00, 0x00, 0x00, 0xff, 0xff, + 0xbb, 0x97, 0x43, 0x36, 0x60, 0x1c, 0x00, 0x00, } diff --git a/vendor/src/github.com/docker/swarmkit/api/control.proto b/vendor/src/github.com/docker/swarmkit/api/control.proto index 6045d1dbb5..b968450094 100644 --- a/vendor/src/github.com/docker/swarmkit/api/control.proto +++ b/vendor/src/github.com/docker/swarmkit/api/control.proto @@ -115,6 +115,7 @@ message UpdateNodeResponse { // RemoveNodeRequest requests to delete the specified node from store. message RemoveNodeRequest { string node_id = 1 [(gogoproto.customname) = "NodeID"]; + bool force = 2; } message RemoveNodeResponse { diff --git a/vendor/src/github.com/docker/swarmkit/ca/server.go b/vendor/src/github.com/docker/swarmkit/ca/server.go index 1574b166c7..a65bdd0098 100644 --- a/vendor/src/github.com/docker/swarmkit/ca/server.go +++ b/vendor/src/github.com/docker/swarmkit/ca/server.go @@ -309,15 +309,6 @@ func (s *Server) Run(ctx context.Context) error { logger := log.G(ctx).WithField("module", "ca") ctx = log.WithLogger(ctx, logger) - // Run() should never be called twice, but just in case, we're - // attempting to close the started channel in a safe way - select { - case <-s.started: - return fmt.Errorf("CA server cannot be started more than once") - default: - close(s.started) - } - // Retrieve the channels to keep track of changes in the cluster // Retrieve all the currently registered nodes var nodes []*api.Node @@ -346,6 +337,7 @@ func (s *Server) Run(ctx context.Context) error { s.mu.Lock() s.ctx, s.cancel = context.WithCancel(ctx) s.mu.Unlock() + close(s.started) if err != nil { log.G(ctx).WithFields(logrus.Fields{ diff --git a/vendor/src/github.com/docker/swarmkit/manager/allocator/network.go b/vendor/src/github.com/docker/swarmkit/manager/allocator/network.go index 8d0b938591..c829db4988 100644 --- a/vendor/src/github.com/docker/swarmkit/manager/allocator/network.go +++ b/vendor/src/github.com/docker/swarmkit/manager/allocator/network.go @@ -377,7 +377,7 @@ func (a *Allocator) doNodeAlloc(ctx context.Context, nc *networkContext, ev even node.Attachment.Network = nc.ingressNetwork.Copy() if err := a.allocateNode(ctx, nc, node); err != nil { - log.G(ctx).Errorf("Fauled to allocate network resources for node %s: %v", node.ID, err) + log.G(ctx).Errorf("Failed to allocate network resources for node %s: %v", node.ID, err) } } } diff --git a/vendor/src/github.com/docker/swarmkit/manager/controlapi/node.go b/vendor/src/github.com/docker/swarmkit/manager/controlapi/node.go index 46b1189720..2a7df6f44f 100644 --- a/vendor/src/github.com/docker/swarmkit/manager/controlapi/node.go +++ b/vendor/src/github.com/docker/swarmkit/manager/controlapi/node.go @@ -283,7 +283,7 @@ func (s *Server) RemoveNode(ctx context.Context, request *api.RemoveNodeRequest) return grpc.Errorf(codes.FailedPrecondition, "node %s is a cluster manager and is a member of the raft cluster. It must be demoted to worker before removal", request.NodeID) } } - if node.Status.State == api.NodeStatus_READY { + if !request.Force && node.Status.State == api.NodeStatus_READY { return grpc.Errorf(codes.FailedPrecondition, "node %s is not down and can't be removed", request.NodeID) } return store.DeleteNode(tx, request.NodeID) diff --git a/vendor/src/github.com/docker/swarmkit/manager/dispatcher/dispatcher.go b/vendor/src/github.com/docker/swarmkit/manager/dispatcher/dispatcher.go index 3617d7e756..a0d984f9fe 100644 --- a/vendor/src/github.com/docker/swarmkit/manager/dispatcher/dispatcher.go +++ b/vendor/src/github.com/docker/swarmkit/manager/dispatcher/dispatcher.go @@ -20,6 +20,7 @@ import ( "github.com/docker/swarmkit/manager/state" "github.com/docker/swarmkit/manager/state/store" "github.com/docker/swarmkit/manager/state/watch" + "github.com/docker/swarmkit/picker" "github.com/docker/swarmkit/protobuf/ptypes" "golang.org/x/net/context" ) @@ -60,8 +61,6 @@ var ( // Config is configuration for Dispatcher. For default you should use // DefautConfig. type Config struct { - // Addr configures the address the dispatcher reports to agents. - Addr string HeartbeatPeriod time.Duration HeartbeatEpsilon time.Duration // RateLimitPeriod specifies how often node with same ID can try to register @@ -90,7 +89,6 @@ type Cluster interface { // Dispatcher is responsible for dispatching tasks and tracking agent health. type Dispatcher struct { mu sync.Mutex - addr string nodes *nodeStore store *store.MemoryStore mgrQueue *watch.Queue @@ -121,7 +119,6 @@ func (b weightedPeerByNodeID) Swap(i, j int) { b[i], b[j] = b[j], b[i] } // NOTE: each handler which does something with raft must add to Dispatcher.wg func New(cluster Cluster, c *Config) *Dispatcher { return &Dispatcher{ - addr: c.Addr, nodes: newNodeStore(c.HeartbeatPeriod, c.HeartbeatEpsilon, c.GracePeriodMultiplier, c.RateLimitPeriod), store: cluster.MemoryStore(), cluster: cluster, @@ -142,7 +139,11 @@ func getWeightedPeers(cluster Cluster) []*api.WeightedPeer { NodeID: m.NodeID, Addr: m.Addr, }, - Weight: 1, + + // TODO(stevvooe): Calculate weight of manager selection based on + // cluster-level observations, such as number of connections and + // load. + Weight: picker.DefaultObservationWeight, }) } return mgrs @@ -574,14 +575,18 @@ func (d *Dispatcher) Tasks(r *api.TasksRequest, stream api.Dispatcher_TasksServe } // bursty events should be processed in batches and sent out snapshot - const modificationBatchLimit = 200 - const eventPausedGap = 50 * time.Millisecond - var modificationCnt int - // eventPaused is true when there have been modifications - // but next event has not arrived within eventPausedGap - eventPaused := false + const ( + modificationBatchLimit = 200 + eventPausedGap = 50 * time.Millisecond + ) + var ( + modificationCnt int + eventPausedTimer *time.Timer + eventPausedTimeout <-chan time.Time + ) - for modificationCnt < modificationBatchLimit && !eventPaused { + batchingLoop: + for modificationCnt < modificationBatchLimit { select { case event := <-nodeTasks: switch v := event.(type) { @@ -602,16 +607,24 @@ func (d *Dispatcher) Tasks(r *api.TasksRequest, stream api.Dispatcher_TasksServe delete(tasksMap, v.Task.ID) modificationCnt++ } - case <-time.After(eventPausedGap): - if modificationCnt > 0 { - eventPaused = true + if eventPausedTimer != nil { + eventPausedTimer.Reset(eventPausedGap) + } else { + eventPausedTimer = time.NewTimer(eventPausedGap) + eventPausedTimeout = eventPausedTimer.C } + case <-eventPausedTimeout: + break batchingLoop case <-stream.Context().Done(): return stream.Context().Err() case <-d.ctx.Done(): return d.ctx.Err() } } + + if eventPausedTimer != nil { + eventPausedTimer.Stop() + } } } diff --git a/vendor/src/github.com/docker/swarmkit/manager/dispatcher/nodes.go b/vendor/src/github.com/docker/swarmkit/manager/dispatcher/nodes.go index 2d0ea076a5..aa962a01b1 100644 --- a/vendor/src/github.com/docker/swarmkit/manager/dispatcher/nodes.go +++ b/vendor/src/github.com/docker/swarmkit/manager/dispatcher/nodes.go @@ -43,26 +43,29 @@ func (rn *registeredNode) checkSessionID(sessionID string) error { } type nodeStore struct { - periodChooser *periodChooser - gracePeriodMultiplier time.Duration - rateLimitPeriod time.Duration - nodes map[string]*registeredNode - mu sync.RWMutex + periodChooser *periodChooser + gracePeriodMultiplierNormal time.Duration + gracePeriodMultiplierUnknown time.Duration + rateLimitPeriod time.Duration + nodes map[string]*registeredNode + mu sync.RWMutex } func newNodeStore(hbPeriod, hbEpsilon time.Duration, graceMultiplier int, rateLimitPeriod time.Duration) *nodeStore { return &nodeStore{ - nodes: make(map[string]*registeredNode), - periodChooser: newPeriodChooser(hbPeriod, hbEpsilon), - gracePeriodMultiplier: time.Duration(graceMultiplier), - rateLimitPeriod: rateLimitPeriod, + nodes: make(map[string]*registeredNode), + periodChooser: newPeriodChooser(hbPeriod, hbEpsilon), + gracePeriodMultiplierNormal: time.Duration(graceMultiplier), + gracePeriodMultiplierUnknown: time.Duration(graceMultiplier) * 2, + rateLimitPeriod: rateLimitPeriod, } } func (s *nodeStore) updatePeriod(hbPeriod, hbEpsilon time.Duration, gracePeriodMultiplier int) { s.mu.Lock() s.periodChooser = newPeriodChooser(hbPeriod, hbEpsilon) - s.gracePeriodMultiplier = time.Duration(gracePeriodMultiplier) + s.gracePeriodMultiplierNormal = time.Duration(gracePeriodMultiplier) + s.gracePeriodMultiplierUnknown = s.gracePeriodMultiplierNormal * 2 s.mu.Unlock() } @@ -79,7 +82,7 @@ func (s *nodeStore) AddUnknown(n *api.Node, expireFunc func()) error { Node: n, } s.nodes[n.ID] = rn - rn.Heartbeat = heartbeat.New(s.periodChooser.Choose()*s.gracePeriodMultiplier, expireFunc) + rn.Heartbeat = heartbeat.New(s.periodChooser.Choose()*s.gracePeriodMultiplierUnknown, expireFunc) return nil } @@ -124,7 +127,7 @@ func (s *nodeStore) Add(n *api.Node, expireFunc func()) *registeredNode { Disconnect: make(chan struct{}), } s.nodes[n.ID] = rn - rn.Heartbeat = heartbeat.New(s.periodChooser.Choose()*s.gracePeriodMultiplier, expireFunc) + rn.Heartbeat = heartbeat.New(s.periodChooser.Choose()*s.gracePeriodMultiplierNormal, expireFunc) return rn } @@ -154,7 +157,7 @@ func (s *nodeStore) Heartbeat(id, sid string) (time.Duration, error) { return 0, err } period := s.periodChooser.Choose() // base period for node - grace := period * time.Duration(s.gracePeriodMultiplier) + grace := period * time.Duration(s.gracePeriodMultiplierNormal) rn.mu.Lock() rn.Heartbeat.Update(grace) rn.Heartbeat.Beat() diff --git a/vendor/src/github.com/docker/swarmkit/manager/manager.go b/vendor/src/github.com/docker/swarmkit/manager/manager.go index 15e1f94981..2b4974bd0a 100644 --- a/vendor/src/github.com/docker/swarmkit/manager/manager.go +++ b/vendor/src/github.com/docker/swarmkit/manager/manager.go @@ -89,9 +89,11 @@ type Manager struct { server *grpc.Server localserver *grpc.Server RaftNode *raft.Node + connSelector *raftpicker.ConnSelector mu sync.Mutex + started chan struct{} stopped chan struct{} } @@ -139,9 +141,6 @@ func New(config *Config) (*Manager, error) { tcpAddr = net.JoinHostPort("0.0.0.0", tcpAddrPort) } - // FIXME(aaronl): Remove this. It appears to be unused. - dispatcherConfig.Addr = tcpAddr - err := os.MkdirAll(filepath.Dir(config.ProtoAddr["unix"]), 0700) if err != nil { return nil, fmt.Errorf("failed to create socket directory: %v", err) @@ -220,6 +219,7 @@ func New(config *Config) (*Manager, error) { server: grpc.NewServer(opts...), localserver: grpc.NewServer(opts...), RaftNode: RaftNode, + started: make(chan struct{}), stopped: make(chan struct{}), } @@ -428,11 +428,12 @@ func (m *Manager) Run(parent context.Context) error { }() proxyOpts := []grpc.DialOption{ - grpc.WithBackoffMaxDelay(2 * time.Second), + grpc.WithBackoffMaxDelay(time.Second), grpc.WithTransportCredentials(m.config.SecurityConfig.ClientTLSCreds), } cs := raftpicker.NewConnSelector(m.RaftNode, proxyOpts...) + m.connSelector = cs authorize := func(ctx context.Context, roles []string) error { // Authorize the remote roles, ensure they can only be forwarded by managers @@ -506,6 +507,8 @@ func (m *Manager) Run(parent context.Context) error { return fmt.Errorf("can't initialize raft node: %v", err) } + close(m.started) + go func() { err := m.RaftNode.Run(ctx) if err != nil { @@ -560,12 +563,15 @@ func (m *Manager) Run(parent context.Context) error { func (m *Manager) Stop(ctx context.Context) { log.G(ctx).Info("Stopping manager") + // It's not safe to start shutting down while the manager is still + // starting up. + <-m.started + // the mutex stops us from trying to stop while we're alrady stopping, or // from returning before we've finished stopping. m.mu.Lock() defer m.mu.Unlock() select { - // check to see that we've already stopped case <-m.stopped: return @@ -600,6 +606,9 @@ func (m *Manager) Stop(ctx context.Context) { m.keyManager.Stop() } + if m.connSelector != nil { + m.connSelector.Stop() + } m.RaftNode.Shutdown() // some time after this point, Run will receive an error from one of these m.server.Stop() diff --git a/vendor/src/github.com/docker/swarmkit/manager/raftpicker/raftpicker.go b/vendor/src/github.com/docker/swarmkit/manager/raftpicker/raftpicker.go index 8a62de69a6..4ead7dc7e7 100644 --- a/vendor/src/github.com/docker/swarmkit/manager/raftpicker/raftpicker.go +++ b/vendor/src/github.com/docker/swarmkit/manager/raftpicker/raftpicker.go @@ -2,6 +2,7 @@ package raftpicker import ( "sync" + "time" "golang.org/x/net/context" "google.golang.org/grpc" @@ -14,46 +15,37 @@ type picker struct { addr string raft AddrSelector conn *grpc.Conn - cc *grpc.ClientConn + + stop chan struct{} + done chan struct{} +} + +func newPicker(raft AddrSelector, addr string) *picker { + return &picker{ + raft: raft, + addr: addr, + + stop: make(chan struct{}), + done: make(chan struct{}), + } } // Init does initial processing for the Picker, e.g., initiate some connections. func (p *picker) Init(cc *grpc.ClientConn) error { - p.cc = cc - return nil -} - -func (p *picker) initConn() error { - if p.conn == nil { - conn, err := grpc.NewConn(p.cc) - if err != nil { - return err - } - p.conn = conn + conn, err := grpc.NewConn(cc) + if err != nil { + return err } + p.conn = conn return nil } // Pick blocks until either a transport.ClientTransport is ready for the upcoming RPC // or some error happens. func (p *picker) Pick(ctx context.Context) (transport.ClientTransport, error) { - p.mu.Lock() - if err := p.initConn(); err != nil { - p.mu.Unlock() + if err := p.updateConn(); err != nil { return nil, err } - p.mu.Unlock() - - addr, err := p.raft.LeaderAddr() - if err != nil { - return nil, err - } - p.mu.Lock() - if p.addr != addr { - p.addr = addr - p.conn.NotifyReset() - } - p.mu.Unlock() return p.conn.Wait(ctx) } @@ -89,15 +81,46 @@ func (p *picker) Reset() error { // Close closes all the Conn's owned by this Picker. func (p *picker) Close() error { + close(p.stop) + <-p.done return p.conn.Close() } +func (p *picker) updateConn() error { + addr, err := p.raft.LeaderAddr() + if err != nil { + return err + } + p.mu.Lock() + if p.addr != addr { + p.addr = addr + p.Reset() + } + p.mu.Unlock() + return nil +} + +func (p *picker) updateLoop() { + defer close(p.done) + ticker := time.NewTicker(1 * time.Second) + defer ticker.Stop() + for { + select { + case <-ticker.C: + p.updateConn() + case <-p.stop: + return + } + } +} + // ConnSelector is struct for obtaining connection with raftpicker. type ConnSelector struct { mu sync.Mutex cc *grpc.ClientConn cluster RaftCluster opts []grpc.DialOption + picker *picker } // NewConnSelector returns new ConnSelector with cluster and grpc.DialOpts which @@ -122,8 +145,9 @@ func (c *ConnSelector) Conn() (*grpc.ClientConn, error) { if err != nil { return nil, err } - picker := &picker{raft: c.cluster, addr: addr} - opts := append(c.opts, grpc.WithPicker(picker)) + c.picker = newPicker(c.cluster, addr) + go c.picker.updateLoop() + opts := append(c.opts, grpc.WithPicker(c.picker)) cc, err := grpc.Dial(addr, opts...) if err != nil { return nil, err @@ -131,3 +155,13 @@ func (c *ConnSelector) Conn() (*grpc.ClientConn, error) { c.cc = cc return c.cc, nil } + +// Stop cancels tracking loop for raftpicker and closes it. +func (c *ConnSelector) Stop() { + c.mu.Lock() + defer c.mu.Unlock() + if c.picker == nil { + return + } + c.picker.Close() +} diff --git a/vendor/src/github.com/docker/swarmkit/manager/state/raft/raft.go b/vendor/src/github.com/docker/swarmkit/manager/state/raft/raft.go index 212c09e1e0..ebb90b4de1 100644 --- a/vendor/src/github.com/docker/swarmkit/manager/state/raft/raft.go +++ b/vendor/src/github.com/docker/swarmkit/manager/state/raft/raft.go @@ -451,6 +451,17 @@ func (n *Node) Shutdown() { } } +// isShutdown indicates if node was shut down. +// This method should be called under n.stopMu to avoid races with n.stop(). +func (n *Node) isShutdown() bool { + select { + case <-n.Ctx.Done(): + return true + default: + return false + } +} + func (n *Node) stop() { n.stopMu.Lock() defer n.stopMu.Unlock() @@ -763,7 +774,10 @@ func (n *Node) ResolveAddress(ctx context.Context, msg *api.ResolveAddressReques func (n *Node) LeaderAddr() (string, error) { n.stopMu.RLock() defer n.stopMu.RUnlock() - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + if n.isShutdown() { + return "", fmt.Errorf("raft node is shut down") + } + ctx, cancel := context.WithTimeout(n.Ctx, 10*time.Second) defer cancel() if err := WaitForLeader(ctx, n); err != nil { return "", ErrNoClusterLeader @@ -1288,7 +1302,7 @@ func (n *Node) ConnectToMember(addr string, timeout time.Duration) (*membership. // SubscribeLeadership returns channel to which events about leadership change // will be sent in form of raft.LeadershipState. Also cancel func is returned - -// it should be called when listener is not longer interested in events. +// it should be called when listener is no longer interested in events. func (n *Node) SubscribeLeadership() (q chan events.Event, cancel func()) { ch := events.NewChannel(0) sink := events.Sink(events.NewQueue(ch)) diff --git a/vendor/src/github.com/docker/swarmkit/picker/picker.go b/vendor/src/github.com/docker/swarmkit/picker/picker.go index 34923e0ef0..2250254352 100644 --- a/vendor/src/github.com/docker/swarmkit/picker/picker.go +++ b/vendor/src/github.com/docker/swarmkit/picker/picker.go @@ -15,6 +15,10 @@ import ( var errRemotesUnavailable = fmt.Errorf("no remote hosts provided") +// DefaultObservationWeight provides a weight to use for positive observations +// that will balance well under repeated observations. +const DefaultObservationWeight = 10 + // Remotes keeps track of remote addresses by weight, informed by // observations. type Remotes interface { @@ -49,7 +53,7 @@ func NewRemotes(peers ...api.Peer) Remotes { } for _, peer := range peers { - mwr.Observe(peer, 1) + mwr.Observe(peer, DefaultObservationWeight) } return mwr @@ -96,7 +100,7 @@ func (mwr *remotesWeightedRandom) Select(excludes ...string) (api.Peer, error) { // bias to zero-weighted remotes have same probability. otherwise, we // always select first entry when all are zero. - const bias = 0.1 + const bias = 0.001 // clear out workspace mwr.cdf = mwr.cdf[:0] @@ -165,7 +169,7 @@ const ( // See // https://en.wikipedia.org/wiki/Exponential_smoothing#Basic_exponential_smoothing // for details. - remoteWeightSmoothingFactor = 0.7 + remoteWeightSmoothingFactor = 0.5 remoteWeightMax = 1 << 8 ) @@ -228,7 +232,7 @@ func (p *Picker) Init(cc *grpc.ClientConn) error { peer := p.peer p.mu.Unlock() - p.r.ObserveIfExists(peer, 1) + p.r.ObserveIfExists(peer, DefaultObservationWeight) c, err := grpc.NewConn(cc) if err != nil { return err @@ -248,7 +252,7 @@ func (p *Picker) Pick(ctx context.Context) (transport.ClientTransport, error) { p.mu.Unlock() transport, err := p.conn.Wait(ctx) if err != nil { - p.r.ObserveIfExists(peer, -1) + p.r.ObserveIfExists(peer, -DefaultObservationWeight) } return transport, err @@ -261,7 +265,7 @@ func (p *Picker) PickAddr() (string, error) { peer := p.peer p.mu.Unlock() - p.r.ObserveIfExists(peer, -1) // downweight the current addr + p.r.ObserveIfExists(peer, -DefaultObservationWeight) // downweight the current addr var err error peer, err = p.r.Select() @@ -299,15 +303,15 @@ func (p *Picker) WaitForStateChange(ctx context.Context, sourceState grpc.Connec // TODO(stevvooe): This is questionable, but we'll see how it works. switch state { case grpc.Idle: - p.r.ObserveIfExists(peer, 1) + p.r.ObserveIfExists(peer, DefaultObservationWeight) case grpc.Connecting: - p.r.ObserveIfExists(peer, 1) + p.r.ObserveIfExists(peer, DefaultObservationWeight) case grpc.Ready: - p.r.ObserveIfExists(peer, 1) + p.r.ObserveIfExists(peer, DefaultObservationWeight) case grpc.TransientFailure: - p.r.ObserveIfExists(peer, -1) + p.r.ObserveIfExists(peer, -DefaultObservationWeight) case grpc.Shutdown: - p.r.ObserveIfExists(peer, -1) + p.r.ObserveIfExists(peer, -DefaultObservationWeight) } return state, err