Add --force to node removal

Signed-off-by: Diogo Monica <diogo.monica@gmail.com>

api/client/node/remove.go

@@ -7,26 +7,36 @@ import (
 
 	"github.com/docker/docker/api/client"
 	"github.com/docker/docker/cli"
+	"github.com/docker/engine-api/types"
 	"github.com/spf13/cobra"
 )
 
+type removeOptions struct {
+	force bool
+}
+
 func newRemoveCommand(dockerCli *client.DockerCli) *cobra.Command {
-	return &cobra.Command{
-		Use:     "rm NODE [NODE...]",
+	opts := removeOptions{}
+
+	cmd := &cobra.Command{
+		Use:     "rm [OPTIONS] NODE [NODE...]",
 		Aliases: []string{"remove"},
 		Short:   "Remove one or more nodes from the swarm",
 		Args:    cli.RequiresMinArgs(1),
 		RunE: func(cmd *cobra.Command, args []string) error {
-			return runRemove(dockerCli, args)
+			return runRemove(dockerCli, args, opts)
 		},
 	}
+	flags := cmd.Flags()
+	flags.BoolVar(&opts.force, "force", false, "Force remove an active node")
+	return cmd
 }
 
-func runRemove(dockerCli *client.DockerCli, args []string) error {
+func runRemove(dockerCli *client.DockerCli, args []string, opts removeOptions) error {
 	client := dockerCli.Client()
 	ctx := context.Background()
 	for _, nodeID := range args {
-		err := client.NodeRemove(ctx, nodeID)
+		err := client.NodeRemove(ctx, nodeID, types.NodeRemoveOptions{Force: opts.force})
 		if err != nil {
 			return err
 		}

api/server/router/swarm/backend.go

@@ -20,7 +20,7 @@ type Backend interface {
 	GetNodes(basictypes.NodeListOptions) ([]types.Node, error)
 	GetNode(string) (types.Node, error)
 	UpdateNode(string, uint64, types.NodeSpec) error
-	RemoveNode(string) error
+	RemoveNode(string, bool) error
 	GetTasks(basictypes.TaskListOptions) ([]types.Task, error)
 	GetTask(string) (types.Task, error)
 }

api/server/router/swarm/cluster_routes.go

@@ -219,7 +219,13 @@ func (sr *swarmRouter) updateNode(ctx context.Context, w http.ResponseWriter, r
 }
 
 func (sr *swarmRouter) removeNode(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
-	if err := sr.backend.RemoveNode(vars["id"]); err != nil {
+	if err := httputils.ParseForm(r); err != nil {
+		return err
+	}
+
+	force := httputils.BoolValue(r, "force")
+
+	if err := sr.backend.RemoveNode(vars["id"], force); err != nil {
 		logrus.Errorf("Error removing node %s: %v", vars["id"], err)
 		return err
 	}

daemon/cluster/cluster.go

@@ -1023,7 +1023,7 @@ func (c *Cluster) UpdateNode(nodeID string, version uint64, spec types.NodeSpec)
 }
 
 // RemoveNode removes a node from a cluster
-func (c *Cluster) RemoveNode(input string) error {
+func (c *Cluster) RemoveNode(input string, force bool) error {
 	c.RLock()
 	defer c.RUnlock()
 
@@ -1039,7 +1039,7 @@ func (c *Cluster) RemoveNode(input string) error {
 		return err
 	}
 
-	if _, err := c.client.RemoveNode(ctx, &swarmapi.RemoveNodeRequest{NodeID: node.ID}); err != nil {
+	if _, err := c.client.RemoveNode(ctx, &swarmapi.RemoveNodeRequest{NodeID: node.ID, Force: force}); err != nil {
 		return err
 	}
 	return nil

docs/reference/commandline/node_rm.md

@@ -11,7 +11,7 @@ parent = "smn_cli"
 # node rm
 
 ```markdown
-Usage:  docker node rm NODE [NODE...]
+Usage:  docker node rm [OPTIONS] NODE [NODE...]
 
 Remove one or more nodes from the swarm
 
@@ -19,6 +19,7 @@ Aliases:
   rm, remove
 
 Options:
+      --force  Force remove an active node
       --help   Print usage
 ```
 
@@ -30,6 +31,24 @@ Example output:
     $ docker node rm swarm-node-02
     Node swarm-node-02 removed from swarm
 
+Removes nodes from the swarm that are in the down state. Attempting to remove
+an active node will result in an error:
+
+```bash
+$ docker node rm swarm-node-03
+Error response from daemon: rpc error: code = 9 desc = node swarm-node-03 is not down and can't be removed
+```
+
+If a worker node becomes compromised, exhibits unexpected or unwanted behavior, or if you lose access to it so
+that a clean shutdown is impossible, you can use the force option.
+
+```bash
+$ docker node rm --force swarm-node-03
+Node swarm-node-03 removed from swarm
+```
+
+Note that manager nodes have to be demoted to worker nodes before they can be removed
+from the cluster.
 
 ## Related information
 

hack/vendor.sh

@@ -60,7 +60,8 @@ clone git golang.org/x/net 2beffdc2e92c8a3027590f898fe88f69af48a3f8 https://gith
 clone git golang.org/x/sys eb2c74142fd19a79b3f237334c7384d5167b1b46 https://github.com/golang/sys.git
 clone git github.com/docker/go-units 651fc226e7441360384da338d0fd37f2440ffbe3
 clone git github.com/docker/go-connections fa2850ff103453a9ad190da0df0af134f0314b3d
-clone git github.com/docker/engine-api 3d1601b9d2436a70b0dfc045a23f6503d19195df
+
+clone git github.com/docker/engine-api 228c7390a733320d48697cb41ae8cde4942cd3e5
 clone git github.com/RackSec/srslog 259aed10dfa74ea2961eddd1d9847619f6e98837
 clone git github.com/imdario/mergo 0.2.1
 
@@ -139,7 +140,7 @@ clone git github.com/docker/docker-credential-helpers v0.3.0
 clone git github.com/docker/containerd 0ac3cd1be170d180b2baed755e8f0da547ceb267
 
 # cluster
-clone git github.com/docker/swarmkit 9d4c2f73124e70f8fa85f9076635b827d17b109f
+clone git github.com/docker/swarmkit e1c0d64515d839b76e2ef33d396c74933753ffaf
 clone git github.com/golang/mock bd3c8e81be01eef76d4b503f5e687d2d1354d2d9
 clone git github.com/gogo/protobuf 43a2e0b1c32252bfbbdf81f7faa7a88fb3fa4028
 clone git github.com/cloudflare/cfssl b895b0549c0ff676f92cf09ba971ae02bb41367b

integration-cli/daemon_swarm.go

@@ -208,6 +208,17 @@ func (d *SwarmDaemon) getNode(c *check.C, id string) *swarm.Node {
 	return &node
 }
 
+func (d *SwarmDaemon) removeNode(c *check.C, id string, force bool) {
+	url := "/nodes/" + id
+	if force {
+		url += "?force=1"
+	}
+
+	status, out, err := d.SockRequest("DELETE", url, nil)
+	c.Assert(status, checker.Equals, http.StatusOK, check.Commentf("output: %q", string(out)))
+	c.Assert(err, checker.IsNil)
+}
+
 func (d *SwarmDaemon) updateNode(c *check.C, id string, f ...nodeConstructor) {
 	for i := 0; ; i++ {
 		node := d.getNode(c, id)

integration-cli/docker_api_swarm_test.go

@@ -544,6 +544,37 @@ func (s *DockerSwarmSuite) TestApiSwarmNodeUpdate(c *check.C) {
 	c.Assert(n.Spec.Availability, checker.Equals, swarm.NodeAvailabilityPause)
 }
 
+func (s *DockerSwarmSuite) TestApiSwarmNodeRemove(c *check.C) {
+	testRequires(c, Network)
+	d1 := s.AddDaemon(c, true, true)
+	d2 := s.AddDaemon(c, true, false)
+	_ = s.AddDaemon(c, true, false)
+
+	nodes := d1.listNodes(c)
+	c.Assert(len(nodes), checker.Equals, 3, check.Commentf("nodes: %#v", nodes))
+
+	// Getting the info so we can take the NodeID
+	d2Info, err := d2.info()
+	c.Assert(err, checker.IsNil)
+
+	// forceful removal of d2 should work
+	d1.removeNode(c, d2Info.NodeID, true)
+
+	nodes = d1.listNodes(c)
+	c.Assert(len(nodes), checker.Equals, 2, check.Commentf("nodes: %#v", nodes))
+
+	// Restart the node that was removed
+	err = d2.Restart()
+	c.Assert(err, checker.IsNil)
+
+	// Give some time for the node to rejoin
+	time.Sleep(1 * time.Second)
+
+	// Make sure the node didn't rejoin
+	nodes = d1.listNodes(c)
+	c.Assert(len(nodes), checker.Equals, 2, check.Commentf("nodes: %#v", nodes))
+}
+
 func (s *DockerSwarmSuite) TestApiSwarmNodeDrainPause(c *check.C) {
 	d1 := s.AddDaemon(c, true, true)
 	d2 := s.AddDaemon(c, true, false)

vendor/src/github.com/docker/engine-api/client/client_darwin.go

@@ -1,4 +0,0 @@
-package client
-
-// DefaultDockerHost defines os specific default if DOCKER_HOST is unset
-const DefaultDockerHost = "tcp://127.0.0.1:2375"

vendor/src/github.com/docker/engine-api/client/client_unix.go

@@ -1,4 +1,4 @@
-// +build linux freebsd solaris openbsd
+// +build linux freebsd solaris openbsd darwin
 
 package client
 

vendor/src/github.com/docker/engine-api/client/interface.go

@@ -94,7 +94,7 @@ type NetworkAPIClient interface {
 type NodeAPIClient interface {
 	NodeInspectWithRaw(ctx context.Context, nodeID string) (swarm.Node, []byte, error)
 	NodeList(ctx context.Context, options types.NodeListOptions) ([]swarm.Node, error)
-	NodeRemove(ctx context.Context, nodeID string) error
+	NodeRemove(ctx context.Context, nodeID string, options types.NodeRemoveOptions) error
 	NodeUpdate(ctx context.Context, nodeID string, version swarm.Version, node swarm.NodeSpec) error
 }
 

vendor/src/github.com/docker/engine-api/client/node_remove.go

@@ -1,10 +1,21 @@
 package client
 
-import "golang.org/x/net/context"
+import (
+	"net/url"
+
+	"github.com/docker/engine-api/types"
+
+	"golang.org/x/net/context"
+)
 
 // NodeRemove removes a Node.
-func (cli *Client) NodeRemove(ctx context.Context, nodeID string) error {
-	resp, err := cli.delete(ctx, "/nodes/"+nodeID, nil, nil)
+func (cli *Client) NodeRemove(ctx context.Context, nodeID string, options types.NodeRemoveOptions) error {
+	query := url.Values{}
+	if options.Force {
+		query.Set("force", "1")
+	}
+
+	resp, err := cli.delete(ctx, "/nodes/"+nodeID, query, nil)
 	ensureReaderClosed(resp)
 	return err
 }

vendor/src/github.com/docker/engine-api/types/client.go

@@ -241,11 +241,16 @@ func (v VersionResponse) ServerOK() bool {
 	return v.Server != nil
 }
 
-// NodeListOptions holds parameters to list  nodes with.
+// NodeListOptions holds parameters to list nodes with.
 type NodeListOptions struct {
 	Filter filters.Args
 }
 
+// NodeRemoveOptions holds parameters to remove nodes with.
+type NodeRemoveOptions struct {
+	Force bool
+}
+
 // ServiceCreateOptions contains the options to use when creating a service.
 type ServiceCreateOptions struct {
 	// EncodedRegistryAuth is the encoded registry authorization credentials to

vendor/src/github.com/docker/swarmkit/agent/agent.go

@@ -12,12 +12,12 @@ import (
 )
 
 const (
-	initialSessionFailureBackoff = time.Second
+	initialSessionFailureBackoff = 100 * time.Millisecond
 	maxSessionFailureBackoff     = 8 * time.Second
 )
 
 // Agent implements the primary node functionality for a member of a swarm
-// cluster. The primary functionality id to run and report on the status of
+// cluster. The primary functionality is to run and report on the status of
 // tasks assigned to the node.
 type Agent struct {
 	config *Config

vendor/src/github.com/docker/swarmkit/agent/node.go

@@ -187,7 +187,7 @@ func (n *Node) run(ctx context.Context) (err error) {
 	if n.config.JoinAddr != "" || n.config.ForceNewCluster {
 		n.remotes = newPersistentRemotes(filepath.Join(n.config.StateDir, stateFilename))
 		if n.config.JoinAddr != "" {
-			n.remotes.Observe(api.Peer{Addr: n.config.JoinAddr}, 1)
+			n.remotes.Observe(api.Peer{Addr: n.config.JoinAddr}, picker.DefaultObservationWeight)
 		}
 	}
 
@@ -647,7 +647,7 @@ func (n *Node) runManager(ctx context.Context, securityConfig *ca.SecurityConfig
 			go func(ready chan struct{}) {
 				select {
 				case <-ready:
-					n.remotes.Observe(api.Peer{NodeID: n.nodeID, Addr: n.config.ListenRemoteAPI}, 5)
+					n.remotes.Observe(api.Peer{NodeID: n.nodeID, Addr: n.config.ListenRemoteAPI}, picker.DefaultObservationWeight)
 				case <-connCtx.Done():
 				}
 			}(ready)

vendor/src/github.com/docker/swarmkit/agent/task.go

@@ -200,7 +200,7 @@ func (tm *taskManager) run(ctx context.Context) {
 				cancel() // cancel outstanding if necessary.
 			} else {
 				// If this channel op fails, it means there is already a
-				// message un the run queue.
+				// message on the run queue.
 				select {
 				case run <- struct{}{}:
 				default:

vendor/src/github.com/docker/swarmkit/api/control.pb.go

@@ -106,6 +106,7 @@ func (*UpdateNodeResponse) Descriptor() ([]byte, []int) { return fileDescriptorC
 // RemoveNodeRequest requests to delete the specified node from store.
 type RemoveNodeRequest struct {
 	NodeID string `protobuf:"bytes,1,opt,name=node_id,json=nodeId,proto3" json:"node_id,omitempty"`
+	Force  bool   `protobuf:"varint,2,opt,name=force,proto3" json:"force,omitempty"`
 }
 
 func (m *RemoveNodeRequest) Reset()                    { *m = RemoveNodeRequest{} }
@@ -786,6 +787,7 @@ func (m *RemoveNodeRequest) Copy() *RemoveNodeRequest {
 
 	o := &RemoveNodeRequest{
 		NodeID: m.NodeID,
+		Force:  m.Force,
 	}
 
 	return o
@@ -1473,9 +1475,10 @@ func (this *RemoveNodeRequest) GoString() string {
 	if this == nil {
 		return "nil"
 	}
-	s := make([]string, 0, 5)
+	s := make([]string, 0, 6)
 	s = append(s, "&api.RemoveNodeRequest{")
 	s = append(s, "NodeID: "+fmt.Sprintf("%#v", this.NodeID)+",\n")
+	s = append(s, "Force: "+fmt.Sprintf("%#v", this.Force)+",\n")
 	s = append(s, "}")
 	return strings.Join(s, "")
 }
@@ -2938,6 +2941,16 @@ func (m *RemoveNodeRequest) MarshalTo(data []byte) (int, error) {
 		i = encodeVarintControl(data, i, uint64(len(m.NodeID)))
 		i += copy(data[i:], m.NodeID)
 	}
+	if m.Force {
+		data[i] = 0x10
+		i++
+		if m.Force {
+			data[i] = 1
+		} else {
+			data[i] = 0
+		}
+		i++
+	}
 	return i, nil
 }
 
@@ -4692,6 +4705,9 @@ func (m *RemoveNodeRequest) Size() (n int) {
 	if l > 0 {
 		n += 1 + l + sovControl(uint64(l))
 	}
+	if m.Force {
+		n += 2
+	}
 	return n
 }
 
@@ -5286,6 +5302,7 @@ func (this *RemoveNodeRequest) String() string {
 	}
 	s := strings.Join([]string{`&RemoveNodeRequest{`,
 		`NodeID:` + fmt.Sprintf("%v", this.NodeID) + `,`,
+		`Force:` + fmt.Sprintf("%v", this.Force) + `,`,
 		`}`,
 	}, "")
 	return s
@@ -6617,6 +6634,26 @@ func (m *RemoveNodeRequest) Unmarshal(data []byte) error {
 			}
 			m.NodeID = string(data[iNdEx:postIndex])
 			iNdEx = postIndex
+		case 2:
+			if wireType != 0 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Force", wireType)
+			}
+			var v int
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowControl
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := data[iNdEx]
+				iNdEx++
+				v |= (int(b) & 0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			m.Force = bool(v != 0)
 		default:
 			iNdEx = preIndex
 			skippy, err := skipControl(data[iNdEx:])
@@ -10521,99 +10558,100 @@ var (
 )
 
 var fileDescriptorControl = []byte{
-	// 1498 bytes of a gzipped FileDescriptorProto
+	// 1512 bytes of a gzipped FileDescriptorProto
 	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xcc, 0x59, 0xcf, 0x6f, 0x1b, 0xc5,
-	0x17, 0xaf, 0x9d, 0x34, 0x8e, 0x9f, 0x6b, 0xb7, 0x9e, 0xba, 0xfa, 0x46, 0x6e, 0xbf, 0x09, 0xda,
+	0x17, 0xaf, 0x9d, 0x34, 0x8e, 0x9f, 0x6b, 0xb7, 0x9e, 0xba, 0xfa, 0x46, 0x6e, 0xbf, 0x0d, 0xda,
 	0xd2, 0x34, 0x91, 0x82, 0x03, 0x8e, 0x2a, 0x02, 0x48, 0x20, 0x9c, 0xd0, 0xca, 0xd0, 0x86, 0x6a,
-	0xd3, 0x02, 0xb7, 0xc8, 0xb1, 0xa7, 0x61, 0xf1, 0x8f, 0x35, 0xbb, 0x9b, 0xb4, 0x11, 0x17, 0x38,
+	0xd3, 0x02, 0xb7, 0xc8, 0xb1, 0x27, 0x61, 0xf1, 0x8f, 0x35, 0xbb, 0x9b, 0xb4, 0x11, 0x17, 0x38,
 	0x20, 0xf1, 0x27, 0x70, 0xe5, 0xca, 0x81, 0x7f, 0x81, 0x6b, 0xc4, 0x89, 0x0b, 0x12, 0xa7, 0x88,
 	0xf6, 0xc4, 0x09, 0xf1, 0x17, 0x20, 0xe6, 0xc7, 0x9b, 0xdd, 0xf5, 0x7a, 0x76, 0x6d, 0x27, 0x41,
-	0xe9, 0xc1, 0xca, 0xee, 0xcc, 0xe7, 0xfd, 0x98, 0x79, 0x9f, 0xf7, 0xf6, 0xcd, 0x04, 0xf2, 0x4d,
-	0xbb, 0xe7, 0x39, 0x76, 0xa7, 0xd2, 0x77, 0x6c, 0xcf, 0x26, 0xa4, 0x65, 0x37, 0xdb, 0xd4, 0xa9,
-	0xb8, 0x4f, 0x1b, 0x4e, 0xb7, 0x6d, 0x79, 0x95, 0x83, 0x37, 0xca, 0x39, 0xb7, 0x4f, 0x9b, 0xae,
-	0x04, 0x94, 0xf3, 0xf6, 0xee, 0x17, 0xb4, 0xe9, 0xa9, 0xd7, 0x9c, 0x77, 0xd8, 0xa7, 0xea, 0xa5,
-	0xb4, 0x67, 0xef, 0xd9, 0xe2, 0x71, 0x95, 0x3f, 0xe1, 0xe8, 0xd5, 0x7e, 0x67, 0x7f, 0xcf, 0xea,
-	0xad, 0xca, 0x3f, 0x72, 0xd0, 0xb8, 0x03, 0x85, 0x7b, 0xd4, 0xdb, 0xb2, 0x5b, 0xd4, 0xa4, 0x5f,
-	0xee, 0x53, 0xd7, 0x23, 0x37, 0x21, 0xd3, 0x63, 0xaf, 0x3b, 0x56, 0x6b, 0x2e, 0xf5, 0x4a, 0x6a,
-	0x29, 0x5b, 0x83, 0x17, 0xc7, 0x0b, 0x33, 0x1c, 0x51, 0xdf, 0x34, 0x67, 0xf8, 0x54, 0xbd, 0x65,
-	0xbc, 0x07, 0x97, 0x7d, 0x31, 0xb7, 0x6f, 0xf7, 0x5c, 0x4a, 0x56, 0x60, 0x9a, 0x4f, 0x0a, 0xa1,
-	0x5c, 0x75, 0xae, 0x32, 0xbc, 0x80, 0x8a, 0xc0, 0x0b, 0x94, 0x71, 0x3c, 0x05, 0x57, 0xee, 0x5b,
-	0xae, 0x50, 0xe1, 0x2a, 0xd3, 0x77, 0x21, 0xf3, 0xc4, 0xea, 0x78, 0xd4, 0x71, 0x51, 0xcb, 0x8a,
-	0x4e, 0x4b, 0x54, 0xac, 0x72, 0x57, 0xca, 0x98, 0x4a, 0xb8, 0xfc, 0xcd, 0x14, 0x64, 0x70, 0x90,
-	0x94, 0xe0, 0x62, 0xaf, 0xd1, 0xa5, 0x5c, 0xe3, 0xd4, 0x52, 0xd6, 0x94, 0x2f, 0x64, 0x15, 0x72,
-	0x56, 0x6b, 0xa7, 0xef, 0xd0, 0x27, 0xd6, 0x33, 0x36, 0x97, 0xe6, 0x73, 0xb5, 0x02, 0x5b, 0x28,
-	0xd4, 0x37, 0x1f, 0xe2, 0xa8, 0x09, 0x56, 0x4b, 0x3d, 0x93, 0x87, 0x30, 0xd3, 0x69, 0xec, 0xd2,
-	0x8e, 0x3b, 0x37, 0xc5, 0xb0, 0xb9, 0xea, 0xfa, 0x24, 0x9e, 0x55, 0xee, 0x0b, 0xd1, 0x0f, 0x58,
-	0x80, 0x0f, 0x4d, 0xd4, 0x43, 0xea, 0x90, 0xeb, 0xd2, 0xee, 0x2e, 0x9b, 0xfe, 0xdc, 0xea, 0xbb,
-	0x73, 0xd3, 0x4c, 0x6d, 0xa1, 0x7a, 0x3b, 0x6e, 0xdb, 0xb6, 0x59, 0xe8, 0x2b, 0x0f, 0x7c, 0xbc,
-	0x19, 0x96, 0x25, 0x55, 0xb8, 0xc8, 0x98, 0xc3, 0xd6, 0x71, 0x51, 0x28, 0xb9, 0x11, 0xbb, 0xf7,
-	0x0c, 0x64, 0x4a, 0x28, 0x0b, 0x73, 0x9e, 0x6f, 0x45, 0xb0, 0x07, 0x33, 0x62, 0x7f, 0x2e, 0xf1,
-	0x41, 0xb5, 0xea, 0xf2, 0x5b, 0x90, 0x0b, 0xb9, 0x4e, 0xae, 0xc0, 0x54, 0x9b, 0x1e, 0x4a, 0x5a,
-	0x98, 0xfc, 0x91, 0xef, 0xee, 0x41, 0xa3, 0xb3, 0x4f, 0xd9, 0x0e, 0xf2, 0x31, 0xf9, 0xf2, 0x76,
-	0x7a, 0x3d, 0x65, 0x6c, 0x40, 0x31, 0xb4, 0x1d, 0xc8, 0x91, 0x0a, 0x0b, 0x06, 0x1f, 0x10, 0xc1,
-	0x48, 0x22, 0x89, 0x84, 0x19, 0x3f, 0xa6, 0xa0, 0xf8, 0xb8, 0xdf, 0x6a, 0x78, 0x74, 0x52, 0x86,
-	0x92, 0x77, 0xe1, 0x92, 0x00, 0x1d, 0xb0, 0x4d, 0xb2, 0xec, 0x9e, 0x70, 0x30, 0x57, 0xbd, 0xae,
-	0xb3, 0xf8, 0x89, 0x84, 0x98, 0x39, 0x2e, 0x80, 0x2f, 0xe4, 0x75, 0x98, 0xe6, 0xe9, 0xc6, 0xc2,
-	0xcd, 0xe5, 0x6e, 0x24, 0xc5, 0xc5, 0x14, 0x48, 0xa3, 0x06, 0x24, 0xec, 0xeb, 0x89, 0xd2, 0x62,
-	0x1d, 0x8a, 0x26, 0xed, 0xda, 0x07, 0x13, 0xaf, 0xd7, 0x28, 0x01, 0x09, 0x4b, 0x4a, 0xeb, 0x98,
-	0xde, 0x8f, 0x1a, 0x6e, 0x3b, 0xa4, 0xcc, 0x63, 0xaf, 0x11, 0x65, 0x1c, 0xc1, 0x95, 0xf1, 0x29,
-	0x3f, 0xbd, 0xa5, 0x58, 0xb0, 0x0e, 0x3e, 0x99, 0xb4, 0x0e, 0x81, 0x17, 0xa8, 0x60, 0x1d, 0x13,
-	0x9b, 0xf6, 0xd7, 0x11, 0xb6, 0x6e, 0xfc, 0x83, 0xe5, 0x82, 0x0f, 0x9e, 0xa0, 0x5c, 0x84, 0xc5,
-	0x86, 0xcb, 0xc5, 0x0f, 0xe7, 0x58, 0x2e, 0x74, 0x9e, 0x69, 0xcb, 0x05, 0x73, 0xc1, 0xa5, 0xce,
-	0x81, 0xd5, 0xe4, 0x3c, 0x90, 0xe5, 0x02, 0x5d, 0xd8, 0x96, 0xc3, 0xf5, 0x4d, 0xe6, 0x02, 0x42,
-	0xea, 0x2d, 0x97, 0x2c, 0xc2, 0x2c, 0xb2, 0x46, 0xd6, 0x85, 0x6c, 0x2d, 0xc7, 0xd0, 0x19, 0x49,
-	0x1b, 0xb6, 0x7a, 0xc9, 0x1b, 0x97, 0x6c, 0x42, 0x81, 0xa5, 0x9a, 0xe5, 0xd0, 0xd6, 0x8e, 0xeb,
-	0x31, 0xf6, 0xca, 0x4a, 0x50, 0xa8, 0xfe, 0x3f, 0x2e, 0xc4, 0xdb, 0x1c, 0x65, 0xe6, 0x51, 0x48,
-	0xbc, 0x69, 0xca, 0x49, 0xe6, 0x3f, 0x29, 0x27, 0xb8, 0x5d, 0x41, 0x39, 0xe1, 0xac, 0x49, 0x2c,
-	0x27, 0x82, 0x46, 0x12, 0x66, 0x7c, 0x04, 0xa5, 0x0d, 0x87, 0x32, 0x7f, 0x71, 0xcb, 0x14, 0x91,
-	0xd6, 0x30, 0xd7, 0x25, 0x8b, 0x16, 0x74, 0x6a, 0x50, 0x22, 0x94, 0xee, 0x5b, 0x70, 0x2d, 0xa2,
-	0x0c, 0xbd, 0xba, 0x03, 0x19, 0x0c, 0x03, 0x2a, 0xbc, 0x9e, 0xa0, 0xd0, 0x54, 0x58, 0xe3, 0x7d,
-	0x28, 0xb2, 0x9c, 0x8b, 0x78, 0xb6, 0x02, 0x10, 0x44, 0x1d, 0xb3, 0x26, 0xcf, 0xc2, 0x98, 0xf5,
-	0x83, 0x6e, 0x66, 0xfd, 0x98, 0xb3, 0xf5, 0x91, 0xb0, 0x8a, 0xd3, 0xf9, 0xf3, 0x73, 0x0a, 0x4a,
-	0xb2, 0x9e, 0x9d, 0xc6, 0x27, 0x46, 0xaf, 0xcb, 0x0a, 0x3d, 0x41, 0x29, 0x2e, 0xa0, 0x8c, 0xaa,
-	0xc6, 0x6b, 0x03, 0xd5, 0x78, 0xfc, 0x08, 0x45, 0x16, 0x70, 0xba, 0x1d, 0xd9, 0x84, 0x92, 0x2c,
-	0x4d, 0xa7, 0x0a, 0xd2, 0xff, 0xe0, 0x5a, 0x44, 0x0b, 0xd6, 0xb8, 0x3f, 0xd3, 0x70, 0x95, 0x73,
-	0x1c, 0xc7, 0xfd, 0x32, 0x57, 0x8f, 0x96, 0xb9, 0xd5, 0xb8, 0x62, 0x12, 0x91, 0x1c, 0xae, 0x74,
-	0xdf, 0xa6, 0xcf, 0xbc, 0xd2, 0x6d, 0x47, 0x2a, 0xdd, 0x3b, 0x13, 0x3a, 0xa7, 0x2d, 0x76, 0x43,
-	0xd5, 0x64, 0xfa, 0x6c, 0xab, 0xc9, 0xc7, 0x50, 0x1a, 0x74, 0x09, 0x89, 0xf1, 0x26, 0xcc, 0x62,
-	0xa0, 0x54, 0x4d, 0x49, 0x64, 0x86, 0x0f, 0x0e, 0x2a, 0xcb, 0x16, 0xf5, 0x9e, 0xda, 0x4e, 0x7b,
-	0x82, 0xca, 0x82, 0x12, 0xba, 0xca, 0xe2, 0x2b, 0x0b, 0x78, 0xdb, 0x93, 0x43, 0x49, 0xbc, 0x55,
-	0x52, 0x0a, 0x6b, 0x3c, 0x16, 0x95, 0x25, 0xe2, 0x19, 0x61, 0x7d, 0x09, 0xdb, 0x4d, 0xdc, 0x2f,
-	0xf1, 0xcc, 0x89, 0x8c, 0x32, 0x9c, 0xc8, 0xe9, 0x80, 0xc8, 0x28, 0xcb, 0x89, 0x8c, 0x00, 0xbf,
-	0xda, 0x9c, 0x91, 0x8f, 0x9f, 0xa9, 0xdc, 0x3a, 0x73, 0x37, 0xfd, 0x7c, 0x8b, 0x78, 0xea, 0xe7,
-	0x1b, 0x8e, 0x9f, 0x20, 0xdf, 0x22, 0x92, 0x2f, 0x57, 0xbe, 0xc5, 0x38, 0x77, 0x9e, 0xf9, 0x16,
-	0xb8, 0x14, 0xe4, 0x1b, 0x06, 0x2a, 0x31, 0xdf, 0x54, 0xe4, 0x7c, 0x30, 0x7e, 0x2c, 0x37, 0x3a,
-	0xfb, 0x2e, 0x5b, 0x53, 0xa8, 0x0e, 0x37, 0xe5, 0x48, 0xa4, 0x0e, 0x23, 0x8e, 0xf3, 0x02, 0x01,
-	0x3e, 0x7d, 0x7d, 0x15, 0x01, 0x7d, 0x11, 0x92, 0x44, 0x5f, 0x25, 0xa5, 0xb0, 0x3e, 0x97, 0x70,
-	0xe2, 0x04, 0x5c, 0x8a, 0x48, 0xbe, 0x5c, 0x5c, 0x8a, 0x71, 0xee, 0x3c, 0xb9, 0x14, 0xb8, 0x14,
-	0x70, 0x09, 0xa3, 0x91, 0xc8, 0x25, 0x15, 0x3a, 0x1f, 0x6c, 0xec, 0x43, 0xf1, 0x43, 0xdb, 0xea,
-	0x3d, 0xb2, 0xdb, 0xb4, 0x67, 0xda, 0xac, 0x9d, 0xe5, 0x0d, 0x47, 0x05, 0xae, 0x3a, 0xfc, 0x99,
-	0xee, 0x70, 0xc2, 0x31, 0x46, 0x79, 0x7c, 0x5a, 0x78, 0x38, 0x6b, 0x16, 0xe5, 0xd4, 0xa7, 0x62,
-	0x46, 0xc8, 0xb1, 0xe3, 0x62, 0x09, 0xf1, 0xdd, 0x46, 0xaf, 0xb1, 0xe7, 0x0b, 0xa4, 0x85, 0x00,
-	0x91, 0x73, 0x0f, 0xe4, 0x94, 0x90, 0x30, 0xbe, 0x4b, 0xab, 0xfe, 0xea, 0x34, 0x34, 0xe6, 0xfd,
-	0x95, 0x42, 0x4f, 0xd2, 0x5f, 0xa1, 0xcc, 0x04, 0xfd, 0x15, 0x5a, 0x0f, 0xbe, 0x53, 0xe4, 0x1e,
-	0xcc, 0x3a, 0xb8, 0x5f, 0x2c, 0xc8, 0x5c, 0xf0, 0x96, 0x4e, 0x70, 0x68, 0x73, 0x6b, 0xd3, 0x47,
-	0xc7, 0x0b, 0x17, 0x4c, 0x5f, 0x38, 0x68, 0xd4, 0xce, 0x26, 0x1b, 0xab, 0xbf, 0x15, 0x21, 0xb3,
-	0x21, 0xaf, 0xd3, 0x88, 0x05, 0x19, 0xbc, 0xa9, 0x22, 0x86, 0x4e, 0x78, 0xf0, 0xf6, 0xab, 0x7c,
-	0x33, 0x11, 0x83, 0x5f, 0x8e, 0x6b, 0xbf, 0xfc, 0xf4, 0xd7, 0xf7, 0xe9, 0xcb, 0x90, 0x17, 0xa0,
-	0xd7, 0x30, 0xe2, 0xc4, 0x86, 0xac, 0x7f, 0xe5, 0x41, 0x5e, 0x1d, 0xe7, 0x82, 0xa8, 0x7c, 0x6b,
-	0x04, 0x2a, 0xd9, 0xa0, 0x03, 0x10, 0xdc, 0x38, 0x10, 0xad, 0xae, 0xa1, 0xdb, 0x93, 0xf2, 0xe2,
-	0x28, 0xd8, 0x48, 0x9b, 0xc1, 0x3d, 0x83, 0xde, 0xe6, 0xd0, 0x0d, 0x86, 0xde, 0xa6, 0xe6, 0xba,
-	0x22, 0xc6, 0xa6, 0x8c, 0x21, 0x3f, 0xc9, 0xc5, 0xc6, 0x30, 0x74, 0xcf, 0x10, 0x1b, 0xc3, 0x81,
-	0x1b, 0x85, 0xe4, 0x18, 0x8a, 0x73, 0x66, 0x7c, 0x0c, 0xc3, 0xa7, 0xf6, 0xf8, 0x18, 0x0e, 0x1c,
-	0x56, 0x47, 0xee, 0xa7, 0x58, 0x5e, 0xc2, 0x7e, 0x86, 0x57, 0xb8, 0x38, 0x0a, 0x36, 0xd2, 0x66,
-	0x70, 0x4e, 0xd4, 0xdb, 0x1c, 0x3a, 0x8a, 0xea, 0x6d, 0x0e, 0x1f, 0x37, 0xe3, 0x6c, 0x3e, 0x83,
-	0x4b, 0xe1, 0x96, 0x9b, 0xdc, 0x1e, 0xf3, 0x9c, 0x50, 0x5e, 0x1a, 0x0d, 0x4c, 0xb6, 0xfc, 0x15,
-	0xe4, 0x07, 0x0e, 0xea, 0x44, 0xab, 0x51, 0x77, 0x31, 0x50, 0x5e, 0x1e, 0x03, 0x39, 0xd2, 0xf8,
-	0xc0, 0x19, 0x54, 0x6f, 0x5c, 0x77, 0xce, 0xd6, 0x1b, 0xd7, 0x1e, 0x68, 0x13, 0x8c, 0x0f, 0x1c,
-	0x35, 0xf5, 0xc6, 0x75, 0x67, 0x5a, 0xbd, 0x71, 0xfd, 0xb9, 0x35, 0x91, 0x64, 0xd8, 0xba, 0xc5,
-	0x92, 0x6c, 0xb0, 0xdd, 0x8f, 0x25, 0x59, 0xb4, 0x77, 0x4f, 0x26, 0x99, 0xea, 0x33, 0xe3, 0x49,
-	0x16, 0x69, 0x8e, 0xe3, 0x49, 0x16, 0x6d, 0x59, 0x47, 0x92, 0x4c, 0x2d, 0x38, 0x81, 0x64, 0x91,
-	0x35, 0x2f, 0x8f, 0x81, 0x1c, 0x33, 0xce, 0x89, 0xc6, 0x75, 0xe7, 0xab, 0xa4, 0x38, 0x8f, 0x69,
-	0x5c, 0xc6, 0x19, 0xbf, 0xc1, 0xb1, 0x71, 0x1e, 0xec, 0x71, 0x62, 0xe3, 0x1c, 0x69, 0x00, 0x46,
-	0xc4, 0x59, 0xf5, 0x80, 0xf1, 0x71, 0x8e, 0x34, 0xae, 0xf1, 0x71, 0x8e, 0xb6, 0x93, 0x23, 0xf3,
-	0x59, 0x2d, 0x38, 0x21, 0x9f, 0x23, 0x6b, 0x5e, 0x1e, 0x03, 0x99, 0x68, 0xbc, 0x76, 0xe3, 0xe8,
-	0xf9, 0xfc, 0x85, 0xdf, 0xd9, 0xef, 0xef, 0xe7, 0xf3, 0xa9, 0xaf, 0x5f, 0xcc, 0xa7, 0x8e, 0xd8,
-	0xef, 0x57, 0xf6, 0xfb, 0x83, 0xfd, 0x76, 0x67, 0xc4, 0x7f, 0xf4, 0xd6, 0xfe, 0x0d, 0x00, 0x00,
-	0xff, 0xff, 0xf3, 0xcc, 0x22, 0xcd, 0x4a, 0x1c, 0x00, 0x00,
+	0xe9, 0xc1, 0xca, 0xee, 0xcc, 0xe7, 0xcd, 0x7b, 0x33, 0x9f, 0xcf, 0xbc, 0x7d, 0x33, 0x81, 0x7c,
+	0xd3, 0xee, 0x79, 0x8e, 0xdd, 0xa9, 0xf4, 0x1d, 0xdb, 0xb3, 0x09, 0x69, 0xd9, 0xcd, 0x36, 0x75,
+	0x2a, 0xee, 0xd3, 0x86, 0xd3, 0x6d, 0x5b, 0x5e, 0xe5, 0xe0, 0x8d, 0x72, 0xce, 0xed, 0xd3, 0xa6,
+	0x2b, 0x01, 0xe5, 0xbc, 0xbd, 0xf3, 0x05, 0x6d, 0x7a, 0xea, 0x35, 0xe7, 0x1d, 0xf6, 0xa9, 0x7a,
+	0x29, 0xed, 0xd9, 0x7b, 0xb6, 0x78, 0x5c, 0xe1, 0x4f, 0xd8, 0x7a, 0xb5, 0xdf, 0xd9, 0xdf, 0xb3,
+	0x7a, 0x2b, 0xf2, 0x8f, 0x6c, 0x34, 0xee, 0x42, 0xe1, 0x3e, 0xf5, 0x36, 0xed, 0x16, 0x35, 0xe9,
+	0x97, 0xfb, 0xd4, 0xf5, 0xc8, 0x2d, 0xc8, 0xf4, 0xd8, 0xeb, 0xb6, 0xd5, 0x9a, 0x4b, 0xbd, 0x92,
+	0x5a, 0xcc, 0xd6, 0xe0, 0xc5, 0xf1, 0xfc, 0x0c, 0x47, 0xd4, 0x37, 0xcc, 0x19, 0xde, 0x55, 0x6f,
+	0x19, 0xef, 0xc1, 0x65, 0xdf, 0xcc, 0xed, 0xdb, 0x3d, 0x97, 0x92, 0x65, 0x98, 0xe6, 0x9d, 0xc2,
+	0x28, 0x57, 0x9d, 0xab, 0x0c, 0x4f, 0xa0, 0x22, 0xf0, 0x02, 0x65, 0x1c, 0x4f, 0xc1, 0x95, 0x07,
+	0x96, 0x2b, 0x86, 0x70, 0x95, 0xeb, 0x7b, 0x90, 0xd9, 0xb5, 0x3a, 0x1e, 0x75, 0x5c, 0x1c, 0x65,
+	0x59, 0x37, 0x4a, 0xd4, 0xac, 0x72, 0x4f, 0xda, 0x98, 0xca, 0xb8, 0xfc, 0xcd, 0x14, 0x64, 0xb0,
+	0x91, 0x94, 0xe0, 0x62, 0xaf, 0xd1, 0xa5, 0x7c, 0xc4, 0xa9, 0xc5, 0xac, 0x29, 0x5f, 0xc8, 0x0a,
+	0xe4, 0xac, 0xd6, 0x76, 0xdf, 0xa1, 0xbb, 0xd6, 0x33, 0xd6, 0x97, 0xe6, 0x7d, 0xb5, 0x02, 0x9b,
+	0x28, 0xd4, 0x37, 0x1e, 0x61, 0xab, 0x09, 0x56, 0x4b, 0x3d, 0x93, 0x47, 0x30, 0xd3, 0x69, 0xec,
+	0xd0, 0x8e, 0x3b, 0x37, 0xc5, 0xb0, 0xb9, 0xea, 0xda, 0x24, 0x91, 0x55, 0x1e, 0x08, 0xd3, 0x0f,
+	0x18, 0xc1, 0x87, 0x26, 0x8e, 0x43, 0xea, 0x90, 0xeb, 0xd2, 0xee, 0x0e, 0xeb, 0xfe, 0xdc, 0xea,
+	0xbb, 0x73, 0xd3, 0x6c, 0xd8, 0x42, 0xf5, 0x4e, 0xdc, 0xb2, 0x6d, 0x31, 0xea, 0x2b, 0x0f, 0x7d,
+	0xbc, 0x19, 0xb6, 0x25, 0x55, 0xb8, 0xc8, 0x94, 0xc3, 0xe6, 0x71, 0x51, 0x0c, 0x72, 0x23, 0x76,
+	0xed, 0x19, 0xc8, 0x94, 0x50, 0x46, 0x73, 0x9e, 0x2f, 0x45, 0xb0, 0x06, 0x33, 0x62, 0x7d, 0x2e,
+	0xf1, 0x46, 0x35, 0xeb, 0xf2, 0x5b, 0x90, 0x0b, 0x85, 0x4e, 0xae, 0xc0, 0x54, 0x9b, 0x1e, 0x4a,
+	0x59, 0x98, 0xfc, 0x91, 0xaf, 0xee, 0x41, 0xa3, 0xb3, 0x4f, 0xd9, 0x0a, 0xf2, 0x36, 0xf9, 0xf2,
+	0x76, 0x7a, 0x2d, 0x65, 0xac, 0x43, 0x31, 0xb4, 0x1c, 0xa8, 0x91, 0x0a, 0x23, 0x83, 0x37, 0x08,
+	0x32, 0x92, 0x44, 0x22, 0x61, 0xc6, 0x8f, 0x29, 0x28, 0x3e, 0xe9, 0xb7, 0x1a, 0x1e, 0x9d, 0x54,
+	0xa1, 0xe4, 0x5d, 0xb8, 0x24, 0x40, 0x07, 0x6c, 0x91, 0x2c, 0xbb, 0x27, 0x02, 0xcc, 0x55, 0xaf,
+	0xeb, 0x3c, 0x7e, 0x22, 0x21, 0x66, 0x8e, 0x1b, 0xe0, 0x0b, 0x79, 0x1d, 0xa6, 0xf9, 0x76, 0x63,
+	0x74, 0x73, 0xbb, 0x1b, 0x49, 0xbc, 0x98, 0x02, 0x69, 0xd4, 0x80, 0x84, 0x63, 0x3d, 0xd1, 0xb6,
+	0xd8, 0x84, 0xa2, 0x49, 0xbb, 0xf6, 0xc1, 0xe4, 0xf3, 0x65, 0x4c, 0xec, 0xda, 0x4e, 0x53, 0x32,
+	0x31, 0x6b, 0xca, 0x17, 0xa3, 0x04, 0x24, 0x3c, 0x9e, 0x8c, 0x09, 0x37, 0xfd, 0xe3, 0x86, 0xdb,
+	0x0e, 0xb9, 0xf0, 0xd8, 0x6b, 0xc4, 0x05, 0x47, 0x70, 0x17, 0xbc, 0xcb, 0xdf, 0xf4, 0xd2, 0x2c,
+	0x98, 0x1d, 0xef, 0x4c, 0x9a, 0x9d, 0xc0, 0x0b, 0x94, 0xb1, 0xa6, 0x66, 0x37, 0xb1, 0x6b, 0x7f,
+	0x1e, 0x61, 0xef, 0xc6, 0x3f, 0x98, 0x44, 0x78, 0xe3, 0x09, 0x92, 0x48, 0xd8, 0x6c, 0x38, 0x89,
+	0xfc, 0x70, 0x8e, 0x49, 0x44, 0x17, 0x99, 0x36, 0x89, 0xb0, 0x10, 0x5c, 0xea, 0x1c, 0x58, 0x4d,
+	0xae, 0x0e, 0x99, 0x44, 0x30, 0x84, 0x2d, 0xd9, 0x5c, 0xdf, 0x60, 0x21, 0x20, 0xa4, 0xde, 0x72,
+	0xc9, 0x02, 0xcc, 0xa2, 0x96, 0x64, 0xb6, 0xc8, 0xd6, 0x72, 0x0c, 0x9d, 0x91, 0x62, 0x62, 0xb3,
+	0x97, 0x6a, 0x72, 0xc9, 0x06, 0x14, 0xd8, 0x06, 0xb4, 0x1c, 0xda, 0xda, 0x76, 0x3d, 0xa6, 0x69,
+	0x99, 0x1f, 0x0a, 0xd5, 0xff, 0xc7, 0x51, 0xbc, 0xc5, 0x51, 0x66, 0x1e, 0x8d, 0xc4, 0x9b, 0x26,
+	0xc9, 0x64, 0xfe, 0x93, 0x24, 0x83, 0xcb, 0x15, 0x24, 0x19, 0xae, 0x9a, 0xc4, 0x24, 0x23, 0x64,
+	0x24, 0x61, 0xc6, 0x47, 0x50, 0x5a, 0x77, 0x28, 0x8b, 0x17, 0x97, 0x4c, 0x09, 0x69, 0x15, 0x33,
+	0x80, 0x54, 0xd1, 0xbc, 0x6e, 0x18, 0xb4, 0x08, 0x25, 0x81, 0x4d, 0xb8, 0x16, 0x19, 0x0c, 0xa3,
+	0xba, 0x0b, 0x19, 0xa4, 0x01, 0x07, 0xbc, 0x9e, 0x30, 0xa0, 0xa9, 0xb0, 0xc6, 0xfb, 0x50, 0x64,
+	0x7b, 0x2e, 0x12, 0xd9, 0x32, 0x40, 0xc0, 0x3a, 0xee, 0x9a, 0x3c, 0xa3, 0x31, 0xeb, 0x93, 0x6e,
+	0x66, 0x7d, 0xce, 0xd9, 0xfc, 0x48, 0x78, 0x88, 0xd3, 0xc5, 0xf3, 0x73, 0x0a, 0x4a, 0x32, 0xcb,
+	0x9d, 0x26, 0x26, 0x26, 0xaf, 0xcb, 0x0a, 0x3d, 0x41, 0x82, 0x2e, 0xa0, 0x8d, 0xca, 0xd1, 0xab,
+	0x03, 0x39, 0x7a, 0x7c, 0x86, 0x22, 0x13, 0x38, 0xdd, 0x8a, 0x6c, 0x40, 0x49, 0xa6, 0xa6, 0x53,
+	0x91, 0xf4, 0x3f, 0xb8, 0x16, 0x19, 0x05, 0x73, 0xdc, 0x9f, 0x69, 0xb8, 0xca, 0x35, 0x8e, 0xed,
+	0x7e, 0x9a, 0xab, 0x47, 0xd3, 0xdc, 0x4a, 0x5c, 0x32, 0x89, 0x58, 0x0e, 0x67, 0xba, 0x6f, 0xd3,
+	0x67, 0x9e, 0xe9, 0xb6, 0x22, 0x99, 0xee, 0x9d, 0x09, 0x83, 0xd3, 0x26, 0xbb, 0xa1, 0x6c, 0x32,
+	0x7d, 0xb6, 0xd9, 0xe4, 0x63, 0x28, 0x0d, 0x86, 0x84, 0xc2, 0x78, 0x13, 0x66, 0x91, 0x28, 0x95,
+	0x53, 0x12, 0x95, 0xe1, 0x83, 0x83, 0xcc, 0xb2, 0x49, 0xbd, 0xa7, 0xb6, 0xd3, 0x9e, 0x20, 0xb3,
+	0xa0, 0x85, 0x2e, 0xb3, 0xf8, 0x83, 0x05, 0xba, 0xed, 0xc9, 0xa6, 0x24, 0xdd, 0x2a, 0x2b, 0x85,
+	0x35, 0x9e, 0x88, 0xcc, 0x12, 0x89, 0x8c, 0xb0, 0x6a, 0x85, 0xad, 0x26, 0xae, 0x97, 0x78, 0xe6,
+	0x42, 0x46, 0x1b, 0x2e, 0xe4, 0x74, 0x20, 0x64, 0xb4, 0xe5, 0x42, 0x46, 0x80, 0x9f, 0x6d, 0xce,
+	0x28, 0xc6, 0xcf, 0xd4, 0xde, 0x3a, 0xf3, 0x30, 0xfd, 0xfd, 0x16, 0x89, 0xd4, 0xdf, 0x6f, 0xd8,
+	0x7e, 0x82, 0xfd, 0x16, 0xb1, 0x7c, 0xb9, 0xf6, 0x5b, 0x4c, 0x70, 0xe7, 0xb9, 0xdf, 0x82, 0x90,
+	0x82, 0xfd, 0x86, 0x44, 0x25, 0xee, 0x37, 0xc5, 0x9c, 0x0f, 0xc6, 0x8f, 0xe5, 0x7a, 0x67, 0xdf,
+	0x65, 0x73, 0x0a, 0xe5, 0xe1, 0xa6, 0x6c, 0x89, 0xe4, 0x61, 0xc4, 0x71, 0x5d, 0x20, 0xc0, 0x97,
+	0xaf, 0x3f, 0x44, 0x20, 0x5f, 0x84, 0x24, 0xc9, 0x57, 0x59, 0x29, 0xac, 0xaf, 0x25, 0xec, 0x38,
+	0x81, 0x96, 0x22, 0x96, 0x2f, 0x97, 0x96, 0x62, 0x82, 0x3b, 0x4f, 0x2d, 0x05, 0x21, 0x05, 0x5a,
+	0x42, 0x36, 0x12, 0xb5, 0xa4, 0xa8, 0xf3, 0xc1, 0xc6, 0x3e, 0x14, 0x3f, 0xb4, 0xad, 0xde, 0x63,
+	0xbb, 0x4d, 0x7b, 0xa6, 0xcd, 0xca, 0x59, 0x5e, 0x70, 0x54, 0xe0, 0xaa, 0xc3, 0x9f, 0xe9, 0x36,
+	0x17, 0x1c, 0x53, 0x94, 0xc7, 0xbb, 0x45, 0x84, 0xb3, 0x66, 0x51, 0x76, 0x7d, 0x2a, 0x7a, 0x84,
+	0x1d, 0x3b, 0x44, 0x96, 0x10, 0xdf, 0x6d, 0xf4, 0x1a, 0x7b, 0xbe, 0x81, 0x3c, 0xa3, 0x11, 0xd9,
+	0xf7, 0x50, 0x76, 0x09, 0x0b, 0xe3, 0xbb, 0xb4, 0xaa, 0xaf, 0x4e, 0x23, 0x63, 0x5e, 0x5f, 0x29,
+	0xf4, 0x24, 0xf5, 0x15, 0xda, 0x4c, 0x50, 0x5f, 0xa1, 0xf7, 0xe0, 0x3b, 0x45, 0xee, 0xc3, 0xac,
+	0x83, 0xeb, 0xc5, 0x48, 0xe6, 0x86, 0xb7, 0x75, 0x86, 0x43, 0x8b, 0x5b, 0x9b, 0x3e, 0x3a, 0x9e,
+	0xbf, 0x60, 0xfa, 0xc6, 0x41, 0xa1, 0x76, 0x36, 0xbb, 0xb1, 0xfa, 0x5b, 0x11, 0x32, 0xeb, 0xf2,
+	0x92, 0x8d, 0x58, 0x90, 0xc1, 0xfb, 0x2b, 0x62, 0xe8, 0x8c, 0x07, 0xef, 0xc4, 0xca, 0xb7, 0x12,
+	0x31, 0xf8, 0xe5, 0xb8, 0xf6, 0xcb, 0x4f, 0x7f, 0x7d, 0x9f, 0xbe, 0x0c, 0x79, 0x01, 0x7a, 0x0d,
+	0x19, 0x27, 0x36, 0x64, 0xfd, 0x8b, 0x10, 0xf2, 0xea, 0x38, 0xd7, 0x46, 0xe5, 0xdb, 0x23, 0x50,
+	0xc9, 0x0e, 0x1d, 0x80, 0xe0, 0x1e, 0x82, 0x68, 0xc7, 0x1a, 0xba, 0x53, 0x29, 0x2f, 0x8c, 0x82,
+	0x8d, 0xf4, 0x19, 0xdc, 0x33, 0xe8, 0x7d, 0x0e, 0xdd, 0x6b, 0xe8, 0x7d, 0x6a, 0xae, 0x2b, 0x62,
+	0x7c, 0x4a, 0x0e, 0xf9, 0x49, 0x2e, 0x96, 0xc3, 0xd0, 0x3d, 0x43, 0x2c, 0x87, 0x03, 0x37, 0x0a,
+	0xc9, 0x1c, 0x8a, 0x73, 0x66, 0x3c, 0x87, 0xe1, 0x53, 0x7b, 0x3c, 0x87, 0x03, 0x87, 0xd5, 0x91,
+	0xeb, 0x29, 0xa6, 0x97, 0xb0, 0x9e, 0xe1, 0x19, 0x2e, 0x8c, 0x82, 0x8d, 0xf4, 0x19, 0x9c, 0x13,
+	0xf5, 0x3e, 0x87, 0x8e, 0xa2, 0x7a, 0x9f, 0xc3, 0xc7, 0xcd, 0x38, 0x9f, 0xcf, 0xe0, 0x52, 0xb8,
+	0xe4, 0x26, 0x77, 0xc6, 0x3c, 0x27, 0x94, 0x17, 0x47, 0x03, 0x93, 0x3d, 0x7f, 0x05, 0xf9, 0x81,
+	0x83, 0x3a, 0xd1, 0x8e, 0xa8, 0xbb, 0x18, 0x28, 0x2f, 0x8d, 0x81, 0x1c, 0xe9, 0x7c, 0xe0, 0x0c,
+	0xaa, 0x77, 0xae, 0x3b, 0x67, 0xeb, 0x9d, 0x6b, 0x0f, 0xb4, 0x09, 0xce, 0x07, 0x8e, 0x9a, 0x7a,
+	0xe7, 0xba, 0x33, 0xad, 0xde, 0xb9, 0xfe, 0xdc, 0x9a, 0x28, 0x32, 0x2c, 0xdd, 0x62, 0x45, 0x36,
+	0x58, 0xee, 0xc7, 0x8a, 0x2c, 0x5a, 0xbb, 0x27, 0x8b, 0x4c, 0xd5, 0x99, 0xf1, 0x22, 0x8b, 0x14,
+	0xc7, 0xf1, 0x22, 0x8b, 0x96, 0xac, 0x23, 0x45, 0xa6, 0x26, 0x9c, 0x20, 0xb2, 0xc8, 0x9c, 0x97,
+	0xc6, 0x40, 0x8e, 0xc9, 0x73, 0xa2, 0x73, 0xdd, 0xf9, 0x2a, 0x89, 0xe7, 0x31, 0x9d, 0x4b, 0x9e,
+	0xf1, 0x1b, 0x1c, 0xcb, 0xf3, 0x60, 0x8d, 0x13, 0xcb, 0x73, 0xa4, 0x00, 0x18, 0xc1, 0xb3, 0xaa,
+	0x01, 0xe3, 0x79, 0x8e, 0x14, 0xae, 0xf1, 0x3c, 0x47, 0xcb, 0xc9, 0x91, 0xfb, 0x59, 0x4d, 0x38,
+	0x61, 0x3f, 0x47, 0xe6, 0xbc, 0x34, 0x06, 0x32, 0xd1, 0x79, 0xed, 0xc6, 0xd1, 0xf3, 0x9b, 0x17,
+	0x7e, 0x67, 0xbf, 0xbf, 0x9f, 0xdf, 0x4c, 0x7d, 0xfd, 0xe2, 0x66, 0xea, 0x88, 0xfd, 0x7e, 0x65,
+	0xbf, 0x3f, 0xd8, 0x6f, 0x67, 0x46, 0xfc, 0x9f, 0x6f, 0xf5, 0xdf, 0x00, 0x00, 0x00, 0xff, 0xff,
+	0xbb, 0x97, 0x43, 0x36, 0x60, 0x1c, 0x00, 0x00,
 }

vendor/src/github.com/docker/swarmkit/api/control.proto

@@ -115,6 +115,7 @@ message UpdateNodeResponse {
 // RemoveNodeRequest requests to delete the specified node from store.
 message RemoveNodeRequest {
 	string node_id = 1 [(gogoproto.customname) = "NodeID"];
+	bool force = 2;
 }
 
 message RemoveNodeResponse {

vendor/src/github.com/docker/swarmkit/ca/server.go

@@ -309,15 +309,6 @@ func (s *Server) Run(ctx context.Context) error {
 	logger := log.G(ctx).WithField("module", "ca")
 	ctx = log.WithLogger(ctx, logger)
 
-	// Run() should never be called twice, but just in case, we're
-	// attempting to close the started channel in a safe way
-	select {
-	case <-s.started:
-		return fmt.Errorf("CA server cannot be started more than once")
-	default:
-		close(s.started)
-	}
-
 	// Retrieve the channels to keep track of changes in the cluster
 	// Retrieve all the currently registered nodes
 	var nodes []*api.Node
@@ -346,6 +337,7 @@ func (s *Server) Run(ctx context.Context) error {
 	s.mu.Lock()
 	s.ctx, s.cancel = context.WithCancel(ctx)
 	s.mu.Unlock()
+	close(s.started)
 
 	if err != nil {
 		log.G(ctx).WithFields(logrus.Fields{

vendor/src/github.com/docker/swarmkit/manager/allocator/network.go

@@ -377,7 +377,7 @@ func (a *Allocator) doNodeAlloc(ctx context.Context, nc *networkContext, ev even
 
 		node.Attachment.Network = nc.ingressNetwork.Copy()
 		if err := a.allocateNode(ctx, nc, node); err != nil {
-			log.G(ctx).Errorf("Fauled to allocate network resources for node %s: %v", node.ID, err)
+			log.G(ctx).Errorf("Failed to allocate network resources for node %s: %v", node.ID, err)
 		}
 	}
 }

vendor/src/github.com/docker/swarmkit/manager/controlapi/node.go

@@ -283,7 +283,7 @@ func (s *Server) RemoveNode(ctx context.Context, request *api.RemoveNodeRequest)
 				return grpc.Errorf(codes.FailedPrecondition, "node %s is a cluster manager and is a member of the raft cluster. It must be demoted to worker before removal", request.NodeID)
 			}
 		}
-		if node.Status.State == api.NodeStatus_READY {
+		if !request.Force && node.Status.State == api.NodeStatus_READY {
 			return grpc.Errorf(codes.FailedPrecondition, "node %s is not down and can't be removed", request.NodeID)
 		}
 		return store.DeleteNode(tx, request.NodeID)

vendor/src/github.com/docker/swarmkit/manager/dispatcher/dispatcher.go

@@ -20,6 +20,7 @@ import (
 	"github.com/docker/swarmkit/manager/state"
 	"github.com/docker/swarmkit/manager/state/store"
 	"github.com/docker/swarmkit/manager/state/watch"
+	"github.com/docker/swarmkit/picker"
 	"github.com/docker/swarmkit/protobuf/ptypes"
 	"golang.org/x/net/context"
 )
@@ -60,8 +61,6 @@ var (
 // Config is configuration for Dispatcher. For default you should use
 // DefautConfig.
 type Config struct {
-	// Addr configures the address the dispatcher reports to agents.
-	Addr             string
 	HeartbeatPeriod  time.Duration
 	HeartbeatEpsilon time.Duration
 	// RateLimitPeriod specifies how often node with same ID can try to register
@@ -90,7 +89,6 @@ type Cluster interface {
 // Dispatcher is responsible for dispatching tasks and tracking agent health.
 type Dispatcher struct {
 	mu                   sync.Mutex
-	addr                 string
 	nodes                *nodeStore
 	store                *store.MemoryStore
 	mgrQueue             *watch.Queue
@@ -121,7 +119,6 @@ func (b weightedPeerByNodeID) Swap(i, j int) { b[i], b[j] = b[j], b[i] }
 // NOTE: each handler which does something with raft must add to Dispatcher.wg
 func New(cluster Cluster, c *Config) *Dispatcher {
 	return &Dispatcher{
-		addr:                      c.Addr,
 		nodes:                     newNodeStore(c.HeartbeatPeriod, c.HeartbeatEpsilon, c.GracePeriodMultiplier, c.RateLimitPeriod),
 		store:                     cluster.MemoryStore(),
 		cluster:                   cluster,
@@ -142,7 +139,11 @@ func getWeightedPeers(cluster Cluster) []*api.WeightedPeer {
 				NodeID: m.NodeID,
 				Addr:   m.Addr,
 			},
-			Weight: 1,
+
+			// TODO(stevvooe): Calculate weight of manager selection based on
+			// cluster-level observations, such as number of connections and
+			// load.
+			Weight: picker.DefaultObservationWeight,
 		})
 	}
 	return mgrs
@@ -574,14 +575,18 @@ func (d *Dispatcher) Tasks(r *api.TasksRequest, stream api.Dispatcher_TasksServe
 		}
 
 		// bursty events should be processed in batches and sent out snapshot
-		const modificationBatchLimit = 200
-		const eventPausedGap = 50 * time.Millisecond
-		var modificationCnt int
-		// eventPaused is true when there have been modifications
-		// but next event has not arrived within eventPausedGap
-		eventPaused := false
-
-		for modificationCnt < modificationBatchLimit && !eventPaused {
+		const (
+			modificationBatchLimit = 200
+			eventPausedGap         = 50 * time.Millisecond
+		)
+		var (
+			modificationCnt    int
+			eventPausedTimer   *time.Timer
+			eventPausedTimeout <-chan time.Time
+		)
+
+	batchingLoop:
+		for modificationCnt < modificationBatchLimit {
 			select {
 			case event := <-nodeTasks:
 				switch v := event.(type) {
@@ -602,16 +607,24 @@ func (d *Dispatcher) Tasks(r *api.TasksRequest, stream api.Dispatcher_TasksServe
 					delete(tasksMap, v.Task.ID)
 					modificationCnt++
 				}
-			case <-time.After(eventPausedGap):
-				if modificationCnt > 0 {
-					eventPaused = true
+				if eventPausedTimer != nil {
+					eventPausedTimer.Reset(eventPausedGap)
+				} else {
+					eventPausedTimer = time.NewTimer(eventPausedGap)
+					eventPausedTimeout = eventPausedTimer.C
 				}
+			case <-eventPausedTimeout:
+				break batchingLoop
 			case <-stream.Context().Done():
 				return stream.Context().Err()
 			case <-d.ctx.Done():
 				return d.ctx.Err()
 			}
 		}
+
+		if eventPausedTimer != nil {
+			eventPausedTimer.Stop()
+		}
 	}
 }
 

vendor/src/github.com/docker/swarmkit/manager/dispatcher/nodes.go

@@ -43,26 +43,29 @@ func (rn *registeredNode) checkSessionID(sessionID string) error {
 }
 
 type nodeStore struct {
-	periodChooser         *periodChooser
-	gracePeriodMultiplier time.Duration
-	rateLimitPeriod       time.Duration
-	nodes                 map[string]*registeredNode
-	mu                    sync.RWMutex
+	periodChooser                *periodChooser
+	gracePeriodMultiplierNormal  time.Duration
+	gracePeriodMultiplierUnknown time.Duration
+	rateLimitPeriod              time.Duration
+	nodes                        map[string]*registeredNode
+	mu                           sync.RWMutex
 }
 
 func newNodeStore(hbPeriod, hbEpsilon time.Duration, graceMultiplier int, rateLimitPeriod time.Duration) *nodeStore {
 	return &nodeStore{
-		nodes:                 make(map[string]*registeredNode),
-		periodChooser:         newPeriodChooser(hbPeriod, hbEpsilon),
-		gracePeriodMultiplier: time.Duration(graceMultiplier),
-		rateLimitPeriod:       rateLimitPeriod,
+		nodes:                        make(map[string]*registeredNode),
+		periodChooser:                newPeriodChooser(hbPeriod, hbEpsilon),
+		gracePeriodMultiplierNormal:  time.Duration(graceMultiplier),
+		gracePeriodMultiplierUnknown: time.Duration(graceMultiplier) * 2,
+		rateLimitPeriod:              rateLimitPeriod,
 	}
 }
 
 func (s *nodeStore) updatePeriod(hbPeriod, hbEpsilon time.Duration, gracePeriodMultiplier int) {
 	s.mu.Lock()
 	s.periodChooser = newPeriodChooser(hbPeriod, hbEpsilon)
-	s.gracePeriodMultiplier = time.Duration(gracePeriodMultiplier)
+	s.gracePeriodMultiplierNormal = time.Duration(gracePeriodMultiplier)
+	s.gracePeriodMultiplierUnknown = s.gracePeriodMultiplierNormal * 2
 	s.mu.Unlock()
 }
 
@@ -79,7 +82,7 @@ func (s *nodeStore) AddUnknown(n *api.Node, expireFunc func()) error {
 		Node: n,
 	}
 	s.nodes[n.ID] = rn
-	rn.Heartbeat = heartbeat.New(s.periodChooser.Choose()*s.gracePeriodMultiplier, expireFunc)
+	rn.Heartbeat = heartbeat.New(s.periodChooser.Choose()*s.gracePeriodMultiplierUnknown, expireFunc)
 	return nil
 }
 
@@ -124,7 +127,7 @@ func (s *nodeStore) Add(n *api.Node, expireFunc func()) *registeredNode {
 		Disconnect: make(chan struct{}),
 	}
 	s.nodes[n.ID] = rn
-	rn.Heartbeat = heartbeat.New(s.periodChooser.Choose()*s.gracePeriodMultiplier, expireFunc)
+	rn.Heartbeat = heartbeat.New(s.periodChooser.Choose()*s.gracePeriodMultiplierNormal, expireFunc)
 	return rn
 }
 
@@ -154,7 +157,7 @@ func (s *nodeStore) Heartbeat(id, sid string) (time.Duration, error) {
 		return 0, err
 	}
 	period := s.periodChooser.Choose() // base period for node
-	grace := period * time.Duration(s.gracePeriodMultiplier)
+	grace := period * time.Duration(s.gracePeriodMultiplierNormal)
 	rn.mu.Lock()
 	rn.Heartbeat.Update(grace)
 	rn.Heartbeat.Beat()

vendor/src/github.com/docker/swarmkit/manager/manager.go

@@ -89,9 +89,11 @@ type Manager struct {
 	server                 *grpc.Server
 	localserver            *grpc.Server
 	RaftNode               *raft.Node
+	connSelector           *raftpicker.ConnSelector
 
 	mu sync.Mutex
 
+	started chan struct{}
 	stopped chan struct{}
 }
 
@@ -139,9 +141,6 @@ func New(config *Config) (*Manager, error) {
 		tcpAddr = net.JoinHostPort("0.0.0.0", tcpAddrPort)
 	}
 
-	// FIXME(aaronl): Remove this. It appears to be unused.
-	dispatcherConfig.Addr = tcpAddr
-
 	err := os.MkdirAll(filepath.Dir(config.ProtoAddr["unix"]), 0700)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create socket directory: %v", err)
@@ -220,6 +219,7 @@ func New(config *Config) (*Manager, error) {
 		server:      grpc.NewServer(opts...),
 		localserver: grpc.NewServer(opts...),
 		RaftNode:    RaftNode,
+		started:     make(chan struct{}),
 		stopped:     make(chan struct{}),
 	}
 
@@ -428,11 +428,12 @@ func (m *Manager) Run(parent context.Context) error {
 	}()
 
 	proxyOpts := []grpc.DialOption{
-		grpc.WithBackoffMaxDelay(2 * time.Second),
+		grpc.WithBackoffMaxDelay(time.Second),
 		grpc.WithTransportCredentials(m.config.SecurityConfig.ClientTLSCreds),
 	}
 
 	cs := raftpicker.NewConnSelector(m.RaftNode, proxyOpts...)
+	m.connSelector = cs
 
 	authorize := func(ctx context.Context, roles []string) error {
 		// Authorize the remote roles, ensure they can only be forwarded by managers
@@ -506,6 +507,8 @@ func (m *Manager) Run(parent context.Context) error {
 		return fmt.Errorf("can't initialize raft node: %v", err)
 	}
 
+	close(m.started)
+
 	go func() {
 		err := m.RaftNode.Run(ctx)
 		if err != nil {
@@ -560,12 +563,15 @@ func (m *Manager) Run(parent context.Context) error {
 func (m *Manager) Stop(ctx context.Context) {
 	log.G(ctx).Info("Stopping manager")
 
+	// It's not safe to start shutting down while the manager is still
+	// starting up.
+	<-m.started
+
 	// the mutex stops us from trying to stop while we're alrady stopping, or
 	// from returning before we've finished stopping.
 	m.mu.Lock()
 	defer m.mu.Unlock()
 	select {
-
 	// check to see that we've already stopped
 	case <-m.stopped:
 		return
@@ -600,6 +606,9 @@ func (m *Manager) Stop(ctx context.Context) {
 		m.keyManager.Stop()
 	}
 
+	if m.connSelector != nil {
+		m.connSelector.Stop()
+	}
 	m.RaftNode.Shutdown()
 	// some time after this point, Run will receive an error from one of these
 	m.server.Stop()

vendor/src/github.com/docker/swarmkit/manager/raftpicker/raftpicker.go

@@ -2,6 +2,7 @@ package raftpicker
 
 import (
 	"sync"
+	"time"
 
 	"golang.org/x/net/context"
 	"google.golang.org/grpc"
@@ -14,46 +15,37 @@ type picker struct {
 	addr string
 	raft AddrSelector
 	conn *grpc.Conn
-	cc   *grpc.ClientConn
+
+	stop chan struct{}
+	done chan struct{}
 }
 
-// Init does initial processing for the Picker, e.g., initiate some connections.
-func (p *picker) Init(cc *grpc.ClientConn) error {
-	p.cc = cc
-	return nil
+func newPicker(raft AddrSelector, addr string) *picker {
+	return &picker{
+		raft: raft,
+		addr: addr,
+
+		stop: make(chan struct{}),
+		done: make(chan struct{}),
+	}
 }
 
-func (p *picker) initConn() error {
-	if p.conn == nil {
-		conn, err := grpc.NewConn(p.cc)
-		if err != nil {
-			return err
-		}
-		p.conn = conn
+// Init does initial processing for the Picker, e.g., initiate some connections.
+func (p *picker) Init(cc *grpc.ClientConn) error {
+	conn, err := grpc.NewConn(cc)
+	if err != nil {
+		return err
 	}
+	p.conn = conn
 	return nil
 }
 
 // Pick blocks until either a transport.ClientTransport is ready for the upcoming RPC
 // or some error happens.
 func (p *picker) Pick(ctx context.Context) (transport.ClientTransport, error) {
-	p.mu.Lock()
-	if err := p.initConn(); err != nil {
-		p.mu.Unlock()
+	if err := p.updateConn(); err != nil {
 		return nil, err
 	}
-	p.mu.Unlock()
-
-	addr, err := p.raft.LeaderAddr()
-	if err != nil {
-		return nil, err
-	}
-	p.mu.Lock()
-	if p.addr != addr {
-		p.addr = addr
-		p.conn.NotifyReset()
-	}
-	p.mu.Unlock()
 	return p.conn.Wait(ctx)
 }
 
@@ -89,15 +81,46 @@ func (p *picker) Reset() error {
 
 // Close closes all the Conn's owned by this Picker.
 func (p *picker) Close() error {
+	close(p.stop)
+	<-p.done
 	return p.conn.Close()
 }
 
+func (p *picker) updateConn() error {
+	addr, err := p.raft.LeaderAddr()
+	if err != nil {
+		return err
+	}
+	p.mu.Lock()
+	if p.addr != addr {
+		p.addr = addr
+		p.Reset()
+	}
+	p.mu.Unlock()
+	return nil
+}
+
+func (p *picker) updateLoop() {
+	defer close(p.done)
+	ticker := time.NewTicker(1 * time.Second)
+	defer ticker.Stop()
+	for {
+		select {
+		case <-ticker.C:
+			p.updateConn()
+		case <-p.stop:
+			return
+		}
+	}
+}
+
 // ConnSelector is struct for obtaining connection with raftpicker.
 type ConnSelector struct {
 	mu      sync.Mutex
 	cc      *grpc.ClientConn
 	cluster RaftCluster
 	opts    []grpc.DialOption
+	picker  *picker
 }
 
 // NewConnSelector returns new ConnSelector with cluster and grpc.DialOpts which
@@ -122,8 +145,9 @@ func (c *ConnSelector) Conn() (*grpc.ClientConn, error) {
 	if err != nil {
 		return nil, err
 	}
-	picker := &picker{raft: c.cluster, addr: addr}
-	opts := append(c.opts, grpc.WithPicker(picker))
+	c.picker = newPicker(c.cluster, addr)
+	go c.picker.updateLoop()
+	opts := append(c.opts, grpc.WithPicker(c.picker))
 	cc, err := grpc.Dial(addr, opts...)
 	if err != nil {
 		return nil, err
@@ -131,3 +155,13 @@ func (c *ConnSelector) Conn() (*grpc.ClientConn, error) {
 	c.cc = cc
 	return c.cc, nil
 }
+
+// Stop cancels tracking loop for raftpicker and closes it.
+func (c *ConnSelector) Stop() {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	if c.picker == nil {
+		return
+	}
+	c.picker.Close()
+}

vendor/src/github.com/docker/swarmkit/manager/state/raft/raft.go

@@ -451,6 +451,17 @@ func (n *Node) Shutdown() {
 	}
 }
 
+// isShutdown indicates if node was shut down.
+// This method should be called under n.stopMu to avoid races with n.stop().
+func (n *Node) isShutdown() bool {
+	select {
+	case <-n.Ctx.Done():
+		return true
+	default:
+		return false
+	}
+}
+
 func (n *Node) stop() {
 	n.stopMu.Lock()
 	defer n.stopMu.Unlock()
@@ -763,7 +774,10 @@ func (n *Node) ResolveAddress(ctx context.Context, msg *api.ResolveAddressReques
 func (n *Node) LeaderAddr() (string, error) {
 	n.stopMu.RLock()
 	defer n.stopMu.RUnlock()
-	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	if n.isShutdown() {
+		return "", fmt.Errorf("raft node is shut down")
+	}
+	ctx, cancel := context.WithTimeout(n.Ctx, 10*time.Second)
 	defer cancel()
 	if err := WaitForLeader(ctx, n); err != nil {
 		return "", ErrNoClusterLeader
@@ -1288,7 +1302,7 @@ func (n *Node) ConnectToMember(addr string, timeout time.Duration) (*membership.
 
 // SubscribeLeadership returns channel to which events about leadership change
 // will be sent in form of raft.LeadershipState. Also cancel func is returned -
-// it should be called when listener is not longer interested in events.
+// it should be called when listener is no longer interested in events.
 func (n *Node) SubscribeLeadership() (q chan events.Event, cancel func()) {
 	ch := events.NewChannel(0)
 	sink := events.Sink(events.NewQueue(ch))

vendor/src/github.com/docker/swarmkit/picker/picker.go

@@ -15,6 +15,10 @@ import (
 
 var errRemotesUnavailable = fmt.Errorf("no remote hosts provided")
 
+// DefaultObservationWeight provides a weight to use for positive observations
+// that will balance well under repeated observations.
+const DefaultObservationWeight = 10
+
 // Remotes keeps track of remote addresses by weight, informed by
 // observations.
 type Remotes interface {
@@ -49,7 +53,7 @@ func NewRemotes(peers ...api.Peer) Remotes {
 	}
 
 	for _, peer := range peers {
-		mwr.Observe(peer, 1)
+		mwr.Observe(peer, DefaultObservationWeight)
 	}
 
 	return mwr
@@ -96,7 +100,7 @@ func (mwr *remotesWeightedRandom) Select(excludes ...string) (api.Peer, error) {
 
 	// bias to zero-weighted remotes have same probability. otherwise, we
 	// always select first entry when all are zero.
-	const bias = 0.1
+	const bias = 0.001
 
 	// clear out workspace
 	mwr.cdf = mwr.cdf[:0]
@@ -165,7 +169,7 @@ const (
 	// See
 	// https://en.wikipedia.org/wiki/Exponential_smoothing#Basic_exponential_smoothing
 	// for details.
-	remoteWeightSmoothingFactor = 0.7
+	remoteWeightSmoothingFactor = 0.5
 	remoteWeightMax             = 1 << 8
 )
 
@@ -228,7 +232,7 @@ func (p *Picker) Init(cc *grpc.ClientConn) error {
 	peer := p.peer
 	p.mu.Unlock()
 
-	p.r.ObserveIfExists(peer, 1)
+	p.r.ObserveIfExists(peer, DefaultObservationWeight)
 	c, err := grpc.NewConn(cc)
 	if err != nil {
 		return err
@@ -248,7 +252,7 @@ func (p *Picker) Pick(ctx context.Context) (transport.ClientTransport, error) {
 	p.mu.Unlock()
 	transport, err := p.conn.Wait(ctx)
 	if err != nil {
-		p.r.ObserveIfExists(peer, -1)
+		p.r.ObserveIfExists(peer, -DefaultObservationWeight)
 	}
 
 	return transport, err
@@ -261,7 +265,7 @@ func (p *Picker) PickAddr() (string, error) {
 	peer := p.peer
 	p.mu.Unlock()
 
-	p.r.ObserveIfExists(peer, -1) // downweight the current addr
+	p.r.ObserveIfExists(peer, -DefaultObservationWeight) // downweight the current addr
 
 	var err error
 	peer, err = p.r.Select()
@@ -299,15 +303,15 @@ func (p *Picker) WaitForStateChange(ctx context.Context, sourceState grpc.Connec
 	// TODO(stevvooe): This is questionable, but we'll see how it works.
 	switch state {
 	case grpc.Idle:
-		p.r.ObserveIfExists(peer, 1)
+		p.r.ObserveIfExists(peer, DefaultObservationWeight)
 	case grpc.Connecting:
-		p.r.ObserveIfExists(peer, 1)
+		p.r.ObserveIfExists(peer, DefaultObservationWeight)
 	case grpc.Ready:
-		p.r.ObserveIfExists(peer, 1)
+		p.r.ObserveIfExists(peer, DefaultObservationWeight)
 	case grpc.TransientFailure:
-		p.r.ObserveIfExists(peer, -1)
+		p.r.ObserveIfExists(peer, -DefaultObservationWeight)
 	case grpc.Shutdown:
-		p.r.ObserveIfExists(peer, -1)
+		p.r.ObserveIfExists(peer, -DefaultObservationWeight)
 	}
 
 	return state, err