Vendor swarmkit in master

Also, update libnetwork and grpc.

Signed-off-by: Aaron Lehmann <aaron.lehmann@docker.com>

daemon/cluster/cluster.go

@@ -26,8 +26,8 @@ import (
 	"github.com/docker/docker/pkg/ioutils"
 	"github.com/docker/docker/pkg/signal"
 	"github.com/docker/docker/runconfig"
-	swarmagent "github.com/docker/swarmkit/agent"
 	swarmapi "github.com/docker/swarmkit/api"
+	swarmnode "github.com/docker/swarmkit/node"
 	"golang.org/x/net/context"
 )
 
@@ -123,7 +123,7 @@ type attacher struct {
 }
 
 type node struct {
-	*swarmagent.Node
+	*swarmnode.Node
 	done           chan struct{}
 	ready          bool
 	conn           *grpc.ClientConn
@@ -279,7 +279,7 @@ func (c *Cluster) startNewNode(forceNewCluster bool, localAddr, remoteAddr, list
 	c.node = nil
 	c.cancelDelay = nil
 	c.stop = false
-	n, err := swarmagent.NewNode(&swarmagent.NodeConfig{
+	n, err := swarmnode.New(&swarmnode.Config{
 		Hostname:           c.config.Name,
 		ForceNewCluster:    forceNewCluster,
 		ListenControlAPI:   filepath.Join(c.runtimeRoot, controlSocket),

hack/vendor.sh

@@ -70,7 +70,7 @@ clone git github.com/RackSec/srslog 365bf33cd9acc21ae1c355209865f17228ca534e
 clone git github.com/imdario/mergo 0.2.1
 
 #get libnetwork packages
-clone git github.com/docker/libnetwork 04025f2a2eebb0d091883e55980dc6916d36842d
+clone git github.com/docker/libnetwork 9fbb4ecbb45af655c4ac3c2f3a849b2294cb447a
 clone git github.com/docker/go-events 18b43f1bc85d9cdd42c05a6cd2d444c7a200a894
 clone git github.com/armon/go-radix e39d623f12e8e41c7b5529e9a9dd67a1e2261f80
 clone git github.com/armon/go-metrics eb0af217e5e9747e41dd5303755356b62d28e3ec
@@ -101,7 +101,7 @@ clone git github.com/pborman/uuid v1.0
 # get desired notary commit, might also need to be updated in Dockerfile
 clone git github.com/docker/notary v0.4.2
 
-clone git google.golang.org/grpc v1.0.1-GA https://github.com/grpc/grpc-go.git
+clone git google.golang.org/grpc v1.0.2 https://github.com/grpc/grpc-go.git
 clone git github.com/miekg/pkcs11 df8ae6ca730422dba20c768ff38ef7d79077a59f
 clone git github.com/docker/go v1.5.1-1-1-gbaf439e
 clone git github.com/agl/ed25519 d2b94fd789ea21d12fac1a4443dd3a3f79cda72c
@@ -146,7 +146,7 @@ clone git github.com/docker/docker-credential-helpers v0.3.0
 clone git github.com/docker/containerd 52ef1ceb4b660c42cf4ea9013180a5663968d4c7
 
 # cluster
-clone git github.com/docker/swarmkit 1fed8d2a2ccd2a9b6d6fb864d4ad3461fc6dc3eb
+clone git github.com/docker/swarmkit 3b221eb0391d34ae0b9dac65df02b5b64de6dff2
 clone git github.com/golang/mock bd3c8e81be01eef76d4b503f5e687d2d1354d2d9
 clone git github.com/gogo/protobuf v0.3
 clone git github.com/cloudflare/cfssl 7fb22c8cba7ecaf98e4082d22d65800cf45e042a

vendor/src/github.com/docker/libnetwork/Makefile

@@ -8,6 +8,14 @@ ciargs = -e CIRCLECI -e "COVERALLS_TOKEN=$$COVERALLS_TOKEN" -e "INSIDECONTAINER=
 cidocker = docker run ${dockerargs} ${ciargs} $$EXTRA_ARGS ${container_env} ${build_image}
 CROSS_PLATFORMS = linux/amd64 linux/386 linux/arm windows/amd64
 export PATH := $(CURDIR)/bin:$(PATH)
+hostOS = ${shell go env GOHOSTOS}
+ifeq (${hostOS}, solaris)
+	gnufind=gfind
+	gnutail=gtail
+else
+	gnufind=find
+	gnutail=tail
+endif
 
 all: ${build_image}.created build check integration-tests clean
 
@@ -62,7 +70,40 @@ check-format:
 run-tests:
 	@echo "Running tests... "
 	@echo "mode: count" > coverage.coverprofile
-	@for dir in $$(find . -maxdepth 10 -not -path './.git*' -not -path '*/_*' -type d); do \
+	@for dir in $$( ${gnufind} . -maxdepth 10 -not -path './.git*' -not -path '*/_*' -type d); do \
+	    if [ ${hostOS} == solaris ]; then \
+	        case "$$dir" in \
+		    "./cmd/dnet" ) \
+		    ;& \
+		    "./cmd/ovrouter" ) \
+		    ;& \
+		    "./ns" ) \
+		    ;& \
+		    "./iptables" ) \
+		    ;& \
+		    "./ipvs" ) \
+		    ;& \
+		    "./drivers/bridge" ) \
+		    ;& \
+		    "./drivers/host" ) \
+		    ;& \
+		    "./drivers/ipvlan" ) \
+		    ;& \
+		    "./drivers/macvlan" ) \
+		    ;& \
+		    "./drivers/overlay" ) \
+		    ;& \
+		    "./drivers/remote" ) \
+		    ;& \
+		    "./drivers/windows" ) \
+			echo "Skipping $$dir on solaris host... "; \
+			continue; \
+			;; \
+		    * )\
+			echo "Entering $$dir ... "; \
+			;; \
+	        esac; \
+	    fi; \
 	    if ls $$dir/*.go &> /dev/null; then \
 		pushd . &> /dev/null ; \
 		cd $$dir ; \
@@ -71,7 +112,7 @@ run-tests:
 		if [ $$ret -ne 0 ]; then exit $$ret; fi ;\
 		popd &> /dev/null; \
 		if [ -f $$dir/profile.tmp ]; then \
-			cat $$dir/profile.tmp | tail -n +2 >> coverage.coverprofile ; \
+			cat $$dir/profile.tmp | ${gnutail} -n +2 >> coverage.coverprofile ; \
 				rm $$dir/profile.tmp ; \
 	    fi ; \
 	fi ; \

vendor/src/github.com/docker/libnetwork/cmd/proxy/tcp_proxy.go

@@ -2,11 +2,10 @@ package main
 
 import (
 	"io"
+	"log"
 	"net"
 	"sync"
 	"syscall"
-
-	"github.com/Sirupsen/logrus"
 )
 
 // TCPProxy is a proxy for TCP connections. It implements the Proxy interface to
@@ -35,7 +34,7 @@ func NewTCPProxy(frontendAddr, backendAddr *net.TCPAddr) (*TCPProxy, error) {
 func (proxy *TCPProxy) clientLoop(client *net.TCPConn, quit chan bool) {
 	backend, err := net.DialTCP("tcp", nil, proxy.backendAddr)
 	if err != nil {
-		logrus.Printf("Can't forward traffic to backend tcp/%v: %s\n", proxy.backendAddr, err)
+		log.Printf("Can't forward traffic to backend tcp/%v: %s\n", proxy.backendAddr, err)
 		client.Close()
 		return
 	}
@@ -79,7 +78,7 @@ func (proxy *TCPProxy) Run() {
 	for {
 		client, err := proxy.listener.Accept()
 		if err != nil {
-			logrus.Printf("Stopping proxy on tcp/%v for tcp/%v (%s)", proxy.frontendAddr, proxy.backendAddr, err)
+			log.Printf("Stopping proxy on tcp/%v for tcp/%v (%s)", proxy.frontendAddr, proxy.backendAddr, err)
 			return
 		}
 		go proxy.clientLoop(client.(*net.TCPConn), quit)

vendor/src/github.com/docker/libnetwork/cmd/proxy/udp_proxy.go

@@ -2,13 +2,12 @@ package main
 
 import (
 	"encoding/binary"
+	"log"
 	"net"
 	"strings"
 	"sync"
 	"syscall"
 	"time"
-
-	"github.com/Sirupsen/logrus"
 )
 
 const (
@@ -112,7 +111,7 @@ func (proxy *UDPProxy) Run() {
 			// ECONNREFUSED like Read do (see comment in
 			// UDPProxy.replyLoop)
 			if !isClosedError(err) {
-				logrus.Printf("Stopping proxy on udp/%v for udp/%v (%s)", proxy.frontendAddr, proxy.backendAddr, err)
+				log.Printf("Stopping proxy on udp/%v for udp/%v (%s)", proxy.frontendAddr, proxy.backendAddr, err)
 			}
 			break
 		}
@@ -123,7 +122,7 @@ func (proxy *UDPProxy) Run() {
 		if !hit {
 			proxyConn, err = net.DialUDP("udp", nil, proxy.backendAddr)
 			if err != nil {
-				logrus.Printf("Can't proxy a datagram to udp/%s: %s\n", proxy.backendAddr, err)
+				log.Printf("Can't proxy a datagram to udp/%s: %s\n", proxy.backendAddr, err)
 				proxy.connTrackLock.Unlock()
 				continue
 			}
@@ -134,7 +133,7 @@ func (proxy *UDPProxy) Run() {
 		for i := 0; i != read; {
 			written, err := proxyConn.Write(readBuf[i:read])
 			if err != nil {
-				logrus.Printf("Can't proxy a datagram to udp/%s: %s\n", proxy.backendAddr, err)
+				log.Printf("Can't proxy a datagram to udp/%s: %s\n", proxy.backendAddr, err)
 				break
 			}
 			i += written

vendor/src/github.com/docker/libnetwork/drivers_solaris.go

@@ -1,5 +1,13 @@
 package libnetwork
 
+import (
+	"github.com/docker/libnetwork/drivers/null"
+	"github.com/docker/libnetwork/drivers/solaris/bridge"
+)
+
 func getInitializers() []initializer {
-	return []initializer{}
+	return []initializer{
+		{bridge.Init, "bridge"},
+		{null.Init, "null"},
+	}
 }

vendor/src/github.com/docker/libnetwork/netutils/utils_solaris.go

@@ -1,13 +1,26 @@
-package netutils
+// +build solaris
 
-// Solaris: TODO
+package netutils
 
 import (
+	"fmt"
 	"net"
+	"os/exec"
+	"strings"
 
 	"github.com/docker/libnetwork/ipamutils"
+	"github.com/vishvananda/netlink"
+)
+
+var (
+	networkGetRoutesFct func(netlink.Link, int) ([]netlink.Route, error)
 )
 
+// CheckRouteOverlaps checks whether the passed network overlaps with any existing routes
+func CheckRouteOverlaps(toCheck *net.IPNet) error {
+	return nil
+}
+
 // ElectInterfaceAddresses looks for an interface on the OS with the specified name
 // and returns its IPv4 and IPv6 addresses in CIDR form. If the interface does not exist,
 // it chooses from a predifined list the first IPv4 address which does not conflict
@@ -15,18 +28,75 @@ import (
 func ElectInterfaceAddresses(name string) (*net.IPNet, []*net.IPNet, error) {
 	var (
 		v4Net *net.IPNet
-		err   error
 	)
 
-	v4Net, err = FindAvailableNetwork(ipamutils.PredefinedBroadNetworks)
+	out, err := exec.Command("/usr/sbin/ipadm", "show-addr",
+		"-p", "-o", "addrobj,addr").Output()
 	if err != nil {
+		fmt.Println("failed to list interfaces on system")
 		return nil, nil, err
 	}
+	alist := strings.Fields(string(out))
+	for _, a := range alist {
+		linkandaddr := strings.SplitN(a, ":", 2)
+		if len(linkandaddr) != 2 {
+			fmt.Println("failed to check interfaces on system: ", a)
+			continue
+		}
+		gw := fmt.Sprintf("%s_gw0", name)
+		link := strings.Split(linkandaddr[0], "/")[0]
+		addr := linkandaddr[1]
+		if gw != link {
+			continue
+		}
+		_, ipnet, err := net.ParseCIDR(addr)
+		if err != nil {
+			fmt.Println("failed to parse address: ", addr)
+			continue
+		}
+		v4Net = ipnet
+		break
+	}
+	if v4Net == nil {
+		v4Net, err = FindAvailableNetwork(ipamutils.PredefinedBroadNetworks)
+		if err != nil {
+			return nil, nil, err
+		}
+	}
 	return v4Net, nil, nil
 }
 
 // FindAvailableNetwork returns a network from the passed list which does not
 // overlap with existing interfaces in the system
 func FindAvailableNetwork(list []*net.IPNet) (*net.IPNet, error) {
-	return list[0], nil
+	out, err := exec.Command("/usr/sbin/ipadm", "show-addr",
+		"-p", "-o", "addr").Output()
+
+	if err != nil {
+		fmt.Println("failed to list interfaces on system")
+		return nil, err
+	}
+	ipaddrs := strings.Fields(string(out))
+	inuse := []*net.IPNet{}
+	for _, ip := range ipaddrs {
+		_, ipnet, err := net.ParseCIDR(ip)
+		if err != nil {
+			fmt.Println("failed to check interfaces on system: ", ip)
+			continue
+		}
+		inuse = append(inuse, ipnet)
+	}
+	for _, avail := range list {
+		is_avail := true
+		for _, ipnet := range inuse {
+			if NetworkOverlaps(avail, ipnet) {
+				is_avail = false
+				break
+			}
+		}
+		if is_avail {
+			return avail, nil
+		}
+	}
+	return nil, fmt.Errorf("no available network")
 }

vendor/src/github.com/docker/libnetwork/networkdb/cluster.go

@@ -270,19 +270,27 @@ func (nDB *NetworkDB) reconnectNode() {
 	nDB.bulkSync([]string{node.Name}, true)
 }
 
+// For timing the entry deletion in the repaer APIs that doesn't use monotonic clock
+// source (time.Now, Sub etc.) should be avoided. Hence we use reapTime in every
+// entry which is set initially to reapInterval and decremented by reapPeriod every time
+// the reaper runs. NOTE nDB.reapTableEntries updates the reapTime with a readlock. This
+// is safe as long as no other concurrent path touches the reapTime field.
 func (nDB *NetworkDB) reapState() {
 	nDB.reapNetworks()
 	nDB.reapTableEntries()
 }
 
 func (nDB *NetworkDB) reapNetworks() {
-	now := time.Now()
 	nDB.Lock()
 	for name, nn := range nDB.networks {
 		for id, n := range nn {
-			if n.leaving && now.Sub(n.leaveTime) > reapInterval {
-				delete(nn, id)
-				nDB.deleteNetworkNode(id, name)
+			if n.leaving {
+				if n.reapTime <= 0 {
+					delete(nn, id)
+					nDB.deleteNetworkNode(id, name)
+					continue
+				}
+				n.reapTime -= reapPeriod
 			}
 		}
 	}
@@ -292,8 +300,6 @@ func (nDB *NetworkDB) reapNetworks() {
 func (nDB *NetworkDB) reapTableEntries() {
 	var paths []string
 
-	now := time.Now()
-
 	nDB.RLock()
 	nDB.indexes[byTable].Walk(func(path string, v interface{}) bool {
 		entry, ok := v.(*entry)
@@ -301,10 +307,13 @@ func (nDB *NetworkDB) reapTableEntries() {
 			return false
 		}
 
-		if !entry.deleting || now.Sub(entry.deleteTime) <= reapInterval {
+		if !entry.deleting {
+			return false
+		}
+		if entry.reapTime > 0 {
+			entry.reapTime -= reapPeriod
 			return false
 		}
-
 		paths = append(paths, path)
 		return false
 	})

vendor/src/github.com/docker/libnetwork/networkdb/delegate.go

@@ -4,7 +4,6 @@ import (
 	"fmt"
 	"net"
 	"strings"
-	"time"
 
 	"github.com/Sirupsen/logrus"
 	"github.com/gogo/protobuf/proto"
@@ -121,7 +120,7 @@ func (nDB *NetworkDB) handleNetworkEvent(nEvent *NetworkEvent) bool {
 		n.ltime = nEvent.LTime
 		n.leaving = nEvent.Type == NetworkEventTypeLeave
 		if n.leaving {
-			n.leaveTime = time.Now()
+			n.reapTime = reapInterval
 		}
 
 		nDB.addNetworkNode(nEvent.NetworkID, nEvent.NodeName)
@@ -178,7 +177,7 @@ func (nDB *NetworkDB) handleTableEvent(tEvent *TableEvent) bool {
 	}
 
 	if e.deleting {
-		e.deleteTime = time.Now()
+		e.reapTime = reapInterval
 	}
 
 	nDB.Lock()

vendor/src/github.com/docker/libnetwork/networkdb/networkdb.go

@@ -107,8 +107,9 @@ type network struct {
 	// Node leave is in progress.
 	leaving bool
 
-	// The time this node knew about the node's network leave.
-	leaveTime time.Time
+	// Number of seconds still left before a deleted network entry gets
+	// removed from networkDB
+	reapTime time.Duration
 
 	// The broadcast queue for table event gossip. This is only
 	// initialized for this node's network attachment entries.
@@ -153,8 +154,9 @@ type entry struct {
 	// the cluster for certain amount of time after deletion.
 	deleting bool
 
-	// The wall clock time when this node learned about this deletion.
-	deleteTime time.Time
+	// Number of seconds still left before a deleted table entry gets
+	// removed from networkDB
+	reapTime time.Duration
 }
 
 // New creates a new instance of NetworkDB using the Config passed by
@@ -286,11 +288,11 @@ func (nDB *NetworkDB) DeleteEntry(tname, nid, key string) error {
 	}
 
 	entry := &entry{
-		ltime:      nDB.tableClock.Increment(),
-		node:       nDB.config.NodeName,
-		value:      value,
-		deleting:   true,
-		deleteTime: time.Now(),
+		ltime:    nDB.tableClock.Increment(),
+		node:     nDB.config.NodeName,
+		value:    value,
+		deleting: true,
+		reapTime: reapInterval,
 	}
 
 	if err := nDB.sendTableEvent(TableEventTypeDelete, nid, tname, key, entry); err != nil {
@@ -339,11 +341,11 @@ func (nDB *NetworkDB) deleteNodeTableEntries(node string) {
 		key := params[2]
 
 		entry := &entry{
-			ltime:      oldEntry.ltime,
-			node:       node,
-			value:      oldEntry.value,
-			deleting:   true,
-			deleteTime: time.Now(),
+			ltime:    oldEntry.ltime,
+			node:     node,
+			value:    oldEntry.value,
+			deleting: true,
+			reapTime: reapInterval,
 		}
 
 		nDB.indexes[byTable].Insert(fmt.Sprintf("/%s/%s/%s", tname, nid, key), entry)

vendor/src/github.com/docker/libnetwork/ns/init_linux.go

@@ -69,8 +69,10 @@ func NlHandle() *netlink.Handle {
 func getSupportedNlFamilies() []int {
 	fams := []int{syscall.NETLINK_ROUTE}
 	if err := loadXfrmModules(); err != nil {
-		log.Warnf("Could not load necessary modules for IPSEC rules: %v", err)
-		return fams
+		if checkXfrmSocket() != nil {
+			log.Warnf("Could not load necessary modules for IPSEC rules: %v", err)
+			return fams
+		}
 	}
 	return append(fams, syscall.NETLINK_XFRM)
 }
@@ -84,3 +86,13 @@ func loadXfrmModules() error {
 	}
 	return nil
 }
+
+// API check on required xfrm modules (xfrm_user, xfrm_algo)
+func checkXfrmSocket() error {
+	fd, err := syscall.Socket(syscall.AF_NETLINK, syscall.SOCK_RAW, syscall.NETLINK_XFRM)
+	if err != nil {
+		return err
+	}
+	syscall.Close(fd)
+	return nil
+}

vendor/src/github.com/docker/libnetwork/osl/sandbox_solaris.go

@@ -0,0 +1,24 @@
+package osl
+
+// NewSandbox provides a new sandbox instance created in an os specific way
+// provided a key which uniquely identifies the sandbox
+func NewSandbox(key string, osCreate, isRestore bool) (Sandbox, error) {
+	return nil, nil
+}
+
+// GenerateKey generates a sandbox key based on the passed
+// container id.
+func GenerateKey(containerID string) string {
+	maxLen := 12
+
+	if len(containerID) < maxLen {
+		maxLen = len(containerID)
+	}
+
+	return containerID[:maxLen]
+}
+
+// InitOSContext initializes OS context while configuring network resources
+func InitOSContext() func() {
+	return func() {}
+}

vendor/src/github.com/docker/libnetwork/osl/sandbox_unsupported.go

@@ -1,4 +1,4 @@
-// +build !linux,!windows,!freebsd
+// +build !linux,!windows,!freebsd,!solaris
 
 package osl
 

vendor/src/github.com/docker/libnetwork/portallocator/portallocator.go

@@ -1,11 +1,9 @@
 package portallocator
 
 import (
-	"bufio"
 	"errors"
 	"fmt"
 	"net"
-	"os"
 	"sync"
 )
 
@@ -106,26 +104,6 @@ func newInstance() *PortAllocator {
 	}
 }
 
-func getDynamicPortRange() (start int, end int, err error) {
-	const portRangeKernelParam = "/proc/sys/net/ipv4/ip_local_port_range"
-	portRangeFallback := fmt.Sprintf("using fallback port range %d-%d", DefaultPortRangeStart, DefaultPortRangeEnd)
-	file, err := os.Open(portRangeKernelParam)
-	if err != nil {
-		return 0, 0, fmt.Errorf("port allocator - %s due to error: %v", portRangeFallback, err)
-	}
-
-	defer file.Close()
-
-	n, err := fmt.Fscanf(bufio.NewReader(file), "%d\t%d", &start, &end)
-	if n != 2 || err != nil {
-		if err == nil {
-			err = fmt.Errorf("unexpected count of parsed numbers (%d)", n)
-		}
-		return 0, 0, fmt.Errorf("port allocator - failed to parse system ephemeral port range from %s - %s: %v", portRangeKernelParam, portRangeFallback, err)
-	}
-	return start, end, nil
-}
-
 // RequestPort requests new port from global ports pool for specified ip and proto.
 // If port is 0 it returns first free port. Otherwise it checks port availability
 // in proto's pool and returns that port or error if port is already busy.

vendor/src/github.com/docker/libnetwork/portallocator/portallocator_linux.go

@@ -0,0 +1,27 @@
+package portallocator
+
+import (
+	"bufio"
+	"fmt"
+	"os"
+)
+
+func getDynamicPortRange() (start int, end int, err error) {
+	const portRangeKernelParam = "/proc/sys/net/ipv4/ip_local_port_range"
+	portRangeFallback := fmt.Sprintf("using fallback port range %d-%d", DefaultPortRangeStart, DefaultPortRangeEnd)
+	file, err := os.Open(portRangeKernelParam)
+	if err != nil {
+		return 0, 0, fmt.Errorf("port allocator - %s due to error: %v", portRangeFallback, err)
+	}
+
+	defer file.Close()
+
+	n, err := fmt.Fscanf(bufio.NewReader(file), "%d\t%d", &start, &end)
+	if n != 2 || err != nil {
+		if err == nil {
+			err = fmt.Errorf("unexpected count of parsed numbers (%d)", n)
+		}
+		return 0, 0, fmt.Errorf("port allocator - failed to parse system ephemeral port range from %s - %s: %v", portRangeKernelParam, portRangeFallback, err)
+	}
+	return start, end, nil
+}

vendor/src/github.com/docker/libnetwork/portallocator/portallocator_solaris.go

@@ -0,0 +1,5 @@
+package portallocator
+
+func getDynamicPortRange() (start int, end int, err error) {
+	return 32768, 65535, nil
+}

vendor/src/github.com/docker/libnetwork/portmapper/proxy.go

@@ -7,8 +7,6 @@ import (
 	"net"
 	"os"
 	"os/exec"
-	"strconv"
-	"syscall"
 	"time"
 )
 
@@ -25,36 +23,6 @@ type proxyCommand struct {
 	cmd *exec.Cmd
 }
 
-func newProxyCommand(proto string, hostIP net.IP, hostPort int, containerIP net.IP, containerPort int, proxyPath string) (userlandProxy, error) {
-	path := proxyPath
-	if proxyPath == "" {
-		cmd, err := exec.LookPath(userlandProxyCommandName)
-		if err != nil {
-			return nil, err
-		}
-		path = cmd
-	}
-
-	args := []string{
-		path,
-		"-proto", proto,
-		"-host-ip", hostIP.String(),
-		"-host-port", strconv.Itoa(hostPort),
-		"-container-ip", containerIP.String(),
-		"-container-port", strconv.Itoa(containerPort),
-	}
-
-	return &proxyCommand{
-		cmd: &exec.Cmd{
-			Path: path,
-			Args: args,
-			SysProcAttr: &syscall.SysProcAttr{
-				Pdeathsig: syscall.SIGTERM, // send a sigterm to the proxy if the daemon process dies
-			},
-		},
-	}, nil
-}
-
 func (p *proxyCommand) Start() error {
 	r, w, err := os.Pipe()
 	if err != nil {

vendor/src/github.com/docker/libnetwork/portmapper/proxy_linux.go

@@ -0,0 +1,38 @@
+package portmapper
+
+import (
+	"net"
+	"os/exec"
+	"strconv"
+	"syscall"
+)
+
+func newProxyCommand(proto string, hostIP net.IP, hostPort int, containerIP net.IP, containerPort int, proxyPath string) (userlandProxy, error) {
+	path := proxyPath
+	if proxyPath == "" {
+		cmd, err := exec.LookPath(userlandProxyCommandName)
+		if err != nil {
+			return nil, err
+		}
+		path = cmd
+	}
+
+	args := []string{
+		path,
+		"-proto", proto,
+		"-host-ip", hostIP.String(),
+		"-host-port", strconv.Itoa(hostPort),
+		"-container-ip", containerIP.String(),
+		"-container-port", strconv.Itoa(containerPort),
+	}
+
+	return &proxyCommand{
+		cmd: &exec.Cmd{
+			Path: path,
+			Args: args,
+			SysProcAttr: &syscall.SysProcAttr{
+				Pdeathsig: syscall.SIGTERM, // send a sigterm to the proxy if the daemon process dies
+			},
+		},
+	}, nil
+}

vendor/src/github.com/docker/libnetwork/portmapper/proxy_solaris.go

@@ -0,0 +1,34 @@
+package portmapper
+
+import (
+	"net"
+	"os/exec"
+	"strconv"
+)
+
+func newProxyCommand(proto string, hostIP net.IP, hostPort int, containerIP net.IP, containerPort int, proxyPath string) (userlandProxy, error) {
+	path := proxyPath
+	if proxyPath == "" {
+		cmd, err := exec.LookPath(userlandProxyCommandName)
+		if err != nil {
+			return nil, err
+		}
+		path = cmd
+	}
+
+	args := []string{
+		path,
+		"-proto", proto,
+		"-host-ip", hostIP.String(),
+		"-host-port", strconv.Itoa(hostPort),
+		"-container-ip", containerIP.String(),
+		"-container-port", strconv.Itoa(containerPort),
+	}
+
+	return &proxyCommand{
+		cmd: &exec.Cmd{
+			Path: path,
+			Args: args,
+		},
+	}, nil
+}

vendor/src/github.com/docker/libnetwork/sandbox.go

@@ -848,8 +848,9 @@ func (sb *sandbox) clearNetworkResources(origEp *endpoint) error {
 		releaseOSSboxResources(osSbox, ep)
 	}
 
-	delete(sb.populatedEndpoints, ep.ID())
 	sb.Lock()
+	delete(sb.populatedEndpoints, ep.ID())
+
 	if len(sb.endpoints) == 0 {
 		// sb.endpoints should never be empty and this is unexpected error condition
 		// We log an error message to note this down for debugging purposes.

vendor/src/github.com/docker/swarmkit/agent/errors.go

@@ -10,8 +10,6 @@ var (
 	ErrClosed = errors.New("agent: closed")
 
 	errNodeNotRegistered = fmt.Errorf("node not registered")
-	errNodeStarted       = errors.New("node: already started")
-	errNodeNotStarted    = errors.New("node: not started")
 
 	errAgentStarted    = errors.New("agent: already started")
 	errAgentNotStarted = errors.New("agent: not started")

vendor/src/github.com/docker/swarmkit/agent/resource.go

@@ -16,7 +16,7 @@ type ResourceAllocator interface {
 	// given a target network and a unique ID representing the
 	// connecting entity and optionally a list of ipv4/ipv6
 	// addresses to be assigned to the attachment. AttachNetwork
-	// returns a unique ID for the attachment if successfull or an
+	// returns a unique ID for the attachment if successful or an
 	// error in case of failure.
 	AttachNetwork(ctx context.Context, id, target string, addresses []string) (string, error)
 

vendor/src/github.com/docker/swarmkit/agent/session.go

@@ -76,7 +76,13 @@ func newSession(ctx context.Context, agent *Agent, delay time.Duration, sessionI
 }
 
 func (s *session) run(ctx context.Context, delay time.Duration, description *api.NodeDescription) {
-	time.Sleep(delay) // delay before registering.
+	timer := time.NewTimer(delay) // delay before registering.
+	defer timer.Stop()
+	select {
+	case <-timer.C:
+	case <-ctx.Done():
+		return
+	}
 
 	if err := s.start(ctx, description); err != nil {
 		select {

vendor/src/github.com/docker/swarmkit/api/control.pb.go

@@ -831,16 +831,12 @@ func (m *ListNodesRequest_Filters) Copy() *ListNodesRequest_Filters {
 
 	if m.Names != nil {
 		o.Names = make([]string, 0, len(m.Names))
-		for _, v := range m.Names {
-			o.Names = append(o.Names, v)
-		}
+		o.Names = append(o.Names, m.Names...)
 	}
 
 	if m.IDPrefixes != nil {
 		o.IDPrefixes = make([]string, 0, len(m.IDPrefixes))
-		for _, v := range m.IDPrefixes {
-			o.IDPrefixes = append(o.IDPrefixes, v)
-		}
+		o.IDPrefixes = append(o.IDPrefixes, m.IDPrefixes...)
 	}
 
 	if m.Labels != nil {
@@ -852,23 +848,17 @@ func (m *ListNodesRequest_Filters) Copy() *ListNodesRequest_Filters {
 
 	if m.Memberships != nil {
 		o.Memberships = make([]NodeSpec_Membership, 0, len(m.Memberships))
-		for _, v := range m.Memberships {
-			o.Memberships = append(o.Memberships, v)
-		}
+		o.Memberships = append(o.Memberships, m.Memberships...)
 	}
 
 	if m.Roles != nil {
 		o.Roles = make([]NodeRole, 0, len(m.Roles))
-		for _, v := range m.Roles {
-			o.Roles = append(o.Roles, v)
-		}
+		o.Roles = append(o.Roles, m.Roles...)
 	}
 
 	if m.NamePrefixes != nil {
 		o.NamePrefixes = make([]string, 0, len(m.NamePrefixes))
-		for _, v := range m.NamePrefixes {
-			o.NamePrefixes = append(o.NamePrefixes, v)
-		}
+		o.NamePrefixes = append(o.NamePrefixes, m.NamePrefixes...)
 	}
 
 	return o
@@ -1007,16 +997,12 @@ func (m *ListTasksRequest_Filters) Copy() *ListTasksRequest_Filters {
 
 	if m.Names != nil {
 		o.Names = make([]string, 0, len(m.Names))
-		for _, v := range m.Names {
-			o.Names = append(o.Names, v)
-		}
+		o.Names = append(o.Names, m.Names...)
 	}
 
 	if m.IDPrefixes != nil {
 		o.IDPrefixes = make([]string, 0, len(m.IDPrefixes))
-		for _, v := range m.IDPrefixes {
-			o.IDPrefixes = append(o.IDPrefixes, v)
-		}
+		o.IDPrefixes = append(o.IDPrefixes, m.IDPrefixes...)
 	}
 
 	if m.Labels != nil {
@@ -1028,30 +1014,22 @@ func (m *ListTasksRequest_Filters) Copy() *ListTasksRequest_Filters {
 
 	if m.ServiceIDs != nil {
 		o.ServiceIDs = make([]string, 0, len(m.ServiceIDs))
-		for _, v := range m.ServiceIDs {
-			o.ServiceIDs = append(o.ServiceIDs, v)
-		}
+		o.ServiceIDs = append(o.ServiceIDs, m.ServiceIDs...)
 	}
 
 	if m.NodeIDs != nil {
 		o.NodeIDs = make([]string, 0, len(m.NodeIDs))
-		for _, v := range m.NodeIDs {
-			o.NodeIDs = append(o.NodeIDs, v)
-		}
+		o.NodeIDs = append(o.NodeIDs, m.NodeIDs...)
 	}
 
 	if m.DesiredStates != nil {
 		o.DesiredStates = make([]TaskState, 0, len(m.DesiredStates))
-		for _, v := range m.DesiredStates {
-			o.DesiredStates = append(o.DesiredStates, v)
-		}
+		o.DesiredStates = append(o.DesiredStates, m.DesiredStates...)
 	}
 
 	if m.NamePrefixes != nil {
 		o.NamePrefixes = make([]string, 0, len(m.NamePrefixes))
-		for _, v := range m.NamePrefixes {
-			o.NamePrefixes = append(o.NamePrefixes, v)
-		}
+		o.NamePrefixes = append(o.NamePrefixes, m.NamePrefixes...)
 	}
 
 	return o
@@ -1191,16 +1169,12 @@ func (m *ListServicesRequest_Filters) Copy() *ListServicesRequest_Filters {
 
 	if m.Names != nil {
 		o.Names = make([]string, 0, len(m.Names))
-		for _, v := range m.Names {
-			o.Names = append(o.Names, v)
-		}
+		o.Names = append(o.Names, m.Names...)
 	}
 
 	if m.IDPrefixes != nil {
 		o.IDPrefixes = make([]string, 0, len(m.IDPrefixes))
-		for _, v := range m.IDPrefixes {
-			o.IDPrefixes = append(o.IDPrefixes, v)
-		}
+		o.IDPrefixes = append(o.IDPrefixes, m.IDPrefixes...)
 	}
 
 	if m.Labels != nil {
@@ -1212,9 +1186,7 @@ func (m *ListServicesRequest_Filters) Copy() *ListServicesRequest_Filters {
 
 	if m.NamePrefixes != nil {
 		o.NamePrefixes = make([]string, 0, len(m.NamePrefixes))
-		for _, v := range m.NamePrefixes {
-			o.NamePrefixes = append(o.NamePrefixes, v)
-		}
+		o.NamePrefixes = append(o.NamePrefixes, m.NamePrefixes...)
 	}
 
 	return o
@@ -1330,16 +1302,12 @@ func (m *ListNetworksRequest_Filters) Copy() *ListNetworksRequest_Filters {
 
 	if m.Names != nil {
 		o.Names = make([]string, 0, len(m.Names))
-		for _, v := range m.Names {
-			o.Names = append(o.Names, v)
-		}
+		o.Names = append(o.Names, m.Names...)
 	}
 
 	if m.IDPrefixes != nil {
 		o.IDPrefixes = make([]string, 0, len(m.IDPrefixes))
-		for _, v := range m.IDPrefixes {
-			o.IDPrefixes = append(o.IDPrefixes, v)
-		}
+		o.IDPrefixes = append(o.IDPrefixes, m.IDPrefixes...)
 	}
 
 	if m.Labels != nil {
@@ -1351,9 +1319,7 @@ func (m *ListNetworksRequest_Filters) Copy() *ListNetworksRequest_Filters {
 
 	if m.NamePrefixes != nil {
 		o.NamePrefixes = make([]string, 0, len(m.NamePrefixes))
-		for _, v := range m.NamePrefixes {
-			o.NamePrefixes = append(o.NamePrefixes, v)
-		}
+		o.NamePrefixes = append(o.NamePrefixes, m.NamePrefixes...)
 	}
 
 	return o
@@ -1421,16 +1387,12 @@ func (m *ListClustersRequest_Filters) Copy() *ListClustersRequest_Filters {
 
 	if m.Names != nil {
 		o.Names = make([]string, 0, len(m.Names))
-		for _, v := range m.Names {
-			o.Names = append(o.Names, v)
-		}
+		o.Names = append(o.Names, m.Names...)
 	}
 
 	if m.IDPrefixes != nil {
 		o.IDPrefixes = make([]string, 0, len(m.IDPrefixes))
-		for _, v := range m.IDPrefixes {
-			o.IDPrefixes = append(o.IDPrefixes, v)
-		}
+		o.IDPrefixes = append(o.IDPrefixes, m.IDPrefixes...)
 	}
 
 	if m.Labels != nil {
@@ -1442,9 +1404,7 @@ func (m *ListClustersRequest_Filters) Copy() *ListClustersRequest_Filters {
 
 	if m.NamePrefixes != nil {
 		o.NamePrefixes = make([]string, 0, len(m.NamePrefixes))
-		for _, v := range m.NamePrefixes {
-			o.NamePrefixes = append(o.NamePrefixes, v)
-		}
+		o.NamePrefixes = append(o.NamePrefixes, m.NamePrefixes...)
 	}
 
 	return o
@@ -1552,16 +1512,12 @@ func (m *ListSecretsRequest_Filters) Copy() *ListSecretsRequest_Filters {
 
 	if m.Names != nil {
 		o.Names = make([]string, 0, len(m.Names))
-		for _, v := range m.Names {
-			o.Names = append(o.Names, v)
-		}
+		o.Names = append(o.Names, m.Names...)
 	}
 
 	if m.IDPrefixes != nil {
 		o.IDPrefixes = make([]string, 0, len(m.IDPrefixes))
-		for _, v := range m.IDPrefixes {
-			o.IDPrefixes = append(o.IDPrefixes, v)
-		}
+		o.IDPrefixes = append(o.IDPrefixes, m.IDPrefixes...)
 	}
 
 	if m.Labels != nil {
@@ -1573,9 +1529,7 @@ func (m *ListSecretsRequest_Filters) Copy() *ListSecretsRequest_Filters {
 
 	if m.NamePrefixes != nil {
 		o.NamePrefixes = make([]string, 0, len(m.NamePrefixes))
-		for _, v := range m.NamePrefixes {
-			o.NamePrefixes = append(o.NamePrefixes, v)
-		}
+		o.NamePrefixes = append(o.NamePrefixes, m.NamePrefixes...)
 	}
 
 	return o

vendor/src/github.com/docker/swarmkit/api/objects.pb.go

@@ -227,6 +227,9 @@ type Cluster struct {
 	// and agents to unambiguously identify the older key to be deleted when
 	// a new key is allocated on key rotation.
 	EncryptionKeyLamportClock uint64 `protobuf:"varint,6,opt,name=encryption_key_lamport_clock,json=encryptionKeyLamportClock,proto3" json:"encryption_key_lamport_clock,omitempty"`
+	// RemovedNodes is the list of nodes that have been removed from the
+	// swarm. Their certificates should effectively be blacklisted.
+	RemovedNodes []*RemovedNode `protobuf:"bytes,7,rep,name=removed_nodes,json=removedNodes" json:"removed_nodes,omitempty"`
 }
 
 func (m *Cluster) Reset()                    { *m = Cluster{} }
@@ -401,16 +404,12 @@ func (m *NetworkAttachment) Copy() *NetworkAttachment {
 
 	if m.Addresses != nil {
 		o.Addresses = make([]string, 0, len(m.Addresses))
-		for _, v := range m.Addresses {
-			o.Addresses = append(o.Addresses, v)
-		}
+		o.Addresses = append(o.Addresses, m.Addresses...)
 	}
 
 	if m.Aliases != nil {
 		o.Aliases = make([]string, 0, len(m.Aliases))
-		for _, v := range m.Aliases {
-			o.Aliases = append(o.Aliases, v)
-		}
+		o.Aliases = append(o.Aliases, m.Aliases...)
 	}
 
 	return o
@@ -452,6 +451,13 @@ func (m *Cluster) Copy() *Cluster {
 		}
 	}
 
+	if m.RemovedNodes != nil {
+		o.RemovedNodes = make([]*RemovedNode, 0, len(m.RemovedNodes))
+		for _, v := range m.RemovedNodes {
+			o.RemovedNodes = append(o.RemovedNodes, v.Copy())
+		}
+	}
+
 	return o
 }
 
@@ -625,7 +631,7 @@ func (this *Cluster) GoString() string {
 	if this == nil {
 		return "nil"
 	}
-	s := make([]string, 0, 10)
+	s := make([]string, 0, 11)
 	s = append(s, "&api.Cluster{")
 	s = append(s, "ID: "+fmt.Sprintf("%#v", this.ID)+",\n")
 	s = append(s, "Meta: "+strings.Replace(this.Meta.GoString(), `&`, ``, 1)+",\n")
@@ -635,6 +641,9 @@ func (this *Cluster) GoString() string {
 		s = append(s, "NetworkBootstrapKeys: "+fmt.Sprintf("%#v", this.NetworkBootstrapKeys)+",\n")
 	}
 	s = append(s, "EncryptionKeyLamportClock: "+fmt.Sprintf("%#v", this.EncryptionKeyLamportClock)+",\n")
+	if this.RemovedNodes != nil {
+		s = append(s, "RemovedNodes: "+fmt.Sprintf("%#v", this.RemovedNodes)+",\n")
+	}
 	s = append(s, "}")
 	return strings.Join(s, "")
 }
@@ -1261,6 +1270,18 @@ func (m *Cluster) MarshalTo(data []byte) (int, error) {
 		i++
 		i = encodeVarintObjects(data, i, uint64(m.EncryptionKeyLamportClock))
 	}
+	if len(m.RemovedNodes) > 0 {
+		for _, msg := range m.RemovedNodes {
+			data[i] = 0x3a
+			i++
+			i = encodeVarintObjects(data, i, uint64(msg.Size()))
+			n, err := msg.MarshalTo(data[i:])
+			if err != nil {
+				return 0, err
+			}
+			i += n
+		}
+	}
 	return i, nil
 }
 
@@ -1575,6 +1596,12 @@ func (m *Cluster) Size() (n int) {
 	if m.EncryptionKeyLamportClock != 0 {
 		n += 1 + sovObjects(uint64(m.EncryptionKeyLamportClock))
 	}
+	if len(m.RemovedNodes) > 0 {
+		for _, e := range m.RemovedNodes {
+			l = e.Size()
+			n += 1 + l + sovObjects(uint64(l))
+		}
+	}
 	return n
 }
 
@@ -1741,6 +1768,7 @@ func (this *Cluster) String() string {
 		`RootCA:` + strings.Replace(strings.Replace(this.RootCA.String(), "RootCA", "RootCA", 1), `&`, ``, 1) + `,`,
 		`NetworkBootstrapKeys:` + strings.Replace(fmt.Sprintf("%v", this.NetworkBootstrapKeys), "EncryptionKey", "EncryptionKey", 1) + `,`,
 		`EncryptionKeyLamportClock:` + fmt.Sprintf("%v", this.EncryptionKeyLamportClock) + `,`,
+		`RemovedNodes:` + strings.Replace(fmt.Sprintf("%v", this.RemovedNodes), "RemovedNode", "RemovedNode", 1) + `,`,
 		`}`,
 	}, "")
 	return s
@@ -3669,6 +3697,37 @@ func (m *Cluster) Unmarshal(data []byte) error {
 					break
 				}
 			}
+		case 7:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field RemovedNodes", wireType)
+			}
+			var msglen int
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowObjects
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := data[iNdEx]
+				iNdEx++
+				msglen |= (int(b) & 0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			if msglen < 0 {
+				return ErrInvalidLengthObjects
+			}
+			postIndex := iNdEx + msglen
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			m.RemovedNodes = append(m.RemovedNodes, &RemovedNode{})
+			if err := m.RemovedNodes[len(m.RemovedNodes)-1].Unmarshal(data[iNdEx:postIndex]); err != nil {
+				return err
+			}
+			iNdEx = postIndex
 		default:
 			iNdEx = preIndex
 			skippy, err := skipObjects(data[iNdEx:])
@@ -4005,74 +4064,76 @@ var (
 func init() { proto.RegisterFile("objects.proto", fileDescriptorObjects) }
 
 var fileDescriptorObjects = []byte{
-	// 1101 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xbc, 0x57, 0x4f, 0x6f, 0x1b, 0xc5,
-	0x1b, 0xee, 0xda, 0x5b, 0xdb, 0xfb, 0x3a, 0x8e, 0xf4, 0x9b, 0x5f, 0x15, 0x6d, 0x43, 0xb0, 0x83,
-	0x2b, 0x50, 0x0f, 0x95, 0x2b, 0x4a, 0x41, 0xad, 0x68, 0x85, 0xfc, 0x27, 0x02, 0xab, 0x04, 0xa2,
-	0x49, 0x49, 0x8f, 0xab, 0xc9, 0xee, 0xd4, 0x0c, 0xb6, 0x77, 0x56, 0x33, 0x63, 0x57, 0xe9, 0x09,
-	0xf1, 0x01, 0xf8, 0x08, 0x7c, 0x15, 0xae, 0x39, 0x70, 0xe0, 0x06, 0x27, 0x8b, 0xf8, 0x80, 0xc4,
-	0x09, 0x3e, 0x02, 0x9a, 0xd9, 0x59, 0xc7, 0x91, 0xd7, 0x21, 0x95, 0xaa, 0xdc, 0xe6, 0xf5, 0x3c,
-	0xcf, 0xf3, 0xfe, 0x99, 0x77, 0xde, 0x59, 0x43, 0x8d, 0x1f, 0x7f, 0x47, 0x43, 0x25, 0x5b, 0x89,
-	0xe0, 0x8a, 0x23, 0x14, 0xf1, 0x70, 0x48, 0x45, 0x4b, 0xbe, 0x22, 0x62, 0x3c, 0x64, 0xaa, 0x35,
-	0xfd, 0x70, 0xbb, 0xaa, 0x4e, 0x12, 0x6a, 0x01, 0xdb, 0x55, 0x99, 0xd0, 0x30, 0x33, 0x6e, 0x2b,
-	0x36, 0xa6, 0x52, 0x91, 0x71, 0x72, 0x7f, 0xb1, 0xb2, 0x5b, 0xb7, 0x06, 0x7c, 0xc0, 0xcd, 0xf2,
-	0xbe, 0x5e, 0xa5, 0xbf, 0x36, 0x7f, 0x76, 0xc0, 0xdd, 0xa7, 0x8a, 0xa0, 0x4f, 0xa1, 0x3c, 0xa5,
-	0x42, 0x32, 0x1e, 0xfb, 0xce, 0xae, 0x73, 0xb7, 0xfa, 0xe0, 0x9d, 0xd6, 0xaa, 0xe7, 0xd6, 0x51,
-	0x0a, 0xe9, 0xb8, 0xa7, 0xb3, 0xc6, 0x0d, 0x9c, 0x31, 0xd0, 0x13, 0x80, 0x50, 0x50, 0xa2, 0x68,
-	0x14, 0x10, 0xe5, 0x17, 0x0c, 0xff, 0xdd, 0x3c, 0xfe, 0xf3, 0x2c, 0x28, 0xec, 0x59, 0x42, 0x5b,
-	0x69, 0xf6, 0x24, 0x89, 0x32, 0x76, 0xf1, 0x4a, 0x6c, 0x4b, 0x68, 0xab, 0xe6, 0x5f, 0x45, 0x70,
-	0xbf, 0xe2, 0x11, 0x45, 0x5b, 0x50, 0x60, 0x91, 0x09, 0xde, 0xeb, 0x94, 0xe6, 0xb3, 0x46, 0xa1,
-	0xdf, 0xc3, 0x05, 0x16, 0xa1, 0x07, 0xe0, 0x8e, 0xa9, 0x22, 0x36, 0x2c, 0x3f, 0x4f, 0x58, 0x57,
-	0xc0, 0xe6, 0x64, 0xb0, 0xe8, 0x13, 0x70, 0x75, 0x59, 0x6d, 0x30, 0x3b, 0x79, 0x1c, 0xed, 0xf3,
-	0x30, 0xa1, 0x61, 0xc6, 0xd3, 0x78, 0xb4, 0x07, 0xd5, 0x88, 0xca, 0x50, 0xb0, 0x44, 0xe9, 0x4a,
-	0xba, 0x86, 0x7e, 0x67, 0x1d, 0xbd, 0x77, 0x0e, 0xc5, 0xcb, 0x3c, 0xf4, 0x04, 0x4a, 0x52, 0x11,
-	0x35, 0x91, 0xfe, 0x4d, 0xa3, 0x50, 0x5f, 0x1b, 0x80, 0x41, 0xd9, 0x10, 0x2c, 0x07, 0x7d, 0x01,
-	0x9b, 0x63, 0x12, 0x93, 0x01, 0x15, 0x81, 0x55, 0x29, 0x19, 0x95, 0xf7, 0x72, 0x53, 0x4f, 0x91,
-	0xa9, 0x10, 0xae, 0x8d, 0x97, 0x4d, 0xb4, 0x07, 0x40, 0x94, 0x22, 0xe1, 0xb7, 0x63, 0x1a, 0x2b,
-	0xbf, 0x6c, 0x54, 0xde, 0xcf, 0x8d, 0x85, 0xaa, 0x57, 0x5c, 0x0c, 0xdb, 0x0b, 0x30, 0x5e, 0x22,
-	0xa2, 0xcf, 0xa1, 0x1a, 0x52, 0xa1, 0xd8, 0x4b, 0x16, 0x12, 0x45, 0xfd, 0x8a, 0xd1, 0x69, 0xe4,
-	0xe9, 0x74, 0xcf, 0x61, 0x36, 0xa9, 0x65, 0x66, 0xf3, 0xb7, 0x02, 0x94, 0x0f, 0xa9, 0x98, 0xb2,
-	0xf0, 0xed, 0x1e, 0xf7, 0xe3, 0x0b, 0xc7, 0x9d, 0x1b, 0x99, 0x75, 0xbb, 0x72, 0xe2, 0x8f, 0xa0,
-	0x42, 0xe3, 0x28, 0xe1, 0x2c, 0x56, 0xf6, 0xb8, 0x73, 0xbb, 0x65, 0xcf, 0x62, 0xf0, 0x02, 0x8d,
-	0xf6, 0xa0, 0x96, 0x76, 0x71, 0x70, 0xe1, 0xac, 0x77, 0xf3, 0xe8, 0xdf, 0x18, 0xa0, 0x3d, 0xa4,
-	0x8d, 0xc9, 0x92, 0x85, 0x7a, 0x50, 0x4b, 0x04, 0x9d, 0x32, 0x3e, 0x91, 0x81, 0x49, 0xa2, 0x74,
-	0xa5, 0x24, 0xf0, 0x46, 0xc6, 0xd2, 0x56, 0xf3, 0xa7, 0x02, 0x54, 0xb2, 0x18, 0xd1, 0x43, 0x5b,
-	0x0e, 0x67, 0x7d, 0x40, 0x19, 0xd6, 0x48, 0xa5, 0x95, 0x78, 0x08, 0x37, 0x13, 0x2e, 0x94, 0xf4,
-	0x0b, 0xbb, 0xc5, 0x75, 0x3d, 0x7b, 0xc0, 0x85, 0xea, 0xf2, 0xf8, 0x25, 0x1b, 0xe0, 0x14, 0x8c,
-	0x5e, 0x40, 0x75, 0xca, 0x84, 0x9a, 0x90, 0x51, 0xc0, 0x12, 0xe9, 0x17, 0x0d, 0xf7, 0x83, 0xcb,
-	0x5c, 0xb6, 0x8e, 0x52, 0x7c, 0xff, 0xa0, 0xb3, 0x39, 0x9f, 0x35, 0x60, 0x61, 0x4a, 0x0c, 0x56,
-	0xaa, 0x9f, 0xc8, 0xed, 0x7d, 0xf0, 0x16, 0x3b, 0xe8, 0x1e, 0x40, 0x9c, 0xb6, 0x68, 0xb0, 0x68,
-	0x9a, 0xda, 0x7c, 0xd6, 0xf0, 0x6c, 0xe3, 0xf6, 0x7b, 0xd8, 0xb3, 0x80, 0x7e, 0x84, 0x10, 0xb8,
-	0x24, 0x8a, 0x84, 0x69, 0x21, 0x0f, 0x9b, 0x75, 0xf3, 0x97, 0x9b, 0xe0, 0x3e, 0x27, 0x72, 0x78,
-	0xdd, 0x63, 0x46, 0xfb, 0x5c, 0x69, 0xba, 0x7b, 0x00, 0x32, 0x3d, 0x4a, 0x9d, 0x8e, 0x7b, 0x9e,
-	0x8e, 0x3d, 0x60, 0x9d, 0x8e, 0x05, 0xa4, 0xe9, 0xc8, 0x11, 0x57, 0xa6, 0xbf, 0x5c, 0x6c, 0xd6,
-	0xe8, 0x0e, 0x94, 0x63, 0x1e, 0x19, 0x7a, 0xc9, 0xd0, 0x61, 0x3e, 0x6b, 0x94, 0xf4, 0x48, 0xe9,
-	0xf7, 0x70, 0x49, 0x6f, 0xf5, 0x23, 0x7d, 0x6f, 0x49, 0x1c, 0x73, 0x45, 0xf4, 0x50, 0x92, 0xf6,
-	0xfe, 0xe7, 0x36, 0x56, 0xfb, 0x1c, 0x96, 0xdd, 0xdb, 0x25, 0x26, 0x3a, 0x82, 0xff, 0x67, 0xf1,
-	0x2e, 0x0b, 0x56, 0xde, 0x44, 0x10, 0x59, 0x85, 0xa5, 0x9d, 0xa5, 0x39, 0xe9, 0xad, 0x9f, 0x93,
-	0xa6, 0x82, 0x79, 0x73, 0xb2, 0x03, 0xb5, 0x88, 0x4a, 0x26, 0x68, 0x64, 0x6e, 0x20, 0xf5, 0x61,
-	0xd7, 0xb9, 0xbb, 0xb9, 0xe6, 0xe9, 0xb1, 0x22, 0x14, 0x6f, 0x58, 0x8e, 0xb1, 0x50, 0x1b, 0x2a,
-	0xb6, 0x6f, 0xa4, 0x5f, 0x35, 0xbd, 0x7b, 0xc5, 0xf9, 0xb8, 0xa0, 0x5d, 0x98, 0x20, 0x1b, 0x6f,
-	0x34, 0x41, 0x1e, 0x03, 0x8c, 0xf8, 0x20, 0x88, 0x04, 0x9b, 0x52, 0xe1, 0xd7, 0x0c, 0x77, 0x3b,
-	0x8f, 0xdb, 0x33, 0x08, 0xec, 0x8d, 0xf8, 0x20, 0x5d, 0x36, 0x7f, 0x70, 0xe0, 0x7f, 0x2b, 0x41,
-	0xa1, 0x8f, 0xa1, 0x6c, 0xc3, 0xba, 0xec, 0x23, 0xc0, 0xf2, 0x70, 0x86, 0x45, 0x3b, 0xe0, 0xe9,
-	0x3b, 0x42, 0xa5, 0xa4, 0xe9, 0xed, 0xf7, 0xf0, 0xf9, 0x0f, 0xc8, 0x87, 0x32, 0x19, 0x31, 0xa2,
-	0xf7, 0x8a, 0x66, 0x2f, 0x33, 0x9b, 0x3f, 0x16, 0xa0, 0x6c, 0xc5, 0xae, 0x7b, 0x9c, 0x5b, 0xb7,
-	0x2b, 0x37, 0xeb, 0x29, 0x6c, 0xa4, 0xe5, 0xb4, 0x2d, 0xe1, 0xfe, 0x67, 0x51, 0xab, 0x29, 0x3e,
-	0x6d, 0x87, 0xa7, 0xe0, 0xb2, 0x84, 0x8c, 0xed, 0x28, 0xcf, 0xf5, 0xdc, 0x3f, 0x68, 0xef, 0x7f,
-	0x9d, 0xa4, 0x9d, 0x5d, 0x99, 0xcf, 0x1a, 0xae, 0xfe, 0x01, 0x1b, 0x5a, 0xf3, 0xef, 0x02, 0x94,
-	0xbb, 0xa3, 0x89, 0x54, 0x54, 0x5c, 0x77, 0x41, 0xac, 0xdb, 0x95, 0x82, 0x74, 0xa1, 0x2c, 0x38,
-	0x57, 0x41, 0x48, 0x2e, 0xab, 0x05, 0xe6, 0x5c, 0x75, 0xdb, 0x9d, 0x4d, 0x4d, 0xd4, 0x83, 0x24,
-	0xb5, 0x71, 0x49, 0x53, 0xbb, 0x04, 0xbd, 0x80, 0xad, 0x6c, 0xfc, 0x1e, 0x73, 0xae, 0xa4, 0x12,
-	0x24, 0x09, 0x86, 0xf4, 0x44, 0xbf, 0x79, 0xc5, 0x75, 0x5f, 0x26, 0x7b, 0x71, 0x28, 0x4e, 0x4c,
-	0xa1, 0x9e, 0xd1, 0x13, 0x7c, 0xcb, 0x0a, 0x74, 0x32, 0xfe, 0x33, 0x7a, 0x22, 0xd1, 0x67, 0xb0,
-	0x43, 0x17, 0x30, 0xad, 0x18, 0x8c, 0xc8, 0x58, 0x3f, 0x2c, 0x41, 0x38, 0xe2, 0xe1, 0xd0, 0xcc,
-	0x36, 0x17, 0xdf, 0xa6, 0xcb, 0x52, 0x5f, 0xa6, 0x88, 0xae, 0x06, 0x34, 0xff, 0x74, 0xa0, 0x74,
-	0x48, 0x43, 0x41, 0xd5, 0x5b, 0x2d, 0xf8, 0xa3, 0x0b, 0x05, 0xaf, 0xe7, 0xbf, 0xc5, 0xda, 0xeb,
-	0x4a, 0xbd, 0xb7, 0xa0, 0x14, 0xb1, 0x01, 0x95, 0xe9, 0xd7, 0x84, 0x87, 0xad, 0x85, 0x9a, 0xe0,
-	0x4a, 0xf6, 0x9a, 0x9a, 0xce, 0x2a, 0xa6, 0x0f, 0x9f, 0x55, 0x60, 0xaf, 0x29, 0x36, 0x7b, 0x68,
-	0x1b, 0x2a, 0x2c, 0x56, 0x54, 0xc4, 0x64, 0x64, 0x32, 0xaf, 0xe0, 0x85, 0xdd, 0xd9, 0x39, 0x3d,
-	0xab, 0xdf, 0xf8, 0xfd, 0xac, 0x7e, 0xe3, 0x9f, 0xb3, 0xba, 0xf3, 0xfd, 0xbc, 0xee, 0x9c, 0xce,
-	0xeb, 0xce, 0xaf, 0xf3, 0xba, 0xf3, 0xc7, 0xbc, 0xee, 0x1c, 0x97, 0xcc, 0xbf, 0x81, 0x8f, 0xfe,
-	0x0d, 0x00, 0x00, 0xff, 0xff, 0x4e, 0x76, 0x6f, 0x66, 0x7d, 0x0c, 0x00, 0x00,
+	// 1126 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xbc, 0x57, 0x4d, 0x6f, 0x1b, 0xc5,
+	0x1b, 0xef, 0xda, 0x5b, 0xbf, 0x3c, 0xb6, 0x23, 0xfd, 0xe7, 0x5f, 0x45, 0xdb, 0x10, 0xec, 0xe0,
+	0x0a, 0xd4, 0x43, 0xe5, 0x8a, 0x52, 0x50, 0x2b, 0x5a, 0x21, 0xbf, 0x44, 0x60, 0x95, 0x40, 0x34,
+	0x29, 0xe9, 0x71, 0x35, 0xd9, 0x9d, 0x9a, 0xc5, 0xf6, 0xce, 0x6a, 0x66, 0xec, 0x2a, 0x3d, 0x21,
+	0x3e, 0x00, 0x1f, 0x81, 0x6f, 0xc1, 0x99, 0x6b, 0x0e, 0x1c, 0xb8, 0xc1, 0xc9, 0x22, 0x3e, 0x20,
+	0x71, 0xe3, 0x23, 0xa0, 0x79, 0x59, 0xdb, 0x91, 0xd7, 0x21, 0x95, 0xaa, 0xdc, 0xe6, 0xf1, 0xfc,
+	0x7e, 0xbf, 0x79, 0xde, 0xe6, 0x99, 0x35, 0xd4, 0xd8, 0xc9, 0x77, 0x34, 0x90, 0xa2, 0x95, 0x70,
+	0x26, 0x19, 0x42, 0x21, 0x0b, 0x86, 0x94, 0xb7, 0xc4, 0x2b, 0xc2, 0xc7, 0xc3, 0x48, 0xb6, 0xa6,
+	0x1f, 0xee, 0x54, 0xe4, 0x69, 0x42, 0x2d, 0x60, 0xa7, 0x22, 0x12, 0x1a, 0xa4, 0xc6, 0x6d, 0x19,
+	0x8d, 0xa9, 0x90, 0x64, 0x9c, 0xdc, 0x5f, 0xac, 0xec, 0xd6, 0xad, 0x01, 0x1b, 0x30, 0xbd, 0xbc,
+	0xaf, 0x56, 0xe6, 0xd7, 0xe6, 0x2f, 0x0e, 0xb8, 0x07, 0x54, 0x12, 0xf4, 0x29, 0x14, 0xa7, 0x94,
+	0x8b, 0x88, 0xc5, 0x9e, 0xb3, 0xe7, 0xdc, 0xad, 0x3c, 0x78, 0xa7, 0xb5, 0x7e, 0x72, 0xeb, 0xd8,
+	0x40, 0x3a, 0xee, 0xd9, 0xac, 0x71, 0x03, 0xa7, 0x0c, 0xf4, 0x04, 0x20, 0xe0, 0x94, 0x48, 0x1a,
+	0xfa, 0x44, 0x7a, 0x39, 0xcd, 0x7f, 0x37, 0x8b, 0xff, 0x3c, 0x75, 0x0a, 0x97, 0x2d, 0xa1, 0x2d,
+	0x15, 0x7b, 0x92, 0x84, 0x29, 0x3b, 0x7f, 0x25, 0xb6, 0x25, 0xb4, 0x65, 0xf3, 0xef, 0x3c, 0xb8,
+	0x5f, 0xb1, 0x90, 0xa2, 0x6d, 0xc8, 0x45, 0xa1, 0x76, 0xbe, 0xdc, 0x29, 0xcc, 0x67, 0x8d, 0x5c,
+	0xbf, 0x87, 0x73, 0x51, 0x88, 0x1e, 0x80, 0x3b, 0xa6, 0x92, 0x58, 0xb7, 0xbc, 0x2c, 0x61, 0x95,
+	0x01, 0x1b, 0x93, 0xc6, 0xa2, 0x4f, 0xc0, 0x55, 0x69, 0xb5, 0xce, 0xec, 0x66, 0x71, 0xd4, 0x99,
+	0x47, 0x09, 0x0d, 0x52, 0x9e, 0xc2, 0xa3, 0x7d, 0xa8, 0x84, 0x54, 0x04, 0x3c, 0x4a, 0xa4, 0xca,
+	0xa4, 0xab, 0xe9, 0x77, 0x36, 0xd1, 0x7b, 0x4b, 0x28, 0x5e, 0xe5, 0xa1, 0x27, 0x50, 0x10, 0x92,
+	0xc8, 0x89, 0xf0, 0x6e, 0x6a, 0x85, 0xfa, 0x46, 0x07, 0x34, 0xca, 0xba, 0x60, 0x39, 0xe8, 0x0b,
+	0xd8, 0x1a, 0x93, 0x98, 0x0c, 0x28, 0xf7, 0xad, 0x4a, 0x41, 0xab, 0xbc, 0x97, 0x19, 0xba, 0x41,
+	0x1a, 0x21, 0x5c, 0x1b, 0xaf, 0x9a, 0x68, 0x1f, 0x80, 0x48, 0x49, 0x82, 0x6f, 0xc7, 0x34, 0x96,
+	0x5e, 0x51, 0xab, 0xbc, 0x9f, 0xe9, 0x0b, 0x95, 0xaf, 0x18, 0x1f, 0xb6, 0x17, 0x60, 0xbc, 0x42,
+	0x44, 0x9f, 0x43, 0x25, 0xa0, 0x5c, 0x46, 0x2f, 0xa3, 0x80, 0x48, 0xea, 0x95, 0xb4, 0x4e, 0x23,
+	0x4b, 0xa7, 0xbb, 0x84, 0xd9, 0xa0, 0x56, 0x99, 0xcd, 0xdf, 0x73, 0x50, 0x3c, 0xa2, 0x7c, 0x1a,
+	0x05, 0x6f, 0xb7, 0xdc, 0x8f, 0x2f, 0x94, 0x3b, 0xd3, 0x33, 0x7b, 0xec, 0x5a, 0xc5, 0x1f, 0x41,
+	0x89, 0xc6, 0x61, 0xc2, 0xa2, 0x58, 0xda, 0x72, 0x67, 0x76, 0xcb, 0xbe, 0xc5, 0xe0, 0x05, 0x1a,
+	0xed, 0x43, 0xcd, 0x74, 0xb1, 0x7f, 0xa1, 0xd6, 0x7b, 0x59, 0xf4, 0x6f, 0x34, 0xd0, 0x16, 0xa9,
+	0x3a, 0x59, 0xb1, 0x50, 0x0f, 0x6a, 0x09, 0xa7, 0xd3, 0x88, 0x4d, 0x84, 0xaf, 0x83, 0x28, 0x5c,
+	0x29, 0x08, 0x5c, 0x4d, 0x59, 0xca, 0x6a, 0xfe, 0x94, 0x83, 0x52, 0xea, 0x23, 0x7a, 0x68, 0xd3,
+	0xe1, 0x6c, 0x76, 0x28, 0xc5, 0x6a, 0x29, 0x93, 0x89, 0x87, 0x70, 0x33, 0x61, 0x5c, 0x0a, 0x2f,
+	0xb7, 0x97, 0xdf, 0xd4, 0xb3, 0x87, 0x8c, 0xcb, 0x2e, 0x8b, 0x5f, 0x46, 0x03, 0x6c, 0xc0, 0xe8,
+	0x05, 0x54, 0xa6, 0x11, 0x97, 0x13, 0x32, 0xf2, 0xa3, 0x44, 0x78, 0x79, 0xcd, 0xfd, 0xe0, 0xb2,
+	0x23, 0x5b, 0xc7, 0x06, 0xdf, 0x3f, 0xec, 0x6c, 0xcd, 0x67, 0x0d, 0x58, 0x98, 0x02, 0x83, 0x95,
+	0xea, 0x27, 0x62, 0xe7, 0x00, 0xca, 0x8b, 0x1d, 0x74, 0x0f, 0x20, 0x36, 0x2d, 0xea, 0x2f, 0x9a,
+	0xa6, 0x36, 0x9f, 0x35, 0xca, 0xb6, 0x71, 0xfb, 0x3d, 0x5c, 0xb6, 0x80, 0x7e, 0x88, 0x10, 0xb8,
+	0x24, 0x0c, 0xb9, 0x6e, 0xa1, 0x32, 0xd6, 0xeb, 0xe6, 0xaf, 0x37, 0xc1, 0x7d, 0x4e, 0xc4, 0xf0,
+	0xba, 0xc7, 0x8c, 0x3a, 0x73, 0xad, 0xe9, 0xee, 0x01, 0x08, 0x53, 0x4a, 0x15, 0x8e, 0xbb, 0x0c,
+	0xc7, 0x16, 0x58, 0x85, 0x63, 0x01, 0x26, 0x1c, 0x31, 0x62, 0x52, 0xf7, 0x97, 0x8b, 0xf5, 0x1a,
+	0xdd, 0x81, 0x62, 0xcc, 0x42, 0x4d, 0x2f, 0x68, 0x3a, 0xcc, 0x67, 0x8d, 0x82, 0x1a, 0x29, 0xfd,
+	0x1e, 0x2e, 0xa8, 0xad, 0x7e, 0xa8, 0xee, 0x2d, 0x89, 0x63, 0x26, 0x89, 0x1a, 0x4a, 0xc2, 0xde,
+	0xff, 0xcc, 0xc6, 0x6a, 0x2f, 0x61, 0xe9, 0xbd, 0x5d, 0x61, 0xa2, 0x63, 0xf8, 0x7f, 0xea, 0xef,
+	0xaa, 0x60, 0xe9, 0x4d, 0x04, 0x91, 0x55, 0x58, 0xd9, 0x59, 0x99, 0x93, 0xe5, 0xcd, 0x73, 0x52,
+	0x67, 0x30, 0x6b, 0x4e, 0x76, 0xa0, 0x16, 0x52, 0x11, 0x71, 0x1a, 0xea, 0x1b, 0x48, 0x3d, 0xd8,
+	0x73, 0xee, 0x6e, 0x6d, 0x78, 0x7a, 0xac, 0x08, 0xc5, 0x55, 0xcb, 0xd1, 0x16, 0x6a, 0x43, 0xc9,
+	0xf6, 0x8d, 0xf0, 0x2a, 0xba, 0x77, 0xaf, 0x38, 0x1f, 0x17, 0xb4, 0x0b, 0x13, 0xa4, 0xfa, 0x46,
+	0x13, 0xe4, 0x31, 0xc0, 0x88, 0x0d, 0xfc, 0x90, 0x47, 0x53, 0xca, 0xbd, 0x9a, 0xe6, 0xee, 0x64,
+	0x71, 0x7b, 0x1a, 0x81, 0xcb, 0x23, 0x36, 0x30, 0xcb, 0xe6, 0x0f, 0x0e, 0xfc, 0x6f, 0xcd, 0x29,
+	0xf4, 0x31, 0x14, 0xad, 0x5b, 0x97, 0x7d, 0x04, 0x58, 0x1e, 0x4e, 0xb1, 0x68, 0x17, 0xca, 0xea,
+	0x8e, 0x50, 0x21, 0xa8, 0xb9, 0xfd, 0x65, 0xbc, 0xfc, 0x01, 0x79, 0x50, 0x24, 0xa3, 0x88, 0xa8,
+	0xbd, 0xbc, 0xde, 0x4b, 0xcd, 0xe6, 0x8f, 0x39, 0x28, 0x5a, 0xb1, 0xeb, 0x1e, 0xe7, 0xf6, 0xd8,
+	0xb5, 0x9b, 0xf5, 0x14, 0xaa, 0x26, 0x9d, 0xb6, 0x25, 0xdc, 0xff, 0x4c, 0x6a, 0xc5, 0xe0, 0x4d,
+	0x3b, 0x3c, 0x05, 0x37, 0x4a, 0xc8, 0xd8, 0x8e, 0xf2, 0xcc, 0x93, 0xfb, 0x87, 0xed, 0x83, 0xaf,
+	0x13, 0xd3, 0xd9, 0xa5, 0xf9, 0xac, 0xe1, 0xaa, 0x1f, 0xb0, 0xa6, 0x35, 0x7f, 0xce, 0x43, 0xb1,
+	0x3b, 0x9a, 0x08, 0x49, 0xf9, 0x75, 0x27, 0xc4, 0x1e, 0xbb, 0x96, 0x90, 0x2e, 0x14, 0x39, 0x63,
+	0xd2, 0x0f, 0xc8, 0x65, 0xb9, 0xc0, 0x8c, 0xc9, 0x6e, 0xbb, 0xb3, 0xa5, 0x88, 0x6a, 0x90, 0x18,
+	0x1b, 0x17, 0x14, 0xb5, 0x4b, 0xd0, 0x0b, 0xd8, 0x4e, 0xc7, 0xef, 0x09, 0x63, 0x52, 0x48, 0x4e,
+	0x12, 0x7f, 0x48, 0x4f, 0xd5, 0x9b, 0x97, 0xdf, 0xf4, 0x65, 0xb2, 0x1f, 0x07, 0xfc, 0x54, 0x27,
+	0xea, 0x19, 0x3d, 0xc5, 0xb7, 0xac, 0x40, 0x27, 0xe5, 0x3f, 0xa3, 0xa7, 0x02, 0x7d, 0x06, 0xbb,
+	0x74, 0x01, 0x53, 0x8a, 0xfe, 0x88, 0x8c, 0xd5, 0xc3, 0xe2, 0x07, 0x23, 0x16, 0x0c, 0xf5, 0x6c,
+	0x73, 0xf1, 0x6d, 0xba, 0x2a, 0xf5, 0xa5, 0x41, 0x74, 0x15, 0x40, 0xbd, 0x9e, 0x9c, 0x8e, 0xd9,
+	0x94, 0x86, 0xbe, 0x1a, 0x7a, 0x6a, 0xc8, 0xe5, 0x37, 0xa5, 0x08, 0x1b, 0xa0, 0x1a, 0x92, 0xb8,
+	0xca, 0x97, 0x86, 0x68, 0xfe, 0xe5, 0x40, 0xe1, 0x88, 0x06, 0x9c, 0xca, 0xb7, 0x5a, 0xb6, 0x47,
+	0x17, 0xca, 0x56, 0xcf, 0x7e, 0xd1, 0xd5, 0xa9, 0x6b, 0x55, 0xdb, 0x86, 0x42, 0x18, 0x0d, 0xa8,
+	0x30, 0xdf, 0x24, 0x65, 0x6c, 0x2d, 0xd4, 0x04, 0x57, 0x44, 0xaf, 0xa9, 0xee, 0xcf, 0xbc, 0x79,
+	0x3e, 0xad, 0x42, 0xf4, 0x9a, 0x62, 0xbd, 0x87, 0x76, 0xa0, 0x14, 0xc5, 0x92, 0xf2, 0x98, 0x8c,
+	0x74, 0xfe, 0x4a, 0x78, 0x61, 0x77, 0x76, 0xcf, 0xce, 0xeb, 0x37, 0xfe, 0x38, 0xaf, 0xdf, 0xf8,
+	0xe7, 0xbc, 0xee, 0x7c, 0x3f, 0xaf, 0x3b, 0x67, 0xf3, 0xba, 0xf3, 0xdb, 0xbc, 0xee, 0xfc, 0x39,
+	0xaf, 0x3b, 0x27, 0x05, 0xfd, 0x9f, 0xe2, 0xa3, 0x7f, 0x03, 0x00, 0x00, 0xff, 0xff, 0xf7, 0x71,
+	0xe0, 0xa4, 0xc3, 0x0c, 0x00, 0x00,
 }

vendor/src/github.com/docker/swarmkit/api/objects.proto

@@ -225,6 +225,10 @@ message Cluster {
 	// and agents to unambiguously identify the older key to be deleted when
 	// a new key is allocated on key rotation.
 	uint64 encryption_key_lamport_clock = 6;
+
+	// RemovedNodes is the list of nodes that have been removed from the
+	// swarm. Their certificates should effectively be blacklisted.
+	repeated RemovedNode removed_nodes = 7;
 }
 
 // Secret represents a secret that should be passed to a container or a node,

vendor/src/github.com/docker/swarmkit/api/raft.pb.go

@@ -528,9 +528,7 @@ func (m *JoinResponse) Copy() *JoinResponse {
 
 	if m.RemovedMembers != nil {
 		o.RemovedMembers = make([]uint64, 0, len(m.RemovedMembers))
-		for _, v := range m.RemovedMembers {
-			o.RemovedMembers = append(o.RemovedMembers, v)
-		}
+		o.RemovedMembers = append(o.RemovedMembers, m.RemovedMembers...)
 	}
 
 	return o

vendor/src/github.com/docker/swarmkit/api/snapshot.pb.go

@@ -151,9 +151,7 @@ func (m *ClusterSnapshot) Copy() *ClusterSnapshot {
 
 	if m.Removed != nil {
 		o.Removed = make([]uint64, 0, len(m.Removed))
-		for _, v := range m.Removed {
-			o.Removed = append(o.Removed, v)
-		}
+		o.Removed = append(o.Removed, m.Removed...)
 	}
 
 	return o

vendor/src/github.com/docker/swarmkit/api/specs.pb.go

@@ -297,6 +297,11 @@ type TaskSpec struct {
 	// configurations (which specify the network and per-network
 	// aliases) that this task spec is bound to.
 	Networks []*NetworkAttachmentConfig `protobuf:"bytes,7,rep,name=networks" json:"networks,omitempty"`
+	// ForceUpdate is a counter that triggers an update even if no relevant
+	// parameters have been changed. We do this to allow forced restarts
+	// using the same reconcilation-based mechanism that performs rolling
+	// updates.
+	ForceUpdate uint64 `protobuf:"varint,9,opt,name=force_update,json=forceUpdate,proto3" json:"force_update,omitempty"`
 }
 
 func (m *TaskSpec) Reset()                    { *m = TaskSpec{} }
@@ -667,10 +672,11 @@ func (m *TaskSpec) Copy() *TaskSpec {
 	}
 
 	o := &TaskSpec{
-		Resources: m.Resources.Copy(),
-		Restart:   m.Restart.Copy(),
-		Placement: m.Placement.Copy(),
-		LogDriver: m.LogDriver.Copy(),
+		Resources:   m.Resources.Copy(),
+		Restart:     m.Restart.Copy(),
+		Placement:   m.Placement.Copy(),
+		LogDriver:   m.LogDriver.Copy(),
+		ForceUpdate: m.ForceUpdate,
 	}
 
 	if m.Networks != nil {
@@ -732,30 +738,22 @@ func (m *ContainerSpec) Copy() *ContainerSpec {
 
 	if m.Command != nil {
 		o.Command = make([]string, 0, len(m.Command))
-		for _, v := range m.Command {
-			o.Command = append(o.Command, v)
-		}
+		o.Command = append(o.Command, m.Command...)
 	}
 
 	if m.Args != nil {
 		o.Args = make([]string, 0, len(m.Args))
-		for _, v := range m.Args {
-			o.Args = append(o.Args, v)
-		}
+		o.Args = append(o.Args, m.Args...)
 	}
 
 	if m.Env != nil {
 		o.Env = make([]string, 0, len(m.Env))
-		for _, v := range m.Env {
-			o.Env = append(o.Env, v)
-		}
+		o.Env = append(o.Env, m.Env...)
 	}
 
 	if m.Groups != nil {
 		o.Groups = make([]string, 0, len(m.Groups))
-		for _, v := range m.Groups {
-			o.Groups = append(o.Groups, v)
-		}
+		o.Groups = append(o.Groups, m.Groups...)
 	}
 
 	if m.Mounts != nil {
@@ -929,7 +927,7 @@ func (this *TaskSpec) GoString() string {
 	if this == nil {
 		return "nil"
 	}
-	s := make([]string, 0, 11)
+	s := make([]string, 0, 12)
 	s = append(s, "&api.TaskSpec{")
 	if this.Runtime != nil {
 		s = append(s, "Runtime: "+fmt.Sprintf("%#v", this.Runtime)+",\n")
@@ -949,6 +947,7 @@ func (this *TaskSpec) GoString() string {
 	if this.Networks != nil {
 		s = append(s, "Networks: "+fmt.Sprintf("%#v", this.Networks)+",\n")
 	}
+	s = append(s, "ForceUpdate: "+fmt.Sprintf("%#v", this.ForceUpdate)+",\n")
 	s = append(s, "}")
 	return strings.Join(s, "")
 }
@@ -1371,6 +1370,11 @@ func (m *TaskSpec) MarshalTo(data []byte) (int, error) {
 			i += n
 		}
 	}
+	if m.ForceUpdate != 0 {
+		data[i] = 0x48
+		i++
+		i = encodeVarintSpecs(data, i, uint64(m.ForceUpdate))
+	}
 	return i, nil
 }
 
@@ -1959,6 +1963,9 @@ func (m *TaskSpec) Size() (n int) {
 			n += 1 + l + sovSpecs(uint64(l))
 		}
 	}
+	if m.ForceUpdate != 0 {
+		n += 1 + sovSpecs(uint64(m.ForceUpdate))
+	}
 	return n
 }
 
@@ -2233,6 +2240,7 @@ func (this *TaskSpec) String() string {
 		`Placement:` + strings.Replace(fmt.Sprintf("%v", this.Placement), "Placement", "Placement", 1) + `,`,
 		`LogDriver:` + strings.Replace(fmt.Sprintf("%v", this.LogDriver), "Driver", "Driver", 1) + `,`,
 		`Networks:` + strings.Replace(fmt.Sprintf("%v", this.Networks), "NetworkAttachmentConfig", "NetworkAttachmentConfig", 1) + `,`,
+		`ForceUpdate:` + fmt.Sprintf("%v", this.ForceUpdate) + `,`,
 		`}`,
 	}, "")
 	return s
@@ -3152,6 +3160,25 @@ func (m *TaskSpec) Unmarshal(data []byte) error {
 			}
 			m.Runtime = &TaskSpec_Attachment{v}
 			iNdEx = postIndex
+		case 9:
+			if wireType != 0 {
+				return fmt.Errorf("proto: wrong wireType = %d for field ForceUpdate", wireType)
+			}
+			m.ForceUpdate = 0
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowSpecs
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := data[iNdEx]
+				iNdEx++
+				m.ForceUpdate |= (uint64(b) & 0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
 		default:
 			iNdEx = preIndex
 			skippy, err := skipSpecs(data[iNdEx:])
@@ -4613,96 +4640,97 @@ var (
 func init() { proto.RegisterFile("specs.proto", fileDescriptorSpecs) }
 
 var fileDescriptorSpecs = []byte{
-	// 1443 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xac, 0x57, 0xcd, 0x6e, 0x1b, 0xc9,
-	0x11, 0xe6, 0x48, 0x23, 0x8a, 0xac, 0xa1, 0x6c, 0xba, 0xe1, 0x9f, 0x31, 0xed, 0x50, 0x34, 0xed,
-	0x38, 0x72, 0x82, 0x48, 0x09, 0x13, 0x38, 0x76, 0x1c, 0x23, 0xe1, 0x5f, 0x64, 0x46, 0x91, 0x4c,
-	0xb4, 0x6c, 0x03, 0x39, 0x11, 0xad, 0x99, 0x16, 0x35, 0xd0, 0x70, 0x7a, 0xd2, 0xd3, 0x43, 0x43,
-	0xb7, 0x1c, 0x0d, 0x1d, 0xf2, 0x06, 0x3a, 0x2d, 0xb0, 0x6f, 0xb0, 0xef, 0xe0, 0xe3, 0x1e, 0xf7,
-	0x24, 0xac, 0x74, 0x5d, 0x2c, 0xb0, 0xc0, 0xbe, 0xc0, 0xa2, 0x7b, 0x9a, 0xe4, 0x70, 0x3d, 0xb2,
-	0x0d, 0xac, 0x6e, 0xd5, 0xd5, 0xdf, 0x57, 0x5d, 0xac, 0xfa, 0xa6, 0xba, 0x09, 0x56, 0x14, 0x52,
-	0x27, 0x5a, 0x0f, 0x39, 0x13, 0x0c, 0x21, 0x97, 0x39, 0x87, 0x94, 0xaf, 0x47, 0x6f, 0x09, 0x1f,
-	0x1d, 0x7a, 0x62, 0x7d, 0xfc, 0xc7, 0x8a, 0x25, 0x8e, 0x42, 0xaa, 0x01, 0x95, 0xeb, 0x43, 0x36,
-	0x64, 0xca, 0xdc, 0x90, 0x96, 0xf6, 0xde, 0x72, 0x63, 0x4e, 0x84, 0xc7, 0x82, 0x8d, 0x89, 0x91,
-	0x6c, 0xd4, 0xff, 0x6f, 0x42, 0x61, 0x87, 0xb9, 0x74, 0x37, 0xa4, 0x0e, 0xda, 0x04, 0x8b, 0x04,
-	0x01, 0x13, 0x0a, 0x10, 0xd9, 0x46, 0xcd, 0x58, 0xb3, 0x1a, 0xab, 0xeb, 0x1f, 0x1e, 0xb9, 0xde,
-	0x9c, 0xc1, 0x5a, 0xe6, 0xfb, 0xd3, 0xd5, 0x1c, 0x4e, 0x33, 0xd1, 0x1f, 0xc0, 0xe4, 0xcc, 0xa7,
-	0xf6, 0x42, 0xcd, 0x58, 0xbb, 0xd2, 0xb8, 0x9b, 0x15, 0x41, 0x1e, 0x8a, 0x99, 0x4f, 0xb1, 0x42,
-	0xa2, 0x4d, 0x80, 0x11, 0x1d, 0xed, 0x51, 0x1e, 0x1d, 0x78, 0xa1, 0xbd, 0xa8, 0x78, 0xbf, 0xb9,
-	0x88, 0x27, 0x93, 0x5d, 0xdf, 0x9e, 0xc2, 0x71, 0x8a, 0x8a, 0xb6, 0xa1, 0x44, 0xc6, 0xc4, 0xf3,
-	0xc9, 0x9e, 0xe7, 0x7b, 0xe2, 0xc8, 0x36, 0x55, 0xa8, 0x47, 0x1f, 0x0d, 0xd5, 0x4c, 0x11, 0xf0,
-	0x1c, 0xbd, 0xee, 0x02, 0xcc, 0x0e, 0x42, 0x0f, 0x61, 0xb9, 0xdf, 0xdd, 0xe9, 0xf4, 0x76, 0x36,
-	0xcb, 0xb9, 0xca, 0xed, 0xe3, 0x93, 0xda, 0x0d, 0x19, 0x63, 0x06, 0xe8, 0xd3, 0xc0, 0xf5, 0x82,
-	0x21, 0x5a, 0x83, 0x42, 0xb3, 0xdd, 0xee, 0xf6, 0x5f, 0x75, 0x3b, 0x65, 0xa3, 0x52, 0x39, 0x3e,
-	0xa9, 0xdd, 0x9c, 0x07, 0x36, 0x1d, 0x87, 0x86, 0x82, 0xba, 0x15, 0xf3, 0xdd, 0x17, 0xd5, 0x5c,
-	0xfd, 0x9d, 0x01, 0xa5, 0x74, 0x12, 0xe8, 0x21, 0xe4, 0x9b, 0xed, 0x57, 0xbd, 0x37, 0xdd, 0x72,
-	0x6e, 0x46, 0x4f, 0x23, 0x9a, 0x8e, 0xf0, 0xc6, 0x14, 0x3d, 0x80, 0xa5, 0x7e, 0xf3, 0xf5, 0x6e,
-	0xb7, 0x6c, 0xcc, 0xd2, 0x49, 0xc3, 0xfa, 0x24, 0x8e, 0x14, 0xaa, 0x83, 0x9b, 0xbd, 0x9d, 0xf2,
-	0x42, 0x36, 0xaa, 0xc3, 0x89, 0x17, 0xe8, 0x54, 0xce, 0x16, 0xc1, 0xda, 0xa5, 0x7c, 0xec, 0x39,
+	// 1457 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xac, 0x57, 0x4d, 0x6f, 0x1b, 0xb9,
+	0x19, 0xd6, 0xd8, 0x63, 0x59, 0x7a, 0x47, 0x4e, 0x14, 0x22, 0x1f, 0x13, 0x25, 0x95, 0x15, 0x25,
+	0x4d, 0x9d, 0x16, 0xb5, 0x5b, 0xb5, 0x48, 0x93, 0xa6, 0x41, 0xab, 0xaf, 0x3a, 0xaa, 0x6b, 0x47,
+	0xa0, 0x93, 0x00, 0x3d, 0x09, 0xf4, 0x0c, 0x2d, 0x0f, 0x3c, 0x1a, 0x4e, 0x39, 0x1c, 0x05, 0xbe,
+	0xed, 0x31, 0xf0, 0x61, 0xff, 0x81, 0x4f, 0x0b, 0xec, 0x3f, 0xd8, 0xff, 0x90, 0xe3, 0x1e, 0xf7,
+	0x64, 0xac, 0x7d, 0x5f, 0x60, 0x81, 0xfd, 0x03, 0x0b, 0x72, 0x28, 0x69, 0xb4, 0x19, 0x27, 0x01,
+	0xd6, 0x37, 0xf2, 0xe5, 0xf3, 0xbc, 0xa4, 0xde, 0xf7, 0x99, 0x87, 0x14, 0x58, 0x51, 0x48, 0x9d,
+	0x68, 0x3d, 0xe4, 0x4c, 0x30, 0x84, 0x5c, 0xe6, 0x1c, 0x52, 0xbe, 0x1e, 0xbd, 0x25, 0x7c, 0x74,
+	0xe8, 0x89, 0xf5, 0xf1, 0x9f, 0x2b, 0x96, 0x38, 0x0a, 0xa9, 0x06, 0x54, 0xae, 0x0f, 0xd9, 0x90,
+	0xa9, 0xe1, 0x86, 0x1c, 0xe9, 0xe8, 0x2d, 0x37, 0xe6, 0x44, 0x78, 0x2c, 0xd8, 0x98, 0x0c, 0x92,
+	0x85, 0xfa, 0x97, 0x26, 0x14, 0x76, 0x98, 0x4b, 0x77, 0x43, 0xea, 0xa0, 0x4d, 0xb0, 0x48, 0x10,
+	0x30, 0xa1, 0x00, 0x91, 0x6d, 0xd4, 0x8c, 0x35, 0xab, 0xb1, 0xba, 0xfe, 0xe1, 0x96, 0xeb, 0xcd,
+	0x19, 0xac, 0x65, 0xbe, 0x3f, 0x5d, 0xcd, 0xe1, 0x34, 0x13, 0xfd, 0x09, 0x4c, 0xce, 0x7c, 0x6a,
+	0x2f, 0xd4, 0x8c, 0xb5, 0x2b, 0x8d, 0xbb, 0x59, 0x19, 0xe4, 0xa6, 0x98, 0xf9, 0x14, 0x2b, 0x24,
+	0xda, 0x04, 0x18, 0xd1, 0xd1, 0x1e, 0xe5, 0xd1, 0x81, 0x17, 0xda, 0x8b, 0x8a, 0xf7, 0xbb, 0x8b,
+	0x78, 0xf2, 0xb0, 0xeb, 0xdb, 0x53, 0x38, 0x4e, 0x51, 0xd1, 0x36, 0x94, 0xc8, 0x98, 0x78, 0x3e,
+	0xd9, 0xf3, 0x7c, 0x4f, 0x1c, 0xd9, 0xa6, 0x4a, 0xf5, 0xe8, 0xa3, 0xa9, 0x9a, 0x29, 0x02, 0x9e,
+	0xa3, 0xd7, 0x5d, 0x80, 0xd9, 0x46, 0xe8, 0x21, 0x2c, 0xf7, 0xbb, 0x3b, 0x9d, 0xde, 0xce, 0x66,
+	0x39, 0x57, 0xb9, 0x7d, 0x7c, 0x52, 0xbb, 0x21, 0x73, 0xcc, 0x00, 0x7d, 0x1a, 0xb8, 0x5e, 0x30,
+	0x44, 0x6b, 0x50, 0x68, 0xb6, 0xdb, 0xdd, 0xfe, 0xab, 0x6e, 0xa7, 0x6c, 0x54, 0x2a, 0xc7, 0x27,
+	0xb5, 0x9b, 0xf3, 0xc0, 0xa6, 0xe3, 0xd0, 0x50, 0x50, 0xb7, 0x62, 0xbe, 0xfb, 0xaa, 0x9a, 0xab,
+	0xbf, 0x33, 0xa0, 0x94, 0x3e, 0x04, 0x7a, 0x08, 0xf9, 0x66, 0xfb, 0x55, 0xef, 0x4d, 0xb7, 0x9c,
+	0x9b, 0xd1, 0xd3, 0x88, 0xa6, 0x23, 0xbc, 0x31, 0x45, 0x0f, 0x60, 0xa9, 0xdf, 0x7c, 0xbd, 0xdb,
+	0x2d, 0x1b, 0xb3, 0xe3, 0xa4, 0x61, 0x7d, 0x12, 0x47, 0x0a, 0xd5, 0xc1, 0xcd, 0xde, 0x4e, 0x79,
+	0x21, 0x1b, 0xd5, 0xe1, 0xc4, 0x0b, 0xf4, 0x51, 0xce, 0x16, 0xc1, 0xda, 0xa5, 0x7c, 0xec, 0x39,
 	0x97, 0xac, 0x89, 0xc7, 0x60, 0x0a, 0x12, 0x1d, 0x2a, 0x4d, 0x58, 0xd9, 0x9a, 0x78, 0x45, 0xa2,
-	0x43, 0x79, 0xa8, 0xa6, 0x2b, 0xbc, 0x54, 0x06, 0xa7, 0xa1, 0xef, 0x39, 0x44, 0x50, 0x57, 0x29,
-	0xc3, 0x6a, 0xfc, 0x3a, 0x8b, 0x8d, 0xa7, 0x28, 0x9d, 0xff, 0x8b, 0x1c, 0x4e, 0x51, 0xd1, 0x33,
-	0xc8, 0x0f, 0x7d, 0xb6, 0x47, 0x7c, 0xa5, 0x09, 0xab, 0x71, 0x2f, 0x2b, 0xc8, 0xa6, 0x42, 0xcc,
-	0x02, 0x68, 0x0a, 0x7a, 0x02, 0xf9, 0x38, 0x74, 0x89, 0xa0, 0x76, 0x5e, 0x91, 0x6b, 0x59, 0xe4,
-	0xd7, 0x0a, 0xd1, 0x66, 0xc1, 0xbe, 0x37, 0xc4, 0x1a, 0x8f, 0xb6, 0xa0, 0x10, 0x50, 0xf1, 0x96,
-	0xf1, 0xc3, 0xc8, 0x5e, 0xae, 0x2d, 0xae, 0x59, 0x8d, 0xdf, 0x65, 0x8a, 0x31, 0xc1, 0x34, 0x85,
-	0x20, 0xce, 0xc1, 0x88, 0x06, 0x22, 0x09, 0xd3, 0x5a, 0xb0, 0x0d, 0x3c, 0x0d, 0x80, 0xfe, 0x06,
-	0x05, 0x1a, 0xb8, 0x21, 0xf3, 0x02, 0x61, 0x17, 0x2e, 0x4e, 0xa4, 0xab, 0x31, 0xb2, 0x98, 0x78,
-	0xca, 0x68, 0xe5, 0xc1, 0x1c, 0x31, 0x97, 0xd6, 0x37, 0xe0, 0xda, 0x07, 0xc5, 0x42, 0x15, 0x28,
-	0xe8, 0x62, 0x25, 0x5d, 0x36, 0xf1, 0x74, 0x5d, 0xbf, 0x0a, 0x2b, 0x73, 0x85, 0xa9, 0x7f, 0xb7,
-	0x08, 0x85, 0x49, 0xb7, 0x50, 0x13, 0x8a, 0x0e, 0x0b, 0x04, 0xf1, 0x02, 0xca, 0xb5, 0x40, 0x32,
-	0x6b, 0xdb, 0x9e, 0x80, 0x24, 0xeb, 0x45, 0x0e, 0xcf, 0x58, 0xe8, 0x9f, 0x50, 0xe4, 0x34, 0x62,
-	0x31, 0x77, 0x68, 0xa4, 0x15, 0xb2, 0x96, 0xdd, 0xe3, 0x04, 0x84, 0xe9, 0x7f, 0x63, 0x8f, 0x53,
-	0x59, 0xa7, 0x08, 0xcf, 0xa8, 0xe8, 0x19, 0x2c, 0x73, 0x1a, 0x09, 0xc2, 0xc5, 0xc7, 0x9a, 0x8c,
-	0x13, 0x48, 0x9f, 0xf9, 0x9e, 0x73, 0x84, 0x27, 0x0c, 0xf4, 0x0c, 0x8a, 0xa1, 0x4f, 0x1c, 0x15,
-	0xd5, 0x5e, 0x52, 0xf4, 0x5f, 0x65, 0xd1, 0xfb, 0x13, 0x10, 0x9e, 0xe1, 0xd1, 0x53, 0x00, 0x9f,
-	0x0d, 0x07, 0x2e, 0xf7, 0xc6, 0x94, 0x6b, 0x91, 0x54, 0xb2, 0xd8, 0x1d, 0x85, 0xc0, 0x45, 0x9f,
-	0x0d, 0x13, 0x13, 0x6d, 0xfe, 0x22, 0x85, 0xa4, 0xd4, 0xb1, 0x05, 0x40, 0xa6, 0xbb, 0x5a, 0x1f,
-	0x8f, 0x3e, 0x2b, 0x94, 0xee, 0x48, 0x8a, 0xde, 0x2a, 0xc2, 0x32, 0x8f, 0x03, 0xe1, 0x8d, 0x68,
-	0x7d, 0x0b, 0x6e, 0x64, 0x32, 0x50, 0x03, 0x4a, 0xd3, 0x1e, 0x0e, 0x3c, 0x57, 0x35, 0xbf, 0xd8,
-	0xba, 0x7a, 0x7e, 0xba, 0x6a, 0x4d, 0x9b, 0xdd, 0xeb, 0x60, 0x6b, 0x0a, 0xea, 0xb9, 0xf5, 0xef,
-	0x4d, 0x58, 0x99, 0x53, 0x02, 0xba, 0x0e, 0x4b, 0xde, 0x88, 0x0c, 0x69, 0x42, 0xc7, 0xc9, 0x02,
-	0x75, 0x21, 0xef, 0x93, 0x3d, 0xea, 0x4b, 0x3d, 0xc8, 0x9a, 0xfc, 0xfe, 0x93, 0x92, 0x5a, 0xff,
-	0xb7, 0xc2, 0x77, 0x03, 0xc1, 0x8f, 0xb0, 0x26, 0x23, 0x1b, 0x96, 0x1d, 0x36, 0x1a, 0x91, 0x40,
-	0xce, 0x8e, 0xc5, 0xb5, 0x22, 0x9e, 0x2c, 0x11, 0x02, 0x93, 0xf0, 0x61, 0x64, 0x9b, 0xca, 0xad,
-	0x6c, 0x54, 0x86, 0x45, 0x1a, 0x8c, 0xed, 0x25, 0xe5, 0x92, 0xa6, 0xf4, 0xb8, 0x5e, 0xd2, 0xd0,
-	0x22, 0x96, 0xa6, 0xe4, 0xc5, 0x11, 0xe5, 0xf6, 0xb2, 0x72, 0x29, 0x1b, 0xfd, 0x05, 0xf2, 0x23,
-	0x16, 0x07, 0x22, 0xb2, 0x0b, 0x2a, 0xd9, 0xdb, 0x59, 0xc9, 0x6e, 0x4b, 0x84, 0x9e, 0x6d, 0x1a,
-	0x8e, 0x5e, 0xc0, 0xb5, 0x48, 0xb0, 0x70, 0x30, 0xe4, 0xc4, 0xa1, 0x83, 0x90, 0x72, 0x8f, 0xb9,
-	0x76, 0xf1, 0xe2, 0x11, 0xd9, 0xd1, 0xd7, 0x37, 0xbe, 0x2a, 0x69, 0x9b, 0x92, 0xd5, 0x57, 0x24,
-	0xd4, 0x87, 0x52, 0x18, 0xfb, 0xfe, 0x80, 0x85, 0xc9, 0xa4, 0x06, 0x15, 0xe4, 0x33, 0xaa, 0xd6,
-	0x8f, 0x7d, 0xff, 0x65, 0x42, 0xc2, 0x56, 0x38, 0x5b, 0xa0, 0x9b, 0x90, 0x1f, 0x72, 0x16, 0x87,
-	0x91, 0x6d, 0xa9, 0x7a, 0xe8, 0x15, 0x7a, 0x0e, 0xcb, 0x11, 0x75, 0x38, 0x15, 0x91, 0x5d, 0x52,
-	0xbf, 0xf6, 0x7e, 0xd6, 0x21, 0xbb, 0x0a, 0x82, 0xe9, 0x3e, 0xe5, 0x34, 0x70, 0x28, 0x9e, 0x70,
-	0x2a, 0x4f, 0xc1, 0x4a, 0x35, 0x4a, 0x16, 0xf8, 0x90, 0x1e, 0xe9, 0xde, 0x4b, 0x53, 0xea, 0x61,
-	0x4c, 0xfc, 0x38, 0x79, 0x3e, 0x14, 0x71, 0xb2, 0xf8, 0xeb, 0xc2, 0x13, 0xa3, 0xd2, 0x00, 0x2b,
-	0x95, 0x2d, 0xba, 0x0f, 0x2b, 0x9c, 0x0e, 0xbd, 0x48, 0xf0, 0xa3, 0x01, 0x89, 0xc5, 0x81, 0xfd,
-	0x0f, 0x45, 0x28, 0x4d, 0x9c, 0xcd, 0x58, 0x1c, 0xd4, 0x7f, 0x34, 0xa0, 0x94, 0x9e, 0x87, 0xa8,
-	0x9d, 0x4c, 0x41, 0x75, 0xe2, 0x95, 0xc6, 0xc6, 0xa7, 0xe6, 0xa7, 0x9a, 0x39, 0x7e, 0x2c, 0x4f,
-	0xdc, 0x96, 0x6f, 0x16, 0x45, 0x46, 0x7f, 0x86, 0xa5, 0x90, 0x71, 0x31, 0x11, 0x67, 0x35, 0x73,
-	0x4e, 0x30, 0x3e, 0xf9, 0x46, 0x13, 0x70, 0xfd, 0x00, 0xae, 0xcc, 0x47, 0x43, 0x0f, 0x60, 0xf1,
-	0x4d, 0xaf, 0x5f, 0xce, 0x55, 0xee, 0x1c, 0x9f, 0xd4, 0x6e, 0xcd, 0x6f, 0xbe, 0xf1, 0xb8, 0x88,
-	0x89, 0xdf, 0xeb, 0xa3, 0xdf, 0xc2, 0x52, 0x67, 0x67, 0x17, 0xe3, 0xb2, 0x51, 0x59, 0x3d, 0x3e,
-	0xa9, 0xdd, 0x99, 0xc7, 0xc9, 0x2d, 0x16, 0x07, 0x2e, 0x66, 0x7b, 0xd3, 0x6b, 0xfc, 0xab, 0x05,
-	0xb0, 0xf4, 0x37, 0x7b, 0xb9, 0xd7, 0xf8, 0xdf, 0x61, 0x25, 0x99, 0x71, 0x03, 0x47, 0xfd, 0x34,
-	0x3d, 0xad, 0x3f, 0x36, 0xea, 0x4a, 0x09, 0x21, 0x29, 0x05, 0xba, 0x07, 0x25, 0x2f, 0x1c, 0x3f,
-	0x1e, 0xd0, 0x80, 0xec, 0xf9, 0xfa, 0x46, 0x2f, 0x60, 0x4b, 0xfa, 0xba, 0x89, 0x4b, 0x5e, 0x45,
-	0x5e, 0x20, 0x28, 0x0f, 0xf4, 0x5d, 0x5d, 0xc0, 0xd3, 0x35, 0x7a, 0x0e, 0xa6, 0x17, 0x92, 0x91,
-	0x9e, 0xcf, 0x99, 0xbf, 0xa0, 0xd7, 0x6f, 0x6e, 0x6b, 0x89, 0xb4, 0x0a, 0xe7, 0xa7, 0xab, 0xa6,
-	0x74, 0x60, 0x45, 0x43, 0xd5, 0xc9, 0x88, 0x94, 0x27, 0xa9, 0xaf, 0xba, 0x80, 0x53, 0x9e, 0xfa,
-	0x97, 0x26, 0x58, 0x6d, 0x3f, 0x8e, 0x84, 0x9e, 0x4d, 0x97, 0x56, 0xb7, 0xff, 0xc0, 0x35, 0xa2,
-	0x1e, 0x7d, 0x24, 0x90, 0x1f, 0xba, 0xba, 0x7a, 0x74, 0xed, 0x1e, 0x64, 0x86, 0x9b, 0x82, 0x93,
-	0x6b, 0xaa, 0x95, 0x97, 0x31, 0x6d, 0x03, 0x97, 0xc9, 0xcf, 0x76, 0xd0, 0x2e, 0xac, 0x30, 0xee,
-	0x1c, 0xd0, 0x48, 0x24, 0xb3, 0x41, 0x3f, 0x92, 0x32, 0x9f, 0xcf, 0x2f, 0xd3, 0x40, 0xfd, 0xc4,
-	0x48, 0xb2, 0x9d, 0x8f, 0x81, 0x9e, 0x80, 0xc9, 0xc9, 0xfe, 0xe4, 0x1a, 0xcd, 0xd4, 0x37, 0x26,
-	0xfb, 0x62, 0x2e, 0x84, 0x62, 0xa0, 0x7f, 0x01, 0xb8, 0x5e, 0x14, 0x12, 0xe1, 0x1c, 0x50, 0xae,
-	0xfb, 0x94, 0xf9, 0x13, 0x3b, 0x53, 0xd4, 0x5c, 0x94, 0x14, 0x1b, 0x6d, 0x41, 0xd1, 0x21, 0x13,
-	0xa5, 0xe5, 0x2f, 0x1e, 0x8b, 0xed, 0xa6, 0x0e, 0x51, 0x96, 0x21, 0xce, 0x4f, 0x57, 0x0b, 0x13,
-	0x0f, 0x2e, 0x38, 0x44, 0x2b, 0x6f, 0x0b, 0x56, 0xe4, 0x8b, 0x72, 0xe0, 0xd2, 0x7d, 0x12, 0xfb,
-	0x22, 0x52, 0x13, 0xfc, 0x82, 0x17, 0x94, 0x7c, 0xdc, 0x74, 0x34, 0x4e, 0xe7, 0x55, 0x12, 0x29,
-	0x5f, 0xdd, 0x03, 0x48, 0x26, 0xdc, 0xe5, 0xca, 0x04, 0x81, 0xe9, 0x12, 0x41, 0x94, 0x32, 0x4a,
-	0x58, 0xd9, 0xad, 0xbb, 0xef, 0xcf, 0xaa, 0xb9, 0x6f, 0xce, 0xaa, 0xb9, 0x1f, 0xce, 0xaa, 0xc6,
-	0xff, 0xce, 0xab, 0xc6, 0xfb, 0xf3, 0xaa, 0xf1, 0xf5, 0x79, 0xd5, 0xf8, 0xf6, 0xbc, 0x6a, 0xec,
-	0xe5, 0xd5, 0x1f, 0xb9, 0x3f, 0xfd, 0x14, 0x00, 0x00, 0xff, 0xff, 0xc3, 0xfc, 0x45, 0x4e, 0x27,
-	0x0e, 0x00, 0x00,
+	0x43, 0xb9, 0xa9, 0xa6, 0x2b, 0xbc, 0x54, 0x06, 0xa7, 0xa1, 0xef, 0x39, 0x44, 0x50, 0x57, 0x29,
+	0xc3, 0x6a, 0xfc, 0x36, 0x8b, 0x8d, 0xa7, 0x28, 0x7d, 0xfe, 0x17, 0x39, 0x9c, 0xa2, 0xa2, 0x67,
+	0x90, 0x1f, 0xfa, 0x6c, 0x8f, 0xf8, 0x4a, 0x13, 0x56, 0xe3, 0x5e, 0x56, 0x92, 0x4d, 0x85, 0x98,
+	0x25, 0xd0, 0x14, 0xf4, 0x04, 0xf2, 0x71, 0xe8, 0x12, 0x41, 0xed, 0xbc, 0x22, 0xd7, 0xb2, 0xc8,
+	0xaf, 0x15, 0xa2, 0xcd, 0x82, 0x7d, 0x6f, 0x88, 0x35, 0x1e, 0x6d, 0x41, 0x21, 0xa0, 0xe2, 0x2d,
+	0xe3, 0x87, 0x91, 0xbd, 0x5c, 0x5b, 0x5c, 0xb3, 0x1a, 0x7f, 0xc8, 0x14, 0x63, 0x82, 0x69, 0x0a,
+	0x41, 0x9c, 0x83, 0x11, 0x0d, 0x44, 0x92, 0xa6, 0xb5, 0x60, 0x1b, 0x78, 0x9a, 0x00, 0xfd, 0x03,
+	0x0a, 0x34, 0x70, 0x43, 0xe6, 0x05, 0xc2, 0x2e, 0x5c, 0x7c, 0x90, 0xae, 0xc6, 0xc8, 0x62, 0xe2,
+	0x29, 0xa3, 0x95, 0x07, 0x73, 0xc4, 0x5c, 0x5a, 0xdf, 0x80, 0x6b, 0x1f, 0x14, 0x0b, 0x55, 0xa0,
+	0xa0, 0x8b, 0x95, 0x74, 0xd9, 0xc4, 0xd3, 0x79, 0xfd, 0x2a, 0xac, 0xcc, 0x15, 0x46, 0xd9, 0xc6,
+	0xa4, 0x5b, 0xa8, 0x09, 0x45, 0x87, 0x05, 0x82, 0x78, 0x01, 0xe5, 0x5a, 0x20, 0x99, 0xb5, 0x6d,
+	0x4f, 0x40, 0x92, 0xf5, 0x22, 0x87, 0x67, 0x2c, 0xf4, 0x6f, 0x28, 0x72, 0x1a, 0xb1, 0x98, 0x3b,
+	0x34, 0xd2, 0x0a, 0x59, 0xcb, 0xee, 0x71, 0x02, 0xc2, 0xf4, 0xff, 0xb1, 0xc7, 0xa9, 0xac, 0x53,
+	0x84, 0x67, 0x54, 0xf4, 0x0c, 0x96, 0x39, 0x8d, 0x04, 0xe1, 0xe2, 0x63, 0x4d, 0xc6, 0x09, 0xa4,
+	0xcf, 0x7c, 0xcf, 0x39, 0xc2, 0x13, 0x06, 0x7a, 0x06, 0xc5, 0xd0, 0x27, 0x8e, 0xca, 0x6a, 0x2f,
+	0x29, 0xfa, 0x6f, 0xb2, 0xe8, 0xfd, 0x09, 0x08, 0xcf, 0xf0, 0xe8, 0x29, 0x80, 0xcf, 0x86, 0x03,
+	0x97, 0x7b, 0x63, 0xca, 0xb5, 0x48, 0x2a, 0x59, 0xec, 0x8e, 0x42, 0xe0, 0xa2, 0xcf, 0x86, 0xc9,
+	0x10, 0x6d, 0xfe, 0x2a, 0x85, 0xa4, 0xd4, 0xb1, 0x05, 0x40, 0xa6, 0xab, 0x5a, 0x1f, 0x8f, 0x3e,
+	0x2b, 0x95, 0xee, 0x48, 0x8a, 0x8e, 0xee, 0x41, 0x69, 0x9f, 0x71, 0x87, 0x0e, 0xb4, 0xee, 0x8b,
+	0x4a, 0x13, 0x96, 0x8a, 0x25, 0x42, 0x6f, 0x15, 0x61, 0x99, 0xc7, 0x81, 0xf0, 0x46, 0xb4, 0xbe,
+	0x05, 0x37, 0x32, 0x93, 0xa2, 0x06, 0x94, 0xa6, 0x6d, 0x1e, 0x78, 0xae, 0xd2, 0x47, 0xb1, 0x75,
+	0xf5, 0xfc, 0x74, 0xd5, 0x9a, 0xea, 0xa1, 0xd7, 0xc1, 0xd6, 0x14, 0xd4, 0x73, 0xeb, 0x3f, 0x98,
+	0xb0, 0x32, 0x27, 0x16, 0x74, 0x1d, 0x96, 0xbc, 0x11, 0x19, 0xd2, 0x84, 0x8e, 0x93, 0x09, 0xea,
+	0x42, 0xde, 0x27, 0x7b, 0xd4, 0x97, 0x92, 0x91, 0x65, 0xfb, 0xe3, 0x27, 0x55, 0xb7, 0xfe, 0x5f,
+	0x85, 0xef, 0x06, 0x82, 0x1f, 0x61, 0x4d, 0x46, 0x36, 0x2c, 0x3b, 0x6c, 0x34, 0x22, 0x81, 0xb4,
+	0x97, 0xc5, 0xb5, 0x22, 0x9e, 0x4c, 0x11, 0x02, 0x93, 0xf0, 0x61, 0x64, 0x9b, 0x2a, 0xac, 0xc6,
+	0xa8, 0x0c, 0x8b, 0x34, 0x18, 0xdb, 0x4b, 0x2a, 0x24, 0x87, 0x32, 0xe2, 0x7a, 0x49, 0xcf, 0x8b,
+	0x58, 0x0e, 0x25, 0x2f, 0x8e, 0x28, 0xb7, 0x97, 0x55, 0x48, 0x8d, 0xd1, 0xdf, 0x20, 0x3f, 0x62,
+	0x71, 0x20, 0x22, 0xbb, 0xa0, 0x0e, 0x7b, 0x3b, 0xeb, 0xb0, 0xdb, 0x12, 0xa1, 0xed, 0x4f, 0xc3,
+	0xd1, 0x0b, 0xb8, 0x16, 0x09, 0x16, 0x0e, 0x86, 0x9c, 0x38, 0x74, 0x10, 0x52, 0xee, 0x31, 0x57,
+	0x75, 0xe3, 0x02, 0x17, 0xed, 0xe8, 0x1b, 0x1e, 0x5f, 0x95, 0xb4, 0x4d, 0xc9, 0xea, 0x2b, 0x12,
+	0xea, 0x43, 0x29, 0x8c, 0x7d, 0x7f, 0xc0, 0xc2, 0xc4, 0xcc, 0x41, 0x25, 0xf9, 0x8c, 0xaa, 0xf5,
+	0x63, 0xdf, 0x7f, 0x99, 0x90, 0xb0, 0x15, 0xce, 0x26, 0xe8, 0x26, 0xe4, 0x87, 0x9c, 0xc5, 0x61,
+	0x64, 0x5b, 0xaa, 0x1e, 0x7a, 0x86, 0x9e, 0xc3, 0x72, 0x44, 0x1d, 0x4e, 0x45, 0x64, 0x97, 0xd4,
+	0xaf, 0xbd, 0x9f, 0xb5, 0xc9, 0xae, 0x82, 0x60, 0xba, 0x4f, 0x39, 0x0d, 0x1c, 0x8a, 0x27, 0x9c,
+	0xca, 0x53, 0xb0, 0x52, 0x8d, 0x92, 0x05, 0x3e, 0xa4, 0x47, 0xba, 0xf7, 0x72, 0x28, 0xf5, 0x30,
+	0x26, 0x7e, 0x9c, 0xbc, 0x30, 0x8a, 0x38, 0x99, 0xfc, 0x7d, 0xe1, 0x89, 0x51, 0x69, 0x80, 0x95,
+	0x3a, 0x2d, 0xba, 0x0f, 0x2b, 0x9c, 0x0e, 0xbd, 0x48, 0xf0, 0xa3, 0x01, 0x89, 0xc5, 0x81, 0xfd,
+	0x2f, 0x45, 0x28, 0x4d, 0x82, 0xcd, 0x58, 0x1c, 0xd4, 0x7f, 0x32, 0xa0, 0x94, 0xb6, 0x4c, 0xd4,
+	0x4e, 0x8c, 0x52, 0xed, 0x78, 0xa5, 0xb1, 0xf1, 0x29, 0x8b, 0x55, 0xb6, 0xe4, 0xc7, 0x72, 0xc7,
+	0x6d, 0xf9, 0xac, 0x51, 0x64, 0xf4, 0x57, 0x58, 0x0a, 0x19, 0x17, 0x13, 0x71, 0x56, 0x33, 0xad,
+	0x84, 0xf1, 0xc9, 0x67, 0x9c, 0x80, 0xeb, 0x07, 0x70, 0x65, 0x3e, 0x1b, 0x7a, 0x00, 0x8b, 0x6f,
+	0x7a, 0xfd, 0x72, 0xae, 0x72, 0xe7, 0xf8, 0xa4, 0x76, 0x6b, 0x7e, 0xf1, 0x8d, 0xc7, 0x45, 0x4c,
+	0xfc, 0x5e, 0x1f, 0xfd, 0x1e, 0x96, 0x3a, 0x3b, 0xbb, 0x18, 0x97, 0x8d, 0xca, 0xea, 0xf1, 0x49,
+	0xed, 0xce, 0x3c, 0x4e, 0x2e, 0xb1, 0x38, 0x70, 0x31, 0xdb, 0x9b, 0xde, 0xf4, 0xdf, 0x2c, 0x80,
+	0xa5, 0xbf, 0xd9, 0xcb, 0xbd, 0xe9, 0xff, 0x09, 0x2b, 0x89, 0x0d, 0x0e, 0x1c, 0xf5, 0xd3, 0xb4,
+	0xa1, 0x7f, 0xcc, 0x0d, 0x4b, 0x09, 0x21, 0x29, 0x85, 0xb4, 0x1e, 0x2f, 0x1c, 0x3f, 0x1e, 0xd0,
+	0x80, 0xec, 0xf9, 0xfa, 0xd2, 0x2f, 0x60, 0x4b, 0xc6, 0xba, 0x49, 0x48, 0xde, 0x56, 0x5e, 0x20,
+	0x28, 0x0f, 0xf4, 0x75, 0x5e, 0xc0, 0xd3, 0x39, 0x7a, 0x0e, 0xa6, 0x17, 0x92, 0x91, 0xb6, 0xf0,
+	0xcc, 0x5f, 0xd0, 0xeb, 0x37, 0xb7, 0xb5, 0x44, 0x5a, 0x85, 0xf3, 0xd3, 0x55, 0x53, 0x06, 0xb0,
+	0xa2, 0xa1, 0xea, 0xc4, 0x45, 0xe5, 0x4e, 0xea, 0xab, 0x2e, 0xe0, 0x54, 0xa4, 0xfe, 0xb5, 0x09,
+	0x56, 0xdb, 0x8f, 0x23, 0xa1, 0xbd, 0xe9, 0xd2, 0xea, 0xf6, 0x3f, 0xb8, 0x46, 0xd4, 0xbb, 0x90,
+	0x04, 0xf2, 0x43, 0x57, 0xb7, 0x93, 0xae, 0xdd, 0x83, 0xcc, 0x74, 0x53, 0x70, 0x72, 0x93, 0xb5,
+	0xf2, 0x32, 0xa7, 0x6d, 0xe0, 0x32, 0xf9, 0xc5, 0x0a, 0xda, 0x85, 0x15, 0xc6, 0x9d, 0x03, 0x1a,
+	0x89, 0xc4, 0x1b, 0xf4, 0x3b, 0x2a, 0xf3, 0x85, 0xfd, 0x32, 0x0d, 0xd4, 0xaf, 0x90, 0xe4, 0xb4,
+	0xf3, 0x39, 0xd0, 0x13, 0x30, 0x39, 0xd9, 0x9f, 0xdc, 0xb4, 0x99, 0xfa, 0xc6, 0x64, 0x5f, 0xcc,
+	0xa5, 0x50, 0x0c, 0xf4, 0x1f, 0x00, 0xd7, 0x8b, 0x42, 0x22, 0x9c, 0x03, 0xca, 0x75, 0x9f, 0x32,
+	0x7f, 0x62, 0x67, 0x8a, 0x9a, 0xcb, 0x92, 0x62, 0xa3, 0x2d, 0x28, 0x3a, 0x64, 0xa2, 0xb4, 0xfc,
+	0xc5, 0xb6, 0xd8, 0x6e, 0xea, 0x14, 0x65, 0x99, 0xe2, 0xfc, 0x74, 0xb5, 0x30, 0x89, 0xe0, 0x82,
+	0x43, 0xb4, 0xf2, 0xb6, 0x60, 0x45, 0x3e, 0x3a, 0x07, 0x2e, 0xdd, 0x27, 0xb1, 0x2f, 0x22, 0xe5,
+	0xe0, 0x17, 0x3c, 0xb2, 0xe4, 0xfb, 0xa7, 0xa3, 0x71, 0xfa, 0x5c, 0x25, 0x91, 0x8a, 0xd5, 0x3d,
+	0x80, 0xc4, 0xe1, 0x2e, 0x57, 0x26, 0x08, 0x4c, 0x97, 0x08, 0xa2, 0x94, 0x51, 0xc2, 0x6a, 0xdc,
+	0xba, 0xfb, 0xfe, 0xac, 0x9a, 0xfb, 0xee, 0xac, 0x9a, 0xfb, 0xf1, 0xac, 0x6a, 0x7c, 0x71, 0x5e,
+	0x35, 0xde, 0x9f, 0x57, 0x8d, 0x6f, 0xcf, 0xab, 0xc6, 0xf7, 0xe7, 0x55, 0x63, 0x2f, 0xaf, 0xfe,
+	0xeb, 0xfd, 0xe5, 0xe7, 0x00, 0x00, 0x00, 0xff, 0xff, 0xb6, 0x26, 0x4a, 0x64, 0x4a, 0x0e, 0x00,
+	0x00,
 }

vendor/src/github.com/docker/swarmkit/api/specs.proto

@@ -112,6 +112,12 @@ message TaskSpec {
 	// configurations (which specify the network and per-network
 	// aliases) that this task spec is bound to.
 	repeated NetworkAttachmentConfig networks = 7;
+
+	// ForceUpdate is a counter that triggers an update even if no relevant
+	// parameters have been changed. We do this to allow forced restarts
+	// using the same reconcilation-based mechanism that performs rolling
+	// updates.
+	uint64 force_update = 9;
 }
 
 // NetworkAttachmentSpec specifies runtime parameters required to attach

vendor/src/github.com/docker/swarmkit/api/types.pb.go

@@ -57,6 +57,7 @@
 		EncryptionKey
 		ManagerStatus
 		SecretReference
+		RemovedNode
 		NodeSpec
 		ServiceSpec
 		ReplicatedService
@@ -1396,6 +1397,19 @@ func (m *SecretReference) Reset()                    { *m = SecretReference{} }
 func (*SecretReference) ProtoMessage()               {}
 func (*SecretReference) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{38} }
 
+// RemovedNode is a record for a node that has been removed from the swarm.
+type RemovedNode struct {
+	// ID is the ID of the removed node.
+	ID string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"`
+	// Expiry is the latest known expiration time of a certificate that
+	// was issued to this node.
+	Expiry *docker_swarmkit_v1.Timestamp `protobuf:"bytes,2,opt,name=expiry" json:"expiry,omitempty"`
+}
+
+func (m *RemovedNode) Reset()                    { *m = RemovedNode{} }
+func (*RemovedNode) ProtoMessage()               {}
+func (*RemovedNode) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{39} }
+
 func init() {
 	proto.RegisterType((*Version)(nil), "docker.swarmkit.v1.Version")
 	proto.RegisterType((*Annotations)(nil), "docker.swarmkit.v1.Annotations")
@@ -1441,6 +1455,7 @@ func init() {
 	proto.RegisterType((*EncryptionKey)(nil), "docker.swarmkit.v1.EncryptionKey")
 	proto.RegisterType((*ManagerStatus)(nil), "docker.swarmkit.v1.ManagerStatus")
 	proto.RegisterType((*SecretReference)(nil), "docker.swarmkit.v1.SecretReference")
+	proto.RegisterType((*RemovedNode)(nil), "docker.swarmkit.v1.RemovedNode")
 	proto.RegisterEnum("docker.swarmkit.v1.TaskState", TaskState_name, TaskState_value)
 	proto.RegisterEnum("docker.swarmkit.v1.NodeRole", NodeRole_name, NodeRole_value)
 	proto.RegisterEnum("docker.swarmkit.v1.RaftMemberStatus_Reachability", RaftMemberStatus_Reachability_name, RaftMemberStatus_Reachability_value)
@@ -1779,16 +1794,12 @@ func (m *NetworkAttachmentConfig) Copy() *NetworkAttachmentConfig {
 
 	if m.Aliases != nil {
 		o.Aliases = make([]string, 0, len(m.Aliases))
-		for _, v := range m.Aliases {
-			o.Aliases = append(o.Aliases, v)
-		}
+		o.Aliases = append(o.Aliases, m.Aliases...)
 	}
 
 	if m.Addresses != nil {
 		o.Addresses = make([]string, 0, len(m.Addresses))
-		for _, v := range m.Addresses {
-			o.Addresses = append(o.Addresses, v)
-		}
+		o.Addresses = append(o.Addresses, m.Addresses...)
 	}
 
 	return o
@@ -2052,9 +2063,7 @@ func (m *Placement) Copy() *Placement {
 
 	if m.Constraints != nil {
 		o.Constraints = make([]string, 0, len(m.Constraints))
-		for _, v := range m.Constraints {
-			o.Constraints = append(o.Constraints, v)
-		}
+		o.Constraints = append(o.Constraints, m.Constraints...)
 	}
 
 	return o
@@ -2149,6 +2158,19 @@ func (m *SecretReference) Copy() *SecretReference {
 	return o
 }
 
+func (m *RemovedNode) Copy() *RemovedNode {
+	if m == nil {
+		return nil
+	}
+
+	o := &RemovedNode{
+		ID:     m.ID,
+		Expiry: m.Expiry.Copy(),
+	}
+
+	return o
+}
+
 func (this *Version) GoString() string {
 	if this == nil {
 		return "nil"
@@ -2800,6 +2822,19 @@ func (this *SecretReference) GoString() string {
 	s = append(s, "}")
 	return strings.Join(s, "")
 }
+func (this *RemovedNode) GoString() string {
+	if this == nil {
+		return "nil"
+	}
+	s := make([]string, 0, 6)
+	s = append(s, "&api.RemovedNode{")
+	s = append(s, "ID: "+fmt.Sprintf("%#v", this.ID)+",\n")
+	if this.Expiry != nil {
+		s = append(s, "Expiry: "+fmt.Sprintf("%#v", this.Expiry)+",\n")
+	}
+	s = append(s, "}")
+	return strings.Join(s, "")
+}
 func valueToGoStringTypes(v interface{}, typ string) string {
 	rv := reflect.ValueOf(v)
 	if rv.IsNil() {
@@ -4560,6 +4595,40 @@ func (m *SecretReference) MarshalTo(data []byte) (int, error) {
 	return i, nil
 }
 
+func (m *RemovedNode) Marshal() (data []byte, err error) {
+	size := m.Size()
+	data = make([]byte, size)
+	n, err := m.MarshalTo(data)
+	if err != nil {
+		return nil, err
+	}
+	return data[:n], nil
+}
+
+func (m *RemovedNode) MarshalTo(data []byte) (int, error) {
+	var i int
+	_ = i
+	var l int
+	_ = l
+	if len(m.ID) > 0 {
+		data[i] = 0xa
+		i++
+		i = encodeVarintTypes(data, i, uint64(len(m.ID)))
+		i += copy(data[i:], m.ID)
+	}
+	if m.Expiry != nil {
+		data[i] = 0x12
+		i++
+		i = encodeVarintTypes(data, i, uint64(m.Expiry.Size()))
+		n27, err := m.Expiry.MarshalTo(data[i:])
+		if err != nil {
+			return 0, err
+		}
+		i += n27
+	}
+	return i, nil
+}
+
 func encodeFixed64Types(data []byte, offset int, v uint64) int {
 	data[offset] = uint8(v)
 	data[offset+1] = uint8(v >> 8)
@@ -5341,6 +5410,20 @@ func (m *SecretReference) Size() (n int) {
 	return n
 }
 
+func (m *RemovedNode) Size() (n int) {
+	var l int
+	_ = l
+	l = len(m.ID)
+	if l > 0 {
+		n += 1 + l + sovTypes(uint64(l))
+	}
+	if m.Expiry != nil {
+		l = m.Expiry.Size()
+		n += 1 + l + sovTypes(uint64(l))
+	}
+	return n
+}
+
 func sovTypes(x uint64) (n int) {
 	for {
 		n++
@@ -5943,6 +6026,17 @@ func (this *SecretReference) String() string {
 	}, "")
 	return s
 }
+func (this *RemovedNode) String() string {
+	if this == nil {
+		return "nil"
+	}
+	s := strings.Join([]string{`&RemovedNode{`,
+		`ID:` + fmt.Sprintf("%v", this.ID) + `,`,
+		`Expiry:` + strings.Replace(fmt.Sprintf("%v", this.Expiry), "Timestamp", "docker_swarmkit_v1.Timestamp", 1) + `,`,
+		`}`,
+	}, "")
+	return s
+}
 func valueToStringTypes(v interface{}) string {
 	rv := reflect.ValueOf(v)
 	if rv.IsNil() {
@@ -11920,6 +12014,118 @@ func (m *SecretReference) Unmarshal(data []byte) error {
 	}
 	return nil
 }
+func (m *RemovedNode) Unmarshal(data []byte) error {
+	l := len(data)
+	iNdEx := 0
+	for iNdEx < l {
+		preIndex := iNdEx
+		var wire uint64
+		for shift := uint(0); ; shift += 7 {
+			if shift >= 64 {
+				return ErrIntOverflowTypes
+			}
+			if iNdEx >= l {
+				return io.ErrUnexpectedEOF
+			}
+			b := data[iNdEx]
+			iNdEx++
+			wire |= (uint64(b) & 0x7F) << shift
+			if b < 0x80 {
+				break
+			}
+		}
+		fieldNum := int32(wire >> 3)
+		wireType := int(wire & 0x7)
+		if wireType == 4 {
+			return fmt.Errorf("proto: RemovedNode: wiretype end group for non-group")
+		}
+		if fieldNum <= 0 {
+			return fmt.Errorf("proto: RemovedNode: illegal tag %d (wire type %d)", fieldNum, wire)
+		}
+		switch fieldNum {
+		case 1:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field ID", wireType)
+			}
+			var stringLen uint64
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowTypes
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := data[iNdEx]
+				iNdEx++
+				stringLen |= (uint64(b) & 0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			intStringLen := int(stringLen)
+			if intStringLen < 0 {
+				return ErrInvalidLengthTypes
+			}
+			postIndex := iNdEx + intStringLen
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			m.ID = string(data[iNdEx:postIndex])
+			iNdEx = postIndex
+		case 2:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Expiry", wireType)
+			}
+			var msglen int
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowTypes
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := data[iNdEx]
+				iNdEx++
+				msglen |= (int(b) & 0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			if msglen < 0 {
+				return ErrInvalidLengthTypes
+			}
+			postIndex := iNdEx + msglen
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			if m.Expiry == nil {
+				m.Expiry = &docker_swarmkit_v1.Timestamp{}
+			}
+			if err := m.Expiry.Unmarshal(data[iNdEx:postIndex]); err != nil {
+				return err
+			}
+			iNdEx = postIndex
+		default:
+			iNdEx = preIndex
+			skippy, err := skipTypes(data[iNdEx:])
+			if err != nil {
+				return err
+			}
+			if skippy < 0 {
+				return ErrInvalidLengthTypes
+			}
+			if (iNdEx + skippy) > l {
+				return io.ErrUnexpectedEOF
+			}
+			iNdEx += skippy
+		}
+	}
+
+	if iNdEx > l {
+		return io.ErrUnexpectedEOF
+	}
+	return nil
+}
 func skipTypes(data []byte) (n int, err error) {
 	l := len(data)
 	iNdEx := 0
@@ -12028,231 +12234,233 @@ var (
 func init() { proto.RegisterFile("types.proto", fileDescriptorTypes) }
 
 var fileDescriptorTypes = []byte{
-	// 3603 bytes of a gzipped FileDescriptorProto
+	// 3635 bytes of a gzipped FileDescriptorProto
 	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xac, 0x59, 0xcd, 0x6f, 0x23, 0x47,
 	0x76, 0x17, 0x3f, 0x45, 0x3e, 0x52, 0x52, 0x4f, 0xcd, 0xec, 0x58, 0x43, 0x8f, 0x25, 0xba, 0xc7,
-	0xb3, 0x1e, 0x7b, 0x1d, 0xda, 0x96, 0x37, 0xc6, 0xac, 0x67, 0xb3, 0x76, 0x8b, 0xa4, 0x66, 0xb8,
-	0x23, 0x51, 0x44, 0x91, 0xd4, 0xc0, 0x08, 0x10, 0xa2, 0xd4, 0x5d, 0x22, 0xdb, 0x6a, 0x76, 0x33,
-	0xdd, 0x45, 0x69, 0x98, 0x20, 0xc0, 0x24, 0x87, 0x24, 0xd0, 0x29, 0xf7, 0x40, 0x58, 0x04, 0x09,
-	0x72, 0xcb, 0x39, 0x40, 0x4e, 0x3e, 0xfa, 0xb8, 0x41, 0x80, 0x60, 0xb1, 0x41, 0x84, 0x58, 0xf9,
-	0x07, 0x16, 0x08, 0x82, 0x3d, 0x24, 0x87, 0xa0, 0x3e, 0xba, 0xd9, 0xe4, 0x50, 0xf2, 0x4c, 0x76,
-	0x4f, 0x64, 0xbd, 0xfa, 0xbd, 0x57, 0xaf, 0xaa, 0x5e, 0xbd, 0xfa, 0xbd, 0x6a, 0x28, 0xb0, 0xc9,
-	0x88, 0x06, 0x95, 0x91, 0xef, 0x31, 0x0f, 0x21, 0xcb, 0x33, 0x8f, 0xa9, 0x5f, 0x09, 0x4e, 0x89,
-	0x3f, 0x3c, 0xb6, 0x59, 0xe5, 0xe4, 0xe3, 0xd2, 0x1d, 0x66, 0x0f, 0x69, 0xc0, 0xc8, 0x70, 0xf4,
-	0x61, 0xf4, 0x4f, 0xc2, 0x4b, 0x6f, 0x58, 0x63, 0x9f, 0x30, 0xdb, 0x73, 0x3f, 0x0c, 0xff, 0xa8,
-	0x8e, 0x5b, 0x7d, 0xaf, 0xef, 0x89, 0xbf, 0x1f, 0xf2, 0x7f, 0x52, 0xaa, 0x6f, 0xc2, 0xf2, 0x01,
-	0xf5, 0x03, 0xdb, 0x73, 0xd1, 0x2d, 0xc8, 0xd8, 0xae, 0x45, 0x9f, 0xaf, 0x27, 0xca, 0x89, 0x07,
-	0x69, 0x2c, 0x1b, 0xfa, 0xdf, 0x24, 0xa0, 0x60, 0xb8, 0xae, 0xc7, 0x84, 0xad, 0x00, 0x21, 0x48,
-	0xbb, 0x64, 0x48, 0x05, 0x28, 0x8f, 0xc5, 0x7f, 0x54, 0x85, 0xac, 0x43, 0x0e, 0xa9, 0x13, 0xac,
-	0x27, 0xcb, 0xa9, 0x07, 0x85, 0xad, 0x1f, 0x54, 0x5e, 0xf6, 0xb9, 0x12, 0x33, 0x52, 0xd9, 0x15,
-	0xe8, 0xba, 0xcb, 0xfc, 0x09, 0x56, 0xaa, 0xa5, 0x1f, 0x41, 0x21, 0x26, 0x46, 0x1a, 0xa4, 0x8e,
-	0xe9, 0x44, 0x0d, 0xc3, 0xff, 0x72, 0xff, 0x4e, 0x88, 0x33, 0xa6, 0xeb, 0x49, 0x21, 0x93, 0x8d,
-	0xcf, 0x92, 0x0f, 0x13, 0xfa, 0x97, 0x90, 0xc7, 0x34, 0xf0, 0xc6, 0xbe, 0x49, 0x03, 0xf4, 0x1e,
-	0xe4, 0x5d, 0xe2, 0x7a, 0x3d, 0x73, 0x34, 0x0e, 0x84, 0x7a, 0x6a, 0xbb, 0x78, 0x79, 0xb1, 0x99,
-	0x6b, 0x12, 0xd7, 0xab, 0xb6, 0xba, 0x01, 0xce, 0xf1, 0xee, 0xea, 0x68, 0x1c, 0xa0, 0xb7, 0xa1,
-	0x38, 0xa4, 0x43, 0xcf, 0x9f, 0xf4, 0x0e, 0x27, 0x8c, 0x06, 0xc2, 0x70, 0x0a, 0x17, 0xa4, 0x6c,
-	0x9b, 0x8b, 0xf4, 0xbf, 0x4a, 0xc0, 0xad, 0xd0, 0x36, 0xa6, 0x7f, 0x38, 0xb6, 0x7d, 0x3a, 0xa4,
-	0x2e, 0x0b, 0xd0, 0xef, 0x42, 0xd6, 0xb1, 0x87, 0x36, 0x93, 0x63, 0x14, 0xb6, 0xde, 0x5a, 0x34,
-	0xe7, 0xc8, 0x2b, 0xac, 0xc0, 0xc8, 0x80, 0xa2, 0x4f, 0x03, 0xea, 0x9f, 0xc8, 0x95, 0x10, 0x43,
-	0x7e, 0xa7, 0xf2, 0x8c, 0x8a, 0xbe, 0x03, 0xb9, 0x96, 0x43, 0xd8, 0x91, 0xe7, 0x0f, 0x91, 0x0e,
-	0x45, 0xe2, 0x9b, 0x03, 0x9b, 0x51, 0x93, 0x8d, 0xfd, 0x70, 0x57, 0x66, 0x64, 0xe8, 0x36, 0x24,
-	0x3d, 0x39, 0x50, 0x7e, 0x3b, 0x7b, 0x79, 0xb1, 0x99, 0xdc, 0x6f, 0xe3, 0xa4, 0x17, 0xe8, 0x8f,
-	0xe0, 0x46, 0xcb, 0x19, 0xf7, 0x6d, 0xb7, 0x46, 0x03, 0xd3, 0xb7, 0x47, 0xdc, 0x3a, 0xdf, 0x5e,
-	0x1e, 0x7c, 0xe1, 0xf6, 0xf2, 0xff, 0xd1, 0x96, 0x27, 0xa7, 0x5b, 0xae, 0xff, 0x45, 0x12, 0x6e,
-	0xd4, 0xdd, 0xbe, 0xed, 0xd2, 0xb8, 0xf6, 0x7d, 0x58, 0xa5, 0x42, 0xd8, 0x3b, 0x91, 0x41, 0xa5,
-	0xec, 0xac, 0x48, 0x69, 0x18, 0x69, 0x8d, 0xb9, 0x78, 0xf9, 0x78, 0xd1, 0xf4, 0x5f, 0xb2, 0xbe,
-	0x28, 0x6a, 0x50, 0x1d, 0x96, 0x47, 0x62, 0x12, 0xc1, 0x7a, 0x4a, 0xd8, 0xba, 0xbf, 0xc8, 0xd6,
-	0x4b, 0xf3, 0xdc, 0x4e, 0x7f, 0x73, 0xb1, 0xb9, 0x84, 0x43, 0xdd, 0xdf, 0x24, 0xf8, 0xfe, 0x33,
-	0x01, 0x6b, 0x4d, 0xcf, 0x9a, 0x59, 0x87, 0x12, 0xe4, 0x06, 0x5e, 0xc0, 0x62, 0x07, 0x25, 0x6a,
-	0xa3, 0x87, 0x90, 0x1b, 0xa9, 0xed, 0x53, 0xbb, 0x7f, 0x77, 0xb1, 0xcb, 0x12, 0x83, 0x23, 0x34,
-	0x7a, 0x04, 0x79, 0x3f, 0x8c, 0x89, 0xf5, 0xd4, 0xab, 0x04, 0xce, 0x14, 0x8f, 0x7e, 0x0f, 0xb2,
-	0x72, 0x13, 0xd6, 0xd3, 0x42, 0xf3, 0xfe, 0x2b, 0xad, 0x39, 0x56, 0x4a, 0xfa, 0x2f, 0x12, 0xa0,
-	0x61, 0x72, 0xc4, 0xf6, 0xe8, 0xf0, 0x90, 0xfa, 0x6d, 0x46, 0xd8, 0x38, 0x40, 0xb7, 0x21, 0xeb,
-	0x50, 0x62, 0x51, 0x5f, 0x4c, 0x32, 0x87, 0x55, 0x0b, 0x75, 0x79, 0x90, 0x13, 0x73, 0x40, 0x0e,
-	0x6d, 0xc7, 0x66, 0x13, 0x31, 0xcd, 0xd5, 0xc5, 0xbb, 0x3c, 0x6f, 0xb3, 0x82, 0x63, 0x8a, 0x78,
-	0xc6, 0x0c, 0x5a, 0x87, 0xe5, 0x21, 0x0d, 0x02, 0xd2, 0xa7, 0x62, 0xf6, 0x79, 0x1c, 0x36, 0xf5,
-	0x47, 0x50, 0x8c, 0xeb, 0xa1, 0x02, 0x2c, 0x77, 0x9b, 0x4f, 0x9b, 0xfb, 0xcf, 0x9a, 0xda, 0x12,
-	0x5a, 0x83, 0x42, 0xb7, 0x89, 0xeb, 0x46, 0xf5, 0x89, 0xb1, 0xbd, 0x5b, 0xd7, 0x12, 0x68, 0x05,
-	0xf2, 0xd3, 0x66, 0x52, 0xff, 0x59, 0x02, 0x80, 0x6f, 0xa0, 0x9a, 0xd4, 0x67, 0x90, 0x09, 0x18,
-	0x61, 0x72, 0xe3, 0x56, 0xb7, 0xde, 0x59, 0xe4, 0xf5, 0x14, 0x5e, 0xe1, 0x3f, 0x14, 0x4b, 0x95,
-	0xb8, 0x87, 0xc9, 0x79, 0x0f, 0x33, 0x02, 0x39, 0xeb, 0x5a, 0x0e, 0xd2, 0x35, 0xfe, 0x2f, 0x81,
-	0xf2, 0x90, 0xc1, 0x75, 0xa3, 0xf6, 0xa5, 0x96, 0x44, 0x1a, 0x14, 0x6b, 0x8d, 0x76, 0x75, 0xbf,
-	0xd9, 0xac, 0x57, 0x3b, 0xf5, 0x9a, 0x96, 0xd2, 0xef, 0x43, 0xa6, 0x31, 0x24, 0x7d, 0x8a, 0xee,
-	0xf2, 0x08, 0x38, 0xa2, 0x3e, 0x75, 0xcd, 0x30, 0xb0, 0xa6, 0x02, 0xfd, 0xe7, 0x79, 0xc8, 0xec,
-	0x79, 0x63, 0x97, 0xa1, 0xad, 0xd8, 0x29, 0x5e, 0xdd, 0xda, 0x58, 0x34, 0x05, 0x01, 0xac, 0x74,
-	0x26, 0x23, 0xaa, 0x4e, 0xf9, 0x6d, 0xc8, 0xca, 0x58, 0x51, 0xae, 0xab, 0x16, 0x97, 0x33, 0xe2,
-	0xf7, 0x29, 0x53, 0x8b, 0xae, 0x5a, 0xe8, 0x01, 0xe4, 0x7c, 0x4a, 0x2c, 0xcf, 0x75, 0x26, 0x22,
-	0xa4, 0x72, 0x32, 0xcd, 0x62, 0x4a, 0xac, 0x7d, 0xd7, 0x99, 0xe0, 0xa8, 0x17, 0x3d, 0x81, 0xe2,
-	0xa1, 0xed, 0x5a, 0x3d, 0x6f, 0x24, 0x73, 0x5e, 0xe6, 0xea, 0x00, 0x94, 0x5e, 0x6d, 0xdb, 0xae,
-	0xb5, 0x2f, 0xc1, 0xb8, 0x70, 0x38, 0x6d, 0xa0, 0x26, 0xac, 0x9e, 0x78, 0xce, 0x78, 0x48, 0x23,
-	0x5b, 0x59, 0x61, 0xeb, 0xdd, 0xab, 0x6d, 0x1d, 0x08, 0x7c, 0x68, 0x6d, 0xe5, 0x24, 0xde, 0x44,
-	0x4f, 0x61, 0x85, 0x0d, 0x47, 0x47, 0x41, 0x64, 0x6e, 0x59, 0x98, 0xfb, 0xfe, 0x35, 0x0b, 0xc6,
-	0xe1, 0xa1, 0xb5, 0x22, 0x8b, 0xb5, 0x4a, 0x7f, 0x96, 0x82, 0x42, 0xcc, 0x73, 0xd4, 0x86, 0xc2,
-	0xc8, 0xf7, 0x46, 0xa4, 0x2f, 0xf2, 0xb6, 0xda, 0x8b, 0x8f, 0x5f, 0x69, 0xd6, 0x95, 0xd6, 0x54,
-	0x11, 0xc7, 0xad, 0xe8, 0xe7, 0x49, 0x28, 0xc4, 0x3a, 0xd1, 0xfb, 0x90, 0xc3, 0x2d, 0xdc, 0x38,
-	0x30, 0x3a, 0x75, 0x6d, 0xa9, 0x74, 0xf7, 0xec, 0xbc, 0xbc, 0x2e, 0xac, 0xc5, 0x0d, 0xb4, 0x7c,
-	0xfb, 0x84, 0x87, 0xde, 0x03, 0x58, 0x0e, 0xa1, 0x89, 0xd2, 0x9b, 0x67, 0xe7, 0xe5, 0x37, 0xe6,
-	0xa1, 0x31, 0x24, 0x6e, 0x3f, 0x31, 0x70, 0xbd, 0xa6, 0x25, 0x17, 0x23, 0x71, 0x7b, 0x40, 0x7c,
-	0x6a, 0xa1, 0xef, 0x43, 0x56, 0x01, 0x53, 0xa5, 0xd2, 0xd9, 0x79, 0xf9, 0xf6, 0x3c, 0x70, 0x8a,
-	0xc3, 0xed, 0x5d, 0xe3, 0xa0, 0xae, 0xa5, 0x17, 0xe3, 0x70, 0xdb, 0x21, 0x27, 0x14, 0xbd, 0x03,
-	0x19, 0x09, 0xcb, 0x94, 0xee, 0x9c, 0x9d, 0x97, 0xbf, 0xf7, 0x92, 0x39, 0x8e, 0x2a, 0xad, 0xff,
-	0xe5, 0xdf, 0x6e, 0x2c, 0xfd, 0xd3, 0xdf, 0x6d, 0x68, 0xf3, 0xdd, 0xa5, 0xff, 0x4d, 0xc0, 0xca,
-	0xcc, 0x96, 0x23, 0x1d, 0xb2, 0xae, 0x67, 0x7a, 0x23, 0x99, 0xce, 0x73, 0xdb, 0x70, 0x79, 0xb1,
-	0x99, 0x6d, 0x7a, 0x55, 0x6f, 0x34, 0xc1, 0xaa, 0x07, 0x3d, 0x9d, 0xbb, 0x90, 0x3e, 0x79, 0xc5,
-	0x78, 0x5a, 0x78, 0x25, 0x7d, 0x0e, 0x2b, 0x96, 0x6f, 0x9f, 0x50, 0xbf, 0x67, 0x7a, 0xee, 0x91,
-	0xdd, 0x57, 0xa9, 0xba, 0xb4, 0xc8, 0x66, 0x4d, 0x00, 0x71, 0x51, 0x2a, 0x54, 0x05, 0xfe, 0x37,
-	0xb8, 0x8c, 0x4a, 0x07, 0x50, 0x8c, 0x47, 0x28, 0x7a, 0x0b, 0x20, 0xb0, 0xff, 0x88, 0x2a, 0x7e,
-	0x23, 0xd8, 0x10, 0xce, 0x73, 0x89, 0x60, 0x37, 0xe8, 0x5d, 0x48, 0x0f, 0x3d, 0x4b, 0xda, 0xc9,
-	0x6c, 0xdf, 0xe4, 0x77, 0xe2, 0x2f, 0x2f, 0x36, 0x0b, 0x5e, 0x50, 0xd9, 0xb1, 0x1d, 0xba, 0xe7,
-	0x59, 0x14, 0x0b, 0x80, 0x7e, 0x02, 0x69, 0x9e, 0x2a, 0xd0, 0x9b, 0x90, 0xde, 0x6e, 0x34, 0x6b,
-	0xda, 0x52, 0xe9, 0xc6, 0xd9, 0x79, 0x79, 0x45, 0x2c, 0x09, 0xef, 0xe0, 0xb1, 0x8b, 0x36, 0x21,
-	0x7b, 0xb0, 0xbf, 0xdb, 0xdd, 0xe3, 0xe1, 0x75, 0xf3, 0xec, 0xbc, 0xbc, 0x16, 0x75, 0xcb, 0x45,
-	0x43, 0x6f, 0x41, 0xa6, 0xb3, 0xd7, 0xda, 0x69, 0x6b, 0xc9, 0x12, 0x3a, 0x3b, 0x2f, 0xaf, 0x46,
-	0xfd, 0xc2, 0xe7, 0xd2, 0x0d, 0xb5, 0xab, 0xf9, 0x48, 0xae, 0xff, 0x4f, 0x12, 0x56, 0x30, 0xe7,
-	0xb7, 0x3e, 0x6b, 0x79, 0x8e, 0x6d, 0x4e, 0x50, 0x0b, 0xf2, 0xa6, 0xe7, 0x5a, 0x76, 0xec, 0x4c,
-	0x6d, 0x5d, 0x71, 0x09, 0x4e, 0xb5, 0xc2, 0x56, 0x35, 0xd4, 0xc4, 0x53, 0x23, 0x68, 0x0b, 0x32,
-	0x16, 0x75, 0xc8, 0xe4, 0xba, 0xdb, 0xb8, 0xa6, 0xb8, 0x34, 0x96, 0x50, 0xc1, 0x1c, 0xc9, 0xf3,
-	0x1e, 0x61, 0x8c, 0x0e, 0x47, 0x4c, 0xde, 0xc6, 0x69, 0x5c, 0x18, 0x92, 0xe7, 0x86, 0x12, 0xa1,
-	0x1f, 0x42, 0xf6, 0xd4, 0x76, 0x2d, 0xef, 0x54, 0x5d, 0xb8, 0xd7, 0xdb, 0x55, 0x58, 0xfd, 0x8c,
-	0xdf, 0xb3, 0x73, 0xce, 0xf2, 0x55, 0x6f, 0xee, 0x37, 0xeb, 0xe1, 0xaa, 0xab, 0xfe, 0x7d, 0xb7,
-	0xe9, 0xb9, 0xfc, 0xc4, 0xc0, 0x7e, 0xb3, 0xb7, 0x63, 0x34, 0x76, 0xbb, 0x98, 0xaf, 0xfc, 0xad,
-	0xb3, 0xf3, 0xb2, 0x16, 0x41, 0x76, 0x88, 0xed, 0x70, 0x12, 0x78, 0x07, 0x52, 0x46, 0xf3, 0x4b,
-	0x2d, 0x59, 0xd2, 0xce, 0xce, 0xcb, 0xc5, 0xa8, 0xdb, 0x70, 0x27, 0xd3, 0xc3, 0x34, 0x3f, 0xae,
-	0xfe, 0xef, 0x49, 0x28, 0x76, 0x47, 0x16, 0x61, 0x54, 0x46, 0x26, 0x2a, 0x43, 0x61, 0x44, 0x7c,
-	0xe2, 0x38, 0xd4, 0xb1, 0x83, 0xa1, 0x2a, 0x14, 0xe2, 0x22, 0xf4, 0xf0, 0x35, 0x16, 0x53, 0x91,
-	0x30, 0xb5, 0xa4, 0x5d, 0x58, 0x3d, 0x92, 0xce, 0xf6, 0x88, 0x29, 0x76, 0x37, 0x25, 0x76, 0xb7,
-	0xb2, 0xc8, 0x44, 0xdc, 0xab, 0x8a, 0x9a, 0xa3, 0x21, 0xb4, 0xf0, 0xca, 0x51, 0xbc, 0x89, 0x3e,
-	0x85, 0xe5, 0xa1, 0xe7, 0xda, 0xcc, 0xf3, 0x5f, 0x69, 0x1f, 0x42, 0x30, 0x7a, 0x1f, 0x6e, 0xf0,
-	0x1d, 0x0e, 0x5d, 0x12, 0xdd, 0xe2, 0xe6, 0x4a, 0xe2, 0xb5, 0x21, 0x79, 0xae, 0xc6, 0xc4, 0x5c,
-	0xac, 0x7f, 0x0a, 0x2b, 0x33, 0x3e, 0xf0, 0xdb, 0xbc, 0x65, 0x74, 0xdb, 0x75, 0x6d, 0x09, 0x15,
-	0x21, 0x57, 0xdd, 0x6f, 0x76, 0x1a, 0xcd, 0x2e, 0xa7, 0x1e, 0x45, 0xc8, 0xe1, 0xfd, 0xdd, 0xdd,
-	0x6d, 0xa3, 0xfa, 0x54, 0x4b, 0xea, 0xff, 0x1d, 0xad, 0xaf, 0xe2, 0x1e, 0xdb, 0xb3, 0xdc, 0xe3,
-	0x83, 0xab, 0xa7, 0xae, 0xd8, 0xc7, 0xb4, 0x11, 0x71, 0x90, 0x1f, 0x03, 0x88, 0x6d, 0xa4, 0x56,
-	0x8f, 0xb0, 0xeb, 0xea, 0x8b, 0x4e, 0x58, 0x39, 0xe2, 0xbc, 0x52, 0x30, 0x18, 0xfa, 0x02, 0x8a,
-	0xa6, 0x37, 0x1c, 0x39, 0x54, 0xe9, 0xa7, 0x5e, 0x45, 0xbf, 0x10, 0xa9, 0x18, 0x2c, 0xce, 0x81,
-	0xd2, 0xb3, 0x1c, 0xe8, 0xcf, 0x13, 0x50, 0x88, 0x39, 0x3c, 0x4b, 0x85, 0x8a, 0x90, 0xeb, 0xb6,
-	0x6a, 0x46, 0xa7, 0xd1, 0x7c, 0xac, 0x25, 0x10, 0x40, 0x56, 0x2c, 0x60, 0x4d, 0x4b, 0x72, 0xba,
-	0x56, 0xdd, 0xdf, 0x6b, 0xed, 0xd6, 0x05, 0x19, 0x42, 0xb7, 0x40, 0x0b, 0x97, 0xb0, 0xd7, 0xee,
-	0x18, 0x98, 0x4b, 0xd3, 0xe8, 0x26, 0xac, 0x45, 0x52, 0xa5, 0x99, 0x41, 0xb7, 0x01, 0x45, 0xc2,
-	0xa9, 0x89, 0xac, 0xfe, 0x27, 0xb0, 0x56, 0xf5, 0x5c, 0x46, 0x6c, 0x37, 0xa2, 0xb2, 0x5b, 0x7c,
-	0xde, 0x4a, 0xd4, 0xb3, 0x2d, 0x99, 0x6d, 0xb7, 0xd7, 0x2e, 0x2f, 0x36, 0x0b, 0x11, 0xb4, 0x51,
-	0xe3, 0x33, 0x0d, 0x1b, 0x16, 0x3f, 0x53, 0x23, 0xdb, 0x52, 0xc9, 0x73, 0xf9, 0xf2, 0x62, 0x33,
-	0xd5, 0x6a, 0xd4, 0x30, 0x97, 0xa1, 0x37, 0x21, 0x4f, 0x9f, 0xdb, 0xac, 0x67, 0xf2, 0xec, 0xca,
-	0xd7, 0x30, 0x83, 0x73, 0x5c, 0x50, 0xe5, 0xc9, 0xf4, 0x4f, 0x93, 0x00, 0x1d, 0x12, 0x1c, 0xab,
-	0xa1, 0x1f, 0x41, 0x3e, 0x2a, 0xe2, 0xaf, 0x2b, 0x26, 0x63, 0xfb, 0x15, 0xe1, 0xd1, 0x27, 0x61,
-	0xc4, 0x48, 0x8e, 0xbd, 0x58, 0x51, 0x8d, 0xb5, 0x88, 0xa6, 0xce, 0x12, 0x69, 0x7e, 0xd7, 0x50,
-	0xdf, 0x57, 0x1b, 0xc7, 0xff, 0xa2, 0xaa, 0xc8, 0xb7, 0x72, 0xce, 0x8a, 0xb9, 0xdd, 0x5b, 0x34,
-	0xc8, 0xdc, 0x82, 0x3e, 0x59, 0xc2, 0x53, 0xbd, 0x6d, 0x0d, 0x56, 0xfd, 0xb1, 0xcb, 0xbd, 0xee,
-	0x05, 0xa2, 0x5b, 0xb7, 0xe1, 0x8d, 0x26, 0x65, 0xa7, 0x9e, 0x7f, 0x6c, 0x30, 0x46, 0xcc, 0x01,
-	0x2f, 0xaa, 0x55, 0x92, 0x99, 0x12, 0xce, 0xc4, 0x0c, 0xe1, 0x5c, 0x87, 0x65, 0xe2, 0xd8, 0x24,
-	0xa0, 0xf2, 0x96, 0xce, 0xe3, 0xb0, 0xc9, 0x69, 0x31, 0xb1, 0x2c, 0x9f, 0x06, 0x01, 0x95, 0x65,
-	0x60, 0x1e, 0x4f, 0x05, 0xfa, 0xbf, 0x24, 0x01, 0x1a, 0x2d, 0x63, 0x4f, 0x99, 0xaf, 0x41, 0xf6,
-	0x88, 0x0c, 0x6d, 0x67, 0x72, 0xdd, 0x21, 0x9b, 0xe2, 0x2b, 0x86, 0x34, 0xb4, 0x23, 0x74, 0xb0,
-	0xd2, 0x15, 0x6c, 0x79, 0x7c, 0xe8, 0x52, 0x16, 0xb1, 0x65, 0xd1, 0xe2, 0x57, 0xb3, 0x4f, 0xdc,
-	0x68, 0x61, 0x65, 0x83, 0xbb, 0xde, 0x27, 0x8c, 0x9e, 0x92, 0x49, 0x78, 0x26, 0x54, 0x13, 0x3d,
-	0xe1, 0x2c, 0x9a, 0x17, 0xf7, 0xd4, 0x5a, 0xcf, 0x08, 0xee, 0xf1, 0x5d, 0xfe, 0x60, 0x05, 0x97,
-	0xa4, 0x23, 0xd2, 0x2e, 0x3d, 0x12, 0x37, 0xe5, 0xb4, 0xeb, 0xb5, 0x8a, 0xd8, 0x8f, 0x60, 0x65,
-	0x66, 0x9e, 0x2f, 0x95, 0x29, 0x8d, 0xd6, 0xc1, 0x0f, 0xb5, 0xb4, 0xfa, 0xf7, 0xa9, 0x96, 0xd5,
-	0xff, 0x2b, 0x01, 0xd0, 0xf2, 0xfc, 0x70, 0xd3, 0x16, 0x3f, 0x0b, 0xe5, 0xc4, 0x23, 0x93, 0xe9,
-	0x39, 0x2a, 0x3c, 0x17, 0xf2, 0xf4, 0xa9, 0x15, 0x4e, 0x7b, 0x05, 0x1c, 0x47, 0x8a, 0x68, 0x13,
-	0x0a, 0x72, 0xff, 0x7b, 0x23, 0xcf, 0x97, 0xf9, 0x68, 0x05, 0x83, 0x14, 0x71, 0x4d, 0x74, 0x1f,
-	0x56, 0x47, 0xe3, 0x43, 0xc7, 0x0e, 0x06, 0xd4, 0x92, 0x98, 0xb4, 0xc0, 0xac, 0x44, 0x52, 0x0e,
-	0xd3, 0x6b, 0x90, 0x0b, 0xad, 0xa3, 0x75, 0x48, 0x75, 0xaa, 0x2d, 0x6d, 0xa9, 0xb4, 0x76, 0x76,
-	0x5e, 0x2e, 0x84, 0xe2, 0x4e, 0xb5, 0xc5, 0x7b, 0xba, 0xb5, 0x96, 0x96, 0x98, 0xed, 0xe9, 0xd6,
-	0x5a, 0xa5, 0x34, 0xbf, 0x25, 0xf5, 0xbf, 0x4e, 0x40, 0x56, 0x72, 0xb6, 0x85, 0x33, 0x36, 0x60,
-	0x39, 0xac, 0x24, 0x24, 0x91, 0x7c, 0xf7, 0x6a, 0xd2, 0x57, 0x51, 0x1c, 0x4d, 0xee, 0x63, 0xa8,
-	0x57, 0xfa, 0x0c, 0x8a, 0xf1, 0x8e, 0xd7, 0xda, 0xc5, 0x3f, 0x86, 0x02, 0x0f, 0x94, 0x90, 0xfc,
-	0x6d, 0x41, 0x56, 0xf2, 0x4a, 0x95, 0x55, 0xae, 0x63, 0xa0, 0x0a, 0x89, 0x1e, 0xc2, 0xb2, 0x64,
-	0xad, 0xe1, 0x7b, 0xca, 0xc6, 0xf5, 0xe1, 0x88, 0x43, 0xb8, 0xfe, 0x39, 0xa4, 0x5b, 0x94, 0xfa,
-	0xe8, 0x1e, 0x2c, 0xbb, 0x9e, 0x45, 0xa7, 0x49, 0x54, 0x11, 0x6e, 0x8b, 0x36, 0x6a, 0x9c, 0x70,
-	0x5b, 0xb4, 0x61, 0xf1, 0xc5, 0xe3, 0x07, 0x34, 0x7c, 0x52, 0xe2, 0xff, 0xf5, 0x0e, 0x14, 0x9f,
-	0x51, 0xbb, 0x3f, 0x60, 0xd4, 0x12, 0x86, 0x3e, 0x80, 0xf4, 0x88, 0x46, 0xce, 0xaf, 0x2f, 0x0c,
-	0x1d, 0x4a, 0x7d, 0x2c, 0x50, 0xfc, 0x40, 0x9e, 0x0a, 0x6d, 0xf5, 0x8a, 0xa7, 0x5a, 0xfa, 0x3f,
-	0x24, 0x61, 0xb5, 0x11, 0x04, 0x63, 0xe2, 0x9a, 0xe1, 0x2d, 0xfb, 0x93, 0xd9, 0x5b, 0xf6, 0xc1,
-	0xc2, 0x19, 0xce, 0xa8, 0xcc, 0x56, 0xf9, 0x2a, 0x49, 0x26, 0xa3, 0x24, 0xa9, 0x7f, 0x93, 0x08,
-	0xcb, 0xfb, 0xfb, 0xb1, 0x73, 0x53, 0x5a, 0x3f, 0x3b, 0x2f, 0xdf, 0x8a, 0x5b, 0xa2, 0x5d, 0xf7,
-	0xd8, 0xf5, 0x4e, 0x5d, 0xf4, 0x36, 0x2f, 0xf7, 0x9b, 0xf5, 0x67, 0x5a, 0xa2, 0x74, 0xfb, 0xec,
-	0xbc, 0x8c, 0x66, 0x40, 0x98, 0xba, 0xf4, 0x94, 0x5b, 0x6a, 0xd5, 0x9b, 0x35, 0x7e, 0x1f, 0x26,
-	0x17, 0x58, 0x6a, 0x51, 0xd7, 0xb2, 0xdd, 0x3e, 0xba, 0x07, 0xd9, 0x46, 0xbb, 0xdd, 0x15, 0x05,
-	0xd8, 0x1b, 0x67, 0xe7, 0xe5, 0x9b, 0x33, 0x28, 0xde, 0xa0, 0x16, 0x07, 0x71, 0x82, 0xc8, 0x6f,
-	0xca, 0x05, 0x20, 0xce, 0x5d, 0xa8, 0xa5, 0x22, 0xfc, 0xdf, 0x92, 0xa0, 0x19, 0xa6, 0x49, 0x47,
-	0x8c, 0xf7, 0x2b, 0xd2, 0xdd, 0x81, 0xdc, 0x88, 0xff, 0xb3, 0x45, 0x11, 0xc1, 0xc3, 0xe2, 0xe1,
-	0xc2, 0x27, 0xde, 0x39, 0xbd, 0x0a, 0xf6, 0x1c, 0x6a, 0x58, 0x43, 0x3b, 0x08, 0x78, 0x71, 0x29,
-	0x64, 0x38, 0xb2, 0x54, 0xfa, 0x55, 0x02, 0x6e, 0x2e, 0x40, 0xa0, 0x8f, 0x20, 0xed, 0x7b, 0x4e,
-	0xb8, 0x3d, 0x77, 0xaf, 0x7a, 0x80, 0xe1, 0xaa, 0x58, 0x20, 0xd1, 0x06, 0x00, 0x19, 0x33, 0x8f,
-	0x88, 0xf1, 0xc5, 0xc6, 0xe4, 0x70, 0x4c, 0x82, 0x9e, 0x41, 0x36, 0xa0, 0xa6, 0x4f, 0x43, 0x3e,
-	0xf3, 0xf9, 0xff, 0xd7, 0xfb, 0x4a, 0x5b, 0x98, 0xc1, 0xca, 0x5c, 0xa9, 0x02, 0x59, 0x29, 0xe1,
-	0x11, 0x6d, 0x11, 0x46, 0x84, 0xd3, 0x45, 0x2c, 0xfe, 0xf3, 0x40, 0x21, 0x4e, 0x3f, 0x0c, 0x14,
-	0xe2, 0xf4, 0xf5, 0x9f, 0x25, 0x01, 0xea, 0xcf, 0x19, 0xf5, 0x5d, 0xe2, 0x54, 0x0d, 0x54, 0x8f,
-	0x65, 0x48, 0x39, 0xdb, 0xf7, 0x16, 0x3e, 0xcb, 0x45, 0x1a, 0x95, 0xaa, 0xb1, 0x20, 0x47, 0xde,
-	0x81, 0xd4, 0xd8, 0x77, 0xd4, 0x13, 0xaf, 0x20, 0x22, 0x5d, 0xbc, 0x8b, 0xb9, 0x0c, 0xd5, 0xa7,
-	0x19, 0x29, 0x75, 0xf5, 0xdb, 0x7c, 0x6c, 0x80, 0xdf, 0x7e, 0x56, 0xfa, 0x00, 0x60, 0xea, 0x35,
-	0xda, 0x80, 0x4c, 0x75, 0xa7, 0xdd, 0xde, 0xd5, 0x96, 0x64, 0x8d, 0x38, 0xed, 0x12, 0x62, 0xfd,
-	0xef, 0x13, 0x90, 0xab, 0x1a, 0xea, 0x56, 0xd9, 0x01, 0x4d, 0xe4, 0x12, 0x93, 0xfa, 0xac, 0x47,
-	0x9f, 0x8f, 0x6c, 0x7f, 0xa2, 0xd2, 0xc1, 0xf5, 0x2c, 0x7e, 0x95, 0x6b, 0x55, 0xa9, 0xcf, 0xea,
-	0x42, 0x07, 0x61, 0x28, 0x52, 0x35, 0xc5, 0x9e, 0x49, 0xc2, 0xe4, 0xbc, 0x71, 0xfd, 0x52, 0x48,
-	0xf6, 0x37, 0x6d, 0x07, 0xb8, 0x10, 0x1a, 0xa9, 0x92, 0x40, 0x3f, 0x80, 0x9b, 0xfb, 0xbe, 0x39,
-	0xa0, 0x01, 0x93, 0x83, 0x2a, 0x97, 0x3f, 0x87, 0xbb, 0x8c, 0x04, 0xc7, 0xbd, 0x81, 0x1d, 0x30,
-	0xcf, 0x9f, 0xf4, 0x7c, 0xca, 0xa8, 0xcb, 0xfb, 0x7b, 0xe2, 0x0b, 0x80, 0xaa, 0xc1, 0xef, 0x70,
-	0xcc, 0x13, 0x09, 0xc1, 0x21, 0x62, 0x97, 0x03, 0xf4, 0x06, 0x14, 0x39, 0x61, 0xab, 0xd1, 0x23,
-	0x32, 0x76, 0x58, 0x80, 0x7e, 0x04, 0xe0, 0x78, 0xfd, 0xde, 0x2b, 0x67, 0xf2, 0xbc, 0xe3, 0xf5,
-	0xe5, 0x5f, 0xfd, 0xf7, 0x41, 0xab, 0xd9, 0xc1, 0x88, 0x30, 0x73, 0x10, 0x3e, 0x2e, 0xa0, 0xc7,
-	0xa0, 0x0d, 0x28, 0xf1, 0xd9, 0x21, 0x25, 0xac, 0x37, 0xa2, 0xbe, 0xed, 0x59, 0xaf, 0xb4, 0xa4,
-	0x6b, 0x91, 0x56, 0x4b, 0x28, 0xe9, 0xbf, 0x4e, 0x00, 0x60, 0x72, 0x14, 0x12, 0x80, 0x1f, 0xc0,
-	0x8d, 0xc0, 0x25, 0xa3, 0x60, 0xe0, 0xb1, 0x9e, 0xed, 0x32, 0xea, 0x9f, 0x10, 0x47, 0x15, 0x88,
-	0x5a, 0xd8, 0xd1, 0x50, 0x72, 0xf4, 0x01, 0xa0, 0x63, 0x4a, 0x47, 0x3d, 0xcf, 0xb1, 0x7a, 0x61,
-	0xa7, 0xfc, 0x44, 0x91, 0xc6, 0x1a, 0xef, 0xd9, 0x77, 0xac, 0x76, 0x28, 0x47, 0xdb, 0xb0, 0xc1,
-	0x57, 0x80, 0xba, 0xcc, 0xb7, 0x69, 0xd0, 0x3b, 0xf2, 0xfc, 0x5e, 0xe0, 0x78, 0xa7, 0xbd, 0x23,
-	0xcf, 0x71, 0xbc, 0x53, 0xea, 0x87, 0xe5, 0x77, 0xc9, 0xf1, 0xfa, 0x75, 0x09, 0xda, 0xf1, 0xfc,
-	0xb6, 0xe3, 0x9d, 0xee, 0x84, 0x08, 0xce, 0x12, 0xa6, 0xd3, 0x66, 0xb6, 0x79, 0x1c, 0xb2, 0x84,
-	0x48, 0xda, 0xb1, 0xcd, 0x63, 0x74, 0x0f, 0x56, 0xa8, 0x43, 0x45, 0x11, 0x27, 0x51, 0x19, 0x81,
-	0x2a, 0x86, 0x42, 0x0e, 0xd2, 0x7f, 0x07, 0xf2, 0x2d, 0x87, 0x98, 0xe2, 0x43, 0x10, 0x2f, 0x89,
-	0x4d, 0xcf, 0xe5, 0x41, 0x60, 0xbb, 0x4c, 0x66, 0xc7, 0x3c, 0x8e, 0x8b, 0xf4, 0x9f, 0x00, 0xfc,
-	0xd4, 0xb3, 0xdd, 0x8e, 0x77, 0x4c, 0x5d, 0xf1, 0x66, 0xce, 0x59, 0xaf, 0xda, 0xca, 0x3c, 0x56,
-	0x2d, 0xc1, 0xc9, 0x89, 0x4b, 0xfa, 0xd4, 0x8f, 0x9e, 0x8e, 0x65, 0x93, 0x5f, 0x2e, 0x59, 0xec,
-	0x79, 0xac, 0x6a, 0xa0, 0x32, 0x64, 0x4d, 0xd2, 0x0b, 0x4f, 0x5e, 0x71, 0x3b, 0x7f, 0x79, 0xb1,
-	0x99, 0xa9, 0x1a, 0x4f, 0xe9, 0x04, 0x67, 0x4c, 0xf2, 0x94, 0x4e, 0xf8, 0xed, 0x6b, 0x12, 0x71,
-	0x5e, 0x84, 0x99, 0xa2, 0xbc, 0x7d, 0xab, 0x06, 0x3f, 0x0c, 0x38, 0x6b, 0x12, 0xfe, 0x8b, 0x3e,
-	0x82, 0xa2, 0x02, 0xf5, 0x06, 0x24, 0x18, 0x48, 0xae, 0xba, 0xbd, 0x7a, 0x79, 0xb1, 0x09, 0x12,
-	0xf9, 0x84, 0x04, 0x03, 0x0c, 0x12, 0xcd, 0xff, 0xa3, 0x3a, 0x14, 0xbe, 0xf2, 0x6c, 0xb7, 0xc7,
-	0xc4, 0x24, 0x54, 0x25, 0xbd, 0xf0, 0xfc, 0x4c, 0xa7, 0xaa, 0xca, 0x7b, 0xf8, 0x2a, 0x92, 0xe8,
-	0xff, 0x9a, 0x80, 0x02, 0xb7, 0x69, 0x1f, 0xd9, 0x26, 0xbf, 0x2d, 0x5f, 0x3f, 0xd3, 0xdf, 0x81,
-	0x94, 0x19, 0xf8, 0x6a, 0x6e, 0x22, 0xd5, 0x55, 0xdb, 0x18, 0x73, 0x19, 0xfa, 0x02, 0xb2, 0xb2,
-	0xb8, 0x50, 0x49, 0x5e, 0xff, 0xee, 0x7b, 0x5d, 0xb9, 0xa8, 0xf4, 0xc4, 0x5e, 0x4e, 0xbd, 0x13,
-	0xb3, 0x2c, 0xe2, 0xb8, 0x08, 0xdd, 0x86, 0xa4, 0xe9, 0x8a, 0xa0, 0x50, 0xdf, 0xd2, 0xaa, 0x4d,
-	0x9c, 0x34, 0x5d, 0xfd, 0x9f, 0x13, 0xb0, 0x52, 0x77, 0x4d, 0x7f, 0x22, 0x92, 0x24, 0xdf, 0x88,
-	0xbb, 0x90, 0x0f, 0xc6, 0x87, 0xc1, 0x24, 0x60, 0x74, 0x18, 0x3e, 0xd5, 0x47, 0x02, 0xd4, 0x80,
-	0x3c, 0x71, 0xfa, 0x9e, 0x6f, 0xb3, 0xc1, 0x50, 0x71, 0xe3, 0xc5, 0x89, 0x39, 0x6e, 0xb3, 0x62,
-	0x84, 0x2a, 0x78, 0xaa, 0x1d, 0xa6, 0xe2, 0x94, 0x70, 0x56, 0xa4, 0xe2, 0xb7, 0xa1, 0xe8, 0x90,
-	0x21, 0xa7, 0xc2, 0x3d, 0x5e, 0x72, 0x89, 0x79, 0xa4, 0x71, 0x41, 0xc9, 0x78, 0x19, 0xa9, 0xeb,
-	0x90, 0x8f, 0x8c, 0xa1, 0x35, 0x28, 0x18, 0xf5, 0x76, 0xef, 0xe3, 0xad, 0x87, 0xbd, 0xc7, 0xd5,
-	0x3d, 0x6d, 0x49, 0x31, 0x81, 0x7f, 0x4c, 0xc0, 0xca, 0x9e, 0x8c, 0x41, 0x45, 0x9c, 0xee, 0xc1,
-	0xb2, 0x4f, 0x8e, 0x58, 0x48, 0xed, 0xd2, 0x32, 0xb8, 0x78, 0x12, 0xe0, 0xd4, 0x8e, 0x77, 0x2d,
-	0xa6, 0x76, 0xb1, 0x0f, 0x45, 0xa9, 0x6b, 0x3f, 0x14, 0xa5, 0x7f, 0x2b, 0x1f, 0x8a, 0xf4, 0x5f,
-	0x26, 0x60, 0x4d, 0x5d, 0xd4, 0xe1, 0xc7, 0x11, 0xf4, 0x1e, 0xe4, 0xe5, 0x9d, 0x3d, 0x25, 0xa6,
-	0xe2, 0x7b, 0x85, 0xc4, 0x35, 0x6a, 0x38, 0x27, 0xbb, 0x1b, 0x16, 0xfa, 0x71, 0xec, 0x55, 0xf4,
-	0x0a, 0x7a, 0x38, 0x67, 0xbd, 0x32, 0x7d, 0x2a, 0xbd, 0xf2, 0x7b, 0xc9, 0x26, 0x14, 0x94, 0x03,
-	0xa2, 0x6c, 0x90, 0x75, 0x20, 0x48, 0x51, 0x93, 0x0c, 0xa9, 0x7e, 0x1f, 0xd2, 0xdc, 0x0c, 0x02,
-	0xc8, 0xb6, 0xbf, 0x6c, 0x77, 0xea, 0x7b, 0xb2, 0xf2, 0xda, 0x69, 0x88, 0x8f, 0x56, 0xcb, 0x90,
-	0xaa, 0x37, 0x0f, 0xb4, 0xe4, 0xfb, 0xbf, 0x4e, 0x41, 0x3e, 0xaa, 0xe8, 0xf9, 0x79, 0xe0, 0x34,
-	0x72, 0x49, 0xbe, 0xeb, 0x45, 0xf2, 0xa6, 0x20, 0x90, 0x79, 0x63, 0x77, 0x77, 0xbf, 0x6a, 0x74,
-	0xea, 0x35, 0xed, 0x0b, 0xc9, 0x33, 0x23, 0x80, 0xe1, 0x38, 0x1e, 0x8f, 0x68, 0x0b, 0xe9, 0x53,
-	0x9e, 0xf9, 0x42, 0xbd, 0x1e, 0x46, 0xa8, 0x90, 0x64, 0xbe, 0x03, 0x39, 0xa3, 0xdd, 0x6e, 0x3c,
-	0x6e, 0xd6, 0x6b, 0xda, 0xd7, 0x89, 0xd2, 0xf7, 0xce, 0xce, 0xcb, 0x37, 0xa6, 0xa6, 0x82, 0xc0,
-	0xee, 0xbb, 0xd4, 0x12, 0xa8, 0x6a, 0xb5, 0xde, 0xe2, 0xe3, 0xbd, 0x48, 0xce, 0xa3, 0x04, 0xbb,
-	0x12, 0x5f, 0x02, 0xf2, 0x2d, 0x5c, 0x6f, 0x19, 0x98, 0x8f, 0xf8, 0x75, 0x72, 0xce, 0xaf, 0x96,
-	0x4f, 0x47, 0xc4, 0xe7, 0x63, 0x6e, 0x84, 0x5f, 0xc4, 0x5e, 0xa4, 0xe4, 0x6b, 0xf1, 0xf4, 0x19,
-	0x83, 0x12, 0x6b, 0xc2, 0x47, 0x13, 0xcf, 0x3f, 0xc2, 0x4c, 0x6a, 0x6e, 0xb4, 0x36, 0x23, 0x3e,
-	0xe3, 0x56, 0x74, 0x58, 0xc6, 0xdd, 0x66, 0x53, 0xcc, 0x2e, 0x3d, 0x37, 0x3b, 0x3c, 0x76, 0x5d,
-	0x8e, 0xb9, 0x0f, 0xb9, 0xf0, 0x75, 0x48, 0xfb, 0x3a, 0x3d, 0xe7, 0x50, 0x35, 0x7c, 0xda, 0x12,
-	0x03, 0x3e, 0xe9, 0x76, 0xc4, 0x07, 0xbb, 0x17, 0x99, 0xf9, 0x01, 0x07, 0x63, 0x66, 0x71, 0x66,
-	0x5f, 0x8e, 0xa8, 0xf6, 0xd7, 0x19, 0xc9, 0x70, 0x22, 0x8c, 0xe4, 0xd9, 0xdc, 0x0e, 0xae, 0xff,
-	0x54, 0x7e, 0xdb, 0x7b, 0x91, 0x9d, 0xb3, 0x83, 0xe9, 0x57, 0xd4, 0x64, 0xd4, 0x9a, 0x3e, 0x86,
-	0x47, 0x5d, 0xef, 0xff, 0x01, 0xe4, 0xc2, 0x6c, 0x88, 0x36, 0x20, 0xfb, 0x6c, 0x1f, 0x3f, 0xad,
-	0x63, 0x6d, 0x49, 0xae, 0x4e, 0xd8, 0xf3, 0x4c, 0x5e, 0x27, 0x65, 0x58, 0xde, 0x33, 0x9a, 0xc6,
-	0xe3, 0x3a, 0x0e, 0x1f, 0xe3, 0x43, 0x80, 0x3a, 0xd2, 0x25, 0x4d, 0x0d, 0x10, 0xd9, 0xdc, 0xbe,
-	0xfb, 0xcd, 0xb7, 0x1b, 0x4b, 0xbf, 0xf8, 0x76, 0x63, 0xe9, 0x57, 0xdf, 0x6e, 0x24, 0x5e, 0x5c,
-	0x6e, 0x24, 0xbe, 0xb9, 0xdc, 0x48, 0xfc, 0xfc, 0x72, 0x23, 0xf1, 0x1f, 0x97, 0x1b, 0x89, 0xc3,
-	0xac, 0xa0, 0x9b, 0x9f, 0xfc, 0x5f, 0x00, 0x00, 0x00, 0xff, 0xff, 0x79, 0xf6, 0x7c, 0x35, 0x9d,
+	0xb3, 0x1e, 0x7b, 0x1d, 0xda, 0x96, 0x77, 0x8d, 0x59, 0xcf, 0x66, 0xed, 0x16, 0x49, 0xcd, 0x70,
+	0x47, 0xa2, 0x88, 0x22, 0x39, 0x03, 0x23, 0x41, 0x88, 0x52, 0x77, 0x89, 0x6a, 0xab, 0xd9, 0xc5,
+	0x74, 0x17, 0xa5, 0x61, 0x82, 0x00, 0x93, 0x1c, 0x92, 0x40, 0xa7, 0xdc, 0x03, 0x61, 0x11, 0x24,
+	0xc8, 0x2d, 0xe7, 0x00, 0x39, 0xf9, 0xe8, 0xe3, 0x06, 0x01, 0x82, 0xc5, 0x06, 0x19, 0xc4, 0xca,
+	0x3f, 0xb0, 0x40, 0x10, 0xec, 0x21, 0x39, 0x04, 0xf5, 0xd1, 0xcd, 0x8f, 0xe1, 0xc8, 0x72, 0xd6,
+	0x27, 0xb2, 0x5e, 0xfd, 0xde, 0xab, 0x57, 0x55, 0xaf, 0x5e, 0xfd, 0x5e, 0x35, 0x14, 0xf8, 0x78,
+	0x48, 0xc3, 0xca, 0x30, 0x60, 0x9c, 0x21, 0xe4, 0x30, 0xfb, 0x98, 0x06, 0x95, 0xf0, 0x94, 0x04,
+	0x83, 0x63, 0x97, 0x57, 0x4e, 0x3e, 0x2c, 0xdd, 0xe2, 0xee, 0x80, 0x86, 0x9c, 0x0c, 0x86, 0xef,
+	0xc7, 0xff, 0x14, 0xbc, 0xf4, 0x9a, 0x33, 0x0a, 0x08, 0x77, 0x99, 0xff, 0x7e, 0xf4, 0x47, 0x77,
+	0xdc, 0xe8, 0xb3, 0x3e, 0x93, 0x7f, 0xdf, 0x17, 0xff, 0x94, 0xd4, 0xdc, 0x84, 0xe5, 0x27, 0x34,
+	0x08, 0x5d, 0xe6, 0xa3, 0x1b, 0x90, 0x71, 0x7d, 0x87, 0x3e, 0x5b, 0x4f, 0x94, 0x13, 0xf7, 0xd2,
+	0x58, 0x35, 0xcc, 0xbf, 0x49, 0x40, 0xc1, 0xf2, 0x7d, 0xc6, 0xa5, 0xad, 0x10, 0x21, 0x48, 0xfb,
+	0x64, 0x40, 0x25, 0x28, 0x8f, 0xe5, 0x7f, 0x54, 0x85, 0xac, 0x47, 0x0e, 0xa8, 0x17, 0xae, 0x27,
+	0xcb, 0xa9, 0x7b, 0x85, 0xad, 0x1f, 0x54, 0x5e, 0xf6, 0xb9, 0x32, 0x65, 0xa4, 0xb2, 0x2b, 0xd1,
+	0x75, 0x9f, 0x07, 0x63, 0xac, 0x55, 0x4b, 0x3f, 0x86, 0xc2, 0x94, 0x18, 0x19, 0x90, 0x3a, 0xa6,
+	0x63, 0x3d, 0x8c, 0xf8, 0x2b, 0xfc, 0x3b, 0x21, 0xde, 0x88, 0xae, 0x27, 0xa5, 0x4c, 0x35, 0x3e,
+	0x49, 0xde, 0x4f, 0x98, 0x9f, 0x43, 0x1e, 0xd3, 0x90, 0x8d, 0x02, 0x9b, 0x86, 0xe8, 0x1d, 0xc8,
+	0xfb, 0xc4, 0x67, 0x3d, 0x7b, 0x38, 0x0a, 0xa5, 0x7a, 0x6a, 0xbb, 0x78, 0xf1, 0x62, 0x33, 0xd7,
+	0x24, 0x3e, 0xab, 0xb6, 0xba, 0x21, 0xce, 0x89, 0xee, 0xea, 0x70, 0x14, 0xa2, 0x37, 0xa1, 0x38,
+	0xa0, 0x03, 0x16, 0x8c, 0x7b, 0x07, 0x63, 0x4e, 0x43, 0x69, 0x38, 0x85, 0x0b, 0x4a, 0xb6, 0x2d,
+	0x44, 0xe6, 0x5f, 0x25, 0xe0, 0x46, 0x64, 0x1b, 0xd3, 0x3f, 0x1c, 0xb9, 0x01, 0x1d, 0x50, 0x9f,
+	0x87, 0xe8, 0x47, 0x90, 0xf5, 0xdc, 0x81, 0xcb, 0xd5, 0x18, 0x85, 0xad, 0x37, 0x16, 0xcd, 0x39,
+	0xf6, 0x0a, 0x6b, 0x30, 0xb2, 0xa0, 0x18, 0xd0, 0x90, 0x06, 0x27, 0x6a, 0x25, 0xe4, 0x90, 0xdf,
+	0xa8, 0x3c, 0xa3, 0x62, 0xee, 0x40, 0xae, 0xe5, 0x11, 0x7e, 0xc8, 0x82, 0x01, 0x32, 0xa1, 0x48,
+	0x02, 0xfb, 0xc8, 0xe5, 0xd4, 0xe6, 0xa3, 0x20, 0xda, 0x95, 0x19, 0x19, 0xba, 0x09, 0x49, 0xa6,
+	0x06, 0xca, 0x6f, 0x67, 0x2f, 0x5e, 0x6c, 0x26, 0xf7, 0xdb, 0x38, 0xc9, 0x42, 0xf3, 0x01, 0x5c,
+	0x6b, 0x79, 0xa3, 0xbe, 0xeb, 0xd7, 0x68, 0x68, 0x07, 0xee, 0x50, 0x58, 0x17, 0xdb, 0x2b, 0x82,
+	0x2f, 0xda, 0x5e, 0xf1, 0x3f, 0xde, 0xf2, 0xe4, 0x64, 0xcb, 0xcd, 0xbf, 0x48, 0xc2, 0xb5, 0xba,
+	0xdf, 0x77, 0x7d, 0x3a, 0xad, 0x7d, 0x17, 0x56, 0xa9, 0x14, 0xf6, 0x4e, 0x54, 0x50, 0x69, 0x3b,
+	0x2b, 0x4a, 0x1a, 0x45, 0x5a, 0x63, 0x2e, 0x5e, 0x3e, 0x5c, 0x34, 0xfd, 0x97, 0xac, 0x2f, 0x8a,
+	0x1a, 0x54, 0x87, 0xe5, 0xa1, 0x9c, 0x44, 0xb8, 0x9e, 0x92, 0xb6, 0xee, 0x2e, 0xb2, 0xf5, 0xd2,
+	0x3c, 0xb7, 0xd3, 0x5f, 0xbd, 0xd8, 0x5c, 0xc2, 0x91, 0xee, 0x6f, 0x13, 0x7c, 0xff, 0x99, 0x80,
+	0xb5, 0x26, 0x73, 0x66, 0xd6, 0xa1, 0x04, 0xb9, 0x23, 0x16, 0xf2, 0xa9, 0x83, 0x12, 0xb7, 0xd1,
+	0x7d, 0xc8, 0x0d, 0xf5, 0xf6, 0xe9, 0xdd, 0xbf, 0xbd, 0xd8, 0x65, 0x85, 0xc1, 0x31, 0x1a, 0x3d,
+	0x80, 0x7c, 0x10, 0xc5, 0xc4, 0x7a, 0xea, 0x2a, 0x81, 0x33, 0xc1, 0xa3, 0xdf, 0x85, 0xac, 0xda,
+	0x84, 0xf5, 0xb4, 0xd4, 0xbc, 0x7b, 0xa5, 0x35, 0xc7, 0x5a, 0xc9, 0xfc, 0x65, 0x02, 0x0c, 0x4c,
+	0x0e, 0xf9, 0x1e, 0x1d, 0x1c, 0xd0, 0xa0, 0xcd, 0x09, 0x1f, 0x85, 0xe8, 0x26, 0x64, 0x3d, 0x4a,
+	0x1c, 0x1a, 0xc8, 0x49, 0xe6, 0xb0, 0x6e, 0xa1, 0xae, 0x08, 0x72, 0x62, 0x1f, 0x91, 0x03, 0xd7,
+	0x73, 0xf9, 0x58, 0x4e, 0x73, 0x75, 0xf1, 0x2e, 0xcf, 0xdb, 0xac, 0xe0, 0x29, 0x45, 0x3c, 0x63,
+	0x06, 0xad, 0xc3, 0xf2, 0x80, 0x86, 0x21, 0xe9, 0x53, 0x39, 0xfb, 0x3c, 0x8e, 0x9a, 0xe6, 0x03,
+	0x28, 0x4e, 0xeb, 0xa1, 0x02, 0x2c, 0x77, 0x9b, 0x8f, 0x9b, 0xfb, 0x4f, 0x9b, 0xc6, 0x12, 0x5a,
+	0x83, 0x42, 0xb7, 0x89, 0xeb, 0x56, 0xf5, 0x91, 0xb5, 0xbd, 0x5b, 0x37, 0x12, 0x68, 0x05, 0xf2,
+	0x93, 0x66, 0xd2, 0xfc, 0x79, 0x02, 0x40, 0x6c, 0xa0, 0x9e, 0xd4, 0x27, 0x90, 0x09, 0x39, 0xe1,
+	0x6a, 0xe3, 0x56, 0xb7, 0xde, 0x5a, 0xe4, 0xf5, 0x04, 0x5e, 0x11, 0x3f, 0x14, 0x2b, 0x95, 0x69,
+	0x0f, 0x93, 0xf3, 0x1e, 0x66, 0x24, 0x72, 0xd6, 0xb5, 0x1c, 0xa4, 0x6b, 0xe2, 0x5f, 0x02, 0xe5,
+	0x21, 0x83, 0xeb, 0x56, 0xed, 0x73, 0x23, 0x89, 0x0c, 0x28, 0xd6, 0x1a, 0xed, 0xea, 0x7e, 0xb3,
+	0x59, 0xaf, 0x76, 0xea, 0x35, 0x23, 0x65, 0xde, 0x85, 0x4c, 0x63, 0x40, 0xfa, 0x14, 0xdd, 0x16,
+	0x11, 0x70, 0x48, 0x03, 0xea, 0xdb, 0x51, 0x60, 0x4d, 0x04, 0xe6, 0x2f, 0xf2, 0x90, 0xd9, 0x63,
+	0x23, 0x9f, 0xa3, 0xad, 0xa9, 0x53, 0xbc, 0xba, 0xb5, 0xb1, 0x68, 0x0a, 0x12, 0x58, 0xe9, 0x8c,
+	0x87, 0x54, 0x9f, 0xf2, 0x9b, 0x90, 0x55, 0xb1, 0xa2, 0x5d, 0xd7, 0x2d, 0x21, 0xe7, 0x24, 0xe8,
+	0x53, 0xae, 0x17, 0x5d, 0xb7, 0xd0, 0x3d, 0xc8, 0x05, 0x94, 0x38, 0xcc, 0xf7, 0xc6, 0x32, 0xa4,
+	0x72, 0x2a, 0xcd, 0x62, 0x4a, 0x9c, 0x7d, 0xdf, 0x1b, 0xe3, 0xb8, 0x17, 0x3d, 0x82, 0xe2, 0x81,
+	0xeb, 0x3b, 0x3d, 0x36, 0x54, 0x39, 0x2f, 0xf3, 0xea, 0x00, 0x54, 0x5e, 0x6d, 0xbb, 0xbe, 0xb3,
+	0xaf, 0xc0, 0xb8, 0x70, 0x30, 0x69, 0xa0, 0x26, 0xac, 0x9e, 0x30, 0x6f, 0x34, 0xa0, 0xb1, 0xad,
+	0xac, 0xb4, 0xf5, 0xf6, 0xab, 0x6d, 0x3d, 0x91, 0xf8, 0xc8, 0xda, 0xca, 0xc9, 0x74, 0x13, 0x3d,
+	0x86, 0x15, 0x3e, 0x18, 0x1e, 0x86, 0xb1, 0xb9, 0x65, 0x69, 0xee, 0xfb, 0x97, 0x2c, 0x98, 0x80,
+	0x47, 0xd6, 0x8a, 0x7c, 0xaa, 0x55, 0xfa, 0xb3, 0x14, 0x14, 0xa6, 0x3c, 0x47, 0x6d, 0x28, 0x0c,
+	0x03, 0x36, 0x24, 0x7d, 0x99, 0xb7, 0xf5, 0x5e, 0x7c, 0x78, 0xa5, 0x59, 0x57, 0x5a, 0x13, 0x45,
+	0x3c, 0x6d, 0xc5, 0x3c, 0x4f, 0x42, 0x61, 0xaa, 0x13, 0xbd, 0x0b, 0x39, 0xdc, 0xc2, 0x8d, 0x27,
+	0x56, 0xa7, 0x6e, 0x2c, 0x95, 0x6e, 0x9f, 0x9d, 0x97, 0xd7, 0xa5, 0xb5, 0x69, 0x03, 0xad, 0xc0,
+	0x3d, 0x11, 0xa1, 0x77, 0x0f, 0x96, 0x23, 0x68, 0xa2, 0xf4, 0xfa, 0xd9, 0x79, 0xf9, 0xb5, 0x79,
+	0xe8, 0x14, 0x12, 0xb7, 0x1f, 0x59, 0xb8, 0x5e, 0x33, 0x92, 0x8b, 0x91, 0xb8, 0x7d, 0x44, 0x02,
+	0xea, 0xa0, 0xef, 0x43, 0x56, 0x03, 0x53, 0xa5, 0xd2, 0xd9, 0x79, 0xf9, 0xe6, 0x3c, 0x70, 0x82,
+	0xc3, 0xed, 0x5d, 0xeb, 0x49, 0xdd, 0x48, 0x2f, 0xc6, 0xe1, 0xb6, 0x47, 0x4e, 0x28, 0x7a, 0x0b,
+	0x32, 0x0a, 0x96, 0x29, 0xdd, 0x3a, 0x3b, 0x2f, 0x7f, 0xef, 0x25, 0x73, 0x02, 0x55, 0x5a, 0xff,
+	0xcb, 0xbf, 0xdd, 0x58, 0xfa, 0xa7, 0xbf, 0xdb, 0x30, 0xe6, 0xbb, 0x4b, 0xff, 0x9b, 0x80, 0x95,
+	0x99, 0x2d, 0x47, 0x26, 0x64, 0x7d, 0x66, 0xb3, 0xa1, 0x4a, 0xe7, 0xb9, 0x6d, 0xb8, 0x78, 0xb1,
+	0x99, 0x6d, 0xb2, 0x2a, 0x1b, 0x8e, 0xb1, 0xee, 0x41, 0x8f, 0xe7, 0x2e, 0xa4, 0x8f, 0xae, 0x18,
+	0x4f, 0x0b, 0xaf, 0xa4, 0x4f, 0x61, 0xc5, 0x09, 0xdc, 0x13, 0x1a, 0xf4, 0x6c, 0xe6, 0x1f, 0xba,
+	0x7d, 0x9d, 0xaa, 0x4b, 0x8b, 0x6c, 0xd6, 0x24, 0x10, 0x17, 0x95, 0x42, 0x55, 0xe2, 0x7f, 0x8b,
+	0xcb, 0xa8, 0xf4, 0x04, 0x8a, 0xd3, 0x11, 0x8a, 0xde, 0x00, 0x08, 0xdd, 0x3f, 0xa2, 0x9a, 0xdf,
+	0x48, 0x36, 0x84, 0xf3, 0x42, 0x22, 0xd9, 0x0d, 0x7a, 0x1b, 0xd2, 0x03, 0xe6, 0x28, 0x3b, 0x99,
+	0xed, 0xeb, 0xe2, 0x4e, 0xfc, 0xd5, 0x8b, 0xcd, 0x02, 0x0b, 0x2b, 0x3b, 0xae, 0x47, 0xf7, 0x98,
+	0x43, 0xb1, 0x04, 0x98, 0x27, 0x90, 0x16, 0xa9, 0x02, 0xbd, 0x0e, 0xe9, 0xed, 0x46, 0xb3, 0x66,
+	0x2c, 0x95, 0xae, 0x9d, 0x9d, 0x97, 0x57, 0xe4, 0x92, 0x88, 0x0e, 0x11, 0xbb, 0x68, 0x13, 0xb2,
+	0x4f, 0xf6, 0x77, 0xbb, 0x7b, 0x22, 0xbc, 0xae, 0x9f, 0x9d, 0x97, 0xd7, 0xe2, 0x6e, 0xb5, 0x68,
+	0xe8, 0x0d, 0xc8, 0x74, 0xf6, 0x5a, 0x3b, 0x6d, 0x23, 0x59, 0x42, 0x67, 0xe7, 0xe5, 0xd5, 0xb8,
+	0x5f, 0xfa, 0x5c, 0xba, 0xa6, 0x77, 0x35, 0x1f, 0xcb, 0xcd, 0xff, 0x49, 0xc2, 0x0a, 0x16, 0xfc,
+	0x36, 0xe0, 0x2d, 0xe6, 0xb9, 0xf6, 0x18, 0xb5, 0x20, 0x6f, 0x33, 0xdf, 0x71, 0xa7, 0xce, 0xd4,
+	0xd6, 0x2b, 0x2e, 0xc1, 0x89, 0x56, 0xd4, 0xaa, 0x46, 0x9a, 0x78, 0x62, 0x04, 0x6d, 0x41, 0xc6,
+	0xa1, 0x1e, 0x19, 0x5f, 0x76, 0x1b, 0xd7, 0x34, 0x97, 0xc6, 0x0a, 0x2a, 0x99, 0x23, 0x79, 0xd6,
+	0x23, 0x9c, 0xd3, 0xc1, 0x90, 0xab, 0xdb, 0x38, 0x8d, 0x0b, 0x03, 0xf2, 0xcc, 0xd2, 0x22, 0xf4,
+	0x43, 0xc8, 0x9e, 0xba, 0xbe, 0xc3, 0x4e, 0xf5, 0x85, 0x7b, 0xb9, 0x5d, 0x8d, 0x35, 0xcf, 0xc4,
+	0x3d, 0x3b, 0xe7, 0xac, 0x58, 0xf5, 0xe6, 0x7e, 0xb3, 0x1e, 0xad, 0xba, 0xee, 0xdf, 0xf7, 0x9b,
+	0xcc, 0x17, 0x27, 0x06, 0xf6, 0x9b, 0xbd, 0x1d, 0xab, 0xb1, 0xdb, 0xc5, 0x62, 0xe5, 0x6f, 0x9c,
+	0x9d, 0x97, 0x8d, 0x18, 0xb2, 0x43, 0x5c, 0x4f, 0x90, 0xc0, 0x5b, 0x90, 0xb2, 0x9a, 0x9f, 0x1b,
+	0xc9, 0x92, 0x71, 0x76, 0x5e, 0x2e, 0xc6, 0xdd, 0x96, 0x3f, 0x9e, 0x1c, 0xa6, 0xf9, 0x71, 0xcd,
+	0x7f, 0x4f, 0x42, 0xb1, 0x3b, 0x74, 0x08, 0xa7, 0x2a, 0x32, 0x51, 0x19, 0x0a, 0x43, 0x12, 0x10,
+	0xcf, 0xa3, 0x9e, 0x1b, 0x0e, 0x74, 0xa1, 0x30, 0x2d, 0x42, 0xf7, 0xbf, 0xc5, 0x62, 0x6a, 0x12,
+	0xa6, 0x97, 0xb4, 0x0b, 0xab, 0x87, 0xca, 0xd9, 0x1e, 0xb1, 0xe5, 0xee, 0xa6, 0xe4, 0xee, 0x56,
+	0x16, 0x99, 0x98, 0xf6, 0xaa, 0xa2, 0xe7, 0x68, 0x49, 0x2d, 0xbc, 0x72, 0x38, 0xdd, 0x44, 0x1f,
+	0xc3, 0xf2, 0x80, 0xf9, 0x2e, 0x67, 0xc1, 0x95, 0xf6, 0x21, 0x02, 0xa3, 0x77, 0xe1, 0x9a, 0xd8,
+	0xe1, 0xc8, 0x25, 0xd9, 0x2d, 0x6f, 0xae, 0x24, 0x5e, 0x1b, 0x90, 0x67, 0x7a, 0x4c, 0x2c, 0xc4,
+	0xe6, 0xc7, 0xb0, 0x32, 0xe3, 0x83, 0xb8, 0xcd, 0x5b, 0x56, 0xb7, 0x5d, 0x37, 0x96, 0x50, 0x11,
+	0x72, 0xd5, 0xfd, 0x66, 0xa7, 0xd1, 0xec, 0x0a, 0xea, 0x51, 0x84, 0x1c, 0xde, 0xdf, 0xdd, 0xdd,
+	0xb6, 0xaa, 0x8f, 0x8d, 0xa4, 0xf9, 0xdf, 0xf1, 0xfa, 0x6a, 0xee, 0xb1, 0x3d, 0xcb, 0x3d, 0xde,
+	0x7b, 0xf5, 0xd4, 0x35, 0xfb, 0x98, 0x34, 0x62, 0x0e, 0xf2, 0x13, 0x00, 0xb9, 0x8d, 0xd4, 0xe9,
+	0x11, 0x7e, 0x59, 0x7d, 0xd1, 0x89, 0x2a, 0x47, 0x9c, 0xd7, 0x0a, 0x16, 0x47, 0x9f, 0x41, 0xd1,
+	0x66, 0x83, 0xa1, 0x47, 0xb5, 0x7e, 0xea, 0x2a, 0xfa, 0x85, 0x58, 0xc5, 0xe2, 0xd3, 0x1c, 0x28,
+	0x3d, 0xcb, 0x81, 0xfe, 0x3c, 0x01, 0x85, 0x29, 0x87, 0x67, 0xa9, 0x50, 0x11, 0x72, 0xdd, 0x56,
+	0xcd, 0xea, 0x34, 0x9a, 0x0f, 0x8d, 0x04, 0x02, 0xc8, 0xca, 0x05, 0xac, 0x19, 0x49, 0x41, 0xd7,
+	0xaa, 0xfb, 0x7b, 0xad, 0xdd, 0xba, 0x24, 0x43, 0xe8, 0x06, 0x18, 0xd1, 0x12, 0xf6, 0xda, 0x1d,
+	0x0b, 0x0b, 0x69, 0x1a, 0x5d, 0x87, 0xb5, 0x58, 0xaa, 0x35, 0x33, 0xe8, 0x26, 0xa0, 0x58, 0x38,
+	0x31, 0x91, 0x35, 0xff, 0x04, 0xd6, 0xaa, 0xcc, 0xe7, 0xc4, 0xf5, 0x63, 0x2a, 0xbb, 0x25, 0xe6,
+	0xad, 0x45, 0x3d, 0xd7, 0x51, 0xd9, 0x76, 0x7b, 0xed, 0xe2, 0xc5, 0x66, 0x21, 0x86, 0x36, 0x6a,
+	0x62, 0xa6, 0x51, 0xc3, 0x11, 0x67, 0x6a, 0xe8, 0x3a, 0x3a, 0x79, 0x2e, 0x5f, 0xbc, 0xd8, 0x4c,
+	0xb5, 0x1a, 0x35, 0x2c, 0x64, 0xe8, 0x75, 0xc8, 0xd3, 0x67, 0x2e, 0xef, 0xd9, 0x22, 0xbb, 0x8a,
+	0x35, 0xcc, 0xe0, 0x9c, 0x10, 0x54, 0x45, 0x32, 0xfd, 0xd3, 0x24, 0x40, 0x87, 0x84, 0xc7, 0x7a,
+	0xe8, 0x07, 0x90, 0x8f, 0x8b, 0xf8, 0xcb, 0x8a, 0xc9, 0xa9, 0xfd, 0x8a, 0xf1, 0xe8, 0xa3, 0x28,
+	0x62, 0x14, 0xc7, 0x5e, 0xac, 0xa8, 0xc7, 0x5a, 0x44, 0x53, 0x67, 0x89, 0xb4, 0xb8, 0x6b, 0x68,
+	0x10, 0xe8, 0x8d, 0x13, 0x7f, 0x51, 0x55, 0xe6, 0x5b, 0x35, 0x67, 0xcd, 0xdc, 0xee, 0x2c, 0x1a,
+	0x64, 0x6e, 0x41, 0x1f, 0x2d, 0xe1, 0x89, 0xde, 0xb6, 0x01, 0xab, 0xc1, 0xc8, 0x17, 0x5e, 0xf7,
+	0x42, 0xd9, 0x6d, 0xba, 0xf0, 0x5a, 0x93, 0xf2, 0x53, 0x16, 0x1c, 0x5b, 0x9c, 0x13, 0xfb, 0x48,
+	0x14, 0xd5, 0x3a, 0xc9, 0x4c, 0x08, 0x67, 0x62, 0x86, 0x70, 0xae, 0xc3, 0x32, 0xf1, 0x5c, 0x12,
+	0x52, 0x75, 0x4b, 0xe7, 0x71, 0xd4, 0x14, 0xb4, 0x98, 0x38, 0x4e, 0x40, 0xc3, 0x90, 0xaa, 0x32,
+	0x30, 0x8f, 0x27, 0x02, 0xf3, 0x5f, 0x92, 0x00, 0x8d, 0x96, 0xb5, 0xa7, 0xcd, 0xd7, 0x20, 0x7b,
+	0x48, 0x06, 0xae, 0x37, 0xbe, 0xec, 0x90, 0x4d, 0xf0, 0x15, 0x4b, 0x19, 0xda, 0x91, 0x3a, 0x58,
+	0xeb, 0x4a, 0xb6, 0x3c, 0x3a, 0xf0, 0x29, 0x8f, 0xd9, 0xb2, 0x6c, 0x89, 0xab, 0x39, 0x20, 0x7e,
+	0xbc, 0xb0, 0xaa, 0x21, 0x5c, 0xef, 0x13, 0x4e, 0x4f, 0xc9, 0x38, 0x3a, 0x13, 0xba, 0x89, 0x1e,
+	0x09, 0x16, 0x2d, 0x8a, 0x7b, 0xea, 0xac, 0x67, 0x24, 0xf7, 0xf8, 0x26, 0x7f, 0xb0, 0x86, 0x2b,
+	0xd2, 0x11, 0x6b, 0x97, 0x1e, 0xc8, 0x9b, 0x72, 0xd2, 0xf5, 0xad, 0x8a, 0xd8, 0x0f, 0x60, 0x65,
+	0x66, 0x9e, 0x2f, 0x95, 0x29, 0x8d, 0xd6, 0x93, 0x1f, 0x1a, 0x69, 0xfd, 0xef, 0x63, 0x23, 0x6b,
+	0xfe, 0x57, 0x02, 0xa0, 0xc5, 0x82, 0x68, 0xd3, 0x16, 0x3f, 0x0b, 0xe5, 0xe4, 0x23, 0x93, 0xcd,
+	0x3c, 0x1d, 0x9e, 0x0b, 0x79, 0xfa, 0xc4, 0x8a, 0xa0, 0xbd, 0x12, 0x8e, 0x63, 0x45, 0xb4, 0x09,
+	0x05, 0xb5, 0xff, 0xbd, 0x21, 0x0b, 0x54, 0x3e, 0x5a, 0xc1, 0xa0, 0x44, 0x42, 0x13, 0xdd, 0x85,
+	0xd5, 0xe1, 0xe8, 0xc0, 0x73, 0xc3, 0x23, 0xea, 0x28, 0x4c, 0x5a, 0x62, 0x56, 0x62, 0xa9, 0x80,
+	0x99, 0x35, 0xc8, 0x45, 0xd6, 0xd1, 0x3a, 0xa4, 0x3a, 0xd5, 0x96, 0xb1, 0x54, 0x5a, 0x3b, 0x3b,
+	0x2f, 0x17, 0x22, 0x71, 0xa7, 0xda, 0x12, 0x3d, 0xdd, 0x5a, 0xcb, 0x48, 0xcc, 0xf6, 0x74, 0x6b,
+	0xad, 0x52, 0x5a, 0xdc, 0x92, 0xe6, 0x5f, 0x27, 0x20, 0xab, 0x38, 0xdb, 0xc2, 0x19, 0x5b, 0xb0,
+	0x1c, 0x55, 0x12, 0x8a, 0x48, 0xbe, 0xfd, 0x6a, 0xd2, 0x57, 0xd1, 0x1c, 0x4d, 0xed, 0x63, 0xa4,
+	0x57, 0xfa, 0x04, 0x8a, 0xd3, 0x1d, 0xdf, 0x6a, 0x17, 0xff, 0x18, 0x0a, 0x22, 0x50, 0x22, 0xf2,
+	0xb7, 0x05, 0x59, 0xc5, 0x2b, 0x75, 0x56, 0xb9, 0x8c, 0x81, 0x6a, 0x24, 0xba, 0x0f, 0xcb, 0x8a,
+	0xb5, 0x46, 0xef, 0x29, 0x1b, 0x97, 0x87, 0x23, 0x8e, 0xe0, 0xe6, 0xa7, 0x90, 0x6e, 0x51, 0x1a,
+	0xa0, 0x3b, 0xb0, 0xec, 0x33, 0x87, 0x4e, 0x92, 0xa8, 0x26, 0xdc, 0x0e, 0x6d, 0xd4, 0x04, 0xe1,
+	0x76, 0x68, 0xc3, 0x11, 0x8b, 0x27, 0x0e, 0x68, 0xf4, 0xa4, 0x24, 0xfe, 0x9b, 0x1d, 0x28, 0x3e,
+	0xa5, 0x6e, 0xff, 0x88, 0x53, 0x47, 0x1a, 0x7a, 0x0f, 0xd2, 0x43, 0x1a, 0x3b, 0xbf, 0xbe, 0x30,
+	0x74, 0x28, 0x0d, 0xb0, 0x44, 0x89, 0x03, 0x79, 0x2a, 0xb5, 0xf5, 0x2b, 0x9e, 0x6e, 0x99, 0xff,
+	0x90, 0x84, 0xd5, 0x46, 0x18, 0x8e, 0x88, 0x6f, 0x47, 0xb7, 0xec, 0x4f, 0x67, 0x6f, 0xd9, 0x7b,
+	0x0b, 0x67, 0x38, 0xa3, 0x32, 0x5b, 0xe5, 0xeb, 0x24, 0x99, 0x8c, 0x93, 0xa4, 0xf9, 0x55, 0x22,
+	0x2a, 0xef, 0xef, 0x4e, 0x9d, 0x9b, 0xd2, 0xfa, 0xd9, 0x79, 0xf9, 0xc6, 0xb4, 0x25, 0xda, 0xf5,
+	0x8f, 0x7d, 0x76, 0xea, 0xa3, 0x37, 0x45, 0xb9, 0xdf, 0xac, 0x3f, 0x35, 0x12, 0xa5, 0x9b, 0x67,
+	0xe7, 0x65, 0x34, 0x03, 0xc2, 0xd4, 0xa7, 0xa7, 0xc2, 0x52, 0xab, 0xde, 0xac, 0x89, 0xfb, 0x30,
+	0xb9, 0xc0, 0x52, 0x8b, 0xfa, 0x8e, 0xeb, 0xf7, 0xd1, 0x1d, 0xc8, 0x36, 0xda, 0xed, 0xae, 0x2c,
+	0xc0, 0x5e, 0x3b, 0x3b, 0x2f, 0x5f, 0x9f, 0x41, 0x89, 0x06, 0x75, 0x04, 0x48, 0x10, 0x44, 0x71,
+	0x53, 0x2e, 0x00, 0x09, 0xee, 0x42, 0x1d, 0x1d, 0xe1, 0xff, 0x96, 0x04, 0xc3, 0xb2, 0x6d, 0x3a,
+	0xe4, 0xa2, 0x5f, 0x93, 0xee, 0x0e, 0xe4, 0x86, 0xe2, 0x9f, 0x2b, 0x8b, 0x08, 0x11, 0x16, 0xf7,
+	0x17, 0x3e, 0xf1, 0xce, 0xe9, 0x55, 0x30, 0xf3, 0xa8, 0xe5, 0x0c, 0xdc, 0x30, 0x14, 0xc5, 0xa5,
+	0x94, 0xe1, 0xd8, 0x52, 0xe9, 0xd7, 0x09, 0xb8, 0xbe, 0x00, 0x81, 0x3e, 0x80, 0x74, 0xc0, 0xbc,
+	0x68, 0x7b, 0x6e, 0xbf, 0xea, 0x01, 0x46, 0xa8, 0x62, 0x89, 0x44, 0x1b, 0x00, 0x64, 0xc4, 0x19,
+	0x91, 0xe3, 0xcb, 0x8d, 0xc9, 0xe1, 0x29, 0x09, 0x7a, 0x0a, 0xd9, 0x90, 0xda, 0x01, 0x8d, 0xf8,
+	0xcc, 0xa7, 0xff, 0x5f, 0xef, 0x2b, 0x6d, 0x69, 0x06, 0x6b, 0x73, 0xa5, 0x0a, 0x64, 0x95, 0x44,
+	0x44, 0xb4, 0x43, 0x38, 0x91, 0x4e, 0x17, 0xb1, 0xfc, 0x2f, 0x02, 0x85, 0x78, 0xfd, 0x28, 0x50,
+	0x88, 0xd7, 0x37, 0x7f, 0x9e, 0x04, 0xa8, 0x3f, 0xe3, 0x34, 0xf0, 0x89, 0x57, 0xb5, 0x50, 0x7d,
+	0x2a, 0x43, 0xaa, 0xd9, 0xbe, 0xb3, 0xf0, 0x59, 0x2e, 0xd6, 0xa8, 0x54, 0xad, 0x05, 0x39, 0xf2,
+	0x16, 0xa4, 0x46, 0x81, 0xa7, 0x9f, 0x78, 0x25, 0x11, 0xe9, 0xe2, 0x5d, 0x2c, 0x64, 0xa8, 0x3e,
+	0xc9, 0x48, 0xa9, 0x57, 0xbf, 0xcd, 0x4f, 0x0d, 0xf0, 0xdd, 0x67, 0xa5, 0xf7, 0x00, 0x26, 0x5e,
+	0xa3, 0x0d, 0xc8, 0x54, 0x77, 0xda, 0xed, 0x5d, 0x63, 0x49, 0xd5, 0x88, 0x93, 0x2e, 0x29, 0x36,
+	0xff, 0x3e, 0x01, 0xb9, 0xaa, 0xa5, 0x6f, 0x95, 0x1d, 0x30, 0x64, 0x2e, 0xb1, 0x69, 0xc0, 0x7b,
+	0xf4, 0xd9, 0xd0, 0x0d, 0xc6, 0x3a, 0x1d, 0x5c, 0xce, 0xe2, 0x57, 0x85, 0x56, 0x95, 0x06, 0xbc,
+	0x2e, 0x75, 0x10, 0x86, 0x22, 0xd5, 0x53, 0xec, 0xd9, 0x24, 0x4a, 0xce, 0x1b, 0x97, 0x2f, 0x85,
+	0x62, 0x7f, 0x93, 0x76, 0x88, 0x0b, 0x91, 0x91, 0x2a, 0x09, 0xcd, 0x27, 0x70, 0x7d, 0x3f, 0xb0,
+	0x8f, 0x68, 0xc8, 0xd5, 0xa0, 0xda, 0xe5, 0x4f, 0xe1, 0x36, 0x27, 0xe1, 0x71, 0xef, 0xc8, 0x0d,
+	0x39, 0x0b, 0xc6, 0xbd, 0x80, 0x72, 0xea, 0x8b, 0xfe, 0x9e, 0xfc, 0x02, 0xa0, 0x6b, 0xf0, 0x5b,
+	0x02, 0xf3, 0x48, 0x41, 0x70, 0x84, 0xd8, 0x15, 0x00, 0xb3, 0x01, 0x45, 0x41, 0xd8, 0x6a, 0xf4,
+	0x90, 0x8c, 0x3c, 0x1e, 0xa2, 0x1f, 0x03, 0x78, 0xac, 0xdf, 0xbb, 0x72, 0x26, 0xcf, 0x7b, 0xac,
+	0xaf, 0xfe, 0x9a, 0xbf, 0x07, 0x46, 0xcd, 0x0d, 0x87, 0x84, 0xdb, 0x47, 0xd1, 0xe3, 0x02, 0x7a,
+	0x08, 0xc6, 0x11, 0x25, 0x01, 0x3f, 0xa0, 0x84, 0xf7, 0x86, 0x34, 0x70, 0x99, 0x73, 0xa5, 0x25,
+	0x5d, 0x8b, 0xb5, 0x5a, 0x52, 0xc9, 0xfc, 0x4d, 0x02, 0x00, 0x93, 0xc3, 0x88, 0x00, 0xfc, 0x00,
+	0xae, 0x85, 0x3e, 0x19, 0x86, 0x47, 0x8c, 0xf7, 0x5c, 0x9f, 0xd3, 0xe0, 0x84, 0x78, 0xba, 0x40,
+	0x34, 0xa2, 0x8e, 0x86, 0x96, 0xa3, 0xf7, 0x00, 0x1d, 0x53, 0x3a, 0xec, 0x31, 0xcf, 0xe9, 0x45,
+	0x9d, 0xea, 0x13, 0x45, 0x1a, 0x1b, 0xa2, 0x67, 0xdf, 0x73, 0xda, 0x91, 0x1c, 0x6d, 0xc3, 0x86,
+	0x58, 0x01, 0xea, 0xf3, 0xc0, 0xa5, 0x61, 0xef, 0x90, 0x05, 0xbd, 0xd0, 0x63, 0xa7, 0xbd, 0x43,
+	0xe6, 0x79, 0xec, 0x94, 0x06, 0x51, 0xf9, 0x5d, 0xf2, 0x58, 0xbf, 0xae, 0x40, 0x3b, 0x2c, 0x68,
+	0x7b, 0xec, 0x74, 0x27, 0x42, 0x08, 0x96, 0x30, 0x99, 0x36, 0x77, 0xed, 0xe3, 0x88, 0x25, 0xc4,
+	0xd2, 0x8e, 0x6b, 0x1f, 0xa3, 0x3b, 0xb0, 0x42, 0x3d, 0x2a, 0x8b, 0x38, 0x85, 0xca, 0x48, 0x54,
+	0x31, 0x12, 0x0a, 0x90, 0xf9, 0x3b, 0x90, 0x6f, 0x79, 0xc4, 0x96, 0x1f, 0x82, 0x44, 0x49, 0x6c,
+	0x33, 0x5f, 0x04, 0x81, 0xeb, 0x73, 0x95, 0x1d, 0xf3, 0x78, 0x5a, 0x64, 0xfe, 0x14, 0xe0, 0x67,
+	0xcc, 0xf5, 0x3b, 0xec, 0x98, 0xfa, 0xf2, 0xcd, 0x5c, 0xb0, 0x5e, 0xbd, 0x95, 0x79, 0xac, 0x5b,
+	0x92, 0x93, 0x13, 0x9f, 0xf4, 0x69, 0x10, 0x3f, 0x1d, 0xab, 0xa6, 0xb8, 0x5c, 0xb2, 0x98, 0x31,
+	0x5e, 0xb5, 0x50, 0x19, 0xb2, 0x36, 0xe9, 0x45, 0x27, 0xaf, 0xb8, 0x9d, 0xbf, 0x78, 0xb1, 0x99,
+	0xa9, 0x5a, 0x8f, 0xe9, 0x18, 0x67, 0x6c, 0xf2, 0x98, 0x8e, 0xc5, 0xed, 0x6b, 0x13, 0x79, 0x5e,
+	0xa4, 0x99, 0xa2, 0xba, 0x7d, 0xab, 0x96, 0x38, 0x0c, 0x38, 0x6b, 0x13, 0xf1, 0x8b, 0x3e, 0x80,
+	0xa2, 0x06, 0xf5, 0x8e, 0x48, 0x78, 0xa4, 0xb8, 0xea, 0xf6, 0xea, 0xc5, 0x8b, 0x4d, 0x50, 0xc8,
+	0x47, 0x24, 0x3c, 0xc2, 0xa0, 0xd0, 0xe2, 0x3f, 0xaa, 0x43, 0xe1, 0x0b, 0xe6, 0xfa, 0x3d, 0x2e,
+	0x27, 0xa1, 0x2b, 0xe9, 0x85, 0xe7, 0x67, 0x32, 0x55, 0x5d, 0xde, 0xc3, 0x17, 0xb1, 0xc4, 0xfc,
+	0xd7, 0x04, 0x14, 0x84, 0x4d, 0xf7, 0xd0, 0xb5, 0xc5, 0x6d, 0xf9, 0xed, 0x33, 0xfd, 0x2d, 0x48,
+	0xd9, 0x61, 0xa0, 0xe7, 0x26, 0x53, 0x5d, 0xb5, 0x8d, 0xb1, 0x90, 0xa1, 0xcf, 0x20, 0xab, 0x8a,
+	0x0b, 0x9d, 0xe4, 0xcd, 0x6f, 0xbe, 0xd7, 0xb5, 0x8b, 0x5a, 0x4f, 0xee, 0xe5, 0xc4, 0x3b, 0x39,
+	0xcb, 0x22, 0x9e, 0x16, 0xa1, 0x9b, 0x90, 0xb4, 0x7d, 0x19, 0x14, 0xfa, 0x5b, 0x5a, 0xb5, 0x89,
+	0x93, 0xb6, 0x6f, 0xfe, 0x73, 0x02, 0x56, 0xea, 0xbe, 0x1d, 0x8c, 0x65, 0x92, 0x14, 0x1b, 0x71,
+	0x1b, 0xf2, 0xe1, 0xe8, 0x20, 0x1c, 0x87, 0x9c, 0x0e, 0xa2, 0xa7, 0xfa, 0x58, 0x80, 0x1a, 0x90,
+	0x27, 0x5e, 0x9f, 0x05, 0x2e, 0x3f, 0x1a, 0x68, 0x6e, 0xbc, 0x38, 0x31, 0x4f, 0xdb, 0xac, 0x58,
+	0x91, 0x0a, 0x9e, 0x68, 0x47, 0xa9, 0x38, 0x25, 0x9d, 0x95, 0xa9, 0xf8, 0x4d, 0x28, 0x7a, 0x64,
+	0x20, 0xa8, 0x70, 0x4f, 0x94, 0x5c, 0x72, 0x1e, 0x69, 0x5c, 0xd0, 0x32, 0x51, 0x46, 0x9a, 0x26,
+	0xe4, 0x63, 0x63, 0x68, 0x0d, 0x0a, 0x56, 0xbd, 0xdd, 0xfb, 0x70, 0xeb, 0x7e, 0xef, 0x61, 0x75,
+	0xcf, 0x58, 0xd2, 0x4c, 0xe0, 0x1f, 0x13, 0xb0, 0xb2, 0xa7, 0x62, 0x50, 0x13, 0xa7, 0x3b, 0xb0,
+	0x1c, 0x90, 0x43, 0x1e, 0x51, 0xbb, 0xb4, 0x0a, 0x2e, 0x91, 0x04, 0x04, 0xb5, 0x13, 0x5d, 0x8b,
+	0xa9, 0xdd, 0xd4, 0x87, 0xa2, 0xd4, 0xa5, 0x1f, 0x8a, 0xd2, 0xdf, 0xc9, 0x87, 0x22, 0xf3, 0x57,
+	0x09, 0x58, 0xd3, 0x17, 0x75, 0xf4, 0x71, 0x04, 0xbd, 0x03, 0x79, 0x75, 0x67, 0x4f, 0x88, 0xa9,
+	0xfc, 0x5e, 0xa1, 0x70, 0x8d, 0x1a, 0xce, 0xa9, 0xee, 0x86, 0x83, 0x7e, 0x32, 0xf5, 0x2a, 0xfa,
+	0x0a, 0x7a, 0x38, 0x67, 0xbd, 0x32, 0x79, 0x2a, 0x7d, 0xe5, 0xf7, 0x92, 0x4d, 0x28, 0x68, 0x07,
+	0x64, 0xd9, 0xa0, 0xea, 0x40, 0x50, 0xa2, 0x26, 0x19, 0x50, 0xf3, 0x2e, 0xa4, 0x85, 0x19, 0x04,
+	0x90, 0x6d, 0x7f, 0xde, 0xee, 0xd4, 0xf7, 0x54, 0xe5, 0xb5, 0xd3, 0x90, 0x1f, 0xad, 0x96, 0x21,
+	0x55, 0x6f, 0x3e, 0x31, 0x92, 0xe6, 0xef, 0x43, 0x01, 0xd3, 0x01, 0x3b, 0xa1, 0x4e, 0x53, 0x0d,
+	0x97, 0x8c, 0x27, 0x24, 0x23, 0xb2, 0x51, 0xc3, 0x49, 0xd7, 0x41, 0x3f, 0x82, 0xac, 0xbe, 0x30,
+	0xaf, 0xf4, 0x04, 0xa4, 0xc1, 0xef, 0xfe, 0x26, 0x05, 0xf9, 0xf8, 0xbd, 0x40, 0x9c, 0x36, 0x41,
+	0x52, 0x97, 0xd4, 0xab, 0x61, 0x2c, 0x6f, 0x4a, 0x7a, 0x9a, 0xb7, 0x76, 0x77, 0xf7, 0xab, 0x56,
+	0xa7, 0x5e, 0x33, 0x3e, 0x53, 0x2c, 0x36, 0x06, 0x58, 0x9e, 0xc7, 0xc4, 0x79, 0x71, 0x90, 0x39,
+	0x61, 0xb1, 0xcf, 0xf5, 0xdb, 0x64, 0x8c, 0x8a, 0x28, 0xec, 0x5b, 0x90, 0xb3, 0xda, 0xed, 0xc6,
+	0xc3, 0x66, 0xbd, 0x66, 0x7c, 0x99, 0x28, 0x7d, 0xef, 0xec, 0xbc, 0x7c, 0x6d, 0x62, 0x2a, 0x0c,
+	0xdd, 0xbe, 0x4f, 0x1d, 0x89, 0xaa, 0x56, 0xeb, 0x2d, 0x31, 0xde, 0xf3, 0xe4, 0x3c, 0x4a, 0x72,
+	0x37, 0xf9, 0x9d, 0x21, 0xdf, 0xc2, 0xf5, 0x96, 0x85, 0xc5, 0x88, 0x5f, 0x26, 0xe7, 0xfc, 0x6a,
+	0x05, 0x74, 0x48, 0x02, 0x31, 0xe6, 0x46, 0xf4, 0xbd, 0xed, 0x79, 0x4a, 0xbd, 0x45, 0x4f, 0x1e,
+	0x49, 0x28, 0x71, 0xc6, 0x62, 0x34, 0xf9, 0xb8, 0x24, 0xcd, 0xa4, 0xe6, 0x46, 0x6b, 0x73, 0x12,
+	0x70, 0x61, 0xc5, 0x84, 0x65, 0xdc, 0x6d, 0x36, 0xe5, 0xec, 0xd2, 0x73, 0xb3, 0xc3, 0x23, 0xdf,
+	0x17, 0x98, 0xbb, 0x90, 0x8b, 0xde, 0x9e, 0x8c, 0x2f, 0xd3, 0x73, 0x0e, 0x55, 0xa3, 0x87, 0x33,
+	0x39, 0xe0, 0xa3, 0x6e, 0x47, 0x7e, 0x0e, 0x7c, 0x9e, 0x99, 0x1f, 0xf0, 0x68, 0xc4, 0x1d, 0x51,
+	0x37, 0x94, 0x63, 0x22, 0xff, 0x65, 0x46, 0xf1, 0xa7, 0x18, 0xa3, 0x58, 0xbc, 0xb0, 0x83, 0xeb,
+	0x3f, 0x53, 0x5f, 0x0e, 0x9f, 0x67, 0xe7, 0xec, 0x60, 0xfa, 0x05, 0xb5, 0x39, 0x75, 0x26, 0x4f,
+	0xed, 0x71, 0xd7, 0xbb, 0x7f, 0x00, 0xb9, 0x28, 0xd7, 0xa2, 0x0d, 0xc8, 0x3e, 0xdd, 0xc7, 0x8f,
+	0xeb, 0xd8, 0x58, 0x52, 0xab, 0x13, 0xf5, 0x3c, 0x55, 0x97, 0x55, 0x19, 0x96, 0xf7, 0xac, 0xa6,
+	0xf5, 0xb0, 0x8e, 0xa3, 0xa7, 0xfe, 0x08, 0xa0, 0x13, 0x46, 0xc9, 0xd0, 0x03, 0xc4, 0x36, 0xb7,
+	0x6f, 0x7f, 0xf5, 0xf5, 0xc6, 0xd2, 0x2f, 0xbf, 0xde, 0x58, 0xfa, 0xf5, 0xd7, 0x1b, 0x89, 0xe7,
+	0x17, 0x1b, 0x89, 0xaf, 0x2e, 0x36, 0x12, 0xbf, 0xb8, 0xd8, 0x48, 0xfc, 0xc7, 0xc5, 0x46, 0xe2,
+	0x20, 0x2b, 0xc9, 0xec, 0x47, 0xff, 0x17, 0x00, 0x00, 0xff, 0xff, 0x1f, 0x9b, 0xee, 0x67, 0xfb,
 	0x22, 0x00, 0x00,
 }

vendor/src/github.com/docker/swarmkit/api/types.proto

@@ -776,3 +776,13 @@ message SecretReference {
 	// lookup/display purposes.  The secret in the reference will be identified by its ID.
 	string secret_name = 4;
 }
+
+// RemovedNode is a record for a node that has been removed from the swarm.
+message RemovedNode {
+	// ID is the ID of the removed node.
+	string id = 1 [(gogoproto.customname) = "ID"];
+
+	// Expiry is the latest known expiration time of a certificate that
+	// was issued to this node.
+	Timestamp expiry = 2;
+}

vendor/src/github.com/docker/swarmkit/ca/auth.go

@@ -7,6 +7,7 @@ import (
 
 	"github.com/Sirupsen/logrus"
 
+	"github.com/docker/swarmkit/api"
 	"github.com/docker/swarmkit/log"
 	"golang.org/x/net/context"
 	"google.golang.org/grpc"
@@ -79,7 +80,7 @@ func certSubjectFromContext(ctx context.Context) (pkix.Name, error) {
 
 // AuthorizeOrgAndRole takes in a context and a list of roles, and returns
 // the Node ID of the node.
-func AuthorizeOrgAndRole(ctx context.Context, org string, ou ...string) (string, error) {
+func AuthorizeOrgAndRole(ctx context.Context, org string, removedNodes []*api.RemovedNode, ou ...string) (string, error) {
 	certSubj, err := certSubjectFromContext(ctx)
 	if err != nil {
 		return "", err
@@ -87,18 +88,19 @@ func AuthorizeOrgAndRole(ctx context.Context, org string, ou ...string) (string,
 	// Check if the current certificate has an OU that authorizes
 	// access to this method
 	if intersectArrays(certSubj.OrganizationalUnit, ou) {
-		return authorizeOrg(ctx, org)
+		return authorizeOrg(certSubj, org, removedNodes)
 	}
 
 	return "", grpc.Errorf(codes.PermissionDenied, "Permission denied: remote certificate not part of OUs: %v", ou)
 }
 
-// authorizeOrg takes in a context and an organization, and returns
+// authorizeOrg takes in a certificate subject and an organization, and returns
 // the Node ID of the node.
-func authorizeOrg(ctx context.Context, org string) (string, error) {
-	certSubj, err := certSubjectFromContext(ctx)
-	if err != nil {
-		return "", err
+func authorizeOrg(certSubj pkix.Name, org string, removedNodes []*api.RemovedNode) (string, error) {
+	for _, removedNode := range removedNodes {
+		if removedNode.ID == certSubj.CommonName {
+			return "", grpc.Errorf(codes.PermissionDenied, "Permission denied: node %s was removed from swarm", certSubj.CommonName)
+		}
 	}
 
 	if len(certSubj.Organization) > 0 && certSubj.Organization[0] == org {
@@ -112,9 +114,9 @@ func authorizeOrg(ctx context.Context, org string) (string, error) {
 // been proxied by a manager, in which case the manager is authenticated and
 // so is the certificate information that it forwarded. It returns the node ID
 // of the original client.
-func AuthorizeForwardedRoleAndOrg(ctx context.Context, authorizedRoles, forwarderRoles []string, org string) (string, error) {
+func AuthorizeForwardedRoleAndOrg(ctx context.Context, authorizedRoles, forwarderRoles []string, org string, removedNodes []*api.RemovedNode) (string, error) {
 	if isForwardedRequest(ctx) {
-		_, err := AuthorizeOrgAndRole(ctx, org, forwarderRoles...)
+		_, err := AuthorizeOrgAndRole(ctx, org, removedNodes, forwarderRoles...)
 		if err != nil {
 			return "", grpc.Errorf(codes.PermissionDenied, "Permission denied: unauthorized forwarder role: %v", err)
 		}
@@ -140,7 +142,7 @@ func AuthorizeForwardedRoleAndOrg(ctx context.Context, authorizedRoles, forwarde
 	}
 
 	// There wasn't any node being forwarded, check if this is a direct call by the expected role
-	nodeID, err := AuthorizeOrgAndRole(ctx, org, authorizedRoles...)
+	nodeID, err := AuthorizeOrgAndRole(ctx, org, removedNodes, authorizedRoles...)
 	if err == nil {
 		return nodeID, nil
 	}

vendor/src/github.com/docker/swarmkit/ca/certificates.go

@@ -69,13 +69,23 @@ const (
 	MinNodeCertExpiration = 1 * time.Hour
 )
 
+// A recoverableErr is an non-fatal error encountered signing a certificate,
+// which means that the certificate issuance may be retried at a later time.
+type recoverableErr struct {
+	err error
+}
+
+func (r recoverableErr) Error() string {
+	return r.err.Error()
+}
+
 // ErrNoLocalRootCA is an error type used to indicate that the local root CA
 // certificate file does not exist.
 var ErrNoLocalRootCA = errors.New("local root CA certificate does not exist")
 
 // ErrNoValidSigner is an error type used to indicate that our RootCA doesn't have the ability to
 // sign certificates.
-var ErrNoValidSigner = errors.New("no valid signer found")
+var ErrNoValidSigner = recoverableErr{err: errors.New("no valid signer found")}
 
 func init() {
 	cflog.Level = 5
@@ -679,7 +689,7 @@ func readCertValidity(paths CertPaths) (time.Time, time.Time, error) {
 	}
 
 	// Create an x509 certificate out of the contents on disk
-	certBlock, _ := pem.Decode([]byte(cert))
+	certBlock, _ := pem.Decode(cert)
 	if certBlock == nil {
 		return zeroTime, zeroTime, errors.New("failed to decode certificate block")
 	}

vendor/src/github.com/docker/swarmkit/ca/external.go

@@ -99,23 +99,23 @@ func (eca *ExternalCA) Sign(req signer.SignRequest) (cert []byte, err error) {
 func makeExternalSignRequest(client *http.Client, url string, csrJSON []byte) (cert []byte, err error) {
 	resp, err := client.Post(url, "application/json", bytes.NewReader(csrJSON))
 	if err != nil {
-		return nil, errors.Wrap(err, "unable to perform certificate signing request")
+		return nil, recoverableErr{err: errors.Wrap(err, "unable to perform certificate signing request")}
 	}
 	defer resp.Body.Close()
 
 	body, err := ioutil.ReadAll(resp.Body)
 	if err != nil {
-		return nil, errors.Wrap(err, "unable to read CSR response body")
+		return nil, recoverableErr{err: errors.Wrap(err, "unable to read CSR response body")}
 	}
 
 	if resp.StatusCode != http.StatusOK {
-		return nil, errors.Errorf("unexpected status code in CSR response: %d - %s", resp.StatusCode, string(body))
+		return nil, recoverableErr{err: errors.Errorf("unexpected status code in CSR response: %d - %s", resp.StatusCode, string(body))}
 	}
 
 	var apiResponse api.Response
 	if err := json.Unmarshal(body, &apiResponse); err != nil {
 		log.Debugf("unable to JSON-parse CFSSL API response body: %s", string(body))
-		return nil, errors.Wrap(err, "unable to parse JSON response")
+		return nil, recoverableErr{err: errors.Wrap(err, "unable to parse JSON response")}
 	}
 
 	if !apiResponse.Success || apiResponse.Result == nil {

vendor/src/github.com/docker/swarmkit/ca/server.go

@@ -3,6 +3,7 @@ package ca
 import (
 	"crypto/subtle"
 	"sync"
+	"time"
 
 	"github.com/Sirupsen/logrus"
 	"github.com/docker/swarmkit/api"
@@ -17,18 +18,27 @@ import (
 	"google.golang.org/grpc/codes"
 )
 
+const (
+	defaultReconciliationRetryInterval = 10 * time.Second
+)
+
 // Server is the CA and NodeCA API gRPC server.
-// TODO(diogo): At some point we may want to have separate implementations of
+// TODO(aaronl): At some point we may want to have separate implementations of
 // CA, NodeCA, and other hypothetical future CA services. At the moment,
 // breaking it apart doesn't seem worth it.
 type Server struct {
-	mu             sync.Mutex
-	wg             sync.WaitGroup
-	ctx            context.Context
-	cancel         func()
-	store          *store.MemoryStore
-	securityConfig *SecurityConfig
-	joinTokens     *api.JoinTokens
+	mu                          sync.Mutex
+	wg                          sync.WaitGroup
+	ctx                         context.Context
+	cancel                      func()
+	store                       *store.MemoryStore
+	securityConfig              *SecurityConfig
+	joinTokens                  *api.JoinTokens
+	reconciliationRetryInterval time.Duration
+
+	// pending is a map of nodes with pending certificates issuance or
+	// renewal. They are indexed by node ID.
+	pending map[string]*api.Node
 
 	// Started is a channel which gets closed once the server is running
 	// and able to service RPCs.
@@ -45,12 +55,20 @@ func DefaultCAConfig() api.CAConfig {
 // NewServer creates a CA API server.
 func NewServer(store *store.MemoryStore, securityConfig *SecurityConfig) *Server {
 	return &Server{
-		store:          store,
-		securityConfig: securityConfig,
-		started:        make(chan struct{}),
+		store:                       store,
+		securityConfig:              securityConfig,
+		pending:                     make(map[string]*api.Node),
+		started:                     make(chan struct{}),
+		reconciliationRetryInterval: defaultReconciliationRetryInterval,
 	}
 }
 
+// SetReconciliationRetryInterval changes the time interval between
+// reconciliation attempts. This function must be called before Run.
+func (s *Server) SetReconciliationRetryInterval(reconciliationRetryInterval time.Duration) {
+	s.reconciliationRetryInterval = reconciliationRetryInterval
+}
+
 // NodeCertificateStatus returns the current issuance status of an issuance request identified by the nodeID
 func (s *Server) NodeCertificateStatus(ctx context.Context, request *api.NodeCertificateStatusRequest) (*api.NodeCertificateStatusResponse, error) {
 	if request.NodeID == "" {
@@ -149,16 +167,33 @@ func (s *Server) IssueNodeCertificate(ctx context.Context, request *api.IssueNod
 	}
 	defer s.doneTask()
 
+	var (
+		removedNodes []*api.RemovedNode
+		clusters     []*api.Cluster
+		err          error
+	)
+
+	s.store.View(func(readTx store.ReadTx) {
+		clusters, err = store.FindClusters(readTx, store.ByName("default"))
+
+	})
+
+	// Not having a cluster object yet means we can't check
+	// the blacklist.
+	if err == nil && len(clusters) == 1 {
+		removedNodes = clusters[0].RemovedNodes
+	}
+
 	// If the remote node is a worker (either forwarded by a manager, or calling directly),
 	// issue a renew worker certificate entry with the correct ID
-	nodeID, err := AuthorizeForwardedRoleAndOrg(ctx, []string{WorkerRole}, []string{ManagerRole}, s.securityConfig.ClientTLSCreds.Organization())
+	nodeID, err := AuthorizeForwardedRoleAndOrg(ctx, []string{WorkerRole}, []string{ManagerRole}, s.securityConfig.ClientTLSCreds.Organization(), removedNodes)
 	if err == nil {
 		return s.issueRenewCertificate(ctx, nodeID, request.CSR)
 	}
 
 	// If the remote node is a manager (either forwarded by another manager, or calling directly),
 	// issue a renew certificate entry with the correct ID
-	nodeID, err = AuthorizeForwardedRoleAndOrg(ctx, []string{ManagerRole}, []string{ManagerRole}, s.securityConfig.ClientTLSCreds.Organization())
+	nodeID, err = AuthorizeForwardedRoleAndOrg(ctx, []string{ManagerRole}, []string{ManagerRole}, s.securityConfig.ClientTLSCreds.Organization(), removedNodes)
 	if err == nil {
 		return s.issueRenewCertificate(ctx, nodeID, request.CSR)
 	}
@@ -240,7 +275,6 @@ func (s *Server) issueRenewCertificate(ctx context.Context, nodeID string, csr [
 		node *api.Node
 	)
 	err := s.store.Update(func(tx store.Tx) error {
-
 		// Attempt to retrieve the node with nodeID
 		node = store.GetNode(tx, nodeID)
 		if node == nil {
@@ -356,6 +390,9 @@ func (s *Server) Run(ctx context.Context) error {
 		}).WithError(err).Errorf("error attempting to reconcile certificates")
 	}
 
+	ticker := time.NewTicker(s.reconciliationRetryInterval)
+	defer ticker.Stop()
+
 	// Watch for new nodes being created, new nodes being updated, and changes
 	// to the cluster
 	for {
@@ -373,7 +410,16 @@ func (s *Server) Run(ctx context.Context) error {
 			case state.EventUpdateCluster:
 				s.updateCluster(ctx, v.Cluster)
 			}
-
+		case <-ticker.C:
+			for _, node := range s.pending {
+				if err := s.evaluateAndSignNodeCert(ctx, node); err != nil {
+					// If this sign operation did not succeed, the rest are
+					// unlikely to. Yield so that we don't hammer an external CA.
+					// Since the map iteration order is randomized, there is no
+					// risk of getting stuck on a problematic CSR.
+					break
+				}
+			}
 		case <-ctx.Done():
 			return ctx.Err()
 		case <-s.ctx.Done():
@@ -493,28 +539,29 @@ func (s *Server) updateCluster(ctx context.Context, cluster *api.Cluster) {
 }
 
 // evaluateAndSignNodeCert implements the logic of which certificates to sign
-func (s *Server) evaluateAndSignNodeCert(ctx context.Context, node *api.Node) {
+func (s *Server) evaluateAndSignNodeCert(ctx context.Context, node *api.Node) error {
 	// If the desired membership and actual state are in sync, there's
 	// nothing to do.
 	if node.Spec.Membership == api.NodeMembershipAccepted && node.Certificate.Status.State == api.IssuanceStateIssued {
-		return
+		return nil
 	}
 
 	// If the certificate state is renew, then it is a server-sided accepted cert (cert renewals)
 	if node.Certificate.Status.State == api.IssuanceStateRenew {
-		s.signNodeCert(ctx, node)
-		return
+		return s.signNodeCert(ctx, node)
 	}
 
 	// Sign this certificate if a user explicitly changed it to Accepted, and
 	// the certificate is in pending state
 	if node.Spec.Membership == api.NodeMembershipAccepted && node.Certificate.Status.State == api.IssuanceStatePending {
-		s.signNodeCert(ctx, node)
+		return s.signNodeCert(ctx, node)
 	}
+
+	return nil
 }
 
 // signNodeCert does the bulk of the work for signing a certificate
-func (s *Server) signNodeCert(ctx context.Context, node *api.Node) {
+func (s *Server) signNodeCert(ctx context.Context, node *api.Node) error {
 	rootCA := s.securityConfig.RootCA()
 	externalCA := s.securityConfig.externalCA
 
@@ -527,9 +574,11 @@ func (s *Server) signNodeCert(ctx context.Context, node *api.Node) {
 			"node.id": node.ID,
 			"method":  "(*Server).signNodeCert",
 		}).WithError(err).Errorf("failed to parse role")
-		return
+		return errors.New("failed to parse role")
 	}
 
+	s.pending[node.ID] = node
+
 	// Attempt to sign the CSR
 	var (
 		rawCSR = node.Certificate.CSR
@@ -550,16 +599,19 @@ func (s *Server) signNodeCert(ctx context.Context, node *api.Node) {
 			"node.id": node.ID,
 			"method":  "(*Server).signNodeCert",
 		}).WithError(err).Errorf("failed to sign CSR")
-		// If this error is due the lack of signer, maybe some other
-		// manager in the future will pick it up. Return without
-		// changing the state of the certificate.
-		if err == ErrNoValidSigner {
-			return
-		}
+
 		// If the current state is already Failed, no need to change it
 		if node.Certificate.Status.State == api.IssuanceStateFailed {
-			return
+			delete(s.pending, node.ID)
+			return errors.New("failed to sign CSR")
+		}
+
+		if _, ok := err.(recoverableErr); ok {
+			// Return without changing the state of the certificate. We may
+			// retry signing it in the future.
+			return errors.New("failed to sign CSR")
 		}
+
 		// We failed to sign this CSR, change the state to FAILED
 		err = s.store.Update(func(tx store.Tx) error {
 			node := store.GetNode(tx, nodeID)
@@ -580,7 +632,9 @@ func (s *Server) signNodeCert(ctx context.Context, node *api.Node) {
 				"method":  "(*Server).signNodeCert",
 			}).WithError(err).Errorf("transaction failed when setting state to FAILED")
 		}
-		return
+
+		delete(s.pending, node.ID)
+		return errors.New("failed to sign CSR")
 	}
 
 	// We were able to successfully sign the new CSR. Let's try to update the nodeStore
@@ -606,6 +660,7 @@ func (s *Server) signNodeCert(ctx context.Context, node *api.Node) {
 				"node.role": node.Certificate.Role,
 				"method":    "(*Server).signNodeCert",
 			}).Debugf("certificate issued")
+			delete(s.pending, node.ID)
 			break
 		}
 		if err == store.ErrSequenceConflict {
@@ -616,8 +671,9 @@ func (s *Server) signNodeCert(ctx context.Context, node *api.Node) {
 			"node.id": nodeID,
 			"method":  "(*Server).signNodeCert",
 		}).WithError(err).Errorf("transaction failed")
-		return
+		return errors.New("transaction failed")
 	}
+	return nil
 }
 
 // reconcileNodeCertificates is a helper method that calles evaluateAndSignNodeCert on all the

vendor/src/github.com/docker/swarmkit/ca/transport.go

@@ -44,6 +44,26 @@ func (c *MutableTLSCreds) Info() credentials.ProtocolInfo {
 	}
 }
 
+// Clone returns new MutableTLSCreds created from underlying *tls.Config.
+// It panics if validation of underlying config fails.
+func (c *MutableTLSCreds) Clone() credentials.TransportCredentials {
+	c.Lock()
+	newCfg, err := NewMutableTLS(c.config)
+	if err != nil {
+		panic("validation error on Clone")
+	}
+	c.Unlock()
+	return newCfg
+}
+
+// OverrideServerName overrides *tls.Config.ServerName.
+func (c *MutableTLSCreds) OverrideServerName(name string) error {
+	c.Lock()
+	c.config.ServerName = name
+	c.Unlock()
+	return nil
+}
+
 // GetRequestMetadata implements the credentials.TransportCredentials interface
 func (c *MutableTLSCreds) GetRequestMetadata(ctx context.Context, uri ...string) (map[string]string, error) {
 	return nil, nil

vendor/src/github.com/docker/swarmkit/identity/randomid.go

@@ -16,16 +16,20 @@ var (
 // parameters for random identifier generation. We can tweak this when there is
 // time for further analysis.
 const (
-	randomIDEntropyBytes = 16
+	randomIDEntropyBytes = 17
 	randomIDBase         = 36
 
 	// To ensure that all identifiers are fixed length, we make sure they
-	// get padded out to 25 characters, which is the maximum for the base36
-	// representation of 128-bit identifiers.
+	// get padded out or truncated to 25 characters.
 	//
 	// For academics,  f5lxx1zz5pnorynqglhzmsp33  == 2^128 - 1. This value
 	// was calculated from floor(log(2^128-1, 36)) + 1.
 	//
+	// While 128 bits is the largest whole-byte size that fits into 25
+	// base-36 characters, we generate an extra byte of entropy to fill
+	// in the high bits, which would otherwise be 0. This gives us a more
+	// even distribution of the first character.
+	//
 	// See http://mathworld.wolfram.com/NumberLength.html for more information.
 	maxRandomIDLength = 25
 )
@@ -34,7 +38,7 @@ const (
 // collision probability are required.
 //
 // With the parameters in this package, the generated identifier will provide
-// 128 bits of entropy encoded with base36. Leading padding is added if the
+// ~129 bits of entropy encoded with base36. Leading padding is added if the
 // string is less 25 bytes. We do not intend to maintain this interface, so
 // identifiers should be treated opaquely.
 func NewID() string {
@@ -44,7 +48,6 @@ func NewID() string {
 		panic(fmt.Errorf("failed to read random bytes: %v", err))
 	}
 
-	var nn big.Int
-	nn.SetBytes(p[:])
-	return fmt.Sprintf("%0[1]*s", maxRandomIDLength, nn.Text(randomIDBase))
+	p[0] |= 0x80 // set high bit to avoid the need for padding
+	return (&big.Int{}).SetBytes(p[:]).Text(randomIDBase)[1 : maxRandomIDLength+1]
 }

vendor/src/github.com/docker/swarmkit/manager/allocator/allocator.go

@@ -125,6 +125,8 @@ func (a *Allocator) Run(ctx context.Context) error {
 		aaCopy := aa
 		actor := func() error {
 			wg.Add(1)
+			defer wg.Done()
+
 			// init might return an allocator specific context
 			// which is a child of the passed in context to hold
 			// allocator specific state
@@ -133,10 +135,10 @@ func (a *Allocator) Run(ctx context.Context) error {
 				// if we are failing in the init of
 				// this allocator.
 				aa.cancel()
-				wg.Done()
 				return err
 			}
 
+			wg.Add(1)
 			go func() {
 				defer wg.Done()
 				a.run(ctx, aaCopy)

vendor/src/github.com/docker/swarmkit/manager/allocator/network.go

@@ -68,7 +68,7 @@ type networkContext struct {
 	unallocatedNetworks map[string]*api.Network
 }
 
-func (a *Allocator) doNetworkInit(ctx context.Context) error {
+func (a *Allocator) doNetworkInit(ctx context.Context) (err error) {
 	na, err := networkallocator.New()
 	if err != nil {
 		return err
@@ -81,6 +81,13 @@ func (a *Allocator) doNetworkInit(ctx context.Context) error {
 		unallocatedNetworks: make(map[string]*api.Network),
 		ingressNetwork:      newIngressNetwork(),
 	}
+	a.netCtx = nc
+	defer func() {
+		// Clear a.netCtx if initialization was unsuccessful.
+		if err != nil {
+			a.netCtx = nil
+		}
+	}()
 
 	// Check if we have the ingress network. If not found create
 	// it before reading all network objects for allocation.
@@ -125,7 +132,7 @@ func (a *Allocator) doNetworkInit(ctx context.Context) error {
 	// that the we can get the preferred subnet for ingress
 	// network.
 	if !na.IsAllocated(nc.ingressNetwork) {
-		if err := a.allocateNetwork(ctx, nc, nc.ingressNetwork); err != nil {
+		if err := a.allocateNetwork(ctx, nc.ingressNetwork); err != nil {
 			log.G(ctx).WithError(err).Error("failed allocating ingress network during init")
 		}
 
@@ -155,7 +162,7 @@ func (a *Allocator) doNetworkInit(ctx context.Context) error {
 			continue
 		}
 
-		if err := a.allocateNetwork(ctx, nc, n); err != nil {
+		if err := a.allocateNetwork(ctx, n); err != nil {
 			log.G(ctx).WithError(err).Errorf("failed allocating network %s during init", n.ID)
 		}
 	}
@@ -179,7 +186,7 @@ func (a *Allocator) doNetworkInit(ctx context.Context) error {
 		}
 
 		node.Attachment.Network = nc.ingressNetwork.Copy()
-		if err := a.allocateNode(ctx, nc, node); err != nil {
+		if err := a.allocateNode(ctx, node); err != nil {
 			log.G(ctx).WithError(err).Errorf("Failed to allocate network resources for node %s during init", node.ID)
 		}
 	}
@@ -198,7 +205,7 @@ func (a *Allocator) doNetworkInit(ctx context.Context) error {
 			continue
 		}
 
-		if err := a.allocateService(ctx, nc, s); err != nil {
+		if err := a.allocateService(ctx, s); err != nil {
 			log.G(ctx).WithError(err).Errorf("failed allocating service %s during init", s.ID)
 		}
 	}
@@ -260,7 +267,7 @@ func (a *Allocator) doNetworkInit(ctx context.Context) error {
 			}
 
 			err := batch.Update(func(tx store.Tx) error {
-				_, err := a.allocateTask(ctx, nc, tx, t)
+				_, err := a.allocateTask(ctx, tx, t)
 				return err
 			})
 			if err != nil {
@@ -274,7 +281,6 @@ func (a *Allocator) doNetworkInit(ctx context.Context) error {
 		return err
 	}
 
-	a.netCtx = nc
 	return nil
 }
 
@@ -288,7 +294,7 @@ func (a *Allocator) doNetworkAlloc(ctx context.Context, ev events.Event) {
 			break
 		}
 
-		if err := a.allocateNetwork(ctx, nc, n); err != nil {
+		if err := a.allocateNetwork(ctx, n); err != nil {
 			log.G(ctx).WithError(err).Errorf("Failed allocation for network %s", n.ID)
 			break
 		}
@@ -309,7 +315,7 @@ func (a *Allocator) doNetworkAlloc(ctx context.Context, ev events.Event) {
 			break
 		}
 
-		if err := a.allocateService(ctx, nc, s); err != nil {
+		if err := a.allocateService(ctx, s); err != nil {
 			log.G(ctx).WithError(err).Errorf("Failed allocation for service %s", s.ID)
 			break
 		}
@@ -320,7 +326,7 @@ func (a *Allocator) doNetworkAlloc(ctx context.Context, ev events.Event) {
 			break
 		}
 
-		if err := a.allocateService(ctx, nc, s); err != nil {
+		if err := a.allocateService(ctx, s); err != nil {
 			log.G(ctx).WithError(err).Errorf("Failed allocation during update of service %s", s.ID)
 			break
 		}
@@ -335,18 +341,18 @@ func (a *Allocator) doNetworkAlloc(ctx context.Context, ev events.Event) {
 		// it's still there.
 		delete(nc.unallocatedServices, s.ID)
 	case state.EventCreateNode, state.EventUpdateNode, state.EventDeleteNode:
-		a.doNodeAlloc(ctx, nc, ev)
+		a.doNodeAlloc(ctx, ev)
 	case state.EventCreateTask, state.EventUpdateTask, state.EventDeleteTask:
-		a.doTaskAlloc(ctx, nc, ev)
+		a.doTaskAlloc(ctx, ev)
 	case state.EventCommit:
-		a.procUnallocatedNetworks(ctx, nc)
-		a.procUnallocatedServices(ctx, nc)
-		a.procUnallocatedTasksNetwork(ctx, nc)
+		a.procUnallocatedNetworks(ctx)
+		a.procUnallocatedServices(ctx)
+		a.procUnallocatedTasksNetwork(ctx)
 		return
 	}
 }
 
-func (a *Allocator) doNodeAlloc(ctx context.Context, nc *networkContext, ev events.Event) {
+func (a *Allocator) doNodeAlloc(ctx context.Context, ev events.Event) {
 	var (
 		isDelete bool
 		node     *api.Node
@@ -362,6 +368,8 @@ func (a *Allocator) doNodeAlloc(ctx context.Context, nc *networkContext, ev even
 		node = v.Node.Copy()
 	}
 
+	nc := a.netCtx
+
 	if isDelete {
 		if nc.nwkAllocator.IsNodeAllocated(node) {
 			if err := nc.nwkAllocator.DeallocateNode(node); err != nil {
@@ -377,7 +385,7 @@ func (a *Allocator) doNodeAlloc(ctx context.Context, nc *networkContext, ev even
 		}
 
 		node.Attachment.Network = nc.ingressNetwork.Copy()
-		if err := a.allocateNode(ctx, nc, node); err != nil {
+		if err := a.allocateNode(ctx, node); err != nil {
 			log.G(ctx).WithError(err).Errorf("Failed to allocate network resources for node %s", node.ID)
 		}
 	}
@@ -445,26 +453,22 @@ func (a *Allocator) taskCreateNetworkAttachments(t *api.Task, s *api.Service) {
 
 		for _, na := range specNetworks {
 			n := store.GetNetwork(tx, na.Target)
-			if n != nil {
-				var aliases []string
-				var addresses []string
+			if n == nil {
+				continue
+			}
 
-				for _, a := range na.Aliases {
-					aliases = append(aliases, a)
-				}
-				for _, a := range na.Addresses {
-					addresses = append(addresses, a)
-				}
+			attachment := api.NetworkAttachment{Network: n}
+			attachment.Aliases = append(attachment.Aliases, na.Aliases...)
+			attachment.Addresses = append(attachment.Addresses, na.Addresses...)
 
-				networks = append(networks, &api.NetworkAttachment{Network: n, Aliases: aliases, Addresses: addresses})
-			}
+			networks = append(networks, &attachment)
 		}
 	})
 
 	taskUpdateNetworks(t, networks)
 }
 
-func (a *Allocator) doTaskAlloc(ctx context.Context, nc *networkContext, ev events.Event) {
+func (a *Allocator) doTaskAlloc(ctx context.Context, ev events.Event) {
 	var (
 		isDelete bool
 		t        *api.Task
@@ -480,6 +484,8 @@ func (a *Allocator) doTaskAlloc(ctx context.Context, nc *networkContext, ev even
 		t = v.Task.Copy()
 	}
 
+	nc := a.netCtx
+
 	// If the task has stopped running or it's being deleted then
 	// we should free the network resources associated with the
 	// task right away.
@@ -530,7 +536,9 @@ func (a *Allocator) doTaskAlloc(ctx context.Context, nc *networkContext, ev even
 	nc.unallocatedTasks[t.ID] = t
 }
 
-func (a *Allocator) allocateNode(ctx context.Context, nc *networkContext, node *api.Node) error {
+func (a *Allocator) allocateNode(ctx context.Context, node *api.Node) error {
+	nc := a.netCtx
+
 	if err := nc.nwkAllocator.AllocateNode(node); err != nil {
 		return err
 	}
@@ -563,7 +571,9 @@ func (a *Allocator) allocateNode(ctx context.Context, nc *networkContext, node *
 	return nil
 }
 
-func (a *Allocator) allocateService(ctx context.Context, nc *networkContext, s *api.Service) error {
+func (a *Allocator) allocateService(ctx context.Context, s *api.Service) error {
+	nc := a.netCtx
+
 	if s.Spec.Endpoint != nil {
 		// service has user-defined endpoint
 		if s.Endpoint == nil {
@@ -648,7 +658,9 @@ func (a *Allocator) allocateService(ctx context.Context, nc *networkContext, s *
 	return nil
 }
 
-func (a *Allocator) allocateNetwork(ctx context.Context, nc *networkContext, n *api.Network) error {
+func (a *Allocator) allocateNetwork(ctx context.Context, n *api.Network) error {
+	nc := a.netCtx
+
 	if err := nc.nwkAllocator.Allocate(n); err != nil {
 		nc.unallocatedNetworks[n.ID] = n
 		return errors.Wrapf(err, "failed during network allocation for network %s", n.ID)
@@ -670,7 +682,7 @@ func (a *Allocator) allocateNetwork(ctx context.Context, nc *networkContext, n *
 	return nil
 }
 
-func (a *Allocator) allocateTask(ctx context.Context, nc *networkContext, tx store.Tx, t *api.Task) (*api.Task, error) {
+func (a *Allocator) allocateTask(ctx context.Context, tx store.Tx, t *api.Task) (*api.Task, error) {
 	taskUpdated := false
 
 	// Get the latest task state from the store before updating.
@@ -679,6 +691,8 @@ func (a *Allocator) allocateTask(ctx context.Context, nc *networkContext, tx sto
 		return nil, fmt.Errorf("could not find task %s while trying to update network allocation", t.ID)
 	}
 
+	nc := a.netCtx
+
 	// We might be here even if a task allocation has already
 	// happened but wasn't successfully committed to store. In such
 	// cases skip allocation and go straight ahead to updating the
@@ -738,10 +752,11 @@ func (a *Allocator) allocateTask(ctx context.Context, nc *networkContext, tx sto
 	return storeT, nil
 }
 
-func (a *Allocator) procUnallocatedNetworks(ctx context.Context, nc *networkContext) {
+func (a *Allocator) procUnallocatedNetworks(ctx context.Context) {
+	nc := a.netCtx
 	for _, n := range nc.unallocatedNetworks {
 		if !nc.nwkAllocator.IsAllocated(n) {
-			if err := a.allocateNetwork(ctx, nc, n); err != nil {
+			if err := a.allocateNetwork(ctx, n); err != nil {
 				log.G(ctx).Debugf("Failed allocation of unallocated network %s: %v", n.ID, err)
 				continue
 			}
@@ -751,10 +766,11 @@ func (a *Allocator) procUnallocatedNetworks(ctx context.Context, nc *networkCont
 	}
 }
 
-func (a *Allocator) procUnallocatedServices(ctx context.Context, nc *networkContext) {
+func (a *Allocator) procUnallocatedServices(ctx context.Context) {
+	nc := a.netCtx
 	for _, s := range nc.unallocatedServices {
 		if !nc.nwkAllocator.IsServiceAllocated(s) {
-			if err := a.allocateService(ctx, nc, s); err != nil {
+			if err := a.allocateService(ctx, s); err != nil {
 				log.G(ctx).Debugf("Failed allocation of unallocated service %s: %v", s.ID, err)
 				continue
 			}
@@ -764,7 +780,8 @@ func (a *Allocator) procUnallocatedServices(ctx context.Context, nc *networkCont
 	}
 }
 
-func (a *Allocator) procUnallocatedTasksNetwork(ctx context.Context, nc *networkContext) {
+func (a *Allocator) procUnallocatedTasksNetwork(ctx context.Context) {
+	nc := a.netCtx
 	tasks := make([]*api.Task, 0, len(nc.unallocatedTasks))
 
 	committed, err := a.store.Batch(func(batch *store.Batch) error {
@@ -772,7 +789,7 @@ func (a *Allocator) procUnallocatedTasksNetwork(ctx context.Context, nc *network
 			var allocatedT *api.Task
 			err := batch.Update(func(tx store.Tx) error {
 				var err error
-				allocatedT, err = a.allocateTask(ctx, nc, tx, t)
+				allocatedT, err = a.allocateTask(ctx, tx, t)
 				return err
 			})
 

vendor/src/github.com/docker/swarmkit/manager/allocator/networkallocator/drivers_darwin.go

@@ -0,0 +1,13 @@
+package networkallocator
+
+import (
+	"github.com/docker/libnetwork/drivers/overlay/ovmanager"
+	"github.com/docker/libnetwork/drivers/remote"
+)
+
+func getInitializers() []initializer {
+	return []initializer{
+		{remote.Init, "remote"},
+		{ovmanager.Init, "overlay"},
+	}
+}

vendor/src/github.com/docker/swarmkit/manager/allocator/networkallocator/drivers_unsupported.go

@@ -1,4 +1,4 @@
-// +build !linux
+// +build !linux,!darwin
 
 package networkallocator
 

vendor/src/github.com/docker/swarmkit/manager/allocator/networkallocator/networkallocator.go

@@ -518,7 +518,7 @@ func (na *NetworkAllocator) allocateNetworkIPs(nAttach *api.NetworkAttachment) e
 	}
 
 	addresses := nAttach.Addresses
-	if addresses == nil {
+	if len(addresses) == 0 {
 		addresses = []string{""}
 	}
 

vendor/src/github.com/docker/swarmkit/manager/controlapi/cluster.go

@@ -202,6 +202,7 @@ func redactClusters(clusters []*api.Cluster) []*api.Cluster {
 				CACertHash: cluster.RootCA.CACertHash,
 				JoinTokens: cluster.RootCA.JoinTokens,
 			},
+			RemovedNodes: cluster.RemovedNodes,
 		}
 
 		redactedClusters = append(redactedClusters, newCluster)

vendor/src/github.com/docker/swarmkit/manager/controlapi/common.go

@@ -64,10 +64,15 @@ func filterMatchLabels(match map[string]string, candidates map[string]string) bo
 func validateAnnotations(m api.Annotations) error {
 	if m.Name == "" {
 		return grpc.Errorf(codes.InvalidArgument, "meta: name must be provided")
-	} else if !isValidName.MatchString(m.Name) {
+	}
+	if !isValidName.MatchString(m.Name) {
 		// if the name doesn't match the regex
 		return grpc.Errorf(codes.InvalidArgument, "name must be valid as a DNS name component")
 	}
+	if len(m.Name) > 63 {
+		// DNS labels are limited to 63 characters
+		return grpc.Errorf(codes.InvalidArgument, "name must be 63 characters or fewer")
+	}
 	return nil
 }
 

vendor/src/github.com/docker/swarmkit/manager/controlapi/node.go

@@ -1,9 +1,13 @@
 package controlapi
 
 import (
+	"crypto/x509"
+	"encoding/pem"
+
 	"github.com/docker/swarmkit/api"
 	"github.com/docker/swarmkit/manager/state/raft/membership"
 	"github.com/docker/swarmkit/manager/state/store"
+	"github.com/docker/swarmkit/protobuf/ptypes"
 	"golang.org/x/net/context"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/codes"
@@ -286,6 +290,40 @@ func (s *Server) RemoveNode(ctx context.Context, request *api.RemoveNodeRequest)
 		if !request.Force && node.Status.State == api.NodeStatus_READY {
 			return grpc.Errorf(codes.FailedPrecondition, "node %s is not down and can't be removed", request.NodeID)
 		}
+
+		// lookup the cluster
+		clusters, err := store.FindClusters(tx, store.ByName("default"))
+		if err != nil {
+			return err
+		}
+		if len(clusters) != 1 {
+			return grpc.Errorf(codes.Internal, "could not fetch cluster object")
+		}
+		cluster := clusters[0]
+
+		removedNode := &api.RemovedNode{ID: node.ID}
+
+		// Set an expiry time for this RemovedNode if a certificate
+		// exists and can be parsed.
+		if len(node.Certificate.Certificate) != 0 {
+			certBlock, _ := pem.Decode(node.Certificate.Certificate)
+			if certBlock != nil {
+				X509Cert, err := x509.ParseCertificate(certBlock.Bytes)
+				if err == nil && !X509Cert.NotAfter.IsZero() {
+					expiry, err := ptypes.TimestampProto(X509Cert.NotAfter)
+					if err == nil {
+						removedNode.Expiry = expiry
+					}
+				}
+			}
+		}
+
+		cluster.RemovedNodes = append(cluster.RemovedNodes, removedNode)
+
+		if err := store.UpdateCluster(tx, cluster); err != nil {
+			return err
+		}
+
 		return store.DeleteNode(tx, request.NodeID)
 	})
 	if err != nil {

vendor/src/github.com/docker/swarmkit/manager/controlapi/secret.go

@@ -48,7 +48,6 @@ func (s *Server) GetSecret(ctx context.Context, request *api.GetSecretRequest) (
 		return nil, grpc.Errorf(codes.NotFound, "secret %s not found", request.SecretID)
 	}
 
-	secret.Spec.Data = nil // clean the actual secret data so it's never returned
 	return &api.GetSecretResponse{Secret: secret}, nil
 }
 

vendor/src/github.com/docker/swarmkit/manager/controlapi/service.go

@@ -158,13 +158,25 @@ func validateEndpointSpec(epSpec *api.EndpointSpec) error {
 		return grpc.Errorf(codes.InvalidArgument, "EndpointSpec: ports can't be used with dnsrr mode")
 	}
 
-	portSet := make(map[uint32]struct{})
+	type portSpec struct {
+		publishedPort uint32
+		protocol      api.PortConfig_Protocol
+	}
+
+	portSet := make(map[portSpec]struct{})
 	for _, port := range epSpec.Ports {
-		if _, ok := portSet[port.PublishedPort]; ok {
+		// If published port is not specified, it does not conflict
+		// with any others.
+		if port.PublishedPort == 0 {
+			continue
+		}
+
+		portSpec := portSpec{publishedPort: port.PublishedPort, protocol: port.Protocol}
+		if _, ok := portSet[portSpec]; ok {
 			return grpc.Errorf(codes.InvalidArgument, "EndpointSpec: duplicate published ports provided")
 		}
 
-		portSet[port.PublishedPort] = struct{}{}
+		portSet[portSpec] = struct{}{}
 	}
 
 	return nil

vendor/src/github.com/docker/swarmkit/manager/dispatcher/dispatcher.go

@@ -1283,8 +1283,3 @@ func (d *Dispatcher) Session(r *api.SessionRequest, stream api.Dispatcher_Sessio
 		}
 	}
 }
-
-// NodeCount returns number of nodes which connected to this dispatcher.
-func (d *Dispatcher) NodeCount() int {
-	return d.nodes.Len()
-}

vendor/src/github.com/docker/swarmkit/manager/manager.go

@@ -9,6 +9,7 @@ import (
 	"path/filepath"
 	"sync"
 	"syscall"
+	"time"
 
 	"github.com/Sirupsen/logrus"
 	"github.com/docker/go-events"
@@ -80,16 +81,17 @@ type Manager struct {
 	listeners map[string]net.Listener
 
 	caserver               *ca.Server
-	Dispatcher             *dispatcher.Dispatcher
+	dispatcher             *dispatcher.Dispatcher
 	replicatedOrchestrator *orchestrator.ReplicatedOrchestrator
 	globalOrchestrator     *orchestrator.GlobalOrchestrator
 	taskReaper             *orchestrator.TaskReaper
+	constraintEnforcer     *orchestrator.ConstraintEnforcer
 	scheduler              *scheduler.Scheduler
 	allocator              *allocator.Allocator
 	keyManager             *keymanager.KeyManager
 	server                 *grpc.Server
 	localserver            *grpc.Server
-	RaftNode               *raft.Node
+	raftNode               *raft.Node
 
 	mu sync.Mutex
 
@@ -210,7 +212,7 @@ func New(config *Config) (*Manager, error) {
 		ForceNewCluster: config.ForceNewCluster,
 		TLSCredentials:  config.SecurityConfig.ClientTLSCreds,
 	}
-	RaftNode := raft.NewNode(context.TODO(), newNodeOpts)
+	raftNode := raft.NewNode(newNodeOpts)
 
 	opts := []grpc.ServerOption{
 		grpc.Creds(config.SecurityConfig.ServerTLSCreds)}
@@ -218,11 +220,11 @@ func New(config *Config) (*Manager, error) {
 	m := &Manager{
 		config:      config,
 		listeners:   listeners,
-		caserver:    ca.NewServer(RaftNode.MemoryStore(), config.SecurityConfig),
-		Dispatcher:  dispatcher.New(RaftNode, dispatcherConfig),
+		caserver:    ca.NewServer(raftNode.MemoryStore(), config.SecurityConfig),
+		dispatcher:  dispatcher.New(raftNode, dispatcherConfig),
 		server:      grpc.NewServer(opts...),
 		localserver: grpc.NewServer(opts...),
-		RaftNode:    RaftNode,
+		raftNode:    raftNode,
 		started:     make(chan struct{}),
 		stopped:     make(chan struct{}),
 	}
@@ -254,36 +256,53 @@ func (m *Manager) Run(parent context.Context) error {
 		}
 	}()
 
-	leadershipCh, cancel := m.RaftNode.SubscribeLeadership()
+	leadershipCh, cancel := m.raftNode.SubscribeLeadership()
 	defer cancel()
 
 	go m.handleLeadershipEvents(ctx, leadershipCh)
 
 	authorize := func(ctx context.Context, roles []string) error {
+		var (
+			removedNodes []*api.RemovedNode
+			clusters     []*api.Cluster
+			err          error
+		)
+
+		m.raftNode.MemoryStore().View(func(readTx store.ReadTx) {
+			clusters, err = store.FindClusters(readTx, store.ByName("default"))
+
+		})
+
+		// Not having a cluster object yet means we can't check
+		// the blacklist.
+		if err == nil && len(clusters) == 1 {
+			removedNodes = clusters[0].RemovedNodes
+		}
+
 		// Authorize the remote roles, ensure they can only be forwarded by managers
-		_, err := ca.AuthorizeForwardedRoleAndOrg(ctx, roles, []string{ca.ManagerRole}, m.config.SecurityConfig.ClientTLSCreds.Organization())
+		_, err = ca.AuthorizeForwardedRoleAndOrg(ctx, roles, []string{ca.ManagerRole}, m.config.SecurityConfig.ClientTLSCreds.Organization(), removedNodes)
 		return err
 	}
 
-	baseControlAPI := controlapi.NewServer(m.RaftNode.MemoryStore(), m.RaftNode, m.config.SecurityConfig.RootCA())
-	baseResourceAPI := resourceapi.New(m.RaftNode.MemoryStore())
+	baseControlAPI := controlapi.NewServer(m.raftNode.MemoryStore(), m.raftNode, m.config.SecurityConfig.RootCA())
+	baseResourceAPI := resourceapi.New(m.raftNode.MemoryStore())
 	healthServer := health.NewHealthServer()
 	localHealthServer := health.NewHealthServer()
 
 	authenticatedControlAPI := api.NewAuthenticatedWrapperControlServer(baseControlAPI, authorize)
 	authenticatedResourceAPI := api.NewAuthenticatedWrapperResourceAllocatorServer(baseResourceAPI, authorize)
-	authenticatedDispatcherAPI := api.NewAuthenticatedWrapperDispatcherServer(m.Dispatcher, authorize)
+	authenticatedDispatcherAPI := api.NewAuthenticatedWrapperDispatcherServer(m.dispatcher, authorize)
 	authenticatedCAAPI := api.NewAuthenticatedWrapperCAServer(m.caserver, authorize)
 	authenticatedNodeCAAPI := api.NewAuthenticatedWrapperNodeCAServer(m.caserver, authorize)
-	authenticatedRaftAPI := api.NewAuthenticatedWrapperRaftServer(m.RaftNode, authorize)
+	authenticatedRaftAPI := api.NewAuthenticatedWrapperRaftServer(m.raftNode, authorize)
 	authenticatedHealthAPI := api.NewAuthenticatedWrapperHealthServer(healthServer, authorize)
-	authenticatedRaftMembershipAPI := api.NewAuthenticatedWrapperRaftMembershipServer(m.RaftNode, authorize)
+	authenticatedRaftMembershipAPI := api.NewAuthenticatedWrapperRaftMembershipServer(m.raftNode, authorize)
 
-	proxyDispatcherAPI := api.NewRaftProxyDispatcherServer(authenticatedDispatcherAPI, m.RaftNode, ca.WithMetadataForwardTLSInfo)
-	proxyCAAPI := api.NewRaftProxyCAServer(authenticatedCAAPI, m.RaftNode, ca.WithMetadataForwardTLSInfo)
-	proxyNodeCAAPI := api.NewRaftProxyNodeCAServer(authenticatedNodeCAAPI, m.RaftNode, ca.WithMetadataForwardTLSInfo)
-	proxyRaftMembershipAPI := api.NewRaftProxyRaftMembershipServer(authenticatedRaftMembershipAPI, m.RaftNode, ca.WithMetadataForwardTLSInfo)
-	proxyResourceAPI := api.NewRaftProxyResourceAllocatorServer(authenticatedResourceAPI, m.RaftNode, ca.WithMetadataForwardTLSInfo)
+	proxyDispatcherAPI := api.NewRaftProxyDispatcherServer(authenticatedDispatcherAPI, m.raftNode, ca.WithMetadataForwardTLSInfo)
+	proxyCAAPI := api.NewRaftProxyCAServer(authenticatedCAAPI, m.raftNode, ca.WithMetadataForwardTLSInfo)
+	proxyNodeCAAPI := api.NewRaftProxyNodeCAServer(authenticatedNodeCAAPI, m.raftNode, ca.WithMetadataForwardTLSInfo)
+	proxyRaftMembershipAPI := api.NewRaftProxyRaftMembershipServer(authenticatedRaftMembershipAPI, m.raftNode, ca.WithMetadataForwardTLSInfo)
+	proxyResourceAPI := api.NewRaftProxyResourceAllocatorServer(authenticatedResourceAPI, m.raftNode, ca.WithMetadataForwardTLSInfo)
 
 	// localProxyControlAPI is a special kind of proxy. It is only wired up
 	// to receive requests from a trusted local socket, and these requests
@@ -292,7 +311,7 @@ func (m *Manager) Run(parent context.Context) error {
 	// this manager rather than forwarded requests (it has no TLS
 	// information to put in the metadata map).
 	forwardAsOwnRequest := func(ctx context.Context) (context.Context, error) { return ctx, nil }
-	localProxyControlAPI := api.NewRaftProxyControlServer(baseControlAPI, m.RaftNode, forwardAsOwnRequest)
+	localProxyControlAPI := api.NewRaftProxyControlServer(baseControlAPI, m.raftNode, forwardAsOwnRequest)
 
 	// Everything registered on m.server should be an authenticated
 	// wrapper, or a proxy wrapping an authenticated wrapper!
@@ -324,7 +343,7 @@ func (m *Manager) Run(parent context.Context) error {
 	// Set the raft server as serving for the health server
 	healthServer.SetServingStatus("Raft", api.HealthCheckResponse_SERVING)
 
-	if err := m.RaftNode.JoinAndStart(); err != nil {
+	if err := m.raftNode.JoinAndStart(ctx); err != nil {
 		return errors.Wrap(err, "can't initialize raft node")
 	}
 
@@ -333,28 +352,28 @@ func (m *Manager) Run(parent context.Context) error {
 	close(m.started)
 
 	go func() {
-		err := m.RaftNode.Run(ctx)
+		err := m.raftNode.Run(ctx)
 		if err != nil {
 			log.G(ctx).Error(err)
 			m.Stop(ctx)
 		}
 	}()
 
-	if err := raft.WaitForLeader(ctx, m.RaftNode); err != nil {
+	if err := raft.WaitForLeader(ctx, m.raftNode); err != nil {
 		return err
 	}
 
-	c, err := raft.WaitForCluster(ctx, m.RaftNode)
+	c, err := raft.WaitForCluster(ctx, m.raftNode)
 	if err != nil {
 		return err
 	}
 	raftConfig := c.Spec.Raft
 
-	if int(raftConfig.ElectionTick) != m.RaftNode.Config.ElectionTick {
-		log.G(ctx).Warningf("election tick value (%ds) is different from the one defined in the cluster config (%vs), the cluster may be unstable", m.RaftNode.Config.ElectionTick, raftConfig.ElectionTick)
+	if int(raftConfig.ElectionTick) != m.raftNode.Config.ElectionTick {
+		log.G(ctx).Warningf("election tick value (%ds) is different from the one defined in the cluster config (%vs), the cluster may be unstable", m.raftNode.Config.ElectionTick, raftConfig.ElectionTick)
 	}
-	if int(raftConfig.HeartbeatTick) != m.RaftNode.Config.HeartbeatTick {
-		log.G(ctx).Warningf("heartbeat tick value (%ds) is different from the one defined in the cluster config (%vs), the cluster may be unstable", m.RaftNode.Config.HeartbeatTick, raftConfig.HeartbeatTick)
+	if int(raftConfig.HeartbeatTick) != m.raftNode.Config.HeartbeatTick {
+		log.G(ctx).Warningf("heartbeat tick value (%ds) is different from the one defined in the cluster config (%vs), the cluster may be unstable", m.raftNode.Config.HeartbeatTick, raftConfig.HeartbeatTick)
 	}
 
 	// wait for an error in serving.
@@ -379,11 +398,12 @@ func (m *Manager) Run(parent context.Context) error {
 	}
 }
 
+const stopTimeout = 8 * time.Second
+
 // Stop stops the manager. It immediately closes all open connections and
 // active RPCs as well as stopping the scheduler.
 func (m *Manager) Stop(ctx context.Context) {
 	log.G(ctx).Info("Stopping manager")
-
 	// It's not safe to start shutting down while the manager is still
 	// starting up.
 	<-m.started
@@ -400,12 +420,17 @@ func (m *Manager) Stop(ctx context.Context) {
 		// do nothing, we're stopping for the first time
 	}
 
-	// once we start stopping, send a signal that we're doing so. this tells
-	// Run that we've started stopping, when it gets the error from errServe
-	// it also prevents the loop from processing any more stuff.
-	close(m.stopped)
+	srvDone, localSrvDone := make(chan struct{}), make(chan struct{})
+	go func() {
+		m.server.GracefulStop()
+		close(srvDone)
+	}()
+	go func() {
+		m.localserver.GracefulStop()
+		close(localSrvDone)
+	}()
 
-	m.Dispatcher.Stop()
+	m.dispatcher.Stop()
 	m.caserver.Stop()
 
 	if m.allocator != nil {
@@ -420,6 +445,9 @@ func (m *Manager) Stop(ctx context.Context) {
 	if m.taskReaper != nil {
 		m.taskReaper.Stop()
 	}
+	if m.constraintEnforcer != nil {
+		m.constraintEnforcer.Stop()
+	}
 	if m.scheduler != nil {
 		m.scheduler.Stop()
 	}
@@ -427,10 +455,26 @@ func (m *Manager) Stop(ctx context.Context) {
 		m.keyManager.Stop()
 	}
 
-	m.RaftNode.Shutdown()
-	// some time after this point, Run will receive an error from one of these
-	m.server.Stop()
-	m.localserver.Stop()
+	// once we start stopping, send a signal that we're doing so. this tells
+	// Run that we've started stopping, when it gets the error from errServe
+	// it also prevents the loop from processing any more stuff.
+	close(m.stopped)
+
+	<-m.raftNode.Done()
+
+	timer := time.AfterFunc(stopTimeout, func() {
+		m.server.Stop()
+		m.localserver.Stop()
+	})
+	defer timer.Stop()
+	// TODO: we're not waiting on ctx because it very well could be passed from Run,
+	// which is already cancelled here. We need to refactor that.
+	select {
+	case <-srvDone:
+		<-localSrvDone
+	case <-localSrvDone:
+		<-srvDone
+	}
 
 	log.G(ctx).Info("Manager shut down")
 	// mutex is released and Run can return now
@@ -452,7 +496,7 @@ func (m *Manager) rotateRootCAKEK(ctx context.Context, clusterID string) error {
 	passphrase := []byte(strPassphrase)
 	passphrasePrev := []byte(strPassphrasePrev)
 
-	s := m.RaftNode.MemoryStore()
+	s := m.raftNode.MemoryStore()
 	var (
 		cluster  *api.Cluster
 		err      error
@@ -468,7 +512,7 @@ func (m *Manager) rotateRootCAKEK(ctx context.Context, clusterID string) error {
 
 	// Try to get the private key from the cluster
 	privKeyPEM := cluster.RootCA.CAKey
-	if privKeyPEM == nil || len(privKeyPEM) == 0 {
+	if len(privKeyPEM) == 0 {
 		// We have no PEM root private key in this cluster.
 		log.G(ctx).Warnf("cluster %s does not have private key material", clusterID)
 		return nil
@@ -575,14 +619,14 @@ func (m *Manager) serveListener(ctx context.Context, errServe chan error, proto
 
 // becomeLeader starts the subsystems that are run on the leader.
 func (m *Manager) becomeLeader(ctx context.Context) {
-	s := m.RaftNode.MemoryStore()
+	s := m.raftNode.MemoryStore()
 
 	rootCA := m.config.SecurityConfig.RootCA()
 	nodeID := m.config.SecurityConfig.ClientTLSCreds.NodeID()
 
 	raftCfg := raft.DefaultRaftConfig()
-	raftCfg.ElectionTick = uint32(m.RaftNode.Config.ElectionTick)
-	raftCfg.HeartbeatTick = uint32(m.RaftNode.Config.HeartbeatTick)
+	raftCfg.ElectionTick = uint32(m.raftNode.Config.ElectionTick)
+	raftCfg.HeartbeatTick = uint32(m.raftNode.Config.HeartbeatTick)
 
 	clusterID := m.config.SecurityConfig.ClientTLSCreds.Organization()
 
@@ -608,6 +652,7 @@ func (m *Manager) becomeLeader(ctx context.Context) {
 	}
 
 	m.replicatedOrchestrator = orchestrator.NewReplicatedOrchestrator(s)
+	m.constraintEnforcer = orchestrator.NewConstraintEnforcer(s)
 	m.globalOrchestrator = orchestrator.NewGlobalOrchestrator(s)
 	m.taskReaper = orchestrator.NewTaskReaper(s)
 	m.scheduler = scheduler.New(s)
@@ -636,7 +681,7 @@ func (m *Manager) becomeLeader(ctx context.Context) {
 		if err := d.Run(ctx); err != nil {
 			log.G(ctx).WithError(err).Error("Dispatcher exited with an error")
 		}
-	}(m.Dispatcher)
+	}(m.dispatcher)
 
 	go func(server *ca.Server) {
 		if err := server.Run(ctx); err != nil {
@@ -661,6 +706,10 @@ func (m *Manager) becomeLeader(ctx context.Context) {
 		}
 	}(m.scheduler)
 
+	go func(constraintEnforcer *orchestrator.ConstraintEnforcer) {
+		constraintEnforcer.Run()
+	}(m.constraintEnforcer)
+
 	go func(taskReaper *orchestrator.TaskReaper) {
 		taskReaper.Run()
 	}(m.taskReaper)
@@ -681,7 +730,7 @@ func (m *Manager) becomeLeader(ctx context.Context) {
 
 // becomeFollower shuts down the subsystems that are only run by the leader.
 func (m *Manager) becomeFollower() {
-	m.Dispatcher.Stop()
+	m.dispatcher.Stop()
 	m.caserver.Stop()
 
 	if m.allocator != nil {
@@ -689,6 +738,9 @@ func (m *Manager) becomeFollower() {
 		m.allocator = nil
 	}
 
+	m.constraintEnforcer.Stop()
+	m.constraintEnforcer = nil
+
 	m.replicatedOrchestrator.Stop()
 	m.replicatedOrchestrator = nil
 

vendor/src/github.com/docker/swarmkit/manager/orchestrator/constraint_enforcer.go

@@ -0,0 +1,157 @@
+package orchestrator
+
+import (
+	"github.com/docker/swarmkit/api"
+	"github.com/docker/swarmkit/log"
+	"github.com/docker/swarmkit/manager/constraint"
+	"github.com/docker/swarmkit/manager/state"
+	"github.com/docker/swarmkit/manager/state/store"
+)
+
+// ConstraintEnforcer watches for updates to nodes and shuts down tasks that no
+// longer satisfy scheduling constraints or resource limits.
+type ConstraintEnforcer struct {
+	store    *store.MemoryStore
+	stopChan chan struct{}
+	doneChan chan struct{}
+}
+
+// NewConstraintEnforcer creates a new ConstraintEnforcer.
+func NewConstraintEnforcer(store *store.MemoryStore) *ConstraintEnforcer {
+	return &ConstraintEnforcer{
+		store:    store,
+		stopChan: make(chan struct{}),
+		doneChan: make(chan struct{}),
+	}
+}
+
+// Run is the ConstraintEnforcer's main loop.
+func (ce *ConstraintEnforcer) Run() {
+	defer close(ce.doneChan)
+
+	watcher, cancelWatch := state.Watch(ce.store.WatchQueue(), state.EventUpdateNode{})
+	defer cancelWatch()
+
+	var (
+		nodes []*api.Node
+		err   error
+	)
+	ce.store.View(func(readTx store.ReadTx) {
+		nodes, err = store.FindNodes(readTx, store.All)
+	})
+	if err != nil {
+		log.L.WithError(err).Error("failed to check nodes for noncompliant tasks")
+	} else {
+		for _, node := range nodes {
+			ce.shutdownNoncompliantTasks(node)
+		}
+	}
+
+	for {
+		select {
+		case event := <-watcher:
+			node := event.(state.EventUpdateNode).Node
+			ce.shutdownNoncompliantTasks(node)
+		case <-ce.stopChan:
+			return
+		}
+	}
+}
+
+func (ce *ConstraintEnforcer) shutdownNoncompliantTasks(node *api.Node) {
+	// If the availability is "drain", the orchestrator will
+	// shut down all tasks.
+	// If the availability is "pause", we shouldn't touch
+	// the tasks on this node.
+	if node.Spec.Availability != api.NodeAvailabilityActive {
+		return
+	}
+
+	var (
+		tasks []*api.Task
+		err   error
+	)
+
+	ce.store.View(func(tx store.ReadTx) {
+		tasks, err = store.FindTasks(tx, store.ByNodeID(node.ID))
+	})
+
+	if err != nil {
+		log.L.WithError(err).Errorf("failed to list tasks for node ID %s", node.ID)
+	}
+
+	var availableMemoryBytes, availableNanoCPUs int64
+	if node.Description != nil && node.Description.Resources != nil {
+		availableMemoryBytes = node.Description.Resources.MemoryBytes
+		availableNanoCPUs = node.Description.Resources.NanoCPUs
+	}
+
+	removeTasks := make(map[string]*api.Task)
+
+	// TODO(aaronl): The set of tasks removed will be
+	// nondeterministic because it depends on the order of
+	// the slice returned from FindTasks. We could do
+	// a separate pass over the tasks for each type of
+	// resource, and sort by the size of the reservation
+	// to remove the most resource-intensive tasks.
+	for _, t := range tasks {
+		if t.DesiredState < api.TaskStateAssigned || t.DesiredState > api.TaskStateRunning {
+			continue
+		}
+
+		// Ensure that the task still meets scheduling
+		// constraints.
+		if t.Spec.Placement != nil && len(t.Spec.Placement.Constraints) != 0 {
+			constraints, _ := constraint.Parse(t.Spec.Placement.Constraints)
+			if !constraint.NodeMatches(constraints, node) {
+				removeTasks[t.ID] = t
+				continue
+			}
+		}
+
+		// Ensure that the task assigned to the node
+		// still satisfies the resource limits.
+		if t.Spec.Resources != nil && t.Spec.Resources.Reservations != nil {
+			if t.Spec.Resources.Reservations.MemoryBytes > availableMemoryBytes {
+				removeTasks[t.ID] = t
+				continue
+			}
+			if t.Spec.Resources.Reservations.NanoCPUs > availableNanoCPUs {
+				removeTasks[t.ID] = t
+				continue
+			}
+			availableMemoryBytes -= t.Spec.Resources.Reservations.MemoryBytes
+			availableNanoCPUs -= t.Spec.Resources.Reservations.NanoCPUs
+		}
+	}
+
+	if len(removeTasks) != 0 {
+		_, err := ce.store.Batch(func(batch *store.Batch) error {
+			for _, t := range removeTasks {
+				err := batch.Update(func(tx store.Tx) error {
+					t = store.GetTask(tx, t.ID)
+					if t == nil || t.DesiredState > api.TaskStateRunning {
+						return nil
+					}
+
+					t.DesiredState = api.TaskStateShutdown
+					return store.UpdateTask(tx, t)
+				})
+				if err != nil {
+					log.L.WithError(err).Errorf("failed to shut down task %s", t.ID)
+				}
+			}
+			return nil
+		})
+
+		if err != nil {
+			log.L.WithError(err).Errorf("failed to shut down tasks")
+		}
+	}
+}
+
+// Stop stops the ConstraintEnforcer and waits for the main loop to exit.
+func (ce *ConstraintEnforcer) Stop() {
+	close(ce.stopChan)
+	<-ce.doneChan
+}

vendor/src/github.com/docker/swarmkit/manager/orchestrator/global.go

@@ -374,11 +374,13 @@ func (g *GlobalOrchestrator) reconcileServicesOneNode(ctx context.Context, servi
 				continue
 			}
 
-			meetsConstraints := constraint.NodeMatches(service.constraints, node)
+			if !constraint.NodeMatches(service.constraints, node) {
+				continue
+			}
 
 			// if restart policy considers this node has finished its task
 			// it should remove all running tasks
-			if completed[serviceID] || !meetsConstraints {
+			if completed[serviceID] {
 				g.removeTasks(ctx, batch, tasks[serviceID])
 				continue
 			}

vendor/src/github.com/docker/swarmkit/manager/orchestrator/services.go

@@ -168,6 +168,9 @@ func (r *ReplicatedOrchestrator) reconcile(ctx context.Context, service *api.Ser
 			r.deleteTasksMap(ctx, batch, deadSlots)
 			return nil
 		})
+		if err != nil {
+			log.G(ctx).WithError(err).Errorf("reconcile batch failed")
+		}
 		// Simple update, no scaling - update all tasks.
 		r.updater.Update(ctx, r.cluster, service, slotsSlice)
 	}

vendor/src/github.com/docker/swarmkit/manager/scheduler/filter.go

@@ -1,6 +1,8 @@
 package scheduler
 
 import (
+	"fmt"
+
 	"github.com/docker/swarmkit/api"
 	"github.com/docker/swarmkit/manager/constraint"
 )
@@ -18,6 +20,9 @@ type Filter interface {
 	// into the given node. This function should not be called if SetTask
 	// returned false.
 	Check(*NodeInfo) bool
+
+	// Explain what a failure of this filter means
+	Explain(nodes int) string
 }
 
 // ReadyFilter checks that the node is ready to schedule tasks.
@@ -35,6 +40,14 @@ func (f *ReadyFilter) Check(n *NodeInfo) bool {
 		n.Spec.Availability == api.NodeAvailabilityActive
 }
 
+// Explain returns an explanation of a failure.
+func (f *ReadyFilter) Explain(nodes int) string {
+	if nodes == 1 {
+		return "1 node not available for new tasks"
+	}
+	return fmt.Sprintf("%d nodes not available for new tasks", nodes)
+}
+
 // ResourceFilter checks that the node has enough resources available to run
 // the task.
 type ResourceFilter struct {
@@ -67,6 +80,14 @@ func (f *ResourceFilter) Check(n *NodeInfo) bool {
 	return true
 }
 
+// Explain returns an explanation of a failure.
+func (f *ResourceFilter) Explain(nodes int) string {
+	if nodes == 1 {
+		return "insufficient resources on 1 node"
+	}
+	return fmt.Sprintf("insufficient resources on %d nodes", nodes)
+}
+
 // PluginFilter checks that the node has a specific volume plugin installed
 type PluginFilter struct {
 	t *api.Task
@@ -133,6 +154,14 @@ func (f *PluginFilter) pluginExistsOnNode(pluginType string, pluginName string,
 	return false
 }
 
+// Explain returns an explanation of a failure.
+func (f *PluginFilter) Explain(nodes int) string {
+	if nodes == 1 {
+		return "missing plugin on 1 node"
+	}
+	return fmt.Sprintf("missing plugin on %d nodes", nodes)
+}
+
 // ConstraintFilter selects only nodes that match certain labels.
 type ConstraintFilter struct {
 	constraints []constraint.Constraint
@@ -159,3 +188,11 @@ func (f *ConstraintFilter) SetTask(t *api.Task) bool {
 func (f *ConstraintFilter) Check(n *NodeInfo) bool {
 	return constraint.NodeMatches(f.constraints, n.Node)
 }
+
+// Explain returns an explanation of a failure.
+func (f *ConstraintFilter) Explain(nodes int) string {
+	if nodes == 1 {
+		return "scheduling constraints not satisfied on 1 node"
+	}
+	return fmt.Sprintf("scheduling constraints not satisfied on %d nodes", nodes)
+}

vendor/src/github.com/docker/swarmkit/manager/scheduler/pipeline.go

@@ -1,6 +1,10 @@
 package scheduler
 
-import "github.com/docker/swarmkit/api"
+import (
+	"sort"
+
+	"github.com/docker/swarmkit/api"
+)
 
 var (
 	defaultFilters = []Filter{
@@ -20,10 +24,21 @@ var (
 type checklistEntry struct {
 	f       Filter
 	enabled bool
+
+	// failureCount counts the number of nodes that this filter failed
+	// against.
+	failureCount int
 }
 
+type checklistByFailures []checklistEntry
+
+func (c checklistByFailures) Len() int           { return len(c) }
+func (c checklistByFailures) Swap(i, j int)      { c[i], c[j] = c[j], c[i] }
+func (c checklistByFailures) Less(i, j int) bool { return c[i].failureCount < c[j].failureCount }
+
 // Pipeline runs a set of filters against nodes.
 type Pipeline struct {
+	// checklist is a slice of filters to run
 	checklist []checklistEntry
 }
 
@@ -41,12 +56,16 @@ func NewPipeline() *Pipeline {
 // Process a node through the filter pipeline.
 // Returns true if all filters pass, false otherwise.
 func (p *Pipeline) Process(n *NodeInfo) bool {
-	for _, entry := range p.checklist {
+	for i, entry := range p.checklist {
 		if entry.enabled && !entry.f.Check(n) {
 			// Immediately stop on first failure.
+			p.checklist[i].failureCount++
 			return false
 		}
 	}
+	for i := range p.checklist {
+		p.checklist[i].failureCount = 0
+	}
 	return true
 }
 
@@ -55,5 +74,28 @@ func (p *Pipeline) Process(n *NodeInfo) bool {
 func (p *Pipeline) SetTask(t *api.Task) {
 	for i := range p.checklist {
 		p.checklist[i].enabled = p.checklist[i].f.SetTask(t)
+		p.checklist[i].failureCount = 0
 	}
 }
+
+// Explain returns a string explaining why a task could not be scheduled.
+func (p *Pipeline) Explain() string {
+	var explanation string
+
+	// Sort from most failures to least
+
+	sortedByFailures := make([]checklistEntry, len(p.checklist))
+	copy(sortedByFailures, p.checklist)
+	sort.Sort(sort.Reverse(checklistByFailures(sortedByFailures)))
+
+	for _, entry := range sortedByFailures {
+		if entry.failureCount > 0 {
+			if len(explanation) > 0 {
+				explanation += "; "
+			}
+			explanation += entry.f.Explain(entry.failureCount)
+		}
+	}
+
+	return explanation
+}

vendor/src/github.com/docker/swarmkit/manager/scheduler/scheduler.go

@@ -298,7 +298,9 @@ func (s *Scheduler) processPreassignedTasks(ctx context.Context) {
 	successful, failed := s.applySchedulingDecisions(ctx, schedulingDecisions)
 
 	for _, decision := range successful {
-		delete(s.preassignedTasks, decision.old.ID)
+		if decision.new.Status.State == api.TaskStateAssigned {
+			delete(s.preassignedTasks, decision.old.ID)
+		}
 	}
 	for _, decision := range failed {
 		s.allTasks[decision.old.ID] = decision.old
@@ -381,11 +383,35 @@ func (s *Scheduler) applySchedulingDecisions(ctx context.Context, schedulingDeci
 
 					t := store.GetTask(tx, taskID)
 					if t == nil {
-						// Task no longer exists. Do nothing.
-						failed = append(failed, decision)
+						// Task no longer exists
+						nodeInfo, err := s.nodeSet.nodeInfo(decision.new.NodeID)
+						if err == nil && nodeInfo.removeTask(decision.new) {
+							s.nodeSet.updateNode(nodeInfo)
+						}
+						delete(s.allTasks, decision.old.ID)
+
 						continue
 					}
 
+					if t.Status.State == decision.new.Status.State && t.Status.Message == decision.new.Status.Message {
+						// No changes, ignore
+						continue
+					}
+
+					if t.Status.State >= api.TaskStateAssigned {
+						nodeInfo, err := s.nodeSet.nodeInfo(decision.new.NodeID)
+						if err != nil {
+							failed = append(failed, decision)
+							continue
+						}
+						node := store.GetNode(tx, decision.new.NodeID)
+						if node == nil || node.Meta.Version != nodeInfo.Meta.Version {
+							// node is out of date
+							failed = append(failed, decision)
+							continue
+						}
+					}
+
 					if err := store.UpdateTask(tx, decision.new); err != nil {
 						log.G(ctx).Debugf("scheduler failed to update task %s; will retry", taskID)
 						failed = append(failed, decision)
@@ -418,12 +444,16 @@ func (s *Scheduler) taskFitNode(ctx context.Context, t *api.Task, nodeID string)
 		// node does not exist in set (it may have been deleted)
 		return nil
 	}
+	newT := *t
 	s.pipeline.SetTask(t)
 	if !s.pipeline.Process(&nodeInfo) {
 		// this node cannot accommodate this task
-		return nil
+		newT.Status.Timestamp = ptypes.MustTimestampProto(time.Now())
+		newT.Status.Message = s.pipeline.Explain()
+		s.allTasks[t.ID] = &newT
+
+		return &newT
 	}
-	newT := *t
 	newT.Status = api.TaskStatus{
 		State:     api.TaskStateAssigned,
 		Timestamp: ptypes.MustTimestampProto(time.Now()),
@@ -468,10 +498,7 @@ func (s *Scheduler) scheduleTaskGroup(ctx context.Context, taskGroup map[string]
 
 	nodes := s.nodeSet.findBestNodes(len(taskGroup), s.pipeline.Process, nodeLess)
 	if len(nodes) == 0 {
-		for _, t := range taskGroup {
-			log.G(ctx).WithField("task.id", t.ID).Debug("no suitable node available for task")
-			s.enqueue(t)
-		}
+		s.noSuitableNode(ctx, taskGroup, schedulingDecisions)
 		return
 	}
 
@@ -518,16 +545,32 @@ func (s *Scheduler) scheduleTaskGroup(ctx context.Context, taskGroup map[string]
 			nodeIter++
 			if nodeIter-origNodeIter == len(nodes) {
 				// None of the nodes meet the constraints anymore.
-				for _, t := range taskGroup {
-					log.G(ctx).WithField("task.id", t.ID).Debug("no suitable node available for task")
-					s.enqueue(t)
-				}
+				s.noSuitableNode(ctx, taskGroup, schedulingDecisions)
 				return
 			}
 		}
 	}
 }
 
+func (s *Scheduler) noSuitableNode(ctx context.Context, taskGroup map[string]*api.Task, schedulingDecisions map[string]schedulingDecision) {
+	explanation := s.pipeline.Explain()
+	for _, t := range taskGroup {
+		log.G(ctx).WithField("task.id", t.ID).Debug("no suitable node available for task")
+
+		newT := *t
+		newT.Status.Timestamp = ptypes.MustTimestampProto(time.Now())
+		if explanation != "" {
+			newT.Status.Message = "no suitable node (" + explanation + ")"
+		} else {
+			newT.Status.Message = "no suitable node"
+		}
+		s.allTasks[t.ID] = &newT
+		schedulingDecisions[t.ID] = schedulingDecision{old: t, new: &newT}
+
+		s.enqueue(&newT)
+	}
+}
+
 func (s *Scheduler) buildNodeSet(tx store.ReadTx, tasksByNode map[string]map[string]*api.Task) error {
 	nodes, err := store.FindNodes(tx, store.All)
 	if err != nil {

vendor/src/github.com/docker/swarmkit/manager/state/raft/membership/cluster.go

@@ -53,7 +53,6 @@ type Cluster struct {
 type Member struct {
 	*api.RaftMember
 
-	api.RaftClient
 	Conn         *grpc.ClientConn
 	tick         int
 	active       bool
@@ -211,8 +210,7 @@ func (c *Cluster) clearMember(id uint64) error {
 	return nil
 }
 
-// ReplaceMemberConnection replaces the member's GRPC connection and GRPC
-// client.
+// ReplaceMemberConnection replaces the member's GRPC connection.
 func (c *Cluster) ReplaceMemberConnection(id uint64, oldConn *Member, newConn *Member, newAddr string, force bool) error {
 	c.mu.Lock()
 	defer c.mu.Unlock()
@@ -236,7 +234,6 @@ func (c *Cluster) ReplaceMemberConnection(id uint64, oldConn *Member, newConn *M
 	newMember.RaftMember = oldMember.RaftMember.Copy()
 	newMember.RaftMember.Addr = newAddr
 	newMember.Conn = newConn.Conn
-	newMember.RaftClient = newConn.RaftClient
 	c.members[id] = &newMember
 
 	return nil

vendor/src/github.com/docker/swarmkit/manager/state/raft/raft.go

@@ -81,10 +81,6 @@ type Node struct {
 	raftNode raft.Node
 	cluster  *membership.Cluster
 
-	Server *grpc.Server
-	Ctx    context.Context
-	cancel func()
-
 	raftStore           *raft.MemoryStorage
 	memoryStore         *store.MemoryStore
 	Config              *raft.Config
@@ -106,7 +102,6 @@ type Node struct {
 	snapshotIndex uint64
 
 	ticker clock.Ticker
-	stopCh chan struct{}
 	doneCh chan struct{}
 	// removeRaftCh notifies about node deletion from raft cluster
 	removeRaftCh        chan struct{}
@@ -121,6 +116,12 @@ type Node struct {
 
 	snapshotInProgress chan uint64
 	asyncTasks         sync.WaitGroup
+
+	// stopped chan is used for notifying grpc handlers that raft node going
+	// to stop.
+	stopped chan struct{}
+
+	lastSendToMember map[uint64]chan struct{}
 }
 
 // NodeOptions provides node-level options.
@@ -155,8 +156,8 @@ func init() {
 	rand.Seed(time.Now().UnixNano())
 }
 
-// NewNode generates a new Raft node.
-func NewNode(ctx context.Context, opts NodeOptions) *Node {
+// NewNode generates a new Raft node
+func NewNode(opts NodeOptions) *Node {
 	cfg := opts.Config
 	if cfg == nil {
 		cfg = DefaultNodeConfig()
@@ -170,11 +171,7 @@ func NewNode(ctx context.Context, opts NodeOptions) *Node {
 
 	raftStore := raft.NewMemoryStorage()
 
-	ctx, cancel := context.WithCancel(ctx)
-
 	n := &Node{
-		Ctx:       ctx,
-		cancel:    cancel,
 		cluster:   membership.NewCluster(2 * cfg.ElectionTick),
 		raftStore: raftStore,
 		opts:      opts,
@@ -186,10 +183,11 @@ func NewNode(ctx context.Context, opts NodeOptions) *Node {
 			MaxInflightMsgs: cfg.MaxInflightMsgs,
 			Logger:          cfg.Logger,
 		},
-		stopCh:              make(chan struct{}),
 		doneCh:              make(chan struct{}),
 		removeRaftCh:        make(chan struct{}),
+		stopped:             make(chan struct{}),
 		leadershipBroadcast: watch.NewQueue(),
+		lastSendToMember:    make(map[uint64]chan struct{}),
 	}
 	n.memoryStore = store.NewMemoryStore(n)
 
@@ -214,15 +212,32 @@ func NewNode(ctx context.Context, opts NodeOptions) *Node {
 	return n
 }
 
+// WithContext returns context which is cancelled when parent context cancelled
+// or node is stopped.
+func (n *Node) WithContext(ctx context.Context) (context.Context, context.CancelFunc) {
+	ctx, cancel := context.WithCancel(ctx)
+
+	go func() {
+		select {
+		case <-ctx.Done():
+		case <-n.stopped:
+			cancel()
+		}
+	}()
+	return ctx, cancel
+}
+
 // JoinAndStart joins and starts the raft server
-func (n *Node) JoinAndStart() (err error) {
+func (n *Node) JoinAndStart(ctx context.Context) (err error) {
+	ctx, cancel := n.WithContext(ctx)
 	defer func() {
+		cancel()
 		if err != nil {
 			n.done()
 		}
 	}()
 
-	loadAndStartErr := n.loadAndStart(n.Ctx, n.opts.ForceNewCluster)
+	loadAndStartErr := n.loadAndStart(ctx, n.opts.ForceNewCluster)
 	if loadAndStartErr != nil && loadAndStartErr != errNoWAL {
 		return loadAndStartErr
 	}
@@ -248,9 +263,9 @@ func (n *Node) JoinAndStart() (err error) {
 				_ = c.Conn.Close()
 			}()
 
-			ctx, cancel := context.WithTimeout(n.Ctx, 10*time.Second)
-			defer cancel()
-			resp, err := client.Join(ctx, &api.JoinRequest{
+			joinCtx, joinCancel := context.WithTimeout(ctx, 10*time.Second)
+			defer joinCancel()
+			resp, err := client.Join(joinCtx, &api.JoinRequest{
 				Addr: n.opts.Addr,
 			})
 			if err != nil {
@@ -267,7 +282,7 @@ func (n *Node) JoinAndStart() (err error) {
 
 			if err := n.registerNodes(resp.Members); err != nil {
 				if walErr := n.wal.Close(); err != nil {
-					n.Config.Logger.Errorf("raft: error closing WAL: %v", walErr)
+					log.G(ctx).WithError(walErr).Error("raft: error closing WAL")
 				}
 				return err
 			}
@@ -286,7 +301,7 @@ func (n *Node) JoinAndStart() (err error) {
 	}
 
 	if n.opts.JoinAddr != "" {
-		n.Config.Logger.Warning("ignoring request to join cluster, because raft state already exists")
+		log.G(ctx).Warning("ignoring request to join cluster, because raft state already exists")
 	}
 	n.campaignWhenAble = true
 	n.raftNode = raft.RestartNode(n.Config)
@@ -340,7 +355,24 @@ func (n *Node) done() {
 // Before running the main loop, it first starts the raft node based on saved
 // cluster state. If no saved state exists, it starts a single-node cluster.
 func (n *Node) Run(ctx context.Context) error {
-	defer n.done()
+	ctx = log.WithLogger(ctx, logrus.WithField("raft_id", fmt.Sprintf("%x", n.Config.ID)))
+	ctx, cancel := context.WithCancel(ctx)
+
+	// nodeRemoved indicates that node was stopped due its removal.
+	nodeRemoved := false
+
+	defer func() {
+		cancel()
+		n.stop(ctx)
+		if nodeRemoved {
+			// Move WAL and snapshot out of the way, since
+			// they are no longer usable.
+			if err := n.moveWALAndSnap(); err != nil {
+				log.G(ctx).WithError(err).Error("failed to move wal after node removal")
+			}
+		}
+		n.done()
+	}()
 
 	wasLeader := false
 
@@ -360,13 +392,13 @@ func (n *Node) Run(ctx context.Context) error {
 
 			// Save entries to storage
 			if err := n.saveToStorage(&raftConfig, rd.HardState, rd.Entries, rd.Snapshot); err != nil {
-				n.Config.Logger.Error(err)
+				log.G(ctx).WithError(err).Error("failed to save entries to storage")
 			}
 
 			if len(rd.Messages) != 0 {
 				// Send raft messages to peers
-				if err := n.send(rd.Messages); err != nil {
-					n.Config.Logger.Error(err)
+				if err := n.send(ctx, rd.Messages); err != nil {
+					log.G(ctx).WithError(err).Error("failed to send message to members")
 				}
 			}
 
@@ -376,7 +408,7 @@ func (n *Node) Run(ctx context.Context) error {
 			if !raft.IsEmptySnap(rd.Snapshot) {
 				// Load the snapshot data into the store
 				if err := n.restoreFromSnapshot(rd.Snapshot.Data, false); err != nil {
-					n.Config.Logger.Error(err)
+					log.G(ctx).WithError(err).Error("failed to restore from snapshot")
 				}
 				n.appliedIndex = rd.Snapshot.Metadata.Index
 				n.snapshotIndex = rd.Snapshot.Metadata.Index
@@ -420,8 +452,8 @@ func (n *Node) Run(ctx context.Context) error {
 
 			// Process committed entries
 			for _, entry := range rd.CommittedEntries {
-				if err := n.processCommitted(entry); err != nil {
-					n.Config.Logger.Error(err)
+				if err := n.processCommitted(ctx, entry); err != nil {
+					log.G(ctx).WithError(err).Error("failed to process committed entries")
 				}
 			}
 
@@ -429,7 +461,7 @@ func (n *Node) Run(ctx context.Context) error {
 			if n.snapshotInProgress == nil &&
 				raftConfig.SnapshotInterval > 0 &&
 				n.appliedIndex-n.snapshotIndex >= raftConfig.SnapshotInterval {
-				n.doSnapshot(&raftConfig)
+				n.doSnapshot(ctx, raftConfig)
 			}
 
 			if wasLeader && atomic.LoadUint32(&n.signalledLeadership) != 1 {
@@ -453,7 +485,7 @@ func (n *Node) Run(ctx context.Context) error {
 					n.campaignWhenAble = false
 				}
 				if len(members) == 1 && members[n.Config.ID] != nil {
-					if err := n.raftNode.Campaign(n.Ctx); err != nil {
+					if err := n.raftNode.Campaign(ctx); err != nil {
 						panic("raft: cannot campaign to be the leader on node restore")
 					}
 				}
@@ -465,49 +497,35 @@ func (n *Node) Run(ctx context.Context) error {
 			}
 			n.snapshotInProgress = nil
 		case <-n.removeRaftCh:
+			nodeRemoved = true
 			// If the node was removed from other members,
 			// send back an error to the caller to start
 			// the shutdown process.
-			n.stop()
-
-			// Move WAL and snapshot out of the way, since
-			// they are no longer usable.
-			if err := n.moveWALAndSnap(); err != nil {
-				n.Config.Logger.Error(err)
-			}
-
 			return ErrMemberRemoved
-		case <-n.stopCh:
-			n.stop()
+		case <-ctx.Done():
 			return nil
 		}
 	}
 }
 
-// Shutdown stops the raft node processing loop.
-// Calling Shutdown on an already stopped node
-// will result in a panic.
-func (n *Node) Shutdown() {
-	select {
-	case <-n.doneCh:
-	default:
-		close(n.stopCh)
-		<-n.doneCh
-	}
+// Done returns channel which is closed when raft node is fully stopped.
+func (n *Node) Done() <-chan struct{} {
+	return n.doneCh
 }
 
-func (n *Node) stop() {
+func (n *Node) stop(ctx context.Context) {
 	n.stopMu.Lock()
 	defer n.stopMu.Unlock()
 
-	n.cancel()
+	close(n.stopped)
+
 	n.waitProp.Wait()
 	n.asyncTasks.Wait()
 
 	n.raftNode.Stop()
 	n.ticker.Stop()
 	if err := n.wal.Close(); err != nil {
-		n.Config.Logger.Errorf("raft: error closing WAL: %v", err)
+		log.G(ctx).WithError(err).Error("raft: failed to close WAL")
 	}
 	atomic.StoreUint32(&n.isMember, 0)
 	// TODO(stevvooe): Handle ctx.Done()
@@ -581,11 +599,13 @@ func (n *Node) Join(ctx context.Context, req *api.JoinRequest) (*api.JoinRespons
 	fields := logrus.Fields{
 		"node.id": nodeInfo.NodeID,
 		"method":  "(*Node).Join",
+		"raft_id": fmt.Sprintf("%x", n.Config.ID),
 	}
 	if nodeInfo.ForwardedBy != nil {
 		fields["forwarder.id"] = nodeInfo.ForwardedBy.NodeID
 	}
 	log := log.G(ctx).WithFields(fields)
+	log.Debug("")
 
 	// can't stop the raft node while an async RPC is in progress
 	n.stopMu.RLock()
@@ -639,7 +659,7 @@ func (n *Node) Join(ctx context.Context, req *api.JoinRequest) (*api.JoinRespons
 
 	err = n.addMember(ctx, remoteAddr, raftID, nodeInfo.NodeID)
 	if err != nil {
-		log.WithError(err).Errorf("failed to add member")
+		log.WithError(err).Errorf("failed to add member %x", raftID)
 		return nil, err
 	}
 
@@ -738,11 +758,12 @@ func (n *Node) Leave(ctx context.Context, req *api.LeaveRequest) (*api.LeaveResp
 	fields := logrus.Fields{
 		"node.id": nodeInfo.NodeID,
 		"method":  "(*Node).Leave",
+		"raft_id": fmt.Sprintf("%x", n.Config.ID),
 	}
 	if nodeInfo.ForwardedBy != nil {
 		fields["forwarder.id"] = nodeInfo.ForwardedBy.NodeID
 	}
-	log.G(ctx).WithFields(fields).Debugf("")
+	log.G(ctx).WithFields(fields).Debug("")
 
 	// can't stop the raft node while an async RPC is in progress
 	n.stopMu.RLock()
@@ -783,8 +804,9 @@ func (n *Node) RemoveMember(ctx context.Context, id uint64) error {
 			NodeID:  id,
 			Context: []byte(""),
 		}
-
+		ctx, cancel := n.WithContext(ctx)
 		err := n.configure(ctx, cc)
+		cancel()
 		return err
 	}
 
@@ -813,11 +835,14 @@ func (n *Node) ProcessRaftMessage(ctx context.Context, msg *api.ProcessRaftMessa
 
 	n.cluster.ReportActive(msg.Message.From, sourceHost)
 
+	ctx, cancel := n.WithContext(ctx)
+	defer cancel()
+
 	// Reject vote requests from unreachable peers
 	if msg.Message.Type == raftpb.MsgVote {
 		member := n.cluster.GetMember(msg.Message.From)
 		if member == nil || member.Conn == nil {
-			n.Config.Logger.Errorf("received vote request from unknown member %x", msg.Message.From)
+			log.G(ctx).Errorf("received vote request from unknown member %x", msg.Message.From)
 			return nil, ErrMemberUnknown
 		}
 
@@ -825,7 +850,7 @@ func (n *Node) ProcessRaftMessage(ctx context.Context, msg *api.ProcessRaftMessa
 		defer cancel()
 
 		if err := member.HealthCheck(healthCtx); err != nil {
-			n.Config.Logger.Warningf("member %x which sent vote request failed health check: %v", msg.Message.From, err)
+			log.G(ctx).WithError(err).Warningf("member %x which sent vote request failed health check", msg.Message.From)
 			return nil, errors.Wrap(err, "member unreachable")
 		}
 	}
@@ -846,7 +871,7 @@ func (n *Node) ProcessRaftMessage(ctx context.Context, msg *api.ProcessRaftMessa
 		return nil, ErrNoRaftMember
 	}
 
-	if err := n.raftNode.Step(n.Ctx, *msg.Message); err != nil {
+	if err := n.raftNode.Step(ctx, *msg.Message); err != nil {
 		return nil, err
 	}
 
@@ -867,11 +892,12 @@ func (n *Node) ResolveAddress(ctx context.Context, msg *api.ResolveAddressReques
 	fields := logrus.Fields{
 		"node.id": nodeInfo.NodeID,
 		"method":  "(*Node).ResolveAddress",
+		"raft_id": fmt.Sprintf("%x", n.Config.ID),
 	}
 	if nodeInfo.ForwardedBy != nil {
 		fields["forwarder.id"] = nodeInfo.ForwardedBy.NodeID
 	}
-	log.G(ctx).WithFields(fields).Debugf("")
+	log.G(ctx).WithFields(fields).Debug("")
 
 	member := n.cluster.GetMember(msg.RaftID)
 	if member == nil {
@@ -948,7 +974,6 @@ func (n *Node) registerNode(node *api.RaftMember) error {
 		// address.
 		if existingMember.Addr != node.Addr {
 			member.RaftMember = node
-			member.RaftClient = existingMember.RaftClient
 			member.Conn = existingMember.Conn
 			n.cluster.AddMember(member)
 		}
@@ -993,6 +1018,8 @@ func (n *Node) registerNodes(nodes []*api.RaftMember) error {
 // ProposeValue calls Propose on the raft and waits
 // on the commit log action before returning a result
 func (n *Node) ProposeValue(ctx context.Context, storeAction []*api.StoreAction, cb func()) error {
+	ctx, cancel := n.WithContext(ctx)
+	defer cancel()
 	_, err := n.processInternalRaftRequest(ctx, &api.InternalRaftRequest{Action: storeAction}, cb)
 	if err != nil {
 		return err
@@ -1083,7 +1110,7 @@ func (n *Node) IsMember() bool {
 // could be submitted and processed.
 func (n *Node) canSubmitProposal() bool {
 	select {
-	case <-n.Ctx.Done():
+	case <-n.stopped:
 		return false
 	default:
 		return true
@@ -1115,7 +1142,7 @@ func (n *Node) saveToStorage(raftConfig *api.RaftConfig, hardState raftpb.HardSt
 }
 
 // Sends a series of messages to members in the raft
-func (n *Node) send(messages []raftpb.Message) error {
+func (n *Node) send(ctx context.Context, messages []raftpb.Message) error {
 	members := n.cluster.Members()
 
 	n.stopMu.RLock()
@@ -1124,7 +1151,7 @@ func (n *Node) send(messages []raftpb.Message) error {
 	for _, m := range messages {
 		// Process locally
 		if m.To == n.Config.ID {
-			if err := n.raftNode.Step(n.Ctx, m); err != nil {
+			if err := n.raftNode.Step(ctx, m); err != nil {
 				return err
 			}
 			continue
@@ -1138,24 +1165,37 @@ func (n *Node) send(messages []raftpb.Message) error {
 			continue
 		}
 
+		ch := make(chan struct{})
+
 		n.asyncTasks.Add(1)
-		go n.sendToMember(members, m)
+		go n.sendToMember(ctx, members, m, n.lastSendToMember[m.To], ch)
+
+		n.lastSendToMember[m.To] = ch
 	}
 
 	return nil
 }
 
-func (n *Node) sendToMember(members map[uint64]*membership.Member, m raftpb.Message) {
+func (n *Node) sendToMember(ctx context.Context, members map[uint64]*membership.Member, m raftpb.Message, lastSend <-chan struct{}, thisSend chan<- struct{}) {
 	defer n.asyncTasks.Done()
+	defer close(thisSend)
+
+	ctx, cancel := context.WithTimeout(ctx, n.opts.SendTimeout)
+	defer cancel()
+
+	if lastSend != nil {
+		select {
+		case <-lastSend:
+		case <-ctx.Done():
+			return
+		}
+	}
 
 	if n.cluster.IsIDRemoved(m.To) {
 		// Should not send to removed members
 		return
 	}
 
-	ctx, cancel := context.WithTimeout(n.Ctx, n.opts.SendTimeout)
-	defer cancel()
-
 	var (
 		conn *membership.Member
 	)
@@ -1165,7 +1205,7 @@ func (n *Node) sendToMember(members map[uint64]*membership.Member, m raftpb.Mess
 		// If we are being asked to send to a member that's not in
 		// our member list, that could indicate that the current leader
 		// was added while we were offline. Try to resolve its address.
-		n.Config.Logger.Warningf("sending message to an unrecognized member ID %x", m.To)
+		log.G(ctx).Warningf("sending message to an unrecognized member ID %x", m.To)
 
 		// Choose a random member
 		var (
@@ -1179,18 +1219,18 @@ func (n *Node) sendToMember(members map[uint64]*membership.Member, m raftpb.Mess
 		}
 
 		if queryMember == nil || queryMember.RaftID == n.Config.ID {
-			n.Config.Logger.Error("could not find cluster member to query for leader address")
+			log.G(ctx).Error("could not find cluster member to query for leader address")
 			return
 		}
 
-		resp, err := queryMember.ResolveAddress(ctx, &api.ResolveAddressRequest{RaftID: m.To})
+		resp, err := api.NewRaftClient(queryMember.Conn).ResolveAddress(ctx, &api.ResolveAddressRequest{RaftID: m.To})
 		if err != nil {
-			n.Config.Logger.Errorf("could not resolve address of member ID %x: %v", m.To, err)
+			log.G(ctx).WithError(err).Errorf("could not resolve address of member ID %x", m.To)
 			return
 		}
 		conn, err = n.ConnectToMember(resp.Addr, n.opts.SendTimeout)
 		if err != nil {
-			n.Config.Logger.Errorf("could connect to member ID %x at %s: %v", m.To, resp.Addr, err)
+			log.G(ctx).WithError(err).Errorf("could connect to member ID %x at %s", m.To, resp.Addr)
 			return
 		}
 		// The temporary connection is only used for this message.
@@ -1199,7 +1239,7 @@ func (n *Node) sendToMember(members map[uint64]*membership.Member, m raftpb.Mess
 		defer conn.Conn.Close()
 	}
 
-	_, err := conn.ProcessRaftMessage(ctx, &api.ProcessRaftMessageRequest{Message: &m})
+	_, err := api.NewRaftClient(conn.Conn).ProcessRaftMessage(ctx, &api.ProcessRaftMessageRequest{Message: &m})
 	if err != nil {
 		if grpc.ErrorDesc(err) == ErrMemberRemoved.Error() {
 			n.removeRaftFunc()
@@ -1219,9 +1259,9 @@ func (n *Node) sendToMember(members map[uint64]*membership.Member, m raftpb.Mess
 			officialHost, officialPort, _ := net.SplitHostPort(conn.Addr)
 			if officialHost != lastSeenHost {
 				reconnectAddr := net.JoinHostPort(lastSeenHost, officialPort)
-				n.Config.Logger.Warningf("detected address change for %x (%s -> %s)", m.To, conn.Addr, reconnectAddr)
-				if err := n.handleAddressChange(conn, reconnectAddr); err != nil {
-					n.Config.Logger.Error(err)
+				log.G(ctx).Warningf("detected address change for %x (%s -> %s)", m.To, conn.Addr, reconnectAddr)
+				if err := n.handleAddressChange(ctx, conn, reconnectAddr); err != nil {
+					log.G(ctx).WithError(err).Error("failed to hande address change")
 				}
 				return
 			}
@@ -1230,12 +1270,12 @@ func (n *Node) sendToMember(members map[uint64]*membership.Member, m raftpb.Mess
 		// Bounce the connection
 		newConn, err := n.ConnectToMember(conn.Addr, 0)
 		if err != nil {
-			n.Config.Logger.Errorf("could connect to member ID %x at %s: %v", m.To, conn.Addr, err)
+			log.G(ctx).WithError(err).Errorf("could connect to member ID %x at %s", m.To, conn.Addr)
 			return
 		}
 		err = n.cluster.ReplaceMemberConnection(m.To, conn, newConn, conn.Addr, false)
 		if err != nil {
-			n.Config.Logger.Errorf("failed to replace connection to raft member: %v", err)
+			log.G(ctx).WithError(err).Error("failed to replace connection to raft member")
 			newConn.Conn.Close()
 		}
 	} else if m.Type == raftpb.MsgSnap {
@@ -1243,13 +1283,13 @@ func (n *Node) sendToMember(members map[uint64]*membership.Member, m raftpb.Mess
 	}
 }
 
-func (n *Node) handleAddressChange(member *membership.Member, reconnectAddr string) error {
+func (n *Node) handleAddressChange(ctx context.Context, member *membership.Member, reconnectAddr string) error {
 	newConn, err := n.ConnectToMember(reconnectAddr, 0)
 	if err != nil {
 		return errors.Wrapf(err, "could connect to member ID %x at observed address %s", member.RaftID, reconnectAddr)
 	}
 
-	healthCtx, cancelHealth := context.WithTimeout(n.Ctx, time.Duration(n.Config.ElectionTick)*n.opts.TickInterval)
+	healthCtx, cancelHealth := context.WithTimeout(ctx, time.Duration(n.Config.ElectionTick)*n.opts.TickInterval)
 	defer cancelHealth()
 
 	if err := newConn.HealthCheck(healthCtx); err != nil {
@@ -1262,7 +1302,7 @@ func (n *Node) handleAddressChange(member *membership.Member, reconnectAddr stri
 	}
 
 	// If we're the leader, write the address change to raft
-	updateCtx, cancelUpdate := context.WithTimeout(n.Ctx, time.Duration(n.Config.ElectionTick)*n.opts.TickInterval)
+	updateCtx, cancelUpdate := context.WithTimeout(ctx, time.Duration(n.Config.ElectionTick)*n.opts.TickInterval)
 	defer cancelUpdate()
 	if err := n.updateMember(updateCtx, reconnectAddr, member.RaftID, member.NodeID); err != nil {
 		return errors.Wrap(err, "failed to update member address in raft")
@@ -1295,7 +1335,7 @@ func (n *Node) processInternalRaftRequest(ctx context.Context, r *api.InternalRa
 
 	// This must be derived from the context which is cancelled by stop()
 	// to avoid a deadlock on shutdown.
-	waitCtx, cancel := context.WithCancel(n.Ctx)
+	waitCtx, cancel := context.WithCancel(ctx)
 
 	ch := n.wait.register(r.ID, cb, cancel)
 
@@ -1361,29 +1401,27 @@ func (n *Node) configure(ctx context.Context, cc raftpb.ConfChange) error {
 	case <-ctx.Done():
 		n.wait.cancel(cc.ID)
 		return ctx.Err()
-	case <-n.Ctx.Done():
-		return ErrStopped
 	}
 }
 
-func (n *Node) processCommitted(entry raftpb.Entry) error {
+func (n *Node) processCommitted(ctx context.Context, entry raftpb.Entry) error {
 	// Process a normal entry
 	if entry.Type == raftpb.EntryNormal && entry.Data != nil {
-		if err := n.processEntry(entry); err != nil {
+		if err := n.processEntry(ctx, entry); err != nil {
 			return err
 		}
 	}
 
 	// Process a configuration change (add/remove node)
 	if entry.Type == raftpb.EntryConfChange {
-		n.processConfChange(entry)
+		n.processConfChange(ctx, entry)
 	}
 
 	n.appliedIndex = entry.Index
 	return nil
 }
 
-func (n *Node) processEntry(entry raftpb.Entry) error {
+func (n *Node) processEntry(ctx context.Context, entry raftpb.Entry) error {
 	r := &api.InternalRaftRequest{}
 	err := proto.Unmarshal(entry.Data, r)
 	if err != nil {
@@ -1409,13 +1447,13 @@ func (n *Node) processEntry(entry raftpb.Entry) error {
 
 		err := n.memoryStore.ApplyStoreActions(r.Action)
 		if err != nil {
-			log.G(context.Background()).Errorf("error applying actions from raft: %v", err)
+			log.G(ctx).WithError(err).Error("failed to apply actions from raft")
 		}
 	}
 	return nil
 }
 
-func (n *Node) processConfChange(entry raftpb.Entry) {
+func (n *Node) processConfChange(ctx context.Context, entry raftpb.Entry) {
 	var (
 		err error
 		cc  raftpb.ConfChange
@@ -1433,9 +1471,9 @@ func (n *Node) processConfChange(entry raftpb.Entry) {
 	case raftpb.ConfChangeAddNode:
 		err = n.applyAddNode(cc)
 	case raftpb.ConfChangeUpdateNode:
-		err = n.applyUpdateNode(cc)
+		err = n.applyUpdateNode(ctx, cc)
 	case raftpb.ConfChangeRemoveNode:
-		err = n.applyRemoveNode(cc)
+		err = n.applyRemoveNode(ctx, cc)
 	}
 
 	if err != nil {
@@ -1469,7 +1507,7 @@ func (n *Node) applyAddNode(cc raftpb.ConfChange) error {
 
 // applyUpdateNode is called when we receive a ConfChange from a member in the
 // raft cluster which update the address of an existing node.
-func (n *Node) applyUpdateNode(cc raftpb.ConfChange) error {
+func (n *Node) applyUpdateNode(ctx context.Context, cc raftpb.ConfChange) error {
 	newMember := &api.RaftMember{}
 	err := proto.Unmarshal(cc.Context, newMember)
 	if err != nil {
@@ -1483,7 +1521,7 @@ func (n *Node) applyUpdateNode(cc raftpb.ConfChange) error {
 	}
 	if oldMember.NodeID != newMember.NodeID {
 		// Should never happen; this is a sanity check
-		n.Config.Logger.Errorf("node ID mismatch on node update (old: %x, new: %x)", oldMember.NodeID, newMember.NodeID)
+		log.G(ctx).Errorf("node ID mismatch on node update (old: %x, new: %x)", oldMember.NodeID, newMember.NodeID)
 		return errors.New("node ID mismatch match on node update")
 	}
 
@@ -1507,13 +1545,13 @@ func (n *Node) applyUpdateNode(cc raftpb.ConfChange) error {
 // applyRemoveNode is called when we receive a ConfChange
 // from a member in the raft cluster, this removes a node
 // from the existing raft cluster
-func (n *Node) applyRemoveNode(cc raftpb.ConfChange) (err error) {
+func (n *Node) applyRemoveNode(ctx context.Context, cc raftpb.ConfChange) (err error) {
 	// If the node from where the remove is issued is
 	// a follower and the leader steps down, Campaign
 	// to be the leader.
 
 	if cc.NodeID == n.leader() && !n.isLeader() {
-		if err = n.raftNode.Campaign(n.Ctx); err != nil {
+		if err = n.raftNode.Campaign(ctx); err != nil {
 			return err
 		}
 	}
@@ -1547,8 +1585,7 @@ func (n *Node) ConnectToMember(addr string, timeout time.Duration) (*membership.
 	}
 
 	return &membership.Member{
-		RaftClient: api.NewRaftClient(conn),
-		Conn:       conn,
+		Conn: conn,
 	}, nil
 }
 
@@ -1579,7 +1616,7 @@ func createConfigChangeEnts(ids []uint64, self uint64, term, index uint64) []raf
 		}
 		data, err := cc.Marshal()
 		if err != nil {
-			log.G(context.Background()).Panicf("marshal configuration change should never fail: %v", err)
+			log.L.WithError(err).Panic("marshal configuration change should never fail")
 		}
 		e := raftpb.Entry{
 			Type:  raftpb.EntryConfChange,
@@ -1594,7 +1631,7 @@ func createConfigChangeEnts(ids []uint64, self uint64, term, index uint64) []raf
 		node := &api.RaftMember{RaftID: self}
 		meta, err := node.Marshal()
 		if err != nil {
-			log.G(context.Background()).Panicf("marshal member should never fail: %v", err)
+			log.L.WithError(err).Panic("marshal member should never fail")
 		}
 		cc := &raftpb.ConfChange{
 			Type:    raftpb.ConfChangeAddNode,
@@ -1603,7 +1640,7 @@ func createConfigChangeEnts(ids []uint64, self uint64, term, index uint64) []raf
 		}
 		data, err := cc.Marshal()
 		if err != nil {
-			log.G(context.Background()).Panicf("marshal configuration change should never fail: %v", err)
+			log.L.WithError(err).Panic("marshal configuration change should never fail")
 		}
 		e := raftpb.Entry{
 			Type:  raftpb.EntryConfChange,
@@ -1637,7 +1674,7 @@ func getIDs(snap *raftpb.Snapshot, ents []raftpb.Entry) []uint64 {
 		}
 		var cc raftpb.ConfChange
 		if err := cc.Unmarshal(e.Data); err != nil {
-			log.G(context.Background()).Panicf("unmarshal configuration change should never fail: %v", err)
+			log.L.WithError(err).Panic("unmarshal configuration change should never fail")
 		}
 		switch cc.Type {
 		case raftpb.ConfChangeAddNode:
@@ -1647,7 +1684,7 @@ func getIDs(snap *raftpb.Snapshot, ents []raftpb.Entry) []uint64 {
 		case raftpb.ConfChangeUpdateNode:
 			// do nothing
 		default:
-			log.G(context.Background()).Panic("ConfChange Type should be either ConfChangeAddNode or ConfChangeRemoveNode!")
+			log.L.Panic("ConfChange Type should be either ConfChangeAddNode or ConfChangeRemoveNode!")
 		}
 	}
 	var sids []uint64

vendor/src/github.com/docker/swarmkit/manager/state/raft/storage.go

@@ -485,7 +485,7 @@ func (n *Node) saveSnapshot(snapshot raftpb.Snapshot, keepOldSnapshots uint64) e
 	return nil
 }
 
-func (n *Node) doSnapshot(raftConfig *api.RaftConfig) {
+func (n *Node) doSnapshot(ctx context.Context, raftConfig api.RaftConfig) {
 	snapshot := api.Snapshot{Version: api.Snapshot_V0}
 	for _, member := range n.cluster.Members() {
 		snapshot.Membership.Members = append(snapshot.Membership.Members,
@@ -515,19 +515,19 @@ func (n *Node) doSnapshot(raftConfig *api.RaftConfig) {
 			snapshot.Store = *storeSnapshot
 		})
 		if err != nil {
-			n.Config.Logger.Error(err)
+			log.G(ctx).WithError(err).Error("failed to read snapshot from store")
 			return
 		}
 
 		d, err := snapshot.Marshal()
 		if err != nil {
-			n.Config.Logger.Error(err)
+			log.G(ctx).WithError(err).Error("failed to marshal snapshot")
 			return
 		}
 		snap, err := n.raftStore.CreateSnapshot(appliedIndex, &n.confState, d)
 		if err == nil {
 			if err := n.saveSnapshot(snap, raftConfig.KeepOldSnapshots); err != nil {
-				n.Config.Logger.Error(err)
+				log.G(ctx).WithError(err).Error("failed to save snapshot")
 				return
 			}
 			snapshotIndex = appliedIndex
@@ -535,11 +535,11 @@ func (n *Node) doSnapshot(raftConfig *api.RaftConfig) {
 			if appliedIndex > raftConfig.LogEntriesForSlowFollowers {
 				err := n.raftStore.Compact(appliedIndex - raftConfig.LogEntriesForSlowFollowers)
 				if err != nil && err != raft.ErrCompacted {
-					n.Config.Logger.Error(err)
+					log.G(ctx).WithError(err).Error("failed to compact snapshot")
 				}
 			}
 		} else if err != raft.ErrSnapOutOfDate {
-			n.Config.Logger.Error(err)
+			log.G(ctx).WithError(err).Error("failed to create snapshot")
 		}
 	}(n.appliedIndex, n.snapshotIndex)
 

vendor/src/github.com/docker/swarmkit/agent/node.go → vendor/src/github.com/docker/swarmkit/node/node.go

@@ -1,4 +1,4 @@
-package agent
+package node
 
 import (
 	"crypto/tls"
@@ -14,6 +14,7 @@ import (
 
 	"github.com/Sirupsen/logrus"
 	"github.com/boltdb/bolt"
+	"github.com/docker/swarmkit/agent"
 	"github.com/docker/swarmkit/agent/exec"
 	"github.com/docker/swarmkit/api"
 	"github.com/docker/swarmkit/ca"
@@ -29,8 +30,13 @@ import (
 
 const stateFilename = "state.json"
 
-// NodeConfig provides values for a Node.
-type NodeConfig struct {
+var (
+	errNodeStarted    = errors.New("node: already started")
+	errNodeNotStarted = errors.New("node: not started")
+)
+
+// Config provides values for a Node.
+type Config struct {
 	// Hostname is the name of host for agent instance.
 	Hostname string
 
@@ -80,7 +86,7 @@ type NodeConfig struct {
 // cluster. Node handles workloads and may also run as a manager.
 type Node struct {
 	sync.RWMutex
-	config               *NodeConfig
+	config               *Config
 	remotes              *persistentRemotes
 	role                 string
 	roleCond             *sync.Cond
@@ -96,7 +102,7 @@ type Node struct {
 	certificateRequested chan struct{} // closed when certificate issue request has been sent by node
 	closed               chan struct{}
 	err                  error
-	agent                *Agent
+	agent                *agent.Agent
 	manager              *manager.Manager
 	roleChangeReq        chan api.NodeRole // used to send role updates from the dispatcher api on promotion/demotion
 }
@@ -116,8 +122,8 @@ func (n *Node) RemoteAPIAddr() (string, error) {
 	return addr.String(), nil
 }
 
-// NewNode returns new Node instance.
-func NewNode(c *NodeConfig) (*Node, error) {
+// New returns new Node instance.
+func New(c *Config) (*Node, error) {
 	if err := os.MkdirAll(c.StateDir, 0700); err != nil {
 		return nil, err
 	}
@@ -369,7 +375,7 @@ func (n *Node) runAgent(ctx context.Context, db *bolt.DB, creds credentials.Tran
 		return ctx.Err()
 	}
 
-	agent, err := New(&Config{
+	a, err := agent.New(&agent.Config{
 		Hostname:         n.config.Hostname,
 		Managers:         n.remotes,
 		Executor:         n.config.Executor,
@@ -380,12 +386,12 @@ func (n *Node) runAgent(ctx context.Context, db *bolt.DB, creds credentials.Tran
 	if err != nil {
 		return err
 	}
-	if err := agent.Start(ctx); err != nil {
+	if err := a.Start(ctx); err != nil {
 		return err
 	}
 
 	n.Lock()
-	n.agent = agent
+	n.agent = a
 	n.Unlock()
 
 	defer func() {
@@ -395,13 +401,13 @@ func (n *Node) runAgent(ctx context.Context, db *bolt.DB, creds credentials.Tran
 	}()
 
 	go func() {
-		<-agent.Ready()
+		<-a.Ready()
 		close(ready)
 	}()
 
 	// todo: manually call stop on context cancellation?
 
-	return agent.Err(context.Background())
+	return a.Err(context.Background())
 }
 
 // Ready returns a channel that is closed after node's initialization has
@@ -483,7 +489,7 @@ func (n *Node) Manager() *manager.Manager {
 }
 
 // Agent returns agent instance started by node. May be nil.
-func (n *Node) Agent() *Agent {
+func (n *Node) Agent() *agent.Agent {
 	n.RLock()
 	defer n.RUnlock()
 	return n.agent

vendor/src/google.golang.org/grpc/balancer.go

@@ -38,6 +38,7 @@ import (
 	"sync"
 
 	"golang.org/x/net/context"
+	"google.golang.org/grpc/credentials"
 	"google.golang.org/grpc/grpclog"
 	"google.golang.org/grpc/naming"
 )
@@ -52,6 +53,14 @@ type Address struct {
 	Metadata interface{}
 }
 
+// BalancerConfig specifies the configurations for Balancer.
+type BalancerConfig struct {
+	// DialCreds is the transport credential the Balancer implementation can
+	// use to dial to a remote load balancer server. The Balancer implementations
+	// can ignore this if it does not need to talk to another party securely.
+	DialCreds credentials.TransportCredentials
+}
+
 // BalancerGetOptions configures a Get call.
 // This is the EXPERIMENTAL API and may be changed or extended in the future.
 type BalancerGetOptions struct {
@@ -66,11 +75,11 @@ type Balancer interface {
 	// Start does the initialization work to bootstrap a Balancer. For example,
 	// this function may start the name resolution and watch the updates. It will
 	// be called when dialing.
-	Start(target string) error
+	Start(target string, config BalancerConfig) error
 	// Up informs the Balancer that gRPC has a connection to the server at
 	// addr. It returns down which is called once the connection to addr gets
 	// lost or closed.
-	// TODO: It is not clear how to construct and take advantage the meaningful error
+	// TODO: It is not clear how to construct and take advantage of the meaningful error
 	// parameter for down. Need realistic demands to guide.
 	Up(addr Address) (down func(error))
 	// Get gets the address of a server for the RPC corresponding to ctx.
@@ -205,7 +214,12 @@ func (rr *roundRobin) watchAddrUpdates() error {
 	return nil
 }
 
-func (rr *roundRobin) Start(target string) error {
+func (rr *roundRobin) Start(target string, config BalancerConfig) error {
+	rr.mu.Lock()
+	defer rr.mu.Unlock()
+	if rr.done {
+		return ErrClientConnClosing
+	}
 	if rr.r == nil {
 		// If there is no name resolver installed, it is not needed to
 		// do name resolution. In this case, target is added into rr.addrs

vendor/src/google.golang.org/grpc/call.go

@@ -96,7 +96,7 @@ func sendRequest(ctx context.Context, codec Codec, compressor Compressor, callHd
 	}
 	outBuf, err := encode(codec, args, compressor, cbuf)
 	if err != nil {
-		return nil, transport.StreamErrorf(codes.Internal, "grpc: %v", err)
+		return nil, Errorf(codes.Internal, "grpc: %v", err)
 	}
 	err = t.Write(stream, outBuf, opts)
 	// t.NewStream(...) could lead to an early rejection of the RPC (e.g., the service/method
@@ -112,7 +112,14 @@ func sendRequest(ctx context.Context, codec Codec, compressor Compressor, callHd
 // Invoke sends the RPC request on the wire and returns after response is received.
 // Invoke is called by generated code. Also users can call Invoke directly when it
 // is really needed in their use cases.
-func Invoke(ctx context.Context, method string, args, reply interface{}, cc *ClientConn, opts ...CallOption) (err error) {
+func Invoke(ctx context.Context, method string, args, reply interface{}, cc *ClientConn, opts ...CallOption) error {
+	if cc.dopts.unaryInt != nil {
+		return cc.dopts.unaryInt(ctx, method, args, reply, cc, invoke, opts...)
+	}
+	return invoke(ctx, method, args, reply, cc, opts...)
+}
+
+func invoke(ctx context.Context, method string, args, reply interface{}, cc *ClientConn, opts ...CallOption) (err error) {
 	c := defaultCallInfo
 	for _, o := range opts {
 		if err := o.before(&c); err != nil {

vendor/src/google.golang.org/grpc/clientconn.go

@@ -83,15 +83,17 @@ var (
 // dialOptions configure a Dial call. dialOptions are set by the DialOption
 // values passed to Dial.
 type dialOptions struct {
-	codec    Codec
-	cp       Compressor
-	dc       Decompressor
-	bs       backoffStrategy
-	balancer Balancer
-	block    bool
-	insecure bool
-	timeout  time.Duration
-	copts    transport.ConnectOptions
+	unaryInt  UnaryClientInterceptor
+	streamInt StreamClientInterceptor
+	codec     Codec
+	cp        Compressor
+	dc        Decompressor
+	bs        backoffStrategy
+	balancer  Balancer
+	block     bool
+	insecure  bool
+	timeout   time.Duration
+	copts     transport.ConnectOptions
 }
 
 // DialOption configures how we set up the connection.
@@ -215,19 +217,48 @@ func WithUserAgent(s string) DialOption {
 	}
 }
 
+// WithUnaryInterceptor returns a DialOption that specifies the interceptor for unary RPCs.
+func WithUnaryInterceptor(f UnaryClientInterceptor) DialOption {
+	return func(o *dialOptions) {
+		o.unaryInt = f
+	}
+}
+
+// WithStreamInterceptor returns a DialOption that specifies the interceptor for streaming RPCs.
+func WithStreamInterceptor(f StreamClientInterceptor) DialOption {
+	return func(o *dialOptions) {
+		o.streamInt = f
+	}
+}
+
 // Dial creates a client connection to the given target.
 func Dial(target string, opts ...DialOption) (*ClientConn, error) {
 	return DialContext(context.Background(), target, opts...)
 }
 
-// DialContext creates a client connection to the given target
-// using the supplied context.
-func DialContext(ctx context.Context, target string, opts ...DialOption) (*ClientConn, error) {
+// DialContext creates a client connection to the given target. ctx can be used to
+// cancel or expire the pending connecting. Once this function returns, the
+// cancellation and expiration of ctx will be noop. Users should call ClientConn.Close
+// to terminate all the pending operations after this function returns.
+// This is the EXPERIMENTAL API.
+func DialContext(ctx context.Context, target string, opts ...DialOption) (conn *ClientConn, err error) {
 	cc := &ClientConn{
 		target: target,
 		conns:  make(map[Address]*addrConn),
 	}
-	cc.ctx, cc.cancel = context.WithCancel(ctx)
+	cc.ctx, cc.cancel = context.WithCancel(context.Background())
+	defer func() {
+		select {
+		case <-ctx.Done():
+			conn, err = nil, ctx.Err()
+		default:
+		}
+
+		if err != nil {
+			cc.Close()
+		}
+	}()
+
 	for _, opt := range opts {
 		opt(&cc.dopts)
 	}
@@ -239,31 +270,47 @@ func DialContext(ctx context.Context, target string, opts ...DialOption) (*Clien
 	if cc.dopts.bs == nil {
 		cc.dopts.bs = DefaultBackoffConfig
 	}
-
-	var (
-		ok    bool
-		addrs []Address
-	)
-	if cc.dopts.balancer == nil {
-		// Connect to target directly if balancer is nil.
-		addrs = append(addrs, Address{Addr: target})
+	creds := cc.dopts.copts.TransportCredentials
+	if creds != nil && creds.Info().ServerName != "" {
+		cc.authority = creds.Info().ServerName
 	} else {
-		if err := cc.dopts.balancer.Start(target); err != nil {
-			return nil, err
+		colonPos := strings.LastIndex(target, ":")
+		if colonPos == -1 {
+			colonPos = len(target)
 		}
-		ch := cc.dopts.balancer.Notify()
-		if ch == nil {
-			// There is no name resolver installed.
+		cc.authority = target[:colonPos]
+	}
+	var ok bool
+	waitC := make(chan error, 1)
+	go func() {
+		var addrs []Address
+		if cc.dopts.balancer == nil {
+			// Connect to target directly if balancer is nil.
 			addrs = append(addrs, Address{Addr: target})
 		} else {
-			addrs, ok = <-ch
-			if !ok || len(addrs) == 0 {
-				return nil, errNoAddr
+			var credsClone credentials.TransportCredentials
+			if creds != nil {
+				credsClone = creds.Clone()
+			}
+			config := BalancerConfig{
+				DialCreds: credsClone,
+			}
+			if err := cc.dopts.balancer.Start(target, config); err != nil {
+				waitC <- err
+				return
+			}
+			ch := cc.dopts.balancer.Notify()
+			if ch == nil {
+				// There is no name resolver installed.
+				addrs = append(addrs, Address{Addr: target})
+			} else {
+				addrs, ok = <-ch
+				if !ok || len(addrs) == 0 {
+					waitC <- errNoAddr
+					return
+				}
 			}
 		}
-	}
-	waitC := make(chan error, 1)
-	go func() {
 		for _, a := range addrs {
 			if err := cc.resetAddrConn(a, false, nil); err != nil {
 				waitC <- err
@@ -277,16 +324,13 @@ func DialContext(ctx context.Context, target string, opts ...DialOption) (*Clien
 		timeoutCh = time.After(cc.dopts.timeout)
 	}
 	select {
+	case <-ctx.Done():
+		return nil, ctx.Err()
 	case err := <-waitC:
 		if err != nil {
-			cc.Close()
 			return nil, err
 		}
-	case <-cc.ctx.Done():
-		cc.Close()
-		return nil, cc.ctx.Err()
 	case <-timeoutCh:
-		cc.Close()
 		return nil, ErrClientConnTimeout
 	}
 	// If balancer is nil or balancer.Notify() is nil, ok will be false here.
@@ -294,11 +338,6 @@ func DialContext(ctx context.Context, target string, opts ...DialOption) (*Clien
 	if ok {
 		go cc.lbWatcher()
 	}
-	colonPos := strings.LastIndex(target, ":")
-	if colonPos == -1 {
-		colonPos = len(target)
-	}
-	cc.authority = target[:colonPos]
 	return cc, nil
 }
 
@@ -652,7 +691,7 @@ func (ac *addrConn) resetTransport(closeTransport bool) error {
 			if e, ok := err.(transport.ConnectionError); ok && !e.Temporary() {
 				return err
 			}
-			grpclog.Printf("grpc: addrConn.resetTransport failed to create client transport: %v; Reconnecting to %q", err, ac.addr)
+			grpclog.Printf("grpc: addrConn.resetTransport failed to create client transport: %v; Reconnecting to %v", err, ac.addr)
 			ac.mu.Lock()
 			if ac.state == Shutdown {
 				// ac.tearDown(...) has been invoked.

vendor/src/google.golang.org/grpc/credentials/credentials.go

@@ -40,6 +40,7 @@ package credentials // import "google.golang.org/grpc/credentials"
 import (
 	"crypto/tls"
 	"crypto/x509"
+	"errors"
 	"fmt"
 	"io/ioutil"
 	"net"
@@ -71,7 +72,7 @@ type PerRPCCredentials interface {
 }
 
 // ProtocolInfo provides information regarding the gRPC wire protocol version,
-// security protocol, security protocol version in use, etc.
+// security protocol, security protocol version in use, server name, etc.
 type ProtocolInfo struct {
 	// ProtocolVersion is the gRPC wire protocol version.
 	ProtocolVersion string
@@ -79,6 +80,8 @@ type ProtocolInfo struct {
 	SecurityProtocol string
 	// SecurityVersion is the security protocol version.
 	SecurityVersion string
+	// ServerName is the user-configured server name.
+	ServerName string
 }
 
 // AuthInfo defines the common interface for the auth information the users are interested in.
@@ -86,6 +89,12 @@ type AuthInfo interface {
 	AuthType() string
 }
 
+var (
+	// ErrConnDispatched indicates that rawConn has been dispatched out of gRPC
+	// and the caller should not close rawConn.
+	ErrConnDispatched = errors.New("credentials: rawConn is dispatched out of gRPC")
+)
+
 // TransportCredentials defines the common interface for all the live gRPC wire
 // protocols and supported transport security protocols (e.g., TLS, SSL).
 type TransportCredentials interface {
@@ -100,6 +109,12 @@ type TransportCredentials interface {
 	ServerHandshake(net.Conn) (net.Conn, AuthInfo, error)
 	// Info provides the ProtocolInfo of this TransportCredentials.
 	Info() ProtocolInfo
+	// Clone makes a copy of this TransportCredentials.
+	Clone() TransportCredentials
+	// OverrideServerName overrides the server name used to verify the hostname on the returned certificates from the server.
+	// gRPC internals also use it to override the virtual hosting name if it is set.
+	// It must be called before dialing. Currently, this is only used by grpclb.
+	OverrideServerName(string) error
 }
 
 // TLSInfo contains the auth information for a TLS authenticated connection.
@@ -123,19 +138,10 @@ func (c tlsCreds) Info() ProtocolInfo {
 	return ProtocolInfo{
 		SecurityProtocol: "tls",
 		SecurityVersion:  "1.2",
+		ServerName:       c.config.ServerName,
 	}
 }
 
-// GetRequestMetadata returns nil, nil since TLS credentials does not have
-// metadata.
-func (c *tlsCreds) GetRequestMetadata(ctx context.Context, uri ...string) (map[string]string, error) {
-	return nil, nil
-}
-
-func (c *tlsCreds) RequireTransportSecurity() bool {
-	return true
-}
-
 func (c *tlsCreds) ClientHandshake(ctx context.Context, addr string, rawConn net.Conn) (_ net.Conn, _ AuthInfo, err error) {
 	// use local cfg to avoid clobbering ServerName if using multiple endpoints
 	cfg := cloneTLSConfig(c.config)
@@ -172,6 +178,15 @@ func (c *tlsCreds) ServerHandshake(rawConn net.Conn) (net.Conn, AuthInfo, error)
 	return conn, TLSInfo{conn.ConnectionState()}, nil
 }
 
+func (c *tlsCreds) Clone() TransportCredentials {
+	return NewTLS(c.config)
+}
+
+func (c *tlsCreds) OverrideServerName(serverNameOverride string) error {
+	c.config.ServerName = serverNameOverride
+	return nil
+}
+
 // NewTLS uses c to construct a TransportCredentials based on TLS.
 func NewTLS(c *tls.Config) TransportCredentials {
 	tc := &tlsCreds{cloneTLSConfig(c)}
@@ -180,12 +195,16 @@ func NewTLS(c *tls.Config) TransportCredentials {
 }
 
 // NewClientTLSFromCert constructs a TLS from the input certificate for client.
-func NewClientTLSFromCert(cp *x509.CertPool, serverName string) TransportCredentials {
-	return NewTLS(&tls.Config{ServerName: serverName, RootCAs: cp})
+// serverNameOverride is for testing only. If set to a non empty string,
+// it will override the virtual host name of authority (e.g. :authority header field) in requests.
+func NewClientTLSFromCert(cp *x509.CertPool, serverNameOverride string) TransportCredentials {
+	return NewTLS(&tls.Config{ServerName: serverNameOverride, RootCAs: cp})
 }
 
 // NewClientTLSFromFile constructs a TLS from the input certificate file for client.
-func NewClientTLSFromFile(certFile, serverName string) (TransportCredentials, error) {
+// serverNameOverride is for testing only. If set to a non empty string,
+// it will override the virtual host name of authority (e.g. :authority header field) in requests.
+func NewClientTLSFromFile(certFile, serverNameOverride string) (TransportCredentials, error) {
 	b, err := ioutil.ReadFile(certFile)
 	if err != nil {
 		return nil, err
@@ -194,7 +213,7 @@ func NewClientTLSFromFile(certFile, serverName string) (TransportCredentials, er
 	if !cp.AppendCertsFromPEM(b) {
 		return nil, fmt.Errorf("credentials: failed to append certificates")
 	}
-	return NewTLS(&tls.Config{ServerName: serverName, RootCAs: cp}), nil
+	return NewTLS(&tls.Config{ServerName: serverNameOverride, RootCAs: cp}), nil
 }
 
 // NewServerTLSFromCert constructs a TLS from the input certificate for server.

vendor/src/google.golang.org/grpc/interceptor.go

@@ -37,6 +37,22 @@ import (
 	"golang.org/x/net/context"
 )
 
+// UnaryInvoker is called by UnaryClientInterceptor to complete RPCs.
+type UnaryInvoker func(ctx context.Context, method string, req, reply interface{}, cc *ClientConn, opts ...CallOption) error
+
+// UnaryClientInterceptor intercepts the execution of a unary RPC on the client. inovker is the handler to complete the RPC
+// and it is the responsibility of the interceptor to call it.
+// This is the EXPERIMENTAL API.
+type UnaryClientInterceptor func(ctx context.Context, method string, req, reply interface{}, cc *ClientConn, invoker UnaryInvoker, opts ...CallOption) error
+
+// Streamer is called by StreamClientInterceptor to create a ClientStream.
+type Streamer func(ctx context.Context, desc *StreamDesc, cc *ClientConn, method string, opts ...CallOption) (ClientStream, error)
+
+// StreamClientInterceptor intercepts the creation of ClientStream. It may return a custom ClientStream to intercept all I/O
+// operations. streamer is the handlder to create a ClientStream and it is the responsibility of the interceptor to call it.
+// This is the EXPERIMENTAL API.
+type StreamClientInterceptor func(ctx context.Context, desc *StreamDesc, cc *ClientConn, method string, streamer Streamer, opts ...CallOption) (ClientStream, error)
+
 // UnaryServerInfo consists of various information about a unary RPC on
 // server side. All per-rpc information may be mutated by the interceptor.
 type UnaryServerInfo struct {

vendor/src/google.golang.org/grpc/metadata/metadata.go

@@ -117,10 +117,17 @@ func (md MD) Len() int {
 
 // Copy returns a copy of md.
 func (md MD) Copy() MD {
+	return Join(md)
+}
+
+// Join joins any number of MDs into a single MD.
+// The order of values for each key is determined by the order in which
+// the MDs containing those values are presented to Join.
+func Join(mds ...MD) MD {
 	out := MD{}
-	for k, v := range md {
-		for _, i := range v {
-			out[k] = append(out[k], i)
+	for _, md := range mds {
+		for k, v := range md {
+			out[k] = append(out[k], v...)
 		}
 	}
 	return out

vendor/src/google.golang.org/grpc/rpc_util.go

@@ -303,10 +303,10 @@ func checkRecvPayload(pf payloadFormat, recvCompress string, dc Decompressor) er
 	case compressionNone:
 	case compressionMade:
 		if dc == nil || recvCompress != dc.Type() {
-			return transport.StreamErrorf(codes.Unimplemented, "grpc: Decompressor is not installed for grpc-encoding %q", recvCompress)
+			return Errorf(codes.Unimplemented, "grpc: Decompressor is not installed for grpc-encoding %q", recvCompress)
 		}
 	default:
-		return transport.StreamErrorf(codes.Internal, "grpc: received unexpected payload format %d", pf)
+		return Errorf(codes.Internal, "grpc: received unexpected payload format %d", pf)
 	}
 	return nil
 }

vendor/src/google.golang.org/grpc/server.go

@@ -324,7 +324,7 @@ func (s *Server) useTransportAuthenticator(rawConn net.Conn) (net.Conn, credenti
 // Serve accepts incoming connections on the listener lis, creating a new
 // ServerTransport and service goroutine for each. The service goroutines
 // read gRPC requests and then call the registered handlers to reply to them.
-// Service returns when lis.Accept fails. lis will be closed when
+// Serve returns when lis.Accept fails. lis will be closed when
 // this method returns.
 func (s *Server) Serve(lis net.Listener) error {
 	s.mu.Lock()
@@ -367,7 +367,10 @@ func (s *Server) handleRawConn(rawConn net.Conn) {
 		s.errorf("ServerHandshake(%q) failed: %v", rawConn.RemoteAddr(), err)
 		s.mu.Unlock()
 		grpclog.Printf("grpc: Server.Serve failed to complete security handshake from %q: %v", rawConn.RemoteAddr(), err)
-		rawConn.Close()
+		// If serverHandShake returns ErrConnDispatched, keep rawConn open.
+		if err != credentials.ErrConnDispatched {
+			rawConn.Close()
+		}
 		return
 	}
 
@@ -544,7 +547,7 @@ func (s *Server) processUnaryRPC(t transport.ServerTransport, stream *transport.
 			return err
 		}
 		if err == io.ErrUnexpectedEOF {
-			err = transport.StreamError{Code: codes.Internal, Desc: "io.ErrUnexpectedEOF"}
+			err = Errorf(codes.Internal, io.ErrUnexpectedEOF.Error())
 		}
 		if err != nil {
 			switch err := err.(type) {
@@ -566,8 +569,8 @@ func (s *Server) processUnaryRPC(t transport.ServerTransport, stream *transport.
 
 		if err := checkRecvPayload(pf, stream.RecvCompress(), s.opts.dc); err != nil {
 			switch err := err.(type) {
-			case transport.StreamError:
-				if err := t.WriteStatus(stream, err.Code, err.Desc); err != nil {
+			case *rpcError:
+				if err := t.WriteStatus(stream, err.code, err.desc); err != nil {
 					grpclog.Printf("grpc: Server.processUnaryRPC failed to write status %v", err)
 				}
 			default:
@@ -870,25 +873,28 @@ func SendHeader(ctx context.Context, md metadata.MD) error {
 	}
 	stream, ok := transport.StreamFromContext(ctx)
 	if !ok {
-		return fmt.Errorf("grpc: failed to fetch the stream from the context %v", ctx)
+		return Errorf(codes.Internal, "grpc: failed to fetch the stream from the context %v", ctx)
 	}
 	t := stream.ServerTransport()
 	if t == nil {
 		grpclog.Fatalf("grpc: SendHeader: %v has no ServerTransport to send header metadata.", stream)
 	}
-	return t.WriteHeader(stream, md)
+	if err := t.WriteHeader(stream, md); err != nil {
+		return toRPCErr(err)
+	}
+	return nil
 }
 
 // SetTrailer sets the trailer metadata that will be sent when an RPC returns.
-// It may be called at most once from a unary RPC handler. The ctx is the RPC
-// handler's Context or one derived from it.
+// When called more than once, all the provided metadata will be merged.
+// The ctx is the RPC handler's Context or one derived from it.
 func SetTrailer(ctx context.Context, md metadata.MD) error {
 	if md.Len() == 0 {
 		return nil
 	}
 	stream, ok := transport.StreamFromContext(ctx)
 	if !ok {
-		return fmt.Errorf("grpc: failed to fetch the stream from the context %v", ctx)
+		return Errorf(codes.Internal, "grpc: failed to fetch the stream from the context %v", ctx)
 	}
 	return stream.SetTrailer(md)
 }

vendor/src/google.golang.org/grpc/stream.go

@@ -97,7 +97,14 @@ type ClientStream interface {
 
 // NewClientStream creates a new Stream for the client side. This is called
 // by generated code.
-func NewClientStream(ctx context.Context, desc *StreamDesc, cc *ClientConn, method string, opts ...CallOption) (_ ClientStream, err error) {
+func NewClientStream(ctx context.Context, desc *StreamDesc, cc *ClientConn, method string, opts ...CallOption) (ClientStream, error) {
+	if cc.dopts.streamInt != nil {
+		return cc.dopts.streamInt(ctx, desc, cc, method, newClientStream, opts...)
+	}
+	return newClientStream(ctx, desc, cc, method, opts...)
+}
+
+func newClientStream(ctx context.Context, desc *StreamDesc, cc *ClientConn, method string, opts ...CallOption) (_ ClientStream, err error) {
 	var (
 		t   transport.ClientTransport
 		s   *transport.Stream
@@ -296,7 +303,7 @@ func (cs *clientStream) SendMsg(m interface{}) (err error) {
 		}
 	}()
 	if err != nil {
-		return transport.StreamErrorf(codes.Internal, "grpc: %v", err)
+		return Errorf(codes.Internal, "grpc: %v", err)
 	}
 	return cs.t.Write(cs.s, out, &transport.Options{Last: false})
 }
@@ -407,8 +414,8 @@ type ServerStream interface {
 	// after SendProto. It fails if called multiple times or if
 	// called after SendProto.
 	SendHeader(metadata.MD) error
-	// SetTrailer sets the trailer metadata which will be sent with the
-	// RPC status.
+	// SetTrailer sets the trailer metadata which will be sent with the RPC status.
+	// When called more than once, all the provided metadata will be merged.
 	SetTrailer(metadata.MD)
 	Stream
 }
@@ -468,10 +475,13 @@ func (ss *serverStream) SendMsg(m interface{}) (err error) {
 		}
 	}()
 	if err != nil {
-		err = transport.StreamErrorf(codes.Internal, "grpc: %v", err)
+		err = Errorf(codes.Internal, "grpc: %v", err)
 		return err
 	}
-	return ss.t.Write(ss.s, out, &transport.Options{Last: false})
+	if err := ss.t.Write(ss.s, out, &transport.Options{Last: false}); err != nil {
+		return toRPCErr(err)
+	}
+	return nil
 }
 
 func (ss *serverStream) RecvMsg(m interface{}) (err error) {
@@ -489,5 +499,14 @@ func (ss *serverStream) RecvMsg(m interface{}) (err error) {
 			ss.mu.Unlock()
 		}
 	}()
-	return recv(ss.p, ss.codec, ss.s, ss.dc, m, ss.maxMsgSize)
+	if err := recv(ss.p, ss.codec, ss.s, ss.dc, m, ss.maxMsgSize); err != nil {
+		if err == io.EOF {
+			return err
+		}
+		if err == io.ErrUnexpectedEOF {
+			err = Errorf(codes.Internal, io.ErrUnexpectedEOF.Error())
+		}
+		return toRPCErr(err)
+	}
+	return nil
 }

vendor/src/google.golang.org/grpc/transport/handler_server.go

@@ -85,7 +85,7 @@ func NewServerHandlerTransport(w http.ResponseWriter, r *http.Request) (ServerTr
 	if v := r.Header.Get("grpc-timeout"); v != "" {
 		to, err := decodeTimeout(v)
 		if err != nil {
-			return nil, StreamErrorf(codes.Internal, "malformed time-out: %v", err)
+			return nil, streamErrorf(codes.Internal, "malformed time-out: %v", err)
 		}
 		st.timeoutSet = true
 		st.timeout = to
@@ -393,5 +393,5 @@ func mapRecvMsgError(err error) error {
 			}
 		}
 	}
-	return ConnectionError{Desc: err.Error()}
+	return connectionErrorf(true, err, err.Error())
 }

vendor/src/google.golang.org/grpc/transport/http2_client.go

@@ -114,14 +114,42 @@ func dial(fn func(context.Context, string) (net.Conn, error), ctx context.Contex
 	return dialContext(ctx, "tcp", addr)
 }
 
+func isTemporary(err error) bool {
+	switch err {
+	case io.EOF:
+		// Connection closures may be resolved upon retry, and are thus
+		// treated as temporary.
+		return true
+	case context.DeadlineExceeded:
+		// In Go 1.7, context.DeadlineExceeded implements Timeout(), and this
+		// special case is not needed. Until then, we need to keep this
+		// clause.
+		return true
+	}
+
+	switch err := err.(type) {
+	case interface {
+		Temporary() bool
+	}:
+		return err.Temporary()
+	case interface {
+		Timeout() bool
+	}:
+		// Timeouts may be resolved upon retry, and are thus treated as
+		// temporary.
+		return err.Timeout()
+	}
+	return false
+}
+
 // newHTTP2Client constructs a connected ClientTransport to addr based on HTTP2
 // and starts to receive messages on it. Non-nil error returns if construction
 // fails.
 func newHTTP2Client(ctx context.Context, addr string, opts ConnectOptions) (_ ClientTransport, err error) {
 	scheme := "http"
-	conn, connErr := dial(opts.Dialer, ctx, addr)
-	if connErr != nil {
-		return nil, ConnectionErrorf(true, connErr, "transport: %v", connErr)
+	conn, err := dial(opts.Dialer, ctx, addr)
+	if err != nil {
+		return nil, connectionErrorf(true, err, "transport: %v", err)
 	}
 	// Any further errors will close the underlying connection
 	defer func(conn net.Conn) {
@@ -132,12 +160,13 @@ func newHTTP2Client(ctx context.Context, addr string, opts ConnectOptions) (_ Cl
 	var authInfo credentials.AuthInfo
 	if creds := opts.TransportCredentials; creds != nil {
 		scheme = "https"
-		conn, authInfo, connErr = creds.ClientHandshake(ctx, addr, conn)
-	}
-	if connErr != nil {
-		// Credentials handshake error is not a temporary error (unless the error
-		// was the connection closing).
-		return nil, ConnectionErrorf(connErr == io.EOF, connErr, "transport: %v", connErr)
+		conn, authInfo, err = creds.ClientHandshake(ctx, addr, conn)
+		if err != nil {
+			// Credentials handshake errors are typically considered permanent
+			// to avoid retrying on e.g. bad certificates.
+			temp := isTemporary(err)
+			return nil, connectionErrorf(temp, err, "transport: %v", err)
+		}
 	}
 	ua := primaryUA
 	if opts.UserAgent != "" {
@@ -176,11 +205,11 @@ func newHTTP2Client(ctx context.Context, addr string, opts ConnectOptions) (_ Cl
 	n, err := t.conn.Write(clientPreface)
 	if err != nil {
 		t.Close()
-		return nil, ConnectionErrorf(true, err, "transport: %v", err)
+		return nil, connectionErrorf(true, err, "transport: %v", err)
 	}
 	if n != len(clientPreface) {
 		t.Close()
-		return nil, ConnectionErrorf(true, err, "transport: preface mismatch, wrote %d bytes; want %d", n, len(clientPreface))
+		return nil, connectionErrorf(true, err, "transport: preface mismatch, wrote %d bytes; want %d", n, len(clientPreface))
 	}
 	if initialWindowSize != defaultWindowSize {
 		err = t.framer.writeSettings(true, http2.Setting{
@@ -192,13 +221,13 @@ func newHTTP2Client(ctx context.Context, addr string, opts ConnectOptions) (_ Cl
 	}
 	if err != nil {
 		t.Close()
-		return nil, ConnectionErrorf(true, err, "transport: %v", err)
+		return nil, connectionErrorf(true, err, "transport: %v", err)
 	}
 	// Adjust the connection flow control window if needed.
 	if delta := uint32(initialConnWindowSize - defaultWindowSize); delta > 0 {
 		if err := t.framer.writeWindowUpdate(true, 0, delta); err != nil {
 			t.Close()
-			return nil, ConnectionErrorf(true, err, "transport: %v", err)
+			return nil, connectionErrorf(true, err, "transport: %v", err)
 		}
 	}
 	go t.controller()
@@ -223,8 +252,10 @@ func (t *http2Client) newStream(ctx context.Context, callHdr *CallHdr) *Stream {
 	s.windowHandler = func(n int) {
 		t.updateWindow(s, uint32(n))
 	}
-	// Make a stream be able to cancel the pending operations by itself.
-	s.ctx, s.cancel = context.WithCancel(ctx)
+	// The client side stream context should have exactly the same life cycle with the user provided context.
+	// That means, s.ctx should be read-only. And s.ctx is done iff ctx is done.
+	// So we use the original context here instead of creating a copy.
+	s.ctx = ctx
 	s.dec = &recvBufferReader{
 		ctx:    s.ctx,
 		goAway: s.goAway,
@@ -236,16 +267,6 @@ func (t *http2Client) newStream(ctx context.Context, callHdr *CallHdr) *Stream {
 // NewStream creates a stream and register it into the transport as "active"
 // streams.
 func (t *http2Client) NewStream(ctx context.Context, callHdr *CallHdr) (_ *Stream, err error) {
-	// Record the timeout value on the context.
-	var timeout time.Duration
-	if dl, ok := ctx.Deadline(); ok {
-		timeout = dl.Sub(time.Now())
-	}
-	select {
-	case <-ctx.Done():
-		return nil, ContextErr(ctx.Err())
-	default:
-	}
 	pr := &peer.Peer{
 		Addr: t.conn.RemoteAddr(),
 	}
@@ -266,12 +287,12 @@ func (t *http2Client) NewStream(ctx context.Context, callHdr *CallHdr) (_ *Strea
 		}
 		pos := strings.LastIndex(callHdr.Method, "/")
 		if pos == -1 {
-			return nil, StreamErrorf(codes.InvalidArgument, "transport: malformed method name: %q", callHdr.Method)
+			return nil, streamErrorf(codes.InvalidArgument, "transport: malformed method name: %q", callHdr.Method)
 		}
 		audience := "https://" + callHdr.Host + port + callHdr.Method[:pos]
 		data, err := c.GetRequestMetadata(ctx, audience)
 		if err != nil {
-			return nil, StreamErrorf(codes.InvalidArgument, "transport: %v", err)
+			return nil, streamErrorf(codes.InvalidArgument, "transport: %v", err)
 		}
 		for k, v := range data {
 			authData[k] = v
@@ -352,9 +373,12 @@ func (t *http2Client) NewStream(ctx context.Context, callHdr *CallHdr) (_ *Strea
 	if callHdr.SendCompress != "" {
 		t.hEnc.WriteField(hpack.HeaderField{Name: "grpc-encoding", Value: callHdr.SendCompress})
 	}
-	if timeout > 0 {
+	if dl, ok := ctx.Deadline(); ok {
+		// Send out timeout regardless its value. The server can detect timeout context by itself.
+		timeout := dl.Sub(time.Now())
 		t.hEnc.WriteField(hpack.HeaderField{Name: "grpc-timeout", Value: encodeTimeout(timeout)})
 	}
+
 	for k, v := range authData {
 		// Capital header names are illegal in HTTP/2.
 		k = strings.ToLower(k)
@@ -408,7 +432,7 @@ func (t *http2Client) NewStream(ctx context.Context, callHdr *CallHdr) (_ *Strea
 		}
 		if err != nil {
 			t.notifyError(err)
-			return nil, ConnectionErrorf(true, err, "transport: %v", err)
+			return nil, connectionErrorf(true, err, "transport: %v", err)
 		}
 	}
 	t.writableChan <- 0
@@ -454,7 +478,7 @@ func (t *http2Client) CloseStream(s *Stream, err error) {
 	}
 	s.state = streamDone
 	s.mu.Unlock()
-	if _, ok := err.(StreamError); ok {
+	if se, ok := err.(StreamError); ok && se.Code != codes.DeadlineExceeded {
 		t.controlBuf.put(&resetStream{s.id, http2.ErrCodeCancel})
 	}
 }
@@ -622,7 +646,7 @@ func (t *http2Client) Write(s *Stream, data []byte, opts *Options) error {
 		// invoked.
 		if err := t.framer.writeData(forceFlush, s.id, endStream, p); err != nil {
 			t.notifyError(err)
-			return ConnectionErrorf(true, err, "transport: %v", err)
+			return connectionErrorf(true, err, "transport: %v", err)
 		}
 		if t.framer.adjustNumWriters(-1) == 0 {
 			t.framer.flushWrite()
@@ -670,7 +694,7 @@ func (t *http2Client) updateWindow(s *Stream, n uint32) {
 func (t *http2Client) handleData(f *http2.DataFrame) {
 	size := len(f.Data())
 	if err := t.fc.onData(uint32(size)); err != nil {
-		t.notifyError(ConnectionErrorf(true, err, "%v", err))
+		t.notifyError(connectionErrorf(true, err, "%v", err))
 		return
 	}
 	// Select the right stream to dispatch.
@@ -776,7 +800,7 @@ func (t *http2Client) handleGoAway(f *http2.GoAwayFrame) {
 	if t.state == reachable || t.state == draining {
 		if f.LastStreamID > 0 && f.LastStreamID%2 != 1 {
 			t.mu.Unlock()
-			t.notifyError(ConnectionErrorf(true, nil, "received illegal http2 GOAWAY frame: stream ID %d is even", f.LastStreamID))
+			t.notifyError(connectionErrorf(true, nil, "received illegal http2 GOAWAY frame: stream ID %d is even", f.LastStreamID))
 			return
 		}
 		select {
@@ -785,7 +809,7 @@ func (t *http2Client) handleGoAway(f *http2.GoAwayFrame) {
 			// t.goAway has been closed (i.e.,multiple GoAways).
 			if id < f.LastStreamID {
 				t.mu.Unlock()
-				t.notifyError(ConnectionErrorf(true, nil, "received illegal http2 GOAWAY frame: previously recv GOAWAY frame with LastStramID %d, currently recv %d", id, f.LastStreamID))
+				t.notifyError(connectionErrorf(true, nil, "received illegal http2 GOAWAY frame: previously recv GOAWAY frame with LastStramID %d, currently recv %d", id, f.LastStreamID))
 				return
 			}
 			t.prevGoAwayID = id
@@ -823,6 +847,12 @@ func (t *http2Client) operateHeaders(frame *http2.MetaHeadersFrame) {
 		state.processHeaderField(hf)
 	}
 	if state.err != nil {
+		s.mu.Lock()
+		if !s.headerDone {
+			close(s.headerChan)
+			s.headerDone = true
+		}
+		s.mu.Unlock()
 		s.write(recvMsg{err: state.err})
 		// Something wrong. Stops reading even when there is remaining.
 		return
@@ -900,7 +930,7 @@ func (t *http2Client) reader() {
 				t.mu.Unlock()
 				if s != nil {
 					// use error detail to provide better err message
-					handleMalformedHTTP2(s, StreamErrorf(http2ErrConvTab[se.Code], "%v", t.framer.errorDetail()))
+					handleMalformedHTTP2(s, streamErrorf(http2ErrConvTab[se.Code], "%v", t.framer.errorDetail()))
 				}
 				continue
 			} else {

vendor/src/google.golang.org/grpc/transport/http2_server.go

@@ -111,12 +111,12 @@ func newHTTP2Server(conn net.Conn, maxStreams uint32, authInfo credentials.AuthI
 			Val: uint32(initialWindowSize)})
 	}
 	if err := framer.writeSettings(true, settings...); err != nil {
-		return nil, ConnectionErrorf(true, err, "transport: %v", err)
+		return nil, connectionErrorf(true, err, "transport: %v", err)
 	}
 	// Adjust the connection flow control window if needed.
 	if delta := uint32(initialConnWindowSize - defaultWindowSize); delta > 0 {
 		if err := framer.writeWindowUpdate(true, 0, delta); err != nil {
-			return nil, ConnectionErrorf(true, err, "transport: %v", err)
+			return nil, connectionErrorf(true, err, "transport: %v", err)
 		}
 	}
 	var buf bytes.Buffer
@@ -448,7 +448,7 @@ func (t *http2Server) writeHeaders(s *Stream, b *bytes.Buffer, endStream bool) e
 		}
 		if err != nil {
 			t.Close()
-			return ConnectionErrorf(true, err, "transport: %v", err)
+			return connectionErrorf(true, err, "transport: %v", err)
 		}
 	}
 	return nil
@@ -544,7 +544,7 @@ func (t *http2Server) Write(s *Stream, data []byte, opts *Options) error {
 	s.mu.Lock()
 	if s.state == streamDone {
 		s.mu.Unlock()
-		return StreamErrorf(codes.Unknown, "the stream has been done")
+		return streamErrorf(codes.Unknown, "the stream has been done")
 	}
 	if !s.headerOk {
 		writeHeaderFrame = true
@@ -568,7 +568,7 @@ func (t *http2Server) Write(s *Stream, data []byte, opts *Options) error {
 		}
 		if err := t.framer.writeHeaders(false, p); err != nil {
 			t.Close()
-			return ConnectionErrorf(true, err, "transport: %v", err)
+			return connectionErrorf(true, err, "transport: %v", err)
 		}
 		t.writableChan <- 0
 	}
@@ -642,7 +642,7 @@ func (t *http2Server) Write(s *Stream, data []byte, opts *Options) error {
 		}
 		if err := t.framer.writeData(forceFlush, s.id, false, p); err != nil {
 			t.Close()
-			return ConnectionErrorf(true, err, "transport: %v", err)
+			return connectionErrorf(true, err, "transport: %v", err)
 		}
 		if t.framer.adjustNumWriters(-1) == 0 {
 			t.framer.flushWrite()

vendor/src/google.golang.org/grpc/transport/http_util.go

@@ -162,7 +162,7 @@ func (d *decodeState) processHeaderField(f hpack.HeaderField) {
 	switch f.Name {
 	case "content-type":
 		if !validContentType(f.Value) {
-			d.setErr(StreamErrorf(codes.FailedPrecondition, "transport: received the unexpected content-type %q", f.Value))
+			d.setErr(streamErrorf(codes.FailedPrecondition, "transport: received the unexpected content-type %q", f.Value))
 			return
 		}
 	case "grpc-encoding":
@@ -170,7 +170,7 @@ func (d *decodeState) processHeaderField(f hpack.HeaderField) {
 	case "grpc-status":
 		code, err := strconv.Atoi(f.Value)
 		if err != nil {
-			d.setErr(StreamErrorf(codes.Internal, "transport: malformed grpc-status: %v", err))
+			d.setErr(streamErrorf(codes.Internal, "transport: malformed grpc-status: %v", err))
 			return
 		}
 		d.statusCode = codes.Code(code)
@@ -181,7 +181,7 @@ func (d *decodeState) processHeaderField(f hpack.HeaderField) {
 		var err error
 		d.timeout, err = decodeTimeout(f.Value)
 		if err != nil {
-			d.setErr(StreamErrorf(codes.Internal, "transport: malformed time-out: %v", err))
+			d.setErr(streamErrorf(codes.Internal, "transport: malformed time-out: %v", err))
 			return
 		}
 	case ":path":
@@ -253,6 +253,9 @@ func div(d, r time.Duration) int64 {
 
 // TODO(zhaoq): It is the simplistic and not bandwidth efficient. Improve it.
 func encodeTimeout(t time.Duration) string {
+	if t <= 0 {
+		return "0n"
+	}
 	if d := div(t, time.Nanosecond); d <= maxTimeoutValue {
 		return strconv.FormatInt(d, 10) + "n"
 	}
@@ -349,7 +352,7 @@ func decodeGrpcMessageUnchecked(msg string) string {
 	for i := 0; i < lenMsg; i++ {
 		c := msg[i]
 		if c == percentByte && i+2 < lenMsg {
-			parsed, err := strconv.ParseInt(msg[i+1:i+3], 16, 8)
+			parsed, err := strconv.ParseUint(msg[i+1:i+3], 16, 8)
 			if err != nil {
 				buf.WriteByte(c)
 			} else {

vendor/src/google.golang.org/grpc/transport/transport.go

@@ -39,7 +39,6 @@ package transport // import "google.golang.org/grpc/transport"
 
 import (
 	"bytes"
-	"errors"
 	"fmt"
 	"io"
 	"net"
@@ -169,7 +168,8 @@ type Stream struct {
 	// nil for client side Stream.
 	st ServerTransport
 	// ctx is the associated context of the stream.
-	ctx    context.Context
+	ctx context.Context
+	// cancel is always nil for client side Stream.
 	cancel context.CancelFunc
 	// done is closed when the final status arrives.
 	done chan struct{}
@@ -286,19 +286,12 @@ func (s *Stream) StatusDesc() string {
 	return s.statusDesc
 }
 
-// ErrIllegalTrailerSet indicates that the trailer has already been set or it
-// is too late to do so.
-var ErrIllegalTrailerSet = errors.New("transport: trailer has been set")
-
 // SetTrailer sets the trailer metadata which will be sent with the RPC status
-// by the server. This can only be called at most once. Server side only.
+// by the server. This can be called multiple times. Server side only.
 func (s *Stream) SetTrailer(md metadata.MD) error {
 	s.mu.Lock()
 	defer s.mu.Unlock()
-	if s.trailer != nil {
-		return ErrIllegalTrailerSet
-	}
-	s.trailer = md.Copy()
+	s.trailer = metadata.Join(s.trailer, md)
 	return nil
 }
 
@@ -476,16 +469,16 @@ type ServerTransport interface {
 	Drain()
 }
 
-// StreamErrorf creates an StreamError with the specified error code and description.
-func StreamErrorf(c codes.Code, format string, a ...interface{}) StreamError {
+// streamErrorf creates an StreamError with the specified error code and description.
+func streamErrorf(c codes.Code, format string, a ...interface{}) StreamError {
 	return StreamError{
 		Code: c,
 		Desc: fmt.Sprintf(format, a...),
 	}
 }
 
-// ConnectionErrorf creates an ConnectionError with the specified error description.
-func ConnectionErrorf(temp bool, e error, format string, a ...interface{}) ConnectionError {
+// connectionErrorf creates an ConnectionError with the specified error description.
+func connectionErrorf(temp bool, e error, format string, a ...interface{}) ConnectionError {
 	return ConnectionError{
 		Desc: fmt.Sprintf(format, a...),
 		temp: temp,
@@ -522,10 +515,10 @@ func (e ConnectionError) Origin() error {
 
 var (
 	// ErrConnClosing indicates that the transport is closing.
-	ErrConnClosing = ConnectionError{Desc: "transport is closing", temp: true}
+	ErrConnClosing = connectionErrorf(true, nil, "transport is closing")
 	// ErrStreamDrain indicates that the stream is rejected by the server because
 	// the server stops accepting new RPCs.
-	ErrStreamDrain = StreamErrorf(codes.Unavailable, "the server stops accepting new RPCs")
+	ErrStreamDrain = streamErrorf(codes.Unavailable, "the server stops accepting new RPCs")
 )
 
 // StreamError is an error that only affects one stream within a connection.
@@ -542,9 +535,9 @@ func (e StreamError) Error() string {
 func ContextErr(err error) StreamError {
 	switch err {
 	case context.DeadlineExceeded:
-		return StreamErrorf(codes.DeadlineExceeded, "%v", err)
+		return streamErrorf(codes.DeadlineExceeded, "%v", err)
 	case context.Canceled:
-		return StreamErrorf(codes.Canceled, "%v", err)
+		return streamErrorf(codes.Canceled, "%v", err)
 	}
 	panic(fmt.Sprintf("Unexpected error from context packet: %v", err))
 }