Vendor swarmkit 46bbd41

Signed-off-by: Aaron Lehmann <aaron.lehmann@docker.com>

integration-cli/docker_api_swarm_test.go

@@ -183,7 +183,15 @@ func (s *DockerSwarmSuite) TestAPISwarmPromoteDemote(c *check.C) {
 	status, out, err := d1.SockRequest("POST", url, node.Spec)
 	c.Assert(err, checker.IsNil)
 	c.Assert(status, checker.Equals, http.StatusInternalServerError, check.Commentf("output: %q", string(out)))
-	c.Assert(string(out), checker.Contains, "last manager of the swarm")
+	// The warning specific to demoting the last manager is best-effort and
+	// won't appear until the Role field of the demoted manager has been
+	// updated.
+	// Yes, I know this looks silly, but checker.Matches is broken, since
+	// it anchors the regexp contrary to the documentation, and this makes
+	// it impossible to match something that includes a line break.
+	if !strings.Contains(string(out), "last manager of the swarm") {
+		c.Assert(string(out), checker.Contains, "this would result in a loss of quorum")
+	}
 	info, err = d1.SwarmInfo()
 	c.Assert(err, checker.IsNil)
 	c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive)

vendor.conf

@@ -104,7 +104,7 @@ github.com/docker/containerd 665e84e6c28653a9c29a6db601636a92d46896f3
 github.com/tonistiigi/fifo 1405643975692217d6720f8b54aeee1bf2cd5cf4
 
 # cluster
-github.com/docker/swarmkit 6bc357e9c5f0ac2cdf801898a43d08c260b4d5d0
+github.com/docker/swarmkit 46bbd41a00b996a13840607772f661a7f5096ca0
 github.com/golang/mock bd3c8e81be01eef76d4b503f5e687d2d1354d2d9
 github.com/gogo/protobuf 8d70fb3182befc465c4a1eac8ad4d38ff49778e2
 github.com/cloudflare/cfssl 7fb22c8cba7ecaf98e4082d22d65800cf45e042a
@@ -125,6 +125,7 @@ github.com/prometheus/procfs abf152e5f3e97f2fafac028d2cc06c1feb87ffa5
 bitbucket.org/ww/goautoneg 75cd24fc2f2c2a2088577d12123ddee5f54e0675
 github.com/matttproud/golang_protobuf_extensions v1.0.0
 github.com/pkg/errors 839d9e913e063e28dfd0e6c7b7512793e0a48be9
+github.com/grpc-ecosystem/go-grpc-prometheus 6b7015e65d366bf3f19b2b2a000a831940f0f7e0
 
 # cli
 github.com/spf13/cobra v1.5.1 https://github.com/dnephin/cobra.git

vendor/github.com/docker/swarmkit/agent/errors.go

@@ -2,14 +2,13 @@ package agent
 
 import (
 	"errors"
-	"fmt"
 )
 
 var (
 	// ErrClosed is returned when an operation fails because the resource is closed.
 	ErrClosed = errors.New("agent: closed")
 
-	errNodeNotRegistered = fmt.Errorf("node not registered")
+	errNodeNotRegistered = errors.New("node not registered")
 
 	errAgentStarted    = errors.New("agent: already started")
 	errAgentNotStarted = errors.New("agent: not started")

vendor/github.com/docker/swarmkit/api/specs.pb.go

@@ -75,7 +75,7 @@ func (NodeSpec_Availability) EnumDescriptor() ([]byte, []int) { return fileDescr
 
 // ResolutionMode specifies the mode of resolution to use for
 // internal loadbalancing between tasks which are all within
-// the cluster. This is sometimes calles east-west data path.
+// the cluster. This is sometimes calls east-west data path.
 type EndpointSpec_ResolutionMode int32
 
 const (
@@ -106,7 +106,7 @@ func (x EndpointSpec_ResolutionMode) String() string {
 	return proto.EnumName(EndpointSpec_ResolutionMode_name, int32(x))
 }
 func (EndpointSpec_ResolutionMode) EnumDescriptor() ([]byte, []int) {
-	return fileDescriptorSpecs, []int{7, 0}
+	return fileDescriptorSpecs, []int{8, 0}
 }
 
 type NodeSpec struct {
@@ -289,6 +289,7 @@ type TaskSpec struct {
 	// Types that are valid to be assigned to Runtime:
 	//	*TaskSpec_Attachment
 	//	*TaskSpec_Container
+	//	*TaskSpec_Plugin
 	Runtime isTaskSpec_Runtime `protobuf_oneof:"runtime"`
 	// Resource requirements for the container.
 	Resources *ResourceRequirements `protobuf:"bytes,2,opt,name=resources" json:"resources,omitempty"`
@@ -326,9 +327,13 @@ type TaskSpec_Attachment struct {
 type TaskSpec_Container struct {
 	Container *ContainerSpec `protobuf:"bytes,1,opt,name=container,oneof"`
 }
+type TaskSpec_Plugin struct {
+	Plugin *PluginSpec `protobuf:"bytes,10,opt,name=plugin,oneof"`
+}
 
 func (*TaskSpec_Attachment) isTaskSpec_Runtime() {}
 func (*TaskSpec_Container) isTaskSpec_Runtime()  {}
+func (*TaskSpec_Plugin) isTaskSpec_Runtime()     {}
 
 func (m *TaskSpec) GetRuntime() isTaskSpec_Runtime {
 	if m != nil {
@@ -351,11 +356,19 @@ func (m *TaskSpec) GetContainer() *ContainerSpec {
 	return nil
 }
 
+func (m *TaskSpec) GetPlugin() *PluginSpec {
+	if x, ok := m.GetRuntime().(*TaskSpec_Plugin); ok {
+		return x.Plugin
+	}
+	return nil
+}
+
 // XXX_OneofFuncs is for the internal use of the proto package.
 func (*TaskSpec) XXX_OneofFuncs() (func(msg proto.Message, b *proto.Buffer) error, func(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error), func(msg proto.Message) (n int), []interface{}) {
 	return _TaskSpec_OneofMarshaler, _TaskSpec_OneofUnmarshaler, _TaskSpec_OneofSizer, []interface{}{
 		(*TaskSpec_Attachment)(nil),
 		(*TaskSpec_Container)(nil),
+		(*TaskSpec_Plugin)(nil),
 	}
 }
 
@@ -373,6 +386,11 @@ func _TaskSpec_OneofMarshaler(msg proto.Message, b *proto.Buffer) error {
 		if err := b.EncodeMessage(x.Container); err != nil {
 			return err
 		}
+	case *TaskSpec_Plugin:
+		_ = b.EncodeVarint(10<<3 | proto.WireBytes)
+		if err := b.EncodeMessage(x.Plugin); err != nil {
+			return err
+		}
 	case nil:
 	default:
 		return fmt.Errorf("TaskSpec.Runtime has unexpected type %T", x)
@@ -399,6 +417,14 @@ func _TaskSpec_OneofUnmarshaler(msg proto.Message, tag, wire int, b *proto.Buffe
 		err := b.DecodeMessage(msg)
 		m.Runtime = &TaskSpec_Container{msg}
 		return true, err
+	case 10: // runtime.plugin
+		if wire != proto.WireBytes {
+			return true, proto.ErrInternalBadWireType
+		}
+		msg := new(PluginSpec)
+		err := b.DecodeMessage(msg)
+		m.Runtime = &TaskSpec_Plugin{msg}
+		return true, err
 	default:
 		return false, nil
 	}
@@ -418,6 +444,11 @@ func _TaskSpec_OneofSizer(msg proto.Message) (n int) {
 		n += proto.SizeVarint(1<<3 | proto.WireBytes)
 		n += proto.SizeVarint(uint64(s))
 		n += s
+	case *TaskSpec_Plugin:
+		s := proto.Size(x.Plugin)
+		n += proto.SizeVarint(10<<3 | proto.WireBytes)
+		n += proto.SizeVarint(uint64(s))
+		n += s
 	case nil:
 	default:
 		panic(fmt.Sprintf("proto: unexpected type %T in oneof", x))
@@ -428,7 +459,7 @@ func _TaskSpec_OneofSizer(msg proto.Message) (n int) {
 // NetworkAttachmentSpec specifies runtime parameters required to attach
 // a container to a network.
 type NetworkAttachmentSpec struct {
-	// ContainerID spcifies a unique ID of the container for which
+	// ContainerID specifies a unique ID of the container for which
 	// this attachment is for.
 	ContainerID string `protobuf:"bytes,1,opt,name=container_id,json=containerId,proto3" json:"container_id,omitempty"`
 }
@@ -559,6 +590,18 @@ func (m *ContainerSpec_DNSConfig) Reset()                    { *m = ContainerSpe
 func (*ContainerSpec_DNSConfig) ProtoMessage()               {}
 func (*ContainerSpec_DNSConfig) Descriptor() ([]byte, []int) { return fileDescriptorSpecs, []int{6, 2} }
 
+// PluginSpec specifies runtime parameters for a plugin.
+type PluginSpec struct {
+	// image defines the image reference, as specified in the
+	// distribution/reference package. This may include a registry host, name,
+	// tag or digest.
+	Image string `protobuf:"bytes,1,opt,name=image,proto3" json:"image,omitempty"`
+}
+
+func (m *PluginSpec) Reset()                    { *m = PluginSpec{} }
+func (*PluginSpec) ProtoMessage()               {}
+func (*PluginSpec) Descriptor() ([]byte, []int) { return fileDescriptorSpecs, []int{7} }
+
 // EndpointSpec defines the properties that can be configured to
 // access and loadbalance the service.
 type EndpointSpec struct {
@@ -570,7 +613,7 @@ type EndpointSpec struct {
 
 func (m *EndpointSpec) Reset()                    { *m = EndpointSpec{} }
 func (*EndpointSpec) ProtoMessage()               {}
-func (*EndpointSpec) Descriptor() ([]byte, []int) { return fileDescriptorSpecs, []int{7} }
+func (*EndpointSpec) Descriptor() ([]byte, []int) { return fileDescriptorSpecs, []int{8} }
 
 // NetworkSpec specifies user defined network parameters.
 type NetworkSpec struct {
@@ -595,7 +638,7 @@ type NetworkSpec struct {
 
 func (m *NetworkSpec) Reset()                    { *m = NetworkSpec{} }
 func (*NetworkSpec) ProtoMessage()               {}
-func (*NetworkSpec) Descriptor() ([]byte, []int) { return fileDescriptorSpecs, []int{8} }
+func (*NetworkSpec) Descriptor() ([]byte, []int) { return fileDescriptorSpecs, []int{9} }
 
 // ClusterSpec specifies global cluster settings.
 type ClusterSpec struct {
@@ -620,7 +663,7 @@ type ClusterSpec struct {
 
 func (m *ClusterSpec) Reset()                    { *m = ClusterSpec{} }
 func (*ClusterSpec) ProtoMessage()               {}
-func (*ClusterSpec) Descriptor() ([]byte, []int) { return fileDescriptorSpecs, []int{9} }
+func (*ClusterSpec) Descriptor() ([]byte, []int) { return fileDescriptorSpecs, []int{10} }
 
 // SecretSpec specifies a user-provided secret.
 type SecretSpec struct {
@@ -631,7 +674,7 @@ type SecretSpec struct {
 
 func (m *SecretSpec) Reset()                    { *m = SecretSpec{} }
 func (*SecretSpec) ProtoMessage()               {}
-func (*SecretSpec) Descriptor() ([]byte, []int) { return fileDescriptorSpecs, []int{10} }
+func (*SecretSpec) Descriptor() ([]byte, []int) { return fileDescriptorSpecs, []int{11} }
 
 func init() {
 	proto.RegisterType((*NodeSpec)(nil), "docker.swarmkit.v1.NodeSpec")
@@ -643,6 +686,7 @@ func init() {
 	proto.RegisterType((*ContainerSpec)(nil), "docker.swarmkit.v1.ContainerSpec")
 	proto.RegisterType((*ContainerSpec_PullOptions)(nil), "docker.swarmkit.v1.ContainerSpec.PullOptions")
 	proto.RegisterType((*ContainerSpec_DNSConfig)(nil), "docker.swarmkit.v1.ContainerSpec.DNSConfig")
+	proto.RegisterType((*PluginSpec)(nil), "docker.swarmkit.v1.PluginSpec")
 	proto.RegisterType((*EndpointSpec)(nil), "docker.swarmkit.v1.EndpointSpec")
 	proto.RegisterType((*NetworkSpec)(nil), "docker.swarmkit.v1.NetworkSpec")
 	proto.RegisterType((*ClusterSpec)(nil), "docker.swarmkit.v1.ClusterSpec")
@@ -798,6 +842,12 @@ func (m *TaskSpec) CopyFrom(src interface{}) {
 			}
 			github_com_docker_swarmkit_api_deepcopy.Copy(v.Container, o.GetContainer())
 			m.Runtime = &v
+		case *TaskSpec_Plugin:
+			v := TaskSpec_Plugin{
+				Plugin: &PluginSpec{},
+			}
+			github_com_docker_swarmkit_api_deepcopy.Copy(v.Plugin, o.GetPlugin())
+			m.Runtime = &v
 		}
 	}
 
@@ -941,6 +991,21 @@ func (m *ContainerSpec_DNSConfig) CopyFrom(src interface{}) {
 
 }
 
+func (m *PluginSpec) Copy() *PluginSpec {
+	if m == nil {
+		return nil
+	}
+	o := &PluginSpec{}
+	o.CopyFrom(m)
+	return o
+}
+
+func (m *PluginSpec) CopyFrom(src interface{}) {
+
+	o := src.(*PluginSpec)
+	*m = *o
+}
+
 func (m *EndpointSpec) Copy() *EndpointSpec {
 	if m == nil {
 		return nil
@@ -1330,6 +1395,20 @@ func (m *TaskSpec_Attachment) MarshalTo(dAtA []byte) (int, error) {
 	}
 	return i, nil
 }
+func (m *TaskSpec_Plugin) MarshalTo(dAtA []byte) (int, error) {
+	i := 0
+	if m.Plugin != nil {
+		dAtA[i] = 0x52
+		i++
+		i = encodeVarintSpecs(dAtA, i, uint64(m.Plugin.Size()))
+		n17, err := m.Plugin.MarshalTo(dAtA[i:])
+		if err != nil {
+			return 0, err
+		}
+		i += n17
+	}
+	return i, nil
+}
 func (m *NetworkAttachmentSpec) Marshal() (dAtA []byte, err error) {
 	size := m.Size()
 	dAtA = make([]byte, size)
@@ -1465,21 +1544,21 @@ func (m *ContainerSpec) MarshalTo(dAtA []byte) (int, error) {
 		dAtA[i] = 0x4a
 		i++
 		i = encodeVarintSpecs(dAtA, i, uint64(m.StopGracePeriod.Size()))
-		n17, err := m.StopGracePeriod.MarshalTo(dAtA[i:])
+		n18, err := m.StopGracePeriod.MarshalTo(dAtA[i:])
 		if err != nil {
 			return 0, err
 		}
-		i += n17
+		i += n18
 	}
 	if m.PullOptions != nil {
 		dAtA[i] = 0x52
 		i++
 		i = encodeVarintSpecs(dAtA, i, uint64(m.PullOptions.Size()))
-		n18, err := m.PullOptions.MarshalTo(dAtA[i:])
+		n19, err := m.PullOptions.MarshalTo(dAtA[i:])
 		if err != nil {
 			return 0, err
 		}
-		i += n18
+		i += n19
 	}
 	if len(m.Groups) > 0 {
 		for _, s := range m.Groups {
@@ -1528,11 +1607,11 @@ func (m *ContainerSpec) MarshalTo(dAtA []byte) (int, error) {
 		dAtA[i] = 0x7a
 		i++
 		i = encodeVarintSpecs(dAtA, i, uint64(m.DNSConfig.Size()))
-		n19, err := m.DNSConfig.MarshalTo(dAtA[i:])
+		n20, err := m.DNSConfig.MarshalTo(dAtA[i:])
 		if err != nil {
 			return 0, err
 		}
-		i += n19
+		i += n20
 	}
 	if m.Healthcheck != nil {
 		dAtA[i] = 0x82
@@ -1540,11 +1619,11 @@ func (m *ContainerSpec) MarshalTo(dAtA []byte) (int, error) {
 		dAtA[i] = 0x1
 		i++
 		i = encodeVarintSpecs(dAtA, i, uint64(m.Healthcheck.Size()))
-		n20, err := m.Healthcheck.MarshalTo(dAtA[i:])
+		n21, err := m.Healthcheck.MarshalTo(dAtA[i:])
 		if err != nil {
 			return 0, err
 		}
-		i += n20
+		i += n21
 	}
 	if len(m.Hosts) > 0 {
 		for _, s := range m.Hosts {
@@ -1687,6 +1766,30 @@ func (m *ContainerSpec_DNSConfig) MarshalTo(dAtA []byte) (int, error) {
 	return i, nil
 }
 
+func (m *PluginSpec) Marshal() (dAtA []byte, err error) {
+	size := m.Size()
+	dAtA = make([]byte, size)
+	n, err := m.MarshalTo(dAtA)
+	if err != nil {
+		return nil, err
+	}
+	return dAtA[:n], nil
+}
+
+func (m *PluginSpec) MarshalTo(dAtA []byte) (int, error) {
+	var i int
+	_ = i
+	var l int
+	_ = l
+	if len(m.Image) > 0 {
+		dAtA[i] = 0xa
+		i++
+		i = encodeVarintSpecs(dAtA, i, uint64(len(m.Image)))
+		i += copy(dAtA[i:], m.Image)
+	}
+	return i, nil
+}
+
 func (m *EndpointSpec) Marshal() (dAtA []byte, err error) {
 	size := m.Size()
 	dAtA = make([]byte, size)
@@ -1740,20 +1843,20 @@ func (m *NetworkSpec) MarshalTo(dAtA []byte) (int, error) {
 	dAtA[i] = 0xa
 	i++
 	i = encodeVarintSpecs(dAtA, i, uint64(m.Annotations.Size()))
-	n21, err := m.Annotations.MarshalTo(dAtA[i:])
+	n22, err := m.Annotations.MarshalTo(dAtA[i:])
 	if err != nil {
 		return 0, err
 	}
-	i += n21
+	i += n22
 	if m.DriverConfig != nil {
 		dAtA[i] = 0x12
 		i++
 		i = encodeVarintSpecs(dAtA, i, uint64(m.DriverConfig.Size()))
-		n22, err := m.DriverConfig.MarshalTo(dAtA[i:])
+		n23, err := m.DriverConfig.MarshalTo(dAtA[i:])
 		if err != nil {
 			return 0, err
 		}
-		i += n22
+		i += n23
 	}
 	if m.Ipv6Enabled {
 		dAtA[i] = 0x18
@@ -1779,11 +1882,11 @@ func (m *NetworkSpec) MarshalTo(dAtA []byte) (int, error) {
 		dAtA[i] = 0x2a
 		i++
 		i = encodeVarintSpecs(dAtA, i, uint64(m.IPAM.Size()))
-		n23, err := m.IPAM.MarshalTo(dAtA[i:])
+		n24, err := m.IPAM.MarshalTo(dAtA[i:])
 		if err != nil {
 			return 0, err
 		}
-		i += n23
+		i += n24
 	}
 	if m.Attachable {
 		dAtA[i] = 0x30
@@ -1816,67 +1919,67 @@ func (m *ClusterSpec) MarshalTo(dAtA []byte) (int, error) {
 	dAtA[i] = 0xa
 	i++
 	i = encodeVarintSpecs(dAtA, i, uint64(m.Annotations.Size()))
-	n24, err := m.Annotations.MarshalTo(dAtA[i:])
-	if err != nil {
-		return 0, err
-	}
-	i += n24
-	dAtA[i] = 0x12
-	i++
-	i = encodeVarintSpecs(dAtA, i, uint64(m.AcceptancePolicy.Size()))
-	n25, err := m.AcceptancePolicy.MarshalTo(dAtA[i:])
+	n25, err := m.Annotations.MarshalTo(dAtA[i:])
 	if err != nil {
 		return 0, err
 	}
 	i += n25
-	dAtA[i] = 0x1a
+	dAtA[i] = 0x12
 	i++
-	i = encodeVarintSpecs(dAtA, i, uint64(m.Orchestration.Size()))
-	n26, err := m.Orchestration.MarshalTo(dAtA[i:])
+	i = encodeVarintSpecs(dAtA, i, uint64(m.AcceptancePolicy.Size()))
+	n26, err := m.AcceptancePolicy.MarshalTo(dAtA[i:])
 	if err != nil {
 		return 0, err
 	}
 	i += n26
-	dAtA[i] = 0x22
+	dAtA[i] = 0x1a
 	i++
-	i = encodeVarintSpecs(dAtA, i, uint64(m.Raft.Size()))
-	n27, err := m.Raft.MarshalTo(dAtA[i:])
+	i = encodeVarintSpecs(dAtA, i, uint64(m.Orchestration.Size()))
+	n27, err := m.Orchestration.MarshalTo(dAtA[i:])
 	if err != nil {
 		return 0, err
 	}
 	i += n27
-	dAtA[i] = 0x2a
+	dAtA[i] = 0x22
 	i++
-	i = encodeVarintSpecs(dAtA, i, uint64(m.Dispatcher.Size()))
-	n28, err := m.Dispatcher.MarshalTo(dAtA[i:])
+	i = encodeVarintSpecs(dAtA, i, uint64(m.Raft.Size()))
+	n28, err := m.Raft.MarshalTo(dAtA[i:])
 	if err != nil {
 		return 0, err
 	}
 	i += n28
-	dAtA[i] = 0x32
+	dAtA[i] = 0x2a
 	i++
-	i = encodeVarintSpecs(dAtA, i, uint64(m.CAConfig.Size()))
-	n29, err := m.CAConfig.MarshalTo(dAtA[i:])
+	i = encodeVarintSpecs(dAtA, i, uint64(m.Dispatcher.Size()))
+	n29, err := m.Dispatcher.MarshalTo(dAtA[i:])
 	if err != nil {
 		return 0, err
 	}
 	i += n29
-	dAtA[i] = 0x3a
+	dAtA[i] = 0x32
 	i++
-	i = encodeVarintSpecs(dAtA, i, uint64(m.TaskDefaults.Size()))
-	n30, err := m.TaskDefaults.MarshalTo(dAtA[i:])
+	i = encodeVarintSpecs(dAtA, i, uint64(m.CAConfig.Size()))
+	n30, err := m.CAConfig.MarshalTo(dAtA[i:])
 	if err != nil {
 		return 0, err
 	}
 	i += n30
-	dAtA[i] = 0x42
+	dAtA[i] = 0x3a
 	i++
-	i = encodeVarintSpecs(dAtA, i, uint64(m.EncryptionConfig.Size()))
-	n31, err := m.EncryptionConfig.MarshalTo(dAtA[i:])
+	i = encodeVarintSpecs(dAtA, i, uint64(m.TaskDefaults.Size()))
+	n31, err := m.TaskDefaults.MarshalTo(dAtA[i:])
 	if err != nil {
 		return 0, err
 	}
 	i += n31
+	dAtA[i] = 0x42
+	i++
+	i = encodeVarintSpecs(dAtA, i, uint64(m.EncryptionConfig.Size()))
+	n32, err := m.EncryptionConfig.MarshalTo(dAtA[i:])
+	if err != nil {
+		return 0, err
+	}
+	i += n32
 	return i, nil
 }
 
@@ -1898,11 +2001,11 @@ func (m *SecretSpec) MarshalTo(dAtA []byte) (int, error) {
 	dAtA[i] = 0xa
 	i++
 	i = encodeVarintSpecs(dAtA, i, uint64(m.Annotations.Size()))
-	n32, err := m.Annotations.MarshalTo(dAtA[i:])
+	n33, err := m.Annotations.MarshalTo(dAtA[i:])
 	if err != nil {
 		return 0, err
 	}
-	i += n32
+	i += n33
 	if len(m.Data) > 0 {
 		dAtA[i] = 0x12
 		i++
@@ -2073,6 +2176,15 @@ func (m *TaskSpec_Attachment) Size() (n int) {
 	}
 	return n
 }
+func (m *TaskSpec_Plugin) Size() (n int) {
+	var l int
+	_ = l
+	if m.Plugin != nil {
+		l = m.Plugin.Size()
+		n += 1 + l + sovSpecs(uint64(l))
+	}
+	return n
+}
 func (m *NetworkAttachmentSpec) Size() (n int) {
 	var l int
 	_ = l
@@ -2218,6 +2330,16 @@ func (m *ContainerSpec_DNSConfig) Size() (n int) {
 	return n
 }
 
+func (m *PluginSpec) Size() (n int) {
+	var l int
+	_ = l
+	l = len(m.Image)
+	if l > 0 {
+		n += 1 + l + sovSpecs(uint64(l))
+	}
+	return n
+}
+
 func (m *EndpointSpec) Size() (n int) {
 	var l int
 	_ = l
@@ -2409,6 +2531,16 @@ func (this *TaskSpec_Attachment) String() string {
 	}, "")
 	return s
 }
+func (this *TaskSpec_Plugin) String() string {
+	if this == nil {
+		return "nil"
+	}
+	s := strings.Join([]string{`&TaskSpec_Plugin{`,
+		`Plugin:` + strings.Replace(fmt.Sprintf("%v", this.Plugin), "PluginSpec", "PluginSpec", 1) + `,`,
+		`}`,
+	}, "")
+	return s
+}
 func (this *NetworkAttachmentSpec) String() string {
 	if this == nil {
 		return "nil"
@@ -2480,6 +2612,16 @@ func (this *ContainerSpec_DNSConfig) String() string {
 	}, "")
 	return s
 }
+func (this *PluginSpec) String() string {
+	if this == nil {
+		return "nil"
+	}
+	s := strings.Join([]string{`&PluginSpec{`,
+		`Image:` + fmt.Sprintf("%v", this.Image) + `,`,
+		`}`,
+	}, "")
+	return s
+}
 func (this *EndpointSpec) String() string {
 	if this == nil {
 		return "nil"
@@ -3377,6 +3519,38 @@ func (m *TaskSpec) Unmarshal(dAtA []byte) error {
 					break
 				}
 			}
+		case 10:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Plugin", wireType)
+			}
+			var msglen int
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowSpecs
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				msglen |= (int(b) & 0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			if msglen < 0 {
+				return ErrInvalidLengthSpecs
+			}
+			postIndex := iNdEx + msglen
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			v := &PluginSpec{}
+			if err := v.Unmarshal(dAtA[iNdEx:postIndex]); err != nil {
+				return err
+			}
+			m.Runtime = &TaskSpec_Plugin{v}
+			iNdEx = postIndex
 		default:
 			iNdEx = preIndex
 			skippy, err := skipSpecs(dAtA[iNdEx:])
@@ -4403,6 +4577,85 @@ func (m *ContainerSpec_DNSConfig) Unmarshal(dAtA []byte) error {
 	}
 	return nil
 }
+func (m *PluginSpec) Unmarshal(dAtA []byte) error {
+	l := len(dAtA)
+	iNdEx := 0
+	for iNdEx < l {
+		preIndex := iNdEx
+		var wire uint64
+		for shift := uint(0); ; shift += 7 {
+			if shift >= 64 {
+				return ErrIntOverflowSpecs
+			}
+			if iNdEx >= l {
+				return io.ErrUnexpectedEOF
+			}
+			b := dAtA[iNdEx]
+			iNdEx++
+			wire |= (uint64(b) & 0x7F) << shift
+			if b < 0x80 {
+				break
+			}
+		}
+		fieldNum := int32(wire >> 3)
+		wireType := int(wire & 0x7)
+		if wireType == 4 {
+			return fmt.Errorf("proto: PluginSpec: wiretype end group for non-group")
+		}
+		if fieldNum <= 0 {
+			return fmt.Errorf("proto: PluginSpec: illegal tag %d (wire type %d)", fieldNum, wire)
+		}
+		switch fieldNum {
+		case 1:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Image", wireType)
+			}
+			var stringLen uint64
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowSpecs
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				stringLen |= (uint64(b) & 0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			intStringLen := int(stringLen)
+			if intStringLen < 0 {
+				return ErrInvalidLengthSpecs
+			}
+			postIndex := iNdEx + intStringLen
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			m.Image = string(dAtA[iNdEx:postIndex])
+			iNdEx = postIndex
+		default:
+			iNdEx = preIndex
+			skippy, err := skipSpecs(dAtA[iNdEx:])
+			if err != nil {
+				return err
+			}
+			if skippy < 0 {
+				return ErrInvalidLengthSpecs
+			}
+			if (iNdEx + skippy) > l {
+				return io.ErrUnexpectedEOF
+			}
+			iNdEx += skippy
+		}
+	}
+
+	if iNdEx > l {
+		return io.ErrUnexpectedEOF
+	}
+	return nil
+}
 func (m *EndpointSpec) Unmarshal(dAtA []byte) error {
 	l := len(dAtA)
 	iNdEx := 0
@@ -5218,112 +5471,114 @@ var (
 func init() { proto.RegisterFile("specs.proto", fileDescriptorSpecs) }
 
 var fileDescriptorSpecs = []byte{
-	// 1707 bytes of a gzipped FileDescriptorProto
+	// 1730 bytes of a gzipped FileDescriptorProto
 	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xac, 0x57, 0x41, 0x73, 0x1b, 0xb7,
-	0x15, 0x26, 0x25, 0x8a, 0x5a, 0xbe, 0xa5, 0x6c, 0x1a, 0x75, 0xd2, 0x35, 0xdd, 0x90, 0x34, 0xe3,
+	0x15, 0x26, 0x25, 0x8a, 0x5a, 0xbe, 0xa5, 0x6c, 0x0a, 0x4d, 0xd2, 0x35, 0xdd, 0x90, 0x34, 0xe3,
 	0xa6, 0x4a, 0x33, 0xa5, 0xa6, 0x6a, 0x27, 0x75, 0xea, 0x66, 0x5a, 0x52, 0x64, 0x65, 0x55, 0x95,
 	0xcc, 0x01, 0x15, 0x77, 0x7c, 0xe2, 0x80, 0xbb, 0x10, 0xb9, 0xa3, 0xe5, 0x62, 0x0b, 0x60, 0x99,
-	0xe1, 0xad, 0xc7, 0x8c, 0x0f, 0x3d, 0xf5, 0xaa, 0xe9, 0xa1, 0xbf, 0xa1, 0xff, 0xc1, 0xc7, 0x1e,
-	0x7b, 0xd2, 0x34, 0xfc, 0x0b, 0xfd, 0x01, 0xed, 0x00, 0x0b, 0x92, 0xcb, 0x64, 0x15, 0x7b, 0x26,
-	0xbe, 0xe1, 0xbd, 0xfd, 0xbe, 0x07, 0xe0, 0xe1, 0xc3, 0xc3, 0x5b, 0xb0, 0x45, 0x44, 0x5d, 0xd1,
-	0x8a, 0x38, 0x93, 0x0c, 0x21, 0x8f, 0xb9, 0x57, 0x94, 0xb7, 0xc4, 0x97, 0x84, 0x4f, 0xaf, 0x7c,
-	0xd9, 0x9a, 0xfd, 0xbc, 0x6a, 0xcb, 0x79, 0x44, 0x0d, 0xa0, 0x7a, 0x7f, 0xcc, 0xc6, 0x4c, 0x0f,
-	0x0f, 0xd4, 0xc8, 0x78, 0x6b, 0x63, 0xc6, 0xc6, 0x01, 0x3d, 0xd0, 0xd6, 0x28, 0xbe, 0x3c, 0xf0,
-	0x62, 0x4e, 0xa4, 0xcf, 0xc2, 0xe4, 0x7b, 0xf3, 0xba, 0x00, 0xd6, 0x39, 0xf3, 0xe8, 0x20, 0xa2,
-	0x2e, 0x3a, 0x06, 0x9b, 0x84, 0x21, 0x93, 0x1a, 0x20, 0x9c, 0x7c, 0x23, 0xbf, 0x6f, 0x1f, 0xd6,
-	0x5b, 0xdf, 0x9e, 0xb9, 0xd5, 0x5e, 0xc3, 0x3a, 0x85, 0xd7, 0x37, 0xf5, 0x1c, 0x4e, 0x33, 0xd1,
-	0x6f, 0xa1, 0xec, 0x51, 0xe1, 0x73, 0xea, 0x0d, 0x39, 0x0b, 0xa8, 0xb3, 0xd5, 0xc8, 0xef, 0xdf,
-	0x39, 0xfc, 0x51, 0x56, 0x24, 0x35, 0x39, 0x66, 0x01, 0xc5, 0xb6, 0x61, 0x28, 0x03, 0x1d, 0x03,
-	0x4c, 0xe9, 0x74, 0x44, 0xb9, 0x98, 0xf8, 0x91, 0xb3, 0xad, 0xe9, 0x3f, 0xb9, 0x8d, 0xae, 0xd6,
-	0xde, 0x3a, 0x5b, 0xc1, 0x71, 0x8a, 0x8a, 0xce, 0xa0, 0x4c, 0x66, 0xc4, 0x0f, 0xc8, 0xc8, 0x0f,
-	0x7c, 0x39, 0x77, 0x0a, 0x3a, 0xd4, 0xc7, 0xdf, 0x19, 0xaa, 0x9d, 0x22, 0xe0, 0x0d, 0x7a, 0xd3,
-	0x03, 0x58, 0x4f, 0x84, 0x3e, 0x82, 0xdd, 0x7e, 0xef, 0xbc, 0x7b, 0x72, 0x7e, 0x5c, 0xc9, 0x55,
-	0x1f, 0xbc, 0xba, 0x6e, 0xbc, 0xa7, 0x62, 0xac, 0x01, 0x7d, 0x1a, 0x7a, 0x7e, 0x38, 0x46, 0xfb,
-	0x60, 0xb5, 0x8f, 0x8e, 0x7a, 0xfd, 0x8b, 0x5e, 0xb7, 0x92, 0xaf, 0x56, 0x5f, 0x5d, 0x37, 0xde,
-	0xdf, 0x04, 0xb6, 0x5d, 0x97, 0x46, 0x92, 0x7a, 0xd5, 0xc2, 0x57, 0xff, 0xa8, 0xe5, 0x9a, 0x5f,
-	0xe5, 0xa1, 0x9c, 0x5e, 0x04, 0xfa, 0x08, 0x8a, 0xed, 0xa3, 0x8b, 0x93, 0x17, 0xbd, 0x4a, 0x6e,
-	0x4d, 0x4f, 0x23, 0xda, 0xae, 0xf4, 0x67, 0x14, 0x3d, 0x86, 0x9d, 0x7e, 0xfb, 0x8b, 0x41, 0xaf,
-	0x92, 0x5f, 0x2f, 0x27, 0x0d, 0xeb, 0x93, 0x58, 0x68, 0x54, 0x17, 0xb7, 0x4f, 0xce, 0x2b, 0x5b,
-	0xd9, 0xa8, 0x2e, 0x27, 0x7e, 0x68, 0x96, 0xf2, 0xf7, 0x02, 0xd8, 0x03, 0xca, 0x67, 0xbe, 0xfb,
-	0x8e, 0x25, 0xf2, 0x29, 0x14, 0x24, 0x11, 0x57, 0x5a, 0x1a, 0x76, 0xb6, 0x34, 0x2e, 0x88, 0xb8,
-	0x52, 0x93, 0x1a, 0xba, 0xc6, 0x2b, 0x65, 0x70, 0x1a, 0x05, 0xbe, 0x4b, 0x24, 0xf5, 0xb4, 0x32,
-	0xec, 0xc3, 0x1f, 0x67, 0xb1, 0xf1, 0x0a, 0x65, 0xd6, 0xff, 0x2c, 0x87, 0x53, 0x54, 0xf4, 0x14,
-	0x8a, 0xe3, 0x80, 0x8d, 0x48, 0xa0, 0x35, 0x61, 0x1f, 0x3e, 0xca, 0x0a, 0x72, 0xac, 0x11, 0xeb,
-	0x00, 0x86, 0x82, 0x9e, 0x40, 0x31, 0x8e, 0x3c, 0x22, 0xa9, 0x53, 0xd4, 0xe4, 0x46, 0x16, 0xf9,
-	0x0b, 0x8d, 0x38, 0x62, 0xe1, 0xa5, 0x3f, 0xc6, 0x06, 0x8f, 0x4e, 0xc1, 0x0a, 0xa9, 0xfc, 0x92,
-	0xf1, 0x2b, 0xe1, 0xec, 0x36, 0xb6, 0xf7, 0xed, 0xc3, 0x4f, 0x32, 0xc5, 0x98, 0x60, 0xda, 0x52,
-	0x12, 0x77, 0x32, 0xa5, 0xa1, 0x4c, 0xc2, 0x74, 0xb6, 0x9c, 0x3c, 0x5e, 0x05, 0x40, 0xbf, 0x01,
-	0x8b, 0x86, 0x5e, 0xc4, 0xfc, 0x50, 0x3a, 0xd6, 0xed, 0x0b, 0xe9, 0x19, 0x8c, 0x4a, 0x26, 0x5e,
-	0x31, 0x14, 0x9b, 0xb3, 0x20, 0x18, 0x11, 0xf7, 0xca, 0x29, 0xbd, 0xe5, 0x36, 0x56, 0x8c, 0x4e,
-	0x11, 0x0a, 0x53, 0xe6, 0xd1, 0xe6, 0x01, 0xdc, 0xfb, 0x56, 0xaa, 0x51, 0x15, 0x2c, 0x93, 0xea,
-	0x44, 0x23, 0x05, 0xbc, 0xb2, 0x9b, 0x77, 0x61, 0x6f, 0x23, 0xad, 0xcd, 0xbf, 0x16, 0xc0, 0x5a,
-	0x9e, 0x35, 0x6a, 0x43, 0xc9, 0x65, 0xa1, 0x24, 0x7e, 0x48, 0xb9, 0x91, 0x57, 0xe6, 0xc9, 0x1c,
-	0x2d, 0x41, 0x8a, 0xf5, 0x2c, 0x87, 0xd7, 0x2c, 0xf4, 0x7b, 0x28, 0x71, 0x2a, 0x58, 0xcc, 0x5d,
-	0x2a, 0x8c, 0xbe, 0xf6, 0xb3, 0x15, 0x92, 0x80, 0x30, 0xfd, 0x73, 0xec, 0x73, 0xaa, 0xb2, 0x2c,
-	0xf0, 0x9a, 0x8a, 0x9e, 0xc2, 0x2e, 0xa7, 0x42, 0x12, 0x2e, 0xbf, 0x4b, 0x22, 0x38, 0x81, 0xf4,
-	0x59, 0xe0, 0xbb, 0x73, 0xbc, 0x64, 0xa0, 0xa7, 0x50, 0x8a, 0x02, 0xe2, 0xea, 0xa8, 0xce, 0x8e,
-	0xa6, 0x7f, 0x90, 0x45, 0xef, 0x2f, 0x41, 0x78, 0x8d, 0x47, 0x9f, 0x01, 0x04, 0x6c, 0x3c, 0xf4,
-	0xb8, 0x3f, 0xa3, 0xdc, 0x48, 0xac, 0x9a, 0xc5, 0xee, 0x6a, 0x04, 0x2e, 0x05, 0x6c, 0x9c, 0x0c,
-	0xd1, 0xf1, 0xf7, 0xd2, 0x57, 0x4a, 0x5b, 0xa7, 0x00, 0x64, 0xf5, 0xd5, 0xa8, 0xeb, 0xe3, 0xb7,
-	0x0a, 0x65, 0x4e, 0x24, 0x45, 0x47, 0x8f, 0xa0, 0x7c, 0xc9, 0xb8, 0x4b, 0x87, 0xe6, 0xd6, 0x94,
-	0xb4, 0x26, 0x6c, 0xed, 0x4b, 0xf4, 0xd5, 0x29, 0xc1, 0x2e, 0x8f, 0x43, 0xe9, 0x4f, 0x69, 0xf3,
-	0x14, 0xde, 0xcb, 0x0c, 0x8a, 0x0e, 0xa1, 0xbc, 0x3a, 0xe6, 0xa1, 0xef, 0x69, 0x7d, 0x94, 0x3a,
-	0x77, 0x17, 0x37, 0x75, 0x7b, 0xa5, 0x87, 0x93, 0x2e, 0xb6, 0x57, 0xa0, 0x13, 0xaf, 0xf9, 0x37,
-	0x0b, 0xf6, 0x36, 0xc4, 0x82, 0xee, 0xc3, 0x8e, 0x3f, 0x25, 0x63, 0x9a, 0xd0, 0x71, 0x62, 0xa0,
-	0x1e, 0x14, 0x03, 0x32, 0xa2, 0x81, 0x92, 0x8c, 0x4a, 0xdb, 0xcf, 0xde, 0xa8, 0xba, 0xd6, 0x1f,
-	0x35, 0xbe, 0x17, 0x4a, 0x3e, 0xc7, 0x86, 0x8c, 0x1c, 0xd8, 0x75, 0xd9, 0x74, 0x4a, 0x42, 0x55,
-	0x9c, 0xb6, 0xf7, 0x4b, 0x78, 0x69, 0x22, 0x04, 0x05, 0xc2, 0xc7, 0xc2, 0x29, 0x68, 0xb7, 0x1e,
-	0xa3, 0x0a, 0x6c, 0xd3, 0x70, 0xe6, 0xec, 0x68, 0x97, 0x1a, 0x2a, 0x8f, 0xe7, 0x27, 0x67, 0x5e,
-	0xc2, 0x6a, 0xa8, 0x78, 0xb1, 0xa0, 0xdc, 0xd9, 0xd5, 0x2e, 0x3d, 0x46, 0xbf, 0x82, 0xe2, 0x94,
-	0xc5, 0xa1, 0x14, 0x8e, 0xa5, 0x17, 0xfb, 0x20, 0x6b, 0xb1, 0x67, 0x0a, 0x61, 0x8a, 0xa7, 0x81,
-	0xa3, 0x1e, 0xdc, 0x13, 0x92, 0x45, 0xc3, 0x31, 0x27, 0x2e, 0x1d, 0x46, 0x94, 0xfb, 0xcc, 0x33,
-	0x97, 0xff, 0x41, 0x2b, 0xe9, 0x15, 0x5a, 0xcb, 0x5e, 0xa1, 0xd5, 0x35, 0xbd, 0x02, 0xbe, 0xab,
-	0x38, 0xc7, 0x8a, 0xd2, 0xd7, 0x0c, 0xd4, 0x87, 0x72, 0x14, 0x07, 0xc1, 0x90, 0x45, 0xc9, 0x3b,
-	0x00, 0x3a, 0xc2, 0x5b, 0xa4, 0xac, 0x1f, 0x07, 0xc1, 0xf3, 0x84, 0x84, 0xed, 0x68, 0x6d, 0xa0,
-	0xf7, 0xa1, 0x38, 0xe6, 0x2c, 0x8e, 0x84, 0x63, 0xeb, 0x64, 0x18, 0x0b, 0x7d, 0x0e, 0xbb, 0x82,
-	0xba, 0x9c, 0x4a, 0xe1, 0x94, 0xf5, 0x56, 0x3f, 0xcc, 0x9a, 0x64, 0xa0, 0x21, 0x98, 0x5e, 0x52,
-	0x4e, 0x43, 0x97, 0xe2, 0x25, 0x07, 0x3d, 0x80, 0x6d, 0x29, 0xe7, 0xce, 0x5e, 0x23, 0xbf, 0x6f,
-	0x75, 0x76, 0x17, 0x37, 0xf5, 0xed, 0x8b, 0x8b, 0x97, 0x58, 0xf9, 0x54, 0x8d, 0x9a, 0x30, 0x21,
-	0x43, 0x32, 0xa5, 0xce, 0x1d, 0x9d, 0xdb, 0x95, 0x8d, 0x5e, 0x02, 0x78, 0xa1, 0x18, 0xba, 0xfa,
-	0x52, 0x38, 0x77, 0xf5, 0xee, 0x3e, 0x79, 0xf3, 0xee, 0xba, 0xe7, 0x03, 0x53, 0xa7, 0xf7, 0x16,
-	0x37, 0xf5, 0xd2, 0xca, 0xc4, 0x25, 0x2f, 0x14, 0xc9, 0x10, 0x75, 0xc0, 0x9e, 0x50, 0x12, 0xc8,
-	0x89, 0x3b, 0xa1, 0xee, 0x95, 0x53, 0xb9, 0xbd, 0xf0, 0x3e, 0xd3, 0x30, 0x13, 0x21, 0x4d, 0x52,
-	0x0a, 0x56, 0x4b, 0x15, 0xce, 0x3d, 0x9d, 0xab, 0xc4, 0x40, 0x1f, 0x00, 0xb0, 0x88, 0x86, 0x43,
-	0x21, 0x3d, 0x3f, 0x74, 0x90, 0xda, 0x32, 0x2e, 0x29, 0xcf, 0x40, 0x39, 0xd0, 0x43, 0x55, 0x16,
-	0x89, 0x37, 0x64, 0x61, 0x30, 0x77, 0x7e, 0xa0, 0xbf, 0x5a, 0xca, 0xf1, 0x3c, 0x0c, 0xe6, 0xa8,
-	0x0e, 0xb6, 0xd6, 0x85, 0xf0, 0xc7, 0x21, 0x09, 0x9c, 0xfb, 0x3a, 0x1f, 0xa0, 0x5c, 0x03, 0xed,
-	0xa9, 0x7e, 0x06, 0x76, 0x4a, 0xee, 0x4a, 0xa6, 0x57, 0x74, 0x6e, 0x6e, 0x90, 0x1a, 0xaa, 0x35,
-	0xcd, 0x48, 0x10, 0x27, 0xcd, 0x5e, 0x09, 0x27, 0xc6, 0xaf, 0xb7, 0x9e, 0xe4, 0xab, 0x87, 0x60,
-	0xa7, 0x8e, 0x1d, 0x7d, 0x08, 0x7b, 0x9c, 0x8e, 0x7d, 0x21, 0xf9, 0x7c, 0x48, 0x62, 0x39, 0x71,
-	0x7e, 0xa7, 0x09, 0xe5, 0xa5, 0xb3, 0x1d, 0xcb, 0x49, 0x75, 0x08, 0xeb, 0xec, 0xa1, 0x06, 0xd8,
-	0xea, 0x54, 0x04, 0xe5, 0x33, 0xca, 0xd5, 0x83, 0xa2, 0x36, 0x9d, 0x76, 0x29, 0xf5, 0x08, 0x4a,
-	0xb8, 0x3b, 0xd1, 0x97, 0xb7, 0x84, 0x8d, 0xa5, 0x6e, 0xe3, 0x52, 0xa2, 0xe6, 0x36, 0x1a, 0xb3,
-	0xf9, 0xdf, 0x3c, 0x94, 0xd3, 0xef, 0x22, 0x3a, 0x4a, 0xde, 0x33, 0xbd, 0xa5, 0x3b, 0x87, 0x07,
-	0x6f, 0x7a, 0x47, 0xf5, 0xeb, 0x11, 0xc4, 0x2a, 0xd8, 0x99, 0x6a, 0x61, 0x35, 0x19, 0xfd, 0x12,
-	0x76, 0x22, 0xc6, 0xe5, 0xb2, 0x86, 0xd4, 0x32, 0x2b, 0x3e, 0xe3, 0xcb, 0x6a, 0x9b, 0x80, 0x9b,
-	0x13, 0xb8, 0xb3, 0x19, 0x0d, 0x3d, 0x86, 0xed, 0x17, 0x27, 0xfd, 0x4a, 0xae, 0xfa, 0xf0, 0xd5,
-	0x75, 0xe3, 0x87, 0x9b, 0x1f, 0x5f, 0xf8, 0x5c, 0xc6, 0x24, 0x38, 0xe9, 0xa3, 0x9f, 0xc2, 0x4e,
-	0xf7, 0x7c, 0x80, 0x71, 0x25, 0x5f, 0xad, 0xbf, 0xba, 0x6e, 0x3c, 0xdc, 0xc4, 0xa9, 0x4f, 0x2c,
-	0x0e, 0x3d, 0xcc, 0x46, 0xab, 0x76, 0xee, 0x9f, 0x5b, 0x60, 0x9b, 0xd2, 0xfa, 0xae, 0x3b, 0xfe,
-	0xbd, 0xe4, 0xb5, 0x5a, 0xde, 0x99, 0xad, 0x37, 0x3e, 0x5a, 0xe5, 0x84, 0x60, 0xce, 0xf8, 0x11,
-	0x94, 0xfd, 0x68, 0xf6, 0xe9, 0x90, 0x86, 0x64, 0x14, 0x98, 0xce, 0xce, 0xc2, 0xb6, 0xf2, 0xf5,
-	0x12, 0x97, 0xba, 0xb0, 0x7e, 0x28, 0x29, 0x0f, 0x4d, 0xcf, 0x66, 0xe1, 0x95, 0x8d, 0x3e, 0x87,
-	0x82, 0x1f, 0x91, 0xa9, 0x79, 0x69, 0x33, 0x77, 0x70, 0xd2, 0x6f, 0x9f, 0x19, 0x0d, 0x76, 0xac,
-	0xc5, 0x4d, 0xbd, 0xa0, 0x1c, 0x58, 0xd3, 0x50, 0x6d, 0xf9, 0xd8, 0xa9, 0x99, 0x74, 0xf1, 0xb5,
-	0x70, 0xca, 0xd3, 0xfc, 0x5f, 0x01, 0xec, 0xa3, 0x20, 0x16, 0xd2, 0x3c, 0x21, 0xef, 0x2c, 0x6f,
-	0x2f, 0xe1, 0x1e, 0xd1, 0xcd, 0x3f, 0x09, 0x55, 0x3d, 0xd6, 0x4d, 0x84, 0xc9, 0xdd, 0xe3, 0xcc,
-	0x70, 0x2b, 0x70, 0xd2, 0x70, 0x74, 0x8a, 0x2a, 0xa6, 0x93, 0xc7, 0x15, 0xf2, 0x8d, 0x2f, 0x68,
-	0x00, 0x7b, 0x8c, 0xbb, 0x13, 0x2a, 0x64, 0x52, 0xc5, 0x4d, 0xb3, 0x9c, 0xf9, 0x1b, 0xf5, 0x3c,
-	0x0d, 0x34, 0x25, 0x2c, 0x59, 0xed, 0x66, 0x0c, 0xf4, 0x04, 0x0a, 0x9c, 0x5c, 0x2e, 0x1b, 0xa2,
-	0x4c, 0x7d, 0x63, 0x72, 0x29, 0x37, 0x42, 0x68, 0x06, 0xfa, 0x03, 0x80, 0xe7, 0x8b, 0x88, 0x48,
-	0x77, 0x42, 0xb9, 0x39, 0xa7, 0xcc, 0x2d, 0x76, 0x57, 0xa8, 0x8d, 0x28, 0x29, 0x36, 0x3a, 0x85,
-	0x92, 0x4b, 0x96, 0x4a, 0x2b, 0xde, 0xfe, 0x07, 0x71, 0xd4, 0x36, 0x21, 0x2a, 0x2a, 0xc4, 0xe2,
-	0xa6, 0x6e, 0x2d, 0x3d, 0xd8, 0x72, 0x89, 0x51, 0xde, 0x29, 0xec, 0xa9, 0x3f, 0x8b, 0xa1, 0x47,
-	0x2f, 0x49, 0x1c, 0x48, 0xa1, 0x1f, 0xda, 0x5b, 0x4a, 0xb2, 0x6a, 0x53, 0xbb, 0x06, 0x67, 0xd6,
-	0x55, 0x96, 0x29, 0x1f, 0xfa, 0x13, 0xdc, 0xa3, 0xa1, 0xcb, 0xe7, 0x5a, 0x67, 0xcb, 0x15, 0x5a,
-	0xb7, 0x6f, 0xb6, 0xb7, 0x02, 0x6f, 0x6c, 0xb6, 0x42, 0xbf, 0xe1, 0x6f, 0xfa, 0x00, 0xc9, 0x23,
-	0xf7, 0x6e, 0xf5, 0x87, 0xa0, 0xe0, 0x11, 0x49, 0xb4, 0xe4, 0xca, 0x58, 0x8f, 0x3b, 0xce, 0xeb,
-	0xaf, 0x6b, 0xb9, 0x7f, 0x7f, 0x5d, 0xcb, 0xfd, 0x65, 0x51, 0xcb, 0xbf, 0x5e, 0xd4, 0xf2, 0xff,
-	0x5a, 0xd4, 0xf2, 0xff, 0x59, 0xd4, 0xf2, 0xa3, 0xa2, 0x6e, 0x0d, 0x7e, 0xf1, 0xff, 0x00, 0x00,
-	0x00, 0xff, 0xff, 0xed, 0xbe, 0x26, 0xe6, 0x9a, 0x10, 0x00, 0x00,
+	0xe1, 0xad, 0xc7, 0x8c, 0xcf, 0xbd, 0x6a, 0x7a, 0xe8, 0x6f, 0xe8, 0x7f, 0xf0, 0xb1, 0xc7, 0x9e,
+	0x34, 0x0d, 0xff, 0x42, 0x7f, 0x40, 0x3b, 0xc0, 0x82, 0xe4, 0x32, 0x59, 0xc5, 0x9e, 0xa9, 0x6f,
+	0xc0, 0xdb, 0xef, 0x7b, 0x78, 0x78, 0xf8, 0xf0, 0xf0, 0x16, 0x6c, 0x11, 0x51, 0x57, 0xb4, 0x22,
+	0xce, 0x24, 0x43, 0xc8, 0x63, 0xee, 0x35, 0xe5, 0x2d, 0xf1, 0x15, 0xe1, 0xd3, 0x6b, 0x5f, 0xb6,
+	0x66, 0x3f, 0xaf, 0xda, 0x72, 0x1e, 0x51, 0x03, 0xa8, 0xbe, 0x37, 0x66, 0x63, 0xa6, 0x87, 0x87,
+	0x6a, 0x64, 0xac, 0xb5, 0x31, 0x63, 0xe3, 0x80, 0x1e, 0xea, 0xd9, 0x28, 0xbe, 0x3a, 0xf4, 0x62,
+	0x4e, 0xa4, 0xcf, 0xc2, 0xe4, 0x7b, 0xf3, 0xa6, 0x00, 0xd6, 0x05, 0xf3, 0xe8, 0x20, 0xa2, 0x2e,
+	0x3a, 0x01, 0x9b, 0x84, 0x21, 0x93, 0x1a, 0x20, 0x9c, 0x7c, 0x23, 0x7f, 0x60, 0x1f, 0xd5, 0x5b,
+	0xdf, 0x5d, 0xb9, 0xd5, 0x5e, 0xc3, 0x3a, 0x85, 0xd7, 0xb7, 0xf5, 0x1c, 0x4e, 0x33, 0xd1, 0x6f,
+	0xa1, 0xec, 0x51, 0xe1, 0x73, 0xea, 0x0d, 0x39, 0x0b, 0xa8, 0xb3, 0xd5, 0xc8, 0x1f, 0xdc, 0x3b,
+	0xfa, 0x51, 0x96, 0x27, 0xb5, 0x38, 0x66, 0x01, 0xc5, 0xb6, 0x61, 0xa8, 0x09, 0x3a, 0x01, 0x98,
+	0xd2, 0xe9, 0x88, 0x72, 0x31, 0xf1, 0x23, 0x67, 0x5b, 0xd3, 0x7f, 0x72, 0x17, 0x5d, 0xc5, 0xde,
+	0x3a, 0x5f, 0xc1, 0x71, 0x8a, 0x8a, 0xce, 0xa1, 0x4c, 0x66, 0xc4, 0x0f, 0xc8, 0xc8, 0x0f, 0x7c,
+	0x39, 0x77, 0x0a, 0xda, 0xd5, 0x27, 0xdf, 0xeb, 0xaa, 0x9d, 0x22, 0xe0, 0x0d, 0x7a, 0xd3, 0x03,
+	0x58, 0x2f, 0x84, 0x3e, 0x86, 0xdd, 0x7e, 0xef, 0xa2, 0x7b, 0x7a, 0x71, 0x52, 0xc9, 0x55, 0x1f,
+	0xbc, 0xba, 0x69, 0xbc, 0xaf, 0x7c, 0xac, 0x01, 0x7d, 0x1a, 0x7a, 0x7e, 0x38, 0x46, 0x07, 0x60,
+	0xb5, 0x8f, 0x8f, 0x7b, 0xfd, 0xcb, 0x5e, 0xb7, 0x92, 0xaf, 0x56, 0x5f, 0xdd, 0x34, 0x3e, 0xd8,
+	0x04, 0xb6, 0x5d, 0x97, 0x46, 0x92, 0x7a, 0xd5, 0xc2, 0xd7, 0x7f, 0xaf, 0xe5, 0x9a, 0x5f, 0xe7,
+	0xa1, 0x9c, 0x0e, 0x02, 0x7d, 0x0c, 0xc5, 0xf6, 0xf1, 0xe5, 0xe9, 0x8b, 0x5e, 0x25, 0xb7, 0xa6,
+	0xa7, 0x11, 0x6d, 0x57, 0xfa, 0x33, 0x8a, 0x1e, 0xc3, 0x4e, 0xbf, 0xfd, 0xe5, 0xa0, 0x57, 0xc9,
+	0xaf, 0xc3, 0x49, 0xc3, 0xfa, 0x24, 0x16, 0x1a, 0xd5, 0xc5, 0xed, 0xd3, 0x8b, 0xca, 0x56, 0x36,
+	0xaa, 0xcb, 0x89, 0x1f, 0x9a, 0x50, 0xfe, 0x56, 0x00, 0x7b, 0x40, 0xf9, 0xcc, 0x77, 0xdf, 0xb1,
+	0x44, 0x3e, 0x83, 0x82, 0x24, 0xe2, 0x5a, 0x4b, 0xc3, 0xce, 0x96, 0xc6, 0x25, 0x11, 0xd7, 0x6a,
+	0x51, 0x43, 0xd7, 0x78, 0xa5, 0x0c, 0x4e, 0xa3, 0xc0, 0x77, 0x89, 0xa4, 0x9e, 0x56, 0x86, 0x7d,
+	0xf4, 0xe3, 0x2c, 0x36, 0x5e, 0xa1, 0x4c, 0xfc, 0xcf, 0x72, 0x38, 0x45, 0x45, 0x4f, 0xa1, 0x38,
+	0x0e, 0xd8, 0x88, 0x04, 0x5a, 0x13, 0xf6, 0xd1, 0xa3, 0x2c, 0x27, 0x27, 0x1a, 0xb1, 0x76, 0x60,
+	0x28, 0xe8, 0x09, 0x14, 0xe3, 0xc8, 0x23, 0x92, 0x3a, 0x45, 0x4d, 0x6e, 0x64, 0x91, 0xbf, 0xd4,
+	0x88, 0x63, 0x16, 0x5e, 0xf9, 0x63, 0x6c, 0xf0, 0xe8, 0x0c, 0xac, 0x90, 0xca, 0xaf, 0x18, 0xbf,
+	0x16, 0xce, 0x6e, 0x63, 0xfb, 0xc0, 0x3e, 0xfa, 0x34, 0x53, 0x8c, 0x09, 0xa6, 0x2d, 0x25, 0x71,
+	0x27, 0x53, 0x1a, 0xca, 0xc4, 0x4d, 0x67, 0xcb, 0xc9, 0xe3, 0x95, 0x03, 0xf4, 0x1b, 0xb0, 0x68,
+	0xe8, 0x45, 0xcc, 0x0f, 0xa5, 0x63, 0xdd, 0x1d, 0x48, 0xcf, 0x60, 0x54, 0x32, 0xf1, 0x8a, 0xa1,
+	0xd8, 0x9c, 0x05, 0xc1, 0x88, 0xb8, 0xd7, 0x4e, 0xe9, 0x2d, 0xb7, 0xb1, 0x62, 0x74, 0x8a, 0x50,
+	0x98, 0x32, 0x8f, 0x36, 0x0f, 0x61, 0xff, 0x3b, 0xa9, 0x46, 0x55, 0xb0, 0x4c, 0xaa, 0x13, 0x8d,
+	0x14, 0xf0, 0x6a, 0xde, 0xbc, 0x0f, 0x7b, 0x1b, 0x69, 0x6d, 0xbe, 0x2e, 0x80, 0xb5, 0x3c, 0x6b,
+	0xd4, 0x86, 0x92, 0xcb, 0x42, 0x49, 0xfc, 0x90, 0x72, 0x23, 0xaf, 0xcc, 0x93, 0x39, 0x5e, 0x82,
+	0x14, 0xeb, 0x59, 0x0e, 0xaf, 0x59, 0xe8, 0xf7, 0x50, 0xe2, 0x54, 0xb0, 0x98, 0xbb, 0x54, 0x18,
+	0x7d, 0x1d, 0x64, 0x2b, 0x24, 0x01, 0x61, 0xfa, 0xe7, 0xd8, 0xe7, 0x54, 0x65, 0x59, 0xe0, 0x35,
+	0x15, 0x3d, 0x85, 0x5d, 0x4e, 0x85, 0x24, 0x5c, 0x7e, 0x9f, 0x44, 0x70, 0x02, 0xe9, 0xb3, 0xc0,
+	0x77, 0xe7, 0x78, 0xc9, 0x40, 0x4f, 0xa1, 0x14, 0x05, 0xc4, 0xd5, 0x5e, 0x9d, 0x1d, 0x4d, 0xff,
+	0x30, 0x8b, 0xde, 0x5f, 0x82, 0xf0, 0x1a, 0x8f, 0x3e, 0x07, 0x08, 0xd8, 0x78, 0xe8, 0x71, 0x7f,
+	0x46, 0xb9, 0x91, 0x58, 0x35, 0x8b, 0xdd, 0xd5, 0x08, 0x5c, 0x0a, 0xd8, 0x38, 0x19, 0xa2, 0x93,
+	0xff, 0x4b, 0x5f, 0x29, 0x6d, 0x9d, 0x01, 0x90, 0xd5, 0x57, 0xa3, 0xae, 0x4f, 0xde, 0xca, 0x95,
+	0x39, 0x91, 0x14, 0x1d, 0x3d, 0x82, 0xf2, 0x15, 0xe3, 0x2e, 0x1d, 0x9a, 0x5b, 0x53, 0xd2, 0x9a,
+	0xb0, 0xb5, 0x2d, 0xd1, 0x97, 0xba, 0x52, 0x51, 0x10, 0x8f, 0xfd, 0xd0, 0x01, 0xbd, 0x56, 0x2d,
+	0x3b, 0x5b, 0x0a, 0x61, 0x16, 0x30, 0xf8, 0x4e, 0x09, 0x76, 0x79, 0x1c, 0x4a, 0x7f, 0x4a, 0x9b,
+	0x67, 0xf0, 0x7e, 0x66, 0x38, 0xe8, 0x08, 0xca, 0x2b, 0x81, 0x0c, 0x7d, 0x4f, 0x2b, 0xab, 0xd4,
+	0xb9, 0xbf, 0xb8, 0xad, 0xdb, 0x2b, 0x25, 0x9d, 0x76, 0xb1, 0xbd, 0x02, 0x9d, 0x7a, 0xcd, 0xbf,
+	0x5a, 0xb0, 0xb7, 0x21, 0x33, 0xf4, 0x1e, 0xec, 0xf8, 0x53, 0x32, 0xa6, 0x09, 0x1d, 0x27, 0x13,
+	0xd4, 0x83, 0x62, 0x40, 0x46, 0x34, 0x50, 0x62, 0x53, 0x09, 0xff, 0xd9, 0x1b, 0xf5, 0xda, 0xfa,
+	0xa3, 0xc6, 0xf7, 0x42, 0xc9, 0xe7, 0xd8, 0x90, 0x91, 0x03, 0xbb, 0x2e, 0x9b, 0x4e, 0x49, 0xa8,
+	0xca, 0xda, 0xf6, 0x41, 0x09, 0x2f, 0xa7, 0x08, 0x41, 0x81, 0xf0, 0xb1, 0x70, 0x0a, 0xda, 0xac,
+	0xc7, 0xa8, 0x02, 0xdb, 0x34, 0x9c, 0x39, 0x3b, 0xda, 0xa4, 0x86, 0xca, 0xe2, 0xf9, 0x89, 0x5a,
+	0x4a, 0x58, 0x0d, 0x15, 0x2f, 0x16, 0x94, 0x3b, 0xbb, 0xda, 0xa4, 0xc7, 0xe8, 0x57, 0x50, 0x9c,
+	0xb2, 0x38, 0x94, 0xc2, 0xb1, 0x74, 0xb0, 0x0f, 0xb2, 0x82, 0x3d, 0x57, 0x08, 0x53, 0x76, 0x0d,
+	0x1c, 0xf5, 0x60, 0x5f, 0x48, 0x16, 0x0d, 0xc7, 0x9c, 0xb8, 0x74, 0x18, 0x51, 0xee, 0x33, 0xcf,
+	0x94, 0x8d, 0x07, 0xad, 0xa4, 0xcb, 0x68, 0x2d, 0xbb, 0x8c, 0x56, 0xd7, 0x74, 0x19, 0xf8, 0xbe,
+	0xe2, 0x9c, 0x28, 0x4a, 0x5f, 0x33, 0x50, 0x1f, 0xca, 0x51, 0x1c, 0x04, 0x43, 0x16, 0x25, 0x2f,
+	0x48, 0x72, 0xd8, 0x6f, 0x91, 0xb2, 0x7e, 0x1c, 0x04, 0xcf, 0x13, 0x12, 0xb6, 0xa3, 0xf5, 0x04,
+	0x7d, 0x00, 0xc5, 0x31, 0x67, 0x71, 0x24, 0x1c, 0x5b, 0x27, 0xc3, 0xcc, 0xd0, 0x17, 0xb0, 0x2b,
+	0xa8, 0xcb, 0xa9, 0x14, 0x4e, 0x59, 0x6f, 0xf5, 0xa3, 0xac, 0x45, 0x06, 0x1a, 0x82, 0xe9, 0x15,
+	0xe5, 0x34, 0x74, 0x29, 0x5e, 0x72, 0xd0, 0x03, 0xd8, 0x96, 0x72, 0xee, 0xec, 0x35, 0xf2, 0x07,
+	0x56, 0x67, 0x77, 0x71, 0x5b, 0xdf, 0xbe, 0xbc, 0x7c, 0x89, 0x95, 0x4d, 0x55, 0xb7, 0x09, 0x13,
+	0x32, 0x24, 0x53, 0xea, 0xdc, 0xd3, 0xb9, 0x5d, 0xcd, 0xd1, 0x4b, 0x00, 0x2f, 0x14, 0x43, 0x57,
+	0x5f, 0x27, 0xe7, 0xbe, 0xde, 0xdd, 0xa7, 0x6f, 0xde, 0x5d, 0xf7, 0x62, 0x60, 0x2a, 0xfc, 0xde,
+	0xe2, 0xb6, 0x5e, 0x5a, 0x4d, 0x71, 0xc9, 0x0b, 0x45, 0x32, 0x44, 0x1d, 0xb0, 0x27, 0x94, 0x04,
+	0x72, 0xe2, 0x4e, 0xa8, 0x7b, 0xed, 0x54, 0xee, 0x2e, 0xd9, 0xcf, 0x34, 0xcc, 0x78, 0x48, 0x93,
+	0x94, 0x82, 0x55, 0xa8, 0xc2, 0xd9, 0xd7, 0xb9, 0x4a, 0x26, 0xe8, 0x43, 0x00, 0x16, 0xd1, 0x70,
+	0x28, 0xa4, 0xe7, 0x87, 0x0e, 0x52, 0x5b, 0xc6, 0x25, 0x65, 0x19, 0x28, 0x03, 0x7a, 0xa8, 0x0a,
+	0x2a, 0xf1, 0x86, 0x2c, 0x0c, 0xe6, 0xce, 0x0f, 0xf4, 0x57, 0x4b, 0x19, 0x9e, 0x87, 0xc1, 0x1c,
+	0xd5, 0xc1, 0xd6, 0xba, 0x10, 0xfe, 0x38, 0x24, 0x81, 0xf3, 0x9e, 0xce, 0x07, 0x28, 0xd3, 0x40,
+	0x5b, 0xaa, 0x9f, 0x83, 0x9d, 0x92, 0xbb, 0x92, 0xe9, 0x35, 0x9d, 0x9b, 0x1b, 0xa4, 0x86, 0x2a,
+	0xa6, 0x19, 0x09, 0xe2, 0xa4, 0x4d, 0x2c, 0xe1, 0x64, 0xf2, 0xeb, 0xad, 0x27, 0xf9, 0xea, 0x11,
+	0xd8, 0xa9, 0x63, 0x47, 0x1f, 0xc1, 0x1e, 0xa7, 0x63, 0x5f, 0x48, 0x3e, 0x1f, 0x92, 0x58, 0x4e,
+	0x9c, 0xdf, 0x69, 0x42, 0x79, 0x69, 0x6c, 0xc7, 0x72, 0x52, 0x1d, 0xc2, 0x3a, 0x7b, 0xa8, 0x01,
+	0xb6, 0x3a, 0x15, 0x41, 0xf9, 0x8c, 0x72, 0xf5, 0x14, 0xa9, 0x4d, 0xa7, 0x4d, 0x4a, 0x3d, 0x82,
+	0x12, 0xee, 0x4e, 0xf4, 0xe5, 0x2d, 0x61, 0x33, 0x53, 0xb7, 0x71, 0x29, 0x51, 0x73, 0x1b, 0xcd,
+	0xb4, 0xd9, 0x04, 0x58, 0x97, 0xa1, 0xec, 0x92, 0xd0, 0xfc, 0x4f, 0x1e, 0xca, 0xe9, 0x57, 0x17,
+	0x1d, 0x27, 0xaf, 0xa5, 0x46, 0xdd, 0x3b, 0x3a, 0x7c, 0xd3, 0x2b, 0xad, 0xdf, 0xa6, 0x20, 0x56,
+	0x0b, 0x9e, 0xab, 0x06, 0x59, 0x93, 0xd1, 0x2f, 0x61, 0x27, 0x62, 0x5c, 0x2e, 0xeb, 0x4c, 0x76,
+	0x85, 0x64, 0x7c, 0x59, 0xcb, 0x13, 0x70, 0x73, 0x02, 0xf7, 0x36, 0xbd, 0xa1, 0xc7, 0xb0, 0xfd,
+	0xe2, 0xb4, 0x5f, 0xc9, 0x55, 0x1f, 0xbe, 0xba, 0x69, 0xfc, 0x70, 0xf3, 0xe3, 0x0b, 0x9f, 0xcb,
+	0x98, 0x04, 0xa7, 0x7d, 0xf4, 0x53, 0xd8, 0xe9, 0x5e, 0x0c, 0x30, 0xae, 0xe4, 0xab, 0xf5, 0x57,
+	0x37, 0x8d, 0x87, 0x9b, 0x38, 0xf5, 0x89, 0xc5, 0xa1, 0x87, 0xd9, 0x68, 0xd5, 0x2c, 0xfe, 0x63,
+	0x0b, 0x6c, 0x53, 0x7e, 0xdf, 0xf5, 0xff, 0xc4, 0x5e, 0xf2, 0x16, 0x2e, 0xef, 0xd5, 0xd6, 0x1b,
+	0x9f, 0xc4, 0x72, 0x42, 0x30, 0x3a, 0x78, 0x04, 0x65, 0x3f, 0x9a, 0x7d, 0x36, 0xa4, 0x21, 0x19,
+	0x05, 0xa6, 0x6f, 0xb4, 0xb0, 0xad, 0x6c, 0xbd, 0xc4, 0xa4, 0x2e, 0xb5, 0x1f, 0x4a, 0xca, 0x43,
+	0xd3, 0x11, 0x5a, 0x78, 0x35, 0x47, 0x5f, 0x40, 0xc1, 0x8f, 0xc8, 0xd4, 0xbc, 0xe3, 0x99, 0x3b,
+	0x38, 0xed, 0xb7, 0xcf, 0x8d, 0x4e, 0x3b, 0xd6, 0xe2, 0xb6, 0x5e, 0x50, 0x06, 0xac, 0x69, 0xa8,
+	0xb6, 0x7c, 0x4a, 0xd5, 0x4a, 0xba, 0x40, 0x5b, 0x38, 0x65, 0x69, 0xfe, 0xb7, 0x00, 0xf6, 0x71,
+	0x10, 0x0b, 0x69, 0x9e, 0x99, 0x77, 0x96, 0xb7, 0x97, 0xb0, 0x4f, 0xf4, 0xaf, 0x05, 0x09, 0x55,
+	0xcd, 0xd6, 0x2d, 0x8a, 0xc9, 0xdd, 0xe3, 0x4c, 0x77, 0x2b, 0x70, 0xd2, 0xce, 0x74, 0x8a, 0xca,
+	0xa7, 0x93, 0xc7, 0x15, 0xf2, 0xad, 0x2f, 0x68, 0x00, 0x7b, 0x8c, 0xbb, 0x13, 0x2a, 0x64, 0x52,
+	0xe9, 0x4d, 0x2b, 0x9e, 0xf9, 0x93, 0xf6, 0x3c, 0x0d, 0x34, 0x65, 0x2e, 0x89, 0x76, 0xd3, 0x07,
+	0x7a, 0x02, 0x05, 0x4e, 0xae, 0x96, 0xed, 0x56, 0xa6, 0xbe, 0x31, 0xb9, 0x92, 0x1b, 0x2e, 0x34,
+	0x03, 0xfd, 0x01, 0xc0, 0xf3, 0x45, 0x44, 0xa4, 0x3b, 0xa1, 0xdc, 0x9c, 0x53, 0xe6, 0x16, 0xbb,
+	0x2b, 0xd4, 0x86, 0x97, 0x14, 0x1b, 0x9d, 0x41, 0xc9, 0x25, 0x4b, 0xa5, 0x15, 0xef, 0xfe, 0x3f,
+	0x39, 0x6e, 0x1b, 0x17, 0x15, 0xe5, 0x62, 0x71, 0x5b, 0xb7, 0x96, 0x16, 0x6c, 0xb9, 0xc4, 0x28,
+	0xef, 0x0c, 0xf6, 0xd4, 0x7f, 0xcb, 0xd0, 0xa3, 0x57, 0x24, 0x0e, 0xa4, 0xd0, 0x8f, 0xf1, 0x1d,
+	0x65, 0x5b, 0x35, 0xc1, 0x5d, 0x83, 0x33, 0x71, 0x95, 0x65, 0xca, 0x86, 0xfe, 0x04, 0xfb, 0x34,
+	0x74, 0xf9, 0x5c, 0xeb, 0x6c, 0x19, 0xa1, 0x75, 0xf7, 0x66, 0x7b, 0x2b, 0xf0, 0xc6, 0x66, 0x2b,
+	0xf4, 0x5b, 0xf6, 0xa6, 0x0f, 0x90, 0x3c, 0x84, 0xef, 0x56, 0x7f, 0x08, 0x0a, 0x1e, 0x91, 0x44,
+	0x4b, 0xae, 0x8c, 0xf5, 0xb8, 0xe3, 0xbc, 0xfe, 0xa6, 0x96, 0xfb, 0xd7, 0x37, 0xb5, 0xdc, 0x5f,
+	0x16, 0xb5, 0xfc, 0xeb, 0x45, 0x2d, 0xff, 0xcf, 0x45, 0x2d, 0xff, 0xef, 0x45, 0x2d, 0x3f, 0x2a,
+	0xea, 0xf6, 0xe1, 0x17, 0xff, 0x0b, 0x00, 0x00, 0xff, 0xff, 0x91, 0xbf, 0x5e, 0xca, 0xf8, 0x10,
+	0x00, 0x00,
 }

vendor/github.com/docker/swarmkit/api/specs.proto

@@ -101,6 +101,7 @@ message TaskSpec {
 	oneof runtime {
 		NetworkAttachmentSpec attachment = 8;
 		ContainerSpec container = 1;
+		PluginSpec plugin = 10;
 	}
 
 	// Resource requirements for the container.
@@ -131,7 +132,7 @@ message TaskSpec {
 // NetworkAttachmentSpec specifies runtime parameters required to attach
 // a container to a network.
 message NetworkAttachmentSpec {
-	// ContainerID spcifies a unique ID of the container for which
+	// ContainerID specifies a unique ID of the container for which
 	// this attachment is for.
 	string container_id = 1;
 }
@@ -266,12 +267,20 @@ message ContainerSpec {
 	HealthConfig healthcheck = 16;
 }
 
+// PluginSpec specifies runtime parameters for a plugin.
+message PluginSpec {
+	// image defines the image reference, as specified in the
+	// distribution/reference package. This may include a registry host, name,
+	// tag or digest.
+	string image = 1;
+}
+
 // EndpointSpec defines the properties that can be configured to
 // access and loadbalance the service.
 message EndpointSpec {
 	// ResolutionMode specifies the mode of resolution to use for
 	// internal loadbalancing between tasks which are all within
-	// the cluster. This is sometimes calles east-west data path.
+	// the cluster. This is sometimes calls east-west data path.
 	enum ResolutionMode {
 		option (gogoproto.goproto_enum_prefix) = false;
 

vendor/github.com/docker/swarmkit/api/types.pb.go

@@ -72,6 +72,7 @@
 		TaskSpec
 		NetworkAttachmentSpec
 		ContainerSpec
+		PluginSpec
 		EndpointSpec
 		NetworkSpec
 		ClusterSpec

vendor/github.com/docker/swarmkit/ca/config.go

@@ -54,7 +54,7 @@ const (
 // RenewTLSExponentialBackoff sets the exponential backoff when trying to renew TLS certificates that have expired
 var RenewTLSExponentialBackoff = events.ExponentialBackoffConfig{
 	Base:   time.Second * 5,
-	Factor: time.Minute,
+	Factor: time.Second * 5,
 	Max:    1 * time.Hour,
 }
 
@@ -454,7 +454,10 @@ func RenewTLSConfig(ctx context.Context, s *SecurityConfig, connBroker *connecti
 	updates := make(chan CertificateUpdate)
 
 	go func() {
-		var retry time.Duration
+		var (
+			retry      time.Duration
+			forceRetry bool
+		)
 		expBackoff := events.NewExponentialBackoff(RenewTLSExponentialBackoff)
 		defer close(updates)
 		for {
@@ -487,6 +490,10 @@ func RenewTLSConfig(ctx context.Context, s *SecurityConfig, connBroker *connecti
 					log.Warn("the current TLS certificate is expired, so an attempt to renew it will be made immediately")
 					// retry immediately(ish) with exponential backoff
 					retry = expBackoff.Proceed(nil)
+				} else if forceRetry {
+					// A forced renewal was requested, but did not succeed yet.
+					// retry immediately(ish) with exponential backoff
+					retry = expBackoff.Proceed(nil)
 				} else {
 					// Random retry time between 50% and 80% of the total time to expiration
 					retry = calculateRandomExpiry(validFrom, validUntil)
@@ -501,6 +508,7 @@ func RenewTLSConfig(ctx context.Context, s *SecurityConfig, connBroker *connecti
 			case <-time.After(retry):
 				log.Info("renewing certificate")
 			case <-renew:
+				forceRetry = true
 				log.Info("forced certificate renewal")
 			case <-ctx.Done():
 				log.Info("shutting down certificate renewal routine")
@@ -515,6 +523,7 @@ func RenewTLSConfig(ctx context.Context, s *SecurityConfig, connBroker *connecti
 			} else {
 				certUpdate.Role = s.ClientTLSCreds.Role()
 				expBackoff = events.NewExponentialBackoff(RenewTLSExponentialBackoff)
+				forceRetry = false
 			}
 
 			select {

vendor/github.com/docker/swarmkit/ca/server.go

@@ -715,7 +715,7 @@ func (s *Server) signNodeCert(ctx context.Context, node *api.Node) error {
 	return nil
 }
 
-// reconcileNodeCertificates is a helper method that calles evaluateAndSignNodeCert on all the
+// reconcileNodeCertificates is a helper method that calls evaluateAndSignNodeCert on all the
 // nodes.
 func (s *Server) reconcileNodeCertificates(ctx context.Context, nodes []*api.Node) error {
 	for _, node := range nodes {

vendor/github.com/docker/swarmkit/connectionbroker/broker.go

@@ -8,6 +8,7 @@ import (
 
 	"github.com/docker/swarmkit/api"
 	"github.com/docker/swarmkit/remotes"
+	grpc_prometheus "github.com/grpc-ecosystem/go-grpc-prometheus"
 	"google.golang.org/grpc"
 )
 
@@ -59,6 +60,10 @@ func (b *Broker) SelectRemote(dialOpts ...grpc.DialOption) (*Conn, error) {
 		return nil, err
 	}
 
+	dialOpts = append(dialOpts,
+		grpc.WithUnaryInterceptor(grpc_prometheus.UnaryClientInterceptor),
+		grpc.WithStreamInterceptor(grpc_prometheus.StreamClientInterceptor))
+
 	cc, err := grpc.Dial(peer.Addr, dialOpts...)
 	if err != nil {
 		b.remotes.ObserveIfExists(peer, -remotes.DefaultObservationWeight)

vendor/github.com/docker/swarmkit/manager/allocator/networkallocator/networkallocator.go

@@ -682,7 +682,7 @@ func (na *NetworkAllocator) resolveDriver(n *api.Network) (driverapi.Driver, str
 func (na *NetworkAllocator) loadDriver(name string) error {
 	pg := na.drvRegistry.GetPluginGetter()
 	if pg == nil {
-		return fmt.Errorf("plugin store is unintialized")
+		return fmt.Errorf("plugin store is uninitialized")
 	}
 	_, err := pg.Get(name, driverapi.NetworkPluginEndpointType, plugingetter.Lookup)
 	return err

vendor/github.com/docker/swarmkit/manager/controlapi/service.go

@@ -117,7 +117,29 @@ func validateUpdate(uc *api.UpdateConfig) error {
 	return nil
 }
 
-func validateContainerSpec(container *api.ContainerSpec) error {
+func validateContainerSpec(taskSpec api.TaskSpec) error {
+	// Building a empty/dummy Task to validate the templating and
+	// the resulting container spec as well. This is a *best effort*
+	// validation.
+	container, err := template.ExpandContainerSpec(&api.Task{
+		Spec:      taskSpec,
+		ServiceID: "serviceid",
+		Slot:      1,
+		NodeID:    "nodeid",
+		Networks:  []*api.NetworkAttachment{},
+		Annotations: api.Annotations{
+			Name: "taskname",
+		},
+		ServiceAnnotations: api.Annotations{
+			Name: "servicename",
+		},
+		Endpoint:  &api.Endpoint{},
+		LogDriver: taskSpec.LogDriver,
+	})
+	if err != nil {
+		return grpc.Errorf(codes.InvalidArgument, err.Error())
+	}
+
 	if container == nil {
 		return grpc.Errorf(codes.InvalidArgument, "ContainerSpec: missing in service spec")
 	}
@@ -141,6 +163,18 @@ func validateContainerSpec(container *api.ContainerSpec) error {
 	return nil
 }
 
+func validatePluginSpec(plugin *api.PluginSpec) error {
+	if plugin.Image == "" {
+		return grpc.Errorf(codes.InvalidArgument, "PluginSpec: image reference must be provided")
+	}
+
+	if _, err := reference.ParseNormalizedNamed(plugin.Image); err != nil {
+		return grpc.Errorf(codes.InvalidArgument, "PluginSpec: %q is not a valid repository/tag", plugin.Image)
+	}
+
+	return nil
+}
+
 func validateTaskSpec(taskSpec api.TaskSpec) error {
 	if err := validateResourceRequirements(taskSpec.Resources); err != nil {
 		return err
@@ -163,36 +197,18 @@ func validateTaskSpec(taskSpec api.TaskSpec) error {
 		return grpc.Errorf(codes.InvalidArgument, "TaskSpec: missing runtime")
 	}
 
-	_, ok := taskSpec.GetRuntime().(*api.TaskSpec_Container)
-	if !ok {
+	switch taskSpec.GetRuntime().(type) {
+	case *api.TaskSpec_Container:
+		if err := validateContainerSpec(taskSpec); err != nil {
+			return err
+		}
+	case *api.TaskSpec_Plugin:
+		if err := validatePluginSpec(taskSpec.GetPlugin()); err != nil {
+			return err
+		}
+	default:
 		return grpc.Errorf(codes.Unimplemented, "RuntimeSpec: unimplemented runtime in service spec")
 	}
-
-	// Building a empty/dummy Task to validate the templating and
-	// the resulting container spec as well. This is a *best effort*
-	// validation.
-	preparedSpec, err := template.ExpandContainerSpec(&api.Task{
-		Spec:      taskSpec,
-		ServiceID: "serviceid",
-		Slot:      1,
-		NodeID:    "nodeid",
-		Networks:  []*api.NetworkAttachment{},
-		Annotations: api.Annotations{
-			Name: "taskname",
-		},
-		ServiceAnnotations: api.Annotations{
-			Name: "servicename",
-		},
-		Endpoint:  &api.Endpoint{},
-		LogDriver: taskSpec.LogDriver,
-	})
-	if err != nil {
-		return grpc.Errorf(codes.InvalidArgument, err.Error())
-	}
-	if err := validateContainerSpec(preparedSpec); err != nil {
-		return err
-	}
-
 	return nil
 }
 

vendor/github.com/docker/swarmkit/manager/manager.go

@@ -38,6 +38,7 @@ import (
 	"github.com/docker/swarmkit/remotes"
 	"github.com/docker/swarmkit/xnet"
 	gogotypes "github.com/gogo/protobuf/types"
+	grpc_prometheus "github.com/grpc-ecosystem/go-grpc-prometheus"
 	"github.com/pkg/errors"
 	"golang.org/x/net/context"
 	"google.golang.org/grpc"
@@ -201,7 +202,10 @@ func New(config *Config) (*Manager, error) {
 	raftNode := raft.NewNode(newNodeOpts)
 
 	opts := []grpc.ServerOption{
-		grpc.Creds(config.SecurityConfig.ServerTLSCreds)}
+		grpc.Creds(config.SecurityConfig.ServerTLSCreds),
+		grpc.StreamInterceptor(grpc_prometheus.StreamServerInterceptor),
+		grpc.UnaryInterceptor(grpc_prometheus.UnaryServerInterceptor),
+	}
 
 	m := &Manager{
 		config:          *config,
@@ -458,6 +462,7 @@ func (m *Manager) Run(parent context.Context) error {
 	api.RegisterLogBrokerServer(m.server, proxyLogBrokerAPI)
 	api.RegisterResourceAllocatorServer(m.server, proxyResourceAPI)
 	api.RegisterDispatcherServer(m.server, proxyDispatcherAPI)
+	grpc_prometheus.Register(m.server)
 
 	api.RegisterControlServer(m.localserver, localProxyControlAPI)
 	api.RegisterLogsServer(m.localserver, localProxyLogsAPI)
@@ -467,6 +472,7 @@ func (m *Manager) Run(parent context.Context) error {
 	api.RegisterNodeCAServer(m.localserver, localProxyNodeCAAPI)
 	api.RegisterResourceAllocatorServer(m.localserver, localProxyResourceAPI)
 	api.RegisterLogBrokerServer(m.localserver, localProxyLogBrokerAPI)
+	grpc_prometheus.Register(m.localserver)
 
 	healthServer.SetServingStatus("Raft", api.HealthCheckResponse_NOT_SERVING)
 	localHealthServer.SetServingStatus("ControlAPI", api.HealthCheckResponse_NOT_SERVING)
@@ -648,6 +654,8 @@ func (m *Manager) updateKEK(ctx context.Context, cluster *api.Cluster) error {
 
 			conn, err := grpc.Dial(
 				m.config.ControlAPI,
+				grpc.WithUnaryInterceptor(grpc_prometheus.UnaryClientInterceptor),
+				grpc.WithStreamInterceptor(grpc_prometheus.StreamClientInterceptor),
 				grpc.WithTransportCredentials(insecureCreds),
 				grpc.WithDialer(
 					func(addr string, timeout time.Duration) (net.Conn, error) {

vendor/github.com/docker/swarmkit/manager/orchestrator/update/updater.go

@@ -21,7 +21,7 @@ import (
 	gogotypes "github.com/gogo/protobuf/types"
 )
 
-const defaultMonitor = 30 * time.Second
+const defaultMonitor = 5 * time.Second
 
 // Supervisor supervises a set of updates. It's responsible for keeping track of updates,
 // shutting them down and replacing them.
@@ -157,7 +157,7 @@ func (u *Updater) Run(ctx context.Context, slots []orchestrator.Slot) {
 	}
 
 	var (
-		parallelism            int
+		parallelism            = 1
 		delay                  time.Duration
 		failureAction          = api.UpdateConfig_PAUSE
 		allowedFailureFraction = float32(0)
@@ -354,7 +354,7 @@ func (u *Updater) worker(ctx context.Context, queue <-chan orchestrator.Slot, de
 
 		if delay != 0 {
 			select {
-			case <-time.After(u.newService.Spec.Update.Delay):
+			case <-time.After(delay):
 			case <-u.stopChan:
 				return
 			}

vendor/github.com/docker/swarmkit/manager/role_manager.go

@@ -136,11 +136,21 @@ func (rm *roleManager) reconcileRole(node *api.Node) {
 			rmCtx, rmCancel := context.WithTimeout(rm.ctx, 5*time.Second)
 			defer rmCancel()
 
+			if member.RaftID == rm.raft.Config.ID {
+				// Don't use rmCtx, because we expect to lose
+				// leadership, which will cancel this context.
+				log.L.Info("demoted; transferring leadership")
+				err := rm.raft.TransferLeadership(context.Background())
+				if err == nil {
+					return
+				}
+				log.L.WithError(err).Info("failed to transfer leadership")
+			}
 			if err := rm.raft.RemoveMember(rmCtx, member.RaftID); err != nil {
 				// TODO(aaronl): Retry later
 				log.L.WithError(err).Debugf("can't demote node %s at this time", node.ID)
-				return
 			}
+			return
 		}
 
 		err := rm.store.Update(func(tx store.Tx) error {

vendor/github.com/docker/swarmkit/manager/scheduler/nodeinfo.go

@@ -11,10 +11,10 @@ import (
 // NodeInfo contains a node and some additional metadata.
 type NodeInfo struct {
 	*api.Node
-	Tasks                             map[string]*api.Task
-	DesiredRunningTasksCount          int
-	DesiredRunningTasksCountByService map[string]int
-	AvailableResources                api.Resources
+	Tasks                     map[string]*api.Task
+	ActiveTasksCount          int
+	ActiveTasksCountByService map[string]int
+	AvailableResources        api.Resources
 
 	// recentFailures is a map from service ID to the timestamps of the
 	// most recent failures the node has experienced from replicas of that
@@ -28,9 +28,9 @@ func newNodeInfo(n *api.Node, tasks map[string]*api.Task, availableResources api
 	nodeInfo := NodeInfo{
 		Node:  n,
 		Tasks: make(map[string]*api.Task),
-		DesiredRunningTasksCountByService: make(map[string]int),
-		AvailableResources:                availableResources,
-		recentFailures:                    make(map[string][]time.Time),
+		ActiveTasksCountByService: make(map[string]int),
+		AvailableResources:        availableResources,
+		recentFailures:            make(map[string][]time.Time),
 	}
 
 	for _, t := range tasks {
@@ -48,9 +48,9 @@ func (nodeInfo *NodeInfo) removeTask(t *api.Task) bool {
 	}
 
 	delete(nodeInfo.Tasks, t.ID)
-	if oldTask.DesiredState == api.TaskStateRunning {
-		nodeInfo.DesiredRunningTasksCount--
-		nodeInfo.DesiredRunningTasksCountByService[t.ServiceID]--
+	if oldTask.DesiredState <= api.TaskStateRunning {
+		nodeInfo.ActiveTasksCount--
+		nodeInfo.ActiveTasksCountByService[t.ServiceID]--
 	}
 
 	reservations := taskReservations(t.Spec)
@@ -65,15 +65,15 @@ func (nodeInfo *NodeInfo) removeTask(t *api.Task) bool {
 func (nodeInfo *NodeInfo) addTask(t *api.Task) bool {
 	oldTask, ok := nodeInfo.Tasks[t.ID]
 	if ok {
-		if t.DesiredState == api.TaskStateRunning && oldTask.DesiredState != api.TaskStateRunning {
+		if t.DesiredState <= api.TaskStateRunning && oldTask.DesiredState > api.TaskStateRunning {
 			nodeInfo.Tasks[t.ID] = t
-			nodeInfo.DesiredRunningTasksCount++
-			nodeInfo.DesiredRunningTasksCountByService[t.ServiceID]++
+			nodeInfo.ActiveTasksCount++
+			nodeInfo.ActiveTasksCountByService[t.ServiceID]++
 			return true
-		} else if t.DesiredState != api.TaskStateRunning && oldTask.DesiredState == api.TaskStateRunning {
+		} else if t.DesiredState > api.TaskStateRunning && oldTask.DesiredState <= api.TaskStateRunning {
 			nodeInfo.Tasks[t.ID] = t
-			nodeInfo.DesiredRunningTasksCount--
-			nodeInfo.DesiredRunningTasksCountByService[t.ServiceID]--
+			nodeInfo.ActiveTasksCount--
+			nodeInfo.ActiveTasksCountByService[t.ServiceID]--
 			return true
 		}
 		return false
@@ -84,9 +84,9 @@ func (nodeInfo *NodeInfo) addTask(t *api.Task) bool {
 	nodeInfo.AvailableResources.MemoryBytes -= reservations.MemoryBytes
 	nodeInfo.AvailableResources.NanoCPUs -= reservations.NanoCPUs
 
-	if t.DesiredState == api.TaskStateRunning {
-		nodeInfo.DesiredRunningTasksCount++
-		nodeInfo.DesiredRunningTasksCountByService[t.ServiceID]++
+	if t.DesiredState <= api.TaskStateRunning {
+		nodeInfo.ActiveTasksCount++
+		nodeInfo.ActiveTasksCountByService[t.ServiceID]++
 	}
 
 	return true

vendor/github.com/docker/swarmkit/manager/scheduler/nodeset.go

@@ -35,8 +35,8 @@ func (ns *nodeSet) addOrUpdateNode(n NodeInfo) {
 	if n.Tasks == nil {
 		n.Tasks = make(map[string]*api.Task)
 	}
-	if n.DesiredRunningTasksCountByService == nil {
-		n.DesiredRunningTasksCountByService = make(map[string]int)
+	if n.ActiveTasksCountByService == nil {
+		n.ActiveTasksCountByService = make(map[string]int)
 	}
 	if n.recentFailures == nil {
 		n.recentFailures = make(map[string][]time.Time)
@@ -96,8 +96,8 @@ func (ns *nodeSet) tree(serviceID string, preferences []*api.PlacementPreference
 			// sure that the tree structure is not affected by
 			// which properties nodes have and don't have.
 
-			if node.DesiredRunningTasksCountByService != nil {
-				tree.tasks += node.DesiredRunningTasksCountByService[serviceID]
+			if node.ActiveTasksCountByService != nil {
+				tree.tasks += node.ActiveTasksCountByService[serviceID]
 			}
 
 			if tree.next == nil {

vendor/github.com/docker/swarmkit/manager/scheduler/scheduler.go

@@ -517,8 +517,8 @@ func (s *Scheduler) scheduleTaskGroup(ctx context.Context, taskGroup map[string]
 			}
 		}
 
-		tasksByServiceA := a.DesiredRunningTasksCountByService[t.ServiceID]
-		tasksByServiceB := b.DesiredRunningTasksCountByService[t.ServiceID]
+		tasksByServiceA := a.ActiveTasksCountByService[t.ServiceID]
+		tasksByServiceB := b.ActiveTasksCountByService[t.ServiceID]
 
 		if tasksByServiceA < tasksByServiceB {
 			return true
@@ -528,7 +528,7 @@ func (s *Scheduler) scheduleTaskGroup(ctx context.Context, taskGroup map[string]
 		}
 
 		// Total number of tasks breaks ties.
-		return a.DesiredRunningTasksCount < b.DesiredRunningTasksCount
+		return a.ActiveTasksCount < b.ActiveTasksCount
 	}
 
 	var prefs []*api.PlacementPreference

vendor/github.com/docker/swarmkit/manager/state/raft/raft.go

@@ -412,7 +412,7 @@ func (n *Node) JoinAndStart(ctx context.Context) (err error) {
 	defer conn.Close()
 	client := api.NewRaftMembershipClient(conn)
 
-	joinCtx, joinCancel := context.WithTimeout(ctx, 10*time.Second)
+	joinCtx, joinCancel := context.WithTimeout(ctx, n.reqTimeout())
 	defer joinCancel()
 	resp, err := client.Join(joinCtx, &api.JoinRequest{
 		Addr: n.opts.Addr,
@@ -1030,6 +1030,10 @@ func (n *Node) UpdateNode(id uint64, addr string) {
 // from a member who is willing to leave its raft
 // membership to an active member of the raft
 func (n *Node) Leave(ctx context.Context, req *api.LeaveRequest) (*api.LeaveResponse, error) {
+	if req.Node == nil {
+		return nil, grpc.Errorf(codes.InvalidArgument, "no node information provided")
+	}
+
 	nodeInfo, err := ca.RemoteNode(ctx)
 	if err != nil {
 		return nil, err
@@ -1100,18 +1104,58 @@ func (n *Node) removeMember(ctx context.Context, id uint64) error {
 
 	n.membershipLock.Lock()
 	defer n.membershipLock.Unlock()
-	if n.CanRemoveMember(id) {
-		cc := raftpb.ConfChange{
-			ID:      id,
-			Type:    raftpb.ConfChangeRemoveNode,
-			NodeID:  id,
-			Context: []byte(""),
-		}
-		err := n.configure(ctx, cc)
-		return err
+	if !n.CanRemoveMember(id) {
+		return ErrCannotRemoveMember
 	}
 
-	return ErrCannotRemoveMember
+	cc := raftpb.ConfChange{
+		ID:      id,
+		Type:    raftpb.ConfChangeRemoveNode,
+		NodeID:  id,
+		Context: []byte(""),
+	}
+	return n.configure(ctx, cc)
+}
+
+// TransferLeadership attempts to transfer leadership to a different node,
+// and wait for the transfer to happen.
+func (n *Node) TransferLeadership(ctx context.Context) error {
+	ctx, cancelTransfer := context.WithTimeout(ctx, n.reqTimeout())
+	defer cancelTransfer()
+
+	n.stopMu.RLock()
+	defer n.stopMu.RUnlock()
+
+	if !n.IsMember() {
+		return ErrNoRaftMember
+	}
+
+	if !n.isLeader() {
+		return ErrLostLeadership
+	}
+
+	transferee, err := n.transport.LongestActive()
+	if err != nil {
+		return errors.Wrap(err, "failed to get longest-active member")
+	}
+	start := time.Now()
+	n.raftNode.TransferLeadership(ctx, n.Config.ID, transferee)
+	ticker := time.NewTicker(n.opts.TickInterval / 10)
+	defer ticker.Stop()
+	var leader uint64
+	for {
+		leader = n.leader()
+		if leader != raft.None && leader != n.Config.ID {
+			break
+		}
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case <-ticker.C:
+		}
+	}
+	log.G(ctx).Infof("raft: transfer leadership %x -> %x finished in %v", n.Config.ID, leader, time.Since(start))
+	return nil
 }
 
 // RemoveMember submits a configuration change to remove a member from the raft cluster
@@ -1726,23 +1770,12 @@ func (n *Node) applyRemoveNode(ctx context.Context, cc raftpb.ConfChange) (err e
 	}
 
 	if cc.NodeID == n.Config.ID {
-		// wait the commit ack to be sent before closing connection
+		// wait for the commit ack to be sent before closing connection
 		n.asyncTasks.Wait()
 
 		n.NodeRemoved()
-		// if there are only 2 nodes in the cluster, and leader is leaving
-		// before closing the connection, leader has to ensure that follower gets
-		// noticed about this raft conf change commit. Otherwise, follower would
-		// assume there are still 2 nodes in the cluster and won't get elected
-		// into the leader by acquiring the majority (2 nodes)
-
-		// while n.asyncTasks.Wait() could be helpful in this case
-		// it's the best-effort strategy, because this send could be fail due to some errors (such as time limit exceeds)
-		// TODO(Runshen Zhu): use leadership transfer to solve this case, after vendoring raft 3.0+
-	} else {
-		if err := n.transport.RemovePeer(cc.NodeID); err != nil {
-			return err
-		}
+	} else if err := n.transport.RemovePeer(cc.NodeID); err != nil {
+		return err
 	}
 
 	return n.cluster.RemoveMember(cc.NodeID)
@@ -1852,3 +1885,7 @@ func getIDs(snap *raftpb.Snapshot, ents []raftpb.Entry) []uint64 {
 	}
 	return sids
 }
+
+func (n *Node) reqTimeout() time.Duration {
+	return 5*time.Second + 2*time.Duration(n.Config.ElectionTick)*n.opts.TickInterval
+}

vendor/github.com/docker/swarmkit/manager/state/raft/storage.go

@@ -134,7 +134,7 @@ func (n *Node) loadAndStart(ctx context.Context, forceNewCluster bool) error {
 		// force commit newly appended entries
 		err := n.raftLogger.SaveEntries(st, toAppEnts)
 		if err != nil {
-			log.G(ctx).WithError(err).Fatalf("failed to save WAL while forcing new cluster")
+			log.G(ctx).WithError(err).Fatal("failed to save WAL while forcing new cluster")
 		}
 		if len(toAppEnts) != 0 {
 			st.Commit = toAppEnts[len(toAppEnts)-1].Index

vendor/github.com/docker/swarmkit/manager/state/raft/transport/transport.go

@@ -8,6 +8,7 @@ import (
 
 	"golang.org/x/net/context"
 
+	grpc_prometheus "github.com/grpc-ecosystem/go-grpc-prometheus"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/credentials"
 
@@ -295,6 +296,19 @@ func (t *Transport) Active(id uint64) bool {
 	return active
 }
 
+// LongestActive returns the ID of the peer that has been active for the longest
+// length of time.
+func (t *Transport) LongestActive() (uint64, error) {
+	p, err := t.longestActive()
+	if err != nil {
+		return 0, err
+	}
+
+	return p.id, nil
+}
+
+// longestActive returns the peer that has been active for the longest length of
+// time.
 func (t *Transport) longestActive() (*peer, error) {
 	var longest *peer
 	var longestTime time.Time
@@ -322,6 +336,8 @@ func (t *Transport) longestActive() (*peer, error) {
 
 func (t *Transport) dial(addr string) (*grpc.ClientConn, error) {
 	grpcOptions := []grpc.DialOption{
+		grpc.WithUnaryInterceptor(grpc_prometheus.UnaryClientInterceptor),
+		grpc.WithStreamInterceptor(grpc_prometheus.StreamClientInterceptor),
 		grpc.WithBackoffMaxDelay(8 * time.Second),
 	}
 	if t.config.Credentials != nil {

vendor/github.com/docker/swarmkit/manager/state/raft/util.go

@@ -8,6 +8,7 @@ import (
 	"github.com/docker/swarmkit/api"
 	"github.com/docker/swarmkit/manager/state"
 	"github.com/docker/swarmkit/manager/state/store"
+	grpc_prometheus "github.com/grpc-ecosystem/go-grpc-prometheus"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/credentials"
 )
@@ -17,6 +18,8 @@ func dial(addr string, protocol string, creds credentials.TransportCredentials,
 	grpcOptions := []grpc.DialOption{
 		grpc.WithBackoffMaxDelay(2 * time.Second),
 		grpc.WithTransportCredentials(creds),
+		grpc.WithUnaryInterceptor(grpc_prometheus.UnaryClientInterceptor),
+		grpc.WithStreamInterceptor(grpc_prometheus.StreamClientInterceptor),
 	}
 
 	if timeout != 0 {

vendor/github.com/docker/swarmkit/node/node.go

@@ -26,6 +26,7 @@ import (
 	"github.com/docker/swarmkit/manager/encryption"
 	"github.com/docker/swarmkit/remotes"
 	"github.com/docker/swarmkit/xnet"
+	grpc_prometheus "github.com/grpc-ecosystem/go-grpc-prometheus"
 	"github.com/pkg/errors"
 	"golang.org/x/net/context"
 	"google.golang.org/grpc"
@@ -640,7 +641,10 @@ func (n *Node) loadSecurityConfig(ctx context.Context) (*ca.SecurityConfig, erro
 }
 
 func (n *Node) initManagerConnection(ctx context.Context, ready chan<- struct{}) error {
-	opts := []grpc.DialOption{}
+	opts := []grpc.DialOption{
+		grpc.WithUnaryInterceptor(grpc_prometheus.UnaryClientInterceptor),
+		grpc.WithStreamInterceptor(grpc_prometheus.StreamClientInterceptor),
+	}
 	insecureCreds := credentials.NewTLS(&tls.Config{InsecureSkipVerify: true})
 	opts = append(opts, grpc.WithTransportCredentials(insecureCreds))
 	addr := n.config.ListenControlAPI

vendor/github.com/grpc-ecosystem/go-grpc-prometheus/LICENSE

@@ -0,0 +1,201 @@
+                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

vendor/github.com/grpc-ecosystem/go-grpc-prometheus/client.go

@@ -0,0 +1,72 @@
+// Copyright 2016 Michal Witkowski. All Rights Reserved.
+// See LICENSE for licensing terms.
+
+// gRPC Prometheus monitoring interceptors for client-side gRPC.
+
+package grpc_prometheus
+
+import (
+	"io"
+
+	"golang.org/x/net/context"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+)
+
+// UnaryClientInterceptor is a gRPC client-side interceptor that provides Prometheus monitoring for Unary RPCs.
+func UnaryClientInterceptor(ctx context.Context, method string, req, reply interface{}, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error {
+	monitor := newClientReporter(Unary, method)
+	monitor.SentMessage()
+	err := invoker(ctx, method, req, reply, cc, opts...)
+	if err != nil {
+		monitor.ReceivedMessage()
+	}
+	monitor.Handled(grpc.Code(err))
+	return err
+}
+
+// StreamServerInterceptor is a gRPC client-side interceptor that provides Prometheus monitoring for Streaming RPCs.
+func StreamClientInterceptor(ctx context.Context, desc *grpc.StreamDesc, cc *grpc.ClientConn, method string, streamer grpc.Streamer, opts ...grpc.CallOption) (grpc.ClientStream, error) {
+	monitor := newClientReporter(clientStreamType(desc), method)
+	clientStream, err := streamer(ctx, desc, cc, method, opts...)
+	if err != nil {
+		monitor.Handled(grpc.Code(err))
+		return nil, err
+	}
+	return &monitoredClientStream{clientStream, monitor}, nil
+}
+
+func clientStreamType(desc *grpc.StreamDesc) grpcType {
+	if desc.ClientStreams && !desc.ServerStreams {
+		return ClientStream
+	} else if !desc.ClientStreams && desc.ServerStreams {
+		return ServerStream
+	}
+	return BidiStream
+}
+
+// monitoredClientStream wraps grpc.ClientStream allowing each Sent/Recv of message to increment counters.
+type monitoredClientStream struct {
+	grpc.ClientStream
+	monitor *clientReporter
+}
+
+func (s *monitoredClientStream) SendMsg(m interface{}) error {
+	err := s.ClientStream.SendMsg(m)
+	if err == nil {
+		s.monitor.SentMessage()
+	}
+	return err
+}
+
+func (s *monitoredClientStream) RecvMsg(m interface{}) error {
+	err := s.ClientStream.RecvMsg(m)
+	if err == nil {
+		s.monitor.ReceivedMessage()
+	} else if err == io.EOF {
+		s.monitor.Handled(codes.OK)
+	} else {
+		s.monitor.Handled(grpc.Code(err))
+	}
+	return err
+}

vendor/github.com/grpc-ecosystem/go-grpc-prometheus/client_reporter.go

@@ -0,0 +1,111 @@
+// Copyright 2016 Michal Witkowski. All Rights Reserved.
+// See LICENSE for licensing terms.
+
+package grpc_prometheus
+
+import (
+	"time"
+
+	"google.golang.org/grpc/codes"
+
+	prom "github.com/prometheus/client_golang/prometheus"
+)
+
+var (
+	clientStartedCounter = prom.NewCounterVec(
+		prom.CounterOpts{
+			Namespace: "grpc",
+			Subsystem: "client",
+			Name:      "started_total",
+			Help:      "Total number of RPCs started on the client.",
+		}, []string{"grpc_type", "grpc_service", "grpc_method"})
+
+	clientHandledCounter = prom.NewCounterVec(
+		prom.CounterOpts{
+			Namespace: "grpc",
+			Subsystem: "client",
+			Name:      "handled_total",
+			Help:      "Total number of RPCs completed by the client, regardless of success or failure.",
+		}, []string{"grpc_type", "grpc_service", "grpc_method", "grpc_code"})
+
+	clientStreamMsgReceived = prom.NewCounterVec(
+		prom.CounterOpts{
+			Namespace: "grpc",
+			Subsystem: "client",
+			Name:      "msg_received_total",
+			Help:      "Total number of RPC stream messages received by the client.",
+		}, []string{"grpc_type", "grpc_service", "grpc_method"})
+
+	clientStreamMsgSent = prom.NewCounterVec(
+		prom.CounterOpts{
+			Namespace: "grpc",
+			Subsystem: "client",
+			Name:      "msg_sent_total",
+			Help:      "Total number of gRPC stream messages sent by the client.",
+		}, []string{"grpc_type", "grpc_service", "grpc_method"})
+
+	clientHandledHistogramEnabled = false
+	clientHandledHistogramOpts    = prom.HistogramOpts{
+		Namespace: "grpc",
+		Subsystem: "client",
+		Name:      "handling_seconds",
+		Help:      "Histogram of response latency (seconds) of the gRPC until it is finished by the application.",
+		Buckets:   prom.DefBuckets,
+	}
+	clientHandledHistogram *prom.HistogramVec
+)
+
+func init() {
+	prom.MustRegister(clientStartedCounter)
+	prom.MustRegister(clientHandledCounter)
+	prom.MustRegister(clientStreamMsgReceived)
+	prom.MustRegister(clientStreamMsgSent)
+}
+
+// EnableClientHandlingTimeHistogram turns on recording of handling time of RPCs.
+// Histogram metrics can be very expensive for Prometheus to retain and query.
+func EnableClientHandlingTimeHistogram(opts ...HistogramOption) {
+	for _, o := range opts {
+		o(&clientHandledHistogramOpts)
+	}
+	if !clientHandledHistogramEnabled {
+		clientHandledHistogram = prom.NewHistogramVec(
+			clientHandledHistogramOpts,
+			[]string{"grpc_type", "grpc_service", "grpc_method"},
+		)
+		prom.Register(clientHandledHistogram)
+	}
+	clientHandledHistogramEnabled = true
+}
+
+type clientReporter struct {
+	rpcType     grpcType
+	serviceName string
+	methodName  string
+	startTime   time.Time
+}
+
+func newClientReporter(rpcType grpcType, fullMethod string) *clientReporter {
+	r := &clientReporter{rpcType: rpcType}
+	if clientHandledHistogramEnabled {
+		r.startTime = time.Now()
+	}
+	r.serviceName, r.methodName = splitMethodName(fullMethod)
+	clientStartedCounter.WithLabelValues(string(r.rpcType), r.serviceName, r.methodName).Inc()
+	return r
+}
+
+func (r *clientReporter) ReceivedMessage() {
+	clientStreamMsgReceived.WithLabelValues(string(r.rpcType), r.serviceName, r.methodName).Inc()
+}
+
+func (r *clientReporter) SentMessage() {
+	clientStreamMsgSent.WithLabelValues(string(r.rpcType), r.serviceName, r.methodName).Inc()
+}
+
+func (r *clientReporter) Handled(code codes.Code) {
+	clientHandledCounter.WithLabelValues(string(r.rpcType), r.serviceName, r.methodName, code.String()).Inc()
+	if clientHandledHistogramEnabled {
+		clientHandledHistogram.WithLabelValues(string(r.rpcType), r.serviceName, r.methodName).Observe(time.Since(r.startTime).Seconds())
+	}
+}

vendor/github.com/grpc-ecosystem/go-grpc-prometheus/server.go

@@ -0,0 +1,74 @@
+// Copyright 2016 Michal Witkowski. All Rights Reserved.
+// See LICENSE for licensing terms.
+
+// gRPC Prometheus monitoring interceptors for server-side gRPC.
+
+package grpc_prometheus
+
+import (
+	"golang.org/x/net/context"
+	"google.golang.org/grpc"
+)
+
+// PreregisterServices takes a gRPC server and pre-initializes all counters to 0.
+// This allows for easier monitoring in Prometheus (no missing metrics), and should be called *after* all services have
+// been registered with the server.
+func Register(server *grpc.Server) {
+	serviceInfo := server.GetServiceInfo()
+	for serviceName, info := range serviceInfo {
+		for _, mInfo := range info.Methods {
+			preRegisterMethod(serviceName, &mInfo)
+		}
+	}
+}
+
+// UnaryServerInterceptor is a gRPC server-side interceptor that provides Prometheus monitoring for Unary RPCs.
+func UnaryServerInterceptor(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) {
+	monitor := newServerReporter(Unary, info.FullMethod)
+	monitor.ReceivedMessage()
+	resp, err := handler(ctx, req)
+	monitor.Handled(grpc.Code(err))
+	if err == nil {
+		monitor.SentMessage()
+	}
+	return resp, err
+}
+
+// StreamServerInterceptor is a gRPC server-side interceptor that provides Prometheus monitoring for Streaming RPCs.
+func StreamServerInterceptor(srv interface{}, ss grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) error {
+	monitor := newServerReporter(streamRpcType(info), info.FullMethod)
+	err := handler(srv, &monitoredServerStream{ss, monitor})
+	monitor.Handled(grpc.Code(err))
+	return err
+}
+
+func streamRpcType(info *grpc.StreamServerInfo) grpcType {
+	if info.IsClientStream && !info.IsServerStream {
+		return ClientStream
+	} else if !info.IsClientStream && info.IsServerStream {
+		return ServerStream
+	}
+	return BidiStream
+}
+
+// monitoredStream wraps grpc.ServerStream allowing each Sent/Recv of message to increment counters.
+type monitoredServerStream struct {
+	grpc.ServerStream
+	monitor *serverReporter
+}
+
+func (s *monitoredServerStream) SendMsg(m interface{}) error {
+	err := s.ServerStream.SendMsg(m)
+	if err == nil {
+		s.monitor.SentMessage()
+	}
+	return err
+}
+
+func (s *monitoredServerStream) RecvMsg(m interface{}) error {
+	err := s.ServerStream.RecvMsg(m)
+	if err == nil {
+		s.monitor.ReceivedMessage()
+	}
+	return err
+}

vendor/github.com/grpc-ecosystem/go-grpc-prometheus/server_reporter.go

@@ -0,0 +1,157 @@
+// Copyright 2016 Michal Witkowski. All Rights Reserved.
+// See LICENSE for licensing terms.
+
+package grpc_prometheus
+
+import (
+	"time"
+
+	"google.golang.org/grpc/codes"
+
+	prom "github.com/prometheus/client_golang/prometheus"
+	"google.golang.org/grpc"
+)
+
+type grpcType string
+
+const (
+	Unary        grpcType = "unary"
+	ClientStream grpcType = "client_stream"
+	ServerStream grpcType = "server_stream"
+	BidiStream   grpcType = "bidi_stream"
+)
+
+var (
+	serverStartedCounter = prom.NewCounterVec(
+		prom.CounterOpts{
+			Namespace: "grpc",
+			Subsystem: "server",
+			Name:      "started_total",
+			Help:      "Total number of RPCs started on the server.",
+		}, []string{"grpc_type", "grpc_service", "grpc_method"})
+
+	serverHandledCounter = prom.NewCounterVec(
+		prom.CounterOpts{
+			Namespace: "grpc",
+			Subsystem: "server",
+			Name:      "handled_total",
+			Help:      "Total number of RPCs completed on the server, regardless of success or failure.",
+		}, []string{"grpc_type", "grpc_service", "grpc_method", "grpc_code"})
+
+	serverStreamMsgReceived = prom.NewCounterVec(
+		prom.CounterOpts{
+			Namespace: "grpc",
+			Subsystem: "server",
+			Name:      "msg_received_total",
+			Help:      "Total number of RPC stream messages received on the server.",
+		}, []string{"grpc_type", "grpc_service", "grpc_method"})
+
+	serverStreamMsgSent = prom.NewCounterVec(
+		prom.CounterOpts{
+			Namespace: "grpc",
+			Subsystem: "server",
+			Name:      "msg_sent_total",
+			Help:      "Total number of gRPC stream messages sent by the server.",
+		}, []string{"grpc_type", "grpc_service", "grpc_method"})
+
+	serverHandledHistogramEnabled = false
+	serverHandledHistogramOpts    = prom.HistogramOpts{
+		Namespace: "grpc",
+		Subsystem: "server",
+		Name:      "handling_seconds",
+		Help:      "Histogram of response latency (seconds) of gRPC that had been application-level handled by the server.",
+		Buckets:   prom.DefBuckets,
+	}
+	serverHandledHistogram *prom.HistogramVec
+)
+
+func init() {
+	prom.MustRegister(serverStartedCounter)
+	prom.MustRegister(serverHandledCounter)
+	prom.MustRegister(serverStreamMsgReceived)
+	prom.MustRegister(serverStreamMsgSent)
+}
+
+type HistogramOption func(*prom.HistogramOpts)
+
+// WithHistogramBuckets allows you to specify custom bucket ranges for histograms if EnableHandlingTimeHistogram is on.
+func WithHistogramBuckets(buckets []float64) HistogramOption {
+	return func(o *prom.HistogramOpts) { o.Buckets = buckets }
+}
+
+// EnableHandlingTimeHistogram turns on recording of handling time of RPCs for server-side interceptors.
+// Histogram metrics can be very expensive for Prometheus to retain and query.
+func EnableHandlingTimeHistogram(opts ...HistogramOption) {
+	for _, o := range opts {
+		o(&serverHandledHistogramOpts)
+	}
+	if !serverHandledHistogramEnabled {
+		serverHandledHistogram = prom.NewHistogramVec(
+			serverHandledHistogramOpts,
+			[]string{"grpc_type", "grpc_service", "grpc_method"},
+		)
+		prom.Register(serverHandledHistogram)
+	}
+	serverHandledHistogramEnabled = true
+}
+
+type serverReporter struct {
+	rpcType     grpcType
+	serviceName string
+	methodName  string
+	startTime   time.Time
+}
+
+func newServerReporter(rpcType grpcType, fullMethod string) *serverReporter {
+	r := &serverReporter{rpcType: rpcType}
+	if serverHandledHistogramEnabled {
+		r.startTime = time.Now()
+	}
+	r.serviceName, r.methodName = splitMethodName(fullMethod)
+	serverStartedCounter.WithLabelValues(string(r.rpcType), r.serviceName, r.methodName).Inc()
+	return r
+}
+
+func (r *serverReporter) ReceivedMessage() {
+	serverStreamMsgReceived.WithLabelValues(string(r.rpcType), r.serviceName, r.methodName).Inc()
+}
+
+func (r *serverReporter) SentMessage() {
+	serverStreamMsgSent.WithLabelValues(string(r.rpcType), r.serviceName, r.methodName).Inc()
+}
+
+func (r *serverReporter) Handled(code codes.Code) {
+	serverHandledCounter.WithLabelValues(string(r.rpcType), r.serviceName, r.methodName, code.String()).Inc()
+	if serverHandledHistogramEnabled {
+		serverHandledHistogram.WithLabelValues(string(r.rpcType), r.serviceName, r.methodName).Observe(time.Since(r.startTime).Seconds())
+	}
+}
+
+// preRegisterMethod is invoked on Register of a Server, allowing all gRPC services labels to be pre-populated.
+func preRegisterMethod(serviceName string, mInfo *grpc.MethodInfo) {
+	methodName := mInfo.Name
+	methodType := string(typeFromMethodInfo(mInfo))
+	// These are just references (no increments), as just referencing will create the labels but not set values.
+	serverStartedCounter.GetMetricWithLabelValues(methodType, serviceName, methodName)
+	serverStreamMsgReceived.GetMetricWithLabelValues(methodType, serviceName, methodName)
+	serverStreamMsgSent.GetMetricWithLabelValues(methodType, serviceName, methodName)
+	if serverHandledHistogramEnabled {
+		serverHandledHistogram.GetMetricWithLabelValues(methodType, serviceName, methodName)
+	}
+	for _, code := range allCodes {
+		serverHandledCounter.GetMetricWithLabelValues(methodType, serviceName, methodName, code.String())
+	}
+}
+
+func typeFromMethodInfo(mInfo *grpc.MethodInfo) grpcType {
+	if mInfo.IsClientStream == false && mInfo.IsServerStream == false {
+		return Unary
+	}
+	if mInfo.IsClientStream == true && mInfo.IsServerStream == false {
+		return ClientStream
+	}
+	if mInfo.IsClientStream == false && mInfo.IsServerStream == true {
+		return ServerStream
+	}
+	return BidiStream
+}

vendor/github.com/grpc-ecosystem/go-grpc-prometheus/util.go

@@ -0,0 +1,27 @@
+// Copyright 2016 Michal Witkowski. All Rights Reserved.
+// See LICENSE for licensing terms.
+
+package grpc_prometheus
+
+import (
+	"strings"
+
+	"google.golang.org/grpc/codes"
+)
+
+var (
+	allCodes = []codes.Code{
+		codes.OK, codes.Canceled, codes.Unknown, codes.InvalidArgument, codes.DeadlineExceeded, codes.NotFound,
+		codes.AlreadyExists, codes.PermissionDenied, codes.Unauthenticated, codes.ResourceExhausted,
+		codes.FailedPrecondition, codes.Aborted, codes.OutOfRange, codes.Unimplemented, codes.Internal,
+		codes.Unavailable, codes.DataLoss,
+	}
+)
+
+func splitMethodName(fullMethodName string) (string, string) {
+	fullMethodName = strings.TrimPrefix(fullMethodName, "/") // remove leading slash
+	if i := strings.Index(fullMethodName, "/"); i >= 0 {
+		return fullMethodName[:i], fullMethodName[i+1:]
+	}
+	return "unknown", "unknown"
+}