Merge pull request #27967 from aaronlehmann/encryption

Encryption at rest of manager keys and raft data

api/server/middleware/debug.go

@@ -64,7 +64,7 @@ func maskSecretKeys(inp interface{}) {
 	if form, ok := inp.(map[string]interface{}); ok {
 	loop0:
 		for k, v := range form {
-			for _, m := range []string{"password", "secret", "jointoken"} {
+			for _, m := range []string{"password", "secret", "jointoken", "unlockkey"} {
 				if strings.EqualFold(m, k) {
 					form[k] = "*****"
 					continue loop0

api/server/router/swarm/backend.go

@@ -12,6 +12,8 @@ type Backend interface {
 	Leave(force bool) error
 	Inspect() (types.Swarm, error)
 	Update(uint64, types.Spec, types.UpdateFlags) error
+	GetUnlockKey() (string, error)
+	UnlockSwarm(req types.UnlockRequest) error
 	GetServices(basictypes.ServiceListOptions) ([]types.Service, error)
 	GetService(string) (types.Service, error)
 	CreateService(types.ServiceSpec, string) (string, error)

api/server/router/swarm/cluster.go

@@ -28,7 +28,9 @@ func (sr *swarmRouter) initRoutes() {
 		router.NewPostRoute("/swarm/join", sr.joinCluster),
 		router.NewPostRoute("/swarm/leave", sr.leaveCluster),
 		router.NewGetRoute("/swarm", sr.inspectCluster),
+		router.NewGetRoute("/swarm/unlockkey", sr.getUnlockKey),
 		router.NewPostRoute("/swarm/update", sr.updateCluster),
+		router.NewPostRoute("/swarm/unlock", sr.unlockCluster),
 		router.NewGetRoute("/services", sr.getServices),
 		router.NewGetRoute("/services/{id:.*}", sr.getService),
 		router.NewPostRoute("/services/create", sr.createService),

api/server/router/swarm/cluster_routes.go

@@ -87,6 +87,15 @@ func (sr *swarmRouter) updateCluster(ctx context.Context, w http.ResponseWriter,
 		flags.RotateManagerToken = rot
 	}
 
+	if value := r.URL.Query().Get("rotateManagerUnlockKey"); value != "" {
+		rot, err := strconv.ParseBool(value)
+		if err != nil {
+			return fmt.Errorf("invalid value for rotateManagerUnlockKey: %s", value)
+		}
+
+		flags.RotateManagerUnlockKey = rot
+	}
+
 	if err := sr.backend.Update(version, swarm, flags); err != nil {
 		logrus.Errorf("Error configuring swarm: %v", err)
 		return err
@@ -94,6 +103,31 @@ func (sr *swarmRouter) updateCluster(ctx context.Context, w http.ResponseWriter,
 	return nil
 }
 
+func (sr *swarmRouter) unlockCluster(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
+	var req types.UnlockRequest
+	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
+		return err
+	}
+
+	if err := sr.backend.UnlockSwarm(req); err != nil {
+		logrus.Errorf("Error unlocking swarm: %v", err)
+		return err
+	}
+	return nil
+}
+
+func (sr *swarmRouter) getUnlockKey(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
+	unlockKey, err := sr.backend.GetUnlockKey()
+	if err != nil {
+		logrus.WithError(err).Errorf("Error retrieving swarm unlock key")
+		return err
+	}
+
+	return httputils.WriteJSON(w, http.StatusOK, &basictypes.SwarmUnlockKeyResponse{
+		UnlockKey: unlockKey,
+	})
+}
+
 func (sr *swarmRouter) getServices(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
 	if err := httputils.ParseForm(r); err != nil {
 		return err

api/types/client.go

@@ -349,3 +349,10 @@ type SecretRequestOption struct {
 	GID    string
 	Mode   os.FileMode
 }
+
+// SwarmUnlockKeyResponse contains the response for Remote API:
+// GET /swarm/unlockkey
+type SwarmUnlockKeyResponse struct {
+	// UnlockKey is the unlock key in ASCII-armored format.
+	UnlockKey string
+}

api/types/swarm/swarm.go

@@ -28,11 +28,12 @@ type JoinTokens struct {
 type Spec struct {
 	Annotations
 
-	Orchestration OrchestrationConfig `json:",omitempty"`
-	Raft          RaftConfig          `json:",omitempty"`
-	Dispatcher    DispatcherConfig    `json:",omitempty"`
-	CAConfig      CAConfig            `json:",omitempty"`
-	TaskDefaults  TaskDefaults        `json:",omitempty"`
+	Orchestration    OrchestrationConfig `json:",omitempty"`
+	Raft             RaftConfig          `json:",omitempty"`
+	Dispatcher       DispatcherConfig    `json:",omitempty"`
+	CAConfig         CAConfig            `json:",omitempty"`
+	TaskDefaults     TaskDefaults        `json:",omitempty"`
+	EncryptionConfig EncryptionConfig    `json:",omitempty"`
 }
 
 // OrchestrationConfig represents orchestration configuration.
@@ -53,6 +54,14 @@ type TaskDefaults struct {
 	LogDriver *Driver `json:",omitempty"`
 }
 
+// EncryptionConfig controls at-rest encryption of data and keys.
+type EncryptionConfig struct {
+	// AutoLockManagers specifies whether or not managers TLS keys and raft data
+	// should be encrypted at rest in such a way that they must be unlocked
+	// before the manager node starts up again.
+	AutoLockManagers bool
+}
+
 // RaftConfig represents raft configuration.
 type RaftConfig struct {
 	// SnapshotInterval is the number of log entries between snapshots.
@@ -121,10 +130,11 @@ type ExternalCA struct {
 
 // InitRequest is the request used to init a swarm.
 type InitRequest struct {
-	ListenAddr      string
-	AdvertiseAddr   string
-	ForceNewCluster bool
-	Spec            Spec
+	ListenAddr       string
+	AdvertiseAddr    string
+	ForceNewCluster  bool
+	Spec             Spec
+	AutoLockManagers bool
 }
 
 // JoinRequest is the request used to join a swarm.
@@ -135,6 +145,12 @@ type JoinRequest struct {
 	JoinToken     string // accept by secret
 }
 
+// UnlockRequest is the request used to unlock a swarm.
+type UnlockRequest struct {
+	// UnlockKey is the unlock key in ASCII-armored format.
+	UnlockKey string
+}
+
 // LocalNodeState represents the state of the local node.
 type LocalNodeState string
 
@@ -147,6 +163,8 @@ const (
 	LocalNodeStateActive LocalNodeState = "active"
 	// LocalNodeStateError ERROR
 	LocalNodeStateError LocalNodeState = "error"
+	// LocalNodeStateLocked LOCKED
+	LocalNodeStateLocked LocalNodeState = "locked"
 )
 
 // Info represents generic information about swarm.
@@ -173,6 +191,7 @@ type Peer struct {
 
 // UpdateFlags contains flags for SwarmUpdate.
 type UpdateFlags struct {
-	RotateWorkerToken  bool
-	RotateManagerToken bool
+	RotateWorkerToken      bool
+	RotateManagerToken     bool
+	RotateManagerUnlockKey bool
 }

cli/command/swarm/cmd.go

@@ -22,8 +22,10 @@ func NewSwarmCommand(dockerCli *command.DockerCli) *cobra.Command {
 		newInitCommand(dockerCli),
 		newJoinCommand(dockerCli),
 		newJoinTokenCommand(dockerCli),
+		newUnlockKeyCommand(dockerCli),
 		newUpdateCommand(dockerCli),
 		newLeaveCommand(dockerCli),
+		newUnlockCommand(dockerCli),
 	)
 	return cmd
 }

cli/command/swarm/init.go

@@ -1,7 +1,6 @@
 package swarm
 
 import (
-	"errors"
 	"fmt"
 	"strings"
 
@@ -10,6 +9,7 @@ import (
 	"github.com/docker/docker/api/types/swarm"
 	"github.com/docker/docker/cli"
 	"github.com/docker/docker/cli/command"
+	"github.com/pkg/errors"
 	"github.com/spf13/cobra"
 	"github.com/spf13/pflag"
 )
@@ -49,10 +49,11 @@ func runInit(dockerCli *command.DockerCli, flags *pflag.FlagSet, opts initOption
 	ctx := context.Background()
 
 	req := swarm.InitRequest{
-		ListenAddr:      opts.listenAddr.String(),
-		AdvertiseAddr:   opts.advertiseAddr,
-		ForceNewCluster: opts.forceNewCluster,
-		Spec:            opts.swarmOptions.ToSpec(flags),
+		ListenAddr:       opts.listenAddr.String(),
+		AdvertiseAddr:    opts.advertiseAddr,
+		ForceNewCluster:  opts.forceNewCluster,
+		Spec:             opts.swarmOptions.ToSpec(flags),
+		AutoLockManagers: opts.swarmOptions.autolock,
 	}
 
 	nodeID, err := client.SwarmInit(ctx, req)
@@ -70,5 +71,14 @@ func runInit(dockerCli *command.DockerCli, flags *pflag.FlagSet, opts initOption
 	}
 
 	fmt.Fprint(dockerCli.Out(), "To add a manager to this swarm, run 'docker swarm join-token manager' and follow the instructions.\n\n")
+
+	if req.AutoLockManagers {
+		unlockKeyResp, err := client.SwarmGetUnlockKey(ctx)
+		if err != nil {
+			return errors.Wrap(err, "could not fetch unlock key")
+		}
+		printUnlockCommand(ctx, dockerCli, unlockKeyResp.UnlockKey)
+	}
+
 	return nil
 }

cli/command/swarm/opts.go

@@ -26,6 +26,8 @@ const (
 	flagExternalCA          = "external-ca"
 	flagMaxSnapshots        = "max-snapshots"
 	flagSnapshotInterval    = "snapshot-interval"
+	flagLockKey             = "lock-key"
+	flagAutolock            = "autolock"
 )
 
 type swarmOptions struct {
@@ -35,6 +37,7 @@ type swarmOptions struct {
 	externalCA          ExternalCAOption
 	maxSnapshots        uint64
 	snapshotInterval    uint64
+	autolock            bool
 }
 
 // NodeAddrOption is a pflag.Value for listening addresses
@@ -173,6 +176,7 @@ func addSwarmFlags(flags *pflag.FlagSet, opts *swarmOptions) {
 	flags.Var(&opts.externalCA, flagExternalCA, "Specifications of one or more certificate signing endpoints")
 	flags.Uint64Var(&opts.maxSnapshots, flagMaxSnapshots, 0, "Number of additional Raft snapshots to retain")
 	flags.Uint64Var(&opts.snapshotInterval, flagSnapshotInterval, 10000, "Number of log entries between Raft snapshots")
+	flags.BoolVar(&opts.autolock, flagAutolock, false, "Enable or disable manager autolocking (requiring an unlock key to start a stopped manager)")
 }
 
 func (opts *swarmOptions) mergeSwarmSpec(spec *swarm.Spec, flags *pflag.FlagSet) {
@@ -194,6 +198,9 @@ func (opts *swarmOptions) mergeSwarmSpec(spec *swarm.Spec, flags *pflag.FlagSet)
 	if flags.Changed(flagSnapshotInterval) {
 		spec.Raft.SnapshotInterval = opts.snapshotInterval
 	}
+	if flags.Changed(flagAutolock) {
+		spec.EncryptionConfig.AutoLockManagers = opts.autolock
+	}
 }
 
 func (opts *swarmOptions) ToSpec(flags *pflag.FlagSet) swarm.Spec {

cli/command/swarm/unlock.go

@@ -0,0 +1,54 @@
+package swarm
+
+import (
+	"bufio"
+	"context"
+	"fmt"
+	"io"
+	"strings"
+
+	"github.com/spf13/cobra"
+	"golang.org/x/crypto/ssh/terminal"
+
+	"github.com/docker/docker/api/types/swarm"
+	"github.com/docker/docker/cli"
+	"github.com/docker/docker/cli/command"
+)
+
+func newUnlockCommand(dockerCli *command.DockerCli) *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "unlock",
+		Short: "Unlock swarm",
+		Args:  cli.ExactArgs(0),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			client := dockerCli.Client()
+			ctx := context.Background()
+
+			key, err := readKey(dockerCli.In(), "Please enter unlock key: ")
+			if err != nil {
+				return err
+			}
+			req := swarm.UnlockRequest{
+				UnlockKey: key,
+			}
+
+			return client.SwarmUnlock(ctx, req)
+		},
+	}
+
+	return cmd
+}
+
+func readKey(in *command.InStream, prompt string) (string, error) {
+	if in.IsTerminal() {
+		fmt.Print(prompt)
+		dt, err := terminal.ReadPassword(int(in.FD()))
+		fmt.Println()
+		return string(dt), err
+	}
+	key, err := bufio.NewReader(in).ReadString('\n')
+	if err == io.EOF {
+		err = nil
+	}
+	return strings.TrimSpace(key), err
+}

cli/command/swarm/unlock_key.go

@@ -0,0 +1,79 @@
+package swarm
+
+import (
+	"fmt"
+
+	"github.com/spf13/cobra"
+
+	"github.com/docker/docker/api/types/swarm"
+	"github.com/docker/docker/cli"
+	"github.com/docker/docker/cli/command"
+	"github.com/pkg/errors"
+	"golang.org/x/net/context"
+)
+
+func newUnlockKeyCommand(dockerCli *command.DockerCli) *cobra.Command {
+	var rotate, quiet bool
+
+	cmd := &cobra.Command{
+		Use:   "unlock-key [OPTIONS]",
+		Short: "Manage the unlock key",
+		Args:  cli.NoArgs,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			client := dockerCli.Client()
+			ctx := context.Background()
+
+			if rotate {
+				flags := swarm.UpdateFlags{RotateManagerUnlockKey: true}
+
+				swarm, err := client.SwarmInspect(ctx)
+				if err != nil {
+					return err
+				}
+
+				if !swarm.Spec.EncryptionConfig.AutoLockManagers {
+					return errors.New("cannot rotate because autolock is not turned on")
+				}
+
+				err = client.SwarmUpdate(ctx, swarm.Version, swarm.Spec, flags)
+				if err != nil {
+					return err
+				}
+				if !quiet {
+					fmt.Fprintf(dockerCli.Out(), "Successfully rotated manager unlock key.\n\n")
+				}
+			}
+
+			unlockKeyResp, err := client.SwarmGetUnlockKey(ctx)
+			if err != nil {
+				return errors.Wrap(err, "could not fetch unlock key")
+			}
+
+			if unlockKeyResp.UnlockKey == "" {
+				return errors.New("no unlock key is set")
+			}
+
+			if quiet {
+				fmt.Fprintln(dockerCli.Out(), unlockKeyResp.UnlockKey)
+			} else {
+				printUnlockCommand(ctx, dockerCli, unlockKeyResp.UnlockKey)
+			}
+			return nil
+		},
+	}
+
+	flags := cmd.Flags()
+	flags.BoolVar(&rotate, flagRotate, false, "Rotate unlock key")
+	flags.BoolVarP(&quiet, flagQuiet, "q", false, "Only display token")
+
+	return cmd
+}
+
+func printUnlockCommand(ctx context.Context, dockerCli *command.DockerCli, unlockKey string) {
+	if len(unlockKey) == 0 {
+		return
+	}
+
+	fmt.Fprintf(dockerCli.Out(), "To unlock a swarm manager after it restarts, run the `docker swarm unlock`\ncommand and provide the following key:\n\n    %s\n\nPlease remember to store this key in a password manager, since without it you\nwill not be able to restart the manager.\n", unlockKey)
+	return
+}

cli/command/swarm/update.go

@@ -8,6 +8,7 @@ import (
 	"github.com/docker/docker/api/types/swarm"
 	"github.com/docker/docker/cli"
 	"github.com/docker/docker/cli/command"
+	"github.com/pkg/errors"
 	"github.com/spf13/cobra"
 	"github.com/spf13/pflag"
 )
@@ -39,8 +40,12 @@ func runUpdate(dockerCli *command.DockerCli, flags *pflag.FlagSet, opts swarmOpt
 		return err
 	}
 
+	prevAutoLock := swarm.Spec.EncryptionConfig.AutoLockManagers
+
 	opts.mergeSwarmSpec(&swarm.Spec, flags)
 
+	curAutoLock := swarm.Spec.EncryptionConfig.AutoLockManagers
+
 	err = client.SwarmUpdate(ctx, swarm.Version, swarm.Spec, updateFlags)
 	if err != nil {
 		return err
@@ -48,5 +53,13 @@ func runUpdate(dockerCli *command.DockerCli, flags *pflag.FlagSet, opts swarmOpt
 
 	fmt.Fprintln(dockerCli.Out(), "Swarm updated.")
 
+	if curAutoLock && !prevAutoLock {
+		unlockKeyResp, err := client.SwarmGetUnlockKey(ctx)
+		if err != nil {
+			return errors.Wrap(err, "could not fetch unlock key")
+		}
+		printUnlockCommand(ctx, dockerCli, unlockKeyResp.UnlockKey)
+	}
+
 	return nil
 }

cli/command/system/info.go

@@ -96,7 +96,7 @@ func prettyPrintInfo(dockerCli *command.DockerCli, info types.Info) error {
 	}
 
 	fmt.Fprintf(dockerCli.Out(), "Swarm: %v\n", info.Swarm.LocalNodeState)
-	if info.Swarm.LocalNodeState != swarm.LocalNodeStateInactive {
+	if info.Swarm.LocalNodeState != swarm.LocalNodeStateInactive && info.Swarm.LocalNodeState != swarm.LocalNodeStateLocked {
 		fmt.Fprintf(dockerCli.Out(), " NodeID: %s\n", info.Swarm.NodeID)
 		if info.Swarm.Error != "" {
 			fmt.Fprintf(dockerCli.Out(), " Error: %v\n", info.Swarm.Error)

client/interface.go

@@ -119,6 +119,8 @@ type ServiceAPIClient interface {
 type SwarmAPIClient interface {
 	SwarmInit(ctx context.Context, req swarm.InitRequest) (string, error)
 	SwarmJoin(ctx context.Context, req swarm.JoinRequest) error
+	SwarmGetUnlockKey(ctx context.Context) (types.SwarmUnlockKeyResponse, error)
+	SwarmUnlock(ctx context.Context, req swarm.UnlockRequest) error
 	SwarmLeave(ctx context.Context, force bool) error
 	SwarmInspect(ctx context.Context) (swarm.Swarm, error)
 	SwarmUpdate(ctx context.Context, version swarm.Version, swarm swarm.Spec, flags swarm.UpdateFlags) error

client/swarm_get_unlock_key.go

@@ -0,0 +1,21 @@
+package client
+
+import (
+	"encoding/json"
+
+	"github.com/docker/docker/api/types"
+	"golang.org/x/net/context"
+)
+
+// SwarmGetUnlockKey retrieves the swarm's unlock key.
+func (cli *Client) SwarmGetUnlockKey(ctx context.Context) (types.SwarmUnlockKeyResponse, error) {
+	serverResp, err := cli.get(ctx, "/swarm/unlockkey", nil, nil)
+	if err != nil {
+		return types.SwarmUnlockKeyResponse{}, err
+	}
+
+	var response types.SwarmUnlockKeyResponse
+	err = json.NewDecoder(serverResp.body).Decode(&response)
+	ensureReaderClosed(serverResp)
+	return response, err
+}

client/swarm_unlock.go

@@ -0,0 +1,17 @@
+package client
+
+import (
+	"github.com/docker/docker/api/types/swarm"
+	"golang.org/x/net/context"
+)
+
+// SwarmUnlock unlockes locked swarm.
+func (cli *Client) SwarmUnlock(ctx context.Context, req swarm.UnlockRequest) error {
+	serverResp, err := cli.post(ctx, "/swarm/unlock", nil, req, nil)
+	if err != nil {
+		return err
+	}
+
+	ensureReaderClosed(serverResp)
+	return err
+}

client/swarm_update.go

@@ -15,6 +15,7 @@ func (cli *Client) SwarmUpdate(ctx context.Context, version swarm.Version, swarm
 	query.Set("version", strconv.FormatUint(version.Index, 10))
 	query.Set("rotateWorkerToken", fmt.Sprintf("%v", flags.RotateWorkerToken))
 	query.Set("rotateManagerToken", fmt.Sprintf("%v", flags.RotateManagerToken))
+	query.Set("rotateManagerUnlockKey", fmt.Sprintf("%v", flags.RotateManagerUnlockKey))
 	resp, err := cli.post(ctx, "/swarm/update", query, swarm, nil)
 	ensureReaderClosed(resp)
 	return err

daemon/cluster/cluster.go

@@ -12,10 +12,8 @@ import (
 	"sync"
 	"time"
 
-	"google.golang.org/grpc"
-
 	"github.com/Sirupsen/logrus"
-	"github.com/docker/docker/api/errors"
+	apierrors "github.com/docker/docker/api/errors"
 	apitypes "github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/filters"
 	"github.com/docker/docker/api/types/network"
@@ -28,8 +26,11 @@ import (
 	"github.com/docker/docker/pkg/signal"
 	"github.com/docker/docker/runconfig"
 	swarmapi "github.com/docker/swarmkit/api"
+	"github.com/docker/swarmkit/manager/encryption"
 	swarmnode "github.com/docker/swarmkit/node"
+	"github.com/pkg/errors"
 	"golang.org/x/net/context"
+	"google.golang.org/grpc"
 )
 
 const swarmDirName = "swarm"
@@ -56,6 +57,9 @@ var ErrPendingSwarmExists = fmt.Errorf("This node is processing an existing join
 // ErrSwarmJoinTimeoutReached is returned when cluster join could not complete before timeout was reached.
 var ErrSwarmJoinTimeoutReached = fmt.Errorf("Timeout was reached before node was joined. The attempt to join the swarm will continue in the background. Use the \"docker info\" command to see the current swarm status of your node.")
 
+// ErrSwarmLocked is returned if the swarm is encrypted and needs a key to unlock it.
+var ErrSwarmLocked = fmt.Errorf("Swarm is encrypted and needs to be unlocked before it can be used. Please use \"docker swarm unlock\" to unlock it.")
+
 // NetworkSubnetsProvider exposes functions for retrieving the subnets
 // of networks managed by Docker, so they can be filtered.
 type NetworkSubnetsProvider interface {
@@ -92,6 +96,8 @@ type Cluster struct {
 	err             error
 	cancelDelay     func()
 	attachers       map[string]*attacher
+	locked          bool
+	lastNodeConfig  *nodeStartConfig
 }
 
 // attacher manages the in-memory attachment state of a container
@@ -133,6 +139,8 @@ type nodeStartConfig struct {
 	joinAddr        string
 	forceNewCluster bool
 	joinToken       string
+	lockKey         []byte
+	autolock        bool
 }
 
 // New creates a new Cluster instance using provided config.
@@ -173,6 +181,10 @@ func New(config Config) (*Cluster, error) {
 		logrus.Error("swarm component could not be started before timeout was reached")
 	case <-n.Ready():
 	case <-n.done:
+		if errors.Cause(c.err) == ErrSwarmLocked {
+			return c, nil
+		}
+
 		return nil, fmt.Errorf("swarm component could not be started: %v", c.err)
 	}
 	go c.reconnectOnFailure(n)
@@ -300,7 +312,10 @@ func (c *Cluster) startNewNode(conf nodeStartConfig) (*node, error) {
 		Executor:           container.NewExecutor(c.config.Backend),
 		HeartbeatTick:      1,
 		ElectionTick:       3,
+		UnlockKey:          conf.lockKey,
+		AutoLockManagers:   conf.autolock,
 	})
+
 	if err != nil {
 		return nil, err
 	}
@@ -320,13 +335,18 @@ func (c *Cluster) startNewNode(conf nodeStartConfig) (*node, error) {
 
 	c.config.Backend.SetClusterProvider(c)
 	go func() {
-		err := n.Err(ctx)
+		err := detectLockedError(n.Err(ctx))
 		if err != nil {
 			logrus.Errorf("cluster exited with error: %v", err)
 		}
 		c.Lock()
 		c.node = nil
 		c.err = err
+		if errors.Cause(err) == ErrSwarmLocked {
+			c.locked = true
+			confClone := conf
+			c.lastNodeConfig = &confClone
+		}
 		c.Unlock()
 		close(node.done)
 	}()
@@ -365,7 +385,7 @@ func (c *Cluster) startNewNode(conf nodeStartConfig) (*node, error) {
 // Init initializes new cluster from user provided request.
 func (c *Cluster) Init(req types.InitRequest) (string, error) {
 	c.Lock()
-	if node := c.node; node != nil {
+	if node := c.node; node != nil || c.locked {
 		if !req.ForceNewCluster {
 			c.Unlock()
 			return "", ErrSwarmExists
@@ -425,6 +445,7 @@ func (c *Cluster) Init(req types.InitRequest) (string, error) {
 	// todo: check current state existing
 	n, err := c.startNewNode(nodeStartConfig{
 		forceNewCluster: req.ForceNewCluster,
+		autolock:        req.AutoLockManagers,
 		LocalAddr:       localAddr,
 		ListenAddr:      net.JoinHostPort(listenHost, listenPort),
 		AdvertiseAddr:   net.JoinHostPort(advertiseHost, advertisePort),
@@ -457,7 +478,7 @@ func (c *Cluster) Init(req types.InitRequest) (string, error) {
 // Join makes current Cluster part of an existing swarm cluster.
 func (c *Cluster) Join(req types.JoinRequest) error {
 	c.Lock()
-	if node := c.node; node != nil {
+	if node := c.node; node != nil || c.locked {
 		c.Unlock()
 		return ErrSwarmExists
 	}
@@ -518,6 +539,66 @@ func (c *Cluster) Join(req types.JoinRequest) error {
 	}
 }
 
+// GetUnlockKey returns the unlock key for the swarm.
+func (c *Cluster) GetUnlockKey() (string, error) {
+	c.RLock()
+	defer c.RUnlock()
+
+	if !c.isActiveManager() {
+		return "", c.errNoManager()
+	}
+
+	ctx, cancel := c.getRequestContext()
+	defer cancel()
+
+	client := swarmapi.NewCAClient(c.conn)
+
+	r, err := client.GetUnlockKey(ctx, &swarmapi.GetUnlockKeyRequest{})
+	if err != nil {
+		return "", err
+	}
+
+	if len(r.UnlockKey) == 0 {
+		// no key
+		return "", nil
+	}
+
+	return encryption.HumanReadableKey(r.UnlockKey), nil
+}
+
+// UnlockSwarm provides a key to decrypt data that is encrypted at rest.
+func (c *Cluster) UnlockSwarm(req types.UnlockRequest) error {
+	key, err := encryption.ParseHumanReadableKey(req.UnlockKey)
+	if err != nil {
+		return err
+	}
+
+	c.Lock()
+	if c.node != nil || c.locked != true {
+		c.Unlock()
+		return errors.New("swarm is not locked")
+	}
+
+	config := *c.lastNodeConfig
+	config.lockKey = key
+	n, err := c.startNewNode(config)
+	if err != nil {
+		c.Unlock()
+		return err
+	}
+	c.Unlock()
+	select {
+	case <-n.Ready():
+	case <-n.done:
+		if errors.Cause(c.err) == ErrSwarmLocked {
+			return errors.New("swarm could not be unlocked: invalid key provided")
+		}
+		return fmt.Errorf("swarm component could not be started: %v", c.err)
+	}
+	go c.reconnectOnFailure(n)
+	return nil
+}
+
 // stopNode is a helper that stops the active c.node and waits until it has
 // shut down. Call while keeping the cluster lock.
 func (c *Cluster) stopNode() error {
@@ -555,47 +636,53 @@ func (c *Cluster) Leave(force bool) error {
 	c.Lock()
 	node := c.node
 	if node == nil {
-		c.Unlock()
-		return ErrNoSwarm
-	}
-
-	if node.Manager() != nil && !force {
-		msg := "You are attempting to leave the swarm on a node that is participating as a manager. "
-		if c.isActiveManager() {
-			active, reachable, unreachable, err := c.managerStats()
-			if err == nil {
-				if active && removingManagerCausesLossOfQuorum(reachable, unreachable) {
-					if isLastManager(reachable, unreachable) {
-						msg += "Removing the last manager erases all current state of the swarm. Use `--force` to ignore this message. "
-						c.Unlock()
-						return fmt.Errorf(msg)
-					}
-					msg += fmt.Sprintf("Removing this node leaves %v managers out of %v. Without a Raft quorum your swarm will be inaccessible. ", reachable-1, reachable+unreachable)
-				}
-			}
+		if c.locked {
+			c.locked = false
+			c.lastNodeConfig = nil
+			c.Unlock()
 		} else {
-			msg += "Doing so may lose the consensus of your cluster. "
+			c.Unlock()
+			return ErrNoSwarm
 		}
+	} else {
+		if node.Manager() != nil && !force {
+			msg := "You are attempting to leave the swarm on a node that is participating as a manager. "
+			if c.isActiveManager() {
+				active, reachable, unreachable, err := c.managerStats()
+				if err == nil {
+					if active && removingManagerCausesLossOfQuorum(reachable, unreachable) {
+						if isLastManager(reachable, unreachable) {
+							msg += "Removing the last manager erases all current state of the swarm. Use `--force` to ignore this message. "
+							c.Unlock()
+							return fmt.Errorf(msg)
+						}
+						msg += fmt.Sprintf("Removing this node leaves %v managers out of %v. Without a Raft quorum your swarm will be inaccessible. ", reachable-1, reachable+unreachable)
+					}
+				}
+			} else {
+				msg += "Doing so may lose the consensus of your cluster. "
+			}
 
-		msg += "The only way to restore a swarm that has lost consensus is to reinitialize it with `--force-new-cluster`. Use `--force` to suppress this message."
-		c.Unlock()
-		return fmt.Errorf(msg)
-	}
-	if err := c.stopNode(); err != nil {
-		logrus.Errorf("failed to shut down cluster node: %v", err)
-		signal.DumpStacks("")
-		c.Unlock()
-		return err
-	}
-	c.Unlock()
-	if nodeID := node.NodeID(); nodeID != "" {
-		nodeContainers, err := c.listContainerForNode(nodeID)
-		if err != nil {
+			msg += "The only way to restore a swarm that has lost consensus is to reinitialize it with `--force-new-cluster`. Use `--force` to suppress this message."
+			c.Unlock()
+			return fmt.Errorf(msg)
+		}
+		if err := c.stopNode(); err != nil {
+			logrus.Errorf("failed to shut down cluster node: %v", err)
+			signal.DumpStacks("")
+			c.Unlock()
 			return err
 		}
-		for _, id := range nodeContainers {
-			if err := c.config.Backend.ContainerRm(id, &apitypes.ContainerRmConfig{ForceRemove: true}); err != nil {
-				logrus.Errorf("error removing %v: %v", id, err)
+		c.Unlock()
+		if nodeID := node.NodeID(); nodeID != "" {
+			nodeContainers, err := c.listContainerForNode(nodeID)
+			if err != nil {
+				return err
+			}
+			for _, id := range nodeContainers {
+				if err := c.config.Backend.ContainerRm(id, &apitypes.ContainerRmConfig{ForceRemove: true}); err != nil {
+					logrus.Errorf("error removing %v: %v", id, err)
+				}
 			}
 		}
 	}
@@ -692,9 +779,10 @@ func (c *Cluster) Update(version uint64, spec types.Spec, flags types.UpdateFlag
 			ClusterVersion: &swarmapi.Version{
 				Index: version,
 			},
-			Rotation: swarmapi.JoinTokenRotation{
-				RotateWorkerToken:  flags.RotateWorkerToken,
-				RotateManagerToken: flags.RotateManagerToken,
+			Rotation: swarmapi.KeyRotation{
+				WorkerJoinToken:  flags.RotateWorkerToken,
+				ManagerJoinToken: flags.RotateManagerToken,
+				ManagerUnlockKey: flags.RotateManagerUnlockKey,
 			},
 		},
 	)
@@ -786,10 +874,15 @@ func (c *Cluster) Info() types.Info {
 		if c.cancelDelay != nil {
 			info.LocalNodeState = types.LocalNodeStateError
 		}
+		if c.locked {
+			info.LocalNodeState = types.LocalNodeStateLocked
+		}
 	} else {
 		info.LocalNodeState = types.LocalNodeStatePending
 		if c.ready == true {
 			info.LocalNodeState = types.LocalNodeStateActive
+		} else if c.locked {
+			info.LocalNodeState = types.LocalNodeStateLocked
 		}
 	}
 	if c.err != nil {
@@ -838,6 +931,9 @@ func (c *Cluster) isActiveManager() bool {
 // Call with read lock.
 func (c *Cluster) errNoManager() error {
 	if c.node == nil {
+		if c.locked {
+			return ErrSwarmLocked
+		}
 		return fmt.Errorf("This node is not a swarm manager. Use \"docker swarm init\" or \"docker swarm join\" to connect this node to swarm and try again.")
 	}
 	if c.node.Manager() != nil {
@@ -1396,7 +1492,7 @@ func (c *Cluster) CreateNetwork(s apitypes.NetworkCreateRequest) (string, error)
 
 	if runconfig.IsPreDefinedNetwork(s.Name) {
 		err := fmt.Errorf("%s is a pre-defined network and cannot be created", s.Name)
-		return "", errors.NewRequestForbiddenError(err)
+		return "", apierrors.NewRequestForbiddenError(err)
 	}
 
 	ctx, cancel := c.getRequestContext()
@@ -1447,7 +1543,7 @@ func (c *Cluster) populateNetworkID(ctx context.Context, client swarmapi.Control
 		if err != nil {
 			if ln, _ := c.config.Backend.FindNetwork(n.Target); ln != nil && !ln.Info().Dynamic() {
 				err = fmt.Errorf("network %s is not eligible for docker services", ln.Name())
-				return errors.NewRequestForbiddenError(err)
+				return apierrors.NewRequestForbiddenError(err)
 			}
 			return err
 		}
@@ -1613,3 +1709,10 @@ func initClusterSpec(node *node, spec types.Spec) error {
 	}
 	return ctx.Err()
 }
+
+func detectLockedError(err error) error {
+	if err == swarmnode.ErrInvalidUnlockKey {
+		return errors.WithStack(ErrSwarmLocked)
+	}
+	return err
+}

daemon/cluster/convert/swarm.go

@@ -26,6 +26,9 @@ func SwarmFromGRPC(c swarmapi.Cluster) types.Swarm {
 					HeartbeatTick:              int(c.Spec.Raft.HeartbeatTick),
 					ElectionTick:               int(c.Spec.Raft.ElectionTick),
 				},
+				EncryptionConfig: types.EncryptionConfig{
+					AutoLockManagers: c.Spec.EncryptionConfig.AutoLockManagers,
+				},
 			},
 		},
 		JoinTokens: types.JoinTokens{
@@ -113,5 +116,7 @@ func MergeSwarmSpecToGRPC(s types.Spec, spec swarmapi.ClusterSpec) (swarmapi.Clu
 		})
 	}
 
+	spec.EncryptionConfig.AutoLockManagers = s.EncryptionConfig.AutoLockManagers
+
 	return spec, nil
 }

docs/reference/api/docker_remote_api_v1.25.md

@@ -4724,18 +4724,21 @@ Inspect swarm
           "ElectionTick" : 3
         },
         "TaskDefaults" : {},
+        "EncryptionConfig" : {
+          "AutoLockManagers": false
+        },
         "Name" : "default"
       },
-     "JoinTokens" : {
+      "JoinTokens" : {
         "Worker" : "SWMTKN-1-1h8aps2yszaiqmz2l3oc5392pgk8e49qhx2aj3nyv0ui0hez2a-6qmn92w6bu3jdvnglku58u11a",
         "Manager" : "SWMTKN-1-1h8aps2yszaiqmz2l3oc5392pgk8e49qhx2aj3nyv0ui0hez2a-8llk83c4wm9lwioey2s316r9l"
-     },
-     "ID" : "70ilmkj2f6sp2137c753w2nmt",
-     "UpdatedAt" : "2016-08-15T16:32:09.623207604Z",
-     "Version" : {
-       "Index" : 51
+      },
+      "ID" : "70ilmkj2f6sp2137c753w2nmt",
+      "UpdatedAt" : "2016-08-15T16:32:09.623207604Z",
+      "Version" : {
+        "Index" : 51
+      }
     }
-  }
 
 **Status codes**:
 
@@ -4761,7 +4764,10 @@ Initialize a new swarm. The body of the HTTP response includes the node ID.
         "Orchestration": {},
         "Raft": {},
         "Dispatcher": {},
-        "CAConfig": {}
+        "CAConfig": {},
+        "EncryptionConfig" : {
+          "AutoLockManagers": false
+        }
       }
     }
 
@@ -4816,6 +4822,9 @@ JSON Parameters:
             - **URL** - URL where certificate signing requests should be sent.
             - **Options** - An object with key/value pairs that are interpreted
               as protocol-specific options for the external CA driver.
+    - **EncryptionConfig** – Parameters related to encryption-at-rest.
+        - **AutoLockManagers**: If set, generate a key and use it to lock data stored on the
+          managers.
 
 ### Join an existing swarm
 
@@ -4885,6 +4894,44 @@ Leave a swarm
 - **200** – no error
 - **406** – node is not part of a swarm
 
+### Retrieve the swarm's unlock key
+
+`GET /swarm/unlockkey`
+
+Get unlock key
+
+**Example response**:
+
+    HTTP/1.1 200 OK
+    Content-Type: application/json
+
+    {
+      "UnlockKey": "SWMKEY-1-7c37Cc8654o6p38HnroywCi19pllOnGtbdZEgtKxZu8"
+    }
+
+**Status codes**:
+
+- **200** - no error
+
+### Unlock a locked manager
+
+`POST /swarm/unlock`
+
+Unlock a manager
+
+**Example request**:
+
+    POST /v1.25/swarm/unlock HTTP/1.1
+    Content-Type: application/json
+
+    {
+      "UnlockKey": "SWMKEY-1-7c37Cc8654o6p38HnroywCi19pllOnGtbdZEgtKxZu8"
+    }
+
+**Status codes**:
+
+- **200** - no error
+
 ### Update a swarm
 
 
@@ -4916,6 +4963,9 @@ Update a swarm
       "JoinTokens": {
         "Worker": "SWMTKN-1-3pu6hszjas19xyp7ghgosyx9k8atbfcr8p2is99znpy26u2lkl-1awxwuwd3z9j1z3puu7rcgdbx",
         "Manager": "SWMTKN-1-3pu6hszjas19xyp7ghgosyx9k8atbfcr8p2is99znpy26u2lkl-7p73s1dx5in4tatdymyhg9hu2"
+      },
+      "EncryptionConfig": {
+        "AutoLockManagers": false
       }
     }
 
@@ -4932,6 +4982,7 @@ Update a swarm
   required to avoid conflicting writes.
 - **rotateWorkerToken** - Set to `true` (or `1`) to rotate the worker join token.
 - **rotateManagerToken** - Set to `true` (or `1`) to rotate the manager join token.
+- **rotateManagerUnlockKey** - Set to `true` (or `1`) to rotate the manager unlock key.
 
 **Status codes**:
 
@@ -4965,6 +5016,9 @@ JSON Parameters:
 - **JoinTokens** - Tokens that can be used by other nodes to join the swarm.
     - **Worker** - Token to use for joining as a worker.
     - **Manager** - Token to use for joining as a manager.
+- **EncryptionConfig** – Parameters related to encryption-at-rest.
+    - **AutoLockManagers**: If set, generate a key and use it to lock data stored on the
+      managers.
 
 ## 3.9 Services
 

docs/reference/commandline/swarm_init.md

@@ -22,6 +22,7 @@ Initialize a swarm
 
 Options:
       --advertise-addr value            Advertised address (format: <ip|interface>[:port])
+      --autolock                        Enable or disable manager autolocking (requiring an unlock key to start a stopped manager)
       --cert-expiry duration            Validity period for node certificates (ns|us|ms|s|m|h) (default 2160h0m0s)
       --dispatcher-heartbeat duration   Dispatcher heartbeat period (ns|us|ms|s|m|h) (default 5s)
       --external-ca value               Specifications of one or more certificate signing endpoints
@@ -57,6 +58,18 @@ to [swarm join](swarm_join.md).
 After you create the swarm, you can display or rotate the token using
 [swarm join-token](swarm_join_token.md).
 
+### `--autolock`
+
+This flag enables automatic locking of managers with an encryption key. The
+private keys and data stored by all managers will be protected by the
+encryption key printed in the output, and will not be accessible without it.
+Thus, it is very important to store this key in order to activate a manager
+after it restarts. The key can be passed to `docker swarm unlock` to reactivate
+the manager. Autolock can be disabled by running
+`docker swarm update --autolock=false`. After disabling it, the encryption key
+is no longer required to start the manager, and it will start up on its own
+without user intervention.
+
 ### `--cert-expiry`
 
 This flag sets the validity period for node certificates.

docs/reference/commandline/swarm_unlock.md

@@ -0,0 +1,41 @@
+---
+title: "swarm unlock"
+description: "The swarm unlock command description and usage"
+keywords: "swarm, unlock"
+---
+
+<!-- This file is maintained within the docker/docker Github
+     repository at https://github.com/docker/docker/. Make all
+     pull requests against that repo. If you see this file in
+     another repository, consider it read-only there, as it will
+     periodically be overwritten by the definitive file. Pull
+     requests which include edits to this file in other repositories
+     will be rejected.
+-->
+
+# swarm unlock
+
+```markdown
+Usage:	docker swarm unlock
+
+Unlock swarm
+
+Options:
+      --help   Print usage
+```
+
+Unlocks a locked manager using a user-supplied unlock key. This command must be
+used to reactivate a manager after its Docker daemon restarts if the autolock
+setting is turned on. The unlock key is printed at the time when autolock is
+enabled, and is also available from the `docker swarm unlock-key` command.
+
+
+```bash
+$ docker swarm unlock
+Please enter unlock key:
+```
+
+## Related information
+
+* [swarm init](swarm_init.md)
+* [swarm update](swarm_update.md)

docs/reference/commandline/swarm_unlock_key.md

@@ -0,0 +1,84 @@
+---
+title: "swarm unlock-key"
+description: "The swarm unlock-keycommand description and usage"
+keywords: "swarm, unlock-key"
+---
+
+<!-- This file is maintained within the docker/docker Github
+     repository at https://github.com/docker/docker/. Make all
+     pull requests against that repo. If you see this file in
+     another repository, consider it read-only there, as it will
+     periodically be overwritten by the definitive file. Pull
+     requests which include edits to this file in other repositories
+     will be rejected.
+-->
+
+# swarm unlock-key
+
+```markdown
+Usage:	docker swarm unlock-key [OPTIONS]
+
+Manage the unlock key
+
+Options:
+      --help     Print usage
+  -q, --quiet    Only display token
+      --rotate   Rotate unlock key
+```
+
+An unlock key is a secret key needed to unlock a manager after its Docker daemon
+restarts. These keys are only used when the autolock feature is enabled for the
+swarm.
+
+You can view or rotate the unlock key using `swarm unlock-key`. To view the key,
+run the `docker swarm unlock-key` command without any arguments:
+
+
+```bash
+$ docker swarm unlock-key
+To unlock a swarm manager after it restarts, run the `docker swarm unlock`
+command and provide the following key:
+
+    SWMKEY-1-fySn8TY4w5lKcWcJPIpKufejh9hxx5KYwx6XZigx3Q4
+
+Please remember to store this key in a password manager, since without it you
+will not be able to restart the manager.
+```
+
+Use the `--rotate` flag to rotate the unlock key to a new, randomly-generated
+key:
+
+```bash
+$ docker swarm unlock-key --rotate
+Successfully rotated manager unlock key.
+
+To unlock a swarm manager after it restarts, run the `docker swarm unlock`
+command and provide the following key:
+
+    SWMKEY-1-7c37Cc8654o6p38HnroywCi19pllOnGtbdZEgtKxZu8
+
+Please remember to store this key in a password manager, since without it you
+will not be able to restart the manager.
+```
+
+The `-q` (or `--quiet`) flag only prints the key:
+
+```bash
+$ docker swarm unlock-key -q
+SWMKEY-1-7c37Cc8654o6p38HnroywCi19pllOnGtbdZEgtKxZu8
+```
+
+### `--rotate`
+
+This flag rotates the unlock key, replacing it with a new randomly-generated
+key. The old unlock key will no longer be accepted.
+
+### `--quiet`
+
+Only print the unlock key, without instructions.
+
+## Related information
+
+* [swarm unlock](swarm_unlock.md)
+* [swarm init](swarm_init.md)
+* [swarm update](swarm_update.md)

docs/reference/commandline/swarm_update.md

@@ -21,6 +21,7 @@ Usage:  docker swarm update [OPTIONS]
 Update the swarm
 
 Options:
+      --autolock                        Enable or disable manager autolocking (requiring an unlock key to start a stopped manager)
       --cert-expiry duration            Validity period for node certificates (ns|us|ms|s|m|h) (default 2160h0m0s)
       --dispatcher-heartbeat duration   Dispatcher heartbeat period (ns|us|ms|s|m|h) (default 5s)
       --external-ca value               Specifications of one or more certificate signing endpoints

integration-cli/daemon.go

@@ -463,11 +463,14 @@ func (d *Daemon) getBaseDeviceSize(c *check.C) int64 {
 // Cmd will execute a docker CLI command against this Daemon.
 // Example: d.Cmd("version") will run docker -H unix://path/to/unix.sock version
 func (d *Daemon) Cmd(args ...string) (string, error) {
-	c := exec.Command(dockerBinary, d.prependHostArg(args)...)
-	b, err := c.CombinedOutput()
+	b, err := d.command(args...).CombinedOutput()
 	return string(b), err
 }
 
+func (d *Daemon) command(args ...string) *exec.Cmd {
+	return exec.Command(dockerBinary, d.prependHostArg(args)...)
+}
+
 func (d *Daemon) prependHostArg(args []string) []string {
 	for _, arg := range args {
 		if arg == "--host" || arg == "-H" {

integration-cli/docker_cli_swarm_test.go

@@ -3,6 +3,7 @@
 package main
 
 import (
+	"bytes"
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
@@ -850,3 +851,180 @@ func (s *DockerSwarmSuite) TestDNSConfigUpdate(c *check.C) {
 	c.Assert(err, checker.IsNil)
 	c.Assert(strings.TrimSpace(out), checker.Equals, "{[1.2.3.4] [example.com] [timeout:3]}")
 }
+
+func (s *DockerSwarmSuite) TestSwarmInitLocked(c *check.C) {
+	d := s.AddDaemon(c, false, false)
+
+	outs, err := d.Cmd("swarm", "init", "--autolock")
+	c.Assert(err, checker.IsNil, check.Commentf("out: %v", outs))
+
+	c.Assert(outs, checker.Contains, "docker swarm unlock")
+
+	var unlockKey string
+	for _, line := range strings.Split(outs, "\n") {
+		if strings.Contains(line, "SWMKEY") {
+			unlockKey = strings.TrimSpace(line)
+			break
+		}
+	}
+
+	c.Assert(unlockKey, checker.Not(checker.Equals), "")
+
+	outs, err = d.Cmd("swarm", "unlock-key", "-q")
+	c.Assert(outs, checker.Equals, unlockKey+"\n")
+
+	info, err := d.info()
+	c.Assert(err, checker.IsNil)
+	c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive)
+
+	c.Assert(d.Restart(), checker.IsNil)
+
+	info, err = d.info()
+	c.Assert(err, checker.IsNil)
+	c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateLocked)
+
+	cmd := d.command("swarm", "unlock")
+	cmd.Stdin = bytes.NewBufferString("wrong-secret-key")
+	out, err := cmd.CombinedOutput()
+	c.Assert(err, checker.NotNil, check.Commentf("out: %v", string(out)))
+	c.Assert(string(out), checker.Contains, "invalid key")
+
+	cmd = d.command("swarm", "unlock")
+	cmd.Stdin = bytes.NewBufferString(unlockKey)
+	out, err = cmd.CombinedOutput()
+	c.Assert(err, checker.IsNil, check.Commentf("out: %v", string(out)))
+
+	info, err = d.info()
+	c.Assert(err, checker.IsNil)
+	c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive)
+
+	outs, err = d.Cmd("node", "ls")
+	c.Assert(err, checker.IsNil)
+	c.Assert(outs, checker.Not(checker.Contains), "Swarm is encrypted and needs to be unlocked")
+
+	outs, err = d.Cmd("swarm", "update", "--autolock=false")
+	c.Assert(err, checker.IsNil, check.Commentf("out: %v", outs))
+
+	// Wait for autolock to be turned off
+	time.Sleep(time.Second)
+
+	c.Assert(d.Restart(), checker.IsNil)
+
+	info, err = d.info()
+	c.Assert(err, checker.IsNil)
+	c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive)
+
+	outs, err = d.Cmd("node", "ls")
+	c.Assert(err, checker.IsNil)
+	c.Assert(outs, checker.Not(checker.Contains), "Swarm is encrypted and needs to be unlocked")
+}
+
+func (s *DockerSwarmSuite) TestSwarmLeaveLocked(c *check.C) {
+	d := s.AddDaemon(c, false, false)
+
+	outs, err := d.Cmd("swarm", "init", "--autolock")
+	c.Assert(err, checker.IsNil, check.Commentf("out: %v", outs))
+
+	c.Assert(d.Restart("--swarm-default-advertise-addr=lo"), checker.IsNil)
+
+	info, err := d.info()
+	c.Assert(err, checker.IsNil)
+	c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateLocked)
+
+	outs, _ = d.Cmd("node", "ls")
+	c.Assert(outs, checker.Contains, "Swarm is encrypted and needs to be unlocked")
+
+	outs, err = d.Cmd("swarm", "leave", "--force")
+	c.Assert(err, checker.IsNil, check.Commentf("out: %v", outs))
+
+	info, err = d.info()
+	c.Assert(err, checker.IsNil)
+	c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateInactive)
+
+	outs, err = d.Cmd("swarm", "init")
+	c.Assert(err, checker.IsNil, check.Commentf("out: %v", outs))
+
+	info, err = d.info()
+	c.Assert(err, checker.IsNil)
+	c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive)
+}
+
+func (s *DockerSwarmSuite) TestSwarmRotateUnlockKey(c *check.C) {
+	d := s.AddDaemon(c, true, true)
+
+	outs, err := d.Cmd("swarm", "update", "--autolock")
+	c.Assert(err, checker.IsNil, check.Commentf("out: %v", outs))
+
+	c.Assert(outs, checker.Contains, "docker swarm unlock")
+
+	var unlockKey string
+	for _, line := range strings.Split(outs, "\n") {
+		if strings.Contains(line, "SWMKEY") {
+			unlockKey = strings.TrimSpace(line)
+			break
+		}
+	}
+
+	c.Assert(unlockKey, checker.Not(checker.Equals), "")
+
+	outs, err = d.Cmd("swarm", "unlock-key", "-q")
+	c.Assert(outs, checker.Equals, unlockKey+"\n")
+
+	// Rotate multiple times
+	for i := 0; i != 3; i++ {
+		outs, err = d.Cmd("swarm", "unlock-key", "-q", "--rotate")
+		c.Assert(err, checker.IsNil, check.Commentf("out: %v", outs))
+		// Strip \n
+		newUnlockKey := outs[:len(outs)-1]
+		c.Assert(newUnlockKey, checker.Not(checker.Equals), "")
+
+		c.Assert(d.Restart(), checker.IsNil)
+
+		info, err := d.info()
+		c.Assert(err, checker.IsNil)
+		c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateLocked)
+
+		outs, _ = d.Cmd("node", "ls")
+		c.Assert(outs, checker.Contains, "Swarm is encrypted and needs to be unlocked")
+
+		cmd := d.command("swarm", "unlock")
+		cmd.Stdin = bytes.NewBufferString(unlockKey)
+		out, err := cmd.CombinedOutput()
+
+		if err == nil {
+			// On occasion, the daemon may not have finished
+			// rotating the KEK before restarting. The test is
+			// intentionally written to explore this behavior.
+			// When this happens, unlocking with the old key will
+			// succeed. If we wait for the rotation to happen and
+			// restart again, the new key should be required this
+			// time.
+
+			time.Sleep(3 * time.Second)
+
+			c.Assert(d.Restart(), checker.IsNil)
+
+			cmd = d.command("swarm", "unlock")
+			cmd.Stdin = bytes.NewBufferString(unlockKey)
+			out, err = cmd.CombinedOutput()
+		}
+		c.Assert(err, checker.NotNil, check.Commentf("out: %v", string(out)))
+		c.Assert(string(out), checker.Contains, "invalid key")
+
+		outs, _ = d.Cmd("node", "ls")
+		c.Assert(outs, checker.Contains, "Swarm is encrypted and needs to be unlocked")
+
+		cmd = d.command("swarm", "unlock")
+		cmd.Stdin = bytes.NewBufferString(newUnlockKey)
+		out, err = cmd.CombinedOutput()
+		c.Assert(err, checker.IsNil, check.Commentf("out: %v", string(out)))
+
+		info, err = d.info()
+		c.Assert(err, checker.IsNil)
+		c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive)
+
+		outs, err = d.Cmd("node", "ls")
+		c.Assert(err, checker.IsNil)
+		c.Assert(outs, checker.Not(checker.Contains), "Swarm is encrypted and needs to be unlocked")
+	}
+}

vendor.conf

@@ -100,7 +100,7 @@ github.com/docker/containerd 8517738ba4b82aff5662c97ca4627e7e4d03b531
 github.com/tonistiigi/fifo 1405643975692217d6720f8b54aeee1bf2cd5cf4
 
 # cluster
-github.com/docker/swarmkit 00890359d8bfba630824b66b848dbf7851149fef
+github.com/docker/swarmkit bddd3f0fb45491987d3dec5fb48311d289d21393
 github.com/golang/mock bd3c8e81be01eef76d4b503f5e687d2d1354d2d9
 github.com/gogo/protobuf v0.3
 github.com/cloudflare/cfssl 7fb22c8cba7ecaf98e4082d22d65800cf45e042a
@@ -120,7 +120,7 @@ github.com/prometheus/common ebdfc6da46522d58825777cf1f90490a5b1ef1d8
 github.com/prometheus/procfs abf152e5f3e97f2fafac028d2cc06c1feb87ffa5
 bitbucket.org/ww/goautoneg 75cd24fc2f2c2a2088577d12123ddee5f54e0675
 github.com/matttproud/golang_protobuf_extensions fc2b8d3a73c4867e51861bbdd5ae3c1f0869dd6a
-github.com/pkg/errors 01fa4104b9c248c8945d14d9f128454d5b28d595
+github.com/pkg/errors 839d9e913e063e28dfd0e6c7b7512793e0a48be9
 
 # cli
 github.com/spf13/cobra v1.5 https://github.com/dnephin/cobra.git

vendor/github.com/docker/swarmkit/api/ca.pb.go

@@ -89,6 +89,22 @@ func (m *GetRootCACertificateResponse) Reset()                    { *m = GetRoot
 func (*GetRootCACertificateResponse) ProtoMessage()               {}
 func (*GetRootCACertificateResponse) Descriptor() ([]byte, []int) { return fileDescriptorCa, []int{5} }
 
+type GetUnlockKeyRequest struct {
+}
+
+func (m *GetUnlockKeyRequest) Reset()                    { *m = GetUnlockKeyRequest{} }
+func (*GetUnlockKeyRequest) ProtoMessage()               {}
+func (*GetUnlockKeyRequest) Descriptor() ([]byte, []int) { return fileDescriptorCa, []int{6} }
+
+type GetUnlockKeyResponse struct {
+	UnlockKey []byte  `protobuf:"bytes,1,opt,name=unlock_key,json=unlockKey,proto3" json:"unlock_key,omitempty"`
+	Version   Version `protobuf:"bytes,2,opt,name=version" json:"version"`
+}
+
+func (m *GetUnlockKeyResponse) Reset()                    { *m = GetUnlockKeyResponse{} }
+func (*GetUnlockKeyResponse) ProtoMessage()               {}
+func (*GetUnlockKeyResponse) Descriptor() ([]byte, []int) { return fileDescriptorCa, []int{7} }
+
 func init() {
 	proto.RegisterType((*NodeCertificateStatusRequest)(nil), "docker.swarmkit.v1.NodeCertificateStatusRequest")
 	proto.RegisterType((*NodeCertificateStatusResponse)(nil), "docker.swarmkit.v1.NodeCertificateStatusResponse")
@@ -96,6 +112,8 @@ func init() {
 	proto.RegisterType((*IssueNodeCertificateResponse)(nil), "docker.swarmkit.v1.IssueNodeCertificateResponse")
 	proto.RegisterType((*GetRootCACertificateRequest)(nil), "docker.swarmkit.v1.GetRootCACertificateRequest")
 	proto.RegisterType((*GetRootCACertificateResponse)(nil), "docker.swarmkit.v1.GetRootCACertificateResponse")
+	proto.RegisterType((*GetUnlockKeyRequest)(nil), "docker.swarmkit.v1.GetUnlockKeyRequest")
+	proto.RegisterType((*GetUnlockKeyResponse)(nil), "docker.swarmkit.v1.GetUnlockKeyResponse")
 }
 
 type authenticatedWrapperCAServer struct {
@@ -115,6 +133,14 @@ func (p *authenticatedWrapperCAServer) GetRootCACertificate(ctx context.Context,
 	return p.local.GetRootCACertificate(ctx, r)
 }
 
+func (p *authenticatedWrapperCAServer) GetUnlockKey(ctx context.Context, r *GetUnlockKeyRequest) (*GetUnlockKeyResponse, error) {
+
+	if err := p.authorize(ctx, []string{"swarm-manager"}); err != nil {
+		return nil, err
+	}
+	return p.local.GetUnlockKey(ctx, r)
+}
+
 type authenticatedWrapperNodeCAServer struct {
 	local     NodeCAServer
 	authorize func(context.Context, []string) error
@@ -211,6 +237,29 @@ func (m *GetRootCACertificateResponse) Copy() *GetRootCACertificateResponse {
 	return o
 }
 
+func (m *GetUnlockKeyRequest) Copy() *GetUnlockKeyRequest {
+	if m == nil {
+		return nil
+	}
+
+	o := &GetUnlockKeyRequest{}
+
+	return o
+}
+
+func (m *GetUnlockKeyResponse) Copy() *GetUnlockKeyResponse {
+	if m == nil {
+		return nil
+	}
+
+	o := &GetUnlockKeyResponse{
+		UnlockKey: m.UnlockKey,
+		Version:   *m.Version.Copy(),
+	}
+
+	return o
+}
+
 func (this *NodeCertificateStatusRequest) GoString() string {
 	if this == nil {
 		return "nil"
@@ -278,6 +327,26 @@ func (this *GetRootCACertificateResponse) GoString() string {
 	s = append(s, "}")
 	return strings.Join(s, "")
 }
+func (this *GetUnlockKeyRequest) GoString() string {
+	if this == nil {
+		return "nil"
+	}
+	s := make([]string, 0, 4)
+	s = append(s, "&api.GetUnlockKeyRequest{")
+	s = append(s, "}")
+	return strings.Join(s, "")
+}
+func (this *GetUnlockKeyResponse) GoString() string {
+	if this == nil {
+		return "nil"
+	}
+	s := make([]string, 0, 6)
+	s = append(s, "&api.GetUnlockKeyResponse{")
+	s = append(s, "UnlockKey: "+fmt.Sprintf("%#v", this.UnlockKey)+",\n")
+	s = append(s, "Version: "+strings.Replace(this.Version.GoString(), `&`, ``, 1)+",\n")
+	s = append(s, "}")
+	return strings.Join(s, "")
+}
 func valueToGoStringCa(v interface{}, typ string) string {
 	rv := reflect.ValueOf(v)
 	if rv.IsNil() {
@@ -317,6 +386,9 @@ const _ = grpc.SupportPackageIsVersion3
 
 type CAClient interface {
 	GetRootCACertificate(ctx context.Context, in *GetRootCACertificateRequest, opts ...grpc.CallOption) (*GetRootCACertificateResponse, error)
+	// GetUnlockKey returns the current unlock key for the cluster for the role of the client
+	// asking.
+	GetUnlockKey(ctx context.Context, in *GetUnlockKeyRequest, opts ...grpc.CallOption) (*GetUnlockKeyResponse, error)
 }
 
 type cAClient struct {
@@ -336,10 +408,22 @@ func (c *cAClient) GetRootCACertificate(ctx context.Context, in *GetRootCACertif
 	return out, nil
 }
 
+func (c *cAClient) GetUnlockKey(ctx context.Context, in *GetUnlockKeyRequest, opts ...grpc.CallOption) (*GetUnlockKeyResponse, error) {
+	out := new(GetUnlockKeyResponse)
+	err := grpc.Invoke(ctx, "/docker.swarmkit.v1.CA/GetUnlockKey", in, out, c.cc, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
 // Server API for CA service
 
 type CAServer interface {
 	GetRootCACertificate(context.Context, *GetRootCACertificateRequest) (*GetRootCACertificateResponse, error)
+	// GetUnlockKey returns the current unlock key for the cluster for the role of the client
+	// asking.
+	GetUnlockKey(context.Context, *GetUnlockKeyRequest) (*GetUnlockKeyResponse, error)
 }
 
 func RegisterCAServer(s *grpc.Server, srv CAServer) {
@@ -364,6 +448,24 @@ func _CA_GetRootCACertificate_Handler(srv interface{}, ctx context.Context, dec
 	return interceptor(ctx, in, info, handler)
 }
 
+func _CA_GetUnlockKey_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(GetUnlockKeyRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(CAServer).GetUnlockKey(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: "/docker.swarmkit.v1.CA/GetUnlockKey",
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(CAServer).GetUnlockKey(ctx, req.(*GetUnlockKeyRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
 var _CA_serviceDesc = grpc.ServiceDesc{
 	ServiceName: "docker.swarmkit.v1.CA",
 	HandlerType: (*CAServer)(nil),
@@ -372,6 +474,10 @@ var _CA_serviceDesc = grpc.ServiceDesc{
 			MethodName: "GetRootCACertificate",
 			Handler:    _CA_GetRootCACertificate_Handler,
 		},
+		{
+			MethodName: "GetUnlockKey",
+			Handler:    _CA_GetUnlockKey_Handler,
+		},
 	},
 	Streams:  []grpc.StreamDesc{},
 	Metadata: fileDescriptorCa,
@@ -642,6 +748,56 @@ func (m *GetRootCACertificateResponse) MarshalTo(data []byte) (int, error) {
 	return i, nil
 }
 
+func (m *GetUnlockKeyRequest) Marshal() (data []byte, err error) {
+	size := m.Size()
+	data = make([]byte, size)
+	n, err := m.MarshalTo(data)
+	if err != nil {
+		return nil, err
+	}
+	return data[:n], nil
+}
+
+func (m *GetUnlockKeyRequest) MarshalTo(data []byte) (int, error) {
+	var i int
+	_ = i
+	var l int
+	_ = l
+	return i, nil
+}
+
+func (m *GetUnlockKeyResponse) Marshal() (data []byte, err error) {
+	size := m.Size()
+	data = make([]byte, size)
+	n, err := m.MarshalTo(data)
+	if err != nil {
+		return nil, err
+	}
+	return data[:n], nil
+}
+
+func (m *GetUnlockKeyResponse) MarshalTo(data []byte) (int, error) {
+	var i int
+	_ = i
+	var l int
+	_ = l
+	if len(m.UnlockKey) > 0 {
+		data[i] = 0xa
+		i++
+		i = encodeVarintCa(data, i, uint64(len(m.UnlockKey)))
+		i += copy(data[i:], m.UnlockKey)
+	}
+	data[i] = 0x12
+	i++
+	i = encodeVarintCa(data, i, uint64(m.Version.Size()))
+	n3, err := m.Version.MarshalTo(data[i:])
+	if err != nil {
+		return 0, err
+	}
+	i += n3
+	return i, nil
+}
+
 func encodeFixed64Ca(data []byte, offset int, v uint64) int {
 	data[offset] = uint8(v)
 	data[offset+1] = uint8(v >> 8)
@@ -767,6 +923,37 @@ func (p *raftProxyCAServer) GetRootCACertificate(ctx context.Context, r *GetRoot
 	return resp, err
 }
 
+func (p *raftProxyCAServer) GetUnlockKey(ctx context.Context, r *GetUnlockKeyRequest) (*GetUnlockKeyResponse, error) {
+
+	conn, err := p.connSelector.LeaderConn(ctx)
+	if err != nil {
+		if err == raftselector.ErrIsLeader {
+			return p.local.GetUnlockKey(ctx, r)
+		}
+		return nil, err
+	}
+	modCtx, err := p.runCtxMods(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	resp, err := NewCAClient(conn).GetUnlockKey(modCtx, r)
+	if err != nil {
+		if !strings.Contains(err.Error(), "is closing") && !strings.Contains(err.Error(), "the connection is unavailable") && !strings.Contains(err.Error(), "connection error") {
+			return resp, err
+		}
+		conn, err := p.pollNewLeaderConn(ctx)
+		if err != nil {
+			if err == raftselector.ErrIsLeader {
+				return p.local.GetUnlockKey(ctx, r)
+			}
+			return nil, err
+		}
+		return NewCAClient(conn).GetUnlockKey(modCtx, r)
+	}
+	return resp, err
+}
+
 type raftProxyNodeCAServer struct {
 	local        NodeCAServer
 	connSelector raftselector.ConnProvider
@@ -965,6 +1152,24 @@ func (m *GetRootCACertificateResponse) Size() (n int) {
 	return n
 }
 
+func (m *GetUnlockKeyRequest) Size() (n int) {
+	var l int
+	_ = l
+	return n
+}
+
+func (m *GetUnlockKeyResponse) Size() (n int) {
+	var l int
+	_ = l
+	l = len(m.UnlockKey)
+	if l > 0 {
+		n += 1 + l + sovCa(uint64(l))
+	}
+	l = m.Version.Size()
+	n += 1 + l + sovCa(uint64(l))
+	return n
+}
+
 func sovCa(x uint64) (n int) {
 	for {
 		n++
@@ -1041,6 +1246,26 @@ func (this *GetRootCACertificateResponse) String() string {
 	}, "")
 	return s
 }
+func (this *GetUnlockKeyRequest) String() string {
+	if this == nil {
+		return "nil"
+	}
+	s := strings.Join([]string{`&GetUnlockKeyRequest{`,
+		`}`,
+	}, "")
+	return s
+}
+func (this *GetUnlockKeyResponse) String() string {
+	if this == nil {
+		return "nil"
+	}
+	s := strings.Join([]string{`&GetUnlockKeyResponse{`,
+		`UnlockKey:` + fmt.Sprintf("%v", this.UnlockKey) + `,`,
+		`Version:` + strings.Replace(strings.Replace(this.Version.String(), "Version", "Version", 1), `&`, ``, 1) + `,`,
+		`}`,
+	}, "")
+	return s
+}
 func valueToStringCa(v interface{}) string {
 	rv := reflect.ValueOf(v)
 	if rv.IsNil() {
@@ -1602,6 +1827,167 @@ func (m *GetRootCACertificateResponse) Unmarshal(data []byte) error {
 	}
 	return nil
 }
+func (m *GetUnlockKeyRequest) Unmarshal(data []byte) error {
+	l := len(data)
+	iNdEx := 0
+	for iNdEx < l {
+		preIndex := iNdEx
+		var wire uint64
+		for shift := uint(0); ; shift += 7 {
+			if shift >= 64 {
+				return ErrIntOverflowCa
+			}
+			if iNdEx >= l {
+				return io.ErrUnexpectedEOF
+			}
+			b := data[iNdEx]
+			iNdEx++
+			wire |= (uint64(b) & 0x7F) << shift
+			if b < 0x80 {
+				break
+			}
+		}
+		fieldNum := int32(wire >> 3)
+		wireType := int(wire & 0x7)
+		if wireType == 4 {
+			return fmt.Errorf("proto: GetUnlockKeyRequest: wiretype end group for non-group")
+		}
+		if fieldNum <= 0 {
+			return fmt.Errorf("proto: GetUnlockKeyRequest: illegal tag %d (wire type %d)", fieldNum, wire)
+		}
+		switch fieldNum {
+		default:
+			iNdEx = preIndex
+			skippy, err := skipCa(data[iNdEx:])
+			if err != nil {
+				return err
+			}
+			if skippy < 0 {
+				return ErrInvalidLengthCa
+			}
+			if (iNdEx + skippy) > l {
+				return io.ErrUnexpectedEOF
+			}
+			iNdEx += skippy
+		}
+	}
+
+	if iNdEx > l {
+		return io.ErrUnexpectedEOF
+	}
+	return nil
+}
+func (m *GetUnlockKeyResponse) Unmarshal(data []byte) error {
+	l := len(data)
+	iNdEx := 0
+	for iNdEx < l {
+		preIndex := iNdEx
+		var wire uint64
+		for shift := uint(0); ; shift += 7 {
+			if shift >= 64 {
+				return ErrIntOverflowCa
+			}
+			if iNdEx >= l {
+				return io.ErrUnexpectedEOF
+			}
+			b := data[iNdEx]
+			iNdEx++
+			wire |= (uint64(b) & 0x7F) << shift
+			if b < 0x80 {
+				break
+			}
+		}
+		fieldNum := int32(wire >> 3)
+		wireType := int(wire & 0x7)
+		if wireType == 4 {
+			return fmt.Errorf("proto: GetUnlockKeyResponse: wiretype end group for non-group")
+		}
+		if fieldNum <= 0 {
+			return fmt.Errorf("proto: GetUnlockKeyResponse: illegal tag %d (wire type %d)", fieldNum, wire)
+		}
+		switch fieldNum {
+		case 1:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field UnlockKey", wireType)
+			}
+			var byteLen int
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowCa
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := data[iNdEx]
+				iNdEx++
+				byteLen |= (int(b) & 0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			if byteLen < 0 {
+				return ErrInvalidLengthCa
+			}
+			postIndex := iNdEx + byteLen
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			m.UnlockKey = append(m.UnlockKey[:0], data[iNdEx:postIndex]...)
+			if m.UnlockKey == nil {
+				m.UnlockKey = []byte{}
+			}
+			iNdEx = postIndex
+		case 2:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Version", wireType)
+			}
+			var msglen int
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowCa
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := data[iNdEx]
+				iNdEx++
+				msglen |= (int(b) & 0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			if msglen < 0 {
+				return ErrInvalidLengthCa
+			}
+			postIndex := iNdEx + msglen
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			if err := m.Version.Unmarshal(data[iNdEx:postIndex]); err != nil {
+				return err
+			}
+			iNdEx = postIndex
+		default:
+			iNdEx = preIndex
+			skippy, err := skipCa(data[iNdEx:])
+			if err != nil {
+				return err
+			}
+			if skippy < 0 {
+				return ErrInvalidLengthCa
+			}
+			if (iNdEx + skippy) > l {
+				return io.ErrUnexpectedEOF
+			}
+			iNdEx += skippy
+		}
+	}
+
+	if iNdEx > l {
+		return io.ErrUnexpectedEOF
+	}
+	return nil
+}
 func skipCa(data []byte) (n int, err error) {
 	l := len(data)
 	iNdEx := 0
@@ -1710,36 +2096,42 @@ var (
 func init() { proto.RegisterFile("ca.proto", fileDescriptorCa) }
 
 var fileDescriptorCa = []byte{
-	// 493 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0x94, 0x94, 0xcf, 0x6e, 0xd3, 0x40,
-	0x10, 0xc6, 0xbb, 0x0e, 0xa4, 0x65, 0x52, 0x05, 0xb4, 0x04, 0x29, 0xa4, 0xa9, 0x53, 0x99, 0x03,
-	0x9c, 0x9c, 0xd6, 0x70, 0xe2, 0x44, 0x62, 0x24, 0x94, 0x03, 0x08, 0x6d, 0x1e, 0x00, 0xb9, 0xf6,
-	0x10, 0xac, 0x24, 0x5e, 0xe3, 0xdd, 0x80, 0xb8, 0x21, 0x81, 0x38, 0x70, 0x47, 0x70, 0xe2, 0x11,
-	0x78, 0x8e, 0x8a, 0x13, 0x47, 0x4e, 0x15, 0xf1, 0x03, 0x20, 0x1e, 0x01, 0xed, 0xda, 0x21, 0xfd,
-	0xb3, 0x89, 0xca, 0xc9, 0x3b, 0xb3, 0xf3, 0x7d, 0xfe, 0xed, 0x8c, 0xd7, 0xb0, 0x15, 0x06, 0x6e,
-	0x9a, 0x71, 0xc9, 0x29, 0x8d, 0x78, 0x38, 0xc6, 0xcc, 0x15, 0xaf, 0x83, 0x6c, 0x3a, 0x8e, 0xa5,
-	0xfb, 0xea, 0xa0, 0x55, 0x93, 0x6f, 0x52, 0x14, 0x45, 0x41, 0xab, 0x26, 0x52, 0x0c, 0x17, 0x41,
-	0x63, 0xc4, 0x47, 0x5c, 0x2f, 0xbb, 0x6a, 0x55, 0x66, 0xaf, 0xa7, 0x93, 0xd9, 0x28, 0x4e, 0xba,
-	0xc5, 0xa3, 0x48, 0x3a, 0x3e, 0xb4, 0x9f, 0xf0, 0x08, 0x7d, 0xcc, 0x64, 0xfc, 0x3c, 0x0e, 0x03,
-	0x89, 0x43, 0x19, 0xc8, 0x99, 0x60, 0xf8, 0x72, 0x86, 0x42, 0xd2, 0x5b, 0xb0, 0x99, 0xf0, 0x08,
-	0x9f, 0xc5, 0x51, 0x93, 0xec, 0x91, 0x3b, 0x57, 0xfa, 0x90, 0x1f, 0x77, 0xaa, 0x4a, 0x32, 0x78,
-	0xc8, 0xaa, 0x6a, 0x6b, 0x10, 0x39, 0x5f, 0x09, 0xec, 0xae, 0x70, 0x11, 0x29, 0x4f, 0x04, 0xd2,
-	0xfb, 0x50, 0x15, 0x3a, 0xa3, 0x5d, 0x6a, 0x9e, 0xe3, 0x9e, 0x3f, 0x90, 0x3b, 0x10, 0x62, 0x16,
-	0x24, 0xe1, 0x42, 0x5b, 0x2a, 0x68, 0x0f, 0x6a, 0xe1, 0xd2, 0xb8, 0x69, 0x69, 0x83, 0x8e, 0xc9,
-	0xe0, 0xc4, 0xfb, 0xd9, 0x49, 0x8d, 0xf3, 0x9e, 0xc0, 0x8e, 0x72, 0xc7, 0x33, 0x94, 0x8b, 0x53,
-	0xde, 0x83, 0x4b, 0x19, 0x9f, 0xa0, 0x86, 0xab, 0x7b, 0x6d, 0x93, 0xb7, 0x52, 0x32, 0x3e, 0xc1,
-	0xbe, 0xd5, 0x24, 0x4c, 0x57, 0xd3, 0x9b, 0x50, 0x09, 0x45, 0xa6, 0x81, 0xb6, 0xfb, 0x9b, 0xf9,
-	0x71, 0xa7, 0xe2, 0x0f, 0x19, 0x53, 0x39, 0xda, 0x80, 0xcb, 0x92, 0x8f, 0x31, 0x69, 0x56, 0x54,
-	0xd3, 0x58, 0x11, 0x38, 0x9f, 0x08, 0xb4, 0xcd, 0x18, 0x65, 0x9b, 0x2e, 0xd2, 0x6d, 0xfa, 0x14,
-	0xae, 0xea, 0xa2, 0x29, 0x4e, 0x0f, 0x31, 0x13, 0x2f, 0xe2, 0x54, 0x23, 0xd4, 0xbd, 0xdb, 0xab,
-	0xb8, 0x87, 0x29, 0x86, 0xee, 0xe3, 0x7f, 0xe5, 0xac, 0xae, 0xf4, 0xcb, 0xd8, 0xd9, 0x85, 0x9d,
-	0x47, 0x28, 0x19, 0xe7, 0xd2, 0xef, 0x9d, 0xef, 0x8e, 0xf3, 0x00, 0xda, 0xe6, 0xed, 0x92, 0x7a,
-	0xef, 0xf4, 0x80, 0x14, 0xf9, 0xf6, 0xa9, 0xfe, 0x7b, 0x1f, 0x09, 0x58, 0x7e, 0x8f, 0xbe, 0x23,
-	0xd0, 0x30, 0x39, 0xd1, 0xae, 0x89, 0x7c, 0x0d, 0x52, 0x6b, 0xff, 0xe2, 0x82, 0x02, 0xd2, 0xd9,
-	0xfa, 0xfe, 0xed, 0xf7, 0x17, 0xcb, 0xba, 0x46, 0xbc, 0xcf, 0x16, 0xe8, 0x96, 0x96, 0x40, 0xa6,
-	0x81, 0x98, 0x81, 0xd6, 0x7c, 0x41, 0x66, 0xa0, 0x75, 0xb3, 0x5e, 0x02, 0xd1, 0x0f, 0x04, 0x6e,
-	0x18, 0xaf, 0x0f, 0xdd, 0x5f, 0x35, 0xd1, 0x55, 0xf7, 0xb5, 0x75, 0xf0, 0x1f, 0x8a, 0xb3, 0x20,
-	0xfd, 0xf6, 0xd1, 0xdc, 0xde, 0xf8, 0x39, 0xb7, 0x37, 0xfe, 0xcc, 0x6d, 0xf2, 0x36, 0xb7, 0xc9,
-	0x51, 0x6e, 0x93, 0x1f, 0xb9, 0x4d, 0x7e, 0xe5, 0x36, 0x39, 0xac, 0xea, 0x3f, 0xc6, 0xdd, 0xbf,
-	0x01, 0x00, 0x00, 0xff, 0xff, 0xb3, 0xf8, 0x41, 0xef, 0x96, 0x04, 0x00, 0x00,
+	// 586 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0x94, 0x54, 0xcb, 0x6e, 0xd3, 0x40,
+	0x14, 0xcd, 0x38, 0x25, 0x69, 0x6f, 0x42, 0x8a, 0xa6, 0x89, 0x14, 0xf2, 0x70, 0x2a, 0xb3, 0x68,
+	0x37, 0x38, 0x6d, 0x60, 0x05, 0x1b, 0x92, 0x20, 0x55, 0x11, 0x02, 0x21, 0x47, 0xb0, 0xad, 0x5c,
+	0x67, 0x08, 0x56, 0x12, 0x8f, 0xf1, 0x8c, 0x0b, 0xd9, 0x21, 0x51, 0xf1, 0x07, 0x08, 0x56, 0x7c,
+	0x02, 0xdf, 0x11, 0xb1, 0x62, 0xc9, 0x2a, 0x22, 0xfe, 0x00, 0xc4, 0x27, 0x20, 0x8f, 0x6d, 0x9a,
+	0x87, 0x13, 0xda, 0x55, 0x3c, 0xd7, 0xe7, 0x9c, 0x7b, 0xee, 0xc9, 0xf5, 0xc0, 0xb6, 0xa1, 0xab,
+	0xb6, 0x43, 0x39, 0xc5, 0xb8, 0x47, 0x8d, 0x01, 0x71, 0x54, 0xf6, 0x56, 0x77, 0x46, 0x03, 0x93,
+	0xab, 0xe7, 0xc7, 0xa5, 0x0c, 0x1f, 0xdb, 0x84, 0x05, 0x80, 0x52, 0x86, 0xd9, 0xc4, 0x88, 0x0e,
+	0xf9, 0x3e, 0xed, 0x53, 0xf1, 0x58, 0xf7, 0x9f, 0xc2, 0xea, 0x9e, 0x3d, 0x74, 0xfb, 0xa6, 0x55,
+	0x0f, 0x7e, 0x82, 0xa2, 0xd2, 0x86, 0xca, 0x33, 0xda, 0x23, 0x6d, 0xe2, 0x70, 0xf3, 0x95, 0x69,
+	0xe8, 0x9c, 0x74, 0xb9, 0xce, 0x5d, 0xa6, 0x91, 0x37, 0x2e, 0x61, 0x1c, 0xdf, 0x81, 0xb4, 0x45,
+	0x7b, 0xe4, 0xd4, 0xec, 0x15, 0xd1, 0x3e, 0x3a, 0xdc, 0x69, 0x81, 0x37, 0xad, 0xa5, 0x7c, 0x4a,
+	0xe7, 0xb1, 0x96, 0xf2, 0x5f, 0x75, 0x7a, 0xca, 0x57, 0x04, 0xd5, 0x35, 0x2a, 0xcc, 0xa6, 0x16,
+	0x23, 0xf8, 0x01, 0xa4, 0x98, 0xa8, 0x08, 0x95, 0x4c, 0x43, 0x51, 0x57, 0x07, 0x52, 0x3b, 0x8c,
+	0xb9, 0xba, 0x65, 0x44, 0xdc, 0x90, 0x81, 0x9b, 0x90, 0x31, 0x2e, 0x85, 0x8b, 0x92, 0x10, 0xa8,
+	0xc5, 0x09, 0xcc, 0xf5, 0xd7, 0xe6, 0x39, 0xca, 0x05, 0x82, 0xb2, 0xaf, 0x4e, 0x96, 0x5c, 0x46,
+	0x53, 0xde, 0x87, 0x2d, 0x87, 0x0e, 0x89, 0x30, 0x97, 0x6b, 0x54, 0xe2, 0xb4, 0x7d, 0xa6, 0x46,
+	0x87, 0xa4, 0x25, 0x15, 0x91, 0x26, 0xd0, 0xf8, 0x36, 0x24, 0x0d, 0xe6, 0x08, 0x43, 0xd9, 0x56,
+	0xda, 0x9b, 0xd6, 0x92, 0xed, 0xae, 0xa6, 0xf9, 0x35, 0x9c, 0x87, 0x1b, 0x9c, 0x0e, 0x88, 0x55,
+	0x4c, 0xfa, 0xa1, 0x69, 0xc1, 0x41, 0xf9, 0x84, 0xa0, 0x12, 0x6f, 0x23, 0x8c, 0xe9, 0x2a, 0x69,
+	0xe3, 0xe7, 0xb0, 0x2b, 0x40, 0x23, 0x32, 0x3a, 0x23, 0x0e, 0x7b, 0x6d, 0xda, 0xc2, 0x42, 0xae,
+	0x71, 0xb0, 0xce, 0x77, 0xd7, 0x26, 0x86, 0xfa, 0xf4, 0x1f, 0x5c, 0xcb, 0xf9, 0xfc, 0xcb, 0xb3,
+	0x52, 0x85, 0xf2, 0x09, 0xe1, 0x1a, 0xa5, 0xbc, 0xdd, 0x5c, 0x4d, 0x47, 0x79, 0x04, 0x95, 0xf8,
+	0xd7, 0xa1, 0xeb, 0xfd, 0xc5, 0x3f, 0xc8, 0x77, 0x9e, 0x5d, 0xcc, 0xbf, 0x00, 0x7b, 0x27, 0x84,
+	0xbf, 0xb0, 0x86, 0xd4, 0x18, 0x3c, 0x21, 0xe3, 0x48, 0xd8, 0x81, 0xfc, 0x62, 0x39, 0x14, 0xac,
+	0x02, 0xb8, 0xa2, 0x78, 0x3a, 0x20, 0xe3, 0x50, 0x6f, 0xc7, 0x8d, 0x60, 0xf8, 0x21, 0xa4, 0xcf,
+	0x89, 0xc3, 0x4c, 0x6a, 0x85, 0xcb, 0x50, 0x8e, 0x1b, 0xfc, 0x65, 0x00, 0x69, 0x6d, 0x4d, 0xa6,
+	0xb5, 0x84, 0x16, 0x31, 0x1a, 0x17, 0x12, 0x48, 0xed, 0x26, 0xfe, 0x80, 0x44, 0xef, 0x95, 0xa1,
+	0x70, 0x3d, 0x4e, 0x6b, 0x43, 0x3a, 0xa5, 0xa3, 0xab, 0x13, 0x82, 0xf1, 0x94, 0xed, 0xef, 0xdf,
+	0x7e, 0x7f, 0x91, 0xa4, 0x5b, 0x08, 0xbf, 0x83, 0xec, 0x7c, 0x00, 0xf8, 0x60, 0x8d, 0xd6, 0x72,
+	0x72, 0xa5, 0xc3, 0xff, 0x03, 0xc3, 0x66, 0x05, 0xd1, 0x6c, 0x17, 0x6e, 0x0a, 0xe4, 0xdd, 0x91,
+	0x6e, 0xe9, 0x7d, 0xe2, 0x34, 0x3e, 0x4b, 0x20, 0xf6, 0x2a, 0x8c, 0x22, 0x6e, 0x2b, 0xe3, 0xa3,
+	0xd8, 0xf0, 0x19, 0xc5, 0x47, 0xb1, 0x69, 0xe1, 0xe7, 0xa2, 0xf8, 0x88, 0xa0, 0x10, 0x7b, 0x87,
+	0xe0, 0xa3, 0x75, 0x6b, 0xbd, 0xee, 0xd2, 0x2a, 0x1d, 0x5f, 0x83, 0xb1, 0x6c, 0xa4, 0x55, 0x99,
+	0xcc, 0xe4, 0xc4, 0xcf, 0x99, 0x9c, 0xf8, 0x33, 0x93, 0xd1, 0x7b, 0x4f, 0x46, 0x13, 0x4f, 0x46,
+	0x3f, 0x3c, 0x19, 0xfd, 0xf2, 0x64, 0x74, 0x96, 0x12, 0xd7, 0xe6, 0xbd, 0xbf, 0x01, 0x00, 0x00,
+	0xff, 0xff, 0xe7, 0x80, 0x3b, 0x00, 0x9b, 0x05, 0x00, 0x00,
 }

vendor/github.com/docker/swarmkit/api/ca.proto

@@ -13,6 +13,11 @@ service CA {
 	rpc GetRootCACertificate(GetRootCACertificateRequest) returns (GetRootCACertificateResponse) {
 		option (docker.protobuf.plugin.tls_authorization) = { insecure: true };
 	};
+	// GetUnlockKey returns the current unlock key for the cluster for the role of the client
+	// asking.
+	rpc GetUnlockKey(GetUnlockKeyRequest) returns (GetUnlockKeyResponse) {
+		option (docker.protobuf.plugin.tls_authorization) = { roles: ["swarm-manager"] };
+	};
 }
 
 service NodeCA {
@@ -55,3 +60,10 @@ message GetRootCACertificateRequest {}
 message GetRootCACertificateResponse {
 	bytes certificate = 1;
 }
+
+message GetUnlockKeyRequest {}
+
+message GetUnlockKeyResponse {
+	bytes unlock_key = 1;
+	Version version = 2 [(gogoproto.nullable) = false];
+}

vendor/github.com/docker/swarmkit/api/control.pb.go

@@ -405,16 +405,19 @@ func (m *ListClustersResponse) Reset()                    { *m = ListClustersRes
 func (*ListClustersResponse) ProtoMessage()               {}
 func (*ListClustersResponse) Descriptor() ([]byte, []int) { return fileDescriptorControl, []int{35} }
 
-type JoinTokenRotation struct {
-	// RotateWorkerToken tells UpdateCluster to rotate the worker secret.
-	RotateWorkerToken bool `protobuf:"varint,1,opt,name=rotate_worker_token,json=rotateWorkerToken,proto3" json:"rotate_worker_token,omitempty"`
-	// RotateManagerSecret tells UpdateCluster to rotate the manager secret.
-	RotateManagerToken bool `protobuf:"varint,2,opt,name=rotate_manager_token,json=rotateManagerToken,proto3" json:"rotate_manager_token,omitempty"`
+// KeyRotation tells UpdateCluster what items to rotate
+type KeyRotation struct {
+	// WorkerJoinToken tells UpdateCluster to rotate the worker secret token.
+	WorkerJoinToken bool `protobuf:"varint,1,opt,name=worker_join_token,json=workerJoinToken,proto3" json:"worker_join_token,omitempty"`
+	// ManagerJoinToken tells UpdateCluster to rotate the manager secret token.
+	ManagerJoinToken bool `protobuf:"varint,2,opt,name=manager_join_token,json=managerJoinToken,proto3" json:"manager_join_token,omitempty"`
+	// ManagerUnlockKey tells UpdateCluster to rotate the manager unlock key
+	ManagerUnlockKey bool `protobuf:"varint,3,opt,name=manager_unlock_key,json=managerUnlockKey,proto3" json:"manager_unlock_key,omitempty"`
 }
 
-func (m *JoinTokenRotation) Reset()                    { *m = JoinTokenRotation{} }
-func (*JoinTokenRotation) ProtoMessage()               {}
-func (*JoinTokenRotation) Descriptor() ([]byte, []int) { return fileDescriptorControl, []int{36} }
+func (m *KeyRotation) Reset()                    { *m = KeyRotation{} }
+func (*KeyRotation) ProtoMessage()               {}
+func (*KeyRotation) Descriptor() ([]byte, []int) { return fileDescriptorControl, []int{36} }
 
 type UpdateClusterRequest struct {
 	// ClusterID is the cluster ID to update.
@@ -423,8 +426,8 @@ type UpdateClusterRequest struct {
 	ClusterVersion *Version `protobuf:"bytes,2,opt,name=cluster_version,json=clusterVersion" json:"cluster_version,omitempty"`
 	// Spec is the new spec to apply to the cluster.
 	Spec *ClusterSpec `protobuf:"bytes,3,opt,name=spec" json:"spec,omitempty"`
-	// Rotation contains flags for join token rotation
-	Rotation JoinTokenRotation `protobuf:"bytes,4,opt,name=rotation" json:"rotation"`
+	// Rotation contains flags for join token and unlock key rotation
+	Rotation KeyRotation `protobuf:"bytes,4,opt,name=rotation" json:"rotation"`
 }
 
 func (m *UpdateClusterRequest) Reset()                    { *m = UpdateClusterRequest{} }
@@ -598,7 +601,7 @@ func init() {
 	proto.RegisterType((*ListClustersRequest)(nil), "docker.swarmkit.v1.ListClustersRequest")
 	proto.RegisterType((*ListClustersRequest_Filters)(nil), "docker.swarmkit.v1.ListClustersRequest.Filters")
 	proto.RegisterType((*ListClustersResponse)(nil), "docker.swarmkit.v1.ListClustersResponse")
-	proto.RegisterType((*JoinTokenRotation)(nil), "docker.swarmkit.v1.JoinTokenRotation")
+	proto.RegisterType((*KeyRotation)(nil), "docker.swarmkit.v1.KeyRotation")
 	proto.RegisterType((*UpdateClusterRequest)(nil), "docker.swarmkit.v1.UpdateClusterRequest")
 	proto.RegisterType((*UpdateClusterResponse)(nil), "docker.swarmkit.v1.UpdateClusterResponse")
 	proto.RegisterType((*GetSecretRequest)(nil), "docker.swarmkit.v1.GetSecretRequest")
@@ -1459,14 +1462,15 @@ func (m *ListClustersResponse) Copy() *ListClustersResponse {
 	return o
 }
 
-func (m *JoinTokenRotation) Copy() *JoinTokenRotation {
+func (m *KeyRotation) Copy() *KeyRotation {
 	if m == nil {
 		return nil
 	}
 
-	o := &JoinTokenRotation{
-		RotateWorkerToken:  m.RotateWorkerToken,
-		RotateManagerToken: m.RotateManagerToken,
+	o := &KeyRotation{
+		WorkerJoinToken:  m.WorkerJoinToken,
+		ManagerJoinToken: m.ManagerJoinToken,
+		ManagerUnlockKey: m.ManagerUnlockKey,
 	}
 
 	return o
@@ -2199,14 +2203,15 @@ func (this *ListClustersResponse) GoString() string {
 	s = append(s, "}")
 	return strings.Join(s, "")
 }
-func (this *JoinTokenRotation) GoString() string {
+func (this *KeyRotation) GoString() string {
 	if this == nil {
 		return "nil"
 	}
-	s := make([]string, 0, 6)
-	s = append(s, "&api.JoinTokenRotation{")
-	s = append(s, "RotateWorkerToken: "+fmt.Sprintf("%#v", this.RotateWorkerToken)+",\n")
-	s = append(s, "RotateManagerToken: "+fmt.Sprintf("%#v", this.RotateManagerToken)+",\n")
+	s := make([]string, 0, 7)
+	s = append(s, "&api.KeyRotation{")
+	s = append(s, "WorkerJoinToken: "+fmt.Sprintf("%#v", this.WorkerJoinToken)+",\n")
+	s = append(s, "ManagerJoinToken: "+fmt.Sprintf("%#v", this.ManagerJoinToken)+",\n")
+	s = append(s, "ManagerUnlockKey: "+fmt.Sprintf("%#v", this.ManagerUnlockKey)+",\n")
 	s = append(s, "}")
 	return strings.Join(s, "")
 }
@@ -4734,7 +4739,7 @@ func (m *ListClustersResponse) MarshalTo(data []byte) (int, error) {
 	return i, nil
 }
 
-func (m *JoinTokenRotation) Marshal() (data []byte, err error) {
+func (m *KeyRotation) Marshal() (data []byte, err error) {
 	size := m.Size()
 	data = make([]byte, size)
 	n, err := m.MarshalTo(data)
@@ -4744,25 +4749,35 @@ func (m *JoinTokenRotation) Marshal() (data []byte, err error) {
 	return data[:n], nil
 }
 
-func (m *JoinTokenRotation) MarshalTo(data []byte) (int, error) {
+func (m *KeyRotation) MarshalTo(data []byte) (int, error) {
 	var i int
 	_ = i
 	var l int
 	_ = l
-	if m.RotateWorkerToken {
+	if m.WorkerJoinToken {
 		data[i] = 0x8
 		i++
-		if m.RotateWorkerToken {
+		if m.WorkerJoinToken {
 			data[i] = 1
 		} else {
 			data[i] = 0
 		}
 		i++
 	}
-	if m.RotateManagerToken {
+	if m.ManagerJoinToken {
 		data[i] = 0x10
 		i++
-		if m.RotateManagerToken {
+		if m.ManagerJoinToken {
+			data[i] = 1
+		} else {
+			data[i] = 0
+		}
+		i++
+	}
+	if m.ManagerUnlockKey {
+		data[i] = 0x18
+		i++
+		if m.ManagerUnlockKey {
 			data[i] = 1
 		} else {
 			data[i] = 0
@@ -6618,13 +6633,16 @@ func (m *ListClustersResponse) Size() (n int) {
 	return n
 }
 
-func (m *JoinTokenRotation) Size() (n int) {
+func (m *KeyRotation) Size() (n int) {
 	var l int
 	_ = l
-	if m.RotateWorkerToken {
+	if m.WorkerJoinToken {
 		n += 2
 	}
-	if m.RotateManagerToken {
+	if m.ManagerJoinToken {
+		n += 2
+	}
+	if m.ManagerUnlockKey {
 		n += 2
 	}
 	return n
@@ -7294,13 +7312,14 @@ func (this *ListClustersResponse) String() string {
 	}, "")
 	return s
 }
-func (this *JoinTokenRotation) String() string {
+func (this *KeyRotation) String() string {
 	if this == nil {
 		return "nil"
 	}
-	s := strings.Join([]string{`&JoinTokenRotation{`,
-		`RotateWorkerToken:` + fmt.Sprintf("%v", this.RotateWorkerToken) + `,`,
-		`RotateManagerToken:` + fmt.Sprintf("%v", this.RotateManagerToken) + `,`,
+	s := strings.Join([]string{`&KeyRotation{`,
+		`WorkerJoinToken:` + fmt.Sprintf("%v", this.WorkerJoinToken) + `,`,
+		`ManagerJoinToken:` + fmt.Sprintf("%v", this.ManagerJoinToken) + `,`,
+		`ManagerUnlockKey:` + fmt.Sprintf("%v", this.ManagerUnlockKey) + `,`,
 		`}`,
 	}, "")
 	return s
@@ -7313,7 +7332,7 @@ func (this *UpdateClusterRequest) String() string {
 		`ClusterID:` + fmt.Sprintf("%v", this.ClusterID) + `,`,
 		`ClusterVersion:` + strings.Replace(fmt.Sprintf("%v", this.ClusterVersion), "Version", "Version", 1) + `,`,
 		`Spec:` + strings.Replace(fmt.Sprintf("%v", this.Spec), "ClusterSpec", "ClusterSpec", 1) + `,`,
-		`Rotation:` + strings.Replace(strings.Replace(this.Rotation.String(), "JoinTokenRotation", "JoinTokenRotation", 1), `&`, ``, 1) + `,`,
+		`Rotation:` + strings.Replace(strings.Replace(this.Rotation.String(), "KeyRotation", "KeyRotation", 1), `&`, ``, 1) + `,`,
 		`}`,
 	}, "")
 	return s
@@ -11855,7 +11874,7 @@ func (m *ListClustersResponse) Unmarshal(data []byte) error {
 	}
 	return nil
 }
-func (m *JoinTokenRotation) Unmarshal(data []byte) error {
+func (m *KeyRotation) Unmarshal(data []byte) error {
 	l := len(data)
 	iNdEx := 0
 	for iNdEx < l {
@@ -11878,15 +11897,15 @@ func (m *JoinTokenRotation) Unmarshal(data []byte) error {
 		fieldNum := int32(wire >> 3)
 		wireType := int(wire & 0x7)
 		if wireType == 4 {
-			return fmt.Errorf("proto: JoinTokenRotation: wiretype end group for non-group")
+			return fmt.Errorf("proto: KeyRotation: wiretype end group for non-group")
 		}
 		if fieldNum <= 0 {
-			return fmt.Errorf("proto: JoinTokenRotation: illegal tag %d (wire type %d)", fieldNum, wire)
+			return fmt.Errorf("proto: KeyRotation: illegal tag %d (wire type %d)", fieldNum, wire)
 		}
 		switch fieldNum {
 		case 1:
 			if wireType != 0 {
-				return fmt.Errorf("proto: wrong wireType = %d for field RotateWorkerToken", wireType)
+				return fmt.Errorf("proto: wrong wireType = %d for field WorkerJoinToken", wireType)
 			}
 			var v int
 			for shift := uint(0); ; shift += 7 {
@@ -11903,10 +11922,10 @@ func (m *JoinTokenRotation) Unmarshal(data []byte) error {
 					break
 				}
 			}
-			m.RotateWorkerToken = bool(v != 0)
+			m.WorkerJoinToken = bool(v != 0)
 		case 2:
 			if wireType != 0 {
-				return fmt.Errorf("proto: wrong wireType = %d for field RotateManagerToken", wireType)
+				return fmt.Errorf("proto: wrong wireType = %d for field ManagerJoinToken", wireType)
 			}
 			var v int
 			for shift := uint(0); ; shift += 7 {
@@ -11923,7 +11942,27 @@ func (m *JoinTokenRotation) Unmarshal(data []byte) error {
 					break
 				}
 			}
-			m.RotateManagerToken = bool(v != 0)
+			m.ManagerJoinToken = bool(v != 0)
+		case 3:
+			if wireType != 0 {
+				return fmt.Errorf("proto: wrong wireType = %d for field ManagerUnlockKey", wireType)
+			}
+			var v int
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowControl
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := data[iNdEx]
+				iNdEx++
+				v |= (int(b) & 0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			m.ManagerUnlockKey = bool(v != 0)
 		default:
 			iNdEx = preIndex
 			skippy, err := skipControl(data[iNdEx:])
@@ -13413,117 +13452,117 @@ var (
 func init() { proto.RegisterFile("control.proto", fileDescriptorControl) }
 
 var fileDescriptorControl = []byte{
-	// 1777 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xcc, 0x5a, 0xcf, 0x6f, 0xdb, 0xc6,
-	0x12, 0x8e, 0x24, 0xdb, 0xb2, 0x47, 0x96, 0x13, 0xaf, 0x95, 0x3c, 0x81, 0xc9, 0x93, 0x03, 0xe6,
-	0xc5, 0x91, 0x81, 0x3c, 0x39, 0x4f, 0x79, 0x41, 0xd3, 0x14, 0xfd, 0x65, 0xbb, 0x71, 0x95, 0x1f,
-	0x6e, 0x40, 0x27, 0x6d, 0x6f, 0x86, 0x2c, 0x6d, 0x5c, 0x56, 0xb2, 0xa8, 0x92, 0xb4, 0x93, 0xa0,
-	0x97, 0x16, 0x68, 0x81, 0xfe, 0x09, 0xbd, 0xf6, 0xda, 0x02, 0x3d, 0xf7, 0xd6, 0x6b, 0xd0, 0x53,
-	0x8f, 0x3d, 0x19, 0x8d, 0x80, 0x02, 0x3d, 0x15, 0xfd, 0x0b, 0x8a, 0x62, 0x77, 0x67, 0x49, 0x8a,
-	0x5a, 0x92, 0x92, 0xe5, 0xc2, 0x39, 0x99, 0x5c, 0x7e, 0xb3, 0x33, 0xbb, 0xf3, 0xed, 0xa7, 0xd9,
-	0x81, 0x21, 0xdf, 0xb0, 0x3a, 0xae, 0x6d, 0xb5, 0x2b, 0x5d, 0xdb, 0x72, 0x2d, 0x42, 0x9a, 0x56,
-	0xa3, 0x45, 0xed, 0x8a, 0xf3, 0xa4, 0x6e, 0xef, 0xb5, 0x4c, 0xb7, 0x72, 0xf0, 0x3f, 0x2d, 0xe7,
-	0x74, 0x69, 0xc3, 0x11, 0x00, 0x2d, 0x6f, 0xed, 0x7c, 0x4c, 0x1b, 0xae, 0x7c, 0xcd, 0xb9, 0xcf,
-	0xba, 0x54, 0xbe, 0x14, 0x76, 0xad, 0x5d, 0x8b, 0x3f, 0xae, 0xb0, 0x27, 0x1c, 0x5d, 0xe8, 0xb6,
-	0xf7, 0x77, 0xcd, 0xce, 0x8a, 0xf8, 0x23, 0x06, 0xf5, 0x1b, 0x30, 0xb7, 0x41, 0xdd, 0x4d, 0xab,
-	0x49, 0x0d, 0xfa, 0xc9, 0x3e, 0x75, 0x5c, 0x72, 0x09, 0xb2, 0x1d, 0xab, 0x49, 0xb7, 0xcd, 0x66,
-	0x31, 0x75, 0x31, 0x55, 0x9e, 0x59, 0x85, 0xde, 0xe1, 0xe2, 0x14, 0x43, 0xd4, 0xd6, 0x8d, 0x29,
-	0xf6, 0xa9, 0xd6, 0xd4, 0xdf, 0x84, 0xd3, 0x9e, 0x99, 0xd3, 0xb5, 0x3a, 0x0e, 0x25, 0x57, 0x61,
-	0x82, 0x7d, 0xe4, 0x46, 0xb9, 0x6a, 0xb1, 0x32, 0xb8, 0x80, 0x0a, 0xc7, 0x73, 0x94, 0x7e, 0x98,
-	0x81, 0x33, 0xf7, 0x4c, 0x87, 0x4f, 0xe1, 0x48, 0xd7, 0xb7, 0x21, 0xfb, 0xd8, 0x6c, 0xbb, 0xd4,
-	0x76, 0x70, 0x96, 0xab, 0xaa, 0x59, 0xc2, 0x66, 0x95, 0xdb, 0xc2, 0xc6, 0x90, 0xc6, 0xda, 0xe7,
-	0x19, 0xc8, 0xe2, 0x20, 0x29, 0xc0, 0x64, 0xa7, 0xbe, 0x47, 0xd9, 0x8c, 0x99, 0xf2, 0x8c, 0x21,
-	0x5e, 0xc8, 0x0a, 0xe4, 0xcc, 0xe6, 0x76, 0xd7, 0xa6, 0x8f, 0xcd, 0xa7, 0xd4, 0x29, 0xa6, 0xd9,
-	0xb7, 0xd5, 0xb9, 0xde, 0xe1, 0x22, 0xd4, 0xd6, 0x1f, 0xe0, 0xa8, 0x01, 0x66, 0x53, 0x3e, 0x93,
-	0x07, 0x30, 0xd5, 0xae, 0xef, 0xd0, 0xb6, 0x53, 0xcc, 0x5c, 0xcc, 0x94, 0x73, 0xd5, 0x9b, 0xa3,
-	0x44, 0x56, 0xb9, 0xc7, 0x4d, 0xdf, 0xe9, 0xb8, 0xf6, 0x33, 0x03, 0xe7, 0x21, 0x35, 0xc8, 0xed,
-	0xd1, 0xbd, 0x1d, 0x6a, 0x3b, 0x1f, 0x99, 0x5d, 0xa7, 0x38, 0x71, 0x31, 0x53, 0x9e, 0xab, 0x5e,
-	0x89, 0xda, 0xb6, 0xad, 0x2e, 0x6d, 0x54, 0xee, 0x7b, 0x78, 0x23, 0x68, 0x4b, 0xaa, 0x30, 0x69,
-	0x5b, 0x6d, 0xea, 0x14, 0x27, 0xf9, 0x24, 0x17, 0x22, 0xf7, 0xde, 0x6a, 0x53, 0x43, 0x40, 0xc9,
-	0x25, 0xc8, 0xb3, 0xad, 0xf0, 0xf7, 0x60, 0x8a, 0xef, 0xcf, 0x2c, 0x1b, 0x94, 0xab, 0xd6, 0x5e,
-	0x85, 0x5c, 0x20, 0x74, 0x72, 0x06, 0x32, 0x2d, 0xfa, 0x4c, 0xd0, 0xc2, 0x60, 0x8f, 0x6c, 0x77,
-	0x0f, 0xea, 0xed, 0x7d, 0x5a, 0x4c, 0xf3, 0x31, 0xf1, 0x72, 0x2b, 0x7d, 0x33, 0xa5, 0xaf, 0xc1,
-	0x7c, 0x60, 0x3b, 0x90, 0x23, 0x15, 0x98, 0x64, 0xd9, 0x17, 0xc9, 0x88, 0x23, 0x89, 0x80, 0xe9,
-	0xdf, 0xa6, 0x60, 0xfe, 0x51, 0xb7, 0x59, 0x77, 0xe9, 0xa8, 0x0c, 0x25, 0x6f, 0xc0, 0x2c, 0x07,
-	0x1d, 0x50, 0xdb, 0x31, 0xad, 0x0e, 0x0f, 0x30, 0x57, 0x3d, 0xaf, 0xf2, 0xf8, 0xbe, 0x80, 0x18,
-	0x39, 0x66, 0x80, 0x2f, 0xe4, 0x1a, 0x4c, 0xb0, 0xe3, 0x56, 0xcc, 0x70, 0xbb, 0x0b, 0x71, 0x79,
-	0x31, 0x38, 0x52, 0x5f, 0x05, 0x12, 0x8c, 0xf5, 0x48, 0xc7, 0x62, 0x13, 0xe6, 0x0d, 0xba, 0x67,
-	0x1d, 0x8c, 0xbe, 0xde, 0x02, 0x4c, 0x3e, 0xb6, 0xec, 0x86, 0xc8, 0xc4, 0xb4, 0x21, 0x5e, 0xf4,
-	0x02, 0x90, 0xe0, 0x7c, 0x22, 0x26, 0x3c, 0xf4, 0x0f, 0xeb, 0x4e, 0x2b, 0xe0, 0xc2, 0xad, 0x3b,
-	0xad, 0x90, 0x0b, 0x86, 0x60, 0x2e, 0xd8, 0x27, 0xef, 0xd0, 0x0b, 0x33, 0x7f, 0x75, 0xec, 0x63,
-	0xdc, 0xea, 0x38, 0x9e, 0xa3, 0xf4, 0x9b, 0x72, 0x75, 0x23, 0xbb, 0xf6, 0xd6, 0x11, 0xf4, 0xae,
-	0xff, 0x85, 0x22, 0xc2, 0x06, 0x8f, 0x20, 0x22, 0x41, 0xb3, 0x41, 0x11, 0xf9, 0xe6, 0x04, 0x45,
-	0x44, 0x15, 0x99, 0x52, 0x44, 0x56, 0x20, 0xe7, 0x50, 0xfb, 0xc0, 0x6c, 0x30, 0x76, 0x08, 0x11,
-	0xc1, 0x10, 0xb6, 0xc4, 0x70, 0x6d, 0xdd, 0x31, 0x00, 0x21, 0xb5, 0xa6, 0x43, 0x96, 0x60, 0x1a,
-	0xb9, 0x24, 0xd4, 0x62, 0x66, 0x35, 0xd7, 0x3b, 0x5c, 0xcc, 0x0a, 0x32, 0x39, 0x46, 0x56, 0xb0,
-	0xc9, 0x21, 0xeb, 0x30, 0xd7, 0xa4, 0x8e, 0x69, 0xd3, 0xe6, 0xb6, 0xe3, 0xd6, 0x5d, 0xd4, 0x87,
-	0xb9, 0xea, 0xbf, 0xa3, 0x52, 0xbc, 0xc5, 0x50, 0x46, 0x1e, 0x8d, 0xf8, 0x9b, 0x42, 0x64, 0xb2,
-	0xff, 0x88, 0xc8, 0xe0, 0x76, 0xf9, 0x22, 0xc3, 0x58, 0x13, 0x2b, 0x32, 0x9c, 0x46, 0x02, 0xa6,
-	0xdf, 0x85, 0xc2, 0x9a, 0x4d, 0xeb, 0x2e, 0xc5, 0x2d, 0x93, 0x44, 0xba, 0x8e, 0x0a, 0x20, 0x58,
-	0xb4, 0xa8, 0x9a, 0x06, 0x2d, 0x02, 0x22, 0xb0, 0x09, 0x67, 0x43, 0x93, 0x61, 0x54, 0x37, 0x20,
-	0x8b, 0x69, 0xc0, 0x09, 0xcf, 0xc7, 0x4c, 0x68, 0x48, 0xac, 0xfe, 0x36, 0xcc, 0x6f, 0x50, 0x37,
-	0x14, 0xd9, 0x55, 0x00, 0x3f, 0xeb, 0x78, 0x6a, 0xf2, 0xbd, 0xc3, 0xc5, 0x19, 0x2f, 0xe9, 0xc6,
-	0x8c, 0x97, 0x73, 0xfd, 0x2e, 0x90, 0xe0, 0x14, 0xe3, 0xc5, 0xf3, 0x63, 0x0a, 0x0a, 0x42, 0xe5,
-	0xc6, 0x89, 0x89, 0xac, 0xc3, 0x69, 0x89, 0x1e, 0x41, 0xa0, 0xe7, 0xd0, 0x46, 0x6a, 0xf4, 0xf5,
-	0x3e, 0x8d, 0x1e, 0x3e, 0x43, 0xa1, 0x05, 0x8c, 0xb7, 0x23, 0xeb, 0x50, 0x10, 0xd2, 0x34, 0x56,
-	0x92, 0xfe, 0x05, 0x67, 0x43, 0xb3, 0xa0, 0xc6, 0xfd, 0x9e, 0x86, 0x05, 0xc6, 0x71, 0x1c, 0xf7,
-	0x64, 0xae, 0x16, 0x96, 0xb9, 0x95, 0x28, 0x31, 0x09, 0x59, 0x0e, 0x2a, 0xdd, 0x97, 0xe9, 0x63,
-	0x57, 0xba, 0xad, 0x90, 0xd2, 0xbd, 0x36, 0x62, 0x70, 0x4a, 0xb1, 0x1b, 0x50, 0x93, 0x89, 0xe3,
-	0x55, 0x93, 0xf7, 0xa0, 0xd0, 0x1f, 0x12, 0x12, 0xe3, 0x15, 0x98, 0xc6, 0x44, 0x49, 0x4d, 0x89,
-	0x65, 0x86, 0x07, 0xf6, 0x95, 0x65, 0x93, 0xba, 0x4f, 0x2c, 0xbb, 0x35, 0x82, 0xb2, 0xa0, 0x85,
-	0x4a, 0x59, 0xbc, 0xc9, 0x7c, 0xde, 0x76, 0xc4, 0x50, 0x1c, 0x6f, 0xa5, 0x95, 0xc4, 0xea, 0x8f,
-	0xb8, 0xb2, 0x84, 0x22, 0x23, 0x30, 0xc1, 0x76, 0x13, 0xf7, 0x8b, 0x3f, 0x33, 0x22, 0xa3, 0x0d,
-	0x23, 0x72, 0xda, 0x27, 0x32, 0xda, 0x32, 0x22, 0x23, 0xc0, 0x53, 0x9b, 0x63, 0x8a, 0xf1, 0x43,
-	0x79, 0xb6, 0x8e, 0x3d, 0x4c, 0xef, 0xbc, 0x85, 0x22, 0xf5, 0xce, 0x1b, 0x8e, 0x1f, 0xe1, 0xbc,
-	0x85, 0x2c, 0x5f, 0xae, 0xf3, 0x16, 0x11, 0xdc, 0x49, 0x9e, 0x37, 0x3f, 0x24, 0xff, 0xbc, 0x61,
-	0xa2, 0x62, 0xcf, 0x9b, 0xcc, 0x9c, 0x07, 0xc6, 0x1f, 0xcb, 0xb5, 0xf6, 0xbe, 0xe3, 0x52, 0x3b,
-	0xa0, 0xc3, 0x0d, 0x31, 0x12, 0xd2, 0x61, 0xc4, 0x31, 0x5e, 0x20, 0xc0, 0xa3, 0xaf, 0x37, 0x85,
-	0x4f, 0x5f, 0x84, 0xc4, 0xd1, 0x57, 0x5a, 0x49, 0xac, 0xc7, 0x25, 0xfc, 0x70, 0x04, 0x2e, 0x85,
-	0x2c, 0x5f, 0x2e, 0x2e, 0x45, 0x04, 0x77, 0x92, 0x5c, 0xf2, 0x43, 0xf2, 0xb9, 0x84, 0xd9, 0x88,
-	0xe5, 0x92, 0x4c, 0x9d, 0x07, 0xd6, 0xf7, 0x61, 0xfe, 0x8e, 0x65, 0x76, 0x1e, 0x5a, 0x2d, 0xda,
-	0x31, 0x2c, 0xb7, 0xee, 0xb2, 0x82, 0xa3, 0x02, 0x0b, 0x36, 0x7b, 0xa6, 0xdb, 0x8c, 0x70, 0xd4,
-	0xde, 0x76, 0xd9, 0x67, 0x1e, 0xe1, 0xb4, 0x31, 0x2f, 0x3e, 0x7d, 0xc0, 0xbf, 0x70, 0x3b, 0x72,
-	0x0d, 0x0a, 0x88, 0xdf, 0xab, 0x77, 0xea, 0xbb, 0x9e, 0x81, 0xb8, 0xa3, 0x11, 0xf1, 0xed, 0xbe,
-	0xf8, 0xc4, 0x2d, 0xf4, 0xaf, 0xd2, 0xb2, 0xbe, 0x1a, 0x87, 0xc6, 0xac, 0xbe, 0x92, 0xe8, 0x51,
-	0xea, 0x2b, 0xb4, 0x19, 0xa1, 0xbe, 0x42, 0xef, 0xfe, 0xef, 0x14, 0xd9, 0x80, 0x69, 0x1b, 0xf7,
-	0xab, 0x38, 0xc1, 0x0d, 0x2f, 0xab, 0x0c, 0x07, 0x36, 0x77, 0x75, 0xe2, 0xf9, 0xe1, 0xe2, 0x29,
-	0xc3, 0x33, 0xf6, 0x0b, 0xb5, 0x63, 0x3a, 0x8d, 0xaf, 0xc3, 0x19, 0x5e, 0x07, 0x37, 0x6c, 0xea,
-	0xca, 0x5d, 0x5d, 0x86, 0x19, 0x87, 0x0f, 0xf8, 0x9b, 0x3a, 0xdb, 0x3b, 0x5c, 0x9c, 0x16, 0xa8,
-	0xda, 0x3a, 0xfb, 0x31, 0xe7, 0x4f, 0x4d, 0x7d, 0x03, 0x2b, 0x71, 0x61, 0x8e, 0xa1, 0x54, 0x61,
-	0x4a, 0x00, 0x30, 0x12, 0x4d, 0x5d, 0x18, 0x70, 0x1b, 0x44, 0xea, 0x3f, 0xa4, 0x60, 0x41, 0x56,
-	0xa0, 0x47, 0x8b, 0x85, 0xac, 0xc2, 0x1c, 0x42, 0x47, 0xc8, 0x6e, 0x5e, 0x98, 0xc8, 0xe4, 0x56,
-	0xfb, 0x92, 0x5b, 0x8a, 0x0e, 0x3c, 0x50, 0x83, 0xdc, 0xf1, 0x8b, 0xff, 0xb1, 0xb7, 0xe1, 0xb7,
-	0x34, 0x10, 0x51, 0x6e, 0xb1, 0x57, 0x4f, 0x1b, 0xdf, 0x0d, 0x6b, 0x63, 0x25, 0xba, 0x74, 0x0c,
-	0x1a, 0x0e, 0x4a, 0xe3, 0x17, 0xc7, 0x2f, 0x8d, 0x46, 0x48, 0x1a, 0x6f, 0x8d, 0x16, 0xdb, 0x89,
-	0x28, 0xe3, 0x5d, 0x79, 0x7f, 0xc0, 0x88, 0x30, 0x65, 0xff, 0x67, 0xb7, 0x1d, 0x3e, 0x84, 0xba,
-	0x18, 0x97, 0x33, 0x09, 0xd5, 0x6b, 0xb0, 0x20, 0xaf, 0xb7, 0x41, 0xea, 0x56, 0xfb, 0x0a, 0xda,
-	0xa1, 0xb9, 0xd4, 0x3f, 0xd5, 0x18, 0x5c, 0x7a, 0x0b, 0x16, 0xe4, 0xed, 0xe9, 0x88, 0xa7, 0xfb,
-	0x9c, 0x7f, 0x8b, 0x0b, 0x46, 0x53, 0xfd, 0xee, 0x1c, 0x64, 0xd7, 0x44, 0x67, 0x9e, 0x98, 0x90,
-	0xc5, 0xa6, 0x37, 0xd1, 0x55, 0x41, 0xf5, 0x37, 0xd2, 0xb5, 0x4b, 0xb1, 0x18, 0x2c, 0x37, 0xcf,
-	0xfe, 0xf4, 0xfd, 0x1f, 0x5f, 0xa7, 0x4f, 0x43, 0x9e, 0x83, 0xfe, 0x8b, 0x3f, 0x13, 0xc4, 0x82,
-	0x19, 0xaf, 0x7b, 0x4a, 0xfe, 0x33, 0x4c, 0xaf, 0x59, 0xbb, 0x9c, 0x80, 0x8a, 0x77, 0x68, 0x03,
-	0xf8, 0xcd, 0x4b, 0xa2, 0x9c, 0x6b, 0xa0, 0x11, 0xab, 0x2d, 0x25, 0xc1, 0x12, 0x7d, 0xfa, 0xcd,
-	0x49, 0xb5, 0xcf, 0x81, 0x66, 0xa8, 0xda, 0xa7, 0xa2, 0xc7, 0x19, 0xe1, 0x53, 0xe4, 0xf0, 0x61,
-	0xdd, 0x69, 0x45, 0xe6, 0x30, 0xd0, 0x9c, 0x8c, 0xcc, 0x61, 0x5f, 0x1b, 0x32, 0x3e, 0x87, 0xbc,
-	0x39, 0x15, 0x9d, 0xc3, 0x60, 0xab, 0x2f, 0x3a, 0x87, 0x7d, 0x1d, 0xae, 0xc4, 0xfd, 0xe4, 0xcb,
-	0x8b, 0xd9, 0xcf, 0xe0, 0x0a, 0x97, 0x92, 0x60, 0x89, 0x3e, 0xfd, 0xe6, 0x92, 0xda, 0xe7, 0x40,
-	0xff, 0x4a, 0xed, 0x73, 0xb0, 0x47, 0x15, 0xe5, 0xf3, 0x29, 0xcc, 0x06, 0xef, 0xe9, 0xe4, 0xca,
-	0x90, 0xcd, 0x05, 0xad, 0x9c, 0x0c, 0x8c, 0xf7, 0xfc, 0x29, 0xe4, 0xfb, 0xba, 0x7b, 0x44, 0x39,
-	0xa3, 0xaa, 0x9b, 0xa8, 0x2d, 0x0f, 0x81, 0x4c, 0x74, 0xde, 0xd7, 0xb8, 0x52, 0x3b, 0x57, 0x35,
-	0xe7, 0xd4, 0xce, 0x95, 0x5d, 0xb0, 0x18, 0xe7, 0x7d, 0xfd, 0x29, 0xb5, 0x73, 0x55, 0x23, 0x4c,
-	0xed, 0x5c, 0xdd, 0xec, 0x8a, 0x25, 0x19, 0xde, 0xf7, 0x22, 0x49, 0xd6, 0xdf, 0x23, 0x88, 0x24,
-	0x59, 0xf8, 0xc2, 0x1f, 0x4f, 0x32, 0x79, 0x39, 0x8d, 0x26, 0x59, 0xe8, 0x46, 0x1d, 0x4d, 0xb2,
-	0xf0, 0x3d, 0x37, 0x91, 0x64, 0x72, 0xc1, 0x31, 0x24, 0x0b, 0xad, 0x79, 0x79, 0x08, 0xe4, 0x90,
-	0x79, 0x8e, 0x75, 0xae, 0x6a, 0xca, 0xc4, 0xe5, 0x79, 0x48, 0xe7, 0x22, 0xcf, 0x58, 0xb8, 0x47,
-	0xe6, 0xb9, 0xff, 0x62, 0x14, 0x99, 0xe7, 0xd0, 0xad, 0x21, 0x21, 0xcf, 0xf2, 0xe2, 0x18, 0x9d,
-	0xe7, 0xd0, 0x6d, 0x37, 0x3a, 0xcf, 0xe1, 0x3b, 0x68, 0xe2, 0x79, 0x96, 0x0b, 0x8e, 0x39, 0xcf,
-	0xa1, 0x35, 0x2f, 0x0f, 0x81, 0x4c, 0xfc, 0x71, 0xf2, 0x6e, 0x33, 0xea, 0x1f, 0xa7, 0xf0, 0x5d,
-	0x49, 0xbb, 0x9c, 0x80, 0x4a, 0xdc, 0xe7, 0xe0, 0xd5, 0x41, 0xbd, 0xcf, 0x8a, 0x6b, 0x91, 0x56,
-	0x4e, 0x06, 0xc6, 0x7b, 0xde, 0x87, 0x5c, 0xa0, 0x00, 0x26, 0x4b, 0xc3, 0xd5, 0xec, 0xda, 0x95,
-	0x44, 0x5c, 0xe2, 0x82, 0x83, 0xf5, 0xad, 0x7a, 0xc1, 0x8a, 0x62, 0x5a, 0x2b, 0x27, 0x03, 0x13,
-	0x3d, 0x07, 0x6b, 0x59, 0xb5, 0x67, 0x45, 0xbd, 0xac, 0x95, 0x93, 0x81, 0xb1, 0x9e, 0x57, 0x2f,
-	0x3c, 0x7f, 0x51, 0x3a, 0xf5, 0xcb, 0x8b, 0xd2, 0xa9, 0x3f, 0x5f, 0x94, 0x52, 0x9f, 0xf5, 0x4a,
-	0xa9, 0xe7, 0xbd, 0x52, 0xea, 0xe7, 0x5e, 0x29, 0xf5, 0x6b, 0xaf, 0x94, 0xda, 0x99, 0xe2, 0xff,
-	0x72, 0x72, 0xfd, 0xef, 0x00, 0x00, 0x00, 0xff, 0xff, 0x2a, 0x7c, 0x4c, 0x3e, 0xeb, 0x22, 0x00,
-	0x00,
+	// 1781 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xcc, 0x5a, 0xcd, 0x6f, 0x1b, 0x45,
+	0x14, 0xaf, 0xed, 0x24, 0x4e, 0x9e, 0xe3, 0x7c, 0x4c, 0xdc, 0x62, 0x6d, 0x8b, 0x53, 0x6d, 0x69,
+	0xea, 0xa0, 0xe0, 0x80, 0x4b, 0x45, 0x29, 0xe2, 0xa3, 0x8e, 0x69, 0x71, 0x53, 0x42, 0xb5, 0x69,
+	0x11, 0xb7, 0xc8, 0xb1, 0xa7, 0x61, 0x6b, 0xc7, 0x6b, 0x76, 0x37, 0x69, 0x23, 0x2e, 0x80, 0xe0,
+	0x4f, 0x40, 0xe2, 0xca, 0x15, 0x24, 0xce, 0xdc, 0xb8, 0x56, 0x9c, 0x38, 0x72, 0xb2, 0xa8, 0x25,
+	0x24, 0x4e, 0x88, 0xbf, 0x00, 0xa1, 0xf9, 0xda, 0x2f, 0xcf, 0xee, 0xda, 0x71, 0x50, 0x7a, 0x8a,
+	0x77, 0xf6, 0xf7, 0xe6, 0xbd, 0x99, 0xf7, 0x9b, 0xdf, 0xbe, 0x79, 0x0a, 0x64, 0x1b, 0x46, 0xc7,
+	0x36, 0x8d, 0x76, 0xa9, 0x6b, 0x1a, 0xb6, 0x81, 0x50, 0xd3, 0x68, 0xb4, 0xb0, 0x59, 0xb2, 0x1e,
+	0xd7, 0xcd, 0xfd, 0x96, 0x6e, 0x97, 0x0e, 0x5f, 0x53, 0x32, 0x56, 0x17, 0x37, 0x2c, 0x06, 0x50,
+	0xb2, 0xc6, 0xee, 0x23, 0xdc, 0xb0, 0xc5, 0x63, 0xc6, 0x3e, 0xea, 0x62, 0xf1, 0x90, 0xdb, 0x33,
+	0xf6, 0x0c, 0xfa, 0x73, 0x9d, 0xfc, 0xe2, 0xa3, 0x4b, 0xdd, 0xf6, 0xc1, 0x9e, 0xde, 0x59, 0x67,
+	0x7f, 0xd8, 0xa0, 0x7a, 0x0d, 0xe6, 0x6e, 0x63, 0x7b, 0xcb, 0x68, 0x62, 0x0d, 0x7f, 0x76, 0x80,
+	0x2d, 0x1b, 0x5d, 0x82, 0x74, 0xc7, 0x68, 0xe2, 0x1d, 0xbd, 0x99, 0x4f, 0x5c, 0x4c, 0x14, 0x67,
+	0x2a, 0xd0, 0xef, 0x2d, 0x4f, 0x11, 0x44, 0xad, 0xaa, 0x4d, 0x91, 0x57, 0xb5, 0xa6, 0xfa, 0x2e,
+	0xcc, 0x3b, 0x66, 0x56, 0xd7, 0xe8, 0x58, 0x18, 0xad, 0xc1, 0x04, 0x79, 0x49, 0x8d, 0x32, 0xe5,
+	0x7c, 0x69, 0x70, 0x01, 0x25, 0x8a, 0xa7, 0x28, 0xb5, 0x97, 0x82, 0x85, 0xbb, 0xba, 0x45, 0xa7,
+	0xb0, 0x84, 0xeb, 0x5b, 0x90, 0x7e, 0xa8, 0xb7, 0x6d, 0x6c, 0x5a, 0x7c, 0x96, 0x35, 0xd9, 0x2c,
+	0x41, 0xb3, 0xd2, 0x2d, 0x66, 0xa3, 0x09, 0x63, 0xe5, 0xcb, 0x14, 0xa4, 0xf9, 0x20, 0xca, 0xc1,
+	0x64, 0xa7, 0xbe, 0x8f, 0xc9, 0x8c, 0xa9, 0xe2, 0x8c, 0xc6, 0x1e, 0xd0, 0x3a, 0x64, 0xf4, 0xe6,
+	0x4e, 0xd7, 0xc4, 0x0f, 0xf5, 0x27, 0xd8, 0xca, 0x27, 0xc9, 0xbb, 0xca, 0x5c, 0xbf, 0xb7, 0x0c,
+	0xb5, 0xea, 0x3d, 0x3e, 0xaa, 0x81, 0xde, 0x14, 0xbf, 0xd1, 0x3d, 0x98, 0x6a, 0xd7, 0x77, 0x71,
+	0xdb, 0xca, 0xa7, 0x2e, 0xa6, 0x8a, 0x99, 0xf2, 0xf5, 0x51, 0x22, 0x2b, 0xdd, 0xa5, 0xa6, 0xef,
+	0x77, 0x6c, 0xf3, 0x48, 0xe3, 0xf3, 0xa0, 0x1a, 0x64, 0xf6, 0xf1, 0xfe, 0x2e, 0x36, 0xad, 0x4f,
+	0xf5, 0xae, 0x95, 0x9f, 0xb8, 0x98, 0x2a, 0xce, 0x95, 0xaf, 0x84, 0x6d, 0xdb, 0x76, 0x17, 0x37,
+	0x4a, 0x1f, 0x3a, 0x78, 0xcd, 0x6b, 0x8b, 0xca, 0x30, 0x69, 0x1a, 0x6d, 0x6c, 0xe5, 0x27, 0xe9,
+	0x24, 0x17, 0x42, 0xf7, 0xde, 0x68, 0x63, 0x8d, 0x41, 0xd1, 0x25, 0xc8, 0x92, 0xad, 0x70, 0xf7,
+	0x60, 0x8a, 0xee, 0xcf, 0x2c, 0x19, 0x14, 0xab, 0x56, 0xde, 0x84, 0x8c, 0x27, 0x74, 0xb4, 0x00,
+	0xa9, 0x16, 0x3e, 0x62, 0xb4, 0xd0, 0xc8, 0x4f, 0xb2, 0xbb, 0x87, 0xf5, 0xf6, 0x01, 0xce, 0x27,
+	0xe9, 0x18, 0x7b, 0xb8, 0x91, 0xbc, 0x9e, 0x50, 0x37, 0x60, 0xd1, 0xb3, 0x1d, 0x9c, 0x23, 0x25,
+	0x98, 0x24, 0xd9, 0x67, 0xc9, 0x88, 0x22, 0x09, 0x83, 0xa9, 0x3f, 0x24, 0x60, 0xf1, 0x41, 0xb7,
+	0x59, 0xb7, 0xf1, 0xa8, 0x0c, 0x45, 0xef, 0xc0, 0x2c, 0x05, 0x1d, 0x62, 0xd3, 0xd2, 0x8d, 0x0e,
+	0x0d, 0x30, 0x53, 0x3e, 0x2f, 0xf3, 0xf8, 0x31, 0x83, 0x68, 0x19, 0x62, 0xc0, 0x1f, 0xd0, 0xab,
+	0x30, 0x41, 0x8e, 0x5b, 0x3e, 0x45, 0xed, 0x2e, 0x44, 0xe5, 0x45, 0xa3, 0x48, 0xb5, 0x02, 0xc8,
+	0x1b, 0xeb, 0xb1, 0x8e, 0xc5, 0x16, 0x2c, 0x6a, 0x78, 0xdf, 0x38, 0x1c, 0x7d, 0xbd, 0x39, 0x98,
+	0x7c, 0x68, 0x98, 0x0d, 0x96, 0x89, 0x69, 0x8d, 0x3d, 0xa8, 0x39, 0x40, 0xde, 0xf9, 0x58, 0x4c,
+	0xfc, 0xd0, 0xdf, 0xaf, 0x5b, 0x2d, 0x8f, 0x0b, 0xbb, 0x6e, 0xb5, 0x02, 0x2e, 0x08, 0x82, 0xb8,
+	0x20, 0xaf, 0x9c, 0x43, 0xcf, 0xcc, 0xdc, 0xd5, 0x91, 0x97, 0x51, 0xab, 0xa3, 0x78, 0x8a, 0x52,
+	0xaf, 0x8b, 0xd5, 0x8d, 0xec, 0xda, 0x59, 0x87, 0xd7, 0xbb, 0xfa, 0x2f, 0x17, 0x11, 0x32, 0x78,
+	0x0c, 0x11, 0xf1, 0x9a, 0x0d, 0x8a, 0xc8, 0xf7, 0xa7, 0x28, 0x22, 0xb2, 0xc8, 0xa4, 0x22, 0xb2,
+	0x0e, 0x19, 0x0b, 0x9b, 0x87, 0x7a, 0x83, 0xb0, 0x83, 0x89, 0x08, 0x0f, 0x61, 0x9b, 0x0d, 0xd7,
+	0xaa, 0x96, 0x06, 0x1c, 0x52, 0x6b, 0x5a, 0x68, 0x05, 0xa6, 0x39, 0x97, 0x98, 0x5a, 0xcc, 0x54,
+	0x32, 0xfd, 0xde, 0x72, 0x9a, 0x91, 0xc9, 0xd2, 0xd2, 0x8c, 0x4d, 0x16, 0xaa, 0xc2, 0x5c, 0x13,
+	0x5b, 0xba, 0x89, 0x9b, 0x3b, 0x96, 0x5d, 0xb7, 0xb9, 0x3e, 0xcc, 0x95, 0x5f, 0x0c, 0x4b, 0xf1,
+	0x36, 0x41, 0x69, 0x59, 0x6e, 0x44, 0x9f, 0x24, 0x22, 0x93, 0xfe, 0x5f, 0x44, 0x86, 0x6f, 0x97,
+	0x2b, 0x32, 0x84, 0x35, 0x91, 0x22, 0x43, 0x69, 0xc4, 0x60, 0xea, 0x26, 0xe4, 0x36, 0x4c, 0x5c,
+	0xb7, 0x31, 0xdf, 0x32, 0x41, 0xa4, 0xab, 0x5c, 0x01, 0x18, 0x8b, 0x96, 0x65, 0xd3, 0x70, 0x0b,
+	0x8f, 0x08, 0x6c, 0xc1, 0xd9, 0xc0, 0x64, 0x3c, 0xaa, 0x6b, 0x90, 0xe6, 0x69, 0xe0, 0x13, 0x9e,
+	0x8f, 0x98, 0x50, 0x13, 0x58, 0xf5, 0x26, 0x2c, 0xde, 0xc6, 0x76, 0x20, 0xb2, 0x35, 0x00, 0x37,
+	0xeb, 0xfc, 0xd4, 0x64, 0xfb, 0xbd, 0xe5, 0x19, 0x27, 0xe9, 0xda, 0x8c, 0x93, 0x73, 0x75, 0x13,
+	0x90, 0x77, 0x8a, 0xf1, 0xe2, 0xf9, 0x25, 0x01, 0x39, 0xa6, 0x72, 0xe3, 0xc4, 0x84, 0xaa, 0x30,
+	0x2f, 0xd0, 0x23, 0x08, 0xf4, 0x1c, 0xb7, 0x11, 0x1a, 0x7d, 0xd5, 0xa7, 0xd1, 0xc3, 0x67, 0x28,
+	0xb0, 0x80, 0xf1, 0x76, 0xa4, 0x0a, 0x39, 0x26, 0x4d, 0x63, 0x25, 0xe9, 0x05, 0x38, 0x1b, 0x98,
+	0x85, 0x6b, 0xdc, 0x5f, 0x49, 0x58, 0x22, 0x1c, 0xe7, 0xe3, 0x8e, 0xcc, 0xd5, 0x82, 0x32, 0xb7,
+	0x1e, 0x26, 0x26, 0x01, 0xcb, 0x41, 0xa5, 0xfb, 0x26, 0x79, 0xe2, 0x4a, 0xb7, 0x1d, 0x50, 0xba,
+	0xb7, 0x46, 0x0c, 0x4e, 0x2a, 0x76, 0x03, 0x6a, 0x32, 0x71, 0xb2, 0x6a, 0xf2, 0x11, 0xe4, 0xfc,
+	0x21, 0x71, 0x62, 0xbc, 0x01, 0xd3, 0x3c, 0x51, 0x42, 0x53, 0x22, 0x99, 0xe1, 0x80, 0x5d, 0x65,
+	0xd9, 0xc2, 0xf6, 0x63, 0xc3, 0x6c, 0x8d, 0xa0, 0x2c, 0xdc, 0x42, 0xa6, 0x2c, 0xce, 0x64, 0x2e,
+	0x6f, 0x3b, 0x6c, 0x28, 0x8a, 0xb7, 0xc2, 0x4a, 0x60, 0xd5, 0x07, 0x54, 0x59, 0x02, 0x91, 0x21,
+	0x98, 0x20, 0xbb, 0xc9, 0xf7, 0x8b, 0xfe, 0x26, 0x44, 0xe6, 0x36, 0x84, 0xc8, 0x49, 0x97, 0xc8,
+	0xdc, 0x96, 0x10, 0x99, 0x03, 0x1c, 0xb5, 0x39, 0xa1, 0x18, 0x3f, 0x11, 0x67, 0xeb, 0xc4, 0xc3,
+	0x74, 0xce, 0x5b, 0x20, 0x52, 0xe7, 0xbc, 0xf1, 0xf1, 0x63, 0x9c, 0xb7, 0x80, 0xe5, 0xf3, 0x75,
+	0xde, 0x42, 0x82, 0x3b, 0xcd, 0xf3, 0xe6, 0x86, 0xe4, 0x9e, 0x37, 0x9e, 0xa8, 0xc8, 0xf3, 0x26,
+	0x32, 0xe7, 0x80, 0xf9, 0xc7, 0x72, 0xa3, 0x7d, 0x60, 0xd9, 0xd8, 0xf4, 0xe8, 0x70, 0x83, 0x8d,
+	0x04, 0x74, 0x98, 0xe3, 0x08, 0x2f, 0x38, 0xc0, 0xa1, 0xaf, 0x33, 0x85, 0x4b, 0x5f, 0x0e, 0x89,
+	0xa2, 0xaf, 0xb0, 0x12, 0x58, 0x87, 0x4b, 0xfc, 0xc5, 0x31, 0xb8, 0x14, 0xb0, 0x7c, 0xbe, 0xb8,
+	0x14, 0x12, 0xdc, 0x69, 0x72, 0xc9, 0x0d, 0xc9, 0xe5, 0x12, 0xcf, 0x46, 0x24, 0x97, 0x44, 0xea,
+	0x1c, 0xb0, 0xfa, 0x6d, 0x02, 0x32, 0x9b, 0xf8, 0x48, 0x33, 0xec, 0xba, 0x4d, 0x6a, 0x8d, 0x97,
+	0x61, 0x91, 0x90, 0x0c, 0x9b, 0x3b, 0x8f, 0x0c, 0xbd, 0xb3, 0x63, 0x1b, 0x2d, 0xdc, 0xa1, 0xa1,
+	0x4d, 0x6b, 0xf3, 0xec, 0xc5, 0x1d, 0x43, 0xef, 0xdc, 0x27, 0xc3, 0x68, 0x0d, 0xd0, 0x7e, 0xbd,
+	0x53, 0xdf, 0xf3, 0x83, 0xd9, 0xc5, 0x6c, 0x81, 0xbf, 0x91, 0xa2, 0x0f, 0x3a, 0x6d, 0xa3, 0xd1,
+	0xda, 0x21, 0xab, 0x4e, 0xf9, 0xd0, 0x0f, 0xe8, 0x8b, 0x4d, 0x7c, 0xa4, 0x7e, 0x95, 0x14, 0x05,
+	0xd8, 0x38, 0x3c, 0x27, 0x05, 0x98, 0x40, 0x8f, 0x52, 0x80, 0x71, 0x9b, 0x11, 0x0a, 0x30, 0xee,
+	0xdd, 0xfd, 0x90, 0xa1, 0x9b, 0x30, 0x6d, 0xf2, 0x5d, 0xcd, 0x4f, 0x84, 0x1b, 0x7a, 0x36, 0xbf,
+	0x32, 0xf1, 0xb4, 0xb7, 0x7c, 0x46, 0x73, 0xcc, 0xdc, 0x1a, 0xee, 0x84, 0x0e, 0xea, 0xdb, 0xb0,
+	0x40, 0x4b, 0xe4, 0x86, 0x89, 0x6d, 0xb1, 0x9f, 0xab, 0x30, 0x63, 0xd1, 0x01, 0x77, 0x3b, 0x67,
+	0xfb, 0xbd, 0xe5, 0x69, 0x86, 0xaa, 0x55, 0xc9, 0x77, 0x9e, 0xfe, 0x6a, 0xaa, 0xb7, 0x79, 0x91,
+	0xce, 0xcc, 0x79, 0x28, 0x65, 0x98, 0x62, 0x00, 0x1e, 0x89, 0x22, 0xaf, 0x19, 0xa8, 0x0d, 0x47,
+	0xaa, 0x3f, 0x27, 0x60, 0x49, 0x14, 0xa7, 0xc7, 0x8b, 0x05, 0x55, 0x60, 0x8e, 0x43, 0x47, 0xc8,
+	0x6b, 0x96, 0x99, 0x88, 0xb4, 0x96, 0x7d, 0x69, 0x2d, 0x84, 0x07, 0xee, 0x29, 0x4f, 0xee, 0xb8,
+	0xf7, 0x82, 0xb1, 0xb7, 0xe1, 0xcf, 0x24, 0x20, 0x56, 0x89, 0x91, 0x47, 0x47, 0x36, 0x3f, 0x08,
+	0xca, 0x66, 0x29, 0xbc, 0xaa, 0xf4, 0x1a, 0x0e, 0xaa, 0xe6, 0xd7, 0x27, 0xaf, 0x9a, 0x5a, 0x40,
+	0x35, 0x6f, 0x8c, 0x16, 0xdb, 0xa9, 0x88, 0xe6, 0xa6, 0xb8, 0x5a, 0xf0, 0x88, 0x78, 0xca, 0x5e,
+	0x27, 0x17, 0x21, 0x3a, 0xc4, 0x25, 0x33, 0x2a, 0x67, 0x02, 0xaa, 0xd6, 0x60, 0x49, 0xdc, 0x7c,
+	0xbd, 0xd4, 0x2d, 0xfb, 0x6a, 0xdd, 0xa1, 0xb9, 0xe4, 0x9f, 0x6a, 0x0c, 0x2e, 0xbd, 0x07, 0x4b,
+	0xe2, 0x62, 0x75, 0xcc, 0xd3, 0x7d, 0xce, 0xbd, 0xe0, 0x79, 0xa3, 0x29, 0xff, 0x78, 0x0e, 0xd2,
+	0x1b, 0xac, 0x69, 0x8f, 0x74, 0x48, 0xf3, 0x7e, 0x38, 0x52, 0x65, 0x41, 0xf9, 0x7b, 0xec, 0xca,
+	0xa5, 0x48, 0x0c, 0xaf, 0x44, 0xcf, 0xfe, 0xfa, 0xd3, 0xdf, 0xdf, 0x25, 0xe7, 0x21, 0x4b, 0x41,
+	0xaf, 0xf0, 0x2f, 0x01, 0x32, 0x60, 0xc6, 0x69, 0xac, 0xa2, 0x97, 0x86, 0x69, 0x43, 0x2b, 0x97,
+	0x63, 0x50, 0xd1, 0x0e, 0x4d, 0x00, 0xb7, 0xaf, 0x89, 0xa4, 0x73, 0x0d, 0xf4, 0x68, 0x95, 0x95,
+	0x38, 0x58, 0xac, 0x4f, 0xb7, 0x6f, 0x29, 0xf7, 0x39, 0xd0, 0x27, 0x95, 0xfb, 0x94, 0xb4, 0x3f,
+	0x43, 0x7c, 0xb2, 0x1c, 0xde, 0xaf, 0x5b, 0xad, 0xd0, 0x1c, 0x7a, 0xfa, 0x96, 0xa1, 0x39, 0xf4,
+	0x75, 0x28, 0xa3, 0x73, 0x48, 0xfb, 0x56, 0xe1, 0x39, 0xf4, 0x76, 0x01, 0xc3, 0x73, 0xe8, 0x6b,
+	0x7e, 0xc5, 0xee, 0x27, 0x5d, 0x5e, 0xc4, 0x7e, 0x7a, 0x57, 0xb8, 0x12, 0x07, 0x8b, 0xf5, 0xe9,
+	0xf6, 0x9d, 0xe4, 0x3e, 0x07, 0x5a, 0x5b, 0x72, 0x9f, 0x83, 0xed, 0xab, 0x30, 0x9f, 0x4f, 0x60,
+	0xd6, 0x7b, 0x85, 0x47, 0x57, 0x86, 0xec, 0x3b, 0x28, 0xc5, 0x78, 0x60, 0xb4, 0xe7, 0xcf, 0x21,
+	0xeb, 0x6b, 0xfc, 0x21, 0xe9, 0x8c, 0xb2, 0x46, 0xa3, 0xb2, 0x3a, 0x04, 0x32, 0xd6, 0xb9, 0xaf,
+	0xa7, 0x25, 0x77, 0x2e, 0xeb, 0xdb, 0xc9, 0x9d, 0x4b, 0x1b, 0x64, 0x11, 0xce, 0x7d, 0xad, 0x2b,
+	0xb9, 0x73, 0x59, 0x8f, 0x4c, 0xee, 0x5c, 0xde, 0x07, 0x8b, 0x24, 0x19, 0xbf, 0x0a, 0x86, 0x92,
+	0xcc, 0xdf, 0x3e, 0x08, 0x25, 0x59, 0xb0, 0x17, 0x10, 0x4d, 0x32, 0x71, 0x6f, 0x0d, 0x27, 0x59,
+	0xe0, 0xb2, 0x1d, 0x4e, 0xb2, 0xe0, 0x15, 0x38, 0x96, 0x64, 0x62, 0xc1, 0x11, 0x24, 0x0b, 0xac,
+	0x79, 0x75, 0x08, 0xe4, 0x90, 0x79, 0x8e, 0x74, 0x2e, 0xeb, 0xd7, 0x44, 0xe5, 0x79, 0x48, 0xe7,
+	0x2c, 0xcf, 0xbc, 0x70, 0x0f, 0xcd, 0xb3, 0xff, 0x4a, 0x14, 0x9a, 0xe7, 0xc0, 0xad, 0x21, 0x26,
+	0xcf, 0xe2, 0x4e, 0x19, 0x9e, 0xe7, 0xc0, 0x45, 0x38, 0x3c, 0xcf, 0xc1, 0xeb, 0x69, 0xec, 0x79,
+	0x16, 0x0b, 0x8e, 0x38, 0xcf, 0x81, 0x35, 0xaf, 0x0e, 0x81, 0x8c, 0xfd, 0x38, 0x39, 0xb7, 0x19,
+	0xf9, 0xc7, 0x29, 0x78, 0x57, 0x52, 0x2e, 0xc7, 0xa0, 0x62, 0xf7, 0xd9, 0x7b, 0x75, 0x90, 0xef,
+	0xb3, 0xe4, 0x5a, 0xa4, 0x14, 0xe3, 0x81, 0xd1, 0x9e, 0x0f, 0x20, 0xe3, 0x29, 0x80, 0xd1, 0xca,
+	0x70, 0x35, 0xbb, 0x72, 0x25, 0x16, 0x17, 0xbb, 0x60, 0x6f, 0x7d, 0x2b, 0x5f, 0xb0, 0xa4, 0x98,
+	0x56, 0x8a, 0xf1, 0xc0, 0x58, 0xcf, 0xde, 0x5a, 0x56, 0xee, 0x59, 0x52, 0x2f, 0x2b, 0xc5, 0x78,
+	0x60, 0xa4, 0xe7, 0xca, 0x85, 0xa7, 0xcf, 0x0a, 0x67, 0x7e, 0x7f, 0x56, 0x38, 0xf3, 0xcf, 0xb3,
+	0x42, 0xe2, 0x8b, 0x7e, 0x21, 0xf1, 0xb4, 0x5f, 0x48, 0xfc, 0xd6, 0x2f, 0x24, 0xfe, 0xe8, 0x17,
+	0x12, 0xbb, 0x53, 0xf4, 0xbf, 0x51, 0xae, 0xfe, 0x17, 0x00, 0x00, 0xff, 0xff, 0x0c, 0xe8, 0xa4,
+	0xf9, 0x06, 0x23, 0x00, 0x00,
 }

vendor/github.com/docker/swarmkit/api/control.proto

@@ -313,12 +313,17 @@ message ListClustersResponse {
 	repeated Cluster clusters = 1;
 }
 
-message JoinTokenRotation {
-	// RotateWorkerToken tells UpdateCluster to rotate the worker secret.
-	bool rotate_worker_token = 1;
+// KeyRotation tells UpdateCluster what items to rotate
+message KeyRotation {
+	// WorkerJoinToken tells UpdateCluster to rotate the worker secret token.
+	bool worker_join_token = 1;
+
+	// ManagerJoinToken tells UpdateCluster to rotate the manager secret token.
+	bool manager_join_token = 2;
+
+	// ManagerUnlockKey tells UpdateCluster to rotate the manager unlock key
+	bool manager_unlock_key = 3;
 
-	// RotateManagerSecret tells UpdateCluster to rotate the manager secret.
-	bool rotate_manager_token = 2;
 }
 
 message UpdateClusterRequest {
@@ -331,8 +336,8 @@ message UpdateClusterRequest {
 	// Spec is the new spec to apply to the cluster.
 	ClusterSpec spec = 3;
 
-	// Rotation contains flags for join token rotation
-	JoinTokenRotation rotation = 4 [(gogoproto.nullable) = false];
+	// Rotation contains flags for join token and unlock key rotation
+	KeyRotation rotation = 4 [(gogoproto.nullable) = false];
 }
 
 message UpdateClusterResponse {

vendor/github.com/docker/swarmkit/api/objects.pb.go

@@ -232,6 +232,12 @@ type Cluster struct {
 	// be honored. It's a mapping from CN -> BlacklistedCertificate.
 	// swarm. Their certificates should effectively be blacklisted.
 	BlacklistedCertificates map[string]*BlacklistedCertificate `protobuf:"bytes,8,rep,name=blacklisted_certificates,json=blacklistedCertificates" json:"blacklisted_certificates,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value"`
+	// UnlockKeys defines the keys that lock node data at rest.  For example,
+	// this would contain the key encrypting key (KEK) that will encrypt the
+	// manager TLS keys at rest and the raft encryption keys at rest.
+	// If the key is empty, the node will be unlocked (will not require a key
+	// to start up from a shut down state).
+	UnlockKeys []*EncryptionKey `protobuf:"bytes,9,rep,name=unlock_keys,json=unlockKeys" json:"unlock_keys,omitempty"`
 }
 
 func (m *Cluster) Reset()                    { *m = Cluster{} }
@@ -460,6 +466,13 @@ func (m *Cluster) Copy() *Cluster {
 		}
 	}
 
+	if m.UnlockKeys != nil {
+		o.UnlockKeys = make([]*EncryptionKey, 0, len(m.UnlockKeys))
+		for _, v := range m.UnlockKeys {
+			o.UnlockKeys = append(o.UnlockKeys, v.Copy())
+		}
+	}
+
 	return o
 }
 
@@ -633,7 +646,7 @@ func (this *Cluster) GoString() string {
 	if this == nil {
 		return "nil"
 	}
-	s := make([]string, 0, 11)
+	s := make([]string, 0, 12)
 	s = append(s, "&api.Cluster{")
 	s = append(s, "ID: "+fmt.Sprintf("%#v", this.ID)+",\n")
 	s = append(s, "Meta: "+strings.Replace(this.Meta.GoString(), `&`, ``, 1)+",\n")
@@ -656,6 +669,9 @@ func (this *Cluster) GoString() string {
 	if this.BlacklistedCertificates != nil {
 		s = append(s, "BlacklistedCertificates: "+mapStringForBlacklistedCertificates+",\n")
 	}
+	if this.UnlockKeys != nil {
+		s = append(s, "UnlockKeys: "+fmt.Sprintf("%#v", this.UnlockKeys)+",\n")
+	}
 	s = append(s, "}")
 	return strings.Join(s, "")
 }
@@ -1310,6 +1326,18 @@ func (m *Cluster) MarshalTo(data []byte) (int, error) {
 			}
 		}
 	}
+	if len(m.UnlockKeys) > 0 {
+		for _, msg := range m.UnlockKeys {
+			data[i] = 0x4a
+			i++
+			i = encodeVarintObjects(data, i, uint64(msg.Size()))
+			n, err := msg.MarshalTo(data[i:])
+			if err != nil {
+				return 0, err
+			}
+			i += n
+		}
+	}
 	return i, nil
 }
 
@@ -1637,6 +1665,12 @@ func (m *Cluster) Size() (n int) {
 			n += mapEntrySize + 1 + sovObjects(uint64(mapEntrySize))
 		}
 	}
+	if len(m.UnlockKeys) > 0 {
+		for _, e := range m.UnlockKeys {
+			l = e.Size()
+			n += 1 + l + sovObjects(uint64(l))
+		}
+	}
 	return n
 }
 
@@ -1814,6 +1848,7 @@ func (this *Cluster) String() string {
 		`NetworkBootstrapKeys:` + strings.Replace(fmt.Sprintf("%v", this.NetworkBootstrapKeys), "EncryptionKey", "EncryptionKey", 1) + `,`,
 		`EncryptionKeyLamportClock:` + fmt.Sprintf("%v", this.EncryptionKeyLamportClock) + `,`,
 		`BlacklistedCertificates:` + mapStringForBlacklistedCertificates + `,`,
+		`UnlockKeys:` + strings.Replace(fmt.Sprintf("%v", this.UnlockKeys), "EncryptionKey", "EncryptionKey", 1) + `,`,
 		`}`,
 	}, "")
 	return s
@@ -3863,6 +3898,37 @@ func (m *Cluster) Unmarshal(data []byte) error {
 				m.BlacklistedCertificates[mapkey] = mapvalue
 			}
 			iNdEx = postIndex
+		case 9:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field UnlockKeys", wireType)
+			}
+			var msglen int
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowObjects
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := data[iNdEx]
+				iNdEx++
+				msglen |= (int(b) & 0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			if msglen < 0 {
+				return ErrInvalidLengthObjects
+			}
+			postIndex := iNdEx + msglen
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			m.UnlockKeys = append(m.UnlockKeys, &EncryptionKey{})
+			if err := m.UnlockKeys[len(m.UnlockKeys)-1].Unmarshal(data[iNdEx:postIndex]); err != nil {
+				return err
+			}
+			iNdEx = postIndex
 		default:
 			iNdEx = preIndex
 			skippy, err := skipObjects(data[iNdEx:])
@@ -4199,79 +4265,80 @@ var (
 func init() { proto.RegisterFile("objects.proto", fileDescriptorObjects) }
 
 var fileDescriptorObjects = []byte{
-	// 1174 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xbc, 0x57, 0x4d, 0x8f, 0x1b, 0x35,
-	0x18, 0xee, 0x24, 0xb3, 0xf9, 0x78, 0xb3, 0x59, 0x81, 0xa9, 0xca, 0x34, 0x2c, 0xc9, 0x92, 0x0a,
-	0x54, 0xa1, 0x2a, 0x15, 0xa5, 0xa0, 0x2d, 0xb4, 0x82, 0x7c, 0x09, 0xa2, 0x52, 0xa8, 0xdc, 0xd2,
-	0x1e, 0x23, 0xef, 0x8c, 0x1b, 0x86, 0x4c, 0xc6, 0x23, 0xdb, 0x49, 0x95, 0x9e, 0x10, 0x3f, 0x80,
-	0x9f, 0xc0, 0x5f, 0xe1, 0xba, 0x07, 0x0e, 0xdc, 0xe0, 0x80, 0x22, 0x36, 0x07, 0x24, 0x6e, 0xfc,
-	0x04, 0x64, 0x8f, 0x27, 0x99, 0x55, 0x26, 0xcb, 0x56, 0xaa, 0xf6, 0xe6, 0x37, 0x7e, 0x9e, 0xc7,
-	0xef, 0x97, 0xdf, 0x71, 0xa0, 0xca, 0x8e, 0xbe, 0xa7, 0xae, 0x14, 0xad, 0x88, 0x33, 0xc9, 0x10,
-	0xf2, 0x98, 0x3b, 0xa6, 0xbc, 0x25, 0x9e, 0x13, 0x3e, 0x19, 0xfb, 0xb2, 0x35, 0xfb, 0xa0, 0x56,
-	0x91, 0xf3, 0x88, 0x1a, 0x40, 0xad, 0x22, 0x22, 0xea, 0x26, 0xc6, 0x55, 0xe9, 0x4f, 0xa8, 0x90,
-	0x64, 0x12, 0xdd, 0x5c, 0xad, 0xcc, 0xd6, 0xe5, 0x11, 0x1b, 0x31, 0xbd, 0xbc, 0xa9, 0x56, 0xf1,
-	0xaf, 0xcd, 0x5f, 0x2c, 0xb0, 0x1f, 0x50, 0x49, 0xd0, 0xa7, 0x50, 0x9c, 0x51, 0x2e, 0x7c, 0x16,
-	0x3a, 0xd6, 0x81, 0x75, 0xbd, 0x72, 0xeb, 0xad, 0xd6, 0xe6, 0xc9, 0xad, 0x27, 0x31, 0xa4, 0x63,
-	0x1f, 0x2f, 0x1a, 0x97, 0x70, 0xc2, 0x40, 0x77, 0x01, 0x5c, 0x4e, 0x89, 0xa4, 0xde, 0x90, 0x48,
-	0x27, 0xa7, 0xf9, 0x6f, 0x67, 0xf1, 0x1f, 0x27, 0x4e, 0xe1, 0xb2, 0x21, 0xb4, 0xa5, 0x62, 0x4f,
-	0x23, 0x2f, 0x61, 0xe7, 0xcf, 0xc5, 0x36, 0x84, 0xb6, 0x6c, 0xfe, 0x93, 0x07, 0xfb, 0x6b, 0xe6,
-	0x51, 0x74, 0x05, 0x72, 0xbe, 0xa7, 0x9d, 0x2f, 0x77, 0x0a, 0xcb, 0x45, 0x23, 0x37, 0xe8, 0xe1,
-	0x9c, 0xef, 0xa1, 0x5b, 0x60, 0x4f, 0xa8, 0x24, 0xc6, 0x2d, 0x27, 0x4b, 0x58, 0x65, 0xc0, 0xc4,
-	0xa4, 0xb1, 0xe8, 0x63, 0xb0, 0x55, 0x5a, 0x8d, 0x33, 0xfb, 0x59, 0x1c, 0x75, 0xe6, 0xa3, 0x88,
-	0xba, 0x09, 0x4f, 0xe1, 0x51, 0x1f, 0x2a, 0x1e, 0x15, 0x2e, 0xf7, 0x23, 0xa9, 0x32, 0x69, 0x6b,
-	0xfa, 0xb5, 0x6d, 0xf4, 0xde, 0x1a, 0x8a, 0xd3, 0x3c, 0x74, 0x17, 0x0a, 0x42, 0x12, 0x39, 0x15,
-	0xce, 0x8e, 0x56, 0xa8, 0x6f, 0x75, 0x40, 0xa3, 0x8c, 0x0b, 0x86, 0x83, 0xbe, 0x84, 0xbd, 0x09,
-	0x09, 0xc9, 0x88, 0xf2, 0xa1, 0x51, 0x29, 0x68, 0x95, 0x77, 0x32, 0x43, 0x8f, 0x91, 0xb1, 0x10,
-	0xae, 0x4e, 0xd2, 0x26, 0xea, 0x03, 0x10, 0x29, 0x89, 0xfb, 0xdd, 0x84, 0x86, 0xd2, 0x29, 0x6a,
-	0x95, 0x77, 0x33, 0x7d, 0xa1, 0xf2, 0x39, 0xe3, 0xe3, 0xf6, 0x0a, 0x8c, 0x53, 0x44, 0xf4, 0x05,
-	0x54, 0x5c, 0xca, 0xa5, 0xff, 0xcc, 0x77, 0x89, 0xa4, 0x4e, 0x49, 0xeb, 0x34, 0xb2, 0x74, 0xba,
-	0x6b, 0x98, 0x09, 0x2a, 0xcd, 0x6c, 0xfe, 0x9e, 0x83, 0xe2, 0x23, 0xca, 0x67, 0xbe, 0xfb, 0x6a,
-	0xcb, 0x7d, 0xe7, 0x54, 0xb9, 0x33, 0x3d, 0x33, 0xc7, 0x6e, 0x54, 0xfc, 0x10, 0x4a, 0x34, 0xf4,
-	0x22, 0xe6, 0x87, 0xd2, 0x94, 0x3b, 0xb3, 0x5b, 0xfa, 0x06, 0x83, 0x57, 0x68, 0xd4, 0x87, 0x6a,
-	0xdc, 0xc5, 0xc3, 0x53, 0xb5, 0x3e, 0xc8, 0xa2, 0x7f, 0xab, 0x81, 0xa6, 0x48, 0xbb, 0xd3, 0x94,
-	0x85, 0x7a, 0x50, 0x8d, 0x38, 0x9d, 0xf9, 0x6c, 0x2a, 0x86, 0x3a, 0x88, 0xc2, 0xb9, 0x82, 0xc0,
-	0xbb, 0x09, 0x4b, 0x59, 0xcd, 0x9f, 0x73, 0x50, 0x4a, 0x7c, 0x44, 0xb7, 0x4d, 0x3a, 0xac, 0xed,
-	0x0e, 0x25, 0x58, 0x2d, 0x15, 0x67, 0xe2, 0x36, 0xec, 0x44, 0x8c, 0x4b, 0xe1, 0xe4, 0x0e, 0xf2,
-	0xdb, 0x7a, 0xf6, 0x21, 0xe3, 0xb2, 0xcb, 0xc2, 0x67, 0xfe, 0x08, 0xc7, 0x60, 0xf4, 0x14, 0x2a,
-	0x33, 0x9f, 0xcb, 0x29, 0x09, 0x86, 0x7e, 0x24, 0x9c, 0xbc, 0xe6, 0xbe, 0x77, 0xd6, 0x91, 0xad,
-	0x27, 0x31, 0x7e, 0xf0, 0xb0, 0xb3, 0xb7, 0x5c, 0x34, 0x60, 0x65, 0x0a, 0x0c, 0x46, 0x6a, 0x10,
-	0x89, 0xda, 0x03, 0x28, 0xaf, 0x76, 0xd0, 0x0d, 0x80, 0x30, 0x6e, 0xd1, 0xe1, 0xaa, 0x69, 0xaa,
-	0xcb, 0x45, 0xa3, 0x6c, 0x1a, 0x77, 0xd0, 0xc3, 0x65, 0x03, 0x18, 0x78, 0x08, 0x81, 0x4d, 0x3c,
-	0x8f, 0xeb, 0x16, 0x2a, 0x63, 0xbd, 0x6e, 0xfe, 0xba, 0x03, 0xf6, 0x63, 0x22, 0xc6, 0x17, 0x3d,
-	0x66, 0xd4, 0x99, 0x1b, 0x4d, 0x77, 0x03, 0x40, 0xc4, 0xa5, 0x54, 0xe1, 0xd8, 0xeb, 0x70, 0x4c,
-	0x81, 0x55, 0x38, 0x06, 0x10, 0x87, 0x23, 0x02, 0x26, 0x75, 0x7f, 0xd9, 0x58, 0xaf, 0xd1, 0x35,
-	0x28, 0x86, 0xcc, 0xd3, 0xf4, 0x82, 0xa6, 0xc3, 0x72, 0xd1, 0x28, 0xa8, 0x91, 0x32, 0xe8, 0xe1,
-	0x82, 0xda, 0x1a, 0x78, 0xea, 0xde, 0x92, 0x30, 0x64, 0x92, 0xa8, 0xa1, 0x24, 0xcc, 0xfd, 0xcf,
-	0x6c, 0xac, 0xf6, 0x1a, 0x96, 0xdc, 0xdb, 0x14, 0x13, 0x3d, 0x81, 0x37, 0x12, 0x7f, 0xd3, 0x82,
-	0xa5, 0x97, 0x11, 0x44, 0x46, 0x21, 0xb5, 0x93, 0x9a, 0x93, 0xe5, 0xed, 0x73, 0x52, 0x67, 0x30,
-	0x6b, 0x4e, 0x76, 0xa0, 0xea, 0x51, 0xe1, 0x73, 0xea, 0xe9, 0x1b, 0x48, 0x1d, 0x38, 0xb0, 0xae,
-	0xef, 0x6d, 0xf9, 0xf4, 0x18, 0x11, 0x8a, 0x77, 0x0d, 0x47, 0x5b, 0xa8, 0x0d, 0x25, 0xd3, 0x37,
-	0xc2, 0xa9, 0xe8, 0xde, 0x3d, 0xe7, 0x7c, 0x5c, 0xd1, 0x4e, 0x4d, 0x90, 0xdd, 0x97, 0x9a, 0x20,
-	0x77, 0x00, 0x02, 0x36, 0x1a, 0x7a, 0xdc, 0x9f, 0x51, 0xee, 0x54, 0x35, 0xb7, 0x96, 0xc5, 0xed,
-	0x69, 0x04, 0x2e, 0x07, 0x6c, 0x14, 0x2f, 0x9b, 0x3f, 0x5a, 0xf0, 0xfa, 0x86, 0x53, 0xe8, 0x23,
-	0x28, 0x1a, 0xb7, 0xce, 0x7a, 0x04, 0x18, 0x1e, 0x4e, 0xb0, 0x68, 0x1f, 0xca, 0xea, 0x8e, 0x50,
-	0x21, 0x68, 0x7c, 0xfb, 0xcb, 0x78, 0xfd, 0x03, 0x72, 0xa0, 0x48, 0x02, 0x9f, 0xa8, 0xbd, 0xbc,
-	0xde, 0x4b, 0xcc, 0xe6, 0x4f, 0x39, 0x28, 0x1a, 0xb1, 0x8b, 0x1e, 0xe7, 0xe6, 0xd8, 0x8d, 0x9b,
-	0x75, 0x0f, 0x76, 0xe3, 0x74, 0x9a, 0x96, 0xb0, 0xff, 0x37, 0xa9, 0x95, 0x18, 0x1f, 0xb7, 0xc3,
-	0x3d, 0xb0, 0xfd, 0x88, 0x4c, 0xcc, 0x28, 0xcf, 0x3c, 0x79, 0xf0, 0xb0, 0xfd, 0xe0, 0x9b, 0x28,
-	0xee, 0xec, 0xd2, 0x72, 0xd1, 0xb0, 0xd5, 0x0f, 0x58, 0xd3, 0x9a, 0x7f, 0xda, 0x50, 0xec, 0x06,
-	0x53, 0x21, 0x29, 0xbf, 0xe8, 0x84, 0x98, 0x63, 0x37, 0x12, 0xd2, 0x85, 0x22, 0x67, 0x4c, 0x0e,
-	0x5d, 0x72, 0x56, 0x2e, 0x30, 0x63, 0xb2, 0xdb, 0xee, 0xec, 0x29, 0xa2, 0x1a, 0x24, 0xb1, 0x8d,
-	0x0b, 0x8a, 0xda, 0x25, 0xe8, 0x29, 0x5c, 0x49, 0xc6, 0xef, 0x11, 0x63, 0x52, 0x48, 0x4e, 0xa2,
-	0xe1, 0x98, 0xce, 0xd5, 0x37, 0x2f, 0xbf, 0xed, 0x65, 0xd2, 0x0f, 0x5d, 0x3e, 0xd7, 0x89, 0xba,
-	0x4f, 0xe7, 0xf8, 0xb2, 0x11, 0xe8, 0x24, 0xfc, 0xfb, 0x74, 0x2e, 0xd0, 0x67, 0xb0, 0x4f, 0x57,
-	0x30, 0xa5, 0x38, 0x0c, 0xc8, 0x44, 0x7d, 0x58, 0x86, 0x6e, 0xc0, 0xdc, 0xb1, 0x9e, 0x6d, 0x36,
-	0xbe, 0x4a, 0xd3, 0x52, 0x5f, 0xc5, 0x88, 0xae, 0x02, 0x20, 0x01, 0xce, 0x51, 0x40, 0xdc, 0x71,
-	0xe0, 0x0b, 0xf5, 0xfe, 0x4c, 0x3d, 0x36, 0xd4, 0x78, 0x52, 0xbe, 0x1d, 0x9e, 0x91, 0xad, 0x56,
-	0x67, 0xcd, 0x4d, 0x3d, 0x5d, 0x44, 0x3f, 0x94, 0x7c, 0x8e, 0xdf, 0x3c, 0xca, 0xde, 0xad, 0xcd,
-	0x60, 0xff, 0x2c, 0x22, 0x7a, 0x0d, 0xf2, 0x63, 0x3a, 0x8f, 0x6b, 0x8f, 0xd5, 0x12, 0x7d, 0x0e,
-	0x3b, 0x33, 0x12, 0x4c, 0xa9, 0xa9, 0xfa, 0xfb, 0x59, 0x3e, 0x65, 0x4b, 0xe2, 0x98, 0xf8, 0x49,
-	0xee, 0xd0, 0x6a, 0xfe, 0x6d, 0x41, 0xe1, 0x11, 0x75, 0x39, 0x95, 0xaf, 0xb4, 0xbb, 0x0e, 0x4f,
-	0x75, 0x57, 0x3d, 0xfb, 0xe1, 0xa1, 0x4e, 0xdd, 0x68, 0xae, 0x2b, 0x50, 0xf0, 0xfc, 0x11, 0x15,
-	0xf1, 0xd3, 0xa9, 0x8c, 0x8d, 0x85, 0x9a, 0x60, 0x0b, 0xff, 0x05, 0xd5, 0xd7, 0x28, 0x1f, 0x7f,
-	0xe5, 0x8d, 0x82, 0xff, 0x82, 0x62, 0xbd, 0x87, 0x6a, 0x50, 0xf2, 0x43, 0x49, 0x79, 0x48, 0x02,
-	0x5d, 0xe6, 0x12, 0x5e, 0xd9, 0x9d, 0xfd, 0xe3, 0x93, 0xfa, 0xa5, 0x3f, 0x4e, 0xea, 0x97, 0xfe,
-	0x3d, 0xa9, 0x5b, 0x3f, 0x2c, 0xeb, 0xd6, 0xf1, 0xb2, 0x6e, 0xfd, 0xb6, 0xac, 0x5b, 0x7f, 0x2d,
-	0xeb, 0xd6, 0x51, 0x41, 0xff, 0xf5, 0xf9, 0xf0, 0xbf, 0x00, 0x00, 0x00, 0xff, 0xff, 0x40, 0xcf,
-	0x57, 0x63, 0x6a, 0x0d, 0x00, 0x00,
+	// 1192 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xbc, 0x57, 0x4f, 0x6f, 0x1b, 0x45,
+	0x14, 0xef, 0xda, 0x1b, 0xdb, 0xfb, 0x1c, 0x47, 0x30, 0x54, 0x65, 0x1b, 0x82, 0x1d, 0x5c, 0x81,
+	0x2a, 0x54, 0xb9, 0xa2, 0x14, 0x94, 0x42, 0x2b, 0xb0, 0x9d, 0x08, 0xac, 0x52, 0xa8, 0xa6, 0xa5,
+	0x3d, 0x5a, 0x93, 0xdd, 0xa9, 0x59, 0xbc, 0xde, 0x59, 0xcd, 0x8c, 0x5d, 0xb9, 0x27, 0xc4, 0x07,
+	0xe0, 0x23, 0x20, 0xbe, 0x09, 0xd7, 0x1e, 0x38, 0x70, 0x83, 0x53, 0x44, 0x7d, 0x40, 0xe2, 0xc6,
+	0x47, 0x40, 0xf3, 0x67, 0x9d, 0x8d, 0xbc, 0x0e, 0xa9, 0x54, 0xe5, 0x36, 0xe3, 0xf9, 0xfd, 0x7e,
+	0xef, 0xcd, 0x9b, 0xdf, 0xbc, 0x1d, 0x43, 0x83, 0x1d, 0x7e, 0x4f, 0x03, 0x29, 0x3a, 0x29, 0x67,
+	0x92, 0x21, 0x14, 0xb2, 0x60, 0x4c, 0x79, 0x47, 0x3c, 0x25, 0x7c, 0x32, 0x8e, 0x64, 0x67, 0xf6,
+	0xc1, 0x76, 0x5d, 0xce, 0x53, 0x6a, 0x01, 0xdb, 0x75, 0x91, 0xd2, 0x20, 0x9b, 0x5c, 0x96, 0xd1,
+	0x84, 0x0a, 0x49, 0x26, 0xe9, 0xf5, 0xe5, 0xc8, 0x2e, 0x5d, 0x1c, 0xb1, 0x11, 0xd3, 0xc3, 0xeb,
+	0x6a, 0x64, 0x7e, 0x6d, 0xff, 0xea, 0x80, 0x7b, 0x8f, 0x4a, 0x82, 0x3e, 0x85, 0xea, 0x8c, 0x72,
+	0x11, 0xb1, 0xc4, 0x77, 0x76, 0x9d, 0xab, 0xf5, 0x1b, 0x6f, 0x75, 0x56, 0x23, 0x77, 0x1e, 0x19,
+	0x48, 0xcf, 0x7d, 0x7e, 0xd4, 0xba, 0x80, 0x33, 0x06, 0xba, 0x0d, 0x10, 0x70, 0x4a, 0x24, 0x0d,
+	0x87, 0x44, 0xfa, 0x25, 0xcd, 0x7f, 0xbb, 0x88, 0xff, 0x30, 0x4b, 0x0a, 0x7b, 0x96, 0xd0, 0x95,
+	0x8a, 0x3d, 0x4d, 0xc3, 0x8c, 0x5d, 0x3e, 0x13, 0xdb, 0x12, 0xba, 0xb2, 0xfd, 0x4f, 0x19, 0xdc,
+	0xaf, 0x59, 0x48, 0xd1, 0x25, 0x28, 0x45, 0xa1, 0x4e, 0xde, 0xeb, 0x55, 0x16, 0x47, 0xad, 0xd2,
+	0x60, 0x1f, 0x97, 0xa2, 0x10, 0xdd, 0x00, 0x77, 0x42, 0x25, 0xb1, 0x69, 0xf9, 0x45, 0xc2, 0xaa,
+	0x02, 0x76, 0x4f, 0x1a, 0x8b, 0x3e, 0x06, 0x57, 0x95, 0xd5, 0x26, 0xb3, 0x53, 0xc4, 0x51, 0x31,
+	0x1f, 0xa4, 0x34, 0xc8, 0x78, 0x0a, 0x8f, 0x0e, 0xa0, 0x1e, 0x52, 0x11, 0xf0, 0x28, 0x95, 0xaa,
+	0x92, 0xae, 0xa6, 0x5f, 0x59, 0x47, 0xdf, 0x3f, 0x86, 0xe2, 0x3c, 0x0f, 0xdd, 0x86, 0x8a, 0x90,
+	0x44, 0x4e, 0x85, 0xbf, 0xa1, 0x15, 0x9a, 0x6b, 0x13, 0xd0, 0x28, 0x9b, 0x82, 0xe5, 0xa0, 0x2f,
+	0x61, 0x6b, 0x42, 0x12, 0x32, 0xa2, 0x7c, 0x68, 0x55, 0x2a, 0x5a, 0xe5, 0x9d, 0xc2, 0xad, 0x1b,
+	0xa4, 0x11, 0xc2, 0x8d, 0x49, 0x7e, 0x8a, 0x0e, 0x00, 0x88, 0x94, 0x24, 0xf8, 0x6e, 0x42, 0x13,
+	0xe9, 0x57, 0xb5, 0xca, 0xbb, 0x85, 0xb9, 0x50, 0xf9, 0x94, 0xf1, 0x71, 0x77, 0x09, 0xc6, 0x39,
+	0x22, 0xfa, 0x02, 0xea, 0x01, 0xe5, 0x32, 0x7a, 0x12, 0x05, 0x44, 0x52, 0xbf, 0xa6, 0x75, 0x5a,
+	0x45, 0x3a, 0xfd, 0x63, 0x98, 0xdd, 0x54, 0x9e, 0xd9, 0xfe, 0xa3, 0x04, 0xd5, 0x07, 0x94, 0xcf,
+	0xa2, 0xe0, 0xd5, 0x1e, 0xf7, 0xad, 0x13, 0xc7, 0x5d, 0x98, 0x99, 0x0d, 0xbb, 0x72, 0xe2, 0x7b,
+	0x50, 0xa3, 0x49, 0x98, 0xb2, 0x28, 0x91, 0xf6, 0xb8, 0x0b, 0xdd, 0x72, 0x60, 0x31, 0x78, 0x89,
+	0x46, 0x07, 0xd0, 0x30, 0x2e, 0x1e, 0x9e, 0x38, 0xeb, 0xdd, 0x22, 0xfa, 0xb7, 0x1a, 0x68, 0x0f,
+	0x69, 0x73, 0x9a, 0x9b, 0xa1, 0x7d, 0x68, 0xa4, 0x9c, 0xce, 0x22, 0x36, 0x15, 0x43, 0xbd, 0x89,
+	0xca, 0x99, 0x36, 0x81, 0x37, 0x33, 0x96, 0x9a, 0xb5, 0x7f, 0x2e, 0x41, 0x2d, 0xcb, 0x11, 0xdd,
+	0xb4, 0xe5, 0x70, 0xd6, 0x27, 0x94, 0x61, 0xb5, 0x94, 0xa9, 0xc4, 0x4d, 0xd8, 0x48, 0x19, 0x97,
+	0xc2, 0x2f, 0xed, 0x96, 0xd7, 0x79, 0xf6, 0x3e, 0xe3, 0xb2, 0xcf, 0x92, 0x27, 0xd1, 0x08, 0x1b,
+	0x30, 0x7a, 0x0c, 0xf5, 0x59, 0xc4, 0xe5, 0x94, 0xc4, 0xc3, 0x28, 0x15, 0x7e, 0x59, 0x73, 0xdf,
+	0x3b, 0x2d, 0x64, 0xe7, 0x91, 0xc1, 0x0f, 0xee, 0xf7, 0xb6, 0x16, 0x47, 0x2d, 0x58, 0x4e, 0x05,
+	0x06, 0x2b, 0x35, 0x48, 0xc5, 0xf6, 0x3d, 0xf0, 0x96, 0x2b, 0xe8, 0x1a, 0x40, 0x62, 0x2c, 0x3a,
+	0x5c, 0x9a, 0xa6, 0xb1, 0x38, 0x6a, 0x79, 0xd6, 0xb8, 0x83, 0x7d, 0xec, 0x59, 0xc0, 0x20, 0x44,
+	0x08, 0x5c, 0x12, 0x86, 0x5c, 0x5b, 0xc8, 0xc3, 0x7a, 0xdc, 0xfe, 0x6d, 0x03, 0xdc, 0x87, 0x44,
+	0x8c, 0xcf, 0xbb, 0xcd, 0xa8, 0x98, 0x2b, 0xa6, 0xbb, 0x06, 0x20, 0xcc, 0x51, 0xaa, 0xed, 0xb8,
+	0xc7, 0xdb, 0xb1, 0x07, 0xac, 0xb6, 0x63, 0x01, 0x66, 0x3b, 0x22, 0x66, 0x52, 0xfb, 0xcb, 0xc5,
+	0x7a, 0x8c, 0xae, 0x40, 0x35, 0x61, 0xa1, 0xa6, 0x57, 0x34, 0x1d, 0x16, 0x47, 0xad, 0x8a, 0x6a,
+	0x29, 0x83, 0x7d, 0x5c, 0x51, 0x4b, 0x83, 0x50, 0xdd, 0x5b, 0x92, 0x24, 0x4c, 0x12, 0xd5, 0x94,
+	0x84, 0xbd, 0xff, 0x85, 0xc6, 0xea, 0x1e, 0xc3, 0xb2, 0x7b, 0x9b, 0x63, 0xa2, 0x47, 0xf0, 0x46,
+	0x96, 0x6f, 0x5e, 0xb0, 0xf6, 0x32, 0x82, 0xc8, 0x2a, 0xe4, 0x56, 0x72, 0x7d, 0xd2, 0x5b, 0xdf,
+	0x27, 0x75, 0x05, 0x8b, 0xfa, 0x64, 0x0f, 0x1a, 0x21, 0x15, 0x11, 0xa7, 0xa1, 0xbe, 0x81, 0xd4,
+	0x87, 0x5d, 0xe7, 0xea, 0xd6, 0x9a, 0x4f, 0x8f, 0x15, 0xa1, 0x78, 0xd3, 0x72, 0xf4, 0x0c, 0x75,
+	0xa1, 0x66, 0x7d, 0x23, 0xfc, 0xba, 0xf6, 0xee, 0x19, 0xfb, 0xe3, 0x92, 0x76, 0xa2, 0x83, 0x6c,
+	0xbe, 0x54, 0x07, 0xb9, 0x05, 0x10, 0xb3, 0xd1, 0x30, 0xe4, 0xd1, 0x8c, 0x72, 0xbf, 0xa1, 0xb9,
+	0xdb, 0x45, 0xdc, 0x7d, 0x8d, 0xc0, 0x5e, 0xcc, 0x46, 0x66, 0xd8, 0xfe, 0xd1, 0x81, 0xd7, 0x57,
+	0x92, 0x42, 0x1f, 0x41, 0xd5, 0xa6, 0x75, 0xda, 0x23, 0xc0, 0xf2, 0x70, 0x86, 0x45, 0x3b, 0xe0,
+	0xa9, 0x3b, 0x42, 0x85, 0xa0, 0xe6, 0xf6, 0x7b, 0xf8, 0xf8, 0x07, 0xe4, 0x43, 0x95, 0xc4, 0x11,
+	0x51, 0x6b, 0x65, 0xbd, 0x96, 0x4d, 0xdb, 0x3f, 0x95, 0xa0, 0x6a, 0xc5, 0xce, 0xbb, 0x9d, 0xdb,
+	0xb0, 0x2b, 0x37, 0xeb, 0x0e, 0x6c, 0x9a, 0x72, 0x5a, 0x4b, 0xb8, 0xff, 0x5b, 0xd4, 0xba, 0xc1,
+	0x1b, 0x3b, 0xdc, 0x01, 0x37, 0x4a, 0xc9, 0xc4, 0xb6, 0xf2, 0xc2, 0xc8, 0x83, 0xfb, 0xdd, 0x7b,
+	0xdf, 0xa4, 0xc6, 0xd9, 0xb5, 0xc5, 0x51, 0xcb, 0x55, 0x3f, 0x60, 0x4d, 0x6b, 0xff, 0xb2, 0x01,
+	0xd5, 0x7e, 0x3c, 0x15, 0x92, 0xf2, 0xf3, 0x2e, 0x88, 0x0d, 0xbb, 0x52, 0x90, 0x3e, 0x54, 0x39,
+	0x63, 0x72, 0x18, 0x90, 0xd3, 0x6a, 0x81, 0x19, 0x93, 0xfd, 0x6e, 0x6f, 0x4b, 0x11, 0x55, 0x23,
+	0x31, 0x73, 0x5c, 0x51, 0xd4, 0x3e, 0x41, 0x8f, 0xe1, 0x52, 0xd6, 0x7e, 0x0f, 0x19, 0x93, 0x42,
+	0x72, 0x92, 0x0e, 0xc7, 0x74, 0xae, 0xbe, 0x79, 0xe5, 0x75, 0x2f, 0x93, 0x83, 0x24, 0xe0, 0x73,
+	0x5d, 0xa8, 0xbb, 0x74, 0x8e, 0x2f, 0x5a, 0x81, 0x5e, 0xc6, 0xbf, 0x4b, 0xe7, 0x02, 0x7d, 0x06,
+	0x3b, 0x74, 0x09, 0x53, 0x8a, 0xc3, 0x98, 0x4c, 0xd4, 0x87, 0x65, 0x18, 0xc4, 0x2c, 0x18, 0xeb,
+	0xde, 0xe6, 0xe2, 0xcb, 0x34, 0x2f, 0xf5, 0x95, 0x41, 0xf4, 0x15, 0x00, 0x09, 0xf0, 0x0f, 0x63,
+	0x12, 0x8c, 0xe3, 0x48, 0xa8, 0xf7, 0x67, 0xee, 0xb1, 0xa1, 0xda, 0x93, 0xca, 0x6d, 0xef, 0x94,
+	0x6a, 0x75, 0x7a, 0xc7, 0xdc, 0xdc, 0xd3, 0x45, 0x1c, 0x24, 0x92, 0xcf, 0xf1, 0x9b, 0x87, 0xc5,
+	0xab, 0xa8, 0x07, 0xf5, 0x69, 0xa2, 0xc2, 0x9b, 0x1a, 0x78, 0x67, 0xad, 0x01, 0x18, 0x96, 0xda,
+	0xf9, 0xf6, 0x0c, 0x76, 0x4e, 0x0b, 0x8e, 0x5e, 0x83, 0xf2, 0x98, 0xce, 0x8d, 0x7f, 0xb0, 0x1a,
+	0xa2, 0xcf, 0x61, 0x63, 0x46, 0xe2, 0x29, 0xb5, 0xce, 0x79, 0xbf, 0x28, 0x5e, 0xb1, 0x24, 0x36,
+	0xc4, 0x4f, 0x4a, 0x7b, 0x4e, 0xfb, 0x6f, 0x07, 0x2a, 0x0f, 0x68, 0xc0, 0xa9, 0x7c, 0xa5, 0x0e,
+	0xdd, 0x3b, 0xe1, 0xd0, 0x66, 0xf1, 0xe3, 0x45, 0x45, 0x5d, 0x31, 0xe8, 0x25, 0xa8, 0x84, 0xd1,
+	0x88, 0x0a, 0xf3, 0xfc, 0xf2, 0xb0, 0x9d, 0xa1, 0x36, 0xb8, 0x22, 0x7a, 0x46, 0xf5, 0x55, 0x2c,
+	0x9b, 0x97, 0x82, 0x55, 0x88, 0x9e, 0x51, 0xac, 0xd7, 0xd0, 0x36, 0xd4, 0xa2, 0x44, 0x52, 0x9e,
+	0x90, 0x58, 0x5b, 0xa5, 0x86, 0x97, 0xf3, 0xde, 0xce, 0xf3, 0x17, 0xcd, 0x0b, 0x7f, 0xbe, 0x68,
+	0x5e, 0xf8, 0xf7, 0x45, 0xd3, 0xf9, 0x61, 0xd1, 0x74, 0x9e, 0x2f, 0x9a, 0xce, 0xef, 0x8b, 0xa6,
+	0xf3, 0xd7, 0xa2, 0xe9, 0x1c, 0x56, 0xf4, 0xdf, 0xa7, 0x0f, 0xff, 0x0b, 0x00, 0x00, 0xff, 0xff,
+	0x49, 0x8f, 0xcd, 0x22, 0xae, 0x0d, 0x00, 0x00,
 }

vendor/github.com/docker/swarmkit/api/objects.proto

@@ -230,6 +230,13 @@ message Cluster {
 	// be honored. It's a mapping from CN -> BlacklistedCertificate.
 	// swarm. Their certificates should effectively be blacklisted.
 	map<string, BlacklistedCertificate> blacklisted_certificates = 8;
+
+	// UnlockKeys defines the keys that lock node data at rest.  For example,
+	// this would contain the key encrypting key (KEK) that will encrypt the
+	// manager TLS keys at rest and the raft encryption keys at rest.
+	// If the key is empty, the node will be unlocked (will not require a key
+	// to start up from a shut down state).
+	repeated EncryptionKey unlock_keys = 9;
 }
 
 // Secret represents a secret that should be passed to a container or a node,

vendor/github.com/docker/swarmkit/api/specs.pb.go

@@ -596,6 +596,8 @@ type ClusterSpec struct {
 	CAConfig CAConfig `protobuf:"bytes,6,opt,name=ca_config,json=caConfig" json:"ca_config"`
 	// TaskDefaults specifies the default values to use for task creation.
 	TaskDefaults TaskDefaults `protobuf:"bytes,7,opt,name=task_defaults,json=taskDefaults" json:"task_defaults"`
+	// EncryptionConfig defines the cluster's encryption settings.
+	EncryptionConfig EncryptionConfig `protobuf:"bytes,8,opt,name=encryption_config,json=encryptionConfig" json:"encryption_config"`
 }
 
 func (m *ClusterSpec) Reset()                    { *m = ClusterSpec{} }
@@ -908,6 +910,7 @@ func (m *ClusterSpec) Copy() *ClusterSpec {
 		Dispatcher:       *m.Dispatcher.Copy(),
 		CAConfig:         *m.CAConfig.Copy(),
 		TaskDefaults:     *m.TaskDefaults.Copy(),
+		EncryptionConfig: *m.EncryptionConfig.Copy(),
 	}
 
 	return o
@@ -1159,7 +1162,7 @@ func (this *ClusterSpec) GoString() string {
 	if this == nil {
 		return "nil"
 	}
-	s := make([]string, 0, 11)
+	s := make([]string, 0, 12)
 	s = append(s, "&api.ClusterSpec{")
 	s = append(s, "Annotations: "+strings.Replace(this.Annotations.GoString(), `&`, ``, 1)+",\n")
 	s = append(s, "AcceptancePolicy: "+strings.Replace(this.AcceptancePolicy.GoString(), `&`, ``, 1)+",\n")
@@ -1168,6 +1171,7 @@ func (this *ClusterSpec) GoString() string {
 	s = append(s, "Dispatcher: "+strings.Replace(this.Dispatcher.GoString(), `&`, ``, 1)+",\n")
 	s = append(s, "CAConfig: "+strings.Replace(this.CAConfig.GoString(), `&`, ``, 1)+",\n")
 	s = append(s, "TaskDefaults: "+strings.Replace(this.TaskDefaults.GoString(), `&`, ``, 1)+",\n")
+	s = append(s, "EncryptionConfig: "+strings.Replace(this.EncryptionConfig.GoString(), `&`, ``, 1)+",\n")
 	s = append(s, "}")
 	return strings.Join(s, "")
 }
@@ -2008,6 +2012,14 @@ func (m *ClusterSpec) MarshalTo(data []byte) (int, error) {
 		return 0, err
 	}
 	i += n29
+	data[i] = 0x42
+	i++
+	i = encodeVarintSpecs(data, i, uint64(m.EncryptionConfig.Size()))
+	n30, err := m.EncryptionConfig.MarshalTo(data[i:])
+	if err != nil {
+		return 0, err
+	}
+	i += n30
 	return i, nil
 }
 
@@ -2029,11 +2041,11 @@ func (m *SecretSpec) MarshalTo(data []byte) (int, error) {
 	data[i] = 0xa
 	i++
 	i = encodeVarintSpecs(data, i, uint64(m.Annotations.Size()))
-	n30, err := m.Annotations.MarshalTo(data[i:])
+	n31, err := m.Annotations.MarshalTo(data[i:])
 	if err != nil {
 		return 0, err
 	}
-	i += n30
+	i += n31
 	if len(m.Data) > 0 {
 		data[i] = 0x12
 		i++
@@ -2392,6 +2404,8 @@ func (m *ClusterSpec) Size() (n int) {
 	n += 1 + l + sovSpecs(uint64(l))
 	l = m.TaskDefaults.Size()
 	n += 1 + l + sovSpecs(uint64(l))
+	l = m.EncryptionConfig.Size()
+	n += 1 + l + sovSpecs(uint64(l))
 	return n
 }
 
@@ -2629,6 +2643,7 @@ func (this *ClusterSpec) String() string {
 		`Dispatcher:` + strings.Replace(strings.Replace(this.Dispatcher.String(), "DispatcherConfig", "DispatcherConfig", 1), `&`, ``, 1) + `,`,
 		`CAConfig:` + strings.Replace(strings.Replace(this.CAConfig.String(), "CAConfig", "CAConfig", 1), `&`, ``, 1) + `,`,
 		`TaskDefaults:` + strings.Replace(strings.Replace(this.TaskDefaults.String(), "TaskDefaults", "TaskDefaults", 1), `&`, ``, 1) + `,`,
+		`EncryptionConfig:` + strings.Replace(strings.Replace(this.EncryptionConfig.String(), "EncryptionConfig", "EncryptionConfig", 1), `&`, ``, 1) + `,`,
 		`}`,
 	}, "")
 	return s
@@ -4956,6 +4971,36 @@ func (m *ClusterSpec) Unmarshal(data []byte) error {
 				return err
 			}
 			iNdEx = postIndex
+		case 8:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field EncryptionConfig", wireType)
+			}
+			var msglen int
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowSpecs
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := data[iNdEx]
+				iNdEx++
+				msglen |= (int(b) & 0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			if msglen < 0 {
+				return ErrInvalidLengthSpecs
+			}
+			postIndex := iNdEx + msglen
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			if err := m.EncryptionConfig.Unmarshal(data[iNdEx:postIndex]); err != nil {
+				return err
+			}
+			iNdEx = postIndex
 		default:
 			iNdEx = preIndex
 			skippy, err := skipSpecs(data[iNdEx:])
@@ -5196,105 +5241,107 @@ var (
 func init() { proto.RegisterFile("specs.proto", fileDescriptorSpecs) }
 
 var fileDescriptorSpecs = []byte{
-	// 1597 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xac, 0x57, 0xcd, 0x6e, 0xe3, 0xc8,
-	0x11, 0x16, 0x2d, 0x59, 0x3f, 0x45, 0x69, 0x46, 0xd3, 0xd8, 0x1f, 0x8e, 0x76, 0x23, 0x69, 0xb4,
-	0x93, 0x8d, 0x37, 0x8b, 0x78, 0x12, 0x25, 0xd8, 0xcc, 0x66, 0xb2, 0x48, 0xf4, 0x17, 0x8f, 0xe2,
-	0xd8, 0x2b, 0xb4, 0xbd, 0x03, 0xcc, 0x49, 0x68, 0x93, 0x6d, 0x89, 0x30, 0xc5, 0x66, 0x9a, 0x4d,
-	0x2d, 0x7c, 0xcb, 0x71, 0x31, 0x87, 0xbc, 0x81, 0x4f, 0x01, 0xf2, 0x06, 0xb9, 0xe4, 0x09, 0xe6,
-	0x98, 0x63, 0x4e, 0x46, 0xac, 0x27, 0x08, 0x90, 0x17, 0x08, 0xba, 0xd9, 0x94, 0xa8, 0x2c, 0xbd,
-	0x5e, 0x20, 0xbe, 0x75, 0x17, 0xbf, 0xaf, 0xd8, 0x5d, 0xf5, 0xb1, 0xaa, 0x08, 0x66, 0x18, 0x50,
-	0x3b, 0xdc, 0x0f, 0x38, 0x13, 0x0c, 0x21, 0x87, 0xd9, 0x17, 0x94, 0xef, 0x87, 0x5f, 0x13, 0xbe,
-	0xb8, 0x70, 0xc5, 0xfe, 0xf2, 0x67, 0x0d, 0x53, 0x5c, 0x06, 0x54, 0x03, 0x1a, 0xef, 0xcc, 0xd8,
-	0x8c, 0xa9, 0xe5, 0x33, 0xb9, 0xd2, 0xd6, 0xf7, 0x9d, 0x88, 0x13, 0xe1, 0x32, 0xff, 0x59, 0xb2,
-	0x88, 0x1f, 0x74, 0xfe, 0x5c, 0x80, 0xf2, 0x31, 0x73, 0xe8, 0x49, 0x40, 0x6d, 0x74, 0x00, 0x26,
-	0xf1, 0x7d, 0x26, 0x14, 0x20, 0xb4, 0x8c, 0xb6, 0xb1, 0x67, 0x76, 0x5b, 0xfb, 0xdf, 0x7e, 0xe5,
-	0x7e, 0x6f, 0x03, 0xeb, 0x17, 0xde, 0x5e, 0xb7, 0x72, 0x38, 0xcd, 0x44, 0x3f, 0x85, 0x02, 0x67,
-	0x1e, 0xb5, 0x76, 0xda, 0xc6, 0xde, 0x83, 0xee, 0x87, 0x59, 0x1e, 0xe4, 0x4b, 0x31, 0xf3, 0x28,
-	0x56, 0x48, 0x74, 0x00, 0xb0, 0xa0, 0x8b, 0x33, 0xca, 0xc3, 0xb9, 0x1b, 0x58, 0x79, 0xc5, 0xfb,
-	0xd1, 0x6d, 0x3c, 0x79, 0xd8, 0xfd, 0xa3, 0x35, 0x1c, 0xa7, 0xa8, 0xe8, 0x08, 0xaa, 0x64, 0x49,
-	0x5c, 0x8f, 0x9c, 0xb9, 0x9e, 0x2b, 0x2e, 0xad, 0x82, 0x72, 0xf5, 0xc9, 0x77, 0xba, 0xea, 0xa5,
-	0x08, 0x78, 0x8b, 0xde, 0x71, 0x00, 0x36, 0x2f, 0x42, 0x1f, 0x43, 0x69, 0x32, 0x3a, 0x1e, 0x8e,
-	0x8f, 0x0f, 0xea, 0xb9, 0xc6, 0xe3, 0x37, 0x57, 0xed, 0x77, 0xa5, 0x8f, 0x0d, 0x60, 0x42, 0x7d,
-	0xc7, 0xf5, 0x67, 0x68, 0x0f, 0xca, 0xbd, 0xc1, 0x60, 0x34, 0x39, 0x1d, 0x0d, 0xeb, 0x46, 0xa3,
-	0xf1, 0xe6, 0xaa, 0xfd, 0xde, 0x36, 0xb0, 0x67, 0xdb, 0x34, 0x10, 0xd4, 0x69, 0x14, 0xbe, 0xf9,
-	0x4b, 0x33, 0xd7, 0xf9, 0xc6, 0x80, 0x6a, 0xfa, 0x10, 0xe8, 0x63, 0x28, 0xf6, 0x06, 0xa7, 0xe3,
-	0x57, 0xa3, 0x7a, 0x6e, 0x43, 0x4f, 0x23, 0x7a, 0xb6, 0x70, 0x97, 0x14, 0x3d, 0x85, 0xdd, 0x49,
-	0xef, 0xab, 0x93, 0x51, 0xdd, 0xd8, 0x1c, 0x27, 0x0d, 0x9b, 0x90, 0x28, 0x54, 0xa8, 0x21, 0xee,
-	0x8d, 0x8f, 0xeb, 0x3b, 0xd9, 0xa8, 0x21, 0x27, 0xae, 0xaf, 0x8f, 0x72, 0x93, 0x07, 0xf3, 0x84,
-	0xf2, 0xa5, 0x6b, 0xdf, 0xb3, 0x26, 0x3e, 0x83, 0x82, 0x20, 0xe1, 0x85, 0xd2, 0x84, 0x99, 0xad,
-	0x89, 0x53, 0x12, 0x5e, 0xc8, 0x97, 0x6a, 0xba, 0xc2, 0x4b, 0x65, 0x70, 0x1a, 0x78, 0xae, 0x4d,
-	0x04, 0x75, 0x94, 0x32, 0xcc, 0xee, 0x0f, 0xb3, 0xd8, 0x78, 0x8d, 0xd2, 0xe7, 0x7f, 0x99, 0xc3,
-	0x29, 0x2a, 0x7a, 0x01, 0xc5, 0x99, 0xc7, 0xce, 0x88, 0xa7, 0x34, 0x61, 0x76, 0x9f, 0x64, 0x39,
-	0x39, 0x50, 0x88, 0x8d, 0x03, 0x4d, 0x41, 0xcf, 0xa1, 0x18, 0x05, 0x0e, 0x11, 0xd4, 0x2a, 0x2a,
-	0x72, 0x3b, 0x8b, 0xfc, 0x95, 0x42, 0x0c, 0x98, 0x7f, 0xee, 0xce, 0xb0, 0xc6, 0xa3, 0x43, 0x28,
-	0xfb, 0x54, 0x7c, 0xcd, 0xf8, 0x45, 0x68, 0x95, 0xda, 0xf9, 0x3d, 0xb3, 0xfb, 0x69, 0xa6, 0x18,
-	0x63, 0x4c, 0x4f, 0x08, 0x62, 0xcf, 0x17, 0xd4, 0x17, 0xb1, 0x9b, 0xfe, 0x8e, 0x65, 0xe0, 0xb5,
-	0x03, 0xf4, 0x6b, 0x28, 0x53, 0xdf, 0x09, 0x98, 0xeb, 0x0b, 0xab, 0x7c, 0xfb, 0x41, 0x46, 0x1a,
-	0x23, 0x83, 0x89, 0xd7, 0x8c, 0x7e, 0x11, 0x0a, 0x0b, 0xe6, 0xd0, 0xce, 0x33, 0x78, 0xf4, 0xad,
-	0x60, 0xa1, 0x06, 0x94, 0x75, 0xb0, 0xe2, 0x2c, 0x17, 0xf0, 0x7a, 0xdf, 0x79, 0x08, 0xb5, 0xad,
-	0xc0, 0xa8, 0xb2, 0x91, 0x64, 0x0b, 0xf5, 0xa0, 0x62, 0x33, 0x5f, 0x10, 0xd7, 0xa7, 0x5c, 0x0b,
-	0x24, 0x33, 0xb6, 0x83, 0x04, 0x24, 0x59, 0x2f, 0x73, 0x78, 0xc3, 0x42, 0xbf, 0x83, 0x0a, 0xa7,
-	0x21, 0x8b, 0xb8, 0x4d, 0x43, 0xad, 0x90, 0xbd, 0xec, 0x1c, 0xc7, 0x20, 0x4c, 0xff, 0x18, 0xb9,
-	0x9c, 0xca, 0x38, 0x85, 0x78, 0x43, 0x45, 0x2f, 0xa0, 0xc4, 0x69, 0x28, 0x08, 0x17, 0xdf, 0x95,
-	0x64, 0x1c, 0x43, 0x26, 0xcc, 0x73, 0xed, 0x4b, 0x9c, 0x30, 0xd0, 0x0b, 0xa8, 0x04, 0x1e, 0xb1,
-	0x95, 0x57, 0x6b, 0x57, 0xd1, 0x7f, 0x90, 0x45, 0x9f, 0x24, 0x20, 0xbc, 0xc1, 0xa3, 0xcf, 0x01,
-	0x3c, 0x36, 0x9b, 0x3a, 0xdc, 0x5d, 0x52, 0xae, 0x45, 0xd2, 0xc8, 0x62, 0x0f, 0x15, 0x02, 0x57,
-	0x3c, 0x36, 0x8b, 0x97, 0xe8, 0xe0, 0xff, 0x52, 0x48, 0x4a, 0x1d, 0x87, 0x00, 0x64, 0xfd, 0x54,
-	0xeb, 0xe3, 0x93, 0xef, 0xe5, 0x4a, 0x67, 0x24, 0x45, 0x47, 0x4f, 0xa0, 0x7a, 0xce, 0xb8, 0x4d,
-	0xa7, 0x5a, 0xf7, 0x15, 0xa5, 0x09, 0x53, 0xd9, 0x62, 0xa1, 0xf7, 0x2b, 0x50, 0xe2, 0x91, 0x2f,
-	0xdc, 0x05, 0xed, 0x1c, 0xc2, 0xbb, 0x99, 0x4e, 0x51, 0x17, 0xaa, 0xeb, 0x34, 0x4f, 0x5d, 0x47,
-	0xe9, 0xa3, 0xd2, 0x7f, 0xb8, 0xba, 0x6e, 0x99, 0x6b, 0x3d, 0x8c, 0x87, 0xd8, 0x5c, 0x83, 0xc6,
-	0x4e, 0xe7, 0xef, 0x25, 0xa8, 0x6d, 0x89, 0x05, 0xbd, 0x03, 0xbb, 0xee, 0x82, 0xcc, 0x68, 0x4c,
-	0xc7, 0xf1, 0x06, 0x8d, 0xa0, 0xe8, 0x91, 0x33, 0xea, 0x49, 0xc9, 0xc8, 0xb0, 0xfd, 0xe4, 0x4e,
-	0xd5, 0xed, 0xff, 0x41, 0xe1, 0x47, 0xbe, 0xe0, 0x97, 0x58, 0x93, 0x91, 0x05, 0x25, 0x9b, 0x2d,
-	0x16, 0xc4, 0x97, 0xe5, 0x25, 0xbf, 0x57, 0xc1, 0xc9, 0x16, 0x21, 0x28, 0x10, 0x3e, 0x0b, 0xad,
-	0x82, 0x32, 0xab, 0x35, 0xaa, 0x43, 0x9e, 0xfa, 0x4b, 0x6b, 0x57, 0x99, 0xe4, 0x52, 0x5a, 0x1c,
-	0x37, 0xce, 0x79, 0x05, 0xcb, 0xa5, 0xe4, 0x45, 0x21, 0xe5, 0x56, 0x49, 0x99, 0xd4, 0x1a, 0xfd,
-	0x12, 0x8a, 0x0b, 0x16, 0xf9, 0x22, 0xb4, 0xca, 0xea, 0xb0, 0x8f, 0xb3, 0x0e, 0x7b, 0x24, 0x11,
-	0xba, 0xfc, 0x69, 0x38, 0x7a, 0x09, 0x8f, 0x42, 0xc1, 0x82, 0xe9, 0x8c, 0x13, 0x9b, 0x4e, 0x03,
-	0xca, 0x5d, 0xe6, 0xa8, 0x6c, 0xdc, 0x52, 0x45, 0x87, 0xba, 0xc3, 0xe3, 0x87, 0x92, 0x76, 0x20,
-	0x59, 0x13, 0x45, 0x42, 0x13, 0xa8, 0x06, 0x91, 0xe7, 0x4d, 0x59, 0x10, 0x17, 0x73, 0x50, 0x4e,
-	0xbe, 0x47, 0xd4, 0x26, 0x91, 0xe7, 0x7d, 0x19, 0x93, 0xb0, 0x19, 0x6c, 0x36, 0xe8, 0x3d, 0x28,
-	0xce, 0x38, 0x8b, 0x82, 0xd0, 0x32, 0x55, 0x3c, 0xf4, 0x0e, 0x7d, 0x01, 0xa5, 0x90, 0xda, 0x9c,
-	0x8a, 0xd0, 0xaa, 0xaa, 0xdb, 0x7e, 0x94, 0xf5, 0x92, 0x13, 0x05, 0xc1, 0xf4, 0x9c, 0x72, 0xea,
-	0xdb, 0x14, 0x27, 0x1c, 0xf4, 0x18, 0xf2, 0x42, 0x5c, 0x5a, 0xb5, 0xb6, 0xb1, 0x57, 0xee, 0x97,
-	0x56, 0xd7, 0xad, 0xfc, 0xe9, 0xe9, 0x6b, 0x2c, 0x6d, 0xb2, 0x4c, 0xcd, 0x59, 0x28, 0x7c, 0xb2,
-	0xa0, 0xd6, 0x03, 0x15, 0xde, 0xf5, 0x1e, 0xbd, 0x06, 0x70, 0xfc, 0x70, 0x6a, 0xab, 0xef, 0xc2,
-	0x7a, 0xa8, 0x6e, 0xf7, 0xe9, 0xdd, 0xb7, 0x1b, 0x1e, 0x9f, 0xe8, 0x62, 0x5b, 0x5b, 0x5d, 0xb7,
-	0x2a, 0xeb, 0x2d, 0xae, 0x38, 0x7e, 0x18, 0x2f, 0x51, 0x1f, 0xcc, 0x39, 0x25, 0x9e, 0x98, 0xdb,
-	0x73, 0x6a, 0x5f, 0x58, 0xf5, 0xdb, 0x6b, 0xef, 0x4b, 0x05, 0xd3, 0x1e, 0xd2, 0x24, 0x29, 0x62,
-	0x79, 0xd4, 0xd0, 0x7a, 0xa4, 0x62, 0x15, 0x6f, 0x1a, 0x9f, 0x83, 0x99, 0x12, 0xa5, 0x14, 0xd3,
-	0x05, 0xbd, 0xd4, 0x3a, 0x97, 0x4b, 0x49, 0x5b, 0x12, 0x2f, 0x8a, 0xa7, 0xa9, 0x0a, 0x8e, 0x37,
-	0xbf, 0xda, 0x79, 0x6e, 0x34, 0xba, 0x60, 0xa6, 0x32, 0x83, 0x3e, 0x82, 0x1a, 0xa7, 0x33, 0x37,
-	0x14, 0xfc, 0x72, 0x4a, 0x22, 0x31, 0xb7, 0x7e, 0xab, 0x08, 0xd5, 0xc4, 0xd8, 0x8b, 0xc4, 0xbc,
-	0x31, 0x85, 0xcd, 0x05, 0x51, 0x1b, 0x4c, 0x19, 0xb8, 0x90, 0xf2, 0x25, 0xe5, 0xb2, 0xec, 0xcb,
-	0x73, 0xa5, 0x4d, 0x32, 0xc1, 0x21, 0x25, 0xdc, 0x9e, 0xab, 0x4f, 0xac, 0x82, 0xf5, 0x4e, 0x7e,
-	0x33, 0x89, 0x8a, 0xf4, 0x37, 0xa3, 0xb7, 0x9d, 0xff, 0x18, 0x50, 0x4d, 0xf7, 0x1f, 0x34, 0x88,
-	0xbb, 0x8e, 0xba, 0xd2, 0x83, 0xee, 0xb3, 0xbb, 0xfa, 0x95, 0xaa, 0xf1, 0x5e, 0x24, 0x9d, 0x1d,
-	0xc9, 0x19, 0x51, 0x91, 0xd1, 0x2f, 0x60, 0x37, 0x60, 0x5c, 0x24, 0x5f, 0x7a, 0x33, 0xb3, 0x2e,
-	0x33, 0x9e, 0xd4, 0xc4, 0x18, 0xdc, 0x99, 0xc3, 0x83, 0x6d, 0x6f, 0xe8, 0x29, 0xe4, 0x5f, 0x8d,
-	0x27, 0xf5, 0x5c, 0xe3, 0x83, 0x37, 0x57, 0xed, 0xf7, 0xb7, 0x1f, 0xbe, 0x72, 0xb9, 0x88, 0x88,
-	0x37, 0x9e, 0xa0, 0x1f, 0xc3, 0xee, 0xf0, 0xf8, 0x04, 0xe3, 0xba, 0xd1, 0x68, 0xbd, 0xb9, 0x6a,
-	0x7f, 0xb0, 0x8d, 0x93, 0x8f, 0x58, 0xe4, 0x3b, 0x98, 0x9d, 0xad, 0xc7, 0xa6, 0xbf, 0xed, 0x80,
-	0xa9, 0x0b, 0xe0, 0xfd, 0x8e, 0x4d, 0xbf, 0x81, 0x5a, 0xdc, 0x53, 0x12, 0x59, 0xef, 0xdc, 0xd9,
-	0x5a, 0xaa, 0x31, 0x41, 0xe7, 0xf8, 0x09, 0x54, 0xdd, 0x60, 0xf9, 0xd9, 0x94, 0xfa, 0xe4, 0xcc,
-	0xd3, 0x13, 0x54, 0x19, 0x9b, 0xd2, 0x36, 0x8a, 0x4d, 0xf2, 0x9b, 0x72, 0x7d, 0x41, 0xb9, 0xaf,
-	0x67, 0xa3, 0x32, 0x5e, 0xef, 0xd1, 0x17, 0x50, 0x70, 0x03, 0xb2, 0xd0, 0xfd, 0x30, 0xf3, 0x06,
-	0xe3, 0x49, 0xef, 0x48, 0x6b, 0xb0, 0x5f, 0x5e, 0x5d, 0xb7, 0x0a, 0xd2, 0x80, 0x15, 0x0d, 0x35,
-	0x93, 0x96, 0x24, 0xdf, 0xa4, 0x4a, 0x64, 0x19, 0xa7, 0x2c, 0x9d, 0xbf, 0x16, 0xc0, 0x1c, 0x78,
-	0x51, 0x28, 0x74, 0xa1, 0xbf, 0xb7, 0xb8, 0xbd, 0x86, 0x47, 0x44, 0x0d, 0xd9, 0xc4, 0x97, 0x55,
-	0x53, 0xb5, 0x7a, 0x1d, 0xbb, 0xa7, 0x99, 0xee, 0xd6, 0xe0, 0x78, 0x2c, 0xe8, 0x17, 0xa5, 0x4f,
-	0xcb, 0xc0, 0x75, 0xf2, 0x3f, 0x4f, 0xd0, 0x09, 0xd4, 0x18, 0xb7, 0xe7, 0x34, 0x14, 0x71, 0xa1,
-	0xd5, 0x43, 0x69, 0xe6, 0xef, 0xca, 0x97, 0x69, 0xa0, 0xae, 0x32, 0xf1, 0x69, 0xb7, 0x7d, 0xa0,
-	0xe7, 0x50, 0xe0, 0xe4, 0x3c, 0x19, 0x5b, 0x32, 0xf5, 0x8d, 0xc9, 0xb9, 0xd8, 0x72, 0xa1, 0x18,
-	0xe8, 0xf7, 0x00, 0x8e, 0x1b, 0x06, 0x44, 0xd8, 0x73, 0xca, 0x75, 0x9e, 0x32, 0xaf, 0x38, 0x5c,
-	0xa3, 0xb6, 0xbc, 0xa4, 0xd8, 0xe8, 0x10, 0x2a, 0x36, 0x49, 0x94, 0x56, 0xbc, 0xbd, 0xc7, 0x0c,
-	0x7a, 0xda, 0x45, 0x5d, 0xba, 0x58, 0x5d, 0xb7, 0xca, 0x89, 0x05, 0x97, 0x6d, 0xa2, 0x95, 0x77,
-	0x08, 0x35, 0x39, 0xc1, 0x4f, 0x1d, 0x7a, 0x4e, 0x22, 0x4f, 0x84, 0xaa, 0x1d, 0xde, 0x52, 0x35,
-	0xe5, 0x30, 0x39, 0xd4, 0x38, 0x7d, 0xae, 0xaa, 0x48, 0xd9, 0x3a, 0x2e, 0x40, 0xdc, 0x2e, 0xee,
-	0x57, 0x26, 0x08, 0x0a, 0x0e, 0x11, 0x44, 0x29, 0xa3, 0x8a, 0xd5, 0xba, 0xff, 0xe1, 0xdb, 0x9b,
-	0x66, 0xee, 0x9f, 0x37, 0xcd, 0xdc, 0xbf, 0x6f, 0x9a, 0xc6, 0x9f, 0x56, 0x4d, 0xe3, 0xed, 0xaa,
-	0x69, 0xfc, 0x63, 0xd5, 0x34, 0xfe, 0xb5, 0x6a, 0x1a, 0x67, 0x45, 0xf5, 0xe3, 0xfc, 0xf3, 0xff,
-	0x06, 0x00, 0x00, 0xff, 0xff, 0x04, 0xd4, 0x09, 0xa4, 0x97, 0x0f, 0x00, 0x00,
+	// 1620 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xac, 0x57, 0xcd, 0x72, 0xdb, 0xc8,
+	0x11, 0x26, 0x24, 0x8a, 0x3f, 0x0d, 0xca, 0xa6, 0xa6, 0xf6, 0x07, 0xe6, 0x6e, 0x48, 0x9a, 0xeb,
+	0x6c, 0xb4, 0xd9, 0x8a, 0x9c, 0x30, 0xa9, 0x8d, 0x37, 0xce, 0x56, 0xc2, 0xbf, 0xc8, 0x8c, 0x22,
+	0x2d, 0x6b, 0xa4, 0x75, 0xca, 0x27, 0xd6, 0x08, 0x18, 0x91, 0x28, 0x81, 0x18, 0x64, 0x30, 0xe0,
+	0x16, 0x6f, 0x39, 0x6e, 0xf9, 0x90, 0x37, 0xf0, 0x29, 0xcf, 0x90, 0x4b, 0x9e, 0xc0, 0xc7, 0x1c,
+	0x73, 0x52, 0xc5, 0x7c, 0x82, 0x54, 0xe5, 0x01, 0x92, 0x9a, 0xc1, 0x00, 0x04, 0x77, 0xa1, 0xb5,
+	0xab, 0xe2, 0xdb, 0x4c, 0xe3, 0xfb, 0x1a, 0x3d, 0x3d, 0x1f, 0xba, 0x1b, 0x60, 0x86, 0x01, 0xb5,
+	0xc3, 0xa3, 0x80, 0x33, 0xc1, 0x10, 0x72, 0x98, 0x7d, 0x4d, 0xf9, 0x51, 0xf8, 0x35, 0xe1, 0x8b,
+	0x6b, 0x57, 0x1c, 0x2d, 0x7f, 0xd6, 0x30, 0xc5, 0x2a, 0xa0, 0x1a, 0xd0, 0x78, 0x67, 0xc6, 0x66,
+	0x4c, 0x2d, 0x1f, 0xca, 0x95, 0xb6, 0xbe, 0xef, 0x44, 0x9c, 0x08, 0x97, 0xf9, 0x0f, 0x93, 0x45,
+	0xfc, 0xa0, 0xf3, 0x97, 0x22, 0x54, 0xce, 0x98, 0x43, 0xcf, 0x03, 0x6a, 0xa3, 0x63, 0x30, 0x89,
+	0xef, 0x33, 0xa1, 0x00, 0xa1, 0x65, 0xb4, 0x8d, 0x43, 0xb3, 0xdb, 0x3a, 0xfa, 0xee, 0x2b, 0x8f,
+	0x7a, 0x1b, 0x58, 0xbf, 0xf8, 0xf2, 0xa6, 0x55, 0xc0, 0x59, 0x26, 0xfa, 0x29, 0x14, 0x39, 0xf3,
+	0xa8, 0xb5, 0xd3, 0x36, 0x0e, 0xef, 0x74, 0x3f, 0xcc, 0xf3, 0x20, 0x5f, 0x8a, 0x99, 0x47, 0xb1,
+	0x42, 0xa2, 0x63, 0x80, 0x05, 0x5d, 0x5c, 0x52, 0x1e, 0xce, 0xdd, 0xc0, 0xda, 0x55, 0xbc, 0x1f,
+	0xdd, 0xc6, 0x93, 0xc1, 0x1e, 0x9d, 0xa6, 0x70, 0x9c, 0xa1, 0xa2, 0x53, 0xa8, 0x91, 0x25, 0x71,
+	0x3d, 0x72, 0xe9, 0x7a, 0xae, 0x58, 0x59, 0x45, 0xe5, 0xea, 0x93, 0xef, 0x75, 0xd5, 0xcb, 0x10,
+	0xf0, 0x16, 0xbd, 0xe3, 0x00, 0x6c, 0x5e, 0x84, 0x3e, 0x86, 0xf2, 0x64, 0x74, 0x36, 0x1c, 0x9f,
+	0x1d, 0xd7, 0x0b, 0x8d, 0x7b, 0xcf, 0x5f, 0xb4, 0xdf, 0x95, 0x3e, 0x36, 0x80, 0x09, 0xf5, 0x1d,
+	0xd7, 0x9f, 0xa1, 0x43, 0xa8, 0xf4, 0x06, 0x83, 0xd1, 0xe4, 0x62, 0x34, 0xac, 0x1b, 0x8d, 0xc6,
+	0xf3, 0x17, 0xed, 0xf7, 0xb6, 0x81, 0x3d, 0xdb, 0xa6, 0x81, 0xa0, 0x4e, 0xa3, 0xf8, 0xcd, 0x5f,
+	0x9b, 0x85, 0xce, 0x37, 0x06, 0xd4, 0xb2, 0x41, 0xa0, 0x8f, 0xa1, 0xd4, 0x1b, 0x5c, 0x8c, 0x9f,
+	0x8e, 0xea, 0x85, 0x0d, 0x3d, 0x8b, 0xe8, 0xd9, 0xc2, 0x5d, 0x52, 0xf4, 0x00, 0xf6, 0x26, 0xbd,
+	0xaf, 0xce, 0x47, 0x75, 0x63, 0x13, 0x4e, 0x16, 0x36, 0x21, 0x51, 0xa8, 0x50, 0x43, 0xdc, 0x1b,
+	0x9f, 0xd5, 0x77, 0xf2, 0x51, 0x43, 0x4e, 0x5c, 0x5f, 0x87, 0xf2, 0x6a, 0x17, 0xcc, 0x73, 0xca,
+	0x97, 0xae, 0xfd, 0x96, 0x35, 0xf1, 0x19, 0x14, 0x05, 0x09, 0xaf, 0x95, 0x26, 0xcc, 0x7c, 0x4d,
+	0x5c, 0x90, 0xf0, 0x5a, 0xbe, 0x54, 0xd3, 0x15, 0x5e, 0x2a, 0x83, 0xd3, 0xc0, 0x73, 0x6d, 0x22,
+	0xa8, 0xa3, 0x94, 0x61, 0x76, 0x7f, 0x98, 0xc7, 0xc6, 0x29, 0x4a, 0xc7, 0xff, 0xa4, 0x80, 0x33,
+	0x54, 0xf4, 0x18, 0x4a, 0x33, 0x8f, 0x5d, 0x12, 0x4f, 0x69, 0xc2, 0xec, 0xde, 0xcf, 0x73, 0x72,
+	0xac, 0x10, 0x1b, 0x07, 0x9a, 0x82, 0x1e, 0x41, 0x29, 0x0a, 0x1c, 0x22, 0xa8, 0x55, 0x52, 0xe4,
+	0x76, 0x1e, 0xf9, 0x2b, 0x85, 0x18, 0x30, 0xff, 0xca, 0x9d, 0x61, 0x8d, 0x47, 0x27, 0x50, 0xf1,
+	0xa9, 0xf8, 0x9a, 0xf1, 0xeb, 0xd0, 0x2a, 0xb7, 0x77, 0x0f, 0xcd, 0xee, 0xa7, 0xb9, 0x62, 0x8c,
+	0x31, 0x3d, 0x21, 0x88, 0x3d, 0x5f, 0x50, 0x5f, 0xc4, 0x6e, 0xfa, 0x3b, 0x96, 0x81, 0x53, 0x07,
+	0xe8, 0xd7, 0x50, 0xa1, 0xbe, 0x13, 0x30, 0xd7, 0x17, 0x56, 0xe5, 0xf6, 0x40, 0x46, 0x1a, 0x23,
+	0x93, 0x89, 0x53, 0x46, 0xbf, 0x04, 0xc5, 0x05, 0x73, 0x68, 0xe7, 0x21, 0x1c, 0x7c, 0x27, 0x59,
+	0xa8, 0x01, 0x15, 0x9d, 0xac, 0xf8, 0x96, 0x8b, 0x38, 0xdd, 0x77, 0xee, 0xc2, 0xfe, 0x56, 0x62,
+	0x54, 0xd9, 0x48, 0x6e, 0x0b, 0xf5, 0xa0, 0x6a, 0x33, 0x5f, 0x10, 0xd7, 0xa7, 0x5c, 0x0b, 0x24,
+	0x37, 0xb7, 0x83, 0x04, 0x24, 0x59, 0x4f, 0x0a, 0x78, 0xc3, 0x42, 0xbf, 0x83, 0x2a, 0xa7, 0x21,
+	0x8b, 0xb8, 0x4d, 0x43, 0xad, 0x90, 0xc3, 0xfc, 0x3b, 0x8e, 0x41, 0x98, 0xfe, 0x29, 0x72, 0x39,
+	0x95, 0x79, 0x0a, 0xf1, 0x86, 0x8a, 0x1e, 0x43, 0x99, 0xd3, 0x50, 0x10, 0x2e, 0xbe, 0xef, 0x92,
+	0x71, 0x0c, 0x99, 0x30, 0xcf, 0xb5, 0x57, 0x38, 0x61, 0xa0, 0xc7, 0x50, 0x0d, 0x3c, 0x62, 0x2b,
+	0xaf, 0xd6, 0x9e, 0xa2, 0xff, 0x20, 0x8f, 0x3e, 0x49, 0x40, 0x78, 0x83, 0x47, 0x9f, 0x03, 0x78,
+	0x6c, 0x36, 0x75, 0xb8, 0xbb, 0xa4, 0x5c, 0x8b, 0xa4, 0x91, 0xc7, 0x1e, 0x2a, 0x04, 0xae, 0x7a,
+	0x6c, 0x16, 0x2f, 0xd1, 0xf1, 0xff, 0xa5, 0x90, 0x8c, 0x3a, 0x4e, 0x00, 0x48, 0xfa, 0x54, 0xeb,
+	0xe3, 0x93, 0x37, 0x72, 0xa5, 0x6f, 0x24, 0x43, 0x47, 0xf7, 0xa1, 0x76, 0xc5, 0xb8, 0x4d, 0xa7,
+	0x5a, 0xf7, 0x55, 0xa5, 0x09, 0x53, 0xd9, 0x62, 0xa1, 0xf7, 0xab, 0x50, 0xe6, 0x91, 0x2f, 0xdc,
+	0x05, 0xed, 0x9c, 0xc0, 0xbb, 0xb9, 0x4e, 0x51, 0x17, 0x6a, 0xe9, 0x35, 0x4f, 0x5d, 0x47, 0xe9,
+	0xa3, 0xda, 0xbf, 0xbb, 0xbe, 0x69, 0x99, 0xa9, 0x1e, 0xc6, 0x43, 0x6c, 0xa6, 0xa0, 0xb1, 0xd3,
+	0xf9, 0x7b, 0x19, 0xf6, 0xb7, 0xc4, 0x82, 0xde, 0x81, 0x3d, 0x77, 0x41, 0x66, 0x34, 0xa6, 0xe3,
+	0x78, 0x83, 0x46, 0x50, 0xf2, 0xc8, 0x25, 0xf5, 0xa4, 0x64, 0x64, 0xda, 0x7e, 0xf2, 0x5a, 0xd5,
+	0x1d, 0xfd, 0x41, 0xe1, 0x47, 0xbe, 0xe0, 0x2b, 0xac, 0xc9, 0xc8, 0x82, 0xb2, 0xcd, 0x16, 0x0b,
+	0xe2, 0xcb, 0xf2, 0xb2, 0x7b, 0x58, 0xc5, 0xc9, 0x16, 0x21, 0x28, 0x12, 0x3e, 0x0b, 0xad, 0xa2,
+	0x32, 0xab, 0x35, 0xaa, 0xc3, 0x2e, 0xf5, 0x97, 0xd6, 0x9e, 0x32, 0xc9, 0xa5, 0xb4, 0x38, 0x6e,
+	0x7c, 0xe7, 0x55, 0x2c, 0x97, 0x92, 0x17, 0x85, 0x94, 0x5b, 0x65, 0x65, 0x52, 0x6b, 0xf4, 0x4b,
+	0x28, 0x2d, 0x58, 0xe4, 0x8b, 0xd0, 0xaa, 0xa8, 0x60, 0xef, 0xe5, 0x05, 0x7b, 0x2a, 0x11, 0xba,
+	0xfc, 0x69, 0x38, 0x7a, 0x02, 0x07, 0xa1, 0x60, 0xc1, 0x74, 0xc6, 0x89, 0x4d, 0xa7, 0x01, 0xe5,
+	0x2e, 0x73, 0xd4, 0x6d, 0xdc, 0x52, 0x45, 0x87, 0xba, 0xc3, 0xe3, 0xbb, 0x92, 0x76, 0x2c, 0x59,
+	0x13, 0x45, 0x42, 0x13, 0xa8, 0x05, 0x91, 0xe7, 0x4d, 0x59, 0x10, 0x17, 0x73, 0x50, 0x4e, 0xde,
+	0x20, 0x6b, 0x93, 0xc8, 0xf3, 0xbe, 0x8c, 0x49, 0xd8, 0x0c, 0x36, 0x1b, 0xf4, 0x1e, 0x94, 0x66,
+	0x9c, 0x45, 0x41, 0x68, 0x99, 0x2a, 0x1f, 0x7a, 0x87, 0xbe, 0x80, 0x72, 0x48, 0x6d, 0x4e, 0x45,
+	0x68, 0xd5, 0xd4, 0x69, 0x3f, 0xca, 0x7b, 0xc9, 0xb9, 0x82, 0x60, 0x7a, 0x45, 0x39, 0xf5, 0x6d,
+	0x8a, 0x13, 0x0e, 0xba, 0x07, 0xbb, 0x42, 0xac, 0xac, 0xfd, 0xb6, 0x71, 0x58, 0xe9, 0x97, 0xd7,
+	0x37, 0xad, 0xdd, 0x8b, 0x8b, 0x67, 0x58, 0xda, 0x64, 0x99, 0x9a, 0xb3, 0x50, 0xf8, 0x64, 0x41,
+	0xad, 0x3b, 0x2a, 0xbd, 0xe9, 0x1e, 0x3d, 0x03, 0x70, 0xfc, 0x70, 0x6a, 0xab, 0xef, 0xc2, 0xba,
+	0xab, 0x4e, 0xf7, 0xe9, 0xeb, 0x4f, 0x37, 0x3c, 0x3b, 0xd7, 0xc5, 0x76, 0x7f, 0x7d, 0xd3, 0xaa,
+	0xa6, 0x5b, 0x5c, 0x75, 0xfc, 0x30, 0x5e, 0xa2, 0x3e, 0x98, 0x73, 0x4a, 0x3c, 0x31, 0xb7, 0xe7,
+	0xd4, 0xbe, 0xb6, 0xea, 0xb7, 0xd7, 0xde, 0x27, 0x0a, 0xa6, 0x3d, 0x64, 0x49, 0x52, 0xc4, 0x32,
+	0xd4, 0xd0, 0x3a, 0x50, 0xb9, 0x8a, 0x37, 0x8d, 0xcf, 0xc1, 0xcc, 0x88, 0x52, 0x8a, 0xe9, 0x9a,
+	0xae, 0xb4, 0xce, 0xe5, 0x52, 0xd2, 0x96, 0xc4, 0x8b, 0xe2, 0x69, 0xaa, 0x8a, 0xe3, 0xcd, 0xaf,
+	0x76, 0x1e, 0x19, 0x8d, 0x2e, 0x98, 0x99, 0x9b, 0x41, 0x1f, 0xc1, 0x3e, 0xa7, 0x33, 0x37, 0x14,
+	0x7c, 0x35, 0x25, 0x91, 0x98, 0x5b, 0xbf, 0x55, 0x84, 0x5a, 0x62, 0xec, 0x45, 0x62, 0xde, 0x98,
+	0xc2, 0xe6, 0x80, 0xa8, 0x0d, 0xa6, 0x4c, 0x5c, 0x48, 0xf9, 0x92, 0x72, 0x59, 0xf6, 0x65, 0x5c,
+	0x59, 0x93, 0xbc, 0xe0, 0x90, 0x12, 0x6e, 0xcf, 0xd5, 0x27, 0x56, 0xc5, 0x7a, 0x27, 0xbf, 0x99,
+	0x44, 0x45, 0xfa, 0x9b, 0xd1, 0xdb, 0xce, 0x7f, 0x0c, 0xa8, 0x65, 0xfb, 0x0f, 0x1a, 0xc4, 0x5d,
+	0x47, 0x1d, 0xe9, 0x4e, 0xf7, 0xe1, 0xeb, 0xfa, 0x95, 0xaa, 0xf1, 0x5e, 0x24, 0x9d, 0x9d, 0xca,
+	0x19, 0x51, 0x91, 0xd1, 0x2f, 0x60, 0x2f, 0x60, 0x5c, 0x24, 0x5f, 0x7a, 0x33, 0xb7, 0x2e, 0x33,
+	0x9e, 0xd4, 0xc4, 0x18, 0xdc, 0x99, 0xc3, 0x9d, 0x6d, 0x6f, 0xe8, 0x01, 0xec, 0x3e, 0x1d, 0x4f,
+	0xea, 0x85, 0xc6, 0x07, 0xcf, 0x5f, 0xb4, 0xdf, 0xdf, 0x7e, 0xf8, 0xd4, 0xe5, 0x22, 0x22, 0xde,
+	0x78, 0x82, 0x7e, 0x0c, 0x7b, 0xc3, 0xb3, 0x73, 0x8c, 0xeb, 0x46, 0xa3, 0xf5, 0xfc, 0x45, 0xfb,
+	0x83, 0x6d, 0x9c, 0x7c, 0xc4, 0x22, 0xdf, 0xc1, 0xec, 0x32, 0x1d, 0x9b, 0xfe, 0xb6, 0x03, 0xa6,
+	0x2e, 0x80, 0x6f, 0x77, 0x6c, 0xfa, 0x0d, 0xec, 0xc7, 0x3d, 0x25, 0x91, 0xf5, 0xce, 0x6b, 0x5b,
+	0x4b, 0x2d, 0x26, 0xe8, 0x3b, 0xbe, 0x0f, 0x35, 0x37, 0x58, 0x7e, 0x36, 0xa5, 0x3e, 0xb9, 0xf4,
+	0xf4, 0x04, 0x55, 0xc1, 0xa6, 0xb4, 0x8d, 0x62, 0x93, 0xfc, 0xa6, 0x5c, 0x5f, 0x50, 0xee, 0xeb,
+	0xd9, 0xa8, 0x82, 0xd3, 0x3d, 0xfa, 0x02, 0x8a, 0x6e, 0x40, 0x16, 0xba, 0x1f, 0xe6, 0x9e, 0x60,
+	0x3c, 0xe9, 0x9d, 0x6a, 0x0d, 0xf6, 0x2b, 0xeb, 0x9b, 0x56, 0x51, 0x1a, 0xb0, 0xa2, 0xa1, 0x66,
+	0xd2, 0x92, 0xe4, 0x9b, 0x54, 0x89, 0xac, 0xe0, 0x8c, 0xa5, 0xf3, 0xdf, 0x22, 0x98, 0x03, 0x2f,
+	0x0a, 0x85, 0x2e, 0xf4, 0x6f, 0x2d, 0x6f, 0xcf, 0xe0, 0x80, 0xa8, 0x21, 0x9b, 0xf8, 0xb2, 0x6a,
+	0xaa, 0x56, 0xaf, 0x73, 0xf7, 0x20, 0xd7, 0x5d, 0x0a, 0x8e, 0xc7, 0x82, 0x7e, 0x49, 0xfa, 0xb4,
+	0x0c, 0x5c, 0x27, 0xdf, 0x7a, 0x82, 0xce, 0x61, 0x9f, 0x71, 0x7b, 0x4e, 0x43, 0x11, 0x17, 0x5a,
+	0x3d, 0x94, 0xe6, 0xfe, 0xae, 0x7c, 0x99, 0x05, 0xea, 0x2a, 0x13, 0x47, 0xbb, 0xed, 0x03, 0x3d,
+	0x82, 0x22, 0x27, 0x57, 0xc9, 0xd8, 0x92, 0xab, 0x6f, 0x4c, 0xae, 0xc4, 0x96, 0x0b, 0xc5, 0x40,
+	0xbf, 0x07, 0x70, 0xdc, 0x30, 0x20, 0xc2, 0x9e, 0x53, 0xae, 0xef, 0x29, 0xf7, 0x88, 0xc3, 0x14,
+	0xb5, 0xe5, 0x25, 0xc3, 0x46, 0x27, 0x50, 0xb5, 0x49, 0xa2, 0xb4, 0xd2, 0xed, 0x3d, 0x66, 0xd0,
+	0xd3, 0x2e, 0xea, 0xd2, 0xc5, 0xfa, 0xa6, 0x55, 0x49, 0x2c, 0xb8, 0x62, 0x13, 0xad, 0xbc, 0x13,
+	0xd8, 0x97, 0x13, 0xfc, 0xd4, 0xa1, 0x57, 0x24, 0xf2, 0x44, 0xa8, 0xda, 0xe1, 0x2d, 0x55, 0x53,
+	0x0e, 0x93, 0x43, 0x8d, 0xd3, 0x71, 0xd5, 0x44, 0xc6, 0x86, 0xfe, 0x08, 0x07, 0xd4, 0xb7, 0xf9,
+	0x4a, 0xe9, 0x2c, 0x89, 0xb0, 0x72, 0xfb, 0x61, 0x47, 0x29, 0x78, 0xeb, 0xb0, 0x75, 0xfa, 0x2d,
+	0x7b, 0xc7, 0x05, 0x88, 0xfb, 0xd0, 0xdb, 0xd5, 0x1f, 0x82, 0xa2, 0x43, 0x04, 0x51, 0x92, 0xab,
+	0x61, 0xb5, 0xee, 0x7f, 0xf8, 0xf2, 0x55, 0xb3, 0xf0, 0xcf, 0x57, 0xcd, 0xc2, 0xbf, 0x5f, 0x35,
+	0x8d, 0x3f, 0xaf, 0x9b, 0xc6, 0xcb, 0x75, 0xd3, 0xf8, 0xc7, 0xba, 0x69, 0xfc, 0x6b, 0xdd, 0x34,
+	0x2e, 0x4b, 0xea, 0x8f, 0xfc, 0xe7, 0xff, 0x0b, 0x00, 0x00, 0xff, 0xff, 0xce, 0x13, 0x97, 0xa2,
+	0xf0, 0x0f, 0x00, 0x00,
 }

vendor/github.com/docker/swarmkit/api/specs.proto

@@ -323,6 +323,9 @@ message ClusterSpec {
 
 	// TaskDefaults specifies the default values to use for task creation.
 	TaskDefaults task_defaults = 7 [(gogoproto.nullable) = false];
+
+	// EncryptionConfig defines the cluster's encryption settings.
+	EncryptionConfig encryption_config = 8 [(gogoproto.nullable) = false];
 }
 
 // SecretSpec specifies a user-provided secret.

vendor/github.com/docker/swarmkit/api/types.pb.go

@@ -52,6 +52,7 @@
 		TaskDefaults
 		DispatcherConfig
 		RaftConfig
+		EncryptionConfig
 		Placement
 		JoinTokens
 		RootCA
@@ -118,7 +119,7 @@
 		GetClusterResponse
 		ListClustersRequest
 		ListClustersResponse
-		JoinTokenRotation
+		KeyRotation
 		UpdateClusterRequest
 		UpdateClusterResponse
 		GetSecretRequest
@@ -149,6 +150,8 @@
 		IssueNodeCertificateResponse
 		GetRootCACertificateRequest
 		GetRootCACertificateResponse
+		GetUnlockKeyRequest
+		GetUnlockKeyResponse
 		StoreSnapshot
 		ClusterSnapshot
 		Snapshot
@@ -662,7 +665,7 @@ func (x EncryptionKey_Algorithm) String() string {
 	return proto.EnumName(EncryptionKey_Algorithm_name, int32(x))
 }
 func (EncryptionKey_Algorithm) EnumDescriptor() ([]byte, []int) {
-	return fileDescriptorTypes, []int{37, 0}
+	return fileDescriptorTypes, []int{38, 0}
 }
 
 type MaybeEncryptedRecord_Algorithm int32
@@ -685,7 +688,7 @@ func (x MaybeEncryptedRecord_Algorithm) String() string {
 	return proto.EnumName(MaybeEncryptedRecord_Algorithm_name, int32(x))
 }
 func (MaybeEncryptedRecord_Algorithm) EnumDescriptor() ([]byte, []int) {
-	return fileDescriptorTypes, []int{42, 0}
+	return fileDescriptorTypes, []int{43, 0}
 }
 
 // Version tracks the last time an object in the store was updated.
@@ -1359,6 +1362,17 @@ func (m *RaftConfig) Reset()                    { *m = RaftConfig{} }
 func (*RaftConfig) ProtoMessage()               {}
 func (*RaftConfig) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{32} }
 
+type EncryptionConfig struct {
+	// AutoLockManagers specifies whether or not managers TLS keys and raft data
+	// should be encrypted at rest in such a way that they must be unlocked
+	// before the manager node starts up again.
+	AutoLockManagers bool `protobuf:"varint,1,opt,name=auto_lock_managers,json=autoLockManagers,proto3" json:"auto_lock_managers,omitempty"`
+}
+
+func (m *EncryptionConfig) Reset()                    { *m = EncryptionConfig{} }
+func (*EncryptionConfig) ProtoMessage()               {}
+func (*EncryptionConfig) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{33} }
+
 // Placement specifies task distribution constraints.
 type Placement struct {
 	// constraints specifies a set of requirements a node should meet for a task.
@@ -1367,7 +1381,7 @@ type Placement struct {
 
 func (m *Placement) Reset()                    { *m = Placement{} }
 func (*Placement) ProtoMessage()               {}
-func (*Placement) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{33} }
+func (*Placement) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{34} }
 
 // JoinToken contains the join tokens for workers and managers.
 type JoinTokens struct {
@@ -1379,7 +1393,7 @@ type JoinTokens struct {
 
 func (m *JoinTokens) Reset()                    { *m = JoinTokens{} }
 func (*JoinTokens) ProtoMessage()               {}
-func (*JoinTokens) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{34} }
+func (*JoinTokens) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{35} }
 
 type RootCA struct {
 	// CAKey is the root CA private key.
@@ -1394,7 +1408,7 @@ type RootCA struct {
 
 func (m *RootCA) Reset()                    { *m = RootCA{} }
 func (*RootCA) ProtoMessage()               {}
-func (*RootCA) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{35} }
+func (*RootCA) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{36} }
 
 type Certificate struct {
 	Role        NodeRole       `protobuf:"varint,1,opt,name=role,proto3,enum=docker.swarmkit.v1.NodeRole" json:"role,omitempty"`
@@ -1407,7 +1421,7 @@ type Certificate struct {
 
 func (m *Certificate) Reset()                    { *m = Certificate{} }
 func (*Certificate) ProtoMessage()               {}
-func (*Certificate) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{36} }
+func (*Certificate) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{37} }
 
 // Symmetric keys to encrypt inter-agent communication.
 type EncryptionKey struct {
@@ -1423,7 +1437,7 @@ type EncryptionKey struct {
 
 func (m *EncryptionKey) Reset()                    { *m = EncryptionKey{} }
 func (*EncryptionKey) ProtoMessage()               {}
-func (*EncryptionKey) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{37} }
+func (*EncryptionKey) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{38} }
 
 // ManagerStatus provides informations about the state of a manager in the cluster.
 type ManagerStatus struct {
@@ -1440,7 +1454,7 @@ type ManagerStatus struct {
 
 func (m *ManagerStatus) Reset()                    { *m = ManagerStatus{} }
 func (*ManagerStatus) ProtoMessage()               {}
-func (*ManagerStatus) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{38} }
+func (*ManagerStatus) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{39} }
 
 // SecretReference is the linkage between a service and a secret that it uses.
 type SecretReference struct {
@@ -1460,7 +1474,7 @@ type SecretReference struct {
 
 func (m *SecretReference) Reset()                    { *m = SecretReference{} }
 func (*SecretReference) ProtoMessage()               {}
-func (*SecretReference) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{39} }
+func (*SecretReference) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{40} }
 
 type isSecretReference_Target interface {
 	isSecretReference_Target()
@@ -1558,7 +1572,7 @@ type SecretReference_FileTarget struct {
 func (m *SecretReference_FileTarget) Reset()      { *m = SecretReference_FileTarget{} }
 func (*SecretReference_FileTarget) ProtoMessage() {}
 func (*SecretReference_FileTarget) Descriptor() ([]byte, []int) {
-	return fileDescriptorTypes, []int{39, 0}
+	return fileDescriptorTypes, []int{40, 0}
 }
 
 // BlacklistedCertificate is a record for a blacklisted certificate. It does not
@@ -1571,7 +1585,7 @@ type BlacklistedCertificate struct {
 
 func (m *BlacklistedCertificate) Reset()                    { *m = BlacklistedCertificate{} }
 func (*BlacklistedCertificate) ProtoMessage()               {}
-func (*BlacklistedCertificate) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{40} }
+func (*BlacklistedCertificate) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{41} }
 
 // HealthConfig holds configuration settings for the HEALTHCHECK feature.
 type HealthConfig struct {
@@ -1595,7 +1609,7 @@ type HealthConfig struct {
 
 func (m *HealthConfig) Reset()                    { *m = HealthConfig{} }
 func (*HealthConfig) ProtoMessage()               {}
-func (*HealthConfig) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{41} }
+func (*HealthConfig) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{42} }
 
 type MaybeEncryptedRecord struct {
 	Algorithm MaybeEncryptedRecord_Algorithm `protobuf:"varint,1,opt,name=algorithm,proto3,enum=docker.swarmkit.v1.MaybeEncryptedRecord_Algorithm" json:"algorithm,omitempty"`
@@ -1605,7 +1619,7 @@ type MaybeEncryptedRecord struct {
 
 func (m *MaybeEncryptedRecord) Reset()                    { *m = MaybeEncryptedRecord{} }
 func (*MaybeEncryptedRecord) ProtoMessage()               {}
-func (*MaybeEncryptedRecord) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{42} }
+func (*MaybeEncryptedRecord) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{43} }
 
 func init() {
 	proto.RegisterType((*Version)(nil), "docker.swarmkit.v1.Version")
@@ -1646,6 +1660,7 @@ func init() {
 	proto.RegisterType((*TaskDefaults)(nil), "docker.swarmkit.v1.TaskDefaults")
 	proto.RegisterType((*DispatcherConfig)(nil), "docker.swarmkit.v1.DispatcherConfig")
 	proto.RegisterType((*RaftConfig)(nil), "docker.swarmkit.v1.RaftConfig")
+	proto.RegisterType((*EncryptionConfig)(nil), "docker.swarmkit.v1.EncryptionConfig")
 	proto.RegisterType((*Placement)(nil), "docker.swarmkit.v1.Placement")
 	proto.RegisterType((*JoinTokens)(nil), "docker.swarmkit.v1.JoinTokens")
 	proto.RegisterType((*RootCA)(nil), "docker.swarmkit.v1.RootCA")
@@ -2276,6 +2291,18 @@ func (m *RaftConfig) Copy() *RaftConfig {
 	return o
 }
 
+func (m *EncryptionConfig) Copy() *EncryptionConfig {
+	if m == nil {
+		return nil
+	}
+
+	o := &EncryptionConfig{
+		AutoLockManagers: m.AutoLockManagers,
+	}
+
+	return o
+}
+
 func (m *Placement) Copy() *Placement {
 	if m == nil {
 		return nil
@@ -3028,6 +3055,16 @@ func (this *RaftConfig) GoString() string {
 	s = append(s, "}")
 	return strings.Join(s, "")
 }
+func (this *EncryptionConfig) GoString() string {
+	if this == nil {
+		return "nil"
+	}
+	s := make([]string, 0, 5)
+	s = append(s, "&api.EncryptionConfig{")
+	s = append(s, "AutoLockManagers: "+fmt.Sprintf("%#v", this.AutoLockManagers)+",\n")
+	s = append(s, "}")
+	return strings.Join(s, "")
+}
 func (this *Placement) GoString() string {
 	if this == nil {
 		return "nil"
@@ -4708,6 +4745,34 @@ func (m *RaftConfig) MarshalTo(data []byte) (int, error) {
 	return i, nil
 }
 
+func (m *EncryptionConfig) Marshal() (data []byte, err error) {
+	size := m.Size()
+	data = make([]byte, size)
+	n, err := m.MarshalTo(data)
+	if err != nil {
+		return nil, err
+	}
+	return data[:n], nil
+}
+
+func (m *EncryptionConfig) MarshalTo(data []byte) (int, error) {
+	var i int
+	_ = i
+	var l int
+	_ = l
+	if m.AutoLockManagers {
+		data[i] = 0x8
+		i++
+		if m.AutoLockManagers {
+			data[i] = 1
+		} else {
+			data[i] = 0
+		}
+		i++
+	}
+	return i, nil
+}
+
 func (m *Placement) Marshal() (data []byte, err error) {
 	size := m.Size()
 	data = make([]byte, size)
@@ -5836,6 +5901,15 @@ func (m *RaftConfig) Size() (n int) {
 	return n
 }
 
+func (m *EncryptionConfig) Size() (n int) {
+	var l int
+	_ = l
+	if m.AutoLockManagers {
+		n += 2
+	}
+	return n
+}
+
 func (m *Placement) Size() (n int) {
 	var l int
 	_ = l
@@ -6569,6 +6643,16 @@ func (this *RaftConfig) String() string {
 	}, "")
 	return s
 }
+func (this *EncryptionConfig) String() string {
+	if this == nil {
+		return "nil"
+	}
+	s := strings.Join([]string{`&EncryptionConfig{`,
+		`AutoLockManagers:` + fmt.Sprintf("%v", this.AutoLockManagers) + `,`,
+		`}`,
+	}, "")
+	return s
+}
 func (this *Placement) String() string {
 	if this == nil {
 		return "nil"
@@ -11863,6 +11947,76 @@ func (m *RaftConfig) Unmarshal(data []byte) error {
 	}
 	return nil
 }
+func (m *EncryptionConfig) Unmarshal(data []byte) error {
+	l := len(data)
+	iNdEx := 0
+	for iNdEx < l {
+		preIndex := iNdEx
+		var wire uint64
+		for shift := uint(0); ; shift += 7 {
+			if shift >= 64 {
+				return ErrIntOverflowTypes
+			}
+			if iNdEx >= l {
+				return io.ErrUnexpectedEOF
+			}
+			b := data[iNdEx]
+			iNdEx++
+			wire |= (uint64(b) & 0x7F) << shift
+			if b < 0x80 {
+				break
+			}
+		}
+		fieldNum := int32(wire >> 3)
+		wireType := int(wire & 0x7)
+		if wireType == 4 {
+			return fmt.Errorf("proto: EncryptionConfig: wiretype end group for non-group")
+		}
+		if fieldNum <= 0 {
+			return fmt.Errorf("proto: EncryptionConfig: illegal tag %d (wire type %d)", fieldNum, wire)
+		}
+		switch fieldNum {
+		case 1:
+			if wireType != 0 {
+				return fmt.Errorf("proto: wrong wireType = %d for field AutoLockManagers", wireType)
+			}
+			var v int
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowTypes
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := data[iNdEx]
+				iNdEx++
+				v |= (int(b) & 0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			m.AutoLockManagers = bool(v != 0)
+		default:
+			iNdEx = preIndex
+			skippy, err := skipTypes(data[iNdEx:])
+			if err != nil {
+				return err
+			}
+			if skippy < 0 {
+				return ErrInvalidLengthTypes
+			}
+			if (iNdEx + skippy) > l {
+				return io.ErrUnexpectedEOF
+			}
+			iNdEx += skippy
+		}
+	}
+
+	if iNdEx > l {
+		return io.ErrUnexpectedEOF
+	}
+	return nil
+}
 func (m *Placement) Unmarshal(data []byte) error {
 	l := len(data)
 	iNdEx := 0
@@ -13478,252 +13632,254 @@ var (
 func init() { proto.RegisterFile("types.proto", fileDescriptorTypes) }
 
 var fileDescriptorTypes = []byte{
-	// 3946 bytes of a gzipped FileDescriptorProto
+	// 3975 bytes of a gzipped FileDescriptorProto
 	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xac, 0x79, 0x4d, 0x6c, 0x1b, 0x49,
-	0x76, 0xbf, 0xf8, 0x29, 0xf2, 0x91, 0x92, 0xda, 0x65, 0xaf, 0x47, 0xe6, 0x78, 0x24, 0x4e, 0xcf,
-	0x78, 0x67, 0xc6, 0x3b, 0x7f, 0x8e, 0xad, 0xf9, 0x80, 0x77, 0xfc, 0xcf, 0x7a, 0x9a, 0x1f, 0xb2,
-	0xb8, 0x96, 0x48, 0xa2, 0x48, 0xd9, 0x19, 0x04, 0x08, 0x51, 0xea, 0x2e, 0x91, 0x3d, 0x6a, 0x76,
-	0x33, 0xdd, 0x45, 0xc9, 0x4c, 0x10, 0xc4, 0xc8, 0x21, 0x09, 0x74, 0xca, 0x3d, 0x10, 0x82, 0x20,
-	0x41, 0x0e, 0x39, 0xec, 0x25, 0x87, 0x00, 0x39, 0x0d, 0x72, 0x9a, 0xe3, 0x26, 0x01, 0x82, 0x45,
-	0x82, 0x18, 0x19, 0xe5, 0x1c, 0x60, 0x2f, 0x8b, 0x1c, 0x92, 0x00, 0x41, 0x7d, 0x74, 0xb3, 0x29,
-	0xd3, 0xb2, 0x27, 0xbb, 0x17, 0xb2, 0xeb, 0xd5, 0xef, 0xbd, 0xfa, 0x7a, 0x55, 0xf5, 0x7b, 0xaf,
-	0xa0, 0xc0, 0xa6, 0x63, 0x1a, 0x54, 0xc6, 0xbe, 0xc7, 0x3c, 0x84, 0x2c, 0xcf, 0x3c, 0xa2, 0x7e,
-	0x25, 0x38, 0x21, 0xfe, 0xe8, 0xc8, 0x66, 0x95, 0xe3, 0xbb, 0xa5, 0x1b, 0xcc, 0x1e, 0xd1, 0x80,
-	0x91, 0xd1, 0xf8, 0xa3, 0xe8, 0x4b, 0xc2, 0x4b, 0x6f, 0x58, 0x13, 0x9f, 0x30, 0xdb, 0x73, 0x3f,
-	0x0a, 0x3f, 0x54, 0xc5, 0xb5, 0x81, 0x37, 0xf0, 0xc4, 0xe7, 0x47, 0xfc, 0x4b, 0x4a, 0xf5, 0x4d,
-	0x58, 0x7e, 0x4c, 0xfd, 0xc0, 0xf6, 0x5c, 0x74, 0x0d, 0x32, 0xb6, 0x6b, 0xd1, 0xa7, 0xeb, 0x89,
-	0x72, 0xe2, 0xfd, 0x34, 0x96, 0x05, 0xfd, 0xcf, 0x12, 0x50, 0x30, 0x5c, 0xd7, 0x63, 0xc2, 0x56,
-	0x80, 0x10, 0xa4, 0x5d, 0x32, 0xa2, 0x02, 0x94, 0xc7, 0xe2, 0x1b, 0xd5, 0x20, 0xeb, 0x90, 0x03,
-	0xea, 0x04, 0xeb, 0xc9, 0x72, 0xea, 0xfd, 0xc2, 0xd6, 0x0f, 0x2a, 0x2f, 0xf6, 0xb9, 0x12, 0x33,
-	0x52, 0xd9, 0x15, 0xe8, 0x86, 0xcb, 0xfc, 0x29, 0x56, 0xaa, 0xa5, 0x1f, 0x42, 0x21, 0x26, 0x46,
-	0x1a, 0xa4, 0x8e, 0xe8, 0x54, 0x35, 0xc3, 0x3f, 0x79, 0xff, 0x8e, 0x89, 0x33, 0xa1, 0xeb, 0x49,
-	0x21, 0x93, 0x85, 0xcf, 0x93, 0xf7, 0x12, 0xfa, 0x97, 0x90, 0xc7, 0x34, 0xf0, 0x26, 0xbe, 0x49,
-	0x03, 0xf4, 0x01, 0xe4, 0x5d, 0xe2, 0x7a, 0x7d, 0x73, 0x3c, 0x09, 0x84, 0x7a, 0xaa, 0x5a, 0x3c,
-	0x7f, 0xbe, 0x99, 0x6b, 0x11, 0xd7, 0xab, 0x75, 0xf6, 0x03, 0x9c, 0xe3, 0xd5, 0xb5, 0xf1, 0x24,
-	0x40, 0x6f, 0x43, 0x71, 0x44, 0x47, 0x9e, 0x3f, 0xed, 0x1f, 0x4c, 0x19, 0x0d, 0x84, 0xe1, 0x14,
-	0x2e, 0x48, 0x59, 0x95, 0x8b, 0xf4, 0x3f, 0x4e, 0xc0, 0xb5, 0xd0, 0x36, 0xa6, 0xbf, 0x35, 0xb1,
-	0x7d, 0x3a, 0xa2, 0x2e, 0x0b, 0xd0, 0xa7, 0x90, 0x75, 0xec, 0x91, 0xcd, 0x64, 0x1b, 0x85, 0xad,
-	0xb7, 0x16, 0x8d, 0x39, 0xea, 0x15, 0x56, 0x60, 0x64, 0x40, 0xd1, 0xa7, 0x01, 0xf5, 0x8f, 0xe5,
-	0x4c, 0x88, 0x26, 0x5f, 0xa9, 0x3c, 0xa7, 0xa2, 0x6f, 0x43, 0xae, 0xe3, 0x10, 0x76, 0xe8, 0xf9,
-	0x23, 0xa4, 0x43, 0x91, 0xf8, 0xe6, 0xd0, 0x66, 0xd4, 0x64, 0x13, 0x3f, 0x5c, 0x95, 0x39, 0x19,
-	0xba, 0x0e, 0x49, 0x4f, 0x36, 0x94, 0xaf, 0x66, 0xcf, 0x9f, 0x6f, 0x26, 0xdb, 0x5d, 0x9c, 0xf4,
-	0x02, 0xfd, 0x3e, 0x5c, 0xe9, 0x38, 0x93, 0x81, 0xed, 0xd6, 0x69, 0x60, 0xfa, 0xf6, 0x98, 0x5b,
-	0xe7, 0xcb, 0xcb, 0x9d, 0x2f, 0x5c, 0x5e, 0xfe, 0x1d, 0x2d, 0x79, 0x72, 0xb6, 0xe4, 0xfa, 0x1f,
-	0x26, 0xe1, 0x4a, 0xc3, 0x1d, 0xd8, 0x2e, 0x8d, 0x6b, 0xdf, 0x82, 0x55, 0x2a, 0x84, 0xfd, 0x63,
-	0xe9, 0x54, 0xca, 0xce, 0x8a, 0x94, 0x86, 0x9e, 0xd6, 0xbc, 0xe0, 0x2f, 0x77, 0x17, 0x0d, 0xff,
-	0x05, 0xeb, 0x8b, 0xbc, 0x06, 0x35, 0x60, 0x79, 0x2c, 0x06, 0x11, 0xac, 0xa7, 0x84, 0xad, 0x5b,
-	0x8b, 0x6c, 0xbd, 0x30, 0xce, 0x6a, 0xfa, 0x9b, 0xe7, 0x9b, 0x4b, 0x38, 0xd4, 0xfd, 0x65, 0x9c,
-	0xef, 0xdf, 0x13, 0xb0, 0xd6, 0xf2, 0xac, 0xb9, 0x79, 0x28, 0x41, 0x6e, 0xe8, 0x05, 0x2c, 0xb6,
-	0x51, 0xa2, 0x32, 0xba, 0x07, 0xb9, 0xb1, 0x5a, 0x3e, 0xb5, 0xfa, 0x37, 0x17, 0x77, 0x59, 0x62,
-	0x70, 0x84, 0x46, 0xf7, 0x21, 0xef, 0x87, 0x3e, 0xb1, 0x9e, 0x7a, 0x1d, 0xc7, 0x99, 0xe1, 0xd1,
-	0xaf, 0x41, 0x56, 0x2e, 0xc2, 0x7a, 0x5a, 0x68, 0xde, 0x7a, 0xad, 0x39, 0xc7, 0x4a, 0x49, 0xff,
-	0x59, 0x02, 0x34, 0x4c, 0x0e, 0xd9, 0x1e, 0x1d, 0x1d, 0x50, 0xbf, 0xcb, 0x08, 0x9b, 0x04, 0xe8,
-	0x3a, 0x64, 0x1d, 0x4a, 0x2c, 0xea, 0x8b, 0x41, 0xe6, 0xb0, 0x2a, 0xa1, 0x7d, 0xee, 0xe4, 0xc4,
-	0x1c, 0x92, 0x03, 0xdb, 0xb1, 0xd9, 0x54, 0x0c, 0x73, 0x75, 0xf1, 0x2a, 0x5f, 0xb4, 0x59, 0xc1,
-	0x31, 0x45, 0x3c, 0x67, 0x06, 0xad, 0xc3, 0xf2, 0x88, 0x06, 0x01, 0x19, 0x50, 0x31, 0xfa, 0x3c,
-	0x0e, 0x8b, 0xfa, 0x7d, 0x28, 0xc6, 0xf5, 0x50, 0x01, 0x96, 0xf7, 0x5b, 0x8f, 0x5a, 0xed, 0x27,
-	0x2d, 0x6d, 0x09, 0xad, 0x41, 0x61, 0xbf, 0x85, 0x1b, 0x46, 0x6d, 0xc7, 0xa8, 0xee, 0x36, 0xb4,
-	0x04, 0x5a, 0x81, 0xfc, 0xac, 0x98, 0xd4, 0xff, 0x3a, 0x01, 0xc0, 0x17, 0x50, 0x0d, 0xea, 0x73,
-	0xc8, 0x04, 0x8c, 0x30, 0xb9, 0x70, 0xab, 0x5b, 0xef, 0x2e, 0xea, 0xf5, 0x0c, 0x5e, 0xe1, 0x7f,
-	0x14, 0x4b, 0x95, 0x78, 0x0f, 0x93, 0x73, 0x3d, 0xe4, 0x7b, 0x88, 0x58, 0x96, 0xaf, 0x3a, 0x2e,
-	0xbe, 0xf5, 0xfb, 0x90, 0x11, 0xda, 0xf3, 0xdd, 0xcd, 0x41, 0xba, 0xce, 0xbf, 0x12, 0x28, 0x0f,
-	0x19, 0xdc, 0x30, 0xea, 0x5f, 0x6a, 0x49, 0xa4, 0x41, 0xb1, 0xde, 0xec, 0xd6, 0xda, 0xad, 0x56,
-	0xa3, 0xd6, 0x6b, 0xd4, 0xb5, 0x94, 0x7e, 0x0b, 0x32, 0xcd, 0x11, 0xb7, 0x7c, 0x93, 0x7b, 0xc5,
-	0x21, 0xf5, 0xa9, 0x6b, 0x86, 0xce, 0x36, 0x13, 0xe8, 0x3f, 0xcd, 0x43, 0x66, 0xcf, 0x9b, 0xb8,
-	0x0c, 0x6d, 0xc5, 0x76, 0xf6, 0xea, 0xd6, 0xc6, 0xa2, 0x61, 0x09, 0x60, 0xa5, 0x37, 0x1d, 0x53,
-	0xb5, 0xf3, 0xaf, 0x43, 0x56, 0xfa, 0x8f, 0x1a, 0x8e, 0x2a, 0x71, 0x39, 0x23, 0xfe, 0x80, 0x32,
-	0x35, 0x1e, 0x55, 0x42, 0xef, 0x43, 0xce, 0xa7, 0xc4, 0xf2, 0x5c, 0x67, 0x2a, 0xdc, 0x2c, 0x27,
-	0x8f, 0x5e, 0x4c, 0x89, 0xd5, 0x76, 0x9d, 0x29, 0x8e, 0x6a, 0xd1, 0x0e, 0x14, 0x0f, 0x6c, 0xd7,
-	0xea, 0x7b, 0x63, 0x79, 0x0e, 0x66, 0x5e, 0xee, 0x94, 0xb2, 0x57, 0x55, 0xdb, 0xb5, 0xda, 0x12,
-	0x8c, 0x0b, 0x07, 0xb3, 0x02, 0x6a, 0xc1, 0xea, 0xb1, 0xe7, 0x4c, 0x46, 0x34, 0xb2, 0x95, 0x15,
-	0xb6, 0xde, 0x7b, 0xb9, 0xad, 0xc7, 0x02, 0x1f, 0x5a, 0x5b, 0x39, 0x8e, 0x17, 0xd1, 0x23, 0x58,
-	0x61, 0xa3, 0xf1, 0x61, 0x10, 0x99, 0x5b, 0x16, 0xe6, 0xbe, 0x7f, 0xc9, 0x84, 0x71, 0x78, 0x68,
-	0xad, 0xc8, 0x62, 0xa5, 0xd2, 0xef, 0xa7, 0xa0, 0x10, 0xeb, 0x39, 0xea, 0x42, 0x61, 0xec, 0x7b,
-	0x63, 0x32, 0x10, 0x67, 0xb9, 0x5a, 0x8b, 0xbb, 0xaf, 0x35, 0xea, 0x4a, 0x67, 0xa6, 0x88, 0xe3,
-	0x56, 0xf4, 0xb3, 0x24, 0x14, 0x62, 0x95, 0xe8, 0x36, 0xe4, 0x70, 0x07, 0x37, 0x1f, 0x1b, 0xbd,
-	0x86, 0xb6, 0x54, 0xba, 0x79, 0x7a, 0x56, 0x5e, 0x17, 0xd6, 0xe2, 0x06, 0x3a, 0xbe, 0x7d, 0xcc,
-	0x5d, 0xef, 0x7d, 0x58, 0x0e, 0xa1, 0x89, 0xd2, 0x9b, 0xa7, 0x67, 0xe5, 0x37, 0x2e, 0x42, 0x63,
-	0x48, 0xdc, 0xdd, 0x31, 0x70, 0xa3, 0xae, 0x25, 0x17, 0x23, 0x71, 0x77, 0x48, 0x7c, 0x6a, 0xa1,
-	0xef, 0x43, 0x56, 0x01, 0x53, 0xa5, 0xd2, 0xe9, 0x59, 0xf9, 0xfa, 0x45, 0xe0, 0x0c, 0x87, 0xbb,
-	0xbb, 0xc6, 0xe3, 0x86, 0x96, 0x5e, 0x8c, 0xc3, 0x5d, 0x87, 0x1c, 0x53, 0xf4, 0x2e, 0x64, 0x24,
-	0x2c, 0x53, 0xba, 0x71, 0x7a, 0x56, 0xfe, 0xde, 0x0b, 0xe6, 0x38, 0xaa, 0xb4, 0xfe, 0x47, 0x7f,
-	0xbe, 0xb1, 0xf4, 0xb7, 0x7f, 0xb1, 0xa1, 0x5d, 0xac, 0x2e, 0xfd, 0x77, 0x02, 0x56, 0xe6, 0x96,
-	0x1c, 0xe9, 0x90, 0x75, 0x3d, 0xd3, 0x1b, 0xcb, 0x23, 0x3e, 0x57, 0x85, 0xf3, 0xe7, 0x9b, 0xd9,
-	0x96, 0x57, 0xf3, 0xc6, 0x53, 0xac, 0x6a, 0xd0, 0xa3, 0x0b, 0x97, 0xd4, 0xc7, 0xaf, 0xe9, 0x4f,
-	0x0b, 0xaf, 0xa9, 0x07, 0xb0, 0x62, 0xf9, 0xf6, 0x31, 0xf5, 0xfb, 0xa6, 0xe7, 0x1e, 0xda, 0x03,
-	0x75, 0x7c, 0x97, 0x16, 0xd9, 0xac, 0x0b, 0x20, 0x2e, 0x4a, 0x85, 0x9a, 0xc0, 0xff, 0x12, 0x17,
-	0x54, 0xe9, 0x31, 0x14, 0xe3, 0x1e, 0x8a, 0xde, 0x02, 0x08, 0xec, 0xdf, 0xa6, 0x8a, 0xf3, 0x08,
-	0x86, 0x84, 0xf3, 0x5c, 0x22, 0x18, 0x0f, 0x7a, 0x0f, 0xd2, 0x23, 0xcf, 0x92, 0x76, 0x56, 0xaa,
-	0x57, 0xf9, 0x3d, 0xf9, 0xcf, 0xcf, 0x37, 0x0b, 0x5e, 0x50, 0xd9, 0xb6, 0x1d, 0xba, 0xe7, 0x59,
-	0x14, 0x0b, 0x80, 0x7e, 0x0c, 0x69, 0x7e, 0x54, 0xa0, 0x37, 0x21, 0x5d, 0x6d, 0xb6, 0xea, 0xda,
-	0x52, 0xe9, 0xca, 0xe9, 0x59, 0x79, 0x45, 0x4c, 0x09, 0xaf, 0xe0, 0xbe, 0x8b, 0x36, 0x21, 0xfb,
-	0xb8, 0xbd, 0xbb, 0xbf, 0xc7, 0xdd, 0xeb, 0xea, 0xe9, 0x59, 0x79, 0x2d, 0xaa, 0x96, 0x93, 0x86,
-	0xde, 0x82, 0x4c, 0x6f, 0xaf, 0xb3, 0xdd, 0xd5, 0x92, 0x25, 0x74, 0x7a, 0x56, 0x5e, 0x8d, 0xea,
-	0x45, 0x9f, 0x4b, 0x57, 0xd4, 0xaa, 0xe6, 0x23, 0xb9, 0xfe, 0x5f, 0x49, 0x58, 0xc1, 0x9c, 0xf3,
-	0xfa, 0xac, 0xe3, 0x39, 0xb6, 0x39, 0x45, 0x1d, 0xc8, 0x9b, 0x9e, 0x6b, 0xd9, 0xb1, 0x3d, 0xb5,
-	0xf5, 0x92, 0x8b, 0x71, 0xa6, 0x15, 0x96, 0x6a, 0xa1, 0x26, 0x9e, 0x19, 0x41, 0x5b, 0x90, 0xb1,
-	0xa8, 0x43, 0xa6, 0x97, 0xdd, 0xd0, 0x75, 0xc5, 0xaf, 0xb1, 0x84, 0x0a, 0x36, 0x49, 0x9e, 0xf6,
-	0x09, 0x63, 0x74, 0x34, 0x66, 0xf2, 0x86, 0x4e, 0xe3, 0xc2, 0x88, 0x3c, 0x35, 0x94, 0x08, 0x7d,
-	0x02, 0xd9, 0x13, 0xdb, 0xb5, 0xbc, 0x13, 0x75, 0x09, 0x5f, 0x6e, 0x57, 0x61, 0xf5, 0x53, 0x7e,
-	0xf7, 0x5e, 0xe8, 0x2c, 0x9f, 0xf5, 0x56, 0xbb, 0xd5, 0x08, 0x67, 0x5d, 0xd5, 0xb7, 0xdd, 0x96,
-	0xe7, 0xf2, 0x1d, 0x03, 0xed, 0x56, 0x7f, 0xdb, 0x68, 0xee, 0xee, 0x63, 0x3e, 0xf3, 0xd7, 0x4e,
-	0xcf, 0xca, 0x5a, 0x04, 0xd9, 0x26, 0xb6, 0xc3, 0x89, 0xe1, 0x0d, 0x48, 0x19, 0xad, 0x2f, 0xb5,
-	0x64, 0x49, 0x3b, 0x3d, 0x2b, 0x17, 0xa3, 0x6a, 0xc3, 0x9d, 0xce, 0x36, 0xd3, 0xc5, 0x76, 0xf5,
-	0x7f, 0x4d, 0x42, 0x71, 0x7f, 0x6c, 0x11, 0x46, 0xa5, 0x67, 0xa2, 0x32, 0x14, 0xc6, 0xc4, 0x27,
-	0x8e, 0x43, 0x1d, 0x3b, 0x18, 0xa9, 0xe0, 0x21, 0x2e, 0x42, 0xf7, 0xbe, 0xc3, 0x64, 0x2a, 0x62,
-	0xa6, 0xa6, 0x74, 0x1f, 0x56, 0x0f, 0x65, 0x67, 0xfb, 0xc4, 0x14, 0xab, 0x9b, 0x12, 0xab, 0x5b,
-	0x59, 0x64, 0x22, 0xde, 0xab, 0x8a, 0x1a, 0xa3, 0x21, 0xb4, 0xf0, 0xca, 0x61, 0xbc, 0x88, 0x3e,
-	0x83, 0xe5, 0x91, 0xe7, 0xda, 0xcc, 0xf3, 0x5f, 0x6b, 0x1d, 0x42, 0x30, 0xba, 0x0d, 0x57, 0xf8,
-	0x0a, 0x87, 0x5d, 0x12, 0xd5, 0xe2, 0xe6, 0x4a, 0xe2, 0xb5, 0x11, 0x79, 0xaa, 0xda, 0xc4, 0x5c,
-	0xac, 0x7f, 0x06, 0x2b, 0x73, 0x7d, 0xe0, 0xb7, 0x79, 0xc7, 0xd8, 0xef, 0x36, 0xb4, 0x25, 0x54,
-	0x84, 0x5c, 0xad, 0xdd, 0xea, 0x35, 0x5b, 0xfb, 0x9c, 0x8e, 0x14, 0x21, 0x87, 0xdb, 0xbb, 0xbb,
-	0x55, 0xa3, 0xf6, 0x48, 0x4b, 0xea, 0xbf, 0x88, 0xe6, 0x57, 0xf1, 0x91, 0xea, 0x3c, 0x1f, 0xf9,
-	0xf0, 0xe5, 0x43, 0x57, 0x8c, 0x64, 0x56, 0x88, 0x78, 0xc9, 0xff, 0x07, 0x10, 0xcb, 0x48, 0xad,
-	0x3e, 0x61, 0x97, 0xc5, 0x1c, 0xbd, 0x30, 0x9a, 0xc4, 0x79, 0xa5, 0x60, 0x30, 0xf4, 0x05, 0x14,
-	0x4d, 0x6f, 0x34, 0x76, 0xa8, 0xd2, 0x4f, 0xbd, 0x8e, 0x7e, 0x21, 0x52, 0x31, 0x58, 0x9c, 0x17,
-	0xa5, 0xe7, 0x99, 0xdb, 0x1f, 0x24, 0xa0, 0x10, 0xeb, 0xf0, 0x3c, 0x15, 0x2a, 0x42, 0x6e, 0xbf,
-	0x53, 0x37, 0x7a, 0xcd, 0xd6, 0x43, 0x2d, 0x81, 0x00, 0xb2, 0x62, 0x02, 0xeb, 0x5a, 0x92, 0x53,
-	0xb8, 0x5a, 0x7b, 0xaf, 0xb3, 0xdb, 0x10, 0x64, 0x08, 0x5d, 0x03, 0x2d, 0x9c, 0xc2, 0x7e, 0xb7,
-	0x67, 0x60, 0x2e, 0x4d, 0xa3, 0xab, 0xb0, 0x16, 0x49, 0x95, 0x66, 0x06, 0x5d, 0x07, 0x14, 0x09,
-	0x67, 0x26, 0xb2, 0xfa, 0xef, 0xc2, 0x5a, 0xcd, 0x73, 0x19, 0xb1, 0xdd, 0x88, 0xde, 0x6e, 0xf1,
-	0x71, 0x2b, 0x51, 0xdf, 0xb6, 0xe4, 0x69, 0x5b, 0x5d, 0x3b, 0x7f, 0xbe, 0x59, 0x88, 0xa0, 0xcd,
-	0x3a, 0x1f, 0x69, 0x58, 0xb0, 0xf8, 0x9e, 0x1a, 0xdb, 0x96, 0x98, 0xe2, 0x4c, 0x75, 0xf9, 0xfc,
-	0xf9, 0x66, 0xaa, 0xd3, 0xac, 0x63, 0x2e, 0x43, 0x6f, 0x42, 0x9e, 0x3e, 0xb5, 0x59, 0xdf, 0xe4,
-	0xa7, 0x2b, 0x9f, 0xc3, 0x0c, 0xce, 0x71, 0x41, 0x8d, 0x1f, 0xa6, 0x55, 0x80, 0x8e, 0xe7, 0x33,
-	0xd5, 0xf2, 0x27, 0x90, 0x19, 0x7b, 0xbe, 0x88, 0x2d, 0xf9, 0xd5, 0xb3, 0x90, 0xac, 0x71, 0xb8,
-	0x74, 0x76, 0x2c, 0xc1, 0xfa, 0xdf, 0x25, 0x01, 0x7a, 0x24, 0x38, 0x52, 0x46, 0xee, 0x43, 0x3e,
-	0x4a, 0x0e, 0x5c, 0x16, 0xa4, 0xc6, 0xd6, 0x3c, 0xc2, 0xa3, 0x8f, 0x43, 0xaf, 0x93, 0xdc, 0x7d,
-	0xb1, 0xa2, 0x6a, 0x6b, 0x11, 0xfd, 0x9d, 0x27, 0xe8, 0xfc, 0xbe, 0xa2, 0xbe, 0xaf, 0x16, 0x9f,
-	0x7f, 0xa2, 0x9a, 0x38, 0xb3, 0xe5, 0xbc, 0x29, 0xf6, 0xf7, 0xce, 0xa2, 0x46, 0x2e, 0x2c, 0xca,
-	0xce, 0x12, 0x9e, 0xe9, 0xa1, 0x07, 0x50, 0xe0, 0x43, 0xef, 0x07, 0xa2, 0x4e, 0x11, 0xbf, 0x97,
-	0xce, 0x96, 0xb4, 0x80, 0x61, 0x1c, 0x7d, 0x57, 0x35, 0x58, 0xf5, 0x27, 0x2e, 0x1f, 0xb6, 0xb2,
-	0xa1, 0xdb, 0xf0, 0x46, 0x8b, 0xb2, 0x13, 0xcf, 0x3f, 0x32, 0x18, 0x23, 0xe6, 0x90, 0x47, 0xfb,
-	0xea, 0xa4, 0x9b, 0xb1, 0xde, 0xc4, 0x1c, 0xeb, 0x5d, 0x87, 0x65, 0xe2, 0xd8, 0x24, 0xa0, 0x92,
-	0x2a, 0xe4, 0x71, 0x58, 0xe4, 0xdc, 0x9c, 0x33, 0x7d, 0x1a, 0x04, 0x54, 0xc6, 0xa7, 0x79, 0x3c,
-	0x13, 0xe8, 0xff, 0x98, 0x04, 0x68, 0x76, 0x8c, 0x3d, 0x65, 0xbe, 0x0e, 0xd9, 0x43, 0x32, 0xb2,
-	0x9d, 0xe9, 0x65, 0x3b, 0x7d, 0x86, 0xaf, 0x18, 0xd2, 0xd0, 0xb6, 0xd0, 0xc1, 0x4a, 0x57, 0x50,
-	0xf6, 0xc9, 0x81, 0x4b, 0x59, 0x44, 0xd9, 0x45, 0x89, 0xf3, 0x03, 0x9f, 0xb8, 0xd1, 0xca, 0xc8,
-	0x02, 0xef, 0xfa, 0x80, 0x30, 0x7a, 0x42, 0xa6, 0xe1, 0xc6, 0x54, 0x45, 0xb4, 0xc3, 0xa9, 0x7c,
-	0x40, 0xfd, 0x63, 0x6a, 0xad, 0x67, 0x84, 0x17, 0xbe, 0xaa, 0x3f, 0x58, 0xc1, 0x25, 0xf3, 0x89,
-	0xb4, 0x4b, 0xf7, 0xc5, 0x75, 0x3d, 0xab, 0xfa, 0x4e, 0xd1, 0xf5, 0x1d, 0x58, 0x99, 0x1b, 0xe7,
-	0x0b, 0xb1, 0x52, 0xb3, 0xf3, 0xf8, 0x13, 0x2d, 0xad, 0xbe, 0x3e, 0xd3, 0xb2, 0xfa, 0x5f, 0xa5,
-	0xe4, 0x56, 0x52, 0xb3, 0xba, 0x38, 0x5f, 0x95, 0x13, 0xd9, 0x2f, 0xd3, 0x73, 0x94, 0x7f, 0xbf,
-	0x77, 0xf9, 0x0e, 0xe3, 0xdc, 0x5b, 0xc0, 0x71, 0xa4, 0x88, 0x36, 0xa1, 0x20, 0xd7, 0xbf, 0xcf,
-	0xfd, 0x49, 0x4c, 0xeb, 0x0a, 0x06, 0x29, 0xe2, 0x9a, 0xe8, 0x16, 0xac, 0x8e, 0x27, 0x07, 0x8e,
-	0x1d, 0x0c, 0xa9, 0x25, 0x31, 0x69, 0x81, 0x59, 0x89, 0xa4, 0x02, 0xb6, 0x07, 0x45, 0x25, 0xe8,
-	0x0b, 0xde, 0x95, 0x11, 0x1d, 0xba, 0xfd, 0xaa, 0x0e, 0x49, 0x15, 0x41, 0xc7, 0x0a, 0xe3, 0x59,
-	0x41, 0xaf, 0x43, 0x2e, 0xec, 0x2c, 0x5a, 0x87, 0x54, 0xaf, 0xd6, 0xd1, 0x96, 0x4a, 0x6b, 0xa7,
-	0x67, 0xe5, 0x42, 0x28, 0xee, 0xd5, 0x3a, 0xbc, 0x66, 0xbf, 0xde, 0xd1, 0x12, 0xf3, 0x35, 0xfb,
-	0xf5, 0x4e, 0x29, 0xcd, 0x6f, 0x7e, 0xfd, 0x10, 0x0a, 0xb1, 0x16, 0xd0, 0x3b, 0xb0, 0xdc, 0x6c,
-	0x3d, 0xc4, 0x8d, 0x6e, 0x57, 0x5b, 0x2a, 0x5d, 0x3f, 0x3d, 0x2b, 0xa3, 0x58, 0x6d, 0xd3, 0x1d,
-	0xf0, 0xf5, 0x41, 0x6f, 0x41, 0x7a, 0xa7, 0xdd, 0xed, 0x85, 0x44, 0x2f, 0x86, 0xd8, 0xf1, 0x02,
-	0x56, 0xba, 0xaa, 0x28, 0x45, 0xdc, 0xb0, 0xfe, 0x27, 0x09, 0xc8, 0x4a, 0xbe, 0xbb, 0x70, 0xa1,
-	0x0c, 0x58, 0x0e, 0xa3, 0x30, 0x49, 0xc2, 0xdf, 0x7b, 0x39, 0x61, 0xae, 0x28, 0x7e, 0x2b, 0xdd,
-	0x2f, 0xd4, 0x2b, 0x7d, 0x0e, 0xc5, 0x78, 0xc5, 0x77, 0x72, 0xbe, 0xdf, 0x81, 0x02, 0xf7, 0xef,
-	0x90, 0x38, 0x6f, 0x41, 0x56, 0x72, 0x72, 0x75, 0x9a, 0x5e, 0xc6, 0xde, 0x15, 0x12, 0xdd, 0x83,
-	0x65, 0xc9, 0xf8, 0xc3, 0xfc, 0xd4, 0xc6, 0xe5, 0xbb, 0x08, 0x87, 0x70, 0xfd, 0x01, 0xa4, 0x3b,
-	0x94, 0xfa, 0x7c, 0xee, 0x5d, 0xcf, 0xa2, 0xb3, 0x0b, 0x48, 0x05, 0x2b, 0x16, 0x6d, 0xd6, 0x79,
-	0xb0, 0x62, 0xd1, 0xa6, 0x15, 0xa5, 0x17, 0x92, 0xb1, 0xf4, 0x42, 0x0f, 0x8a, 0x4f, 0xa8, 0x3d,
-	0x18, 0x32, 0x6a, 0x09, 0x43, 0x1f, 0x42, 0x7a, 0x4c, 0xa3, 0xce, 0xaf, 0x2f, 0x74, 0x30, 0x4a,
-	0x7d, 0x2c, 0x50, 0xfc, 0x1c, 0x39, 0x11, 0xda, 0x2a, 0x2b, 0xaa, 0x4a, 0xfa, 0x3f, 0x24, 0x61,
-	0xb5, 0x19, 0x04, 0x13, 0xe2, 0x9a, 0x21, 0x43, 0xf9, 0xd1, 0x3c, 0x43, 0x79, 0x7f, 0xe1, 0x08,
-	0xe7, 0x54, 0xe6, 0xb3, 0x26, 0xea, 0x72, 0x48, 0x46, 0x97, 0x83, 0xfe, 0x1f, 0x89, 0x30, 0x35,
-	0x72, 0x2b, 0xb6, 0xdd, 0x4b, 0xeb, 0xa7, 0x67, 0xe5, 0x6b, 0x71, 0x4b, 0x74, 0xdf, 0x3d, 0x72,
-	0xbd, 0x13, 0x17, 0xbd, 0x0d, 0x19, 0xdc, 0x68, 0x35, 0x9e, 0x68, 0x09, 0xe9, 0x9e, 0x73, 0x20,
-	0x4c, 0x5d, 0x7a, 0xc2, 0x2d, 0x75, 0x1a, 0xad, 0x3a, 0xe7, 0x12, 0xc9, 0x05, 0x96, 0x3a, 0xd4,
-	0xb5, 0x6c, 0x77, 0x80, 0xde, 0x81, 0x6c, 0xb3, 0xdb, 0xdd, 0x17, 0xc1, 0xeb, 0x1b, 0xa7, 0x67,
-	0xe5, 0xab, 0x73, 0x28, 0x5e, 0xa0, 0x16, 0x07, 0x71, 0x72, 0xcd, 0x59, 0xc6, 0x02, 0x10, 0xe7,
-	0x7d, 0x12, 0x84, 0xdb, 0x3d, 0x1e, 0x59, 0x67, 0x16, 0x80, 0xb0, 0xc7, 0x7f, 0xd5, 0x76, 0xfb,
-	0x97, 0x24, 0x68, 0x86, 0x69, 0xd2, 0x31, 0xe3, 0xf5, 0x2a, 0xaa, 0xe9, 0x41, 0x6e, 0xcc, 0xbf,
-	0x6c, 0x1a, 0xf2, 0x80, 0x7b, 0x0b, 0xf3, 0xea, 0x17, 0xf4, 0x2a, 0xd8, 0x73, 0xa8, 0x61, 0x8d,
-	0xec, 0x20, 0xe0, 0xd1, 0xbb, 0x90, 0xe1, 0xc8, 0x52, 0xe9, 0xe7, 0x09, 0xb8, 0xba, 0x00, 0x81,
-	0xee, 0x40, 0xda, 0xf7, 0x9c, 0x70, 0x0d, 0x6f, 0xbe, 0x2c, 0xeb, 0xc5, 0x55, 0xb1, 0x40, 0xa2,
-	0x0d, 0x00, 0x32, 0x61, 0x1e, 0x11, 0xed, 0x8b, 0xd5, 0xcb, 0xe1, 0x98, 0x04, 0x3d, 0x81, 0x6c,
-	0x40, 0x4d, 0x9f, 0x86, 0x84, 0xf1, 0xc1, 0xff, 0xb5, 0xf7, 0x95, 0xae, 0x30, 0x83, 0x95, 0xb9,
-	0x52, 0x05, 0xb2, 0x52, 0xc2, 0xdd, 0xde, 0x22, 0x8c, 0x88, 0x4e, 0x17, 0xb1, 0xf8, 0xe6, 0xde,
-	0x44, 0x9c, 0x41, 0xe8, 0x4d, 0xc4, 0x19, 0xe8, 0x7f, 0x9a, 0x04, 0x68, 0x3c, 0x65, 0xd4, 0x77,
-	0x89, 0x53, 0x33, 0x50, 0x23, 0x76, 0xfa, 0xcb, 0xd1, 0x7e, 0xb0, 0x30, 0x17, 0x1a, 0x69, 0x54,
-	0x6a, 0xc6, 0x82, 0xf3, 0xff, 0x06, 0xa4, 0x26, 0xbe, 0xa3, 0xf2, 0xea, 0x82, 0xe9, 0xed, 0xe3,
-	0x5d, 0xcc, 0x65, 0xa8, 0x31, 0x3b, 0xb6, 0x52, 0x2f, 0x7f, 0x10, 0x89, 0x35, 0xf0, 0xab, 0x3f,
-	0xba, 0x3e, 0x04, 0x98, 0xf5, 0x1a, 0x6d, 0x40, 0xa6, 0xb6, 0xdd, 0xed, 0xee, 0x6a, 0x4b, 0xf2,
-	0x6c, 0x9e, 0x55, 0x09, 0xb1, 0xfe, 0x97, 0x09, 0xc8, 0xd5, 0x0c, 0x75, 0x63, 0x6e, 0x83, 0x26,
-	0x0e, 0x1c, 0x93, 0xfa, 0xac, 0x4f, 0x9f, 0x8e, 0x6d, 0x7f, 0xaa, 0xce, 0x8c, 0xcb, 0xc3, 0xa4,
-	0x55, 0xae, 0x55, 0xa3, 0x3e, 0x6b, 0x08, 0x1d, 0x84, 0xa1, 0x48, 0xd5, 0x10, 0xfb, 0x26, 0x09,
-	0x4f, 0xf0, 0x8d, 0xcb, 0xa7, 0x42, 0xd2, 0xeb, 0x59, 0x39, 0xc0, 0x85, 0xd0, 0x48, 0x8d, 0x04,
-	0xfa, 0x63, 0xb8, 0xda, 0xf6, 0xcd, 0x21, 0x0d, 0x98, 0x6c, 0x54, 0x75, 0xf9, 0x01, 0xdc, 0x64,
-	0x24, 0x38, 0xea, 0x0f, 0xed, 0x80, 0x79, 0xfe, 0xb4, 0xef, 0x53, 0x46, 0x5d, 0x5e, 0xdf, 0x17,
-	0xcf, 0x2e, 0x2a, 0xc9, 0x71, 0x83, 0x63, 0x76, 0x24, 0x04, 0x87, 0x88, 0x5d, 0x0e, 0xd0, 0x9b,
-	0x50, 0xe4, 0x6c, 0xb6, 0x4e, 0x0f, 0xc9, 0xc4, 0x61, 0x01, 0xfa, 0x21, 0x80, 0xe3, 0x0d, 0xfa,
-	0xaf, 0x7d, 0xdc, 0xe7, 0x1d, 0x6f, 0x20, 0x3f, 0xf5, 0xdf, 0x00, 0xad, 0x6e, 0x07, 0x63, 0xc2,
-	0xcc, 0x61, 0x98, 0xbd, 0x41, 0x0f, 0x41, 0x1b, 0x52, 0xe2, 0xb3, 0x03, 0x4a, 0x58, 0x7f, 0x4c,
-	0x7d, 0xdb, 0xb3, 0x5e, 0x6b, 0x4a, 0xd7, 0x22, 0xad, 0x8e, 0x50, 0xd2, 0xff, 0x33, 0x01, 0x80,
-	0xc9, 0x61, 0x48, 0x6e, 0x7e, 0x00, 0x57, 0x02, 0x97, 0x8c, 0x83, 0xa1, 0xc7, 0xfa, 0xb6, 0xcb,
-	0xa8, 0x7f, 0x4c, 0x1c, 0x15, 0x81, 0x6b, 0x61, 0x45, 0x53, 0xc9, 0xd1, 0x87, 0x80, 0x8e, 0x28,
-	0x1d, 0xf7, 0x3d, 0xc7, 0xea, 0x87, 0x95, 0xf2, 0x5d, 0x28, 0x8d, 0x35, 0x5e, 0xd3, 0x76, 0xac,
-	0x6e, 0x28, 0x47, 0x55, 0xd8, 0xe0, 0x33, 0x40, 0x5d, 0xe6, 0xdb, 0x34, 0xe8, 0x1f, 0x7a, 0x7e,
-	0x3f, 0x70, 0xbc, 0x93, 0xfe, 0xa1, 0xe7, 0x38, 0xde, 0x09, 0xf5, 0xc3, 0xfc, 0x46, 0xc9, 0xf1,
-	0x06, 0x0d, 0x09, 0xda, 0xf6, 0xfc, 0xae, 0xe3, 0x9d, 0x6c, 0x87, 0x08, 0xce, 0x80, 0x66, 0xc3,
-	0x66, 0xb6, 0x79, 0x14, 0x32, 0xa0, 0x48, 0xda, 0xb3, 0xcd, 0x23, 0xf4, 0x0e, 0xac, 0x50, 0x87,
-	0x8a, 0x28, 0x59, 0xa2, 0x32, 0x02, 0x55, 0x0c, 0x85, 0x1c, 0xa4, 0xff, 0x3f, 0xc8, 0x77, 0x1c,
-	0x62, 0x8a, 0xd7, 0x37, 0x54, 0x06, 0x1e, 0x74, 0x71, 0x27, 0xb0, 0x5d, 0x15, 0x25, 0xe5, 0x71,
-	0x5c, 0xa4, 0xff, 0x08, 0xe0, 0xc7, 0x9e, 0xed, 0xf6, 0xbc, 0x23, 0xea, 0x8a, 0x87, 0x0a, 0xce,
-	0xe8, 0xd5, 0x52, 0xe6, 0xb1, 0x2a, 0x89, 0x80, 0x85, 0xb8, 0x64, 0x40, 0xfd, 0x28, 0x5f, 0x2f,
-	0x8b, 0xfa, 0x37, 0x09, 0xc8, 0x62, 0xcf, 0x63, 0x35, 0x03, 0x95, 0x21, 0x6b, 0x92, 0x7e, 0xb8,
-	0xf3, 0x8a, 0xd5, 0xfc, 0xf9, 0xf3, 0xcd, 0x4c, 0xcd, 0x78, 0x44, 0xa7, 0x38, 0x63, 0x92, 0x47,
-	0x74, 0xca, 0xaf, 0x68, 0x93, 0x88, 0xfd, 0x22, 0xcc, 0x14, 0xe5, 0x15, 0x5d, 0x33, 0xf8, 0x66,
-	0xc0, 0x59, 0x93, 0xf0, 0x7f, 0x74, 0x07, 0x8a, 0x0a, 0xd4, 0x1f, 0x92, 0x60, 0x28, 0x79, 0x78,
-	0x75, 0xf5, 0xfc, 0xf9, 0x26, 0x48, 0xe4, 0x0e, 0x09, 0x86, 0x18, 0x24, 0x9a, 0x7f, 0xa3, 0x06,
-	0x14, 0xbe, 0xf2, 0x6c, 0xb7, 0xcf, 0xc4, 0x20, 0x54, 0xaa, 0x62, 0xe1, 0xfe, 0x99, 0x0d, 0x55,
-	0xe5, 0x4f, 0xe0, 0xab, 0x48, 0xa2, 0xff, 0x53, 0x02, 0x0a, 0xdc, 0xa6, 0x7d, 0x68, 0x9b, 0xfc,
-	0x4a, 0xfd, 0xee, 0x27, 0xfd, 0x0d, 0x48, 0x99, 0x81, 0xaf, 0xc6, 0x26, 0x8e, 0xba, 0x5a, 0x17,
-	0x63, 0x2e, 0x43, 0x5f, 0x40, 0x56, 0x05, 0x5f, 0xf2, 0x90, 0xd7, 0x5f, 0x7d, 0xf9, 0xab, 0x2e,
-	0x2a, 0x3d, 0xb1, 0x96, 0xb3, 0xde, 0x89, 0x51, 0x16, 0x71, 0x5c, 0x84, 0xae, 0x43, 0xd2, 0x74,
-	0x85, 0x53, 0xa8, 0x07, 0xcc, 0x5a, 0x0b, 0x27, 0x4d, 0x57, 0xff, 0xfb, 0x04, 0xac, 0x34, 0x5c,
-	0xd3, 0x9f, 0x8a, 0x43, 0x92, 0x2f, 0xc4, 0x4d, 0xc8, 0x07, 0x93, 0x83, 0x60, 0x1a, 0x30, 0x3a,
-	0x0a, 0xdf, 0x42, 0x22, 0x01, 0x6a, 0x42, 0x9e, 0x38, 0x03, 0xcf, 0xb7, 0xd9, 0x70, 0xa4, 0x78,
-	0xff, 0xe2, 0x83, 0x39, 0x6e, 0xb3, 0x62, 0x84, 0x2a, 0x78, 0xa6, 0x1d, 0x1e, 0xc5, 0x29, 0xd1,
-	0x59, 0x71, 0x14, 0xbf, 0x0d, 0x45, 0x87, 0x8c, 0x44, 0x34, 0xca, 0xc3, 0x49, 0x31, 0x8e, 0x34,
-	0x2e, 0x28, 0x19, 0x8f, 0xb1, 0x75, 0x1d, 0xf2, 0x91, 0x31, 0xb4, 0x06, 0x05, 0xa3, 0xd1, 0xed,
-	0xdf, 0xdd, 0xba, 0xd7, 0x7f, 0x58, 0xdb, 0xd3, 0x96, 0x14, 0x13, 0xf8, 0x9b, 0x04, 0xac, 0xec,
-	0x49, 0x1f, 0x54, 0xec, 0xea, 0x1d, 0x58, 0xf6, 0xc9, 0x21, 0x0b, 0xf9, 0x5f, 0x5a, 0x3a, 0x17,
-	0x3f, 0x04, 0x38, 0xff, 0xe3, 0x55, 0x8b, 0xf9, 0x5f, 0xec, 0x75, 0x2e, 0x75, 0xe9, 0xeb, 0x5c,
-	0xfa, 0x57, 0xf2, 0x3a, 0xa7, 0xff, 0x24, 0x09, 0x6b, 0xea, 0xa2, 0x0e, 0x5f, 0x9f, 0xd0, 0x07,
-	0x90, 0x97, 0x77, 0xf6, 0x8c, 0xbd, 0x8a, 0x07, 0x21, 0x89, 0x6b, 0xd6, 0x71, 0x4e, 0x56, 0x37,
-	0x2d, 0x1e, 0x4e, 0x29, 0x68, 0xec, 0xad, 0x19, 0xa4, 0xa8, 0xc5, 0x63, 0x81, 0x3a, 0xa4, 0x0f,
-	0x6d, 0x87, 0x2a, 0x3f, 0x5b, 0x98, 0x01, 0xbc, 0xd0, 0xbc, 0x48, 0x58, 0xf7, 0x44, 0x40, 0xb6,
-	0xb3, 0x84, 0x85, 0x76, 0xe9, 0xf7, 0x00, 0x66, 0xd2, 0x85, 0x31, 0x07, 0xbf, 0xd7, 0x55, 0x06,
-	0x27, 0xbc, 0xd7, 0x9b, 0x75, 0xcc, 0x65, 0xbc, 0x6a, 0x60, 0x5b, 0x6a, 0xe7, 0x8a, 0xaa, 0x87,
-	0xbc, 0x6a, 0x60, 0x5b, 0x51, 0xd6, 0x3c, 0xfd, 0x8a, 0xac, 0x79, 0x35, 0x17, 0x26, 0x11, 0xf4,
-	0x36, 0x5c, 0xaf, 0x3a, 0xc4, 0x3c, 0x72, 0xec, 0x80, 0x51, 0x2b, 0xbe, 0x43, 0x3f, 0x85, 0xec,
-	0xdc, 0xbd, 0xfb, 0x8a, 0xb4, 0x8d, 0x02, 0xeb, 0x3f, 0x49, 0x40, 0x71, 0x87, 0x12, 0x87, 0x0d,
-	0x67, 0xb1, 0x2f, 0xa3, 0x01, 0x53, 0xe7, 0xa3, 0xf8, 0x46, 0xf7, 0x20, 0x17, 0xdd, 0x14, 0xaf,
-	0x93, 0xdc, 0x8e, 0xd0, 0xe8, 0x33, 0x58, 0xe6, 0x9e, 0xed, 0x4d, 0x42, 0x42, 0xf7, 0x8a, 0xac,
-	0xa9, 0x02, 0xf3, 0x43, 0xd6, 0xa7, 0xe2, 0x82, 0x10, 0xb3, 0x93, 0xc1, 0x61, 0x51, 0xff, 0x9f,
-	0x04, 0x5c, 0xdb, 0x23, 0xd3, 0x03, 0xaa, 0x76, 0x1c, 0xb5, 0x30, 0x35, 0x3d, 0xdf, 0x42, 0x9d,
-	0xf8, 0x4e, 0xbd, 0x24, 0xa1, 0xbf, 0x48, 0x79, 0xf1, 0x86, 0x0d, 0x99, 0x62, 0x32, 0xc6, 0x14,
-	0xaf, 0x41, 0xc6, 0xf5, 0x5c, 0x93, 0xaa, 0x6d, 0x2c, 0x0b, 0xba, 0x1d, 0xdf, 0xa5, 0xa5, 0x28,
-	0xcb, 0x2e, 0x72, 0xe4, 0x2d, 0x8f, 0x45, 0xad, 0xa1, 0x2f, 0xa0, 0xd4, 0x6d, 0xd4, 0x70, 0xa3,
-	0x57, 0x6d, 0xff, 0x7a, 0xbf, 0x6b, 0xec, 0x76, 0x8d, 0xad, 0x3b, 0xfd, 0x4e, 0x7b, 0xf7, 0xcb,
-	0xbb, 0x1f, 0xdf, 0xf9, 0x54, 0x4b, 0x94, 0xca, 0xa7, 0x67, 0xe5, 0x9b, 0x2d, 0xa3, 0xb6, 0x2b,
-	0xdd, 0xf2, 0xc0, 0x7b, 0xda, 0x25, 0x4e, 0x40, 0xb6, 0xee, 0x74, 0x3c, 0x67, 0xca, 0x31, 0xb7,
-	0x7f, 0x91, 0x82, 0x7c, 0x94, 0x44, 0xe3, 0xde, 0xc5, 0x23, 0x18, 0xd5, 0x54, 0x24, 0x6f, 0xd1,
-	0x13, 0xf4, 0xf6, 0x2c, 0x76, 0xf9, 0x42, 0x26, 0xf3, 0xa3, 0xea, 0x30, 0x6e, 0x79, 0x17, 0x72,
-	0x46, 0xb7, 0xdb, 0x7c, 0xd8, 0x6a, 0xd4, 0xb5, 0xaf, 0x13, 0xa5, 0xef, 0x9d, 0x9e, 0x95, 0xaf,
-	0x44, 0x20, 0x23, 0x08, 0xec, 0x81, 0x4b, 0x2d, 0x81, 0xaa, 0xd5, 0x1a, 0x9d, 0x5e, 0xa3, 0xae,
-	0x3d, 0x4b, 0x5e, 0x44, 0x09, 0x2e, 0x2e, 0x1e, 0xe6, 0xf2, 0x1d, 0xdc, 0xe8, 0x18, 0x98, 0x37,
-	0xf8, 0x75, 0x52, 0x86, 0x54, 0xb3, 0x16, 0x7d, 0x3a, 0x26, 0x3e, 0x6f, 0x73, 0x23, 0x7c, 0xa0,
-	0x7e, 0x96, 0x92, 0x8f, 0x37, 0xb3, 0x8c, 0x20, 0x25, 0xd6, 0x94, 0xb7, 0x26, 0xb2, 0xb1, 0xc2,
-	0x4c, 0xea, 0x42, 0x6b, 0x5d, 0x46, 0x7c, 0xc6, 0xad, 0xe8, 0xb0, 0x8c, 0xf7, 0x5b, 0x2d, 0x0e,
-	0x7a, 0x96, 0xbe, 0x30, 0x3a, 0x3c, 0x71, 0x5d, 0x8e, 0xb9, 0x05, 0xb9, 0x30, 0x59, 0xab, 0x7d,
-	0x9d, 0xbe, 0xd0, 0xa1, 0x5a, 0x98, 0x69, 0x16, 0x0d, 0xee, 0xec, 0xf7, 0xc4, 0xfb, 0xf9, 0xb3,
-	0xcc, 0xc5, 0x06, 0x87, 0x13, 0x66, 0xf1, 0x60, 0xb1, 0x1c, 0x45, 0x6f, 0x5f, 0x67, 0x24, 0x1f,
-	0x8e, 0x30, 0x2a, 0x74, 0x7b, 0x17, 0x72, 0xb8, 0xf1, 0x63, 0xf9, 0xd4, 0xfe, 0x2c, 0x7b, 0xc1,
-	0x0e, 0xa6, 0x5f, 0x51, 0x53, 0xb5, 0xd6, 0xc6, 0x9d, 0x1d, 0x43, 0x4c, 0xf9, 0x45, 0x54, 0xdb,
-	0x1f, 0x0f, 0x89, 0x4b, 0xad, 0xd9, 0x0b, 0x56, 0x54, 0x75, 0xfb, 0x37, 0x21, 0x17, 0xde, 0xb0,
-	0x68, 0x03, 0xb2, 0x4f, 0xda, 0xf8, 0x51, 0x03, 0x6b, 0x4b, 0x72, 0x0e, 0xc3, 0x9a, 0x27, 0x92,
-	0xa2, 0x94, 0x61, 0x79, 0xcf, 0x68, 0x19, 0x0f, 0x1b, 0x38, 0x4c, 0xac, 0x84, 0x00, 0x75, 0x4d,
-	0x94, 0x34, 0xd5, 0x40, 0x64, 0xb3, 0x7a, 0xf3, 0x9b, 0x6f, 0x37, 0x96, 0x7e, 0xf6, 0xed, 0xc6,
-	0xd2, 0xcf, 0xbf, 0xdd, 0x48, 0x3c, 0x3b, 0xdf, 0x48, 0x7c, 0x73, 0xbe, 0x91, 0xf8, 0xe9, 0xf9,
-	0x46, 0xe2, 0xdf, 0xce, 0x37, 0x12, 0x07, 0x59, 0x11, 0xc2, 0x7c, 0xfc, 0xbf, 0x01, 0x00, 0x00,
-	0xff, 0xff, 0x6b, 0x1c, 0x13, 0xe7, 0x66, 0x26, 0x00, 0x00,
+	0x76, 0xbf, 0xf8, 0x29, 0xf2, 0x91, 0x92, 0xda, 0x65, 0xaf, 0x47, 0xe6, 0x78, 0x24, 0x4e, 0x7b,
+	0xbc, 0xe3, 0xf1, 0xfa, 0xcf, 0xb1, 0x35, 0x1f, 0xf0, 0x8e, 0xff, 0x59, 0xbb, 0xf9, 0x21, 0x8b,
+	0x6b, 0x89, 0x24, 0x8a, 0x94, 0x9d, 0x41, 0x80, 0x10, 0xa5, 0xee, 0x12, 0xd5, 0xa3, 0x66, 0x37,
+	0xd3, 0x5d, 0x94, 0xcc, 0x04, 0x41, 0x8c, 0x1c, 0x92, 0x40, 0xa7, 0xdc, 0x03, 0x21, 0x08, 0x12,
+	0xe4, 0x90, 0xc3, 0x5e, 0x72, 0x08, 0x90, 0xd3, 0x20, 0xa7, 0x39, 0x6e, 0x12, 0x20, 0x58, 0x24,
+	0x88, 0x91, 0x51, 0xce, 0x01, 0xf6, 0xb2, 0xc8, 0x21, 0x09, 0x10, 0xd4, 0x47, 0x37, 0x9b, 0x32,
+	0x2d, 0x7b, 0xb2, 0x7b, 0x21, 0xbb, 0x5e, 0xfd, 0xde, 0xab, 0xaf, 0x57, 0x55, 0xbf, 0xf7, 0x0a,
+	0x0a, 0x6c, 0x32, 0xa2, 0x41, 0x65, 0xe4, 0x7b, 0xcc, 0x43, 0xc8, 0xf2, 0xcc, 0x43, 0xea, 0x57,
+	0x82, 0x63, 0xe2, 0x0f, 0x0f, 0x6d, 0x56, 0x39, 0xba, 0x57, 0xba, 0xc6, 0xec, 0x21, 0x0d, 0x18,
+	0x19, 0x8e, 0x3e, 0x8e, 0xbe, 0x24, 0xbc, 0xf4, 0x8e, 0x35, 0xf6, 0x09, 0xb3, 0x3d, 0xf7, 0xe3,
+	0xf0, 0x43, 0x55, 0x5c, 0x19, 0x78, 0x03, 0x4f, 0x7c, 0x7e, 0xcc, 0xbf, 0xa4, 0x54, 0x5f, 0x87,
+	0xc5, 0xa7, 0xd4, 0x0f, 0x6c, 0xcf, 0x45, 0x57, 0x20, 0x63, 0xbb, 0x16, 0x7d, 0xbe, 0x9a, 0x28,
+	0x27, 0x6e, 0xa5, 0xb1, 0x2c, 0xe8, 0x7f, 0x96, 0x80, 0x82, 0xe1, 0xba, 0x1e, 0x13, 0xb6, 0x02,
+	0x84, 0x20, 0xed, 0x92, 0x21, 0x15, 0xa0, 0x3c, 0x16, 0xdf, 0xa8, 0x06, 0x59, 0x87, 0xec, 0x51,
+	0x27, 0x58, 0x4d, 0x96, 0x53, 0xb7, 0x0a, 0x1b, 0x3f, 0xa8, 0xbc, 0xda, 0xe7, 0x4a, 0xcc, 0x48,
+	0x65, 0x5b, 0xa0, 0x1b, 0x2e, 0xf3, 0x27, 0x58, 0xa9, 0x96, 0x7e, 0x08, 0x85, 0x98, 0x18, 0x69,
+	0x90, 0x3a, 0xa4, 0x13, 0xd5, 0x0c, 0xff, 0xe4, 0xfd, 0x3b, 0x22, 0xce, 0x98, 0xae, 0x26, 0x85,
+	0x4c, 0x16, 0xbe, 0x48, 0xde, 0x4f, 0xe8, 0x5f, 0x42, 0x1e, 0xd3, 0xc0, 0x1b, 0xfb, 0x26, 0x0d,
+	0xd0, 0x47, 0x90, 0x77, 0x89, 0xeb, 0xf5, 0xcd, 0xd1, 0x38, 0x10, 0xea, 0xa9, 0x6a, 0xf1, 0xec,
+	0xe5, 0x7a, 0xae, 0x45, 0x5c, 0xaf, 0xd6, 0xd9, 0x0d, 0x70, 0x8e, 0x57, 0xd7, 0x46, 0xe3, 0x00,
+	0xbd, 0x0f, 0xc5, 0x21, 0x1d, 0x7a, 0xfe, 0xa4, 0xbf, 0x37, 0x61, 0x34, 0x10, 0x86, 0x53, 0xb8,
+	0x20, 0x65, 0x55, 0x2e, 0xd2, 0xff, 0x38, 0x01, 0x57, 0x42, 0xdb, 0x98, 0xfe, 0xd6, 0xd8, 0xf6,
+	0xe9, 0x90, 0xba, 0x2c, 0x40, 0x9f, 0x41, 0xd6, 0xb1, 0x87, 0x36, 0x93, 0x6d, 0x14, 0x36, 0xde,
+	0x9b, 0x37, 0xe6, 0xa8, 0x57, 0x58, 0x81, 0x91, 0x01, 0x45, 0x9f, 0x06, 0xd4, 0x3f, 0x92, 0x33,
+	0x21, 0x9a, 0x7c, 0xa3, 0xf2, 0x8c, 0x8a, 0xbe, 0x09, 0xb9, 0x8e, 0x43, 0xd8, 0xbe, 0xe7, 0x0f,
+	0x91, 0x0e, 0x45, 0xe2, 0x9b, 0x07, 0x36, 0xa3, 0x26, 0x1b, 0xfb, 0xe1, 0xaa, 0xcc, 0xc8, 0xd0,
+	0x55, 0x48, 0x7a, 0xb2, 0xa1, 0x7c, 0x35, 0x7b, 0xf6, 0x72, 0x3d, 0xd9, 0xee, 0xe2, 0xa4, 0x17,
+	0xe8, 0x0f, 0xe0, 0x52, 0xc7, 0x19, 0x0f, 0x6c, 0xb7, 0x4e, 0x03, 0xd3, 0xb7, 0x47, 0xdc, 0x3a,
+	0x5f, 0x5e, 0xee, 0x7c, 0xe1, 0xf2, 0xf2, 0xef, 0x68, 0xc9, 0x93, 0xd3, 0x25, 0xd7, 0xff, 0x30,
+	0x09, 0x97, 0x1a, 0xee, 0xc0, 0x76, 0x69, 0x5c, 0xfb, 0x26, 0x2c, 0x53, 0x21, 0xec, 0x1f, 0x49,
+	0xa7, 0x52, 0x76, 0x96, 0xa4, 0x34, 0xf4, 0xb4, 0xe6, 0x39, 0x7f, 0xb9, 0x37, 0x6f, 0xf8, 0xaf,
+	0x58, 0x9f, 0xe7, 0x35, 0xa8, 0x01, 0x8b, 0x23, 0x31, 0x88, 0x60, 0x35, 0x25, 0x6c, 0xdd, 0x9c,
+	0x67, 0xeb, 0x95, 0x71, 0x56, 0xd3, 0xdf, 0xbc, 0x5c, 0x5f, 0xc0, 0xa1, 0xee, 0x2f, 0xe3, 0x7c,
+	0xff, 0x9e, 0x80, 0x95, 0x96, 0x67, 0xcd, 0xcc, 0x43, 0x09, 0x72, 0x07, 0x5e, 0xc0, 0x62, 0x1b,
+	0x25, 0x2a, 0xa3, 0xfb, 0x90, 0x1b, 0xa9, 0xe5, 0x53, 0xab, 0x7f, 0x7d, 0x7e, 0x97, 0x25, 0x06,
+	0x47, 0x68, 0xf4, 0x00, 0xf2, 0x7e, 0xe8, 0x13, 0xab, 0xa9, 0xb7, 0x71, 0x9c, 0x29, 0x1e, 0xfd,
+	0x1a, 0x64, 0xe5, 0x22, 0xac, 0xa6, 0x85, 0xe6, 0xcd, 0xb7, 0x9a, 0x73, 0xac, 0x94, 0xf4, 0x9f,
+	0x25, 0x40, 0xc3, 0x64, 0x9f, 0xed, 0xd0, 0xe1, 0x1e, 0xf5, 0xbb, 0x8c, 0xb0, 0x71, 0x80, 0xae,
+	0x42, 0xd6, 0xa1, 0xc4, 0xa2, 0xbe, 0x18, 0x64, 0x0e, 0xab, 0x12, 0xda, 0xe5, 0x4e, 0x4e, 0xcc,
+	0x03, 0xb2, 0x67, 0x3b, 0x36, 0x9b, 0x88, 0x61, 0x2e, 0xcf, 0x5f, 0xe5, 0xf3, 0x36, 0x2b, 0x38,
+	0xa6, 0x88, 0x67, 0xcc, 0xa0, 0x55, 0x58, 0x1c, 0xd2, 0x20, 0x20, 0x03, 0x2a, 0x46, 0x9f, 0xc7,
+	0x61, 0x51, 0x7f, 0x00, 0xc5, 0xb8, 0x1e, 0x2a, 0xc0, 0xe2, 0x6e, 0xeb, 0x49, 0xab, 0xfd, 0xac,
+	0xa5, 0x2d, 0xa0, 0x15, 0x28, 0xec, 0xb6, 0x70, 0xc3, 0xa8, 0x6d, 0x19, 0xd5, 0xed, 0x86, 0x96,
+	0x40, 0x4b, 0x90, 0x9f, 0x16, 0x93, 0xfa, 0x5f, 0x27, 0x00, 0xf8, 0x02, 0xaa, 0x41, 0x7d, 0x01,
+	0x99, 0x80, 0x11, 0x26, 0x17, 0x6e, 0x79, 0xe3, 0x83, 0x79, 0xbd, 0x9e, 0xc2, 0x2b, 0xfc, 0x8f,
+	0x62, 0xa9, 0x12, 0xef, 0x61, 0x72, 0xa6, 0x87, 0x7c, 0x0f, 0x11, 0xcb, 0xf2, 0x55, 0xc7, 0xc5,
+	0xb7, 0xfe, 0x00, 0x32, 0x42, 0x7b, 0xb6, 0xbb, 0x39, 0x48, 0xd7, 0xf9, 0x57, 0x02, 0xe5, 0x21,
+	0x83, 0x1b, 0x46, 0xfd, 0x4b, 0x2d, 0x89, 0x34, 0x28, 0xd6, 0x9b, 0xdd, 0x5a, 0xbb, 0xd5, 0x6a,
+	0xd4, 0x7a, 0x8d, 0xba, 0x96, 0xd2, 0x6f, 0x42, 0xa6, 0x39, 0xe4, 0x96, 0xaf, 0x73, 0xaf, 0xd8,
+	0xa7, 0x3e, 0x75, 0xcd, 0xd0, 0xd9, 0xa6, 0x02, 0xfd, 0xa7, 0x79, 0xc8, 0xec, 0x78, 0x63, 0x97,
+	0xa1, 0x8d, 0xd8, 0xce, 0x5e, 0xde, 0x58, 0x9b, 0x37, 0x2c, 0x01, 0xac, 0xf4, 0x26, 0x23, 0xaa,
+	0x76, 0xfe, 0x55, 0xc8, 0x4a, 0xff, 0x51, 0xc3, 0x51, 0x25, 0x2e, 0x67, 0xc4, 0x1f, 0x50, 0xa6,
+	0xc6, 0xa3, 0x4a, 0xe8, 0x16, 0xe4, 0x7c, 0x4a, 0x2c, 0xcf, 0x75, 0x26, 0xc2, 0xcd, 0x72, 0xf2,
+	0xe8, 0xc5, 0x94, 0x58, 0x6d, 0xd7, 0x99, 0xe0, 0xa8, 0x16, 0x6d, 0x41, 0x71, 0xcf, 0x76, 0xad,
+	0xbe, 0x37, 0x92, 0xe7, 0x60, 0xe6, 0xf5, 0x4e, 0x29, 0x7b, 0x55, 0xb5, 0x5d, 0xab, 0x2d, 0xc1,
+	0xb8, 0xb0, 0x37, 0x2d, 0xa0, 0x16, 0x2c, 0x1f, 0x79, 0xce, 0x78, 0x48, 0x23, 0x5b, 0x59, 0x61,
+	0xeb, 0xc3, 0xd7, 0xdb, 0x7a, 0x2a, 0xf0, 0xa1, 0xb5, 0xa5, 0xa3, 0x78, 0x11, 0x3d, 0x81, 0x25,
+	0x36, 0x1c, 0xed, 0x07, 0x91, 0xb9, 0x45, 0x61, 0xee, 0xfb, 0x17, 0x4c, 0x18, 0x87, 0x87, 0xd6,
+	0x8a, 0x2c, 0x56, 0x2a, 0xfd, 0x7e, 0x0a, 0x0a, 0xb1, 0x9e, 0xa3, 0x2e, 0x14, 0x46, 0xbe, 0x37,
+	0x22, 0x03, 0x71, 0x96, 0xab, 0xb5, 0xb8, 0xf7, 0x56, 0xa3, 0xae, 0x74, 0xa6, 0x8a, 0x38, 0x6e,
+	0x45, 0x3f, 0x4d, 0x42, 0x21, 0x56, 0x89, 0x6e, 0x43, 0x0e, 0x77, 0x70, 0xf3, 0xa9, 0xd1, 0x6b,
+	0x68, 0x0b, 0xa5, 0xeb, 0x27, 0xa7, 0xe5, 0x55, 0x61, 0x2d, 0x6e, 0xa0, 0xe3, 0xdb, 0x47, 0xdc,
+	0xf5, 0x6e, 0xc1, 0x62, 0x08, 0x4d, 0x94, 0xde, 0x3d, 0x39, 0x2d, 0xbf, 0x73, 0x1e, 0x1a, 0x43,
+	0xe2, 0xee, 0x96, 0x81, 0x1b, 0x75, 0x2d, 0x39, 0x1f, 0x89, 0xbb, 0x07, 0xc4, 0xa7, 0x16, 0xfa,
+	0x3e, 0x64, 0x15, 0x30, 0x55, 0x2a, 0x9d, 0x9c, 0x96, 0xaf, 0x9e, 0x07, 0x4e, 0x71, 0xb8, 0xbb,
+	0x6d, 0x3c, 0x6d, 0x68, 0xe9, 0xf9, 0x38, 0xdc, 0x75, 0xc8, 0x11, 0x45, 0x1f, 0x40, 0x46, 0xc2,
+	0x32, 0xa5, 0x6b, 0x27, 0xa7, 0xe5, 0xef, 0xbd, 0x62, 0x8e, 0xa3, 0x4a, 0xab, 0x7f, 0xf4, 0xe7,
+	0x6b, 0x0b, 0x7f, 0xfb, 0x17, 0x6b, 0xda, 0xf9, 0xea, 0xd2, 0x7f, 0x27, 0x60, 0x69, 0x66, 0xc9,
+	0x91, 0x0e, 0x59, 0xd7, 0x33, 0xbd, 0x91, 0x3c, 0xe2, 0x73, 0x55, 0x38, 0x7b, 0xb9, 0x9e, 0x6d,
+	0x79, 0x35, 0x6f, 0x34, 0xc1, 0xaa, 0x06, 0x3d, 0x39, 0x77, 0x49, 0x7d, 0xf2, 0x96, 0xfe, 0x34,
+	0xf7, 0x9a, 0x7a, 0x08, 0x4b, 0x96, 0x6f, 0x1f, 0x51, 0xbf, 0x6f, 0x7a, 0xee, 0xbe, 0x3d, 0x50,
+	0xc7, 0x77, 0x69, 0x9e, 0xcd, 0xba, 0x00, 0xe2, 0xa2, 0x54, 0xa8, 0x09, 0xfc, 0x2f, 0x71, 0x41,
+	0x95, 0x9e, 0x42, 0x31, 0xee, 0xa1, 0xe8, 0x3d, 0x80, 0xc0, 0xfe, 0x6d, 0xaa, 0x38, 0x8f, 0x60,
+	0x48, 0x38, 0xcf, 0x25, 0x82, 0xf1, 0xa0, 0x0f, 0x21, 0x3d, 0xf4, 0x2c, 0x69, 0x67, 0xa9, 0x7a,
+	0x99, 0xdf, 0x93, 0xff, 0xfc, 0x72, 0xbd, 0xe0, 0x05, 0x95, 0x4d, 0xdb, 0xa1, 0x3b, 0x9e, 0x45,
+	0xb1, 0x00, 0xe8, 0x47, 0x90, 0xe6, 0x47, 0x05, 0x7a, 0x17, 0xd2, 0xd5, 0x66, 0xab, 0xae, 0x2d,
+	0x94, 0x2e, 0x9d, 0x9c, 0x96, 0x97, 0xc4, 0x94, 0xf0, 0x0a, 0xee, 0xbb, 0x68, 0x1d, 0xb2, 0x4f,
+	0xdb, 0xdb, 0xbb, 0x3b, 0xdc, 0xbd, 0x2e, 0x9f, 0x9c, 0x96, 0x57, 0xa2, 0x6a, 0x39, 0x69, 0xe8,
+	0x3d, 0xc8, 0xf4, 0x76, 0x3a, 0x9b, 0x5d, 0x2d, 0x59, 0x42, 0x27, 0xa7, 0xe5, 0xe5, 0xa8, 0x5e,
+	0xf4, 0xb9, 0x74, 0x49, 0xad, 0x6a, 0x3e, 0x92, 0xeb, 0xff, 0x95, 0x84, 0x25, 0xcc, 0x39, 0xaf,
+	0xcf, 0x3a, 0x9e, 0x63, 0x9b, 0x13, 0xd4, 0x81, 0xbc, 0xe9, 0xb9, 0x96, 0x1d, 0xdb, 0x53, 0x1b,
+	0xaf, 0xb9, 0x18, 0xa7, 0x5a, 0x61, 0xa9, 0x16, 0x6a, 0xe2, 0xa9, 0x11, 0xb4, 0x01, 0x19, 0x8b,
+	0x3a, 0x64, 0x72, 0xd1, 0x0d, 0x5d, 0x57, 0xfc, 0x1a, 0x4b, 0xa8, 0x60, 0x93, 0xe4, 0x79, 0x9f,
+	0x30, 0x46, 0x87, 0x23, 0x26, 0x6f, 0xe8, 0x34, 0x2e, 0x0c, 0xc9, 0x73, 0x43, 0x89, 0xd0, 0xa7,
+	0x90, 0x3d, 0xb6, 0x5d, 0xcb, 0x3b, 0x56, 0x97, 0xf0, 0xc5, 0x76, 0x15, 0x56, 0x3f, 0xe1, 0x77,
+	0xef, 0xb9, 0xce, 0xf2, 0x59, 0x6f, 0xb5, 0x5b, 0x8d, 0x70, 0xd6, 0x55, 0x7d, 0xdb, 0x6d, 0x79,
+	0x2e, 0xdf, 0x31, 0xd0, 0x6e, 0xf5, 0x37, 0x8d, 0xe6, 0xf6, 0x2e, 0xe6, 0x33, 0x7f, 0xe5, 0xe4,
+	0xb4, 0xac, 0x45, 0x90, 0x4d, 0x62, 0x3b, 0x9c, 0x18, 0x5e, 0x83, 0x94, 0xd1, 0xfa, 0x52, 0x4b,
+	0x96, 0xb4, 0x93, 0xd3, 0x72, 0x31, 0xaa, 0x36, 0xdc, 0xc9, 0x74, 0x33, 0x9d, 0x6f, 0x57, 0xff,
+	0xd7, 0x24, 0x14, 0x77, 0x47, 0x16, 0x61, 0x54, 0x7a, 0x26, 0x2a, 0x43, 0x61, 0x44, 0x7c, 0xe2,
+	0x38, 0xd4, 0xb1, 0x83, 0xa1, 0x0a, 0x1e, 0xe2, 0x22, 0x74, 0xff, 0x3b, 0x4c, 0xa6, 0x22, 0x66,
+	0x6a, 0x4a, 0x77, 0x61, 0x79, 0x5f, 0x76, 0xb6, 0x4f, 0x4c, 0xb1, 0xba, 0x29, 0xb1, 0xba, 0x95,
+	0x79, 0x26, 0xe2, 0xbd, 0xaa, 0xa8, 0x31, 0x1a, 0x42, 0x0b, 0x2f, 0xed, 0xc7, 0x8b, 0xe8, 0x73,
+	0x58, 0x1c, 0x7a, 0xae, 0xcd, 0x3c, 0xff, 0xad, 0xd6, 0x21, 0x04, 0xa3, 0xdb, 0x70, 0x89, 0xaf,
+	0x70, 0xd8, 0x25, 0x51, 0x2d, 0x6e, 0xae, 0x24, 0x5e, 0x19, 0x92, 0xe7, 0xaa, 0x4d, 0xcc, 0xc5,
+	0xfa, 0xe7, 0xb0, 0x34, 0xd3, 0x07, 0x7e, 0x9b, 0x77, 0x8c, 0xdd, 0x6e, 0x43, 0x5b, 0x40, 0x45,
+	0xc8, 0xd5, 0xda, 0xad, 0x5e, 0xb3, 0xb5, 0xcb, 0xe9, 0x48, 0x11, 0x72, 0xb8, 0xbd, 0xbd, 0x5d,
+	0x35, 0x6a, 0x4f, 0xb4, 0xa4, 0xfe, 0x8b, 0x68, 0x7e, 0x15, 0x1f, 0xa9, 0xce, 0xf2, 0x91, 0x3b,
+	0xaf, 0x1f, 0xba, 0x62, 0x24, 0xd3, 0x42, 0xc4, 0x4b, 0xfe, 0x3f, 0x80, 0x58, 0x46, 0x6a, 0xf5,
+	0x09, 0xbb, 0x28, 0xe6, 0xe8, 0x85, 0xd1, 0x24, 0xce, 0x2b, 0x05, 0x83, 0xa1, 0x47, 0x50, 0x34,
+	0xbd, 0xe1, 0xc8, 0xa1, 0x4a, 0x3f, 0xf5, 0x36, 0xfa, 0x85, 0x48, 0xc5, 0x60, 0x71, 0x5e, 0x94,
+	0x9e, 0x65, 0x6e, 0x7f, 0x90, 0x80, 0x42, 0xac, 0xc3, 0xb3, 0x54, 0xa8, 0x08, 0xb9, 0xdd, 0x4e,
+	0xdd, 0xe8, 0x35, 0x5b, 0x8f, 0xb5, 0x04, 0x02, 0xc8, 0x8a, 0x09, 0xac, 0x6b, 0x49, 0x4e, 0xe1,
+	0x6a, 0xed, 0x9d, 0xce, 0x76, 0x43, 0x90, 0x21, 0x74, 0x05, 0xb4, 0x70, 0x0a, 0xfb, 0xdd, 0x9e,
+	0x81, 0xb9, 0x34, 0x8d, 0x2e, 0xc3, 0x4a, 0x24, 0x55, 0x9a, 0x19, 0x74, 0x15, 0x50, 0x24, 0x9c,
+	0x9a, 0xc8, 0xea, 0xbf, 0x0b, 0x2b, 0x35, 0xcf, 0x65, 0xc4, 0x76, 0x23, 0x7a, 0xbb, 0xc1, 0xc7,
+	0xad, 0x44, 0x7d, 0xdb, 0x92, 0xa7, 0x6d, 0x75, 0xe5, 0xec, 0xe5, 0x7a, 0x21, 0x82, 0x36, 0xeb,
+	0x7c, 0xa4, 0x61, 0xc1, 0xe2, 0x7b, 0x6a, 0x64, 0x5b, 0x62, 0x8a, 0x33, 0xd5, 0xc5, 0xb3, 0x97,
+	0xeb, 0xa9, 0x4e, 0xb3, 0x8e, 0xb9, 0x0c, 0xbd, 0x0b, 0x79, 0xfa, 0xdc, 0x66, 0x7d, 0x93, 0x9f,
+	0xae, 0x7c, 0x0e, 0x33, 0x38, 0xc7, 0x05, 0x35, 0x7e, 0x98, 0x56, 0x01, 0x3a, 0x9e, 0xcf, 0x54,
+	0xcb, 0x9f, 0x42, 0x66, 0xe4, 0xf9, 0x22, 0xb6, 0xe4, 0x57, 0xcf, 0x5c, 0xb2, 0xc6, 0xe1, 0xd2,
+	0xd9, 0xb1, 0x04, 0xeb, 0x7f, 0x97, 0x04, 0xe8, 0x91, 0xe0, 0x50, 0x19, 0x79, 0x00, 0xf9, 0x28,
+	0x39, 0x70, 0x51, 0x90, 0x1a, 0x5b, 0xf3, 0x08, 0x8f, 0x3e, 0x09, 0xbd, 0x4e, 0x72, 0xf7, 0xf9,
+	0x8a, 0xaa, 0xad, 0x79, 0xf4, 0x77, 0x96, 0xa0, 0xf3, 0xfb, 0x8a, 0xfa, 0xbe, 0x5a, 0x7c, 0xfe,
+	0x89, 0x6a, 0xe2, 0xcc, 0x96, 0xf3, 0xa6, 0xd8, 0xdf, 0x8d, 0x79, 0x8d, 0x9c, 0x5b, 0x94, 0xad,
+	0x05, 0x3c, 0xd5, 0x43, 0x0f, 0xa1, 0xc0, 0x87, 0xde, 0x0f, 0x44, 0x9d, 0x22, 0x7e, 0xaf, 0x9d,
+	0x2d, 0x69, 0x01, 0xc3, 0x28, 0xfa, 0xae, 0x6a, 0xb0, 0xec, 0x8f, 0x5d, 0x3e, 0x6c, 0x65, 0x43,
+	0xb7, 0xe1, 0x9d, 0x16, 0x65, 0xc7, 0x9e, 0x7f, 0x68, 0x30, 0x46, 0xcc, 0x03, 0x1e, 0xed, 0xab,
+	0x93, 0x6e, 0xca, 0x7a, 0x13, 0x33, 0xac, 0x77, 0x15, 0x16, 0x89, 0x63, 0x93, 0x80, 0x4a, 0xaa,
+	0x90, 0xc7, 0x61, 0x91, 0x73, 0x73, 0xce, 0xf4, 0x69, 0x10, 0x50, 0x19, 0x9f, 0xe6, 0xf1, 0x54,
+	0xa0, 0xff, 0x63, 0x12, 0xa0, 0xd9, 0x31, 0x76, 0x94, 0xf9, 0x3a, 0x64, 0xf7, 0xc9, 0xd0, 0x76,
+	0x26, 0x17, 0xed, 0xf4, 0x29, 0xbe, 0x62, 0x48, 0x43, 0x9b, 0x42, 0x07, 0x2b, 0x5d, 0x41, 0xd9,
+	0xc7, 0x7b, 0x2e, 0x65, 0x11, 0x65, 0x17, 0x25, 0xce, 0x0f, 0x7c, 0xe2, 0x46, 0x2b, 0x23, 0x0b,
+	0xbc, 0xeb, 0x03, 0xc2, 0xe8, 0x31, 0x99, 0x84, 0x1b, 0x53, 0x15, 0xd1, 0x16, 0xa7, 0xf2, 0x01,
+	0xf5, 0x8f, 0xa8, 0xb5, 0x9a, 0x11, 0x5e, 0xf8, 0xa6, 0xfe, 0x60, 0x05, 0x97, 0xcc, 0x27, 0xd2,
+	0x2e, 0x3d, 0x10, 0xd7, 0xf5, 0xb4, 0xea, 0x3b, 0x45, 0xd7, 0x77, 0x61, 0x69, 0x66, 0x9c, 0xaf,
+	0xc4, 0x4a, 0xcd, 0xce, 0xd3, 0x4f, 0xb5, 0xb4, 0xfa, 0xfa, 0x5c, 0xcb, 0xea, 0x7f, 0x95, 0x92,
+	0x5b, 0x49, 0xcd, 0xea, 0xfc, 0x7c, 0x55, 0x4e, 0x64, 0xbf, 0x4c, 0xcf, 0x51, 0xfe, 0xfd, 0xe1,
+	0xc5, 0x3b, 0x8c, 0x73, 0x6f, 0x01, 0xc7, 0x91, 0x22, 0x5a, 0x87, 0x82, 0x5c, 0xff, 0x3e, 0xf7,
+	0x27, 0x31, 0xad, 0x4b, 0x18, 0xa4, 0x88, 0x6b, 0xa2, 0x9b, 0xb0, 0x3c, 0x1a, 0xef, 0x39, 0x76,
+	0x70, 0x40, 0x2d, 0x89, 0x49, 0x0b, 0xcc, 0x52, 0x24, 0x15, 0xb0, 0x1d, 0x28, 0x2a, 0x41, 0x5f,
+	0xf0, 0xae, 0x8c, 0xe8, 0xd0, 0xed, 0x37, 0x75, 0x48, 0xaa, 0x08, 0x3a, 0x56, 0x18, 0x4d, 0x0b,
+	0x7a, 0x1d, 0x72, 0x61, 0x67, 0xd1, 0x2a, 0xa4, 0x7a, 0xb5, 0x8e, 0xb6, 0x50, 0x5a, 0x39, 0x39,
+	0x2d, 0x17, 0x42, 0x71, 0xaf, 0xd6, 0xe1, 0x35, 0xbb, 0xf5, 0x8e, 0x96, 0x98, 0xad, 0xd9, 0xad,
+	0x77, 0x4a, 0x69, 0x7e, 0xf3, 0xeb, 0xfb, 0x50, 0x88, 0xb5, 0x80, 0x6e, 0xc0, 0x62, 0xb3, 0xf5,
+	0x18, 0x37, 0xba, 0x5d, 0x6d, 0xa1, 0x74, 0xf5, 0xe4, 0xb4, 0x8c, 0x62, 0xb5, 0x4d, 0x77, 0xc0,
+	0xd7, 0x07, 0xbd, 0x07, 0xe9, 0xad, 0x76, 0xb7, 0x17, 0x12, 0xbd, 0x18, 0x62, 0xcb, 0x0b, 0x58,
+	0xe9, 0xb2, 0xa2, 0x14, 0x71, 0xc3, 0xfa, 0x9f, 0x24, 0x20, 0x2b, 0xf9, 0xee, 0xdc, 0x85, 0x32,
+	0x60, 0x31, 0x8c, 0xc2, 0x24, 0x09, 0xff, 0xf0, 0xf5, 0x84, 0xb9, 0xa2, 0xf8, 0xad, 0x74, 0xbf,
+	0x50, 0xaf, 0xf4, 0x05, 0x14, 0xe3, 0x15, 0xdf, 0xc9, 0xf9, 0x7e, 0x07, 0x0a, 0xdc, 0xbf, 0x43,
+	0xe2, 0xbc, 0x01, 0x59, 0xc9, 0xc9, 0xd5, 0x69, 0x7a, 0x11, 0x7b, 0x57, 0x48, 0x74, 0x1f, 0x16,
+	0x25, 0xe3, 0x0f, 0xf3, 0x53, 0x6b, 0x17, 0xef, 0x22, 0x1c, 0xc2, 0xf5, 0x87, 0x90, 0xee, 0x50,
+	0xea, 0xf3, 0xb9, 0x77, 0x3d, 0x8b, 0x4e, 0x2f, 0x20, 0x15, 0xac, 0x58, 0xb4, 0x59, 0xe7, 0xc1,
+	0x8a, 0x45, 0x9b, 0x56, 0x94, 0x5e, 0x48, 0xc6, 0xd2, 0x0b, 0x3d, 0x28, 0x3e, 0xa3, 0xf6, 0xe0,
+	0x80, 0x51, 0x4b, 0x18, 0xba, 0x03, 0xe9, 0x11, 0x8d, 0x3a, 0xbf, 0x3a, 0xd7, 0xc1, 0x28, 0xf5,
+	0xb1, 0x40, 0xf1, 0x73, 0xe4, 0x58, 0x68, 0xab, 0xac, 0xa8, 0x2a, 0xe9, 0xff, 0x90, 0x84, 0xe5,
+	0x66, 0x10, 0x8c, 0x89, 0x6b, 0x86, 0x0c, 0xe5, 0x47, 0xb3, 0x0c, 0xe5, 0xd6, 0xdc, 0x11, 0xce,
+	0xa8, 0xcc, 0x66, 0x4d, 0xd4, 0xe5, 0x90, 0x8c, 0x2e, 0x07, 0xfd, 0x3f, 0x12, 0x61, 0x6a, 0xe4,
+	0x66, 0x6c, 0xbb, 0x97, 0x56, 0x4f, 0x4e, 0xcb, 0x57, 0xe2, 0x96, 0xe8, 0xae, 0x7b, 0xe8, 0x7a,
+	0xc7, 0x2e, 0x7a, 0x1f, 0x32, 0xb8, 0xd1, 0x6a, 0x3c, 0xd3, 0x12, 0xd2, 0x3d, 0x67, 0x40, 0x98,
+	0xba, 0xf4, 0x98, 0x5b, 0xea, 0x34, 0x5a, 0x75, 0xce, 0x25, 0x92, 0x73, 0x2c, 0x75, 0xa8, 0x6b,
+	0xd9, 0xee, 0x00, 0xdd, 0x80, 0x6c, 0xb3, 0xdb, 0xdd, 0x15, 0xc1, 0xeb, 0x3b, 0x27, 0xa7, 0xe5,
+	0xcb, 0x33, 0x28, 0x5e, 0xa0, 0x16, 0x07, 0x71, 0x72, 0xcd, 0x59, 0xc6, 0x1c, 0x10, 0xe7, 0x7d,
+	0x12, 0x84, 0xdb, 0x3d, 0x1e, 0x59, 0x67, 0xe6, 0x80, 0xb0, 0xc7, 0x7f, 0xd5, 0x76, 0xfb, 0x97,
+	0x24, 0x68, 0x86, 0x69, 0xd2, 0x11, 0xe3, 0xf5, 0x2a, 0xaa, 0xe9, 0x41, 0x6e, 0xc4, 0xbf, 0x6c,
+	0x1a, 0xf2, 0x80, 0xfb, 0x73, 0xf3, 0xea, 0xe7, 0xf4, 0x2a, 0xd8, 0x73, 0xa8, 0x61, 0x0d, 0xed,
+	0x20, 0xe0, 0xd1, 0xbb, 0x90, 0xe1, 0xc8, 0x52, 0xe9, 0xe7, 0x09, 0xb8, 0x3c, 0x07, 0x81, 0xee,
+	0x42, 0xda, 0xf7, 0x9c, 0x70, 0x0d, 0xaf, 0xbf, 0x2e, 0xeb, 0xc5, 0x55, 0xb1, 0x40, 0xa2, 0x35,
+	0x00, 0x32, 0x66, 0x1e, 0x11, 0xed, 0x8b, 0xd5, 0xcb, 0xe1, 0x98, 0x04, 0x3d, 0x83, 0x6c, 0x40,
+	0x4d, 0x9f, 0x86, 0x84, 0xf1, 0xe1, 0xff, 0xb5, 0xf7, 0x95, 0xae, 0x30, 0x83, 0x95, 0xb9, 0x52,
+	0x05, 0xb2, 0x52, 0xc2, 0xdd, 0xde, 0x22, 0x8c, 0x88, 0x4e, 0x17, 0xb1, 0xf8, 0xe6, 0xde, 0x44,
+	0x9c, 0x41, 0xe8, 0x4d, 0xc4, 0x19, 0xe8, 0x7f, 0x9a, 0x04, 0x68, 0x3c, 0x67, 0xd4, 0x77, 0x89,
+	0x53, 0x33, 0x50, 0x23, 0x76, 0xfa, 0xcb, 0xd1, 0x7e, 0x34, 0x37, 0x17, 0x1a, 0x69, 0x54, 0x6a,
+	0xc6, 0x9c, 0xf3, 0xff, 0x1a, 0xa4, 0xc6, 0xbe, 0xa3, 0xf2, 0xea, 0x82, 0xe9, 0xed, 0xe2, 0x6d,
+	0xcc, 0x65, 0xa8, 0x31, 0x3d, 0xb6, 0x52, 0xaf, 0x7f, 0x10, 0x89, 0x35, 0xf0, 0xab, 0x3f, 0xba,
+	0xee, 0x00, 0x4c, 0x7b, 0x8d, 0xd6, 0x20, 0x53, 0xdb, 0xec, 0x76, 0xb7, 0xb5, 0x05, 0x79, 0x36,
+	0x4f, 0xab, 0x84, 0x58, 0xff, 0xcb, 0x04, 0xe4, 0x6a, 0x86, 0xba, 0x31, 0x37, 0x41, 0x13, 0x07,
+	0x8e, 0x49, 0x7d, 0xd6, 0xa7, 0xcf, 0x47, 0xb6, 0x3f, 0x51, 0x67, 0xc6, 0xc5, 0x61, 0xd2, 0x32,
+	0xd7, 0xaa, 0x51, 0x9f, 0x35, 0x84, 0x0e, 0xc2, 0x50, 0xa4, 0x6a, 0x88, 0x7d, 0x93, 0x84, 0x27,
+	0xf8, 0xda, 0xc5, 0x53, 0x21, 0xe9, 0xf5, 0xb4, 0x1c, 0xe0, 0x42, 0x68, 0xa4, 0x46, 0x02, 0xfd,
+	0x29, 0x5c, 0x6e, 0xfb, 0xe6, 0x01, 0x0d, 0x98, 0x6c, 0x54, 0x75, 0xf9, 0x21, 0x5c, 0x67, 0x24,
+	0x38, 0xec, 0x1f, 0xd8, 0x01, 0xf3, 0xfc, 0x49, 0xdf, 0xa7, 0x8c, 0xba, 0xbc, 0xbe, 0x2f, 0x9e,
+	0x5d, 0x54, 0x92, 0xe3, 0x1a, 0xc7, 0x6c, 0x49, 0x08, 0x0e, 0x11, 0xdb, 0x1c, 0xa0, 0x37, 0xa1,
+	0xc8, 0xd9, 0x6c, 0x9d, 0xee, 0x93, 0xb1, 0xc3, 0x02, 0xf4, 0x43, 0x00, 0xc7, 0x1b, 0xf4, 0xdf,
+	0xfa, 0xb8, 0xcf, 0x3b, 0xde, 0x40, 0x7e, 0xea, 0xbf, 0x01, 0x5a, 0xdd, 0x0e, 0x46, 0x84, 0x99,
+	0x07, 0x61, 0xf6, 0x06, 0x3d, 0x06, 0xed, 0x80, 0x12, 0x9f, 0xed, 0x51, 0xc2, 0xfa, 0x23, 0xea,
+	0xdb, 0x9e, 0xf5, 0x56, 0x53, 0xba, 0x12, 0x69, 0x75, 0x84, 0x92, 0xfe, 0x9f, 0x09, 0x00, 0x4c,
+	0xf6, 0x43, 0x72, 0xf3, 0x03, 0xb8, 0x14, 0xb8, 0x64, 0x14, 0x1c, 0x78, 0xac, 0x6f, 0xbb, 0x8c,
+	0xfa, 0x47, 0xc4, 0x51, 0x11, 0xb8, 0x16, 0x56, 0x34, 0x95, 0x1c, 0xdd, 0x01, 0x74, 0x48, 0xe9,
+	0xa8, 0xef, 0x39, 0x56, 0x3f, 0xac, 0x94, 0xef, 0x42, 0x69, 0xac, 0xf1, 0x9a, 0xb6, 0x63, 0x75,
+	0x43, 0x39, 0xaa, 0xc2, 0x1a, 0x9f, 0x01, 0xea, 0x32, 0xdf, 0xa6, 0x41, 0x7f, 0xdf, 0xf3, 0xfb,
+	0x81, 0xe3, 0x1d, 0xf7, 0xf7, 0x3d, 0xc7, 0xf1, 0x8e, 0xa9, 0x1f, 0xe6, 0x37, 0x4a, 0x8e, 0x37,
+	0x68, 0x48, 0xd0, 0xa6, 0xe7, 0x77, 0x1d, 0xef, 0x78, 0x33, 0x44, 0x70, 0x06, 0x34, 0x1d, 0x36,
+	0xb3, 0xcd, 0xc3, 0x90, 0x01, 0x45, 0xd2, 0x9e, 0x6d, 0x1e, 0xa2, 0x1b, 0xb0, 0x44, 0x1d, 0x2a,
+	0xa2, 0x64, 0x89, 0xca, 0x08, 0x54, 0x31, 0x14, 0x72, 0x90, 0xfe, 0x08, 0xb4, 0x86, 0x6b, 0xfa,
+	0x93, 0x51, 0x6c, 0xd9, 0xef, 0x00, 0xe2, 0xe7, 0x4d, 0xdf, 0xf1, 0xcc, 0xc3, 0xfe, 0x90, 0xb8,
+	0x64, 0xc0, 0xfb, 0x25, 0xdf, 0x22, 0x34, 0x5e, 0xb3, 0xed, 0x99, 0x87, 0x3b, 0x4a, 0xae, 0xff,
+	0x3f, 0xc8, 0x77, 0x1c, 0x62, 0x8a, 0xf7, 0x3b, 0x54, 0x06, 0x1e, 0xb6, 0x71, 0x37, 0xb2, 0x5d,
+	0x15, 0x67, 0xe5, 0x71, 0x5c, 0xa4, 0xff, 0x08, 0xe0, 0xc7, 0x9e, 0xed, 0xf6, 0xbc, 0x43, 0xea,
+	0x8a, 0xa7, 0x0e, 0x1e, 0x13, 0x28, 0x67, 0xc8, 0x63, 0x55, 0x12, 0x21, 0x8f, 0x6c, 0x20, 0xca,
+	0xf8, 0xcb, 0xa2, 0xfe, 0x4d, 0x02, 0xb2, 0xd8, 0xf3, 0x58, 0xcd, 0x40, 0x65, 0xc8, 0x9a, 0xa4,
+	0x1f, 0xee, 0xdd, 0x62, 0x35, 0x7f, 0xf6, 0x72, 0x3d, 0x53, 0x33, 0x9e, 0xd0, 0x09, 0xce, 0x98,
+	0xe4, 0x09, 0x9d, 0xf0, 0x4b, 0xde, 0x24, 0x62, 0xc7, 0x09, 0x33, 0x45, 0x79, 0xc9, 0xd7, 0x0c,
+	0xbe, 0x9d, 0x70, 0xd6, 0x24, 0xfc, 0x1f, 0xdd, 0x85, 0xa2, 0x02, 0xf5, 0x0f, 0x48, 0x70, 0x20,
+	0x99, 0x7c, 0x75, 0xf9, 0xec, 0xe5, 0x3a, 0x48, 0xe4, 0x16, 0x09, 0x0e, 0x30, 0x48, 0x34, 0xff,
+	0x46, 0x0d, 0x28, 0x7c, 0xe5, 0xd9, 0x6e, 0x9f, 0x89, 0x41, 0xa8, 0x64, 0xc7, 0xdc, 0x1d, 0x38,
+	0x1d, 0xaa, 0xca, 0xc0, 0xc0, 0x57, 0x91, 0x44, 0xff, 0xa7, 0x04, 0x14, 0xb8, 0x4d, 0x7b, 0xdf,
+	0x36, 0xf9, 0xa5, 0xfc, 0xdd, 0xef, 0x8a, 0x6b, 0x90, 0x32, 0x03, 0x5f, 0x8d, 0x4d, 0x1c, 0x96,
+	0xb5, 0x2e, 0xc6, 0x5c, 0x86, 0x1e, 0x41, 0x56, 0x85, 0x6f, 0xf2, 0x9a, 0xd0, 0xdf, 0x4c, 0x1f,
+	0x54, 0x17, 0x95, 0x9e, 0x58, 0xcb, 0x69, 0xef, 0xc4, 0x28, 0x8b, 0x38, 0x2e, 0x42, 0x57, 0x21,
+	0x69, 0xba, 0xc2, 0xad, 0xd4, 0x13, 0x68, 0xad, 0x85, 0x93, 0xa6, 0xab, 0xff, 0x7d, 0x02, 0x96,
+	0xa6, 0x5e, 0xc5, 0x17, 0xe2, 0x3a, 0xe4, 0x83, 0xf1, 0x5e, 0x30, 0x09, 0x18, 0x1d, 0x86, 0xaf,
+	0x29, 0x91, 0x00, 0x35, 0x21, 0x4f, 0x9c, 0x81, 0xe7, 0xdb, 0xec, 0x60, 0xa8, 0x22, 0x87, 0xf9,
+	0x47, 0x7b, 0xdc, 0x66, 0xc5, 0x08, 0x55, 0xf0, 0x54, 0x3b, 0x3c, 0xcc, 0x53, 0xa2, 0xb3, 0xe2,
+	0x30, 0x7f, 0x1f, 0x8a, 0x0e, 0x19, 0x8a, 0x78, 0x96, 0x07, 0xa4, 0x62, 0x1c, 0x69, 0x5c, 0x50,
+	0x32, 0x1e, 0xa5, 0xeb, 0x3a, 0xe4, 0x23, 0x63, 0x68, 0x05, 0x0a, 0x46, 0xa3, 0xdb, 0xbf, 0xb7,
+	0x71, 0xbf, 0xff, 0xb8, 0xb6, 0xa3, 0x2d, 0x28, 0x2e, 0xf1, 0x37, 0x09, 0x58, 0x52, 0x3e, 0xaf,
+	0xf8, 0xd9, 0x0d, 0x58, 0xf4, 0xc9, 0x3e, 0x0b, 0x19, 0x64, 0x5a, 0x3a, 0x17, 0x3f, 0x46, 0x38,
+	0x83, 0xe4, 0x55, 0xf3, 0x19, 0x64, 0xec, 0x7d, 0x2f, 0x75, 0xe1, 0xfb, 0x5e, 0xfa, 0x57, 0xf2,
+	0xbe, 0xa7, 0xff, 0x24, 0x09, 0x2b, 0xea, 0xaa, 0x0f, 0xdf, 0xaf, 0xd0, 0x47, 0x90, 0x97, 0xb7,
+	0xfe, 0x94, 0xff, 0x8a, 0x27, 0x25, 0x89, 0x6b, 0xd6, 0x71, 0x4e, 0x56, 0x37, 0x2d, 0x1e, 0x90,
+	0x29, 0x68, 0xec, 0xb5, 0x1a, 0xa4, 0xa8, 0xc5, 0xa3, 0x89, 0x3a, 0xa4, 0xf7, 0x6d, 0x87, 0x2a,
+	0x3f, 0x9b, 0x9b, 0x43, 0x3c, 0xd7, 0xbc, 0x48, 0x79, 0xf7, 0x44, 0x48, 0xb7, 0xb5, 0x80, 0x85,
+	0x76, 0xe9, 0xf7, 0x00, 0xa6, 0xd2, 0xb9, 0x51, 0x0b, 0x67, 0x06, 0x2a, 0x07, 0x14, 0x32, 0x83,
+	0x66, 0x1d, 0x73, 0x19, 0xaf, 0x1a, 0xd8, 0x96, 0xda, 0xb9, 0xa2, 0xea, 0x31, 0xaf, 0x1a, 0xd8,
+	0x56, 0x94, 0x77, 0x4f, 0xbf, 0x21, 0xef, 0x5e, 0xcd, 0x85, 0x69, 0x08, 0xbd, 0x0d, 0x57, 0xab,
+	0x0e, 0x31, 0x0f, 0x1d, 0x3b, 0x60, 0xd4, 0x8a, 0xef, 0xd0, 0xcf, 0x20, 0x3b, 0x73, 0x73, 0xbf,
+	0x21, 0xf1, 0xa3, 0xc0, 0xfa, 0x4f, 0x12, 0x50, 0xdc, 0xa2, 0xc4, 0x61, 0x07, 0xd3, 0xe8, 0x99,
+	0xd1, 0x80, 0xa9, 0xf3, 0x51, 0x7c, 0xa3, 0xfb, 0x90, 0x8b, 0xee, 0x9a, 0xb7, 0x49, 0x8f, 0x47,
+	0x68, 0xf4, 0x39, 0x2c, 0x72, 0xcf, 0xf6, 0xc6, 0x21, 0x25, 0x7c, 0x43, 0xde, 0x55, 0x81, 0xf9,
+	0x21, 0xeb, 0x53, 0x71, 0xc5, 0x88, 0xd9, 0xc9, 0xe0, 0xb0, 0xa8, 0xff, 0x4f, 0x02, 0xae, 0xec,
+	0x90, 0xc9, 0x1e, 0x55, 0x3b, 0x8e, 0x5a, 0x98, 0x9a, 0x9e, 0x6f, 0xa1, 0x4e, 0x7c, 0xa7, 0x5e,
+	0xf0, 0x24, 0x30, 0x4f, 0x79, 0xfe, 0x86, 0x0d, 0xb9, 0x66, 0x32, 0xc6, 0x35, 0xaf, 0x40, 0xc6,
+	0xf5, 0x5c, 0x93, 0xaa, 0x6d, 0x2c, 0x0b, 0xba, 0x1d, 0xdf, 0xa5, 0xa5, 0x28, 0x4f, 0x2f, 0xb2,
+	0xec, 0x2d, 0x8f, 0x45, 0xad, 0xa1, 0x47, 0x50, 0xea, 0x36, 0x6a, 0xb8, 0xd1, 0xab, 0xb6, 0x7f,
+	0xbd, 0xdf, 0x35, 0xb6, 0xbb, 0xc6, 0xc6, 0xdd, 0x7e, 0xa7, 0xbd, 0xfd, 0xe5, 0xbd, 0x4f, 0xee,
+	0x7e, 0xa6, 0x25, 0x4a, 0xe5, 0x93, 0xd3, 0xf2, 0xf5, 0x96, 0x51, 0xdb, 0x96, 0x6e, 0xb9, 0xe7,
+	0x3d, 0xef, 0x12, 0x27, 0x20, 0x1b, 0x77, 0x3b, 0x9e, 0x33, 0xe1, 0x98, 0xdb, 0xbf, 0x48, 0x41,
+	0x3e, 0x4a, 0xc3, 0x71, 0xef, 0xe2, 0x31, 0x90, 0x6a, 0x2a, 0x92, 0xb7, 0xe8, 0x31, 0x7a, 0x7f,
+	0x1a, 0xfd, 0x3c, 0x92, 0xcf, 0x01, 0x51, 0x75, 0x18, 0xf9, 0x7c, 0x00, 0x39, 0xa3, 0xdb, 0x6d,
+	0x3e, 0x6e, 0x35, 0xea, 0xda, 0xd7, 0x89, 0xd2, 0xf7, 0x4e, 0x4e, 0xcb, 0x97, 0x22, 0x90, 0x11,
+	0x04, 0xf6, 0xc0, 0xa5, 0x96, 0x40, 0xd5, 0x6a, 0x8d, 0x4e, 0xaf, 0x51, 0xd7, 0x5e, 0x24, 0xcf,
+	0xa3, 0x04, 0x9b, 0x17, 0x4f, 0x7b, 0xf9, 0x0e, 0x6e, 0x74, 0x0c, 0xcc, 0x1b, 0xfc, 0x3a, 0x29,
+	0x83, 0xb2, 0x69, 0x8b, 0x3e, 0x1d, 0x11, 0x9f, 0xb7, 0xb9, 0x16, 0x3e, 0x71, 0xbf, 0x48, 0xc9,
+	0xe7, 0x9f, 0x69, 0x4e, 0x91, 0x12, 0x6b, 0xc2, 0x5b, 0x13, 0xf9, 0x5c, 0x61, 0x26, 0x75, 0xae,
+	0xb5, 0x2e, 0x23, 0x3e, 0xe3, 0x56, 0x74, 0x58, 0xc4, 0xbb, 0xad, 0x16, 0x07, 0xbd, 0x48, 0x9f,
+	0x1b, 0x1d, 0x1e, 0xbb, 0x2e, 0xc7, 0xdc, 0x84, 0x5c, 0x98, 0xee, 0xd5, 0xbe, 0x4e, 0x9f, 0xeb,
+	0x50, 0x2d, 0xcc, 0x55, 0x8b, 0x06, 0xb7, 0x76, 0x7b, 0xe2, 0x05, 0xfe, 0x45, 0xe6, 0x7c, 0x83,
+	0x07, 0x63, 0x66, 0xf1, 0x70, 0xb3, 0x1c, 0xc5, 0x7f, 0x5f, 0x67, 0x24, 0xa3, 0x8e, 0x30, 0x2a,
+	0xf8, 0xfb, 0x00, 0x72, 0xb8, 0xf1, 0x63, 0xf9, 0x58, 0xff, 0x22, 0x7b, 0xce, 0x0e, 0xa6, 0x5f,
+	0x51, 0x53, 0xb5, 0xd6, 0xc6, 0x9d, 0x2d, 0x43, 0x4c, 0xf9, 0x79, 0x54, 0xdb, 0x1f, 0x1d, 0x10,
+	0x97, 0x5a, 0xd3, 0x37, 0xb0, 0xa8, 0xea, 0xf6, 0x6f, 0x42, 0x2e, 0xbc, 0x61, 0xd1, 0x1a, 0x64,
+	0x9f, 0xb5, 0xf1, 0x93, 0x06, 0xd6, 0x16, 0xe4, 0x1c, 0x86, 0x35, 0xcf, 0x24, 0x45, 0x29, 0xc3,
+	0xe2, 0x8e, 0xd1, 0x32, 0x1e, 0x37, 0x70, 0x98, 0x9a, 0x09, 0x01, 0xea, 0x9a, 0x28, 0x69, 0xaa,
+	0x81, 0xc8, 0x66, 0xf5, 0xfa, 0x37, 0xdf, 0xae, 0x2d, 0xfc, 0xec, 0xdb, 0xb5, 0x85, 0x9f, 0x7f,
+	0xbb, 0x96, 0x78, 0x71, 0xb6, 0x96, 0xf8, 0xe6, 0x6c, 0x2d, 0xf1, 0xd3, 0xb3, 0xb5, 0xc4, 0xbf,
+	0x9d, 0xad, 0x25, 0xf6, 0xb2, 0x22, 0x08, 0xfa, 0xe4, 0x7f, 0x03, 0x00, 0x00, 0xff, 0xff, 0x7d,
+	0xfe, 0xa7, 0xa7, 0xa8, 0x26, 0x00, 0x00,
 }

vendor/github.com/docker/swarmkit/api/types.proto

@@ -694,6 +694,13 @@ message RaftConfig {
 	uint32 election_tick = 5;
 }
 
+message EncryptionConfig {
+	// AutoLockManagers specifies whether or not managers TLS keys and raft data
+	// should be encrypted at rest in such a way that they must be unlocked
+	// before the manager node starts up again.
+	bool auto_lock_managers = 1;
+}
+
 // Placement specifies task distribution constraints.
 message Placement {
 	// constraints specifies a set of requirements a node should meet for a task.

vendor/github.com/docker/swarmkit/ca/certificates.go

@@ -23,12 +23,12 @@ import (
 	"github.com/docker/distribution/digest"
 	"github.com/docker/go-events"
 	"github.com/docker/swarmkit/api"
-	"github.com/docker/swarmkit/identity"
 	"github.com/docker/swarmkit/ioutils"
 	"github.com/docker/swarmkit/remotes"
 	"github.com/pkg/errors"
 	"golang.org/x/net/context"
 	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/credentials"
 )
 
@@ -122,8 +122,8 @@ func (rca *RootCA) CanSign() bool {
 
 // IssueAndSaveNewCertificates generates a new key-pair, signs it with the local root-ca, and returns a
 // tls certificate
-func (rca *RootCA) IssueAndSaveNewCertificates(paths CertPaths, cn, ou, org string) (*tls.Certificate, error) {
-	csr, key, err := GenerateAndWriteNewKey(paths)
+func (rca *RootCA) IssueAndSaveNewCertificates(kw KeyWriter, cn, ou, org string) (*tls.Certificate, error) {
+	csr, key, err := GenerateNewCSR()
 	if err != nil {
 		return nil, errors.Wrap(err, "error when generating new node certs")
 	}
@@ -138,33 +138,24 @@ func (rca *RootCA) IssueAndSaveNewCertificates(paths CertPaths, cn, ou, org stri
 		return nil, errors.Wrap(err, "failed to sign node certificate")
 	}
 
-	// Ensure directory exists
-	err = os.MkdirAll(filepath.Dir(paths.Cert), 0755)
-	if err != nil {
-		return nil, err
-	}
-
-	// Write the chain to disk
-	if err := ioutils.AtomicWriteFile(paths.Cert, certChain, 0644); err != nil {
-		return nil, err
-	}
-
 	// Create a valid TLSKeyPair out of the PEM encoded private key and certificate
 	tlsKeyPair, err := tls.X509KeyPair(certChain, key)
 	if err != nil {
 		return nil, err
 	}
 
+	if err := kw.Write(certChain, key, nil); err != nil {
+		return nil, err
+	}
+
 	return &tlsKeyPair, nil
 }
 
 // RequestAndSaveNewCertificates gets new certificates issued, either by signing them locally if a signer is
 // available, or by requesting them from the remote server at remoteAddr.
-func (rca *RootCA) RequestAndSaveNewCertificates(ctx context.Context, paths CertPaths, token string, remotes remotes.Remotes, transport credentials.TransportCredentials, nodeInfo chan<- api.IssueNodeCertificateResponse) (*tls.Certificate, error) {
-	// Create a new key/pair and CSR for the new manager
-	// Write the new CSR and the new key to a temporary location so we can survive crashes on rotation
-	tempPaths := genTempPaths(paths)
-	csr, key, err := GenerateAndWriteNewKey(tempPaths)
+func (rca *RootCA) RequestAndSaveNewCertificates(ctx context.Context, kw KeyWriter, token string, r remotes.Remotes, transport credentials.TransportCredentials, nodeInfo chan<- api.IssueNodeCertificateResponse) (*tls.Certificate, error) {
+	// Create a new key/pair and CSR
+	csr, key, err := GenerateNewCSR()
 	if err != nil {
 		return nil, errors.Wrap(err, "error when generating new node certs")
 	}
@@ -174,7 +165,7 @@ func (rca *RootCA) RequestAndSaveNewCertificates(ctx context.Context, paths Cert
 	// responding properly (for example, it may have just been demoted).
 	var signedCert []byte
 	for i := 0; i != 5; i++ {
-		signedCert, err = GetRemoteSignedCertificate(ctx, csr, token, rca.Pool, remotes, transport, nodeInfo)
+		signedCert, err = GetRemoteSignedCertificate(ctx, csr, token, rca.Pool, r, transport, nodeInfo)
 		if err == nil {
 			break
 		}
@@ -184,7 +175,7 @@ func (rca *RootCA) RequestAndSaveNewCertificates(ctx context.Context, paths Cert
 	}
 
 	// Доверяй, но проверяй.
-	// Before we overwrite our local certificate, let's make sure the server gave us one that is valid
+	// Before we overwrite our local key + certificate, let's make sure the server gave us one that is valid
 	// Create an X509Cert so we can .Verify()
 	certBlock, _ := pem.Decode(signedCert)
 	if certBlock == nil {
@@ -209,25 +200,62 @@ func (rca *RootCA) RequestAndSaveNewCertificates(ctx context.Context, paths Cert
 		return nil, err
 	}
 
-	// Ensure directory exists
-	err = os.MkdirAll(filepath.Dir(paths.Cert), 0755)
+	var kekUpdate *KEKData
+	for i := 0; i < 5; i++ {
+		kekUpdate, err = rca.getKEKUpdate(ctx, X509Cert, tlsKeyPair, r)
+		if err == nil {
+			break
+		}
+	}
 	if err != nil {
 		return nil, err
 	}
 
-	// Write the chain to disk
-	if err := ioutils.AtomicWriteFile(paths.Cert, signedCert, 0644); err != nil {
-		return nil, err
-	}
-
-	// Move the new key to the final location
-	if err := os.Rename(tempPaths.Key, paths.Key); err != nil {
+	if err := kw.Write(signedCert, key, kekUpdate); err != nil {
 		return nil, err
 	}
 
 	return &tlsKeyPair, nil
 }
 
+func (rca *RootCA) getKEKUpdate(ctx context.Context, cert *x509.Certificate, keypair tls.Certificate, r remotes.Remotes) (*KEKData, error) {
+	var managerRole bool
+	for _, ou := range cert.Subject.OrganizationalUnit {
+		if ou == ManagerRole {
+			managerRole = true
+			break
+		}
+	}
+
+	if managerRole {
+		mtlsCreds := credentials.NewTLS(&tls.Config{ServerName: CARole, RootCAs: rca.Pool, Certificates: []tls.Certificate{keypair}})
+		conn, peer, err := getGRPCConnection(mtlsCreds, r)
+		if err != nil {
+			return nil, err
+		}
+		defer conn.Close()
+
+		client := api.NewCAClient(conn)
+		ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
+		defer cancel()
+		response, err := client.GetUnlockKey(ctx, &api.GetUnlockKeyRequest{})
+		if err != nil {
+			if grpc.Code(err) == codes.Unimplemented { // if the server does not support keks, return as if no encryption key was specified
+				return &KEKData{}, nil
+			}
+
+			r.Observe(peer, -remotes.DefaultObservationWeight)
+			return nil, err
+		}
+		r.Observe(peer, remotes.DefaultObservationWeight)
+		return &KEKData{KEK: response.UnlockKey, Version: response.Version.Index}, nil
+	}
+
+	// If this is a worker, set to never encrypt. We always want to set to the lock key to nil,
+	// in case this was a manager that was demoted to a worker.
+	return &KEKData{}, nil
+}
+
 // PrepareCSR creates a CFSSL Sign Request based on the given raw CSR and
 // overrides the Subject and Hosts with the given extra args.
 func PrepareCSR(csrBytes []byte, cn, ou, org string) cfsigner.SignRequest {
@@ -388,11 +416,9 @@ func ensureCertKeyMatch(cert *x509.Certificate, key crypto.PublicKey) error {
 
 // GetLocalRootCA validates if the contents of the file are a valid self-signed
 // CA certificate, and returns the PEM-encoded Certificate if so
-func GetLocalRootCA(baseDir string) (RootCA, error) {
-	paths := NewConfigPaths(baseDir)
-
+func GetLocalRootCA(paths CertPaths) (RootCA, error) {
 	// Check if we have a Certificate file
-	cert, err := ioutil.ReadFile(paths.RootCA.Cert)
+	cert, err := ioutil.ReadFile(paths.Cert)
 	if err != nil {
 		if os.IsNotExist(err) {
 			err = ErrNoLocalRootCA
@@ -401,7 +427,7 @@ func GetLocalRootCA(baseDir string) (RootCA, error) {
 		return RootCA{}, err
 	}
 
-	key, err := ioutil.ReadFile(paths.RootCA.Key)
+	key, err := ioutil.ReadFile(paths.Key)
 	if err != nil {
 		if !os.IsNotExist(err) {
 			return RootCA{}, err
@@ -414,24 +440,31 @@ func GetLocalRootCA(baseDir string) (RootCA, error) {
 	return NewRootCA(cert, key, DefaultNodeCertExpiration)
 }
 
-// GetRemoteCA returns the remote endpoint's CA certificate
-func GetRemoteCA(ctx context.Context, d digest.Digest, r remotes.Remotes) (RootCA, error) {
-	// This TLS Config is intentionally using InsecureSkipVerify. Either we're
-	// doing TOFU, in which case we don't validate the remote CA, or we're using
-	// a user supplied hash to check the integrity of the CA certificate.
-	insecureCreds := credentials.NewTLS(&tls.Config{InsecureSkipVerify: true})
+func getGRPCConnection(creds credentials.TransportCredentials, r remotes.Remotes) (*grpc.ClientConn, api.Peer, error) {
+	peer, err := r.Select()
+	if err != nil {
+		return nil, api.Peer{}, err
+	}
+
 	opts := []grpc.DialOption{
-		grpc.WithTransportCredentials(insecureCreds),
+		grpc.WithTransportCredentials(creds),
 		grpc.WithTimeout(5 * time.Second),
 		grpc.WithBackoffMaxDelay(5 * time.Second),
 	}
 
-	peer, err := r.Select()
-	if err != nil {
-		return RootCA{}, err
-	}
-
 	conn, err := grpc.Dial(peer.Addr, opts...)
+	if err != nil {
+		return nil, api.Peer{}, err
+	}
+	return conn, peer, nil
+}
+
+// GetRemoteCA returns the remote endpoint's CA certificate
+func GetRemoteCA(ctx context.Context, d digest.Digest, r remotes.Remotes) (RootCA, error) {
+	// This TLS Config is intentionally using InsecureSkipVerify. We use the
+	// digest instead to check the integrity of the CA certificate.
+	insecureCreds := credentials.NewTLS(&tls.Config{InsecureSkipVerify: true})
+	conn, peer, err := getGRPCConnection(insecureCreds, r)
 	if err != nil {
 		return RootCA{}, err
 	}
@@ -481,9 +514,9 @@ func GetRemoteCA(ctx context.Context, d digest.Digest, r remotes.Remotes) (RootC
 	return RootCA{Cert: response.Certificate, Digest: digest.FromBytes(response.Certificate), Pool: pool}, nil
 }
 
-// CreateAndWriteRootCA creates a Certificate authority for a new Swarm Cluster, potentially
+// CreateRootCA creates a Certificate authority for a new Swarm Cluster, potentially
 // overwriting any existing CAs.
-func CreateAndWriteRootCA(rootCN string, paths CertPaths) (RootCA, error) {
+func CreateRootCA(rootCN string, paths CertPaths) (RootCA, error) {
 	// Create a simple CSR for the CA using the default CA validator and policy
 	req := cfcsr.CertificateRequest{
 		CN:         rootCN,
@@ -497,99 +530,17 @@ func CreateAndWriteRootCA(rootCN string, paths CertPaths) (RootCA, error) {
 		return RootCA{}, err
 	}
 
-	// Ensure directory exists
-	err = os.MkdirAll(filepath.Dir(paths.Cert), 0755)
+	rootCA, err := NewRootCA(cert, key, DefaultNodeCertExpiration)
 	if err != nil {
 		return RootCA{}, err
 	}
 
-	// Write the Private Key and Certificate to disk, using decent permissions
-	if err := ioutils.AtomicWriteFile(paths.Cert, cert, 0644); err != nil {
-		return RootCA{}, err
-	}
-	if err := ioutils.AtomicWriteFile(paths.Key, key, 0600); err != nil {
+	// save the cert to disk
+	if err := saveRootCA(rootCA, paths); err != nil {
 		return RootCA{}, err
 	}
 
-	return NewRootCA(cert, key, DefaultNodeCertExpiration)
-}
-
-// BootstrapCluster receives a directory and creates both new Root CA key material
-// and a ManagerRole key/certificate pair to be used by the initial cluster manager
-func BootstrapCluster(baseCertDir string) error {
-	paths := NewConfigPaths(baseCertDir)
-
-	rootCA, err := CreateAndWriteRootCA(rootCN, paths.RootCA)
-	if err != nil {
-		return err
-	}
-
-	nodeID := identity.NewID()
-	newOrg := identity.NewID()
-	_, err = GenerateAndSignNewTLSCert(rootCA, nodeID, ManagerRole, newOrg, paths.Node)
-
-	return err
-}
-
-// GenerateAndSignNewTLSCert creates a new keypair, signs the certificate using signer,
-// and saves the certificate and key to disk. This method is used to bootstrap the first
-// manager TLS certificates.
-func GenerateAndSignNewTLSCert(rootCA RootCA, cn, ou, org string, paths CertPaths) (*tls.Certificate, error) {
-	// Generate and new keypair and CSR
-	csr, key, err := generateNewCSR()
-	if err != nil {
-		return nil, err
-	}
-
-	// Obtain a signed Certificate
-	certChain, err := rootCA.ParseValidateAndSignCSR(csr, cn, ou, org)
-	if err != nil {
-		return nil, errors.Wrap(err, "failed to sign node certificate")
-	}
-
-	// Ensure directory exists
-	err = os.MkdirAll(filepath.Dir(paths.Cert), 0755)
-	if err != nil {
-		return nil, err
-	}
-
-	// Write both the chain and key to disk
-	if err := ioutils.AtomicWriteFile(paths.Cert, certChain, 0644); err != nil {
-		return nil, err
-	}
-	if err := ioutils.AtomicWriteFile(paths.Key, key, 0600); err != nil {
-		return nil, err
-	}
-
-	// Load a valid tls.Certificate from the chain and the key
-	serverCert, err := tls.X509KeyPair(certChain, key)
-	if err != nil {
-		return nil, err
-	}
-
-	return &serverCert, nil
-}
-
-// GenerateAndWriteNewKey generates a new pub/priv key pair, writes it to disk
-// and returns the CSR and the private key material
-func GenerateAndWriteNewKey(paths CertPaths) (csr, key []byte, err error) {
-	// Generate a new key pair
-	csr, key, err = generateNewCSR()
-	if err != nil {
-		return
-	}
-
-	// Ensure directory exists
-	err = os.MkdirAll(filepath.Dir(paths.Key), 0755)
-	if err != nil {
-		return
-	}
-
-	if err = ioutils.AtomicWriteFile(paths.Key, key, 0600); err != nil {
-		return
-	}
-
-	return
+	return rootCA, nil
 }
 
 // GetRemoteSignedCertificate submits a CSR to a remote CA server address,
@@ -605,18 +556,7 @@ func GetRemoteSignedCertificate(ctx context.Context, csr []byte, token string, r
 		creds = credentials.NewTLS(&tls.Config{ServerName: CARole, RootCAs: rootCAPool})
 	}
 
-	peer, err := r.Select()
-	if err != nil {
-		return nil, err
-	}
-
-	opts := []grpc.DialOption{
-		grpc.WithTransportCredentials(creds),
-		grpc.WithTimeout(5 * time.Second),
-		grpc.WithBackoffMaxDelay(5 * time.Second),
-	}
-
-	conn, err := grpc.Dial(peer.Addr, opts...)
+	conn, peer, err := getGRPCConnection(creds, r)
 	if err != nil {
 		return nil, err
 	}
@@ -681,10 +621,10 @@ func GetRemoteSignedCertificate(ctx context.Context, csr []byte, token string, r
 }
 
 // readCertValidity returns the certificate issue and expiration time
-func readCertValidity(paths CertPaths) (time.Time, time.Time, error) {
+func readCertValidity(kr KeyReader) (time.Time, time.Time, error) {
 	var zeroTime time.Time
 	// Read the Cert
-	cert, err := ioutil.ReadFile(paths.Cert)
+	cert, _, err := kr.Read()
 	if err != nil {
 		return zeroTime, zeroTime, err
 	}
@@ -714,7 +654,8 @@ func saveRootCA(rootCA RootCA, paths CertPaths) error {
 	return ioutils.AtomicWriteFile(paths.Cert, rootCA.Cert, 0644)
 }
 
-func generateNewCSR() (csr, key []byte, err error) {
+// GenerateNewCSR returns a newly generated key and CSR signed with said key
+func GenerateNewCSR() (csr, key []byte, err error) {
 	req := &cfcsr.CertificateRequest{
 		KeyRequest: cfcsr.NewBasicKeyRequest(),
 	}

vendor/github.com/docker/swarmkit/ca/config.go

@@ -6,7 +6,6 @@ import (
 	"crypto/x509"
 	"encoding/pem"
 	"fmt"
-	"io/ioutil"
 	"math/big"
 	"math/rand"
 	"path/filepath"
@@ -33,7 +32,8 @@ const (
 	nodeTLSKeyFilename  = "swarm-node.key"
 	nodeCSRFilename     = "swarm-node.csr"
 
-	rootCN = "swarm-ca"
+	// DefaultRootCN represents the root CN that we should create roots CAs with by default
+	DefaultRootCN = "swarm-ca"
 	// ManagerRole represents the Manager node type, and is used for authorization to endpoints
 	ManagerRole = "swarm-manager"
 	// WorkerRole represents the Worker node type, and is used for authorization to endpoints
@@ -54,8 +54,9 @@ const (
 type SecurityConfig struct {
 	mu sync.Mutex
 
-	rootCA     *RootCA
-	externalCA *ExternalCA
+	rootCA        *RootCA
+	externalCA    *ExternalCA
+	keyReadWriter *KeyReadWriter
 
 	ServerTLSCreds *MutableTLSCreds
 	ClientTLSCreds *MutableTLSCreds
@@ -69,7 +70,7 @@ type CertificateUpdate struct {
 }
 
 // NewSecurityConfig initializes and returns a new SecurityConfig.
-func NewSecurityConfig(rootCA *RootCA, clientTLSCreds, serverTLSCreds *MutableTLSCreds) *SecurityConfig {
+func NewSecurityConfig(rootCA *RootCA, krw *KeyReadWriter, clientTLSCreds, serverTLSCreds *MutableTLSCreds) *SecurityConfig {
 	// Make a new TLS config for the external CA client without a
 	// ServerName value set.
 	clientTLSConfig := clientTLSCreds.Config()
@@ -82,6 +83,7 @@ func NewSecurityConfig(rootCA *RootCA, clientTLSCreds, serverTLSCreds *MutableTL
 
 	return &SecurityConfig{
 		rootCA:         rootCA,
+		keyReadWriter:  krw,
 		externalCA:     NewExternalCA(rootCA, externalCATLSConfig),
 		ClientTLSCreds: clientTLSCreds,
 		ServerTLSCreds: serverTLSCreds,
@@ -96,6 +98,16 @@ func (s *SecurityConfig) RootCA() *RootCA {
 	return s.rootCA
 }
 
+// KeyWriter returns the object that can write keys to disk
+func (s *SecurityConfig) KeyWriter() KeyWriter {
+	return s.keyReadWriter
+}
+
+// KeyReader returns the object that can read keys from disk
+func (s *SecurityConfig) KeyReader() KeyReader {
+	return s.keyReadWriter
+}
+
 // UpdateRootCA replaces the root CA with a new root CA based on the specified
 // certificate, key, and the number of hours the certificates issue should last.
 func (s *SecurityConfig) UpdateRootCA(cert, key []byte, certExpiry time.Duration) error {
@@ -181,70 +193,63 @@ func getCAHashFromToken(token string) (digest.Digest, error) {
 	return digest.ParseDigest(fmt.Sprintf("sha256:%0[1]*s", 64, digestInt.Text(16)))
 }
 
+// DownloadRootCA tries to retrieve a remote root CA and matches the digest against the provided token.
+func DownloadRootCA(ctx context.Context, paths CertPaths, token string, r remotes.Remotes) (RootCA, error) {
+	var rootCA RootCA
+	// Get a digest for the optional CA hash string that we've been provided
+	// If we were provided a non-empty string, and it is an invalid hash, return
+	// otherwise, allow the invalid digest through.
+	var (
+		d   digest.Digest
+		err error
+	)
+	if token != "" {
+		d, err = getCAHashFromToken(token)
+		if err != nil {
+			return RootCA{}, err
+		}
+	}
+	// Get the remote CA certificate, verify integrity with the
+	// hash provided. Retry up to 5 times, in case the manager we
+	// first try to contact is not responding properly (it may have
+	// just been demoted, for example).
+
+	for i := 0; i != 5; i++ {
+		rootCA, err = GetRemoteCA(ctx, d, r)
+		if err == nil {
+			break
+		}
+		log.G(ctx).WithError(err).Errorf("failed to retrieve remote root CA certificate")
+	}
+	if err != nil {
+		return RootCA{}, err
+	}
+
+	// Save root CA certificate to disk
+	if err = saveRootCA(rootCA, paths); err != nil {
+		return RootCA{}, err
+	}
+
+	log.G(ctx).Debugf("retrieved remote CA certificate: %s", paths.Cert)
+	return rootCA, nil
+}
+
 // LoadOrCreateSecurityConfig encapsulates the security logic behind joining a cluster.
 // Every node requires at least a set of TLS certificates with which to join the cluster with.
 // In the case of a manager, these certificates will be used both for client and server credentials.
-func LoadOrCreateSecurityConfig(ctx context.Context, baseCertDir, token, proposedRole string, remotes remotes.Remotes, nodeInfo chan<- api.IssueNodeCertificateResponse) (*SecurityConfig, error) {
+func LoadOrCreateSecurityConfig(ctx context.Context, rootCA RootCA, token, proposedRole string, remotes remotes.Remotes, nodeInfo chan<- api.IssueNodeCertificateResponse, krw *KeyReadWriter) (*SecurityConfig, error) {
 	ctx = log.WithModule(ctx, "tls")
-	paths := NewConfigPaths(baseCertDir)
-
-	var (
-		rootCA                         RootCA
-		serverTLSCreds, clientTLSCreds *MutableTLSCreds
-		err                            error
-	)
-
-	// Check if we already have a CA certificate on disk. We need a CA to have a valid SecurityConfig
-	rootCA, err = GetLocalRootCA(baseCertDir)
-	switch err {
-	case nil:
-		log.G(ctx).Debug("loaded CA certificate")
-	case ErrNoLocalRootCA:
-		log.G(ctx).WithError(err).Debugf("failed to load local CA certificate")
-
-		// Get a digest for the optional CA hash string that we've been provided
-		// If we were provided a non-empty string, and it is an invalid hash, return
-		// otherwise, allow the invalid digest through.
-		var d digest.Digest
-		if token != "" {
-			d, err = getCAHashFromToken(token)
-			if err != nil {
-				return nil, err
-			}
-		}
-
-		// Get the remote CA certificate, verify integrity with the
-		// hash provided. Retry up to 5 times, in case the manager we
-		// first try to contact is not responding properly (it may have
-		// just been demoted, for example).
-
-		for i := 0; i != 5; i++ {
-			rootCA, err = GetRemoteCA(ctx, d, remotes)
-			if err == nil {
-				break
-			}
-			log.G(ctx).WithError(err).Errorf("failed to retrieve remote root CA certificate")
-		}
-		if err != nil {
-			return nil, err
-		}
-
-		// Save root CA certificate to disk
-		if err = saveRootCA(rootCA, paths.RootCA); err != nil {
-			return nil, err
-		}
-
-		log.G(ctx).Debugf("retrieved remote CA certificate: %s", paths.RootCA.Cert)
-	default:
-		return nil, err
-	}
 
 	// At this point we've successfully loaded the CA details from disk, or
 	// successfully downloaded them remotely. The next step is to try to
 	// load our certificates.
-	clientTLSCreds, serverTLSCreds, err = LoadTLSCreds(rootCA, paths.Node)
+	clientTLSCreds, serverTLSCreds, err := LoadTLSCreds(rootCA, krw)
 	if err != nil {
-		log.G(ctx).WithError(err).Debugf("no node credentials found in: %s", paths.Node.Cert)
+		if _, ok := errors.Cause(err).(ErrInvalidKEK); ok {
+			return nil, err
+		}
+
+		log.G(ctx).WithError(err).Debugf("no node credentials found in: %s", krw.Target())
 
 		var (
 			tlsKeyPair *tls.Certificate
@@ -262,7 +267,7 @@ func LoadOrCreateSecurityConfig(ctx context.Context, baseCertDir, token, propose
 					NodeMembership: api.NodeMembershipAccepted,
 				}
 			}
-			tlsKeyPair, err = rootCA.IssueAndSaveNewCertificates(paths.Node, cn, proposedRole, org)
+			tlsKeyPair, err = rootCA.IssueAndSaveNewCertificates(krw, cn, proposedRole, org)
 			if err != nil {
 				log.G(ctx).WithFields(logrus.Fields{
 					"node.id":   cn,
@@ -278,7 +283,7 @@ func LoadOrCreateSecurityConfig(ctx context.Context, baseCertDir, token, propose
 		} else {
 			// There was an error loading our Credentials, let's get a new certificate issued
 			// Last argument is nil because at this point we don't have any valid TLS creds
-			tlsKeyPair, err = rootCA.RequestAndSaveNewCertificates(ctx, paths.Node, token, remotes, nil, nodeInfo)
+			tlsKeyPair, err = rootCA.RequestAndSaveNewCertificates(ctx, krw, token, remotes, nil, nodeInfo)
 			if err != nil {
 				log.G(ctx).WithError(err).Error("failed to request save new certificate")
 				return nil, err
@@ -299,7 +304,7 @@ func LoadOrCreateSecurityConfig(ctx context.Context, baseCertDir, token, propose
 		log.G(ctx).WithFields(logrus.Fields{
 			"node.id":   clientTLSCreds.NodeID(),
 			"node.role": clientTLSCreds.Role(),
-		}).Debugf("new node credentials generated: %s", paths.Node.Cert)
+		}).Debugf("new node credentials generated: %s", krw.Target())
 	} else {
 		if nodeInfo != nil {
 			nodeInfo <- api.IssueNodeCertificateResponse{
@@ -313,13 +318,66 @@ func LoadOrCreateSecurityConfig(ctx context.Context, baseCertDir, token, propose
 		}).Debug("loaded node credentials")
 	}
 
-	return NewSecurityConfig(&rootCA, clientTLSCreds, serverTLSCreds), nil
+	return NewSecurityConfig(&rootCA, krw, clientTLSCreds, serverTLSCreds), nil
+}
+
+// RenewTLSConfigNow gets a new TLS cert and key, and updates the security config if provided.  This is similar to
+// RenewTLSConfig, except while that monitors for expiry, and periodically renews, this renews once and is blocking
+func RenewTLSConfigNow(ctx context.Context, s *SecurityConfig, r remotes.Remotes) error {
+	ctx = log.WithModule(ctx, "tls")
+	log := log.G(ctx).WithFields(logrus.Fields{
+		"node.id":   s.ClientTLSCreds.NodeID(),
+		"node.role": s.ClientTLSCreds.Role(),
+	})
+
+	// Let's request new certs. Renewals don't require a token.
+	rootCA := s.RootCA()
+	tlsKeyPair, err := rootCA.RequestAndSaveNewCertificates(ctx,
+		s.KeyWriter(),
+		"",
+		r,
+		s.ClientTLSCreds,
+		nil)
+	if err != nil {
+		log.WithError(err).Errorf("failed to renew the certificate")
+		return err
+	}
+
+	clientTLSConfig, err := NewClientTLSConfig(tlsKeyPair, rootCA.Pool, CARole)
+	if err != nil {
+		log.WithError(err).Errorf("failed to create a new client config")
+		return err
+	}
+	serverTLSConfig, err := NewServerTLSConfig(tlsKeyPair, rootCA.Pool)
+	if err != nil {
+		log.WithError(err).Errorf("failed to create a new server config")
+		return err
+	}
+
+	if err = s.ClientTLSCreds.LoadNewTLSConfig(clientTLSConfig); err != nil {
+		log.WithError(err).Errorf("failed to update the client credentials")
+		return err
+	}
+
+	// Update the external CA to use the new client TLS
+	// config using a copy without a serverName specified.
+	s.externalCA.UpdateTLSConfig(&tls.Config{
+		Certificates: clientTLSConfig.Certificates,
+		RootCAs:      clientTLSConfig.RootCAs,
+		MinVersion:   tls.VersionTLS12,
+	})
+
+	if err = s.ServerTLSCreds.LoadNewTLSConfig(serverTLSConfig); err != nil {
+		log.WithError(err).Errorf("failed to update the server TLS credentials")
+		return err
+	}
+
+	return nil
 }
 
 // RenewTLSConfig will continuously monitor for the necessity of renewing the local certificates, either by
 // issuing them locally if key-material is available, or requesting them from a remote CA.
-func RenewTLSConfig(ctx context.Context, s *SecurityConfig, baseCertDir string, remotes remotes.Remotes, renew <-chan struct{}) <-chan CertificateUpdate {
-	paths := NewConfigPaths(baseCertDir)
+func RenewTLSConfig(ctx context.Context, s *SecurityConfig, remotes remotes.Remotes, renew <-chan struct{}) <-chan CertificateUpdate {
 	updates := make(chan CertificateUpdate)
 
 	go func() {
@@ -337,10 +395,10 @@ func RenewTLSConfig(ctx context.Context, s *SecurityConfig, baseCertDir string,
 			// Since the expiration of the certificate is managed remotely we should update our
 			// retry timer on every iteration of this loop.
 			// Retrieve the current certificate expiration information.
-			validFrom, validUntil, err := readCertValidity(paths.Node)
+			validFrom, validUntil, err := readCertValidity(s.KeyReader())
 			if err != nil {
 				// We failed to read the expiration, let's stick with the starting default
-				log.Errorf("failed to read the expiration of the TLS certificate in: %s", paths.Node.Cert)
+				log.Errorf("failed to read the expiration of the TLS certificate in: %s", s.KeyReader().Target())
 				updates <- CertificateUpdate{Err: errors.New("failed to read certificate expiration")}
 			} else {
 				// If we have an expired certificate, we let's stick with the starting default in
@@ -368,52 +426,12 @@ func RenewTLSConfig(ctx context.Context, s *SecurityConfig, baseCertDir string,
 				return
 			}
 
-			// Let's request new certs. Renewals don't require a token.
-			rootCA := s.RootCA()
-			tlsKeyPair, err := rootCA.RequestAndSaveNewCertificates(ctx,
-				paths.Node,
-				"",
-				remotes,
-				s.ClientTLSCreds,
-				nil)
-			if err != nil {
-				log.WithError(err).Errorf("failed to renew the certificate")
+			// ignore errors - it will just try again laster
+			if err := RenewTLSConfigNow(ctx, s, remotes); err != nil {
 				updates <- CertificateUpdate{Err: err}
-				continue
+			} else {
+				updates <- CertificateUpdate{Role: s.ClientTLSCreds.Role()}
 			}
-
-			clientTLSConfig, err := NewClientTLSConfig(tlsKeyPair, rootCA.Pool, CARole)
-			if err != nil {
-				log.WithError(err).Errorf("failed to create a new client config")
-				updates <- CertificateUpdate{Err: err}
-			}
-			serverTLSConfig, err := NewServerTLSConfig(tlsKeyPair, rootCA.Pool)
-			if err != nil {
-				log.WithError(err).Errorf("failed to create a new server config")
-				updates <- CertificateUpdate{Err: err}
-			}
-
-			err = s.ClientTLSCreds.LoadNewTLSConfig(clientTLSConfig)
-			if err != nil {
-				log.WithError(err).Errorf("failed to update the client credentials")
-				updates <- CertificateUpdate{Err: err}
-			}
-
-			// Update the external CA to use the new client TLS
-			// config using a copy without a serverName specified.
-			s.externalCA.UpdateTLSConfig(&tls.Config{
-				Certificates: clientTLSConfig.Certificates,
-				RootCAs:      clientTLSConfig.RootCAs,
-				MinVersion:   tls.VersionTLS12,
-			})
-
-			err = s.ServerTLSCreds.LoadNewTLSConfig(serverTLSConfig)
-			if err != nil {
-				log.WithError(err).Errorf("failed to update the server TLS credentials")
-				updates <- CertificateUpdate{Err: err}
-			}
-
-			updates <- CertificateUpdate{Role: s.ClientTLSCreds.Role()}
 		}
 	}()
 
@@ -447,13 +465,9 @@ func calculateRandomExpiry(validFrom, validUntil time.Time) time.Duration {
 
 // LoadTLSCreds loads tls credentials from the specified path and verifies that
 // thay are valid for the RootCA.
-func LoadTLSCreds(rootCA RootCA, paths CertPaths) (*MutableTLSCreds, *MutableTLSCreds, error) {
+func LoadTLSCreds(rootCA RootCA, kr KeyReader) (*MutableTLSCreds, *MutableTLSCreds, error) {
 	// Read both the Cert and Key from disk
-	cert, err := ioutil.ReadFile(paths.Cert)
-	if err != nil {
-		return nil, nil, err
-	}
-	key, err := ioutil.ReadFile(paths.Key)
+	cert, key, err := kr.Read()
 	if err != nil {
 		return nil, nil, err
 	}
@@ -482,24 +496,9 @@ func LoadTLSCreds(rootCA RootCA, paths CertPaths) (*MutableTLSCreds, *MutableTLS
 
 	// Now that we know this certificate is valid, create a TLS Certificate for our
 	// credentials
-	var (
-		keyPair tls.Certificate
-		newErr  error
-	)
-	keyPair, err = tls.X509KeyPair(cert, key)
+	keyPair, err := tls.X509KeyPair(cert, key)
 	if err != nil {
-		// This current keypair isn't valid. It's possible we crashed before we
-		// overwrote the current key. Let's try loading it from disk.
-		tempPaths := genTempPaths(paths)
-		key, newErr = ioutil.ReadFile(tempPaths.Key)
-		if newErr != nil {
-			return nil, nil, err
-		}
-
-		keyPair, newErr = tls.X509KeyPair(cert, key)
-		if newErr != nil {
-			return nil, nil, err
-		}
+		return nil, nil, err
 	}
 
 	// Load the Certificates as server credentials
@@ -519,13 +518,6 @@ func LoadTLSCreds(rootCA RootCA, paths CertPaths) (*MutableTLSCreds, *MutableTLS
 	return clientTLSCreds, serverTLSCreds, nil
 }
 
-func genTempPaths(path CertPaths) CertPaths {
-	return CertPaths{
-		Key:  filepath.Join(filepath.Dir(path.Key), "."+filepath.Base(path.Key)),
-		Cert: filepath.Join(filepath.Dir(path.Cert), "."+filepath.Base(path.Cert)),
-	}
-}
-
 // NewServerTLSConfig returns a tls.Config configured for a TLS Server, given a tls.Certificate
 // and the PEM-encoded root CA Certificate
 func NewServerTLSConfig(cert *tls.Certificate, rootCAPool *x509.CertPool) (*tls.Config, error) {

vendor/github.com/docker/swarmkit/ca/keyreadwriter.go

@@ -0,0 +1,388 @@
+package ca
+
+import (
+	"crypto/rand"
+	"crypto/x509"
+	"encoding/pem"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"sync"
+
+	"crypto/tls"
+
+	"github.com/docker/swarmkit/ioutils"
+	"github.com/pkg/errors"
+)
+
+const (
+	// keyPerms are the permissions used to write the TLS keys
+	keyPerms = 0600
+	// certPerms are the permissions used to write TLS certificates
+	certPerms = 0644
+	// versionHeader is the TLS PEM key header that contains the KEK version
+	versionHeader = "kek-version"
+)
+
+// PEMKeyHeaders is something that needs to know about PEM headers when reading
+// or writing TLS keys.
+type PEMKeyHeaders interface {
+	// UnmarshalHeaders loads the headers map given the current KEK
+	UnmarshalHeaders(map[string]string, KEKData) (PEMKeyHeaders, error)
+	// MarshalHeaders returns a header map given the current KEK
+	MarshalHeaders(KEKData) (map[string]string, error)
+	// UpdateKEK may get a new PEMKeyHeaders if the KEK changes
+	UpdateKEK(KEKData, KEKData) PEMKeyHeaders
+}
+
+// KeyReader reads a TLS cert and key from disk
+type KeyReader interface {
+	Read() ([]byte, []byte, error)
+	Target() string
+}
+
+// KeyWriter writes a TLS key and cert to disk
+type KeyWriter interface {
+	Write([]byte, []byte, *KEKData) error
+	ViewAndUpdateHeaders(func(PEMKeyHeaders) (PEMKeyHeaders, error)) error
+	ViewAndRotateKEK(func(KEKData, PEMKeyHeaders) (KEKData, PEMKeyHeaders, error)) error
+	GetCurrentState() (PEMKeyHeaders, KEKData)
+	Target() string
+}
+
+// KEKData provides an optional update to the kek when writing.  The structure
+// is needed so that we can tell the difference between "do not encrypt anymore"
+// and there is "no update".
+type KEKData struct {
+	KEK     []byte
+	Version uint64
+}
+
+// ErrInvalidKEK means that we cannot decrypt the TLS key for some reason
+type ErrInvalidKEK struct {
+	Wrapped error
+}
+
+func (e ErrInvalidKEK) Error() string {
+	return e.Wrapped.Error()
+}
+
+// KeyReadWriter is an object that knows how to read and write TLS keys and certs to disk,
+// optionally encrypted and optionally updating PEM headers.
+type KeyReadWriter struct {
+	mu         sync.Mutex
+	kekData    KEKData
+	paths      CertPaths
+	headersObj PEMKeyHeaders
+}
+
+// NewKeyReadWriter creates a new KeyReadWriter
+func NewKeyReadWriter(paths CertPaths, kek []byte, headersObj PEMKeyHeaders) *KeyReadWriter {
+	return &KeyReadWriter{
+		kekData:    KEKData{KEK: kek},
+		paths:      paths,
+		headersObj: headersObj,
+	}
+}
+
+// Migrate checks to see if a temporary key file exists.  Older versions of
+// swarmkit wrote temporary keys instead of temporary certificates, so
+// migrate that temporary key if it exists.  We want to write temporary certificates,
+// instead of temporary keys, because we may need to periodically re-encrypt the
+// keys and modify the headers, and it's easier to have a single canonical key
+// location than two possible key locations.
+func (k *KeyReadWriter) Migrate() error {
+	tmpPaths := k.genTempPaths()
+	keyBytes, err := ioutil.ReadFile(tmpPaths.Key)
+	if err != nil {
+		return nil // no key?  no migration
+	}
+
+	// it does exist - no need to decrypt, because previous versions of swarmkit
+	// which supported this temporary key did not support encrypting TLS keys
+	cert, err := ioutil.ReadFile(k.paths.Cert)
+	if err != nil {
+		return os.RemoveAll(tmpPaths.Key) // no cert?  no migration
+	}
+
+	// nope, this does not match the cert
+	if _, err = tls.X509KeyPair(cert, keyBytes); err != nil {
+		return os.RemoveAll(tmpPaths.Key)
+	}
+
+	return os.Rename(tmpPaths.Key, k.paths.Key)
+}
+
+// Read will read a TLS cert and key from the given paths
+func (k *KeyReadWriter) Read() ([]byte, []byte, error) {
+	k.mu.Lock()
+	defer k.mu.Unlock()
+	keyBlock, err := k.readKey()
+	if err != nil {
+		return nil, nil, err
+	}
+
+	if version, ok := keyBlock.Headers[versionHeader]; ok {
+		if versionInt, err := strconv.ParseUint(version, 10, 64); err == nil {
+			k.kekData.Version = versionInt
+		}
+	}
+	delete(keyBlock.Headers, versionHeader)
+
+	if k.headersObj != nil {
+		newHeaders, err := k.headersObj.UnmarshalHeaders(keyBlock.Headers, k.kekData)
+		if err != nil {
+			return nil, nil, errors.Wrap(err, "unable to read TLS key headers")
+		}
+		k.headersObj = newHeaders
+	}
+
+	keyBytes := pem.EncodeToMemory(keyBlock)
+	cert, err := ioutil.ReadFile(k.paths.Cert)
+	// The cert is written to a temporary file first, then the key, and then
+	// the cert gets renamed - so, if interrupted, it's possible to end up with
+	// a cert that only exists in the temporary location.
+	switch {
+	case err == nil:
+		_, err = tls.X509KeyPair(cert, keyBytes)
+	case os.IsNotExist(err): //continue to try temp location
+		break
+	default:
+		return nil, nil, err
+	}
+
+	// either the cert doesn't exist, or it doesn't match the key - try the temp file, if it exists
+	if err != nil {
+		var tempErr error
+		tmpPaths := k.genTempPaths()
+		cert, tempErr = ioutil.ReadFile(tmpPaths.Cert)
+		if tempErr != nil {
+			return nil, nil, err // return the original error
+		}
+		if _, tempErr := tls.X509KeyPair(cert, keyBytes); tempErr != nil {
+			os.RemoveAll(tmpPaths.Cert) // nope, it doesn't match either - remove and return the original error
+			return nil, nil, err
+		}
+		os.Rename(tmpPaths.Cert, k.paths.Cert) // try to move the temp cert back to the regular location
+
+	}
+
+	return cert, keyBytes, nil
+}
+
+// ViewAndRotateKEK re-encrypts the key with a new KEK
+func (k *KeyReadWriter) ViewAndRotateKEK(cb func(KEKData, PEMKeyHeaders) (KEKData, PEMKeyHeaders, error)) error {
+	k.mu.Lock()
+	defer k.mu.Unlock()
+
+	updatedKEK, updatedHeaderObj, err := cb(k.kekData, k.headersObj)
+	if err != nil {
+		return err
+	}
+
+	keyBlock, err := k.readKey()
+	if err != nil {
+		return err
+	}
+
+	if err := k.writeKey(keyBlock, updatedKEK, updatedHeaderObj); err != nil {
+		return err
+	}
+	return nil
+}
+
+// ViewAndUpdateHeaders updates the header manager, and updates any headers on the existing key
+func (k *KeyReadWriter) ViewAndUpdateHeaders(cb func(PEMKeyHeaders) (PEMKeyHeaders, error)) error {
+	k.mu.Lock()
+	defer k.mu.Unlock()
+
+	pkh, err := cb(k.headersObj)
+	if err != nil {
+		return err
+	}
+
+	keyBlock, err := k.readKeyblock()
+	if err != nil {
+		return err
+	}
+
+	headers := make(map[string]string)
+	if pkh != nil {
+		var err error
+		headers, err = pkh.MarshalHeaders(k.kekData)
+		if err != nil {
+			return err
+		}
+	}
+	// we WANT any original encryption headers
+	for key, value := range keyBlock.Headers {
+		normalizedKey := strings.TrimSpace(strings.ToLower(key))
+		if normalizedKey == "proc-type" || normalizedKey == "dek-info" {
+			headers[key] = value
+		}
+	}
+	headers[versionHeader] = strconv.FormatUint(k.kekData.Version, 10)
+	keyBlock.Headers = headers
+
+	if err = ioutils.AtomicWriteFile(k.paths.Key, pem.EncodeToMemory(keyBlock), keyPerms); err != nil {
+		return err
+	}
+	k.headersObj = pkh
+	return nil
+}
+
+// GetCurrentState returns the current KEK data, including version
+func (k *KeyReadWriter) GetCurrentState() (PEMKeyHeaders, KEKData) {
+	k.mu.Lock()
+	defer k.mu.Unlock()
+	return k.headersObj, k.kekData
+}
+
+// Write attempts write a cert and key to text.  This can also optionally update
+// the KEK while writing, if an updated KEK is provided.  If the pointer to the
+// update KEK is nil, then we don't update. If the updated KEK itself is nil,
+// then we update the KEK to be nil (data should be unencrypted).
+func (k *KeyReadWriter) Write(certBytes, plaintextKeyBytes []byte, kekData *KEKData) error {
+	k.mu.Lock()
+	defer k.mu.Unlock()
+
+	// current assumption is that the cert and key will be in the same directory
+	if err := os.MkdirAll(filepath.Dir(k.paths.Key), 0755); err != nil {
+		return err
+	}
+
+	// Ensure that we will have a keypair on disk at all times by writing the cert to a
+	// temp path first.  This is because we want to have only a single copy of the key
+	// for rotation and header modification.
+	tmpPaths := k.genTempPaths()
+	if err := ioutils.AtomicWriteFile(tmpPaths.Cert, certBytes, certPerms); err != nil {
+		return err
+	}
+
+	keyBlock, _ := pem.Decode(plaintextKeyBytes)
+	if keyBlock == nil {
+		return errors.New("invalid PEM-encoded private key")
+	}
+
+	if kekData == nil {
+		kekData = &k.kekData
+	}
+	pkh := k.headersObj
+	if k.headersObj != nil {
+		pkh = k.headersObj.UpdateKEK(k.kekData, *kekData)
+	}
+
+	if err := k.writeKey(keyBlock, *kekData, pkh); err != nil {
+		return err
+	}
+	return os.Rename(tmpPaths.Cert, k.paths.Cert)
+}
+
+func (k *KeyReadWriter) genTempPaths() CertPaths {
+	return CertPaths{
+		Key:  filepath.Join(filepath.Dir(k.paths.Key), "."+filepath.Base(k.paths.Key)),
+		Cert: filepath.Join(filepath.Dir(k.paths.Cert), "."+filepath.Base(k.paths.Cert)),
+	}
+}
+
+// Target returns a string representation of this KeyReadWriter, namely where
+// it is writing to
+func (k *KeyReadWriter) Target() string {
+	return k.paths.Cert
+}
+
+func (k *KeyReadWriter) readKeyblock() (*pem.Block, error) {
+	key, err := ioutil.ReadFile(k.paths.Key)
+	if err != nil {
+		return nil, err
+	}
+
+	// Decode the PEM private key
+	keyBlock, _ := pem.Decode(key)
+	if keyBlock == nil {
+		return nil, errors.New("invalid PEM-encoded private key")
+	}
+
+	return keyBlock, nil
+}
+
+// readKey returns the decrypted key pem bytes, and enforces the KEK if applicable
+// (writes it back with the correct encryption if it is not correctly encrypted)
+func (k *KeyReadWriter) readKey() (*pem.Block, error) {
+	keyBlock, err := k.readKeyblock()
+	if err != nil {
+		return nil, err
+	}
+
+	if !x509.IsEncryptedPEMBlock(keyBlock) {
+		return keyBlock, nil
+	}
+
+	// If it's encrypted, we can't read without a passphrase (we're assuming
+	// empty passphrases iare invalid)
+	if k.kekData.KEK == nil {
+		return nil, ErrInvalidKEK{Wrapped: x509.IncorrectPasswordError}
+	}
+
+	derBytes, err := x509.DecryptPEMBlock(keyBlock, k.kekData.KEK)
+	if err != nil {
+		return nil, ErrInvalidKEK{Wrapped: err}
+	}
+	// remove encryption PEM headers
+	headers := make(map[string]string)
+	mergePEMHeaders(headers, keyBlock.Headers)
+
+	return &pem.Block{
+		Type:    keyBlock.Type, // the key type doesn't change
+		Bytes:   derBytes,
+		Headers: headers,
+	}, nil
+}
+
+// writeKey takes an unencrypted keyblock and, if the kek is not nil, encrypts it before
+// writing it to disk.  If the kek is nil, writes it to disk unencrypted.
+func (k *KeyReadWriter) writeKey(keyBlock *pem.Block, kekData KEKData, pkh PEMKeyHeaders) error {
+	if kekData.KEK != nil {
+		encryptedPEMBlock, err := x509.EncryptPEMBlock(rand.Reader,
+			keyBlock.Type,
+			keyBlock.Bytes,
+			kekData.KEK,
+			x509.PEMCipherAES256)
+		if err != nil {
+			return err
+		}
+		if encryptedPEMBlock.Headers == nil {
+			return errors.New("unable to encrypt key - invalid PEM file produced")
+		}
+		keyBlock = encryptedPEMBlock
+	}
+
+	if pkh != nil {
+		headers, err := pkh.MarshalHeaders(kekData)
+		if err != nil {
+			return err
+		}
+		mergePEMHeaders(keyBlock.Headers, headers)
+	}
+	keyBlock.Headers[versionHeader] = strconv.FormatUint(kekData.Version, 10)
+
+	if err := ioutils.AtomicWriteFile(k.paths.Key, pem.EncodeToMemory(keyBlock), keyPerms); err != nil {
+		return err
+	}
+	k.kekData = kekData
+	k.headersObj = pkh
+	return nil
+}
+
+// merges one set of PEM headers onto another, excepting for key encryption value
+// "proc-type" and "dek-info"
+func mergePEMHeaders(original, newSet map[string]string) {
+	for key, value := range newSet {
+		normalizedKey := strings.TrimSpace(strings.ToLower(key))
+		if normalizedKey != "proc-type" && normalizedKey != "dek-info" {
+			original[key] = value
+		}
+	}
+}

vendor/github.com/docker/swarmkit/ca/server.go

@@ -69,6 +69,33 @@ func (s *Server) SetReconciliationRetryInterval(reconciliationRetryInterval time
 	s.reconciliationRetryInterval = reconciliationRetryInterval
 }
 
+// GetUnlockKey is responsible for returning the current unlock key used for encrypting TLS private keys and
+// other at rest data.  Access to this RPC call should only be allowed via mutual TLS from managers.
+func (s *Server) GetUnlockKey(ctx context.Context, request *api.GetUnlockKeyRequest) (*api.GetUnlockKeyResponse, error) {
+	// This directly queries the store, rather than storing the unlock key and version on
+	// the `Server` object and updating it `updateCluster` is called, because we need this
+	// API to return the latest version of the key.  Otherwise, there might be a slight delay
+	// between when the cluster gets updated, and when this function returns the latest key.
+	// This delay is currently unacceptable because this RPC call is the only way, after a
+	// cluster update, to get the actual value of the unlock key, and we don't want to return
+	// a cached value.
+	resp := api.GetUnlockKeyResponse{}
+	s.store.View(func(tx store.ReadTx) {
+		cluster := store.GetCluster(tx, s.securityConfig.ClientTLSCreds.Organization())
+		resp.Version = cluster.Meta.Version
+		if cluster.Spec.EncryptionConfig.AutoLockManagers {
+			for _, encryptionKey := range cluster.UnlockKeys {
+				if encryptionKey.Subsystem == ManagerRole {
+					resp.UnlockKey = encryptionKey.Key
+					return
+				}
+			}
+		}
+	})
+
+	return &resp, nil
+}
+
 // NodeCertificateStatus returns the current issuance status of an issuance request identified by the nodeID
 func (s *Server) NodeCertificateStatus(ctx context.Context, request *api.NodeCertificateStatusRequest) (*api.NodeCertificateStatusResponse, error) {
 	if request.NodeID == "" {

vendor/github.com/docker/swarmkit/manager/controlapi/cluster.go

@@ -6,6 +6,7 @@ import (
 
 	"github.com/docker/swarmkit/api"
 	"github.com/docker/swarmkit/ca"
+	"github.com/docker/swarmkit/manager/encryption"
 	"github.com/docker/swarmkit/manager/state/store"
 	"github.com/docker/swarmkit/protobuf/ptypes"
 	"golang.org/x/net/context"
@@ -107,12 +108,38 @@ func (s *Server) UpdateCluster(ctx context.Context, request *api.UpdateClusterRe
 
 		expireBlacklistedCerts(cluster)
 
-		if request.Rotation.RotateWorkerToken {
+		if request.Rotation.WorkerJoinToken {
 			cluster.RootCA.JoinTokens.Worker = ca.GenerateJoinToken(s.rootCA)
 		}
-		if request.Rotation.RotateManagerToken {
+		if request.Rotation.ManagerJoinToken {
 			cluster.RootCA.JoinTokens.Manager = ca.GenerateJoinToken(s.rootCA)
 		}
+
+		var unlockKeys []*api.EncryptionKey
+		var managerKey *api.EncryptionKey
+		for _, eKey := range cluster.UnlockKeys {
+			if eKey.Subsystem == ca.ManagerRole {
+				if !cluster.Spec.EncryptionConfig.AutoLockManagers {
+					continue
+				}
+				managerKey = eKey
+			}
+			unlockKeys = append(unlockKeys, eKey)
+		}
+
+		switch {
+		case !cluster.Spec.EncryptionConfig.AutoLockManagers:
+			break
+		case managerKey == nil:
+			unlockKeys = append(unlockKeys, &api.EncryptionKey{
+				Subsystem: ca.ManagerRole,
+				Key:       encryption.GenerateSecretKey(),
+			})
+		case request.Rotation.ManagerUnlockKey:
+			managerKey.Key = encryption.GenerateSecretKey()
+		}
+		cluster.UnlockKeys = unlockKeys
+
 		return store.UpdateCluster(tx, cluster)
 	})
 	if err != nil {

vendor/github.com/docker/swarmkit/manager/deks.go

@@ -0,0 +1,269 @@
+package manager
+
+import (
+	"crypto/subtle"
+	"encoding/base64"
+	"fmt"
+
+	"github.com/docker/swarmkit/ca"
+	"github.com/docker/swarmkit/manager/encryption"
+	"github.com/docker/swarmkit/manager/state/raft"
+)
+
+const (
+	// the raft DEK (data encryption key) is stored in the TLS key as a header
+	// these are the header values
+	pemHeaderRaftDEK              = "raft-dek"
+	pemHeaderRaftPendingDEK       = "raft-dek-pending"
+	pemHeaderRaftDEKNeedsRotation = "raft-dek-needs-rotation"
+)
+
+// RaftDEKData contains all the data stored in TLS pem headers
+type RaftDEKData struct {
+	raft.EncryptionKeys
+	NeedsRotation bool
+}
+
+// UnmarshalHeaders loads the state of the DEK manager given the current TLS headers
+func (r RaftDEKData) UnmarshalHeaders(headers map[string]string, kekData ca.KEKData) (ca.PEMKeyHeaders, error) {
+	var (
+		currentDEK, pendingDEK []byte
+		err                    error
+	)
+
+	if currentDEKStr, ok := headers[pemHeaderRaftDEK]; ok {
+		currentDEK, err = decodePEMHeaderValue(currentDEKStr, kekData.KEK)
+		if err != nil {
+			return nil, err
+		}
+	}
+	if pendingDEKStr, ok := headers[pemHeaderRaftPendingDEK]; ok {
+		pendingDEK, err = decodePEMHeaderValue(pendingDEKStr, kekData.KEK)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	if pendingDEK != nil && currentDEK == nil {
+		return nil, fmt.Errorf("there is a pending DEK, but no current DEK")
+	}
+
+	_, ok := headers[pemHeaderRaftDEKNeedsRotation]
+	return RaftDEKData{
+		NeedsRotation: ok,
+		EncryptionKeys: raft.EncryptionKeys{
+			CurrentDEK: currentDEK,
+			PendingDEK: pendingDEK,
+		},
+	}, nil
+}
+
+// MarshalHeaders returns new headers given the current KEK
+func (r RaftDEKData) MarshalHeaders(kekData ca.KEKData) (map[string]string, error) {
+	headers := make(map[string]string)
+	for headerKey, contents := range map[string][]byte{
+		pemHeaderRaftDEK:        r.CurrentDEK,
+		pemHeaderRaftPendingDEK: r.PendingDEK,
+	} {
+		if contents != nil {
+			dekStr, err := encodePEMHeaderValue(contents, kekData.KEK)
+			if err != nil {
+				return nil, err
+			}
+			headers[headerKey] = dekStr
+		}
+	}
+
+	if r.NeedsRotation {
+		headers[pemHeaderRaftDEKNeedsRotation] = "true"
+	}
+
+	// return a function that updates the dek data on write success
+	return headers, nil
+}
+
+// UpdateKEK optionally sets NeedRotation to true if we go from unlocked to locked
+func (r RaftDEKData) UpdateKEK(oldKEK, candidateKEK ca.KEKData) ca.PEMKeyHeaders {
+	if _, unlockedToLocked, err := compareKEKs(oldKEK, candidateKEK); err == nil && unlockedToLocked {
+		return RaftDEKData{
+			EncryptionKeys: r.EncryptionKeys,
+			NeedsRotation:  true,
+		}
+	}
+	return r
+}
+
+// Returns whether the old KEK should be replaced with the new KEK, whether we went from
+// unlocked to locked, and whether there was an error (the versions are the same, but the
+// keks are different)
+func compareKEKs(oldKEK, candidateKEK ca.KEKData) (bool, bool, error) {
+	keksEqual := subtle.ConstantTimeCompare(oldKEK.KEK, candidateKEK.KEK) == 1
+	switch {
+	case oldKEK.Version == candidateKEK.Version && !keksEqual:
+		return false, false, fmt.Errorf("candidate KEK has the same version as the current KEK, but a different KEK value")
+	case oldKEK.Version >= candidateKEK.Version || keksEqual:
+		return false, false, nil
+	default:
+		return true, oldKEK.KEK == nil, nil
+	}
+}
+
+// RaftDEKManager manages the raft DEK keys using TLS headers
+type RaftDEKManager struct {
+	kw         ca.KeyWriter
+	rotationCh chan struct{}
+}
+
+var errNoUpdateNeeded = fmt.Errorf("don't need to rotate or update")
+
+// this error is returned if the KeyReadWriter's PEMKeyHeaders object is no longer a RaftDEKData object -
+// this can happen if the node is no longer a manager, for example
+var errNotUsingRaftDEKData = fmt.Errorf("RaftDEKManager can no longer store and manage TLS key headers")
+
+// NewRaftDEKManager returns a RaftDEKManager that uses the current key writer
+// and header manager
+func NewRaftDEKManager(kw ca.KeyWriter) (*RaftDEKManager, error) {
+	// If there is no current DEK, generate one and write it to disk
+	err := kw.ViewAndUpdateHeaders(func(h ca.PEMKeyHeaders) (ca.PEMKeyHeaders, error) {
+		dekData, ok := h.(RaftDEKData)
+		// it wasn't a raft DEK manager before - just replace it
+		if !ok || dekData.CurrentDEK == nil {
+			return RaftDEKData{
+				EncryptionKeys: raft.EncryptionKeys{
+					CurrentDEK: encryption.GenerateSecretKey(),
+				},
+			}, nil
+		}
+		return nil, errNoUpdateNeeded
+	})
+	if err != nil && err != errNoUpdateNeeded {
+		return nil, err
+	}
+	return &RaftDEKManager{
+		kw:         kw,
+		rotationCh: make(chan struct{}, 1),
+	}, nil
+}
+
+// NeedsRotation returns a boolean about whether we should do a rotation
+func (r *RaftDEKManager) NeedsRotation() bool {
+	h, _ := r.kw.GetCurrentState()
+	data, ok := h.(RaftDEKData)
+	if !ok {
+		return false
+	}
+	return data.NeedsRotation || data.EncryptionKeys.PendingDEK != nil
+}
+
+// GetKeys returns the current set of DEKs.  If NeedsRotation is true, and there
+// is no existing PendingDEK, it will try to create one.  If there are any errors
+// doing so, just return the original.
+func (r *RaftDEKManager) GetKeys() raft.EncryptionKeys {
+	var newKeys, originalKeys raft.EncryptionKeys
+	err := r.kw.ViewAndUpdateHeaders(func(h ca.PEMKeyHeaders) (ca.PEMKeyHeaders, error) {
+		data, ok := h.(RaftDEKData)
+		if !ok {
+			return nil, errNotUsingRaftDEKData
+		}
+		originalKeys = data.EncryptionKeys
+		if !data.NeedsRotation || data.PendingDEK != nil {
+			return nil, errNoUpdateNeeded
+		}
+		newKeys = raft.EncryptionKeys{
+			CurrentDEK: data.CurrentDEK,
+			PendingDEK: encryption.GenerateSecretKey(),
+		}
+		return RaftDEKData{EncryptionKeys: newKeys}, nil
+	})
+	if err != nil {
+		return originalKeys
+	}
+	return newKeys
+}
+
+// RotationNotify the channel used to notify subscribers as to whether there
+// should be a rotation done
+func (r *RaftDEKManager) RotationNotify() chan struct{} {
+	return r.rotationCh
+}
+
+// UpdateKeys will set the updated encryption keys in the headers.  This finishes
+// a rotation, and is expected to set the CurrentDEK to the previous PendingDEK.
+func (r *RaftDEKManager) UpdateKeys(newKeys raft.EncryptionKeys) error {
+	return r.kw.ViewAndUpdateHeaders(func(h ca.PEMKeyHeaders) (ca.PEMKeyHeaders, error) {
+		data, ok := h.(RaftDEKData)
+		if !ok {
+			return nil, errNotUsingRaftDEKData
+		}
+		// If there is no current DEK, we are basically wiping out all DEKs (no header object)
+		if newKeys.CurrentDEK == nil {
+			return nil, nil
+		}
+		return RaftDEKData{
+			EncryptionKeys: newKeys,
+			NeedsRotation:  data.NeedsRotation,
+		}, nil
+	})
+}
+
+// MaybeUpdateKEK does a KEK rotation if one is required.  Returns whether
+// the kek was updated, whether it went from unlocked to locked, and any errors.
+func (r *RaftDEKManager) MaybeUpdateKEK(candidateKEK ca.KEKData) (bool, bool, error) {
+	var updated, unlockedToLocked bool
+	err := r.kw.ViewAndRotateKEK(func(currentKEK ca.KEKData, h ca.PEMKeyHeaders) (ca.KEKData, ca.PEMKeyHeaders, error) {
+		var err error
+		updated, unlockedToLocked, err = compareKEKs(currentKEK, candidateKEK)
+		if err == nil && !updated { // if we don't need to rotate the KEK, don't bother updating
+			err = errNoUpdateNeeded
+		}
+		if err != nil {
+			return ca.KEKData{}, nil, err
+		}
+
+		data, ok := h.(RaftDEKData)
+		if !ok {
+			return ca.KEKData{}, nil, errNotUsingRaftDEKData
+		}
+
+		if unlockedToLocked {
+			data.NeedsRotation = true
+		}
+		return candidateKEK, data, nil
+	})
+	if err == errNoUpdateNeeded {
+		err = nil
+	}
+
+	if err == nil && unlockedToLocked {
+		r.rotationCh <- struct{}{}
+	}
+	return updated, unlockedToLocked, err
+}
+
+func decodePEMHeaderValue(headerValue string, kek []byte) ([]byte, error) {
+	var decrypter encryption.Decrypter = encryption.NoopCrypter
+	if kek != nil {
+		_, decrypter = encryption.Defaults(kek)
+	}
+	valueBytes, err := base64.StdEncoding.DecodeString(headerValue)
+	if err != nil {
+		return nil, err
+	}
+	result, err := encryption.Decrypt(valueBytes, decrypter)
+	if err != nil {
+		return nil, ca.ErrInvalidKEK{Wrapped: err}
+	}
+	return result, nil
+}
+
+func encodePEMHeaderValue(headerValue []byte, kek []byte) (string, error) {
+	var encrypter encryption.Encrypter = encryption.NoopCrypter
+	if kek != nil {
+		encrypter, _ = encryption.Defaults(kek)
+	}
+	encrypted, err := encryption.Encrypt(headerValue, encrypter)
+	if err != nil {
+		return "", err
+	}
+	return base64.StdEncoding.EncodeToString(encrypted), nil
+}

vendor/github.com/docker/swarmkit/manager/encryption/encryption.go

@@ -0,0 +1,132 @@
+package encryption
+
+import (
+	"crypto/rand"
+	"encoding/base64"
+	"fmt"
+	"io"
+	"strings"
+
+	"github.com/docker/swarmkit/api"
+	"github.com/gogo/protobuf/proto"
+	"github.com/pkg/errors"
+)
+
+// This package defines the interfaces and encryption package
+
+const humanReadablePrefix = "SWMKEY-1-"
+
+// ErrCannotDecrypt is the type of error returned when some data cannot be decryptd as plaintext
+type ErrCannotDecrypt struct {
+	msg string
+}
+
+func (e ErrCannotDecrypt) Error() string {
+	return e.msg
+}
+
+// A Decrypter can decrypt an encrypted record
+type Decrypter interface {
+	Decrypt(api.MaybeEncryptedRecord) ([]byte, error)
+}
+
+// A Encrypter can encrypt some bytes into an encrypted record
+type Encrypter interface {
+	Encrypt(data []byte) (*api.MaybeEncryptedRecord, error)
+}
+
+type noopCrypter struct{}
+
+func (n noopCrypter) Decrypt(e api.MaybeEncryptedRecord) ([]byte, error) {
+	if e.Algorithm != n.Algorithm() {
+		return nil, fmt.Errorf("record is encrypted")
+	}
+	return e.Data, nil
+}
+
+func (n noopCrypter) Encrypt(data []byte) (*api.MaybeEncryptedRecord, error) {
+	return &api.MaybeEncryptedRecord{
+		Algorithm: n.Algorithm(),
+		Data:      data,
+	}, nil
+}
+
+func (n noopCrypter) Algorithm() api.MaybeEncryptedRecord_Algorithm {
+	return api.MaybeEncryptedRecord_NotEncrypted
+}
+
+// NoopCrypter is just a pass-through crypter - it does not actually encrypt or
+// decrypt any data
+var NoopCrypter = noopCrypter{}
+
+// Decrypt turns a slice of bytes serialized as an MaybeEncryptedRecord into a slice of plaintext bytes
+func Decrypt(encryptd []byte, decrypter Decrypter) ([]byte, error) {
+	if decrypter == nil {
+		return nil, ErrCannotDecrypt{msg: "no decrypter specified"}
+	}
+	r := api.MaybeEncryptedRecord{}
+	if err := proto.Unmarshal(encryptd, &r); err != nil {
+		// nope, this wasn't marshalled as a MaybeEncryptedRecord
+		return nil, ErrCannotDecrypt{msg: "unable to unmarshal as MaybeEncryptedRecord"}
+	}
+	plaintext, err := decrypter.Decrypt(r)
+	if err != nil {
+		return nil, ErrCannotDecrypt{msg: err.Error()}
+	}
+	return plaintext, nil
+}
+
+// Encrypt turns a slice of bytes into a serialized MaybeEncryptedRecord slice of bytes
+func Encrypt(plaintext []byte, encrypter Encrypter) ([]byte, error) {
+	if encrypter == nil {
+		return nil, fmt.Errorf("no encrypter specified")
+	}
+
+	encryptedRecord, err := encrypter.Encrypt(plaintext)
+	if err != nil {
+		return nil, errors.Wrap(err, "unable to encrypt data")
+	}
+
+	data, err := proto.Marshal(encryptedRecord)
+	if err != nil {
+		return nil, errors.Wrap(err, "unable to marshal as MaybeEncryptedRecord")
+	}
+
+	return data, nil
+}
+
+// Defaults returns a default encrypter and decrypter
+func Defaults(key []byte) (Encrypter, Decrypter) {
+	n := NewNACLSecretbox(key)
+	return n, n
+}
+
+// GenerateSecretKey generates a secret key that can be used for encrypting data
+// using this package
+func GenerateSecretKey() []byte {
+	secretData := make([]byte, naclSecretboxKeySize)
+	if _, err := io.ReadFull(rand.Reader, secretData); err != nil {
+		// panic if we can't read random data
+		panic(errors.Wrap(err, "failed to read random bytes"))
+	}
+	return secretData
+}
+
+// HumanReadableKey displays a secret key in a human readable way
+func HumanReadableKey(key []byte) string {
+	// base64-encode the key
+	return humanReadablePrefix + base64.RawStdEncoding.EncodeToString(key)
+}
+
+// ParseHumanReadableKey returns a key as bytes from recognized serializations of
+// said keys
+func ParseHumanReadableKey(key string) ([]byte, error) {
+	if !strings.HasPrefix(key, humanReadablePrefix) {
+		return nil, fmt.Errorf("invalid key string")
+	}
+	keyBytes, err := base64.RawStdEncoding.DecodeString(strings.TrimPrefix(key, humanReadablePrefix))
+	if err != nil {
+		return nil, fmt.Errorf("invalid key string")
+	}
+	return keyBytes, nil
+}

vendor/github.com/docker/swarmkit/manager/encryption/nacl.go

@@ -0,0 +1,73 @@
+package encryption
+
+import (
+	"crypto/rand"
+	"fmt"
+	"io"
+
+	"github.com/docker/swarmkit/api"
+
+	"golang.org/x/crypto/nacl/secretbox"
+)
+
+const naclSecretboxKeySize = 32
+const naclSecretboxNonceSize = 24
+
+// This provides the default implementation of an encrypter and decrypter, as well
+// as the default KDF function.
+
+// NACLSecretbox is an implementation of an encrypter/decrypter.  Encrypting
+// generates random Nonces.
+type NACLSecretbox struct {
+	key [naclSecretboxKeySize]byte
+}
+
+// NewNACLSecretbox returns a new NACL secretbox encrypter/decrypter with the given key
+func NewNACLSecretbox(key []byte) NACLSecretbox {
+	secretbox := NACLSecretbox{}
+	copy(secretbox.key[:], key)
+	return secretbox
+}
+
+// Algorithm returns the type of algorhtm this is (NACL Secretbox using XSalsa20 and Poly1305)
+func (n NACLSecretbox) Algorithm() api.MaybeEncryptedRecord_Algorithm {
+	return api.MaybeEncryptedRecord_NACLSecretboxSalsa20Poly1305
+}
+
+// Encrypt encrypts some bytes and returns an encrypted record
+func (n NACLSecretbox) Encrypt(data []byte) (*api.MaybeEncryptedRecord, error) {
+	var nonce [24]byte
+	if _, err := io.ReadFull(rand.Reader, nonce[:]); err != nil {
+		return nil, err
+	}
+
+	// Seal's first argument is an "out", the data that the new encrypted message should be
+	// appended to.  Since we don't want to append anything, we pass nil.
+	encrypted := secretbox.Seal(nil, data, &nonce, &n.key)
+	return &api.MaybeEncryptedRecord{
+		Algorithm: n.Algorithm(),
+		Data:      encrypted,
+		Nonce:     nonce[:],
+	}, nil
+}
+
+// Decrypt decrypts a MaybeEncryptedRecord and returns some bytes
+func (n NACLSecretbox) Decrypt(record api.MaybeEncryptedRecord) ([]byte, error) {
+	if record.Algorithm != n.Algorithm() {
+		return nil, fmt.Errorf("not a NACL secretbox record")
+	}
+	if len(record.Nonce) != naclSecretboxNonceSize {
+		return nil, fmt.Errorf("invalid nonce size for NACL secretbox: require 24, got %d", len(record.Nonce))
+	}
+
+	var decryptNonce [naclSecretboxNonceSize]byte
+	copy(decryptNonce[:], record.Nonce[:naclSecretboxNonceSize])
+
+	// Open's first argument is an "out", the data that the decrypted message should be
+	// appended to.  Since we don't want to append anything, we pass nil.
+	decrypted, ok := secretbox.Open(nil, record.Data, &decryptNonce, &n.key)
+	if !ok {
+		return nil, fmt.Errorf("decryption error using NACL secretbox")
+	}
+	return decrypted, nil
+}

vendor/github.com/docker/swarmkit/manager/manager.go

@@ -29,9 +29,11 @@ import (
 	"github.com/docker/swarmkit/manager/orchestrator/taskreaper"
 	"github.com/docker/swarmkit/manager/resourceapi"
 	"github.com/docker/swarmkit/manager/scheduler"
+	"github.com/docker/swarmkit/manager/state"
 	"github.com/docker/swarmkit/manager/state/raft"
 	"github.com/docker/swarmkit/manager/state/store"
 	"github.com/docker/swarmkit/protobuf/ptypes"
+	"github.com/docker/swarmkit/remotes"
 	"github.com/docker/swarmkit/xnet"
 	"github.com/pkg/errors"
 	"golang.org/x/net/context"
@@ -86,6 +88,17 @@ type Config struct {
 	// HeartbeatTick defines the amount of ticks between each
 	// heartbeat sent to other members for health-check purposes
 	HeartbeatTick uint32
+
+	// AutoLockManagers determines whether or not managers require an unlock key
+	// when starting from a stopped state.  This configuration parameter is only
+	// applicable when bootstrapping a new cluster for the first time.
+	AutoLockManagers bool
+
+	// UnlockKey is the key to unlock a node - used for decrypting manager TLS keys
+	// as well as the raft data encryption key (DEK).  It is applicable when
+	// bootstrapping a cluster for the first time (it's a cluster-wide setting),
+	// and also when loading up any raft data on disk (as a KEK for the raft DEK).
+	UnlockKey []byte
 }
 
 // Manager is the cluster manager for Swarm.
@@ -108,6 +121,7 @@ type Manager struct {
 	server                 *grpc.Server
 	localserver            *grpc.Server
 	raftNode               *raft.Node
+	dekRotator             *RaftDEKManager
 
 	cancelFunc context.CancelFunc
 
@@ -217,6 +231,11 @@ func New(config *Config) (*Manager, error) {
 		raftCfg.HeartbeatTick = int(config.HeartbeatTick)
 	}
 
+	dekRotator, err := NewRaftDEKManager(config.SecurityConfig.KeyWriter())
+	if err != nil {
+		return nil, err
+	}
+
 	newNodeOpts := raft.NodeOptions{
 		ID:              config.SecurityConfig.ClientTLSCreds.NodeID(),
 		Addr:            advertiseAddr,
@@ -225,6 +244,7 @@ func New(config *Config) (*Manager, error) {
 		StateDir:        raftStateDir,
 		ForceNewCluster: config.ForceNewCluster,
 		TLSCredentials:  config.SecurityConfig.ClientTLSCreds,
+		KeyRotator:      dekRotator,
 	}
 	raftNode := raft.NewNode(newNodeOpts)
 
@@ -241,6 +261,7 @@ func New(config *Config) (*Manager, error) {
 		localserver: grpc.NewServer(opts...),
 		raftNode:    raftNode,
 		started:     make(chan struct{}),
+		dekRotator:  dekRotator,
 	}
 
 	return m, nil
@@ -320,6 +341,7 @@ func (m *Manager) Run(parent context.Context) error {
 	forwardAsOwnRequest := func(ctx context.Context) (context.Context, error) { return ctx, nil }
 	localProxyControlAPI := api.NewRaftProxyControlServer(baseControlAPI, m.raftNode, forwardAsOwnRequest)
 	localProxyLogsAPI := api.NewRaftProxyLogsServer(m.logbroker, m.raftNode, forwardAsOwnRequest)
+	localCAAPI := api.NewRaftProxyCAServer(m.caserver, m.raftNode, forwardAsOwnRequest)
 
 	// Everything registered on m.server should be an authenticated
 	// wrapper, or a proxy wrapping an authenticated wrapper!
@@ -337,6 +359,7 @@ func (m *Manager) Run(parent context.Context) error {
 	api.RegisterControlServer(m.localserver, localProxyControlAPI)
 	api.RegisterLogsServer(m.localserver, localProxyLogsAPI)
 	api.RegisterHealthServer(m.localserver, localHealthServer)
+	api.RegisterCAServer(m.localserver, localCAAPI)
 
 	healthServer.SetServingStatus("Raft", api.HealthCheckResponse_NOT_SERVING)
 	localHealthServer.SetServingStatus("ControlAPI", api.HealthCheckResponse_NOT_SERVING)
@@ -362,8 +385,12 @@ func (m *Manager) Run(parent context.Context) error {
 
 	close(m.started)
 
+	watchDone := make(chan struct{})
+	watchCtx, watchCtxCancel := context.WithCancel(parent)
 	go func() {
 		err := m.raftNode.Run(ctx)
+		watchCtxCancel()
+		<-watchDone
 		if err != nil {
 			log.G(ctx).Error(err)
 			m.Stop(ctx)
@@ -380,6 +407,10 @@ func (m *Manager) Run(parent context.Context) error {
 	}
 	raftConfig := c.Spec.Raft
 
+	if err := m.watchForKEKChanges(watchCtx, watchDone); err != nil {
+		return err
+	}
+
 	if int(raftConfig.ElectionTick) != m.raftNode.Config.ElectionTick {
 		log.G(ctx).Warningf("election tick value (%ds) is different from the one defined in the cluster config (%vs), the cluster may be unstable", m.raftNode.Config.ElectionTick, raftConfig.ElectionTick)
 	}
@@ -475,6 +506,78 @@ func (m *Manager) Stop(ctx context.Context) {
 	// mutex is released and Run can return now
 }
 
+func (m *Manager) updateKEK(ctx context.Context, cluster *api.Cluster) error {
+	securityConfig := m.config.SecurityConfig
+	nodeID := m.config.SecurityConfig.ClientTLSCreds.NodeID()
+	logger := log.G(ctx).WithFields(logrus.Fields{
+		"node.id":   nodeID,
+		"node.role": ca.ManagerRole,
+	})
+
+	// we are our own peer from which we get certs - try to connect over the local socket
+	r := remotes.NewRemotes(api.Peer{Addr: m.Addr(), NodeID: nodeID})
+
+	kekData := ca.KEKData{Version: cluster.Meta.Version.Index}
+	for _, encryptionKey := range cluster.UnlockKeys {
+		if encryptionKey.Subsystem == ca.ManagerRole {
+			kekData.KEK = encryptionKey.Key
+			break
+		}
+	}
+	updated, unlockedToLocked, err := m.dekRotator.MaybeUpdateKEK(kekData)
+	if err != nil {
+		logger.WithError(err).Errorf("failed to re-encrypt TLS key with a new KEK")
+		return err
+	}
+	if updated {
+		logger.Debug("successfully rotated KEK")
+	}
+	if unlockedToLocked {
+		// a best effort attempt to update the TLS certificate - if it fails, it'll be updated the next time it renews;
+		// don't wait because it might take a bit
+		go func() {
+			if err := ca.RenewTLSConfigNow(ctx, securityConfig, r); err != nil {
+				logger.WithError(err).Errorf("failed to download new TLS certificate after locking the cluster")
+			}
+		}()
+	}
+	return nil
+}
+
+func (m *Manager) watchForKEKChanges(ctx context.Context, watchDone chan struct{}) error {
+	defer close(watchDone)
+	clusterID := m.config.SecurityConfig.ClientTLSCreds.Organization()
+	clusterWatch, clusterWatchCancel, err := store.ViewAndWatch(m.raftNode.MemoryStore(),
+		func(tx store.ReadTx) error {
+			cluster := store.GetCluster(tx, clusterID)
+			if cluster == nil {
+				return fmt.Errorf("unable to get current cluster")
+			}
+			return m.updateKEK(ctx, cluster)
+		},
+		state.EventUpdateCluster{
+			Cluster: &api.Cluster{ID: clusterID},
+			Checks:  []state.ClusterCheckFunc{state.ClusterCheckID},
+		},
+	)
+	if err != nil {
+		return err
+	}
+	go func() {
+		for {
+			select {
+			case event := <-clusterWatch:
+				clusterEvent := event.(state.EventUpdateCluster)
+				m.updateKEK(ctx, clusterEvent.Cluster)
+			case <-ctx.Done():
+				clusterWatchCancel()
+				return
+			}
+		}
+	}()
+	return nil
+}
+
 // rotateRootCAKEK will attempt to rotate the key-encryption-key for root CA key-material in raft.
 // If there is no passphrase set in ENV, it returns.
 // If there is plain-text root key-material, and a passphrase set, it encrypts it.
@@ -625,12 +728,26 @@ func (m *Manager) becomeLeader(ctx context.Context) {
 	initialCAConfig := ca.DefaultCAConfig()
 	initialCAConfig.ExternalCAs = m.config.ExternalCAs
 
+	var unlockKeys []*api.EncryptionKey
+	if m.config.AutoLockManagers {
+		unlockKeys = []*api.EncryptionKey{{
+			Subsystem: ca.ManagerRole,
+			Key:       m.config.UnlockKey,
+		}}
+	}
+
 	s.Update(func(tx store.Tx) error {
 		// Add a default cluster object to the
 		// store. Don't check the error because
 		// we expect this to fail unless this
 		// is a brand new cluster.
-		store.CreateCluster(tx, defaultClusterObject(clusterID, initialCAConfig, raftCfg, rootCA))
+		store.CreateCluster(tx, defaultClusterObject(
+			clusterID,
+			initialCAConfig,
+			raftCfg,
+			api.EncryptionConfig{AutoLockManagers: m.config.AutoLockManagers},
+			unlockKeys,
+			rootCA))
 		// Add Node entry for ourself, if one
 		// doesn't exist already.
 		store.CreateNode(tx, managerNode(nodeID))
@@ -759,7 +876,14 @@ func (m *Manager) becomeFollower() {
 }
 
 // defaultClusterObject creates a default cluster.
-func defaultClusterObject(clusterID string, initialCAConfig api.CAConfig, raftCfg api.RaftConfig, rootCA *ca.RootCA) *api.Cluster {
+func defaultClusterObject(
+	clusterID string,
+	initialCAConfig api.CAConfig,
+	raftCfg api.RaftConfig,
+	encryptionConfig api.EncryptionConfig,
+	initialUnlockKeys []*api.EncryptionKey,
+	rootCA *ca.RootCA) *api.Cluster {
+
 	return &api.Cluster{
 		ID: clusterID,
 		Spec: api.ClusterSpec{
@@ -772,8 +896,9 @@ func defaultClusterObject(clusterID string, initialCAConfig api.CAConfig, raftCf
 			Dispatcher: api.DispatcherConfig{
 				HeartbeatPeriod: ptypes.DurationProto(dispatcher.DefaultHeartBeatPeriod),
 			},
-			Raft:     raftCfg,
-			CAConfig: initialCAConfig,
+			Raft:             raftCfg,
+			CAConfig:         initialCAConfig,
+			EncryptionConfig: encryptionConfig,
 		},
 		RootCA: api.RootCA{
 			CAKey:      rootCA.Key,
@@ -784,6 +909,7 @@ func defaultClusterObject(clusterID string, initialCAConfig api.CAConfig, raftCf
 				Manager: ca.GenerateJoinToken(rootCA),
 			},
 		},
+		UnlockKeys: initialUnlockKeys,
 	}
 }
 

vendor/github.com/docker/swarmkit/manager/state/raft/raft.go

@@ -20,14 +20,13 @@ import (
 	"github.com/coreos/etcd/pkg/idutil"
 	"github.com/coreos/etcd/raft"
 	"github.com/coreos/etcd/raft/raftpb"
-	"github.com/coreos/etcd/snap"
-	"github.com/coreos/etcd/wal"
 	"github.com/docker/go-events"
 	"github.com/docker/swarmkit/api"
 	"github.com/docker/swarmkit/ca"
 	"github.com/docker/swarmkit/log"
 	"github.com/docker/swarmkit/manager/raftselector"
 	"github.com/docker/swarmkit/manager/state/raft/membership"
+	"github.com/docker/swarmkit/manager/state/raft/storage"
 	"github.com/docker/swarmkit/manager/state/store"
 	"github.com/docker/swarmkit/watch"
 	"github.com/gogo/protobuf/proto"
@@ -75,6 +74,21 @@ const (
 	IsFollower
 )
 
+// EncryptionKeys are the current and, if necessary, pending DEKs with which to
+// encrypt raft data
+type EncryptionKeys struct {
+	CurrentDEK []byte
+	PendingDEK []byte
+}
+
+// EncryptionKeyRotator is an interface to find out if any keys need rotating.
+type EncryptionKeyRotator interface {
+	GetKeys() EncryptionKeys
+	UpdateKeys(EncryptionKeys) error
+	NeedsRotation() bool
+	RotationNotify() chan struct{}
+}
+
 // Node represents the Raft Node useful
 // configuration.
 type Node struct {
@@ -87,8 +101,6 @@ type Node struct {
 	opts                NodeOptions
 	reqIDGen            *idutil.Generator
 	wait                *wait
-	wal                 *wal.WAL
-	snapshotter         *snap.Snapshotter
 	campaignWhenAble    bool
 	signalledLeadership uint32
 	isMember            uint32
@@ -122,6 +134,9 @@ type Node struct {
 	stopped chan struct{}
 
 	lastSendToMember map[uint64]chan struct{}
+	raftLogger       *storage.EncryptedRaftLogger
+	keyRotator       EncryptionKeyRotator
+	rotationQueued   bool
 }
 
 // NodeOptions provides node-level options.
@@ -150,6 +165,8 @@ type NodeOptions struct {
 	// nodes. Leave this as 0 to get the default value.
 	SendTimeout    time.Duration
 	TLSCredentials credentials.TransportCredentials
+
+	KeyRotator EncryptionKeyRotator
 }
 
 func init() {
@@ -188,6 +205,7 @@ func NewNode(opts NodeOptions) *Node {
 		stopped:             make(chan struct{}),
 		leadershipBroadcast: watch.NewQueue(),
 		lastSendToMember:    make(map[uint64]chan struct{}),
+		keyRotator:          opts.KeyRotator,
 	}
 	n.memoryStore = store.NewMemoryStore(n)
 
@@ -238,7 +256,7 @@ func (n *Node) JoinAndStart(ctx context.Context) (err error) {
 	}()
 
 	loadAndStartErr := n.loadAndStart(ctx, n.opts.ForceNewCluster)
-	if loadAndStartErr != nil && loadAndStartErr != errNoWAL {
+	if loadAndStartErr != nil && loadAndStartErr != storage.ErrNoWAL {
 		return loadAndStartErr
 	}
 
@@ -252,7 +270,7 @@ func (n *Node) JoinAndStart(ctx context.Context) (err error) {
 	n.appliedIndex = snapshot.Metadata.Index
 	n.snapshotIndex = snapshot.Metadata.Index
 
-	if loadAndStartErr == errNoWAL {
+	if loadAndStartErr == storage.ErrNoWAL {
 		if n.opts.JoinAddr != "" {
 			c, err := n.ConnectToMember(n.opts.JoinAddr, 10*time.Second)
 			if err != nil {
@@ -274,22 +292,20 @@ func (n *Node) JoinAndStart(ctx context.Context) (err error) {
 
 			n.Config.ID = resp.RaftID
 
-			if _, err := n.createWAL(n.opts.ID); err != nil {
+			if _, err := n.newRaftLogs(n.opts.ID); err != nil {
 				return err
 			}
 
 			n.raftNode = raft.StartNode(n.Config, []raft.Peer{})
 
 			if err := n.registerNodes(resp.Members); err != nil {
-				if walErr := n.wal.Close(); err != nil {
-					log.G(ctx).WithError(walErr).Error("raft: error closing WAL")
-				}
+				n.raftLogger.Close(ctx)
 				return err
 			}
 		} else {
 			// First member in the cluster, self-assign ID
 			n.Config.ID = uint64(rand.Int63()) + 1
-			peer, err := n.createWAL(n.opts.ID)
+			peer, err := n.newRaftLogs(n.opts.ID)
 			if err != nil {
 				return err
 			}
@@ -367,9 +383,13 @@ func (n *Node) Run(ctx context.Context) error {
 		if nodeRemoved {
 			// Move WAL and snapshot out of the way, since
 			// they are no longer usable.
-			if err := n.moveWALAndSnap(); err != nil {
+			if err := n.raftLogger.Clear(ctx); err != nil {
 				log.G(ctx).WithError(err).Error("failed to move wal after node removal")
 			}
+			// clear out the DEKs
+			if err := n.keyRotator.UpdateKeys(EncryptionKeys{}); err != nil {
+				log.G(ctx).WithError(err).Error("could not remove DEKs")
+			}
 		}
 		n.done()
 	}()
@@ -382,16 +402,10 @@ func (n *Node) Run(ctx context.Context) error {
 			n.raftNode.Tick()
 			n.cluster.Tick()
 		case rd := <-n.raftNode.Ready():
-			raftConfig := DefaultRaftConfig()
-			n.memoryStore.View(func(readTx store.ReadTx) {
-				clusters, err := store.FindClusters(readTx, store.ByName(store.DefaultClusterName))
-				if err == nil && len(clusters) == 1 {
-					raftConfig = clusters[0].Spec.Raft
-				}
-			})
+			raftConfig := n.getCurrentRaftConfig()
 
 			// Save entries to storage
-			if err := n.saveToStorage(&raftConfig, rd.HardState, rd.Entries, rd.Snapshot); err != nil {
+			if err := n.saveToStorage(ctx, &raftConfig, rd.HardState, rd.Entries, rd.Snapshot); err != nil {
 				log.G(ctx).WithError(err).Error("failed to save entries to storage")
 			}
 
@@ -459,8 +473,8 @@ func (n *Node) Run(ctx context.Context) error {
 
 			// Trigger a snapshot every once in awhile
 			if n.snapshotInProgress == nil &&
-				raftConfig.SnapshotInterval > 0 &&
-				n.appliedIndex-n.snapshotIndex >= raftConfig.SnapshotInterval {
+				(n.keyRotator.NeedsRotation() || raftConfig.SnapshotInterval > 0 &&
+					n.appliedIndex-n.snapshotIndex >= raftConfig.SnapshotInterval) {
 				n.doSnapshot(ctx, raftConfig)
 			}
 
@@ -496,6 +510,24 @@ func (n *Node) Run(ctx context.Context) error {
 				n.snapshotIndex = snapshotIndex
 			}
 			n.snapshotInProgress = nil
+			if n.rotationQueued {
+				// there was a key rotation that took place before while the snapshot
+				// was in progress - we have to take another snapshot and encrypt with the new key
+				n.doSnapshot(ctx, n.getCurrentRaftConfig())
+			}
+		case <-n.keyRotator.RotationNotify():
+			// There are 2 separate checks:  rotationQueued, and keyRotator.NeedsRotation().
+			// We set rotationQueued so that when we are notified of a rotation, we try to
+			// do a snapshot as soon as possible.  However, if there is an error while doing
+			// the snapshot, we don't want to hammer the node attempting to do snapshots over
+			// and over.  So if doing a snapshot fails, wait until the next entry comes in to
+			// try again.
+			switch {
+			case n.snapshotInProgress != nil:
+				n.rotationQueued = true
+			case n.keyRotator.NeedsRotation():
+				n.doSnapshot(ctx, n.getCurrentRaftConfig())
+			}
 		case <-n.removeRaftCh:
 			nodeRemoved = true
 			// If the node was removed from other members,
@@ -508,6 +540,17 @@ func (n *Node) Run(ctx context.Context) error {
 	}
 }
 
+func (n *Node) getCurrentRaftConfig() api.RaftConfig {
+	raftConfig := DefaultRaftConfig()
+	n.memoryStore.View(func(readTx store.ReadTx) {
+		clusters, err := store.FindClusters(readTx, store.ByName(store.DefaultClusterName))
+		if err == nil && len(clusters) == 1 {
+			raftConfig = clusters[0].Spec.Raft
+		}
+	})
+	return raftConfig
+}
+
 // Done returns channel which is closed when raft node is fully stopped.
 func (n *Node) Done() <-chan struct{} {
 	return n.doneCh
@@ -524,9 +567,7 @@ func (n *Node) stop(ctx context.Context) {
 
 	n.raftNode.Stop()
 	n.ticker.Stop()
-	if err := n.wal.Close(); err != nil {
-		log.G(ctx).WithError(err).Error("raft: failed to close WAL")
-	}
+	n.raftLogger.Close(ctx)
 	atomic.StoreUint32(&n.isMember, 0)
 	// TODO(stevvooe): Handle ctx.Done()
 }
@@ -1123,17 +1164,27 @@ func (n *Node) canSubmitProposal() bool {
 }
 
 // Saves a log entry to our Store
-func (n *Node) saveToStorage(raftConfig *api.RaftConfig, hardState raftpb.HardState, entries []raftpb.Entry, snapshot raftpb.Snapshot) (err error) {
+func (n *Node) saveToStorage(
+	ctx context.Context,
+	raftConfig *api.RaftConfig,
+	hardState raftpb.HardState,
+	entries []raftpb.Entry,
+	snapshot raftpb.Snapshot,
+) (err error) {
+
 	if !raft.IsEmptySnap(snapshot) {
-		if err := n.saveSnapshot(snapshot, raftConfig.KeepOldSnapshots); err != nil {
+		if err := n.raftLogger.SaveSnapshot(snapshot); err != nil {
 			return ErrApplySnapshot
 		}
+		if err := n.raftLogger.GC(snapshot.Metadata.Index, snapshot.Metadata.Term, raftConfig.KeepOldSnapshots); err != nil {
+			log.G(ctx).WithError(err).Error("unable to clean old snapshots and WALs")
+		}
 		if err = n.raftStore.ApplySnapshot(snapshot); err != nil {
 			return ErrApplySnapshot
 		}
 	}
 
-	if err := n.wal.Save(hardState, entries); err != nil {
+	if err := n.raftLogger.SaveEntries(hardState, entries); err != nil {
 		// TODO(aaronl): These error types should really wrap more
 		// detailed errors.
 		return ErrApplySnapshot

vendor/github.com/docker/swarmkit/manager/state/raft/storage.go

@@ -2,84 +2,53 @@ package raft
 
 import (
 	"fmt"
-	"io"
-	"io/ioutil"
-	"os"
-	"path/filepath"
-	"sort"
-	"strings"
 
-	"github.com/coreos/etcd/pkg/fileutil"
 	"github.com/coreos/etcd/raft"
 	"github.com/coreos/etcd/raft/raftpb"
-	"github.com/coreos/etcd/snap"
-	"github.com/coreos/etcd/wal"
-	"github.com/coreos/etcd/wal/walpb"
 	"github.com/docker/swarmkit/api"
 	"github.com/docker/swarmkit/log"
+	"github.com/docker/swarmkit/manager/encryption"
 	"github.com/docker/swarmkit/manager/state/raft/membership"
+	"github.com/docker/swarmkit/manager/state/raft/storage"
 	"github.com/docker/swarmkit/manager/state/store"
 	"github.com/pkg/errors"
 	"golang.org/x/net/context"
 )
 
-var errNoWAL = errors.New("no WAL present")
+func (n *Node) readFromDisk(ctx context.Context) (*raftpb.Snapshot, storage.WALData, error) {
+	keys := n.keyRotator.GetKeys()
 
-func (n *Node) legacyWALDir() string {
-	return filepath.Join(n.opts.StateDir, "wal")
-}
+	n.raftLogger = &storage.EncryptedRaftLogger{
+		StateDir:      n.opts.StateDir,
+		EncryptionKey: keys.CurrentDEK,
+	}
+	if keys.PendingDEK != nil {
+		n.raftLogger.EncryptionKey = keys.PendingDEK
+	}
 
-func (n *Node) walDir() string {
-	return filepath.Join(n.opts.StateDir, "wal-v3")
-}
+	snap, walData, err := n.raftLogger.BootstrapFromDisk(ctx)
 
-func (n *Node) legacySnapDir() string {
-	return filepath.Join(n.opts.StateDir, "snap")
-}
-
-func (n *Node) snapDir() string {
-	return filepath.Join(n.opts.StateDir, "snap-v3")
-}
-
-func (n *Node) loadAndStart(ctx context.Context, forceNewCluster bool) error {
-	walDir := n.walDir()
-	snapDir := n.snapDir()
-
-	if !fileutil.Exist(snapDir) {
-		// If snapshots created by the etcd-v2 code exist, hard link
-		// them at the new path. This prevents etc-v2 creating
-		// snapshots that are visible to us, but out of sync with our
-		// WALs, after a downgrade.
-		legacySnapDir := n.legacySnapDir()
-		if fileutil.Exist(legacySnapDir) {
-			if err := migrateSnapshots(legacySnapDir, snapDir); err != nil {
-				return err
+	if keys.PendingDEK != nil {
+		switch errors.Cause(err).(type) {
+		case nil:
+			if err = n.keyRotator.UpdateKeys(EncryptionKeys{CurrentDEK: keys.PendingDEK}); err != nil {
+				err = errors.Wrap(err, "previous key rotation was successful, but unable mark rotation as complete")
 			}
-		} else if err := os.MkdirAll(snapDir, 0700); err != nil {
-			return errors.Wrap(err, "failed to create snapshot directory")
+		case encryption.ErrCannotDecrypt:
+			snap, walData, err = n.raftLogger.BootstrapFromDisk(ctx, keys.CurrentDEK)
 		}
 	}
 
-	// Create a snapshotter
-	n.snapshotter = snap.New(snapDir)
-
-	if !wal.Exist(walDir) {
-		// If wals created by the etcd-v2 wal code exist, copy them to
-		// the new path to avoid adding backwards-incompatible entries
-		// to those files.
-		legacyWALDir := n.legacyWALDir()
-		if !wal.Exist(legacyWALDir) {
-			return errNoWAL
-		}
-
-		if err := migrateWALs(legacyWALDir, walDir); err != nil {
-			return err
-		}
+	if err != nil {
+		return nil, storage.WALData{}, err
 	}
+	return snap, walData, nil
+}
 
-	// Load snapshot data
-	snapshot, err := n.snapshotter.Load()
-	if err != nil && err != snap.ErrNoSnapshot {
+// bootstraps a node's raft store from the raft logs and snapshots on disk
+func (n *Node) loadAndStart(ctx context.Context, forceNewCluster bool) error {
+	snapshot, waldata, err := n.readFromDisk(ctx)
+	if err != nil {
 		return err
 	}
 
@@ -91,193 +60,14 @@ func (n *Node) loadAndStart(ctx context.Context, forceNewCluster bool) error {
 	}
 
 	// Read logs to fully catch up store
-	if err := n.readWAL(ctx, snapshot, forceNewCluster); err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func migrateWALs(legacyWALDir, walDir string) error {
-	// keep temporary wal directory so WAL initialization appears atomic
-	tmpdirpath := filepath.Clean(walDir) + ".tmp"
-	if fileutil.Exist(tmpdirpath) {
-		if err := os.RemoveAll(tmpdirpath); err != nil {
-			return errors.Wrap(err, "could not remove temporary WAL directory")
-		}
-	}
-	if err := fileutil.CreateDirAll(tmpdirpath); err != nil {
-		return errors.Wrap(err, "could not create temporary WAL directory")
-	}
-
-	walNames, err := fileutil.ReadDir(legacyWALDir)
-	if err != nil {
-		return errors.Wrapf(err, "could not list WAL directory %s", legacyWALDir)
-	}
-
-	for _, fname := range walNames {
-		_, err := copyFile(filepath.Join(legacyWALDir, fname), filepath.Join(tmpdirpath, fname), 0600)
-		if err != nil {
-			return errors.Wrap(err, "error copying WAL file")
-		}
-	}
-
-	if err := os.Rename(tmpdirpath, walDir); err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func migrateSnapshots(legacySnapDir, snapDir string) error {
-	// use temporary snaphot directory so initialization appears atomic
-	tmpdirpath := filepath.Clean(snapDir) + ".tmp"
-	if fileutil.Exist(tmpdirpath) {
-		if err := os.RemoveAll(tmpdirpath); err != nil {
-			return errors.Wrap(err, "could not remove temporary snapshot directory")
-		}
-	}
-	if err := fileutil.CreateDirAll(tmpdirpath); err != nil {
-		return errors.Wrap(err, "could not create temporary snapshot directory")
-	}
-
-	snapshotNames, err := fileutil.ReadDir(legacySnapDir)
-	if err != nil {
-		return errors.Wrapf(err, "could not list snapshot directory %s", legacySnapDir)
-	}
-
-	for _, fname := range snapshotNames {
-		err := os.Link(filepath.Join(legacySnapDir, fname), filepath.Join(tmpdirpath, fname))
-		if err != nil {
-			return errors.Wrap(err, "error linking snapshot file")
-		}
-	}
-
-	if err := os.Rename(tmpdirpath, snapDir); err != nil {
-		return err
-	}
-
-	return nil
-}
-
-// copyFile copies from src to dst until either EOF is reached
-// on src or an error occurs. It verifies src exists and removes
-// the dst if it exists.
-func copyFile(src, dst string, perm os.FileMode) (int64, error) {
-	cleanSrc := filepath.Clean(src)
-	cleanDst := filepath.Clean(dst)
-	if cleanSrc == cleanDst {
-		return 0, nil
-	}
-	sf, err := os.Open(cleanSrc)
-	if err != nil {
-		return 0, err
-	}
-	defer sf.Close()
-	if err := os.Remove(cleanDst); err != nil && !os.IsNotExist(err) {
-		return 0, err
-	}
-	df, err := os.OpenFile(cleanDst, os.O_RDWR|os.O_CREATE|os.O_TRUNC, perm)
-	if err != nil {
-		return 0, err
-	}
-	defer df.Close()
-	return io.Copy(df, sf)
-}
-
-func (n *Node) createWAL(nodeID string) (raft.Peer, error) {
-	raftNode := &api.RaftMember{
-		RaftID: n.Config.ID,
-		NodeID: nodeID,
-		Addr:   n.opts.Addr,
-	}
-	metadata, err := raftNode.Marshal()
-	if err != nil {
-		return raft.Peer{}, errors.Wrap(err, "error marshalling raft node")
-	}
-	n.wal, err = wal.Create(n.walDir(), metadata)
-	if err != nil {
-		return raft.Peer{}, errors.Wrap(err, "failed to create WAL")
-	}
-
-	n.cluster.AddMember(&membership.Member{RaftMember: raftNode})
-	return raft.Peer{ID: n.Config.ID, Context: metadata}, nil
-}
-
-// moveWALAndSnap moves away the WAL and snapshot because we were removed
-// from the cluster and will need to recreate them if we are readded.
-func (n *Node) moveWALAndSnap() error {
-	newWALDir, err := ioutil.TempDir(n.opts.StateDir, "wal.")
-	if err != nil {
-		return err
-	}
-	err = os.Rename(n.walDir(), newWALDir)
-	if err != nil {
-		return err
-	}
-
-	newSnapDir, err := ioutil.TempDir(n.opts.StateDir, "snap.")
-	if err != nil {
-		return err
-	}
-	err = os.Rename(n.snapDir(), newSnapDir)
-	if err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func (n *Node) readWAL(ctx context.Context, snapshot *raftpb.Snapshot, forceNewCluster bool) (err error) {
-	var (
-		walsnap  walpb.Snapshot
-		metadata []byte
-		st       raftpb.HardState
-		ents     []raftpb.Entry
-	)
-
-	if snapshot != nil {
-		walsnap.Index = snapshot.Metadata.Index
-		walsnap.Term = snapshot.Metadata.Term
-	}
-
-	repaired := false
-	for {
-		if n.wal, err = wal.Open(n.walDir(), walsnap); err != nil {
-			return errors.Wrap(err, "failed to open WAL")
-		}
-		if metadata, st, ents, err = n.wal.ReadAll(); err != nil {
-			if err := n.wal.Close(); err != nil {
-				return err
-			}
-			// we can only repair ErrUnexpectedEOF and we never repair twice.
-			if repaired || err != io.ErrUnexpectedEOF {
-				return errors.Wrap(err, "irreparable WAL error")
-			}
-			if !wal.Repair(n.walDir()) {
-				return errors.Wrap(err, "WAL error cannot be repaired")
-			}
-			log.G(ctx).WithError(err).Info("repaired WAL error")
-			repaired = true
-			continue
-		}
-		break
-	}
-
-	defer func() {
-		if err != nil {
-			if walErr := n.wal.Close(); walErr != nil {
-				log.G(ctx).WithError(walErr).Error("error closing raft WAL")
-			}
-		}
-	}()
-
 	var raftNode api.RaftMember
-	if err := raftNode.Unmarshal(metadata); err != nil {
+	if err := raftNode.Unmarshal(waldata.Metadata); err != nil {
 		return errors.Wrap(err, "failed to unmarshal WAL metadata")
 	}
 	n.Config.ID = raftNode.RaftID
 
+	ents, st := waldata.Entries, waldata.HardState
+
 	// All members that are no longer part of the cluster must be added to
 	// the removed list right away, so that we don't try to connect to them
 	// before processing the configuration change entries, which could make
@@ -326,7 +116,7 @@ func (n *Node) readWAL(ctx context.Context, snapshot *raftpb.Snapshot, forceNewC
 		ents = append(ents, toAppEnts...)
 
 		// force commit newly appended entries
-		err := n.wal.Save(st, toAppEnts)
+		err := n.raftLogger.SaveEntries(st, toAppEnts)
 		if err != nil {
 			log.G(ctx).WithError(err).Fatalf("failed to save WAL while forcing new cluster")
 		}
@@ -343,146 +133,24 @@ func (n *Node) readWAL(ctx context.Context, snapshot *raftpb.Snapshot, forceNewC
 	if err := n.raftStore.SetHardState(st); err != nil {
 		return err
 	}
-	if err := n.raftStore.Append(ents); err != nil {
-		return err
-	}
-
-	return nil
+	return n.raftStore.Append(ents)
 }
 
-func (n *Node) saveSnapshot(snapshot raftpb.Snapshot, keepOldSnapshots uint64) error {
-	err := n.wal.SaveSnapshot(walpb.Snapshot{
-		Index: snapshot.Metadata.Index,
-		Term:  snapshot.Metadata.Term,
-	})
+func (n *Node) newRaftLogs(nodeID string) (raft.Peer, error) {
+	raftNode := &api.RaftMember{
+		RaftID: n.Config.ID,
+		NodeID: nodeID,
+		Addr:   n.opts.Addr,
+	}
+	metadata, err := raftNode.Marshal()
 	if err != nil {
-		return err
+		return raft.Peer{}, errors.Wrap(err, "error marshalling raft node")
 	}
-	err = n.snapshotter.SaveSnap(snapshot)
-	if err != nil {
-		return err
+	if err := n.raftLogger.BootstrapNew(metadata); err != nil {
+		return raft.Peer{}, err
 	}
-	err = n.wal.ReleaseLockTo(snapshot.Metadata.Index)
-	if err != nil {
-		return err
-	}
-
-	// Delete any older snapshots
-	curSnapshot := fmt.Sprintf("%016x-%016x%s", snapshot.Metadata.Term, snapshot.Metadata.Index, ".snap")
-
-	dirents, err := ioutil.ReadDir(n.snapDir())
-	if err != nil {
-		return err
-	}
-
-	var snapshots []string
-	for _, dirent := range dirents {
-		if strings.HasSuffix(dirent.Name(), ".snap") {
-			snapshots = append(snapshots, dirent.Name())
-		}
-	}
-
-	// Sort snapshot filenames in reverse lexical order
-	sort.Sort(sort.Reverse(sort.StringSlice(snapshots)))
-
-	// Ignore any snapshots that are older than the current snapshot.
-	// Delete the others. Rather than doing lexical comparisons, we look
-	// at what exists before/after the current snapshot in the slice.
-	// This means that if the current snapshot doesn't appear in the
-	// directory for some strange reason, we won't delete anything, which
-	// is the safe behavior.
-	curSnapshotIdx := -1
-	var (
-		removeErr      error
-		oldestSnapshot string
-	)
-
-	for i, snapFile := range snapshots {
-		if curSnapshotIdx >= 0 && i > curSnapshotIdx {
-			if uint64(i-curSnapshotIdx) > keepOldSnapshots {
-				err := os.Remove(filepath.Join(n.snapDir(), snapFile))
-				if err != nil && removeErr == nil {
-					removeErr = err
-				}
-				continue
-			}
-		} else if snapFile == curSnapshot {
-			curSnapshotIdx = i
-		}
-		oldestSnapshot = snapFile
-	}
-
-	if removeErr != nil {
-		return removeErr
-	}
-
-	// Remove any WAL files that only contain data from before the oldest
-	// remaining snapshot.
-
-	if oldestSnapshot == "" {
-		return nil
-	}
-
-	// Parse index out of oldest snapshot's filename
-	var snapTerm, snapIndex uint64
-	_, err = fmt.Sscanf(oldestSnapshot, "%016x-%016x.snap", &snapTerm, &snapIndex)
-	if err != nil {
-		return errors.Wrapf(err, "malformed snapshot filename %s", oldestSnapshot)
-	}
-
-	// List the WALs
-	dirents, err = ioutil.ReadDir(n.walDir())
-	if err != nil {
-		return err
-	}
-
-	var wals []string
-	for _, dirent := range dirents {
-		if strings.HasSuffix(dirent.Name(), ".wal") {
-			wals = append(wals, dirent.Name())
-		}
-	}
-
-	// Sort WAL filenames in lexical order
-	sort.Sort(sort.StringSlice(wals))
-
-	found := false
-	deleteUntil := -1
-
-	for i, walName := range wals {
-		var walSeq, walIndex uint64
-		_, err = fmt.Sscanf(walName, "%016x-%016x.wal", &walSeq, &walIndex)
-		if err != nil {
-			return errors.Wrapf(err, "could not parse WAL name %s", walName)
-		}
-
-		if walIndex >= snapIndex {
-			deleteUntil = i - 1
-			found = true
-			break
-		}
-	}
-
-	// If all WAL files started with indices below the oldest snapshot's
-	// index, we can delete all but the newest WAL file.
-	if !found && len(wals) != 0 {
-		deleteUntil = len(wals) - 1
-	}
-
-	for i := 0; i < deleteUntil; i++ {
-		walPath := filepath.Join(n.walDir(), wals[i])
-		l, err := fileutil.TryLockFile(walPath, os.O_WRONLY, fileutil.PrivateFileMode)
-		if err != nil {
-			return errors.Wrapf(err, "could not lock old WAL file %s for removal", wals[i])
-		}
-		err = os.Remove(walPath)
-		l.Close()
-		if err != nil {
-			return errors.Wrapf(err, "error removing old WAL file %s", wals[i])
-		}
-	}
-
-	return nil
+	n.cluster.AddMember(&membership.Member{RaftMember: raftNode})
+	return raft.Peer{ID: n.Config.ID, Context: metadata}, nil
 }
 
 func (n *Node) doSnapshot(ctx context.Context, raftConfig api.RaftConfig) {
@@ -497,6 +165,17 @@ func (n *Node) doSnapshot(ctx context.Context, raftConfig api.RaftConfig) {
 	}
 	snapshot.Membership.Removed = n.cluster.Removed()
 
+	// maybe start rotation
+	n.rotationQueued = false
+	var newEncryptionKeys *EncryptionKeys
+	if n.keyRotator.NeedsRotation() {
+		keys := n.keyRotator.GetKeys()
+		if keys.PendingDEK != nil {
+			n.raftLogger.RotateEncryptionKey(keys.PendingDEK)
+			newEncryptionKeys = &EncryptionKeys{CurrentDEK: keys.PendingDEK}
+		}
+	}
+
 	viewStarted := make(chan struct{})
 	n.asyncTasks.Add(1)
 	n.snapshotInProgress = make(chan uint64, 1) // buffered in case Shutdown is called during the snapshot
@@ -505,7 +184,6 @@ func (n *Node) doSnapshot(ctx context.Context, raftConfig api.RaftConfig) {
 			n.asyncTasks.Done()
 			n.snapshotInProgress <- snapshotIndex
 		}()
-
 		var err error
 		n.memoryStore.View(func(tx store.ReadTx) {
 			close(viewStarted)
@@ -526,11 +204,18 @@ func (n *Node) doSnapshot(ctx context.Context, raftConfig api.RaftConfig) {
 		}
 		snap, err := n.raftStore.CreateSnapshot(appliedIndex, &n.confState, d)
 		if err == nil {
-			if err := n.saveSnapshot(snap, raftConfig.KeepOldSnapshots); err != nil {
+			if err := n.raftLogger.SaveSnapshot(snap); err != nil {
 				log.G(ctx).WithError(err).Error("failed to save snapshot")
 				return
 			}
 			snapshotIndex = appliedIndex
+			if newEncryptionKeys != nil {
+				// this means we tried to rotate - so finish the rotation
+				if err := n.keyRotator.UpdateKeys(*newEncryptionKeys); err != nil {
+					log.G(ctx).WithError(err).Error(
+						"failed to update encryption keys after a rotation - will wait for the next snapshot")
+				}
+			}
 
 			if appliedIndex > raftConfig.LogEntriesForSlowFollowers {
 				err := n.raftStore.Compact(appliedIndex - raftConfig.LogEntriesForSlowFollowers)
@@ -538,6 +223,10 @@ func (n *Node) doSnapshot(ctx context.Context, raftConfig api.RaftConfig) {
 					log.G(ctx).WithError(err).Error("failed to compact snapshot")
 				}
 			}
+
+			if err := n.raftLogger.GC(snap.Metadata.Index, snap.Metadata.Term, raftConfig.KeepOldSnapshots); err != nil {
+				log.G(ctx).WithError(err).Error("failed to clean up old snapshots and WALs")
+			}
 		} else if err != raft.ErrSnapOutOfDate {
 			log.G(ctx).WithError(err).Error("failed to create snapshot")
 		}

vendor/github.com/docker/swarmkit/manager/state/raft/storage/snapwrap.go

@@ -0,0 +1,158 @@
+package storage
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+
+	"github.com/coreos/etcd/pkg/fileutil"
+	"github.com/coreos/etcd/raft/raftpb"
+	"github.com/coreos/etcd/snap"
+	"github.com/docker/swarmkit/manager/encryption"
+	"github.com/pkg/errors"
+)
+
+// This package wraps the github.com/coreos/etcd/snap package, and encrypts
+// the bytes of whatever snapshot is passed to it, and decrypts the bytes of
+// whatever snapshot it reads.
+
+// Snapshotter is the interface presented by github.com/coreos/etcd/snap.Snapshotter that we depend upon
+type Snapshotter interface {
+	SaveSnap(snapshot raftpb.Snapshot) error
+	Load() (*raftpb.Snapshot, error)
+}
+
+// SnapFactory provides an interface for the different ways to get a Snapshotter object.
+// For instance, the etcd/snap package itself provides this
+type SnapFactory interface {
+	New(dirpath string) Snapshotter
+}
+
+var _ Snapshotter = &wrappedSnap{}
+var _ Snapshotter = &snap.Snapshotter{}
+var _ SnapFactory = snapCryptor{}
+
+// wrappedSnap wraps a github.com/coreos/etcd/snap.Snapshotter, and handles
+// encrypting/decrypting.
+type wrappedSnap struct {
+	*snap.Snapshotter
+	encrypter encryption.Encrypter
+	decrypter encryption.Decrypter
+}
+
+// SaveSnap encrypts the snapshot data (if an encrypter is exists) before passing it onto the
+// wrapped snap.Snapshotter's SaveSnap function.
+func (s *wrappedSnap) SaveSnap(snapshot raftpb.Snapshot) error {
+	toWrite := snapshot
+	var err error
+	toWrite.Data, err = encryption.Encrypt(snapshot.Data, s.encrypter)
+	if err != nil {
+		return err
+	}
+	return s.Snapshotter.SaveSnap(toWrite)
+}
+
+// Load decrypts the snapshot data (if a decrypter is exists) after reading it using the
+// wrapped snap.Snapshotter's Load function.
+func (s *wrappedSnap) Load() (*raftpb.Snapshot, error) {
+	snapshot, err := s.Snapshotter.Load()
+	if err != nil {
+		return nil, err
+	}
+	snapshot.Data, err = encryption.Decrypt(snapshot.Data, s.decrypter)
+	if err != nil {
+		return nil, err
+	}
+
+	return snapshot, nil
+}
+
+// snapCryptor is an object that provides the same functions as `etcd/wal`
+// and `etcd/snap` that we need to open a WAL object or Snapshotter object
+type snapCryptor struct {
+	encrypter encryption.Encrypter
+	decrypter encryption.Decrypter
+}
+
+// NewSnapFactory returns a new object that can read from and write to encrypted
+// snapshots on disk
+func NewSnapFactory(encrypter encryption.Encrypter, decrypter encryption.Decrypter) SnapFactory {
+	return snapCryptor{
+		encrypter: encrypter,
+		decrypter: decrypter,
+	}
+}
+
+// NewSnapshotter returns a new Snapshotter with the given encrypters and decrypters
+func (sc snapCryptor) New(dirpath string) Snapshotter {
+	return &wrappedSnap{
+		Snapshotter: snap.New(dirpath),
+		encrypter:   sc.encrypter,
+		decrypter:   sc.decrypter,
+	}
+}
+
+type originalSnap struct{}
+
+func (o originalSnap) New(dirpath string) Snapshotter {
+	return snap.New(dirpath)
+}
+
+// OriginalSnap is the original `snap` package as an implemntation of the SnapFactory interface
+var OriginalSnap SnapFactory = originalSnap{}
+
+// MigrateSnapshot reads the latest existing snapshot from one directory, encoded one way, and writes
+// it to a new directory, encoded a different way
+func MigrateSnapshot(oldDir, newDir string, oldFactory, newFactory SnapFactory) error {
+	// use temporary snapshot directory so initialization appears atomic
+	oldSnapshotter := oldFactory.New(oldDir)
+	snapshot, err := oldSnapshotter.Load()
+	switch err {
+	case snap.ErrNoSnapshot: // if there's no snapshot, the migration succeeded
+		return nil
+	case nil:
+		break
+	default:
+		return err
+	}
+
+	tmpdirpath := filepath.Clean(newDir) + ".tmp"
+	if fileutil.Exist(tmpdirpath) {
+		if err := os.RemoveAll(tmpdirpath); err != nil {
+			return errors.Wrap(err, "could not remove temporary snapshot directory")
+		}
+	}
+	if err := fileutil.CreateDirAll(tmpdirpath); err != nil {
+		return errors.Wrap(err, "could not create temporary snapshot directory")
+	}
+	tmpSnapshotter := newFactory.New(tmpdirpath)
+
+	// write the new snapshot to the temporary location
+	if err = tmpSnapshotter.SaveSnap(*snapshot); err != nil {
+		return err
+	}
+
+	return os.Rename(tmpdirpath, newDir)
+}
+
+// ListSnapshots lists all the snapshot files in a particular directory and returns
+// the snapshot files in reverse lexical order (newest first)
+func ListSnapshots(dirpath string) ([]string, error) {
+	dirents, err := ioutil.ReadDir(dirpath)
+	if err != nil {
+		return nil, err
+	}
+
+	var snapshots []string
+	for _, dirent := range dirents {
+		if strings.HasSuffix(dirent.Name(), ".snap") {
+			snapshots = append(snapshots, dirent.Name())
+		}
+	}
+
+	// Sort snapshot filenames in reverse lexical order
+	sort.Sort(sort.Reverse(sort.StringSlice(snapshots)))
+	return snapshots, nil
+}

vendor/github.com/docker/swarmkit/manager/state/raft/storage/storage.go

@@ -0,0 +1,391 @@
+package storage
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"sync"
+
+	"golang.org/x/net/context"
+
+	"github.com/coreos/etcd/pkg/fileutil"
+	"github.com/coreos/etcd/raft/raftpb"
+	"github.com/coreos/etcd/snap"
+	"github.com/coreos/etcd/wal"
+	"github.com/coreos/etcd/wal/walpb"
+	"github.com/docker/swarmkit/api"
+	"github.com/docker/swarmkit/log"
+	"github.com/docker/swarmkit/manager/encryption"
+	"github.com/pkg/errors"
+)
+
+// ErrNoWAL is returned if there are no WALs on disk
+var ErrNoWAL = errors.New("no WAL present")
+
+type walSnapDirs struct {
+	wal  string
+	snap string
+}
+
+// the wal/snap directories in decreasing order of preference/version
+var versionedWALSnapDirs = []walSnapDirs{
+	{wal: "wal-v3-encrypted", snap: "snap-v3-encrypted"},
+	{wal: "wal-v3", snap: "snap-v3"},
+	{wal: "wal", snap: "snap"},
+}
+
+// MultiDecrypter attempts to decrypt with a list of decrypters
+type MultiDecrypter []encryption.Decrypter
+
+// Decrypt tries to decrypt using all the decrypters
+func (m MultiDecrypter) Decrypt(r api.MaybeEncryptedRecord) (result []byte, err error) {
+	for _, d := range m {
+		result, err = d.Decrypt(r)
+		if err == nil {
+			return
+		}
+	}
+	return
+}
+
+// EncryptedRaftLogger saves raft data to disk
+type EncryptedRaftLogger struct {
+	StateDir      string
+	EncryptionKey []byte
+
+	// mutex is locked for writing only when we need to replace the wal object and snapshotter
+	// object, not when we're writing snapshots or wals (in which case it's locked for reading)
+	encoderMu   sync.RWMutex
+	wal         WAL
+	snapshotter Snapshotter
+}
+
+// BootstrapFromDisk creates a new snapshotter and wal, and also reads the latest snapshot and WALs from disk
+func (e *EncryptedRaftLogger) BootstrapFromDisk(ctx context.Context, oldEncryptionKeys ...[]byte) (*raftpb.Snapshot, WALData, error) {
+	e.encoderMu.Lock()
+	defer e.encoderMu.Unlock()
+
+	walDir := e.walDir()
+	snapDir := e.snapDir()
+
+	encrypter, decrypter := encryption.Defaults(e.EncryptionKey)
+	if oldEncryptionKeys != nil {
+		decrypters := []encryption.Decrypter{decrypter}
+		for _, key := range oldEncryptionKeys {
+			_, d := encryption.Defaults(key)
+			decrypters = append(decrypters, d)
+		}
+		decrypter = MultiDecrypter(decrypters)
+	}
+
+	snapFactory := NewSnapFactory(encrypter, decrypter)
+
+	if !fileutil.Exist(snapDir) {
+		// If snapshots created by the etcd-v2 code exist, or by swarmkit development version,
+		// read the latest snapshot and write it encoded to the new path.  The new path
+		// prevents etc-v2 creating snapshots that are visible to us, but not encoded and
+		// out of sync with our WALs, after a downgrade.
+		for _, dirs := range versionedWALSnapDirs[1:] {
+			legacySnapDir := filepath.Join(e.StateDir, dirs.snap)
+			if fileutil.Exist(legacySnapDir) {
+				if err := MigrateSnapshot(legacySnapDir, snapDir, OriginalSnap, snapFactory); err != nil {
+					return nil, WALData{}, err
+				}
+				break
+			}
+		}
+	}
+	// ensure the new directory exists
+	if err := os.MkdirAll(snapDir, 0700); err != nil {
+		return nil, WALData{}, errors.Wrap(err, "failed to create snapshot directory")
+	}
+
+	var (
+		snapshotter Snapshotter
+		walObj      WAL
+		err         error
+	)
+
+	// Create a snapshotter and load snapshot data
+	snapshotter = snapFactory.New(snapDir)
+	snapshot, err := snapshotter.Load()
+	if err != nil && err != snap.ErrNoSnapshot {
+		return nil, WALData{}, err
+	}
+
+	walFactory := NewWALFactory(encrypter, decrypter)
+	var walsnap walpb.Snapshot
+	if snapshot != nil {
+		walsnap.Index = snapshot.Metadata.Index
+		walsnap.Term = snapshot.Metadata.Term
+	}
+
+	if !wal.Exist(walDir) {
+		var walExists bool
+		// If wals created by the etcd-v2 wal code exist, read the latest ones based
+		// on this snapshot and encode them to wals in the new path to avoid adding
+		// backwards-incompatible entries to those files.
+		for _, dirs := range versionedWALSnapDirs[1:] {
+			legacyWALDir := filepath.Join(e.StateDir, dirs.wal)
+			if !wal.Exist(legacyWALDir) {
+				continue
+			}
+			if err = MigrateWALs(ctx, legacyWALDir, walDir, OriginalWAL, walFactory, walsnap); err != nil {
+				return nil, WALData{}, err
+			}
+			walExists = true
+			break
+		}
+		if !walExists {
+			return nil, WALData{}, ErrNoWAL
+		}
+	}
+
+	walObj, waldata, err := ReadRepairWAL(ctx, walDir, walsnap, walFactory)
+	if err != nil {
+		return nil, WALData{}, err
+	}
+
+	e.snapshotter = snapshotter
+	e.wal = walObj
+
+	return snapshot, waldata, nil
+}
+
+// BootstrapNew creates a new snapshotter and WAL writer, expecting that there is nothing on disk
+func (e *EncryptedRaftLogger) BootstrapNew(metadata []byte) error {
+	e.encoderMu.Lock()
+	defer e.encoderMu.Unlock()
+	encrypter, decrypter := encryption.Defaults(e.EncryptionKey)
+	walFactory := NewWALFactory(encrypter, decrypter)
+
+	for _, dirpath := range []string{e.walDir(), e.snapDir()} {
+		if err := os.MkdirAll(dirpath, 0700); err != nil {
+			return errors.Wrapf(err, "failed to create %s", dirpath)
+		}
+	}
+	var err error
+	e.wal, err = walFactory.Create(e.walDir(), metadata)
+	if err != nil {
+		return errors.Wrap(err, "failed to create WAL")
+	}
+
+	e.snapshotter = NewSnapFactory(encrypter, decrypter).New(e.snapDir())
+	return nil
+}
+
+func (e *EncryptedRaftLogger) walDir() string {
+	return filepath.Join(e.StateDir, versionedWALSnapDirs[0].wal)
+}
+
+func (e *EncryptedRaftLogger) snapDir() string {
+	return filepath.Join(e.StateDir, versionedWALSnapDirs[0].snap)
+}
+
+// RotateEncryptionKey swaps out the encoders and decoders used by the wal and snapshotter
+func (e *EncryptedRaftLogger) RotateEncryptionKey(newKey []byte) {
+	e.encoderMu.Lock()
+	defer e.encoderMu.Unlock()
+
+	if e.wal != nil { // if the wal exists, the snapshotter exists
+		// We don't want to have to close the WAL, because we can't open a new one.
+		// We need to know the previous snapshot, because when you open a WAL you
+		// have to read out all the entries from a particular snapshot, or you can't
+		// write.  So just rotate the encoders out from under it.  We already
+		// have a lock on writing to snapshots and WALs.
+		wrapped, ok := e.wal.(*wrappedWAL)
+		if !ok {
+			panic(fmt.Errorf("EncryptedRaftLogger's WAL is not a wrappedWAL"))
+		}
+
+		wrapped.encrypter, wrapped.decrypter = encryption.Defaults(newKey)
+
+		e.snapshotter = NewSnapFactory(wrapped.encrypter, wrapped.decrypter).New(e.snapDir())
+	}
+	e.EncryptionKey = newKey
+}
+
+// SaveSnapshot actually saves a given snapshot to both the WAL and the snapshot.
+func (e *EncryptedRaftLogger) SaveSnapshot(snapshot raftpb.Snapshot) error {
+
+	walsnap := walpb.Snapshot{
+		Index: snapshot.Metadata.Index,
+		Term:  snapshot.Metadata.Term,
+	}
+
+	e.encoderMu.RLock()
+	if err := e.wal.SaveSnapshot(walsnap); err != nil {
+		e.encoderMu.RUnlock()
+		return err
+	}
+
+	snapshotter := e.snapshotter
+	e.encoderMu.RUnlock()
+
+	if err := snapshotter.SaveSnap(snapshot); err != nil {
+		return err
+	}
+	if err := e.wal.ReleaseLockTo(snapshot.Metadata.Index); err != nil {
+		return err
+	}
+	return nil
+}
+
+// GC garbage collects snapshots and wals older than the provided index and term
+func (e *EncryptedRaftLogger) GC(index uint64, term uint64, keepOldSnapshots uint64) error {
+	// Delete any older snapshots
+	curSnapshot := fmt.Sprintf("%016x-%016x%s", term, index, ".snap")
+
+	snapshots, err := ListSnapshots(e.snapDir())
+	if err != nil {
+		return err
+	}
+
+	// Ignore any snapshots that are older than the current snapshot.
+	// Delete the others. Rather than doing lexical comparisons, we look
+	// at what exists before/after the current snapshot in the slice.
+	// This means that if the current snapshot doesn't appear in the
+	// directory for some strange reason, we won't delete anything, which
+	// is the safe behavior.
+	curSnapshotIdx := -1
+	var (
+		removeErr      error
+		oldestSnapshot string
+	)
+
+	for i, snapFile := range snapshots {
+		if curSnapshotIdx >= 0 && i > curSnapshotIdx {
+			if uint64(i-curSnapshotIdx) > keepOldSnapshots {
+				err := os.Remove(filepath.Join(e.snapDir(), snapFile))
+				if err != nil && removeErr == nil {
+					removeErr = err
+				}
+				continue
+			}
+		} else if snapFile == curSnapshot {
+			curSnapshotIdx = i
+		}
+		oldestSnapshot = snapFile
+	}
+
+	if removeErr != nil {
+		return removeErr
+	}
+
+	// Remove any WAL files that only contain data from before the oldest
+	// remaining snapshot.
+
+	if oldestSnapshot == "" {
+		return nil
+	}
+
+	// Parse index out of oldest snapshot's filename
+	var snapTerm, snapIndex uint64
+	_, err = fmt.Sscanf(oldestSnapshot, "%016x-%016x.snap", &snapTerm, &snapIndex)
+	if err != nil {
+		return errors.Wrapf(err, "malformed snapshot filename %s", oldestSnapshot)
+	}
+
+	wals, err := ListWALs(e.walDir())
+	if err != nil {
+		return err
+	}
+
+	found := false
+	deleteUntil := -1
+
+	for i, walName := range wals {
+		var walSeq, walIndex uint64
+		_, err = fmt.Sscanf(walName, "%016x-%016x.wal", &walSeq, &walIndex)
+		if err != nil {
+			return errors.Wrapf(err, "could not parse WAL name %s", walName)
+		}
+
+		if walIndex >= snapIndex {
+			deleteUntil = i - 1
+			found = true
+			break
+		}
+	}
+
+	// If all WAL files started with indices below the oldest snapshot's
+	// index, we can delete all but the newest WAL file.
+	if !found && len(wals) != 0 {
+		deleteUntil = len(wals) - 1
+	}
+
+	for i := 0; i < deleteUntil; i++ {
+		walPath := filepath.Join(e.walDir(), wals[i])
+		l, err := fileutil.TryLockFile(walPath, os.O_WRONLY, fileutil.PrivateFileMode)
+		if err != nil {
+			return errors.Wrapf(err, "could not lock old WAL file %s for removal", wals[i])
+		}
+		err = os.Remove(walPath)
+		l.Close()
+		if err != nil {
+			return errors.Wrapf(err, "error removing old WAL file %s", wals[i])
+		}
+	}
+
+	return nil
+}
+
+// SaveEntries saves only entries to disk
+func (e *EncryptedRaftLogger) SaveEntries(st raftpb.HardState, entries []raftpb.Entry) error {
+	e.encoderMu.RLock()
+	defer e.encoderMu.RUnlock()
+
+	if e.wal == nil {
+		return fmt.Errorf("raft WAL has either been closed or has never been created")
+	}
+	return e.wal.Save(st, entries)
+}
+
+// Close closes the logger - it will have to be bootstrapped again to start writing
+func (e *EncryptedRaftLogger) Close(ctx context.Context) {
+	e.encoderMu.Lock()
+	defer e.encoderMu.Unlock()
+
+	if e.wal != nil {
+		if err := e.wal.Close(); err != nil {
+			log.G(ctx).WithError(err).Error("error closing raft WAL")
+		}
+	}
+
+	e.wal = nil
+	e.snapshotter = nil
+}
+
+// Clear closes the existing WAL and moves away the WAL and snapshot.
+func (e *EncryptedRaftLogger) Clear(ctx context.Context) error {
+	e.encoderMu.Lock()
+	defer e.encoderMu.Unlock()
+
+	if e.wal != nil {
+		if err := e.wal.Close(); err != nil {
+			log.G(ctx).WithError(err).Error("error closing raft WAL")
+		}
+	}
+	e.snapshotter = nil
+
+	newWALDir, err := ioutil.TempDir(e.StateDir, "wal.")
+	if err != nil {
+		return err
+	}
+	err = os.Rename(e.walDir(), newWALDir)
+	if err != nil {
+		return err
+	}
+
+	newSnapDir, err := ioutil.TempDir(e.StateDir, "snap.")
+	if err != nil {
+		return err
+	}
+	err = os.Rename(e.snapDir(), newSnapDir)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}

vendor/github.com/docker/swarmkit/manager/state/raft/storage/walwrap.go

@@ -0,0 +1,253 @@
+package storage
+
+import (
+	"context"
+	"io"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+
+	"github.com/coreos/etcd/pkg/fileutil"
+	"github.com/coreos/etcd/raft/raftpb"
+	"github.com/coreos/etcd/wal"
+	"github.com/coreos/etcd/wal/walpb"
+	"github.com/docker/swarmkit/log"
+	"github.com/docker/swarmkit/manager/encryption"
+	"github.com/pkg/errors"
+)
+
+// This package wraps the github.com/coreos/etcd/wal package, and encrypts
+// the bytes of whatever entry is passed to it, and decrypts the bytes of
+// whatever entry it reads.
+
+// WAL is the interface presented by github.com/coreos/etcd/wal.WAL that we depend upon
+type WAL interface {
+	ReadAll() ([]byte, raftpb.HardState, []raftpb.Entry, error)
+	ReleaseLockTo(index uint64) error
+	Close() error
+	Save(st raftpb.HardState, ents []raftpb.Entry) error
+	SaveSnapshot(e walpb.Snapshot) error
+}
+
+// WALFactory provides an interface for the different ways to get a WAL object.
+// For instance, the etcd/wal package itself provides this
+type WALFactory interface {
+	Create(dirpath string, metadata []byte) (WAL, error)
+	Open(dirpath string, walsnap walpb.Snapshot) (WAL, error)
+}
+
+var _ WAL = &wrappedWAL{}
+var _ WAL = &wal.WAL{}
+var _ WALFactory = walCryptor{}
+
+// wrappedWAL wraps a github.com/coreos/etcd/wal.WAL, and handles encrypting/decrypting
+type wrappedWAL struct {
+	*wal.WAL
+	encrypter encryption.Encrypter
+	decrypter encryption.Decrypter
+}
+
+// ReadAll wraps the wal.WAL.ReadAll() function, but it first checks to see if the
+// metadata indicates that the entries are encryptd, and if so, decrypts them.
+func (w *wrappedWAL) ReadAll() ([]byte, raftpb.HardState, []raftpb.Entry, error) {
+	metadata, state, ents, err := w.WAL.ReadAll()
+	if err != nil {
+		return metadata, state, ents, err
+	}
+	for i, ent := range ents {
+		ents[i].Data, err = encryption.Decrypt(ent.Data, w.decrypter)
+		if err != nil {
+			return nil, raftpb.HardState{}, nil, err
+		}
+	}
+
+	return metadata, state, ents, nil
+}
+
+// Save encrypts the entry data (if an encrypter is exists) before passing it onto the
+// wrapped wal.WAL's Save function.
+func (w *wrappedWAL) Save(st raftpb.HardState, ents []raftpb.Entry) error {
+	var writeEnts []raftpb.Entry
+	for _, ent := range ents {
+		data, err := encryption.Encrypt(ent.Data, w.encrypter)
+		if err != nil {
+			return err
+		}
+		writeEnts = append(writeEnts, raftpb.Entry{
+			Index: ent.Index,
+			Term:  ent.Term,
+			Type:  ent.Type,
+			Data:  data,
+		})
+	}
+
+	return w.WAL.Save(st, writeEnts)
+}
+
+// walCryptor is an object that provides the same functions as `etcd/wal`
+// and `etcd/snap` that we need to open a WAL object or Snapshotter object
+type walCryptor struct {
+	encrypter encryption.Encrypter
+	decrypter encryption.Decrypter
+}
+
+// NewWALFactory returns an object that can be used to produce objects that
+// will read from and write to encrypted WALs on disk.
+func NewWALFactory(encrypter encryption.Encrypter, decrypter encryption.Decrypter) WALFactory {
+	return walCryptor{
+		encrypter: encrypter,
+		decrypter: decrypter,
+	}
+}
+
+// Create returns a new WAL object with the given encrypters and decrypters.
+func (wc walCryptor) Create(dirpath string, metadata []byte) (WAL, error) {
+	w, err := wal.Create(dirpath, metadata)
+	if err != nil {
+		return nil, err
+	}
+	return &wrappedWAL{
+		WAL:       w,
+		encrypter: wc.encrypter,
+		decrypter: wc.decrypter,
+	}, nil
+}
+
+// Open returns a new WAL object with the given encrypters and decrypters.
+func (wc walCryptor) Open(dirpath string, snap walpb.Snapshot) (WAL, error) {
+	w, err := wal.Open(dirpath, snap)
+	if err != nil {
+		return nil, err
+	}
+	return &wrappedWAL{
+		WAL:       w,
+		encrypter: wc.encrypter,
+		decrypter: wc.decrypter,
+	}, nil
+}
+
+type originalWAL struct{}
+
+func (o originalWAL) Create(dirpath string, metadata []byte) (WAL, error) {
+	return wal.Create(dirpath, metadata)
+}
+func (o originalWAL) Open(dirpath string, walsnap walpb.Snapshot) (WAL, error) {
+	return wal.Open(dirpath, walsnap)
+}
+
+// OriginalWAL is the original `wal` package as an implemntation of the WALFactory interface
+var OriginalWAL WALFactory = originalWAL{}
+
+// WALData contains all the data returned by a WAL's ReadAll() function
+// (metadata, hardwate, and entries)
+type WALData struct {
+	Metadata  []byte
+	HardState raftpb.HardState
+	Entries   []raftpb.Entry
+}
+
+// ReadRepairWAL opens a WAL for reading, and attempts to read it.  If we can't read it, attempts to repair
+// and read again.
+func ReadRepairWAL(
+	ctx context.Context,
+	walDir string,
+	walsnap walpb.Snapshot,
+	factory WALFactory,
+) (WAL, WALData, error) {
+	var (
+		reader   WAL
+		metadata []byte
+		st       raftpb.HardState
+		ents     []raftpb.Entry
+		err      error
+	)
+	repaired := false
+	for {
+		if reader, err = factory.Open(walDir, walsnap); err != nil {
+			return nil, WALData{}, errors.Wrap(err, "failed to open WAL")
+		}
+		if metadata, st, ents, err = reader.ReadAll(); err != nil {
+			if closeErr := reader.Close(); closeErr != nil {
+				return nil, WALData{}, closeErr
+			}
+			if _, ok := err.(encryption.ErrCannotDecrypt); ok {
+				return nil, WALData{}, errors.Wrap(err, "failed to decrypt WAL")
+			}
+			// we can only repair ErrUnexpectedEOF and we never repair twice.
+			if repaired || err != io.ErrUnexpectedEOF {
+				return nil, WALData{}, errors.Wrap(err, "irreparable WAL error")
+			}
+			if !wal.Repair(walDir) {
+				return nil, WALData{}, errors.Wrap(err, "WAL error cannot be repaired")
+			}
+			log.G(ctx).WithError(err).Info("repaired WAL error")
+			repaired = true
+			continue
+		}
+		break
+	}
+	return reader, WALData{
+		Metadata:  metadata,
+		HardState: st,
+		Entries:   ents,
+	}, nil
+}
+
+// MigrateWALs reads existing WALs (from a particular snapshot and beyond) from one directory, encoded one way,
+// and writes them to a new directory, encoded a different way
+func MigrateWALs(ctx context.Context, oldDir, newDir string, oldFactory, newFactory WALFactory, snapshot walpb.Snapshot) error {
+	oldReader, waldata, err := ReadRepairWAL(ctx, oldDir, snapshot, oldFactory)
+	if err != nil {
+		return err
+	}
+	oldReader.Close()
+
+	// keep temporary wal directory so WAL initialization appears atomic
+	tmpdirpath := filepath.Clean(newDir) + ".tmp"
+	if fileutil.Exist(tmpdirpath) {
+		if err := os.RemoveAll(tmpdirpath); err != nil {
+			return errors.Wrap(err, "could not remove temporary WAL directory")
+		}
+	}
+	if err := fileutil.CreateDirAll(tmpdirpath); err != nil {
+		return errors.Wrap(err, "could not create temporary WAL directory")
+	}
+
+	tmpWAL, err := newFactory.Create(tmpdirpath, waldata.Metadata)
+	if err != nil {
+		return errors.Wrap(err, "could not create new WAL in temporary WAL directory")
+	}
+	defer tmpWAL.Close()
+
+	if err := tmpWAL.SaveSnapshot(snapshot); err != nil {
+		return errors.Wrap(err, "could not write WAL snapshot in temporary directory")
+	}
+
+	if err := tmpWAL.Save(waldata.HardState, waldata.Entries); err != nil {
+		return errors.Wrap(err, "could not migrate WALs to temporary directory")
+	}
+
+	return os.Rename(tmpdirpath, newDir)
+}
+
+// ListWALs lists all the wals in a directory and returns the list in lexical
+// order (oldest first)
+func ListWALs(dirpath string) ([]string, error) {
+	dirents, err := ioutil.ReadDir(dirpath)
+	if err != nil {
+		return nil, err
+	}
+
+	var wals []string
+	for _, dirent := range dirents {
+		if strings.HasSuffix(dirent.Name(), ".wal") {
+			wals = append(wals, dirent.Name())
+		}
+	}
+
+	// Sort WAL filenames in lexical order
+	sort.Sort(sort.StringSlice(wals))
+	return wals, nil
+}

vendor/github.com/docker/swarmkit/node/node.go

@@ -21,6 +21,7 @@ import (
 	"github.com/docker/swarmkit/ioutils"
 	"github.com/docker/swarmkit/log"
 	"github.com/docker/swarmkit/manager"
+	"github.com/docker/swarmkit/manager/encryption"
 	"github.com/docker/swarmkit/remotes"
 	"github.com/docker/swarmkit/xnet"
 	"github.com/pkg/errors"
@@ -34,6 +35,10 @@ const stateFilename = "state.json"
 var (
 	errNodeStarted    = errors.New("node: already started")
 	errNodeNotStarted = errors.New("node: not started")
+	certDirectory     = "certificates"
+
+	// ErrInvalidUnlockKey is returned when we can't decrypt the TLS certificate
+	ErrInvalidUnlockKey = errors.New("node is locked, and needs a valid unlock key")
 )
 
 // Config provides values for a Node.
@@ -81,6 +86,14 @@ type Config struct {
 	// HeartbeatTick defines the amount of ticks between each
 	// heartbeat sent to other members for health-check purposes
 	HeartbeatTick uint32
+
+	// AutoLockManagers determines whether or not an unlock key will be generated
+	// when bootstrapping a new cluster for the first time
+	AutoLockManagers bool
+
+	// UnlockKey is the key to unlock a node - used for decrypting at rest.  This
+	// only applies to nodes that have already joined a cluster.
+	UnlockKey []byte
 }
 
 // Node implements the primary node functionality for a member of a swarm
@@ -106,6 +119,7 @@ type Node struct {
 	agent                *agent.Agent
 	manager              *manager.Manager
 	notifyNodeChange     chan *api.Node // used to send role updates from the dispatcher api on promotion/demotion
+	unlockKey            []byte
 }
 
 // RemoteAPIAddr returns address on which remote manager api listens.
@@ -150,12 +164,18 @@ func New(c *Config) (*Node, error) {
 		ready:                make(chan struct{}),
 		certificateRequested: make(chan struct{}),
 		notifyNodeChange:     make(chan *api.Node, 1),
+		unlockKey:            c.UnlockKey,
 	}
+
+	if n.config.JoinAddr != "" || n.config.ForceNewCluster {
+		n.remotes = newPersistentRemotes(filepath.Join(n.config.StateDir, stateFilename))
+		if n.config.JoinAddr != "" {
+			n.remotes.Observe(api.Peer{Addr: n.config.JoinAddr}, remotes.DefaultObservationWeight)
+		}
+	}
+
 	n.roleCond = sync.NewCond(n.RLocker())
 	n.connCond = sync.NewCond(n.RLocker())
-	if err := n.loadCertificates(); err != nil {
-		return nil, err
-	}
 	return n, nil
 }
 
@@ -189,46 +209,7 @@ func (n *Node) run(ctx context.Context) (err error) {
 		}
 	}()
 
-	// NOTE: When this node is created by NewNode(), our nodeID is set if
-	// n.loadCertificates() succeeded in loading TLS credentials.
-	if n.config.JoinAddr == "" && n.nodeID == "" {
-		if err := n.bootstrapCA(); err != nil {
-			return err
-		}
-	}
-
-	if n.config.JoinAddr != "" || n.config.ForceNewCluster {
-		n.remotes = newPersistentRemotes(filepath.Join(n.config.StateDir, stateFilename))
-		if n.config.JoinAddr != "" {
-			n.remotes.Observe(api.Peer{Addr: n.config.JoinAddr}, remotes.DefaultObservationWeight)
-		}
-	}
-
-	// Obtain new certs and setup TLS certificates renewal for this node:
-	// - We call LoadOrCreateSecurityConfig which blocks until a valid certificate has been issued
-	// - We retrieve the nodeID from LoadOrCreateSecurityConfig through the info channel. This allows
-	// us to display the ID before the certificate gets issued (for potential approval).
-	// - We wait for LoadOrCreateSecurityConfig to finish since we need a certificate to operate.
-	// - Given a valid certificate, spin a renewal go-routine that will ensure that certificates stay
-	// up to date.
-	issueResponseChan := make(chan api.IssueNodeCertificateResponse, 1)
-	go func() {
-		select {
-		case <-ctx.Done():
-		case resp := <-issueResponseChan:
-			log.G(log.WithModule(ctx, "tls")).WithFields(logrus.Fields{
-				"node.id": resp.NodeID,
-			}).Debugf("requesting certificate")
-			n.Lock()
-			n.nodeID = resp.NodeID
-			n.nodeMembership = resp.NodeMembership
-			n.Unlock()
-			close(n.certificateRequested)
-		}
-	}()
-
-	certDir := filepath.Join(n.config.StateDir, "certificates")
-	securityConfig, err := ca.LoadOrCreateSecurityConfig(ctx, certDir, n.config.JoinToken, ca.ManagerRole, n.remotes, issueResponseChan)
+	securityConfig, err := n.loadSecurityConfig(ctx)
 	if err != nil {
 		return err
 	}
@@ -244,10 +225,6 @@ func (n *Node) run(ctx context.Context) (err error) {
 	}
 	defer db.Close()
 
-	if err := n.loadCertificates(); err != nil {
-		return err
-	}
-
 	forceCertRenewal := make(chan struct{})
 	renewCert := func() {
 		select {
@@ -289,7 +266,7 @@ func (n *Node) run(ctx context.Context) (err error) {
 		}
 	}()
 
-	updates := ca.RenewTLSConfig(ctx, securityConfig, certDir, n.remotes, forceCertRenewal)
+	updates := ca.RenewTLSConfig(ctx, securityConfig, n.remotes, forceCertRenewal)
 	go func() {
 		for {
 			select {
@@ -515,40 +492,100 @@ func (n *Node) Remotes() []api.Peer {
 	return remotes
 }
 
-func (n *Node) loadCertificates() error {
-	certDir := filepath.Join(n.config.StateDir, "certificates")
-	rootCA, err := ca.GetLocalRootCA(certDir)
-	if err != nil {
-		if err == ca.ErrNoLocalRootCA {
-			return nil
-		}
-		return err
+func (n *Node) loadSecurityConfig(ctx context.Context) (*ca.SecurityConfig, error) {
+	paths := ca.NewConfigPaths(filepath.Join(n.config.StateDir, certDirectory))
+	var securityConfig *ca.SecurityConfig
+
+	krw := ca.NewKeyReadWriter(paths.Node, n.unlockKey, &manager.RaftDEKData{})
+	if err := krw.Migrate(); err != nil {
+		return nil, err
 	}
-	configPaths := ca.NewConfigPaths(certDir)
-	clientTLSCreds, _, err := ca.LoadTLSCreds(rootCA, configPaths.Node)
-	if err != nil {
-		if os.IsNotExist(err) {
-			return nil
+
+	// Check if we already have a valid certificates on disk.
+	rootCA, err := ca.GetLocalRootCA(paths.RootCA)
+	if err != nil && err != ca.ErrNoLocalRootCA {
+		return nil, err
+	}
+	if err == nil {
+		clientTLSCreds, serverTLSCreds, err := ca.LoadTLSCreds(rootCA, krw)
+		_, ok := errors.Cause(err).(ca.ErrInvalidKEK)
+		switch {
+		case err == nil:
+			securityConfig = ca.NewSecurityConfig(&rootCA, krw, clientTLSCreds, serverTLSCreds)
+			log.G(ctx).Debug("loaded CA and TLS certificates")
+		case ok:
+			return nil, ErrInvalidUnlockKey
+		case os.IsNotExist(err):
+			break
+		default:
+			return nil, errors.Wrapf(err, "error while loading TLS certificate in %s", paths.Node.Cert)
+		}
+	}
+
+	if securityConfig == nil {
+		if n.config.JoinAddr == "" {
+			// if we're not joining a cluster, bootstrap a new one - and we have to set the unlock key
+			n.unlockKey = nil
+			if n.config.AutoLockManagers {
+				n.unlockKey = encryption.GenerateSecretKey()
+			}
+			krw = ca.NewKeyReadWriter(paths.Node, n.unlockKey, &manager.RaftDEKData{})
+			rootCA, err = ca.CreateRootCA(ca.DefaultRootCN, paths.RootCA)
+			if err != nil {
+				return nil, err
+			}
+			log.G(ctx).Debug("generated CA key and certificate")
+		} else if err == ca.ErrNoLocalRootCA { // from previous error loading the root CA from disk
+			rootCA, err = ca.DownloadRootCA(ctx, paths.RootCA, n.config.JoinToken, n.remotes)
+			if err != nil {
+				return nil, err
+			}
+			log.G(ctx).Debug("downloaded CA certificate")
 		}
 
-		return errors.Wrapf(err, "error while loading TLS Certificate in %s", configPaths.Node.Cert)
+		// Obtain new certs and setup TLS certificates renewal for this node:
+		// - We call LoadOrCreateSecurityConfig which blocks until a valid certificate has been issued
+		// - We retrieve the nodeID from LoadOrCreateSecurityConfig through the info channel. This allows
+		// us to display the ID before the certificate gets issued (for potential approval).
+		// - We wait for LoadOrCreateSecurityConfig to finish since we need a certificate to operate.
+		// - Given a valid certificate, spin a renewal go-routine that will ensure that certificates stay
+		// up to date.
+		issueResponseChan := make(chan api.IssueNodeCertificateResponse, 1)
+		go func() {
+			select {
+			case <-ctx.Done():
+			case resp := <-issueResponseChan:
+				log.G(log.WithModule(ctx, "tls")).WithFields(logrus.Fields{
+					"node.id": resp.NodeID,
+				}).Debugf("loaded TLS certificate")
+				n.Lock()
+				n.nodeID = resp.NodeID
+				n.nodeMembership = resp.NodeMembership
+				n.Unlock()
+				close(n.certificateRequested)
+			}
+		}()
+
+		// LoadOrCreateSecurityConfig is the point at which a new node joining a cluster will retrieve TLS
+		// certificates and write them to disk
+		securityConfig, err = ca.LoadOrCreateSecurityConfig(
+			ctx, rootCA, n.config.JoinToken, ca.ManagerRole, n.remotes, issueResponseChan, krw)
+		if err != nil {
+			if _, ok := errors.Cause(err).(ca.ErrInvalidKEK); ok {
+				return nil, ErrInvalidUnlockKey
+			}
+			return nil, err
+		}
 	}
-	// todo: try csr if no cert or store nodeID/role in some other way
+
 	n.Lock()
-	n.role = clientTLSCreds.Role()
-	n.nodeID = clientTLSCreds.NodeID()
+	n.role = securityConfig.ClientTLSCreds.Role()
+	n.nodeID = securityConfig.ClientTLSCreds.NodeID()
 	n.nodeMembership = api.NodeMembershipAccepted
 	n.roleCond.Broadcast()
 	n.Unlock()
 
-	return nil
-}
-
-func (n *Node) bootstrapCA() error {
-	if err := ca.BootstrapCluster(filepath.Join(n.config.StateDir, "certificates")); err != nil {
-		return err
-	}
-	return n.loadCertificates()
+	return securityConfig, nil
 }
 
 func (n *Node) initManagerConnection(ctx context.Context, ready chan<- struct{}) error {
@@ -626,13 +663,15 @@ func (n *Node) runManager(ctx context.Context, securityConfig *ca.SecurityConfig
 				ListenAddr:    n.config.ListenRemoteAPI,
 				AdvertiseAddr: n.config.AdvertiseRemoteAPI,
 			},
-			ControlAPI:     n.config.ListenControlAPI,
-			SecurityConfig: securityConfig,
-			ExternalCAs:    n.config.ExternalCAs,
-			JoinRaft:       remoteAddr.Addr,
-			StateDir:       n.config.StateDir,
-			HeartbeatTick:  n.config.HeartbeatTick,
-			ElectionTick:   n.config.ElectionTick,
+			ControlAPI:       n.config.ListenControlAPI,
+			SecurityConfig:   securityConfig,
+			ExternalCAs:      n.config.ExternalCAs,
+			JoinRaft:         remoteAddr.Addr,
+			StateDir:         n.config.StateDir,
+			HeartbeatTick:    n.config.HeartbeatTick,
+			ElectionTick:     n.config.ElectionTick,
+			AutoLockManagers: n.config.AutoLockManagers,
+			UnlockKey:        n.unlockKey,
 		})
 		if err != nil {
 			return err

vendor/github.com/pkg/errors/LICENSE

@@ -21,4 +21,3 @@ SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
 CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
 OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-

vendor/github.com/pkg/errors/errors.go

@@ -14,13 +14,18 @@
 // Adding context to an error
 //
 // The errors.Wrap function returns a new error that adds context to the
-// original error. For example
+// original error by recording a stack trace at the point Wrap is called,
+// and the supplied message. For example
 //
 //     _, err := ioutil.ReadAll(r)
 //     if err != nil {
 //             return errors.Wrap(err, "read failed")
 //     }
 //
+// If additional control is required the errors.WithStack and errors.WithMessage
+// functions destructure errors.Wrap into its component operations of annotating
+// an error with a stack trace and an a message, respectively.
+//
 // Retrieving the cause of an error
 //
 // Using errors.Wrap constructs a stack of errors, adding context to the
@@ -28,7 +33,7 @@
 // to reverse the operation of errors.Wrap to retrieve the original error
 // for inspection. Any error value which implements this interface
 //
-//     type Causer interface {
+//     type causer interface {
 //             Cause() error
 //     }
 //
@@ -43,6 +48,9 @@
 //             // unknown error
 //     }
 //
+// causer interface is not exported by this package, but is considered a part
+// of stable public API.
+//
 // Formatted printing of errors
 //
 // All error values returned from this package implement fmt.Formatter and can
@@ -59,7 +67,7 @@
 // New, Errorf, Wrap, and Wrapf record a stack trace at the point they are
 // invoked. This information can be retrieved with the following interface.
 //
-//     type StackTrace interface {
+//     type stackTracer interface {
 //             StackTrace() errors.StackTrace
 //     }
 //
@@ -67,16 +75,19 @@
 //
 //     type StackTrace []Frame
 //
-// The Frame type represents a call site in the stacktrace. Frame supports
+// The Frame type represents a call site in the stack trace. Frame supports
 // the fmt.Formatter interface that can be used for printing information about
-// the stacktrace of this error. For example:
+// the stack trace of this error. For example:
 //
-//     if err, ok := err.(StackTrace); ok {
+//     if err, ok := err.(stackTracer); ok {
 //             for _, f := range err.StackTrace() {
 //                     fmt.Printf("%+s:%d", f)
 //             }
 //     }
 //
+// stackTracer interface is not exported by this package, but is considered a part
+// of stable public API.
+//
 // See the documentation for Frame.Format for more details.
 package errors
 
@@ -85,102 +96,149 @@ import (
 	"io"
 )
 
-// _error is an error implementation returned by New and Errorf
-// that implements its own fmt.Formatter.
-type _error struct {
-	msg string
-	*stack
-}
-
-func (e _error) Error() string { return e.msg }
-
-func (e _error) Format(s fmt.State, verb rune) {
-	switch verb {
-	case 'v':
-		if s.Flag('+') {
-			io.WriteString(s, e.msg)
-			fmt.Fprintf(s, "%+v", e.StackTrace())
-			return
-		}
-		fallthrough
-	case 's':
-		io.WriteString(s, e.msg)
-	}
-}
-
 // New returns an error with the supplied message.
+// New also records the stack trace at the point it was called.
 func New(message string) error {
-	return _error{
-		message,
-		callers(),
+	return &fundamental{
+		msg:   message,
+		stack: callers(),
 	}
 }
 
 // Errorf formats according to a format specifier and returns the string
 // as a value that satisfies error.
+// Errorf also records the stack trace at the point it was called.
 func Errorf(format string, args ...interface{}) error {
-	return _error{
-		fmt.Sprintf(format, args...),
+	return &fundamental{
+		msg:   fmt.Sprintf(format, args...),
+		stack: callers(),
+	}
+}
+
+// fundamental is an error that has a message and a stack, but no caller.
+type fundamental struct {
+	msg string
+	*stack
+}
+
+func (f *fundamental) Error() string { return f.msg }
+
+func (f *fundamental) Format(s fmt.State, verb rune) {
+	switch verb {
+	case 'v':
+		if s.Flag('+') {
+			io.WriteString(s, f.msg)
+			f.stack.Format(s, verb)
+			return
+		}
+		fallthrough
+	case 's':
+		io.WriteString(s, f.msg)
+	case 'q':
+		fmt.Fprintf(s, "%q", f.msg)
+	}
+}
+
+// WithStack annotates err with a stack trace at the point WithStack was called.
+// If err is nil, WithStack returns nil.
+func WithStack(err error) error {
+	if err == nil {
+		return nil
+	}
+	return &withStack{
+		err,
 		callers(),
 	}
 }
 
-type cause struct {
-	cause error
-	msg   string
-}
-
-func (c cause) Error() string { return fmt.Sprintf("%s: %v", c.msg, c.Cause()) }
-func (c cause) Cause() error  { return c.cause }
-
-// wrapper is an error implementation returned by Wrap and Wrapf
-// that implements its own fmt.Formatter.
-type wrapper struct {
-	cause
+type withStack struct {
+	error
 	*stack
 }
 
-func (w wrapper) Format(s fmt.State, verb rune) {
+func (w *withStack) Cause() error { return w.error }
+
+func (w *withStack) Format(s fmt.State, verb rune) {
 	switch verb {
 	case 'v':
 		if s.Flag('+') {
-			fmt.Fprintf(s, "%+v\n", w.Cause())
-			fmt.Fprintf(s, "%+v: %s", w.StackTrace()[0], w.msg)
+			fmt.Fprintf(s, "%+v", w.Cause())
+			w.stack.Format(s, verb)
 			return
 		}
 		fallthrough
 	case 's':
 		io.WriteString(s, w.Error())
+	case 'q':
+		fmt.Fprintf(s, "%q", w.Error())
 	}
 }
 
-// Wrap returns an error annotating err with message.
+// Wrap returns an error annotating err with a stack trace
+// at the point Wrap is called, and the supplied message.
 // If err is nil, Wrap returns nil.
 func Wrap(err error, message string) error {
 	if err == nil {
 		return nil
 	}
-	return wrapper{
-		cause: cause{
-			cause: err,
-			msg:   message,
-		},
-		stack: callers(),
+	err = &withMessage{
+		cause: err,
+		msg:   message,
+	}
+	return &withStack{
+		err,
+		callers(),
 	}
 }
 
-// Wrapf returns an error annotating err with the format specifier.
+// Wrapf returns an error annotating err with a stack trace
+// at the point Wrapf is call, and the format specifier.
 // If err is nil, Wrapf returns nil.
 func Wrapf(err error, format string, args ...interface{}) error {
 	if err == nil {
 		return nil
 	}
-	return wrapper{
-		cause: cause{
-			cause: err,
-			msg:   fmt.Sprintf(format, args...),
-		},
-		stack: callers(),
+	err = &withMessage{
+		cause: err,
+		msg:   fmt.Sprintf(format, args...),
+	}
+	return &withStack{
+		err,
+		callers(),
+	}
+}
+
+// WithMessage annotates err with a new message.
+// If err is nil, WithMessage returns nil.
+func WithMessage(err error, message string) error {
+	if err == nil {
+		return nil
+	}
+	return &withMessage{
+		cause: err,
+		msg:   message,
+	}
+}
+
+type withMessage struct {
+	cause error
+	msg   string
+}
+
+func (w *withMessage) Error() string { return w.msg + ": " + w.cause.Error() }
+func (w *withMessage) Cause() error  { return w.cause }
+
+func (w *withMessage) Format(s fmt.State, verb rune) {
+	switch verb {
+	case 'v':
+		if s.Flag('+') {
+			fmt.Fprintf(s, "%+v\n", w.Cause())
+			io.WriteString(s, w.msg)
+			return
+		}
+		fallthrough
+	case 's', 'q':
+		io.WriteString(s, w.Error())
 	}
 }
 
@@ -188,7 +246,7 @@ func Wrapf(err error, format string, args ...interface{}) error {
 // An error value has a cause if it implements the following
 // interface:
 //
-//     type Causer interface {
+//     type causer interface {
 //            Cause() error
 //     }
 //

vendor/github.com/pkg/errors/stack.go

@@ -100,6 +100,19 @@ func (st StackTrace) Format(s fmt.State, verb rune) {
 // stack represents a stack of program counters.
 type stack []uintptr
 
+func (s *stack) Format(st fmt.State, verb rune) {
+	switch verb {
+	case 'v':
+		switch {
+		case st.Flag('+'):
+			for _, pc := range *s {
+				f := Frame(pc)
+				fmt.Fprintf(st, "\n%+v", f)
+			}
+		}
+	}
+}
+
 func (s *stack) StackTrace() StackTrace {
 	f := make([]Frame, len(*s))
 	for i := 0; i < len(f); i++ {

vendor/golang.org/x/crypto/nacl/secretbox/secretbox.go

@@ -0,0 +1,149 @@
+// Copyright 2012 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+/*
+Package secretbox encrypts and authenticates small messages.
+
+Secretbox uses XSalsa20 and Poly1305 to encrypt and authenticate messages with
+secret-key cryptography. The length of messages is not hidden.
+
+It is the caller's responsibility to ensure the uniqueness of nonces—for
+example, by using nonce 1 for the first message, nonce 2 for the second
+message, etc. Nonces are long enough that randomly generated nonces have
+negligible risk of collision.
+
+This package is interoperable with NaCl: http://nacl.cr.yp.to/secretbox.html.
+*/
+package secretbox // import "golang.org/x/crypto/nacl/secretbox"
+
+import (
+	"golang.org/x/crypto/poly1305"
+	"golang.org/x/crypto/salsa20/salsa"
+)
+
+// Overhead is the number of bytes of overhead when boxing a message.
+const Overhead = poly1305.TagSize
+
+// setup produces a sub-key and Salsa20 counter given a nonce and key.
+func setup(subKey *[32]byte, counter *[16]byte, nonce *[24]byte, key *[32]byte) {
+	// We use XSalsa20 for encryption so first we need to generate a
+	// key and nonce with HSalsa20.
+	var hNonce [16]byte
+	copy(hNonce[:], nonce[:])
+	salsa.HSalsa20(subKey, &hNonce, key, &salsa.Sigma)
+
+	// The final 8 bytes of the original nonce form the new nonce.
+	copy(counter[:], nonce[16:])
+}
+
+// sliceForAppend takes a slice and a requested number of bytes. It returns a
+// slice with the contents of the given slice followed by that many bytes and a
+// second slice that aliases into it and contains only the extra bytes. If the
+// original slice has sufficient capacity then no allocation is performed.
+func sliceForAppend(in []byte, n int) (head, tail []byte) {
+	if total := len(in) + n; cap(in) >= total {
+		head = in[:total]
+	} else {
+		head = make([]byte, total)
+		copy(head, in)
+	}
+	tail = head[len(in):]
+	return
+}
+
+// Seal appends an encrypted and authenticated copy of message to out, which
+// must not overlap message. The key and nonce pair must be unique for each
+// distinct message and the output will be Overhead bytes longer than message.
+func Seal(out, message []byte, nonce *[24]byte, key *[32]byte) []byte {
+	var subKey [32]byte
+	var counter [16]byte
+	setup(&subKey, &counter, nonce, key)
+
+	// The Poly1305 key is generated by encrypting 32 bytes of zeros. Since
+	// Salsa20 works with 64-byte blocks, we also generate 32 bytes of
+	// keystream as a side effect.
+	var firstBlock [64]byte
+	salsa.XORKeyStream(firstBlock[:], firstBlock[:], &counter, &subKey)
+
+	var poly1305Key [32]byte
+	copy(poly1305Key[:], firstBlock[:])
+
+	ret, out := sliceForAppend(out, len(message)+poly1305.TagSize)
+
+	// We XOR up to 32 bytes of message with the keystream generated from
+	// the first block.
+	firstMessageBlock := message
+	if len(firstMessageBlock) > 32 {
+		firstMessageBlock = firstMessageBlock[:32]
+	}
+
+	tagOut := out
+	out = out[poly1305.TagSize:]
+	for i, x := range firstMessageBlock {
+		out[i] = firstBlock[32+i] ^ x
+	}
+	message = message[len(firstMessageBlock):]
+	ciphertext := out
+	out = out[len(firstMessageBlock):]
+
+	// Now encrypt the rest.
+	counter[8] = 1
+	salsa.XORKeyStream(out, message, &counter, &subKey)
+
+	var tag [poly1305.TagSize]byte
+	poly1305.Sum(&tag, ciphertext, &poly1305Key)
+	copy(tagOut, tag[:])
+
+	return ret
+}
+
+// Open authenticates and decrypts a box produced by Seal and appends the
+// message to out, which must not overlap box. The output will be Overhead
+// bytes smaller than box.
+func Open(out []byte, box []byte, nonce *[24]byte, key *[32]byte) ([]byte, bool) {
+	if len(box) < Overhead {
+		return nil, false
+	}
+
+	var subKey [32]byte
+	var counter [16]byte
+	setup(&subKey, &counter, nonce, key)
+
+	// The Poly1305 key is generated by encrypting 32 bytes of zeros. Since
+	// Salsa20 works with 64-byte blocks, we also generate 32 bytes of
+	// keystream as a side effect.
+	var firstBlock [64]byte
+	salsa.XORKeyStream(firstBlock[:], firstBlock[:], &counter, &subKey)
+
+	var poly1305Key [32]byte
+	copy(poly1305Key[:], firstBlock[:])
+	var tag [poly1305.TagSize]byte
+	copy(tag[:], box)
+
+	if !poly1305.Verify(&tag, box[poly1305.TagSize:], &poly1305Key) {
+		return nil, false
+	}
+
+	ret, out := sliceForAppend(out, len(box)-Overhead)
+
+	// We XOR up to 32 bytes of box with the keystream generated from
+	// the first block.
+	box = box[Overhead:]
+	firstMessageBlock := box
+	if len(firstMessageBlock) > 32 {
+		firstMessageBlock = firstMessageBlock[:32]
+	}
+	for i, x := range firstMessageBlock {
+		out[i] = firstBlock[32+i] ^ x
+	}
+
+	box = box[len(firstMessageBlock):]
+	out = out[len(firstMessageBlock):]
+
+	// Now decrypt the rest.
+	counter[8] = 1
+	salsa.XORKeyStream(out, box, &counter, &subKey)
+
+	return ret, true
+}

vendor/golang.org/x/crypto/poly1305/const_amd64.s

@@ -0,0 +1,45 @@
+// Copyright 2012 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// This code was translated into a form compatible with 6a from the public
+// domain sources in SUPERCOP: http://bench.cr.yp.to/supercop.html
+
+// +build amd64,!gccgo,!appengine
+
+DATA ·SCALE(SB)/8, $0x37F4000000000000
+GLOBL ·SCALE(SB), 8, $8
+DATA ·TWO32(SB)/8, $0x41F0000000000000
+GLOBL ·TWO32(SB), 8, $8
+DATA ·TWO64(SB)/8, $0x43F0000000000000
+GLOBL ·TWO64(SB), 8, $8
+DATA ·TWO96(SB)/8, $0x45F0000000000000
+GLOBL ·TWO96(SB), 8, $8
+DATA ·ALPHA32(SB)/8, $0x45E8000000000000
+GLOBL ·ALPHA32(SB), 8, $8
+DATA ·ALPHA64(SB)/8, $0x47E8000000000000
+GLOBL ·ALPHA64(SB), 8, $8
+DATA ·ALPHA96(SB)/8, $0x49E8000000000000
+GLOBL ·ALPHA96(SB), 8, $8
+DATA ·ALPHA130(SB)/8, $0x4C08000000000000
+GLOBL ·ALPHA130(SB), 8, $8
+DATA ·DOFFSET0(SB)/8, $0x4330000000000000
+GLOBL ·DOFFSET0(SB), 8, $8
+DATA ·DOFFSET1(SB)/8, $0x4530000000000000
+GLOBL ·DOFFSET1(SB), 8, $8
+DATA ·DOFFSET2(SB)/8, $0x4730000000000000
+GLOBL ·DOFFSET2(SB), 8, $8
+DATA ·DOFFSET3(SB)/8, $0x4930000000000000
+GLOBL ·DOFFSET3(SB), 8, $8
+DATA ·DOFFSET3MINUSTWO128(SB)/8, $0x492FFFFE00000000
+GLOBL ·DOFFSET3MINUSTWO128(SB), 8, $8
+DATA ·HOFFSET0(SB)/8, $0x43300001FFFFFFFB
+GLOBL ·HOFFSET0(SB), 8, $8
+DATA ·HOFFSET1(SB)/8, $0x45300001FFFFFFFE
+GLOBL ·HOFFSET1(SB), 8, $8
+DATA ·HOFFSET2(SB)/8, $0x47300001FFFFFFFE
+GLOBL ·HOFFSET2(SB), 8, $8
+DATA ·HOFFSET3(SB)/8, $0x49300003FFFFFFFE
+GLOBL ·HOFFSET3(SB), 8, $8
+DATA ·ROUNDING(SB)/2, $0x137f
+GLOBL ·ROUNDING(SB), 8, $2

vendor/golang.org/x/crypto/poly1305/poly1305.go

@@ -0,0 +1,32 @@
+// Copyright 2012 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+/*
+Package poly1305 implements Poly1305 one-time message authentication code as specified in http://cr.yp.to/mac/poly1305-20050329.pdf.
+
+Poly1305 is a fast, one-time authentication function. It is infeasible for an
+attacker to generate an authenticator for a message without the key. However, a
+key must only be used for a single message. Authenticating two different
+messages with the same key allows an attacker to forge authenticators for other
+messages with the same key.
+
+Poly1305 was originally coupled with AES in order to make Poly1305-AES. AES was
+used with a fixed key in order to generate one-time keys from an nonce.
+However, in this package AES isn't used and the one-time key is specified
+directly.
+*/
+package poly1305 // import "golang.org/x/crypto/poly1305"
+
+import "crypto/subtle"
+
+// TagSize is the size, in bytes, of a poly1305 authenticator.
+const TagSize = 16
+
+// Verify returns true if mac is a valid authenticator for m with the given
+// key.
+func Verify(mac *[16]byte, m []byte, key *[32]byte) bool {
+	var tmp [16]byte
+	Sum(&tmp, m, key)
+	return subtle.ConstantTimeCompare(tmp[:], mac[:]) == 1
+}

vendor/golang.org/x/crypto/poly1305/poly1305_amd64.s

@@ -0,0 +1,497 @@
+// Copyright 2012 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// This code was translated into a form compatible with 6a from the public
+// domain sources in SUPERCOP: http://bench.cr.yp.to/supercop.html
+
+// +build amd64,!gccgo,!appengine
+
+// func poly1305(out *[16]byte, m *byte, mlen uint64, key *[32]key)
+TEXT ·poly1305(SB),0,$224-32
+	MOVQ out+0(FP),DI
+	MOVQ m+8(FP),SI
+	MOVQ mlen+16(FP),DX
+	MOVQ key+24(FP),CX
+
+	MOVQ SP,R11
+	MOVQ $31,R9
+	NOTQ R9
+	ANDQ R9,SP
+	ADDQ $32,SP
+
+	MOVQ R11,32(SP)
+	MOVQ R12,40(SP)
+	MOVQ R13,48(SP)
+	MOVQ R14,56(SP)
+	MOVQ R15,64(SP)
+	MOVQ BX,72(SP)
+	MOVQ BP,80(SP)
+	FLDCW ·ROUNDING(SB)
+	MOVL 0(CX),R8
+	MOVL 4(CX),R9
+	MOVL 8(CX),AX
+	MOVL 12(CX),R10
+	MOVQ DI,88(SP)
+	MOVQ CX,96(SP)
+	MOVL $0X43300000,108(SP)
+	MOVL $0X45300000,116(SP)
+	MOVL $0X47300000,124(SP)
+	MOVL $0X49300000,132(SP)
+	ANDL $0X0FFFFFFF,R8
+	ANDL $0X0FFFFFFC,R9
+	ANDL $0X0FFFFFFC,AX
+	ANDL $0X0FFFFFFC,R10
+	MOVL R8,104(SP)
+	MOVL R9,112(SP)
+	MOVL AX,120(SP)
+	MOVL R10,128(SP)
+	FMOVD 104(SP), F0
+	FSUBD ·DOFFSET0(SB), F0
+	FMOVD 112(SP), F0
+	FSUBD ·DOFFSET1(SB), F0
+	FMOVD 120(SP), F0
+	FSUBD ·DOFFSET2(SB), F0
+	FMOVD 128(SP), F0
+	FSUBD ·DOFFSET3(SB), F0
+	FXCHD F0, F3
+	FMOVDP F0, 136(SP)
+	FXCHD F0, F1
+	FMOVD F0, 144(SP)
+	FMULD ·SCALE(SB), F0
+	FMOVDP F0, 152(SP)
+	FMOVD F0, 160(SP)
+	FMULD ·SCALE(SB), F0
+	FMOVDP F0, 168(SP)
+	FMOVD F0, 176(SP)
+	FMULD ·SCALE(SB), F0
+	FMOVDP F0, 184(SP)
+	FLDZ
+	FLDZ
+	FLDZ
+	FLDZ
+	CMPQ DX,$16
+	JB ADDATMOST15BYTES
+	INITIALATLEAST16BYTES:
+	MOVL 12(SI),DI
+	MOVL 8(SI),CX
+	MOVL 4(SI),R8
+	MOVL 0(SI),R9
+	MOVL DI,128(SP)
+	MOVL CX,120(SP)
+	MOVL R8,112(SP)
+	MOVL R9,104(SP)
+	ADDQ $16,SI
+	SUBQ $16,DX
+	FXCHD F0, F3
+	FADDD 128(SP), F0
+	FSUBD ·DOFFSET3MINUSTWO128(SB), F0
+	FXCHD F0, F1
+	FADDD 112(SP), F0
+	FSUBD ·DOFFSET1(SB), F0
+	FXCHD F0, F2
+	FADDD 120(SP), F0
+	FSUBD ·DOFFSET2(SB), F0
+	FXCHD F0, F3
+	FADDD 104(SP), F0
+	FSUBD ·DOFFSET0(SB), F0
+	CMPQ DX,$16
+	JB MULTIPLYADDATMOST15BYTES
+	MULTIPLYADDATLEAST16BYTES:
+	MOVL 12(SI),DI
+	MOVL 8(SI),CX
+	MOVL 4(SI),R8
+	MOVL 0(SI),R9
+	MOVL DI,128(SP)
+	MOVL CX,120(SP)
+	MOVL R8,112(SP)
+	MOVL R9,104(SP)
+	ADDQ $16,SI
+	SUBQ $16,DX
+	FMOVD ·ALPHA130(SB), F0
+	FADDD F2,F0
+	FSUBD ·ALPHA130(SB), F0
+	FSUBD F0,F2
+	FMULD ·SCALE(SB), F0
+	FMOVD ·ALPHA32(SB), F0
+	FADDD F2,F0
+	FSUBD ·ALPHA32(SB), F0
+	FSUBD F0,F2
+	FXCHD F0, F2
+	FADDDP F0,F1
+	FMOVD ·ALPHA64(SB), F0
+	FADDD F4,F0
+	FSUBD ·ALPHA64(SB), F0
+	FSUBD F0,F4
+	FMOVD ·ALPHA96(SB), F0
+	FADDD F6,F0
+	FSUBD ·ALPHA96(SB), F0
+	FSUBD F0,F6
+	FXCHD F0, F6
+	FADDDP F0,F1
+	FXCHD F0, F3
+	FADDDP F0,F5
+	FXCHD F0, F3
+	FADDDP F0,F1
+	FMOVD 176(SP), F0
+	FMULD F3,F0
+	FMOVD 160(SP), F0
+	FMULD F4,F0
+	FMOVD 144(SP), F0
+	FMULD F5,F0
+	FMOVD 136(SP), F0
+	FMULDP F0,F6
+	FMOVD 160(SP), F0
+	FMULD F4,F0
+	FADDDP F0,F3
+	FMOVD 144(SP), F0
+	FMULD F4,F0
+	FADDDP F0,F2
+	FMOVD 136(SP), F0
+	FMULD F4,F0
+	FADDDP F0,F1
+	FMOVD 184(SP), F0
+	FMULDP F0,F4
+	FXCHD F0, F3
+	FADDDP F0,F5
+	FMOVD 144(SP), F0
+	FMULD F4,F0
+	FADDDP F0,F2
+	FMOVD 136(SP), F0
+	FMULD F4,F0
+	FADDDP F0,F1
+	FMOVD 184(SP), F0
+	FMULD F4,F0
+	FADDDP F0,F3
+	FMOVD 168(SP), F0
+	FMULDP F0,F4
+	FXCHD F0, F3
+	FADDDP F0,F4
+	FMOVD 136(SP), F0
+	FMULD F5,F0
+	FADDDP F0,F1
+	FXCHD F0, F3
+	FMOVD 184(SP), F0
+	FMULD F5,F0
+	FADDDP F0,F3
+	FXCHD F0, F1
+	FMOVD 168(SP), F0
+	FMULD F5,F0
+	FADDDP F0,F1
+	FMOVD 152(SP), F0
+	FMULDP F0,F5
+	FXCHD F0, F4
+	FADDDP F0,F1
+	CMPQ DX,$16
+	FXCHD F0, F2
+	FMOVD 128(SP), F0
+	FSUBD ·DOFFSET3MINUSTWO128(SB), F0
+	FADDDP F0,F1
+	FXCHD F0, F1
+	FMOVD 120(SP), F0
+	FSUBD ·DOFFSET2(SB), F0
+	FADDDP F0,F1
+	FXCHD F0, F3
+	FMOVD 112(SP), F0
+	FSUBD ·DOFFSET1(SB), F0
+	FADDDP F0,F1
+	FXCHD F0, F2
+	FMOVD 104(SP), F0
+	FSUBD ·DOFFSET0(SB), F0
+	FADDDP F0,F1
+	JAE MULTIPLYADDATLEAST16BYTES
+	MULTIPLYADDATMOST15BYTES:
+	FMOVD ·ALPHA130(SB), F0
+	FADDD F2,F0
+	FSUBD ·ALPHA130(SB), F0
+	FSUBD F0,F2
+	FMULD ·SCALE(SB), F0
+	FMOVD ·ALPHA32(SB), F0
+	FADDD F2,F0
+	FSUBD ·ALPHA32(SB), F0
+	FSUBD F0,F2
+	FMOVD ·ALPHA64(SB), F0
+	FADDD F5,F0
+	FSUBD ·ALPHA64(SB), F0
+	FSUBD F0,F5
+	FMOVD ·ALPHA96(SB), F0
+	FADDD F7,F0
+	FSUBD ·ALPHA96(SB), F0
+	FSUBD F0,F7
+	FXCHD F0, F7
+	FADDDP F0,F1
+	FXCHD F0, F5
+	FADDDP F0,F1
+	FXCHD F0, F3
+	FADDDP F0,F5
+	FADDDP F0,F1
+	FMOVD 176(SP), F0
+	FMULD F1,F0
+	FMOVD 160(SP), F0
+	FMULD F2,F0
+	FMOVD 144(SP), F0
+	FMULD F3,F0
+	FMOVD 136(SP), F0
+	FMULDP F0,F4
+	FMOVD 160(SP), F0
+	FMULD F5,F0
+	FADDDP F0,F3
+	FMOVD 144(SP), F0
+	FMULD F5,F0
+	FADDDP F0,F2
+	FMOVD 136(SP), F0
+	FMULD F5,F0
+	FADDDP F0,F1
+	FMOVD 184(SP), F0
+	FMULDP F0,F5
+	FXCHD F0, F4
+	FADDDP F0,F3
+	FMOVD 144(SP), F0
+	FMULD F5,F0
+	FADDDP F0,F2
+	FMOVD 136(SP), F0
+	FMULD F5,F0
+	FADDDP F0,F1
+	FMOVD 184(SP), F0
+	FMULD F5,F0
+	FADDDP F0,F4
+	FMOVD 168(SP), F0
+	FMULDP F0,F5
+	FXCHD F0, F4
+	FADDDP F0,F2
+	FMOVD 136(SP), F0
+	FMULD F5,F0
+	FADDDP F0,F1
+	FMOVD 184(SP), F0
+	FMULD F5,F0
+	FADDDP F0,F4
+	FMOVD 168(SP), F0
+	FMULD F5,F0
+	FADDDP F0,F3
+	FMOVD 152(SP), F0
+	FMULDP F0,F5
+	FXCHD F0, F4
+	FADDDP F0,F1
+	ADDATMOST15BYTES:
+	CMPQ DX,$0
+	JE NOMOREBYTES
+	MOVL $0,0(SP)
+	MOVL $0, 4 (SP)
+	MOVL $0, 8 (SP)
+	MOVL $0, 12 (SP)
+	LEAQ 0(SP),DI
+	MOVQ DX,CX
+	REP; MOVSB
+	MOVB $1,0(DI)
+	MOVL  12 (SP),DI
+	MOVL  8 (SP),SI
+	MOVL  4 (SP),DX
+	MOVL 0(SP),CX
+	MOVL DI,128(SP)
+	MOVL SI,120(SP)
+	MOVL DX,112(SP)
+	MOVL CX,104(SP)
+	FXCHD F0, F3
+	FADDD 128(SP), F0
+	FSUBD ·DOFFSET3(SB), F0
+	FXCHD F0, F2
+	FADDD 120(SP), F0
+	FSUBD ·DOFFSET2(SB), F0
+	FXCHD F0, F1
+	FADDD 112(SP), F0
+	FSUBD ·DOFFSET1(SB), F0
+	FXCHD F0, F3
+	FADDD 104(SP), F0
+	FSUBD ·DOFFSET0(SB), F0
+	FMOVD ·ALPHA130(SB), F0
+	FADDD F3,F0
+	FSUBD ·ALPHA130(SB), F0
+	FSUBD F0,F3
+	FMULD ·SCALE(SB), F0
+	FMOVD ·ALPHA32(SB), F0
+	FADDD F2,F0
+	FSUBD ·ALPHA32(SB), F0
+	FSUBD F0,F2
+	FMOVD ·ALPHA64(SB), F0
+	FADDD F6,F0
+	FSUBD ·ALPHA64(SB), F0
+	FSUBD F0,F6
+	FMOVD ·ALPHA96(SB), F0
+	FADDD F5,F0
+	FSUBD ·ALPHA96(SB), F0
+	FSUBD F0,F5
+	FXCHD F0, F4
+	FADDDP F0,F3
+	FXCHD F0, F6
+	FADDDP F0,F1
+	FXCHD F0, F3
+	FADDDP F0,F5
+	FXCHD F0, F3
+	FADDDP F0,F1
+	FMOVD 176(SP), F0
+	FMULD F3,F0
+	FMOVD 160(SP), F0
+	FMULD F4,F0
+	FMOVD 144(SP), F0
+	FMULD F5,F0
+	FMOVD 136(SP), F0
+	FMULDP F0,F6
+	FMOVD 160(SP), F0
+	FMULD F5,F0
+	FADDDP F0,F3
+	FMOVD 144(SP), F0
+	FMULD F5,F0
+	FADDDP F0,F2
+	FMOVD 136(SP), F0
+	FMULD F5,F0
+	FADDDP F0,F1
+	FMOVD 184(SP), F0
+	FMULDP F0,F5
+	FXCHD F0, F4
+	FADDDP F0,F5
+	FMOVD 144(SP), F0
+	FMULD F6,F0
+	FADDDP F0,F2
+	FMOVD 136(SP), F0
+	FMULD F6,F0
+	FADDDP F0,F1
+	FMOVD 184(SP), F0
+	FMULD F6,F0
+	FADDDP F0,F4
+	FMOVD 168(SP), F0
+	FMULDP F0,F6
+	FXCHD F0, F5
+	FADDDP F0,F4
+	FMOVD 136(SP), F0
+	FMULD F2,F0
+	FADDDP F0,F1
+	FMOVD 184(SP), F0
+	FMULD F2,F0
+	FADDDP F0,F5
+	FMOVD 168(SP), F0
+	FMULD F2,F0
+	FADDDP F0,F3
+	FMOVD 152(SP), F0
+	FMULDP F0,F2
+	FXCHD F0, F1
+	FADDDP F0,F3
+	FXCHD F0, F3
+	FXCHD F0, F2
+	NOMOREBYTES:
+	MOVL $0,R10
+	FMOVD ·ALPHA130(SB), F0
+	FADDD F4,F0
+	FSUBD ·ALPHA130(SB), F0
+	FSUBD F0,F4
+	FMULD ·SCALE(SB), F0
+	FMOVD ·ALPHA32(SB), F0
+	FADDD F2,F0
+	FSUBD ·ALPHA32(SB), F0
+	FSUBD F0,F2
+	FMOVD ·ALPHA64(SB), F0
+	FADDD F4,F0
+	FSUBD ·ALPHA64(SB), F0
+	FSUBD F0,F4
+	FMOVD ·ALPHA96(SB), F0
+	FADDD F6,F0
+	FSUBD ·ALPHA96(SB), F0
+	FXCHD F0, F6
+	FSUBD F6,F0
+	FXCHD F0, F4
+	FADDDP F0,F3
+	FXCHD F0, F4
+	FADDDP F0,F1
+	FXCHD F0, F2
+	FADDDP F0,F3
+	FXCHD F0, F4
+	FADDDP F0,F3
+	FXCHD F0, F3
+	FADDD ·HOFFSET0(SB), F0
+	FXCHD F0, F3
+	FADDD ·HOFFSET1(SB), F0
+	FXCHD F0, F1
+	FADDD ·HOFFSET2(SB), F0
+	FXCHD F0, F2
+	FADDD ·HOFFSET3(SB), F0
+	FXCHD F0, F3
+	FMOVDP F0, 104(SP)
+	FMOVDP F0, 112(SP)
+	FMOVDP F0, 120(SP)
+	FMOVDP F0, 128(SP)
+	MOVL 108(SP),DI
+	ANDL $63,DI
+	MOVL 116(SP),SI
+	ANDL $63,SI
+	MOVL 124(SP),DX
+	ANDL $63,DX
+	MOVL 132(SP),CX
+	ANDL $63,CX
+	MOVL 112(SP),R8
+	ADDL DI,R8
+	MOVQ R8,112(SP)
+	MOVL 120(SP),DI
+	ADCL SI,DI
+	MOVQ DI,120(SP)
+	MOVL 128(SP),DI
+	ADCL DX,DI
+	MOVQ DI,128(SP)
+	MOVL R10,DI
+	ADCL CX,DI
+	MOVQ DI,136(SP)
+	MOVQ $5,DI
+	MOVL 104(SP),SI
+	ADDL SI,DI
+	MOVQ DI,104(SP)
+	MOVL R10,DI
+	MOVQ 112(SP),DX
+	ADCL DX,DI
+	MOVQ DI,112(SP)
+	MOVL R10,DI
+	MOVQ 120(SP),CX
+	ADCL CX,DI
+	MOVQ DI,120(SP)
+	MOVL R10,DI
+	MOVQ 128(SP),R8
+	ADCL R8,DI
+	MOVQ DI,128(SP)
+	MOVQ $0XFFFFFFFC,DI
+	MOVQ 136(SP),R9
+	ADCL R9,DI
+	SARL $16,DI
+	MOVQ DI,R9
+	XORL $0XFFFFFFFF,R9
+	ANDQ DI,SI
+	MOVQ 104(SP),AX
+	ANDQ R9,AX
+	ORQ AX,SI
+	ANDQ DI,DX
+	MOVQ 112(SP),AX
+	ANDQ R9,AX
+	ORQ AX,DX
+	ANDQ DI,CX
+	MOVQ 120(SP),AX
+	ANDQ R9,AX
+	ORQ AX,CX
+	ANDQ DI,R8
+	MOVQ 128(SP),DI
+	ANDQ R9,DI
+	ORQ DI,R8
+	MOVQ 88(SP),DI
+	MOVQ 96(SP),R9
+	ADDL 16(R9),SI
+	ADCL 20(R9),DX
+	ADCL 24(R9),CX
+	ADCL 28(R9),R8
+	MOVL SI,0(DI)
+	MOVL DX,4(DI)
+	MOVL CX,8(DI)
+	MOVL R8,12(DI)
+	MOVQ 32(SP),R11
+	MOVQ 40(SP),R12
+	MOVQ 48(SP),R13
+	MOVQ 56(SP),R14
+	MOVQ 64(SP),R15
+	MOVQ 72(SP),BX
+	MOVQ 80(SP),BP
+	MOVQ R11,SP
+	RET

vendor/golang.org/x/crypto/poly1305/poly1305_arm.s

@@ -0,0 +1,379 @@
+// Copyright 2015 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// This code was translated into a form compatible with 5a from the public
+// domain source by Andrew Moon: github.com/floodyberry/poly1305-opt/blob/master/app/extensions/poly1305.
+
+// +build arm,!gccgo,!appengine
+
+DATA poly1305_init_constants_armv6<>+0x00(SB)/4, $0x3ffffff
+DATA poly1305_init_constants_armv6<>+0x04(SB)/4, $0x3ffff03
+DATA poly1305_init_constants_armv6<>+0x08(SB)/4, $0x3ffc0ff
+DATA poly1305_init_constants_armv6<>+0x0c(SB)/4, $0x3f03fff
+DATA poly1305_init_constants_armv6<>+0x10(SB)/4, $0x00fffff
+GLOBL poly1305_init_constants_armv6<>(SB), 8, $20
+
+// Warning: the linker may use R11 to synthesize certain instructions. Please
+// take care and verify that no synthetic instructions use it.
+
+TEXT poly1305_init_ext_armv6<>(SB),4,$-4
+  MOVM.DB.W [R4-R11], (R13)
+  MOVM.IA.W (R1), [R2-R5]
+  MOVW $poly1305_init_constants_armv6<>(SB), R7
+  MOVW R2, R8
+  MOVW R2>>26, R9
+  MOVW R3>>20, g
+  MOVW R4>>14, R11
+  MOVW R5>>8, R12
+  ORR R3<<6, R9, R9
+  ORR R4<<12, g, g
+  ORR R5<<18, R11, R11
+  MOVM.IA (R7), [R2-R6]
+  AND R8, R2, R2
+  AND R9, R3, R3
+  AND g, R4, R4
+  AND R11, R5, R5
+  AND R12, R6, R6
+  MOVM.IA.W [R2-R6], (R0)
+  EOR R2, R2, R2
+  EOR R3, R3, R3
+  EOR R4, R4, R4
+  EOR R5, R5, R5
+  EOR R6, R6, R6
+  MOVM.IA.W [R2-R6], (R0)
+  MOVM.IA.W (R1), [R2-R5]
+  MOVM.IA [R2-R6], (R0)
+  MOVM.IA.W (R13), [R4-R11]
+  RET
+
+#define MOVW_UNALIGNED(Rsrc, Rdst, Rtmp, offset) \
+  MOVBU (offset+0)(Rsrc), Rtmp; \
+  MOVBU Rtmp, (offset+0)(Rdst); \
+  MOVBU (offset+1)(Rsrc), Rtmp; \
+  MOVBU Rtmp, (offset+1)(Rdst); \
+  MOVBU (offset+2)(Rsrc), Rtmp; \
+  MOVBU Rtmp, (offset+2)(Rdst); \
+  MOVBU (offset+3)(Rsrc), Rtmp; \
+  MOVBU Rtmp, (offset+3)(Rdst)
+
+TEXT poly1305_blocks_armv6<>(SB),4,$-4
+  MOVM.DB.W [R4, R5, R6, R7, R8, R9, g, R11, R14], (R13)
+  SUB $128, R13
+  MOVW R0, 36(R13)
+  MOVW R1, 40(R13)
+  MOVW R2, 44(R13)
+  MOVW R1, R14
+  MOVW R2, R12
+  MOVW 56(R0), R8
+  WORD $0xe1180008 // TST R8, R8 not working see issue 5921
+  EOR R6, R6, R6
+  MOVW.EQ $(1<<24), R6
+  MOVW R6, 32(R13)
+  ADD $64, R13, g
+  MOVM.IA (R0), [R0-R9]
+  MOVM.IA [R0-R4], (g)
+  CMP $16, R12
+  BLO poly1305_blocks_armv6_done
+poly1305_blocks_armv6_mainloop:
+  WORD $0xe31e0003 // TST R14, #3 not working see issue 5921
+  BEQ poly1305_blocks_armv6_mainloop_aligned
+  ADD $48, R13, g
+  MOVW_UNALIGNED(R14, g, R0, 0)
+  MOVW_UNALIGNED(R14, g, R0, 4)
+  MOVW_UNALIGNED(R14, g, R0, 8)
+  MOVW_UNALIGNED(R14, g, R0, 12)
+  MOVM.IA (g), [R0-R3]
+  ADD $16, R14
+  B poly1305_blocks_armv6_mainloop_loaded
+poly1305_blocks_armv6_mainloop_aligned:
+  MOVM.IA.W (R14), [R0-R3]
+poly1305_blocks_armv6_mainloop_loaded:
+  MOVW R0>>26, g
+  MOVW R1>>20, R11
+  MOVW R2>>14, R12
+  MOVW R14, 40(R13)
+  MOVW R3>>8, R4
+  ORR R1<<6, g, g
+  ORR R2<<12, R11, R11
+  ORR R3<<18, R12, R12
+  BIC $0xfc000000, R0, R0
+  BIC $0xfc000000, g, g
+  MOVW 32(R13), R3
+  BIC $0xfc000000, R11, R11
+  BIC $0xfc000000, R12, R12
+  ADD R0, R5, R5
+  ADD g, R6, R6
+  ORR R3, R4, R4
+  ADD R11, R7, R7
+  ADD $64, R13, R14
+  ADD R12, R8, R8
+  ADD R4, R9, R9
+  MOVM.IA (R14), [R0-R4]
+  MULLU R4, R5, (R11, g)
+  MULLU R3, R5, (R14, R12)
+  MULALU R3, R6, (R11, g)
+  MULALU R2, R6, (R14, R12)
+  MULALU R2, R7, (R11, g)
+  MULALU R1, R7, (R14, R12)
+  ADD R4<<2, R4, R4
+  ADD R3<<2, R3, R3
+  MULALU R1, R8, (R11, g)
+  MULALU R0, R8, (R14, R12)
+  MULALU R0, R9, (R11, g)
+  MULALU R4, R9, (R14, R12)
+  MOVW g, 24(R13)
+  MOVW R11, 28(R13)
+  MOVW R12, 16(R13)
+  MOVW R14, 20(R13)
+  MULLU R2, R5, (R11, g)
+  MULLU R1, R5, (R14, R12)
+  MULALU R1, R6, (R11, g)
+  MULALU R0, R6, (R14, R12)
+  MULALU R0, R7, (R11, g)
+  MULALU R4, R7, (R14, R12)
+  ADD R2<<2, R2, R2
+  ADD R1<<2, R1, R1
+  MULALU R4, R8, (R11, g)
+  MULALU R3, R8, (R14, R12)
+  MULALU R3, R9, (R11, g)
+  MULALU R2, R9, (R14, R12)
+  MOVW g, 8(R13)
+  MOVW R11, 12(R13)
+  MOVW R12, 0(R13)
+  MOVW R14, w+4(SP)
+  MULLU R0, R5, (R11, g)
+  MULALU R4, R6, (R11, g)
+  MULALU R3, R7, (R11, g)
+  MULALU R2, R8, (R11, g)
+  MULALU R1, R9, (R11, g)
+  MOVM.IA (R13), [R0-R7]
+  MOVW g>>26, R12
+  MOVW R4>>26, R14
+  ORR R11<<6, R12, R12
+  ORR R5<<6, R14, R14
+  BIC $0xfc000000, g, g
+  BIC $0xfc000000, R4, R4
+  ADD.S R12, R0, R0
+  ADC $0, R1, R1
+  ADD.S R14, R6, R6
+  ADC $0, R7, R7
+  MOVW R0>>26, R12
+  MOVW R6>>26, R14
+  ORR R1<<6, R12, R12
+  ORR R7<<6, R14, R14
+  BIC $0xfc000000, R0, R0
+  BIC $0xfc000000, R6, R6
+  ADD R14<<2, R14, R14
+  ADD.S R12, R2, R2
+  ADC $0, R3, R3
+  ADD R14, g, g
+  MOVW R2>>26, R12
+  MOVW g>>26, R14
+  ORR R3<<6, R12, R12
+  BIC $0xfc000000, g, R5
+  BIC $0xfc000000, R2, R7
+  ADD R12, R4, R4
+  ADD R14, R0, R0
+  MOVW R4>>26, R12
+  BIC $0xfc000000, R4, R8
+  ADD R12, R6, R9
+  MOVW w+44(SP), R12
+  MOVW w+40(SP), R14
+  MOVW R0, R6
+  CMP $32, R12
+  SUB $16, R12, R12
+  MOVW R12, 44(R13)
+  BHS poly1305_blocks_armv6_mainloop
+poly1305_blocks_armv6_done:
+  MOVW 36(R13), R12
+  MOVW R5, 20(R12)
+  MOVW R6, 24(R12)
+  MOVW R7, 28(R12)
+  MOVW R8, 32(R12)
+  MOVW R9, 36(R12)
+  ADD $128, R13, R13
+  MOVM.IA.W (R13), [R4, R5, R6, R7, R8, R9, g, R11, R14]
+  RET
+
+#define MOVHUP_UNALIGNED(Rsrc, Rdst, Rtmp) \
+  MOVBU.P 1(Rsrc), Rtmp; \
+  MOVBU.P Rtmp, 1(Rdst); \
+  MOVBU.P 1(Rsrc), Rtmp; \
+  MOVBU.P Rtmp, 1(Rdst)
+
+#define MOVWP_UNALIGNED(Rsrc, Rdst, Rtmp) \
+  MOVHUP_UNALIGNED(Rsrc, Rdst, Rtmp); \
+  MOVHUP_UNALIGNED(Rsrc, Rdst, Rtmp)
+
+TEXT poly1305_finish_ext_armv6<>(SB),4,$-4
+  MOVM.DB.W [R4, R5, R6, R7, R8, R9, g, R11, R14], (R13)
+  SUB $16, R13, R13
+  MOVW R0, R5
+  MOVW R1, R6
+  MOVW R2, R7
+  MOVW R3, R8
+  AND.S R2, R2, R2
+  BEQ poly1305_finish_ext_armv6_noremaining
+  EOR R0, R0
+  MOVW R13, R9
+  MOVW R0, 0(R13)
+  MOVW R0, 4(R13)
+  MOVW R0, 8(R13)
+  MOVW R0, 12(R13)
+  WORD $0xe3110003 // TST R1, #3 not working see issue 5921
+  BEQ poly1305_finish_ext_armv6_aligned
+  WORD $0xe3120008 // TST R2, #8 not working see issue 5921
+  BEQ poly1305_finish_ext_armv6_skip8
+  MOVWP_UNALIGNED(R1, R9, g)
+  MOVWP_UNALIGNED(R1, R9, g)
+poly1305_finish_ext_armv6_skip8:
+  WORD $0xe3120004 // TST $4, R2 not working see issue 5921
+  BEQ poly1305_finish_ext_armv6_skip4
+  MOVWP_UNALIGNED(R1, R9, g)
+poly1305_finish_ext_armv6_skip4:
+  WORD $0xe3120002 // TST $2, R2 not working see issue 5921
+  BEQ poly1305_finish_ext_armv6_skip2
+  MOVHUP_UNALIGNED(R1, R9, g)
+  B poly1305_finish_ext_armv6_skip2
+poly1305_finish_ext_armv6_aligned:
+  WORD $0xe3120008 // TST R2, #8 not working see issue 5921
+  BEQ poly1305_finish_ext_armv6_skip8_aligned
+  MOVM.IA.W (R1), [g-R11]
+  MOVM.IA.W [g-R11], (R9)
+poly1305_finish_ext_armv6_skip8_aligned:
+  WORD $0xe3120004 // TST $4, R2 not working see issue 5921
+  BEQ poly1305_finish_ext_armv6_skip4_aligned
+  MOVW.P 4(R1), g
+  MOVW.P g, 4(R9)
+poly1305_finish_ext_armv6_skip4_aligned:
+  WORD $0xe3120002 // TST $2, R2 not working see issue 5921
+  BEQ poly1305_finish_ext_armv6_skip2
+  MOVHU.P 2(R1), g
+  MOVH.P g, 2(R9)
+poly1305_finish_ext_armv6_skip2:
+  WORD $0xe3120001 // TST $1, R2 not working see issue 5921
+  BEQ poly1305_finish_ext_armv6_skip1
+  MOVBU.P 1(R1), g
+  MOVBU.P g, 1(R9)
+poly1305_finish_ext_armv6_skip1:
+  MOVW $1, R11
+  MOVBU R11, 0(R9)
+  MOVW R11, 56(R5)
+  MOVW R5, R0
+  MOVW R13, R1
+  MOVW $16, R2
+  BL poly1305_blocks_armv6<>(SB)
+poly1305_finish_ext_armv6_noremaining:
+  MOVW 20(R5), R0
+  MOVW 24(R5), R1
+  MOVW 28(R5), R2
+  MOVW 32(R5), R3
+  MOVW 36(R5), R4
+  MOVW R4>>26, R12
+  BIC $0xfc000000, R4, R4
+  ADD R12<<2, R12, R12
+  ADD R12, R0, R0
+  MOVW R0>>26, R12
+  BIC $0xfc000000, R0, R0
+  ADD R12, R1, R1
+  MOVW R1>>26, R12
+  BIC $0xfc000000, R1, R1
+  ADD R12, R2, R2
+  MOVW R2>>26, R12
+  BIC $0xfc000000, R2, R2
+  ADD R12, R3, R3
+  MOVW R3>>26, R12
+  BIC $0xfc000000, R3, R3
+  ADD R12, R4, R4
+  ADD $5, R0, R6
+  MOVW R6>>26, R12
+  BIC $0xfc000000, R6, R6
+  ADD R12, R1, R7
+  MOVW R7>>26, R12
+  BIC $0xfc000000, R7, R7
+  ADD R12, R2, g
+  MOVW g>>26, R12
+  BIC $0xfc000000, g, g
+  ADD R12, R3, R11
+  MOVW $-(1<<26), R12
+  ADD R11>>26, R12, R12
+  BIC $0xfc000000, R11, R11
+  ADD R12, R4, R14
+  MOVW R14>>31, R12
+  SUB $1, R12
+  AND R12, R6, R6
+  AND R12, R7, R7
+  AND R12, g, g
+  AND R12, R11, R11
+  AND R12, R14, R14
+  MVN R12, R12
+  AND R12, R0, R0
+  AND R12, R1, R1
+  AND R12, R2, R2
+  AND R12, R3, R3
+  AND R12, R4, R4
+  ORR R6, R0, R0
+  ORR R7, R1, R1
+  ORR g, R2, R2
+  ORR R11, R3, R3
+  ORR R14, R4, R4
+  ORR R1<<26, R0, R0
+  MOVW R1>>6, R1
+  ORR R2<<20, R1, R1
+  MOVW R2>>12, R2
+  ORR R3<<14, R2, R2
+  MOVW R3>>18, R3
+  ORR R4<<8, R3, R3
+  MOVW 40(R5), R6
+  MOVW 44(R5), R7
+  MOVW 48(R5), g
+  MOVW 52(R5), R11
+  ADD.S R6, R0, R0
+  ADC.S R7, R1, R1
+  ADC.S g, R2, R2
+  ADC.S R11, R3, R3
+  MOVM.IA [R0-R3], (R8)
+  MOVW R5, R12
+  EOR R0, R0, R0
+  EOR R1, R1, R1
+  EOR R2, R2, R2
+  EOR R3, R3, R3
+  EOR R4, R4, R4
+  EOR R5, R5, R5
+  EOR R6, R6, R6
+  EOR R7, R7, R7
+  MOVM.IA.W [R0-R7], (R12)
+  MOVM.IA [R0-R7], (R12)
+  ADD $16, R13, R13
+  MOVM.IA.W (R13), [R4, R5, R6, R7, R8, R9, g, R11, R14]
+  RET
+
+// func poly1305_auth_armv6(out *[16]byte, m *byte, mlen uint32, key *[32]key)
+TEXT ·poly1305_auth_armv6(SB),0,$280-16
+  MOVW  out+0(FP), R4
+  MOVW  m+4(FP), R5
+  MOVW  mlen+8(FP), R6
+  MOVW  key+12(FP), R7
+
+  MOVW R13, R8
+  BIC $63, R13
+  SUB $64, R13, R13
+  MOVW  R13, R0
+  MOVW  R7, R1
+  BL poly1305_init_ext_armv6<>(SB)
+  BIC.S $15, R6, R2
+  BEQ poly1305_auth_armv6_noblocks
+  MOVW R13, R0
+  MOVW R5, R1
+  ADD R2, R5, R5
+  SUB R2, R6, R6
+  BL poly1305_blocks_armv6<>(SB)
+poly1305_auth_armv6_noblocks:
+  MOVW R13, R0
+  MOVW R5, R1
+  MOVW R6, R2
+  MOVW R4, R3
+  BL poly1305_finish_ext_armv6<>(SB)
+  MOVW R8, R13
+  RET

vendor/golang.org/x/crypto/poly1305/sum_amd64.go

@@ -0,0 +1,24 @@
+// Copyright 2012 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build amd64,!gccgo,!appengine
+
+package poly1305
+
+// This function is implemented in poly1305_amd64.s
+
+//go:noescape
+
+func poly1305(out *[16]byte, m *byte, mlen uint64, key *[32]byte)
+
+// Sum generates an authenticator for m using a one-time key and puts the
+// 16-byte result into out. Authenticating two different messages with the same
+// key allows an attacker to forge messages at will.
+func Sum(out *[16]byte, m []byte, key *[32]byte) {
+	var mPtr *byte
+	if len(m) > 0 {
+		mPtr = &m[0]
+	}
+	poly1305(out, mPtr, uint64(len(m)), key)
+}

vendor/golang.org/x/crypto/poly1305/sum_arm.go

@@ -0,0 +1,24 @@
+// Copyright 2015 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build arm,!gccgo,!appengine
+
+package poly1305
+
+// This function is implemented in poly1305_arm.s
+
+//go:noescape
+
+func poly1305_auth_armv6(out *[16]byte, m *byte, mlen uint32, key *[32]byte)
+
+// Sum generates an authenticator for m using a one-time key and puts the
+// 16-byte result into out. Authenticating two different messages with the same
+// key allows an attacker to forge messages at will.
+func Sum(out *[16]byte, m []byte, key *[32]byte) {
+	var mPtr *byte
+	if len(m) > 0 {
+		mPtr = &m[0]
+	}
+	poly1305_auth_armv6(out, mPtr, uint32(len(m)), key)
+}

vendor/golang.org/x/crypto/salsa20/salsa/hsalsa20.go

@@ -0,0 +1,144 @@
+// Copyright 2012 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package salsa provides low-level access to functions in the Salsa family.
+package salsa // import "golang.org/x/crypto/salsa20/salsa"
+
+// Sigma is the Salsa20 constant for 256-bit keys.
+var Sigma = [16]byte{'e', 'x', 'p', 'a', 'n', 'd', ' ', '3', '2', '-', 'b', 'y', 't', 'e', ' ', 'k'}
+
+// HSalsa20 applies the HSalsa20 core function to a 16-byte input in, 32-byte
+// key k, and 16-byte constant c, and puts the result into the 32-byte array
+// out.
+func HSalsa20(out *[32]byte, in *[16]byte, k *[32]byte, c *[16]byte) {
+	x0 := uint32(c[0]) | uint32(c[1])<<8 | uint32(c[2])<<16 | uint32(c[3])<<24
+	x1 := uint32(k[0]) | uint32(k[1])<<8 | uint32(k[2])<<16 | uint32(k[3])<<24
+	x2 := uint32(k[4]) | uint32(k[5])<<8 | uint32(k[6])<<16 | uint32(k[7])<<24
+	x3 := uint32(k[8]) | uint32(k[9])<<8 | uint32(k[10])<<16 | uint32(k[11])<<24
+	x4 := uint32(k[12]) | uint32(k[13])<<8 | uint32(k[14])<<16 | uint32(k[15])<<24
+	x5 := uint32(c[4]) | uint32(c[5])<<8 | uint32(c[6])<<16 | uint32(c[7])<<24
+	x6 := uint32(in[0]) | uint32(in[1])<<8 | uint32(in[2])<<16 | uint32(in[3])<<24
+	x7 := uint32(in[4]) | uint32(in[5])<<8 | uint32(in[6])<<16 | uint32(in[7])<<24
+	x8 := uint32(in[8]) | uint32(in[9])<<8 | uint32(in[10])<<16 | uint32(in[11])<<24
+	x9 := uint32(in[12]) | uint32(in[13])<<8 | uint32(in[14])<<16 | uint32(in[15])<<24
+	x10 := uint32(c[8]) | uint32(c[9])<<8 | uint32(c[10])<<16 | uint32(c[11])<<24
+	x11 := uint32(k[16]) | uint32(k[17])<<8 | uint32(k[18])<<16 | uint32(k[19])<<24
+	x12 := uint32(k[20]) | uint32(k[21])<<8 | uint32(k[22])<<16 | uint32(k[23])<<24
+	x13 := uint32(k[24]) | uint32(k[25])<<8 | uint32(k[26])<<16 | uint32(k[27])<<24
+	x14 := uint32(k[28]) | uint32(k[29])<<8 | uint32(k[30])<<16 | uint32(k[31])<<24
+	x15 := uint32(c[12]) | uint32(c[13])<<8 | uint32(c[14])<<16 | uint32(c[15])<<24
+
+	for i := 0; i < 20; i += 2 {
+		u := x0 + x12
+		x4 ^= u<<7 | u>>(32-7)
+		u = x4 + x0
+		x8 ^= u<<9 | u>>(32-9)
+		u = x8 + x4
+		x12 ^= u<<13 | u>>(32-13)
+		u = x12 + x8
+		x0 ^= u<<18 | u>>(32-18)
+
+		u = x5 + x1
+		x9 ^= u<<7 | u>>(32-7)
+		u = x9 + x5
+		x13 ^= u<<9 | u>>(32-9)
+		u = x13 + x9
+		x1 ^= u<<13 | u>>(32-13)
+		u = x1 + x13
+		x5 ^= u<<18 | u>>(32-18)
+
+		u = x10 + x6
+		x14 ^= u<<7 | u>>(32-7)
+		u = x14 + x10
+		x2 ^= u<<9 | u>>(32-9)
+		u = x2 + x14
+		x6 ^= u<<13 | u>>(32-13)
+		u = x6 + x2
+		x10 ^= u<<18 | u>>(32-18)
+
+		u = x15 + x11
+		x3 ^= u<<7 | u>>(32-7)
+		u = x3 + x15
+		x7 ^= u<<9 | u>>(32-9)
+		u = x7 + x3
+		x11 ^= u<<13 | u>>(32-13)
+		u = x11 + x7
+		x15 ^= u<<18 | u>>(32-18)
+
+		u = x0 + x3
+		x1 ^= u<<7 | u>>(32-7)
+		u = x1 + x0
+		x2 ^= u<<9 | u>>(32-9)
+		u = x2 + x1
+		x3 ^= u<<13 | u>>(32-13)
+		u = x3 + x2
+		x0 ^= u<<18 | u>>(32-18)
+
+		u = x5 + x4
+		x6 ^= u<<7 | u>>(32-7)
+		u = x6 + x5
+		x7 ^= u<<9 | u>>(32-9)
+		u = x7 + x6
+		x4 ^= u<<13 | u>>(32-13)
+		u = x4 + x7
+		x5 ^= u<<18 | u>>(32-18)
+
+		u = x10 + x9
+		x11 ^= u<<7 | u>>(32-7)
+		u = x11 + x10
+		x8 ^= u<<9 | u>>(32-9)
+		u = x8 + x11
+		x9 ^= u<<13 | u>>(32-13)
+		u = x9 + x8
+		x10 ^= u<<18 | u>>(32-18)
+
+		u = x15 + x14
+		x12 ^= u<<7 | u>>(32-7)
+		u = x12 + x15
+		x13 ^= u<<9 | u>>(32-9)
+		u = x13 + x12
+		x14 ^= u<<13 | u>>(32-13)
+		u = x14 + x13
+		x15 ^= u<<18 | u>>(32-18)
+	}
+	out[0] = byte(x0)
+	out[1] = byte(x0 >> 8)
+	out[2] = byte(x0 >> 16)
+	out[3] = byte(x0 >> 24)
+
+	out[4] = byte(x5)
+	out[5] = byte(x5 >> 8)
+	out[6] = byte(x5 >> 16)
+	out[7] = byte(x5 >> 24)
+
+	out[8] = byte(x10)
+	out[9] = byte(x10 >> 8)
+	out[10] = byte(x10 >> 16)
+	out[11] = byte(x10 >> 24)
+
+	out[12] = byte(x15)
+	out[13] = byte(x15 >> 8)
+	out[14] = byte(x15 >> 16)
+	out[15] = byte(x15 >> 24)
+
+	out[16] = byte(x6)
+	out[17] = byte(x6 >> 8)
+	out[18] = byte(x6 >> 16)
+	out[19] = byte(x6 >> 24)
+
+	out[20] = byte(x7)
+	out[21] = byte(x7 >> 8)
+	out[22] = byte(x7 >> 16)
+	out[23] = byte(x7 >> 24)
+
+	out[24] = byte(x8)
+	out[25] = byte(x8 >> 8)
+	out[26] = byte(x8 >> 16)
+	out[27] = byte(x8 >> 24)
+
+	out[28] = byte(x9)
+	out[29] = byte(x9 >> 8)
+	out[30] = byte(x9 >> 16)
+	out[31] = byte(x9 >> 24)
+}

vendor/golang.org/x/crypto/salsa20/salsa/salsa2020_amd64.s

@@ -0,0 +1,902 @@
+// Copyright 2012 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build amd64,!appengine,!gccgo
+
+// This code was translated into a form compatible with 6a from the public
+// domain sources in SUPERCOP: http://bench.cr.yp.to/supercop.html
+
+// func salsa2020XORKeyStream(out, in *byte, n uint64, nonce, key *byte)
+TEXT ·salsa2020XORKeyStream(SB),0,$512-40
+	MOVQ out+0(FP),DI
+	MOVQ in+8(FP),SI
+	MOVQ n+16(FP),DX
+	MOVQ nonce+24(FP),CX
+	MOVQ key+32(FP),R8
+
+	MOVQ SP,R11
+	MOVQ $31,R9
+	NOTQ R9
+	ANDQ R9,SP
+	ADDQ $32,SP
+
+	MOVQ R11,352(SP)
+	MOVQ R12,360(SP)
+	MOVQ R13,368(SP)
+	MOVQ R14,376(SP)
+	MOVQ R15,384(SP)
+	MOVQ BX,392(SP)
+	MOVQ BP,400(SP)
+	MOVQ DX,R9
+	MOVQ CX,DX
+	MOVQ R8,R10
+	CMPQ R9,$0
+	JBE DONE
+	START:
+	MOVL 20(R10),CX
+	MOVL 0(R10),R8
+	MOVL 0(DX),AX
+	MOVL 16(R10),R11
+	MOVL CX,0(SP)
+	MOVL R8, 4 (SP)
+	MOVL AX, 8 (SP)
+	MOVL R11, 12 (SP)
+	MOVL 8(DX),CX
+	MOVL 24(R10),R8
+	MOVL 4(R10),AX
+	MOVL 4(DX),R11
+	MOVL CX,16(SP)
+	MOVL R8, 20 (SP)
+	MOVL AX, 24 (SP)
+	MOVL R11, 28 (SP)
+	MOVL 12(DX),CX
+	MOVL 12(R10),DX
+	MOVL 28(R10),R8
+	MOVL 8(R10),AX
+	MOVL DX,32(SP)
+	MOVL CX, 36 (SP)
+	MOVL R8, 40 (SP)
+	MOVL AX, 44 (SP)
+	MOVQ $1634760805,DX
+	MOVQ $857760878,CX
+	MOVQ $2036477234,R8
+	MOVQ $1797285236,AX
+	MOVL DX,48(SP)
+	MOVL CX, 52 (SP)
+	MOVL R8, 56 (SP)
+	MOVL AX, 60 (SP)
+	CMPQ R9,$256
+	JB BYTESBETWEEN1AND255
+	MOVOA 48(SP),X0
+	PSHUFL $0X55,X0,X1
+	PSHUFL $0XAA,X0,X2
+	PSHUFL $0XFF,X0,X3
+	PSHUFL $0X00,X0,X0
+	MOVOA X1,64(SP)
+	MOVOA X2,80(SP)
+	MOVOA X3,96(SP)
+	MOVOA X0,112(SP)
+	MOVOA 0(SP),X0
+	PSHUFL $0XAA,X0,X1
+	PSHUFL $0XFF,X0,X2
+	PSHUFL $0X00,X0,X3
+	PSHUFL $0X55,X0,X0
+	MOVOA X1,128(SP)
+	MOVOA X2,144(SP)
+	MOVOA X3,160(SP)
+	MOVOA X0,176(SP)
+	MOVOA 16(SP),X0
+	PSHUFL $0XFF,X0,X1
+	PSHUFL $0X55,X0,X2
+	PSHUFL $0XAA,X0,X0
+	MOVOA X1,192(SP)
+	MOVOA X2,208(SP)
+	MOVOA X0,224(SP)
+	MOVOA 32(SP),X0
+	PSHUFL $0X00,X0,X1
+	PSHUFL $0XAA,X0,X2
+	PSHUFL $0XFF,X0,X0
+	MOVOA X1,240(SP)
+	MOVOA X2,256(SP)
+	MOVOA X0,272(SP)
+	BYTESATLEAST256:
+	MOVL 16(SP),DX
+	MOVL  36 (SP),CX
+	MOVL DX,288(SP)
+	MOVL CX,304(SP)
+	ADDQ $1,DX
+	SHLQ $32,CX
+	ADDQ CX,DX
+	MOVQ DX,CX
+	SHRQ $32,CX
+	MOVL DX, 292 (SP)
+	MOVL CX, 308 (SP)
+	ADDQ $1,DX
+	SHLQ $32,CX
+	ADDQ CX,DX
+	MOVQ DX,CX
+	SHRQ $32,CX
+	MOVL DX, 296 (SP)
+	MOVL CX, 312 (SP)
+	ADDQ $1,DX
+	SHLQ $32,CX
+	ADDQ CX,DX
+	MOVQ DX,CX
+	SHRQ $32,CX
+	MOVL DX, 300 (SP)
+	MOVL CX, 316 (SP)
+	ADDQ $1,DX
+	SHLQ $32,CX
+	ADDQ CX,DX
+	MOVQ DX,CX
+	SHRQ $32,CX
+	MOVL DX,16(SP)
+	MOVL CX, 36 (SP)
+	MOVQ R9,408(SP)
+	MOVQ $20,DX
+	MOVOA 64(SP),X0
+	MOVOA 80(SP),X1
+	MOVOA 96(SP),X2
+	MOVOA 256(SP),X3
+	MOVOA 272(SP),X4
+	MOVOA 128(SP),X5
+	MOVOA 144(SP),X6
+	MOVOA 176(SP),X7
+	MOVOA 192(SP),X8
+	MOVOA 208(SP),X9
+	MOVOA 224(SP),X10
+	MOVOA 304(SP),X11
+	MOVOA 112(SP),X12
+	MOVOA 160(SP),X13
+	MOVOA 240(SP),X14
+	MOVOA 288(SP),X15
+	MAINLOOP1:
+	MOVOA X1,320(SP)
+	MOVOA X2,336(SP)
+	MOVOA X13,X1
+	PADDL X12,X1
+	MOVOA X1,X2
+	PSLLL $7,X1
+	PXOR X1,X14
+	PSRLL $25,X2
+	PXOR X2,X14
+	MOVOA X7,X1
+	PADDL X0,X1
+	MOVOA X1,X2
+	PSLLL $7,X1
+	PXOR X1,X11
+	PSRLL $25,X2
+	PXOR X2,X11
+	MOVOA X12,X1
+	PADDL X14,X1
+	MOVOA X1,X2
+	PSLLL $9,X1
+	PXOR X1,X15
+	PSRLL $23,X2
+	PXOR X2,X15
+	MOVOA X0,X1
+	PADDL X11,X1
+	MOVOA X1,X2
+	PSLLL $9,X1
+	PXOR X1,X9
+	PSRLL $23,X2
+	PXOR X2,X9
+	MOVOA X14,X1
+	PADDL X15,X1
+	MOVOA X1,X2
+	PSLLL $13,X1
+	PXOR X1,X13
+	PSRLL $19,X2
+	PXOR X2,X13
+	MOVOA X11,X1
+	PADDL X9,X1
+	MOVOA X1,X2
+	PSLLL $13,X1
+	PXOR X1,X7
+	PSRLL $19,X2
+	PXOR X2,X7
+	MOVOA X15,X1
+	PADDL X13,X1
+	MOVOA X1,X2
+	PSLLL $18,X1
+	PXOR X1,X12
+	PSRLL $14,X2
+	PXOR X2,X12
+	MOVOA 320(SP),X1
+	MOVOA X12,320(SP)
+	MOVOA X9,X2
+	PADDL X7,X2
+	MOVOA X2,X12
+	PSLLL $18,X2
+	PXOR X2,X0
+	PSRLL $14,X12
+	PXOR X12,X0
+	MOVOA X5,X2
+	PADDL X1,X2
+	MOVOA X2,X12
+	PSLLL $7,X2
+	PXOR X2,X3
+	PSRLL $25,X12
+	PXOR X12,X3
+	MOVOA 336(SP),X2
+	MOVOA X0,336(SP)
+	MOVOA X6,X0
+	PADDL X2,X0
+	MOVOA X0,X12
+	PSLLL $7,X0
+	PXOR X0,X4
+	PSRLL $25,X12
+	PXOR X12,X4
+	MOVOA X1,X0
+	PADDL X3,X0
+	MOVOA X0,X12
+	PSLLL $9,X0
+	PXOR X0,X10
+	PSRLL $23,X12
+	PXOR X12,X10
+	MOVOA X2,X0
+	PADDL X4,X0
+	MOVOA X0,X12
+	PSLLL $9,X0
+	PXOR X0,X8
+	PSRLL $23,X12
+	PXOR X12,X8
+	MOVOA X3,X0
+	PADDL X10,X0
+	MOVOA X0,X12
+	PSLLL $13,X0
+	PXOR X0,X5
+	PSRLL $19,X12
+	PXOR X12,X5
+	MOVOA X4,X0
+	PADDL X8,X0
+	MOVOA X0,X12
+	PSLLL $13,X0
+	PXOR X0,X6
+	PSRLL $19,X12
+	PXOR X12,X6
+	MOVOA X10,X0
+	PADDL X5,X0
+	MOVOA X0,X12
+	PSLLL $18,X0
+	PXOR X0,X1
+	PSRLL $14,X12
+	PXOR X12,X1
+	MOVOA 320(SP),X0
+	MOVOA X1,320(SP)
+	MOVOA X4,X1
+	PADDL X0,X1
+	MOVOA X1,X12
+	PSLLL $7,X1
+	PXOR X1,X7
+	PSRLL $25,X12
+	PXOR X12,X7
+	MOVOA X8,X1
+	PADDL X6,X1
+	MOVOA X1,X12
+	PSLLL $18,X1
+	PXOR X1,X2
+	PSRLL $14,X12
+	PXOR X12,X2
+	MOVOA 336(SP),X12
+	MOVOA X2,336(SP)
+	MOVOA X14,X1
+	PADDL X12,X1
+	MOVOA X1,X2
+	PSLLL $7,X1
+	PXOR X1,X5
+	PSRLL $25,X2
+	PXOR X2,X5
+	MOVOA X0,X1
+	PADDL X7,X1
+	MOVOA X1,X2
+	PSLLL $9,X1
+	PXOR X1,X10
+	PSRLL $23,X2
+	PXOR X2,X10
+	MOVOA X12,X1
+	PADDL X5,X1
+	MOVOA X1,X2
+	PSLLL $9,X1
+	PXOR X1,X8
+	PSRLL $23,X2
+	PXOR X2,X8
+	MOVOA X7,X1
+	PADDL X10,X1
+	MOVOA X1,X2
+	PSLLL $13,X1
+	PXOR X1,X4
+	PSRLL $19,X2
+	PXOR X2,X4
+	MOVOA X5,X1
+	PADDL X8,X1
+	MOVOA X1,X2
+	PSLLL $13,X1
+	PXOR X1,X14
+	PSRLL $19,X2
+	PXOR X2,X14
+	MOVOA X10,X1
+	PADDL X4,X1
+	MOVOA X1,X2
+	PSLLL $18,X1
+	PXOR X1,X0
+	PSRLL $14,X2
+	PXOR X2,X0
+	MOVOA 320(SP),X1
+	MOVOA X0,320(SP)
+	MOVOA X8,X0
+	PADDL X14,X0
+	MOVOA X0,X2
+	PSLLL $18,X0
+	PXOR X0,X12
+	PSRLL $14,X2
+	PXOR X2,X12
+	MOVOA X11,X0
+	PADDL X1,X0
+	MOVOA X0,X2
+	PSLLL $7,X0
+	PXOR X0,X6
+	PSRLL $25,X2
+	PXOR X2,X6
+	MOVOA 336(SP),X2
+	MOVOA X12,336(SP)
+	MOVOA X3,X0
+	PADDL X2,X0
+	MOVOA X0,X12
+	PSLLL $7,X0
+	PXOR X0,X13
+	PSRLL $25,X12
+	PXOR X12,X13
+	MOVOA X1,X0
+	PADDL X6,X0
+	MOVOA X0,X12
+	PSLLL $9,X0
+	PXOR X0,X15
+	PSRLL $23,X12
+	PXOR X12,X15
+	MOVOA X2,X0
+	PADDL X13,X0
+	MOVOA X0,X12
+	PSLLL $9,X0
+	PXOR X0,X9
+	PSRLL $23,X12
+	PXOR X12,X9
+	MOVOA X6,X0
+	PADDL X15,X0
+	MOVOA X0,X12
+	PSLLL $13,X0
+	PXOR X0,X11
+	PSRLL $19,X12
+	PXOR X12,X11
+	MOVOA X13,X0
+	PADDL X9,X0
+	MOVOA X0,X12
+	PSLLL $13,X0
+	PXOR X0,X3
+	PSRLL $19,X12
+	PXOR X12,X3
+	MOVOA X15,X0
+	PADDL X11,X0
+	MOVOA X0,X12
+	PSLLL $18,X0
+	PXOR X0,X1
+	PSRLL $14,X12
+	PXOR X12,X1
+	MOVOA X9,X0
+	PADDL X3,X0
+	MOVOA X0,X12
+	PSLLL $18,X0
+	PXOR X0,X2
+	PSRLL $14,X12
+	PXOR X12,X2
+	MOVOA 320(SP),X12
+	MOVOA 336(SP),X0
+	SUBQ $2,DX
+	JA MAINLOOP1
+	PADDL 112(SP),X12
+	PADDL 176(SP),X7
+	PADDL 224(SP),X10
+	PADDL 272(SP),X4
+	MOVD X12,DX
+	MOVD X7,CX
+	MOVD X10,R8
+	MOVD X4,R9
+	PSHUFL $0X39,X12,X12
+	PSHUFL $0X39,X7,X7
+	PSHUFL $0X39,X10,X10
+	PSHUFL $0X39,X4,X4
+	XORL 0(SI),DX
+	XORL 4(SI),CX
+	XORL 8(SI),R8
+	XORL 12(SI),R9
+	MOVL DX,0(DI)
+	MOVL CX,4(DI)
+	MOVL R8,8(DI)
+	MOVL R9,12(DI)
+	MOVD X12,DX
+	MOVD X7,CX
+	MOVD X10,R8
+	MOVD X4,R9
+	PSHUFL $0X39,X12,X12
+	PSHUFL $0X39,X7,X7
+	PSHUFL $0X39,X10,X10
+	PSHUFL $0X39,X4,X4
+	XORL 64(SI),DX
+	XORL 68(SI),CX
+	XORL 72(SI),R8
+	XORL 76(SI),R9
+	MOVL DX,64(DI)
+	MOVL CX,68(DI)
+	MOVL R8,72(DI)
+	MOVL R9,76(DI)
+	MOVD X12,DX
+	MOVD X7,CX
+	MOVD X10,R8
+	MOVD X4,R9
+	PSHUFL $0X39,X12,X12
+	PSHUFL $0X39,X7,X7
+	PSHUFL $0X39,X10,X10
+	PSHUFL $0X39,X4,X4
+	XORL 128(SI),DX
+	XORL 132(SI),CX
+	XORL 136(SI),R8
+	XORL 140(SI),R9
+	MOVL DX,128(DI)
+	MOVL CX,132(DI)
+	MOVL R8,136(DI)
+	MOVL R9,140(DI)
+	MOVD X12,DX
+	MOVD X7,CX
+	MOVD X10,R8
+	MOVD X4,R9
+	XORL 192(SI),DX
+	XORL 196(SI),CX
+	XORL 200(SI),R8
+	XORL 204(SI),R9
+	MOVL DX,192(DI)
+	MOVL CX,196(DI)
+	MOVL R8,200(DI)
+	MOVL R9,204(DI)
+	PADDL 240(SP),X14
+	PADDL 64(SP),X0
+	PADDL 128(SP),X5
+	PADDL 192(SP),X8
+	MOVD X14,DX
+	MOVD X0,CX
+	MOVD X5,R8
+	MOVD X8,R9
+	PSHUFL $0X39,X14,X14
+	PSHUFL $0X39,X0,X0
+	PSHUFL $0X39,X5,X5
+	PSHUFL $0X39,X8,X8
+	XORL 16(SI),DX
+	XORL 20(SI),CX
+	XORL 24(SI),R8
+	XORL 28(SI),R9
+	MOVL DX,16(DI)
+	MOVL CX,20(DI)
+	MOVL R8,24(DI)
+	MOVL R9,28(DI)
+	MOVD X14,DX
+	MOVD X0,CX
+	MOVD X5,R8
+	MOVD X8,R9
+	PSHUFL $0X39,X14,X14
+	PSHUFL $0X39,X0,X0
+	PSHUFL $0X39,X5,X5
+	PSHUFL $0X39,X8,X8
+	XORL 80(SI),DX
+	XORL 84(SI),CX
+	XORL 88(SI),R8
+	XORL 92(SI),R9
+	MOVL DX,80(DI)
+	MOVL CX,84(DI)
+	MOVL R8,88(DI)
+	MOVL R9,92(DI)
+	MOVD X14,DX
+	MOVD X0,CX
+	MOVD X5,R8
+	MOVD X8,R9
+	PSHUFL $0X39,X14,X14
+	PSHUFL $0X39,X0,X0
+	PSHUFL $0X39,X5,X5
+	PSHUFL $0X39,X8,X8
+	XORL 144(SI),DX
+	XORL 148(SI),CX
+	XORL 152(SI),R8
+	XORL 156(SI),R9
+	MOVL DX,144(DI)
+	MOVL CX,148(DI)
+	MOVL R8,152(DI)
+	MOVL R9,156(DI)
+	MOVD X14,DX
+	MOVD X0,CX
+	MOVD X5,R8
+	MOVD X8,R9
+	XORL 208(SI),DX
+	XORL 212(SI),CX
+	XORL 216(SI),R8
+	XORL 220(SI),R9
+	MOVL DX,208(DI)
+	MOVL CX,212(DI)
+	MOVL R8,216(DI)
+	MOVL R9,220(DI)
+	PADDL 288(SP),X15
+	PADDL 304(SP),X11
+	PADDL 80(SP),X1
+	PADDL 144(SP),X6
+	MOVD X15,DX
+	MOVD X11,CX
+	MOVD X1,R8
+	MOVD X6,R9
+	PSHUFL $0X39,X15,X15
+	PSHUFL $0X39,X11,X11
+	PSHUFL $0X39,X1,X1
+	PSHUFL $0X39,X6,X6
+	XORL 32(SI),DX
+	XORL 36(SI),CX
+	XORL 40(SI),R8
+	XORL 44(SI),R9
+	MOVL DX,32(DI)
+	MOVL CX,36(DI)
+	MOVL R8,40(DI)
+	MOVL R9,44(DI)
+	MOVD X15,DX
+	MOVD X11,CX
+	MOVD X1,R8
+	MOVD X6,R9
+	PSHUFL $0X39,X15,X15
+	PSHUFL $0X39,X11,X11
+	PSHUFL $0X39,X1,X1
+	PSHUFL $0X39,X6,X6
+	XORL 96(SI),DX
+	XORL 100(SI),CX
+	XORL 104(SI),R8
+	XORL 108(SI),R9
+	MOVL DX,96(DI)
+	MOVL CX,100(DI)
+	MOVL R8,104(DI)
+	MOVL R9,108(DI)
+	MOVD X15,DX
+	MOVD X11,CX
+	MOVD X1,R8
+	MOVD X6,R9
+	PSHUFL $0X39,X15,X15
+	PSHUFL $0X39,X11,X11
+	PSHUFL $0X39,X1,X1
+	PSHUFL $0X39,X6,X6
+	XORL 160(SI),DX
+	XORL 164(SI),CX
+	XORL 168(SI),R8
+	XORL 172(SI),R9
+	MOVL DX,160(DI)
+	MOVL CX,164(DI)
+	MOVL R8,168(DI)
+	MOVL R9,172(DI)
+	MOVD X15,DX
+	MOVD X11,CX
+	MOVD X1,R8
+	MOVD X6,R9
+	XORL 224(SI),DX
+	XORL 228(SI),CX
+	XORL 232(SI),R8
+	XORL 236(SI),R9
+	MOVL DX,224(DI)
+	MOVL CX,228(DI)
+	MOVL R8,232(DI)
+	MOVL R9,236(DI)
+	PADDL 160(SP),X13
+	PADDL 208(SP),X9
+	PADDL 256(SP),X3
+	PADDL 96(SP),X2
+	MOVD X13,DX
+	MOVD X9,CX
+	MOVD X3,R8
+	MOVD X2,R9
+	PSHUFL $0X39,X13,X13
+	PSHUFL $0X39,X9,X9
+	PSHUFL $0X39,X3,X3
+	PSHUFL $0X39,X2,X2
+	XORL 48(SI),DX
+	XORL 52(SI),CX
+	XORL 56(SI),R8
+	XORL 60(SI),R9
+	MOVL DX,48(DI)
+	MOVL CX,52(DI)
+	MOVL R8,56(DI)
+	MOVL R9,60(DI)
+	MOVD X13,DX
+	MOVD X9,CX
+	MOVD X3,R8
+	MOVD X2,R9
+	PSHUFL $0X39,X13,X13
+	PSHUFL $0X39,X9,X9
+	PSHUFL $0X39,X3,X3
+	PSHUFL $0X39,X2,X2
+	XORL 112(SI),DX
+	XORL 116(SI),CX
+	XORL 120(SI),R8
+	XORL 124(SI),R9
+	MOVL DX,112(DI)
+	MOVL CX,116(DI)
+	MOVL R8,120(DI)
+	MOVL R9,124(DI)
+	MOVD X13,DX
+	MOVD X9,CX
+	MOVD X3,R8
+	MOVD X2,R9
+	PSHUFL $0X39,X13,X13
+	PSHUFL $0X39,X9,X9
+	PSHUFL $0X39,X3,X3
+	PSHUFL $0X39,X2,X2
+	XORL 176(SI),DX
+	XORL 180(SI),CX
+	XORL 184(SI),R8
+	XORL 188(SI),R9
+	MOVL DX,176(DI)
+	MOVL CX,180(DI)
+	MOVL R8,184(DI)
+	MOVL R9,188(DI)
+	MOVD X13,DX
+	MOVD X9,CX
+	MOVD X3,R8
+	MOVD X2,R9
+	XORL 240(SI),DX
+	XORL 244(SI),CX
+	XORL 248(SI),R8
+	XORL 252(SI),R9
+	MOVL DX,240(DI)
+	MOVL CX,244(DI)
+	MOVL R8,248(DI)
+	MOVL R9,252(DI)
+	MOVQ 408(SP),R9
+	SUBQ $256,R9
+	ADDQ $256,SI
+	ADDQ $256,DI
+	CMPQ R9,$256
+	JAE BYTESATLEAST256
+	CMPQ R9,$0
+	JBE DONE
+	BYTESBETWEEN1AND255:
+	CMPQ R9,$64
+	JAE NOCOPY
+	MOVQ DI,DX
+	LEAQ 416(SP),DI
+	MOVQ R9,CX
+	REP; MOVSB
+	LEAQ 416(SP),DI
+	LEAQ 416(SP),SI
+	NOCOPY:
+	MOVQ R9,408(SP)
+	MOVOA 48(SP),X0
+	MOVOA 0(SP),X1
+	MOVOA 16(SP),X2
+	MOVOA 32(SP),X3
+	MOVOA X1,X4
+	MOVQ $20,CX
+	MAINLOOP2:
+	PADDL X0,X4
+	MOVOA X0,X5
+	MOVOA X4,X6
+	PSLLL $7,X4
+	PSRLL $25,X6
+	PXOR X4,X3
+	PXOR X6,X3
+	PADDL X3,X5
+	MOVOA X3,X4
+	MOVOA X5,X6
+	PSLLL $9,X5
+	PSRLL $23,X6
+	PXOR X5,X2
+	PSHUFL $0X93,X3,X3
+	PXOR X6,X2
+	PADDL X2,X4
+	MOVOA X2,X5
+	MOVOA X4,X6
+	PSLLL $13,X4
+	PSRLL $19,X6
+	PXOR X4,X1
+	PSHUFL $0X4E,X2,X2
+	PXOR X6,X1
+	PADDL X1,X5
+	MOVOA X3,X4
+	MOVOA X5,X6
+	PSLLL $18,X5
+	PSRLL $14,X6
+	PXOR X5,X0
+	PSHUFL $0X39,X1,X1
+	PXOR X6,X0
+	PADDL X0,X4
+	MOVOA X0,X5
+	MOVOA X4,X6
+	PSLLL $7,X4
+	PSRLL $25,X6
+	PXOR X4,X1
+	PXOR X6,X1
+	PADDL X1,X5
+	MOVOA X1,X4
+	MOVOA X5,X6
+	PSLLL $9,X5
+	PSRLL $23,X6
+	PXOR X5,X2
+	PSHUFL $0X93,X1,X1
+	PXOR X6,X2
+	PADDL X2,X4
+	MOVOA X2,X5
+	MOVOA X4,X6
+	PSLLL $13,X4
+	PSRLL $19,X6
+	PXOR X4,X3
+	PSHUFL $0X4E,X2,X2
+	PXOR X6,X3
+	PADDL X3,X5
+	MOVOA X1,X4
+	MOVOA X5,X6
+	PSLLL $18,X5
+	PSRLL $14,X6
+	PXOR X5,X0
+	PSHUFL $0X39,X3,X3
+	PXOR X6,X0
+	PADDL X0,X4
+	MOVOA X0,X5
+	MOVOA X4,X6
+	PSLLL $7,X4
+	PSRLL $25,X6
+	PXOR X4,X3
+	PXOR X6,X3
+	PADDL X3,X5
+	MOVOA X3,X4
+	MOVOA X5,X6
+	PSLLL $9,X5
+	PSRLL $23,X6
+	PXOR X5,X2
+	PSHUFL $0X93,X3,X3
+	PXOR X6,X2
+	PADDL X2,X4
+	MOVOA X2,X5
+	MOVOA X4,X6
+	PSLLL $13,X4
+	PSRLL $19,X6
+	PXOR X4,X1
+	PSHUFL $0X4E,X2,X2
+	PXOR X6,X1
+	PADDL X1,X5
+	MOVOA X3,X4
+	MOVOA X5,X6
+	PSLLL $18,X5
+	PSRLL $14,X6
+	PXOR X5,X0
+	PSHUFL $0X39,X1,X1
+	PXOR X6,X0
+	PADDL X0,X4
+	MOVOA X0,X5
+	MOVOA X4,X6
+	PSLLL $7,X4
+	PSRLL $25,X6
+	PXOR X4,X1
+	PXOR X6,X1
+	PADDL X1,X5
+	MOVOA X1,X4
+	MOVOA X5,X6
+	PSLLL $9,X5
+	PSRLL $23,X6
+	PXOR X5,X2
+	PSHUFL $0X93,X1,X1
+	PXOR X6,X2
+	PADDL X2,X4
+	MOVOA X2,X5
+	MOVOA X4,X6
+	PSLLL $13,X4
+	PSRLL $19,X6
+	PXOR X4,X3
+	PSHUFL $0X4E,X2,X2
+	PXOR X6,X3
+	SUBQ $4,CX
+	PADDL X3,X5
+	MOVOA X1,X4
+	MOVOA X5,X6
+	PSLLL $18,X5
+	PXOR X7,X7
+	PSRLL $14,X6
+	PXOR X5,X0
+	PSHUFL $0X39,X3,X3
+	PXOR X6,X0
+	JA MAINLOOP2
+	PADDL 48(SP),X0
+	PADDL 0(SP),X1
+	PADDL 16(SP),X2
+	PADDL 32(SP),X3
+	MOVD X0,CX
+	MOVD X1,R8
+	MOVD X2,R9
+	MOVD X3,AX
+	PSHUFL $0X39,X0,X0
+	PSHUFL $0X39,X1,X1
+	PSHUFL $0X39,X2,X2
+	PSHUFL $0X39,X3,X3
+	XORL 0(SI),CX
+	XORL 48(SI),R8
+	XORL 32(SI),R9
+	XORL 16(SI),AX
+	MOVL CX,0(DI)
+	MOVL R8,48(DI)
+	MOVL R9,32(DI)
+	MOVL AX,16(DI)
+	MOVD X0,CX
+	MOVD X1,R8
+	MOVD X2,R9
+	MOVD X3,AX
+	PSHUFL $0X39,X0,X0
+	PSHUFL $0X39,X1,X1
+	PSHUFL $0X39,X2,X2
+	PSHUFL $0X39,X3,X3
+	XORL 20(SI),CX
+	XORL 4(SI),R8
+	XORL 52(SI),R9
+	XORL 36(SI),AX
+	MOVL CX,20(DI)
+	MOVL R8,4(DI)
+	MOVL R9,52(DI)
+	MOVL AX,36(DI)
+	MOVD X0,CX
+	MOVD X1,R8
+	MOVD X2,R9
+	MOVD X3,AX
+	PSHUFL $0X39,X0,X0
+	PSHUFL $0X39,X1,X1
+	PSHUFL $0X39,X2,X2
+	PSHUFL $0X39,X3,X3
+	XORL 40(SI),CX
+	XORL 24(SI),R8
+	XORL 8(SI),R9
+	XORL 56(SI),AX
+	MOVL CX,40(DI)
+	MOVL R8,24(DI)
+	MOVL R9,8(DI)
+	MOVL AX,56(DI)
+	MOVD X0,CX
+	MOVD X1,R8
+	MOVD X2,R9
+	MOVD X3,AX
+	XORL 60(SI),CX
+	XORL 44(SI),R8
+	XORL 28(SI),R9
+	XORL 12(SI),AX
+	MOVL CX,60(DI)
+	MOVL R8,44(DI)
+	MOVL R9,28(DI)
+	MOVL AX,12(DI)
+	MOVQ 408(SP),R9
+	MOVL 16(SP),CX
+	MOVL  36 (SP),R8
+	ADDQ $1,CX
+	SHLQ $32,R8
+	ADDQ R8,CX
+	MOVQ CX,R8
+	SHRQ $32,R8
+	MOVL CX,16(SP)
+	MOVL R8, 36 (SP)
+	CMPQ R9,$64
+	JA BYTESATLEAST65
+	JAE BYTESATLEAST64
+	MOVQ DI,SI
+	MOVQ DX,DI
+	MOVQ R9,CX
+	REP; MOVSB
+	BYTESATLEAST64:
+	DONE:
+	MOVQ 352(SP),R11
+	MOVQ 360(SP),R12
+	MOVQ 368(SP),R13
+	MOVQ 376(SP),R14
+	MOVQ 384(SP),R15
+	MOVQ 392(SP),BX
+	MOVQ 400(SP),BP
+	MOVQ R11,SP
+	RET
+	BYTESATLEAST65:
+	SUBQ $64,R9
+	ADDQ $64,DI
+	ADDQ $64,SI
+	JMP BYTESBETWEEN1AND255

vendor/golang.org/x/crypto/salsa20/salsa/salsa208.go

@@ -0,0 +1,199 @@
+// Copyright 2012 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package salsa
+
+// Core208 applies the Salsa20/8 core function to the 64-byte array in and puts
+// the result into the 64-byte array out. The input and output may be the same array.
+func Core208(out *[64]byte, in *[64]byte) {
+	j0 := uint32(in[0]) | uint32(in[1])<<8 | uint32(in[2])<<16 | uint32(in[3])<<24
+	j1 := uint32(in[4]) | uint32(in[5])<<8 | uint32(in[6])<<16 | uint32(in[7])<<24
+	j2 := uint32(in[8]) | uint32(in[9])<<8 | uint32(in[10])<<16 | uint32(in[11])<<24
+	j3 := uint32(in[12]) | uint32(in[13])<<8 | uint32(in[14])<<16 | uint32(in[15])<<24
+	j4 := uint32(in[16]) | uint32(in[17])<<8 | uint32(in[18])<<16 | uint32(in[19])<<24
+	j5 := uint32(in[20]) | uint32(in[21])<<8 | uint32(in[22])<<16 | uint32(in[23])<<24
+	j6 := uint32(in[24]) | uint32(in[25])<<8 | uint32(in[26])<<16 | uint32(in[27])<<24
+	j7 := uint32(in[28]) | uint32(in[29])<<8 | uint32(in[30])<<16 | uint32(in[31])<<24
+	j8 := uint32(in[32]) | uint32(in[33])<<8 | uint32(in[34])<<16 | uint32(in[35])<<24
+	j9 := uint32(in[36]) | uint32(in[37])<<8 | uint32(in[38])<<16 | uint32(in[39])<<24
+	j10 := uint32(in[40]) | uint32(in[41])<<8 | uint32(in[42])<<16 | uint32(in[43])<<24
+	j11 := uint32(in[44]) | uint32(in[45])<<8 | uint32(in[46])<<16 | uint32(in[47])<<24
+	j12 := uint32(in[48]) | uint32(in[49])<<8 | uint32(in[50])<<16 | uint32(in[51])<<24
+	j13 := uint32(in[52]) | uint32(in[53])<<8 | uint32(in[54])<<16 | uint32(in[55])<<24
+	j14 := uint32(in[56]) | uint32(in[57])<<8 | uint32(in[58])<<16 | uint32(in[59])<<24
+	j15 := uint32(in[60]) | uint32(in[61])<<8 | uint32(in[62])<<16 | uint32(in[63])<<24
+
+	x0, x1, x2, x3, x4, x5, x6, x7, x8 := j0, j1, j2, j3, j4, j5, j6, j7, j8
+	x9, x10, x11, x12, x13, x14, x15 := j9, j10, j11, j12, j13, j14, j15
+
+	for i := 0; i < 8; i += 2 {
+		u := x0 + x12
+		x4 ^= u<<7 | u>>(32-7)
+		u = x4 + x0
+		x8 ^= u<<9 | u>>(32-9)
+		u = x8 + x4
+		x12 ^= u<<13 | u>>(32-13)
+		u = x12 + x8
+		x0 ^= u<<18 | u>>(32-18)
+
+		u = x5 + x1
+		x9 ^= u<<7 | u>>(32-7)
+		u = x9 + x5
+		x13 ^= u<<9 | u>>(32-9)
+		u = x13 + x9
+		x1 ^= u<<13 | u>>(32-13)
+		u = x1 + x13
+		x5 ^= u<<18 | u>>(32-18)
+
+		u = x10 + x6
+		x14 ^= u<<7 | u>>(32-7)
+		u = x14 + x10
+		x2 ^= u<<9 | u>>(32-9)
+		u = x2 + x14
+		x6 ^= u<<13 | u>>(32-13)
+		u = x6 + x2
+		x10 ^= u<<18 | u>>(32-18)
+
+		u = x15 + x11
+		x3 ^= u<<7 | u>>(32-7)
+		u = x3 + x15
+		x7 ^= u<<9 | u>>(32-9)
+		u = x7 + x3
+		x11 ^= u<<13 | u>>(32-13)
+		u = x11 + x7
+		x15 ^= u<<18 | u>>(32-18)
+
+		u = x0 + x3
+		x1 ^= u<<7 | u>>(32-7)
+		u = x1 + x0
+		x2 ^= u<<9 | u>>(32-9)
+		u = x2 + x1
+		x3 ^= u<<13 | u>>(32-13)
+		u = x3 + x2
+		x0 ^= u<<18 | u>>(32-18)
+
+		u = x5 + x4
+		x6 ^= u<<7 | u>>(32-7)
+		u = x6 + x5
+		x7 ^= u<<9 | u>>(32-9)
+		u = x7 + x6
+		x4 ^= u<<13 | u>>(32-13)
+		u = x4 + x7
+		x5 ^= u<<18 | u>>(32-18)
+
+		u = x10 + x9
+		x11 ^= u<<7 | u>>(32-7)
+		u = x11 + x10
+		x8 ^= u<<9 | u>>(32-9)
+		u = x8 + x11
+		x9 ^= u<<13 | u>>(32-13)
+		u = x9 + x8
+		x10 ^= u<<18 | u>>(32-18)
+
+		u = x15 + x14
+		x12 ^= u<<7 | u>>(32-7)
+		u = x12 + x15
+		x13 ^= u<<9 | u>>(32-9)
+		u = x13 + x12
+		x14 ^= u<<13 | u>>(32-13)
+		u = x14 + x13
+		x15 ^= u<<18 | u>>(32-18)
+	}
+	x0 += j0
+	x1 += j1
+	x2 += j2
+	x3 += j3
+	x4 += j4
+	x5 += j5
+	x6 += j6
+	x7 += j7
+	x8 += j8
+	x9 += j9
+	x10 += j10
+	x11 += j11
+	x12 += j12
+	x13 += j13
+	x14 += j14
+	x15 += j15
+
+	out[0] = byte(x0)
+	out[1] = byte(x0 >> 8)
+	out[2] = byte(x0 >> 16)
+	out[3] = byte(x0 >> 24)
+
+	out[4] = byte(x1)
+	out[5] = byte(x1 >> 8)
+	out[6] = byte(x1 >> 16)
+	out[7] = byte(x1 >> 24)
+
+	out[8] = byte(x2)
+	out[9] = byte(x2 >> 8)
+	out[10] = byte(x2 >> 16)
+	out[11] = byte(x2 >> 24)
+
+	out[12] = byte(x3)
+	out[13] = byte(x3 >> 8)
+	out[14] = byte(x3 >> 16)
+	out[15] = byte(x3 >> 24)
+
+	out[16] = byte(x4)
+	out[17] = byte(x4 >> 8)
+	out[18] = byte(x4 >> 16)
+	out[19] = byte(x4 >> 24)
+
+	out[20] = byte(x5)
+	out[21] = byte(x5 >> 8)
+	out[22] = byte(x5 >> 16)
+	out[23] = byte(x5 >> 24)
+
+	out[24] = byte(x6)
+	out[25] = byte(x6 >> 8)
+	out[26] = byte(x6 >> 16)
+	out[27] = byte(x6 >> 24)
+
+	out[28] = byte(x7)
+	out[29] = byte(x7 >> 8)
+	out[30] = byte(x7 >> 16)
+	out[31] = byte(x7 >> 24)
+
+	out[32] = byte(x8)
+	out[33] = byte(x8 >> 8)
+	out[34] = byte(x8 >> 16)
+	out[35] = byte(x8 >> 24)
+
+	out[36] = byte(x9)
+	out[37] = byte(x9 >> 8)
+	out[38] = byte(x9 >> 16)
+	out[39] = byte(x9 >> 24)
+
+	out[40] = byte(x10)
+	out[41] = byte(x10 >> 8)
+	out[42] = byte(x10 >> 16)
+	out[43] = byte(x10 >> 24)
+
+	out[44] = byte(x11)
+	out[45] = byte(x11 >> 8)
+	out[46] = byte(x11 >> 16)
+	out[47] = byte(x11 >> 24)
+
+	out[48] = byte(x12)
+	out[49] = byte(x12 >> 8)
+	out[50] = byte(x12 >> 16)
+	out[51] = byte(x12 >> 24)
+
+	out[52] = byte(x13)
+	out[53] = byte(x13 >> 8)
+	out[54] = byte(x13 >> 16)
+	out[55] = byte(x13 >> 24)
+
+	out[56] = byte(x14)
+	out[57] = byte(x14 >> 8)
+	out[58] = byte(x14 >> 16)
+	out[59] = byte(x14 >> 24)
+
+	out[60] = byte(x15)
+	out[61] = byte(x15 >> 8)
+	out[62] = byte(x15 >> 16)
+	out[63] = byte(x15 >> 24)
+}

vendor/golang.org/x/crypto/salsa20/salsa/salsa20_amd64.go

@@ -0,0 +1,23 @@
+// Copyright 2012 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build amd64,!appengine,!gccgo
+
+package salsa
+
+// This function is implemented in salsa2020_amd64.s.
+
+//go:noescape
+
+func salsa2020XORKeyStream(out, in *byte, n uint64, nonce, key *byte)
+
+// XORKeyStream crypts bytes from in to out using the given key and counters.
+// In and out may be the same slice but otherwise should not overlap. Counter
+// contains the raw salsa20 counter bytes (both nonce and block counter).
+func XORKeyStream(out, in []byte, counter *[16]byte, key *[32]byte) {
+	if len(in) == 0 {
+		return
+	}
+	salsa2020XORKeyStream(&out[0], &in[0], uint64(len(in)), &counter[0], &key[0])
+}

vendor/golang.org/x/crypto/salsa20/salsa/salsa20_ref.go

@@ -0,0 +1,234 @@
+// Copyright 2012 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build !amd64 appengine gccgo
+
+package salsa
+
+const rounds = 20
+
+// core applies the Salsa20 core function to 16-byte input in, 32-byte key k,
+// and 16-byte constant c, and puts the result into 64-byte array out.
+func core(out *[64]byte, in *[16]byte, k *[32]byte, c *[16]byte) {
+	j0 := uint32(c[0]) | uint32(c[1])<<8 | uint32(c[2])<<16 | uint32(c[3])<<24
+	j1 := uint32(k[0]) | uint32(k[1])<<8 | uint32(k[2])<<16 | uint32(k[3])<<24
+	j2 := uint32(k[4]) | uint32(k[5])<<8 | uint32(k[6])<<16 | uint32(k[7])<<24
+	j3 := uint32(k[8]) | uint32(k[9])<<8 | uint32(k[10])<<16 | uint32(k[11])<<24
+	j4 := uint32(k[12]) | uint32(k[13])<<8 | uint32(k[14])<<16 | uint32(k[15])<<24
+	j5 := uint32(c[4]) | uint32(c[5])<<8 | uint32(c[6])<<16 | uint32(c[7])<<24
+	j6 := uint32(in[0]) | uint32(in[1])<<8 | uint32(in[2])<<16 | uint32(in[3])<<24
+	j7 := uint32(in[4]) | uint32(in[5])<<8 | uint32(in[6])<<16 | uint32(in[7])<<24
+	j8 := uint32(in[8]) | uint32(in[9])<<8 | uint32(in[10])<<16 | uint32(in[11])<<24
+	j9 := uint32(in[12]) | uint32(in[13])<<8 | uint32(in[14])<<16 | uint32(in[15])<<24
+	j10 := uint32(c[8]) | uint32(c[9])<<8 | uint32(c[10])<<16 | uint32(c[11])<<24
+	j11 := uint32(k[16]) | uint32(k[17])<<8 | uint32(k[18])<<16 | uint32(k[19])<<24
+	j12 := uint32(k[20]) | uint32(k[21])<<8 | uint32(k[22])<<16 | uint32(k[23])<<24
+	j13 := uint32(k[24]) | uint32(k[25])<<8 | uint32(k[26])<<16 | uint32(k[27])<<24
+	j14 := uint32(k[28]) | uint32(k[29])<<8 | uint32(k[30])<<16 | uint32(k[31])<<24
+	j15 := uint32(c[12]) | uint32(c[13])<<8 | uint32(c[14])<<16 | uint32(c[15])<<24
+
+	x0, x1, x2, x3, x4, x5, x6, x7, x8 := j0, j1, j2, j3, j4, j5, j6, j7, j8
+	x9, x10, x11, x12, x13, x14, x15 := j9, j10, j11, j12, j13, j14, j15
+
+	for i := 0; i < rounds; i += 2 {
+		u := x0 + x12
+		x4 ^= u<<7 | u>>(32-7)
+		u = x4 + x0
+		x8 ^= u<<9 | u>>(32-9)
+		u = x8 + x4
+		x12 ^= u<<13 | u>>(32-13)
+		u = x12 + x8
+		x0 ^= u<<18 | u>>(32-18)
+
+		u = x5 + x1
+		x9 ^= u<<7 | u>>(32-7)
+		u = x9 + x5
+		x13 ^= u<<9 | u>>(32-9)
+		u = x13 + x9
+		x1 ^= u<<13 | u>>(32-13)
+		u = x1 + x13
+		x5 ^= u<<18 | u>>(32-18)
+
+		u = x10 + x6
+		x14 ^= u<<7 | u>>(32-7)
+		u = x14 + x10
+		x2 ^= u<<9 | u>>(32-9)
+		u = x2 + x14
+		x6 ^= u<<13 | u>>(32-13)
+		u = x6 + x2
+		x10 ^= u<<18 | u>>(32-18)
+
+		u = x15 + x11
+		x3 ^= u<<7 | u>>(32-7)
+		u = x3 + x15
+		x7 ^= u<<9 | u>>(32-9)
+		u = x7 + x3
+		x11 ^= u<<13 | u>>(32-13)
+		u = x11 + x7
+		x15 ^= u<<18 | u>>(32-18)
+
+		u = x0 + x3
+		x1 ^= u<<7 | u>>(32-7)
+		u = x1 + x0
+		x2 ^= u<<9 | u>>(32-9)
+		u = x2 + x1
+		x3 ^= u<<13 | u>>(32-13)
+		u = x3 + x2
+		x0 ^= u<<18 | u>>(32-18)
+
+		u = x5 + x4
+		x6 ^= u<<7 | u>>(32-7)
+		u = x6 + x5
+		x7 ^= u<<9 | u>>(32-9)
+		u = x7 + x6
+		x4 ^= u<<13 | u>>(32-13)
+		u = x4 + x7
+		x5 ^= u<<18 | u>>(32-18)
+
+		u = x10 + x9
+		x11 ^= u<<7 | u>>(32-7)
+		u = x11 + x10
+		x8 ^= u<<9 | u>>(32-9)
+		u = x8 + x11
+		x9 ^= u<<13 | u>>(32-13)
+		u = x9 + x8
+		x10 ^= u<<18 | u>>(32-18)
+
+		u = x15 + x14
+		x12 ^= u<<7 | u>>(32-7)
+		u = x12 + x15
+		x13 ^= u<<9 | u>>(32-9)
+		u = x13 + x12
+		x14 ^= u<<13 | u>>(32-13)
+		u = x14 + x13
+		x15 ^= u<<18 | u>>(32-18)
+	}
+	x0 += j0
+	x1 += j1
+	x2 += j2
+	x3 += j3
+	x4 += j4
+	x5 += j5
+	x6 += j6
+	x7 += j7
+	x8 += j8
+	x9 += j9
+	x10 += j10
+	x11 += j11
+	x12 += j12
+	x13 += j13
+	x14 += j14
+	x15 += j15
+
+	out[0] = byte(x0)
+	out[1] = byte(x0 >> 8)
+	out[2] = byte(x0 >> 16)
+	out[3] = byte(x0 >> 24)
+
+	out[4] = byte(x1)
+	out[5] = byte(x1 >> 8)
+	out[6] = byte(x1 >> 16)
+	out[7] = byte(x1 >> 24)
+
+	out[8] = byte(x2)
+	out[9] = byte(x2 >> 8)
+	out[10] = byte(x2 >> 16)
+	out[11] = byte(x2 >> 24)
+
+	out[12] = byte(x3)
+	out[13] = byte(x3 >> 8)
+	out[14] = byte(x3 >> 16)
+	out[15] = byte(x3 >> 24)
+
+	out[16] = byte(x4)
+	out[17] = byte(x4 >> 8)
+	out[18] = byte(x4 >> 16)
+	out[19] = byte(x4 >> 24)
+
+	out[20] = byte(x5)
+	out[21] = byte(x5 >> 8)
+	out[22] = byte(x5 >> 16)
+	out[23] = byte(x5 >> 24)
+
+	out[24] = byte(x6)
+	out[25] = byte(x6 >> 8)
+	out[26] = byte(x6 >> 16)
+	out[27] = byte(x6 >> 24)
+
+	out[28] = byte(x7)
+	out[29] = byte(x7 >> 8)
+	out[30] = byte(x7 >> 16)
+	out[31] = byte(x7 >> 24)
+
+	out[32] = byte(x8)
+	out[33] = byte(x8 >> 8)
+	out[34] = byte(x8 >> 16)
+	out[35] = byte(x8 >> 24)
+
+	out[36] = byte(x9)
+	out[37] = byte(x9 >> 8)
+	out[38] = byte(x9 >> 16)
+	out[39] = byte(x9 >> 24)
+
+	out[40] = byte(x10)
+	out[41] = byte(x10 >> 8)
+	out[42] = byte(x10 >> 16)
+	out[43] = byte(x10 >> 24)
+
+	out[44] = byte(x11)
+	out[45] = byte(x11 >> 8)
+	out[46] = byte(x11 >> 16)
+	out[47] = byte(x11 >> 24)
+
+	out[48] = byte(x12)
+	out[49] = byte(x12 >> 8)
+	out[50] = byte(x12 >> 16)
+	out[51] = byte(x12 >> 24)
+
+	out[52] = byte(x13)
+	out[53] = byte(x13 >> 8)
+	out[54] = byte(x13 >> 16)
+	out[55] = byte(x13 >> 24)
+
+	out[56] = byte(x14)
+	out[57] = byte(x14 >> 8)
+	out[58] = byte(x14 >> 16)
+	out[59] = byte(x14 >> 24)
+
+	out[60] = byte(x15)
+	out[61] = byte(x15 >> 8)
+	out[62] = byte(x15 >> 16)
+	out[63] = byte(x15 >> 24)
+}
+
+// XORKeyStream crypts bytes from in to out using the given key and counters.
+// In and out may be the same slice but otherwise should not overlap. Counter
+// contains the raw salsa20 counter bytes (both nonce and block counter).
+func XORKeyStream(out, in []byte, counter *[16]byte, key *[32]byte) {
+	var block [64]byte
+	var counterCopy [16]byte
+	copy(counterCopy[:], counter[:])
+
+	for len(in) >= 64 {
+		core(&block, &counterCopy, key, &Sigma)
+		for i, x := range block {
+			out[i] = in[i] ^ x
+		}
+		u := uint32(1)
+		for i := 8; i < 16; i++ {
+			u += uint32(counterCopy[i])
+			counterCopy[i] = byte(u)
+			u >>= 8
+		}
+		in = in[64:]
+		out = out[64:]
+	}
+
+	if len(in) > 0 {
+		core(&block, &counterCopy, key, &Sigma)
+		for i, v := range in {
+			out[i] = v ^ block[i]
+		}
+	}
+}

vendor/golang.org/x/crypto/ssh/terminal/terminal.go

@@ -0,0 +1,892 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package terminal
+
+import (
+	"bytes"
+	"io"
+	"sync"
+	"unicode/utf8"
+)
+
+// EscapeCodes contains escape sequences that can be written to the terminal in
+// order to achieve different styles of text.
+type EscapeCodes struct {
+	// Foreground colors
+	Black, Red, Green, Yellow, Blue, Magenta, Cyan, White []byte
+
+	// Reset all attributes
+	Reset []byte
+}
+
+var vt100EscapeCodes = EscapeCodes{
+	Black:   []byte{keyEscape, '[', '3', '0', 'm'},
+	Red:     []byte{keyEscape, '[', '3', '1', 'm'},
+	Green:   []byte{keyEscape, '[', '3', '2', 'm'},
+	Yellow:  []byte{keyEscape, '[', '3', '3', 'm'},
+	Blue:    []byte{keyEscape, '[', '3', '4', 'm'},
+	Magenta: []byte{keyEscape, '[', '3', '5', 'm'},
+	Cyan:    []byte{keyEscape, '[', '3', '6', 'm'},
+	White:   []byte{keyEscape, '[', '3', '7', 'm'},
+
+	Reset: []byte{keyEscape, '[', '0', 'm'},
+}
+
+// Terminal contains the state for running a VT100 terminal that is capable of
+// reading lines of input.
+type Terminal struct {
+	// AutoCompleteCallback, if non-null, is called for each keypress with
+	// the full input line and the current position of the cursor (in
+	// bytes, as an index into |line|). If it returns ok=false, the key
+	// press is processed normally. Otherwise it returns a replacement line
+	// and the new cursor position.
+	AutoCompleteCallback func(line string, pos int, key rune) (newLine string, newPos int, ok bool)
+
+	// Escape contains a pointer to the escape codes for this terminal.
+	// It's always a valid pointer, although the escape codes themselves
+	// may be empty if the terminal doesn't support them.
+	Escape *EscapeCodes
+
+	// lock protects the terminal and the state in this object from
+	// concurrent processing of a key press and a Write() call.
+	lock sync.Mutex
+
+	c      io.ReadWriter
+	prompt []rune
+
+	// line is the current line being entered.
+	line []rune
+	// pos is the logical position of the cursor in line
+	pos int
+	// echo is true if local echo is enabled
+	echo bool
+	// pasteActive is true iff there is a bracketed paste operation in
+	// progress.
+	pasteActive bool
+
+	// cursorX contains the current X value of the cursor where the left
+	// edge is 0. cursorY contains the row number where the first row of
+	// the current line is 0.
+	cursorX, cursorY int
+	// maxLine is the greatest value of cursorY so far.
+	maxLine int
+
+	termWidth, termHeight int
+
+	// outBuf contains the terminal data to be sent.
+	outBuf []byte
+	// remainder contains the remainder of any partial key sequences after
+	// a read. It aliases into inBuf.
+	remainder []byte
+	inBuf     [256]byte
+
+	// history contains previously entered commands so that they can be
+	// accessed with the up and down keys.
+	history stRingBuffer
+	// historyIndex stores the currently accessed history entry, where zero
+	// means the immediately previous entry.
+	historyIndex int
+	// When navigating up and down the history it's possible to return to
+	// the incomplete, initial line. That value is stored in
+	// historyPending.
+	historyPending string
+}
+
+// NewTerminal runs a VT100 terminal on the given ReadWriter. If the ReadWriter is
+// a local terminal, that terminal must first have been put into raw mode.
+// prompt is a string that is written at the start of each input line (i.e.
+// "> ").
+func NewTerminal(c io.ReadWriter, prompt string) *Terminal {
+	return &Terminal{
+		Escape:       &vt100EscapeCodes,
+		c:            c,
+		prompt:       []rune(prompt),
+		termWidth:    80,
+		termHeight:   24,
+		echo:         true,
+		historyIndex: -1,
+	}
+}
+
+const (
+	keyCtrlD     = 4
+	keyCtrlU     = 21
+	keyEnter     = '\r'
+	keyEscape    = 27
+	keyBackspace = 127
+	keyUnknown   = 0xd800 /* UTF-16 surrogate area */ + iota
+	keyUp
+	keyDown
+	keyLeft
+	keyRight
+	keyAltLeft
+	keyAltRight
+	keyHome
+	keyEnd
+	keyDeleteWord
+	keyDeleteLine
+	keyClearScreen
+	keyPasteStart
+	keyPasteEnd
+)
+
+var pasteStart = []byte{keyEscape, '[', '2', '0', '0', '~'}
+var pasteEnd = []byte{keyEscape, '[', '2', '0', '1', '~'}
+
+// bytesToKey tries to parse a key sequence from b. If successful, it returns
+// the key and the remainder of the input. Otherwise it returns utf8.RuneError.
+func bytesToKey(b []byte, pasteActive bool) (rune, []byte) {
+	if len(b) == 0 {
+		return utf8.RuneError, nil
+	}
+
+	if !pasteActive {
+		switch b[0] {
+		case 1: // ^A
+			return keyHome, b[1:]
+		case 5: // ^E
+			return keyEnd, b[1:]
+		case 8: // ^H
+			return keyBackspace, b[1:]
+		case 11: // ^K
+			return keyDeleteLine, b[1:]
+		case 12: // ^L
+			return keyClearScreen, b[1:]
+		case 23: // ^W
+			return keyDeleteWord, b[1:]
+		}
+	}
+
+	if b[0] != keyEscape {
+		if !utf8.FullRune(b) {
+			return utf8.RuneError, b
+		}
+		r, l := utf8.DecodeRune(b)
+		return r, b[l:]
+	}
+
+	if !pasteActive && len(b) >= 3 && b[0] == keyEscape && b[1] == '[' {
+		switch b[2] {
+		case 'A':
+			return keyUp, b[3:]
+		case 'B':
+			return keyDown, b[3:]
+		case 'C':
+			return keyRight, b[3:]
+		case 'D':
+			return keyLeft, b[3:]
+		case 'H':
+			return keyHome, b[3:]
+		case 'F':
+			return keyEnd, b[3:]
+		}
+	}
+
+	if !pasteActive && len(b) >= 6 && b[0] == keyEscape && b[1] == '[' && b[2] == '1' && b[3] == ';' && b[4] == '3' {
+		switch b[5] {
+		case 'C':
+			return keyAltRight, b[6:]
+		case 'D':
+			return keyAltLeft, b[6:]
+		}
+	}
+
+	if !pasteActive && len(b) >= 6 && bytes.Equal(b[:6], pasteStart) {
+		return keyPasteStart, b[6:]
+	}
+
+	if pasteActive && len(b) >= 6 && bytes.Equal(b[:6], pasteEnd) {
+		return keyPasteEnd, b[6:]
+	}
+
+	// If we get here then we have a key that we don't recognise, or a
+	// partial sequence. It's not clear how one should find the end of a
+	// sequence without knowing them all, but it seems that [a-zA-Z~] only
+	// appears at the end of a sequence.
+	for i, c := range b[0:] {
+		if c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' || c == '~' {
+			return keyUnknown, b[i+1:]
+		}
+	}
+
+	return utf8.RuneError, b
+}
+
+// queue appends data to the end of t.outBuf
+func (t *Terminal) queue(data []rune) {
+	t.outBuf = append(t.outBuf, []byte(string(data))...)
+}
+
+var eraseUnderCursor = []rune{' ', keyEscape, '[', 'D'}
+var space = []rune{' '}
+
+func isPrintable(key rune) bool {
+	isInSurrogateArea := key >= 0xd800 && key <= 0xdbff
+	return key >= 32 && !isInSurrogateArea
+}
+
+// moveCursorToPos appends data to t.outBuf which will move the cursor to the
+// given, logical position in the text.
+func (t *Terminal) moveCursorToPos(pos int) {
+	if !t.echo {
+		return
+	}
+
+	x := visualLength(t.prompt) + pos
+	y := x / t.termWidth
+	x = x % t.termWidth
+
+	up := 0
+	if y < t.cursorY {
+		up = t.cursorY - y
+	}
+
+	down := 0
+	if y > t.cursorY {
+		down = y - t.cursorY
+	}
+
+	left := 0
+	if x < t.cursorX {
+		left = t.cursorX - x
+	}
+
+	right := 0
+	if x > t.cursorX {
+		right = x - t.cursorX
+	}
+
+	t.cursorX = x
+	t.cursorY = y
+	t.move(up, down, left, right)
+}
+
+func (t *Terminal) move(up, down, left, right int) {
+	movement := make([]rune, 3*(up+down+left+right))
+	m := movement
+	for i := 0; i < up; i++ {
+		m[0] = keyEscape
+		m[1] = '['
+		m[2] = 'A'
+		m = m[3:]
+	}
+	for i := 0; i < down; i++ {
+		m[0] = keyEscape
+		m[1] = '['
+		m[2] = 'B'
+		m = m[3:]
+	}
+	for i := 0; i < left; i++ {
+		m[0] = keyEscape
+		m[1] = '['
+		m[2] = 'D'
+		m = m[3:]
+	}
+	for i := 0; i < right; i++ {
+		m[0] = keyEscape
+		m[1] = '['
+		m[2] = 'C'
+		m = m[3:]
+	}
+
+	t.queue(movement)
+}
+
+func (t *Terminal) clearLineToRight() {
+	op := []rune{keyEscape, '[', 'K'}
+	t.queue(op)
+}
+
+const maxLineLength = 4096
+
+func (t *Terminal) setLine(newLine []rune, newPos int) {
+	if t.echo {
+		t.moveCursorToPos(0)
+		t.writeLine(newLine)
+		for i := len(newLine); i < len(t.line); i++ {
+			t.writeLine(space)
+		}
+		t.moveCursorToPos(newPos)
+	}
+	t.line = newLine
+	t.pos = newPos
+}
+
+func (t *Terminal) advanceCursor(places int) {
+	t.cursorX += places
+	t.cursorY += t.cursorX / t.termWidth
+	if t.cursorY > t.maxLine {
+		t.maxLine = t.cursorY
+	}
+	t.cursorX = t.cursorX % t.termWidth
+
+	if places > 0 && t.cursorX == 0 {
+		// Normally terminals will advance the current position
+		// when writing a character. But that doesn't happen
+		// for the last character in a line. However, when
+		// writing a character (except a new line) that causes
+		// a line wrap, the position will be advanced two
+		// places.
+		//
+		// So, if we are stopping at the end of a line, we
+		// need to write a newline so that our cursor can be
+		// advanced to the next line.
+		t.outBuf = append(t.outBuf, '\n')
+	}
+}
+
+func (t *Terminal) eraseNPreviousChars(n int) {
+	if n == 0 {
+		return
+	}
+
+	if t.pos < n {
+		n = t.pos
+	}
+	t.pos -= n
+	t.moveCursorToPos(t.pos)
+
+	copy(t.line[t.pos:], t.line[n+t.pos:])
+	t.line = t.line[:len(t.line)-n]
+	if t.echo {
+		t.writeLine(t.line[t.pos:])
+		for i := 0; i < n; i++ {
+			t.queue(space)
+		}
+		t.advanceCursor(n)
+		t.moveCursorToPos(t.pos)
+	}
+}
+
+// countToLeftWord returns then number of characters from the cursor to the
+// start of the previous word.
+func (t *Terminal) countToLeftWord() int {
+	if t.pos == 0 {
+		return 0
+	}
+
+	pos := t.pos - 1
+	for pos > 0 {
+		if t.line[pos] != ' ' {
+			break
+		}
+		pos--
+	}
+	for pos > 0 {
+		if t.line[pos] == ' ' {
+			pos++
+			break
+		}
+		pos--
+	}
+
+	return t.pos - pos
+}
+
+// countToRightWord returns then number of characters from the cursor to the
+// start of the next word.
+func (t *Terminal) countToRightWord() int {
+	pos := t.pos
+	for pos < len(t.line) {
+		if t.line[pos] == ' ' {
+			break
+		}
+		pos++
+	}
+	for pos < len(t.line) {
+		if t.line[pos] != ' ' {
+			break
+		}
+		pos++
+	}
+	return pos - t.pos
+}
+
+// visualLength returns the number of visible glyphs in s.
+func visualLength(runes []rune) int {
+	inEscapeSeq := false
+	length := 0
+
+	for _, r := range runes {
+		switch {
+		case inEscapeSeq:
+			if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') {
+				inEscapeSeq = false
+			}
+		case r == '\x1b':
+			inEscapeSeq = true
+		default:
+			length++
+		}
+	}
+
+	return length
+}
+
+// handleKey processes the given key and, optionally, returns a line of text
+// that the user has entered.
+func (t *Terminal) handleKey(key rune) (line string, ok bool) {
+	if t.pasteActive && key != keyEnter {
+		t.addKeyToLine(key)
+		return
+	}
+
+	switch key {
+	case keyBackspace:
+		if t.pos == 0 {
+			return
+		}
+		t.eraseNPreviousChars(1)
+	case keyAltLeft:
+		// move left by a word.
+		t.pos -= t.countToLeftWord()
+		t.moveCursorToPos(t.pos)
+	case keyAltRight:
+		// move right by a word.
+		t.pos += t.countToRightWord()
+		t.moveCursorToPos(t.pos)
+	case keyLeft:
+		if t.pos == 0 {
+			return
+		}
+		t.pos--
+		t.moveCursorToPos(t.pos)
+	case keyRight:
+		if t.pos == len(t.line) {
+			return
+		}
+		t.pos++
+		t.moveCursorToPos(t.pos)
+	case keyHome:
+		if t.pos == 0 {
+			return
+		}
+		t.pos = 0
+		t.moveCursorToPos(t.pos)
+	case keyEnd:
+		if t.pos == len(t.line) {
+			return
+		}
+		t.pos = len(t.line)
+		t.moveCursorToPos(t.pos)
+	case keyUp:
+		entry, ok := t.history.NthPreviousEntry(t.historyIndex + 1)
+		if !ok {
+			return "", false
+		}
+		if t.historyIndex == -1 {
+			t.historyPending = string(t.line)
+		}
+		t.historyIndex++
+		runes := []rune(entry)
+		t.setLine(runes, len(runes))
+	case keyDown:
+		switch t.historyIndex {
+		case -1:
+			return
+		case 0:
+			runes := []rune(t.historyPending)
+			t.setLine(runes, len(runes))
+			t.historyIndex--
+		default:
+			entry, ok := t.history.NthPreviousEntry(t.historyIndex - 1)
+			if ok {
+				t.historyIndex--
+				runes := []rune(entry)
+				t.setLine(runes, len(runes))
+			}
+		}
+	case keyEnter:
+		t.moveCursorToPos(len(t.line))
+		t.queue([]rune("\r\n"))
+		line = string(t.line)
+		ok = true
+		t.line = t.line[:0]
+		t.pos = 0
+		t.cursorX = 0
+		t.cursorY = 0
+		t.maxLine = 0
+	case keyDeleteWord:
+		// Delete zero or more spaces and then one or more characters.
+		t.eraseNPreviousChars(t.countToLeftWord())
+	case keyDeleteLine:
+		// Delete everything from the current cursor position to the
+		// end of line.
+		for i := t.pos; i < len(t.line); i++ {
+			t.queue(space)
+			t.advanceCursor(1)
+		}
+		t.line = t.line[:t.pos]
+		t.moveCursorToPos(t.pos)
+	case keyCtrlD:
+		// Erase the character under the current position.
+		// The EOF case when the line is empty is handled in
+		// readLine().
+		if t.pos < len(t.line) {
+			t.pos++
+			t.eraseNPreviousChars(1)
+		}
+	case keyCtrlU:
+		t.eraseNPreviousChars(t.pos)
+	case keyClearScreen:
+		// Erases the screen and moves the cursor to the home position.
+		t.queue([]rune("\x1b[2J\x1b[H"))
+		t.queue(t.prompt)
+		t.cursorX, t.cursorY = 0, 0
+		t.advanceCursor(visualLength(t.prompt))
+		t.setLine(t.line, t.pos)
+	default:
+		if t.AutoCompleteCallback != nil {
+			prefix := string(t.line[:t.pos])
+			suffix := string(t.line[t.pos:])
+
+			t.lock.Unlock()
+			newLine, newPos, completeOk := t.AutoCompleteCallback(prefix+suffix, len(prefix), key)
+			t.lock.Lock()
+
+			if completeOk {
+				t.setLine([]rune(newLine), utf8.RuneCount([]byte(newLine)[:newPos]))
+				return
+			}
+		}
+		if !isPrintable(key) {
+			return
+		}
+		if len(t.line) == maxLineLength {
+			return
+		}
+		t.addKeyToLine(key)
+	}
+	return
+}
+
+// addKeyToLine inserts the given key at the current position in the current
+// line.
+func (t *Terminal) addKeyToLine(key rune) {
+	if len(t.line) == cap(t.line) {
+		newLine := make([]rune, len(t.line), 2*(1+len(t.line)))
+		copy(newLine, t.line)
+		t.line = newLine
+	}
+	t.line = t.line[:len(t.line)+1]
+	copy(t.line[t.pos+1:], t.line[t.pos:])
+	t.line[t.pos] = key
+	if t.echo {
+		t.writeLine(t.line[t.pos:])
+	}
+	t.pos++
+	t.moveCursorToPos(t.pos)
+}
+
+func (t *Terminal) writeLine(line []rune) {
+	for len(line) != 0 {
+		remainingOnLine := t.termWidth - t.cursorX
+		todo := len(line)
+		if todo > remainingOnLine {
+			todo = remainingOnLine
+		}
+		t.queue(line[:todo])
+		t.advanceCursor(visualLength(line[:todo]))
+		line = line[todo:]
+	}
+}
+
+func (t *Terminal) Write(buf []byte) (n int, err error) {
+	t.lock.Lock()
+	defer t.lock.Unlock()
+
+	if t.cursorX == 0 && t.cursorY == 0 {
+		// This is the easy case: there's nothing on the screen that we
+		// have to move out of the way.
+		return t.c.Write(buf)
+	}
+
+	// We have a prompt and possibly user input on the screen. We
+	// have to clear it first.
+	t.move(0 /* up */, 0 /* down */, t.cursorX /* left */, 0 /* right */)
+	t.cursorX = 0
+	t.clearLineToRight()
+
+	for t.cursorY > 0 {
+		t.move(1 /* up */, 0, 0, 0)
+		t.cursorY--
+		t.clearLineToRight()
+	}
+
+	if _, err = t.c.Write(t.outBuf); err != nil {
+		return
+	}
+	t.outBuf = t.outBuf[:0]
+
+	if n, err = t.c.Write(buf); err != nil {
+		return
+	}
+
+	t.writeLine(t.prompt)
+	if t.echo {
+		t.writeLine(t.line)
+	}
+
+	t.moveCursorToPos(t.pos)
+
+	if _, err = t.c.Write(t.outBuf); err != nil {
+		return
+	}
+	t.outBuf = t.outBuf[:0]
+	return
+}
+
+// ReadPassword temporarily changes the prompt and reads a password, without
+// echo, from the terminal.
+func (t *Terminal) ReadPassword(prompt string) (line string, err error) {
+	t.lock.Lock()
+	defer t.lock.Unlock()
+
+	oldPrompt := t.prompt
+	t.prompt = []rune(prompt)
+	t.echo = false
+
+	line, err = t.readLine()
+
+	t.prompt = oldPrompt
+	t.echo = true
+
+	return
+}
+
+// ReadLine returns a line of input from the terminal.
+func (t *Terminal) ReadLine() (line string, err error) {
+	t.lock.Lock()
+	defer t.lock.Unlock()
+
+	return t.readLine()
+}
+
+func (t *Terminal) readLine() (line string, err error) {
+	// t.lock must be held at this point
+
+	if t.cursorX == 0 && t.cursorY == 0 {
+		t.writeLine(t.prompt)
+		t.c.Write(t.outBuf)
+		t.outBuf = t.outBuf[:0]
+	}
+
+	lineIsPasted := t.pasteActive
+
+	for {
+		rest := t.remainder
+		lineOk := false
+		for !lineOk {
+			var key rune
+			key, rest = bytesToKey(rest, t.pasteActive)
+			if key == utf8.RuneError {
+				break
+			}
+			if !t.pasteActive {
+				if key == keyCtrlD {
+					if len(t.line) == 0 {
+						return "", io.EOF
+					}
+				}
+				if key == keyPasteStart {
+					t.pasteActive = true
+					if len(t.line) == 0 {
+						lineIsPasted = true
+					}
+					continue
+				}
+			} else if key == keyPasteEnd {
+				t.pasteActive = false
+				continue
+			}
+			if !t.pasteActive {
+				lineIsPasted = false
+			}
+			line, lineOk = t.handleKey(key)
+		}
+		if len(rest) > 0 {
+			n := copy(t.inBuf[:], rest)
+			t.remainder = t.inBuf[:n]
+		} else {
+			t.remainder = nil
+		}
+		t.c.Write(t.outBuf)
+		t.outBuf = t.outBuf[:0]
+		if lineOk {
+			if t.echo {
+				t.historyIndex = -1
+				t.history.Add(line)
+			}
+			if lineIsPasted {
+				err = ErrPasteIndicator
+			}
+			return
+		}
+
+		// t.remainder is a slice at the beginning of t.inBuf
+		// containing a partial key sequence
+		readBuf := t.inBuf[len(t.remainder):]
+		var n int
+
+		t.lock.Unlock()
+		n, err = t.c.Read(readBuf)
+		t.lock.Lock()
+
+		if err != nil {
+			return
+		}
+
+		t.remainder = t.inBuf[:n+len(t.remainder)]
+	}
+
+	panic("unreachable") // for Go 1.0.
+}
+
+// SetPrompt sets the prompt to be used when reading subsequent lines.
+func (t *Terminal) SetPrompt(prompt string) {
+	t.lock.Lock()
+	defer t.lock.Unlock()
+
+	t.prompt = []rune(prompt)
+}
+
+func (t *Terminal) clearAndRepaintLinePlusNPrevious(numPrevLines int) {
+	// Move cursor to column zero at the start of the line.
+	t.move(t.cursorY, 0, t.cursorX, 0)
+	t.cursorX, t.cursorY = 0, 0
+	t.clearLineToRight()
+	for t.cursorY < numPrevLines {
+		// Move down a line
+		t.move(0, 1, 0, 0)
+		t.cursorY++
+		t.clearLineToRight()
+	}
+	// Move back to beginning.
+	t.move(t.cursorY, 0, 0, 0)
+	t.cursorX, t.cursorY = 0, 0
+
+	t.queue(t.prompt)
+	t.advanceCursor(visualLength(t.prompt))
+	t.writeLine(t.line)
+	t.moveCursorToPos(t.pos)
+}
+
+func (t *Terminal) SetSize(width, height int) error {
+	t.lock.Lock()
+	defer t.lock.Unlock()
+
+	if width == 0 {
+		width = 1
+	}
+
+	oldWidth := t.termWidth
+	t.termWidth, t.termHeight = width, height
+
+	switch {
+	case width == oldWidth:
+		// If the width didn't change then nothing else needs to be
+		// done.
+		return nil
+	case len(t.line) == 0 && t.cursorX == 0 && t.cursorY == 0:
+		// If there is nothing on current line and no prompt printed,
+		// just do nothing
+		return nil
+	case width < oldWidth:
+		// Some terminals (e.g. xterm) will truncate lines that were
+		// too long when shinking. Others, (e.g. gnome-terminal) will
+		// attempt to wrap them. For the former, repainting t.maxLine
+		// works great, but that behaviour goes badly wrong in the case
+		// of the latter because they have doubled every full line.
+
+		// We assume that we are working on a terminal that wraps lines
+		// and adjust the cursor position based on every previous line
+		// wrapping and turning into two. This causes the prompt on
+		// xterms to move upwards, which isn't great, but it avoids a
+		// huge mess with gnome-terminal.
+		if t.cursorX >= t.termWidth {
+			t.cursorX = t.termWidth - 1
+		}
+		t.cursorY *= 2
+		t.clearAndRepaintLinePlusNPrevious(t.maxLine * 2)
+	case width > oldWidth:
+		// If the terminal expands then our position calculations will
+		// be wrong in the future because we think the cursor is
+		// |t.pos| chars into the string, but there will be a gap at
+		// the end of any wrapped line.
+		//
+		// But the position will actually be correct until we move, so
+		// we can move back to the beginning and repaint everything.
+		t.clearAndRepaintLinePlusNPrevious(t.maxLine)
+	}
+
+	_, err := t.c.Write(t.outBuf)
+	t.outBuf = t.outBuf[:0]
+	return err
+}
+
+type pasteIndicatorError struct{}
+
+func (pasteIndicatorError) Error() string {
+	return "terminal: ErrPasteIndicator not correctly handled"
+}
+
+// ErrPasteIndicator may be returned from ReadLine as the error, in addition
+// to valid line data. It indicates that bracketed paste mode is enabled and
+// that the returned line consists only of pasted data. Programs may wish to
+// interpret pasted data more literally than typed data.
+var ErrPasteIndicator = pasteIndicatorError{}
+
+// SetBracketedPasteMode requests that the terminal bracket paste operations
+// with markers. Not all terminals support this but, if it is supported, then
+// enabling this mode will stop any autocomplete callback from running due to
+// pastes. Additionally, any lines that are completely pasted will be returned
+// from ReadLine with the error set to ErrPasteIndicator.
+func (t *Terminal) SetBracketedPasteMode(on bool) {
+	if on {
+		io.WriteString(t.c, "\x1b[?2004h")
+	} else {
+		io.WriteString(t.c, "\x1b[?2004l")
+	}
+}
+
+// stRingBuffer is a ring buffer of strings.
+type stRingBuffer struct {
+	// entries contains max elements.
+	entries []string
+	max     int
+	// head contains the index of the element most recently added to the ring.
+	head int
+	// size contains the number of elements in the ring.
+	size int
+}
+
+func (s *stRingBuffer) Add(a string) {
+	if s.entries == nil {
+		const defaultNumEntries = 100
+		s.entries = make([]string, defaultNumEntries)
+		s.max = defaultNumEntries
+	}
+
+	s.head = (s.head + 1) % s.max
+	s.entries[s.head] = a
+	if s.size < s.max {
+		s.size++
+	}
+}
+
+// NthPreviousEntry returns the value passed to the nth previous call to Add.
+// If n is zero then the immediately prior value is returned, if one, then the
+// next most recent, and so on. If such an element doesn't exist then ok is
+// false.
+func (s *stRingBuffer) NthPreviousEntry(n int) (value string, ok bool) {
+	if n >= s.size {
+		return "", false
+	}
+	index := s.head - n
+	if index < 0 {
+		index += s.max
+	}
+	return s.entries[index], true
+}

vendor/golang.org/x/crypto/ssh/terminal/util.go

@@ -0,0 +1,128 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build darwin dragonfly freebsd linux,!appengine netbsd openbsd
+
+// Package terminal provides support functions for dealing with terminals, as
+// commonly found on UNIX systems.
+//
+// Putting a terminal into raw mode is the most common requirement:
+//
+// 	oldState, err := terminal.MakeRaw(0)
+// 	if err != nil {
+// 	        panic(err)
+// 	}
+// 	defer terminal.Restore(0, oldState)
+package terminal // import "golang.org/x/crypto/ssh/terminal"
+
+import (
+	"io"
+	"syscall"
+	"unsafe"
+)
+
+// State contains the state of a terminal.
+type State struct {
+	termios syscall.Termios
+}
+
+// IsTerminal returns true if the given file descriptor is a terminal.
+func IsTerminal(fd int) bool {
+	var termios syscall.Termios
+	_, _, err := syscall.Syscall6(syscall.SYS_IOCTL, uintptr(fd), ioctlReadTermios, uintptr(unsafe.Pointer(&termios)), 0, 0, 0)
+	return err == 0
+}
+
+// MakeRaw put the terminal connected to the given file descriptor into raw
+// mode and returns the previous state of the terminal so that it can be
+// restored.
+func MakeRaw(fd int) (*State, error) {
+	var oldState State
+	if _, _, err := syscall.Syscall6(syscall.SYS_IOCTL, uintptr(fd), ioctlReadTermios, uintptr(unsafe.Pointer(&oldState.termios)), 0, 0, 0); err != 0 {
+		return nil, err
+	}
+
+	newState := oldState.termios
+	newState.Iflag &^= syscall.ISTRIP | syscall.INLCR | syscall.ICRNL | syscall.IGNCR | syscall.IXON | syscall.IXOFF
+	newState.Lflag &^= syscall.ECHO | syscall.ICANON | syscall.ISIG
+	if _, _, err := syscall.Syscall6(syscall.SYS_IOCTL, uintptr(fd), ioctlWriteTermios, uintptr(unsafe.Pointer(&newState)), 0, 0, 0); err != 0 {
+		return nil, err
+	}
+
+	return &oldState, nil
+}
+
+// GetState returns the current state of a terminal which may be useful to
+// restore the terminal after a signal.
+func GetState(fd int) (*State, error) {
+	var oldState State
+	if _, _, err := syscall.Syscall6(syscall.SYS_IOCTL, uintptr(fd), ioctlReadTermios, uintptr(unsafe.Pointer(&oldState.termios)), 0, 0, 0); err != 0 {
+		return nil, err
+	}
+
+	return &oldState, nil
+}
+
+// Restore restores the terminal connected to the given file descriptor to a
+// previous state.
+func Restore(fd int, state *State) error {
+	_, _, err := syscall.Syscall6(syscall.SYS_IOCTL, uintptr(fd), ioctlWriteTermios, uintptr(unsafe.Pointer(&state.termios)), 0, 0, 0)
+	return err
+}
+
+// GetSize returns the dimensions of the given terminal.
+func GetSize(fd int) (width, height int, err error) {
+	var dimensions [4]uint16
+
+	if _, _, err := syscall.Syscall6(syscall.SYS_IOCTL, uintptr(fd), uintptr(syscall.TIOCGWINSZ), uintptr(unsafe.Pointer(&dimensions)), 0, 0, 0); err != 0 {
+		return -1, -1, err
+	}
+	return int(dimensions[1]), int(dimensions[0]), nil
+}
+
+// ReadPassword reads a line of input from a terminal without local echo.  This
+// is commonly used for inputting passwords and other sensitive data. The slice
+// returned does not include the \n.
+func ReadPassword(fd int) ([]byte, error) {
+	var oldState syscall.Termios
+	if _, _, err := syscall.Syscall6(syscall.SYS_IOCTL, uintptr(fd), ioctlReadTermios, uintptr(unsafe.Pointer(&oldState)), 0, 0, 0); err != 0 {
+		return nil, err
+	}
+
+	newState := oldState
+	newState.Lflag &^= syscall.ECHO
+	newState.Lflag |= syscall.ICANON | syscall.ISIG
+	newState.Iflag |= syscall.ICRNL
+	if _, _, err := syscall.Syscall6(syscall.SYS_IOCTL, uintptr(fd), ioctlWriteTermios, uintptr(unsafe.Pointer(&newState)), 0, 0, 0); err != 0 {
+		return nil, err
+	}
+
+	defer func() {
+		syscall.Syscall6(syscall.SYS_IOCTL, uintptr(fd), ioctlWriteTermios, uintptr(unsafe.Pointer(&oldState)), 0, 0, 0)
+	}()
+
+	var buf [16]byte
+	var ret []byte
+	for {
+		n, err := syscall.Read(fd, buf[:])
+		if err != nil {
+			return nil, err
+		}
+		if n == 0 {
+			if len(ret) == 0 {
+				return nil, io.EOF
+			}
+			break
+		}
+		if buf[n-1] == '\n' {
+			n--
+		}
+		ret = append(ret, buf[:n]...)
+		if n < len(buf) {
+			break
+		}
+	}
+
+	return ret, nil
+}

vendor/golang.org/x/crypto/ssh/terminal/util_bsd.go

@@ -0,0 +1,12 @@
+// Copyright 2013 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build darwin dragonfly freebsd netbsd openbsd
+
+package terminal
+
+import "syscall"
+
+const ioctlReadTermios = syscall.TIOCGETA
+const ioctlWriteTermios = syscall.TIOCSETA

vendor/golang.org/x/crypto/ssh/terminal/util_linux.go

@@ -0,0 +1,11 @@
+// Copyright 2013 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package terminal
+
+// These constants are declared here, rather than importing
+// them from the syscall package as some syscall packages, even
+// on linux, for example gccgo, do not declare them.
+const ioctlReadTermios = 0x5401  // syscall.TCGETS
+const ioctlWriteTermios = 0x5402 // syscall.TCSETS

vendor/golang.org/x/crypto/ssh/terminal/util_plan9.go

@@ -0,0 +1,58 @@
+// Copyright 2016 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package terminal provides support functions for dealing with terminals, as
+// commonly found on UNIX systems.
+//
+// Putting a terminal into raw mode is the most common requirement:
+//
+// 	oldState, err := terminal.MakeRaw(0)
+// 	if err != nil {
+// 	        panic(err)
+// 	}
+// 	defer terminal.Restore(0, oldState)
+package terminal
+
+import (
+	"fmt"
+	"runtime"
+)
+
+type State struct{}
+
+// IsTerminal returns true if the given file descriptor is a terminal.
+func IsTerminal(fd int) bool {
+	return false
+}
+
+// MakeRaw put the terminal connected to the given file descriptor into raw
+// mode and returns the previous state of the terminal so that it can be
+// restored.
+func MakeRaw(fd int) (*State, error) {
+	return nil, fmt.Errorf("terminal: MakeRaw not implemented on %s/%s", runtime.GOOS, runtime.GOARCH)
+}
+
+// GetState returns the current state of a terminal which may be useful to
+// restore the terminal after a signal.
+func GetState(fd int) (*State, error) {
+	return nil, fmt.Errorf("terminal: GetState not implemented on %s/%s", runtime.GOOS, runtime.GOARCH)
+}
+
+// Restore restores the terminal connected to the given file descriptor to a
+// previous state.
+func Restore(fd int, state *State) error {
+	return fmt.Errorf("terminal: Restore not implemented on %s/%s", runtime.GOOS, runtime.GOARCH)
+}
+
+// GetSize returns the dimensions of the given terminal.
+func GetSize(fd int) (width, height int, err error) {
+	return 0, 0, fmt.Errorf("terminal: GetSize not implemented on %s/%s", runtime.GOOS, runtime.GOARCH)
+}
+
+// ReadPassword reads a line of input from a terminal without local echo.  This
+// is commonly used for inputting passwords and other sensitive data. The slice
+// returned does not include the \n.
+func ReadPassword(fd int) ([]byte, error) {
+	return nil, fmt.Errorf("terminal: ReadPassword not implemented on %s/%s", runtime.GOOS, runtime.GOARCH)
+}

vendor/golang.org/x/crypto/ssh/terminal/util_windows.go

@@ -0,0 +1,174 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build windows
+
+// Package terminal provides support functions for dealing with terminals, as
+// commonly found on UNIX systems.
+//
+// Putting a terminal into raw mode is the most common requirement:
+//
+// 	oldState, err := terminal.MakeRaw(0)
+// 	if err != nil {
+// 	        panic(err)
+// 	}
+// 	defer terminal.Restore(0, oldState)
+package terminal
+
+import (
+	"io"
+	"syscall"
+	"unsafe"
+)
+
+const (
+	enableLineInput       = 2
+	enableEchoInput       = 4
+	enableProcessedInput  = 1
+	enableWindowInput     = 8
+	enableMouseInput      = 16
+	enableInsertMode      = 32
+	enableQuickEditMode   = 64
+	enableExtendedFlags   = 128
+	enableAutoPosition    = 256
+	enableProcessedOutput = 1
+	enableWrapAtEolOutput = 2
+)
+
+var kernel32 = syscall.NewLazyDLL("kernel32.dll")
+
+var (
+	procGetConsoleMode             = kernel32.NewProc("GetConsoleMode")
+	procSetConsoleMode             = kernel32.NewProc("SetConsoleMode")
+	procGetConsoleScreenBufferInfo = kernel32.NewProc("GetConsoleScreenBufferInfo")
+)
+
+type (
+	short int16
+	word  uint16
+
+	coord struct {
+		x short
+		y short
+	}
+	smallRect struct {
+		left   short
+		top    short
+		right  short
+		bottom short
+	}
+	consoleScreenBufferInfo struct {
+		size              coord
+		cursorPosition    coord
+		attributes        word
+		window            smallRect
+		maximumWindowSize coord
+	}
+)
+
+type State struct {
+	mode uint32
+}
+
+// IsTerminal returns true if the given file descriptor is a terminal.
+func IsTerminal(fd int) bool {
+	var st uint32
+	r, _, e := syscall.Syscall(procGetConsoleMode.Addr(), 2, uintptr(fd), uintptr(unsafe.Pointer(&st)), 0)
+	return r != 0 && e == 0
+}
+
+// MakeRaw put the terminal connected to the given file descriptor into raw
+// mode and returns the previous state of the terminal so that it can be
+// restored.
+func MakeRaw(fd int) (*State, error) {
+	var st uint32
+	_, _, e := syscall.Syscall(procGetConsoleMode.Addr(), 2, uintptr(fd), uintptr(unsafe.Pointer(&st)), 0)
+	if e != 0 {
+		return nil, error(e)
+	}
+	raw := st &^ (enableEchoInput | enableProcessedInput | enableLineInput | enableProcessedOutput)
+	_, _, e = syscall.Syscall(procSetConsoleMode.Addr(), 2, uintptr(fd), uintptr(raw), 0)
+	if e != 0 {
+		return nil, error(e)
+	}
+	return &State{st}, nil
+}
+
+// GetState returns the current state of a terminal which may be useful to
+// restore the terminal after a signal.
+func GetState(fd int) (*State, error) {
+	var st uint32
+	_, _, e := syscall.Syscall(procGetConsoleMode.Addr(), 2, uintptr(fd), uintptr(unsafe.Pointer(&st)), 0)
+	if e != 0 {
+		return nil, error(e)
+	}
+	return &State{st}, nil
+}
+
+// Restore restores the terminal connected to the given file descriptor to a
+// previous state.
+func Restore(fd int, state *State) error {
+	_, _, err := syscall.Syscall(procSetConsoleMode.Addr(), 2, uintptr(fd), uintptr(state.mode), 0)
+	return err
+}
+
+// GetSize returns the dimensions of the given terminal.
+func GetSize(fd int) (width, height int, err error) {
+	var info consoleScreenBufferInfo
+	_, _, e := syscall.Syscall(procGetConsoleScreenBufferInfo.Addr(), 2, uintptr(fd), uintptr(unsafe.Pointer(&info)), 0)
+	if e != 0 {
+		return 0, 0, error(e)
+	}
+	return int(info.size.x), int(info.size.y), nil
+}
+
+// ReadPassword reads a line of input from a terminal without local echo.  This
+// is commonly used for inputting passwords and other sensitive data. The slice
+// returned does not include the \n.
+func ReadPassword(fd int) ([]byte, error) {
+	var st uint32
+	_, _, e := syscall.Syscall(procGetConsoleMode.Addr(), 2, uintptr(fd), uintptr(unsafe.Pointer(&st)), 0)
+	if e != 0 {
+		return nil, error(e)
+	}
+	old := st
+
+	st &^= (enableEchoInput)
+	st |= (enableProcessedInput | enableLineInput | enableProcessedOutput)
+	_, _, e = syscall.Syscall(procSetConsoleMode.Addr(), 2, uintptr(fd), uintptr(st), 0)
+	if e != 0 {
+		return nil, error(e)
+	}
+
+	defer func() {
+		syscall.Syscall(procSetConsoleMode.Addr(), 2, uintptr(fd), uintptr(old), 0)
+	}()
+
+	var buf [16]byte
+	var ret []byte
+	for {
+		n, err := syscall.Read(syscall.Handle(fd), buf[:])
+		if err != nil {
+			return nil, err
+		}
+		if n == 0 {
+			if len(ret) == 0 {
+				return nil, io.EOF
+			}
+			break
+		}
+		if buf[n-1] == '\n' {
+			n--
+		}
+		if n > 0 && buf[n-1] == '\r' {
+			n--
+		}
+		ret = append(ret, buf[:n]...)
+		if n < len(buf) {
+			break
+		}
+	}
+
+	return ret, nil
+}