Merge pull request #24943 from aaronlehmann/rolling-updates

Add failure action for rolling updates

api/client/service/inspect.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"io"
 	"strings"
+	"time"
 
 	"golang.org/x/net/context"
 
@@ -101,6 +102,17 @@ func printService(out io.Writer, service swarm.Service) {
 			fmt.Fprintf(out, " Replicas:\t%d\n", *service.Spec.Mode.Replicated.Replicas)
 		}
 	}
+
+	if service.UpdateStatus.State != "" {
+		fmt.Fprintln(out, "Update status:")
+		fmt.Fprintf(out, " State:\t\t%s\n", service.UpdateStatus.State)
+		fmt.Fprintf(out, " Started:\t%s ago\n", strings.ToLower(units.HumanDuration(time.Since(service.UpdateStatus.StartedAt))))
+		if service.UpdateStatus.State == swarm.UpdateStateCompleted {
+			fmt.Fprintf(out, " Completed:\t%s ago\n", strings.ToLower(units.HumanDuration(time.Since(service.UpdateStatus.CompletedAt))))
+		}
+		fmt.Fprintf(out, " Message:\t%s\n", service.UpdateStatus.Message)
+	}
+
 	fmt.Fprintln(out, "Placement:")
 	if service.Spec.TaskTemplate.Placement != nil && len(service.Spec.TaskTemplate.Placement.Constraints) > 0 {
 		ioutils.FprintfIfNotEmpty(out, " Constraints\t: %s\n", strings.Join(service.Spec.TaskTemplate.Placement.Constraints, ", "))
@@ -110,6 +122,7 @@ func printService(out io.Writer, service swarm.Service) {
 	if service.Spec.UpdateConfig.Delay.Nanoseconds() > 0 {
 		fmt.Fprintf(out, " Delay:\t\t%s\n", service.Spec.UpdateConfig.Delay)
 	}
+	fmt.Fprintf(out, " On failure:\t%s\n", service.Spec.UpdateConfig.FailureAction)
 	fmt.Fprintf(out, "ContainerSpec:\n")
 	printContainerSpec(out, service.Spec.TaskTemplate.ContainerSpec)
 

api/client/service/opts.go

@@ -274,6 +274,7 @@ func (m *MountOpt) Value() []swarm.Mount {
 type updateOptions struct {
 	parallelism uint64
 	delay       time.Duration
+	onFailure   string
 }
 
 type resourceOptions struct {
@@ -455,8 +456,9 @@ func (opts *serviceOptions) ToService() (swarm.ServiceSpec, error) {
 		},
 		Mode: swarm.ServiceMode{},
 		UpdateConfig: &swarm.UpdateConfig{
-			Parallelism: opts.update.parallelism,
-			Delay:       opts.update.delay,
+			Parallelism:   opts.update.parallelism,
+			Delay:         opts.update.delay,
+			FailureAction: opts.update.onFailure,
 		},
 		Networks:     convertNetworks(opts.networks),
 		EndpointSpec: opts.endpoint.ToEndpointSpec(),
@@ -503,6 +505,7 @@ func addServiceFlags(cmd *cobra.Command, opts *serviceOptions) {
 
 	flags.Uint64Var(&opts.update.parallelism, flagUpdateParallelism, 1, "Maximum number of tasks updated simultaneously (0 to update all at once)")
 	flags.DurationVar(&opts.update.delay, flagUpdateDelay, time.Duration(0), "Delay between updates")
+	flags.StringVar(&opts.update.onFailure, flagUpdateFailureAction, "pause", "Action on update failure (pause|continue)")
 
 	flags.StringVar(&opts.endpoint.mode, flagEndpointMode, "", "Endpoint mode (vip or dnsrr)")
 
@@ -513,41 +516,42 @@ func addServiceFlags(cmd *cobra.Command, opts *serviceOptions) {
 }
 
 const (
-	flagConstraint         = "constraint"
-	flagConstraintRemove   = "constraint-rm"
-	flagConstraintAdd      = "constraint-add"
-	flagEndpointMode       = "endpoint-mode"
-	flagEnv                = "env"
-	flagEnvRemove          = "env-rm"
-	flagEnvAdd             = "env-add"
-	flagLabel              = "label"
-	flagLabelRemove        = "label-rm"
-	flagLabelAdd           = "label-add"
-	flagLimitCPU           = "limit-cpu"
-	flagLimitMemory        = "limit-memory"
-	flagMode               = "mode"
-	flagMount              = "mount"
-	flagMountRemove        = "mount-rm"
-	flagMountAdd           = "mount-add"
-	flagName               = "name"
-	flagNetwork            = "network"
-	flagNetworkRemove      = "network-rm"
-	flagNetworkAdd         = "network-add"
-	flagPublish            = "publish"
-	flagPublishRemove      = "publish-rm"
-	flagPublishAdd         = "publish-add"
-	flagReplicas           = "replicas"
-	flagReserveCPU         = "reserve-cpu"
-	flagReserveMemory      = "reserve-memory"
-	flagRestartCondition   = "restart-condition"
-	flagRestartDelay       = "restart-delay"
-	flagRestartMaxAttempts = "restart-max-attempts"
-	flagRestartWindow      = "restart-window"
-	flagStopGracePeriod    = "stop-grace-period"
-	flagUpdateDelay        = "update-delay"
-	flagUpdateParallelism  = "update-parallelism"
-	flagUser               = "user"
-	flagRegistryAuth       = "with-registry-auth"
-	flagLogDriver          = "log-driver"
-	flagLogOpt             = "log-opt"
+	flagConstraint          = "constraint"
+	flagConstraintRemove    = "constraint-rm"
+	flagConstraintAdd       = "constraint-add"
+	flagEndpointMode        = "endpoint-mode"
+	flagEnv                 = "env"
+	flagEnvRemove           = "env-rm"
+	flagEnvAdd              = "env-add"
+	flagLabel               = "label"
+	flagLabelRemove         = "label-rm"
+	flagLabelAdd            = "label-add"
+	flagLimitCPU            = "limit-cpu"
+	flagLimitMemory         = "limit-memory"
+	flagMode                = "mode"
+	flagMount               = "mount"
+	flagMountRemove         = "mount-rm"
+	flagMountAdd            = "mount-add"
+	flagName                = "name"
+	flagNetwork             = "network"
+	flagNetworkRemove       = "network-rm"
+	flagNetworkAdd          = "network-add"
+	flagPublish             = "publish"
+	flagPublishRemove       = "publish-rm"
+	flagPublishAdd          = "publish-add"
+	flagReplicas            = "replicas"
+	flagReserveCPU          = "reserve-cpu"
+	flagReserveMemory       = "reserve-memory"
+	flagRestartCondition    = "restart-condition"
+	flagRestartDelay        = "restart-delay"
+	flagRestartMaxAttempts  = "restart-max-attempts"
+	flagRestartWindow       = "restart-window"
+	flagStopGracePeriod     = "stop-grace-period"
+	flagUpdateDelay         = "update-delay"
+	flagUpdateFailureAction = "update-failure-action"
+	flagUpdateParallelism   = "update-parallelism"
+	flagUser                = "user"
+	flagRegistryAuth        = "with-registry-auth"
+	flagLogDriver           = "log-driver"
+	flagLogOpt              = "log-opt"
 )

api/client/service/update.go

@@ -191,12 +191,13 @@ func updateService(flags *pflag.FlagSet, spec *swarm.ServiceSpec) error {
 		return err
 	}
 
-	if anyChanged(flags, flagUpdateParallelism, flagUpdateDelay) {
+	if anyChanged(flags, flagUpdateParallelism, flagUpdateDelay, flagUpdateFailureAction) {
 		if spec.UpdateConfig == nil {
 			spec.UpdateConfig = &swarm.UpdateConfig{}
 		}
 		updateUint64(flagUpdateParallelism, &spec.UpdateConfig.Parallelism)
 		updateDuration(flagUpdateDelay, &spec.UpdateConfig.Delay)
+		updateString(flagUpdateFailureAction, &spec.UpdateConfig.FailureAction)
 	}
 
 	updateNetworks(flags, &spec.Networks)

contrib/completion/bash/docker

@@ -1726,6 +1726,7 @@ _docker_service_update() {
 		--restart-window
 		--stop-grace-period
 		--update-delay
+		--update-failure-action
 		--update-parallelism
 		--user -u
 		--workdir -w

contrib/completion/zsh/_docker

@@ -1094,6 +1094,7 @@ __docker_service_subcommand() {
         "($help)--restart-window=[Window used to evaluate the restart policy]:window: "
         "($help)--stop-grace-period=[Time to wait before force killing a container]:grace period: "
         "($help)--update-delay=[Delay between updates]:delay: "
+        "($help)--update-failure-action=[Action on update failure]:mode:(pause continue)"
         "($help)--update-parallelism=[Maximum number of tasks updated simultaneously]:number: "
         "($help -u --user)"{-u=,--user=}"[Username or UID]:user:_users"
         "($help)--with-registry-auth[Send registry authentication details to swarm agents]"

daemon/cluster/convert/service.go

@@ -53,9 +53,16 @@ func ServiceFromGRPC(s swarmapi.Service) types.Service {
 		}
 
 		service.Spec.UpdateConfig.Delay, _ = ptypes.Duration(&s.Spec.Update.Delay)
+
+		switch s.Spec.Update.FailureAction {
+		case swarmapi.UpdateConfig_PAUSE:
+			service.Spec.UpdateConfig.FailureAction = types.UpdateFailureActionPause
+		case swarmapi.UpdateConfig_CONTINUE:
+			service.Spec.UpdateConfig.FailureAction = types.UpdateFailureActionContinue
+		}
 	}
 
-	//Mode
+	// Mode
 	switch t := s.Spec.GetMode().(type) {
 	case *swarmapi.ServiceSpec_Global:
 		service.Spec.Mode.Global = &types.GlobalService{}
@@ -65,6 +72,23 @@ func ServiceFromGRPC(s swarmapi.Service) types.Service {
 		}
 	}
 
+	// UpdateStatus
+	service.UpdateStatus = types.UpdateStatus{}
+	if s.UpdateStatus != nil {
+		switch s.UpdateStatus.State {
+		case swarmapi.UpdateStatus_UPDATING:
+			service.UpdateStatus.State = types.UpdateStateUpdating
+		case swarmapi.UpdateStatus_PAUSED:
+			service.UpdateStatus.State = types.UpdateStatePaused
+		case swarmapi.UpdateStatus_COMPLETED:
+			service.UpdateStatus.State = types.UpdateStateCompleted
+		}
+
+		service.UpdateStatus.StartedAt, _ = ptypes.Timestamp(s.UpdateStatus.StartedAt)
+		service.UpdateStatus.CompletedAt, _ = ptypes.Timestamp(s.UpdateStatus.CompletedAt)
+		service.UpdateStatus.Message = s.UpdateStatus.Message
+	}
+
 	return service
 }
 
@@ -111,9 +135,19 @@ func ServiceSpecToGRPC(s types.ServiceSpec) (swarmapi.ServiceSpec, error) {
 	}
 
 	if s.UpdateConfig != nil {
+		var failureAction swarmapi.UpdateConfig_FailureAction
+		switch s.UpdateConfig.FailureAction {
+		case types.UpdateFailureActionPause, "":
+			failureAction = swarmapi.UpdateConfig_PAUSE
+		case types.UpdateFailureActionContinue:
+			failureAction = swarmapi.UpdateConfig_CONTINUE
+		default:
+			return swarmapi.ServiceSpec{}, fmt.Errorf("unrecongized update failure action %s", s.UpdateConfig.FailureAction)
+		}
 		spec.Update = &swarmapi.UpdateConfig{
-			Parallelism: s.UpdateConfig.Parallelism,
-			Delay:       *ptypes.DurationProto(s.UpdateConfig.Delay),
+			Parallelism:   s.UpdateConfig.Parallelism,
+			Delay:         *ptypes.DurationProto(s.UpdateConfig.Delay),
+			FailureAction: failureAction,
 		}
 	}
 

docs/reference/api/docker_remote_api_v1.24.md

@@ -3966,7 +3966,8 @@ Create a service
       },
       "UpdateConfig": {
         "Delay": 30000000000.0,
-        "Parallelism": 2
+        "Parallelism": 2,
+        "FailureAction": "pause"
       },
       "EndpointSpec": {
         "Ports": [
@@ -4056,6 +4057,8 @@ JSON Parameters:
     - **Parallelism** – Maximum number of tasks to be updated in one iteration (0 means unlimited
       parallelism).
     - **Delay** – Amount of time between updates.
+    - **FailureAction** - Action to take if an updated task fails to run, or stops running during the
+      update. Values are `continue` and `pause`.
 - **Networks** – Array of network names or IDs to attach the service to.
 - **Endpoint** – Properties that can be configured to access and load balance a service.
     - **Spec** –

docs/reference/api/docker_remote_api_v1.25.md

@@ -3967,7 +3967,8 @@ Create a service
       },
       "UpdateConfig": {
         "Delay": 30000000000.0,
-        "Parallelism": 2
+        "Parallelism": 2,
+        "FailureAction": "pause"
       },
       "EndpointSpec": {
         "Ports": [
@@ -4057,6 +4058,8 @@ JSON Parameters:
     - **Parallelism** – Maximum number of tasks to be updated in one iteration (0 means unlimited
       parallelism).
     - **Delay** – Amount of time between updates.
+    - **FailureAction** - Action to take if an updated task fails to run, or stops running during the
+      update. Values are `continue` and `pause`.
 - **Networks** – Array of network names or IDs to attach the service to.
 - **Endpoint** – Properties that can be configured to access and load balance a service.
     - **Spec** –

docs/reference/commandline/service_create.md

@@ -17,33 +17,34 @@ Usage:  docker service create [OPTIONS] IMAGE [COMMAND] [ARG...]
 Create a new service
 
 Options:
-      --constraint value             Placement constraints (default [])
-      --endpoint-mode string         Endpoint mode (vip or dnsrr)
-  -e, --env value                    Set environment variables (default [])
-      --help                         Print usage
-  -l, --label value                  Service labels (default [])
-      --limit-cpu value              Limit CPUs (default 0.000)
-      --limit-memory value           Limit Memory (default 0 B)
-      --log-driver string            Logging driver for service
-      --log-opt value                Logging driver options (default [])
-      --mode string                  Service mode (replicated or global) (default "replicated")
-      --mount value                  Attach a mount to the service
-      --name string                  Service name
-      --network value                Network attachments (default [])
-  -p, --publish value                Publish a port as a node port (default [])
-      --replicas value               Number of tasks (default none)
-      --reserve-cpu value            Reserve CPUs (default 0.000)
-      --reserve-memory value         Reserve Memory (default 0 B)
-      --restart-condition string     Restart when condition is met (none, on-failure, or any)
-      --restart-delay value          Delay between restart attempts (default none)
-      --restart-max-attempts value   Maximum number of restarts before giving up (default none)
-      --restart-window value         Window used to evaluate the restart policy (default none)
-      --stop-grace-period value      Time to wait before force killing a container (default none)
-      --update-delay duration        Delay between updates
-      --update-parallelism uint      Maximum number of tasks updated simultaneously (0 to update all at once) (default 1)
-  -u, --user string                  Username or UID
-      --with-registry-auth           Send registry authentication details to Swarm agents
-  -w, --workdir string               Working directory inside the container
+      --constraint value               Placement constraints (default [])
+      --endpoint-mode string           Endpoint mode (vip or dnsrr)
+  -e, --env value                      Set environment variables (default [])
+      --help                           Print usage
+  -l, --label value                    Service labels (default [])
+      --limit-cpu value                Limit CPUs (default 0.000)
+      --limit-memory value             Limit Memory (default 0 B)
+      --log-driver string              Logging driver for service
+      --log-opt value                  Logging driver options (default [])
+      --mode string                    Service mode (replicated or global) (default "replicated")
+      --mount value                    Attach a mount to the service
+      --name string                    Service name
+      --network value                  Network attachments (default [])
+  -p, --publish value                  Publish a port as a node port (default [])
+      --replicas value                 Number of tasks (default none)
+      --reserve-cpu value              Reserve CPUs (default 0.000)
+      --reserve-memory value           Reserve Memory (default 0 B)
+      --restart-condition string       Restart when condition is met (none, on-failure, or any)
+      --restart-delay value            Delay between restart attempts (default none)
+      --restart-max-attempts value     Maximum number of restarts before giving up (default none)
+      --restart-window value           Window used to evaluate the restart policy (default none)
+      --stop-grace-period value        Time to wait before force killing a container (default none)
+      --update-delay duration          Delay between updates
+      --update-failure-action string   Action on update failure (pause|continue) (default "pause")
+      --update-parallelism uint        Maximum number of tasks updated simultaneously (0 to update all at once) (default 1)
+  -u, --user string                    Username or UID
+      --with-registry-auth             Send registry authentication details to Swarm agents
+  -w, --workdir string                 Working directory inside the container
 ```
 
 Creates a service as described by the specified parameters. This command has to

docs/reference/commandline/service_update.md

@@ -17,40 +17,41 @@ Usage:  docker service update [OPTIONS] SERVICE
 Update a service
 
 Options:
-      --args string                  Service command args
-      --constraint-add value         Add or update placement constraints (default [])
-      --constraint-rm value          Remove a constraint (default [])
-      --endpoint-mode string         Endpoint mode (vip or dnsrr)
-      --env-add value                Add or update environment variables (default [])
-      --env-rm value                 Remove an environment variable (default [])
-      --help                         Print usage
-      --image string                 Service image tag
-      --label-add value              Add or update service labels (default [])
-      --label-rm value               Remove a label by its key (default [])
-      --limit-cpu value              Limit CPUs (default 0.000)
-      --limit-memory value           Limit Memory (default 0 B)
-      --log-driver string            Logging driver for service
-      --log-opt value                Logging driver options (default [])
-      --mount-add value              Add or update a mount on a service
-      --mount-rm value               Remove a mount by its target path (default [])
-      --name string                  Service name
-      --network-add value            Add or update network attachments (default [])
-      --network-rm value             Remove a network by name (default [])
-      --publish-add value            Add or update a published port (default [])
-      --publish-rm value             Remove a published port by its target port (default [])
-      --replicas value               Number of tasks (default none)
-      --reserve-cpu value            Reserve CPUs (default 0.000)
-      --reserve-memory value         Reserve Memory (default 0 B)
-      --restart-condition string     Restart when condition is met (none, on-failure, or any)
-      --restart-delay value          Delay between restart attempts (default none)
-      --restart-max-attempts value   Maximum number of restarts before giving up (default none)
-      --restart-window value         Window used to evaluate the restart policy (default none)
-      --stop-grace-period value      Time to wait before force killing a container (default none)
-      --update-delay duration        Delay between updates
-      --update-parallelism uint      Maximum number of tasks updated simultaneously (0 to update all at once) (default 1)
-  -u, --user string                  Username or UID
-      --with-registry-auth           Send registry authentication details to Swarm agents
-  -w, --workdir string               Working directory inside the container
+      --args string                    Service command args
+      --constraint-add value           Add or update placement constraints (default [])
+      --constraint-rm value            Remove a constraint (default [])
+      --endpoint-mode string           Endpoint mode (vip or dnsrr)
+      --env-add value                  Add or update environment variables (default [])
+      --env-rm value                   Remove an environment variable (default [])
+      --help                           Print usage
+      --image string                   Service image tag
+      --label-add value                Add or update service labels (default [])
+      --label-rm value                 Remove a label by its key (default [])
+      --limit-cpu value                Limit CPUs (default 0.000)
+      --limit-memory value             Limit Memory (default 0 B)
+      --log-driver string              Logging driver for service
+      --log-opt value                  Logging driver options (default [])
+      --mount-add value                Add or update a mount on a service
+      --mount-rm value                 Remove a mount by its target path (default [])
+      --name string                    Service name
+      --network-add value              Add or update network attachments (default [])
+      --network-rm value               Remove a network by name (default [])
+      --publish-add value              Add or update a published port (default [])
+      --publish-rm value               Remove a published port by its target port (default [])
+      --replicas value                 Number of tasks (default none)
+      --reserve-cpu value              Reserve CPUs (default 0.000)
+      --reserve-memory value           Reserve Memory (default 0 B)
+      --restart-condition string       Restart when condition is met (none, on-failure, or any)
+      --restart-delay value            Delay between restart attempts (default none)
+      --restart-max-attempts value     Maximum number of restarts before giving up (default none)
+      --restart-window value           Window used to evaluate the restart policy (default none)
+      --stop-grace-period value        Time to wait before force killing a container (default none)
+      --update-delay duration          Delay between updates
+      --update-failure-action string   Action on update failure (pause|continue) (default "pause")
+      --update-parallelism uint        Maximum number of tasks updated simultaneously (0 to update all at once) (default 1)
+  -u, --user string                    Username or UID
+      --with-registry-auth             Send registry authentication details to Swarm agents
+  -w, --workdir string                 Working directory inside the container
 ```
 
 Updates a service as described by the specified parameters. This command has to be run targeting a manager node.

hack/vendor.sh

@@ -60,7 +60,7 @@ clone git golang.org/x/net 2beffdc2e92c8a3027590f898fe88f69af48a3f8 https://gith
 clone git golang.org/x/sys eb2c74142fd19a79b3f237334c7384d5167b1b46 https://github.com/golang/sys.git
 clone git github.com/docker/go-units 651fc226e7441360384da338d0fd37f2440ffbe3
 clone git github.com/docker/go-connections fa2850ff103453a9ad190da0df0af134f0314b3d
-clone git github.com/docker/engine-api ebb728a1346926edc2ad9418f9b6045901810b20
+clone git github.com/docker/engine-api 53b6b19ee622c8584c28fdde0e3893383b290da3
 clone git github.com/RackSec/srslog 259aed10dfa74ea2961eddd1d9847619f6e98837
 clone git github.com/imdario/mergo 0.2.1
 

integration-cli/docker_api_swarm_test.go

@@ -791,8 +791,9 @@ func serviceForUpdate(s *swarm.Service) {
 			},
 		},
 		UpdateConfig: &swarm.UpdateConfig{
-			Parallelism: 2,
-			Delay:       8 * time.Second,
+			Parallelism:   2,
+			Delay:         8 * time.Second,
+			FailureAction: swarm.UpdateFailureActionContinue,
 		},
 	}
 	s.Spec.Name = "updatetest"

vendor/src/github.com/docker/engine-api/types/swarm/service.go

@@ -6,8 +6,9 @@ import "time"
 type Service struct {
 	ID string
 	Meta
-	Spec     ServiceSpec `json:",omitempty"`
-	Endpoint Endpoint    `json:",omitempty"`
+	Spec         ServiceSpec  `json:",omitempty"`
+	Endpoint     Endpoint     `json:",omitempty"`
+	UpdateStatus UpdateStatus `json:",omitempty"`
 }
 
 // ServiceSpec represents the spec of a service.
@@ -29,6 +30,26 @@ type ServiceMode struct {
 	Global     *GlobalService     `json:",omitempty"`
 }
 
+// UpdateState is the state of a service update.
+type UpdateState string
+
+const (
+	// UpdateStateUpdating is the updating state.
+	UpdateStateUpdating UpdateState = "updating"
+	// UpdateStatePaused is the paused state.
+	UpdateStatePaused UpdateState = "paused"
+	// UpdateStateCompleted is the completed state.
+	UpdateStateCompleted UpdateState = "completed"
+)
+
+// UpdateStatus reports the status of a service update.
+type UpdateStatus struct {
+	State       UpdateState `json:",omitempty"`
+	StartedAt   time.Time   `json:",omitempty"`
+	CompletedAt time.Time   `json:",omitempty"`
+	Message     string      `json:",omitempty"`
+}
+
 // ReplicatedService is a kind of ServiceMode.
 type ReplicatedService struct {
 	Replicas *uint64 `json:",omitempty"`
@@ -37,8 +58,16 @@ type ReplicatedService struct {
 // GlobalService is a kind of ServiceMode.
 type GlobalService struct{}
 
+const (
+	// UpdateFailureActionPause PAUSE
+	UpdateFailureActionPause = "pause"
+	// UpdateFailureActionContinue CONTINUE
+	UpdateFailureActionContinue = "continue"
+)
+
 // UpdateConfig represents the update configuration.
 type UpdateConfig struct {
-	Parallelism uint64        `json:",omitempty"`
-	Delay       time.Duration `json:",omitempty"`
+	Parallelism   uint64        `json:",omitempty"`
+	Delay         time.Duration `json:",omitempty"`
+	FailureAction string        `json:",omitempty"`
 }