Retry service updates on out of sequence errors

Code retrying service update operations when receiving "update out of
sequence" errors was removed because of a misunderstanding, which has
made tests flaky. This re-adds the "CmdRetryOutOfSequence" method, and
uses it in TestSwarmPublishAdd to avoid flaky behavior.

Signed-off-by: Drew Erny <drew.erny@docker.com>

integration-cli/daemon/daemon_swarm.go

@@ -189,3 +189,25 @@ func (d *Daemon) CheckLeader(c *check.C) (interface{}, check.CommentInterface) {
 	}
 	return fmt.Errorf("no leader"), check.Commentf("could not find leader")
 }
+
+// CmdRetryOutOfSequence tries the specified command against the current daemon
+// up to 10 times, retrying if it encounters an "update out of sequence" error.
+func (d *Daemon) CmdRetryOutOfSequence(args ...string) (string, error) {
+	var (
+		output string
+		err    error
+	)
+
+	for i := 0; i < 10; i++ {
+		output, err = d.Cmd(args...)
+		// error, no error, whatever. if we don't have "update out of
+		// sequence", we don't retry, we just return.
+		if !strings.Contains(output, "update out of sequence") {
+			return output, err
+		}
+	}
+
+	// otherwise, once all of our attempts have been exhausted, just return
+	// whatever the last values were.
+	return output, err
+}

integration-cli/docker_cli_swarm_test.go

@@ -277,19 +277,23 @@ func (s *DockerSwarmSuite) TestSwarmPublishAdd(c *check.C) {
 	d := s.AddDaemon(c, true, true)
 
 	name := "top"
+	// this first command does not have to be retried because service creates
+	// don't return out of sequence errors.
 	out, err := d.Cmd("service", "create", "--detach", "--no-resolve-image", "--name", name, "--label", "x=y", "busybox", "top")
 	assert.NilError(c, err, out)
 	assert.Assert(c, strings.TrimSpace(out) != "")
 
-	out, err = d.Cmd("service", "update", "--detach", "--publish-add", "80:80", name)
+	out, err = d.CmdRetryOutOfSequence("service", "update", "--detach", "--publish-add", "80:80", name)
 	assert.NilError(c, err, out)
 
-	out, err = d.Cmd("service", "update", "--detach", "--publish-add", "80:80", name)
+	out, err = d.CmdRetryOutOfSequence("service", "update", "--detach", "--publish-add", "80:80", name)
 	assert.NilError(c, err, out)
 
-	_, err = d.Cmd("service", "update", "--detach", "--publish-add", "80:80", "--publish-add", "80:20", name)
+	_, err = d.CmdRetryOutOfSequence("service", "update", "--detach", "--publish-add", "80:80", "--publish-add", "80:20", name)
 	assert.ErrorContains(c, err, "")
 
+	// this last command does not have to be retried because service inspect
+	// does not return out of sequence errors.
 	out, err = d.Cmd("service", "inspect", "--format", "{{ .Spec.EndpointSpec.Ports }}", name)
 	assert.NilError(c, err, out)
 	assert.Equal(c, strings.TrimSpace(out), "[{ tcp 80 80 ingress}]")