Update the swarm cluster lock/unlock tests to be less flakey.

Signed-off-by: cyli <cyli@twistedmatrix.com>

integration-cli/docker_cli_swarm_test.go

@@ -884,6 +884,36 @@ func getNodeStatus(c *check.C, d *SwarmDaemon) swarm.LocalNodeState {
 	return info.LocalNodeState
 }
 
+func checkSwarmLockedToUnlocked(c *check.C, d *SwarmDaemon, unlockKey string) {
+	c.Assert(d.Restart(), checker.IsNil)
+	status := getNodeStatus(c, d)
+	if status == swarm.LocalNodeStateLocked {
+		// it must not have updated to be unlocked in time - unlock, wait 3 seconds, and try again
+		cmd := d.command("swarm", "unlock")
+		cmd.Stdin = bytes.NewBufferString(unlockKey)
+		out, err := cmd.CombinedOutput()
+		c.Assert(err, checker.IsNil, check.Commentf("out: %v", string(out)))
+
+		c.Assert(getNodeStatus(c, d), checker.Equals, swarm.LocalNodeStateActive)
+
+		time.Sleep(3 * time.Second)
+		c.Assert(d.Restart(), checker.IsNil)
+	}
+
+	c.Assert(getNodeStatus(c, d), checker.Equals, swarm.LocalNodeStateActive)
+}
+
+func checkSwarmUnlockedToLocked(c *check.C, d *SwarmDaemon) {
+	c.Assert(d.Restart(), checker.IsNil)
+	status := getNodeStatus(c, d)
+	if status == swarm.LocalNodeStateActive {
+		// it must not have updated to be unlocked in time - wait 3 seconds, and try again
+		time.Sleep(3 * time.Second)
+		c.Assert(d.Restart(), checker.IsNil)
+	}
+	c.Assert(getNodeStatus(c, d), checker.Equals, swarm.LocalNodeStateLocked)
+}
+
 func (s *DockerSwarmSuite) TestSwarmInitLocked(c *check.C) {
 	d := s.AddDaemon(c, false, false)
 
@@ -907,8 +937,8 @@ func (s *DockerSwarmSuite) TestSwarmInitLocked(c *check.C) {
 
 	c.Assert(getNodeStatus(c, d), checker.Equals, swarm.LocalNodeStateActive)
 
+	// It starts off locked
 	c.Assert(d.Restart(), checker.IsNil)
-
 	c.Assert(getNodeStatus(c, d), checker.Equals, swarm.LocalNodeStateLocked)
 
 	cmd := d.command("swarm", "unlock")
@@ -933,23 +963,7 @@ func (s *DockerSwarmSuite) TestSwarmInitLocked(c *check.C) {
 	outs, err = d.Cmd("swarm", "update", "--autolock=false")
 	c.Assert(err, checker.IsNil, check.Commentf("out: %v", outs))
 
-	// Wait for autolock to be turned off
-	c.Assert(d.Restart(), checker.IsNil)
-	status := getNodeStatus(c, d)
-	if status == swarm.LocalNodeStateLocked {
-		// it must not have updated in time - unlock, wait 3 seconds, and try again
-		cmd := d.command("swarm", "unlock")
-		cmd.Stdin = bytes.NewBufferString(unlockKey)
-		out, err := cmd.CombinedOutput()
-		c.Assert(err, checker.IsNil, check.Commentf("out: %v", string(out)))
-
-		c.Assert(getNodeStatus(c, d), checker.Equals, swarm.LocalNodeStateActive)
-
-		time.Sleep(3 * time.Second)
-		c.Assert(d.Restart(), checker.IsNil)
-	}
-
-	c.Assert(getNodeStatus(c, d), checker.Equals, swarm.LocalNodeStateActive)
+	checkSwarmLockedToUnlocked(c, d, unlockKey)
 
 	outs, err = d.Cmd("node", "ls")
 	c.Assert(err, checker.IsNil)
@@ -962,6 +976,7 @@ func (s *DockerSwarmSuite) TestSwarmLeaveLocked(c *check.C) {
 	outs, err := d.Cmd("swarm", "init", "--autolock")
 	c.Assert(err, checker.IsNil, check.Commentf("out: %v", outs))
 
+	// It starts off locked
 	c.Assert(d.Restart("--swarm-default-advertise-addr=lo"), checker.IsNil)
 
 	info, err := d.info()
@@ -1019,8 +1034,7 @@ func (s *DockerSwarmSuite) TestSwarmLockUnlockCluster(c *check.C) {
 
 	// The ones that got the cluster update should be set to locked
 	for _, d := range []*SwarmDaemon{d1, d3} {
-		c.Assert(d.Restart(), checker.IsNil)
-		c.Assert(getNodeStatus(c, d), checker.Equals, swarm.LocalNodeStateLocked)
+		checkSwarmUnlockedToLocked(c, d)
 
 		cmd := d.command("swarm", "unlock")
 		cmd.Stdin = bytes.NewBufferString(unlockKey)
@@ -1034,14 +1048,7 @@ func (s *DockerSwarmSuite) TestSwarmLockUnlockCluster(c *check.C) {
 	c.Assert(getNodeStatus(c, d2), checker.Equals, swarm.LocalNodeStateActive)
 
 	// d2 is now set to lock
-	c.Assert(d2.Restart(), checker.IsNil)
-	status := getNodeStatus(c, d2)
-	if status == swarm.LocalNodeStateActive {
-		// it must not have updated in time - wait 3 seconds, and try again
-		time.Sleep(3 * time.Second)
-		c.Assert(d2.Restart(), checker.IsNil)
-	}
-	c.Assert(getNodeStatus(c, d2), checker.Equals, swarm.LocalNodeStateLocked)
+	checkSwarmUnlockedToLocked(c, d2)
 
 	// leave it locked, and set the cluster to no longer autolock
 	outs, err = d1.Cmd("swarm", "update", "--autolock=false")
@@ -1049,23 +1056,7 @@ func (s *DockerSwarmSuite) TestSwarmLockUnlockCluster(c *check.C) {
 
 	// the ones that got the update are now set to unlocked
 	for _, d := range []*SwarmDaemon{d1, d3} {
-		c.Assert(d.Restart(), checker.IsNil)
-		status := getNodeStatus(c, d)
-		if status == swarm.LocalNodeStateLocked {
-			// it must not have updated to be unlocked in time - unlock, wait 3 seconds, and try again
-			cmd := d.command("swarm", "unlock")
-			cmd.Stdin = bytes.NewBufferString(unlockKey)
-			out, err := cmd.CombinedOutput()
-			c.Assert(err, checker.IsNil, check.Commentf("out: %v", string(out)))
-
-			c.Assert(getNodeStatus(c, d), checker.Equals, swarm.LocalNodeStateActive)
-
-			time.Sleep(3 * time.Second)
-			c.Assert(d.Restart(), checker.IsNil)
-		}
-
-		c.Assert(d.Restart(), checker.IsNil)
-		c.Assert(getNodeStatus(c, d), checker.Equals, swarm.LocalNodeStateActive)
+		checkSwarmLockedToUnlocked(c, d, unlockKey)
 	}
 
 	// d2 still locked
@@ -1079,10 +1070,9 @@ func (s *DockerSwarmSuite) TestSwarmLockUnlockCluster(c *check.C) {
 	c.Assert(getNodeStatus(c, d2), checker.Equals, swarm.LocalNodeStateActive)
 
 	// once it's caught up, d2 is set to not be locked
-	c.Assert(d2.Restart(), checker.IsNil)
-	c.Assert(getNodeStatus(c, d2), checker.Equals, swarm.LocalNodeStateActive)
+	checkSwarmLockedToUnlocked(c, d2, unlockKey)
 
-	// managers who join now are also unlocked
+	// managers who join now are never set to locked in the first place
 	d4 := s.AddDaemon(c, true, true)
 	c.Assert(d4.Restart(), checker.IsNil)
 	c.Assert(getNodeStatus(c, d4), checker.Equals, swarm.LocalNodeStateActive)
@@ -1125,14 +1115,7 @@ func (s *DockerSwarmSuite) TestSwarmJoinPromoteLocked(c *check.C) {
 
 	// both new nodes are locked
 	for _, d := range []*SwarmDaemon{d2, d3} {
-		c.Assert(d.Restart(), checker.IsNil)
-		status := getNodeStatus(c, d)
-		if status == swarm.LocalNodeStateActive {
-			// it must not have updated in time - wait 3 seconds, and try again
-			time.Sleep(3 * time.Second)
-			c.Assert(d.Restart(), checker.IsNil)
-		}
-		c.Assert(getNodeStatus(c, d), checker.Equals, swarm.LocalNodeStateLocked)
+		checkSwarmUnlockedToLocked(c, d)
 
 		cmd := d.command("swarm", "unlock")
 		cmd.Stdin = bytes.NewBufferString(unlockKey)
@@ -1154,21 +1137,7 @@ func (s *DockerSwarmSuite) TestSwarmJoinPromoteLocked(c *check.C) {
 	columns := strings.Fields(lines[len(lines)-1])
 	c.Assert(columns, checker.HasLen, 4) // if it was a manager it'd have a manager status field
 
-	c.Assert(d3.Restart(), checker.IsNil)
-	status := getNodeStatus(c, d3)
-	if status == swarm.LocalNodeStateLocked {
-		// it must not have updated in time - unlock, wait 3 seconds, and try again
-		cmd := d3.command("swarm", "unlock")
-		cmd.Stdin = bytes.NewBufferString(unlockKey)
-		out, err := cmd.CombinedOutput()
-		c.Assert(err, checker.IsNil, check.Commentf("out: %v", string(out)))
-
-		c.Assert(getNodeStatus(c, d3), checker.Equals, swarm.LocalNodeStateActive)
-
-		time.Sleep(3 * time.Second)
-		c.Assert(d3.Restart(), checker.IsNil)
-	}
-	c.Assert(getNodeStatus(c, d3), checker.Equals, swarm.LocalNodeStateActive)
+	checkSwarmLockedToUnlocked(c, d3, unlockKey)
 }
 
 func (s *DockerSwarmSuite) TestSwarmRotateUnlockKey(c *check.C) {
@@ -1354,16 +1323,7 @@ func (s *DockerSwarmSuite) TestSwarmAlternateLockUnlock(c *check.C) {
 		}
 
 		c.Assert(unlockKey, checker.Not(checker.Equals), "")
-
-		c.Assert(d.Restart(), checker.IsNil)
-		status := getNodeStatus(c, d)
-		if status == swarm.LocalNodeStateActive {
-			// hasn't updated yet - wait and try again
-			time.Sleep(3 * time.Second)
-			c.Assert(d.Restart(), checker.IsNil)
-			status = getNodeStatus(c, d)
-		}
-		c.Assert(status, checker.Equals, swarm.LocalNodeStateLocked)
+		checkSwarmUnlockedToLocked(c, d)
 
 		cmd := d.command("swarm", "unlock")
 		cmd.Stdin = bytes.NewBufferString(unlockKey)
@@ -1375,22 +1335,7 @@ func (s *DockerSwarmSuite) TestSwarmAlternateLockUnlock(c *check.C) {
 		outs, err = d.Cmd("swarm", "update", "--autolock=false")
 		c.Assert(err, checker.IsNil, check.Commentf("out: %v", outs))
 
-		c.Assert(d.Restart(), checker.IsNil)
-		status = getNodeStatus(c, d)
-		if status == swarm.LocalNodeStateLocked {
-			// it must not have updated to be unlocked in time - unlock, wait 3 seconds, and try again
-			cmd := d.command("swarm", "unlock")
-			cmd.Stdin = bytes.NewBufferString(unlockKey)
-			out, err := cmd.CombinedOutput()
-			c.Assert(err, checker.IsNil, check.Commentf("out: %v", string(out)))
-
-			c.Assert(getNodeStatus(c, d), checker.Equals, swarm.LocalNodeStateActive)
-
-			time.Sleep(3 * time.Second)
-			c.Assert(d.Restart(), checker.IsNil)
-			status = getNodeStatus(c, d)
-		}
-		c.Assert(getNodeStatus(c, d), checker.Equals, swarm.LocalNodeStateActive)
+		checkSwarmLockedToUnlocked(c, d, unlockKey)
 	}
 }