Merge pull request #38127 from kolyshkin/int-fixes

Integration tests fixes and cleanups

integration-cli/check_test.go

@@ -333,7 +333,7 @@ func (s *DockerSwarmSuite) AddDaemon(c *check.C, joinSwarm, manager bool) *daemo
 			d.StartAndSwarmInit(c)
 		}
 	} else {
-		d.StartWithBusybox(c, "--iptables=false", "--swarm-default-advertise-addr=lo")
+		d.StartNode(c)
 	}
 
 	s.portIndex++

integration-cli/docker_api_swarm_node_test.go

@@ -62,7 +62,7 @@ func (s *DockerSwarmSuite) TestAPISwarmNodeRemove(c *check.C) {
 	c.Assert(len(nodes), checker.Equals, 2, check.Commentf("nodes: %#v", nodes))
 
 	// Restart the node that was removed
-	d2.Restart(c)
+	d2.RestartNode(c)
 
 	// Give some time for the node to rejoin
 	time.Sleep(1 * time.Second)

integration-cli/docker_api_swarm_test.go

@@ -66,8 +66,8 @@ func (s *DockerSwarmSuite) TestAPISwarmInit(c *check.C) {
 	d1.Stop(c)
 	d2.Stop(c)
 
-	d1.Start(c)
+	d1.StartNode(c)
-	d2.Start(c)
+	d2.StartNode(c)
 
 	info = d1.SwarmInfo(c)
 	c.Assert(info.ControlAvailable, checker.True)
@@ -332,6 +332,7 @@ func (s *DockerSwarmSuite) TestAPISwarmLeaderElection(c *check.C) {
 	}
 
 	// wait for an election to occur
+	c.Logf("Waiting for election to occur...")
 	waitAndAssert(c, defaultReconciliationTimeout, checkLeader(d2, d3), checker.True)
 
 	// assert that we have a new leader
@@ -341,11 +342,10 @@ func (s *DockerSwarmSuite) TestAPISwarmLeaderElection(c *check.C) {
 	stableleader := leader
 
 	// add the d1, the initial leader, back
-	d1.Start(c)
+	d1.StartNode(c)
-
-	// TODO(stevvooe): may need to wait for rejoin here
 
 	// wait for possible election
+	c.Logf("Waiting for possible election...")
 	waitAndAssert(c, defaultReconciliationTimeout, checkLeader(d1, d2, d3), checker.True)
 	// pick out the leader and the followers again
 
@@ -387,7 +387,7 @@ func (s *DockerSwarmSuite) TestAPISwarmRaftQuorum(c *check.C) {
 		return err.Error(), nil
 	}, checker.Contains, "Make sure more than half of the managers are online.")
 
-	d2.Start(c)
+	d2.StartNode(c)
 
 	// make sure there is a leader
 	waitAndAssert(c, defaultReconciliationTimeout, d1.CheckLeader, checker.IsNil)
@@ -463,8 +463,7 @@ func (s *DockerSwarmSuite) TestAPISwarmRestoreOnPendingJoin(c *check.C) {
 
 	waitAndAssert(c, defaultReconciliationTimeout, d.CheckLocalNodeState, checker.Equals, swarm.LocalNodeStatePending)
 
-	d.Stop(c)
+	d.RestartNode(c)
-	d.Start(c)
 
 	info := d.SwarmInfo(c)
 	c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateInactive)
@@ -477,25 +476,22 @@ func (s *DockerSwarmSuite) TestAPISwarmManagerRestore(c *check.C) {
 	id := d1.CreateService(c, simpleTestService, setInstances(instances))
 
 	d1.GetService(c, id)
-	d1.Stop(c)
+	d1.RestartNode(c)
-	d1.Start(c)
 	d1.GetService(c, id)
 
 	d2 := s.AddDaemon(c, true, true)
 	d2.GetService(c, id)
-	d2.Stop(c)
+	d2.RestartNode(c)
-	d2.Start(c)
 	d2.GetService(c, id)
 
 	d3 := s.AddDaemon(c, true, true)
 	d3.GetService(c, id)
-	d3.Stop(c)
+	d3.RestartNode(c)
-	d3.Start(c)
 	d3.GetService(c, id)
 
 	d3.Kill()
 	time.Sleep(1 * time.Second) // time to handle signal
-	d3.Start(c)
+	d3.StartNode(c)
 	d3.GetService(c, id)
 }
 

integration-cli/docker_cli_swarm_test.go

@@ -163,7 +163,7 @@ func (s *DockerSwarmSuite) TestSwarmIncompatibleDaemon(c *check.C) {
 	c.Assert(err, checker.IsNil)
 	c.Assert(string(content), checker.Contains, "--live-restore daemon configuration is incompatible with swarm mode")
 	// restart for teardown
-	d.Start(c)
+	d.StartNode(c)
 }
 
 func (s *DockerSwarmSuite) TestSwarmServiceTemplatingHostname(c *check.C) {
@@ -330,7 +330,7 @@ func (s *DockerSwarmSuite) TestSwarmContainerAutoStart(c *check.C) {
 	c.Assert(err, checker.IsNil, check.Commentf("%s", out))
 	c.Assert(strings.TrimSpace(out), checker.Not(checker.Equals), "")
 
-	d.Restart(c)
+	d.RestartNode(c)
 
 	out, err = d.Cmd("ps", "-q")
 	c.Assert(err, checker.IsNil, check.Commentf("%s", out))
@@ -1009,11 +1009,11 @@ func checkKeyIsEncrypted(d *daemon.Daemon) func(*check.C) (interface{}, check.Co
 	}
 }
 
-func checkSwarmLockedToUnlocked(c *check.C, d *daemon.Daemon, unlockKey string) {
+func checkSwarmLockedToUnlocked(c *check.C, d *daemon.Daemon) {
 	// Wait for the PEM file to become unencrypted
 	waitAndAssert(c, defaultReconciliationTimeout, checkKeyIsEncrypted(d), checker.Equals, false)
 
-	d.Restart(c)
+	d.RestartNode(c)
 	c.Assert(getNodeStatus(c, d), checker.Equals, swarm.LocalNodeStateActive)
 }
 
@@ -1021,7 +1021,7 @@ func checkSwarmUnlockedToLocked(c *check.C, d *daemon.Daemon) {
 	// Wait for the PEM file to become encrypted
 	waitAndAssert(c, defaultReconciliationTimeout, checkKeyIsEncrypted(d), checker.Equals, true)
 
-	d.Restart(c)
+	d.RestartNode(c)
 	c.Assert(getNodeStatus(c, d), checker.Equals, swarm.LocalNodeStateLocked)
 }
 
@@ -1055,27 +1055,12 @@ func (s *DockerSwarmSuite) TestSwarmInitLocked(c *check.C) {
 
 	outs, err := d.Cmd("swarm", "init", "--autolock")
 	c.Assert(err, checker.IsNil, check.Commentf("%s", outs))
-
+	unlockKey := getUnlockKey(d, c, outs)
-	c.Assert(outs, checker.Contains, "docker swarm unlock")
-
-	var unlockKey string
-	for _, line := range strings.Split(outs, "\n") {
-		if strings.Contains(line, "SWMKEY") {
-			unlockKey = strings.TrimSpace(line)
-			break
-		}
-	}
-
-	c.Assert(unlockKey, checker.Not(checker.Equals), "")
-
-	outs, err = d.Cmd("swarm", "unlock-key", "-q")
-	c.Assert(err, checker.IsNil, check.Commentf("%s", outs))
-	c.Assert(outs, checker.Equals, unlockKey+"\n")
 
 	c.Assert(getNodeStatus(c, d), checker.Equals, swarm.LocalNodeStateActive)
 
 	// It starts off locked
-	d.Restart(c)
+	d.RestartNode(c)
 	c.Assert(getNodeStatus(c, d), checker.Equals, swarm.LocalNodeStateLocked)
 
 	cmd := d.Command("swarm", "unlock")
@@ -1100,7 +1085,7 @@ func (s *DockerSwarmSuite) TestSwarmInitLocked(c *check.C) {
 	outs, err = d.Cmd("swarm", "update", "--autolock=false")
 	c.Assert(err, checker.IsNil, check.Commentf("%s", outs))
 
-	checkSwarmLockedToUnlocked(c, d, unlockKey)
+	checkSwarmLockedToUnlocked(c, d)
 
 	outs, err = d.Cmd("node", "ls")
 	c.Assert(err, checker.IsNil, check.Commentf("%s", outs))
@@ -1114,7 +1099,7 @@ func (s *DockerSwarmSuite) TestSwarmLeaveLocked(c *check.C) {
 	c.Assert(err, checker.IsNil, check.Commentf("%s", outs))
 
 	// It starts off locked
-	d.Restart(c, "--swarm-default-advertise-addr=lo")
+	d.RestartNode(c)
 
 	info := d.SwarmInfo(c)
 	c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateLocked)
@@ -1146,7 +1131,7 @@ func (s *DockerSwarmSuite) TestSwarmLockUnlockCluster(c *check.C) {
 	d3 := s.AddDaemon(c, true, true)
 
 	// they start off unlocked
-	d2.Restart(c)
+	d2.RestartNode(c)
 	c.Assert(getNodeStatus(c, d2), checker.Equals, swarm.LocalNodeStateActive)
 
 	// stop this one so it does not get autolock info
@@ -1155,22 +1140,7 @@ func (s *DockerSwarmSuite) TestSwarmLockUnlockCluster(c *check.C) {
 	// enable autolock
 	outs, err := d1.Cmd("swarm", "update", "--autolock")
 	c.Assert(err, checker.IsNil, check.Commentf("%s", outs))
-
+	unlockKey := getUnlockKey(d1, c, outs)
-	c.Assert(outs, checker.Contains, "docker swarm unlock")
-
-	var unlockKey string
-	for _, line := range strings.Split(outs, "\n") {
-		if strings.Contains(line, "SWMKEY") {
-			unlockKey = strings.TrimSpace(line)
-			break
-		}
-	}
-
-	c.Assert(unlockKey, checker.Not(checker.Equals), "")
-
-	outs, err = d1.Cmd("swarm", "unlock-key", "-q")
-	c.Assert(err, checker.IsNil)
-	c.Assert(outs, checker.Equals, unlockKey+"\n")
 
 	// The ones that got the cluster update should be set to locked
 	for _, d := range []*daemon.Daemon{d1, d3} {
@@ -1183,7 +1153,7 @@ func (s *DockerSwarmSuite) TestSwarmLockUnlockCluster(c *check.C) {
 	}
 
 	// d2 never got the cluster update, so it is still set to unlocked
-	d2.Start(c)
+	d2.StartNode(c)
 	c.Assert(getNodeStatus(c, d2), checker.Equals, swarm.LocalNodeStateActive)
 
 	// d2 is now set to lock
@@ -1195,7 +1165,7 @@ func (s *DockerSwarmSuite) TestSwarmLockUnlockCluster(c *check.C) {
 
 	// the ones that got the update are now set to unlocked
 	for _, d := range []*daemon.Daemon{d1, d3} {
-		checkSwarmLockedToUnlocked(c, d, unlockKey)
+		checkSwarmLockedToUnlocked(c, d)
 	}
 
 	// d2 still locked
@@ -1208,11 +1178,11 @@ func (s *DockerSwarmSuite) TestSwarmLockUnlockCluster(c *check.C) {
 	c.Assert(getNodeStatus(c, d2), checker.Equals, swarm.LocalNodeStateActive)
 
 	// once it's caught up, d2 is set to not be locked
-	checkSwarmLockedToUnlocked(c, d2, unlockKey)
+	checkSwarmLockedToUnlocked(c, d2)
 
 	// managers who join now are never set to locked in the first place
 	d4 := s.AddDaemon(c, true, true)
-	d4.Restart(c)
+	d4.RestartNode(c)
 	c.Assert(getNodeStatus(c, d4), checker.Equals, swarm.LocalNodeStateActive)
 }
 
@@ -1222,26 +1192,11 @@ func (s *DockerSwarmSuite) TestSwarmJoinPromoteLocked(c *check.C) {
 	// enable autolock
 	outs, err := d1.Cmd("swarm", "update", "--autolock")
 	c.Assert(err, checker.IsNil, check.Commentf("out: %v", outs))
-
+	unlockKey := getUnlockKey(d1, c, outs)
-	c.Assert(outs, checker.Contains, "docker swarm unlock")
-
-	var unlockKey string
-	for _, line := range strings.Split(outs, "\n") {
-		if strings.Contains(line, "SWMKEY") {
-			unlockKey = strings.TrimSpace(line)
-			break
-		}
-	}
-
-	c.Assert(unlockKey, checker.Not(checker.Equals), "")
-
-	outs, err = d1.Cmd("swarm", "unlock-key", "-q")
-	c.Assert(err, checker.IsNil)
-	c.Assert(outs, checker.Equals, unlockKey+"\n")
 
 	// joined workers start off unlocked
 	d2 := s.AddDaemon(c, true, false)
-	d2.Restart(c)
+	d2.RestartNode(c)
 	c.Assert(getNodeStatus(c, d2), checker.Equals, swarm.LocalNodeStateActive)
 
 	// promote worker
@@ -1286,7 +1241,7 @@ func (s *DockerSwarmSuite) TestSwarmJoinPromoteLocked(c *check.C) {
 	}, checker.Equals, "swarm-worker")
 
 	// by now, it should *never* be locked on restart
-	d3.Restart(c)
+	d3.RestartNode(c)
 	c.Assert(getNodeStatus(c, d3), checker.Equals, swarm.LocalNodeStateActive)
 }
 
@@ -1295,22 +1250,7 @@ func (s *DockerSwarmSuite) TestSwarmRotateUnlockKey(c *check.C) {
 
 	outs, err := d.Cmd("swarm", "update", "--autolock")
 	c.Assert(err, checker.IsNil, check.Commentf("out: %v", outs))
-
+	unlockKey := getUnlockKey(d, c, outs)
-	c.Assert(outs, checker.Contains, "docker swarm unlock")
-
-	var unlockKey string
-	for _, line := range strings.Split(outs, "\n") {
-		if strings.Contains(line, "SWMKEY") {
-			unlockKey = strings.TrimSpace(line)
-			break
-		}
-	}
-
-	c.Assert(unlockKey, checker.Not(checker.Equals), "")
-
-	outs, err = d.Cmd("swarm", "unlock-key", "-q")
-	c.Assert(err, checker.IsNil)
-	c.Assert(outs, checker.Equals, unlockKey+"\n")
 
 	// Rotate multiple times
 	for i := 0; i != 3; i++ {
@@ -1321,7 +1261,7 @@ func (s *DockerSwarmSuite) TestSwarmRotateUnlockKey(c *check.C) {
 		c.Assert(newUnlockKey, checker.Not(checker.Equals), "")
 		c.Assert(newUnlockKey, checker.Not(checker.Equals), unlockKey)
 
-		d.Restart(c)
+		d.RestartNode(c)
 		c.Assert(getNodeStatus(c, d), checker.Equals, swarm.LocalNodeStateLocked)
 
 		outs, _ = d.Cmd("node", "ls")
@@ -1342,7 +1282,7 @@ func (s *DockerSwarmSuite) TestSwarmRotateUnlockKey(c *check.C) {
 
 			time.Sleep(3 * time.Second)
 
-			d.Restart(c)
+			d.RestartNode(c)
 
 			cmd = d.Command("swarm", "unlock")
 			cmd.Stdin = bytes.NewBufferString(unlockKey)
@@ -1380,22 +1320,7 @@ func (s *DockerSwarmSuite) TestSwarmClusterRotateUnlockKey(c *check.C) {
 
 	outs, err := d1.Cmd("swarm", "update", "--autolock")
 	c.Assert(err, checker.IsNil, check.Commentf("%s", outs))
-
+	unlockKey := getUnlockKey(d1, c, outs)
-	c.Assert(outs, checker.Contains, "docker swarm unlock")
-
-	var unlockKey string
-	for _, line := range strings.Split(outs, "\n") {
-		if strings.Contains(line, "SWMKEY") {
-			unlockKey = strings.TrimSpace(line)
-			break
-		}
-	}
-
-	c.Assert(unlockKey, checker.Not(checker.Equals), "")
-
-	outs, err = d1.Cmd("swarm", "unlock-key", "-q")
-	c.Assert(err, checker.IsNil, check.Commentf("%s", outs))
-	c.Assert(outs, checker.Equals, unlockKey+"\n")
 
 	// Rotate multiple times
 	for i := 0; i != 3; i++ {
@@ -1406,8 +1331,8 @@ func (s *DockerSwarmSuite) TestSwarmClusterRotateUnlockKey(c *check.C) {
 		c.Assert(newUnlockKey, checker.Not(checker.Equals), "")
 		c.Assert(newUnlockKey, checker.Not(checker.Equals), unlockKey)
 
-		d2.Restart(c)
+		d2.RestartNode(c)
-		d3.Restart(c)
+		d3.RestartNode(c)
 
 		for _, d := range []*daemon.Daemon{d2, d3} {
 			c.Assert(getNodeStatus(c, d), checker.Equals, swarm.LocalNodeStateLocked)
@@ -1430,7 +1355,7 @@ func (s *DockerSwarmSuite) TestSwarmClusterRotateUnlockKey(c *check.C) {
 
 				time.Sleep(3 * time.Second)
 
-				d.Restart(c)
+				d.RestartNode(c)
 
 				cmd = d.Command("swarm", "unlock")
 				cmd.Stdin = bytes.NewBufferString(unlockKey)
@@ -1462,21 +1387,13 @@ func (s *DockerSwarmSuite) TestSwarmClusterRotateUnlockKey(c *check.C) {
 func (s *DockerSwarmSuite) TestSwarmAlternateLockUnlock(c *check.C) {
 	d := s.AddDaemon(c, true, true)
 
-	var unlockKey string
 	for i := 0; i < 2; i++ {
 		// set to lock
 		outs, err := d.Cmd("swarm", "update", "--autolock")
 		c.Assert(err, checker.IsNil, check.Commentf("out: %v", outs))
 		c.Assert(outs, checker.Contains, "docker swarm unlock")
+		unlockKey := getUnlockKey(d, c, outs)
 
-		for _, line := range strings.Split(outs, "\n") {
-			if strings.Contains(line, "SWMKEY") {
-				unlockKey = strings.TrimSpace(line)
-				break
-			}
-		}
-
-		c.Assert(unlockKey, checker.Not(checker.Equals), "")
 		checkSwarmUnlockedToLocked(c, d)
 
 		cmd := d.Command("swarm", "unlock")
@@ -1488,7 +1405,7 @@ func (s *DockerSwarmSuite) TestSwarmAlternateLockUnlock(c *check.C) {
 		outs, err = d.Cmd("swarm", "update", "--autolock=false")
 		c.Assert(err, checker.IsNil, check.Commentf("out: %v", outs))
 
-		checkSwarmLockedToUnlocked(c, d, unlockKey)
+		checkSwarmLockedToUnlocked(c, d)
 	}
 }
 
@@ -2065,3 +1982,16 @@ func (s *DockerSwarmSuite) TestSwarmClusterEventsConfig(c *check.C) {
 	// filtered by config
 	waitForEvent(c, d, t1, "-f type=config", "config remove "+id, defaultRetryCount)
 }
+
+func getUnlockKey(d *daemon.Daemon, c *check.C, autolockOutput string) string {
+	unlockKey, err := d.Cmd("swarm", "unlock-key", "-q")
+	c.Assert(err, checker.IsNil, check.Commentf("%s", unlockKey))
+	unlockKey = strings.TrimSuffix(unlockKey, "\n")
+
+	// Check that "docker swarm init --autolock" or "docker swarm update --autolock"
+	// contains all the expected strings, including the unlock key
+	c.Assert(autolockOutput, checker.Contains, "docker swarm unlock")
+	c.Assert(autolockOutput, checker.Contains, unlockKey)
+
+	return unlockKey
+}

integration-cli/docker_utils_test.go

@@ -419,6 +419,12 @@ func getErrorMessage(c *check.C, body []byte) string {
 }
 
 func waitAndAssert(c *check.C, timeout time.Duration, f checkF, checker check.Checker, args ...interface{}) {
+	t1 := time.Now()
+	defer func() {
+		t2 := time.Now()
+		c.Logf("waited for %v (out of %v)", t2.Sub(t1), timeout)
+	}()
+
 	after := time.After(timeout)
 	for {
 		v, comment := f(c)

internal/test/daemon/daemon.go

@@ -285,7 +285,10 @@ func (d *Daemon) StartWithLogFile(out *os.File, providedArgs ...string) error {
 
 	d.Wait = wait
 
-	tick := time.Tick(500 * time.Millisecond)
+	ticker := time.NewTicker(500 * time.Millisecond)
+	defer ticker.Stop()
+	tick := ticker.C
+
 	// make sure daemon is ready to receive requests
 	startTime := time.Now().Unix()
 	for {
@@ -423,7 +426,9 @@ func (d *Daemon) StopWithError() error {
 	}()
 
 	i := 1
-	tick := time.Tick(time.Second)
+	ticker := time.NewTicker(time.Second)
+	defer ticker.Stop()
+	tick := ticker.C
 
 	if err := d.cmd.Process.Signal(os.Interrupt); err != nil {
 		if strings.Contains(err.Error(), "os: process already finished") {

internal/test/daemon/daemon_unix.go

@@ -21,7 +21,7 @@ func cleanupNetworkNamespace(t testingT, execRoot string) {
 	// new exec root.
 	netnsPath := filepath.Join(execRoot, "netns")
 	filepath.Walk(netnsPath, func(path string, info os.FileInfo, err error) error {
-		if err := unix.Unmount(path, unix.MNT_FORCE); err != nil {
+		if err := unix.Unmount(path, unix.MNT_DETACH); err != nil && err != unix.EINVAL && err != unix.ENOENT {
 			t.Logf("unmount of %s failed: %v", path, err)
 		}
 		os.Remove(path)

internal/test/daemon/node.go

@@ -23,7 +23,7 @@ func (d *Daemon) GetNode(t assert.TestingT, id string) *swarm.Node {
 	defer cli.Close()
 
 	node, _, err := cli.NodeInspectWithRaw(context.Background(), id)
-	assert.NilError(t, err)
+	assert.NilError(t, err, "[%s] (*Daemon).GetNode: NodeInspectWithRaw(%q) failed", d.id, id)
 	assert.Check(t, node.ID == id)
 	return &node
 }

internal/test/daemon/swarm.go

@@ -16,26 +16,38 @@ const (
 	defaultSwarmListenAddr = "0.0.0.0"
 )
 
-// StartAndSwarmInit starts the daemon (with busybox) and init the swarm
+var (
-func (d *Daemon) StartAndSwarmInit(t testingT) {
+	startArgs = []string{"--iptables=false", "--swarm-default-advertise-addr=lo"}
+)
+
+// StartNode starts daemon to be used as a swarm node
+func (d *Daemon) StartNode(t testingT) {
 	if ht, ok := t.(test.HelperT); ok {
 		ht.Helper()
 	}
 	// avoid networking conflicts
-	args := []string{"--iptables=false", "--swarm-default-advertise-addr=lo"}
+	d.StartWithBusybox(t, startArgs...)
-	d.StartWithBusybox(t, args...)
+}
 
+// RestartNode restarts a daemon to be used as a swarm node
+func (d *Daemon) RestartNode(t testingT) {
+	if ht, ok := t.(test.HelperT); ok {
+		ht.Helper()
+	}
+	// avoid networking conflicts
+	d.Stop(t)
+	d.StartWithBusybox(t, startArgs...)
+}
+
+// StartAndSwarmInit starts the daemon (with busybox) and init the swarm
+func (d *Daemon) StartAndSwarmInit(t testingT) {
+	d.StartNode(t)
 	d.SwarmInit(t, swarm.InitRequest{})
 }
 
 // StartAndSwarmJoin starts the daemon (with busybox) and join the specified swarm as worker or manager
 func (d *Daemon) StartAndSwarmJoin(t testingT, leader *Daemon, manager bool) {
-	if ht, ok := t.(test.HelperT); ok {
+	d.StartNode(t)
-		ht.Helper()
-	}
-	// avoid networking conflicts
-	args := []string{"--iptables=false", "--swarm-default-advertise-addr=lo"}
-	d.StartWithBusybox(t, args...)
 
 	tokens := leader.JoinTokens(t)
 	token := tokens.Worker