Add some swarm encryption tests, and modify some existing ones, to test locking the whole cluster

Signed-off-by: cyli <cyli@twistedmatrix.com>

integration-cli/docker_cli_swarm_test.go

@@ -849,6 +849,12 @@ func (s *DockerSwarmSuite) TestDNSConfigUpdate(c *check.C) {
 	c.Assert(strings.TrimSpace(out), checker.Equals, "{[1.2.3.4] [example.com] [timeout:3]}")
 }
 
+func getNodeStatus(c *check.C, d *SwarmDaemon) swarm.LocalNodeState {
+	info, err := d.info()
+	c.Assert(err, checker.IsNil)
+	return info.LocalNodeState
+}
+
 func (s *DockerSwarmSuite) TestSwarmInitLocked(c *check.C) {
 	d := s.AddDaemon(c, false, false)
 
@@ -870,15 +876,11 @@ func (s *DockerSwarmSuite) TestSwarmInitLocked(c *check.C) {
 	outs, err = d.Cmd("swarm", "unlock-key", "-q")
 	c.Assert(outs, checker.Equals, unlockKey+"\n")
 
-	info, err := d.info()
-	c.Assert(err, checker.IsNil)
-	c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive)
+	c.Assert(getNodeStatus(c, d), checker.Equals, swarm.LocalNodeStateActive)
 
 	c.Assert(d.Restart(), checker.IsNil)
 
-	info, err = d.info()
-	c.Assert(err, checker.IsNil)
-	c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateLocked)
+	c.Assert(getNodeStatus(c, d), checker.Equals, swarm.LocalNodeStateLocked)
 
 	cmd := d.command("swarm", "unlock")
 	cmd.Stdin = bytes.NewBufferString("wrong-secret-key")
@@ -886,14 +888,14 @@ func (s *DockerSwarmSuite) TestSwarmInitLocked(c *check.C) {
 	c.Assert(err, checker.NotNil, check.Commentf("out: %v", string(out)))
 	c.Assert(string(out), checker.Contains, "invalid key")
 
+	c.Assert(getNodeStatus(c, d), checker.Equals, swarm.LocalNodeStateLocked)
+
 	cmd = d.command("swarm", "unlock")
 	cmd.Stdin = bytes.NewBufferString(unlockKey)
 	out, err = cmd.CombinedOutput()
 	c.Assert(err, checker.IsNil, check.Commentf("out: %v", string(out)))
 
-	info, err = d.info()
-	c.Assert(err, checker.IsNil)
-	c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive)
+	c.Assert(getNodeStatus(c, d), checker.Equals, swarm.LocalNodeStateActive)
 
 	outs, err = d.Cmd("node", "ls")
 	c.Assert(err, checker.IsNil)
@@ -903,13 +905,22 @@ func (s *DockerSwarmSuite) TestSwarmInitLocked(c *check.C) {
 	c.Assert(err, checker.IsNil, check.Commentf("out: %v", outs))
 
 	// Wait for autolock to be turned off
-	time.Sleep(time.Second)
-
 	c.Assert(d.Restart(), checker.IsNil)
+	status := getNodeStatus(c, d)
+	if status == swarm.LocalNodeStateLocked {
+		// it must not have updated in time - unlock, wait 3 seconds, and try again
+		cmd := d.command("swarm", "unlock")
+		cmd.Stdin = bytes.NewBufferString(unlockKey)
+		out, err := cmd.CombinedOutput()
+		c.Assert(err, checker.IsNil, check.Commentf("out: %v", string(out)))
 
-	info, err = d.info()
-	c.Assert(err, checker.IsNil)
-	c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive)
+		c.Assert(getNodeStatus(c, d), checker.Equals, swarm.LocalNodeStateActive)
+
+		time.Sleep(3 * time.Second)
+		c.Assert(d.Restart(), checker.IsNil)
+	}
+
+	c.Assert(getNodeStatus(c, d), checker.Equals, swarm.LocalNodeStateActive)
 
 	outs, err = d.Cmd("node", "ls")
 	c.Assert(err, checker.IsNil)
@@ -946,6 +957,191 @@ func (s *DockerSwarmSuite) TestSwarmLeaveLocked(c *check.C) {
 	c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive)
 }
 
+func (s *DockerSwarmSuite) TestSwarmLockUnlockCluster(c *check.C) {
+	d1 := s.AddDaemon(c, true, true)
+	d2 := s.AddDaemon(c, true, true)
+	d3 := s.AddDaemon(c, true, true)
+
+	// they start off unlocked
+	c.Assert(d2.Restart(), checker.IsNil)
+	c.Assert(getNodeStatus(c, d2), checker.Equals, swarm.LocalNodeStateActive)
+
+	// stop this one so it does not get autolock info
+	c.Assert(d2.Stop(), checker.IsNil)
+
+	// enable autolock
+	outs, err := d1.Cmd("swarm", "update", "--autolock")
+	c.Assert(err, checker.IsNil, check.Commentf("out: %v", outs))
+
+	c.Assert(outs, checker.Contains, "docker swarm unlock")
+
+	var unlockKey string
+	for _, line := range strings.Split(outs, "\n") {
+		if strings.Contains(line, "SWMKEY") {
+			unlockKey = strings.TrimSpace(line)
+			break
+		}
+	}
+
+	c.Assert(unlockKey, checker.Not(checker.Equals), "")
+
+	outs, err = d1.Cmd("swarm", "unlock-key", "-q")
+	c.Assert(outs, checker.Equals, unlockKey+"\n")
+
+	// The ones that got the cluster update should be set to locked
+	for _, d := range []*SwarmDaemon{d1, d3} {
+		c.Assert(d.Restart(), checker.IsNil)
+		c.Assert(getNodeStatus(c, d), checker.Equals, swarm.LocalNodeStateLocked)
+
+		cmd := d.command("swarm", "unlock")
+		cmd.Stdin = bytes.NewBufferString(unlockKey)
+		out, err := cmd.CombinedOutput()
+		c.Assert(err, checker.IsNil, check.Commentf("out: %v", string(out)))
+		c.Assert(getNodeStatus(c, d), checker.Equals, swarm.LocalNodeStateActive)
+	}
+
+	// d2 never got the cluster update, so it is still set to unlocked
+	c.Assert(d2.Start(), checker.IsNil)
+	c.Assert(getNodeStatus(c, d2), checker.Equals, swarm.LocalNodeStateActive)
+
+	// d2 is now set to lock
+	c.Assert(d2.Restart(), checker.IsNil)
+	status := getNodeStatus(c, d2)
+	if status == swarm.LocalNodeStateActive {
+		// it must not have updated in time - wait 3 seconds, and try again
+		time.Sleep(3 * time.Second)
+		c.Assert(d2.Restart(), checker.IsNil)
+	}
+	c.Assert(getNodeStatus(c, d2), checker.Equals, swarm.LocalNodeStateLocked)
+
+	// leave it locked, and set the cluster to no longer autolock
+	outs, err = d1.Cmd("swarm", "update", "--autolock=false")
+	c.Assert(err, checker.IsNil, check.Commentf("out: %v", outs))
+
+	// the ones that got the update are now set to unlocked
+	for _, d := range []*SwarmDaemon{d1, d3} {
+		c.Assert(d.Restart(), checker.IsNil)
+		status := getNodeStatus(c, d)
+		if status == swarm.LocalNodeStateLocked {
+			// it must not have updated to be unlocked in time - unlock, wait 3 seconds, and try again
+			cmd := d.command("swarm", "unlock")
+			cmd.Stdin = bytes.NewBufferString(unlockKey)
+			out, err := cmd.CombinedOutput()
+			c.Assert(err, checker.IsNil, check.Commentf("out: %v", string(out)))
+
+			c.Assert(getNodeStatus(c, d), checker.Equals, swarm.LocalNodeStateActive)
+
+			time.Sleep(3 * time.Second)
+			c.Assert(d.Restart(), checker.IsNil)
+		}
+
+		c.Assert(d.Restart(), checker.IsNil)
+		c.Assert(getNodeStatus(c, d), checker.Equals, swarm.LocalNodeStateActive)
+	}
+
+	// d2 still locked
+	c.Assert(getNodeStatus(c, d2), checker.Equals, swarm.LocalNodeStateLocked)
+
+	// unlock it
+	cmd := d2.command("swarm", "unlock")
+	cmd.Stdin = bytes.NewBufferString(unlockKey)
+	out, err := cmd.CombinedOutput()
+	c.Assert(err, checker.IsNil, check.Commentf("out: %v", string(out)))
+	c.Assert(getNodeStatus(c, d2), checker.Equals, swarm.LocalNodeStateActive)
+
+	// once it's caught up, d2 is set to not be locked
+	c.Assert(d2.Restart(), checker.IsNil)
+	c.Assert(getNodeStatus(c, d2), checker.Equals, swarm.LocalNodeStateActive)
+
+	// managers who join now are also unlocked
+	d4 := s.AddDaemon(c, true, true)
+	c.Assert(d4.Restart(), checker.IsNil)
+	c.Assert(getNodeStatus(c, d4), checker.Equals, swarm.LocalNodeStateActive)
+}
+
+func (s *DockerSwarmSuite) TestSwarmJoinPromoteLocked(c *check.C) {
+	d1 := s.AddDaemon(c, true, true)
+
+	// enable autolock
+	outs, err := d1.Cmd("swarm", "update", "--autolock")
+	c.Assert(err, checker.IsNil, check.Commentf("out: %v", outs))
+
+	c.Assert(outs, checker.Contains, "docker swarm unlock")
+
+	var unlockKey string
+	for _, line := range strings.Split(outs, "\n") {
+		if strings.Contains(line, "SWMKEY") {
+			unlockKey = strings.TrimSpace(line)
+			break
+		}
+	}
+
+	c.Assert(unlockKey, checker.Not(checker.Equals), "")
+
+	outs, err = d1.Cmd("swarm", "unlock-key", "-q")
+	c.Assert(outs, checker.Equals, unlockKey+"\n")
+
+	// joined workers start off unlocked
+	d2 := s.AddDaemon(c, true, false)
+	c.Assert(d2.Restart(), checker.IsNil)
+	c.Assert(getNodeStatus(c, d2), checker.Equals, swarm.LocalNodeStateActive)
+
+	// promote worker
+	outs, err = d1.Cmd("node", "promote", d2.Info.NodeID)
+	c.Assert(err, checker.IsNil)
+	c.Assert(outs, checker.Contains, "promoted to a manager in the swarm")
+
+	// join new manager node
+	d3 := s.AddDaemon(c, true, true)
+
+	// both new nodes are locked
+	for _, d := range []*SwarmDaemon{d2, d3} {
+		c.Assert(d.Restart(), checker.IsNil)
+		status := getNodeStatus(c, d)
+		if status == swarm.LocalNodeStateActive {
+			// it must not have updated in time - wait 3 seconds, and try again
+			time.Sleep(3 * time.Second)
+			c.Assert(d.Restart(), checker.IsNil)
+		}
+		c.Assert(getNodeStatus(c, d), checker.Equals, swarm.LocalNodeStateLocked)
+
+		cmd := d.command("swarm", "unlock")
+		cmd.Stdin = bytes.NewBufferString(unlockKey)
+		out, err := cmd.CombinedOutput()
+		c.Assert(err, checker.IsNil, check.Commentf("out: %v", string(out)))
+		c.Assert(getNodeStatus(c, d), checker.Equals, swarm.LocalNodeStateActive)
+	}
+
+	// demote manager back to worker - workers are not locked
+	outs, err = d1.Cmd("node", "demote", d3.Info.NodeID)
+	c.Assert(err, checker.IsNil)
+	c.Assert(outs, checker.Contains, "demoted in the swarm")
+
+	// verify that it's been demoted
+	out, err := d1.Cmd("node", "ls", "--filter", "id="+d3.Info.NodeID)
+	c.Assert(err, checker.IsNil)
+	lines := strings.Split(strings.TrimSpace(out), "\n")
+	c.Assert(len(lines), checker.GreaterThan, 0)
+	columns := strings.Fields(lines[len(lines)-1])
+	c.Assert(columns, checker.HasLen, 4) // if it was a manager it'd have a manager status field
+
+	c.Assert(d3.Restart(), checker.IsNil)
+	status := getNodeStatus(c, d3)
+	if status == swarm.LocalNodeStateLocked {
+		// it must not have updated in time - unlock, wait 3 seconds, and try again
+		cmd := d3.command("swarm", "unlock")
+		cmd.Stdin = bytes.NewBufferString(unlockKey)
+		out, err := cmd.CombinedOutput()
+		c.Assert(err, checker.IsNil, check.Commentf("out: %v", string(out)))
+
+		c.Assert(getNodeStatus(c, d3), checker.Equals, swarm.LocalNodeStateActive)
+
+		time.Sleep(3 * time.Second)
+		c.Assert(d3.Restart(), checker.IsNil)
+	}
+	c.Assert(getNodeStatus(c, d3), checker.Equals, swarm.LocalNodeStateActive)
+}
+
 func (s *DockerSwarmSuite) TestSwarmRotateUnlockKey(c *check.C) {
 	d := s.AddDaemon(c, true, true)
 
@@ -977,10 +1173,7 @@ func (s *DockerSwarmSuite) TestSwarmRotateUnlockKey(c *check.C) {
 		c.Assert(newUnlockKey, checker.Not(checker.Equals), unlockKey)
 
 		c.Assert(d.Restart(), checker.IsNil)
-
-		info, err := d.info()
-		c.Assert(err, checker.IsNil)
-		c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateLocked)
+		c.Assert(getNodeStatus(c, d), checker.Equals, swarm.LocalNodeStateLocked)
 
 		outs, _ = d.Cmd("node", "ls")
 		c.Assert(outs, checker.Contains, "Swarm is encrypted and needs to be unlocked")
@@ -1017,9 +1210,7 @@ func (s *DockerSwarmSuite) TestSwarmRotateUnlockKey(c *check.C) {
 		out, err = cmd.CombinedOutput()
 		c.Assert(err, checker.IsNil, check.Commentf("out: %v", string(out)))
 
-		info, err = d.info()
-		c.Assert(err, checker.IsNil)
-		c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive)
+		c.Assert(getNodeStatus(c, d), checker.Equals, swarm.LocalNodeStateActive)
 
 		outs, err = d.Cmd("node", "ls")
 		c.Assert(err, checker.IsNil)
@@ -1029,6 +1220,151 @@ func (s *DockerSwarmSuite) TestSwarmRotateUnlockKey(c *check.C) {
 	}
 }
 
+// This differs from `TestSwarmRotateUnlockKey` because that one rotates a single node, which is the leader.
+// This one keeps the leader up, and asserts that other manager nodes in the cluster also have their unlock
+// key rotated.
+func (s *DockerSwarmSuite) TestSwarmClusterRotateUnlockKey(c *check.C) {
+	d1 := s.AddDaemon(c, true, true) // leader - don't restart this one, we don't want leader election delays
+	d2 := s.AddDaemon(c, true, true)
+	d3 := s.AddDaemon(c, true, true)
+
+	outs, err := d1.Cmd("swarm", "update", "--autolock")
+	c.Assert(err, checker.IsNil, check.Commentf("out: %v", outs))
+
+	c.Assert(outs, checker.Contains, "docker swarm unlock")
+
+	var unlockKey string
+	for _, line := range strings.Split(outs, "\n") {
+		if strings.Contains(line, "SWMKEY") {
+			unlockKey = strings.TrimSpace(line)
+			break
+		}
+	}
+
+	c.Assert(unlockKey, checker.Not(checker.Equals), "")
+
+	outs, err = d1.Cmd("swarm", "unlock-key", "-q")
+	c.Assert(outs, checker.Equals, unlockKey+"\n")
+
+	// Rotate multiple times
+	for i := 0; i != 3; i++ {
+		outs, err = d1.Cmd("swarm", "unlock-key", "-q", "--rotate")
+		c.Assert(err, checker.IsNil, check.Commentf("out: %v", outs))
+		// Strip \n
+		newUnlockKey := outs[:len(outs)-1]
+		c.Assert(newUnlockKey, checker.Not(checker.Equals), "")
+		c.Assert(newUnlockKey, checker.Not(checker.Equals), unlockKey)
+
+		c.Assert(d2.Restart(), checker.IsNil)
+		c.Assert(d3.Restart(), checker.IsNil)
+
+		for _, d := range []*SwarmDaemon{d2, d3} {
+			c.Assert(getNodeStatus(c, d), checker.Equals, swarm.LocalNodeStateLocked)
+
+			outs, _ := d.Cmd("node", "ls")
+			c.Assert(outs, checker.Contains, "Swarm is encrypted and needs to be unlocked")
+
+			cmd := d.command("swarm", "unlock")
+			cmd.Stdin = bytes.NewBufferString(unlockKey)
+			out, err := cmd.CombinedOutput()
+
+			if err == nil {
+				// On occasion, the daemon may not have finished
+				// rotating the KEK before restarting. The test is
+				// intentionally written to explore this behavior.
+				// When this happens, unlocking with the old key will
+				// succeed. If we wait for the rotation to happen and
+				// restart again, the new key should be required this
+				// time.
+
+				time.Sleep(3 * time.Second)
+
+				c.Assert(d.Restart(), checker.IsNil)
+
+				cmd = d.command("swarm", "unlock")
+				cmd.Stdin = bytes.NewBufferString(unlockKey)
+				out, err = cmd.CombinedOutput()
+			}
+			c.Assert(err, checker.NotNil, check.Commentf("out: %v", string(out)))
+			c.Assert(string(out), checker.Contains, "invalid key")
+
+			outs, _ = d.Cmd("node", "ls")
+			c.Assert(outs, checker.Contains, "Swarm is encrypted and needs to be unlocked")
+
+			cmd = d.command("swarm", "unlock")
+			cmd.Stdin = bytes.NewBufferString(newUnlockKey)
+			out, err = cmd.CombinedOutput()
+			c.Assert(err, checker.IsNil, check.Commentf("out: %v", string(out)))
+
+			c.Assert(getNodeStatus(c, d), checker.Equals, swarm.LocalNodeStateActive)
+
+			outs, err = d.Cmd("node", "ls")
+			c.Assert(err, checker.IsNil)
+			c.Assert(outs, checker.Not(checker.Contains), "Swarm is encrypted and needs to be unlocked")
+		}
+
+		unlockKey = newUnlockKey
+	}
+}
+
+func (s *DockerSwarmSuite) TestSwarmAlternateLockUnlock(c *check.C) {
+	d := s.AddDaemon(c, true, true)
+
+	var unlockKey string
+	for i := 0; i < 2; i++ {
+		// set to lock
+		outs, err := d.Cmd("swarm", "update", "--autolock")
+		c.Assert(err, checker.IsNil, check.Commentf("out: %v", outs))
+		c.Assert(outs, checker.Contains, "docker swarm unlock")
+
+		for _, line := range strings.Split(outs, "\n") {
+			if strings.Contains(line, "SWMKEY") {
+				unlockKey = strings.TrimSpace(line)
+				break
+			}
+		}
+
+		c.Assert(unlockKey, checker.Not(checker.Equals), "")
+
+		c.Assert(d.Restart(), checker.IsNil)
+		status := getNodeStatus(c, d)
+		if status == swarm.LocalNodeStateActive {
+			// hasn't updated yet - wait and try again
+			time.Sleep(3 * time.Second)
+			c.Assert(d.Restart(), checker.IsNil)
+			status = getNodeStatus(c, d)
+		}
+		c.Assert(status, checker.Equals, swarm.LocalNodeStateLocked)
+
+		cmd := d.command("swarm", "unlock")
+		cmd.Stdin = bytes.NewBufferString(unlockKey)
+		out, err := cmd.CombinedOutput()
+		c.Assert(err, checker.IsNil, check.Commentf("out: %v", string(out)))
+
+		c.Assert(getNodeStatus(c, d), checker.Equals, swarm.LocalNodeStateActive)
+
+		outs, err = d.Cmd("swarm", "update", "--autolock=false")
+		c.Assert(err, checker.IsNil, check.Commentf("out: %v", outs))
+
+		c.Assert(d.Restart(), checker.IsNil)
+		status = getNodeStatus(c, d)
+		if status == swarm.LocalNodeStateLocked {
+			// it must not have updated to be unlocked in time - unlock, wait 3 seconds, and try again
+			cmd := d.command("swarm", "unlock")
+			cmd.Stdin = bytes.NewBufferString(unlockKey)
+			out, err := cmd.CombinedOutput()
+			c.Assert(err, checker.IsNil, check.Commentf("out: %v", string(out)))
+
+			c.Assert(getNodeStatus(c, d), checker.Equals, swarm.LocalNodeStateActive)
+
+			time.Sleep(3 * time.Second)
+			c.Assert(d.Restart(), checker.IsNil)
+			status = getNodeStatus(c, d)
+		}
+		c.Assert(getNodeStatus(c, d), checker.Equals, swarm.LocalNodeStateActive)
+	}
+}
+
 func (s *DockerSwarmSuite) TestExtraHosts(c *check.C) {
 	d := s.AddDaemon(c, true, true)