docker_api_swarm_test.go 33 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046
  1. // +build !windows
  2. package main
  3. import (
  4. "context"
  5. "fmt"
  6. "io/ioutil"
  7. "net"
  8. "net/http"
  9. "path/filepath"
  10. "runtime"
  11. "strings"
  12. "sync"
  13. "time"
  14. "github.com/cloudflare/cfssl/csr"
  15. "github.com/cloudflare/cfssl/helpers"
  16. "github.com/cloudflare/cfssl/initca"
  17. "github.com/docker/docker/api/types"
  18. "github.com/docker/docker/api/types/container"
  19. "github.com/docker/docker/api/types/swarm"
  20. "github.com/docker/docker/client"
  21. "github.com/docker/docker/integration-cli/checker"
  22. "github.com/docker/docker/integration-cli/daemon"
  23. testdaemon "github.com/docker/docker/internal/test/daemon"
  24. "github.com/docker/docker/internal/test/request"
  25. "github.com/docker/swarmkit/ca"
  26. "github.com/go-check/check"
  27. "gotest.tools/assert"
  28. is "gotest.tools/assert/cmp"
  29. )
  30. var defaultReconciliationTimeout = 30 * time.Second
  31. func (s *DockerSwarmSuite) TestAPISwarmInit(c *check.C) {
  32. // todo: should find a better way to verify that components are running than /info
  33. d1 := s.AddDaemon(c, true, true)
  34. info := d1.SwarmInfo(c)
  35. c.Assert(info.ControlAvailable, checker.True)
  36. c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive)
  37. c.Assert(info.Cluster.RootRotationInProgress, checker.False)
  38. d2 := s.AddDaemon(c, true, false)
  39. info = d2.SwarmInfo(c)
  40. c.Assert(info.ControlAvailable, checker.False)
  41. c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive)
  42. // Leaving cluster
  43. c.Assert(d2.SwarmLeave(c, false), checker.IsNil)
  44. info = d2.SwarmInfo(c)
  45. c.Assert(info.ControlAvailable, checker.False)
  46. c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateInactive)
  47. d2.SwarmJoin(c, swarm.JoinRequest{
  48. ListenAddr: d1.SwarmListenAddr(),
  49. JoinToken: d1.JoinTokens(c).Worker,
  50. RemoteAddrs: []string{d1.SwarmListenAddr()},
  51. })
  52. info = d2.SwarmInfo(c)
  53. c.Assert(info.ControlAvailable, checker.False)
  54. c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive)
  55. // Current state restoring after restarts
  56. d1.Stop(c)
  57. d2.Stop(c)
  58. d1.Start(c)
  59. d2.Start(c)
  60. info = d1.SwarmInfo(c)
  61. c.Assert(info.ControlAvailable, checker.True)
  62. c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive)
  63. info = d2.SwarmInfo(c)
  64. c.Assert(info.ControlAvailable, checker.False)
  65. c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive)
  66. }
  67. func (s *DockerSwarmSuite) TestAPISwarmJoinToken(c *check.C) {
  68. d1 := s.AddDaemon(c, false, false)
  69. d1.SwarmInit(c, swarm.InitRequest{})
  70. // todo: error message differs depending if some components of token are valid
  71. d2 := s.AddDaemon(c, false, false)
  72. c2 := d2.NewClientT(c)
  73. err := c2.SwarmJoin(context.Background(), swarm.JoinRequest{
  74. ListenAddr: d2.SwarmListenAddr(),
  75. RemoteAddrs: []string{d1.SwarmListenAddr()},
  76. })
  77. c.Assert(err, checker.NotNil)
  78. c.Assert(err.Error(), checker.Contains, "join token is necessary")
  79. info := d2.SwarmInfo(c)
  80. c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateInactive)
  81. err = c2.SwarmJoin(context.Background(), swarm.JoinRequest{
  82. ListenAddr: d2.SwarmListenAddr(),
  83. JoinToken: "foobaz",
  84. RemoteAddrs: []string{d1.SwarmListenAddr()},
  85. })
  86. c.Assert(err, checker.NotNil)
  87. c.Assert(err.Error(), checker.Contains, "invalid join token")
  88. info = d2.SwarmInfo(c)
  89. c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateInactive)
  90. workerToken := d1.JoinTokens(c).Worker
  91. d2.SwarmJoin(c, swarm.JoinRequest{
  92. ListenAddr: d2.SwarmListenAddr(),
  93. JoinToken: workerToken,
  94. RemoteAddrs: []string{d1.SwarmListenAddr()},
  95. })
  96. info = d2.SwarmInfo(c)
  97. c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive)
  98. c.Assert(d2.SwarmLeave(c, false), checker.IsNil)
  99. info = d2.SwarmInfo(c)
  100. c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateInactive)
  101. // change tokens
  102. d1.RotateTokens(c)
  103. err = c2.SwarmJoin(context.Background(), swarm.JoinRequest{
  104. ListenAddr: d2.SwarmListenAddr(),
  105. JoinToken: workerToken,
  106. RemoteAddrs: []string{d1.SwarmListenAddr()},
  107. })
  108. c.Assert(err, checker.NotNil)
  109. c.Assert(err.Error(), checker.Contains, "join token is necessary")
  110. info = d2.SwarmInfo(c)
  111. c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateInactive)
  112. workerToken = d1.JoinTokens(c).Worker
  113. d2.SwarmJoin(c, swarm.JoinRequest{JoinToken: workerToken, RemoteAddrs: []string{d1.SwarmListenAddr()}})
  114. info = d2.SwarmInfo(c)
  115. c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive)
  116. c.Assert(d2.SwarmLeave(c, false), checker.IsNil)
  117. info = d2.SwarmInfo(c)
  118. c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateInactive)
  119. // change spec, don't change tokens
  120. d1.UpdateSwarm(c, func(s *swarm.Spec) {})
  121. err = c2.SwarmJoin(context.Background(), swarm.JoinRequest{
  122. ListenAddr: d2.SwarmListenAddr(),
  123. RemoteAddrs: []string{d1.SwarmListenAddr()},
  124. })
  125. c.Assert(err, checker.NotNil)
  126. c.Assert(err.Error(), checker.Contains, "join token is necessary")
  127. info = d2.SwarmInfo(c)
  128. c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateInactive)
  129. d2.SwarmJoin(c, swarm.JoinRequest{JoinToken: workerToken, RemoteAddrs: []string{d1.SwarmListenAddr()}})
  130. info = d2.SwarmInfo(c)
  131. c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive)
  132. c.Assert(d2.SwarmLeave(c, false), checker.IsNil)
  133. info = d2.SwarmInfo(c)
  134. c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateInactive)
  135. }
  136. func (s *DockerSwarmSuite) TestUpdateSwarmAddExternalCA(c *check.C) {
  137. d1 := s.AddDaemon(c, false, false)
  138. d1.SwarmInit(c, swarm.InitRequest{})
  139. d1.UpdateSwarm(c, func(s *swarm.Spec) {
  140. s.CAConfig.ExternalCAs = []*swarm.ExternalCA{
  141. {
  142. Protocol: swarm.ExternalCAProtocolCFSSL,
  143. URL: "https://thishasnoca.org",
  144. },
  145. {
  146. Protocol: swarm.ExternalCAProtocolCFSSL,
  147. URL: "https://thishasacacert.org",
  148. CACert: "cacert",
  149. },
  150. }
  151. })
  152. info := d1.SwarmInfo(c)
  153. c.Assert(info.Cluster.Spec.CAConfig.ExternalCAs, checker.HasLen, 2)
  154. c.Assert(info.Cluster.Spec.CAConfig.ExternalCAs[0].CACert, checker.Equals, "")
  155. c.Assert(info.Cluster.Spec.CAConfig.ExternalCAs[1].CACert, checker.Equals, "cacert")
  156. }
  157. func (s *DockerSwarmSuite) TestAPISwarmCAHash(c *check.C) {
  158. d1 := s.AddDaemon(c, true, true)
  159. d2 := s.AddDaemon(c, false, false)
  160. splitToken := strings.Split(d1.JoinTokens(c).Worker, "-")
  161. splitToken[2] = "1kxftv4ofnc6mt30lmgipg6ngf9luhwqopfk1tz6bdmnkubg0e"
  162. replacementToken := strings.Join(splitToken, "-")
  163. c2 := d2.NewClientT(c)
  164. err := c2.SwarmJoin(context.Background(), swarm.JoinRequest{
  165. ListenAddr: d2.SwarmListenAddr(),
  166. JoinToken: replacementToken,
  167. RemoteAddrs: []string{d1.SwarmListenAddr()},
  168. })
  169. c.Assert(err, checker.NotNil)
  170. c.Assert(err.Error(), checker.Contains, "remote CA does not match fingerprint")
  171. }
  172. func (s *DockerSwarmSuite) TestAPISwarmPromoteDemote(c *check.C) {
  173. d1 := s.AddDaemon(c, false, false)
  174. d1.SwarmInit(c, swarm.InitRequest{})
  175. d2 := s.AddDaemon(c, true, false)
  176. info := d2.SwarmInfo(c)
  177. c.Assert(info.ControlAvailable, checker.False)
  178. c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive)
  179. d1.UpdateNode(c, d2.NodeID(), func(n *swarm.Node) {
  180. n.Spec.Role = swarm.NodeRoleManager
  181. })
  182. waitAndAssert(c, defaultReconciliationTimeout, d2.CheckControlAvailable, checker.True)
  183. d1.UpdateNode(c, d2.NodeID(), func(n *swarm.Node) {
  184. n.Spec.Role = swarm.NodeRoleWorker
  185. })
  186. waitAndAssert(c, defaultReconciliationTimeout, d2.CheckControlAvailable, checker.False)
  187. // Wait for the role to change to worker in the cert. This is partially
  188. // done because it's something worth testing in its own right, and
  189. // partially because changing the role from manager to worker and then
  190. // back to manager quickly might cause the node to pause for awhile
  191. // while waiting for the role to change to worker, and the test can
  192. // time out during this interval.
  193. waitAndAssert(c, defaultReconciliationTimeout, func(c *check.C) (interface{}, check.CommentInterface) {
  194. certBytes, err := ioutil.ReadFile(filepath.Join(d2.Folder, "root", "swarm", "certificates", "swarm-node.crt"))
  195. if err != nil {
  196. return "", check.Commentf("error: %v", err)
  197. }
  198. certs, err := helpers.ParseCertificatesPEM(certBytes)
  199. if err == nil && len(certs) > 0 && len(certs[0].Subject.OrganizationalUnit) > 0 {
  200. return certs[0].Subject.OrganizationalUnit[0], nil
  201. }
  202. return "", check.Commentf("could not get organizational unit from certificate")
  203. }, checker.Equals, "swarm-worker")
  204. // Demoting last node should fail
  205. node := d1.GetNode(c, d1.NodeID())
  206. node.Spec.Role = swarm.NodeRoleWorker
  207. url := fmt.Sprintf("/nodes/%s/update?version=%d", node.ID, node.Version.Index)
  208. res, body, err := request.Post(url, request.Host(d1.Sock()), request.JSONBody(node.Spec))
  209. c.Assert(err, checker.IsNil)
  210. b, err := request.ReadBody(body)
  211. c.Assert(err, checker.IsNil)
  212. c.Assert(res.StatusCode, checker.Equals, http.StatusBadRequest, check.Commentf("output: %q", string(b)))
  213. // The warning specific to demoting the last manager is best-effort and
  214. // won't appear until the Role field of the demoted manager has been
  215. // updated.
  216. // Yes, I know this looks silly, but checker.Matches is broken, since
  217. // it anchors the regexp contrary to the documentation, and this makes
  218. // it impossible to match something that includes a line break.
  219. if !strings.Contains(string(b), "last manager of the swarm") {
  220. c.Assert(string(b), checker.Contains, "this would result in a loss of quorum")
  221. }
  222. info = d1.SwarmInfo(c)
  223. c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive)
  224. c.Assert(info.ControlAvailable, checker.True)
  225. // Promote already demoted node
  226. d1.UpdateNode(c, d2.NodeID(), func(n *swarm.Node) {
  227. n.Spec.Role = swarm.NodeRoleManager
  228. })
  229. waitAndAssert(c, defaultReconciliationTimeout, d2.CheckControlAvailable, checker.True)
  230. }
  231. func (s *DockerSwarmSuite) TestAPISwarmLeaderProxy(c *check.C) {
  232. // add three managers, one of these is leader
  233. d1 := s.AddDaemon(c, true, true)
  234. d2 := s.AddDaemon(c, true, true)
  235. d3 := s.AddDaemon(c, true, true)
  236. // start a service by hitting each of the 3 managers
  237. d1.CreateService(c, simpleTestService, func(s *swarm.Service) {
  238. s.Spec.Name = "test1"
  239. })
  240. d2.CreateService(c, simpleTestService, func(s *swarm.Service) {
  241. s.Spec.Name = "test2"
  242. })
  243. d3.CreateService(c, simpleTestService, func(s *swarm.Service) {
  244. s.Spec.Name = "test3"
  245. })
  246. // 3 services should be started now, because the requests were proxied to leader
  247. // query each node and make sure it returns 3 services
  248. for _, d := range []*daemon.Daemon{d1, d2, d3} {
  249. services := d.ListServices(c)
  250. c.Assert(services, checker.HasLen, 3)
  251. }
  252. }
  253. func (s *DockerSwarmSuite) TestAPISwarmLeaderElection(c *check.C) {
  254. if runtime.GOARCH == "s390x" {
  255. c.Skip("Disabled on s390x")
  256. }
  257. if runtime.GOARCH == "ppc64le" {
  258. c.Skip("Disabled on ppc64le")
  259. }
  260. // Create 3 nodes
  261. d1 := s.AddDaemon(c, true, true)
  262. d2 := s.AddDaemon(c, true, true)
  263. d3 := s.AddDaemon(c, true, true)
  264. // assert that the first node we made is the leader, and the other two are followers
  265. c.Assert(d1.GetNode(c, d1.NodeID()).ManagerStatus.Leader, checker.True)
  266. c.Assert(d1.GetNode(c, d2.NodeID()).ManagerStatus.Leader, checker.False)
  267. c.Assert(d1.GetNode(c, d3.NodeID()).ManagerStatus.Leader, checker.False)
  268. d1.Stop(c)
  269. var (
  270. leader *daemon.Daemon // keep track of leader
  271. followers []*daemon.Daemon // keep track of followers
  272. )
  273. checkLeader := func(nodes ...*daemon.Daemon) checkF {
  274. return func(c *check.C) (interface{}, check.CommentInterface) {
  275. // clear these out before each run
  276. leader = nil
  277. followers = nil
  278. for _, d := range nodes {
  279. if d.GetNode(c, d.NodeID()).ManagerStatus.Leader {
  280. leader = d
  281. } else {
  282. followers = append(followers, d)
  283. }
  284. }
  285. if leader == nil {
  286. return false, check.Commentf("no leader elected")
  287. }
  288. return true, check.Commentf("elected %v", leader.ID())
  289. }
  290. }
  291. // wait for an election to occur
  292. waitAndAssert(c, defaultReconciliationTimeout, checkLeader(d2, d3), checker.True)
  293. // assert that we have a new leader
  294. c.Assert(leader, checker.NotNil)
  295. // Keep track of the current leader, since we want that to be chosen.
  296. stableleader := leader
  297. // add the d1, the initial leader, back
  298. d1.Start(c)
  299. // TODO(stevvooe): may need to wait for rejoin here
  300. // wait for possible election
  301. waitAndAssert(c, defaultReconciliationTimeout, checkLeader(d1, d2, d3), checker.True)
  302. // pick out the leader and the followers again
  303. // verify that we still only have 1 leader and 2 followers
  304. c.Assert(leader, checker.NotNil)
  305. c.Assert(followers, checker.HasLen, 2)
  306. // and that after we added d1 back, the leader hasn't changed
  307. c.Assert(leader.NodeID(), checker.Equals, stableleader.NodeID())
  308. }
  309. func (s *DockerSwarmSuite) TestAPISwarmRaftQuorum(c *check.C) {
  310. if runtime.GOARCH == "s390x" {
  311. c.Skip("Disabled on s390x")
  312. }
  313. if runtime.GOARCH == "ppc64le" {
  314. c.Skip("Disabled on ppc64le")
  315. }
  316. d1 := s.AddDaemon(c, true, true)
  317. d2 := s.AddDaemon(c, true, true)
  318. d3 := s.AddDaemon(c, true, true)
  319. d1.CreateService(c, simpleTestService)
  320. d2.Stop(c)
  321. // make sure there is a leader
  322. waitAndAssert(c, defaultReconciliationTimeout, d1.CheckLeader, checker.IsNil)
  323. d1.CreateService(c, simpleTestService, func(s *swarm.Service) {
  324. s.Spec.Name = "top1"
  325. })
  326. d3.Stop(c)
  327. var service swarm.Service
  328. simpleTestService(&service)
  329. service.Spec.Name = "top2"
  330. cli := d1.NewClientT(c)
  331. defer cli.Close()
  332. // d1 will eventually step down from leader because there is no longer an active quorum, wait for that to happen
  333. waitAndAssert(c, defaultReconciliationTimeout, func(c *check.C) (interface{}, check.CommentInterface) {
  334. _, err := cli.ServiceCreate(context.Background(), service.Spec, types.ServiceCreateOptions{})
  335. return err.Error(), nil
  336. }, checker.Contains, "Make sure more than half of the managers are online.")
  337. d2.Start(c)
  338. // make sure there is a leader
  339. waitAndAssert(c, defaultReconciliationTimeout, d1.CheckLeader, checker.IsNil)
  340. d1.CreateService(c, simpleTestService, func(s *swarm.Service) {
  341. s.Spec.Name = "top3"
  342. })
  343. }
  344. func (s *DockerSwarmSuite) TestAPISwarmLeaveRemovesContainer(c *check.C) {
  345. d := s.AddDaemon(c, true, true)
  346. instances := 2
  347. d.CreateService(c, simpleTestService, setInstances(instances))
  348. id, err := d.Cmd("run", "-d", "busybox", "top")
  349. c.Assert(err, checker.IsNil, check.Commentf("%s", id))
  350. id = strings.TrimSpace(id)
  351. waitAndAssert(c, defaultReconciliationTimeout, d.CheckActiveContainerCount, checker.Equals, instances+1)
  352. c.Assert(d.SwarmLeave(c, false), checker.NotNil)
  353. c.Assert(d.SwarmLeave(c, true), checker.IsNil)
  354. waitAndAssert(c, defaultReconciliationTimeout, d.CheckActiveContainerCount, checker.Equals, 1)
  355. id2, err := d.Cmd("ps", "-q")
  356. c.Assert(err, checker.IsNil, check.Commentf("%s", id2))
  357. c.Assert(id, checker.HasPrefix, strings.TrimSpace(id2))
  358. }
  359. // #23629
  360. func (s *DockerSwarmSuite) TestAPISwarmLeaveOnPendingJoin(c *check.C) {
  361. testRequires(c, Network)
  362. s.AddDaemon(c, true, true)
  363. d2 := s.AddDaemon(c, false, false)
  364. id, err := d2.Cmd("run", "-d", "busybox", "top")
  365. c.Assert(err, checker.IsNil, check.Commentf("%s", id))
  366. id = strings.TrimSpace(id)
  367. c2 := d2.NewClientT(c)
  368. err = c2.SwarmJoin(context.Background(), swarm.JoinRequest{
  369. ListenAddr: d2.SwarmListenAddr(),
  370. RemoteAddrs: []string{"123.123.123.123:1234"},
  371. })
  372. c.Assert(err, check.NotNil)
  373. c.Assert(err.Error(), checker.Contains, "Timeout was reached")
  374. info := d2.SwarmInfo(c)
  375. c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStatePending)
  376. c.Assert(d2.SwarmLeave(c, true), checker.IsNil)
  377. waitAndAssert(c, defaultReconciliationTimeout, d2.CheckActiveContainerCount, checker.Equals, 1)
  378. id2, err := d2.Cmd("ps", "-q")
  379. c.Assert(err, checker.IsNil, check.Commentf("%s", id2))
  380. c.Assert(id, checker.HasPrefix, strings.TrimSpace(id2))
  381. }
  382. // #23705
  383. func (s *DockerSwarmSuite) TestAPISwarmRestoreOnPendingJoin(c *check.C) {
  384. testRequires(c, Network)
  385. d := s.AddDaemon(c, false, false)
  386. client := d.NewClientT(c)
  387. err := client.SwarmJoin(context.Background(), swarm.JoinRequest{
  388. ListenAddr: d.SwarmListenAddr(),
  389. RemoteAddrs: []string{"123.123.123.123:1234"},
  390. })
  391. c.Assert(err, check.NotNil)
  392. c.Assert(err.Error(), checker.Contains, "Timeout was reached")
  393. waitAndAssert(c, defaultReconciliationTimeout, d.CheckLocalNodeState, checker.Equals, swarm.LocalNodeStatePending)
  394. d.Stop(c)
  395. d.Start(c)
  396. info := d.SwarmInfo(c)
  397. c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateInactive)
  398. }
  399. func (s *DockerSwarmSuite) TestAPISwarmManagerRestore(c *check.C) {
  400. d1 := s.AddDaemon(c, true, true)
  401. instances := 2
  402. id := d1.CreateService(c, simpleTestService, setInstances(instances))
  403. d1.GetService(c, id)
  404. d1.Stop(c)
  405. d1.Start(c)
  406. d1.GetService(c, id)
  407. d2 := s.AddDaemon(c, true, true)
  408. d2.GetService(c, id)
  409. d2.Stop(c)
  410. d2.Start(c)
  411. d2.GetService(c, id)
  412. d3 := s.AddDaemon(c, true, true)
  413. d3.GetService(c, id)
  414. d3.Stop(c)
  415. d3.Start(c)
  416. d3.GetService(c, id)
  417. err := d3.Kill()
  418. assert.NilError(c, err)
  419. time.Sleep(1 * time.Second) // time to handle signal
  420. d3.Start(c)
  421. d3.GetService(c, id)
  422. }
  423. func (s *DockerSwarmSuite) TestAPISwarmScaleNoRollingUpdate(c *check.C) {
  424. d := s.AddDaemon(c, true, true)
  425. instances := 2
  426. id := d.CreateService(c, simpleTestService, setInstances(instances))
  427. waitAndAssert(c, defaultReconciliationTimeout, d.CheckActiveContainerCount, checker.Equals, instances)
  428. containers := d.ActiveContainers(c)
  429. instances = 4
  430. d.UpdateService(c, d.GetService(c, id), setInstances(instances))
  431. waitAndAssert(c, defaultReconciliationTimeout, d.CheckActiveContainerCount, checker.Equals, instances)
  432. containers2 := d.ActiveContainers(c)
  433. loop0:
  434. for _, c1 := range containers {
  435. for _, c2 := range containers2 {
  436. if c1 == c2 {
  437. continue loop0
  438. }
  439. }
  440. c.Errorf("container %v not found in new set %#v", c1, containers2)
  441. }
  442. }
  443. func (s *DockerSwarmSuite) TestAPISwarmInvalidAddress(c *check.C) {
  444. d := s.AddDaemon(c, false, false)
  445. req := swarm.InitRequest{
  446. ListenAddr: "",
  447. }
  448. res, _, err := request.Post("/swarm/init", request.Host(d.Sock()), request.JSONBody(req))
  449. c.Assert(err, checker.IsNil)
  450. c.Assert(res.StatusCode, checker.Equals, http.StatusBadRequest)
  451. req2 := swarm.JoinRequest{
  452. ListenAddr: "0.0.0.0:2377",
  453. RemoteAddrs: []string{""},
  454. }
  455. res, _, err = request.Post("/swarm/join", request.Host(d.Sock()), request.JSONBody(req2))
  456. c.Assert(err, checker.IsNil)
  457. c.Assert(res.StatusCode, checker.Equals, http.StatusBadRequest)
  458. }
  459. func (s *DockerSwarmSuite) TestAPISwarmForceNewCluster(c *check.C) {
  460. d1 := s.AddDaemon(c, true, true)
  461. d2 := s.AddDaemon(c, true, true)
  462. instances := 2
  463. id := d1.CreateService(c, simpleTestService, setInstances(instances))
  464. waitAndAssert(c, defaultReconciliationTimeout, reducedCheck(sumAsIntegers, d1.CheckActiveContainerCount, d2.CheckActiveContainerCount), checker.Equals, instances)
  465. // drain d2, all containers should move to d1
  466. d1.UpdateNode(c, d2.NodeID(), func(n *swarm.Node) {
  467. n.Spec.Availability = swarm.NodeAvailabilityDrain
  468. })
  469. waitAndAssert(c, defaultReconciliationTimeout, d1.CheckActiveContainerCount, checker.Equals, instances)
  470. waitAndAssert(c, defaultReconciliationTimeout, d2.CheckActiveContainerCount, checker.Equals, 0)
  471. d2.Stop(c)
  472. d1.SwarmInit(c, swarm.InitRequest{
  473. ForceNewCluster: true,
  474. Spec: swarm.Spec{},
  475. })
  476. waitAndAssert(c, defaultReconciliationTimeout, d1.CheckActiveContainerCount, checker.Equals, instances)
  477. d3 := s.AddDaemon(c, true, true)
  478. info := d3.SwarmInfo(c)
  479. c.Assert(info.ControlAvailable, checker.True)
  480. c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive)
  481. instances = 4
  482. d3.UpdateService(c, d3.GetService(c, id), setInstances(instances))
  483. waitAndAssert(c, defaultReconciliationTimeout, reducedCheck(sumAsIntegers, d1.CheckActiveContainerCount, d3.CheckActiveContainerCount), checker.Equals, instances)
  484. }
  485. func simpleTestService(s *swarm.Service) {
  486. ureplicas := uint64(1)
  487. restartDelay := time.Duration(100 * time.Millisecond)
  488. s.Spec = swarm.ServiceSpec{
  489. TaskTemplate: swarm.TaskSpec{
  490. ContainerSpec: &swarm.ContainerSpec{
  491. Image: "busybox:latest",
  492. Command: []string{"/bin/top"},
  493. },
  494. RestartPolicy: &swarm.RestartPolicy{
  495. Delay: &restartDelay,
  496. },
  497. },
  498. Mode: swarm.ServiceMode{
  499. Replicated: &swarm.ReplicatedService{
  500. Replicas: &ureplicas,
  501. },
  502. },
  503. }
  504. s.Spec.Name = "top"
  505. }
  506. func serviceForUpdate(s *swarm.Service) {
  507. ureplicas := uint64(1)
  508. restartDelay := time.Duration(100 * time.Millisecond)
  509. s.Spec = swarm.ServiceSpec{
  510. TaskTemplate: swarm.TaskSpec{
  511. ContainerSpec: &swarm.ContainerSpec{
  512. Image: "busybox:latest",
  513. Command: []string{"/bin/top"},
  514. },
  515. RestartPolicy: &swarm.RestartPolicy{
  516. Delay: &restartDelay,
  517. },
  518. },
  519. Mode: swarm.ServiceMode{
  520. Replicated: &swarm.ReplicatedService{
  521. Replicas: &ureplicas,
  522. },
  523. },
  524. UpdateConfig: &swarm.UpdateConfig{
  525. Parallelism: 2,
  526. Delay: 4 * time.Second,
  527. FailureAction: swarm.UpdateFailureActionContinue,
  528. },
  529. RollbackConfig: &swarm.UpdateConfig{
  530. Parallelism: 3,
  531. Delay: 4 * time.Second,
  532. FailureAction: swarm.UpdateFailureActionContinue,
  533. },
  534. }
  535. s.Spec.Name = "updatetest"
  536. }
  537. func setInstances(replicas int) testdaemon.ServiceConstructor {
  538. ureplicas := uint64(replicas)
  539. return func(s *swarm.Service) {
  540. s.Spec.Mode = swarm.ServiceMode{
  541. Replicated: &swarm.ReplicatedService{
  542. Replicas: &ureplicas,
  543. },
  544. }
  545. }
  546. }
  547. func setUpdateOrder(order string) testdaemon.ServiceConstructor {
  548. return func(s *swarm.Service) {
  549. if s.Spec.UpdateConfig == nil {
  550. s.Spec.UpdateConfig = &swarm.UpdateConfig{}
  551. }
  552. s.Spec.UpdateConfig.Order = order
  553. }
  554. }
  555. func setRollbackOrder(order string) testdaemon.ServiceConstructor {
  556. return func(s *swarm.Service) {
  557. if s.Spec.RollbackConfig == nil {
  558. s.Spec.RollbackConfig = &swarm.UpdateConfig{}
  559. }
  560. s.Spec.RollbackConfig.Order = order
  561. }
  562. }
  563. func setImage(image string) testdaemon.ServiceConstructor {
  564. return func(s *swarm.Service) {
  565. if s.Spec.TaskTemplate.ContainerSpec == nil {
  566. s.Spec.TaskTemplate.ContainerSpec = &swarm.ContainerSpec{}
  567. }
  568. s.Spec.TaskTemplate.ContainerSpec.Image = image
  569. }
  570. }
  571. func setFailureAction(failureAction string) testdaemon.ServiceConstructor {
  572. return func(s *swarm.Service) {
  573. s.Spec.UpdateConfig.FailureAction = failureAction
  574. }
  575. }
  576. func setMaxFailureRatio(maxFailureRatio float32) testdaemon.ServiceConstructor {
  577. return func(s *swarm.Service) {
  578. s.Spec.UpdateConfig.MaxFailureRatio = maxFailureRatio
  579. }
  580. }
  581. func setParallelism(parallelism uint64) testdaemon.ServiceConstructor {
  582. return func(s *swarm.Service) {
  583. s.Spec.UpdateConfig.Parallelism = parallelism
  584. }
  585. }
  586. func setConstraints(constraints []string) testdaemon.ServiceConstructor {
  587. return func(s *swarm.Service) {
  588. if s.Spec.TaskTemplate.Placement == nil {
  589. s.Spec.TaskTemplate.Placement = &swarm.Placement{}
  590. }
  591. s.Spec.TaskTemplate.Placement.Constraints = constraints
  592. }
  593. }
  594. func setPlacementPrefs(prefs []swarm.PlacementPreference) testdaemon.ServiceConstructor {
  595. return func(s *swarm.Service) {
  596. if s.Spec.TaskTemplate.Placement == nil {
  597. s.Spec.TaskTemplate.Placement = &swarm.Placement{}
  598. }
  599. s.Spec.TaskTemplate.Placement.Preferences = prefs
  600. }
  601. }
  602. func setGlobalMode(s *swarm.Service) {
  603. s.Spec.Mode = swarm.ServiceMode{
  604. Global: &swarm.GlobalService{},
  605. }
  606. }
  607. func checkClusterHealth(c *check.C, cl []*daemon.Daemon, managerCount, workerCount int) {
  608. var totalMCount, totalWCount int
  609. for _, d := range cl {
  610. var (
  611. info swarm.Info
  612. )
  613. // check info in a waitAndAssert, because if the cluster doesn't have a leader, `info` will return an error
  614. checkInfo := func(c *check.C) (interface{}, check.CommentInterface) {
  615. client := d.NewClientT(c)
  616. daemonInfo, err := client.Info(context.Background())
  617. info = daemonInfo.Swarm
  618. return err, check.Commentf("cluster not ready in time")
  619. }
  620. waitAndAssert(c, defaultReconciliationTimeout, checkInfo, checker.IsNil)
  621. if !info.ControlAvailable {
  622. totalWCount++
  623. continue
  624. }
  625. var leaderFound bool
  626. totalMCount++
  627. var mCount, wCount int
  628. for _, n := range d.ListNodes(c) {
  629. waitReady := func(c *check.C) (interface{}, check.CommentInterface) {
  630. if n.Status.State == swarm.NodeStateReady {
  631. return true, nil
  632. }
  633. nn := d.GetNode(c, n.ID)
  634. n = *nn
  635. return n.Status.State == swarm.NodeStateReady, check.Commentf("state of node %s, reported by %s", n.ID, d.NodeID())
  636. }
  637. waitAndAssert(c, defaultReconciliationTimeout, waitReady, checker.True)
  638. waitActive := func(c *check.C) (interface{}, check.CommentInterface) {
  639. if n.Spec.Availability == swarm.NodeAvailabilityActive {
  640. return true, nil
  641. }
  642. nn := d.GetNode(c, n.ID)
  643. n = *nn
  644. return n.Spec.Availability == swarm.NodeAvailabilityActive, check.Commentf("availability of node %s, reported by %s", n.ID, d.NodeID())
  645. }
  646. waitAndAssert(c, defaultReconciliationTimeout, waitActive, checker.True)
  647. if n.Spec.Role == swarm.NodeRoleManager {
  648. c.Assert(n.ManagerStatus, checker.NotNil, check.Commentf("manager status of node %s (manager), reported by %s", n.ID, d.NodeID()))
  649. if n.ManagerStatus.Leader {
  650. leaderFound = true
  651. }
  652. mCount++
  653. } else {
  654. c.Assert(n.ManagerStatus, checker.IsNil, check.Commentf("manager status of node %s (worker), reported by %s", n.ID, d.NodeID()))
  655. wCount++
  656. }
  657. }
  658. c.Assert(leaderFound, checker.True, check.Commentf("lack of leader reported by node %s", info.NodeID))
  659. c.Assert(mCount, checker.Equals, managerCount, check.Commentf("managers count reported by node %s", info.NodeID))
  660. c.Assert(wCount, checker.Equals, workerCount, check.Commentf("workers count reported by node %s", info.NodeID))
  661. }
  662. c.Assert(totalMCount, checker.Equals, managerCount)
  663. c.Assert(totalWCount, checker.Equals, workerCount)
  664. }
  665. func (s *DockerSwarmSuite) TestAPISwarmRestartCluster(c *check.C) {
  666. mCount, wCount := 5, 1
  667. var nodes []*daemon.Daemon
  668. for i := 0; i < mCount; i++ {
  669. manager := s.AddDaemon(c, true, true)
  670. info := manager.SwarmInfo(c)
  671. c.Assert(info.ControlAvailable, checker.True)
  672. c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive)
  673. nodes = append(nodes, manager)
  674. }
  675. for i := 0; i < wCount; i++ {
  676. worker := s.AddDaemon(c, true, false)
  677. info := worker.SwarmInfo(c)
  678. c.Assert(info.ControlAvailable, checker.False)
  679. c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive)
  680. nodes = append(nodes, worker)
  681. }
  682. // stop whole cluster
  683. {
  684. var wg sync.WaitGroup
  685. wg.Add(len(nodes))
  686. errs := make(chan error, len(nodes))
  687. for _, d := range nodes {
  688. go func(daemon *daemon.Daemon) {
  689. defer wg.Done()
  690. if err := daemon.StopWithError(); err != nil {
  691. errs <- err
  692. }
  693. }(d)
  694. }
  695. wg.Wait()
  696. close(errs)
  697. for err := range errs {
  698. c.Assert(err, check.IsNil)
  699. }
  700. }
  701. // start whole cluster
  702. {
  703. var wg sync.WaitGroup
  704. wg.Add(len(nodes))
  705. errs := make(chan error, len(nodes))
  706. for _, d := range nodes {
  707. go func(daemon *daemon.Daemon) {
  708. defer wg.Done()
  709. if err := daemon.StartWithError("--iptables=false"); err != nil {
  710. errs <- err
  711. }
  712. }(d)
  713. }
  714. wg.Wait()
  715. close(errs)
  716. for err := range errs {
  717. c.Assert(err, check.IsNil)
  718. }
  719. }
  720. checkClusterHealth(c, nodes, mCount, wCount)
  721. }
  722. func (s *DockerSwarmSuite) TestAPISwarmServicesUpdateWithName(c *check.C) {
  723. d := s.AddDaemon(c, true, true)
  724. instances := 2
  725. id := d.CreateService(c, simpleTestService, setInstances(instances))
  726. waitAndAssert(c, defaultReconciliationTimeout, d.CheckActiveContainerCount, checker.Equals, instances)
  727. service := d.GetService(c, id)
  728. instances = 5
  729. setInstances(instances)(service)
  730. cli := d.NewClientT(c)
  731. defer cli.Close()
  732. _, err := cli.ServiceUpdate(context.Background(), service.Spec.Name, service.Version, service.Spec, types.ServiceUpdateOptions{})
  733. c.Assert(err, checker.IsNil)
  734. waitAndAssert(c, defaultReconciliationTimeout, d.CheckActiveContainerCount, checker.Equals, instances)
  735. }
  736. // Unlocking an unlocked swarm results in an error
  737. func (s *DockerSwarmSuite) TestAPISwarmUnlockNotLocked(c *check.C) {
  738. d := s.AddDaemon(c, true, true)
  739. err := d.SwarmUnlock(c, swarm.UnlockRequest{UnlockKey: "wrong-key"})
  740. c.Assert(err, checker.NotNil)
  741. c.Assert(err.Error(), checker.Contains, "swarm is not locked")
  742. }
  743. // #29885
  744. func (s *DockerSwarmSuite) TestAPISwarmErrorHandling(c *check.C) {
  745. ln, err := net.Listen("tcp", fmt.Sprintf(":%d", defaultSwarmPort))
  746. c.Assert(err, checker.IsNil)
  747. defer ln.Close()
  748. d := s.AddDaemon(c, false, false)
  749. client := d.NewClientT(c)
  750. _, err = client.SwarmInit(context.Background(), swarm.InitRequest{
  751. ListenAddr: d.SwarmListenAddr(),
  752. })
  753. c.Assert(err, checker.NotNil)
  754. c.Assert(err.Error(), checker.Contains, "address already in use")
  755. }
  756. // Test case for 30242, where duplicate networks, with different drivers `bridge` and `overlay`,
  757. // caused both scopes to be `swarm` for `docker network inspect` and `docker network ls`.
  758. // This test makes sure the fixes correctly output scopes instead.
  759. func (s *DockerSwarmSuite) TestAPIDuplicateNetworks(c *check.C) {
  760. d := s.AddDaemon(c, true, true)
  761. cli := d.NewClientT(c)
  762. defer cli.Close()
  763. name := "foo"
  764. networkCreate := types.NetworkCreate{
  765. CheckDuplicate: false,
  766. }
  767. networkCreate.Driver = "bridge"
  768. n1, err := cli.NetworkCreate(context.Background(), name, networkCreate)
  769. c.Assert(err, checker.IsNil)
  770. networkCreate.Driver = "overlay"
  771. n2, err := cli.NetworkCreate(context.Background(), name, networkCreate)
  772. c.Assert(err, checker.IsNil)
  773. r1, err := cli.NetworkInspect(context.Background(), n1.ID, types.NetworkInspectOptions{})
  774. c.Assert(err, checker.IsNil)
  775. c.Assert(r1.Scope, checker.Equals, "local")
  776. r2, err := cli.NetworkInspect(context.Background(), n2.ID, types.NetworkInspectOptions{})
  777. c.Assert(err, checker.IsNil)
  778. c.Assert(r2.Scope, checker.Equals, "swarm")
  779. }
  780. // Test case for 30178
  781. func (s *DockerSwarmSuite) TestAPISwarmHealthcheckNone(c *check.C) {
  782. // Issue #36386 can be a independent one, which is worth further investigation.
  783. c.Skip("Root cause of Issue #36386 is needed")
  784. d := s.AddDaemon(c, true, true)
  785. out, err := d.Cmd("network", "create", "-d", "overlay", "lb")
  786. c.Assert(err, checker.IsNil, check.Commentf("%s", out))
  787. instances := 1
  788. d.CreateService(c, simpleTestService, setInstances(instances), func(s *swarm.Service) {
  789. if s.Spec.TaskTemplate.ContainerSpec == nil {
  790. s.Spec.TaskTemplate.ContainerSpec = &swarm.ContainerSpec{}
  791. }
  792. s.Spec.TaskTemplate.ContainerSpec.Healthcheck = &container.HealthConfig{}
  793. s.Spec.TaskTemplate.Networks = []swarm.NetworkAttachmentConfig{
  794. {Target: "lb"},
  795. }
  796. })
  797. waitAndAssert(c, defaultReconciliationTimeout, d.CheckActiveContainerCount, checker.Equals, instances)
  798. containers := d.ActiveContainers(c)
  799. out, err = d.Cmd("exec", containers[0], "ping", "-c1", "-W3", "top")
  800. c.Assert(err, checker.IsNil, check.Commentf("%s", out))
  801. }
  802. func (s *DockerSwarmSuite) TestSwarmRepeatedRootRotation(c *check.C) {
  803. m := s.AddDaemon(c, true, true)
  804. w := s.AddDaemon(c, true, false)
  805. info := m.SwarmInfo(c)
  806. currentTrustRoot := info.Cluster.TLSInfo.TrustRoot
  807. // rotate multiple times
  808. for i := 0; i < 4; i++ {
  809. var err error
  810. var cert, key []byte
  811. if i%2 != 0 {
  812. cert, _, key, err = initca.New(&csr.CertificateRequest{
  813. CN: "newRoot",
  814. KeyRequest: csr.NewBasicKeyRequest(),
  815. CA: &csr.CAConfig{Expiry: ca.RootCAExpiration},
  816. })
  817. c.Assert(err, checker.IsNil)
  818. }
  819. expectedCert := string(cert)
  820. m.UpdateSwarm(c, func(s *swarm.Spec) {
  821. s.CAConfig.SigningCACert = expectedCert
  822. s.CAConfig.SigningCAKey = string(key)
  823. s.CAConfig.ForceRotate++
  824. })
  825. // poll to make sure update succeeds
  826. var clusterTLSInfo swarm.TLSInfo
  827. for j := 0; j < 18; j++ {
  828. info := m.SwarmInfo(c)
  829. // the desired CA cert and key is always redacted
  830. c.Assert(info.Cluster.Spec.CAConfig.SigningCAKey, checker.Equals, "")
  831. c.Assert(info.Cluster.Spec.CAConfig.SigningCACert, checker.Equals, "")
  832. clusterTLSInfo = info.Cluster.TLSInfo
  833. // if root rotation is done and the trust root has changed, we don't have to poll anymore
  834. if !info.Cluster.RootRotationInProgress && clusterTLSInfo.TrustRoot != currentTrustRoot {
  835. break
  836. }
  837. // root rotation not done
  838. time.Sleep(250 * time.Millisecond)
  839. }
  840. if cert != nil {
  841. c.Assert(clusterTLSInfo.TrustRoot, checker.Equals, expectedCert)
  842. }
  843. // could take another second or two for the nodes to trust the new roots after they've all gotten
  844. // new TLS certificates
  845. for j := 0; j < 18; j++ {
  846. mInfo := m.GetNode(c, m.NodeID()).Description.TLSInfo
  847. wInfo := m.GetNode(c, w.NodeID()).Description.TLSInfo
  848. if mInfo.TrustRoot == clusterTLSInfo.TrustRoot && wInfo.TrustRoot == clusterTLSInfo.TrustRoot {
  849. break
  850. }
  851. // nodes don't trust root certs yet
  852. time.Sleep(250 * time.Millisecond)
  853. }
  854. c.Assert(m.GetNode(c, m.NodeID()).Description.TLSInfo, checker.DeepEquals, clusterTLSInfo)
  855. c.Assert(m.GetNode(c, w.NodeID()).Description.TLSInfo, checker.DeepEquals, clusterTLSInfo)
  856. currentTrustRoot = clusterTLSInfo.TrustRoot
  857. }
  858. }
  859. func (s *DockerSwarmSuite) TestAPINetworkInspectWithScope(c *check.C) {
  860. d := s.AddDaemon(c, true, true)
  861. name := "test-scoped-network"
  862. ctx := context.Background()
  863. apiclient := d.NewClientT(c)
  864. resp, err := apiclient.NetworkCreate(ctx, name, types.NetworkCreate{Driver: "overlay"})
  865. assert.NilError(c, err)
  866. network, err := apiclient.NetworkInspect(ctx, name, types.NetworkInspectOptions{})
  867. assert.NilError(c, err)
  868. assert.Check(c, is.Equal("swarm", network.Scope))
  869. assert.Check(c, is.Equal(resp.ID, network.ID))
  870. _, err = apiclient.NetworkInspect(ctx, name, types.NetworkInspectOptions{Scope: "local"})
  871. assert.Check(c, client.IsErrNotFound(err))
  872. }