docker_api_swarm_test.go 29 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911
  1. // +build !windows
  2. package main
  3. import (
  4. "encoding/json"
  5. "fmt"
  6. "io/ioutil"
  7. "net"
  8. "net/http"
  9. "os"
  10. "path/filepath"
  11. "strings"
  12. "sync"
  13. "time"
  14. "github.com/cloudflare/cfssl/helpers"
  15. "github.com/docker/docker/api/types"
  16. "github.com/docker/docker/api/types/container"
  17. "github.com/docker/docker/api/types/swarm"
  18. "github.com/docker/docker/integration-cli/checker"
  19. "github.com/docker/docker/integration-cli/daemon"
  20. "github.com/go-check/check"
  21. )
  22. var defaultReconciliationTimeout = 30 * time.Second
  23. func (s *DockerSwarmSuite) TestAPISwarmInit(c *check.C) {
  24. // todo: should find a better way to verify that components are running than /info
  25. d1 := s.AddDaemon(c, true, true)
  26. info, err := d1.SwarmInfo()
  27. c.Assert(err, checker.IsNil)
  28. c.Assert(info.ControlAvailable, checker.True)
  29. c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive)
  30. d2 := s.AddDaemon(c, true, false)
  31. info, err = d2.SwarmInfo()
  32. c.Assert(err, checker.IsNil)
  33. c.Assert(info.ControlAvailable, checker.False)
  34. c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive)
  35. // Leaving cluster
  36. c.Assert(d2.Leave(false), checker.IsNil)
  37. info, err = d2.SwarmInfo()
  38. c.Assert(err, checker.IsNil)
  39. c.Assert(info.ControlAvailable, checker.False)
  40. c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateInactive)
  41. c.Assert(d2.Join(swarm.JoinRequest{JoinToken: d1.JoinTokens(c).Worker, RemoteAddrs: []string{d1.ListenAddr}}), checker.IsNil)
  42. info, err = d2.SwarmInfo()
  43. c.Assert(err, checker.IsNil)
  44. c.Assert(info.ControlAvailable, checker.False)
  45. c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive)
  46. // Current state restoring after restarts
  47. d1.Stop(c)
  48. d2.Stop(c)
  49. d1.Start(c)
  50. d2.Start(c)
  51. info, err = d1.SwarmInfo()
  52. c.Assert(err, checker.IsNil)
  53. c.Assert(info.ControlAvailable, checker.True)
  54. c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive)
  55. info, err = d2.SwarmInfo()
  56. c.Assert(err, checker.IsNil)
  57. c.Assert(info.ControlAvailable, checker.False)
  58. c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive)
  59. }
  60. func (s *DockerSwarmSuite) TestAPISwarmJoinToken(c *check.C) {
  61. d1 := s.AddDaemon(c, false, false)
  62. c.Assert(d1.Init(swarm.InitRequest{}), checker.IsNil)
  63. // todo: error message differs depending if some components of token are valid
  64. d2 := s.AddDaemon(c, false, false)
  65. err := d2.Join(swarm.JoinRequest{RemoteAddrs: []string{d1.ListenAddr}})
  66. c.Assert(err, checker.NotNil)
  67. c.Assert(err.Error(), checker.Contains, "join token is necessary")
  68. info, err := d2.SwarmInfo()
  69. c.Assert(err, checker.IsNil)
  70. c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateInactive)
  71. err = d2.Join(swarm.JoinRequest{JoinToken: "foobaz", RemoteAddrs: []string{d1.ListenAddr}})
  72. c.Assert(err, checker.NotNil)
  73. c.Assert(err.Error(), checker.Contains, "invalid join token")
  74. info, err = d2.SwarmInfo()
  75. c.Assert(err, checker.IsNil)
  76. c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateInactive)
  77. workerToken := d1.JoinTokens(c).Worker
  78. c.Assert(d2.Join(swarm.JoinRequest{JoinToken: workerToken, RemoteAddrs: []string{d1.ListenAddr}}), checker.IsNil)
  79. info, err = d2.SwarmInfo()
  80. c.Assert(err, checker.IsNil)
  81. c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive)
  82. c.Assert(d2.Leave(false), checker.IsNil)
  83. info, err = d2.SwarmInfo()
  84. c.Assert(err, checker.IsNil)
  85. c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateInactive)
  86. // change tokens
  87. d1.RotateTokens(c)
  88. err = d2.Join(swarm.JoinRequest{JoinToken: workerToken, RemoteAddrs: []string{d1.ListenAddr}})
  89. c.Assert(err, checker.NotNil)
  90. c.Assert(err.Error(), checker.Contains, "join token is necessary")
  91. info, err = d2.SwarmInfo()
  92. c.Assert(err, checker.IsNil)
  93. c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateInactive)
  94. workerToken = d1.JoinTokens(c).Worker
  95. c.Assert(d2.Join(swarm.JoinRequest{JoinToken: workerToken, RemoteAddrs: []string{d1.ListenAddr}}), checker.IsNil)
  96. info, err = d2.SwarmInfo()
  97. c.Assert(err, checker.IsNil)
  98. c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive)
  99. c.Assert(d2.Leave(false), checker.IsNil)
  100. info, err = d2.SwarmInfo()
  101. c.Assert(err, checker.IsNil)
  102. c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateInactive)
  103. // change spec, don't change tokens
  104. d1.UpdateSwarm(c, func(s *swarm.Spec) {})
  105. err = d2.Join(swarm.JoinRequest{RemoteAddrs: []string{d1.ListenAddr}})
  106. c.Assert(err, checker.NotNil)
  107. c.Assert(err.Error(), checker.Contains, "join token is necessary")
  108. info, err = d2.SwarmInfo()
  109. c.Assert(err, checker.IsNil)
  110. c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateInactive)
  111. c.Assert(d2.Join(swarm.JoinRequest{JoinToken: workerToken, RemoteAddrs: []string{d1.ListenAddr}}), checker.IsNil)
  112. info, err = d2.SwarmInfo()
  113. c.Assert(err, checker.IsNil)
  114. c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive)
  115. c.Assert(d2.Leave(false), checker.IsNil)
  116. info, err = d2.SwarmInfo()
  117. c.Assert(err, checker.IsNil)
  118. c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateInactive)
  119. }
  120. func (s *DockerSwarmSuite) TestAPISwarmCAHash(c *check.C) {
  121. d1 := s.AddDaemon(c, true, true)
  122. d2 := s.AddDaemon(c, false, false)
  123. splitToken := strings.Split(d1.JoinTokens(c).Worker, "-")
  124. splitToken[2] = "1kxftv4ofnc6mt30lmgipg6ngf9luhwqopfk1tz6bdmnkubg0e"
  125. replacementToken := strings.Join(splitToken, "-")
  126. err := d2.Join(swarm.JoinRequest{JoinToken: replacementToken, RemoteAddrs: []string{d1.ListenAddr}})
  127. c.Assert(err, checker.NotNil)
  128. c.Assert(err.Error(), checker.Contains, "remote CA does not match fingerprint")
  129. }
  130. func (s *DockerSwarmSuite) TestAPISwarmPromoteDemote(c *check.C) {
  131. d1 := s.AddDaemon(c, false, false)
  132. c.Assert(d1.Init(swarm.InitRequest{}), checker.IsNil)
  133. d2 := s.AddDaemon(c, true, false)
  134. info, err := d2.SwarmInfo()
  135. c.Assert(err, checker.IsNil)
  136. c.Assert(info.ControlAvailable, checker.False)
  137. c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive)
  138. d1.UpdateNode(c, d2.NodeID, func(n *swarm.Node) {
  139. n.Spec.Role = swarm.NodeRoleManager
  140. })
  141. waitAndAssert(c, defaultReconciliationTimeout, d2.CheckControlAvailable, checker.True)
  142. d1.UpdateNode(c, d2.NodeID, func(n *swarm.Node) {
  143. n.Spec.Role = swarm.NodeRoleWorker
  144. })
  145. waitAndAssert(c, defaultReconciliationTimeout, d2.CheckControlAvailable, checker.False)
  146. // Wait for the role to change to worker in the cert. This is partially
  147. // done because it's something worth testing in its own right, and
  148. // partially because changing the role from manager to worker and then
  149. // back to manager quickly might cause the node to pause for awhile
  150. // while waiting for the role to change to worker, and the test can
  151. // time out during this interval.
  152. waitAndAssert(c, defaultReconciliationTimeout, func(c *check.C) (interface{}, check.CommentInterface) {
  153. certBytes, err := ioutil.ReadFile(filepath.Join(d2.Folder, "root", "swarm", "certificates", "swarm-node.crt"))
  154. if err != nil {
  155. return "", check.Commentf("error: %v", err)
  156. }
  157. certs, err := helpers.ParseCertificatesPEM(certBytes)
  158. if err == nil && len(certs) > 0 && len(certs[0].Subject.OrganizationalUnit) > 0 {
  159. return certs[0].Subject.OrganizationalUnit[0], nil
  160. }
  161. return "", check.Commentf("could not get organizational unit from certificate")
  162. }, checker.Equals, "swarm-worker")
  163. // Demoting last node should fail
  164. node := d1.GetNode(c, d1.NodeID)
  165. node.Spec.Role = swarm.NodeRoleWorker
  166. url := fmt.Sprintf("/nodes/%s/update?version=%d", node.ID, node.Version.Index)
  167. status, out, err := d1.SockRequest("POST", url, node.Spec)
  168. c.Assert(err, checker.IsNil)
  169. c.Assert(status, checker.Equals, http.StatusInternalServerError, check.Commentf("output: %q", string(out)))
  170. // The warning specific to demoting the last manager is best-effort and
  171. // won't appear until the Role field of the demoted manager has been
  172. // updated.
  173. // Yes, I know this looks silly, but checker.Matches is broken, since
  174. // it anchors the regexp contrary to the documentation, and this makes
  175. // it impossible to match something that includes a line break.
  176. if !strings.Contains(string(out), "last manager of the swarm") {
  177. c.Assert(string(out), checker.Contains, "this would result in a loss of quorum")
  178. }
  179. info, err = d1.SwarmInfo()
  180. c.Assert(err, checker.IsNil)
  181. c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive)
  182. c.Assert(info.ControlAvailable, checker.True)
  183. // Promote already demoted node
  184. d1.UpdateNode(c, d2.NodeID, func(n *swarm.Node) {
  185. n.Spec.Role = swarm.NodeRoleManager
  186. })
  187. waitAndAssert(c, defaultReconciliationTimeout, d2.CheckControlAvailable, checker.True)
  188. }
  189. func (s *DockerSwarmSuite) TestAPISwarmLeaderProxy(c *check.C) {
  190. // add three managers, one of these is leader
  191. d1 := s.AddDaemon(c, true, true)
  192. d2 := s.AddDaemon(c, true, true)
  193. d3 := s.AddDaemon(c, true, true)
  194. // start a service by hitting each of the 3 managers
  195. d1.CreateService(c, simpleTestService, func(s *swarm.Service) {
  196. s.Spec.Name = "test1"
  197. })
  198. d2.CreateService(c, simpleTestService, func(s *swarm.Service) {
  199. s.Spec.Name = "test2"
  200. })
  201. d3.CreateService(c, simpleTestService, func(s *swarm.Service) {
  202. s.Spec.Name = "test3"
  203. })
  204. // 3 services should be started now, because the requests were proxied to leader
  205. // query each node and make sure it returns 3 services
  206. for _, d := range []*daemon.Swarm{d1, d2, d3} {
  207. services := d.ListServices(c)
  208. c.Assert(services, checker.HasLen, 3)
  209. }
  210. }
  211. func (s *DockerSwarmSuite) TestAPISwarmLeaderElection(c *check.C) {
  212. // Create 3 nodes
  213. d1 := s.AddDaemon(c, true, true)
  214. d2 := s.AddDaemon(c, true, true)
  215. d3 := s.AddDaemon(c, true, true)
  216. // assert that the first node we made is the leader, and the other two are followers
  217. c.Assert(d1.GetNode(c, d1.NodeID).ManagerStatus.Leader, checker.True)
  218. c.Assert(d1.GetNode(c, d2.NodeID).ManagerStatus.Leader, checker.False)
  219. c.Assert(d1.GetNode(c, d3.NodeID).ManagerStatus.Leader, checker.False)
  220. d1.Stop(c)
  221. var (
  222. leader *daemon.Swarm // keep track of leader
  223. followers []*daemon.Swarm // keep track of followers
  224. )
  225. checkLeader := func(nodes ...*daemon.Swarm) checkF {
  226. return func(c *check.C) (interface{}, check.CommentInterface) {
  227. // clear these out before each run
  228. leader = nil
  229. followers = nil
  230. for _, d := range nodes {
  231. if d.GetNode(c, d.NodeID).ManagerStatus.Leader {
  232. leader = d
  233. } else {
  234. followers = append(followers, d)
  235. }
  236. }
  237. if leader == nil {
  238. return false, check.Commentf("no leader elected")
  239. }
  240. return true, check.Commentf("elected %v", leader.ID())
  241. }
  242. }
  243. // wait for an election to occur
  244. waitAndAssert(c, defaultReconciliationTimeout, checkLeader(d2, d3), checker.True)
  245. // assert that we have a new leader
  246. c.Assert(leader, checker.NotNil)
  247. // Keep track of the current leader, since we want that to be chosen.
  248. stableleader := leader
  249. // add the d1, the initial leader, back
  250. d1.Start(c)
  251. // TODO(stevvooe): may need to wait for rejoin here
  252. // wait for possible election
  253. waitAndAssert(c, defaultReconciliationTimeout, checkLeader(d1, d2, d3), checker.True)
  254. // pick out the leader and the followers again
  255. // verify that we still only have 1 leader and 2 followers
  256. c.Assert(leader, checker.NotNil)
  257. c.Assert(followers, checker.HasLen, 2)
  258. // and that after we added d1 back, the leader hasn't changed
  259. c.Assert(leader.NodeID, checker.Equals, stableleader.NodeID)
  260. }
  261. func (s *DockerSwarmSuite) TestAPISwarmRaftQuorum(c *check.C) {
  262. d1 := s.AddDaemon(c, true, true)
  263. d2 := s.AddDaemon(c, true, true)
  264. d3 := s.AddDaemon(c, true, true)
  265. d1.CreateService(c, simpleTestService)
  266. d2.Stop(c)
  267. // make sure there is a leader
  268. waitAndAssert(c, defaultReconciliationTimeout, d1.CheckLeader, checker.IsNil)
  269. d1.CreateService(c, simpleTestService, func(s *swarm.Service) {
  270. s.Spec.Name = "top1"
  271. })
  272. d3.Stop(c)
  273. // make sure there is a leader
  274. waitAndAssert(c, defaultReconciliationTimeout, d1.CheckLeader, checker.IsNil)
  275. var service swarm.Service
  276. simpleTestService(&service)
  277. service.Spec.Name = "top2"
  278. status, out, err := d1.SockRequest("POST", "/services/create", service.Spec)
  279. c.Assert(err, checker.IsNil)
  280. c.Assert(status, checker.Equals, http.StatusInternalServerError, check.Commentf("deadline exceeded", string(out)))
  281. d2.Start(c)
  282. // make sure there is a leader
  283. waitAndAssert(c, defaultReconciliationTimeout, d1.CheckLeader, checker.IsNil)
  284. d1.CreateService(c, simpleTestService, func(s *swarm.Service) {
  285. s.Spec.Name = "top3"
  286. })
  287. }
  288. func (s *DockerSwarmSuite) TestAPISwarmLeaveRemovesContainer(c *check.C) {
  289. d := s.AddDaemon(c, true, true)
  290. instances := 2
  291. d.CreateService(c, simpleTestService, setInstances(instances))
  292. id, err := d.Cmd("run", "-d", "busybox", "top")
  293. c.Assert(err, checker.IsNil)
  294. id = strings.TrimSpace(id)
  295. waitAndAssert(c, defaultReconciliationTimeout, d.CheckActiveContainerCount, checker.Equals, instances+1)
  296. c.Assert(d.Leave(false), checker.NotNil)
  297. c.Assert(d.Leave(true), checker.IsNil)
  298. waitAndAssert(c, defaultReconciliationTimeout, d.CheckActiveContainerCount, checker.Equals, 1)
  299. id2, err := d.Cmd("ps", "-q")
  300. c.Assert(err, checker.IsNil)
  301. c.Assert(id, checker.HasPrefix, strings.TrimSpace(id2))
  302. }
  303. // #23629
  304. func (s *DockerSwarmSuite) TestAPISwarmLeaveOnPendingJoin(c *check.C) {
  305. testRequires(c, Network)
  306. s.AddDaemon(c, true, true)
  307. d2 := s.AddDaemon(c, false, false)
  308. id, err := d2.Cmd("run", "-d", "busybox", "top")
  309. c.Assert(err, checker.IsNil)
  310. id = strings.TrimSpace(id)
  311. err = d2.Join(swarm.JoinRequest{
  312. RemoteAddrs: []string{"123.123.123.123:1234"},
  313. })
  314. c.Assert(err, check.NotNil)
  315. c.Assert(err.Error(), checker.Contains, "Timeout was reached")
  316. info, err := d2.SwarmInfo()
  317. c.Assert(err, checker.IsNil)
  318. c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStatePending)
  319. c.Assert(d2.Leave(true), checker.IsNil)
  320. waitAndAssert(c, defaultReconciliationTimeout, d2.CheckActiveContainerCount, checker.Equals, 1)
  321. id2, err := d2.Cmd("ps", "-q")
  322. c.Assert(err, checker.IsNil)
  323. c.Assert(id, checker.HasPrefix, strings.TrimSpace(id2))
  324. }
  325. // #23705
  326. func (s *DockerSwarmSuite) TestAPISwarmRestoreOnPendingJoin(c *check.C) {
  327. testRequires(c, Network)
  328. d := s.AddDaemon(c, false, false)
  329. err := d.Join(swarm.JoinRequest{
  330. RemoteAddrs: []string{"123.123.123.123:1234"},
  331. })
  332. c.Assert(err, check.NotNil)
  333. c.Assert(err.Error(), checker.Contains, "Timeout was reached")
  334. waitAndAssert(c, defaultReconciliationTimeout, d.CheckLocalNodeState, checker.Equals, swarm.LocalNodeStatePending)
  335. d.Stop(c)
  336. d.Start(c)
  337. info, err := d.SwarmInfo()
  338. c.Assert(err, checker.IsNil)
  339. c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateInactive)
  340. }
  341. func (s *DockerSwarmSuite) TestAPISwarmManagerRestore(c *check.C) {
  342. d1 := s.AddDaemon(c, true, true)
  343. instances := 2
  344. id := d1.CreateService(c, simpleTestService, setInstances(instances))
  345. d1.GetService(c, id)
  346. d1.Stop(c)
  347. d1.Start(c)
  348. d1.GetService(c, id)
  349. d2 := s.AddDaemon(c, true, true)
  350. d2.GetService(c, id)
  351. d2.Stop(c)
  352. d2.Start(c)
  353. d2.GetService(c, id)
  354. d3 := s.AddDaemon(c, true, true)
  355. d3.GetService(c, id)
  356. d3.Stop(c)
  357. d3.Start(c)
  358. d3.GetService(c, id)
  359. d3.Kill()
  360. time.Sleep(1 * time.Second) // time to handle signal
  361. d3.Start(c)
  362. d3.GetService(c, id)
  363. }
  364. func (s *DockerSwarmSuite) TestAPISwarmScaleNoRollingUpdate(c *check.C) {
  365. d := s.AddDaemon(c, true, true)
  366. instances := 2
  367. id := d.CreateService(c, simpleTestService, setInstances(instances))
  368. waitAndAssert(c, defaultReconciliationTimeout, d.CheckActiveContainerCount, checker.Equals, instances)
  369. containers := d.ActiveContainers()
  370. instances = 4
  371. d.UpdateService(c, d.GetService(c, id), setInstances(instances))
  372. waitAndAssert(c, defaultReconciliationTimeout, d.CheckActiveContainerCount, checker.Equals, instances)
  373. containers2 := d.ActiveContainers()
  374. loop0:
  375. for _, c1 := range containers {
  376. for _, c2 := range containers2 {
  377. if c1 == c2 {
  378. continue loop0
  379. }
  380. }
  381. c.Errorf("container %v not found in new set %#v", c1, containers2)
  382. }
  383. }
  384. func (s *DockerSwarmSuite) TestAPISwarmInvalidAddress(c *check.C) {
  385. d := s.AddDaemon(c, false, false)
  386. req := swarm.InitRequest{
  387. ListenAddr: "",
  388. }
  389. status, _, err := d.SockRequest("POST", "/swarm/init", req)
  390. c.Assert(err, checker.IsNil)
  391. c.Assert(status, checker.Equals, http.StatusBadRequest)
  392. req2 := swarm.JoinRequest{
  393. ListenAddr: "0.0.0.0:2377",
  394. RemoteAddrs: []string{""},
  395. }
  396. status, _, err = d.SockRequest("POST", "/swarm/join", req2)
  397. c.Assert(err, checker.IsNil)
  398. c.Assert(status, checker.Equals, http.StatusBadRequest)
  399. }
  400. func (s *DockerSwarmSuite) TestAPISwarmForceNewCluster(c *check.C) {
  401. d1 := s.AddDaemon(c, true, true)
  402. d2 := s.AddDaemon(c, true, true)
  403. instances := 2
  404. id := d1.CreateService(c, simpleTestService, setInstances(instances))
  405. waitAndAssert(c, defaultReconciliationTimeout, reducedCheck(sumAsIntegers, d1.CheckActiveContainerCount, d2.CheckActiveContainerCount), checker.Equals, instances)
  406. // drain d2, all containers should move to d1
  407. d1.UpdateNode(c, d2.NodeID, func(n *swarm.Node) {
  408. n.Spec.Availability = swarm.NodeAvailabilityDrain
  409. })
  410. waitAndAssert(c, defaultReconciliationTimeout, d1.CheckActiveContainerCount, checker.Equals, instances)
  411. waitAndAssert(c, defaultReconciliationTimeout, d2.CheckActiveContainerCount, checker.Equals, 0)
  412. d2.Stop(c)
  413. c.Assert(d1.Init(swarm.InitRequest{
  414. ForceNewCluster: true,
  415. Spec: swarm.Spec{},
  416. }), checker.IsNil)
  417. waitAndAssert(c, defaultReconciliationTimeout, d1.CheckActiveContainerCount, checker.Equals, instances)
  418. d3 := s.AddDaemon(c, true, true)
  419. info, err := d3.SwarmInfo()
  420. c.Assert(err, checker.IsNil)
  421. c.Assert(info.ControlAvailable, checker.True)
  422. c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive)
  423. instances = 4
  424. d3.UpdateService(c, d3.GetService(c, id), setInstances(instances))
  425. waitAndAssert(c, defaultReconciliationTimeout, reducedCheck(sumAsIntegers, d1.CheckActiveContainerCount, d3.CheckActiveContainerCount), checker.Equals, instances)
  426. }
  427. func simpleTestService(s *swarm.Service) {
  428. ureplicas := uint64(1)
  429. restartDelay := time.Duration(100 * time.Millisecond)
  430. s.Spec = swarm.ServiceSpec{
  431. TaskTemplate: swarm.TaskSpec{
  432. ContainerSpec: swarm.ContainerSpec{
  433. Image: "busybox:latest",
  434. Command: []string{"/bin/top"},
  435. },
  436. RestartPolicy: &swarm.RestartPolicy{
  437. Delay: &restartDelay,
  438. },
  439. },
  440. Mode: swarm.ServiceMode{
  441. Replicated: &swarm.ReplicatedService{
  442. Replicas: &ureplicas,
  443. },
  444. },
  445. }
  446. s.Spec.Name = "top"
  447. }
  448. func serviceForUpdate(s *swarm.Service) {
  449. ureplicas := uint64(1)
  450. restartDelay := time.Duration(100 * time.Millisecond)
  451. s.Spec = swarm.ServiceSpec{
  452. TaskTemplate: swarm.TaskSpec{
  453. ContainerSpec: swarm.ContainerSpec{
  454. Image: "busybox:latest",
  455. Command: []string{"/bin/top"},
  456. },
  457. RestartPolicy: &swarm.RestartPolicy{
  458. Delay: &restartDelay,
  459. },
  460. },
  461. Mode: swarm.ServiceMode{
  462. Replicated: &swarm.ReplicatedService{
  463. Replicas: &ureplicas,
  464. },
  465. },
  466. UpdateConfig: &swarm.UpdateConfig{
  467. Parallelism: 2,
  468. Delay: 4 * time.Second,
  469. FailureAction: swarm.UpdateFailureActionContinue,
  470. },
  471. RollbackConfig: &swarm.UpdateConfig{
  472. Parallelism: 3,
  473. Delay: 4 * time.Second,
  474. FailureAction: swarm.UpdateFailureActionContinue,
  475. },
  476. }
  477. s.Spec.Name = "updatetest"
  478. }
  479. func setInstances(replicas int) daemon.ServiceConstructor {
  480. ureplicas := uint64(replicas)
  481. return func(s *swarm.Service) {
  482. s.Spec.Mode = swarm.ServiceMode{
  483. Replicated: &swarm.ReplicatedService{
  484. Replicas: &ureplicas,
  485. },
  486. }
  487. }
  488. }
  489. func setUpdateOrder(order string) daemon.ServiceConstructor {
  490. return func(s *swarm.Service) {
  491. if s.Spec.UpdateConfig == nil {
  492. s.Spec.UpdateConfig = &swarm.UpdateConfig{}
  493. }
  494. s.Spec.UpdateConfig.Order = order
  495. }
  496. }
  497. func setRollbackOrder(order string) daemon.ServiceConstructor {
  498. return func(s *swarm.Service) {
  499. if s.Spec.RollbackConfig == nil {
  500. s.Spec.RollbackConfig = &swarm.UpdateConfig{}
  501. }
  502. s.Spec.RollbackConfig.Order = order
  503. }
  504. }
  505. func setImage(image string) daemon.ServiceConstructor {
  506. return func(s *swarm.Service) {
  507. s.Spec.TaskTemplate.ContainerSpec.Image = image
  508. }
  509. }
  510. func setFailureAction(failureAction string) daemon.ServiceConstructor {
  511. return func(s *swarm.Service) {
  512. s.Spec.UpdateConfig.FailureAction = failureAction
  513. }
  514. }
  515. func setMaxFailureRatio(maxFailureRatio float32) daemon.ServiceConstructor {
  516. return func(s *swarm.Service) {
  517. s.Spec.UpdateConfig.MaxFailureRatio = maxFailureRatio
  518. }
  519. }
  520. func setParallelism(parallelism uint64) daemon.ServiceConstructor {
  521. return func(s *swarm.Service) {
  522. s.Spec.UpdateConfig.Parallelism = parallelism
  523. }
  524. }
  525. func setConstraints(constraints []string) daemon.ServiceConstructor {
  526. return func(s *swarm.Service) {
  527. if s.Spec.TaskTemplate.Placement == nil {
  528. s.Spec.TaskTemplate.Placement = &swarm.Placement{}
  529. }
  530. s.Spec.TaskTemplate.Placement.Constraints = constraints
  531. }
  532. }
  533. func setPlacementPrefs(prefs []swarm.PlacementPreference) daemon.ServiceConstructor {
  534. return func(s *swarm.Service) {
  535. if s.Spec.TaskTemplate.Placement == nil {
  536. s.Spec.TaskTemplate.Placement = &swarm.Placement{}
  537. }
  538. s.Spec.TaskTemplate.Placement.Preferences = prefs
  539. }
  540. }
  541. func setGlobalMode(s *swarm.Service) {
  542. s.Spec.Mode = swarm.ServiceMode{
  543. Global: &swarm.GlobalService{},
  544. }
  545. }
  546. func checkClusterHealth(c *check.C, cl []*daemon.Swarm, managerCount, workerCount int) {
  547. var totalMCount, totalWCount int
  548. for _, d := range cl {
  549. var (
  550. info swarm.Info
  551. err error
  552. )
  553. // check info in a waitAndAssert, because if the cluster doesn't have a leader, `info` will return an error
  554. checkInfo := func(c *check.C) (interface{}, check.CommentInterface) {
  555. info, err = d.SwarmInfo()
  556. return err, check.Commentf("cluster not ready in time")
  557. }
  558. waitAndAssert(c, defaultReconciliationTimeout, checkInfo, checker.IsNil)
  559. if !info.ControlAvailable {
  560. totalWCount++
  561. continue
  562. }
  563. var leaderFound bool
  564. totalMCount++
  565. var mCount, wCount int
  566. for _, n := range d.ListNodes(c) {
  567. waitReady := func(c *check.C) (interface{}, check.CommentInterface) {
  568. if n.Status.State == swarm.NodeStateReady {
  569. return true, nil
  570. }
  571. nn := d.GetNode(c, n.ID)
  572. n = *nn
  573. return n.Status.State == swarm.NodeStateReady, check.Commentf("state of node %s, reported by %s", n.ID, d.Info.NodeID)
  574. }
  575. waitAndAssert(c, defaultReconciliationTimeout, waitReady, checker.True)
  576. waitActive := func(c *check.C) (interface{}, check.CommentInterface) {
  577. if n.Spec.Availability == swarm.NodeAvailabilityActive {
  578. return true, nil
  579. }
  580. nn := d.GetNode(c, n.ID)
  581. n = *nn
  582. return n.Spec.Availability == swarm.NodeAvailabilityActive, check.Commentf("availability of node %s, reported by %s", n.ID, d.Info.NodeID)
  583. }
  584. waitAndAssert(c, defaultReconciliationTimeout, waitActive, checker.True)
  585. if n.Spec.Role == swarm.NodeRoleManager {
  586. c.Assert(n.ManagerStatus, checker.NotNil, check.Commentf("manager status of node %s (manager), reported by %s", n.ID, d.Info.NodeID))
  587. if n.ManagerStatus.Leader {
  588. leaderFound = true
  589. }
  590. mCount++
  591. } else {
  592. c.Assert(n.ManagerStatus, checker.IsNil, check.Commentf("manager status of node %s (worker), reported by %s", n.ID, d.Info.NodeID))
  593. wCount++
  594. }
  595. }
  596. c.Assert(leaderFound, checker.True, check.Commentf("lack of leader reported by node %s", info.NodeID))
  597. c.Assert(mCount, checker.Equals, managerCount, check.Commentf("managers count reported by node %s", info.NodeID))
  598. c.Assert(wCount, checker.Equals, workerCount, check.Commentf("workers count reported by node %s", info.NodeID))
  599. }
  600. c.Assert(totalMCount, checker.Equals, managerCount)
  601. c.Assert(totalWCount, checker.Equals, workerCount)
  602. }
  603. func (s *DockerSwarmSuite) TestAPISwarmRestartCluster(c *check.C) {
  604. mCount, wCount := 5, 1
  605. var nodes []*daemon.Swarm
  606. for i := 0; i < mCount; i++ {
  607. manager := s.AddDaemon(c, true, true)
  608. info, err := manager.SwarmInfo()
  609. c.Assert(err, checker.IsNil)
  610. c.Assert(info.ControlAvailable, checker.True)
  611. c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive)
  612. nodes = append(nodes, manager)
  613. }
  614. for i := 0; i < wCount; i++ {
  615. worker := s.AddDaemon(c, true, false)
  616. info, err := worker.SwarmInfo()
  617. c.Assert(err, checker.IsNil)
  618. c.Assert(info.ControlAvailable, checker.False)
  619. c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive)
  620. nodes = append(nodes, worker)
  621. }
  622. // stop whole cluster
  623. {
  624. var wg sync.WaitGroup
  625. wg.Add(len(nodes))
  626. errs := make(chan error, len(nodes))
  627. for _, d := range nodes {
  628. go func(daemon *daemon.Swarm) {
  629. defer wg.Done()
  630. if err := daemon.StopWithError(); err != nil {
  631. errs <- err
  632. }
  633. // FIXME(vdemeester) This is duplicated…
  634. if root := os.Getenv("DOCKER_REMAP_ROOT"); root != "" {
  635. daemon.Root = filepath.Dir(daemon.Root)
  636. }
  637. }(d)
  638. }
  639. wg.Wait()
  640. close(errs)
  641. for err := range errs {
  642. c.Assert(err, check.IsNil)
  643. }
  644. }
  645. // start whole cluster
  646. {
  647. var wg sync.WaitGroup
  648. wg.Add(len(nodes))
  649. errs := make(chan error, len(nodes))
  650. for _, d := range nodes {
  651. go func(daemon *daemon.Swarm) {
  652. defer wg.Done()
  653. if err := daemon.StartWithError("--iptables=false"); err != nil {
  654. errs <- err
  655. }
  656. }(d)
  657. }
  658. wg.Wait()
  659. close(errs)
  660. for err := range errs {
  661. c.Assert(err, check.IsNil)
  662. }
  663. }
  664. checkClusterHealth(c, nodes, mCount, wCount)
  665. }
  666. func (s *DockerSwarmSuite) TestAPISwarmServicesUpdateWithName(c *check.C) {
  667. d := s.AddDaemon(c, true, true)
  668. instances := 2
  669. id := d.CreateService(c, simpleTestService, setInstances(instances))
  670. waitAndAssert(c, defaultReconciliationTimeout, d.CheckActiveContainerCount, checker.Equals, instances)
  671. service := d.GetService(c, id)
  672. instances = 5
  673. setInstances(instances)(service)
  674. url := fmt.Sprintf("/services/%s/update?version=%d", service.Spec.Name, service.Version.Index)
  675. status, out, err := d.SockRequest("POST", url, service.Spec)
  676. c.Assert(err, checker.IsNil)
  677. c.Assert(status, checker.Equals, http.StatusOK, check.Commentf("output: %q", string(out)))
  678. waitAndAssert(c, defaultReconciliationTimeout, d.CheckActiveContainerCount, checker.Equals, instances)
  679. }
  680. // Unlocking an unlocked swarm results in an error
  681. func (s *DockerSwarmSuite) TestAPISwarmUnlockNotLocked(c *check.C) {
  682. d := s.AddDaemon(c, true, true)
  683. err := d.Unlock(swarm.UnlockRequest{UnlockKey: "wrong-key"})
  684. c.Assert(err, checker.NotNil)
  685. c.Assert(err.Error(), checker.Contains, "swarm is not locked")
  686. }
  687. // #29885
  688. func (s *DockerSwarmSuite) TestAPISwarmErrorHandling(c *check.C) {
  689. ln, err := net.Listen("tcp", fmt.Sprintf(":%d", defaultSwarmPort))
  690. c.Assert(err, checker.IsNil)
  691. defer ln.Close()
  692. d := s.AddDaemon(c, false, false)
  693. err = d.Init(swarm.InitRequest{})
  694. c.Assert(err, checker.NotNil)
  695. c.Assert(err.Error(), checker.Contains, "address already in use")
  696. }
  697. // Test case for 30242, where duplicate networks, with different drivers `bridge` and `overlay`,
  698. // caused both scopes to be `swarm` for `docker network inspect` and `docker network ls`.
  699. // This test makes sure the fixes correctly output scopes instead.
  700. func (s *DockerSwarmSuite) TestAPIDuplicateNetworks(c *check.C) {
  701. d := s.AddDaemon(c, true, true)
  702. name := "foo"
  703. networkCreateRequest := types.NetworkCreateRequest{
  704. Name: name,
  705. NetworkCreate: types.NetworkCreate{
  706. CheckDuplicate: false,
  707. },
  708. }
  709. var n1 types.NetworkCreateResponse
  710. networkCreateRequest.NetworkCreate.Driver = "bridge"
  711. status, out, err := d.SockRequest("POST", "/networks/create", networkCreateRequest)
  712. c.Assert(err, checker.IsNil, check.Commentf(string(out)))
  713. c.Assert(status, checker.Equals, http.StatusCreated, check.Commentf(string(out)))
  714. c.Assert(json.Unmarshal(out, &n1), checker.IsNil)
  715. var n2 types.NetworkCreateResponse
  716. networkCreateRequest.NetworkCreate.Driver = "overlay"
  717. status, out, err = d.SockRequest("POST", "/networks/create", networkCreateRequest)
  718. c.Assert(err, checker.IsNil, check.Commentf(string(out)))
  719. c.Assert(status, checker.Equals, http.StatusCreated, check.Commentf(string(out)))
  720. c.Assert(json.Unmarshal(out, &n2), checker.IsNil)
  721. var r1 types.NetworkResource
  722. status, out, err = d.SockRequest("GET", "/networks/"+n1.ID, nil)
  723. c.Assert(err, checker.IsNil, check.Commentf(string(out)))
  724. c.Assert(status, checker.Equals, http.StatusOK, check.Commentf(string(out)))
  725. c.Assert(json.Unmarshal(out, &r1), checker.IsNil)
  726. c.Assert(r1.Scope, checker.Equals, "local")
  727. var r2 types.NetworkResource
  728. status, out, err = d.SockRequest("GET", "/networks/"+n2.ID, nil)
  729. c.Assert(err, checker.IsNil, check.Commentf(string(out)))
  730. c.Assert(status, checker.Equals, http.StatusOK, check.Commentf(string(out)))
  731. c.Assert(json.Unmarshal(out, &r2), checker.IsNil)
  732. c.Assert(r2.Scope, checker.Equals, "swarm")
  733. }
  734. // Test case for 30178
  735. func (s *DockerSwarmSuite) TestAPISwarmHealthcheckNone(c *check.C) {
  736. d := s.AddDaemon(c, true, true)
  737. out, err := d.Cmd("network", "create", "-d", "overlay", "lb")
  738. c.Assert(err, checker.IsNil, check.Commentf(out))
  739. instances := 1
  740. d.CreateService(c, simpleTestService, setInstances(instances), func(s *swarm.Service) {
  741. s.Spec.TaskTemplate.ContainerSpec.Healthcheck = &container.HealthConfig{}
  742. s.Spec.TaskTemplate.Networks = []swarm.NetworkAttachmentConfig{
  743. {Target: "lb"},
  744. }
  745. })
  746. waitAndAssert(c, defaultReconciliationTimeout, d.CheckActiveContainerCount, checker.Equals, instances)
  747. containers := d.ActiveContainers()
  748. out, err = d.Cmd("exec", containers[0], "ping", "-c1", "-W3", "top")
  749. c.Assert(err, checker.IsNil, check.Commentf(out))
  750. }