docker_api_swarm_test.go 26 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823
  1. // +build !windows
  2. package main
  3. import (
  4. "fmt"
  5. "net/http"
  6. "strconv"
  7. "strings"
  8. "syscall"
  9. "time"
  10. "github.com/docker/docker/pkg/integration/checker"
  11. "github.com/docker/engine-api/types/swarm"
  12. "github.com/go-check/check"
  13. )
  14. var defaultReconciliationTimeout = 30 * time.Second
  15. func (s *DockerSwarmSuite) TestApiSwarmInit(c *check.C) {
  16. testRequires(c, Network)
  17. // todo: should find a better way to verify that components are running than /info
  18. d1 := s.AddDaemon(c, true, true)
  19. info, err := d1.info()
  20. c.Assert(err, checker.IsNil)
  21. c.Assert(info.ControlAvailable, checker.Equals, true)
  22. c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive)
  23. d2 := s.AddDaemon(c, true, false)
  24. info, err = d2.info()
  25. c.Assert(err, checker.IsNil)
  26. c.Assert(info.ControlAvailable, checker.Equals, false)
  27. c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive)
  28. // Leaving cluster
  29. c.Assert(d2.Leave(false), checker.IsNil)
  30. info, err = d2.info()
  31. c.Assert(err, checker.IsNil)
  32. c.Assert(info.ControlAvailable, checker.Equals, false)
  33. c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateInactive)
  34. c.Assert(d2.Join(swarm.JoinRequest{RemoteAddrs: []string{d1.listenAddr}}), checker.IsNil)
  35. info, err = d2.info()
  36. c.Assert(err, checker.IsNil)
  37. c.Assert(info.ControlAvailable, checker.Equals, false)
  38. c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive)
  39. // Current state restoring after restarts
  40. err = d1.Stop()
  41. c.Assert(err, checker.IsNil)
  42. err = d2.Stop()
  43. c.Assert(err, checker.IsNil)
  44. err = d1.Start()
  45. c.Assert(err, checker.IsNil)
  46. err = d2.Start()
  47. c.Assert(err, checker.IsNil)
  48. info, err = d1.info()
  49. c.Assert(err, checker.IsNil)
  50. c.Assert(info.ControlAvailable, checker.Equals, true)
  51. c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive)
  52. info, err = d2.info()
  53. c.Assert(err, checker.IsNil)
  54. c.Assert(info.ControlAvailable, checker.Equals, false)
  55. c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive)
  56. }
  57. func (s *DockerSwarmSuite) TestApiSwarmManualAcceptance(c *check.C) {
  58. testRequires(c, Network)
  59. s.testAPISwarmManualAcceptance(c, "")
  60. }
  61. func (s *DockerSwarmSuite) TestApiSwarmManualAcceptanceSecret(c *check.C) {
  62. testRequires(c, Network)
  63. s.testAPISwarmManualAcceptance(c, "foobaz")
  64. }
  65. func (s *DockerSwarmSuite) testAPISwarmManualAcceptance(c *check.C, secret string) {
  66. d1 := s.AddDaemon(c, false, false)
  67. c.Assert(d1.Init(swarm.InitRequest{
  68. Spec: swarm.Spec{
  69. AcceptancePolicy: swarm.AcceptancePolicy{
  70. Policies: []swarm.Policy{
  71. {Role: swarm.NodeRoleWorker, Secret: &secret},
  72. {Role: swarm.NodeRoleManager, Secret: &secret},
  73. },
  74. },
  75. },
  76. }), checker.IsNil)
  77. d2 := s.AddDaemon(c, false, false)
  78. err := d2.Join(swarm.JoinRequest{RemoteAddrs: []string{d1.listenAddr}})
  79. c.Assert(err, checker.NotNil)
  80. if secret == "" {
  81. c.Assert(err.Error(), checker.Contains, "needs to be accepted")
  82. info, err := d2.info()
  83. c.Assert(err, checker.IsNil)
  84. c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStatePending)
  85. c.Assert(d2.Leave(false), checker.IsNil)
  86. info, err = d2.info()
  87. c.Assert(err, checker.IsNil)
  88. c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateInactive)
  89. } else {
  90. c.Assert(err.Error(), checker.Contains, "valid secret token is necessary")
  91. info, err := d2.info()
  92. c.Assert(err, checker.IsNil)
  93. c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateInactive)
  94. }
  95. d3 := s.AddDaemon(c, false, false)
  96. c.Assert(d3.Join(swarm.JoinRequest{Secret: secret, RemoteAddrs: []string{d1.listenAddr}}), checker.NotNil)
  97. info, err := d3.info()
  98. c.Assert(err, checker.IsNil)
  99. c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStatePending)
  100. c.Assert(len(info.NodeID), checker.GreaterThan, 5)
  101. d1.updateNode(c, info.NodeID, func(n *swarm.Node) {
  102. n.Spec.Membership = swarm.NodeMembershipAccepted
  103. })
  104. for i := 0; ; i++ {
  105. info, err := d3.info()
  106. c.Assert(err, checker.IsNil)
  107. if info.LocalNodeState == swarm.LocalNodeStateActive {
  108. break
  109. }
  110. if i > 100 {
  111. c.Fatalf("node did not become active")
  112. }
  113. time.Sleep(200 * time.Millisecond)
  114. }
  115. }
  116. func (s *DockerSwarmSuite) TestApiSwarmSecretAcceptance(c *check.C) {
  117. testRequires(c, Network)
  118. d1 := s.AddDaemon(c, false, false)
  119. secret := "foobar"
  120. c.Assert(d1.Init(swarm.InitRequest{
  121. Spec: swarm.Spec{
  122. AcceptancePolicy: swarm.AcceptancePolicy{
  123. Policies: []swarm.Policy{
  124. {Role: swarm.NodeRoleWorker, Autoaccept: true, Secret: &secret},
  125. {Role: swarm.NodeRoleManager, Secret: &secret},
  126. },
  127. },
  128. },
  129. }), checker.IsNil)
  130. d2 := s.AddDaemon(c, false, false)
  131. err := d2.Join(swarm.JoinRequest{RemoteAddrs: []string{d1.listenAddr}})
  132. c.Assert(err, checker.NotNil)
  133. c.Assert(err.Error(), checker.Contains, "secret token is necessary")
  134. info, err := d2.info()
  135. c.Assert(err, checker.IsNil)
  136. c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateInactive)
  137. err = d2.Join(swarm.JoinRequest{Secret: "foobaz", RemoteAddrs: []string{d1.listenAddr}})
  138. c.Assert(err, checker.NotNil)
  139. c.Assert(err.Error(), checker.Contains, "secret token is necessary")
  140. info, err = d2.info()
  141. c.Assert(err, checker.IsNil)
  142. c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateInactive)
  143. c.Assert(d2.Join(swarm.JoinRequest{Secret: "foobar", RemoteAddrs: []string{d1.listenAddr}}), checker.IsNil)
  144. info, err = d2.info()
  145. c.Assert(err, checker.IsNil)
  146. c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive)
  147. c.Assert(d2.Leave(false), checker.IsNil)
  148. info, err = d2.info()
  149. c.Assert(err, checker.IsNil)
  150. c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateInactive)
  151. // change secret
  152. d1.updateSwarm(c, func(s *swarm.Spec) {
  153. for i := range s.AcceptancePolicy.Policies {
  154. p := "foobaz"
  155. s.AcceptancePolicy.Policies[i].Secret = &p
  156. }
  157. })
  158. err = d2.Join(swarm.JoinRequest{Secret: "foobar", RemoteAddrs: []string{d1.listenAddr}})
  159. c.Assert(err, checker.NotNil)
  160. c.Assert(err.Error(), checker.Contains, "secret token is necessary")
  161. info, err = d2.info()
  162. c.Assert(err, checker.IsNil)
  163. c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateInactive)
  164. c.Assert(d2.Join(swarm.JoinRequest{Secret: "foobaz", RemoteAddrs: []string{d1.listenAddr}}), checker.IsNil)
  165. info, err = d2.info()
  166. c.Assert(err, checker.IsNil)
  167. c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive)
  168. c.Assert(d2.Leave(false), checker.IsNil)
  169. info, err = d2.info()
  170. c.Assert(err, checker.IsNil)
  171. c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateInactive)
  172. // change policy, don't change secret
  173. d1.updateSwarm(c, func(s *swarm.Spec) {
  174. for i, p := range s.AcceptancePolicy.Policies {
  175. if p.Role == swarm.NodeRoleManager {
  176. s.AcceptancePolicy.Policies[i].Autoaccept = false
  177. }
  178. s.AcceptancePolicy.Policies[i].Secret = nil
  179. }
  180. })
  181. err = d2.Join(swarm.JoinRequest{RemoteAddrs: []string{d1.listenAddr}})
  182. c.Assert(err, checker.NotNil)
  183. c.Assert(err.Error(), checker.Contains, "secret token is necessary")
  184. info, err = d2.info()
  185. c.Assert(err, checker.IsNil)
  186. c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateInactive)
  187. c.Assert(d2.Join(swarm.JoinRequest{Secret: "foobaz", RemoteAddrs: []string{d1.listenAddr}}), checker.IsNil)
  188. info, err = d2.info()
  189. c.Assert(err, checker.IsNil)
  190. c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive)
  191. c.Assert(d2.Leave(false), checker.IsNil)
  192. info, err = d2.info()
  193. c.Assert(err, checker.IsNil)
  194. c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateInactive)
  195. // clear secret
  196. d1.updateSwarm(c, func(s *swarm.Spec) {
  197. for i := range s.AcceptancePolicy.Policies {
  198. p := ""
  199. s.AcceptancePolicy.Policies[i].Secret = &p
  200. }
  201. })
  202. c.Assert(d2.Join(swarm.JoinRequest{RemoteAddrs: []string{d1.listenAddr}}), checker.IsNil)
  203. info, err = d2.info()
  204. c.Assert(err, checker.IsNil)
  205. c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive)
  206. c.Assert(d2.Leave(false), checker.IsNil)
  207. info, err = d2.info()
  208. c.Assert(err, checker.IsNil)
  209. c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateInactive)
  210. }
  211. func (s *DockerSwarmSuite) TestApiSwarmCAHash(c *check.C) {
  212. testRequires(c, Network)
  213. d1 := s.AddDaemon(c, true, true)
  214. d2 := s.AddDaemon(c, false, false)
  215. err := d2.Join(swarm.JoinRequest{CACertHash: "foobar", RemoteAddrs: []string{d1.listenAddr}})
  216. c.Assert(err, checker.NotNil)
  217. c.Assert(err.Error(), checker.Contains, "invalid checksum digest format")
  218. c.Assert(len(d1.CACertHash), checker.GreaterThan, 0)
  219. c.Assert(d2.Join(swarm.JoinRequest{CACertHash: d1.CACertHash, RemoteAddrs: []string{d1.listenAddr}}), checker.IsNil)
  220. }
  221. func (s *DockerSwarmSuite) TestApiSwarmPromoteDemote(c *check.C) {
  222. testRequires(c, Network)
  223. d1 := s.AddDaemon(c, false, false)
  224. c.Assert(d1.Init(swarm.InitRequest{
  225. Spec: swarm.Spec{
  226. AcceptancePolicy: swarm.AcceptancePolicy{
  227. Policies: []swarm.Policy{
  228. {Role: swarm.NodeRoleWorker, Autoaccept: true},
  229. {Role: swarm.NodeRoleManager},
  230. },
  231. },
  232. },
  233. }), checker.IsNil)
  234. d2 := s.AddDaemon(c, true, false)
  235. info, err := d2.info()
  236. c.Assert(err, checker.IsNil)
  237. c.Assert(info.ControlAvailable, checker.Equals, false)
  238. c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive)
  239. d1.updateNode(c, d2.NodeID, func(n *swarm.Node) {
  240. n.Spec.Role = swarm.NodeRoleManager
  241. })
  242. for i := 0; ; i++ {
  243. info, err := d2.info()
  244. c.Assert(err, checker.IsNil)
  245. c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive)
  246. if info.ControlAvailable {
  247. break
  248. }
  249. if i > 100 {
  250. c.Errorf("node did not turn into manager")
  251. }
  252. time.Sleep(100 * time.Millisecond)
  253. }
  254. d1.updateNode(c, d2.NodeID, func(n *swarm.Node) {
  255. n.Spec.Role = swarm.NodeRoleWorker
  256. })
  257. for i := 0; ; i++ {
  258. info, err := d2.info()
  259. c.Assert(err, checker.IsNil)
  260. c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive)
  261. if !info.ControlAvailable {
  262. break
  263. }
  264. if i > 100 {
  265. c.Errorf("node did not turn into worker")
  266. }
  267. time.Sleep(100 * time.Millisecond)
  268. }
  269. // Demoting last node should fail
  270. node := d1.getNode(c, d1.NodeID)
  271. node.Spec.Role = swarm.NodeRoleWorker
  272. url := fmt.Sprintf("/nodes/%s/update?version=%d", node.ID, node.Version.Index)
  273. status, out, err := d1.SockRequest("POST", url, node.Spec)
  274. c.Assert(err, checker.IsNil)
  275. c.Assert(status, checker.Equals, http.StatusInternalServerError, check.Commentf("output: %q", string(out)))
  276. c.Assert(string(out), checker.Contains, "last manager of the swarm")
  277. info, err = d1.info()
  278. c.Assert(err, checker.IsNil)
  279. c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive)
  280. c.Assert(info.ControlAvailable, checker.Equals, true)
  281. // Promote already demoted node
  282. d1.updateNode(c, d2.NodeID, func(n *swarm.Node) {
  283. n.Spec.Role = swarm.NodeRoleManager
  284. })
  285. for i := 0; ; i++ {
  286. info, err := d2.info()
  287. c.Assert(err, checker.IsNil)
  288. c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive)
  289. if info.ControlAvailable {
  290. break
  291. }
  292. if i > 100 {
  293. c.Errorf("node did not turn into manager")
  294. }
  295. time.Sleep(100 * time.Millisecond)
  296. }
  297. }
  298. func (s *DockerSwarmSuite) TestApiSwarmServicesCreate(c *check.C) {
  299. testRequires(c, Network)
  300. d := s.AddDaemon(c, true, true)
  301. instances := 2
  302. id := d.createService(c, simpleTestService, setInstances(instances))
  303. waitAndAssert(c, defaultReconciliationTimeout, d.checkActiveContainerCount, checker.Equals, instances)
  304. service := d.getService(c, id)
  305. instances = 5
  306. d.updateService(c, service, setInstances(instances))
  307. waitAndAssert(c, defaultReconciliationTimeout, d.checkActiveContainerCount, checker.Equals, instances)
  308. d.removeService(c, service.ID)
  309. waitAndAssert(c, defaultReconciliationTimeout, d.checkActiveContainerCount, checker.Equals, 0)
  310. }
  311. func (s *DockerSwarmSuite) TestApiSwarmServicesMultipleAgents(c *check.C) {
  312. testRequires(c, Network)
  313. d1 := s.AddDaemon(c, true, true)
  314. d2 := s.AddDaemon(c, true, false)
  315. d3 := s.AddDaemon(c, true, false)
  316. time.Sleep(1 * time.Second) // make sure all daemons are ready to accept tasks
  317. instances := 9
  318. id := d1.createService(c, simpleTestService, setInstances(instances))
  319. waitAndAssert(c, defaultReconciliationTimeout, d1.checkActiveContainerCount, checker.GreaterThan, 0)
  320. waitAndAssert(c, defaultReconciliationTimeout, d2.checkActiveContainerCount, checker.GreaterThan, 0)
  321. waitAndAssert(c, defaultReconciliationTimeout, d3.checkActiveContainerCount, checker.GreaterThan, 0)
  322. waitAndAssert(c, defaultReconciliationTimeout, reducedCheck(sumAsIntegers, d1.checkActiveContainerCount, d2.checkActiveContainerCount, d3.checkActiveContainerCount), checker.Equals, instances)
  323. // reconciliation on d2 node down
  324. c.Assert(d2.Stop(), checker.IsNil)
  325. waitAndAssert(c, defaultReconciliationTimeout, reducedCheck(sumAsIntegers, d1.checkActiveContainerCount, d3.checkActiveContainerCount), checker.Equals, instances)
  326. // test downscaling
  327. instances = 5
  328. d1.updateService(c, d1.getService(c, id), setInstances(instances))
  329. waitAndAssert(c, defaultReconciliationTimeout, reducedCheck(sumAsIntegers, d1.checkActiveContainerCount, d3.checkActiveContainerCount), checker.Equals, instances)
  330. }
  331. func (s *DockerSwarmSuite) TestApiSwarmServicesCreateGlobal(c *check.C) {
  332. testRequires(c, Network)
  333. d1 := s.AddDaemon(c, true, true)
  334. d2 := s.AddDaemon(c, true, false)
  335. d3 := s.AddDaemon(c, true, false)
  336. d1.createService(c, simpleTestService, setGlobalMode)
  337. waitAndAssert(c, defaultReconciliationTimeout, d1.checkActiveContainerCount, checker.Equals, 1)
  338. waitAndAssert(c, defaultReconciliationTimeout, d2.checkActiveContainerCount, checker.Equals, 1)
  339. waitAndAssert(c, defaultReconciliationTimeout, d3.checkActiveContainerCount, checker.Equals, 1)
  340. d4 := s.AddDaemon(c, true, false)
  341. d5 := s.AddDaemon(c, true, false)
  342. waitAndAssert(c, defaultReconciliationTimeout, d4.checkActiveContainerCount, checker.Equals, 1)
  343. waitAndAssert(c, defaultReconciliationTimeout, d5.checkActiveContainerCount, checker.Equals, 1)
  344. }
  345. func (s *DockerSwarmSuite) TestApiSwarmServicesStateReporting(c *check.C) {
  346. testRequires(c, Network)
  347. testRequires(c, SameHostDaemon)
  348. testRequires(c, DaemonIsLinux)
  349. d1 := s.AddDaemon(c, true, true)
  350. d2 := s.AddDaemon(c, true, true)
  351. d3 := s.AddDaemon(c, true, false)
  352. time.Sleep(1 * time.Second) // make sure all daemons are ready to accept
  353. instances := 9
  354. d1.createService(c, simpleTestService, setInstances(instances))
  355. waitAndAssert(c, defaultReconciliationTimeout, reducedCheck(sumAsIntegers, d1.checkActiveContainerCount, d2.checkActiveContainerCount, d3.checkActiveContainerCount), checker.Equals, instances)
  356. getContainers := func() map[string]*SwarmDaemon {
  357. m := make(map[string]*SwarmDaemon)
  358. for _, d := range []*SwarmDaemon{d1, d2, d3} {
  359. for _, id := range d.activeContainers() {
  360. m[id] = d
  361. }
  362. }
  363. return m
  364. }
  365. containers := getContainers()
  366. c.Assert(containers, checker.HasLen, instances)
  367. var toRemove string
  368. for i := range containers {
  369. toRemove = i
  370. }
  371. _, err := containers[toRemove].Cmd("stop", toRemove)
  372. c.Assert(err, checker.IsNil)
  373. waitAndAssert(c, defaultReconciliationTimeout, reducedCheck(sumAsIntegers, d1.checkActiveContainerCount, d2.checkActiveContainerCount, d3.checkActiveContainerCount), checker.Equals, instances)
  374. containers2 := getContainers()
  375. c.Assert(containers2, checker.HasLen, instances)
  376. for i := range containers {
  377. if i == toRemove {
  378. c.Assert(containers2[i], checker.IsNil)
  379. } else {
  380. c.Assert(containers2[i], checker.NotNil)
  381. }
  382. }
  383. containers = containers2
  384. for i := range containers {
  385. toRemove = i
  386. }
  387. // try with killing process outside of docker
  388. pidStr, err := containers[toRemove].Cmd("inspect", "-f", "{{.State.Pid}}", toRemove)
  389. c.Assert(err, checker.IsNil)
  390. pid, err := strconv.Atoi(strings.TrimSpace(pidStr))
  391. c.Assert(err, checker.IsNil)
  392. c.Assert(syscall.Kill(pid, syscall.SIGKILL), checker.IsNil)
  393. time.Sleep(time.Second) // give some time to handle the signal
  394. waitAndAssert(c, defaultReconciliationTimeout, reducedCheck(sumAsIntegers, d1.checkActiveContainerCount, d2.checkActiveContainerCount, d3.checkActiveContainerCount), checker.Equals, instances)
  395. containers2 = getContainers()
  396. c.Assert(containers2, checker.HasLen, instances)
  397. for i := range containers {
  398. if i == toRemove {
  399. c.Assert(containers2[i], checker.IsNil)
  400. } else {
  401. c.Assert(containers2[i], checker.NotNil)
  402. }
  403. }
  404. }
  405. func (s *DockerSwarmSuite) TestApiSwarmRaftQuorum(c *check.C) {
  406. testRequires(c, Network)
  407. d1 := s.AddDaemon(c, true, true)
  408. d2 := s.AddDaemon(c, true, true)
  409. d3 := s.AddDaemon(c, true, true)
  410. d1.createService(c, simpleTestService)
  411. c.Assert(d2.Stop(), checker.IsNil)
  412. d1.createService(c, simpleTestService, func(s *swarm.Service) {
  413. s.Spec.Name = "top1"
  414. })
  415. c.Assert(d3.Stop(), checker.IsNil)
  416. var service swarm.Service
  417. simpleTestService(&service)
  418. service.Spec.Name = "top2"
  419. status, out, err := d1.SockRequest("POST", "/services/create", service.Spec)
  420. c.Assert(err, checker.IsNil)
  421. c.Assert(status, checker.Equals, http.StatusInternalServerError, check.Commentf("deadline exceeded", string(out)))
  422. c.Assert(d2.Start(), checker.IsNil)
  423. d1.createService(c, simpleTestService, func(s *swarm.Service) {
  424. s.Spec.Name = "top3"
  425. })
  426. }
  427. func (s *DockerSwarmSuite) TestApiSwarmListNodes(c *check.C) {
  428. testRequires(c, Network)
  429. d1 := s.AddDaemon(c, true, true)
  430. d2 := s.AddDaemon(c, true, false)
  431. d3 := s.AddDaemon(c, true, false)
  432. nodes := d1.listNodes(c)
  433. c.Assert(len(nodes), checker.Equals, 3, check.Commentf("nodes: %#v", nodes))
  434. loop0:
  435. for _, n := range nodes {
  436. for _, d := range []*SwarmDaemon{d1, d2, d3} {
  437. if n.ID == d.NodeID {
  438. continue loop0
  439. }
  440. }
  441. c.Errorf("unknown nodeID %v", n.ID)
  442. }
  443. }
  444. func (s *DockerSwarmSuite) TestApiSwarmNodeUpdate(c *check.C) {
  445. testRequires(c, Network)
  446. d := s.AddDaemon(c, true, true)
  447. nodes := d.listNodes(c)
  448. d.updateNode(c, nodes[0].ID, func(n *swarm.Node) {
  449. n.Spec.Availability = swarm.NodeAvailabilityPause
  450. })
  451. n := d.getNode(c, nodes[0].ID)
  452. c.Assert(n.Spec.Availability, checker.Equals, swarm.NodeAvailabilityPause)
  453. }
  454. func (s *DockerSwarmSuite) TestApiSwarmNodeDrainPause(c *check.C) {
  455. testRequires(c, Network)
  456. d1 := s.AddDaemon(c, true, true)
  457. d2 := s.AddDaemon(c, true, false)
  458. time.Sleep(1 * time.Second) // make sure all daemons are ready to accept tasks
  459. // start a service, expect balanced distribution
  460. instances := 8
  461. id := d1.createService(c, simpleTestService, setInstances(instances))
  462. waitAndAssert(c, defaultReconciliationTimeout, d1.checkActiveContainerCount, checker.GreaterThan, 0)
  463. waitAndAssert(c, defaultReconciliationTimeout, d2.checkActiveContainerCount, checker.GreaterThan, 0)
  464. waitAndAssert(c, defaultReconciliationTimeout, reducedCheck(sumAsIntegers, d1.checkActiveContainerCount, d2.checkActiveContainerCount), checker.Equals, instances)
  465. // drain d2, all containers should move to d1
  466. d1.updateNode(c, d2.NodeID, func(n *swarm.Node) {
  467. n.Spec.Availability = swarm.NodeAvailabilityDrain
  468. })
  469. waitAndAssert(c, defaultReconciliationTimeout, d1.checkActiveContainerCount, checker.Equals, instances)
  470. waitAndAssert(c, defaultReconciliationTimeout, d2.checkActiveContainerCount, checker.Equals, 0)
  471. // set d2 back to active
  472. d1.updateNode(c, d2.NodeID, func(n *swarm.Node) {
  473. n.Spec.Availability = swarm.NodeAvailabilityActive
  474. })
  475. instances = 1
  476. d1.updateService(c, d1.getService(c, id), setInstances(instances))
  477. waitAndAssert(c, defaultReconciliationTimeout*2, reducedCheck(sumAsIntegers, d1.checkActiveContainerCount, d2.checkActiveContainerCount), checker.Equals, instances)
  478. instances = 8
  479. d1.updateService(c, d1.getService(c, id), setInstances(instances))
  480. // drained node first so we don't get any old containers
  481. waitAndAssert(c, defaultReconciliationTimeout, d2.checkActiveContainerCount, checker.GreaterThan, 0)
  482. waitAndAssert(c, defaultReconciliationTimeout, d1.checkActiveContainerCount, checker.GreaterThan, 0)
  483. waitAndAssert(c, defaultReconciliationTimeout*2, reducedCheck(sumAsIntegers, d1.checkActiveContainerCount, d2.checkActiveContainerCount), checker.Equals, instances)
  484. d2ContainerCount := len(d2.activeContainers())
  485. // set d2 to paused, scale service up, only d1 gets new tasks
  486. d1.updateNode(c, d2.NodeID, func(n *swarm.Node) {
  487. n.Spec.Availability = swarm.NodeAvailabilityPause
  488. })
  489. instances = 14
  490. d1.updateService(c, d1.getService(c, id), setInstances(instances))
  491. waitAndAssert(c, defaultReconciliationTimeout, d1.checkActiveContainerCount, checker.Equals, instances-d2ContainerCount)
  492. waitAndAssert(c, defaultReconciliationTimeout, d2.checkActiveContainerCount, checker.Equals, d2ContainerCount)
  493. }
  494. func (s *DockerSwarmSuite) TestApiSwarmLeaveRemovesContainer(c *check.C) {
  495. testRequires(c, Network)
  496. d := s.AddDaemon(c, true, true)
  497. instances := 2
  498. d.createService(c, simpleTestService, setInstances(instances))
  499. id, err := d.Cmd("run", "-d", "busybox", "top")
  500. c.Assert(err, checker.IsNil)
  501. id = strings.TrimSpace(id)
  502. waitAndAssert(c, defaultReconciliationTimeout, d.checkActiveContainerCount, checker.Equals, instances+1)
  503. c.Assert(d.Leave(false), checker.NotNil)
  504. c.Assert(d.Leave(true), checker.IsNil)
  505. waitAndAssert(c, defaultReconciliationTimeout, d.checkActiveContainerCount, checker.Equals, 1)
  506. id2, err := d.Cmd("ps", "-q")
  507. c.Assert(err, checker.IsNil)
  508. c.Assert(id, checker.HasPrefix, strings.TrimSpace(id2))
  509. }
  510. // #23629
  511. func (s *DockerSwarmSuite) TestApiSwarmLeaveOnPendingJoin(c *check.C) {
  512. s.AddDaemon(c, true, true)
  513. d2 := s.AddDaemon(c, false, false)
  514. id, err := d2.Cmd("run", "-d", "busybox", "top")
  515. c.Assert(err, checker.IsNil)
  516. id = strings.TrimSpace(id)
  517. go d2.Join(swarm.JoinRequest{
  518. RemoteAddrs: []string{"nosuchhost:1234"},
  519. }) // will block on pending state
  520. for i := 0; ; i++ {
  521. info, err := d2.info()
  522. c.Assert(err, checker.IsNil)
  523. if info.LocalNodeState == swarm.LocalNodeStatePending {
  524. break
  525. }
  526. if i > 100 {
  527. c.Fatalf("node did not go to pending state: %v", info.LocalNodeState)
  528. }
  529. time.Sleep(100 * time.Millisecond)
  530. }
  531. c.Assert(d2.Leave(true), checker.IsNil)
  532. waitAndAssert(c, defaultReconciliationTimeout, d2.checkActiveContainerCount, checker.Equals, 1)
  533. id2, err := d2.Cmd("ps", "-q")
  534. c.Assert(err, checker.IsNil)
  535. c.Assert(id, checker.HasPrefix, strings.TrimSpace(id2))
  536. }
  537. // #23705
  538. func (s *DockerSwarmSuite) TestApiSwarmRestoreOnPendingJoin(c *check.C) {
  539. d := s.AddDaemon(c, false, false)
  540. go d.Join(swarm.JoinRequest{
  541. RemoteAddrs: []string{"nosuchhost:1234"},
  542. }) // will block on pending state
  543. for i := 0; ; i++ {
  544. info, err := d.info()
  545. c.Assert(err, checker.IsNil)
  546. if info.LocalNodeState == swarm.LocalNodeStatePending {
  547. break
  548. }
  549. if i > 100 {
  550. c.Fatalf("node did not go to pending state: %v", info.LocalNodeState)
  551. }
  552. time.Sleep(100 * time.Millisecond)
  553. }
  554. c.Assert(d.Stop(), checker.IsNil)
  555. c.Assert(d.Start(), checker.IsNil)
  556. info, err := d.info()
  557. c.Assert(err, checker.IsNil)
  558. c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateInactive)
  559. }
  560. func (s *DockerSwarmSuite) TestApiSwarmManagerRestore(c *check.C) {
  561. testRequires(c, Network)
  562. d1 := s.AddDaemon(c, true, true)
  563. instances := 2
  564. id := d1.createService(c, simpleTestService, setInstances(instances))
  565. d1.getService(c, id)
  566. d1.Stop()
  567. d1.Start()
  568. d1.getService(c, id)
  569. d2 := s.AddDaemon(c, true, true)
  570. d2.getService(c, id)
  571. d2.Stop()
  572. d2.Start()
  573. d2.getService(c, id)
  574. d3 := s.AddDaemon(c, true, true)
  575. d3.getService(c, id)
  576. d3.Stop()
  577. d3.Start()
  578. d3.getService(c, id)
  579. d3.Kill()
  580. time.Sleep(1 * time.Second) // time to handle signal
  581. d3.Start()
  582. d3.getService(c, id)
  583. }
  584. func (s *DockerSwarmSuite) TestApiSwarmScaleNoRollingUpdate(c *check.C) {
  585. testRequires(c, Network)
  586. d := s.AddDaemon(c, true, true)
  587. instances := 2
  588. id := d.createService(c, simpleTestService, setInstances(instances))
  589. waitAndAssert(c, defaultReconciliationTimeout, d.checkActiveContainerCount, checker.Equals, instances)
  590. containers := d.activeContainers()
  591. instances = 4
  592. d.updateService(c, d.getService(c, id), setInstances(instances))
  593. waitAndAssert(c, defaultReconciliationTimeout, d.checkActiveContainerCount, checker.Equals, instances)
  594. containers2 := d.activeContainers()
  595. loop0:
  596. for _, c1 := range containers {
  597. for _, c2 := range containers2 {
  598. if c1 == c2 {
  599. continue loop0
  600. }
  601. }
  602. c.Errorf("container %v not found in new set %#v", c1, containers2)
  603. }
  604. }
  605. func (s *DockerSwarmSuite) TestApiSwarmInvalidAddress(c *check.C) {
  606. d := s.AddDaemon(c, false, false)
  607. req := swarm.InitRequest{
  608. ListenAddr: "",
  609. }
  610. status, _, err := d.SockRequest("POST", "/swarm/init", req)
  611. c.Assert(err, checker.IsNil)
  612. c.Assert(status, checker.Equals, http.StatusInternalServerError)
  613. req2 := swarm.JoinRequest{
  614. ListenAddr: "0.0.0.0:2377",
  615. RemoteAddrs: []string{""},
  616. }
  617. status, _, err = d.SockRequest("POST", "/swarm/join", req2)
  618. c.Assert(err, checker.IsNil)
  619. c.Assert(status, checker.Equals, http.StatusInternalServerError)
  620. }
  621. func (s *DockerSwarmSuite) TestApiSwarmForceNewCluster(c *check.C) {
  622. d1 := s.AddDaemon(c, true, true)
  623. d2 := s.AddDaemon(c, true, true)
  624. instances := 2
  625. id := d1.createService(c, simpleTestService, setInstances(instances))
  626. waitAndAssert(c, defaultReconciliationTimeout, reducedCheck(sumAsIntegers, d1.checkActiveContainerCount, d2.checkActiveContainerCount), checker.Equals, instances)
  627. c.Assert(d2.Stop(), checker.IsNil)
  628. time.Sleep(5 * time.Second)
  629. c.Assert(d1.Init(swarm.InitRequest{
  630. ForceNewCluster: true,
  631. Spec: swarm.Spec{
  632. AcceptancePolicy: autoAcceptPolicy,
  633. },
  634. }), checker.IsNil)
  635. waitAndAssert(c, defaultReconciliationTimeout, d1.checkActiveContainerCount, checker.Equals, instances)
  636. d3 := s.AddDaemon(c, true, true)
  637. info, err := d3.info()
  638. c.Assert(err, checker.IsNil)
  639. c.Assert(info.ControlAvailable, checker.Equals, true)
  640. c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive)
  641. instances = 4
  642. d3.updateService(c, d3.getService(c, id), setInstances(instances))
  643. waitAndAssert(c, defaultReconciliationTimeout, reducedCheck(sumAsIntegers, d1.checkActiveContainerCount, d3.checkActiveContainerCount), checker.Equals, instances)
  644. }
  645. func simpleTestService(s *swarm.Service) {
  646. var ureplicas uint64
  647. ureplicas = 1
  648. s.Spec = swarm.ServiceSpec{
  649. TaskTemplate: swarm.TaskSpec{
  650. ContainerSpec: swarm.ContainerSpec{
  651. Image: "busybox:latest",
  652. Command: []string{"/bin/top"},
  653. },
  654. },
  655. Mode: swarm.ServiceMode{
  656. Replicated: &swarm.ReplicatedService{
  657. Replicas: &ureplicas,
  658. },
  659. },
  660. }
  661. s.Spec.Name = "top"
  662. }
  663. func setInstances(replicas int) serviceConstructor {
  664. ureplicas := uint64(replicas)
  665. return func(s *swarm.Service) {
  666. s.Spec.Mode = swarm.ServiceMode{
  667. Replicated: &swarm.ReplicatedService{
  668. Replicas: &ureplicas,
  669. },
  670. }
  671. }
  672. }
  673. func setGlobalMode(s *swarm.Service) {
  674. s.Spec.Mode = swarm.ServiceMode{
  675. Global: &swarm.GlobalService{},
  676. }
  677. }