docker_api_swarm_service_test.go 21 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588
  1. // +build !windows
  2. package main
  3. import (
  4. "fmt"
  5. "strconv"
  6. "strings"
  7. "syscall"
  8. "time"
  9. "github.com/docker/docker/api/types/swarm"
  10. "github.com/docker/docker/integration-cli/checker"
  11. "github.com/docker/docker/integration-cli/daemon"
  12. "github.com/go-check/check"
  13. )
  14. func setPortConfig(portConfig []swarm.PortConfig) daemon.ServiceConstructor {
  15. return func(s *swarm.Service) {
  16. if s.Spec.EndpointSpec == nil {
  17. s.Spec.EndpointSpec = &swarm.EndpointSpec{}
  18. }
  19. s.Spec.EndpointSpec.Ports = portConfig
  20. }
  21. }
  22. func (s *DockerSwarmSuite) TestAPIServiceUpdatePort(c *check.C) {
  23. d := s.AddDaemon(c, true, true)
  24. // Create a service with a port mapping of 8080:8081.
  25. portConfig := []swarm.PortConfig{{TargetPort: 8081, PublishedPort: 8080}}
  26. serviceID := d.CreateService(c, simpleTestService, setInstances(1), setPortConfig(portConfig))
  27. waitAndAssert(c, defaultReconciliationTimeout, d.CheckActiveContainerCount, checker.Equals, 1)
  28. // Update the service: changed the port mapping from 8080:8081 to 8082:8083.
  29. updatedPortConfig := []swarm.PortConfig{{TargetPort: 8083, PublishedPort: 8082}}
  30. remoteService := d.GetService(c, serviceID)
  31. d.UpdateService(c, remoteService, setPortConfig(updatedPortConfig))
  32. // Inspect the service and verify port mapping.
  33. updatedService := d.GetService(c, serviceID)
  34. c.Assert(updatedService.Spec.EndpointSpec, check.NotNil)
  35. c.Assert(len(updatedService.Spec.EndpointSpec.Ports), check.Equals, 1)
  36. c.Assert(updatedService.Spec.EndpointSpec.Ports[0].TargetPort, check.Equals, uint32(8083))
  37. c.Assert(updatedService.Spec.EndpointSpec.Ports[0].PublishedPort, check.Equals, uint32(8082))
  38. }
  39. func (s *DockerSwarmSuite) TestAPISwarmServicesEmptyList(c *check.C) {
  40. d := s.AddDaemon(c, true, true)
  41. services := d.ListServices(c)
  42. c.Assert(services, checker.NotNil)
  43. c.Assert(len(services), checker.Equals, 0, check.Commentf("services: %#v", services))
  44. }
  45. func (s *DockerSwarmSuite) TestAPISwarmServicesCreate(c *check.C) {
  46. d := s.AddDaemon(c, true, true)
  47. instances := 2
  48. id := d.CreateService(c, simpleTestService, setInstances(instances))
  49. waitAndAssert(c, defaultReconciliationTimeout, d.CheckActiveContainerCount, checker.Equals, instances)
  50. service := d.GetService(c, id)
  51. instances = 5
  52. d.UpdateService(c, service, setInstances(instances))
  53. waitAndAssert(c, defaultReconciliationTimeout, d.CheckActiveContainerCount, checker.Equals, instances)
  54. d.RemoveService(c, service.ID)
  55. waitAndAssert(c, defaultReconciliationTimeout, d.CheckActiveContainerCount, checker.Equals, 0)
  56. }
  57. func (s *DockerSwarmSuite) TestAPISwarmServicesMultipleAgents(c *check.C) {
  58. d1 := s.AddDaemon(c, true, true)
  59. d2 := s.AddDaemon(c, true, false)
  60. d3 := s.AddDaemon(c, true, false)
  61. time.Sleep(1 * time.Second) // make sure all daemons are ready to accept tasks
  62. instances := 9
  63. id := d1.CreateService(c, simpleTestService, setInstances(instances))
  64. waitAndAssert(c, defaultReconciliationTimeout, d1.CheckActiveContainerCount, checker.GreaterThan, 0)
  65. waitAndAssert(c, defaultReconciliationTimeout, d2.CheckActiveContainerCount, checker.GreaterThan, 0)
  66. waitAndAssert(c, defaultReconciliationTimeout, d3.CheckActiveContainerCount, checker.GreaterThan, 0)
  67. waitAndAssert(c, defaultReconciliationTimeout, reducedCheck(sumAsIntegers, d1.CheckActiveContainerCount, d2.CheckActiveContainerCount, d3.CheckActiveContainerCount), checker.Equals, instances)
  68. // reconciliation on d2 node down
  69. d2.Stop(c)
  70. waitAndAssert(c, defaultReconciliationTimeout, reducedCheck(sumAsIntegers, d1.CheckActiveContainerCount, d3.CheckActiveContainerCount), checker.Equals, instances)
  71. // test downscaling
  72. instances = 5
  73. d1.UpdateService(c, d1.GetService(c, id), setInstances(instances))
  74. waitAndAssert(c, defaultReconciliationTimeout, reducedCheck(sumAsIntegers, d1.CheckActiveContainerCount, d3.CheckActiveContainerCount), checker.Equals, instances)
  75. }
  76. func (s *DockerSwarmSuite) TestAPISwarmServicesCreateGlobal(c *check.C) {
  77. d1 := s.AddDaemon(c, true, true)
  78. d2 := s.AddDaemon(c, true, false)
  79. d3 := s.AddDaemon(c, true, false)
  80. d1.CreateService(c, simpleTestService, setGlobalMode)
  81. waitAndAssert(c, defaultReconciliationTimeout, d1.CheckActiveContainerCount, checker.Equals, 1)
  82. waitAndAssert(c, defaultReconciliationTimeout, d2.CheckActiveContainerCount, checker.Equals, 1)
  83. waitAndAssert(c, defaultReconciliationTimeout, d3.CheckActiveContainerCount, checker.Equals, 1)
  84. d4 := s.AddDaemon(c, true, false)
  85. d5 := s.AddDaemon(c, true, false)
  86. waitAndAssert(c, defaultReconciliationTimeout, d4.CheckActiveContainerCount, checker.Equals, 1)
  87. waitAndAssert(c, defaultReconciliationTimeout, d5.CheckActiveContainerCount, checker.Equals, 1)
  88. }
  89. func (s *DockerSwarmSuite) TestAPISwarmServicesUpdate(c *check.C) {
  90. const nodeCount = 3
  91. var daemons [nodeCount]*daemon.Swarm
  92. for i := 0; i < nodeCount; i++ {
  93. daemons[i] = s.AddDaemon(c, true, i == 0)
  94. }
  95. // wait for nodes ready
  96. waitAndAssert(c, 5*time.Second, daemons[0].CheckNodeReadyCount, checker.Equals, nodeCount)
  97. // service image at start
  98. image1 := "busybox:latest"
  99. // target image in update
  100. image2 := "busybox:test"
  101. // create a different tag
  102. for _, d := range daemons {
  103. out, err := d.Cmd("tag", image1, image2)
  104. c.Assert(err, checker.IsNil, check.Commentf(out))
  105. }
  106. // create service
  107. instances := 5
  108. parallelism := 2
  109. rollbackParallelism := 3
  110. id := daemons[0].CreateService(c, serviceForUpdate, setInstances(instances))
  111. // wait for tasks ready
  112. waitAndAssert(c, defaultReconciliationTimeout, daemons[0].CheckRunningTaskImages, checker.DeepEquals,
  113. map[string]int{image1: instances})
  114. // issue service update
  115. service := daemons[0].GetService(c, id)
  116. daemons[0].UpdateService(c, service, setImage(image2))
  117. // first batch
  118. waitAndAssert(c, defaultReconciliationTimeout, daemons[0].CheckRunningTaskImages, checker.DeepEquals,
  119. map[string]int{image1: instances - parallelism, image2: parallelism})
  120. // 2nd batch
  121. waitAndAssert(c, defaultReconciliationTimeout, daemons[0].CheckRunningTaskImages, checker.DeepEquals,
  122. map[string]int{image1: instances - 2*parallelism, image2: 2 * parallelism})
  123. // 3nd batch
  124. waitAndAssert(c, defaultReconciliationTimeout, daemons[0].CheckRunningTaskImages, checker.DeepEquals,
  125. map[string]int{image2: instances})
  126. // Roll back to the previous version. This uses the CLI because
  127. // rollback used to be a client-side operation.
  128. out, err := daemons[0].Cmd("service", "update", "--rollback", id)
  129. c.Assert(err, checker.IsNil, check.Commentf(out))
  130. // first batch
  131. waitAndAssert(c, defaultReconciliationTimeout, daemons[0].CheckRunningTaskImages, checker.DeepEquals,
  132. map[string]int{image2: instances - rollbackParallelism, image1: rollbackParallelism})
  133. // 2nd batch
  134. waitAndAssert(c, defaultReconciliationTimeout, daemons[0].CheckRunningTaskImages, checker.DeepEquals,
  135. map[string]int{image1: instances})
  136. }
  137. func (s *DockerSwarmSuite) TestAPISwarmServicesUpdateStartFirst(c *check.C) {
  138. d := s.AddDaemon(c, true, true)
  139. // service image at start
  140. image1 := "busybox:latest"
  141. // target image in update
  142. image2 := "testhealth"
  143. // service started from this image won't pass health check
  144. _, _, err := d.BuildImageWithOut(image2,
  145. `FROM busybox
  146. HEALTHCHECK --interval=1s --timeout=1s --retries=1024\
  147. CMD cat /status`,
  148. true)
  149. c.Check(err, check.IsNil)
  150. // create service
  151. instances := 5
  152. parallelism := 2
  153. rollbackParallelism := 3
  154. id := d.CreateService(c, serviceForUpdate, setInstances(instances), setUpdateOrder(swarm.UpdateOrderStartFirst), setRollbackOrder(swarm.UpdateOrderStartFirst))
  155. checkStartingTasks := func(expected int) []swarm.Task {
  156. var startingTasks []swarm.Task
  157. waitAndAssert(c, defaultReconciliationTimeout, func(c *check.C) (interface{}, check.CommentInterface) {
  158. tasks := d.GetServiceTasks(c, id)
  159. startingTasks = nil
  160. for _, t := range tasks {
  161. if t.Status.State == swarm.TaskStateStarting {
  162. startingTasks = append(startingTasks, t)
  163. }
  164. }
  165. return startingTasks, nil
  166. }, checker.HasLen, expected)
  167. return startingTasks
  168. }
  169. makeTasksHealthy := func(tasks []swarm.Task) {
  170. for _, t := range tasks {
  171. containerID := t.Status.ContainerStatus.ContainerID
  172. d.Cmd("exec", containerID, "touch", "/status")
  173. }
  174. }
  175. // wait for tasks ready
  176. waitAndAssert(c, defaultReconciliationTimeout, d.CheckRunningTaskImages, checker.DeepEquals,
  177. map[string]int{image1: instances})
  178. // issue service update
  179. service := d.GetService(c, id)
  180. d.UpdateService(c, service, setImage(image2))
  181. // first batch
  182. // The old tasks should be running, and the new ones should be starting.
  183. startingTasks := checkStartingTasks(parallelism)
  184. waitAndAssert(c, defaultReconciliationTimeout, d.CheckRunningTaskImages, checker.DeepEquals,
  185. map[string]int{image1: instances})
  186. // make it healthy
  187. makeTasksHealthy(startingTasks)
  188. waitAndAssert(c, defaultReconciliationTimeout, d.CheckRunningTaskImages, checker.DeepEquals,
  189. map[string]int{image1: instances - parallelism, image2: parallelism})
  190. // 2nd batch
  191. // The old tasks should be running, and the new ones should be starting.
  192. startingTasks = checkStartingTasks(parallelism)
  193. waitAndAssert(c, defaultReconciliationTimeout, d.CheckRunningTaskImages, checker.DeepEquals,
  194. map[string]int{image1: instances - parallelism, image2: parallelism})
  195. // make it healthy
  196. makeTasksHealthy(startingTasks)
  197. waitAndAssert(c, defaultReconciliationTimeout, d.CheckRunningTaskImages, checker.DeepEquals,
  198. map[string]int{image1: instances - 2*parallelism, image2: 2 * parallelism})
  199. // 3nd batch
  200. // The old tasks should be running, and the new ones should be starting.
  201. startingTasks = checkStartingTasks(1)
  202. waitAndAssert(c, defaultReconciliationTimeout, d.CheckRunningTaskImages, checker.DeepEquals,
  203. map[string]int{image1: instances - 2*parallelism, image2: 2 * parallelism})
  204. // make it healthy
  205. makeTasksHealthy(startingTasks)
  206. waitAndAssert(c, defaultReconciliationTimeout, d.CheckRunningTaskImages, checker.DeepEquals,
  207. map[string]int{image2: instances})
  208. // Roll back to the previous version. This uses the CLI because
  209. // rollback is a client-side operation.
  210. out, err := d.Cmd("service", "update", "--rollback", id)
  211. c.Assert(err, checker.IsNil, check.Commentf(out))
  212. // first batch
  213. waitAndAssert(c, defaultReconciliationTimeout, d.CheckRunningTaskImages, checker.DeepEquals,
  214. map[string]int{image2: instances - rollbackParallelism, image1: rollbackParallelism})
  215. // 2nd batch
  216. waitAndAssert(c, defaultReconciliationTimeout, d.CheckRunningTaskImages, checker.DeepEquals,
  217. map[string]int{image1: instances})
  218. }
  219. func (s *DockerSwarmSuite) TestAPISwarmServicesFailedUpdate(c *check.C) {
  220. const nodeCount = 3
  221. var daemons [nodeCount]*daemon.Swarm
  222. for i := 0; i < nodeCount; i++ {
  223. daemons[i] = s.AddDaemon(c, true, i == 0)
  224. }
  225. // wait for nodes ready
  226. waitAndAssert(c, 5*time.Second, daemons[0].CheckNodeReadyCount, checker.Equals, nodeCount)
  227. // service image at start
  228. image1 := "busybox:latest"
  229. // target image in update
  230. image2 := "busybox:badtag"
  231. // create service
  232. instances := 5
  233. id := daemons[0].CreateService(c, serviceForUpdate, setInstances(instances))
  234. // wait for tasks ready
  235. waitAndAssert(c, defaultReconciliationTimeout, daemons[0].CheckRunningTaskImages, checker.DeepEquals,
  236. map[string]int{image1: instances})
  237. // issue service update
  238. service := daemons[0].GetService(c, id)
  239. daemons[0].UpdateService(c, service, setImage(image2), setFailureAction(swarm.UpdateFailureActionPause), setMaxFailureRatio(0.25), setParallelism(1))
  240. // should update 2 tasks and then pause
  241. waitAndAssert(c, defaultReconciliationTimeout, daemons[0].CheckServiceUpdateState(id), checker.Equals, swarm.UpdateStatePaused)
  242. v, _ := daemons[0].CheckServiceRunningTasks(id)(c)
  243. c.Assert(v, checker.Equals, instances-2)
  244. // Roll back to the previous version. This uses the CLI because
  245. // rollback used to be a client-side operation.
  246. out, err := daemons[0].Cmd("service", "update", "--rollback", id)
  247. c.Assert(err, checker.IsNil, check.Commentf(out))
  248. waitAndAssert(c, defaultReconciliationTimeout, daemons[0].CheckRunningTaskImages, checker.DeepEquals,
  249. map[string]int{image1: instances})
  250. }
  251. func (s *DockerSwarmSuite) TestAPISwarmServiceConstraintRole(c *check.C) {
  252. const nodeCount = 3
  253. var daemons [nodeCount]*daemon.Swarm
  254. for i := 0; i < nodeCount; i++ {
  255. daemons[i] = s.AddDaemon(c, true, i == 0)
  256. }
  257. // wait for nodes ready
  258. waitAndAssert(c, 5*time.Second, daemons[0].CheckNodeReadyCount, checker.Equals, nodeCount)
  259. // create service
  260. constraints := []string{"node.role==worker"}
  261. instances := 3
  262. id := daemons[0].CreateService(c, simpleTestService, setConstraints(constraints), setInstances(instances))
  263. // wait for tasks ready
  264. waitAndAssert(c, defaultReconciliationTimeout, daemons[0].CheckServiceRunningTasks(id), checker.Equals, instances)
  265. // validate tasks are running on worker nodes
  266. tasks := daemons[0].GetServiceTasks(c, id)
  267. for _, task := range tasks {
  268. node := daemons[0].GetNode(c, task.NodeID)
  269. c.Assert(node.Spec.Role, checker.Equals, swarm.NodeRoleWorker)
  270. }
  271. //remove service
  272. daemons[0].RemoveService(c, id)
  273. // create service
  274. constraints = []string{"node.role!=worker"}
  275. id = daemons[0].CreateService(c, simpleTestService, setConstraints(constraints), setInstances(instances))
  276. // wait for tasks ready
  277. waitAndAssert(c, defaultReconciliationTimeout, daemons[0].CheckServiceRunningTasks(id), checker.Equals, instances)
  278. tasks = daemons[0].GetServiceTasks(c, id)
  279. // validate tasks are running on manager nodes
  280. for _, task := range tasks {
  281. node := daemons[0].GetNode(c, task.NodeID)
  282. c.Assert(node.Spec.Role, checker.Equals, swarm.NodeRoleManager)
  283. }
  284. //remove service
  285. daemons[0].RemoveService(c, id)
  286. // create service
  287. constraints = []string{"node.role==nosuchrole"}
  288. id = daemons[0].CreateService(c, simpleTestService, setConstraints(constraints), setInstances(instances))
  289. // wait for tasks created
  290. waitAndAssert(c, defaultReconciliationTimeout, daemons[0].CheckServiceTasks(id), checker.Equals, instances)
  291. // let scheduler try
  292. time.Sleep(250 * time.Millisecond)
  293. // validate tasks are not assigned to any node
  294. tasks = daemons[0].GetServiceTasks(c, id)
  295. for _, task := range tasks {
  296. c.Assert(task.NodeID, checker.Equals, "")
  297. }
  298. }
  299. func (s *DockerSwarmSuite) TestAPISwarmServiceConstraintLabel(c *check.C) {
  300. const nodeCount = 3
  301. var daemons [nodeCount]*daemon.Swarm
  302. for i := 0; i < nodeCount; i++ {
  303. daemons[i] = s.AddDaemon(c, true, i == 0)
  304. }
  305. // wait for nodes ready
  306. waitAndAssert(c, 5*time.Second, daemons[0].CheckNodeReadyCount, checker.Equals, nodeCount)
  307. nodes := daemons[0].ListNodes(c)
  308. c.Assert(len(nodes), checker.Equals, nodeCount)
  309. // add labels to nodes
  310. daemons[0].UpdateNode(c, nodes[0].ID, func(n *swarm.Node) {
  311. n.Spec.Annotations.Labels = map[string]string{
  312. "security": "high",
  313. }
  314. })
  315. for i := 1; i < nodeCount; i++ {
  316. daemons[0].UpdateNode(c, nodes[i].ID, func(n *swarm.Node) {
  317. n.Spec.Annotations.Labels = map[string]string{
  318. "security": "low",
  319. }
  320. })
  321. }
  322. // create service
  323. instances := 3
  324. constraints := []string{"node.labels.security==high"}
  325. id := daemons[0].CreateService(c, simpleTestService, setConstraints(constraints), setInstances(instances))
  326. // wait for tasks ready
  327. waitAndAssert(c, defaultReconciliationTimeout, daemons[0].CheckServiceRunningTasks(id), checker.Equals, instances)
  328. tasks := daemons[0].GetServiceTasks(c, id)
  329. // validate all tasks are running on nodes[0]
  330. for _, task := range tasks {
  331. c.Assert(task.NodeID, checker.Equals, nodes[0].ID)
  332. }
  333. //remove service
  334. daemons[0].RemoveService(c, id)
  335. // create service
  336. constraints = []string{"node.labels.security!=high"}
  337. id = daemons[0].CreateService(c, simpleTestService, setConstraints(constraints), setInstances(instances))
  338. // wait for tasks ready
  339. waitAndAssert(c, defaultReconciliationTimeout, daemons[0].CheckServiceRunningTasks(id), checker.Equals, instances)
  340. tasks = daemons[0].GetServiceTasks(c, id)
  341. // validate all tasks are NOT running on nodes[0]
  342. for _, task := range tasks {
  343. c.Assert(task.NodeID, checker.Not(checker.Equals), nodes[0].ID)
  344. }
  345. //remove service
  346. daemons[0].RemoveService(c, id)
  347. constraints = []string{"node.labels.security==medium"}
  348. id = daemons[0].CreateService(c, simpleTestService, setConstraints(constraints), setInstances(instances))
  349. // wait for tasks created
  350. waitAndAssert(c, defaultReconciliationTimeout, daemons[0].CheckServiceTasks(id), checker.Equals, instances)
  351. // let scheduler try
  352. time.Sleep(250 * time.Millisecond)
  353. tasks = daemons[0].GetServiceTasks(c, id)
  354. // validate tasks are not assigned
  355. for _, task := range tasks {
  356. c.Assert(task.NodeID, checker.Equals, "")
  357. }
  358. //remove service
  359. daemons[0].RemoveService(c, id)
  360. // multiple constraints
  361. constraints = []string{
  362. "node.labels.security==high",
  363. fmt.Sprintf("node.id==%s", nodes[1].ID),
  364. }
  365. id = daemons[0].CreateService(c, simpleTestService, setConstraints(constraints), setInstances(instances))
  366. // wait for tasks created
  367. waitAndAssert(c, defaultReconciliationTimeout, daemons[0].CheckServiceTasks(id), checker.Equals, instances)
  368. // let scheduler try
  369. time.Sleep(250 * time.Millisecond)
  370. tasks = daemons[0].GetServiceTasks(c, id)
  371. // validate tasks are not assigned
  372. for _, task := range tasks {
  373. c.Assert(task.NodeID, checker.Equals, "")
  374. }
  375. // make nodes[1] fulfills the constraints
  376. daemons[0].UpdateNode(c, nodes[1].ID, func(n *swarm.Node) {
  377. n.Spec.Annotations.Labels = map[string]string{
  378. "security": "high",
  379. }
  380. })
  381. // wait for tasks ready
  382. waitAndAssert(c, defaultReconciliationTimeout, daemons[0].CheckServiceRunningTasks(id), checker.Equals, instances)
  383. tasks = daemons[0].GetServiceTasks(c, id)
  384. for _, task := range tasks {
  385. c.Assert(task.NodeID, checker.Equals, nodes[1].ID)
  386. }
  387. }
  388. func (s *DockerSwarmSuite) TestAPISwarmServicePlacementPrefs(c *check.C) {
  389. const nodeCount = 3
  390. var daemons [nodeCount]*daemon.Swarm
  391. for i := 0; i < nodeCount; i++ {
  392. daemons[i] = s.AddDaemon(c, true, i == 0)
  393. }
  394. // wait for nodes ready
  395. waitAndAssert(c, 5*time.Second, daemons[0].CheckNodeReadyCount, checker.Equals, nodeCount)
  396. nodes := daemons[0].ListNodes(c)
  397. c.Assert(len(nodes), checker.Equals, nodeCount)
  398. // add labels to nodes
  399. daemons[0].UpdateNode(c, nodes[0].ID, func(n *swarm.Node) {
  400. n.Spec.Annotations.Labels = map[string]string{
  401. "rack": "a",
  402. }
  403. })
  404. for i := 1; i < nodeCount; i++ {
  405. daemons[0].UpdateNode(c, nodes[i].ID, func(n *swarm.Node) {
  406. n.Spec.Annotations.Labels = map[string]string{
  407. "rack": "b",
  408. }
  409. })
  410. }
  411. // create service
  412. instances := 4
  413. prefs := []swarm.PlacementPreference{{Spread: &swarm.SpreadOver{SpreadDescriptor: "node.labels.rack"}}}
  414. id := daemons[0].CreateService(c, simpleTestService, setPlacementPrefs(prefs), setInstances(instances))
  415. // wait for tasks ready
  416. waitAndAssert(c, defaultReconciliationTimeout, daemons[0].CheckServiceRunningTasks(id), checker.Equals, instances)
  417. tasks := daemons[0].GetServiceTasks(c, id)
  418. // validate all tasks are running on nodes[0]
  419. tasksOnNode := make(map[string]int)
  420. for _, task := range tasks {
  421. tasksOnNode[task.NodeID]++
  422. }
  423. c.Assert(tasksOnNode[nodes[0].ID], checker.Equals, 2)
  424. c.Assert(tasksOnNode[nodes[1].ID], checker.Equals, 1)
  425. c.Assert(tasksOnNode[nodes[2].ID], checker.Equals, 1)
  426. }
  427. func (s *DockerSwarmSuite) TestAPISwarmServicesStateReporting(c *check.C) {
  428. testRequires(c, SameHostDaemon)
  429. testRequires(c, DaemonIsLinux)
  430. d1 := s.AddDaemon(c, true, true)
  431. d2 := s.AddDaemon(c, true, true)
  432. d3 := s.AddDaemon(c, true, false)
  433. time.Sleep(1 * time.Second) // make sure all daemons are ready to accept
  434. instances := 9
  435. d1.CreateService(c, simpleTestService, setInstances(instances))
  436. waitAndAssert(c, defaultReconciliationTimeout, reducedCheck(sumAsIntegers, d1.CheckActiveContainerCount, d2.CheckActiveContainerCount, d3.CheckActiveContainerCount), checker.Equals, instances)
  437. getContainers := func() map[string]*daemon.Swarm {
  438. m := make(map[string]*daemon.Swarm)
  439. for _, d := range []*daemon.Swarm{d1, d2, d3} {
  440. for _, id := range d.ActiveContainers() {
  441. m[id] = d
  442. }
  443. }
  444. return m
  445. }
  446. containers := getContainers()
  447. c.Assert(containers, checker.HasLen, instances)
  448. var toRemove string
  449. for i := range containers {
  450. toRemove = i
  451. }
  452. _, err := containers[toRemove].Cmd("stop", toRemove)
  453. c.Assert(err, checker.IsNil)
  454. waitAndAssert(c, defaultReconciliationTimeout, reducedCheck(sumAsIntegers, d1.CheckActiveContainerCount, d2.CheckActiveContainerCount, d3.CheckActiveContainerCount), checker.Equals, instances)
  455. containers2 := getContainers()
  456. c.Assert(containers2, checker.HasLen, instances)
  457. for i := range containers {
  458. if i == toRemove {
  459. c.Assert(containers2[i], checker.IsNil)
  460. } else {
  461. c.Assert(containers2[i], checker.NotNil)
  462. }
  463. }
  464. containers = containers2
  465. for i := range containers {
  466. toRemove = i
  467. }
  468. // try with killing process outside of docker
  469. pidStr, err := containers[toRemove].Cmd("inspect", "-f", "{{.State.Pid}}", toRemove)
  470. c.Assert(err, checker.IsNil)
  471. pid, err := strconv.Atoi(strings.TrimSpace(pidStr))
  472. c.Assert(err, checker.IsNil)
  473. c.Assert(syscall.Kill(pid, syscall.SIGKILL), checker.IsNil)
  474. time.Sleep(time.Second) // give some time to handle the signal
  475. waitAndAssert(c, defaultReconciliationTimeout, reducedCheck(sumAsIntegers, d1.CheckActiveContainerCount, d2.CheckActiveContainerCount, d3.CheckActiveContainerCount), checker.Equals, instances)
  476. containers2 = getContainers()
  477. c.Assert(containers2, checker.HasLen, instances)
  478. for i := range containers {
  479. if i == toRemove {
  480. c.Assert(containers2[i], checker.IsNil)
  481. } else {
  482. c.Assert(containers2[i], checker.NotNil)
  483. }
  484. }
  485. }