docker_api_swarm_service_test.go 24 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608
  1. //go:build !windows
  2. package main
  3. import (
  4. "fmt"
  5. "strconv"
  6. "strings"
  7. "testing"
  8. "time"
  9. "github.com/docker/docker/api/types"
  10. "github.com/docker/docker/api/types/swarm"
  11. "github.com/docker/docker/integration-cli/checker"
  12. "github.com/docker/docker/integration-cli/cli"
  13. "github.com/docker/docker/integration-cli/cli/build"
  14. "github.com/docker/docker/integration-cli/daemon"
  15. "github.com/docker/docker/testutil"
  16. testdaemon "github.com/docker/docker/testutil/daemon"
  17. "golang.org/x/sys/unix"
  18. "gotest.tools/v3/assert"
  19. "gotest.tools/v3/icmd"
  20. "gotest.tools/v3/poll"
  21. )
  22. func setPortConfig(portConfig []swarm.PortConfig) testdaemon.ServiceConstructor {
  23. return func(s *swarm.Service) {
  24. if s.Spec.EndpointSpec == nil {
  25. s.Spec.EndpointSpec = &swarm.EndpointSpec{}
  26. }
  27. s.Spec.EndpointSpec.Ports = portConfig
  28. }
  29. }
  30. func (s *DockerSwarmSuite) TestAPIServiceUpdatePort(c *testing.T) {
  31. ctx := testutil.GetContext(c)
  32. d := s.AddDaemon(ctx, c, true, true)
  33. // Create a service with a port mapping of 8080:8081.
  34. portConfig := []swarm.PortConfig{{TargetPort: 8081, PublishedPort: 8080}}
  35. serviceID := d.CreateService(ctx, c, simpleTestService, setInstances(1), setPortConfig(portConfig))
  36. poll.WaitOn(c, pollCheck(c, d.CheckActiveContainerCount(ctx), checker.Equals(1)), poll.WithTimeout(defaultReconciliationTimeout))
  37. // Update the service: changed the port mapping from 8080:8081 to 8082:8083.
  38. updatedPortConfig := []swarm.PortConfig{{TargetPort: 8083, PublishedPort: 8082}}
  39. remoteService := d.GetService(ctx, c, serviceID)
  40. d.UpdateService(ctx, c, remoteService, setPortConfig(updatedPortConfig))
  41. // Inspect the service and verify port mapping.
  42. updatedService := d.GetService(ctx, c, serviceID)
  43. assert.Assert(c, updatedService.Spec.EndpointSpec != nil)
  44. assert.Equal(c, len(updatedService.Spec.EndpointSpec.Ports), 1)
  45. assert.Equal(c, updatedService.Spec.EndpointSpec.Ports[0].TargetPort, uint32(8083))
  46. assert.Equal(c, updatedService.Spec.EndpointSpec.Ports[0].PublishedPort, uint32(8082))
  47. }
  48. func (s *DockerSwarmSuite) TestAPISwarmServicesEmptyList(c *testing.T) {
  49. ctx := testutil.GetContext(c)
  50. d := s.AddDaemon(ctx, c, true, true)
  51. services := d.ListServices(ctx, c)
  52. assert.Assert(c, services != nil)
  53. assert.Assert(c, len(services) == 0, "services: %#v", services)
  54. }
  55. func (s *DockerSwarmSuite) TestAPISwarmServicesCreate(c *testing.T) {
  56. ctx := testutil.GetContext(c)
  57. d := s.AddDaemon(ctx, c, true, true)
  58. instances := 2
  59. id := d.CreateService(ctx, c, simpleTestService, setInstances(instances))
  60. poll.WaitOn(c, pollCheck(c, d.CheckActiveContainerCount(ctx), checker.Equals(instances)), poll.WithTimeout(defaultReconciliationTimeout))
  61. client := d.NewClientT(c)
  62. defer client.Close()
  63. options := types.ServiceInspectOptions{InsertDefaults: true}
  64. // insertDefaults inserts UpdateConfig when service is fetched by ID
  65. resp, _, err := client.ServiceInspectWithRaw(ctx, id, options)
  66. out := fmt.Sprintf("%+v", resp)
  67. assert.NilError(c, err)
  68. assert.Assert(c, strings.Contains(out, "UpdateConfig"))
  69. // insertDefaults inserts UpdateConfig when service is fetched by ID
  70. resp, _, err = client.ServiceInspectWithRaw(ctx, "top", options)
  71. out = fmt.Sprintf("%+v", resp)
  72. assert.NilError(c, err)
  73. assert.Assert(c, strings.Contains(out, "UpdateConfig"))
  74. service := d.GetService(ctx, c, id)
  75. instances = 5
  76. d.UpdateService(ctx, c, service, setInstances(instances))
  77. poll.WaitOn(c, pollCheck(c, d.CheckActiveContainerCount(ctx), checker.Equals(instances)), poll.WithTimeout(defaultReconciliationTimeout))
  78. d.RemoveService(ctx, c, service.ID)
  79. poll.WaitOn(c, pollCheck(c, d.CheckActiveContainerCount(ctx), checker.Equals(0)), poll.WithTimeout(defaultReconciliationTimeout))
  80. }
  81. func (s *DockerSwarmSuite) TestAPISwarmServicesMultipleAgents(c *testing.T) {
  82. ctx := testutil.GetContext(c)
  83. d1 := s.AddDaemon(ctx, c, true, true)
  84. d2 := s.AddDaemon(ctx, c, true, false)
  85. d3 := s.AddDaemon(ctx, c, true, false)
  86. time.Sleep(1 * time.Second) // make sure all daemons are ready to accept tasks
  87. instances := 9
  88. id := d1.CreateService(ctx, c, simpleTestService, setInstances(instances))
  89. poll.WaitOn(c, pollCheck(c, d1.CheckActiveContainerCount(ctx), checker.GreaterThan(0)), poll.WithTimeout(defaultReconciliationTimeout))
  90. poll.WaitOn(c, pollCheck(c, d2.CheckActiveContainerCount(ctx), checker.GreaterThan(0)), poll.WithTimeout(defaultReconciliationTimeout))
  91. poll.WaitOn(c, pollCheck(c, d3.CheckActiveContainerCount(ctx), checker.GreaterThan(0)), poll.WithTimeout(defaultReconciliationTimeout))
  92. poll.WaitOn(c, pollCheck(c, reducedCheck(sumAsIntegers, d1.CheckActiveContainerCount(ctx), d2.CheckActiveContainerCount(ctx), d3.CheckActiveContainerCount(ctx)), checker.Equals(instances)), poll.WithTimeout(defaultReconciliationTimeout))
  93. // reconciliation on d2 node down
  94. d2.Stop(c)
  95. poll.WaitOn(c, pollCheck(c, reducedCheck(sumAsIntegers, d1.CheckActiveContainerCount(ctx), d3.CheckActiveContainerCount(ctx)), checker.Equals(instances)), poll.WithTimeout(defaultReconciliationTimeout))
  96. // test downscaling
  97. instances = 5
  98. d1.UpdateService(ctx, c, d1.GetService(ctx, c, id), setInstances(instances))
  99. poll.WaitOn(c, pollCheck(c, reducedCheck(sumAsIntegers, d1.CheckActiveContainerCount(ctx), d3.CheckActiveContainerCount(ctx)), checker.Equals(instances)), poll.WithTimeout(defaultReconciliationTimeout))
  100. }
  101. func (s *DockerSwarmSuite) TestAPISwarmServicesCreateGlobal(c *testing.T) {
  102. ctx := testutil.GetContext(c)
  103. d1 := s.AddDaemon(ctx, c, true, true)
  104. d2 := s.AddDaemon(ctx, c, true, false)
  105. d3 := s.AddDaemon(ctx, c, true, false)
  106. d1.CreateService(ctx, c, simpleTestService, setGlobalMode)
  107. poll.WaitOn(c, pollCheck(c, d1.CheckActiveContainerCount(ctx), checker.Equals(1)), poll.WithTimeout(defaultReconciliationTimeout))
  108. poll.WaitOn(c, pollCheck(c, d2.CheckActiveContainerCount(ctx), checker.Equals(1)), poll.WithTimeout(defaultReconciliationTimeout))
  109. poll.WaitOn(c, pollCheck(c, d3.CheckActiveContainerCount(ctx), checker.Equals(1)), poll.WithTimeout(defaultReconciliationTimeout))
  110. d4 := s.AddDaemon(ctx, c, true, false)
  111. d5 := s.AddDaemon(ctx, c, true, false)
  112. poll.WaitOn(c, pollCheck(c, d4.CheckActiveContainerCount(ctx), checker.Equals(1)), poll.WithTimeout(defaultReconciliationTimeout))
  113. poll.WaitOn(c, pollCheck(c, d5.CheckActiveContainerCount(ctx), checker.Equals(1)), poll.WithTimeout(defaultReconciliationTimeout))
  114. }
  115. func (s *DockerSwarmSuite) TestAPISwarmServicesUpdate(c *testing.T) {
  116. ctx := testutil.GetContext(c)
  117. const nodeCount = 3
  118. var daemons [nodeCount]*daemon.Daemon
  119. for i := 0; i < nodeCount; i++ {
  120. daemons[i] = s.AddDaemon(ctx, c, true, i == 0)
  121. }
  122. // wait for nodes ready
  123. poll.WaitOn(c, pollCheck(c, daemons[0].CheckNodeReadyCount(ctx), checker.Equals(nodeCount)), poll.WithTimeout(5*time.Second))
  124. // service image at start
  125. image1 := "busybox:latest"
  126. // target image in update
  127. image2 := "busybox:test"
  128. // create a different tag
  129. for _, d := range daemons {
  130. out, err := d.Cmd("tag", image1, image2)
  131. assert.NilError(c, err, out)
  132. }
  133. // create service
  134. instances := 5
  135. parallelism := 2
  136. rollbackParallelism := 3
  137. id := daemons[0].CreateService(ctx, c, serviceForUpdate, setInstances(instances))
  138. // wait for tasks ready
  139. poll.WaitOn(c, pollCheck(c, daemons[0].CheckRunningTaskImages(ctx), checker.DeepEquals(map[string]int{image1: instances})), poll.WithTimeout(defaultReconciliationTimeout))
  140. // issue service update
  141. service := daemons[0].GetService(ctx, c, id)
  142. daemons[0].UpdateService(ctx, c, service, setImage(image2))
  143. // first batch
  144. poll.WaitOn(c, pollCheck(c, daemons[0].CheckRunningTaskImages(ctx), checker.DeepEquals(map[string]int{image1: instances - parallelism, image2: parallelism})), poll.WithTimeout(defaultReconciliationTimeout))
  145. // 2nd batch
  146. poll.WaitOn(c, pollCheck(c, daemons[0].CheckRunningTaskImages(ctx), checker.DeepEquals(map[string]int{image1: instances - 2*parallelism, image2: 2 * parallelism})), poll.WithTimeout(defaultReconciliationTimeout))
  147. // 3nd batch
  148. poll.WaitOn(c, pollCheck(c, daemons[0].CheckRunningTaskImages(ctx), checker.DeepEquals(map[string]int{image2: instances})), poll.WithTimeout(defaultReconciliationTimeout))
  149. // Roll back to the previous version. This uses the CLI because
  150. // rollback used to be a client-side operation.
  151. out, err := daemons[0].Cmd("service", "update", "--detach", "--rollback", id)
  152. assert.NilError(c, err, out)
  153. // first batch
  154. poll.WaitOn(c, pollCheck(c, daemons[0].CheckRunningTaskImages(ctx), checker.DeepEquals(map[string]int{image2: instances - rollbackParallelism, image1: rollbackParallelism})), poll.WithTimeout(defaultReconciliationTimeout))
  155. // 2nd batch
  156. poll.WaitOn(c, pollCheck(c, daemons[0].CheckRunningTaskImages(ctx), checker.DeepEquals(map[string]int{image1: instances})), poll.WithTimeout(defaultReconciliationTimeout))
  157. }
  158. func (s *DockerSwarmSuite) TestAPISwarmServicesUpdateStartFirst(c *testing.T) {
  159. ctx := testutil.GetContext(c)
  160. d := s.AddDaemon(ctx, c, true, true)
  161. // service image at start
  162. image1 := "busybox:latest"
  163. // target image in update
  164. image2 := "testhealth:latest"
  165. // service started from this image won't pass health check
  166. result := cli.BuildCmd(c, image2, cli.Daemon(d),
  167. build.WithDockerfile(`FROM busybox
  168. HEALTHCHECK --interval=1s --timeout=30s --retries=1024 \
  169. CMD cat /status`),
  170. )
  171. result.Assert(c, icmd.Success)
  172. // create service
  173. instances := 5
  174. parallelism := 2
  175. rollbackParallelism := 3
  176. id := d.CreateService(ctx, c, serviceForUpdate, setInstances(instances), setUpdateOrder(swarm.UpdateOrderStartFirst), setRollbackOrder(swarm.UpdateOrderStartFirst))
  177. checkStartingTasks := func(expected int) []swarm.Task {
  178. var startingTasks []swarm.Task
  179. poll.WaitOn(c, pollCheck(c, func(c *testing.T) (interface{}, string) {
  180. tasks := d.GetServiceTasks(ctx, c, id)
  181. startingTasks = nil
  182. for _, t := range tasks {
  183. if t.Status.State == swarm.TaskStateStarting {
  184. startingTasks = append(startingTasks, t)
  185. }
  186. }
  187. return startingTasks, ""
  188. }, checker.HasLen(expected)), poll.WithTimeout(defaultReconciliationTimeout))
  189. return startingTasks
  190. }
  191. makeTasksHealthy := func(tasks []swarm.Task) {
  192. for _, t := range tasks {
  193. containerID := t.Status.ContainerStatus.ContainerID
  194. d.Cmd("exec", containerID, "touch", "/status")
  195. }
  196. }
  197. // wait for tasks ready
  198. poll.WaitOn(c, pollCheck(c, d.CheckRunningTaskImages(ctx), checker.DeepEquals(map[string]int{image1: instances})), poll.WithTimeout(defaultReconciliationTimeout))
  199. // issue service update
  200. service := d.GetService(ctx, c, id)
  201. d.UpdateService(ctx, c, service, setImage(image2))
  202. // first batch
  203. // The old tasks should be running, and the new ones should be starting.
  204. startingTasks := checkStartingTasks(parallelism)
  205. poll.WaitOn(c, pollCheck(c, d.CheckRunningTaskImages(ctx), checker.DeepEquals(map[string]int{image1: instances})), poll.WithTimeout(defaultReconciliationTimeout))
  206. // make it healthy
  207. makeTasksHealthy(startingTasks)
  208. poll.WaitOn(c, pollCheck(c, d.CheckRunningTaskImages(ctx), checker.DeepEquals(map[string]int{image1: instances - parallelism, image2: parallelism})), poll.WithTimeout(defaultReconciliationTimeout))
  209. // 2nd batch
  210. // The old tasks should be running, and the new ones should be starting.
  211. startingTasks = checkStartingTasks(parallelism)
  212. poll.WaitOn(c, pollCheck(c, d.CheckRunningTaskImages(ctx), checker.DeepEquals(map[string]int{image1: instances - parallelism, image2: parallelism})), poll.WithTimeout(defaultReconciliationTimeout))
  213. // make it healthy
  214. makeTasksHealthy(startingTasks)
  215. poll.WaitOn(c, pollCheck(c, d.CheckRunningTaskImages(ctx), checker.DeepEquals(map[string]int{image1: instances - 2*parallelism, image2: 2 * parallelism})), poll.WithTimeout(defaultReconciliationTimeout))
  216. // 3nd batch
  217. // The old tasks should be running, and the new ones should be starting.
  218. startingTasks = checkStartingTasks(1)
  219. poll.WaitOn(c, pollCheck(c, d.CheckRunningTaskImages(ctx), checker.DeepEquals(map[string]int{image1: instances - 2*parallelism, image2: 2 * parallelism})), poll.WithTimeout(defaultReconciliationTimeout))
  220. // make it healthy
  221. makeTasksHealthy(startingTasks)
  222. poll.WaitOn(c, pollCheck(c, d.CheckRunningTaskImages(ctx), checker.DeepEquals(map[string]int{image2: instances})), poll.WithTimeout(defaultReconciliationTimeout))
  223. // Roll back to the previous version. This uses the CLI because
  224. // rollback is a client-side operation.
  225. out, err := d.Cmd("service", "update", "--detach", "--rollback", id)
  226. assert.NilError(c, err, out)
  227. // first batch
  228. poll.WaitOn(c, pollCheck(c, d.CheckRunningTaskImages(ctx), checker.DeepEquals(map[string]int{image2: instances - rollbackParallelism, image1: rollbackParallelism})), poll.WithTimeout(defaultReconciliationTimeout))
  229. // 2nd batch
  230. poll.WaitOn(c, pollCheck(c, d.CheckRunningTaskImages(ctx), checker.DeepEquals(map[string]int{image1: instances})), poll.WithTimeout(defaultReconciliationTimeout))
  231. }
  232. func (s *DockerSwarmSuite) TestAPISwarmServicesFailedUpdate(c *testing.T) {
  233. ctx := testutil.GetContext(c)
  234. const nodeCount = 3
  235. var daemons [nodeCount]*daemon.Daemon
  236. for i := 0; i < nodeCount; i++ {
  237. daemons[i] = s.AddDaemon(ctx, c, true, i == 0)
  238. }
  239. // wait for nodes ready
  240. poll.WaitOn(c, pollCheck(c, daemons[0].CheckNodeReadyCount(ctx), checker.Equals(nodeCount)), poll.WithTimeout(5*time.Second))
  241. // service image at start
  242. image1 := "busybox:latest"
  243. // target image in update
  244. image2 := "busybox:badtag"
  245. // create service
  246. instances := 5
  247. id := daemons[0].CreateService(ctx, c, serviceForUpdate, setInstances(instances))
  248. // wait for tasks ready
  249. poll.WaitOn(c, pollCheck(c, daemons[0].CheckRunningTaskImages(ctx), checker.DeepEquals(map[string]int{image1: instances})), poll.WithTimeout(defaultReconciliationTimeout))
  250. // issue service update
  251. service := daemons[0].GetService(ctx, c, id)
  252. daemons[0].UpdateService(ctx, c, service, setImage(image2), setFailureAction(swarm.UpdateFailureActionPause), setMaxFailureRatio(0.25), setParallelism(1))
  253. // should update 2 tasks and then pause
  254. poll.WaitOn(c, pollCheck(c, daemons[0].CheckServiceUpdateState(ctx, id), checker.Equals(swarm.UpdateStatePaused)), poll.WithTimeout(defaultReconciliationTimeout))
  255. v, _ := daemons[0].CheckServiceRunningTasks(ctx, id)(c)
  256. assert.Assert(c, v == instances-2)
  257. // Roll back to the previous version. This uses the CLI because
  258. // rollback used to be a client-side operation.
  259. out, err := daemons[0].Cmd("service", "update", "--detach", "--rollback", id)
  260. assert.NilError(c, err, out)
  261. poll.WaitOn(c, pollCheck(c, daemons[0].CheckRunningTaskImages(ctx), checker.DeepEquals(map[string]int{image1: instances})), poll.WithTimeout(defaultReconciliationTimeout))
  262. }
  263. func (s *DockerSwarmSuite) TestAPISwarmServiceConstraintRole(c *testing.T) {
  264. ctx := testutil.GetContext(c)
  265. const nodeCount = 3
  266. var daemons [nodeCount]*daemon.Daemon
  267. for i := 0; i < nodeCount; i++ {
  268. daemons[i] = s.AddDaemon(ctx, c, true, i == 0)
  269. }
  270. // wait for nodes ready
  271. poll.WaitOn(c, pollCheck(c, daemons[0].CheckNodeReadyCount(ctx), checker.Equals(nodeCount)), poll.WithTimeout(5*time.Second))
  272. // create service
  273. constraints := []string{"node.role==worker"}
  274. instances := 3
  275. id := daemons[0].CreateService(ctx, c, simpleTestService, setConstraints(constraints), setInstances(instances))
  276. // wait for tasks ready
  277. poll.WaitOn(c, pollCheck(c, daemons[0].CheckServiceRunningTasks(ctx, id), checker.Equals(instances)), poll.WithTimeout(defaultReconciliationTimeout))
  278. // validate tasks are running on worker nodes
  279. tasks := daemons[0].GetServiceTasks(ctx, c, id)
  280. for _, task := range tasks {
  281. node := daemons[0].GetNode(ctx, c, task.NodeID)
  282. assert.Equal(c, node.Spec.Role, swarm.NodeRoleWorker)
  283. }
  284. // remove service
  285. daemons[0].RemoveService(ctx, c, id)
  286. // create service
  287. constraints = []string{"node.role!=worker"}
  288. id = daemons[0].CreateService(ctx, c, simpleTestService, setConstraints(constraints), setInstances(instances))
  289. // wait for tasks ready
  290. poll.WaitOn(c, pollCheck(c, daemons[0].CheckServiceRunningTasks(ctx, id), checker.Equals(instances)), poll.WithTimeout(defaultReconciliationTimeout))
  291. tasks = daemons[0].GetServiceTasks(ctx, c, id)
  292. // validate tasks are running on manager nodes
  293. for _, task := range tasks {
  294. node := daemons[0].GetNode(ctx, c, task.NodeID)
  295. assert.Equal(c, node.Spec.Role, swarm.NodeRoleManager)
  296. }
  297. // remove service
  298. daemons[0].RemoveService(ctx, c, id)
  299. // create service
  300. constraints = []string{"node.role==nosuchrole"}
  301. id = daemons[0].CreateService(ctx, c, simpleTestService, setConstraints(constraints), setInstances(instances))
  302. // wait for tasks created
  303. poll.WaitOn(c, pollCheck(c, daemons[0].CheckServiceTasks(ctx, id), checker.Equals(instances)), poll.WithTimeout(defaultReconciliationTimeout))
  304. // let scheduler try
  305. time.Sleep(250 * time.Millisecond)
  306. // validate tasks are not assigned to any node
  307. tasks = daemons[0].GetServiceTasks(ctx, c, id)
  308. for _, task := range tasks {
  309. assert.Equal(c, task.NodeID, "")
  310. }
  311. }
  312. func (s *DockerSwarmSuite) TestAPISwarmServiceConstraintLabel(c *testing.T) {
  313. ctx := testutil.GetContext(c)
  314. const nodeCount = 3
  315. var daemons [nodeCount]*daemon.Daemon
  316. for i := 0; i < nodeCount; i++ {
  317. daemons[i] = s.AddDaemon(ctx, c, true, i == 0)
  318. }
  319. // wait for nodes ready
  320. poll.WaitOn(c, pollCheck(c, daemons[0].CheckNodeReadyCount(ctx), checker.Equals(nodeCount)), poll.WithTimeout(5*time.Second))
  321. nodes := daemons[0].ListNodes(ctx, c)
  322. assert.Equal(c, len(nodes), nodeCount)
  323. // add labels to nodes
  324. daemons[0].UpdateNode(ctx, c, nodes[0].ID, func(n *swarm.Node) {
  325. n.Spec.Annotations.Labels = map[string]string{
  326. "security": "high",
  327. }
  328. })
  329. for i := 1; i < nodeCount; i++ {
  330. daemons[0].UpdateNode(ctx, c, nodes[i].ID, func(n *swarm.Node) {
  331. n.Spec.Annotations.Labels = map[string]string{
  332. "security": "low",
  333. }
  334. })
  335. }
  336. // create service
  337. instances := 3
  338. constraints := []string{"node.labels.security==high"}
  339. id := daemons[0].CreateService(ctx, c, simpleTestService, setConstraints(constraints), setInstances(instances))
  340. // wait for tasks ready
  341. poll.WaitOn(c, pollCheck(c, daemons[0].CheckServiceRunningTasks(ctx, id), checker.Equals(instances)), poll.WithTimeout(defaultReconciliationTimeout))
  342. tasks := daemons[0].GetServiceTasks(ctx, c, id)
  343. // validate all tasks are running on nodes[0]
  344. for _, task := range tasks {
  345. assert.Assert(c, task.NodeID == nodes[0].ID)
  346. }
  347. // remove service
  348. daemons[0].RemoveService(ctx, c, id)
  349. // create service
  350. constraints = []string{"node.labels.security!=high"}
  351. id = daemons[0].CreateService(ctx, c, simpleTestService, setConstraints(constraints), setInstances(instances))
  352. // wait for tasks ready
  353. poll.WaitOn(c, pollCheck(c, daemons[0].CheckServiceRunningTasks(ctx, id), checker.Equals(instances)), poll.WithTimeout(defaultReconciliationTimeout))
  354. tasks = daemons[0].GetServiceTasks(ctx, c, id)
  355. // validate all tasks are NOT running on nodes[0]
  356. for _, task := range tasks {
  357. assert.Assert(c, task.NodeID != nodes[0].ID)
  358. }
  359. // remove service
  360. daemons[0].RemoveService(ctx, c, id)
  361. constraints = []string{"node.labels.security==medium"}
  362. id = daemons[0].CreateService(ctx, c, simpleTestService, setConstraints(constraints), setInstances(instances))
  363. // wait for tasks created
  364. poll.WaitOn(c, pollCheck(c, daemons[0].CheckServiceTasks(ctx, id), checker.Equals(instances)), poll.WithTimeout(defaultReconciliationTimeout))
  365. // let scheduler try
  366. time.Sleep(250 * time.Millisecond)
  367. tasks = daemons[0].GetServiceTasks(ctx, c, id)
  368. // validate tasks are not assigned
  369. for _, task := range tasks {
  370. assert.Assert(c, task.NodeID == "")
  371. }
  372. // remove service
  373. daemons[0].RemoveService(ctx, c, id)
  374. // multiple constraints
  375. constraints = []string{
  376. "node.labels.security==high",
  377. fmt.Sprintf("node.id==%s", nodes[1].ID),
  378. }
  379. id = daemons[0].CreateService(ctx, c, simpleTestService, setConstraints(constraints), setInstances(instances))
  380. // wait for tasks created
  381. poll.WaitOn(c, pollCheck(c, daemons[0].CheckServiceTasks(ctx, id), checker.Equals(instances)), poll.WithTimeout(defaultReconciliationTimeout))
  382. // let scheduler try
  383. time.Sleep(250 * time.Millisecond)
  384. tasks = daemons[0].GetServiceTasks(ctx, c, id)
  385. // validate tasks are not assigned
  386. for _, task := range tasks {
  387. assert.Assert(c, task.NodeID == "")
  388. }
  389. // make nodes[1] fulfills the constraints
  390. daemons[0].UpdateNode(ctx, c, nodes[1].ID, func(n *swarm.Node) {
  391. n.Spec.Annotations.Labels = map[string]string{
  392. "security": "high",
  393. }
  394. })
  395. // wait for tasks ready
  396. poll.WaitOn(c, pollCheck(c, daemons[0].CheckServiceRunningTasks(ctx, id), checker.Equals(instances)), poll.WithTimeout(defaultReconciliationTimeout))
  397. tasks = daemons[0].GetServiceTasks(ctx, c, id)
  398. for _, task := range tasks {
  399. assert.Assert(c, task.NodeID == nodes[1].ID)
  400. }
  401. }
  402. func (s *DockerSwarmSuite) TestAPISwarmServicePlacementPrefs(c *testing.T) {
  403. ctx := testutil.GetContext(c)
  404. const nodeCount = 3
  405. var daemons [nodeCount]*daemon.Daemon
  406. for i := 0; i < nodeCount; i++ {
  407. daemons[i] = s.AddDaemon(ctx, c, true, i == 0)
  408. }
  409. // wait for nodes ready
  410. poll.WaitOn(c, pollCheck(c, daemons[0].CheckNodeReadyCount(ctx), checker.Equals(nodeCount)), poll.WithTimeout(5*time.Second))
  411. nodes := daemons[0].ListNodes(ctx, c)
  412. assert.Equal(c, len(nodes), nodeCount)
  413. // add labels to nodes
  414. daemons[0].UpdateNode(ctx, c, nodes[0].ID, func(n *swarm.Node) {
  415. n.Spec.Annotations.Labels = map[string]string{
  416. "rack": "a",
  417. }
  418. })
  419. for i := 1; i < nodeCount; i++ {
  420. daemons[0].UpdateNode(ctx, c, nodes[i].ID, func(n *swarm.Node) {
  421. n.Spec.Annotations.Labels = map[string]string{
  422. "rack": "b",
  423. }
  424. })
  425. }
  426. // create service
  427. instances := 4
  428. prefs := []swarm.PlacementPreference{{Spread: &swarm.SpreadOver{SpreadDescriptor: "node.labels.rack"}}}
  429. id := daemons[0].CreateService(ctx, c, simpleTestService, setPlacementPrefs(prefs), setInstances(instances))
  430. // wait for tasks ready
  431. poll.WaitOn(c, pollCheck(c, daemons[0].CheckServiceRunningTasks(ctx, id), checker.Equals(instances)), poll.WithTimeout(defaultReconciliationTimeout))
  432. tasks := daemons[0].GetServiceTasks(ctx, c, id)
  433. // validate all tasks are running on nodes[0]
  434. tasksOnNode := make(map[string]int)
  435. for _, task := range tasks {
  436. tasksOnNode[task.NodeID]++
  437. }
  438. assert.Assert(c, tasksOnNode[nodes[0].ID] == 2)
  439. assert.Assert(c, tasksOnNode[nodes[1].ID] == 1)
  440. assert.Assert(c, tasksOnNode[nodes[2].ID] == 1)
  441. }
  442. func (s *DockerSwarmSuite) TestAPISwarmServicesStateReporting(c *testing.T) {
  443. testRequires(c, testEnv.IsLocalDaemon)
  444. testRequires(c, DaemonIsLinux)
  445. ctx := testutil.GetContext(c)
  446. d1 := s.AddDaemon(ctx, c, true, true)
  447. d2 := s.AddDaemon(ctx, c, true, true)
  448. d3 := s.AddDaemon(ctx, c, true, false)
  449. time.Sleep(1 * time.Second) // make sure all daemons are ready to accept
  450. instances := 9
  451. d1.CreateService(ctx, c, simpleTestService, setInstances(instances))
  452. poll.WaitOn(c, pollCheck(c, reducedCheck(sumAsIntegers, d1.CheckActiveContainerCount(ctx), d2.CheckActiveContainerCount(ctx), d3.CheckActiveContainerCount(ctx)), checker.Equals(instances)), poll.WithTimeout(defaultReconciliationTimeout))
  453. getContainers := func() map[string]*daemon.Daemon {
  454. m := make(map[string]*daemon.Daemon)
  455. for _, d := range []*daemon.Daemon{d1, d2, d3} {
  456. for _, id := range d.ActiveContainers(testutil.GetContext(c), c) {
  457. m[id] = d
  458. }
  459. }
  460. return m
  461. }
  462. containers := getContainers()
  463. assert.Assert(c, len(containers) == instances)
  464. var toRemove string
  465. for i := range containers {
  466. toRemove = i
  467. }
  468. _, err := containers[toRemove].Cmd("stop", toRemove)
  469. assert.NilError(c, err)
  470. poll.WaitOn(c, pollCheck(c, reducedCheck(sumAsIntegers, d1.CheckActiveContainerCount(ctx), d2.CheckActiveContainerCount(ctx), d3.CheckActiveContainerCount(ctx)), checker.Equals(instances)), poll.WithTimeout(defaultReconciliationTimeout))
  471. containers2 := getContainers()
  472. assert.Assert(c, len(containers2) == instances)
  473. for i := range containers {
  474. if i == toRemove {
  475. assert.Assert(c, containers2[i] == nil)
  476. } else {
  477. assert.Assert(c, containers2[i] != nil)
  478. }
  479. }
  480. containers = containers2
  481. for i := range containers {
  482. toRemove = i
  483. }
  484. // try with killing process outside of docker
  485. pidStr, err := containers[toRemove].Cmd("inspect", "-f", "{{.State.Pid}}", toRemove)
  486. assert.NilError(c, err)
  487. pid, err := strconv.Atoi(strings.TrimSpace(pidStr))
  488. assert.NilError(c, err)
  489. assert.NilError(c, unix.Kill(pid, unix.SIGKILL))
  490. time.Sleep(time.Second) // give some time to handle the signal
  491. poll.WaitOn(c, pollCheck(c, reducedCheck(sumAsIntegers, d1.CheckActiveContainerCount(ctx), d2.CheckActiveContainerCount(ctx), d3.CheckActiveContainerCount(ctx)), checker.Equals(instances)), poll.WithTimeout(defaultReconciliationTimeout))
  492. containers2 = getContainers()
  493. assert.Assert(c, len(containers2) == instances)
  494. for i := range containers {
  495. if i == toRemove {
  496. assert.Assert(c, containers2[i] == nil)
  497. } else {
  498. assert.Assert(c, containers2[i] != nil)
  499. }
  500. }
  501. }