daemon_swarm.go 19 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664
  1. package daemon
  2. import (
  3. "encoding/json"
  4. "fmt"
  5. "net/http"
  6. "strings"
  7. "time"
  8. "github.com/docker/docker/api/types"
  9. "github.com/docker/docker/api/types/filters"
  10. "github.com/docker/docker/api/types/swarm"
  11. "github.com/docker/docker/integration-cli/checker"
  12. "github.com/go-check/check"
  13. "github.com/pkg/errors"
  14. "golang.org/x/net/context"
  15. )
  16. // Swarm is a test daemon with helpers for participating in a swarm.
  17. type Swarm struct {
  18. *Daemon
  19. swarm.Info
  20. Port int
  21. ListenAddr string
  22. }
  23. // Init initializes a new swarm cluster.
  24. func (d *Swarm) Init(req swarm.InitRequest) error {
  25. if req.ListenAddr == "" {
  26. req.ListenAddr = d.ListenAddr
  27. }
  28. cli, err := d.NewClient()
  29. if err != nil {
  30. return fmt.Errorf("initializing swarm: failed to create client %v", err)
  31. }
  32. defer cli.Close()
  33. _, err = cli.SwarmInit(context.Background(), req)
  34. if err != nil {
  35. return fmt.Errorf("initializing swarm: %v", err)
  36. }
  37. info, err := d.SwarmInfo()
  38. if err != nil {
  39. return err
  40. }
  41. d.Info = info
  42. return nil
  43. }
  44. // Join joins a daemon to an existing cluster.
  45. func (d *Swarm) Join(req swarm.JoinRequest) error {
  46. if req.ListenAddr == "" {
  47. req.ListenAddr = d.ListenAddr
  48. }
  49. cli, err := d.NewClient()
  50. if err != nil {
  51. return fmt.Errorf("joining swarm: failed to create client %v", err)
  52. }
  53. defer cli.Close()
  54. err = cli.SwarmJoin(context.Background(), req)
  55. if err != nil {
  56. return fmt.Errorf("joining swarm: %v", err)
  57. }
  58. info, err := d.SwarmInfo()
  59. if err != nil {
  60. return err
  61. }
  62. d.Info = info
  63. return nil
  64. }
  65. // Leave forces daemon to leave current cluster.
  66. func (d *Swarm) Leave(force bool) error {
  67. cli, err := d.NewClient()
  68. if err != nil {
  69. return fmt.Errorf("leaving swarm: failed to create client %v", err)
  70. }
  71. defer cli.Close()
  72. err = cli.SwarmLeave(context.Background(), force)
  73. if err != nil {
  74. err = fmt.Errorf("leaving swarm: %v", err)
  75. }
  76. return err
  77. }
  78. // SwarmInfo returns the swarm information of the daemon
  79. func (d *Swarm) SwarmInfo() (swarm.Info, error) {
  80. cli, err := d.NewClient()
  81. if err != nil {
  82. return swarm.Info{}, fmt.Errorf("get swarm info: %v", err)
  83. }
  84. info, err := cli.Info(context.Background())
  85. if err != nil {
  86. return swarm.Info{}, fmt.Errorf("get swarm info: %v", err)
  87. }
  88. return info.Swarm, nil
  89. }
  90. // Unlock tries to unlock a locked swarm
  91. func (d *Swarm) Unlock(req swarm.UnlockRequest) error {
  92. cli, err := d.NewClient()
  93. if err != nil {
  94. return fmt.Errorf("unlocking swarm: failed to create client %v", err)
  95. }
  96. defer cli.Close()
  97. err = cli.SwarmUnlock(context.Background(), req)
  98. if err != nil {
  99. err = errors.Wrap(err, "unlocking swarm")
  100. }
  101. return err
  102. }
  103. // ServiceConstructor defines a swarm service constructor function
  104. type ServiceConstructor func(*swarm.Service)
  105. // NodeConstructor defines a swarm node constructor
  106. type NodeConstructor func(*swarm.Node)
  107. // SecretConstructor defines a swarm secret constructor
  108. type SecretConstructor func(*swarm.Secret)
  109. // ConfigConstructor defines a swarm config constructor
  110. type ConfigConstructor func(*swarm.Config)
  111. // SpecConstructor defines a swarm spec constructor
  112. type SpecConstructor func(*swarm.Spec)
  113. // CreateServiceWithOptions creates a swarm service given the specified service constructors
  114. // and auth config
  115. func (d *Swarm) CreateServiceWithOptions(c *check.C, opts types.ServiceCreateOptions, f ...ServiceConstructor) string {
  116. var service swarm.Service
  117. for _, fn := range f {
  118. fn(&service)
  119. }
  120. cli, err := d.NewClient()
  121. c.Assert(err, checker.IsNil)
  122. defer cli.Close()
  123. ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
  124. defer cancel()
  125. res, err := cli.ServiceCreate(ctx, service.Spec, opts)
  126. c.Assert(err, checker.IsNil)
  127. return res.ID
  128. }
  129. // CreateService creates a swarm service given the specified service constructor
  130. func (d *Swarm) CreateService(c *check.C, f ...ServiceConstructor) string {
  131. return d.CreateServiceWithOptions(c, types.ServiceCreateOptions{}, f...)
  132. }
  133. // GetService returns the swarm service corresponding to the specified id
  134. func (d *Swarm) GetService(c *check.C, id string) *swarm.Service {
  135. cli, err := d.NewClient()
  136. c.Assert(err, checker.IsNil)
  137. defer cli.Close()
  138. service, _, err := cli.ServiceInspectWithRaw(context.Background(), id, types.ServiceInspectOptions{})
  139. c.Assert(err, checker.IsNil)
  140. return &service
  141. }
  142. // GetServiceTasks returns the swarm tasks for the specified service
  143. func (d *Swarm) GetServiceTasks(c *check.C, service string) []swarm.Task {
  144. cli, err := d.NewClient()
  145. c.Assert(err, checker.IsNil)
  146. defer cli.Close()
  147. filterArgs := filters.NewArgs()
  148. filterArgs.Add("desired-state", "running")
  149. filterArgs.Add("service", service)
  150. options := types.TaskListOptions{
  151. Filters: filterArgs,
  152. }
  153. tasks, err := cli.TaskList(context.Background(), options)
  154. c.Assert(err, checker.IsNil)
  155. return tasks
  156. }
  157. // CheckServiceTasksInState returns the number of tasks with a matching state,
  158. // and optional message substring.
  159. func (d *Swarm) CheckServiceTasksInState(service string, state swarm.TaskState, message string) func(*check.C) (interface{}, check.CommentInterface) {
  160. return func(c *check.C) (interface{}, check.CommentInterface) {
  161. tasks := d.GetServiceTasks(c, service)
  162. var count int
  163. for _, task := range tasks {
  164. if task.Status.State == state {
  165. if message == "" || strings.Contains(task.Status.Message, message) {
  166. count++
  167. }
  168. }
  169. }
  170. return count, nil
  171. }
  172. }
  173. // CheckServiceTasksInStateWithError returns the number of tasks with a matching state,
  174. // and optional message substring.
  175. func (d *Swarm) CheckServiceTasksInStateWithError(service string, state swarm.TaskState, errorMessage string) func(*check.C) (interface{}, check.CommentInterface) {
  176. return func(c *check.C) (interface{}, check.CommentInterface) {
  177. tasks := d.GetServiceTasks(c, service)
  178. var count int
  179. for _, task := range tasks {
  180. if task.Status.State == state {
  181. if errorMessage == "" || strings.Contains(task.Status.Err, errorMessage) {
  182. count++
  183. }
  184. }
  185. }
  186. return count, nil
  187. }
  188. }
  189. // CheckServiceRunningTasks returns the number of running tasks for the specified service
  190. func (d *Swarm) CheckServiceRunningTasks(service string) func(*check.C) (interface{}, check.CommentInterface) {
  191. return d.CheckServiceTasksInState(service, swarm.TaskStateRunning, "")
  192. }
  193. // CheckServiceUpdateState returns the current update state for the specified service
  194. func (d *Swarm) CheckServiceUpdateState(service string) func(*check.C) (interface{}, check.CommentInterface) {
  195. return func(c *check.C) (interface{}, check.CommentInterface) {
  196. service := d.GetService(c, service)
  197. if service.UpdateStatus == nil {
  198. return "", nil
  199. }
  200. return service.UpdateStatus.State, nil
  201. }
  202. }
  203. // CheckPluginRunning returns the runtime state of the plugin
  204. func (d *Swarm) CheckPluginRunning(plugin string) func(c *check.C) (interface{}, check.CommentInterface) {
  205. return func(c *check.C) (interface{}, check.CommentInterface) {
  206. status, out, err := d.SockRequest("GET", "/plugins/"+plugin+"/json", nil)
  207. c.Assert(err, checker.IsNil, check.Commentf(string(out)))
  208. if status != http.StatusOK {
  209. return false, nil
  210. }
  211. var p types.Plugin
  212. c.Assert(json.Unmarshal(out, &p), checker.IsNil, check.Commentf(string(out)))
  213. return p.Enabled, check.Commentf("%+v", p)
  214. }
  215. }
  216. // CheckPluginImage returns the runtime state of the plugin
  217. func (d *Swarm) CheckPluginImage(plugin string) func(c *check.C) (interface{}, check.CommentInterface) {
  218. return func(c *check.C) (interface{}, check.CommentInterface) {
  219. status, out, err := d.SockRequest("GET", "/plugins/"+plugin+"/json", nil)
  220. c.Assert(err, checker.IsNil, check.Commentf(string(out)))
  221. if status != http.StatusOK {
  222. return false, nil
  223. }
  224. var p types.Plugin
  225. c.Assert(json.Unmarshal(out, &p), checker.IsNil, check.Commentf(string(out)))
  226. return p.PluginReference, check.Commentf("%+v", p)
  227. }
  228. }
  229. // CheckServiceTasks returns the number of tasks for the specified service
  230. func (d *Swarm) CheckServiceTasks(service string) func(*check.C) (interface{}, check.CommentInterface) {
  231. return func(c *check.C) (interface{}, check.CommentInterface) {
  232. tasks := d.GetServiceTasks(c, service)
  233. return len(tasks), nil
  234. }
  235. }
  236. // CheckRunningTaskNetworks returns the number of times each network is referenced from a task.
  237. func (d *Swarm) CheckRunningTaskNetworks(c *check.C) (interface{}, check.CommentInterface) {
  238. cli, err := d.NewClient()
  239. c.Assert(err, checker.IsNil)
  240. defer cli.Close()
  241. filterArgs := filters.NewArgs()
  242. filterArgs.Add("desired-state", "running")
  243. options := types.TaskListOptions{
  244. Filters: filterArgs,
  245. }
  246. tasks, err := cli.TaskList(context.Background(), options)
  247. c.Assert(err, checker.IsNil)
  248. result := make(map[string]int)
  249. for _, task := range tasks {
  250. for _, network := range task.Spec.Networks {
  251. result[network.Target]++
  252. }
  253. }
  254. return result, nil
  255. }
  256. // CheckRunningTaskImages returns the times each image is running as a task.
  257. func (d *Swarm) CheckRunningTaskImages(c *check.C) (interface{}, check.CommentInterface) {
  258. cli, err := d.NewClient()
  259. c.Assert(err, checker.IsNil)
  260. defer cli.Close()
  261. filterArgs := filters.NewArgs()
  262. filterArgs.Add("desired-state", "running")
  263. options := types.TaskListOptions{
  264. Filters: filterArgs,
  265. }
  266. tasks, err := cli.TaskList(context.Background(), options)
  267. c.Assert(err, checker.IsNil)
  268. result := make(map[string]int)
  269. for _, task := range tasks {
  270. if task.Status.State == swarm.TaskStateRunning && task.Spec.ContainerSpec != nil {
  271. result[task.Spec.ContainerSpec.Image]++
  272. }
  273. }
  274. return result, nil
  275. }
  276. // CheckNodeReadyCount returns the number of ready node on the swarm
  277. func (d *Swarm) CheckNodeReadyCount(c *check.C) (interface{}, check.CommentInterface) {
  278. nodes := d.ListNodes(c)
  279. var readyCount int
  280. for _, node := range nodes {
  281. if node.Status.State == swarm.NodeStateReady {
  282. readyCount++
  283. }
  284. }
  285. return readyCount, nil
  286. }
  287. // GetTask returns the swarm task identified by the specified id
  288. func (d *Swarm) GetTask(c *check.C, id string) swarm.Task {
  289. cli, err := d.NewClient()
  290. c.Assert(err, checker.IsNil)
  291. defer cli.Close()
  292. task, _, err := cli.TaskInspectWithRaw(context.Background(), id)
  293. c.Assert(err, checker.IsNil)
  294. return task
  295. }
  296. // UpdateService updates a swarm service with the specified service constructor
  297. func (d *Swarm) UpdateService(c *check.C, service *swarm.Service, f ...ServiceConstructor) {
  298. cli, err := d.NewClient()
  299. c.Assert(err, checker.IsNil)
  300. defer cli.Close()
  301. for _, fn := range f {
  302. fn(service)
  303. }
  304. _, err = cli.ServiceUpdate(context.Background(), service.ID, service.Version, service.Spec, types.ServiceUpdateOptions{})
  305. c.Assert(err, checker.IsNil)
  306. }
  307. // RemoveService removes the specified service
  308. func (d *Swarm) RemoveService(c *check.C, id string) {
  309. cli, err := d.NewClient()
  310. c.Assert(err, checker.IsNil)
  311. defer cli.Close()
  312. err = cli.ServiceRemove(context.Background(), id)
  313. c.Assert(err, checker.IsNil)
  314. }
  315. // GetNode returns a swarm node identified by the specified id
  316. func (d *Swarm) GetNode(c *check.C, id string) *swarm.Node {
  317. cli, err := d.NewClient()
  318. c.Assert(err, checker.IsNil)
  319. defer cli.Close()
  320. node, _, err := cli.NodeInspectWithRaw(context.Background(), id)
  321. c.Assert(err, checker.IsNil)
  322. c.Assert(node.ID, checker.Equals, id)
  323. return &node
  324. }
  325. // RemoveNode removes the specified node
  326. func (d *Swarm) RemoveNode(c *check.C, id string, force bool) {
  327. cli, err := d.NewClient()
  328. c.Assert(err, checker.IsNil)
  329. defer cli.Close()
  330. options := types.NodeRemoveOptions{
  331. Force: force,
  332. }
  333. err = cli.NodeRemove(context.Background(), id, options)
  334. c.Assert(err, checker.IsNil)
  335. }
  336. // UpdateNode updates a swarm node with the specified node constructor
  337. func (d *Swarm) UpdateNode(c *check.C, id string, f ...NodeConstructor) {
  338. cli, err := d.NewClient()
  339. c.Assert(err, checker.IsNil)
  340. defer cli.Close()
  341. for i := 0; ; i++ {
  342. node := d.GetNode(c, id)
  343. for _, fn := range f {
  344. fn(node)
  345. }
  346. err = cli.NodeUpdate(context.Background(), node.ID, node.Version, node.Spec)
  347. if i < 10 && err != nil && strings.Contains(err.Error(), "update out of sequence") {
  348. time.Sleep(100 * time.Millisecond)
  349. continue
  350. }
  351. c.Assert(err, checker.IsNil)
  352. return
  353. }
  354. }
  355. // ListNodes returns the list of the current swarm nodes
  356. func (d *Swarm) ListNodes(c *check.C) []swarm.Node {
  357. cli, err := d.NewClient()
  358. c.Assert(err, checker.IsNil)
  359. defer cli.Close()
  360. nodes, err := cli.NodeList(context.Background(), types.NodeListOptions{})
  361. c.Assert(err, checker.IsNil)
  362. return nodes
  363. }
  364. // ListServices returns the list of the current swarm services
  365. func (d *Swarm) ListServices(c *check.C) []swarm.Service {
  366. cli, err := d.NewClient()
  367. c.Assert(err, checker.IsNil)
  368. defer cli.Close()
  369. services, err := cli.ServiceList(context.Background(), types.ServiceListOptions{})
  370. c.Assert(err, checker.IsNil)
  371. return services
  372. }
  373. // CreateSecret creates a secret given the specified spec
  374. func (d *Swarm) CreateSecret(c *check.C, secretSpec swarm.SecretSpec) string {
  375. cli, err := d.NewClient()
  376. c.Assert(err, checker.IsNil)
  377. defer cli.Close()
  378. scr, err := cli.SecretCreate(context.Background(), secretSpec)
  379. c.Assert(err, checker.IsNil)
  380. return scr.ID
  381. }
  382. // ListSecrets returns the list of the current swarm secrets
  383. func (d *Swarm) ListSecrets(c *check.C) []swarm.Secret {
  384. cli, err := d.NewClient()
  385. c.Assert(err, checker.IsNil)
  386. defer cli.Close()
  387. secrets, err := cli.SecretList(context.Background(), types.SecretListOptions{})
  388. c.Assert(err, checker.IsNil)
  389. return secrets
  390. }
  391. // GetSecret returns a swarm secret identified by the specified id
  392. func (d *Swarm) GetSecret(c *check.C, id string) *swarm.Secret {
  393. cli, err := d.NewClient()
  394. c.Assert(err, checker.IsNil)
  395. defer cli.Close()
  396. secret, _, err := cli.SecretInspectWithRaw(context.Background(), id)
  397. c.Assert(err, checker.IsNil)
  398. return &secret
  399. }
  400. // DeleteSecret removes the swarm secret identified by the specified id
  401. func (d *Swarm) DeleteSecret(c *check.C, id string) {
  402. cli, err := d.NewClient()
  403. c.Assert(err, checker.IsNil)
  404. defer cli.Close()
  405. err = cli.SecretRemove(context.Background(), id)
  406. c.Assert(err, checker.IsNil)
  407. }
  408. // UpdateSecret updates the swarm secret identified by the specified id
  409. // Currently, only label update is supported.
  410. func (d *Swarm) UpdateSecret(c *check.C, id string, f ...SecretConstructor) {
  411. cli, err := d.NewClient()
  412. c.Assert(err, checker.IsNil)
  413. defer cli.Close()
  414. secret := d.GetSecret(c, id)
  415. for _, fn := range f {
  416. fn(secret)
  417. }
  418. err = cli.SecretUpdate(context.Background(), secret.ID, secret.Version, secret.Spec)
  419. c.Assert(err, checker.IsNil)
  420. }
  421. // CreateConfig creates a config given the specified spec
  422. func (d *Swarm) CreateConfig(c *check.C, configSpec swarm.ConfigSpec) string {
  423. cli, err := d.NewClient()
  424. c.Assert(err, checker.IsNil)
  425. defer cli.Close()
  426. scr, err := cli.ConfigCreate(context.Background(), configSpec)
  427. c.Assert(err, checker.IsNil)
  428. return scr.ID
  429. }
  430. // ListConfigs returns the list of the current swarm configs
  431. func (d *Swarm) ListConfigs(c *check.C) []swarm.Config {
  432. cli, err := d.NewClient()
  433. c.Assert(err, checker.IsNil)
  434. defer cli.Close()
  435. configs, err := cli.ConfigList(context.Background(), types.ConfigListOptions{})
  436. c.Assert(err, checker.IsNil)
  437. return configs
  438. }
  439. // GetConfig returns a swarm config identified by the specified id
  440. func (d *Swarm) GetConfig(c *check.C, id string) *swarm.Config {
  441. cli, err := d.NewClient()
  442. c.Assert(err, checker.IsNil)
  443. defer cli.Close()
  444. config, _, err := cli.ConfigInspectWithRaw(context.Background(), id)
  445. c.Assert(err, checker.IsNil)
  446. return &config
  447. }
  448. // DeleteConfig removes the swarm config identified by the specified id
  449. func (d *Swarm) DeleteConfig(c *check.C, id string) {
  450. cli, err := d.NewClient()
  451. c.Assert(err, checker.IsNil)
  452. defer cli.Close()
  453. err = cli.ConfigRemove(context.Background(), id)
  454. c.Assert(err, checker.IsNil)
  455. }
  456. // UpdateConfig updates the swarm config identified by the specified id
  457. // Currently, only label update is supported.
  458. func (d *Swarm) UpdateConfig(c *check.C, id string, f ...ConfigConstructor) {
  459. cli, err := d.NewClient()
  460. c.Assert(err, checker.IsNil)
  461. defer cli.Close()
  462. config := d.GetConfig(c, id)
  463. for _, fn := range f {
  464. fn(config)
  465. }
  466. err = cli.ConfigUpdate(context.Background(), config.ID, config.Version, config.Spec)
  467. c.Assert(err, checker.IsNil)
  468. }
  469. // GetSwarm returns the current swarm object
  470. func (d *Swarm) GetSwarm(c *check.C) swarm.Swarm {
  471. cli, err := d.NewClient()
  472. c.Assert(err, checker.IsNil)
  473. defer cli.Close()
  474. sw, err := cli.SwarmInspect(context.Background())
  475. c.Assert(err, checker.IsNil)
  476. return sw
  477. }
  478. // UpdateSwarm updates the current swarm object with the specified spec constructors
  479. func (d *Swarm) UpdateSwarm(c *check.C, f ...SpecConstructor) {
  480. cli, err := d.NewClient()
  481. c.Assert(err, checker.IsNil)
  482. defer cli.Close()
  483. sw := d.GetSwarm(c)
  484. for _, fn := range f {
  485. fn(&sw.Spec)
  486. }
  487. err = cli.SwarmUpdate(context.Background(), sw.Version, sw.Spec, swarm.UpdateFlags{})
  488. c.Assert(err, checker.IsNil)
  489. }
  490. // RotateTokens update the swarm to rotate tokens
  491. func (d *Swarm) RotateTokens(c *check.C) {
  492. cli, err := d.NewClient()
  493. c.Assert(err, checker.IsNil)
  494. defer cli.Close()
  495. sw, err := cli.SwarmInspect(context.Background())
  496. c.Assert(err, checker.IsNil)
  497. flags := swarm.UpdateFlags{
  498. RotateManagerToken: true,
  499. RotateWorkerToken: true,
  500. }
  501. err = cli.SwarmUpdate(context.Background(), sw.Version, sw.Spec, flags)
  502. c.Assert(err, checker.IsNil)
  503. }
  504. // JoinTokens returns the current swarm join tokens
  505. func (d *Swarm) JoinTokens(c *check.C) swarm.JoinTokens {
  506. cli, err := d.NewClient()
  507. c.Assert(err, checker.IsNil)
  508. defer cli.Close()
  509. sw, err := cli.SwarmInspect(context.Background())
  510. c.Assert(err, checker.IsNil)
  511. return sw.JoinTokens
  512. }
  513. // CheckLocalNodeState returns the current swarm node state
  514. func (d *Swarm) CheckLocalNodeState(c *check.C) (interface{}, check.CommentInterface) {
  515. info, err := d.SwarmInfo()
  516. c.Assert(err, checker.IsNil)
  517. return info.LocalNodeState, nil
  518. }
  519. // CheckControlAvailable returns the current swarm control available
  520. func (d *Swarm) CheckControlAvailable(c *check.C) (interface{}, check.CommentInterface) {
  521. info, err := d.SwarmInfo()
  522. c.Assert(err, checker.IsNil)
  523. c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive)
  524. return info.ControlAvailable, nil
  525. }
  526. // CheckLeader returns whether there is a leader on the swarm or not
  527. func (d *Swarm) CheckLeader(c *check.C) (interface{}, check.CommentInterface) {
  528. cli, err := d.NewClient()
  529. c.Assert(err, checker.IsNil)
  530. defer cli.Close()
  531. errList := check.Commentf("could not get node list")
  532. ls, err := cli.NodeList(context.Background(), types.NodeListOptions{})
  533. if err != nil {
  534. return err, errList
  535. }
  536. for _, node := range ls {
  537. if node.ManagerStatus != nil && node.ManagerStatus.Leader {
  538. return nil, nil
  539. }
  540. }
  541. return fmt.Errorf("no leader"), check.Commentf("could not find leader")
  542. }
  543. // CmdRetryOutOfSequence tries the specified command against the current daemon for 10 times
  544. func (d *Swarm) CmdRetryOutOfSequence(args ...string) (string, error) {
  545. for i := 0; ; i++ {
  546. out, err := d.Cmd(args...)
  547. if err != nil {
  548. if strings.Contains(out, "update out of sequence") {
  549. if i < 10 {
  550. continue
  551. }
  552. }
  553. }
  554. return out, err
  555. }
  556. }