docker_utils.go 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620
  1. package main
  2. import (
  3. "errors"
  4. "fmt"
  5. "io"
  6. "io/ioutil"
  7. "net"
  8. "net/http"
  9. "net/http/httptest"
  10. "net/http/httputil"
  11. "os"
  12. "os/exec"
  13. "path"
  14. "path/filepath"
  15. "strconv"
  16. "strings"
  17. "testing"
  18. "time"
  19. )
  20. // Daemon represents a Docker daemon for the testing framework.
  21. type Daemon struct {
  22. t *testing.T
  23. logFile *os.File
  24. folder string
  25. stdin io.WriteCloser
  26. stdout, stderr io.ReadCloser
  27. cmd *exec.Cmd
  28. storageDriver string
  29. execDriver string
  30. wait chan error
  31. }
  32. // NewDaemon returns a Daemon instance to be used for testing.
  33. // This will create a directory such as daemon123456789 in the folder specified by $DEST.
  34. // The daemon will not automatically start.
  35. func NewDaemon(t *testing.T) *Daemon {
  36. dest := os.Getenv("DEST")
  37. if dest == "" {
  38. t.Fatal("Please set the DEST environment variable")
  39. }
  40. dir := filepath.Join(dest, fmt.Sprintf("daemon%d", time.Now().Unix()))
  41. daemonFolder, err := filepath.Abs(dir)
  42. if err != nil {
  43. t.Fatal("Could not make '%s' an absolute path: %v", dir, err)
  44. }
  45. if err := os.MkdirAll(filepath.Join(daemonFolder, "graph"), 0600); err != nil {
  46. t.Fatal("Could not create %s/graph directory", daemonFolder)
  47. }
  48. return &Daemon{
  49. t: t,
  50. folder: daemonFolder,
  51. storageDriver: os.Getenv("DOCKER_GRAPHDRIVER"),
  52. execDriver: os.Getenv("DOCKER_EXECDRIVER"),
  53. }
  54. }
  55. // Start will start the daemon and return once it is ready to receive requests.
  56. // You can specify additional daemon flags.
  57. func (d *Daemon) Start(arg ...string) error {
  58. dockerBinary, err := exec.LookPath(dockerBinary)
  59. if err != nil {
  60. d.t.Fatalf("could not find docker binary in $PATH: %v", err)
  61. }
  62. args := []string{
  63. "--host", d.sock(),
  64. "--daemon", "--debug",
  65. "--graph", fmt.Sprintf("%s/graph", d.folder),
  66. "--pidfile", fmt.Sprintf("%s/docker.pid", d.folder),
  67. }
  68. if d.storageDriver != "" {
  69. args = append(args, "--storage-driver", d.storageDriver)
  70. }
  71. if d.execDriver != "" {
  72. args = append(args, "--exec-driver", d.execDriver)
  73. }
  74. args = append(args, arg...)
  75. d.cmd = exec.Command(dockerBinary, args...)
  76. d.logFile, err = os.OpenFile(filepath.Join(d.folder, "docker.log"), os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0600)
  77. if err != nil {
  78. d.t.Fatalf("Could not create %s/docker.log: %v", d.folder, err)
  79. }
  80. d.cmd.Stdout = d.logFile
  81. d.cmd.Stderr = d.logFile
  82. if err := d.cmd.Start(); err != nil {
  83. return fmt.Errorf("Could not start daemon container: %v", err)
  84. }
  85. wait := make(chan error)
  86. go func() {
  87. wait <- d.cmd.Wait()
  88. d.t.Log("exiting daemon")
  89. close(wait)
  90. }()
  91. d.wait = wait
  92. tick := time.Tick(500 * time.Millisecond)
  93. // make sure daemon is ready to receive requests
  94. for {
  95. d.t.Log("waiting for daemon to start")
  96. select {
  97. case <-time.After(2 * time.Second):
  98. return errors.New("timeout: daemon does not respond")
  99. case <-tick:
  100. c, err := net.Dial("unix", filepath.Join(d.folder, "docker.sock"))
  101. if err != nil {
  102. continue
  103. }
  104. client := httputil.NewClientConn(c, nil)
  105. defer client.Close()
  106. req, err := http.NewRequest("GET", "/_ping", nil)
  107. if err != nil {
  108. d.t.Fatalf("could not create new request: %v", err)
  109. }
  110. resp, err := client.Do(req)
  111. if err != nil {
  112. continue
  113. }
  114. if resp.StatusCode != http.StatusOK {
  115. d.t.Logf("received status != 200 OK: %s", resp.Status)
  116. }
  117. d.t.Log("daemon started")
  118. return nil
  119. }
  120. }
  121. }
  122. // StartWithBusybox will first start the daemon with Daemon.Start()
  123. // then save the busybox image from the main daemon and load it into this Daemon instance.
  124. func (d *Daemon) StartWithBusybox(arg ...string) error {
  125. if err := d.Start(arg...); err != nil {
  126. return err
  127. }
  128. bb := filepath.Join(d.folder, "busybox.tar")
  129. if _, err := os.Stat(bb); err != nil {
  130. if !os.IsNotExist(err) {
  131. return fmt.Errorf("unexpected error on busybox.tar stat: %v", err)
  132. }
  133. // saving busybox image from main daemon
  134. if err := exec.Command(dockerBinary, "save", "--output", bb, "busybox:latest").Run(); err != nil {
  135. return fmt.Errorf("could not save busybox image: %v", err)
  136. }
  137. }
  138. // loading busybox image to this daemon
  139. if _, err := d.Cmd("load", "--input", bb); err != nil {
  140. return fmt.Errorf("could not load busybox image: %v", err)
  141. }
  142. if err := os.Remove(bb); err != nil {
  143. d.t.Logf("Could not remove %s: %v", bb, err)
  144. }
  145. return nil
  146. }
  147. // Stop will send a SIGINT every second and wait for the daemon to stop.
  148. // If it timeouts, a SIGKILL is sent.
  149. // Stop will not delete the daemon directory. If a purged daemon is needed,
  150. // instantiate a new one with NewDaemon.
  151. func (d *Daemon) Stop() error {
  152. if d.cmd == nil || d.wait == nil {
  153. return errors.New("Daemon not started")
  154. }
  155. defer func() {
  156. d.logFile.Close()
  157. d.cmd = nil
  158. }()
  159. i := 1
  160. tick := time.Tick(time.Second)
  161. if err := d.cmd.Process.Signal(os.Interrupt); err != nil {
  162. return fmt.Errorf("Could not send signal: %v", err)
  163. }
  164. out:
  165. for {
  166. select {
  167. case err := <-d.wait:
  168. return err
  169. case <-time.After(20 * time.Second):
  170. d.t.Log("timeout")
  171. break out
  172. case <-tick:
  173. d.t.Logf("Attempt #%d: daemon is still running with pid %d", i+1, d.cmd.Process.Pid)
  174. if err := d.cmd.Process.Signal(os.Interrupt); err != nil {
  175. return fmt.Errorf("Could not send signal: %v", err)
  176. }
  177. i++
  178. }
  179. }
  180. if err := d.cmd.Process.Kill(); err != nil {
  181. d.t.Logf("Could not kill daemon: %v", err)
  182. return err
  183. }
  184. return nil
  185. }
  186. // Restart will restart the daemon by first stopping it and then starting it.
  187. func (d *Daemon) Restart(arg ...string) error {
  188. d.Stop()
  189. return d.Start(arg...)
  190. }
  191. func (d *Daemon) sock() string {
  192. return fmt.Sprintf("unix://%s/docker.sock", d.folder)
  193. }
  194. // Cmd will execute a docker CLI command against this Daemon.
  195. // Example: d.Cmd("version") will run docker -H unix://path/to/unix.sock version
  196. func (d *Daemon) Cmd(name string, arg ...string) (string, error) {
  197. args := []string{"--host", d.sock(), name}
  198. args = append(args, arg...)
  199. c := exec.Command(dockerBinary, args...)
  200. b, err := c.CombinedOutput()
  201. return string(b), err
  202. }
  203. func deleteContainer(container string) error {
  204. container = strings.Replace(container, "\n", " ", -1)
  205. container = strings.Trim(container, " ")
  206. killArgs := fmt.Sprintf("kill %v", container)
  207. killSplitArgs := strings.Split(killArgs, " ")
  208. killCmd := exec.Command(dockerBinary, killSplitArgs...)
  209. runCommand(killCmd)
  210. rmArgs := fmt.Sprintf("rm %v", container)
  211. rmSplitArgs := strings.Split(rmArgs, " ")
  212. rmCmd := exec.Command(dockerBinary, rmSplitArgs...)
  213. exitCode, err := runCommand(rmCmd)
  214. // set error manually if not set
  215. if exitCode != 0 && err == nil {
  216. err = fmt.Errorf("failed to remove container: `docker rm` exit is non-zero")
  217. }
  218. return err
  219. }
  220. func getAllContainers() (string, error) {
  221. getContainersCmd := exec.Command(dockerBinary, "ps", "-q", "-a")
  222. out, exitCode, err := runCommandWithOutput(getContainersCmd)
  223. if exitCode != 0 && err == nil {
  224. err = fmt.Errorf("failed to get a list of containers: %v\n", out)
  225. }
  226. return out, err
  227. }
  228. func deleteAllContainers() error {
  229. containers, err := getAllContainers()
  230. if err != nil {
  231. fmt.Println(containers)
  232. return err
  233. }
  234. if err = deleteContainer(containers); err != nil {
  235. return err
  236. }
  237. return nil
  238. }
  239. func deleteImages(images string) error {
  240. rmiCmd := exec.Command(dockerBinary, "rmi", images)
  241. exitCode, err := runCommand(rmiCmd)
  242. // set error manually if not set
  243. if exitCode != 0 && err == nil {
  244. err = fmt.Errorf("failed to remove image: `docker rmi` exit is non-zero")
  245. }
  246. return err
  247. }
  248. func imageExists(image string) error {
  249. inspectCmd := exec.Command(dockerBinary, "inspect", image)
  250. exitCode, err := runCommand(inspectCmd)
  251. if exitCode != 0 && err == nil {
  252. err = fmt.Errorf("couldn't find image '%s'", image)
  253. }
  254. return err
  255. }
  256. func pullImageIfNotExist(image string) (err error) {
  257. if err := imageExists(image); err != nil {
  258. pullCmd := exec.Command(dockerBinary, "pull", image)
  259. _, exitCode, err := runCommandWithOutput(pullCmd)
  260. if err != nil || exitCode != 0 {
  261. err = fmt.Errorf("image '%s' wasn't found locally and it couldn't be pulled: %s", image, err)
  262. }
  263. }
  264. return
  265. }
  266. // deprecated, use dockerCmd instead
  267. func cmd(t *testing.T, args ...string) (string, int, error) {
  268. return dockerCmd(t, args...)
  269. }
  270. func dockerCmd(t *testing.T, args ...string) (string, int, error) {
  271. out, status, err := runCommandWithOutput(exec.Command(dockerBinary, args...))
  272. errorOut(err, t, fmt.Sprintf("'%s' failed with errors: %v (%v)", strings.Join(args, " "), err, out))
  273. return out, status, err
  274. }
  275. // execute a docker command in a directory
  276. func dockerCmdInDir(t *testing.T, path string, args ...string) (string, int, error) {
  277. dockerCommand := exec.Command(dockerBinary, args...)
  278. dockerCommand.Dir = path
  279. out, status, err := runCommandWithOutput(dockerCommand)
  280. errorOut(err, t, fmt.Sprintf("'%s' failed with errors: %v (%v)", strings.Join(args, " "), err, out))
  281. return out, status, err
  282. }
  283. func findContainerIp(t *testing.T, id string) string {
  284. cmd := exec.Command(dockerBinary, "inspect", "--format='{{ .NetworkSettings.IPAddress }}'", id)
  285. out, _, err := runCommandWithOutput(cmd)
  286. if err != nil {
  287. t.Fatal(err, out)
  288. }
  289. return strings.Trim(out, " \r\n'")
  290. }
  291. func getContainerCount() (int, error) {
  292. const containers = "Containers:"
  293. cmd := exec.Command(dockerBinary, "info")
  294. out, _, err := runCommandWithOutput(cmd)
  295. if err != nil {
  296. return 0, err
  297. }
  298. lines := strings.Split(out, "\n")
  299. for _, line := range lines {
  300. if strings.Contains(line, containers) {
  301. output := stripTrailingCharacters(line)
  302. output = strings.TrimLeft(output, containers)
  303. output = strings.Trim(output, " ")
  304. containerCount, err := strconv.Atoi(output)
  305. if err != nil {
  306. return 0, err
  307. }
  308. return containerCount, nil
  309. }
  310. }
  311. return 0, fmt.Errorf("couldn't find the Container count in the output")
  312. }
  313. type FakeContext struct {
  314. Dir string
  315. }
  316. func (f *FakeContext) Add(file, content string) error {
  317. filepath := path.Join(f.Dir, file)
  318. dirpath := path.Dir(filepath)
  319. if dirpath != "." {
  320. if err := os.MkdirAll(dirpath, 0755); err != nil {
  321. return err
  322. }
  323. }
  324. return ioutil.WriteFile(filepath, []byte(content), 0644)
  325. }
  326. func (f *FakeContext) Delete(file string) error {
  327. filepath := path.Join(f.Dir, file)
  328. return os.RemoveAll(filepath)
  329. }
  330. func (f *FakeContext) Close() error {
  331. return os.RemoveAll(f.Dir)
  332. }
  333. func fakeContext(dockerfile string, files map[string]string) (*FakeContext, error) {
  334. tmp, err := ioutil.TempDir("", "fake-context")
  335. if err != nil {
  336. return nil, err
  337. }
  338. ctx := &FakeContext{tmp}
  339. for file, content := range files {
  340. if err := ctx.Add(file, content); err != nil {
  341. ctx.Close()
  342. return nil, err
  343. }
  344. }
  345. if err := ctx.Add("Dockerfile", dockerfile); err != nil {
  346. ctx.Close()
  347. return nil, err
  348. }
  349. return ctx, nil
  350. }
  351. type FakeStorage struct {
  352. *FakeContext
  353. *httptest.Server
  354. }
  355. func (f *FakeStorage) Close() error {
  356. f.Server.Close()
  357. return f.FakeContext.Close()
  358. }
  359. func fakeStorage(files map[string]string) (*FakeStorage, error) {
  360. tmp, err := ioutil.TempDir("", "fake-storage")
  361. if err != nil {
  362. return nil, err
  363. }
  364. ctx := &FakeContext{tmp}
  365. for file, content := range files {
  366. if err := ctx.Add(file, content); err != nil {
  367. ctx.Close()
  368. return nil, err
  369. }
  370. }
  371. handler := http.FileServer(http.Dir(ctx.Dir))
  372. server := httptest.NewServer(handler)
  373. return &FakeStorage{
  374. FakeContext: ctx,
  375. Server: server,
  376. }, nil
  377. }
  378. func inspectField(name, field string) (string, error) {
  379. format := fmt.Sprintf("{{.%s}}", field)
  380. inspectCmd := exec.Command(dockerBinary, "inspect", "-f", format, name)
  381. out, exitCode, err := runCommandWithOutput(inspectCmd)
  382. if err != nil || exitCode != 0 {
  383. return "", fmt.Errorf("failed to inspect %s: %s", name, out)
  384. }
  385. return strings.TrimSpace(out), nil
  386. }
  387. func inspectFieldJSON(name, field string) (string, error) {
  388. format := fmt.Sprintf("{{json .%s}}", field)
  389. inspectCmd := exec.Command(dockerBinary, "inspect", "-f", format, name)
  390. out, exitCode, err := runCommandWithOutput(inspectCmd)
  391. if err != nil || exitCode != 0 {
  392. return "", fmt.Errorf("failed to inspect %s: %s", name, out)
  393. }
  394. return strings.TrimSpace(out), nil
  395. }
  396. func getIDByName(name string) (string, error) {
  397. return inspectField(name, "Id")
  398. }
  399. func buildImageWithOut(name, dockerfile string, useCache bool) (string, string, error) {
  400. args := []string{"build", "-t", name}
  401. if !useCache {
  402. args = append(args, "--no-cache")
  403. }
  404. args = append(args, "-")
  405. buildCmd := exec.Command(dockerBinary, args...)
  406. buildCmd.Stdin = strings.NewReader(dockerfile)
  407. out, exitCode, err := runCommandWithOutput(buildCmd)
  408. if err != nil || exitCode != 0 {
  409. return "", out, fmt.Errorf("failed to build the image: %s", out)
  410. }
  411. id, err := getIDByName(name)
  412. if err != nil {
  413. return "", out, err
  414. }
  415. return id, out, nil
  416. }
  417. func buildImage(name, dockerfile string, useCache bool) (string, error) {
  418. id, _, err := buildImageWithOut(name, dockerfile, useCache)
  419. return id, err
  420. }
  421. func buildImageFromContext(name string, ctx *FakeContext, useCache bool) (string, error) {
  422. args := []string{"build", "-t", name}
  423. if !useCache {
  424. args = append(args, "--no-cache")
  425. }
  426. args = append(args, ".")
  427. buildCmd := exec.Command(dockerBinary, args...)
  428. buildCmd.Dir = ctx.Dir
  429. out, exitCode, err := runCommandWithOutput(buildCmd)
  430. if err != nil || exitCode != 0 {
  431. return "", fmt.Errorf("failed to build the image: %s", out)
  432. }
  433. return getIDByName(name)
  434. }
  435. func buildImageFromPath(name, path string, useCache bool) (string, error) {
  436. args := []string{"build", "-t", name}
  437. if !useCache {
  438. args = append(args, "--no-cache")
  439. }
  440. args = append(args, path)
  441. buildCmd := exec.Command(dockerBinary, args...)
  442. out, exitCode, err := runCommandWithOutput(buildCmd)
  443. if err != nil || exitCode != 0 {
  444. return "", fmt.Errorf("failed to build the image: %s", out)
  445. }
  446. return getIDByName(name)
  447. }
  448. type FakeGIT struct {
  449. *httptest.Server
  450. Root string
  451. RepoURL string
  452. }
  453. func (g *FakeGIT) Close() {
  454. g.Server.Close()
  455. os.RemoveAll(g.Root)
  456. }
  457. func fakeGIT(name string, files map[string]string) (*FakeGIT, error) {
  458. tmp, err := ioutil.TempDir("", "fake-git-repo")
  459. if err != nil {
  460. return nil, err
  461. }
  462. ctx := &FakeContext{tmp}
  463. for file, content := range files {
  464. if err := ctx.Add(file, content); err != nil {
  465. ctx.Close()
  466. return nil, err
  467. }
  468. }
  469. defer ctx.Close()
  470. curdir, err := os.Getwd()
  471. if err != nil {
  472. return nil, err
  473. }
  474. defer os.Chdir(curdir)
  475. if output, err := exec.Command("git", "init", ctx.Dir).CombinedOutput(); err != nil {
  476. return nil, fmt.Errorf("Error trying to init repo: %s (%s)", err, output)
  477. }
  478. err = os.Chdir(ctx.Dir)
  479. if err != nil {
  480. return nil, err
  481. }
  482. if output, err := exec.Command("git", "add", "*").CombinedOutput(); err != nil {
  483. return nil, fmt.Errorf("Error trying to add files to repo: %s (%s)", err, output)
  484. }
  485. if output, err := exec.Command("git", "commit", "-a", "-m", "Initial commit").CombinedOutput(); err != nil {
  486. return nil, fmt.Errorf("Error trying to commit to repo: %s (%s)", err, output)
  487. }
  488. root, err := ioutil.TempDir("", "docker-test-git-repo")
  489. if err != nil {
  490. return nil, err
  491. }
  492. repoPath := filepath.Join(root, name+".git")
  493. if output, err := exec.Command("git", "clone", "--bare", ctx.Dir, repoPath).CombinedOutput(); err != nil {
  494. os.RemoveAll(root)
  495. return nil, fmt.Errorf("Error trying to clone --bare: %s (%s)", err, output)
  496. }
  497. err = os.Chdir(repoPath)
  498. if err != nil {
  499. os.RemoveAll(root)
  500. return nil, err
  501. }
  502. if output, err := exec.Command("git", "update-server-info").CombinedOutput(); err != nil {
  503. os.RemoveAll(root)
  504. return nil, fmt.Errorf("Error trying to git update-server-info: %s (%s)", err, output)
  505. }
  506. err = os.Chdir(curdir)
  507. if err != nil {
  508. os.RemoveAll(root)
  509. return nil, err
  510. }
  511. handler := http.FileServer(http.Dir(root))
  512. server := httptest.NewServer(handler)
  513. return &FakeGIT{
  514. Server: server,
  515. Root: root,
  516. RepoURL: fmt.Sprintf("%s/%s.git", server.URL, name),
  517. }, nil
  518. }
  519. // Write `content` to the file at path `dst`, creating it if necessary,
  520. // as well as any missing directories.
  521. // The file is truncated if it already exists.
  522. // Call t.Fatal() at the first error.
  523. func writeFile(dst, content string, t *testing.T) {
  524. // Create subdirectories if necessary
  525. if err := os.MkdirAll(path.Dir(dst), 0700); err != nil && !os.IsExist(err) {
  526. t.Fatal(err)
  527. }
  528. f, err := os.OpenFile(dst, os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0700)
  529. if err != nil {
  530. t.Fatal(err)
  531. }
  532. // Write content (truncate if it exists)
  533. if _, err := io.Copy(f, strings.NewReader(content)); err != nil {
  534. t.Fatal(err)
  535. }
  536. }
  537. // Return the contents of file at path `src`.
  538. // Call t.Fatal() at the first error (including if the file doesn't exist)
  539. func readFile(src string, t *testing.T) (content string) {
  540. f, err := os.Open(src)
  541. if err != nil {
  542. t.Fatal(err)
  543. }
  544. data, err := ioutil.ReadAll(f)
  545. if err != nil {
  546. t.Fatal(err)
  547. }
  548. return string(data)
  549. }