docker_utils.go 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615
  1. package main
  2. import (
  3. "errors"
  4. "fmt"
  5. "io"
  6. "io/ioutil"
  7. "net"
  8. "net/http"
  9. "net/http/httptest"
  10. "net/http/httputil"
  11. "os"
  12. "os/exec"
  13. "path"
  14. "path/filepath"
  15. "strconv"
  16. "strings"
  17. "testing"
  18. "time"
  19. )
  20. // Daemon represents a Docker daemon for the testing framework.
  21. type Daemon struct {
  22. t *testing.T
  23. logFile *os.File
  24. folder string
  25. stdin io.WriteCloser
  26. stdout, stderr io.ReadCloser
  27. cmd *exec.Cmd
  28. storageDriver string
  29. execDriver string
  30. wait chan error
  31. }
  32. // NewDaemon returns a Daemon instance to be used for testing.
  33. // This will create a directory such as daemon123456789 in the folder specified by $DEST.
  34. // The daemon will not automatically start.
  35. func NewDaemon(t *testing.T) *Daemon {
  36. dest := os.Getenv("DEST")
  37. if dest == "" {
  38. t.Fatal("Please set the DEST environment variable")
  39. }
  40. dir := filepath.Join(dest, fmt.Sprintf("daemon%d", time.Now().Unix()))
  41. daemonFolder, err := filepath.Abs(dir)
  42. if err != nil {
  43. t.Fatal("Could not make '%s' an absolute path: %v", dir, err)
  44. }
  45. if err := os.MkdirAll(filepath.Join(daemonFolder, "graph"), 0600); err != nil {
  46. t.Fatal("Could not create %s/graph directory", daemonFolder)
  47. }
  48. return &Daemon{
  49. t: t,
  50. folder: daemonFolder,
  51. storageDriver: os.Getenv("DOCKER_GRAPHDRIVER"),
  52. execDriver: os.Getenv("DOCKER_EXECDRIVER"),
  53. }
  54. }
  55. // Start will start the daemon and return once it is ready to receive requests.
  56. // You can specify additional daemon flags (e.g. "--restart=false").
  57. func (d *Daemon) Start(arg ...string) error {
  58. dockerBinary, err := exec.LookPath(dockerBinary)
  59. if err != nil {
  60. d.t.Fatalf("could not find docker binary in $PATH: %v", err)
  61. }
  62. args := []string{
  63. "--host", d.sock(),
  64. "--daemon", "--debug",
  65. "--graph", fmt.Sprintf("%s/graph", d.folder),
  66. "--storage-driver", d.storageDriver,
  67. "--exec-driver", d.execDriver,
  68. "--pidfile", fmt.Sprintf("%s/docker.pid", d.folder),
  69. }
  70. args = append(args, arg...)
  71. d.cmd = exec.Command(dockerBinary, args...)
  72. d.logFile, err = os.OpenFile(filepath.Join(d.folder, "docker.log"), os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0600)
  73. if err != nil {
  74. d.t.Fatalf("Could not create %s/docker.log: %v", d.folder, err)
  75. }
  76. d.cmd.Stdout = d.logFile
  77. d.cmd.Stderr = d.logFile
  78. if err := d.cmd.Start(); err != nil {
  79. return fmt.Errorf("Could not start daemon container: %v", err)
  80. }
  81. wait := make(chan error)
  82. go func() {
  83. wait <- d.cmd.Wait()
  84. d.t.Log("exiting daemon")
  85. close(wait)
  86. }()
  87. d.wait = wait
  88. tick := time.Tick(500 * time.Millisecond)
  89. // make sure daemon is ready to receive requests
  90. for {
  91. d.t.Log("waiting for daemon to start")
  92. select {
  93. case <-time.After(2 * time.Second):
  94. return errors.New("timeout: daemon does not respond")
  95. case <-tick:
  96. c, err := net.Dial("unix", filepath.Join(d.folder, "docker.sock"))
  97. if err != nil {
  98. continue
  99. }
  100. client := httputil.NewClientConn(c, nil)
  101. defer client.Close()
  102. req, err := http.NewRequest("GET", "/_ping", nil)
  103. if err != nil {
  104. d.t.Fatalf("could not create new request: %v", err)
  105. }
  106. resp, err := client.Do(req)
  107. if err != nil {
  108. continue
  109. }
  110. if resp.StatusCode != http.StatusOK {
  111. d.t.Logf("received status != 200 OK: %s", resp.Status)
  112. }
  113. d.t.Log("daemon started")
  114. return nil
  115. }
  116. }
  117. }
  118. // StartWithBusybox will first start the daemon with Daemon.Start()
  119. // then save the busybox image from the main daemon and load it into this Daemon instance.
  120. func (d *Daemon) StartWithBusybox(arg ...string) error {
  121. if err := d.Start(arg...); err != nil {
  122. return err
  123. }
  124. bb := filepath.Join(d.folder, "busybox.tar")
  125. if _, err := os.Stat(bb); err != nil {
  126. if !os.IsNotExist(err) {
  127. return fmt.Errorf("unexpected error on busybox.tar stat: %v", err)
  128. }
  129. // saving busybox image from main daemon
  130. if err := exec.Command(dockerBinary, "save", "--output", bb, "busybox:latest").Run(); err != nil {
  131. return fmt.Errorf("could not save busybox image: %v", err)
  132. }
  133. }
  134. // loading busybox image to this daemon
  135. if _, err := d.Cmd("load", "--input", bb); err != nil {
  136. return fmt.Errorf("could not load busybox image: %v", err)
  137. }
  138. if err := os.Remove(bb); err != nil {
  139. d.t.Logf("Could not remove %s: %v", bb, err)
  140. }
  141. return nil
  142. }
  143. // Stop will send a SIGINT every second and wait for the daemon to stop.
  144. // If it timeouts, a SIGKILL is sent.
  145. // Stop will not delete the daemon directory. If a purged daemon is needed,
  146. // instantiate a new one with NewDaemon.
  147. func (d *Daemon) Stop() error {
  148. if d.cmd == nil || d.wait == nil {
  149. return errors.New("Daemon not started")
  150. }
  151. defer func() {
  152. d.logFile.Close()
  153. d.cmd = nil
  154. }()
  155. i := 1
  156. tick := time.Tick(time.Second)
  157. if err := d.cmd.Process.Signal(os.Interrupt); err != nil {
  158. return fmt.Errorf("Could not send signal: %v", err)
  159. }
  160. out:
  161. for {
  162. select {
  163. case err := <-d.wait:
  164. return err
  165. case <-time.After(20 * time.Second):
  166. d.t.Log("timeout")
  167. break out
  168. case <-tick:
  169. d.t.Logf("Attempt #%d: daemon is still running with pid %d", i+1, d.cmd.Process.Pid)
  170. if err := d.cmd.Process.Signal(os.Interrupt); err != nil {
  171. return fmt.Errorf("Could not send signal: %v", err)
  172. }
  173. i++
  174. }
  175. }
  176. if err := d.cmd.Process.Kill(); err != nil {
  177. d.t.Logf("Could not kill daemon: %v", err)
  178. return err
  179. }
  180. return nil
  181. }
  182. // Restart will restart the daemon by first stopping it and then starting it.
  183. func (d *Daemon) Restart(arg ...string) error {
  184. d.Stop()
  185. return d.Start(arg...)
  186. }
  187. func (d *Daemon) sock() string {
  188. return fmt.Sprintf("unix://%s/docker.sock", d.folder)
  189. }
  190. // Cmd will execute a docker CLI command against this Daemon.
  191. // Example: d.Cmd("version") will run docker -H unix://path/to/unix.sock version
  192. func (d *Daemon) Cmd(name string, arg ...string) (string, error) {
  193. args := []string{"--host", d.sock(), name}
  194. args = append(args, arg...)
  195. c := exec.Command(dockerBinary, args...)
  196. b, err := c.CombinedOutput()
  197. return string(b), err
  198. }
  199. func deleteContainer(container string) error {
  200. container = strings.Replace(container, "\n", " ", -1)
  201. container = strings.Trim(container, " ")
  202. killArgs := fmt.Sprintf("kill %v", container)
  203. killSplitArgs := strings.Split(killArgs, " ")
  204. killCmd := exec.Command(dockerBinary, killSplitArgs...)
  205. runCommand(killCmd)
  206. rmArgs := fmt.Sprintf("rm %v", container)
  207. rmSplitArgs := strings.Split(rmArgs, " ")
  208. rmCmd := exec.Command(dockerBinary, rmSplitArgs...)
  209. exitCode, err := runCommand(rmCmd)
  210. // set error manually if not set
  211. if exitCode != 0 && err == nil {
  212. err = fmt.Errorf("failed to remove container: `docker rm` exit is non-zero")
  213. }
  214. return err
  215. }
  216. func getAllContainers() (string, error) {
  217. getContainersCmd := exec.Command(dockerBinary, "ps", "-q", "-a")
  218. out, exitCode, err := runCommandWithOutput(getContainersCmd)
  219. if exitCode != 0 && err == nil {
  220. err = fmt.Errorf("failed to get a list of containers: %v\n", out)
  221. }
  222. return out, err
  223. }
  224. func deleteAllContainers() error {
  225. containers, err := getAllContainers()
  226. if err != nil {
  227. fmt.Println(containers)
  228. return err
  229. }
  230. if err = deleteContainer(containers); err != nil {
  231. return err
  232. }
  233. return nil
  234. }
  235. func deleteImages(images string) error {
  236. rmiCmd := exec.Command(dockerBinary, "rmi", images)
  237. exitCode, err := runCommand(rmiCmd)
  238. // set error manually if not set
  239. if exitCode != 0 && err == nil {
  240. err = fmt.Errorf("failed to remove image: `docker rmi` exit is non-zero")
  241. }
  242. return err
  243. }
  244. func imageExists(image string) error {
  245. inspectCmd := exec.Command(dockerBinary, "inspect", image)
  246. exitCode, err := runCommand(inspectCmd)
  247. if exitCode != 0 && err == nil {
  248. err = fmt.Errorf("couldn't find image '%s'", image)
  249. }
  250. return err
  251. }
  252. func pullImageIfNotExist(image string) (err error) {
  253. if err := imageExists(image); err != nil {
  254. pullCmd := exec.Command(dockerBinary, "pull", image)
  255. _, exitCode, err := runCommandWithOutput(pullCmd)
  256. if err != nil || exitCode != 0 {
  257. err = fmt.Errorf("image '%s' wasn't found locally and it couldn't be pulled: %s", image, err)
  258. }
  259. }
  260. return
  261. }
  262. // deprecated, use dockerCmd instead
  263. func cmd(t *testing.T, args ...string) (string, int, error) {
  264. return dockerCmd(t, args...)
  265. }
  266. func dockerCmd(t *testing.T, args ...string) (string, int, error) {
  267. out, status, err := runCommandWithOutput(exec.Command(dockerBinary, args...))
  268. errorOut(err, t, fmt.Sprintf("'%s' failed with errors: %v (%v)", strings.Join(args, " "), err, out))
  269. return out, status, err
  270. }
  271. // execute a docker command in a directory
  272. func dockerCmdInDir(t *testing.T, path string, args ...string) (string, int, error) {
  273. dockerCommand := exec.Command(dockerBinary, args...)
  274. dockerCommand.Dir = path
  275. out, status, err := runCommandWithOutput(dockerCommand)
  276. errorOut(err, t, fmt.Sprintf("'%s' failed with errors: %v (%v)", strings.Join(args, " "), err, out))
  277. return out, status, err
  278. }
  279. func findContainerIp(t *testing.T, id string) string {
  280. cmd := exec.Command(dockerBinary, "inspect", "--format='{{ .NetworkSettings.IPAddress }}'", id)
  281. out, _, err := runCommandWithOutput(cmd)
  282. if err != nil {
  283. t.Fatal(err, out)
  284. }
  285. return strings.Trim(out, " \r\n'")
  286. }
  287. func getContainerCount() (int, error) {
  288. const containers = "Containers:"
  289. cmd := exec.Command(dockerBinary, "info")
  290. out, _, err := runCommandWithOutput(cmd)
  291. if err != nil {
  292. return 0, err
  293. }
  294. lines := strings.Split(out, "\n")
  295. for _, line := range lines {
  296. if strings.Contains(line, containers) {
  297. output := stripTrailingCharacters(line)
  298. output = strings.TrimLeft(output, containers)
  299. output = strings.Trim(output, " ")
  300. containerCount, err := strconv.Atoi(output)
  301. if err != nil {
  302. return 0, err
  303. }
  304. return containerCount, nil
  305. }
  306. }
  307. return 0, fmt.Errorf("couldn't find the Container count in the output")
  308. }
  309. type FakeContext struct {
  310. Dir string
  311. }
  312. func (f *FakeContext) Add(file, content string) error {
  313. filepath := path.Join(f.Dir, file)
  314. dirpath := path.Dir(filepath)
  315. if dirpath != "." {
  316. if err := os.MkdirAll(dirpath, 0755); err != nil {
  317. return err
  318. }
  319. }
  320. return ioutil.WriteFile(filepath, []byte(content), 0644)
  321. }
  322. func (f *FakeContext) Delete(file string) error {
  323. filepath := path.Join(f.Dir, file)
  324. return os.RemoveAll(filepath)
  325. }
  326. func (f *FakeContext) Close() error {
  327. return os.RemoveAll(f.Dir)
  328. }
  329. func fakeContext(dockerfile string, files map[string]string) (*FakeContext, error) {
  330. tmp, err := ioutil.TempDir("", "fake-context")
  331. if err != nil {
  332. return nil, err
  333. }
  334. ctx := &FakeContext{tmp}
  335. for file, content := range files {
  336. if err := ctx.Add(file, content); err != nil {
  337. ctx.Close()
  338. return nil, err
  339. }
  340. }
  341. if err := ctx.Add("Dockerfile", dockerfile); err != nil {
  342. ctx.Close()
  343. return nil, err
  344. }
  345. return ctx, nil
  346. }
  347. type FakeStorage struct {
  348. *FakeContext
  349. *httptest.Server
  350. }
  351. func (f *FakeStorage) Close() error {
  352. f.Server.Close()
  353. return f.FakeContext.Close()
  354. }
  355. func fakeStorage(files map[string]string) (*FakeStorage, error) {
  356. tmp, err := ioutil.TempDir("", "fake-storage")
  357. if err != nil {
  358. return nil, err
  359. }
  360. ctx := &FakeContext{tmp}
  361. for file, content := range files {
  362. if err := ctx.Add(file, content); err != nil {
  363. ctx.Close()
  364. return nil, err
  365. }
  366. }
  367. handler := http.FileServer(http.Dir(ctx.Dir))
  368. server := httptest.NewServer(handler)
  369. return &FakeStorage{
  370. FakeContext: ctx,
  371. Server: server,
  372. }, nil
  373. }
  374. func inspectField(name, field string) (string, error) {
  375. format := fmt.Sprintf("{{.%s}}", field)
  376. inspectCmd := exec.Command(dockerBinary, "inspect", "-f", format, name)
  377. out, exitCode, err := runCommandWithOutput(inspectCmd)
  378. if err != nil || exitCode != 0 {
  379. return "", fmt.Errorf("failed to inspect %s: %s", name, out)
  380. }
  381. return strings.TrimSpace(out), nil
  382. }
  383. func inspectFieldJSON(name, field string) (string, error) {
  384. format := fmt.Sprintf("{{json .%s}}", field)
  385. inspectCmd := exec.Command(dockerBinary, "inspect", "-f", format, name)
  386. out, exitCode, err := runCommandWithOutput(inspectCmd)
  387. if err != nil || exitCode != 0 {
  388. return "", fmt.Errorf("failed to inspect %s: %s", name, out)
  389. }
  390. return strings.TrimSpace(out), nil
  391. }
  392. func getIDByName(name string) (string, error) {
  393. return inspectField(name, "Id")
  394. }
  395. func buildImageWithOut(name, dockerfile string, useCache bool) (string, string, error) {
  396. args := []string{"build", "-t", name}
  397. if !useCache {
  398. args = append(args, "--no-cache")
  399. }
  400. args = append(args, "-")
  401. buildCmd := exec.Command(dockerBinary, args...)
  402. buildCmd.Stdin = strings.NewReader(dockerfile)
  403. out, exitCode, err := runCommandWithOutput(buildCmd)
  404. if err != nil || exitCode != 0 {
  405. return "", out, fmt.Errorf("failed to build the image: %s", out)
  406. }
  407. id, err := getIDByName(name)
  408. if err != nil {
  409. return "", out, err
  410. }
  411. return id, out, nil
  412. }
  413. func buildImage(name, dockerfile string, useCache bool) (string, error) {
  414. id, _, err := buildImageWithOut(name, dockerfile, useCache)
  415. return id, err
  416. }
  417. func buildImageFromContext(name string, ctx *FakeContext, useCache bool) (string, error) {
  418. args := []string{"build", "-t", name}
  419. if !useCache {
  420. args = append(args, "--no-cache")
  421. }
  422. args = append(args, ".")
  423. buildCmd := exec.Command(dockerBinary, args...)
  424. buildCmd.Dir = ctx.Dir
  425. out, exitCode, err := runCommandWithOutput(buildCmd)
  426. if err != nil || exitCode != 0 {
  427. return "", fmt.Errorf("failed to build the image: %s", out)
  428. }
  429. return getIDByName(name)
  430. }
  431. func buildImageFromPath(name, path string, useCache bool) (string, error) {
  432. args := []string{"build", "-t", name}
  433. if !useCache {
  434. args = append(args, "--no-cache")
  435. }
  436. args = append(args, path)
  437. buildCmd := exec.Command(dockerBinary, args...)
  438. out, exitCode, err := runCommandWithOutput(buildCmd)
  439. if err != nil || exitCode != 0 {
  440. return "", fmt.Errorf("failed to build the image: %s", out)
  441. }
  442. return getIDByName(name)
  443. }
  444. type FakeGIT struct {
  445. *httptest.Server
  446. Root string
  447. RepoURL string
  448. }
  449. func (g *FakeGIT) Close() {
  450. g.Server.Close()
  451. os.RemoveAll(g.Root)
  452. }
  453. func fakeGIT(name string, files map[string]string) (*FakeGIT, error) {
  454. tmp, err := ioutil.TempDir("", "fake-git-repo")
  455. if err != nil {
  456. return nil, err
  457. }
  458. ctx := &FakeContext{tmp}
  459. for file, content := range files {
  460. if err := ctx.Add(file, content); err != nil {
  461. ctx.Close()
  462. return nil, err
  463. }
  464. }
  465. defer ctx.Close()
  466. curdir, err := os.Getwd()
  467. if err != nil {
  468. return nil, err
  469. }
  470. defer os.Chdir(curdir)
  471. if output, err := exec.Command("git", "init", ctx.Dir).CombinedOutput(); err != nil {
  472. return nil, fmt.Errorf("Error trying to init repo: %s (%s)", err, output)
  473. }
  474. err = os.Chdir(ctx.Dir)
  475. if err != nil {
  476. return nil, err
  477. }
  478. if output, err := exec.Command("git", "add", "*").CombinedOutput(); err != nil {
  479. return nil, fmt.Errorf("Error trying to add files to repo: %s (%s)", err, output)
  480. }
  481. if output, err := exec.Command("git", "commit", "-a", "-m", "Initial commit").CombinedOutput(); err != nil {
  482. return nil, fmt.Errorf("Error trying to commit to repo: %s (%s)", err, output)
  483. }
  484. root, err := ioutil.TempDir("", "docker-test-git-repo")
  485. if err != nil {
  486. return nil, err
  487. }
  488. repoPath := filepath.Join(root, name+".git")
  489. if output, err := exec.Command("git", "clone", "--bare", ctx.Dir, repoPath).CombinedOutput(); err != nil {
  490. os.RemoveAll(root)
  491. return nil, fmt.Errorf("Error trying to clone --bare: %s (%s)", err, output)
  492. }
  493. err = os.Chdir(repoPath)
  494. if err != nil {
  495. os.RemoveAll(root)
  496. return nil, err
  497. }
  498. if output, err := exec.Command("git", "update-server-info").CombinedOutput(); err != nil {
  499. os.RemoveAll(root)
  500. return nil, fmt.Errorf("Error trying to git update-server-info: %s (%s)", err, output)
  501. }
  502. err = os.Chdir(curdir)
  503. if err != nil {
  504. os.RemoveAll(root)
  505. return nil, err
  506. }
  507. handler := http.FileServer(http.Dir(root))
  508. server := httptest.NewServer(handler)
  509. return &FakeGIT{
  510. Server: server,
  511. Root: root,
  512. RepoURL: fmt.Sprintf("%s/%s.git", server.URL, name),
  513. }, nil
  514. }
  515. // Write `content` to the file at path `dst`, creating it if necessary,
  516. // as well as any missing directories.
  517. // The file is truncated if it already exists.
  518. // Call t.Fatal() at the first error.
  519. func writeFile(dst, content string, t *testing.T) {
  520. // Create subdirectories if necessary
  521. if err := os.MkdirAll(path.Dir(dst), 0700); err != nil && !os.IsExist(err) {
  522. t.Fatal(err)
  523. }
  524. f, err := os.OpenFile(dst, os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0700)
  525. if err != nil {
  526. t.Fatal(err)
  527. }
  528. // Write content (truncate if it exists)
  529. if _, err := io.Copy(f, strings.NewReader(content)); err != nil {
  530. t.Fatal(err)
  531. }
  532. }
  533. // Return the contents of file at path `src`.
  534. // Call t.Fatal() at the first error (including if the file doesn't exist)
  535. func readFile(src string, t *testing.T) (content string) {
  536. f, err := os.Open(src)
  537. if err != nil {
  538. t.Fatal(err)
  539. }
  540. data, err := ioutil.ReadAll(f)
  541. if err != nil {
  542. t.Fatal(err)
  543. }
  544. return string(data)
  545. }