internals.go 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657
  1. package dockerfile
  2. // internals for handling commands. Covers many areas and a lot of
  3. // non-contiguous functionality. Please read the comments.
  4. import (
  5. "crypto/sha256"
  6. "encoding/hex"
  7. "errors"
  8. "fmt"
  9. "io"
  10. "io/ioutil"
  11. "net/http"
  12. "net/url"
  13. "os"
  14. "path/filepath"
  15. "sort"
  16. "strings"
  17. "sync"
  18. "time"
  19. "github.com/Sirupsen/logrus"
  20. "github.com/docker/docker/api/types"
  21. "github.com/docker/docker/api/types/backend"
  22. "github.com/docker/docker/api/types/container"
  23. "github.com/docker/docker/api/types/strslice"
  24. "github.com/docker/docker/builder"
  25. "github.com/docker/docker/builder/dockerfile/parser"
  26. "github.com/docker/docker/pkg/archive"
  27. "github.com/docker/docker/pkg/httputils"
  28. "github.com/docker/docker/pkg/ioutils"
  29. "github.com/docker/docker/pkg/jsonmessage"
  30. "github.com/docker/docker/pkg/progress"
  31. "github.com/docker/docker/pkg/streamformatter"
  32. "github.com/docker/docker/pkg/stringid"
  33. "github.com/docker/docker/pkg/system"
  34. "github.com/docker/docker/pkg/tarsum"
  35. "github.com/docker/docker/pkg/urlutil"
  36. "github.com/docker/docker/runconfig/opts"
  37. )
  38. func (b *Builder) commit(id string, autoCmd strslice.StrSlice, comment string) error {
  39. if b.disableCommit {
  40. return nil
  41. }
  42. if b.image == "" && !b.noBaseImage {
  43. return fmt.Errorf("Please provide a source image with `from` prior to commit")
  44. }
  45. b.runConfig.Image = b.image
  46. if id == "" {
  47. cmd := b.runConfig.Cmd
  48. b.runConfig.Cmd = strslice.StrSlice(append(getShell(b.runConfig), "#(nop) ", comment))
  49. defer func(cmd strslice.StrSlice) { b.runConfig.Cmd = cmd }(cmd)
  50. hit, err := b.probeCache()
  51. if err != nil {
  52. return err
  53. } else if hit {
  54. return nil
  55. }
  56. id, err = b.create()
  57. if err != nil {
  58. return err
  59. }
  60. }
  61. // Note: Actually copy the struct
  62. autoConfig := *b.runConfig
  63. autoConfig.Cmd = autoCmd
  64. commitCfg := &backend.ContainerCommitConfig{
  65. ContainerCommitConfig: types.ContainerCommitConfig{
  66. Author: b.maintainer,
  67. Pause: true,
  68. Config: &autoConfig,
  69. },
  70. }
  71. // Commit the container
  72. imageID, err := b.docker.Commit(id, commitCfg)
  73. if err != nil {
  74. return err
  75. }
  76. b.image = imageID
  77. return nil
  78. }
  79. type copyInfo struct {
  80. builder.FileInfo
  81. decompress bool
  82. }
  83. func (b *Builder) runContextCommand(args []string, allowRemote bool, allowLocalDecompression bool, cmdName string) error {
  84. if b.context == nil {
  85. return fmt.Errorf("No context given. Impossible to use %s", cmdName)
  86. }
  87. if len(args) < 2 {
  88. return fmt.Errorf("Invalid %s format - at least two arguments required", cmdName)
  89. }
  90. // Work in daemon-specific filepath semantics
  91. dest := filepath.FromSlash(args[len(args)-1]) // last one is always the dest
  92. b.runConfig.Image = b.image
  93. var infos []copyInfo
  94. // Loop through each src file and calculate the info we need to
  95. // do the copy (e.g. hash value if cached). Don't actually do
  96. // the copy until we've looked at all src files
  97. var err error
  98. for _, orig := range args[0 : len(args)-1] {
  99. var fi builder.FileInfo
  100. decompress := allowLocalDecompression
  101. if urlutil.IsURL(orig) {
  102. if !allowRemote {
  103. return fmt.Errorf("Source can't be a URL for %s", cmdName)
  104. }
  105. fi, err = b.download(orig)
  106. if err != nil {
  107. return err
  108. }
  109. defer os.RemoveAll(filepath.Dir(fi.Path()))
  110. decompress = false
  111. infos = append(infos, copyInfo{fi, decompress})
  112. continue
  113. }
  114. // not a URL
  115. subInfos, err := b.calcCopyInfo(cmdName, orig, allowLocalDecompression, true)
  116. if err != nil {
  117. return err
  118. }
  119. infos = append(infos, subInfos...)
  120. }
  121. if len(infos) == 0 {
  122. return fmt.Errorf("No source files were specified")
  123. }
  124. if len(infos) > 1 && !strings.HasSuffix(dest, string(os.PathSeparator)) {
  125. return fmt.Errorf("When using %s with more than one source file, the destination must be a directory and end with a /", cmdName)
  126. }
  127. // For backwards compat, if there's just one info then use it as the
  128. // cache look-up string, otherwise hash 'em all into one
  129. var srcHash string
  130. var origPaths string
  131. if len(infos) == 1 {
  132. fi := infos[0].FileInfo
  133. origPaths = fi.Name()
  134. if hfi, ok := fi.(builder.Hashed); ok {
  135. srcHash = hfi.Hash()
  136. }
  137. } else {
  138. var hashs []string
  139. var origs []string
  140. for _, info := range infos {
  141. fi := info.FileInfo
  142. origs = append(origs, fi.Name())
  143. if hfi, ok := fi.(builder.Hashed); ok {
  144. hashs = append(hashs, hfi.Hash())
  145. }
  146. }
  147. hasher := sha256.New()
  148. hasher.Write([]byte(strings.Join(hashs, ",")))
  149. srcHash = "multi:" + hex.EncodeToString(hasher.Sum(nil))
  150. origPaths = strings.Join(origs, " ")
  151. }
  152. cmd := b.runConfig.Cmd
  153. b.runConfig.Cmd = strslice.StrSlice(append(getShell(b.runConfig), fmt.Sprintf("#(nop) %s %s in %s ", cmdName, srcHash, dest)))
  154. defer func(cmd strslice.StrSlice) { b.runConfig.Cmd = cmd }(cmd)
  155. if hit, err := b.probeCache(); err != nil {
  156. return err
  157. } else if hit {
  158. return nil
  159. }
  160. container, err := b.docker.ContainerCreate(types.ContainerCreateConfig{Config: b.runConfig}, true)
  161. if err != nil {
  162. return err
  163. }
  164. b.tmpContainers[container.ID] = struct{}{}
  165. comment := fmt.Sprintf("%s %s in %s", cmdName, origPaths, dest)
  166. // Twiddle the destination when its a relative path - meaning, make it
  167. // relative to the WORKINGDIR
  168. if dest, err = normaliseDest(cmdName, b.runConfig.WorkingDir, dest); err != nil {
  169. return err
  170. }
  171. for _, info := range infos {
  172. if err := b.docker.CopyOnBuild(container.ID, dest, info.FileInfo, info.decompress); err != nil {
  173. return err
  174. }
  175. }
  176. return b.commit(container.ID, cmd, comment)
  177. }
  178. func (b *Builder) download(srcURL string) (fi builder.FileInfo, err error) {
  179. // get filename from URL
  180. u, err := url.Parse(srcURL)
  181. if err != nil {
  182. return
  183. }
  184. path := filepath.FromSlash(u.Path) // Ensure in platform semantics
  185. if strings.HasSuffix(path, string(os.PathSeparator)) {
  186. path = path[:len(path)-1]
  187. }
  188. parts := strings.Split(path, string(os.PathSeparator))
  189. filename := parts[len(parts)-1]
  190. if filename == "" {
  191. err = fmt.Errorf("cannot determine filename from url: %s", u)
  192. return
  193. }
  194. // Initiate the download
  195. resp, err := httputils.Download(srcURL)
  196. if err != nil {
  197. return
  198. }
  199. // Prepare file in a tmp dir
  200. tmpDir, err := ioutils.TempDir("", "docker-remote")
  201. if err != nil {
  202. return
  203. }
  204. defer func() {
  205. if err != nil {
  206. os.RemoveAll(tmpDir)
  207. }
  208. }()
  209. tmpFileName := filepath.Join(tmpDir, filename)
  210. tmpFile, err := os.OpenFile(tmpFileName, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0600)
  211. if err != nil {
  212. return
  213. }
  214. stdoutFormatter := b.Stdout.(*streamformatter.StdoutFormatter)
  215. progressOutput := stdoutFormatter.StreamFormatter.NewProgressOutput(stdoutFormatter.Writer, true)
  216. progressReader := progress.NewProgressReader(resp.Body, progressOutput, resp.ContentLength, "", "Downloading")
  217. // Download and dump result to tmp file
  218. if _, err = io.Copy(tmpFile, progressReader); err != nil {
  219. tmpFile.Close()
  220. return
  221. }
  222. fmt.Fprintln(b.Stdout)
  223. // ignoring error because the file was already opened successfully
  224. tmpFileSt, err := tmpFile.Stat()
  225. if err != nil {
  226. tmpFile.Close()
  227. return
  228. }
  229. // Set the mtime to the Last-Modified header value if present
  230. // Otherwise just remove atime and mtime
  231. mTime := time.Time{}
  232. lastMod := resp.Header.Get("Last-Modified")
  233. if lastMod != "" {
  234. // If we can't parse it then just let it default to 'zero'
  235. // otherwise use the parsed time value
  236. if parsedMTime, err := http.ParseTime(lastMod); err == nil {
  237. mTime = parsedMTime
  238. }
  239. }
  240. tmpFile.Close()
  241. if err = system.Chtimes(tmpFileName, mTime, mTime); err != nil {
  242. return
  243. }
  244. // Calc the checksum, even if we're using the cache
  245. r, err := archive.Tar(tmpFileName, archive.Uncompressed)
  246. if err != nil {
  247. return
  248. }
  249. tarSum, err := tarsum.NewTarSum(r, true, tarsum.Version1)
  250. if err != nil {
  251. return
  252. }
  253. if _, err = io.Copy(ioutil.Discard, tarSum); err != nil {
  254. return
  255. }
  256. hash := tarSum.Sum(nil)
  257. r.Close()
  258. return &builder.HashedFileInfo{FileInfo: builder.PathFileInfo{FileInfo: tmpFileSt, FilePath: tmpFileName}, FileHash: hash}, nil
  259. }
  260. func (b *Builder) calcCopyInfo(cmdName, origPath string, allowLocalDecompression, allowWildcards bool) ([]copyInfo, error) {
  261. // Work in daemon-specific OS filepath semantics
  262. origPath = filepath.FromSlash(origPath)
  263. if origPath != "" && origPath[0] == os.PathSeparator && len(origPath) > 1 {
  264. origPath = origPath[1:]
  265. }
  266. origPath = strings.TrimPrefix(origPath, "."+string(os.PathSeparator))
  267. // Deal with wildcards
  268. if allowWildcards && containsWildcards(origPath) {
  269. var copyInfos []copyInfo
  270. if err := b.context.Walk("", func(path string, info builder.FileInfo, err error) error {
  271. if err != nil {
  272. return err
  273. }
  274. if info.Name() == "" {
  275. // Why are we doing this check?
  276. return nil
  277. }
  278. if match, _ := filepath.Match(origPath, path); !match {
  279. return nil
  280. }
  281. // Note we set allowWildcards to false in case the name has
  282. // a * in it
  283. subInfos, err := b.calcCopyInfo(cmdName, path, allowLocalDecompression, false)
  284. if err != nil {
  285. return err
  286. }
  287. copyInfos = append(copyInfos, subInfos...)
  288. return nil
  289. }); err != nil {
  290. return nil, err
  291. }
  292. return copyInfos, nil
  293. }
  294. // Must be a dir or a file
  295. statPath, fi, err := b.context.Stat(origPath)
  296. if err != nil {
  297. return nil, err
  298. }
  299. copyInfos := []copyInfo{{FileInfo: fi, decompress: allowLocalDecompression}}
  300. hfi, handleHash := fi.(builder.Hashed)
  301. if !handleHash {
  302. return copyInfos, nil
  303. }
  304. // Deal with the single file case
  305. if !fi.IsDir() {
  306. hfi.SetHash("file:" + hfi.Hash())
  307. return copyInfos, nil
  308. }
  309. // Must be a dir
  310. var subfiles []string
  311. err = b.context.Walk(statPath, func(path string, info builder.FileInfo, err error) error {
  312. if err != nil {
  313. return err
  314. }
  315. // we already checked handleHash above
  316. subfiles = append(subfiles, info.(builder.Hashed).Hash())
  317. return nil
  318. })
  319. if err != nil {
  320. return nil, err
  321. }
  322. sort.Strings(subfiles)
  323. hasher := sha256.New()
  324. hasher.Write([]byte(strings.Join(subfiles, ",")))
  325. hfi.SetHash("dir:" + hex.EncodeToString(hasher.Sum(nil)))
  326. return copyInfos, nil
  327. }
  328. func (b *Builder) processImageFrom(img builder.Image) error {
  329. if img != nil {
  330. b.image = img.ImageID()
  331. if img.RunConfig() != nil {
  332. b.runConfig = img.RunConfig()
  333. }
  334. }
  335. // Check to see if we have a default PATH, note that windows won't
  336. // have one as its set by HCS
  337. if system.DefaultPathEnv != "" {
  338. // Convert the slice of strings that represent the current list
  339. // of env vars into a map so we can see if PATH is already set.
  340. // If its not set then go ahead and give it our default value
  341. configEnv := opts.ConvertKVStringsToMap(b.runConfig.Env)
  342. if _, ok := configEnv["PATH"]; !ok {
  343. b.runConfig.Env = append(b.runConfig.Env,
  344. "PATH="+system.DefaultPathEnv)
  345. }
  346. }
  347. if img == nil {
  348. // Typically this means they used "FROM scratch"
  349. return nil
  350. }
  351. // Process ONBUILD triggers if they exist
  352. if nTriggers := len(b.runConfig.OnBuild); nTriggers != 0 {
  353. word := "trigger"
  354. if nTriggers > 1 {
  355. word = "triggers"
  356. }
  357. fmt.Fprintf(b.Stderr, "# Executing %d build %s...\n", nTriggers, word)
  358. }
  359. // Copy the ONBUILD triggers, and remove them from the config, since the config will be comitted.
  360. onBuildTriggers := b.runConfig.OnBuild
  361. b.runConfig.OnBuild = []string{}
  362. // parse the ONBUILD triggers by invoking the parser
  363. for _, step := range onBuildTriggers {
  364. ast, err := parser.Parse(strings.NewReader(step), &b.directive)
  365. if err != nil {
  366. return err
  367. }
  368. total := len(ast.Children)
  369. for _, n := range ast.Children {
  370. if err := b.checkDispatch(n, true); err != nil {
  371. return err
  372. }
  373. }
  374. for i, n := range ast.Children {
  375. if err := b.dispatch(i, total, n); err != nil {
  376. return err
  377. }
  378. }
  379. }
  380. return nil
  381. }
  382. // probeCache checks if cache match can be found for current build instruction.
  383. // If an image is found, probeCache returns `(true, nil)`.
  384. // If no image is found, it returns `(false, nil)`.
  385. // If there is any error, it returns `(false, err)`.
  386. func (b *Builder) probeCache() (bool, error) {
  387. c := b.imageCache
  388. if c == nil || b.options.NoCache || b.cacheBusted {
  389. return false, nil
  390. }
  391. cache, err := c.GetCache(b.image, b.runConfig)
  392. if err != nil {
  393. return false, err
  394. }
  395. if len(cache) == 0 {
  396. logrus.Debugf("[BUILDER] Cache miss: %s", b.runConfig.Cmd)
  397. b.cacheBusted = true
  398. return false, nil
  399. }
  400. fmt.Fprintf(b.Stdout, " ---> Using cache\n")
  401. logrus.Debugf("[BUILDER] Use cached version: %s", b.runConfig.Cmd)
  402. b.image = string(cache)
  403. return true, nil
  404. }
  405. func (b *Builder) create() (string, error) {
  406. if b.image == "" && !b.noBaseImage {
  407. return "", fmt.Errorf("Please provide a source image with `from` prior to run")
  408. }
  409. b.runConfig.Image = b.image
  410. resources := container.Resources{
  411. CgroupParent: b.options.CgroupParent,
  412. CPUShares: b.options.CPUShares,
  413. CPUPeriod: b.options.CPUPeriod,
  414. CPUQuota: b.options.CPUQuota,
  415. CpusetCpus: b.options.CPUSetCPUs,
  416. CpusetMems: b.options.CPUSetMems,
  417. Memory: b.options.Memory,
  418. MemorySwap: b.options.MemorySwap,
  419. Ulimits: b.options.Ulimits,
  420. }
  421. // TODO: why not embed a hostconfig in builder?
  422. hostConfig := &container.HostConfig{
  423. SecurityOpt: b.options.SecurityOpt,
  424. Isolation: b.options.Isolation,
  425. ShmSize: b.options.ShmSize,
  426. Resources: resources,
  427. }
  428. config := *b.runConfig
  429. // Create the container
  430. c, err := b.docker.ContainerCreate(types.ContainerCreateConfig{
  431. Config: b.runConfig,
  432. HostConfig: hostConfig,
  433. }, true)
  434. if err != nil {
  435. return "", err
  436. }
  437. for _, warning := range c.Warnings {
  438. fmt.Fprintf(b.Stdout, " ---> [Warning] %s\n", warning)
  439. }
  440. b.tmpContainers[c.ID] = struct{}{}
  441. fmt.Fprintf(b.Stdout, " ---> Running in %s\n", stringid.TruncateID(c.ID))
  442. // override the entry point that may have been picked up from the base image
  443. if err := b.docker.ContainerUpdateCmdOnBuild(c.ID, config.Cmd); err != nil {
  444. return "", err
  445. }
  446. return c.ID, nil
  447. }
  448. var errCancelled = errors.New("build cancelled")
  449. func (b *Builder) run(cID string) (err error) {
  450. errCh := make(chan error)
  451. go func() {
  452. errCh <- b.docker.ContainerAttachRaw(cID, nil, b.Stdout, b.Stderr, true)
  453. }()
  454. finished := make(chan struct{})
  455. var once sync.Once
  456. finish := func() { close(finished) }
  457. cancelErrCh := make(chan error, 1)
  458. defer once.Do(finish)
  459. go func() {
  460. select {
  461. case <-b.clientCtx.Done():
  462. logrus.Debugln("Build cancelled, killing and removing container:", cID)
  463. b.docker.ContainerKill(cID, 0)
  464. b.removeContainer(cID)
  465. cancelErrCh <- errCancelled
  466. case <-finished:
  467. cancelErrCh <- nil
  468. }
  469. }()
  470. if err := b.docker.ContainerStart(cID, nil, true, ""); err != nil {
  471. return err
  472. }
  473. // Block on reading output from container, stop on err or chan closed
  474. if err := <-errCh; err != nil {
  475. return err
  476. }
  477. if ret, _ := b.docker.ContainerWait(cID, -1); ret != 0 {
  478. // TODO: change error type, because jsonmessage.JSONError assumes HTTP
  479. return &jsonmessage.JSONError{
  480. Message: fmt.Sprintf("The command '%s' returned a non-zero code: %d", strings.Join(b.runConfig.Cmd, " "), ret),
  481. Code: ret,
  482. }
  483. }
  484. once.Do(finish)
  485. return <-cancelErrCh
  486. }
  487. func (b *Builder) removeContainer(c string) error {
  488. rmConfig := &types.ContainerRmConfig{
  489. ForceRemove: true,
  490. RemoveVolume: true,
  491. }
  492. if err := b.docker.ContainerRm(c, rmConfig); err != nil {
  493. fmt.Fprintf(b.Stdout, "Error removing intermediate container %s: %v\n", stringid.TruncateID(c), err)
  494. return err
  495. }
  496. return nil
  497. }
  498. func (b *Builder) clearTmp() {
  499. for c := range b.tmpContainers {
  500. if err := b.removeContainer(c); err != nil {
  501. return
  502. }
  503. delete(b.tmpContainers, c)
  504. fmt.Fprintf(b.Stdout, "Removing intermediate container %s\n", stringid.TruncateID(c))
  505. }
  506. }
  507. // readDockerfile reads a Dockerfile from the current context.
  508. func (b *Builder) readDockerfile() error {
  509. // If no -f was specified then look for 'Dockerfile'. If we can't find
  510. // that then look for 'dockerfile'. If neither are found then default
  511. // back to 'Dockerfile' and use that in the error message.
  512. if b.options.Dockerfile == "" {
  513. b.options.Dockerfile = builder.DefaultDockerfileName
  514. if _, _, err := b.context.Stat(b.options.Dockerfile); os.IsNotExist(err) {
  515. lowercase := strings.ToLower(b.options.Dockerfile)
  516. if _, _, err := b.context.Stat(lowercase); err == nil {
  517. b.options.Dockerfile = lowercase
  518. }
  519. }
  520. }
  521. err := b.parseDockerfile()
  522. if err != nil {
  523. return err
  524. }
  525. // After the Dockerfile has been parsed, we need to check the .dockerignore
  526. // file for either "Dockerfile" or ".dockerignore", and if either are
  527. // present then erase them from the build context. These files should never
  528. // have been sent from the client but we did send them to make sure that
  529. // we had the Dockerfile to actually parse, and then we also need the
  530. // .dockerignore file to know whether either file should be removed.
  531. // Note that this assumes the Dockerfile has been read into memory and
  532. // is now safe to be removed.
  533. if dockerIgnore, ok := b.context.(builder.DockerIgnoreContext); ok {
  534. dockerIgnore.Process([]string{b.options.Dockerfile})
  535. }
  536. return nil
  537. }
  538. func (b *Builder) parseDockerfile() error {
  539. f, err := b.context.Open(b.options.Dockerfile)
  540. if err != nil {
  541. if os.IsNotExist(err) {
  542. return fmt.Errorf("Cannot locate specified Dockerfile: %s", b.options.Dockerfile)
  543. }
  544. return err
  545. }
  546. defer f.Close()
  547. if f, ok := f.(*os.File); ok {
  548. // ignoring error because Open already succeeded
  549. fi, err := f.Stat()
  550. if err != nil {
  551. return fmt.Errorf("Unexpected error reading Dockerfile: %v", err)
  552. }
  553. if fi.Size() == 0 {
  554. return fmt.Errorf("The Dockerfile (%s) cannot be empty", b.options.Dockerfile)
  555. }
  556. }
  557. b.dockerfile, err = parser.Parse(f, &b.directive)
  558. if err != nil {
  559. return err
  560. }
  561. return nil
  562. }
  563. // determine if build arg is part of built-in args or user
  564. // defined args in Dockerfile at any point in time.
  565. func (b *Builder) isBuildArgAllowed(arg string) bool {
  566. if _, ok := BuiltinAllowedBuildArgs[arg]; ok {
  567. return true
  568. }
  569. if _, ok := b.allowedBuildArgs[arg]; ok {
  570. return true
  571. }
  572. return false
  573. }