copy.go 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567
  1. package dockerfile // import "github.com/docker/docker/builder/dockerfile"
  2. import (
  3. "archive/tar"
  4. "fmt"
  5. "io"
  6. "mime"
  7. "net/http"
  8. "net/url"
  9. "os"
  10. "path/filepath"
  11. "runtime"
  12. "sort"
  13. "strings"
  14. "time"
  15. "github.com/docker/docker/builder"
  16. "github.com/docker/docker/builder/remotecontext"
  17. "github.com/docker/docker/pkg/archive"
  18. "github.com/docker/docker/pkg/containerfs"
  19. "github.com/docker/docker/pkg/idtools"
  20. "github.com/docker/docker/pkg/ioutils"
  21. "github.com/docker/docker/pkg/progress"
  22. "github.com/docker/docker/pkg/streamformatter"
  23. "github.com/docker/docker/pkg/system"
  24. "github.com/docker/docker/pkg/urlutil"
  25. specs "github.com/opencontainers/image-spec/specs-go/v1"
  26. "github.com/pkg/errors"
  27. )
  28. const unnamedFilename = "__unnamed__"
  29. type pathCache interface {
  30. Load(key interface{}) (value interface{}, ok bool)
  31. Store(key, value interface{})
  32. }
  33. // copyInfo is a data object which stores the metadata about each source file in
  34. // a copyInstruction
  35. type copyInfo struct {
  36. root containerfs.ContainerFS
  37. path string
  38. hash string
  39. noDecompress bool
  40. }
  41. func (c copyInfo) fullPath() (string, error) {
  42. return c.root.ResolveScopedPath(c.path, true)
  43. }
  44. func newCopyInfoFromSource(source builder.Source, path string, hash string) copyInfo {
  45. return copyInfo{root: source.Root(), path: path, hash: hash}
  46. }
  47. func newCopyInfos(copyInfos ...copyInfo) []copyInfo {
  48. return copyInfos
  49. }
  50. // copyInstruction is a fully parsed COPY or ADD command that is passed to
  51. // Builder.performCopy to copy files into the image filesystem
  52. type copyInstruction struct {
  53. cmdName string
  54. infos []copyInfo
  55. dest string
  56. chownStr string
  57. allowLocalDecompression bool
  58. }
  59. // copier reads a raw COPY or ADD command, fetches remote sources using a downloader,
  60. // and creates a copyInstruction
  61. type copier struct {
  62. imageSource *imageMount
  63. source builder.Source
  64. pathCache pathCache
  65. download sourceDownloader
  66. platform *specs.Platform
  67. // for cleanup. TODO: having copier.cleanup() is error prone and hard to
  68. // follow. Code calling performCopy should manage the lifecycle of its params.
  69. // Copier should take override source as input, not imageMount.
  70. activeLayer builder.RWLayer
  71. tmpPaths []string
  72. }
  73. func copierFromDispatchRequest(req dispatchRequest, download sourceDownloader, imageSource *imageMount) copier {
  74. return copier{
  75. source: req.source,
  76. pathCache: req.builder.pathCache,
  77. download: download,
  78. imageSource: imageSource,
  79. platform: req.builder.options.Platform,
  80. }
  81. }
  82. func (o *copier) createCopyInstruction(args []string, cmdName string) (copyInstruction, error) {
  83. inst := copyInstruction{cmdName: cmdName}
  84. last := len(args) - 1
  85. // Work in platform-specific filepath semantics
  86. // TODO: This OS switch for paths is NOT correct and should not be supported.
  87. // Maintained for backwards compatibility
  88. pathOS := runtime.GOOS
  89. if o.platform != nil {
  90. pathOS = o.platform.OS
  91. }
  92. inst.dest = fromSlash(args[last], pathOS)
  93. separator := string(separator(pathOS))
  94. infos, err := o.getCopyInfosForSourcePaths(args[0:last], inst.dest)
  95. if err != nil {
  96. return inst, errors.Wrapf(err, "%s failed", cmdName)
  97. }
  98. if len(infos) > 1 && !strings.HasSuffix(inst.dest, separator) {
  99. return inst, errors.Errorf("When using %s with more than one source file, the destination must be a directory and end with a /", cmdName)
  100. }
  101. inst.infos = infos
  102. return inst, nil
  103. }
  104. // getCopyInfosForSourcePaths iterates over the source files and calculate the info
  105. // needed to copy (e.g. hash value if cached)
  106. // The dest is used in case source is URL (and ends with "/")
  107. func (o *copier) getCopyInfosForSourcePaths(sources []string, dest string) ([]copyInfo, error) {
  108. var infos []copyInfo
  109. for _, orig := range sources {
  110. subinfos, err := o.getCopyInfoForSourcePath(orig, dest)
  111. if err != nil {
  112. return nil, err
  113. }
  114. infos = append(infos, subinfos...)
  115. }
  116. if len(infos) == 0 {
  117. return nil, errors.New("no source files were specified")
  118. }
  119. return infos, nil
  120. }
  121. func (o *copier) getCopyInfoForSourcePath(orig, dest string) ([]copyInfo, error) {
  122. if !urlutil.IsURL(orig) {
  123. return o.calcCopyInfo(orig, true)
  124. }
  125. remote, path, err := o.download(orig)
  126. if err != nil {
  127. return nil, err
  128. }
  129. // If path == "" then we are unable to determine filename from src
  130. // We have to make sure dest is available
  131. if path == "" {
  132. if strings.HasSuffix(dest, "/") {
  133. return nil, errors.Errorf("cannot determine filename for source %s", orig)
  134. }
  135. path = unnamedFilename
  136. }
  137. o.tmpPaths = append(o.tmpPaths, remote.Root().Path())
  138. hash, err := remote.Hash(path)
  139. ci := newCopyInfoFromSource(remote, path, hash)
  140. ci.noDecompress = true // data from http shouldn't be extracted even on ADD
  141. return newCopyInfos(ci), err
  142. }
  143. // Cleanup removes any temporary directories created as part of downloading
  144. // remote files.
  145. func (o *copier) Cleanup() {
  146. for _, path := range o.tmpPaths {
  147. os.RemoveAll(path)
  148. }
  149. o.tmpPaths = []string{}
  150. if o.activeLayer != nil {
  151. o.activeLayer.Release()
  152. o.activeLayer = nil
  153. }
  154. }
  155. // TODO: allowWildcards can probably be removed by refactoring this function further.
  156. func (o *copier) calcCopyInfo(origPath string, allowWildcards bool) ([]copyInfo, error) {
  157. imageSource := o.imageSource
  158. // TODO: do this when creating copier. Requires validateCopySourcePath
  159. // (and other below) to be aware of the difference sources. Why is it only
  160. // done on image Source?
  161. if imageSource != nil && o.activeLayer == nil {
  162. // this needs to be protected against repeated calls as wildcard copy
  163. // will call it multiple times for a single COPY
  164. var err error
  165. rwLayer, err := imageSource.NewRWLayer()
  166. if err != nil {
  167. return nil, err
  168. }
  169. o.activeLayer = rwLayer
  170. o.source, err = remotecontext.NewLazySource(rwLayer.Root())
  171. if err != nil {
  172. return nil, errors.Wrapf(err, "failed to create context for copy from %s", rwLayer.Root().Path())
  173. }
  174. }
  175. if o.source == nil {
  176. return nil, errors.Errorf("missing build context")
  177. }
  178. root := o.source.Root()
  179. if err := validateCopySourcePath(imageSource, origPath, root.OS()); err != nil {
  180. return nil, err
  181. }
  182. // Work in source OS specific filepath semantics
  183. // For LCOW, this is NOT the daemon OS.
  184. origPath = root.FromSlash(origPath)
  185. origPath = strings.TrimPrefix(origPath, string(root.Separator()))
  186. origPath = strings.TrimPrefix(origPath, "."+string(root.Separator()))
  187. // Deal with wildcards
  188. if allowWildcards && containsWildcards(origPath, root.OS()) {
  189. return o.copyWithWildcards(origPath)
  190. }
  191. if imageSource != nil && imageSource.ImageID() != "" {
  192. // return a cached copy if one exists
  193. if h, ok := o.pathCache.Load(imageSource.ImageID() + origPath); ok {
  194. return newCopyInfos(newCopyInfoFromSource(o.source, origPath, h.(string))), nil
  195. }
  196. }
  197. // Deal with the single file case
  198. copyInfo, err := copyInfoForFile(o.source, origPath)
  199. switch {
  200. case err != nil:
  201. return nil, err
  202. case copyInfo.hash != "":
  203. o.storeInPathCache(imageSource, origPath, copyInfo.hash)
  204. return newCopyInfos(copyInfo), err
  205. }
  206. // TODO: remove, handle dirs in Hash()
  207. subfiles, err := walkSource(o.source, origPath)
  208. if err != nil {
  209. return nil, err
  210. }
  211. hash := hashStringSlice("dir", subfiles)
  212. o.storeInPathCache(imageSource, origPath, hash)
  213. return newCopyInfos(newCopyInfoFromSource(o.source, origPath, hash)), nil
  214. }
  215. func containsWildcards(name, platform string) bool {
  216. isWindows := platform == "windows"
  217. for i := 0; i < len(name); i++ {
  218. ch := name[i]
  219. if ch == '\\' && !isWindows {
  220. i++
  221. } else if ch == '*' || ch == '?' || ch == '[' {
  222. return true
  223. }
  224. }
  225. return false
  226. }
  227. func (o *copier) storeInPathCache(im *imageMount, path string, hash string) {
  228. if im != nil {
  229. o.pathCache.Store(im.ImageID()+path, hash)
  230. }
  231. }
  232. func (o *copier) copyWithWildcards(origPath string) ([]copyInfo, error) {
  233. root := o.source.Root()
  234. var copyInfos []copyInfo
  235. if err := root.Walk(root.Path(), func(path string, info os.FileInfo, err error) error {
  236. if err != nil {
  237. return err
  238. }
  239. rel, err := remotecontext.Rel(root, path)
  240. if err != nil {
  241. return err
  242. }
  243. if rel == "." {
  244. return nil
  245. }
  246. if match, _ := root.Match(origPath, rel); !match {
  247. return nil
  248. }
  249. // Note we set allowWildcards to false in case the name has
  250. // a * in it
  251. subInfos, err := o.calcCopyInfo(rel, false)
  252. if err != nil {
  253. return err
  254. }
  255. copyInfos = append(copyInfos, subInfos...)
  256. return nil
  257. }); err != nil {
  258. return nil, err
  259. }
  260. return copyInfos, nil
  261. }
  262. func copyInfoForFile(source builder.Source, path string) (copyInfo, error) {
  263. fi, err := remotecontext.StatAt(source, path)
  264. if err != nil {
  265. return copyInfo{}, err
  266. }
  267. if fi.IsDir() {
  268. return copyInfo{}, nil
  269. }
  270. hash, err := source.Hash(path)
  271. if err != nil {
  272. return copyInfo{}, err
  273. }
  274. return newCopyInfoFromSource(source, path, "file:"+hash), nil
  275. }
  276. // TODO: dedupe with copyWithWildcards()
  277. func walkSource(source builder.Source, origPath string) ([]string, error) {
  278. fp, err := remotecontext.FullPath(source, origPath)
  279. if err != nil {
  280. return nil, err
  281. }
  282. // Must be a dir
  283. var subfiles []string
  284. err = source.Root().Walk(fp, func(path string, info os.FileInfo, err error) error {
  285. if err != nil {
  286. return err
  287. }
  288. rel, err := remotecontext.Rel(source.Root(), path)
  289. if err != nil {
  290. return err
  291. }
  292. if rel == "." {
  293. return nil
  294. }
  295. hash, err := source.Hash(rel)
  296. if err != nil {
  297. return nil
  298. }
  299. // we already checked handleHash above
  300. subfiles = append(subfiles, hash)
  301. return nil
  302. })
  303. if err != nil {
  304. return nil, err
  305. }
  306. sort.Strings(subfiles)
  307. return subfiles, nil
  308. }
  309. type sourceDownloader func(string) (builder.Source, string, error)
  310. func newRemoteSourceDownloader(output, stdout io.Writer) sourceDownloader {
  311. return func(url string) (builder.Source, string, error) {
  312. return downloadSource(output, stdout, url)
  313. }
  314. }
  315. func errOnSourceDownload(_ string) (builder.Source, string, error) {
  316. return nil, "", errors.New("source can't be a URL for COPY")
  317. }
  318. func getFilenameForDownload(path string, resp *http.Response) string {
  319. // Guess filename based on source
  320. if path != "" && !strings.HasSuffix(path, "/") {
  321. if filename := filepath.Base(filepath.FromSlash(path)); filename != "" {
  322. return filename
  323. }
  324. }
  325. // Guess filename based on Content-Disposition
  326. if contentDisposition := resp.Header.Get("Content-Disposition"); contentDisposition != "" {
  327. if _, params, err := mime.ParseMediaType(contentDisposition); err == nil {
  328. if params["filename"] != "" && !strings.HasSuffix(params["filename"], "/") {
  329. if filename := filepath.Base(filepath.FromSlash(params["filename"])); filename != "" {
  330. return filename
  331. }
  332. }
  333. }
  334. }
  335. return ""
  336. }
  337. func downloadSource(output io.Writer, stdout io.Writer, srcURL string) (remote builder.Source, p string, err error) {
  338. u, err := url.Parse(srcURL)
  339. if err != nil {
  340. return
  341. }
  342. resp, err := remotecontext.GetWithStatusError(srcURL)
  343. if err != nil {
  344. return
  345. }
  346. filename := getFilenameForDownload(u.Path, resp)
  347. // Prepare file in a tmp dir
  348. tmpDir, err := ioutils.TempDir("", "docker-remote")
  349. if err != nil {
  350. return
  351. }
  352. defer func() {
  353. if err != nil {
  354. os.RemoveAll(tmpDir)
  355. }
  356. }()
  357. // If filename is empty, the returned filename will be "" but
  358. // the tmp filename will be created as "__unnamed__"
  359. tmpFileName := filename
  360. if filename == "" {
  361. tmpFileName = unnamedFilename
  362. }
  363. tmpFileName = filepath.Join(tmpDir, tmpFileName)
  364. tmpFile, err := os.OpenFile(tmpFileName, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0600)
  365. if err != nil {
  366. return
  367. }
  368. progressOutput := streamformatter.NewJSONProgressOutput(output, true)
  369. progressReader := progress.NewProgressReader(resp.Body, progressOutput, resp.ContentLength, "", "Downloading")
  370. // Download and dump result to tmp file
  371. // TODO: add filehash directly
  372. if _, err = io.Copy(tmpFile, progressReader); err != nil {
  373. tmpFile.Close()
  374. return
  375. }
  376. // TODO: how important is this random blank line to the output?
  377. fmt.Fprintln(stdout)
  378. // Set the mtime to the Last-Modified header value if present
  379. // Otherwise just remove atime and mtime
  380. mTime := time.Time{}
  381. lastMod := resp.Header.Get("Last-Modified")
  382. if lastMod != "" {
  383. // If we can't parse it then just let it default to 'zero'
  384. // otherwise use the parsed time value
  385. if parsedMTime, err := http.ParseTime(lastMod); err == nil {
  386. mTime = parsedMTime
  387. }
  388. }
  389. tmpFile.Close()
  390. if err = system.Chtimes(tmpFileName, mTime, mTime); err != nil {
  391. return
  392. }
  393. lc, err := remotecontext.NewLazySource(containerfs.NewLocalContainerFS(tmpDir))
  394. return lc, filename, err
  395. }
  396. type copyFileOptions struct {
  397. decompress bool
  398. chownPair idtools.IDPair
  399. archiver Archiver
  400. }
  401. type copyEndpoint struct {
  402. driver containerfs.Driver
  403. path string
  404. }
  405. func performCopyForInfo(dest copyInfo, source copyInfo, options copyFileOptions) error {
  406. srcPath, err := source.fullPath()
  407. if err != nil {
  408. return err
  409. }
  410. destPath, err := dest.fullPath()
  411. if err != nil {
  412. return err
  413. }
  414. archiver := options.archiver
  415. srcEndpoint := &copyEndpoint{driver: source.root, path: srcPath}
  416. destEndpoint := &copyEndpoint{driver: dest.root, path: destPath}
  417. src, err := source.root.Stat(srcPath)
  418. if err != nil {
  419. return errors.Wrapf(err, "source path not found")
  420. }
  421. if src.IsDir() {
  422. return copyDirectory(archiver, srcEndpoint, destEndpoint, options.chownPair)
  423. }
  424. if options.decompress && isArchivePath(source.root, srcPath) && !source.noDecompress {
  425. return archiver.UntarPath(srcPath, destPath)
  426. }
  427. destExistsAsDir, err := isExistingDirectory(destEndpoint)
  428. if err != nil {
  429. return err
  430. }
  431. // dest.path must be used because destPath has already been cleaned of any
  432. // trailing slash
  433. if endsInSlash(dest.root, dest.path) || destExistsAsDir {
  434. // source.path must be used to get the correct filename when the source
  435. // is a symlink
  436. destPath = dest.root.Join(destPath, source.root.Base(source.path))
  437. destEndpoint = &copyEndpoint{driver: dest.root, path: destPath}
  438. }
  439. return copyFile(archiver, srcEndpoint, destEndpoint, options.chownPair)
  440. }
  441. func isArchivePath(driver containerfs.ContainerFS, path string) bool {
  442. file, err := driver.Open(path)
  443. if err != nil {
  444. return false
  445. }
  446. defer file.Close()
  447. rdr, err := archive.DecompressStream(file)
  448. if err != nil {
  449. return false
  450. }
  451. r := tar.NewReader(rdr)
  452. _, err = r.Next()
  453. return err == nil
  454. }
  455. func copyDirectory(archiver Archiver, source, dest *copyEndpoint, chownPair idtools.IDPair) error {
  456. destExists, err := isExistingDirectory(dest)
  457. if err != nil {
  458. return errors.Wrapf(err, "failed to query destination path")
  459. }
  460. if err := archiver.CopyWithTar(source.path, dest.path); err != nil {
  461. return errors.Wrapf(err, "failed to copy directory")
  462. }
  463. // TODO: @gupta-ak. Investigate how LCOW permission mappings will work.
  464. return fixPermissions(source.path, dest.path, chownPair, !destExists)
  465. }
  466. func copyFile(archiver Archiver, source, dest *copyEndpoint, chownPair idtools.IDPair) error {
  467. if runtime.GOOS == "windows" && dest.driver.OS() == "linux" {
  468. // LCOW
  469. if err := dest.driver.MkdirAll(dest.driver.Dir(dest.path), 0755); err != nil {
  470. return errors.Wrapf(err, "failed to create new directory")
  471. }
  472. } else {
  473. if err := idtools.MkdirAllAndChownNew(filepath.Dir(dest.path), 0755, chownPair); err != nil {
  474. // Normal containers
  475. return errors.Wrapf(err, "failed to create new directory")
  476. }
  477. }
  478. if err := archiver.CopyFileWithTar(source.path, dest.path); err != nil {
  479. return errors.Wrapf(err, "failed to copy file")
  480. }
  481. // TODO: @gupta-ak. Investigate how LCOW permission mappings will work.
  482. return fixPermissions(source.path, dest.path, chownPair, false)
  483. }
  484. func endsInSlash(driver containerfs.Driver, path string) bool {
  485. return strings.HasSuffix(path, string(driver.Separator()))
  486. }
  487. // isExistingDirectory returns true if the path exists and is a directory
  488. func isExistingDirectory(point *copyEndpoint) (bool, error) {
  489. destStat, err := point.driver.Stat(point.path)
  490. switch {
  491. case os.IsNotExist(err):
  492. return false, nil
  493. case err != nil:
  494. return false, err
  495. }
  496. return destStat.IsDir(), nil
  497. }