archive.go 37 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271
  1. package archive
  2. import (
  3. "archive/tar"
  4. "bufio"
  5. "bytes"
  6. "compress/bzip2"
  7. "compress/gzip"
  8. "errors"
  9. "fmt"
  10. "io"
  11. "io/ioutil"
  12. "os"
  13. "os/exec"
  14. "path/filepath"
  15. "runtime"
  16. "strings"
  17. "syscall"
  18. "github.com/Sirupsen/logrus"
  19. "github.com/docker/docker/pkg/fileutils"
  20. "github.com/docker/docker/pkg/idtools"
  21. "github.com/docker/docker/pkg/ioutils"
  22. "github.com/docker/docker/pkg/pools"
  23. "github.com/docker/docker/pkg/promise"
  24. "github.com/docker/docker/pkg/system"
  25. )
  26. type (
  27. // Compression is the state represents if compressed or not.
  28. Compression int
  29. // WhiteoutFormat is the format of whiteouts unpacked
  30. WhiteoutFormat int
  31. // TarChownOptions wraps the chown options UID and GID.
  32. TarChownOptions struct {
  33. UID, GID int
  34. }
  35. // TarOptions wraps the tar options.
  36. TarOptions struct {
  37. IncludeFiles []string
  38. ExcludePatterns []string
  39. Compression Compression
  40. NoLchown bool
  41. UIDMaps []idtools.IDMap
  42. GIDMaps []idtools.IDMap
  43. ChownOpts *TarChownOptions
  44. IncludeSourceDir bool
  45. // WhiteoutFormat is the expected on disk format for whiteout files.
  46. // This format will be converted to the standard format on pack
  47. // and from the standard format on unpack.
  48. WhiteoutFormat WhiteoutFormat
  49. // When unpacking, specifies whether overwriting a directory with a
  50. // non-directory is allowed and vice versa.
  51. NoOverwriteDirNonDir bool
  52. // For each include when creating an archive, the included name will be
  53. // replaced with the matching name from this map.
  54. RebaseNames map[string]string
  55. InUserNS bool
  56. }
  57. // Archiver allows the reuse of most utility functions of this package
  58. // with a pluggable Untar function. Also, to facilitate the passing of
  59. // specific id mappings for untar, an archiver can be created with maps
  60. // which will then be passed to Untar operations
  61. Archiver struct {
  62. Untar func(io.Reader, string, *TarOptions) error
  63. UIDMaps []idtools.IDMap
  64. GIDMaps []idtools.IDMap
  65. }
  66. // breakoutError is used to differentiate errors related to breaking out
  67. // When testing archive breakout in the unit tests, this error is expected
  68. // in order for the test to pass.
  69. breakoutError error
  70. )
  71. var (
  72. // ErrNotImplemented is the error message of function not implemented.
  73. ErrNotImplemented = errors.New("Function not implemented")
  74. defaultArchiver = &Archiver{Untar: Untar, UIDMaps: nil, GIDMaps: nil}
  75. )
  76. const (
  77. // HeaderSize is the size in bytes of a tar header
  78. HeaderSize = 512
  79. )
  80. const (
  81. // Uncompressed represents the uncompressed.
  82. Uncompressed Compression = iota
  83. // Bzip2 is bzip2 compression algorithm.
  84. Bzip2
  85. // Gzip is gzip compression algorithm.
  86. Gzip
  87. // Xz is xz compression algorithm.
  88. Xz
  89. )
  90. const (
  91. // AUFSWhiteoutFormat is the default format for whiteouts
  92. AUFSWhiteoutFormat WhiteoutFormat = iota
  93. // OverlayWhiteoutFormat formats whiteout according to the overlay
  94. // standard.
  95. OverlayWhiteoutFormat
  96. )
  97. // IsArchive checks for the magic bytes of a tar or any supported compression
  98. // algorithm.
  99. func IsArchive(header []byte) bool {
  100. compression := DetectCompression(header)
  101. if compression != Uncompressed {
  102. return true
  103. }
  104. r := tar.NewReader(bytes.NewBuffer(header))
  105. _, err := r.Next()
  106. return err == nil
  107. }
  108. // IsArchivePath checks if the (possibly compressed) file at the given path
  109. // starts with a tar file header.
  110. func IsArchivePath(path string) bool {
  111. file, err := os.Open(path)
  112. if err != nil {
  113. return false
  114. }
  115. defer file.Close()
  116. rdr, err := DecompressStream(file)
  117. if err != nil {
  118. return false
  119. }
  120. r := tar.NewReader(rdr)
  121. _, err = r.Next()
  122. return err == nil
  123. }
  124. // DetectCompression detects the compression algorithm of the source.
  125. func DetectCompression(source []byte) Compression {
  126. for compression, m := range map[Compression][]byte{
  127. Bzip2: {0x42, 0x5A, 0x68},
  128. Gzip: {0x1F, 0x8B, 0x08},
  129. Xz: {0xFD, 0x37, 0x7A, 0x58, 0x5A, 0x00},
  130. } {
  131. if len(source) < len(m) {
  132. logrus.Debug("Len too short")
  133. continue
  134. }
  135. if bytes.Equal(m, source[:len(m)]) {
  136. return compression
  137. }
  138. }
  139. return Uncompressed
  140. }
  141. func xzDecompress(archive io.Reader) (io.ReadCloser, <-chan struct{}, error) {
  142. args := []string{"xz", "-d", "-c", "-q"}
  143. return cmdStream(exec.Command(args[0], args[1:]...), archive)
  144. }
  145. // DecompressStream decompresses the archive and returns a ReaderCloser with the decompressed archive.
  146. func DecompressStream(archive io.Reader) (io.ReadCloser, error) {
  147. p := pools.BufioReader32KPool
  148. buf := p.Get(archive)
  149. bs, err := buf.Peek(10)
  150. if err != nil && err != io.EOF {
  151. // Note: we'll ignore any io.EOF error because there are some odd
  152. // cases where the layer.tar file will be empty (zero bytes) and
  153. // that results in an io.EOF from the Peek() call. So, in those
  154. // cases we'll just treat it as a non-compressed stream and
  155. // that means just create an empty layer.
  156. // See Issue 18170
  157. return nil, err
  158. }
  159. compression := DetectCompression(bs)
  160. switch compression {
  161. case Uncompressed:
  162. readBufWrapper := p.NewReadCloserWrapper(buf, buf)
  163. return readBufWrapper, nil
  164. case Gzip:
  165. gzReader, err := gzip.NewReader(buf)
  166. if err != nil {
  167. return nil, err
  168. }
  169. readBufWrapper := p.NewReadCloserWrapper(buf, gzReader)
  170. return readBufWrapper, nil
  171. case Bzip2:
  172. bz2Reader := bzip2.NewReader(buf)
  173. readBufWrapper := p.NewReadCloserWrapper(buf, bz2Reader)
  174. return readBufWrapper, nil
  175. case Xz:
  176. xzReader, chdone, err := xzDecompress(buf)
  177. if err != nil {
  178. return nil, err
  179. }
  180. readBufWrapper := p.NewReadCloserWrapper(buf, xzReader)
  181. return ioutils.NewReadCloserWrapper(readBufWrapper, func() error {
  182. <-chdone
  183. return readBufWrapper.Close()
  184. }), nil
  185. default:
  186. return nil, fmt.Errorf("Unsupported compression format %s", (&compression).Extension())
  187. }
  188. }
  189. // CompressStream compresseses the dest with specified compression algorithm.
  190. func CompressStream(dest io.Writer, compression Compression) (io.WriteCloser, error) {
  191. p := pools.BufioWriter32KPool
  192. buf := p.Get(dest)
  193. switch compression {
  194. case Uncompressed:
  195. writeBufWrapper := p.NewWriteCloserWrapper(buf, buf)
  196. return writeBufWrapper, nil
  197. case Gzip:
  198. gzWriter := gzip.NewWriter(dest)
  199. writeBufWrapper := p.NewWriteCloserWrapper(buf, gzWriter)
  200. return writeBufWrapper, nil
  201. case Bzip2, Xz:
  202. // archive/bzip2 does not support writing, and there is no xz support at all
  203. // However, this is not a problem as docker only currently generates gzipped tars
  204. return nil, fmt.Errorf("Unsupported compression format %s", (&compression).Extension())
  205. default:
  206. return nil, fmt.Errorf("Unsupported compression format %s", (&compression).Extension())
  207. }
  208. }
  209. // TarModifierFunc is a function that can be passed to ReplaceFileTarWrapper to
  210. // modify the contents or header of an entry in the archive. If the file already
  211. // exists in the archive the TarModifierFunc will be called with the Header and
  212. // a reader which will return the files content. If the file does not exist both
  213. // header and content will be nil.
  214. type TarModifierFunc func(path string, header *tar.Header, content io.Reader) (*tar.Header, []byte, error)
  215. // ReplaceFileTarWrapper converts inputTarStream to a new tar stream. Files in the
  216. // tar stream are modified if they match any of the keys in mods.
  217. func ReplaceFileTarWrapper(inputTarStream io.ReadCloser, mods map[string]TarModifierFunc) io.ReadCloser {
  218. pipeReader, pipeWriter := io.Pipe()
  219. go func() {
  220. tarReader := tar.NewReader(inputTarStream)
  221. tarWriter := tar.NewWriter(pipeWriter)
  222. defer inputTarStream.Close()
  223. defer tarWriter.Close()
  224. modify := func(name string, original *tar.Header, modifier TarModifierFunc, tarReader io.Reader) error {
  225. header, data, err := modifier(name, original, tarReader)
  226. switch {
  227. case err != nil:
  228. return err
  229. case header == nil:
  230. return nil
  231. }
  232. header.Name = name
  233. header.Size = int64(len(data))
  234. if err := tarWriter.WriteHeader(header); err != nil {
  235. return err
  236. }
  237. if len(data) != 0 {
  238. if _, err := tarWriter.Write(data); err != nil {
  239. return err
  240. }
  241. }
  242. return nil
  243. }
  244. var err error
  245. var originalHeader *tar.Header
  246. for {
  247. originalHeader, err = tarReader.Next()
  248. if err == io.EOF {
  249. break
  250. }
  251. if err != nil {
  252. pipeWriter.CloseWithError(err)
  253. return
  254. }
  255. modifier, ok := mods[originalHeader.Name]
  256. if !ok {
  257. // No modifiers for this file, copy the header and data
  258. if err := tarWriter.WriteHeader(originalHeader); err != nil {
  259. pipeWriter.CloseWithError(err)
  260. return
  261. }
  262. if _, err := pools.Copy(tarWriter, tarReader); err != nil {
  263. pipeWriter.CloseWithError(err)
  264. return
  265. }
  266. continue
  267. }
  268. delete(mods, originalHeader.Name)
  269. if err := modify(originalHeader.Name, originalHeader, modifier, tarReader); err != nil {
  270. pipeWriter.CloseWithError(err)
  271. return
  272. }
  273. }
  274. // Apply the modifiers that haven't matched any files in the archive
  275. for name, modifier := range mods {
  276. if err := modify(name, nil, modifier, nil); err != nil {
  277. pipeWriter.CloseWithError(err)
  278. return
  279. }
  280. }
  281. pipeWriter.Close()
  282. }()
  283. return pipeReader
  284. }
  285. // Extension returns the extension of a file that uses the specified compression algorithm.
  286. func (compression *Compression) Extension() string {
  287. switch *compression {
  288. case Uncompressed:
  289. return "tar"
  290. case Bzip2:
  291. return "tar.bz2"
  292. case Gzip:
  293. return "tar.gz"
  294. case Xz:
  295. return "tar.xz"
  296. }
  297. return ""
  298. }
  299. // FileInfoHeader creates a populated Header from fi.
  300. // Compared to archive pkg this function fills in more information.
  301. func FileInfoHeader(path, name string, fi os.FileInfo) (*tar.Header, error) {
  302. var link string
  303. if fi.Mode()&os.ModeSymlink != 0 {
  304. var err error
  305. link, err = os.Readlink(path)
  306. if err != nil {
  307. return nil, err
  308. }
  309. }
  310. hdr, err := tar.FileInfoHeader(fi, link)
  311. if err != nil {
  312. return nil, err
  313. }
  314. hdr.Mode = int64(chmodTarEntry(os.FileMode(hdr.Mode)))
  315. name, err = canonicalTarName(name, fi.IsDir())
  316. if err != nil {
  317. return nil, fmt.Errorf("tar: cannot canonicalize path: %v", err)
  318. }
  319. hdr.Name = name
  320. if err := setHeaderForSpecialDevice(hdr, name, fi.Sys()); err != nil {
  321. return nil, err
  322. }
  323. capability, _ := system.Lgetxattr(path, "security.capability")
  324. if capability != nil {
  325. hdr.Xattrs = make(map[string]string)
  326. hdr.Xattrs["security.capability"] = string(capability)
  327. }
  328. return hdr, nil
  329. }
  330. type tarWhiteoutConverter interface {
  331. ConvertWrite(*tar.Header, string, os.FileInfo) (*tar.Header, error)
  332. ConvertRead(*tar.Header, string) (bool, error)
  333. }
  334. type tarAppender struct {
  335. TarWriter *tar.Writer
  336. Buffer *bufio.Writer
  337. // for hardlink mapping
  338. SeenFiles map[uint64]string
  339. UIDMaps []idtools.IDMap
  340. GIDMaps []idtools.IDMap
  341. // For packing and unpacking whiteout files in the
  342. // non standard format. The whiteout files defined
  343. // by the AUFS standard are used as the tar whiteout
  344. // standard.
  345. WhiteoutConverter tarWhiteoutConverter
  346. }
  347. // canonicalTarName provides a platform-independent and consistent posix-style
  348. //path for files and directories to be archived regardless of the platform.
  349. func canonicalTarName(name string, isDir bool) (string, error) {
  350. name, err := CanonicalTarNameForPath(name)
  351. if err != nil {
  352. return "", err
  353. }
  354. // suffix with '/' for directories
  355. if isDir && !strings.HasSuffix(name, "/") {
  356. name += "/"
  357. }
  358. return name, nil
  359. }
  360. // addTarFile adds to the tar archive a file from `path` as `name`
  361. func (ta *tarAppender) addTarFile(path, name string) error {
  362. fi, err := os.Lstat(path)
  363. if err != nil {
  364. return err
  365. }
  366. hdr, err := FileInfoHeader(path, name, fi)
  367. if err != nil {
  368. return err
  369. }
  370. // if it's not a directory and has more than 1 link,
  371. // it's hard linked, so set the type flag accordingly
  372. if !fi.IsDir() && hasHardlinks(fi) {
  373. inode, err := getInodeFromStat(fi.Sys())
  374. if err != nil {
  375. return err
  376. }
  377. // a link should have a name that it links too
  378. // and that linked name should be first in the tar archive
  379. if oldpath, ok := ta.SeenFiles[inode]; ok {
  380. hdr.Typeflag = tar.TypeLink
  381. hdr.Linkname = oldpath
  382. hdr.Size = 0 // This Must be here for the writer math to add up!
  383. } else {
  384. ta.SeenFiles[inode] = name
  385. }
  386. }
  387. //handle re-mapping container ID mappings back to host ID mappings before
  388. //writing tar headers/files. We skip whiteout files because they were written
  389. //by the kernel and already have proper ownership relative to the host
  390. if !strings.HasPrefix(filepath.Base(hdr.Name), WhiteoutPrefix) && (ta.UIDMaps != nil || ta.GIDMaps != nil) {
  391. uid, gid, err := getFileUIDGID(fi.Sys())
  392. if err != nil {
  393. return err
  394. }
  395. xUID, err := idtools.ToContainer(uid, ta.UIDMaps)
  396. if err != nil {
  397. return err
  398. }
  399. xGID, err := idtools.ToContainer(gid, ta.GIDMaps)
  400. if err != nil {
  401. return err
  402. }
  403. hdr.Uid = xUID
  404. hdr.Gid = xGID
  405. }
  406. if ta.WhiteoutConverter != nil {
  407. wo, err := ta.WhiteoutConverter.ConvertWrite(hdr, path, fi)
  408. if err != nil {
  409. return err
  410. }
  411. // If a new whiteout file exists, write original hdr, then
  412. // replace hdr with wo to be written after. Whiteouts should
  413. // always be written after the original. Note the original
  414. // hdr may have been updated to be a whiteout with returning
  415. // a whiteout header
  416. if wo != nil {
  417. if err := ta.TarWriter.WriteHeader(hdr); err != nil {
  418. return err
  419. }
  420. if hdr.Typeflag == tar.TypeReg && hdr.Size > 0 {
  421. return fmt.Errorf("tar: cannot use whiteout for non-empty file")
  422. }
  423. hdr = wo
  424. }
  425. }
  426. if err := ta.TarWriter.WriteHeader(hdr); err != nil {
  427. return err
  428. }
  429. if hdr.Typeflag == tar.TypeReg && hdr.Size > 0 {
  430. // We use system.OpenSequential to ensure we use sequential file
  431. // access on Windows to avoid depleting the standby list.
  432. // On Linux, this equates to a regular os.Open.
  433. file, err := system.OpenSequential(path)
  434. if err != nil {
  435. return err
  436. }
  437. ta.Buffer.Reset(ta.TarWriter)
  438. defer ta.Buffer.Reset(nil)
  439. _, err = io.Copy(ta.Buffer, file)
  440. file.Close()
  441. if err != nil {
  442. return err
  443. }
  444. err = ta.Buffer.Flush()
  445. if err != nil {
  446. return err
  447. }
  448. }
  449. return nil
  450. }
  451. func createTarFile(path, extractDir string, hdr *tar.Header, reader io.Reader, Lchown bool, chownOpts *TarChownOptions, inUserns bool) error {
  452. // hdr.Mode is in linux format, which we can use for sycalls,
  453. // but for os.Foo() calls we need the mode converted to os.FileMode,
  454. // so use hdrInfo.Mode() (they differ for e.g. setuid bits)
  455. hdrInfo := hdr.FileInfo()
  456. switch hdr.Typeflag {
  457. case tar.TypeDir:
  458. // Create directory unless it exists as a directory already.
  459. // In that case we just want to merge the two
  460. if fi, err := os.Lstat(path); !(err == nil && fi.IsDir()) {
  461. if err := os.Mkdir(path, hdrInfo.Mode()); err != nil {
  462. return err
  463. }
  464. }
  465. case tar.TypeReg, tar.TypeRegA:
  466. // Source is regular file. We use system.OpenFileSequential to use sequential
  467. // file access to avoid depleting the standby list on Windows.
  468. // On Linux, this equates to a regular os.OpenFile
  469. file, err := system.OpenFileSequential(path, os.O_CREATE|os.O_WRONLY, hdrInfo.Mode())
  470. if err != nil {
  471. return err
  472. }
  473. if _, err := io.Copy(file, reader); err != nil {
  474. file.Close()
  475. return err
  476. }
  477. file.Close()
  478. case tar.TypeBlock, tar.TypeChar:
  479. if inUserns { // cannot create devices in a userns
  480. return nil
  481. }
  482. // Handle this is an OS-specific way
  483. if err := handleTarTypeBlockCharFifo(hdr, path); err != nil {
  484. return err
  485. }
  486. case tar.TypeFifo:
  487. // Handle this is an OS-specific way
  488. if err := handleTarTypeBlockCharFifo(hdr, path); err != nil {
  489. return err
  490. }
  491. case tar.TypeLink:
  492. targetPath := filepath.Join(extractDir, hdr.Linkname)
  493. // check for hardlink breakout
  494. if !strings.HasPrefix(targetPath, extractDir) {
  495. return breakoutError(fmt.Errorf("invalid hardlink %q -> %q", targetPath, hdr.Linkname))
  496. }
  497. if err := os.Link(targetPath, path); err != nil {
  498. return err
  499. }
  500. case tar.TypeSymlink:
  501. // path -> hdr.Linkname = targetPath
  502. // e.g. /extractDir/path/to/symlink -> ../2/file = /extractDir/path/2/file
  503. targetPath := filepath.Join(filepath.Dir(path), hdr.Linkname)
  504. // the reason we don't need to check symlinks in the path (with FollowSymlinkInScope) is because
  505. // that symlink would first have to be created, which would be caught earlier, at this very check:
  506. if !strings.HasPrefix(targetPath, extractDir) {
  507. return breakoutError(fmt.Errorf("invalid symlink %q -> %q", path, hdr.Linkname))
  508. }
  509. if err := os.Symlink(hdr.Linkname, path); err != nil {
  510. return err
  511. }
  512. case tar.TypeXGlobalHeader:
  513. logrus.Debug("PAX Global Extended Headers found and ignored")
  514. return nil
  515. default:
  516. return fmt.Errorf("Unhandled tar header type %d\n", hdr.Typeflag)
  517. }
  518. // Lchown is not supported on Windows.
  519. if Lchown && runtime.GOOS != "windows" {
  520. if chownOpts == nil {
  521. chownOpts = &TarChownOptions{UID: hdr.Uid, GID: hdr.Gid}
  522. }
  523. if err := os.Lchown(path, chownOpts.UID, chownOpts.GID); err != nil {
  524. return err
  525. }
  526. }
  527. var errors []string
  528. for key, value := range hdr.Xattrs {
  529. if err := system.Lsetxattr(path, key, []byte(value), 0); err != nil {
  530. if err == syscall.ENOTSUP {
  531. // We ignore errors here because not all graphdrivers support
  532. // xattrs *cough* old versions of AUFS *cough*. However only
  533. // ENOTSUP should be emitted in that case, otherwise we still
  534. // bail.
  535. errors = append(errors, err.Error())
  536. continue
  537. }
  538. return err
  539. }
  540. }
  541. if len(errors) > 0 {
  542. logrus.WithFields(logrus.Fields{
  543. "errors": errors,
  544. }).Warn("ignored xattrs in archive: underlying filesystem doesn't support them")
  545. }
  546. // There is no LChmod, so ignore mode for symlink. Also, this
  547. // must happen after chown, as that can modify the file mode
  548. if err := handleLChmod(hdr, path, hdrInfo); err != nil {
  549. return err
  550. }
  551. aTime := hdr.AccessTime
  552. if aTime.Before(hdr.ModTime) {
  553. // Last access time should never be before last modified time.
  554. aTime = hdr.ModTime
  555. }
  556. // system.Chtimes doesn't support a NOFOLLOW flag atm
  557. if hdr.Typeflag == tar.TypeLink {
  558. if fi, err := os.Lstat(hdr.Linkname); err == nil && (fi.Mode()&os.ModeSymlink == 0) {
  559. if err := system.Chtimes(path, aTime, hdr.ModTime); err != nil {
  560. return err
  561. }
  562. }
  563. } else if hdr.Typeflag != tar.TypeSymlink {
  564. if err := system.Chtimes(path, aTime, hdr.ModTime); err != nil {
  565. return err
  566. }
  567. } else {
  568. ts := []syscall.Timespec{timeToTimespec(aTime), timeToTimespec(hdr.ModTime)}
  569. if err := system.LUtimesNano(path, ts); err != nil && err != system.ErrNotSupportedPlatform {
  570. return err
  571. }
  572. }
  573. return nil
  574. }
  575. // Tar creates an archive from the directory at `path`, and returns it as a
  576. // stream of bytes.
  577. func Tar(path string, compression Compression) (io.ReadCloser, error) {
  578. return TarWithOptions(path, &TarOptions{Compression: compression})
  579. }
  580. // TarWithOptions creates an archive from the directory at `path`, only including files whose relative
  581. // paths are included in `options.IncludeFiles` (if non-nil) or not in `options.ExcludePatterns`.
  582. func TarWithOptions(srcPath string, options *TarOptions) (io.ReadCloser, error) {
  583. // Fix the source path to work with long path names. This is a no-op
  584. // on platforms other than Windows.
  585. srcPath = fixVolumePathPrefix(srcPath)
  586. pm, err := fileutils.NewPatternMatcher(options.ExcludePatterns)
  587. if err != nil {
  588. return nil, err
  589. }
  590. pipeReader, pipeWriter := io.Pipe()
  591. compressWriter, err := CompressStream(pipeWriter, options.Compression)
  592. if err != nil {
  593. return nil, err
  594. }
  595. go func() {
  596. ta := &tarAppender{
  597. TarWriter: tar.NewWriter(compressWriter),
  598. Buffer: pools.BufioWriter32KPool.Get(nil),
  599. SeenFiles: make(map[uint64]string),
  600. UIDMaps: options.UIDMaps,
  601. GIDMaps: options.GIDMaps,
  602. WhiteoutConverter: getWhiteoutConverter(options.WhiteoutFormat),
  603. }
  604. defer func() {
  605. // Make sure to check the error on Close.
  606. if err := ta.TarWriter.Close(); err != nil {
  607. logrus.Errorf("Can't close tar writer: %s", err)
  608. }
  609. if err := compressWriter.Close(); err != nil {
  610. logrus.Errorf("Can't close compress writer: %s", err)
  611. }
  612. if err := pipeWriter.Close(); err != nil {
  613. logrus.Errorf("Can't close pipe writer: %s", err)
  614. }
  615. }()
  616. // this buffer is needed for the duration of this piped stream
  617. defer pools.BufioWriter32KPool.Put(ta.Buffer)
  618. // In general we log errors here but ignore them because
  619. // during e.g. a diff operation the container can continue
  620. // mutating the filesystem and we can see transient errors
  621. // from this
  622. stat, err := os.Lstat(srcPath)
  623. if err != nil {
  624. return
  625. }
  626. if !stat.IsDir() {
  627. // We can't later join a non-dir with any includes because the
  628. // 'walk' will error if "file/." is stat-ed and "file" is not a
  629. // directory. So, we must split the source path and use the
  630. // basename as the include.
  631. if len(options.IncludeFiles) > 0 {
  632. logrus.Warn("Tar: Can't archive a file with includes")
  633. }
  634. dir, base := SplitPathDirEntry(srcPath)
  635. srcPath = dir
  636. options.IncludeFiles = []string{base}
  637. }
  638. if len(options.IncludeFiles) == 0 {
  639. options.IncludeFiles = []string{"."}
  640. }
  641. seen := make(map[string]bool)
  642. for _, include := range options.IncludeFiles {
  643. rebaseName := options.RebaseNames[include]
  644. walkRoot := getWalkRoot(srcPath, include)
  645. filepath.Walk(walkRoot, func(filePath string, f os.FileInfo, err error) error {
  646. if err != nil {
  647. logrus.Errorf("Tar: Can't stat file %s to tar: %s", srcPath, err)
  648. return nil
  649. }
  650. relFilePath, err := filepath.Rel(srcPath, filePath)
  651. if err != nil || (!options.IncludeSourceDir && relFilePath == "." && f.IsDir()) {
  652. // Error getting relative path OR we are looking
  653. // at the source directory path. Skip in both situations.
  654. return nil
  655. }
  656. if options.IncludeSourceDir && include == "." && relFilePath != "." {
  657. relFilePath = strings.Join([]string{".", relFilePath}, string(filepath.Separator))
  658. }
  659. skip := false
  660. // If "include" is an exact match for the current file
  661. // then even if there's an "excludePatterns" pattern that
  662. // matches it, don't skip it. IOW, assume an explicit 'include'
  663. // is asking for that file no matter what - which is true
  664. // for some files, like .dockerignore and Dockerfile (sometimes)
  665. if include != relFilePath {
  666. skip, err = pm.Matches(relFilePath)
  667. if err != nil {
  668. logrus.Errorf("Error matching %s: %v", relFilePath, err)
  669. return err
  670. }
  671. }
  672. if skip {
  673. // If we want to skip this file and its a directory
  674. // then we should first check to see if there's an
  675. // excludes pattern (e.g. !dir/file) that starts with this
  676. // dir. If so then we can't skip this dir.
  677. // Its not a dir then so we can just return/skip.
  678. if !f.IsDir() {
  679. return nil
  680. }
  681. // No exceptions (!...) in patterns so just skip dir
  682. if !pm.Exclusions() {
  683. return filepath.SkipDir
  684. }
  685. dirSlash := relFilePath + string(filepath.Separator)
  686. for _, pat := range pm.Patterns() {
  687. if !pat.Exclusion() {
  688. continue
  689. }
  690. if strings.HasPrefix(pat.String()+string(filepath.Separator), dirSlash) {
  691. // found a match - so can't skip this dir
  692. return nil
  693. }
  694. }
  695. // No matching exclusion dir so just skip dir
  696. return filepath.SkipDir
  697. }
  698. if seen[relFilePath] {
  699. return nil
  700. }
  701. seen[relFilePath] = true
  702. // Rename the base resource.
  703. if rebaseName != "" {
  704. var replacement string
  705. if rebaseName != string(filepath.Separator) {
  706. // Special case the root directory to replace with an
  707. // empty string instead so that we don't end up with
  708. // double slashes in the paths.
  709. replacement = rebaseName
  710. }
  711. relFilePath = strings.Replace(relFilePath, include, replacement, 1)
  712. }
  713. if err := ta.addTarFile(filePath, relFilePath); err != nil {
  714. logrus.Errorf("Can't add file %s to tar: %s", filePath, err)
  715. // if pipe is broken, stop writing tar stream to it
  716. if err == io.ErrClosedPipe {
  717. return err
  718. }
  719. }
  720. return nil
  721. })
  722. }
  723. }()
  724. return pipeReader, nil
  725. }
  726. // Unpack unpacks the decompressedArchive to dest with options.
  727. func Unpack(decompressedArchive io.Reader, dest string, options *TarOptions) error {
  728. tr := tar.NewReader(decompressedArchive)
  729. trBuf := pools.BufioReader32KPool.Get(nil)
  730. defer pools.BufioReader32KPool.Put(trBuf)
  731. var dirs []*tar.Header
  732. remappedRootUID, remappedRootGID, err := idtools.GetRootUIDGID(options.UIDMaps, options.GIDMaps)
  733. if err != nil {
  734. return err
  735. }
  736. whiteoutConverter := getWhiteoutConverter(options.WhiteoutFormat)
  737. // Iterate through the files in the archive.
  738. loop:
  739. for {
  740. hdr, err := tr.Next()
  741. if err == io.EOF {
  742. // end of tar archive
  743. break
  744. }
  745. if err != nil {
  746. return err
  747. }
  748. // Normalize name, for safety and for a simple is-root check
  749. // This keeps "../" as-is, but normalizes "/../" to "/". Or Windows:
  750. // This keeps "..\" as-is, but normalizes "\..\" to "\".
  751. hdr.Name = filepath.Clean(hdr.Name)
  752. for _, exclude := range options.ExcludePatterns {
  753. if strings.HasPrefix(hdr.Name, exclude) {
  754. continue loop
  755. }
  756. }
  757. // After calling filepath.Clean(hdr.Name) above, hdr.Name will now be in
  758. // the filepath format for the OS on which the daemon is running. Hence
  759. // the check for a slash-suffix MUST be done in an OS-agnostic way.
  760. if !strings.HasSuffix(hdr.Name, string(os.PathSeparator)) {
  761. // Not the root directory, ensure that the parent directory exists
  762. parent := filepath.Dir(hdr.Name)
  763. parentPath := filepath.Join(dest, parent)
  764. if _, err := os.Lstat(parentPath); err != nil && os.IsNotExist(err) {
  765. err = idtools.MkdirAllNewAs(parentPath, 0777, remappedRootUID, remappedRootGID)
  766. if err != nil {
  767. return err
  768. }
  769. }
  770. }
  771. path := filepath.Join(dest, hdr.Name)
  772. rel, err := filepath.Rel(dest, path)
  773. if err != nil {
  774. return err
  775. }
  776. if strings.HasPrefix(rel, ".."+string(os.PathSeparator)) {
  777. return breakoutError(fmt.Errorf("%q is outside of %q", hdr.Name, dest))
  778. }
  779. // If path exits we almost always just want to remove and replace it
  780. // The only exception is when it is a directory *and* the file from
  781. // the layer is also a directory. Then we want to merge them (i.e.
  782. // just apply the metadata from the layer).
  783. if fi, err := os.Lstat(path); err == nil {
  784. if options.NoOverwriteDirNonDir && fi.IsDir() && hdr.Typeflag != tar.TypeDir {
  785. // If NoOverwriteDirNonDir is true then we cannot replace
  786. // an existing directory with a non-directory from the archive.
  787. return fmt.Errorf("cannot overwrite directory %q with non-directory %q", path, dest)
  788. }
  789. if options.NoOverwriteDirNonDir && !fi.IsDir() && hdr.Typeflag == tar.TypeDir {
  790. // If NoOverwriteDirNonDir is true then we cannot replace
  791. // an existing non-directory with a directory from the archive.
  792. return fmt.Errorf("cannot overwrite non-directory %q with directory %q", path, dest)
  793. }
  794. if fi.IsDir() && hdr.Name == "." {
  795. continue
  796. }
  797. if !(fi.IsDir() && hdr.Typeflag == tar.TypeDir) {
  798. if err := os.RemoveAll(path); err != nil {
  799. return err
  800. }
  801. }
  802. }
  803. trBuf.Reset(tr)
  804. // if the options contain a uid & gid maps, convert header uid/gid
  805. // entries using the maps such that lchown sets the proper mapped
  806. // uid/gid after writing the file. We only perform this mapping if
  807. // the file isn't already owned by the remapped root UID or GID, as
  808. // that specific uid/gid has no mapping from container -> host, and
  809. // those files already have the proper ownership for inside the
  810. // container.
  811. if hdr.Uid != remappedRootUID {
  812. xUID, err := idtools.ToHost(hdr.Uid, options.UIDMaps)
  813. if err != nil {
  814. return err
  815. }
  816. hdr.Uid = xUID
  817. }
  818. if hdr.Gid != remappedRootGID {
  819. xGID, err := idtools.ToHost(hdr.Gid, options.GIDMaps)
  820. if err != nil {
  821. return err
  822. }
  823. hdr.Gid = xGID
  824. }
  825. if whiteoutConverter != nil {
  826. writeFile, err := whiteoutConverter.ConvertRead(hdr, path)
  827. if err != nil {
  828. return err
  829. }
  830. if !writeFile {
  831. continue
  832. }
  833. }
  834. if err := createTarFile(path, dest, hdr, trBuf, !options.NoLchown, options.ChownOpts, options.InUserNS); err != nil {
  835. return err
  836. }
  837. // Directory mtimes must be handled at the end to avoid further
  838. // file creation in them to modify the directory mtime
  839. if hdr.Typeflag == tar.TypeDir {
  840. dirs = append(dirs, hdr)
  841. }
  842. }
  843. for _, hdr := range dirs {
  844. path := filepath.Join(dest, hdr.Name)
  845. if err := system.Chtimes(path, hdr.AccessTime, hdr.ModTime); err != nil {
  846. return err
  847. }
  848. }
  849. return nil
  850. }
  851. // Untar reads a stream of bytes from `archive`, parses it as a tar archive,
  852. // and unpacks it into the directory at `dest`.
  853. // The archive may be compressed with one of the following algorithms:
  854. // identity (uncompressed), gzip, bzip2, xz.
  855. // FIXME: specify behavior when target path exists vs. doesn't exist.
  856. func Untar(tarArchive io.Reader, dest string, options *TarOptions) error {
  857. return untarHandler(tarArchive, dest, options, true)
  858. }
  859. // UntarUncompressed reads a stream of bytes from `archive`, parses it as a tar archive,
  860. // and unpacks it into the directory at `dest`.
  861. // The archive must be an uncompressed stream.
  862. func UntarUncompressed(tarArchive io.Reader, dest string, options *TarOptions) error {
  863. return untarHandler(tarArchive, dest, options, false)
  864. }
  865. // Handler for teasing out the automatic decompression
  866. func untarHandler(tarArchive io.Reader, dest string, options *TarOptions, decompress bool) error {
  867. if tarArchive == nil {
  868. return fmt.Errorf("Empty archive")
  869. }
  870. dest = filepath.Clean(dest)
  871. if options == nil {
  872. options = &TarOptions{}
  873. }
  874. if options.ExcludePatterns == nil {
  875. options.ExcludePatterns = []string{}
  876. }
  877. r := tarArchive
  878. if decompress {
  879. decompressedArchive, err := DecompressStream(tarArchive)
  880. if err != nil {
  881. return err
  882. }
  883. defer decompressedArchive.Close()
  884. r = decompressedArchive
  885. }
  886. return Unpack(r, dest, options)
  887. }
  888. // TarUntar is a convenience function which calls Tar and Untar, with the output of one piped into the other.
  889. // If either Tar or Untar fails, TarUntar aborts and returns the error.
  890. func (archiver *Archiver) TarUntar(src, dst string) error {
  891. logrus.Debugf("TarUntar(%s %s)", src, dst)
  892. archive, err := TarWithOptions(src, &TarOptions{Compression: Uncompressed})
  893. if err != nil {
  894. return err
  895. }
  896. defer archive.Close()
  897. var options *TarOptions
  898. if archiver.UIDMaps != nil || archiver.GIDMaps != nil {
  899. options = &TarOptions{
  900. UIDMaps: archiver.UIDMaps,
  901. GIDMaps: archiver.GIDMaps,
  902. }
  903. }
  904. return archiver.Untar(archive, dst, options)
  905. }
  906. // TarUntar is a convenience function which calls Tar and Untar, with the output of one piped into the other.
  907. // If either Tar or Untar fails, TarUntar aborts and returns the error.
  908. func TarUntar(src, dst string) error {
  909. return defaultArchiver.TarUntar(src, dst)
  910. }
  911. // UntarPath untar a file from path to a destination, src is the source tar file path.
  912. func (archiver *Archiver) UntarPath(src, dst string) error {
  913. archive, err := os.Open(src)
  914. if err != nil {
  915. return err
  916. }
  917. defer archive.Close()
  918. var options *TarOptions
  919. if archiver.UIDMaps != nil || archiver.GIDMaps != nil {
  920. options = &TarOptions{
  921. UIDMaps: archiver.UIDMaps,
  922. GIDMaps: archiver.GIDMaps,
  923. }
  924. }
  925. return archiver.Untar(archive, dst, options)
  926. }
  927. // UntarPath is a convenience function which looks for an archive
  928. // at filesystem path `src`, and unpacks it at `dst`.
  929. func UntarPath(src, dst string) error {
  930. return defaultArchiver.UntarPath(src, dst)
  931. }
  932. // CopyWithTar creates a tar archive of filesystem path `src`, and
  933. // unpacks it at filesystem path `dst`.
  934. // The archive is streamed directly with fixed buffering and no
  935. // intermediary disk IO.
  936. func (archiver *Archiver) CopyWithTar(src, dst string) error {
  937. srcSt, err := os.Stat(src)
  938. if err != nil {
  939. return err
  940. }
  941. if !srcSt.IsDir() {
  942. return archiver.CopyFileWithTar(src, dst)
  943. }
  944. // if this archiver is set up with ID mapping we need to create
  945. // the new destination directory with the remapped root UID/GID pair
  946. // as owner
  947. rootUID, rootGID, err := idtools.GetRootUIDGID(archiver.UIDMaps, archiver.GIDMaps)
  948. if err != nil {
  949. return err
  950. }
  951. // Create dst, copy src's content into it
  952. logrus.Debugf("Creating dest directory: %s", dst)
  953. if err := idtools.MkdirAllNewAs(dst, 0755, rootUID, rootGID); err != nil {
  954. return err
  955. }
  956. logrus.Debugf("Calling TarUntar(%s, %s)", src, dst)
  957. return archiver.TarUntar(src, dst)
  958. }
  959. // CopyWithTar creates a tar archive of filesystem path `src`, and
  960. // unpacks it at filesystem path `dst`.
  961. // The archive is streamed directly with fixed buffering and no
  962. // intermediary disk IO.
  963. func CopyWithTar(src, dst string) error {
  964. return defaultArchiver.CopyWithTar(src, dst)
  965. }
  966. // CopyFileWithTar emulates the behavior of the 'cp' command-line
  967. // for a single file. It copies a regular file from path `src` to
  968. // path `dst`, and preserves all its metadata.
  969. func (archiver *Archiver) CopyFileWithTar(src, dst string) (err error) {
  970. logrus.Debugf("CopyFileWithTar(%s, %s)", src, dst)
  971. srcSt, err := os.Stat(src)
  972. if err != nil {
  973. return err
  974. }
  975. if srcSt.IsDir() {
  976. return fmt.Errorf("Can't copy a directory")
  977. }
  978. // Clean up the trailing slash. This must be done in an operating
  979. // system specific manner.
  980. if dst[len(dst)-1] == os.PathSeparator {
  981. dst = filepath.Join(dst, filepath.Base(src))
  982. }
  983. // Create the holding directory if necessary
  984. if err := system.MkdirAll(filepath.Dir(dst), 0700); err != nil {
  985. return err
  986. }
  987. r, w := io.Pipe()
  988. errC := promise.Go(func() error {
  989. defer w.Close()
  990. srcF, err := os.Open(src)
  991. if err != nil {
  992. return err
  993. }
  994. defer srcF.Close()
  995. hdr, err := tar.FileInfoHeader(srcSt, "")
  996. if err != nil {
  997. return err
  998. }
  999. hdr.Name = filepath.Base(dst)
  1000. hdr.Mode = int64(chmodTarEntry(os.FileMode(hdr.Mode)))
  1001. remappedRootUID, remappedRootGID, err := idtools.GetRootUIDGID(archiver.UIDMaps, archiver.GIDMaps)
  1002. if err != nil {
  1003. return err
  1004. }
  1005. // only perform mapping if the file being copied isn't already owned by the
  1006. // uid or gid of the remapped root in the container
  1007. if remappedRootUID != hdr.Uid {
  1008. xUID, err := idtools.ToHost(hdr.Uid, archiver.UIDMaps)
  1009. if err != nil {
  1010. return err
  1011. }
  1012. hdr.Uid = xUID
  1013. }
  1014. if remappedRootGID != hdr.Gid {
  1015. xGID, err := idtools.ToHost(hdr.Gid, archiver.GIDMaps)
  1016. if err != nil {
  1017. return err
  1018. }
  1019. hdr.Gid = xGID
  1020. }
  1021. tw := tar.NewWriter(w)
  1022. defer tw.Close()
  1023. if err := tw.WriteHeader(hdr); err != nil {
  1024. return err
  1025. }
  1026. if _, err := io.Copy(tw, srcF); err != nil {
  1027. return err
  1028. }
  1029. return nil
  1030. })
  1031. defer func() {
  1032. if er := <-errC; err == nil && er != nil {
  1033. err = er
  1034. }
  1035. }()
  1036. err = archiver.Untar(r, filepath.Dir(dst), nil)
  1037. if err != nil {
  1038. r.CloseWithError(err)
  1039. }
  1040. return err
  1041. }
  1042. // CopyFileWithTar emulates the behavior of the 'cp' command-line
  1043. // for a single file. It copies a regular file from path `src` to
  1044. // path `dst`, and preserves all its metadata.
  1045. //
  1046. // Destination handling is in an operating specific manner depending
  1047. // where the daemon is running. If `dst` ends with a trailing slash
  1048. // the final destination path will be `dst/base(src)` (Linux) or
  1049. // `dst\base(src)` (Windows).
  1050. func CopyFileWithTar(src, dst string) (err error) {
  1051. return defaultArchiver.CopyFileWithTar(src, dst)
  1052. }
  1053. // cmdStream executes a command, and returns its stdout as a stream.
  1054. // If the command fails to run or doesn't complete successfully, an error
  1055. // will be returned, including anything written on stderr.
  1056. func cmdStream(cmd *exec.Cmd, input io.Reader) (io.ReadCloser, <-chan struct{}, error) {
  1057. chdone := make(chan struct{})
  1058. cmd.Stdin = input
  1059. pipeR, pipeW := io.Pipe()
  1060. cmd.Stdout = pipeW
  1061. var errBuf bytes.Buffer
  1062. cmd.Stderr = &errBuf
  1063. // Run the command and return the pipe
  1064. if err := cmd.Start(); err != nil {
  1065. return nil, nil, err
  1066. }
  1067. // Copy stdout to the returned pipe
  1068. go func() {
  1069. if err := cmd.Wait(); err != nil {
  1070. pipeW.CloseWithError(fmt.Errorf("%s: %s", err, errBuf.String()))
  1071. } else {
  1072. pipeW.Close()
  1073. }
  1074. close(chdone)
  1075. }()
  1076. return pipeR, chdone, nil
  1077. }
  1078. // NewTempArchive reads the content of src into a temporary file, and returns the contents
  1079. // of that file as an archive. The archive can only be read once - as soon as reading completes,
  1080. // the file will be deleted.
  1081. func NewTempArchive(src io.Reader, dir string) (*TempArchive, error) {
  1082. f, err := ioutil.TempFile(dir, "")
  1083. if err != nil {
  1084. return nil, err
  1085. }
  1086. if _, err := io.Copy(f, src); err != nil {
  1087. return nil, err
  1088. }
  1089. if _, err := f.Seek(0, 0); err != nil {
  1090. return nil, err
  1091. }
  1092. st, err := f.Stat()
  1093. if err != nil {
  1094. return nil, err
  1095. }
  1096. size := st.Size()
  1097. return &TempArchive{File: f, Size: size}, nil
  1098. }
  1099. // TempArchive is a temporary archive. The archive can only be read once - as soon as reading completes,
  1100. // the file will be deleted.
  1101. type TempArchive struct {
  1102. *os.File
  1103. Size int64 // Pre-computed from Stat().Size() as a convenience
  1104. read int64
  1105. closed bool
  1106. }
  1107. // Close closes the underlying file if it's still open, or does a no-op
  1108. // to allow callers to try to close the TempArchive multiple times safely.
  1109. func (archive *TempArchive) Close() error {
  1110. if archive.closed {
  1111. return nil
  1112. }
  1113. archive.closed = true
  1114. return archive.File.Close()
  1115. }
  1116. func (archive *TempArchive) Read(data []byte) (int, error) {
  1117. n, err := archive.File.Read(data)
  1118. archive.read += int64(n)
  1119. if err != nil || archive.read == archive.Size {
  1120. archive.Close()
  1121. os.Remove(archive.File.Name())
  1122. }
  1123. return n, err
  1124. }