writer.go 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416
  1. // Copyright 2009 The Go Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style
  3. // license that can be found in the LICENSE file.
  4. package tar
  5. // TODO(dsymonds):
  6. // - catch more errors (no first header, etc.)
  7. import (
  8. "bytes"
  9. "errors"
  10. "fmt"
  11. "io"
  12. "path"
  13. "sort"
  14. "strconv"
  15. "strings"
  16. "time"
  17. )
  18. var (
  19. ErrWriteTooLong = errors.New("archive/tar: write too long")
  20. ErrFieldTooLong = errors.New("archive/tar: header field too long")
  21. ErrWriteAfterClose = errors.New("archive/tar: write after close")
  22. errInvalidHeader = errors.New("archive/tar: header field too long or contains invalid values")
  23. )
  24. // A Writer provides sequential writing of a tar archive in POSIX.1 format.
  25. // A tar archive consists of a sequence of files.
  26. // Call WriteHeader to begin a new file, and then call Write to supply that file's data,
  27. // writing at most hdr.Size bytes in total.
  28. type Writer struct {
  29. w io.Writer
  30. err error
  31. nb int64 // number of unwritten bytes for current file entry
  32. pad int64 // amount of padding to write after current file entry
  33. closed bool
  34. usedBinary bool // whether the binary numeric field extension was used
  35. preferPax bool // use pax header instead of binary numeric header
  36. hdrBuff [blockSize]byte // buffer to use in writeHeader when writing a regular header
  37. paxHdrBuff [blockSize]byte // buffer to use in writeHeader when writing a pax header
  38. }
  39. type formatter struct {
  40. err error // Last error seen
  41. }
  42. // NewWriter creates a new Writer writing to w.
  43. func NewWriter(w io.Writer) *Writer { return &Writer{w: w} }
  44. // Flush finishes writing the current file (optional).
  45. func (tw *Writer) Flush() error {
  46. if tw.nb > 0 {
  47. tw.err = fmt.Errorf("archive/tar: missed writing %d bytes", tw.nb)
  48. return tw.err
  49. }
  50. n := tw.nb + tw.pad
  51. for n > 0 && tw.err == nil {
  52. nr := n
  53. if nr > blockSize {
  54. nr = blockSize
  55. }
  56. var nw int
  57. nw, tw.err = tw.w.Write(zeroBlock[0:nr])
  58. n -= int64(nw)
  59. }
  60. tw.nb = 0
  61. tw.pad = 0
  62. return tw.err
  63. }
  64. // Write s into b, terminating it with a NUL if there is room.
  65. func (f *formatter) formatString(b []byte, s string) {
  66. if len(s) > len(b) {
  67. f.err = ErrFieldTooLong
  68. return
  69. }
  70. ascii := toASCII(s)
  71. copy(b, ascii)
  72. if len(ascii) < len(b) {
  73. b[len(ascii)] = 0
  74. }
  75. }
  76. // Encode x as an octal ASCII string and write it into b with leading zeros.
  77. func (f *formatter) formatOctal(b []byte, x int64) {
  78. s := strconv.FormatInt(x, 8)
  79. // leading zeros, but leave room for a NUL.
  80. for len(s)+1 < len(b) {
  81. s = "0" + s
  82. }
  83. f.formatString(b, s)
  84. }
  85. // fitsInBase256 reports whether x can be encoded into n bytes using base-256
  86. // encoding. Unlike octal encoding, base-256 encoding does not require that the
  87. // string ends with a NUL character. Thus, all n bytes are available for output.
  88. //
  89. // If operating in binary mode, this assumes strict GNU binary mode; which means
  90. // that the first byte can only be either 0x80 or 0xff. Thus, the first byte is
  91. // equivalent to the sign bit in two's complement form.
  92. func fitsInBase256(n int, x int64) bool {
  93. var binBits = uint(n-1) * 8
  94. return n >= 9 || (x >= -1<<binBits && x < 1<<binBits)
  95. }
  96. // Write x into b, as binary (GNUtar/star extension).
  97. func (f *formatter) formatNumeric(b []byte, x int64) {
  98. if fitsInBase256(len(b), x) {
  99. for i := len(b) - 1; i >= 0; i-- {
  100. b[i] = byte(x)
  101. x >>= 8
  102. }
  103. b[0] |= 0x80 // Highest bit indicates binary format
  104. return
  105. }
  106. f.formatOctal(b, 0) // Last resort, just write zero
  107. f.err = ErrFieldTooLong
  108. }
  109. var (
  110. minTime = time.Unix(0, 0)
  111. // There is room for 11 octal digits (33 bits) of mtime.
  112. maxTime = minTime.Add((1<<33 - 1) * time.Second)
  113. )
  114. // WriteHeader writes hdr and prepares to accept the file's contents.
  115. // WriteHeader calls Flush if it is not the first header.
  116. // Calling after a Close will return ErrWriteAfterClose.
  117. func (tw *Writer) WriteHeader(hdr *Header) error {
  118. return tw.writeHeader(hdr, true)
  119. }
  120. // WriteHeader writes hdr and prepares to accept the file's contents.
  121. // WriteHeader calls Flush if it is not the first header.
  122. // Calling after a Close will return ErrWriteAfterClose.
  123. // As this method is called internally by writePax header to allow it to
  124. // suppress writing the pax header.
  125. func (tw *Writer) writeHeader(hdr *Header, allowPax bool) error {
  126. if tw.closed {
  127. return ErrWriteAfterClose
  128. }
  129. if tw.err == nil {
  130. tw.Flush()
  131. }
  132. if tw.err != nil {
  133. return tw.err
  134. }
  135. // a map to hold pax header records, if any are needed
  136. paxHeaders := make(map[string]string)
  137. // TODO(shanemhansen): we might want to use PAX headers for
  138. // subsecond time resolution, but for now let's just capture
  139. // too long fields or non ascii characters
  140. var f formatter
  141. var header []byte
  142. // We need to select which scratch buffer to use carefully,
  143. // since this method is called recursively to write PAX headers.
  144. // If allowPax is true, this is the non-recursive call, and we will use hdrBuff.
  145. // If allowPax is false, we are being called by writePAXHeader, and hdrBuff is
  146. // already being used by the non-recursive call, so we must use paxHdrBuff.
  147. header = tw.hdrBuff[:]
  148. if !allowPax {
  149. header = tw.paxHdrBuff[:]
  150. }
  151. copy(header, zeroBlock)
  152. s := slicer(header)
  153. // Wrappers around formatter that automatically sets paxHeaders if the
  154. // argument extends beyond the capacity of the input byte slice.
  155. var formatString = func(b []byte, s string, paxKeyword string) {
  156. needsPaxHeader := paxKeyword != paxNone && len(s) > len(b) || !isASCII(s)
  157. if needsPaxHeader {
  158. paxHeaders[paxKeyword] = s
  159. return
  160. }
  161. f.formatString(b, s)
  162. }
  163. var formatNumeric = func(b []byte, x int64, paxKeyword string) {
  164. // Try octal first.
  165. s := strconv.FormatInt(x, 8)
  166. if len(s) < len(b) {
  167. f.formatOctal(b, x)
  168. return
  169. }
  170. // If it is too long for octal, and PAX is preferred, use a PAX header.
  171. if paxKeyword != paxNone && tw.preferPax {
  172. f.formatOctal(b, 0)
  173. s := strconv.FormatInt(x, 10)
  174. paxHeaders[paxKeyword] = s
  175. return
  176. }
  177. tw.usedBinary = true
  178. f.formatNumeric(b, x)
  179. }
  180. // keep a reference to the filename to allow to overwrite it later if we detect that we can use ustar longnames instead of pax
  181. pathHeaderBytes := s.next(fileNameSize)
  182. formatString(pathHeaderBytes, hdr.Name, paxPath)
  183. // Handle out of range ModTime carefully.
  184. var modTime int64
  185. if !hdr.ModTime.Before(minTime) && !hdr.ModTime.After(maxTime) {
  186. modTime = hdr.ModTime.Unix()
  187. }
  188. f.formatOctal(s.next(8), hdr.Mode) // 100:108
  189. formatNumeric(s.next(8), int64(hdr.Uid), paxUid) // 108:116
  190. formatNumeric(s.next(8), int64(hdr.Gid), paxGid) // 116:124
  191. formatNumeric(s.next(12), hdr.Size, paxSize) // 124:136
  192. formatNumeric(s.next(12), modTime, paxNone) // 136:148 --- consider using pax for finer granularity
  193. s.next(8) // chksum (148:156)
  194. s.next(1)[0] = hdr.Typeflag // 156:157
  195. formatString(s.next(100), hdr.Linkname, paxLinkpath)
  196. copy(s.next(8), []byte("ustar\x0000")) // 257:265
  197. formatString(s.next(32), hdr.Uname, paxUname) // 265:297
  198. formatString(s.next(32), hdr.Gname, paxGname) // 297:329
  199. formatNumeric(s.next(8), hdr.Devmajor, paxNone) // 329:337
  200. formatNumeric(s.next(8), hdr.Devminor, paxNone) // 337:345
  201. // keep a reference to the prefix to allow to overwrite it later if we detect that we can use ustar longnames instead of pax
  202. prefixHeaderBytes := s.next(155)
  203. formatString(prefixHeaderBytes, "", paxNone) // 345:500 prefix
  204. // Use the GNU magic instead of POSIX magic if we used any GNU extensions.
  205. if tw.usedBinary {
  206. copy(header[257:265], []byte("ustar \x00"))
  207. }
  208. _, paxPathUsed := paxHeaders[paxPath]
  209. // try to use a ustar header when only the name is too long
  210. if !tw.preferPax && len(paxHeaders) == 1 && paxPathUsed {
  211. prefix, suffix, ok := splitUSTARPath(hdr.Name)
  212. if ok {
  213. // Since we can encode in USTAR format, disable PAX header.
  214. delete(paxHeaders, paxPath)
  215. // Update the path fields
  216. formatString(pathHeaderBytes, suffix, paxNone)
  217. formatString(prefixHeaderBytes, prefix, paxNone)
  218. }
  219. }
  220. // The chksum field is terminated by a NUL and a space.
  221. // This is different from the other octal fields.
  222. chksum, _ := checksum(header)
  223. f.formatOctal(header[148:155], chksum) // Never fails
  224. header[155] = ' '
  225. // Check if there were any formatting errors.
  226. if f.err != nil {
  227. tw.err = f.err
  228. return tw.err
  229. }
  230. if allowPax {
  231. for k, v := range hdr.Xattrs {
  232. paxHeaders[paxXattr+k] = v
  233. }
  234. }
  235. if len(paxHeaders) > 0 {
  236. if !allowPax {
  237. return errInvalidHeader
  238. }
  239. if err := tw.writePAXHeader(hdr, paxHeaders); err != nil {
  240. return err
  241. }
  242. }
  243. tw.nb = int64(hdr.Size)
  244. tw.pad = (blockSize - (tw.nb % blockSize)) % blockSize
  245. _, tw.err = tw.w.Write(header)
  246. return tw.err
  247. }
  248. // splitUSTARPath splits a path according to USTAR prefix and suffix rules.
  249. // If the path is not splittable, then it will return ("", "", false).
  250. func splitUSTARPath(name string) (prefix, suffix string, ok bool) {
  251. length := len(name)
  252. if length <= fileNameSize || !isASCII(name) {
  253. return "", "", false
  254. } else if length > fileNamePrefixSize+1 {
  255. length = fileNamePrefixSize + 1
  256. } else if name[length-1] == '/' {
  257. length--
  258. }
  259. i := strings.LastIndex(name[:length], "/")
  260. nlen := len(name) - i - 1 // nlen is length of suffix
  261. plen := i // plen is length of prefix
  262. if i <= 0 || nlen > fileNameSize || nlen == 0 || plen > fileNamePrefixSize {
  263. return "", "", false
  264. }
  265. return name[:i], name[i+1:], true
  266. }
  267. // writePaxHeader writes an extended pax header to the
  268. // archive.
  269. func (tw *Writer) writePAXHeader(hdr *Header, paxHeaders map[string]string) error {
  270. // Prepare extended header
  271. ext := new(Header)
  272. ext.Typeflag = TypeXHeader
  273. // Setting ModTime is required for reader parsing to
  274. // succeed, and seems harmless enough.
  275. ext.ModTime = hdr.ModTime
  276. // The spec asks that we namespace our pseudo files
  277. // with the current pid. However, this results in differing outputs
  278. // for identical inputs. As such, the constant 0 is now used instead.
  279. // golang.org/issue/12358
  280. dir, file := path.Split(hdr.Name)
  281. fullName := path.Join(dir, "PaxHeaders.0", file)
  282. ascii := toASCII(fullName)
  283. if len(ascii) > 100 {
  284. ascii = ascii[:100]
  285. }
  286. ext.Name = ascii
  287. // Construct the body
  288. var buf bytes.Buffer
  289. // Keys are sorted before writing to body to allow deterministic output.
  290. var keys []string
  291. for k := range paxHeaders {
  292. keys = append(keys, k)
  293. }
  294. sort.Strings(keys)
  295. for _, k := range keys {
  296. fmt.Fprint(&buf, formatPAXRecord(k, paxHeaders[k]))
  297. }
  298. ext.Size = int64(len(buf.Bytes()))
  299. if err := tw.writeHeader(ext, false); err != nil {
  300. return err
  301. }
  302. if _, err := tw.Write(buf.Bytes()); err != nil {
  303. return err
  304. }
  305. if err := tw.Flush(); err != nil {
  306. return err
  307. }
  308. return nil
  309. }
  310. // formatPAXRecord formats a single PAX record, prefixing it with the
  311. // appropriate length.
  312. func formatPAXRecord(k, v string) string {
  313. const padding = 3 // Extra padding for ' ', '=', and '\n'
  314. size := len(k) + len(v) + padding
  315. size += len(strconv.Itoa(size))
  316. record := fmt.Sprintf("%d %s=%s\n", size, k, v)
  317. // Final adjustment if adding size field increased the record size.
  318. if len(record) != size {
  319. size = len(record)
  320. record = fmt.Sprintf("%d %s=%s\n", size, k, v)
  321. }
  322. return record
  323. }
  324. // Write writes to the current entry in the tar archive.
  325. // Write returns the error ErrWriteTooLong if more than
  326. // hdr.Size bytes are written after WriteHeader.
  327. func (tw *Writer) Write(b []byte) (n int, err error) {
  328. if tw.closed {
  329. err = ErrWriteAfterClose
  330. return
  331. }
  332. overwrite := false
  333. if int64(len(b)) > tw.nb {
  334. b = b[0:tw.nb]
  335. overwrite = true
  336. }
  337. n, err = tw.w.Write(b)
  338. tw.nb -= int64(n)
  339. if err == nil && overwrite {
  340. err = ErrWriteTooLong
  341. return
  342. }
  343. tw.err = err
  344. return
  345. }
  346. // Close closes the tar archive, flushing any unwritten
  347. // data to the underlying writer.
  348. func (tw *Writer) Close() error {
  349. if tw.err != nil || tw.closed {
  350. return tw.err
  351. }
  352. tw.Flush()
  353. tw.closed = true
  354. if tw.err != nil {
  355. return tw.err
  356. }
  357. // trailer: two zero blocks
  358. for i := 0; i < 2; i++ {
  359. _, tw.err = tw.w.Write(zeroBlock)
  360. if tw.err != nil {
  361. break
  362. }
  363. }
  364. return tw.err
  365. }