12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064 |
- // Copyright 2009 The Go Authors. All rights reserved.
- // Use of this source code is governed by a BSD-style
- // license that can be found in the LICENSE file.
- package tar
- // TODO(dsymonds):
- // - pax extensions
- import (
- "bytes"
- "errors"
- "io"
- "io/ioutil"
- "math"
- "os"
- "strconv"
- "strings"
- "time"
- )
- var (
- ErrHeader = errors.New("archive/tar: invalid tar header")
- )
- const maxNanoSecondIntSize = 9
- // A Reader provides sequential access to the contents of a tar archive.
- // A tar archive consists of a sequence of files.
- // The Next method advances to the next file in the archive (including the first),
- // and then it can be treated as an io.Reader to access the file's data.
- type Reader struct {
- r io.Reader
- err error
- pad int64 // amount of padding (ignored) after current file entry
- curr numBytesReader // reader for current file entry
- hdrBuff [blockSize]byte // buffer to use in readHeader
- RawAccounting bool // Whether to enable the access needed to reassemble the tar from raw bytes. Some performance/memory hit for this.
- rawBytes *bytes.Buffer // last raw bits
- }
- type parser struct {
- err error // Last error seen
- }
- // RawBytes accesses the raw bytes of the archive, apart from the file payload itself.
- // This includes the header and padding.
- //
- // This call resets the current rawbytes buffer
- //
- // Only when RawAccounting is enabled, otherwise this returns nil
- func (tr *Reader) RawBytes() []byte {
- if !tr.RawAccounting {
- return nil
- }
- if tr.rawBytes == nil {
- tr.rawBytes = bytes.NewBuffer(nil)
- }
- // if we've read them, then flush them.
- defer tr.rawBytes.Reset()
- return tr.rawBytes.Bytes()
- }
- // A numBytesReader is an io.Reader with a numBytes method, returning the number
- // of bytes remaining in the underlying encoded data.
- type numBytesReader interface {
- io.Reader
- numBytes() int64
- }
- // A regFileReader is a numBytesReader for reading file data from a tar archive.
- type regFileReader struct {
- r io.Reader // underlying reader
- nb int64 // number of unread bytes for current file entry
- }
- // A sparseFileReader is a numBytesReader for reading sparse file data from a
- // tar archive.
- type sparseFileReader struct {
- rfr numBytesReader // Reads the sparse-encoded file data
- sp []sparseEntry // The sparse map for the file
- pos int64 // Keeps track of file position
- total int64 // Total size of the file
- }
- // A sparseEntry holds a single entry in a sparse file's sparse map.
- //
- // Sparse files are represented using a series of sparseEntrys.
- // Despite the name, a sparseEntry represents an actual data fragment that
- // references data found in the underlying archive stream. All regions not
- // covered by a sparseEntry are logically filled with zeros.
- //
- // For example, if the underlying raw file contains the 10-byte data:
- // var compactData = "abcdefgh"
- //
- // And the sparse map has the following entries:
- // var sp = []sparseEntry{
- // {offset: 2, numBytes: 5} // Data fragment for [2..7]
- // {offset: 18, numBytes: 3} // Data fragment for [18..21]
- // }
- //
- // Then the content of the resulting sparse file with a "real" size of 25 is:
- // var sparseData = "\x00"*2 + "abcde" + "\x00"*11 + "fgh" + "\x00"*4
- type sparseEntry struct {
- offset int64 // Starting position of the fragment
- numBytes int64 // Length of the fragment
- }
- // Keywords for GNU sparse files in a PAX extended header
- const (
- paxGNUSparseNumBlocks = "GNU.sparse.numblocks"
- paxGNUSparseOffset = "GNU.sparse.offset"
- paxGNUSparseNumBytes = "GNU.sparse.numbytes"
- paxGNUSparseMap = "GNU.sparse.map"
- paxGNUSparseName = "GNU.sparse.name"
- paxGNUSparseMajor = "GNU.sparse.major"
- paxGNUSparseMinor = "GNU.sparse.minor"
- paxGNUSparseSize = "GNU.sparse.size"
- paxGNUSparseRealSize = "GNU.sparse.realsize"
- )
- // Keywords for old GNU sparse headers
- const (
- oldGNUSparseMainHeaderOffset = 386
- oldGNUSparseMainHeaderIsExtendedOffset = 482
- oldGNUSparseMainHeaderNumEntries = 4
- oldGNUSparseExtendedHeaderIsExtendedOffset = 504
- oldGNUSparseExtendedHeaderNumEntries = 21
- oldGNUSparseOffsetSize = 12
- oldGNUSparseNumBytesSize = 12
- )
- // NewReader creates a new Reader reading from r.
- func NewReader(r io.Reader) *Reader { return &Reader{r: r} }
- // Next advances to the next entry in the tar archive.
- //
- // io.EOF is returned at the end of the input.
- func (tr *Reader) Next() (*Header, error) {
- if tr.RawAccounting {
- if tr.rawBytes == nil {
- tr.rawBytes = bytes.NewBuffer(nil)
- } else {
- tr.rawBytes.Reset()
- }
- }
- if tr.err != nil {
- return nil, tr.err
- }
- var hdr *Header
- var extHdrs map[string]string
- // Externally, Next iterates through the tar archive as if it is a series of
- // files. Internally, the tar format often uses fake "files" to add meta
- // data that describes the next file. These meta data "files" should not
- // normally be visible to the outside. As such, this loop iterates through
- // one or more "header files" until it finds a "normal file".
- loop:
- for {
- tr.err = tr.skipUnread()
- if tr.err != nil {
- return nil, tr.err
- }
- hdr = tr.readHeader()
- if tr.err != nil {
- return nil, tr.err
- }
- // Check for PAX/GNU special headers and files.
- switch hdr.Typeflag {
- case TypeXHeader:
- extHdrs, tr.err = parsePAX(tr)
- if tr.err != nil {
- return nil, tr.err
- }
- continue loop // This is a meta header affecting the next header
- case TypeGNULongName, TypeGNULongLink:
- var realname []byte
- realname, tr.err = ioutil.ReadAll(tr)
- if tr.err != nil {
- return nil, tr.err
- }
- if tr.RawAccounting {
- if _, tr.err = tr.rawBytes.Write(realname); tr.err != nil {
- return nil, tr.err
- }
- }
- // Convert GNU extensions to use PAX headers.
- if extHdrs == nil {
- extHdrs = make(map[string]string)
- }
- var p parser
- switch hdr.Typeflag {
- case TypeGNULongName:
- extHdrs[paxPath] = p.parseString(realname)
- case TypeGNULongLink:
- extHdrs[paxLinkpath] = p.parseString(realname)
- }
- if p.err != nil {
- tr.err = p.err
- return nil, tr.err
- }
- continue loop // This is a meta header affecting the next header
- default:
- mergePAX(hdr, extHdrs)
- // Check for a PAX format sparse file
- sp, err := tr.checkForGNUSparsePAXHeaders(hdr, extHdrs)
- if err != nil {
- tr.err = err
- return nil, err
- }
- if sp != nil {
- // Current file is a PAX format GNU sparse file.
- // Set the current file reader to a sparse file reader.
- tr.curr, tr.err = newSparseFileReader(tr.curr, sp, hdr.Size)
- if tr.err != nil {
- return nil, tr.err
- }
- }
- break loop // This is a file, so stop
- }
- }
- return hdr, nil
- }
- // checkForGNUSparsePAXHeaders checks the PAX headers for GNU sparse headers. If they are found, then
- // this function reads the sparse map and returns it. Unknown sparse formats are ignored, causing the file to
- // be treated as a regular file.
- func (tr *Reader) checkForGNUSparsePAXHeaders(hdr *Header, headers map[string]string) ([]sparseEntry, error) {
- var sparseFormat string
- // Check for sparse format indicators
- major, majorOk := headers[paxGNUSparseMajor]
- minor, minorOk := headers[paxGNUSparseMinor]
- sparseName, sparseNameOk := headers[paxGNUSparseName]
- _, sparseMapOk := headers[paxGNUSparseMap]
- sparseSize, sparseSizeOk := headers[paxGNUSparseSize]
- sparseRealSize, sparseRealSizeOk := headers[paxGNUSparseRealSize]
- // Identify which, if any, sparse format applies from which PAX headers are set
- if majorOk && minorOk {
- sparseFormat = major + "." + minor
- } else if sparseNameOk && sparseMapOk {
- sparseFormat = "0.1"
- } else if sparseSizeOk {
- sparseFormat = "0.0"
- } else {
- // Not a PAX format GNU sparse file.
- return nil, nil
- }
- // Check for unknown sparse format
- if sparseFormat != "0.0" && sparseFormat != "0.1" && sparseFormat != "1.0" {
- return nil, nil
- }
- // Update hdr from GNU sparse PAX headers
- if sparseNameOk {
- hdr.Name = sparseName
- }
- if sparseSizeOk {
- realSize, err := strconv.ParseInt(sparseSize, 10, 0)
- if err != nil {
- return nil, ErrHeader
- }
- hdr.Size = realSize
- } else if sparseRealSizeOk {
- realSize, err := strconv.ParseInt(sparseRealSize, 10, 0)
- if err != nil {
- return nil, ErrHeader
- }
- hdr.Size = realSize
- }
- // Set up the sparse map, according to the particular sparse format in use
- var sp []sparseEntry
- var err error
- switch sparseFormat {
- case "0.0", "0.1":
- sp, err = readGNUSparseMap0x1(headers)
- case "1.0":
- sp, err = readGNUSparseMap1x0(tr.curr)
- }
- return sp, err
- }
- // mergePAX merges well known headers according to PAX standard.
- // In general headers with the same name as those found
- // in the header struct overwrite those found in the header
- // struct with higher precision or longer values. Esp. useful
- // for name and linkname fields.
- func mergePAX(hdr *Header, headers map[string]string) error {
- for k, v := range headers {
- switch k {
- case paxPath:
- hdr.Name = v
- case paxLinkpath:
- hdr.Linkname = v
- case paxGname:
- hdr.Gname = v
- case paxUname:
- hdr.Uname = v
- case paxUid:
- uid, err := strconv.ParseInt(v, 10, 0)
- if err != nil {
- return err
- }
- hdr.Uid = int(uid)
- case paxGid:
- gid, err := strconv.ParseInt(v, 10, 0)
- if err != nil {
- return err
- }
- hdr.Gid = int(gid)
- case paxAtime:
- t, err := parsePAXTime(v)
- if err != nil {
- return err
- }
- hdr.AccessTime = t
- case paxMtime:
- t, err := parsePAXTime(v)
- if err != nil {
- return err
- }
- hdr.ModTime = t
- case paxCtime:
- t, err := parsePAXTime(v)
- if err != nil {
- return err
- }
- hdr.ChangeTime = t
- case paxSize:
- size, err := strconv.ParseInt(v, 10, 0)
- if err != nil {
- return err
- }
- hdr.Size = int64(size)
- default:
- if strings.HasPrefix(k, paxXattr) {
- if hdr.Xattrs == nil {
- hdr.Xattrs = make(map[string]string)
- }
- hdr.Xattrs[k[len(paxXattr):]] = v
- }
- }
- }
- return nil
- }
- // parsePAXTime takes a string of the form %d.%d as described in
- // the PAX specification.
- func parsePAXTime(t string) (time.Time, error) {
- buf := []byte(t)
- pos := bytes.IndexByte(buf, '.')
- var seconds, nanoseconds int64
- var err error
- if pos == -1 {
- seconds, err = strconv.ParseInt(t, 10, 0)
- if err != nil {
- return time.Time{}, err
- }
- } else {
- seconds, err = strconv.ParseInt(string(buf[:pos]), 10, 0)
- if err != nil {
- return time.Time{}, err
- }
- nano_buf := string(buf[pos+1:])
- // Pad as needed before converting to a decimal.
- // For example .030 -> .030000000 -> 30000000 nanoseconds
- if len(nano_buf) < maxNanoSecondIntSize {
- // Right pad
- nano_buf += strings.Repeat("0", maxNanoSecondIntSize-len(nano_buf))
- } else if len(nano_buf) > maxNanoSecondIntSize {
- // Right truncate
- nano_buf = nano_buf[:maxNanoSecondIntSize]
- }
- nanoseconds, err = strconv.ParseInt(string(nano_buf), 10, 0)
- if err != nil {
- return time.Time{}, err
- }
- }
- ts := time.Unix(seconds, nanoseconds)
- return ts, nil
- }
- // parsePAX parses PAX headers.
- // If an extended header (type 'x') is invalid, ErrHeader is returned
- func parsePAX(r io.Reader) (map[string]string, error) {
- buf, err := ioutil.ReadAll(r)
- if err != nil {
- return nil, err
- }
- // leaving this function for io.Reader makes it more testable
- if tr, ok := r.(*Reader); ok && tr.RawAccounting {
- if _, err = tr.rawBytes.Write(buf); err != nil {
- return nil, err
- }
- }
- sbuf := string(buf)
- // For GNU PAX sparse format 0.0 support.
- // This function transforms the sparse format 0.0 headers into sparse format 0.1 headers.
- var sparseMap bytes.Buffer
- headers := make(map[string]string)
- // Each record is constructed as
- // "%d %s=%s\n", length, keyword, value
- for len(sbuf) > 0 {
- key, value, residual, err := parsePAXRecord(sbuf)
- if err != nil {
- return nil, ErrHeader
- }
- sbuf = residual
- keyStr := string(key)
- if keyStr == paxGNUSparseOffset || keyStr == paxGNUSparseNumBytes {
- // GNU sparse format 0.0 special key. Write to sparseMap instead of using the headers map.
- sparseMap.WriteString(value)
- sparseMap.Write([]byte{','})
- } else {
- // Normal key. Set the value in the headers map.
- headers[keyStr] = string(value)
- }
- }
- if sparseMap.Len() != 0 {
- // Add sparse info to headers, chopping off the extra comma
- sparseMap.Truncate(sparseMap.Len() - 1)
- headers[paxGNUSparseMap] = sparseMap.String()
- }
- return headers, nil
- }
- // parsePAXRecord parses the input PAX record string into a key-value pair.
- // If parsing is successful, it will slice off the currently read record and
- // return the remainder as r.
- //
- // A PAX record is of the following form:
- // "%d %s=%s\n" % (size, key, value)
- func parsePAXRecord(s string) (k, v, r string, err error) {
- // The size field ends at the first space.
- sp := strings.IndexByte(s, ' ')
- if sp == -1 {
- return "", "", s, ErrHeader
- }
- // Parse the first token as a decimal integer.
- n, perr := strconv.ParseInt(s[:sp], 10, 0) // Intentionally parse as native int
- if perr != nil || n < 5 || int64(len(s)) < n {
- return "", "", s, ErrHeader
- }
- // Extract everything between the space and the final newline.
- rec, nl, rem := s[sp+1:n-1], s[n-1:n], s[n:]
- if nl != "\n" {
- return "", "", s, ErrHeader
- }
- // The first equals separates the key from the value.
- eq := strings.IndexByte(rec, '=')
- if eq == -1 {
- return "", "", s, ErrHeader
- }
- return rec[:eq], rec[eq+1:], rem, nil
- }
- // parseString parses bytes as a NUL-terminated C-style string.
- // If a NUL byte is not found then the whole slice is returned as a string.
- func (*parser) parseString(b []byte) string {
- n := 0
- for n < len(b) && b[n] != 0 {
- n++
- }
- return string(b[0:n])
- }
- // parseNumeric parses the input as being encoded in either base-256 or octal.
- // This function may return negative numbers.
- // If parsing fails or an integer overflow occurs, err will be set.
- func (p *parser) parseNumeric(b []byte) int64 {
- // Check for base-256 (binary) format first.
- // If the first bit is set, then all following bits constitute a two's
- // complement encoded number in big-endian byte order.
- if len(b) > 0 && b[0]&0x80 != 0 {
- // Handling negative numbers relies on the following identity:
- // -a-1 == ^a
- //
- // If the number is negative, we use an inversion mask to invert the
- // data bytes and treat the value as an unsigned number.
- var inv byte // 0x00 if positive or zero, 0xff if negative
- if b[0]&0x40 != 0 {
- inv = 0xff
- }
- var x uint64
- for i, c := range b {
- c ^= inv // Inverts c only if inv is 0xff, otherwise does nothing
- if i == 0 {
- c &= 0x7f // Ignore signal bit in first byte
- }
- if (x >> 56) > 0 {
- p.err = ErrHeader // Integer overflow
- return 0
- }
- x = x<<8 | uint64(c)
- }
- if (x >> 63) > 0 {
- p.err = ErrHeader // Integer overflow
- return 0
- }
- if inv == 0xff {
- return ^int64(x)
- }
- return int64(x)
- }
- // Normal case is base-8 (octal) format.
- return p.parseOctal(b)
- }
- func (p *parser) parseOctal(b []byte) int64 {
- // Because unused fields are filled with NULs, we need
- // to skip leading NULs. Fields may also be padded with
- // spaces or NULs.
- // So we remove leading and trailing NULs and spaces to
- // be sure.
- b = bytes.Trim(b, " \x00")
- if len(b) == 0 {
- return 0
- }
- x, perr := strconv.ParseUint(p.parseString(b), 8, 64)
- if perr != nil {
- p.err = ErrHeader
- }
- return int64(x)
- }
- // skipUnread skips any unread bytes in the existing file entry, as well as any
- // alignment padding. It returns io.ErrUnexpectedEOF if any io.EOF is
- // encountered in the data portion; it is okay to hit io.EOF in the padding.
- //
- // Note that this function still works properly even when sparse files are being
- // used since numBytes returns the bytes remaining in the underlying io.Reader.
- func (tr *Reader) skipUnread() error {
- dataSkip := tr.numBytes() // Number of data bytes to skip
- totalSkip := dataSkip + tr.pad // Total number of bytes to skip
- tr.curr, tr.pad = nil, 0
- if tr.RawAccounting {
- _, tr.err = io.CopyN(tr.rawBytes, tr.r, totalSkip)
- return tr.err
- }
- // If possible, Seek to the last byte before the end of the data section.
- // Do this because Seek is often lazy about reporting errors; this will mask
- // the fact that the tar stream may be truncated. We can rely on the
- // io.CopyN done shortly afterwards to trigger any IO errors.
- var seekSkipped int64 // Number of bytes skipped via Seek
- if sr, ok := tr.r.(io.Seeker); ok && dataSkip > 1 {
- // Not all io.Seeker can actually Seek. For example, os.Stdin implements
- // io.Seeker, but calling Seek always returns an error and performs
- // no action. Thus, we try an innocent seek to the current position
- // to see if Seek is really supported.
- pos1, err := sr.Seek(0, os.SEEK_CUR)
- if err == nil {
- // Seek seems supported, so perform the real Seek.
- pos2, err := sr.Seek(dataSkip-1, os.SEEK_CUR)
- if err != nil {
- tr.err = err
- return tr.err
- }
- seekSkipped = pos2 - pos1
- }
- }
- var copySkipped int64 // Number of bytes skipped via CopyN
- copySkipped, tr.err = io.CopyN(ioutil.Discard, tr.r, totalSkip-seekSkipped)
- if tr.err == io.EOF && seekSkipped+copySkipped < dataSkip {
- tr.err = io.ErrUnexpectedEOF
- }
- return tr.err
- }
- func (tr *Reader) verifyChecksum(header []byte) bool {
- if tr.err != nil {
- return false
- }
- var p parser
- given := p.parseOctal(header[148:156])
- unsigned, signed := checksum(header)
- return p.err == nil && (given == unsigned || given == signed)
- }
- // readHeader reads the next block header and assumes that the underlying reader
- // is already aligned to a block boundary.
- //
- // The err will be set to io.EOF only when one of the following occurs:
- // * Exactly 0 bytes are read and EOF is hit.
- // * Exactly 1 block of zeros is read and EOF is hit.
- // * At least 2 blocks of zeros are read.
- func (tr *Reader) readHeader() *Header {
- header := tr.hdrBuff[:]
- copy(header, zeroBlock)
- if n, err := io.ReadFull(tr.r, header); err != nil {
- tr.err = err
- // because it could read some of the block, but reach EOF first
- if tr.err == io.EOF && tr.RawAccounting {
- if _, err := tr.rawBytes.Write(header[:n]); err != nil {
- tr.err = err
- }
- }
- return nil // io.EOF is okay here
- }
- if tr.RawAccounting {
- if _, tr.err = tr.rawBytes.Write(header); tr.err != nil {
- return nil
- }
- }
- // Two blocks of zero bytes marks the end of the archive.
- if bytes.Equal(header, zeroBlock[0:blockSize]) {
- if n, err := io.ReadFull(tr.r, header); err != nil {
- tr.err = err
- // because it could read some of the block, but reach EOF first
- if tr.err == io.EOF && tr.RawAccounting {
- if _, err := tr.rawBytes.Write(header[:n]); err != nil {
- tr.err = err
- }
- }
- return nil // io.EOF is okay here
- }
- if tr.RawAccounting {
- if _, tr.err = tr.rawBytes.Write(header); tr.err != nil {
- return nil
- }
- }
- if bytes.Equal(header, zeroBlock[0:blockSize]) {
- tr.err = io.EOF
- } else {
- tr.err = ErrHeader // zero block and then non-zero block
- }
- return nil
- }
- if !tr.verifyChecksum(header) {
- tr.err = ErrHeader
- return nil
- }
- // Unpack
- var p parser
- hdr := new(Header)
- s := slicer(header)
- hdr.Name = p.parseString(s.next(100))
- hdr.Mode = p.parseNumeric(s.next(8))
- hdr.Uid = int(p.parseNumeric(s.next(8)))
- hdr.Gid = int(p.parseNumeric(s.next(8)))
- hdr.Size = p.parseNumeric(s.next(12))
- hdr.ModTime = time.Unix(p.parseNumeric(s.next(12)), 0)
- s.next(8) // chksum
- hdr.Typeflag = s.next(1)[0]
- hdr.Linkname = p.parseString(s.next(100))
- // The remainder of the header depends on the value of magic.
- // The original (v7) version of tar had no explicit magic field,
- // so its magic bytes, like the rest of the block, are NULs.
- magic := string(s.next(8)) // contains version field as well.
- var format string
- switch {
- case magic[:6] == "ustar\x00": // POSIX tar (1003.1-1988)
- if string(header[508:512]) == "tar\x00" {
- format = "star"
- } else {
- format = "posix"
- }
- case magic == "ustar \x00": // old GNU tar
- format = "gnu"
- }
- switch format {
- case "posix", "gnu", "star":
- hdr.Uname = p.parseString(s.next(32))
- hdr.Gname = p.parseString(s.next(32))
- devmajor := s.next(8)
- devminor := s.next(8)
- if hdr.Typeflag == TypeChar || hdr.Typeflag == TypeBlock {
- hdr.Devmajor = p.parseNumeric(devmajor)
- hdr.Devminor = p.parseNumeric(devminor)
- }
- var prefix string
- switch format {
- case "posix", "gnu":
- prefix = p.parseString(s.next(155))
- case "star":
- prefix = p.parseString(s.next(131))
- hdr.AccessTime = time.Unix(p.parseNumeric(s.next(12)), 0)
- hdr.ChangeTime = time.Unix(p.parseNumeric(s.next(12)), 0)
- }
- if len(prefix) > 0 {
- hdr.Name = prefix + "/" + hdr.Name
- }
- }
- if p.err != nil {
- tr.err = p.err
- return nil
- }
- nb := hdr.Size
- if isHeaderOnlyType(hdr.Typeflag) {
- nb = 0
- }
- if nb < 0 {
- tr.err = ErrHeader
- return nil
- }
- // Set the current file reader.
- tr.pad = -nb & (blockSize - 1) // blockSize is a power of two
- tr.curr = ®FileReader{r: tr.r, nb: nb}
- // Check for old GNU sparse format entry.
- if hdr.Typeflag == TypeGNUSparse {
- // Get the real size of the file.
- hdr.Size = p.parseNumeric(header[483:495])
- if p.err != nil {
- tr.err = p.err
- return nil
- }
- // Read the sparse map.
- sp := tr.readOldGNUSparseMap(header)
- if tr.err != nil {
- return nil
- }
- // Current file is a GNU sparse file. Update the current file reader.
- tr.curr, tr.err = newSparseFileReader(tr.curr, sp, hdr.Size)
- if tr.err != nil {
- return nil
- }
- }
- return hdr
- }
- // readOldGNUSparseMap reads the sparse map as stored in the old GNU sparse format.
- // The sparse map is stored in the tar header if it's small enough. If it's larger than four entries,
- // then one or more extension headers are used to store the rest of the sparse map.
- func (tr *Reader) readOldGNUSparseMap(header []byte) []sparseEntry {
- var p parser
- isExtended := header[oldGNUSparseMainHeaderIsExtendedOffset] != 0
- spCap := oldGNUSparseMainHeaderNumEntries
- if isExtended {
- spCap += oldGNUSparseExtendedHeaderNumEntries
- }
- sp := make([]sparseEntry, 0, spCap)
- s := slicer(header[oldGNUSparseMainHeaderOffset:])
- // Read the four entries from the main tar header
- for i := 0; i < oldGNUSparseMainHeaderNumEntries; i++ {
- offset := p.parseNumeric(s.next(oldGNUSparseOffsetSize))
- numBytes := p.parseNumeric(s.next(oldGNUSparseNumBytesSize))
- if p.err != nil {
- tr.err = p.err
- return nil
- }
- if offset == 0 && numBytes == 0 {
- break
- }
- sp = append(sp, sparseEntry{offset: offset, numBytes: numBytes})
- }
- for isExtended {
- // There are more entries. Read an extension header and parse its entries.
- sparseHeader := make([]byte, blockSize)
- if _, tr.err = io.ReadFull(tr.r, sparseHeader); tr.err != nil {
- return nil
- }
- if tr.RawAccounting {
- if _, tr.err = tr.rawBytes.Write(sparseHeader); tr.err != nil {
- return nil
- }
- }
- isExtended = sparseHeader[oldGNUSparseExtendedHeaderIsExtendedOffset] != 0
- s = slicer(sparseHeader)
- for i := 0; i < oldGNUSparseExtendedHeaderNumEntries; i++ {
- offset := p.parseNumeric(s.next(oldGNUSparseOffsetSize))
- numBytes := p.parseNumeric(s.next(oldGNUSparseNumBytesSize))
- if p.err != nil {
- tr.err = p.err
- return nil
- }
- if offset == 0 && numBytes == 0 {
- break
- }
- sp = append(sp, sparseEntry{offset: offset, numBytes: numBytes})
- }
- }
- return sp
- }
- // readGNUSparseMap1x0 reads the sparse map as stored in GNU's PAX sparse format
- // version 1.0. The format of the sparse map consists of a series of
- // newline-terminated numeric fields. The first field is the number of entries
- // and is always present. Following this are the entries, consisting of two
- // fields (offset, numBytes). This function must stop reading at the end
- // boundary of the block containing the last newline.
- //
- // Note that the GNU manual says that numeric values should be encoded in octal
- // format. However, the GNU tar utility itself outputs these values in decimal.
- // As such, this library treats values as being encoded in decimal.
- func readGNUSparseMap1x0(r io.Reader) ([]sparseEntry, error) {
- var cntNewline int64
- var buf bytes.Buffer
- var blk = make([]byte, blockSize)
- // feedTokens copies data in numBlock chunks from r into buf until there are
- // at least cnt newlines in buf. It will not read more blocks than needed.
- var feedTokens = func(cnt int64) error {
- for cntNewline < cnt {
- if _, err := io.ReadFull(r, blk); err != nil {
- if err == io.EOF {
- err = io.ErrUnexpectedEOF
- }
- return err
- }
- buf.Write(blk)
- for _, c := range blk {
- if c == '\n' {
- cntNewline++
- }
- }
- }
- return nil
- }
- // nextToken gets the next token delimited by a newline. This assumes that
- // at least one newline exists in the buffer.
- var nextToken = func() string {
- cntNewline--
- tok, _ := buf.ReadString('\n')
- return tok[:len(tok)-1] // Cut off newline
- }
- // Parse for the number of entries.
- // Use integer overflow resistant math to check this.
- if err := feedTokens(1); err != nil {
- return nil, err
- }
- numEntries, err := strconv.ParseInt(nextToken(), 10, 0) // Intentionally parse as native int
- if err != nil || numEntries < 0 || int(2*numEntries) < int(numEntries) {
- return nil, ErrHeader
- }
- // Parse for all member entries.
- // numEntries is trusted after this since a potential attacker must have
- // committed resources proportional to what this library used.
- if err := feedTokens(2 * numEntries); err != nil {
- return nil, err
- }
- sp := make([]sparseEntry, 0, numEntries)
- for i := int64(0); i < numEntries; i++ {
- offset, err := strconv.ParseInt(nextToken(), 10, 64)
- if err != nil {
- return nil, ErrHeader
- }
- numBytes, err := strconv.ParseInt(nextToken(), 10, 64)
- if err != nil {
- return nil, ErrHeader
- }
- sp = append(sp, sparseEntry{offset: offset, numBytes: numBytes})
- }
- return sp, nil
- }
- // readGNUSparseMap0x1 reads the sparse map as stored in GNU's PAX sparse format
- // version 0.1. The sparse map is stored in the PAX headers.
- func readGNUSparseMap0x1(extHdrs map[string]string) ([]sparseEntry, error) {
- // Get number of entries.
- // Use integer overflow resistant math to check this.
- numEntriesStr := extHdrs[paxGNUSparseNumBlocks]
- numEntries, err := strconv.ParseInt(numEntriesStr, 10, 0) // Intentionally parse as native int
- if err != nil || numEntries < 0 || int(2*numEntries) < int(numEntries) {
- return nil, ErrHeader
- }
- // There should be two numbers in sparseMap for each entry.
- sparseMap := strings.Split(extHdrs[paxGNUSparseMap], ",")
- if int64(len(sparseMap)) != 2*numEntries {
- return nil, ErrHeader
- }
- // Loop through the entries in the sparse map.
- // numEntries is trusted now.
- sp := make([]sparseEntry, 0, numEntries)
- for i := int64(0); i < numEntries; i++ {
- offset, err := strconv.ParseInt(sparseMap[2*i], 10, 64)
- if err != nil {
- return nil, ErrHeader
- }
- numBytes, err := strconv.ParseInt(sparseMap[2*i+1], 10, 64)
- if err != nil {
- return nil, ErrHeader
- }
- sp = append(sp, sparseEntry{offset: offset, numBytes: numBytes})
- }
- return sp, nil
- }
- // numBytes returns the number of bytes left to read in the current file's entry
- // in the tar archive, or 0 if there is no current file.
- func (tr *Reader) numBytes() int64 {
- if tr.curr == nil {
- // No current file, so no bytes
- return 0
- }
- return tr.curr.numBytes()
- }
- // Read reads from the current entry in the tar archive.
- // It returns 0, io.EOF when it reaches the end of that entry,
- // until Next is called to advance to the next entry.
- //
- // Calling Read on special types like TypeLink, TypeSymLink, TypeChar,
- // TypeBlock, TypeDir, and TypeFifo returns 0, io.EOF regardless of what
- // the Header.Size claims.
- func (tr *Reader) Read(b []byte) (n int, err error) {
- if tr.err != nil {
- return 0, tr.err
- }
- if tr.curr == nil {
- return 0, io.EOF
- }
- n, err = tr.curr.Read(b)
- if err != nil && err != io.EOF {
- tr.err = err
- }
- return
- }
- func (rfr *regFileReader) Read(b []byte) (n int, err error) {
- if rfr.nb == 0 {
- // file consumed
- return 0, io.EOF
- }
- if int64(len(b)) > rfr.nb {
- b = b[0:rfr.nb]
- }
- n, err = rfr.r.Read(b)
- rfr.nb -= int64(n)
- if err == io.EOF && rfr.nb > 0 {
- err = io.ErrUnexpectedEOF
- }
- return
- }
- // numBytes returns the number of bytes left to read in the file's data in the tar archive.
- func (rfr *regFileReader) numBytes() int64 {
- return rfr.nb
- }
- // newSparseFileReader creates a new sparseFileReader, but validates all of the
- // sparse entries before doing so.
- func newSparseFileReader(rfr numBytesReader, sp []sparseEntry, total int64) (*sparseFileReader, error) {
- if total < 0 {
- return nil, ErrHeader // Total size cannot be negative
- }
- // Validate all sparse entries. These are the same checks as performed by
- // the BSD tar utility.
- for i, s := range sp {
- switch {
- case s.offset < 0 || s.numBytes < 0:
- return nil, ErrHeader // Negative values are never okay
- case s.offset > math.MaxInt64-s.numBytes:
- return nil, ErrHeader // Integer overflow with large length
- case s.offset+s.numBytes > total:
- return nil, ErrHeader // Region extends beyond the "real" size
- case i > 0 && sp[i-1].offset+sp[i-1].numBytes > s.offset:
- return nil, ErrHeader // Regions can't overlap and must be in order
- }
- }
- return &sparseFileReader{rfr: rfr, sp: sp, total: total}, nil
- }
- // readHole reads a sparse hole ending at endOffset.
- func (sfr *sparseFileReader) readHole(b []byte, endOffset int64) int {
- n64 := endOffset - sfr.pos
- if n64 > int64(len(b)) {
- n64 = int64(len(b))
- }
- n := int(n64)
- for i := 0; i < n; i++ {
- b[i] = 0
- }
- sfr.pos += n64
- return n
- }
- // Read reads the sparse file data in expanded form.
- func (sfr *sparseFileReader) Read(b []byte) (n int, err error) {
- // Skip past all empty fragments.
- for len(sfr.sp) > 0 && sfr.sp[0].numBytes == 0 {
- sfr.sp = sfr.sp[1:]
- }
- // If there are no more fragments, then it is possible that there
- // is one last sparse hole.
- if len(sfr.sp) == 0 {
- // This behavior matches the BSD tar utility.
- // However, GNU tar stops returning data even if sfr.total is unmet.
- if sfr.pos < sfr.total {
- return sfr.readHole(b, sfr.total), nil
- }
- return 0, io.EOF
- }
- // In front of a data fragment, so read a hole.
- if sfr.pos < sfr.sp[0].offset {
- return sfr.readHole(b, sfr.sp[0].offset), nil
- }
- // In a data fragment, so read from it.
- // This math is overflow free since we verify that offset and numBytes can
- // be safely added when creating the sparseFileReader.
- endPos := sfr.sp[0].offset + sfr.sp[0].numBytes // End offset of fragment
- bytesLeft := endPos - sfr.pos // Bytes left in fragment
- if int64(len(b)) > bytesLeft {
- b = b[:bytesLeft]
- }
- n, err = sfr.rfr.Read(b)
- sfr.pos += int64(n)
- if err == io.EOF {
- if sfr.pos < endPos {
- err = io.ErrUnexpectedEOF // There was supposed to be more data
- } else if sfr.pos < sfr.total {
- err = nil // There is still an implicit sparse hole at the end
- }
- }
- if sfr.pos == endPos {
- sfr.sp = sfr.sp[1:] // We are done with this fragment, so pop it
- }
- return n, err
- }
- // numBytes returns the number of bytes left to read in the sparse file's
- // sparse-encoded data in the tar archive.
- func (sfr *sparseFileReader) numBytes() int64 {
- return sfr.rfr.numBytes()
- }
|