From 444a087ac2ca34776b2b3e4433aa35c27cdfd57a Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Mon, 6 Jan 2014 17:37:49 -0800 Subject: [PATCH] Make CopyFileWithTar use a pipe instead of a buffer. Save more than 1Gb of ram for a 500Mb file. --- archive/archive.go | 55 ++++++++++++++++++++++++++++------------------ 1 file changed, 34 insertions(+), 21 deletions(-) diff --git a/archive/archive.go b/archive/archive.go index f8fcf0b163..1b8076543f 100644 --- a/archive/archive.go +++ b/archive/archive.go @@ -3,8 +3,8 @@ package archive import ( "archive/tar" "bytes" - "compress/gzip" "compress/bzip2" + "compress/gzip" "fmt" "github.com/dotcloud/docker/utils" "io" @@ -299,7 +299,7 @@ func CopyWithTar(src, dst string) error { // // If `dst` ends with a trailing slash '/', the final destination path // will be `dst/base(src)`. -func CopyFileWithTar(src, dst string) error { +func CopyFileWithTar(src, dst string) (err error) { utils.Debugf("CopyFileWithTar(%s, %s)", src, dst) srcSt, err := os.Stat(src) if err != nil { @@ -316,25 +316,38 @@ func CopyFileWithTar(src, dst string) error { if err := os.MkdirAll(filepath.Dir(dst), 0700); err != nil && !os.IsExist(err) { return err } - buf := new(bytes.Buffer) - tw := tar.NewWriter(buf) - hdr, err := tar.FileInfoHeader(srcSt, "") - if err != nil { - return err - } - hdr.Name = filepath.Base(dst) - if err := tw.WriteHeader(hdr); err != nil { - return err - } - srcF, err := os.Open(src) - if err != nil { - return err - } - if _, err := io.Copy(tw, srcF); err != nil { - return err - } - tw.Close() - return Untar(buf, filepath.Dir(dst), nil) + + r, w := io.Pipe() + errC := utils.Go(func() error { + defer w.Close() + + srcF, err := os.Open(src) + if err != nil { + return err + } + defer srcF.Close() + + tw := tar.NewWriter(w) + hdr, err := tar.FileInfoHeader(srcSt, "") + if err != nil { + return err + } + hdr.Name = filepath.Base(dst) + if err := tw.WriteHeader(hdr); err != nil { + return err + } + if _, err := io.Copy(tw, srcF); err != nil { + return err + } + tw.Close() + return nil + }) + defer func() { + if er := <-errC; err != nil { + err = er + } + }() + return Untar(r, filepath.Dir(dst), nil) } // CmdStream executes a command, and returns its stdout as a stream.