From c6350bcc2411dfc5039976239e143f6b9da64444 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Tue, 7 Jan 2014 17:46:04 -0800 Subject: [PATCH 1/5] Disable compression for build. More space usage but much faster upload Docker-DCO-1.0-Signed-off-by: Guillaume J. Charmes (github: creack) --- buildfile.go | 3 ++- utils/tarsum.go | 44 +++++++++++++++++++++++++++++++++----------- 2 files changed, 35 insertions(+), 12 deletions(-) diff --git a/buildfile.go b/buildfile.go index ef4b95c064..dc86026226 100644 --- a/buildfile.go +++ b/buildfile.go @@ -604,11 +604,12 @@ func (b *buildFile) Build(context io.Reader) (string, error) { if err != nil { return "", err } - b.context = &utils.TarSum{Reader: context} + b.context = &utils.TarSum{Reader: context, DisableCompression: true} if err := archive.Untar(b.context, tmpdirPath, nil); err != nil { return "", err } defer os.RemoveAll(tmpdirPath) + b.contextPath = tmpdirPath filename := path.Join(tmpdirPath, "Dockerfile") if _, err := os.Stat(filename); os.IsNotExist(err) { diff --git a/utils/tarsum.go b/utils/tarsum.go index 7de3dbe5be..786196b6b4 100644 --- a/utils/tarsum.go +++ b/utils/tarsum.go @@ -15,16 +15,34 @@ import ( type TarSum struct { io.Reader - tarR *tar.Reader - tarW *tar.Writer - gz *gzip.Writer - bufTar *bytes.Buffer - bufGz *bytes.Buffer - h hash.Hash - sums map[string]string - currentFile string - finished bool - first bool + tarR *tar.Reader + tarW *tar.Writer + gz writeCloseFlusher + bufTar *bytes.Buffer + bufGz *bytes.Buffer + h hash.Hash + sums map[string]string + currentFile string + finished bool + first bool + DisableCompression bool +} + +type writeCloseFlusher interface { + io.WriteCloser + Flush() error +} + +type nopCloseFlusher struct { + io.Writer +} + +func (n *nopCloseFlusher) Close() error { + return nil +} + +func (n *nopCloseFlusher) Flush() error { + return nil } func (ts *TarSum) encodeHeader(h *tar.Header) error { @@ -57,7 +75,11 @@ func (ts *TarSum) Read(buf []byte) (int, error) { ts.bufGz = bytes.NewBuffer([]byte{}) ts.tarR = tar.NewReader(ts.Reader) ts.tarW = tar.NewWriter(ts.bufTar) - ts.gz = gzip.NewWriter(ts.bufGz) + if !ts.DisableCompression { + ts.gz = gzip.NewWriter(ts.bufGz) + } else { + ts.gz = &nopCloseFlusher{Writer: ts.bufGz} + } ts.h = sha256.New() ts.h.Reset() ts.first = true From 93ff70a3e70e14762314f04e2b9f61c8710d5353 Mon Sep 17 00:00:00 2001 From: Frank Macreery Date: Mon, 6 Jan 2014 22:42:57 -0500 Subject: [PATCH 2/5] Potential fix for ADD . Docker-DCO-1.0-Signed-off-by: Guillaume J. Charmes (github: creack) --- buildfile.go | 4 +++- integration/buildfile_test.go | 20 ++++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/buildfile.go b/buildfile.go index dc86026226..d28d499bfa 100644 --- a/buildfile.go +++ b/buildfile.go @@ -421,7 +421,9 @@ func (b *buildFile) CmdAdd(args string) error { } else if fi.IsDir() { var subfiles []string for file, sum := range sums { - if strings.HasPrefix(file, origPath) { + absFile := path.Join(b.contextPath, file) + absOrigPath := path.Join(b.contextPath, origPath) + if strings.HasPrefix(absFile, absOrigPath) { subfiles = append(subfiles, sum) } } diff --git a/integration/buildfile_test.go b/integration/buildfile_test.go index ef51777390..5dd403274e 100644 --- a/integration/buildfile_test.go +++ b/integration/buildfile_test.go @@ -592,6 +592,26 @@ func TestBuildADDLocalFileWithoutCache(t *testing.T) { checkCacheBehavior(t, template, false) } +func TestBuildADDCurrentDirectoryWithCache(t *testing.T) { + template := testContextTemplate{` + from {IMAGE} + maintainer dockerio + add . /usr/lib/bla + `, + nil, nil} + checkCacheBehavior(t, template, true) +} + +func TestBuildADDCurrentDirectoryWithoutCache(t *testing.T) { + template := testContextTemplate{` + from {IMAGE} + maintainer dockerio + add . /usr/lib/bla + `, + nil, nil} + checkCacheBehavior(t, template, false) +} + func TestBuildADDRemoteFileWithCache(t *testing.T) { template := testContextTemplate{` from {IMAGE} From e24e9c09f8533b662d1d829123431540bb95ea51 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Wed, 8 Jan 2014 12:16:23 -0800 Subject: [PATCH 3/5] Make sure file are cached correctly during build Docker-DCO-1.0-Signed-off-by: Guillaume J. Charmes (github: creack) --- buildfile.go | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/buildfile.go b/buildfile.go index d28d499bfa..2243962f9e 100644 --- a/buildfile.go +++ b/buildfile.go @@ -408,14 +408,6 @@ func (b *buildFile) CmdAdd(args string) error { sums = b.context.GetSums() ) - // Has tarsum strips the '.' and './', we put it back for comparaison. - for file, sum := range sums { - if len(file) == 0 || file[0] != '.' && file[0] != '/' { - delete(sums, file) - sums["./"+file] = sum - } - } - if fi, err := os.Stat(path.Join(b.contextPath, origPath)); err != nil { return err } else if fi.IsDir() { @@ -432,7 +424,13 @@ func (b *buildFile) CmdAdd(args string) error { hasher.Write([]byte(strings.Join(subfiles, ","))) hash = "dir:" + hex.EncodeToString(hasher.Sum(nil)) } else { - hash = "file:" + sums[origPath] + if origPath[0] == '/' && len(origPath) > 1 { + origPath = origPath[1:] + } + origPath = strings.TrimPrefix(origPath, "./") + if h, ok := sums[origPath]; ok { + hash = "file:" + h + } } b.config.Cmd = []string{"/bin/sh", "-c", fmt.Sprintf("#(nop) ADD %s in %s", hash, dest)} hit, err := b.probeCache() From 762a0cc4723917b4e6ca3731356e43c5e9a13f5f Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Wed, 8 Jan 2014 14:20:30 -0800 Subject: [PATCH 4/5] Cleanup utils.Download Docker-DCO-1.0-Signed-off-by: Guillaume J. Charmes (github: creack) --- utils/utils.go | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/utils/utils.go b/utils/utils.go index d5dbf35f0a..4dfadb793f 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -6,7 +6,6 @@ import ( "crypto/sha256" "encoding/hex" "encoding/json" - "errors" "fmt" "index/suffixarray" "io" @@ -46,14 +45,12 @@ func Go(f func() error) chan error { } // Request a given URL and return an io.Reader -func Download(url string) (*http.Response, error) { - var resp *http.Response - var err error +func Download(url string) (resp *http.Response, err error) { if resp, err = http.Get(url); err != nil { return nil, err } if resp.StatusCode >= 400 { - return nil, errors.New("Got HTTP status code >= 400: " + resp.Status) + return nil, fmt.Errorf("Got HTTP status code >= 400: %s", resp.Status) } return resp, nil } From 0fd9c98de3fc2c8fb1da6dc3ef277b3325106b26 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Wed, 8 Jan 2014 14:20:50 -0800 Subject: [PATCH 5/5] Make sure the remote ADD are cached properly Docker-DCO-1.0-Signed-off-by: Guillaume J. Charmes (github: creack) --- buildfile.go | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/buildfile.go b/buildfile.go index 2243962f9e..de03e5879f 100644 --- a/buildfile.go +++ b/buildfile.go @@ -353,8 +353,9 @@ func (b *buildFile) CmdAdd(args string) error { // FIXME: do we really need this? var ( - origPath = orig - destPath = dest + origPath = orig + destPath = dest + remoteHash string ) if utils.IsURL(orig) { @@ -373,11 +374,20 @@ func (b *buildFile) CmdAdd(args string) error { } defer os.RemoveAll(tmpDirName) if _, err = io.Copy(tmpFile, resp.Body); err != nil { + tmpFile.Close() return err } origPath = path.Join(filepath.Base(tmpDirName), filepath.Base(tmpFileName)) tmpFile.Close() + // Process the checksum + r, err := archive.Tar(tmpFileName, archive.Uncompressed) + if err != nil { + return err + } + tarSum := utils.TarSum{Reader: r, DisableCompression: true} + remoteHash = tarSum.Sum(nil) + // If the destination is a directory, figure out the filename. if strings.HasSuffix(dest, "/") { u, err := url.Parse(orig) @@ -408,7 +418,9 @@ func (b *buildFile) CmdAdd(args string) error { sums = b.context.GetSums() ) - if fi, err := os.Stat(path.Join(b.contextPath, origPath)); err != nil { + if remoteHash != "" { + hash = remoteHash + } else if fi, err := os.Stat(path.Join(b.contextPath, origPath)); err != nil { return err } else if fi.IsDir() { var subfiles []string