build: accept -f - to read Dockerfile from stdin

Heavily based on implementation by David Sheets

Signed-off-by: David Sheets <sheets@alum.mit.edu>
Signed-off-by: Tonis Tiigi <tonistiigi@gmail.com>

cli/command/image/build.go

@@ -6,10 +6,12 @@ import (
 	"bytes"
 	"fmt"
 	"io"
+	"io/ioutil"
 	"os"
 	"path/filepath"
 	"regexp"
 	"runtime"
+	"time"
 
 	"github.com/docker/distribution/reference"
 	"github.com/docker/docker/api"
@@ -25,6 +27,7 @@ import (
 	"github.com/docker/docker/pkg/jsonmessage"
 	"github.com/docker/docker/pkg/progress"
 	"github.com/docker/docker/pkg/streamformatter"
+	"github.com/docker/docker/pkg/stringid"
 	"github.com/docker/docker/pkg/urlutil"
 	runconfigopts "github.com/docker/docker/runconfig/opts"
 	units "github.com/docker/go-units"
@@ -141,6 +144,7 @@ func (out *lastProgressOutput) WriteProgress(prog progress.Progress) error {
 func runBuild(dockerCli *command.DockerCli, options buildOptions) error {
 	var (
 		buildCtx      io.ReadCloser
+		dockerfileCtx io.ReadCloser
 		err           error
 		contextDir    string
 		tempDir       string
@@ -157,6 +161,13 @@ func runBuild(dockerCli *command.DockerCli, options buildOptions) error {
 		buildBuff = bytes.NewBuffer(nil)
 	}
 
+	if options.dockerfileName == "-" {
+		if specifiedContext == "-" {
+			return errors.New("invalid argument: can't use stdin for both build context and dockerfile")
+		}
+		dockerfileCtx = dockerCli.In()
+	}
+
 	switch {
 	case specifiedContext == "-":
 		buildCtx, relDockerfile, err = build.GetContextFromReader(dockerCli.In(), options.dockerfileName)
@@ -214,11 +225,11 @@ func runBuild(dockerCli *command.DockerCli, options buildOptions) error {
 		// removed. The daemon will remove them for us, if needed, after it
 		// parses the Dockerfile. Ignore errors here, as they will have been
 		// caught by validateContextDirectory above.
-		var includes = []string{"."}
-		keepThem1, _ := fileutils.Matches(".dockerignore", excludes)
-		keepThem2, _ := fileutils.Matches(relDockerfile, excludes)
-		if keepThem1 || keepThem2 {
-			includes = append(includes, ".dockerignore", relDockerfile)
+		if keep, _ := fileutils.Matches(".dockerignore", excludes); keep {
+			excludes = append(excludes, "!.dockerignore")
+		}
+		if keep, _ := fileutils.Matches(relDockerfile, excludes); keep && dockerfileCtx == nil {
+			excludes = append(excludes, "!"+relDockerfile)
 		}
 
 		compression := archive.Uncompressed
@@ -228,13 +239,56 @@ func runBuild(dockerCli *command.DockerCli, options buildOptions) error {
 		buildCtx, err = archive.TarWithOptions(contextDir, &archive.TarOptions{
 			Compression:     compression,
 			ExcludePatterns: excludes,
-			IncludeFiles:    includes,
 		})
 		if err != nil {
 			return err
 		}
 	}
 
+	// replace Dockerfile if added dynamically
+	if dockerfileCtx != nil {
+		file, err := ioutil.ReadAll(dockerfileCtx)
+		dockerfileCtx.Close()
+		if err != nil {
+			return err
+		}
+		now := time.Now()
+		hdrTmpl := &tar.Header{
+			Mode:       0600,
+			Uid:        0,
+			Gid:        0,
+			ModTime:    now,
+			Typeflag:   tar.TypeReg,
+			AccessTime: now,
+			ChangeTime: now,
+		}
+		randomName := ".dockerfile." + stringid.GenerateRandomID()[:20]
+
+		buildCtx = archive.ReplaceFileTarWrapper(buildCtx, map[string]archive.TarModifierFunc{
+			randomName: func(_ string, h *tar.Header, content io.Reader) (*tar.Header, []byte, error) {
+				return hdrTmpl, file, nil
+			},
+			".dockerignore": func(_ string, h *tar.Header, content io.Reader) (*tar.Header, []byte, error) {
+				if h == nil {
+					h = hdrTmpl
+				}
+				extraIgnore := randomName + "\n"
+				b := &bytes.Buffer{}
+				if content != nil {
+					_, err := b.ReadFrom(content)
+					if err != nil {
+						return nil, nil, err
+					}
+				} else {
+					extraIgnore += ".dockerignore\n"
+				}
+				b.Write([]byte("\n" + extraIgnore))
+				return h, b.Bytes(), nil
+			},
+		})
+		relDockerfile = randomName
+	}
+
 	ctx := context.Background()
 
 	var resolvedTags []*resolvedTag

cli/command/image/build/context.go

@@ -89,6 +89,10 @@ func GetContextFromReader(r io.ReadCloser, dockerfileName string) (out io.ReadCl
 		return ioutils.NewReadCloserWrapper(buf, func() error { return r.Close() }), dockerfileName, nil
 	}
 
+	if dockerfileName == "-" {
+		return nil, "", errors.New("build context is not an archive")
+	}
+
 	// Input should be read as a Dockerfile.
 	tmpDir, err := ioutil.TempDir("", "docker-build-context-")
 	if err != nil {
@@ -166,7 +170,7 @@ func GetContextFromLocalDir(localDir, dockerfileName string) (absContextDir, rel
 	// When using a local context directory, when the Dockerfile is specified
 	// with the `-f/--file` option then it is considered relative to the
 	// current directory and not the context directory.
-	if dockerfileName != "" {
+	if dockerfileName != "" && dockerfileName != "-" {
 		if dockerfileName, err = filepath.Abs(dockerfileName); err != nil {
 			return "", "", errors.Errorf("unable to get absolute path to Dockerfile: %v", err)
 		}
@@ -220,6 +224,8 @@ func getDockerfileRelPath(givenContextDir, givenDockerfile string) (absContextDi
 				absDockerfile = altPath
 			}
 		}
+	} else if absDockerfile == "-" {
+		absDockerfile = filepath.Join(absContextDir, DefaultDockerfileName)
 	}
 
 	// If not already an absolute path, the Dockerfile path should be joined to
@@ -234,18 +240,21 @@ func getDockerfileRelPath(givenContextDir, givenDockerfile string) (absContextDi
 	// an issue in golang. On Windows, EvalSymLinks does not work on UNC file
 	// paths (those starting with \\). This hack means that when using links
 	// on UNC paths, they will not be followed.
-	if !isUNC(absDockerfile) {
-		absDockerfile, err = filepath.EvalSymlinks(absDockerfile)
-		if err != nil {
-			return "", "", errors.Errorf("unable to evaluate symlinks in Dockerfile path: %v", err)
+	if givenDockerfile != "-" {
+		if !isUNC(absDockerfile) {
+			absDockerfile, err = filepath.EvalSymlinks(absDockerfile)
+			if err != nil {
+				return "", "", errors.Errorf("unable to evaluate symlinks in Dockerfile path: %v", err)
+
+			}
 		}
-	}
 
-	if _, err := os.Lstat(absDockerfile); err != nil {
-		if os.IsNotExist(err) {
-			return "", "", errors.Errorf("Cannot locate Dockerfile: %q", absDockerfile)
+		if _, err := os.Lstat(absDockerfile); err != nil {
+			if os.IsNotExist(err) {
+				return "", "", errors.Errorf("Cannot locate Dockerfile: %q", absDockerfile)
+			}
+			return "", "", errors.Errorf("unable to stat Dockerfile: %v", err)
 		}
-		return "", "", errors.Errorf("unable to stat Dockerfile: %v", err)
 	}
 
 	if relDockerfile, err = filepath.Rel(absContextDir, absDockerfile); err != nil {

integration-cli/docker_cli_build_test.go

@@ -2024,6 +2024,81 @@ func (s *DockerSuite) TestBuildNoContext(c *check.C) {
 	}
 }
 
+func (s *DockerSuite) TestBuildDockerfileStdin(c *check.C) {
+	name := "stdindockerfile"
+	tmpDir, err := ioutil.TempDir("", "fake-context")
+	c.Assert(err, check.IsNil)
+	err = ioutil.WriteFile(filepath.Join(tmpDir, "foo"), []byte("bar"), 0600)
+	c.Assert(err, check.IsNil)
+
+	icmd.RunCmd(icmd.Cmd{
+		Command: []string{dockerBinary, "build", "-t", name, "-f", "-", tmpDir},
+		Stdin: strings.NewReader(
+			`FROM busybox
+ADD foo /foo
+CMD ["cat", "/foo"]`),
+	}).Assert(c, icmd.Success)
+
+	res := inspectField(c, name, "Config.Cmd")
+	c.Assert(strings.TrimSpace(string(res)), checker.Equals, `[cat /foo]`)
+}
+
+func (s *DockerSuite) TestBuildDockerfileStdinConflict(c *check.C) {
+	name := "stdindockerfiletarcontext"
+	icmd.RunCmd(icmd.Cmd{
+		Command: []string{dockerBinary, "build", "-t", name, "-f", "-", "-"},
+	}).Assert(c, icmd.Expected{
+		ExitCode: 1,
+		Err:      "use stdin for both build context and dockerfile",
+	})
+}
+
+func (s *DockerSuite) TestBuildDockerfileStdinNoExtraFiles(c *check.C) {
+	s.testBuildDockerfileStdinNoExtraFiles(c, false, false)
+}
+
+func (s *DockerSuite) TestBuildDockerfileStdinDockerignore(c *check.C) {
+	s.testBuildDockerfileStdinNoExtraFiles(c, true, false)
+}
+
+func (s *DockerSuite) TestBuildDockerfileStdinDockerignoreIgnored(c *check.C) {
+	s.testBuildDockerfileStdinNoExtraFiles(c, true, true)
+}
+
+func (s *DockerSuite) testBuildDockerfileStdinNoExtraFiles(c *check.C, hasDockerignore, ignoreDockerignore bool) {
+	name := "stdindockerfilenoextra"
+	tmpDir, err := ioutil.TempDir("", "fake-context")
+	c.Assert(err, check.IsNil)
+	err = ioutil.WriteFile(filepath.Join(tmpDir, "foo"), []byte("bar"), 0600)
+	c.Assert(err, check.IsNil)
+	if hasDockerignore {
+		// test that this file is removed
+		err = ioutil.WriteFile(filepath.Join(tmpDir, "Dockerfile"), []byte(""), 0600)
+		c.Assert(err, check.IsNil)
+		ignores := "Dockerfile\n"
+		if ignoreDockerignore {
+			ignores += ".dockerignore\n"
+		}
+		err = ioutil.WriteFile(filepath.Join(tmpDir, ".dockerignore"), []byte(ignores), 0600)
+		c.Assert(err, check.IsNil)
+	}
+
+	icmd.RunCmd(icmd.Cmd{
+		Command: []string{dockerBinary, "build", "-t", name, "-f", "-", tmpDir},
+		Stdin: strings.NewReader(
+			`FROM busybox
+COPY . /baz`),
+	}).Assert(c, icmd.Success)
+
+	out, _ := dockerCmd(c, "run", "--rm", name, "ls", "-A", "/baz")
+	if hasDockerignore && !ignoreDockerignore {
+		c.Assert(strings.TrimSpace(string(out)), checker.Equals, ".dockerignore\nfoo")
+	} else {
+		c.Assert(strings.TrimSpace(string(out)), checker.Equals, "foo")
+	}
+
+}
+
 func (s *DockerSuite) TestBuildWithVolumeOwnership(c *check.C) {
 	testRequires(c, DaemonIsLinux)
 	name := "testbuildimg"

pkg/archive/archive.go

@@ -14,6 +14,7 @@ import (
 	"os/exec"
 	"path/filepath"
 	"runtime"
+	"sort"
 	"strings"
 	"syscall"
 
@@ -225,6 +226,91 @@ func CompressStream(dest io.Writer, compression Compression) (io.WriteCloser, er
 	}
 }
 
+// TarModifierFunc is a function that can be passed to ReplaceFileTarWrapper to
+// define a modification step for a single path
+type TarModifierFunc func(path string, header *tar.Header, content io.Reader) (*tar.Header, []byte, error)
+
+// ReplaceFileTarWrapper converts inputTarStream to a new tar stream
+// while replacing a single file called header.Name with new contents.
+// If the file with header.Name does not exist it is added to the tar stream.
+// TODO: make this into a generic tar conversion function with walkFn argument
+func ReplaceFileTarWrapper(inputTarStream io.ReadCloser, mods map[string]TarModifierFunc) io.ReadCloser {
+	pipeReader, pipeWriter := io.Pipe()
+
+	modKeys := make([]string, 0, len(mods))
+	for key := range mods {
+		modKeys = append(modKeys, key)
+	}
+	sort.Strings(modKeys)
+
+	go func() {
+		tarReader := tar.NewReader(inputTarStream)
+		tarWriter := tar.NewWriter(pipeWriter)
+
+		defer inputTarStream.Close()
+
+	loop0:
+		for {
+			hdr, err := tarReader.Next()
+			for len(modKeys) > 0 && (err == io.EOF || err == nil && hdr.Name >= modKeys[0]) {
+				var h *tar.Header
+				var rdr io.Reader
+				if hdr != nil && hdr.Name == modKeys[0] {
+					h = hdr
+					rdr = tarReader
+				}
+
+				h2, dt, err := mods[modKeys[0]](modKeys[0], h, rdr)
+				if err != nil {
+					pipeWriter.CloseWithError(err)
+					return
+				}
+				if h2 != nil {
+					h2.Name = modKeys[0]
+					h2.Size = int64(len(dt))
+					if err := tarWriter.WriteHeader(h2); err != nil {
+						pipeWriter.CloseWithError(err)
+						return
+					}
+					if len(dt) != 0 {
+						if _, err := tarWriter.Write(dt); err != nil {
+							pipeWriter.CloseWithError(err)
+							return
+						}
+					}
+				}
+				modKeys = modKeys[1:]
+				if h != nil {
+					continue loop0
+				}
+			}
+
+			if err == io.EOF {
+				tarWriter.Close()
+				pipeWriter.Close()
+				return
+			}
+
+			if err != nil {
+				pipeWriter.CloseWithError(err)
+				return
+			}
+
+			if err := tarWriter.WriteHeader(hdr); err != nil {
+				pipeWriter.CloseWithError(err)
+				return
+			}
+
+			if _, err := pools.Copy(tarWriter, tarReader); err != nil {
+				pipeWriter.CloseWithError(err)
+				return
+			}
+
+		}
+	}()
+	return pipeReader
+}
+
 // Extension returns the extension of a file that uses the specified compression algorithm.
 func (compression *Compression) Extension() string {
 	switch *compression {

pkg/archive/archive_test.go

@@ -1160,3 +1160,59 @@ func TestTempArchiveCloseMultipleTimes(t *testing.T) {
 		}
 	}
 }
+
+func testReplaceFileTarWrapper(t *testing.T, name string) {
+	srcDir, err := ioutil.TempDir("", "docker-test-srcDir")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(srcDir)
+
+	destDir, err := ioutil.TempDir("", "docker-test-destDir")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(destDir)
+
+	_, err = prepareUntarSourceDirectory(20, srcDir, false)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	archive, err := TarWithOptions(srcDir, &TarOptions{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer archive.Close()
+
+	archive2 := ReplaceFileTarWrapper(archive, map[string]TarModifierFunc{name: func(path string, header *tar.Header, content io.Reader) (*tar.Header, []byte, error) {
+		return &tar.Header{
+			Mode:     0600,
+			Typeflag: tar.TypeReg,
+		}, []byte("foobar"), nil
+	}})
+
+	if err := Untar(archive2, destDir, nil); err != nil {
+		t.Fatal(err)
+	}
+
+	dt, err := ioutil.ReadFile(filepath.Join(destDir, name))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if expected, actual := "foobar", string(dt); actual != expected {
+		t.Fatalf("file contents mismatch, expected: %q, got %q", expected, actual)
+	}
+}
+
+func TestReplaceFileTarWrapperNewFile(t *testing.T) {
+	testReplaceFileTarWrapper(t, "abc")
+}
+
+func TestReplaceFileTarWrapperReplaceFile(t *testing.T) {
+	testReplaceFileTarWrapper(t, "file-2")
+}
+
+func TestReplaceFileTarWrapperLastFile(t *testing.T) {
+	testReplaceFileTarWrapper(t, "file-999")
+}