Merge pull request #31236 from tonistiigi/docker-stdin

build: accept -f - to read Dockerfile from stdin

cli/command/image/build.go

@@ -6,10 +6,12 @@ import (
 	"bytes"
 	"fmt"
 	"io"
+	"io/ioutil"
 	"os"
 	"path/filepath"
 	"regexp"
 	"runtime"
+	"time"
 
 	"github.com/docker/distribution/reference"
 	"github.com/docker/docker/api"
@@ -25,6 +27,7 @@ import (
 	"github.com/docker/docker/pkg/jsonmessage"
 	"github.com/docker/docker/pkg/progress"
 	"github.com/docker/docker/pkg/streamformatter"
+	"github.com/docker/docker/pkg/stringid"
 	"github.com/docker/docker/pkg/urlutil"
 	runconfigopts "github.com/docker/docker/runconfig/opts"
 	units "github.com/docker/go-units"
@@ -141,6 +144,7 @@ func (out *lastProgressOutput) WriteProgress(prog progress.Progress) error {
 func runBuild(dockerCli *command.DockerCli, options buildOptions) error {
 	var (
 		buildCtx      io.ReadCloser
+		dockerfileCtx io.ReadCloser
 		err           error
 		contextDir    string
 		tempDir       string
@@ -157,6 +161,13 @@ func runBuild(dockerCli *command.DockerCli, options buildOptions) error {
 		buildBuff = bytes.NewBuffer(nil)
 	}
 
+	if options.dockerfileName == "-" {
+		if specifiedContext == "-" {
+			return errors.New("invalid argument: can't use stdin for both build context and dockerfile")
+		}
+		dockerfileCtx = dockerCli.In()
+	}
+
 	switch {
 	case specifiedContext == "-":
 		buildCtx, relDockerfile, err = build.GetContextFromReader(dockerCli.In(), options.dockerfileName)
@@ -207,18 +218,19 @@ func runBuild(dockerCli *command.DockerCli, options buildOptions) error {
 			return errors.Errorf("Error checking context: '%s'.", err)
 		}
 
-		// If .dockerignore mentions .dockerignore or the Dockerfile
-		// then make sure we send both files over to the daemon
-		// because Dockerfile is, obviously, needed no matter what, and
-		// .dockerignore is needed to know if either one needs to be
-		// removed. The daemon will remove them for us, if needed, after it
-		// parses the Dockerfile. Ignore errors here, as they will have been
-		// caught by validateContextDirectory above.
-		var includes = []string{"."}
-		keepThem1, _ := fileutils.Matches(".dockerignore", excludes)
-		keepThem2, _ := fileutils.Matches(relDockerfile, excludes)
-		if keepThem1 || keepThem2 {
-			includes = append(includes, ".dockerignore", relDockerfile)
+		// If .dockerignore mentions .dockerignore or the Dockerfile then make
+		// sure we send both files over to the daemon because Dockerfile is,
+		// obviously, needed no matter what, and .dockerignore is needed to know
+		// if either one needs to be removed. The daemon will remove them
+		// if necessary, after it parses the Dockerfile. Ignore errors here, as
+		// they will have been caught by validateContextDirectory above.
+		// Excludes are used instead of includes to maintain the order of files
+		// in the archive.
+		if keep, _ := fileutils.Matches(".dockerignore", excludes); keep {
+			excludes = append(excludes, "!.dockerignore")
+		}
+		if keep, _ := fileutils.Matches(relDockerfile, excludes); keep && dockerfileCtx == nil {
+			excludes = append(excludes, "!"+relDockerfile)
 		}
 
 		compression := archive.Uncompressed
@@ -228,13 +240,20 @@ func runBuild(dockerCli *command.DockerCli, options buildOptions) error {
 		buildCtx, err = archive.TarWithOptions(contextDir, &archive.TarOptions{
 			Compression:     compression,
 			ExcludePatterns: excludes,
-			IncludeFiles:    includes,
 		})
 		if err != nil {
 			return err
 		}
 	}
 
+	// replace Dockerfile if added dynamically
+	if dockerfileCtx != nil {
+		buildCtx, relDockerfile, err = addDockerfileToBuildContext(dockerfileCtx, buildCtx)
+		if err != nil {
+			return err
+		}
+	}
+
 	ctx := context.Background()
 
 	var resolvedTags []*resolvedTag
@@ -338,6 +357,50 @@ func runBuild(dockerCli *command.DockerCli, options buildOptions) error {
 	return nil
 }
 
+func addDockerfileToBuildContext(dockerfileCtx io.ReadCloser, buildCtx io.ReadCloser) (io.ReadCloser, string, error) {
+	file, err := ioutil.ReadAll(dockerfileCtx)
+	dockerfileCtx.Close()
+	if err != nil {
+		return nil, "", err
+	}
+	now := time.Now()
+	hdrTmpl := &tar.Header{
+		Mode:       0600,
+		Uid:        0,
+		Gid:        0,
+		ModTime:    now,
+		Typeflag:   tar.TypeReg,
+		AccessTime: now,
+		ChangeTime: now,
+	}
+	randomName := ".dockerfile." + stringid.GenerateRandomID()[:20]
+
+	buildCtx = archive.ReplaceFileTarWrapper(buildCtx, map[string]archive.TarModifierFunc{
+		// Add the dockerfile with a random filename
+		randomName: func(_ string, h *tar.Header, content io.Reader) (*tar.Header, []byte, error) {
+			return hdrTmpl, file, nil
+		},
+		// Update .dockerignore to include the random filename
+		".dockerignore": func(_ string, h *tar.Header, content io.Reader) (*tar.Header, []byte, error) {
+			if h == nil {
+				h = hdrTmpl
+			}
+
+			b := &bytes.Buffer{}
+			if content != nil {
+				if _, err := b.ReadFrom(content); err != nil {
+					return nil, nil, err
+				}
+			} else {
+				b.WriteString(".dockerignore")
+			}
+			b.WriteString("\n" + randomName + "\n")
+			return h, b.Bytes(), nil
+		},
+	})
+	return buildCtx, randomName, nil
+}
+
 func isLocalDir(c string) bool {
 	_, err := os.Stat(c)
 	return err == nil

cli/command/image/build/context.go

@@ -89,6 +89,10 @@ func GetContextFromReader(r io.ReadCloser, dockerfileName string) (out io.ReadCl
 		return ioutils.NewReadCloserWrapper(buf, func() error { return r.Close() }), dockerfileName, nil
 	}
 
+	if dockerfileName == "-" {
+		return nil, "", errors.New("build context is not an archive")
+	}
+
 	// Input should be read as a Dockerfile.
 	tmpDir, err := ioutil.TempDir("", "docker-build-context-")
 	if err != nil {
@@ -166,7 +170,7 @@ func GetContextFromLocalDir(localDir, dockerfileName string) (absContextDir, rel
 	// When using a local context directory, when the Dockerfile is specified
 	// with the `-f/--file` option then it is considered relative to the
 	// current directory and not the context directory.
-	if dockerfileName != "" {
+	if dockerfileName != "" && dockerfileName != "-" {
 		if dockerfileName, err = filepath.Abs(dockerfileName); err != nil {
 			return "", "", errors.Errorf("unable to get absolute path to Dockerfile: %v", err)
 		}
@@ -220,6 +224,8 @@ func getDockerfileRelPath(givenContextDir, givenDockerfile string) (absContextDi
 				absDockerfile = altPath
 			}
 		}
+	} else if absDockerfile == "-" {
+		absDockerfile = filepath.Join(absContextDir, DefaultDockerfileName)
 	}
 
 	// If not already an absolute path, the Dockerfile path should be joined to
@@ -234,18 +240,21 @@ func getDockerfileRelPath(givenContextDir, givenDockerfile string) (absContextDi
 	// an issue in golang. On Windows, EvalSymLinks does not work on UNC file
 	// paths (those starting with \\). This hack means that when using links
 	// on UNC paths, they will not be followed.
-	if !isUNC(absDockerfile) {
-		absDockerfile, err = filepath.EvalSymlinks(absDockerfile)
-		if err != nil {
-			return "", "", errors.Errorf("unable to evaluate symlinks in Dockerfile path: %v", err)
-		}
-	}
+	if givenDockerfile != "-" {
+		if !isUNC(absDockerfile) {
+			absDockerfile, err = filepath.EvalSymlinks(absDockerfile)
+			if err != nil {
+				return "", "", errors.Errorf("unable to evaluate symlinks in Dockerfile path: %v", err)
 
-	if _, err := os.Lstat(absDockerfile); err != nil {
-		if os.IsNotExist(err) {
-			return "", "", errors.Errorf("Cannot locate Dockerfile: %q", absDockerfile)
+			}
+		}
+
+		if _, err := os.Lstat(absDockerfile); err != nil {
+			if os.IsNotExist(err) {
+				return "", "", errors.Errorf("Cannot locate Dockerfile: %q", absDockerfile)
+			}
+			return "", "", errors.Errorf("unable to stat Dockerfile: %v", err)
 		}
-		return "", "", errors.Errorf("unable to stat Dockerfile: %v", err)
 	}
 
 	if relDockerfile, err = filepath.Rel(absContextDir, absDockerfile); err != nil {

integration-cli/docker_cli_build_test.go

@@ -2024,6 +2024,87 @@ func (s *DockerSuite) TestBuildNoContext(c *check.C) {
 	}
 }
 
+func (s *DockerSuite) TestBuildDockerfileStdin(c *check.C) {
+	name := "stdindockerfile"
+	tmpDir, err := ioutil.TempDir("", "fake-context")
+	c.Assert(err, check.IsNil)
+	err = ioutil.WriteFile(filepath.Join(tmpDir, "foo"), []byte("bar"), 0600)
+	c.Assert(err, check.IsNil)
+
+	icmd.RunCmd(icmd.Cmd{
+		Command: []string{dockerBinary, "build", "-t", name, "-f", "-", tmpDir},
+		Stdin: strings.NewReader(
+			`FROM busybox
+ADD foo /foo
+CMD ["cat", "/foo"]`),
+	}).Assert(c, icmd.Success)
+
+	res := inspectField(c, name, "Config.Cmd")
+	c.Assert(strings.TrimSpace(string(res)), checker.Equals, `[cat /foo]`)
+}
+
+func (s *DockerSuite) TestBuildDockerfileStdinConflict(c *check.C) {
+	name := "stdindockerfiletarcontext"
+	icmd.RunCmd(icmd.Cmd{
+		Command: []string{dockerBinary, "build", "-t", name, "-f", "-", "-"},
+	}).Assert(c, icmd.Expected{
+		ExitCode: 1,
+		Err:      "use stdin for both build context and dockerfile",
+	})
+}
+
+func (s *DockerSuite) TestBuildDockerfileStdinNoExtraFiles(c *check.C) {
+	s.testBuildDockerfileStdinNoExtraFiles(c, false, false)
+}
+
+func (s *DockerSuite) TestBuildDockerfileStdinDockerignore(c *check.C) {
+	s.testBuildDockerfileStdinNoExtraFiles(c, true, false)
+}
+
+func (s *DockerSuite) TestBuildDockerfileStdinDockerignoreIgnored(c *check.C) {
+	s.testBuildDockerfileStdinNoExtraFiles(c, true, true)
+}
+
+func (s *DockerSuite) testBuildDockerfileStdinNoExtraFiles(c *check.C, hasDockerignore, ignoreDockerignore bool) {
+	name := "stdindockerfilenoextra"
+	tmpDir, err := ioutil.TempDir("", "fake-context")
+	c.Assert(err, check.IsNil)
+	defer os.RemoveAll(tmpDir)
+
+	writeFile := func(filename, content string) {
+		err = ioutil.WriteFile(filepath.Join(tmpDir, filename), []byte(content), 0600)
+		c.Assert(err, check.IsNil)
+	}
+
+	writeFile("foo", "bar")
+
+	if hasDockerignore {
+		// Add an empty Dockerfile to verify that it is not added to the image
+		writeFile("Dockerfile", "")
+
+		ignores := "Dockerfile\n"
+		if ignoreDockerignore {
+			ignores += ".dockerignore\n"
+		}
+		writeFile(".dockerignore", ignores)
+	}
+
+	result := icmd.RunCmd(icmd.Cmd{
+		Command: []string{dockerBinary, "build", "-t", name, "-f", "-", tmpDir},
+		Stdin: strings.NewReader(
+			`FROM busybox
+COPY . /baz`),
+	})
+	result.Assert(c, icmd.Success)
+
+	result = cli.DockerCmd(c, "run", "--rm", name, "ls", "-A", "/baz")
+	if hasDockerignore && !ignoreDockerignore {
+		c.Assert(result.Stdout(), checker.Equals, ".dockerignore\nfoo\n")
+	} else {
+		c.Assert(result.Stdout(), checker.Equals, "foo\n")
+	}
+}
+
 func (s *DockerSuite) TestBuildWithVolumeOwnership(c *check.C) {
 	testRequires(c, DaemonIsLinux)
 	name := "testbuildimg"

pkg/archive/archive.go

@@ -225,6 +225,93 @@ func CompressStream(dest io.Writer, compression Compression) (io.WriteCloser, er
 	}
 }
 
+// TarModifierFunc is a function that can be passed to ReplaceFileTarWrapper to
+// modify the contents or header of an entry in the archive. If the file already
+// exists in the archive the TarModifierFunc will be called with the Header and
+// a reader which will return the files content. If the file does not exist both
+// header and content will be nil.
+type TarModifierFunc func(path string, header *tar.Header, content io.Reader) (*tar.Header, []byte, error)
+
+// ReplaceFileTarWrapper converts inputTarStream to a new tar stream. Files in the
+// tar stream are modified if they match any of the keys in mods.
+func ReplaceFileTarWrapper(inputTarStream io.ReadCloser, mods map[string]TarModifierFunc) io.ReadCloser {
+	pipeReader, pipeWriter := io.Pipe()
+
+	go func() {
+		tarReader := tar.NewReader(inputTarStream)
+		tarWriter := tar.NewWriter(pipeWriter)
+		defer inputTarStream.Close()
+		defer tarWriter.Close()
+
+		modify := func(name string, original *tar.Header, modifier TarModifierFunc, tarReader io.Reader) error {
+			header, data, err := modifier(name, original, tarReader)
+			switch {
+			case err != nil:
+				return err
+			case header == nil:
+				return nil
+			}
+
+			header.Name = name
+			header.Size = int64(len(data))
+			if err := tarWriter.WriteHeader(header); err != nil {
+				return err
+			}
+			if len(data) != 0 {
+				if _, err := tarWriter.Write(data); err != nil {
+					return err
+				}
+			}
+			return nil
+		}
+
+		var err error
+		var originalHeader *tar.Header
+		for {
+			originalHeader, err = tarReader.Next()
+			if err == io.EOF {
+				break
+			}
+			if err != nil {
+				pipeWriter.CloseWithError(err)
+				return
+			}
+
+			modifier, ok := mods[originalHeader.Name]
+			if !ok {
+				// No modifiers for this file, copy the header and data
+				if err := tarWriter.WriteHeader(originalHeader); err != nil {
+					pipeWriter.CloseWithError(err)
+					return
+				}
+				if _, err := pools.Copy(tarWriter, tarReader); err != nil {
+					pipeWriter.CloseWithError(err)
+					return
+				}
+				continue
+			}
+			delete(mods, originalHeader.Name)
+
+			if err := modify(originalHeader.Name, originalHeader, modifier, tarReader); err != nil {
+				pipeWriter.CloseWithError(err)
+				return
+			}
+		}
+
+		// Apply the modifiers that haven't matched any files in the archive
+		for name, modifier := range mods {
+			if err := modify(name, nil, modifier, nil); err != nil {
+				pipeWriter.CloseWithError(err)
+				return
+			}
+		}
+
+		pipeWriter.Close()
+
+	}()
+	return pipeReader
+}
+
 // Extension returns the extension of a file that uses the specified compression algorithm.
 func (compression *Compression) Extension() string {
 	switch *compression {

pkg/archive/archive_test.go

@@ -4,6 +4,7 @@ import (
 	"archive/tar"
 	"bytes"
 	"fmt"
+	"github.com/docker/docker/pkg/testutil/assert"
 	"io"
 	"io/ioutil"
 	"os"
@@ -1160,3 +1161,111 @@ func TestTempArchiveCloseMultipleTimes(t *testing.T) {
 		}
 	}
 }
+
+func TestReplaceFileTarWrapper(t *testing.T) {
+	filesInArchive := 20
+	testcases := []struct {
+		doc       string
+		filename  string
+		modifier  TarModifierFunc
+		expected  string
+		fileCount int
+	}{
+		{
+			doc:       "Modifier creates a new file",
+			filename:  "newfile",
+			modifier:  createModifier(t),
+			expected:  "the new content",
+			fileCount: filesInArchive + 1,
+		},
+		{
+			doc:       "Modifier replaces a file",
+			filename:  "file-2",
+			modifier:  createOrReplaceModifier,
+			expected:  "the new content",
+			fileCount: filesInArchive,
+		},
+		{
+			doc:       "Modifier replaces the last file",
+			filename:  fmt.Sprintf("file-%d", filesInArchive-1),
+			modifier:  createOrReplaceModifier,
+			expected:  "the new content",
+			fileCount: filesInArchive,
+		},
+		{
+			doc:       "Modifier appends to a file",
+			filename:  "file-3",
+			modifier:  appendModifier,
+			expected:  "fooo\nnext line",
+			fileCount: filesInArchive,
+		},
+	}
+
+	for _, testcase := range testcases {
+		sourceArchive, cleanup := buildSourceArchive(t, filesInArchive)
+		defer cleanup()
+
+		resultArchive := ReplaceFileTarWrapper(
+			sourceArchive,
+			map[string]TarModifierFunc{testcase.filename: testcase.modifier})
+
+		actual := readFileFromArchive(t, resultArchive, testcase.filename, testcase.fileCount, testcase.doc)
+		assert.Equal(t, actual, testcase.expected, testcase.doc)
+	}
+}
+
+func buildSourceArchive(t *testing.T, numberOfFiles int) (io.ReadCloser, func()) {
+	srcDir, err := ioutil.TempDir("", "docker-test-srcDir")
+	assert.NilError(t, err)
+
+	_, err = prepareUntarSourceDirectory(numberOfFiles, srcDir, false)
+	assert.NilError(t, err)
+
+	sourceArchive, err := TarWithOptions(srcDir, &TarOptions{})
+	assert.NilError(t, err)
+	return sourceArchive, func() {
+		os.RemoveAll(srcDir)
+		sourceArchive.Close()
+	}
+}
+
+func createOrReplaceModifier(path string, header *tar.Header, content io.Reader) (*tar.Header, []byte, error) {
+	return &tar.Header{
+		Mode:     0600,
+		Typeflag: tar.TypeReg,
+	}, []byte("the new content"), nil
+}
+
+func createModifier(t *testing.T) TarModifierFunc {
+	return func(path string, header *tar.Header, content io.Reader) (*tar.Header, []byte, error) {
+		assert.Nil(t, content)
+		return createOrReplaceModifier(path, header, content)
+	}
+}
+
+func appendModifier(path string, header *tar.Header, content io.Reader) (*tar.Header, []byte, error) {
+	buffer := bytes.Buffer{}
+	if content != nil {
+		if _, err := buffer.ReadFrom(content); err != nil {
+			return nil, nil, err
+		}
+	}
+	buffer.WriteString("\nnext line")
+	return &tar.Header{Mode: 0600, Typeflag: tar.TypeReg}, buffer.Bytes(), nil
+}
+
+func readFileFromArchive(t *testing.T, archive io.ReadCloser, name string, expectedCount int, doc string) string {
+	destDir, err := ioutil.TempDir("", "docker-test-destDir")
+	assert.NilError(t, err)
+	defer os.RemoveAll(destDir)
+
+	err = Untar(archive, destDir, nil)
+	assert.NilError(t, err)
+
+	files, _ := ioutil.ReadDir(destDir)
+	assert.Equal(t, len(files), expectedCount, doc)
+
+	content, err := ioutil.ReadFile(filepath.Join(destDir, name))
+	assert.NilError(t, err)
+	return string(content)
+}

pkg/testutil/assert/assert.go

@@ -20,9 +20,9 @@ type TestingT interface {
 
 // Equal compare the actual value to the expected value and fails the test if
 // they are not equal.
-func Equal(t TestingT, actual, expected interface{}) {
+func Equal(t TestingT, actual, expected interface{}, extra ...string) {
 	if expected != actual {
-		fatal(t, "Expected '%v' (%T) got '%v' (%T)", expected, expected, actual, actual)
+		fatalWithExtra(t, extra, "Expected '%v' (%T) got '%v' (%T)", expected, expected, actual, actual)
 	}
 }
 
@@ -103,10 +103,25 @@ func NotNil(t TestingT, obj interface{}) {
 	}
 }
 
+// Nil fails the test if the object is not nil
+func Nil(t TestingT, obj interface{}) {
+	if obj != nil {
+		fatal(t, "Expected nil value, got (%T) %s", obj, obj)
+	}
+}
+
 func fatal(t TestingT, format string, args ...interface{}) {
 	t.Fatalf(errorSource()+format, args...)
 }
 
+func fatalWithExtra(t TestingT, extra []string, format string, args ...interface{}) {
+	msg := fmt.Sprintf(errorSource()+format, args...)
+	if len(extra) > 0 {
+		msg += ": " + strings.Join(extra, ", ")
+	}
+	t.Fatalf(msg)
+}
+
 // See testing.decorate()
 func errorSource() string {
 	_, filename, line, ok := runtime.Caller(3)