moby/integration-cli/docker_api_build_test.go

package main

import (
	"archive/tar"
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"regexp"
	"strings"

	"github.com/docker/docker/api/types"
	"github.com/docker/docker/integration-cli/checker"
	"github.com/docker/docker/integration-cli/cli/build/fakecontext"
	"github.com/docker/docker/integration-cli/cli/build/fakegit"
	"github.com/docker/docker/integration-cli/cli/build/fakestorage"
	"github.com/docker/docker/integration-cli/request"
	"github.com/go-check/check"
	"github.com/moby/buildkit/session"
	"github.com/moby/buildkit/session/filesync"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
	"golang.org/x/net/context"
	"golang.org/x/sync/errgroup"
)

func (s *DockerSuite) TestBuildAPIDockerFileRemote(c *check.C) {
	testRequires(c, NotUserNamespace)

	var testD string
	if testEnv.OSType == "windows" {
		testD = `FROM busybox
RUN find / -name ba*
RUN find /tmp/`
	} else {
		// -xdev is required because sysfs can cause EPERM
		testD = `FROM busybox
RUN find / -xdev -name ba*
RUN find /tmp/`
	}
	server := fakestorage.New(c, "", fakecontext.WithFiles(map[string]string{"testD": testD}))
	defer server.Close()

	res, body, err := request.Post("/build?dockerfile=baz&remote="+server.URL()+"/testD", request.JSON)
	c.Assert(err, checker.IsNil)
	c.Assert(res.StatusCode, checker.Equals, http.StatusOK)

	buf, err := request.ReadBody(body)
	c.Assert(err, checker.IsNil)

	// Make sure Dockerfile exists.
	// Make sure 'baz' doesn't exist ANYWHERE despite being mentioned in the URL
	out := string(buf)
	c.Assert(out, checker.Contains, "RUN find /tmp")
	c.Assert(out, checker.Not(checker.Contains), "baz")
}

func (s *DockerSuite) TestBuildAPIRemoteTarballContext(c *check.C) {
	buffer := new(bytes.Buffer)
	tw := tar.NewWriter(buffer)
	defer tw.Close()

	dockerfile := []byte("FROM busybox")
	err := tw.WriteHeader(&tar.Header{
		Name: "Dockerfile",
		Size: int64(len(dockerfile)),
	})
	// failed to write tar file header
	c.Assert(err, checker.IsNil)

	_, err = tw.Write(dockerfile)
	// failed to write tar file content
	c.Assert(err, checker.IsNil)

	// failed to close tar archive
	c.Assert(tw.Close(), checker.IsNil)

	server := fakestorage.New(c, "", fakecontext.WithBinaryFiles(map[string]*bytes.Buffer{
		"testT.tar": buffer,
	}))
	defer server.Close()

	res, b, err := request.Post("/build?remote="+server.URL()+"/testT.tar", request.ContentType("application/tar"))
	c.Assert(err, checker.IsNil)
	c.Assert(res.StatusCode, checker.Equals, http.StatusOK)
	b.Close()
}

func (s *DockerSuite) TestBuildAPIRemoteTarballContextWithCustomDockerfile(c *check.C) {
	buffer := new(bytes.Buffer)
	tw := tar.NewWriter(buffer)
	defer tw.Close()

	dockerfile := []byte(`FROM busybox
RUN echo 'wrong'`)
	err := tw.WriteHeader(&tar.Header{
		Name: "Dockerfile",
		Size: int64(len(dockerfile)),
	})
	// failed to write tar file header
	c.Assert(err, checker.IsNil)

	_, err = tw.Write(dockerfile)
	// failed to write tar file content
	c.Assert(err, checker.IsNil)

	custom := []byte(`FROM busybox
RUN echo 'right'
`)
	err = tw.WriteHeader(&tar.Header{
		Name: "custom",
		Size: int64(len(custom)),
	})

	// failed to write tar file header
	c.Assert(err, checker.IsNil)

	_, err = tw.Write(custom)
	// failed to write tar file content
	c.Assert(err, checker.IsNil)

	// failed to close tar archive
	c.Assert(tw.Close(), checker.IsNil)

	server := fakestorage.New(c, "", fakecontext.WithBinaryFiles(map[string]*bytes.Buffer{
		"testT.tar": buffer,
	}))
	defer server.Close()

	url := "/build?dockerfile=custom&remote=" + server.URL() + "/testT.tar"
	res, body, err := request.Post(url, request.ContentType("application/tar"))
	c.Assert(err, checker.IsNil)
	c.Assert(res.StatusCode, checker.Equals, http.StatusOK)

	defer body.Close()
	content, err := request.ReadBody(body)
	c.Assert(err, checker.IsNil)

	// Build used the wrong dockerfile.
	c.Assert(string(content), checker.Not(checker.Contains), "wrong")
}

func (s *DockerSuite) TestBuildAPILowerDockerfile(c *check.C) {
	git := fakegit.New(c, "repo", map[string]string{
		"dockerfile": `FROM busybox
RUN echo from dockerfile`,
	}, false)
	defer git.Close()

	res, body, err := request.Post("/build?remote="+git.RepoURL, request.JSON)
	c.Assert(err, checker.IsNil)
	c.Assert(res.StatusCode, checker.Equals, http.StatusOK)

	buf, err := request.ReadBody(body)
	c.Assert(err, checker.IsNil)

	out := string(buf)
	c.Assert(out, checker.Contains, "from dockerfile")
}

func (s *DockerSuite) TestBuildAPIBuildGitWithF(c *check.C) {
	git := fakegit.New(c, "repo", map[string]string{
		"baz": `FROM busybox
RUN echo from baz`,
		"Dockerfile": `FROM busybox
RUN echo from Dockerfile`,
	}, false)
	defer git.Close()

	// Make sure it tries to 'dockerfile' query param value
	res, body, err := request.Post("/build?dockerfile=baz&remote="+git.RepoURL, request.JSON)
	c.Assert(err, checker.IsNil)
	c.Assert(res.StatusCode, checker.Equals, http.StatusOK)

	buf, err := request.ReadBody(body)
	c.Assert(err, checker.IsNil)

	out := string(buf)
	c.Assert(out, checker.Contains, "from baz")
}

func (s *DockerSuite) TestBuildAPIDoubleDockerfile(c *check.C) {
	testRequires(c, UnixCli) // dockerfile overwrites Dockerfile on Windows
	git := fakegit.New(c, "repo", map[string]string{
		"Dockerfile": `FROM busybox
RUN echo from Dockerfile`,
		"dockerfile": `FROM busybox
RUN echo from dockerfile`,
	}, false)
	defer git.Close()

	// Make sure it tries to 'dockerfile' query param value
	res, body, err := request.Post("/build?remote="+git.RepoURL, request.JSON)
	c.Assert(err, checker.IsNil)
	c.Assert(res.StatusCode, checker.Equals, http.StatusOK)

	buf, err := request.ReadBody(body)
	c.Assert(err, checker.IsNil)

	out := string(buf)
	c.Assert(out, checker.Contains, "from Dockerfile")
}

func (s *DockerSuite) TestBuildAPIUnnormalizedTarPaths(c *check.C) {
	// Make sure that build context tars with entries of the form
	// x/./y don't cause caching false positives.

	buildFromTarContext := func(fileContents []byte) string {
		buffer := new(bytes.Buffer)
		tw := tar.NewWriter(buffer)
		defer tw.Close()

		dockerfile := []byte(`FROM busybox
	COPY dir /dir/`)
		err := tw.WriteHeader(&tar.Header{
			Name: "Dockerfile",
			Size: int64(len(dockerfile)),
		})
		//failed to write tar file header
		c.Assert(err, checker.IsNil)

		_, err = tw.Write(dockerfile)
		// failed to write Dockerfile in tar file content
		c.Assert(err, checker.IsNil)

		err = tw.WriteHeader(&tar.Header{
			Name: "dir/./file",
			Size: int64(len(fileContents)),
		})
		//failed to write tar file header
		c.Assert(err, checker.IsNil)

		_, err = tw.Write(fileContents)
		// failed to write file contents in tar file content
		c.Assert(err, checker.IsNil)

		// failed to close tar archive
		c.Assert(tw.Close(), checker.IsNil)

		res, body, err := request.Post("/build", request.RawContent(ioutil.NopCloser(buffer)), request.ContentType("application/x-tar"))
		c.Assert(err, checker.IsNil)
		c.Assert(res.StatusCode, checker.Equals, http.StatusOK)

		out, err := request.ReadBody(body)
		c.Assert(err, checker.IsNil)
		lines := strings.Split(string(out), "\n")
		c.Assert(len(lines), checker.GreaterThan, 1)
		c.Assert(lines[len(lines)-2], checker.Matches, ".*Successfully built [0-9a-f]{12}.*")

		re := regexp.MustCompile("Successfully built ([0-9a-f]{12})")
		matches := re.FindStringSubmatch(lines[len(lines)-2])
		return matches[1]
	}

	imageA := buildFromTarContext([]byte("abc"))
	imageB := buildFromTarContext([]byte("def"))

	c.Assert(imageA, checker.Not(checker.Equals), imageB)
}

func (s *DockerSuite) TestBuildOnBuildWithCopy(c *check.C) {
	dockerfile := `
		FROM ` + minimalBaseImage() + ` as onbuildbase
		ONBUILD COPY file /file

		FROM onbuildbase
	`
	ctx := fakecontext.New(c, "",
		fakecontext.WithDockerfile(dockerfile),
		fakecontext.WithFile("file", "some content"),
	)
	defer ctx.Close()

	res, body, err := request.Post(
		"/build",
		request.RawContent(ctx.AsTarReader(c)),
		request.ContentType("application/x-tar"))
	c.Assert(err, checker.IsNil)
	c.Assert(res.StatusCode, checker.Equals, http.StatusOK)

	out, err := request.ReadBody(body)
	c.Assert(err, checker.IsNil)
	c.Assert(string(out), checker.Contains, "Successfully built")
}

func (s *DockerSuite) TestBuildOnBuildCache(c *check.C) {
	build := func(dockerfile string) []byte {
		ctx := fakecontext.New(c, "",
			fakecontext.WithDockerfile(dockerfile),
		)
		defer ctx.Close()

		res, body, err := request.Post(
			"/build",
			request.RawContent(ctx.AsTarReader(c)),
			request.ContentType("application/x-tar"))
		require.NoError(c, err)
		assert.Equal(c, http.StatusOK, res.StatusCode)

		out, err := request.ReadBody(body)
		require.NoError(c, err)
		assert.Contains(c, string(out), "Successfully built")
		return out
	}

	dockerfile := `
		FROM ` + minimalBaseImage() + ` as onbuildbase
		ENV something=bar
		ONBUILD ENV foo=bar
	`
	build(dockerfile)

	dockerfile += "FROM onbuildbase"
	out := build(dockerfile)

	imageIDs := getImageIDsFromBuild(c, out)
	assert.Len(c, imageIDs, 2)
	parentID, childID := imageIDs[0], imageIDs[1]

	client, err := request.NewClient()
	require.NoError(c, err)

	// check parentID is correct
	image, _, err := client.ImageInspectWithRaw(context.Background(), childID)
	require.NoError(c, err)
	assert.Equal(c, parentID, image.Parent)
}

func (s *DockerRegistrySuite) TestBuildCopyFromForcePull(c *check.C) {
	client, err := request.NewClient()
	require.NoError(c, err)

	repoName := fmt.Sprintf("%v/dockercli/busybox", privateRegistryURL)
	// tag the image to upload it to the private registry
	err = client.ImageTag(context.TODO(), "busybox", repoName)
	assert.Nil(c, err)
	// push the image to the registry
	rc, err := client.ImagePush(context.TODO(), repoName, types.ImagePushOptions{RegistryAuth: "{}"})
	assert.Nil(c, err)
	_, err = io.Copy(ioutil.Discard, rc)
	assert.Nil(c, err)

	dockerfile := fmt.Sprintf(`
		FROM %s AS foo
		RUN touch abc
		FROM %s
		COPY --from=foo /abc /
		`, repoName, repoName)

	ctx := fakecontext.New(c, "",
		fakecontext.WithDockerfile(dockerfile),
	)
	defer ctx.Close()

	res, body, err := request.Post(
		"/build?pull=1",
		request.RawContent(ctx.AsTarReader(c)),
		request.ContentType("application/x-tar"))
	require.NoError(c, err)
	assert.Equal(c, http.StatusOK, res.StatusCode)

	out, err := request.ReadBody(body)
	require.NoError(c, err)
	assert.Contains(c, string(out), "Successfully built")
}

func (s *DockerSuite) TestBuildAddRemoteNoDecompress(c *check.C) {
	buffer := new(bytes.Buffer)
	tw := tar.NewWriter(buffer)
	dt := []byte("contents")
	err := tw.WriteHeader(&tar.Header{
		Name:     "foo",
		Size:     int64(len(dt)),
		Mode:     0600,
		Typeflag: tar.TypeReg,
	})
	require.NoError(c, err)
	_, err = tw.Write(dt)
	require.NoError(c, err)
	err = tw.Close()
	require.NoError(c, err)

	server := fakestorage.New(c, "", fakecontext.WithBinaryFiles(map[string]*bytes.Buffer{
		"test.tar": buffer,
	}))
	defer server.Close()

	dockerfile := fmt.Sprintf(`
		FROM busybox
		ADD %s/test.tar /
		RUN [ -f test.tar ]
		`, server.URL())

	ctx := fakecontext.New(c, "",
		fakecontext.WithDockerfile(dockerfile),
	)
	defer ctx.Close()

	res, body, err := request.Post(
		"/build",
		request.RawContent(ctx.AsTarReader(c)),
		request.ContentType("application/x-tar"))
	require.NoError(c, err)
	assert.Equal(c, http.StatusOK, res.StatusCode)

	out, err := request.ReadBody(body)
	require.NoError(c, err)
	assert.Contains(c, string(out), "Successfully built")
}

func (s *DockerSuite) TestBuildChownOnCopy(c *check.C) {
	testRequires(c, DaemonIsLinux)
	dockerfile := `FROM busybox
		RUN echo 'test1:x:1001:1001::/bin:/bin/false' >> /etc/passwd
		RUN echo 'test1:x:1001:' >> /etc/group
		RUN echo 'test2:x:1002:' >> /etc/group
		COPY --chown=test1:1002 . /new_dir
		RUN ls -l /
		RUN [ $(ls -l / | grep new_dir | awk '{print $3":"$4}') = 'test1:test2' ]
		RUN [ $(ls -nl / | grep new_dir | awk '{print $3":"$4}') = '1001:1002' ]
	`
	ctx := fakecontext.New(c, "",
		fakecontext.WithDockerfile(dockerfile),
		fakecontext.WithFile("test_file1", "some test content"),
	)
	defer ctx.Close()

	res, body, err := request.Post(
		"/build",
		request.RawContent(ctx.AsTarReader(c)),
		request.ContentType("application/x-tar"))
	c.Assert(err, checker.IsNil)
	c.Assert(res.StatusCode, checker.Equals, http.StatusOK)

	out, err := request.ReadBody(body)
	require.NoError(c, err)
	assert.Contains(c, string(out), "Successfully built")
}

func (s *DockerSuite) TestBuildCopyCacheOnFileChange(c *check.C) {

	dockerfile := `FROM busybox
COPY file /file`

	ctx1 := fakecontext.New(c, "",
		fakecontext.WithDockerfile(dockerfile),
		fakecontext.WithFile("file", "foo"))
	ctx2 := fakecontext.New(c, "",
		fakecontext.WithDockerfile(dockerfile),
		fakecontext.WithFile("file", "bar"))

	var build = func(ctx *fakecontext.Fake) string {
		res, body, err := request.Post("/build",
			request.RawContent(ctx.AsTarReader(c)),
			request.ContentType("application/x-tar"))

		require.NoError(c, err)
		assert.Equal(c, http.StatusOK, res.StatusCode)

		out, err := request.ReadBody(body)
		require.NoError(c, err)

		ids := getImageIDsFromBuild(c, out)
		return ids[len(ids)-1]
	}

	id1 := build(ctx1)
	id2 := build(ctx1)
	id3 := build(ctx2)

	if id1 != id2 {
		c.Fatal("didn't use the cache")
	}
	if id1 == id3 {
		c.Fatal("COPY With different source file should not share same cache")
	}
}

func (s *DockerSuite) TestBuildAddCacheOnFileChange(c *check.C) {

	dockerfile := `FROM busybox
ADD file /file`

	ctx1 := fakecontext.New(c, "",
		fakecontext.WithDockerfile(dockerfile),
		fakecontext.WithFile("file", "foo"))
	ctx2 := fakecontext.New(c, "",
		fakecontext.WithDockerfile(dockerfile),
		fakecontext.WithFile("file", "bar"))

	var build = func(ctx *fakecontext.Fake) string {
		res, body, err := request.Post("/build",
			request.RawContent(ctx.AsTarReader(c)),
			request.ContentType("application/x-tar"))

		require.NoError(c, err)
		assert.Equal(c, http.StatusOK, res.StatusCode)

		out, err := request.ReadBody(body)
		require.NoError(c, err)

		ids := getImageIDsFromBuild(c, out)
		return ids[len(ids)-1]
	}

	id1 := build(ctx1)
	id2 := build(ctx1)
	id3 := build(ctx2)

	if id1 != id2 {
		c.Fatal("didn't use the cache")
	}
	if id1 == id3 {
		c.Fatal("COPY With different source file should not share same cache")
	}
}

func (s *DockerSuite) TestBuildWithSession(c *check.C) {
	testRequires(c, ExperimentalDaemon)

	dockerfile := `
		FROM busybox
		COPY file /
		RUN cat /file
	`

	fctx := fakecontext.New(c, "",
		fakecontext.WithFile("file", "some content"),
	)
	defer fctx.Close()

	out := testBuildWithSession(c, fctx.Dir, dockerfile)
	assert.Contains(c, out, "some content")

	fctx.Add("second", "contentcontent")

	dockerfile += `
	COPY second /
	RUN cat /second
	`

	out = testBuildWithSession(c, fctx.Dir, dockerfile)
	assert.Equal(c, strings.Count(out, "Using cache"), 2)
	assert.Contains(c, out, "contentcontent")

	client, err := request.NewClient()
	require.NoError(c, err)

	du, err := client.DiskUsage(context.TODO())
	assert.Nil(c, err)
	assert.True(c, du.BuilderSize > 10)

	out = testBuildWithSession(c, fctx.Dir, dockerfile)
	assert.Equal(c, strings.Count(out, "Using cache"), 4)

	du2, err := client.DiskUsage(context.TODO())
	assert.Nil(c, err)
	assert.Equal(c, du.BuilderSize, du2.BuilderSize)

	// rebuild with regular tar, confirm cache still applies
	fctx.Add("Dockerfile", dockerfile)
	res, body, err := request.Post(
		"/build",
		request.RawContent(fctx.AsTarReader(c)),
		request.ContentType("application/x-tar"))
	require.NoError(c, err)
	assert.Equal(c, http.StatusOK, res.StatusCode)

	outBytes, err := request.ReadBody(body)
	require.NoError(c, err)
	assert.Contains(c, string(outBytes), "Successfully built")
	assert.Equal(c, strings.Count(string(outBytes), "Using cache"), 4)

	_, err = client.BuildCachePrune(context.TODO())
	assert.Nil(c, err)

	du, err = client.DiskUsage(context.TODO())
	assert.Nil(c, err)
	assert.Equal(c, du.BuilderSize, int64(0))
}

func testBuildWithSession(c *check.C, dir, dockerfile string) (outStr string) {
	client, err := request.NewClient()
	require.NoError(c, err)

	sess, err := session.NewSession("foo1", "foo")
	assert.Nil(c, err)

	fsProvider := filesync.NewFSSyncProvider([]filesync.SyncedDir{
		{Dir: dir},
	})
	sess.Allow(fsProvider)

	g, ctx := errgroup.WithContext(context.Background())

	g.Go(func() error {
		return sess.Run(ctx, client.DialSession)
	})

	g.Go(func() error {
		res, body, err := request.Post("/build?remote=client-session&session="+sess.ID(), func(req *http.Request) error {
			req.Body = ioutil.NopCloser(strings.NewReader(dockerfile))
			return nil
		})
		if err != nil {
			return err
		}
		assert.Equal(c, res.StatusCode, http.StatusOK)
		out, err := request.ReadBody(body)
		require.NoError(c, err)
		assert.Contains(c, string(out), "Successfully built")
		sess.Close()
		outStr = string(out)
		return nil
	})

	err = g.Wait()
	assert.Nil(c, err)
	return
}

func (s *DockerSuite) TestBuildScratchCopy(c *check.C) {
	testRequires(c, DaemonIsLinux)
	dockerfile := `FROM scratch
ADD Dockerfile /
ENV foo bar`
	ctx := fakecontext.New(c, "",
		fakecontext.WithDockerfile(dockerfile),
	)
	defer ctx.Close()

	res, body, err := request.Post(
		"/build",
		request.RawContent(ctx.AsTarReader(c)),
		request.ContentType("application/x-tar"))
	c.Assert(err, checker.IsNil)
	c.Assert(res.StatusCode, checker.Equals, http.StatusOK)

	out, err := request.ReadBody(body)
	require.NoError(c, err)
	assert.Contains(c, string(out), "Successfully built")
}

type buildLine struct {
	Stream string
	Aux    struct {
		ID string
	}
}

func getImageIDsFromBuild(c *check.C, output []byte) []string {
	ids := []string{}
	for _, line := range bytes.Split(output, []byte("\n")) {
		if len(line) == 0 {
			continue
		}
		entry := buildLine{}
		require.NoError(c, json.Unmarshal(line, &entry))
		if entry.Aux.ID != "" {
			ids = append(ids, entry.Aux.ID)
		}
	}
	return ids
}