From 5894bc1abf8186802d360d20739b57bfffed51df Mon Sep 17 00:00:00 2001 From: Ian Campbell Date: Thu, 6 Apr 2017 13:33:56 +0100 Subject: [PATCH] Add `docker build --iidfile=FILE` This is synonymous with `docker run --cidfile=FILE` and writes the digest of the newly built image to the named file. This is intended to be used by build systems which want to avoid tagging (perhaps because they are in CI or otherwise want to avoid fixed names which can clash) by enabling e.g. Makefile constructs like: image.id: Dockerfile docker build --iidfile=image.id . do-some-more-stuff: image.id do-stuff-with = v1.29 the caller will now see a `JSONMessage` with the `Aux` field containing a `types.BuildResult` in the output stream for each image/layer produced during the build, with the final one being the end product. Having all of the intermediate images might be interesting in some cases. In silent mode (with `-q`) there is no change, on success the only output will be the resulting image digest as it was previosuly. There was no wrapper to just output an Aux section without enclosing it in a Progress, so add one here. Added some tests to integration cli tests. Signed-off-by: Ian Campbell --- api/server/router/build/build_routes.go | 17 +++++++-- api/types/backend/build.go | 2 + api/types/types.go | 5 +++ builder/dockerfile/builder.go | 25 +++++++++++++ cli/command/image/build.go | 32 +++++++++++++++- docs/api/version-history.md | 1 + docs/reference/commandline/build.md | 1 + integration-cli/docker_cli_build_test.go | 47 ++++++++++++++++++++++++ man/docker-build.1.md | 4 ++ pkg/jsonmessage/jsonmessage.go | 2 +- pkg/streamformatter/streamformatter.go | 25 +++++++++++++ 11 files changed, 155 insertions(+), 6 deletions(-) diff --git a/api/server/router/build/build_routes.go b/api/server/router/build/build_routes.go index d927674436..c90b5965b0 100644 --- a/api/server/router/build/build_routes.go +++ b/api/server/router/build/build_routes.go @@ -132,7 +132,10 @@ func newImageBuildOptions(ctx context.Context, r *http.Request) (*types.ImageBui } func (br *buildRouter) postBuild(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error { - var notVerboseBuffer = bytes.NewBuffer(nil) + var ( + notVerboseBuffer = bytes.NewBuffer(nil) + version = httputils.VersionFromContext(ctx) + ) w.Header().Set("Content-Type", "application/json") @@ -177,10 +180,12 @@ func (br *buildRouter) postBuild(ctx context.Context, w http.ResponseWriter, r * return progress.NewProgressReader(in, progressOutput, r.ContentLength, "Downloading context", buildOptions.RemoteContext) } + wantAux := versions.GreaterThanOrEqualTo(version, "1.30") + imgID, err := br.backend.Build(ctx, backend.BuildConfig{ Source: r.Body, Options: buildOptions, - ProgressWriter: buildProgressWriter(out, createProgressReader), + ProgressWriter: buildProgressWriter(out, wantAux, createProgressReader), }) if err != nil { return errf(err) @@ -221,13 +226,19 @@ func (s *syncWriter) Write(b []byte) (count int, err error) { return } -func buildProgressWriter(out io.Writer, createProgressReader func(io.ReadCloser) io.ReadCloser) backend.ProgressWriter { +func buildProgressWriter(out io.Writer, wantAux bool, createProgressReader func(io.ReadCloser) io.ReadCloser) backend.ProgressWriter { out = &syncWriter{w: out} + var aux *streamformatter.AuxFormatter + if wantAux { + aux = &streamformatter.AuxFormatter{Writer: out} + } + return backend.ProgressWriter{ Output: out, StdoutFormatter: streamformatter.NewStdoutWriter(out), StderrFormatter: streamformatter.NewStderrWriter(out), + AuxFormatter: aux, ProgressReaderFunc: createProgressReader, } } diff --git a/api/types/backend/build.go b/api/types/backend/build.go index adbd6b344a..01fcbd1a73 100644 --- a/api/types/backend/build.go +++ b/api/types/backend/build.go @@ -4,6 +4,7 @@ import ( "io" "github.com/docker/docker/api/types" + "github.com/docker/docker/pkg/streamformatter" ) // ProgressWriter is a data object to transport progress streams to the client @@ -11,6 +12,7 @@ type ProgressWriter struct { Output io.Writer StdoutFormatter io.Writer StderrFormatter io.Writer + AuxFormatter *streamformatter.AuxFormatter ProgressReaderFunc func(io.ReadCloser) io.ReadCloser } diff --git a/api/types/types.go b/api/types/types.go index 9493bd95e2..75aaab157d 100644 --- a/api/types/types.go +++ b/api/types/types.go @@ -530,3 +530,8 @@ type PushResult struct { Digest string Size int } + +// BuildResult contains the image id of a successful build +type BuildResult struct { + ID string +} diff --git a/builder/dockerfile/builder.go b/builder/dockerfile/builder.go index d484682d53..917e06575e 100644 --- a/builder/dockerfile/builder.go +++ b/builder/dockerfile/builder.go @@ -15,6 +15,7 @@ import ( "github.com/docker/docker/builder/dockerfile/command" "github.com/docker/docker/builder/dockerfile/parser" "github.com/docker/docker/builder/remotecontext" + "github.com/docker/docker/pkg/streamformatter" "github.com/docker/docker/pkg/stringid" "github.com/pkg/errors" "golang.org/x/net/context" @@ -89,6 +90,7 @@ type Builder struct { Stdout io.Writer Stderr io.Writer + Aux *streamformatter.AuxFormatter Output io.Writer docker builder.Backend @@ -114,6 +116,7 @@ func newBuilder(clientCtx context.Context, options builderOptions) *Builder { options: config, Stdout: options.ProgressWriter.StdoutFormatter, Stderr: options.ProgressWriter.StderrFormatter, + Aux: options.ProgressWriter.AuxFormatter, Output: options.ProgressWriter.Output, docker: options.Backend, tmpContainers: map[string]struct{}{}, @@ -161,6 +164,13 @@ func (b *Builder) build(source builder.Source, dockerfile *parser.Result) (*buil return &builder.Result{ImageID: dispatchState.imageID, FromImage: dispatchState.baseImage}, nil } +func emitImageID(aux *streamformatter.AuxFormatter, state *dispatchState) error { + if aux == nil || state.imageID == "" { + return nil + } + return aux.Emit(types.BuildResult{ID: state.imageID}) +} + func (b *Builder) dispatchDockerfileWithCancellation(dockerfile *parser.Result) (*dispatchState, error) { shlex := NewShellLex(dockerfile.EscapeToken) state := newDispatchState() @@ -176,6 +186,15 @@ func (b *Builder) dispatchDockerfileWithCancellation(dockerfile *parser.Result) // Not cancelled yet, keep going... } + // If this is a FROM and we have a previous image then + // emit an aux message for that image since it is the + // end of the previous stage + if n.Value == command.From { + if err := emitImageID(b.Aux, state); err != nil { + return nil, err + } + } + if n.Value == command.From && state.isCurrentStage(b.options.Target) { break } @@ -198,6 +217,12 @@ func (b *Builder) dispatchDockerfileWithCancellation(dockerfile *parser.Result) b.clearTmp() } } + + // Emit a final aux message for the final image + if err := emitImageID(b.Aux, state); err != nil { + return nil, err + } + return state, nil } diff --git a/cli/command/image/build.go b/cli/command/image/build.go index cdb116f169..de12afdad3 100644 --- a/cli/command/image/build.go +++ b/cli/command/image/build.go @@ -4,6 +4,7 @@ import ( "archive/tar" "bufio" "bytes" + "encoding/json" "fmt" "io" "io/ioutil" @@ -65,6 +66,7 @@ type buildOptions struct { networkMode string squash bool target string + imageIDFile string } // NewBuildCommand creates a new `docker build` command @@ -117,6 +119,7 @@ func NewBuildCommand(dockerCli *command.DockerCli) *cobra.Command { flags.SetAnnotation("network", "version", []string{"1.25"}) flags.Var(&options.extraHosts, "add-host", "Add a custom host-to-IP mapping (host:ip)") flags.StringVar(&options.target, "target", "", "Set the target build stage to build.") + flags.StringVar(&options.imageIDFile, "iidfile", "", "Write the image ID to the file") command.AddTrustVerificationFlags(flags) @@ -162,6 +165,12 @@ func runBuild(dockerCli *command.DockerCli, options buildOptions) error { progBuff = bytes.NewBuffer(nil) buildBuff = bytes.NewBuffer(nil) } + if options.imageIDFile != "" { + // Avoid leaving a stale file if we eventually fail + if err := os.Remove(options.imageIDFile); err != nil && !os.IsNotExist(err) { + return errors.Wrap(err, "Removing image ID file") + } + } if options.dockerfileName == "-" { if specifiedContext == "-" { @@ -316,7 +325,17 @@ func runBuild(dockerCli *command.DockerCli, options buildOptions) error { } defer response.Body.Close() - err = jsonmessage.DisplayJSONMessagesStream(response.Body, buildBuff, dockerCli.Out().FD(), dockerCli.Out().IsTerminal(), nil) + imageID := "" + aux := func(auxJSON *json.RawMessage) { + var result types.BuildResult + if err := json.Unmarshal(*auxJSON, &result); err != nil { + fmt.Fprintf(dockerCli.Err(), "Failed to parse aux message: %s", err) + } else { + imageID = result.ID + } + } + + err = jsonmessage.DisplayJSONMessagesStream(response.Body, buildBuff, dockerCli.Out().FD(), dockerCli.Out().IsTerminal(), aux) if err != nil { if jerr, ok := err.(*jsonmessage.JSONError); ok { // If no error code is set, default to 1 @@ -344,9 +363,18 @@ func runBuild(dockerCli *command.DockerCli, options buildOptions) error { // Everything worked so if -q was provided the output from the daemon // should be just the image ID and we'll print that to stdout. if options.quiet { - fmt.Fprintf(dockerCli.Out(), "%s", buildBuff) + imageID = fmt.Sprintf("%s", buildBuff) + fmt.Fprintf(dockerCli.Out(), imageID) } + if options.imageIDFile != "" { + if imageID == "" { + return errors.Errorf("Server did not provide an image ID. Cannot write %s", options.imageIDFile) + } + if err := ioutil.WriteFile(options.imageIDFile, []byte(imageID), 0666); err != nil { + return err + } + } if command.IsTrusted() { // Since the build was successful, now we must tag any of the resolved // images from the above Dockerfile rewrite. diff --git a/docs/api/version-history.md b/docs/api/version-history.md index 85bbcf1e5d..e501c58658 100644 --- a/docs/api/version-history.md +++ b/docs/api/version-history.md @@ -18,6 +18,7 @@ keywords: "API, Docker, rcli, REST, documentation" [Docker Engine API v1.30](https://docs.docker.com/engine/api/v1.30/) documentation * `GET /info` now returns the list of supported logging drivers, including plugins. +* `POST /build/` now (when not silent) produces an `Aux` message in the JSON output stream with payload `types.BuildResult` for each image produced. The final such message will reference the image resulting from the build. ## v1.29 API changes diff --git a/docs/reference/commandline/build.md b/docs/reference/commandline/build.md index 697c63647d..d228853472 100644 --- a/docs/reference/commandline/build.md +++ b/docs/reference/commandline/build.md @@ -35,6 +35,7 @@ Options: -f, --file string Name of the Dockerfile (Default is 'PATH/Dockerfile') --force-rm Always remove intermediate containers --help Print usage + --iidfile string Write the image ID to the file --isolation string Container isolation technology --label value Set metadata for an image (default []) -m, --memory string Memory limit diff --git a/integration-cli/docker_cli_build_test.go b/integration-cli/docker_cli_build_test.go index 701572908b..754a0c0f05 100644 --- a/integration-cli/docker_cli_build_test.go +++ b/integration-cli/docker_cli_build_test.go @@ -28,6 +28,7 @@ import ( "github.com/docker/docker/pkg/testutil" icmd "github.com/docker/docker/pkg/testutil/cmd" "github.com/go-check/check" + "github.com/opencontainers/go-digest" ) func (s *DockerSuite) TestBuildJSONEmptyRun(c *check.C) { @@ -6398,3 +6399,49 @@ CMD echo foo out, _ := dockerCmd(c, "inspect", "--format", "{{ json .Config.Cmd }}", "build2") c.Assert(strings.TrimSpace(out), checker.Equals, `["/bin/sh","-c","echo foo"]`) } + +func (s *DockerSuite) TestBuildIidFile(c *check.C) { + tmpDir, err := ioutil.TempDir("", "TestBuildIidFile") + if err != nil { + c.Fatal(err) + } + defer os.RemoveAll(tmpDir) + tmpIidFile := filepath.Join(tmpDir, "iid") + + name := "testbuildiidfile" + // Use a Dockerfile with multiple stages to ensure we get the last one + cli.BuildCmd(c, name, + build.WithDockerfile(`FROM `+minimalBaseImage()+` AS stage1 +ENV FOO FOO +FROM `+minimalBaseImage()+` +ENV BAR BAZ`), + cli.WithFlags("--iidfile", tmpIidFile)) + + id, err := ioutil.ReadFile(tmpIidFile) + c.Assert(err, check.IsNil) + d, err := digest.Parse(string(id)) + c.Assert(err, check.IsNil) + c.Assert(d.String(), checker.Equals, getIDByName(c, name)) +} + +func (s *DockerSuite) TestBuildIidFileCleanupOnFail(c *check.C) { + tmpDir, err := ioutil.TempDir("", "TestBuildIidFileCleanupOnFail") + if err != nil { + c.Fatal(err) + } + defer os.RemoveAll(tmpDir) + tmpIidFile := filepath.Join(tmpDir, "iid") + + err = ioutil.WriteFile(tmpIidFile, []byte("Dummy"), 0666) + c.Assert(err, check.IsNil) + + cli.Docker(cli.Build("testbuildiidfilecleanuponfail"), + build.WithDockerfile(`FROM `+minimalBaseImage()+` + RUN /non/existing/command`), + cli.WithFlags("--iidfile", tmpIidFile)).Assert(c, icmd.Expected{ + ExitCode: 1, + }) + _, err = os.Stat(tmpIidFile) + c.Assert(err, check.NotNil) + c.Assert(os.IsNotExist(err), check.Equals, true) +} diff --git a/man/docker-build.1.md b/man/docker-build.1.md index b650fc3aa2..59eba003a0 100644 --- a/man/docker-build.1.md +++ b/man/docker-build.1.md @@ -11,6 +11,7 @@ docker-build - Build an image from a Dockerfile [**--cpu-shares**[=*0*]] [**--cgroup-parent**[=*CGROUP-PARENT*]] [**--help**] +[**--iidfile**[=*CIDFILE*]] [**-f**|**--file**[=*PATH/Dockerfile*]] [**-squash**] *Experimental* [**--force-rm**] @@ -104,6 +105,9 @@ option can be set multiple times. **--no-cache**=*true*|*false* Do not use cache when building the image. The default is *false*. +**--iidfile**="" + Write the image ID to the file + **--help** Print usage statement diff --git a/pkg/jsonmessage/jsonmessage.go b/pkg/jsonmessage/jsonmessage.go index c3b1371cde..2b8e98c429 100644 --- a/pkg/jsonmessage/jsonmessage.go +++ b/pkg/jsonmessage/jsonmessage.go @@ -109,7 +109,7 @@ type JSONMessage struct { TimeNano int64 `json:"timeNano,omitempty"` Error *JSONError `json:"errorDetail,omitempty"` ErrorMessage string `json:"error,omitempty"` //deprecated - // Aux contains out-of-band data, such as digests for push signing. + // Aux contains out-of-band data, such as digests for push signing and image id after building. Aux *json.RawMessage `json:"aux,omitempty"` } diff --git a/pkg/streamformatter/streamformatter.go b/pkg/streamformatter/streamformatter.go index fa79828172..48ba65503c 100644 --- a/pkg/streamformatter/streamformatter.go +++ b/pkg/streamformatter/streamformatter.go @@ -132,3 +132,28 @@ func (out *progressOutput) WriteProgress(prog progress.Progress) error { return nil } + +// AuxFormatter is a streamFormatter that writes aux progress messages +type AuxFormatter struct { + io.Writer +} + +// Emit emits the given interface as an aux progress message +func (sf *AuxFormatter) Emit(aux interface{}) error { + auxJSONBytes, err := json.Marshal(aux) + if err != nil { + return err + } + auxJSON := new(json.RawMessage) + *auxJSON = auxJSONBytes + msgJSON, err := json.Marshal(&jsonmessage.JSONMessage{Aux: auxJSON}) + if err != nil { + return err + } + msgJSON = appendNewline(msgJSON) + n, err := sf.Writer.Write(msgJSON) + if n != len(msgJSON) { + return io.ErrShortWrite + } + return err +}