beenull/moby
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Merge pull request #32610 from dnephin/builder-move-tag-and-squash

Browse source 
Extract squash and tagging from the Dockerfile builder
This commit is contained in:
Daniel Nephin 2017-05-01 18:25:36 -04:00 committed by GitHub
commit 73abe0c682
20 changed files with 761 additions and 259 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

70
api/server/backend/build/backend.go Normal file
View file

@ -0,0 +1,70 @@
package build
import (
"fmt"
"github.com/docker/distribution/reference"
"github.com/docker/docker/api/types/backend"
"github.com/docker/docker/builder"
"github.com/docker/docker/builder/dockerfile"
"github.com/docker/docker/image"
"github.com/docker/docker/pkg/stringid"
"github.com/pkg/errors"
"golang.org/x/net/context"
)
// ImageComponent provides an interface for working with images
type ImageComponent interface {
SquashImage(from string, to string) (string, error)
TagImageWithReference(image.ID, reference.Named) error
}
// Backend provides build functionality to the API router
type Backend struct {
manager *dockerfile.BuildManager
imageComponent ImageComponent
}
// NewBackend creates a new build backend from components
func NewBackend(components ImageComponent, builderBackend builder.Backend) *Backend {
manager := dockerfile.NewBuildManager(builderBackend)
return &Backend{imageComponent: components, manager: manager}
}
// Build builds an image from a Source
func (b *Backend) Build(ctx context.Context, config backend.BuildConfig) (string, error) {
options := config.Options
tagger, err := NewTagger(b.imageComponent, config.ProgressWriter.StdoutFormatter, options.Tags)
if err != nil {
return "", err
}
build, err := b.manager.Build(ctx, config)
if err != nil {
return "", err
}
var imageID = build.ImageID
if options.Squash {
if imageID, err = squashBuild(build, b.imageComponent); err != nil {
return "", err
}
}
stdout := config.ProgressWriter.StdoutFormatter
fmt.Fprintf(stdout, "Successfully built %s\n", stringid.TruncateID(imageID))
err = tagger.TagImages(image.ID(imageID))
return imageID, err
}
func squashBuild(build *builder.Result, imageComponent ImageComponent) (string, error) {
var fromID string
if build.FromImage != nil {
fromID = build.FromImage.ImageID()
}
imageID, err := imageComponent.SquashImage(build.ImageID, fromID)
if err != nil {
return "", errors.Wrap(err, "error squashing image")
}
return imageID, nil
}

77
api/server/backend/build/tag.go Normal file
View file

@ -0,0 +1,77 @@
package build
import (
"fmt"
"io"
"github.com/docker/distribution/reference"
"github.com/docker/docker/image"
"github.com/pkg/errors"
)
// Tagger is responsible for tagging an image created by a builder
type Tagger struct {
imageComponent ImageComponent
stdout io.Writer
repoAndTags []reference.Named
}
// NewTagger returns a new Tagger for tagging the images of a build.
// If any of the names are invalid tags an error is returned.
func NewTagger(backend ImageComponent, stdout io.Writer, names []string) (*Tagger, error) {
reposAndTags, err := sanitizeRepoAndTags(names)
if err != nil {
return nil, err
}
return &Tagger{
imageComponent: backend,
stdout: stdout,
repoAndTags: reposAndTags,
}, nil
}
// TagImages creates image tags for the imageID
func (bt *Tagger) TagImages(imageID image.ID) error {
for _, rt := range bt.repoAndTags {
if err := bt.imageComponent.TagImageWithReference(imageID, rt); err != nil {
return err
}
fmt.Fprintf(bt.stdout, "Successfully tagged %s\n", reference.FamiliarString(rt))
}
return nil
}
// sanitizeRepoAndTags parses the raw "t" parameter received from the client
// to a slice of repoAndTag.
// It also validates each repoName and tag.
func sanitizeRepoAndTags(names []string) ([]reference.Named, error) {
var (
repoAndTags []reference.Named
// This map is used for deduplicating the "-t" parameter.
uniqNames = make(map[string]struct{})
)
for _, repo := range names {
if repo == "" {
continue
}
ref, err := reference.ParseNormalizedNamed(repo)
if err != nil {
return nil, err
}
if _, isCanonical := ref.(reference.Canonical); isCanonical {
return nil, errors.New("build tag cannot contain a digest")
}
ref = reference.TagNameOnly(ref)
nameWithTag := ref.String()
if _, exists := uniqNames[nameWithTag]; !exists {
uniqNames[nameWithTag] = struct{}{}
repoAndTags = append(repoAndTags, ref)
}
}
return repoAndTags, nil
}

15
api/server/router/build/backend.go
View file

@ -1,20 +1,17 @@
package build
import (
"io"
"github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/backend"
"golang.org/x/net/context"
)
// Backend abstracts an image builder whose only purpose is to build an image referenced by an imageID.
type Backend interface {
// BuildFromContext builds a Docker image referenced by an imageID string.
//
// Note: Tagging an image should not be done by a Builder, it should instead be done
// by the caller.
//
// Build a Docker image returning the id of the image
// TODO: make this return a reference instead of string
BuildFromContext(ctx context.Context, src io.ReadCloser, buildOptions *types.ImageBuildOptions, pg backend.ProgressWriter) (string, error)
Build(context.Context, backend.BuildConfig) (string, error)
}
type experimentalProvider interface {
HasExperimental() bool
}

7
api/server/router/build/build.go
View file

@ -5,14 +5,13 @@ import "github.com/docker/docker/api/server/router"
// buildRouter is a router to talk with the build controller
type buildRouter struct {
backend Backend
daemon experimentalProvider
routes []router.Route
}
// NewRouter initializes a new build router
func NewRouter(b Backend) router.Router {
r := &buildRouter{
backend: b,
}
func NewRouter(b Backend, d experimentalProvider) router.Router {
r := &buildRouter{backend: b, daemon: d}
r.initRoutes()
return r
}

104
api/server/router/build/build_routes.go
View file

@ -13,6 +13,7 @@ import (
"sync"
"github.com/Sirupsen/logrus"
apierrors "github.com/docker/docker/api/errors"
"github.com/docker/docker/api/server/httputils"
"github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/backend"
@ -22,6 +23,7 @@ import (
"github.com/docker/docker/pkg/progress"
"github.com/docker/docker/pkg/streamformatter"
units "github.com/docker/go-units"
"github.com/pkg/errors"
"golang.org/x/net/context"
)
@ -87,9 +89,6 @@ func newImageBuildOptions(ctx context.Context, r *http.Request) (*types.ImageBui
options.Ulimits = buildUlimits
}
var buildArgs = map[string]*string{}
buildArgsJSON := r.FormValue("buildargs")
// Note that there are two ways a --build-arg might appear in the
// json of the query param:
// "foo":"bar"
@ -102,25 +101,27 @@ func newImageBuildOptions(ctx context.Context, r *http.Request) (*types.ImageBui
// the fact they mentioned it, we need to pass that along to the builder
// so that it can print a warning about "foo" being unused if there is
// no "ARG foo" in the Dockerfile.
buildArgsJSON := r.FormValue("buildargs")
if buildArgsJSON != "" {
var buildArgs = map[string]*string{}
if err := json.Unmarshal([]byte(buildArgsJSON), &buildArgs); err != nil {
return nil, err
}
options.BuildArgs = buildArgs
}
var labels = map[string]string{}
labelsJSON := r.FormValue("labels")
if labelsJSON != "" {
var labels = map[string]string{}
if err := json.Unmarshal([]byte(labelsJSON), &labels); err != nil {
return nil, err
}
options.Labels = labels
}
var cacheFrom = []string{}
cacheFromJSON := r.FormValue("cachefrom")
if cacheFromJSON != "" {
var cacheFrom = []string{}
if err := json.Unmarshal([]byte(cacheFromJSON), &cacheFrom); err != nil {
return nil, err
}
@ -130,33 +131,8 @@ func newImageBuildOptions(ctx context.Context, r *http.Request) (*types.ImageBui
return options, nil
}
type syncWriter struct {
w io.Writer
mu sync.Mutex
}
func (s *syncWriter) Write(b []byte) (count int, err error) {
s.mu.Lock()
count, err = s.w.Write(b)
s.mu.Unlock()
return
}
func (br *buildRouter) postBuild(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
var (
authConfigs = map[string]types.AuthConfig{}
authConfigsEncoded = r.Header.Get("X-Registry-Config")
notVerboseBuffer = bytes.NewBuffer(nil)
)
if authConfigsEncoded != "" {
authConfigsJSON := base64.NewDecoder(base64.URLEncoding, strings.NewReader(authConfigsEncoded))
if err := json.NewDecoder(authConfigsJSON).Decode(&authConfigs); err != nil {
// for a pull it is not an error if no auth was given
// to increase compatibility with the existing api it is defaulting
// to be empty.
}
}
var notVerboseBuffer = bytes.NewBuffer(nil)
w.Header().Set("Content-Type", "application/json")
@ -183,7 +159,12 @@ func (br *buildRouter) postBuild(ctx context.Context, w http.ResponseWriter, r *
if err != nil {
return errf(err)
}
buildOptions.AuthConfigs = authConfigs
buildOptions.AuthConfigs = getAuthConfigs(r.Header)
if buildOptions.Squash && !br.daemon.HasExperimental() {
return apierrors.NewBadRequestError(
errors.New("squash is only supported with experimental mode"))
}
// Currently, only used if context is from a remote url.
// Look at code in DetectContextFromRemoteURL for more information.
@ -199,18 +180,12 @@ func (br *buildRouter) postBuild(ctx context.Context, w http.ResponseWriter, r *
if buildOptions.SuppressOutput {
out = notVerboseBuffer
}
out = &syncWriter{w: out}
stdout := &streamformatter.StdoutFormatter{Writer: out, StreamFormatter: sf}
stderr := &streamformatter.StderrFormatter{Writer: out, StreamFormatter: sf}
pg := backend.ProgressWriter{
Output: out,
StdoutFormatter: stdout,
StderrFormatter: stderr,
ProgressReaderFunc: createProgressReader,
}
imgID, err := br.backend.BuildFromContext(ctx, r.Body, buildOptions, pg)
imgID, err := br.backend.Build(ctx, backend.BuildConfig{
Source: r.Body,
Options: buildOptions,
ProgressWriter: buildProgressWriter(out, sf, createProgressReader),
})
if err != nil {
return errf(err)
}
@ -219,8 +194,47 @@ func (br *buildRouter) postBuild(ctx context.Context, w http.ResponseWriter, r *
// should be just the image ID and we'll print that to stdout.
if buildOptions.SuppressOutput {
stdout := &streamformatter.StdoutFormatter{Writer: output, StreamFormatter: sf}
fmt.Fprintf(stdout, "%s\n", string(imgID))
fmt.Fprintln(stdout, imgID)
}
return nil
}
func getAuthConfigs(header http.Header) map[string]types.AuthConfig {
authConfigs := map[string]types.AuthConfig{}
authConfigsEncoded := header.Get("X-Registry-Config")
if authConfigsEncoded == "" {
return authConfigs
}
authConfigsJSON := base64.NewDecoder(base64.URLEncoding, strings.NewReader(authConfigsEncoded))
// Pulling an image does not error when no auth is provided so to remain
// consistent with the existing api decode errors are ignored
json.NewDecoder(authConfigsJSON).Decode(&authConfigs)
return authConfigs
}
type syncWriter struct {
w io.Writer
mu sync.Mutex
}
func (s *syncWriter) Write(b []byte) (count int, err error) {
s.mu.Lock()
count, err = s.w.Write(b)
s.mu.Unlock()
return
}
func buildProgressWriter(out io.Writer, sf *streamformatter.StreamFormatter, createProgressReader func(io.ReadCloser) io.ReadCloser) backend.ProgressWriter {
out = &syncWriter{w: out}
stdout := &streamformatter.StdoutFormatter{Writer: out, StreamFormatter: sf}
stderr := &streamformatter.StderrFormatter{Writer: out, StreamFormatter: sf}
return backend.ProgressWriter{
Output: out,
StdoutFormatter: stdout,
StderrFormatter: stderr,
ProgressReaderFunc: createProgressReader,
}
}

9
api/types/backend/backend.go
View file

@ -6,7 +6,6 @@ import (
"time"
"github.com/docker/docker/api/types"
"github.com/docker/docker/pkg/streamformatter"
)
// ContainerAttachConfig holds the streams to use when connecting to a container to view logs.
@ -99,11 +98,3 @@ type ContainerCommitConfig struct {
types.ContainerCommitConfig
Changes []string
}
// ProgressWriter is a data object to transport progress streams to the client
type ProgressWriter struct {
Output io.Writer
StdoutFormatter *streamformatter.StdoutFormatter
StderrFormatter *streamformatter.StderrFormatter
ProgressReaderFunc func(io.ReadCloser) io.ReadCloser
}

23
api/types/backend/build.go Normal file
View file

@ -0,0 +1,23 @@
package backend
import (
"io"
"github.com/docker/docker/api/types"
"github.com/docker/docker/pkg/streamformatter"
)
// ProgressWriter is a data object to transport progress streams to the client
type ProgressWriter struct {
Output io.Writer
StdoutFormatter *streamformatter.StdoutFormatter
StderrFormatter *streamformatter.StderrFormatter
ProgressReaderFunc func(io.ReadCloser) io.ReadCloser
}
// BuildConfig is the configuration used by a BuildManager to start a build
type BuildConfig struct {
Source io.ReadCloser
ProgressWriter ProgressWriter
Options *types.ImageBuildOptions
}

16
builder/builder.go
View file

@ -8,11 +8,9 @@ import (
"io"
"time"
"github.com/docker/distribution/reference"
"github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/backend"
"github.com/docker/docker/api/types/container"
"github.com/docker/docker/image"
"golang.org/x/net/context"
)
@ -40,8 +38,6 @@ type Backend interface {
// GetImageOnBuild looks up a Docker image referenced by `name`.
GetImageOnBuild(name string) (Image, error)
// TagImageWithReference tags an image with newTag
TagImageWithReference(image.ID, reference.Named) error
// PullOnBuild tells Docker to pull image referenced by `name`.
PullOnBuild(ctx context.Context, name string, authConfigs map[string]types.AuthConfig, output io.Writer) (Image, error)
// ContainerAttachRaw attaches to container.
@ -69,12 +65,6 @@ type Backend interface {
// TODO: use containerd/fs.changestream instead as a source
CopyOnBuild(containerID string, destPath string, srcRoot string, srcPath string, decompress bool) error
// HasExperimental checks if the backend supports experimental features
HasExperimental() bool
// SquashImage squashes the fs layers from the provided image down to the specified `to` image
SquashImage(from string, to string) (string, error)
// MountImage returns mounted path with rootfs of an image.
MountImage(name string) (string, func() error, error)
}
@ -85,6 +75,12 @@ type Image interface {
RunConfig() *container.Config
}
// Result is the output produced by a Builder
type Result struct {
ImageID string
FromImage Image
}
// ImageCacheBuilder represents a generator for stateful image cache.
type ImageCacheBuilder interface {
// MakeImageCache creates a stateful image cache.

210
builder/dockerfile/builder.go
View file

@ -5,12 +5,9 @@ import (
"fmt"
"io"
"io/ioutil"
"os"
"strings"
"github.com/Sirupsen/logrus"
"github.com/docker/distribution/reference"
apierrors "github.com/docker/docker/api/errors"
"github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/backend"
"github.com/docker/docker/api/types/container"
@ -18,10 +15,10 @@ import (
"github.com/docker/docker/builder/dockerfile/command"
"github.com/docker/docker/builder/dockerfile/parser"
"github.com/docker/docker/builder/remotecontext"
"github.com/docker/docker/image"
"github.com/docker/docker/pkg/stringid"
"github.com/pkg/errors"
"golang.org/x/net/context"
"golang.org/x/sync/syncmap"
)
var validCommitCommands = map[string]bool{
@ -39,6 +36,52 @@ var validCommitCommands = map[string]bool{
var defaultLogConfig = container.LogConfig{Type: "none"}
// BuildManager is shared across all Builder objects
type BuildManager struct {
backend builder.Backend
pathCache pathCache // TODO: make this persistent
}
// NewBuildManager creates a BuildManager
func NewBuildManager(b builder.Backend) *BuildManager {
return &BuildManager{backend: b, pathCache: &syncmap.Map{}}
}
// Build starts a new build from a BuildConfig
func (bm *BuildManager) Build(ctx context.Context, config backend.BuildConfig) (*builder.Result, error) {
if config.Options.Dockerfile == "" {
config.Options.Dockerfile = builder.DefaultDockerfileName
}
source, dockerfile, err := remotecontext.Detect(config)
if err != nil {
return nil, err
}
if source != nil {
defer func() {
if err := source.Close(); err != nil {
logrus.Debugf("[BUILDER] failed to remove temporary context: %v", err)
}
}()
}
builderOptions := builderOptions{
Options: config.Options,
ProgressWriter: config.ProgressWriter,
Backend: bm.backend,
PathCache: bm.pathCache,
}
return newBuilder(ctx, builderOptions).build(source, dockerfile)
}
// builderOptions are the dependencies required by the builder
type builderOptions struct {
Options *types.ImageBuildOptions
Backend builder.Backend
ProgressWriter backend.ProgressWriter
PathCache pathCache
}
// Builder is a Dockerfile builder
// It implements the builder.Backend interface.
type Builder struct {
@ -68,65 +111,25 @@ type Builder struct {
from builder.Image
}
// BuildManager implements builder.Backend and is shared across all Builder objects.
type BuildManager struct {
backend builder.Backend
pathCache *pathCache // TODO: make this persistent
}
// NewBuildManager creates a BuildManager.
func NewBuildManager(b builder.Backend) (bm *BuildManager) {
return &BuildManager{backend: b, pathCache: &pathCache{}}
}
// BuildFromContext builds a new image from a given context.
func (bm *BuildManager) BuildFromContext(ctx context.Context, src io.ReadCloser, buildOptions *types.ImageBuildOptions, pg backend.ProgressWriter) (string, error) {
if buildOptions.Squash && !bm.backend.HasExperimental() {
return "", apierrors.NewBadRequestError(errors.New("squash is only supported with experimental mode"))
}
if buildOptions.Dockerfile == "" {
buildOptions.Dockerfile = builder.DefaultDockerfileName
}
source, dockerfile, err := remotecontext.Detect(ctx, buildOptions.RemoteContext, buildOptions.Dockerfile, src, pg.ProgressReaderFunc)
if err != nil {
return "", err
}
if source != nil {
defer func() {
if err := source.Close(); err != nil {
logrus.Debugf("[BUILDER] failed to remove temporary context: %v", err)
}
}()
}
b, err := NewBuilder(ctx, buildOptions, bm.backend, source)
if err != nil {
return "", err
}
b.imageContexts.cache = bm.pathCache
return b.build(dockerfile, pg.StdoutFormatter, pg.StderrFormatter, pg.Output)
}
// NewBuilder creates a new Dockerfile builder from an optional dockerfile and a Config.
// If dockerfile is nil, the Dockerfile specified by Config.DockerfileName,
// will be read from the Context passed to Build().
func NewBuilder(clientCtx context.Context, config *types.ImageBuildOptions, backend builder.Backend, source builder.Source) (b *Builder, err error) {
// newBuilder creates a new Dockerfile builder from an optional dockerfile and a Options.
func newBuilder(clientCtx context.Context, options builderOptions) *Builder {
config := options.Options
if config == nil {
config = new(types.ImageBuildOptions)
}
b = &Builder{
b := &Builder{
clientCtx: clientCtx,
options: config,
Stdout: os.Stdout,
Stderr: os.Stderr,
docker: backend,
source: source,
Stdout: options.ProgressWriter.StdoutFormatter,
Stderr: options.ProgressWriter.StderrFormatter,
Output: options.ProgressWriter.Output,
docker: options.Backend,
runConfig: new(container.Config),
tmpContainers: map[string]struct{}{},
buildArgs: newBuildArgs(config.BuildArgs),
}
b.imageContexts = &imageContexts{b: b}
return b, nil
b.imageContexts = &imageContexts{b: b, cache: options.PathCache}
return b
}
func (b *Builder) resetImageCache() {
@ -137,83 +140,31 @@ func (b *Builder) resetImageCache() {
b.cacheBusted = false
}
// sanitizeRepoAndTags parses the raw "t" parameter received from the client
// to a slice of repoAndTag.
// It also validates each repoName and tag.
func sanitizeRepoAndTags(names []string) ([]reference.Named, error) {
var (
repoAndTags []reference.Named
// This map is used for deduplicating the "-t" parameter.
uniqNames = make(map[string]struct{})
)
for _, repo := range names {
if repo == "" {
continue
}
ref, err := reference.ParseNormalizedNamed(repo)
if err != nil {
return nil, err
}
if _, isCanonical := ref.(reference.Canonical); isCanonical {
return nil, errors.New("build tag cannot contain a digest")
}
ref = reference.TagNameOnly(ref)
nameWithTag := ref.String()
if _, exists := uniqNames[nameWithTag]; !exists {
uniqNames[nameWithTag] = struct{}{}
repoAndTags = append(repoAndTags, ref)
}
}
return repoAndTags, nil
}
// build runs the Dockerfile builder from a context and a docker object that allows to make calls
// to Docker.
func (b *Builder) build(dockerfile *parser.Result, stdout io.Writer, stderr io.Writer, out io.Writer) (string, error) {
// Build runs the Dockerfile builder by parsing the Dockerfile and executing
// the instructions from the file.
func (b *Builder) build(source builder.Source, dockerfile *parser.Result) (*builder.Result, error) {
defer b.imageContexts.unmount()
b.Stdout = stdout
b.Stderr = stderr
b.Output = out
repoAndTags, err := sanitizeRepoAndTags(b.options.Tags)
if err != nil {
return "", err
}
// TODO: Remove source field from Builder
b.source = source
addNodesForLabelOption(dockerfile.AST, b.options.Labels)
if err := checkDispatchDockerfile(dockerfile.AST); err != nil {
return "", err
return nil, err
}
imageID, err := b.dispatchDockerfileWithCancellation(dockerfile)
if err != nil {
return "", err
return nil, err
}
b.warnOnUnusedBuildArgs()
if imageID == "" {
return "", errors.New("No image was generated. Is your Dockerfile empty?")
return nil, errors.New("No image was generated. Is your Dockerfile empty?")
}
if b.options.Squash {
if imageID, err = b.squashBuild(imageID); err != nil {
return "", err
}
}
fmt.Fprintf(b.Stdout, "Successfully built %s\n", stringid.TruncateID(imageID))
if err := b.tagImages(imageID, repoAndTags); err != nil {
return "", err
}
return imageID, nil
return &builder.Result{ImageID: imageID, FromImage: b.from}, nil
}
func (b *Builder) dispatchDockerfileWithCancellation(dockerfile *parser.Result) (string, error) {
@ -258,19 +209,6 @@ func (b *Builder) dispatchDockerfileWithCancellation(dockerfile *parser.Result)
return imageID, nil
}
func (b *Builder) squashBuild(imageID string) (string, error) {
var fromID string
var err error
if b.from != nil {
fromID = b.from.ImageID()
}
imageID, err = b.docker.SquashImage(imageID, fromID)
if err != nil {
return "", errors.Wrap(err, "error squashing image")
}
return imageID, nil
}
func addNodesForLabelOption(dockerfile *parser.Node, labels map[string]string) {
if len(labels) == 0 {
return
@ -289,17 +227,6 @@ func (b *Builder) warnOnUnusedBuildArgs() {
}
}
func (b *Builder) tagImages(id string, repoAndTags []reference.Named) error {
imageID := image.ID(id)
for _, rt := range repoAndTags {
if err := b.docker.TagImageWithReference(imageID, rt); err != nil {
return err
}
fmt.Fprintf(b.Stdout, "Successfully tagged %s\n", reference.FamiliarString(rt))
}
return nil
}
// hasFromImage returns true if the builder has processed a `FROM <image>` line
// TODO: move to DispatchState
func (b *Builder) hasFromImage() bool {
@ -316,10 +243,7 @@ func (b *Builder) hasFromImage() bool {
//
// TODO: Remove?
func BuildFromConfig(config *container.Config, changes []string) (*container.Config, error) {
b, err := NewBuilder(context.Background(), nil, nil, nil)
if err != nil {
return nil, err
}
b := newBuilder(context.Background(), builderOptions{})
result, err := parser.Parse(bytes.NewBufferString(strings.Join(changes, "\n")))
if err != nil {

2
builder/dockerfile/dispatchers.go
View file

@ -595,7 +595,7 @@ func healthcheck(req dispatchRequest) error {
// to /usr/sbin/nginx. Uses the default shell if not in JSON format.
//
// Handles command processing similar to CMD and RUN, only req.runConfig.Entrypoint
// is initialized at NewBuilder time instead of through argument parsing.
// is initialized at newBuilder time instead of through argument parsing.
//
func entrypoint(req dispatchRequest) error {
if err := req.flags.Parse(); err != nil {

2
builder/dockerfile/evaluator.go
View file

@ -2,7 +2,7 @@
//
// It incorporates a dispatch table based on the parser.Node values (see the
// parser package for more information) that are yielded from the parser itself.
// Calling NewBuilder with the BuildOpts struct can be used to customize the
// Calling newBuilder with the BuildOpts struct can be used to customize the
// experience for execution purposes only. Parsing is controlled in the parser
// package, and this division of responsibility should be respected.
//

37
builder/dockerfile/imagecontext.go
View file

@ -3,7 +3,6 @@ package dockerfile
import (
"strconv"
"strings"
"sync"
"github.com/Sirupsen/logrus"
"github.com/docker/docker/api/types/container"
@ -12,13 +11,18 @@ import (
"github.com/pkg/errors"
)
type pathCache interface {
Load(key interface{}) (value interface{}, ok bool)
Store(key, value interface{})
}
// imageContexts is a helper for stacking up built image rootfs and reusing
// them as contexts
type imageContexts struct {
b *Builder
list []*imageMount
byName map[string]*imageMount
cache *pathCache
cache pathCache
currentName string
}
@ -104,14 +108,14 @@ func (ic *imageContexts) getCache(id, path string) (interface{}, bool) {
if id == "" {
return nil, false
}
return ic.cache.get(id + path)
return ic.cache.Load(id + path)
}
return nil, false
}
func (ic *imageContexts) setCache(id, path string, v interface{}) {
if ic.cache != nil {
ic.cache.set(id+path, v)
ic.cache.Store(id+path, v)
}
}
@ -160,28 +164,3 @@ func (im *imageMount) ImageID() string {
func (im *imageMount) RunConfig() *container.Config {
return im.runConfig
}
type pathCache struct {
mu sync.Mutex
items map[string]interface{}
}
func (c *pathCache) set(k string, v interface{}) {
c.mu.Lock()
if c.items == nil {
c.items = make(map[string]interface{})
}
c.items[k] = v
c.mu.Unlock()
}
func (c *pathCache) get(k string) (interface{}, bool) {
c.mu.Lock()
if c.items == nil {
c.mu.Unlock()
return nil, false
}
v, ok := c.items[k]
c.mu.Unlock()
return v, ok
}

9
builder/dockerfile/internals_test.go
View file

@ -1,10 +1,11 @@
package dockerfile
import (
"context"
"fmt"
"testing"
"github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/backend"
"github.com/docker/docker/api/types/container"
"github.com/docker/docker/builder"
"github.com/docker/docker/builder/remotecontext"
@ -69,7 +70,11 @@ func readAndCheckDockerfile(t *testing.T, testName, contextDir, dockerfilePath,
dockerfilePath = builder.DefaultDockerfileName
}
_, _, err = remotecontext.Detect(context.Background(), "", dockerfilePath, tarStream, nil)
config := backend.BuildConfig{
Options: &types.ImageBuildOptions{Dockerfile: dockerfilePath},
Source: tarStream,
}
_, _, err = remotecontext.Detect(config)
assert.EqualError(t, err, expectedError)
}

11
builder/remotecontext/detect.go
View file

@ -2,7 +2,6 @@ package remotecontext
import (
"bufio"
"context"
"fmt"
"io"
"os"
@ -10,6 +9,7 @@ import (
"strings"
"github.com/Sirupsen/logrus"
"github.com/docker/docker/api/types/backend"
"github.com/docker/docker/builder"
"github.com/docker/docker/builder/dockerfile/parser"
"github.com/docker/docker/builder/dockerignore"
@ -23,14 +23,17 @@ import (
// Detect returns a context and dockerfile from remote location or local
// archive. progressReader is only used if remoteURL is actually a URL
// (not empty, and not a Git endpoint).
func Detect(ctx context.Context, remoteURL string, dockerfilePath string, r io.ReadCloser, progressReader func(in io.ReadCloser) io.ReadCloser) (remote builder.Source, dockerfile *parser.Result, err error) {
func Detect(config backend.BuildConfig) (remote builder.Source, dockerfile *parser.Result, err error) {
remoteURL := config.Options.RemoteContext
dockerfilePath := config.Options.Dockerfile
switch {
case remoteURL == "":
remote, dockerfile, err = newArchiveRemote(r, dockerfilePath)
remote, dockerfile, err = newArchiveRemote(config.Source, dockerfilePath)
case urlutil.IsGitURL(remoteURL):
remote, dockerfile, err = newGitRemote(remoteURL, dockerfilePath)
case urlutil.IsURL(remoteURL):
remote, dockerfile, err = newURLRemote(remoteURL, dockerfilePath, progressReader)
remote, dockerfile, err = newURLRemote(remoteURL, dockerfilePath, config.ProgressWriter.ProgressReaderFunc)
default:
err = fmt.Errorf("remoteURL (%s) could not be recognized as URL", remoteURL)
}

4
cmd/dockerd/daemon.go
View file

@ -14,6 +14,7 @@ import (
"github.com/docker/distribution/uuid"
"github.com/docker/docker/api"
apiserver "github.com/docker/docker/api/server"
buildbackend "github.com/docker/docker/api/server/backend/build"
"github.com/docker/docker/api/server/middleware"
"github.com/docker/docker/api/server/router"
"github.com/docker/docker/api/server/router/build"
@ -25,7 +26,6 @@ import (
swarmrouter "github.com/docker/docker/api/server/router/swarm"
systemrouter "github.com/docker/docker/api/server/router/system"
"github.com/docker/docker/api/server/router/volume"
"github.com/docker/docker/builder/dockerfile"
cliconfig "github.com/docker/docker/cli/config"
"github.com/docker/docker/cli/debug"
cliflags "github.com/docker/docker/cli/flags"
@ -484,7 +484,7 @@ func initRouter(s *apiserver.Server, d *daemon.Daemon, c *cluster.Cluster) {
image.NewRouter(d, decoder),
systemrouter.NewRouter(d, c),
volume.NewRouter(d),
build.NewRouter(dockerfile.NewBuildManager(d)),
build.NewRouter(buildbackend.NewBackend(d, d), d),
swarmrouter.NewRouter(c),
pluginrouter.NewRouter(d.PluginManager()),
}

1
vendor.conf
View file

@ -22,6 +22,7 @@ github.com/stretchr/testify 4d4bfba8f1d1027c4fdbe371823030df51419987
github.com/RackSec/srslog 456df3a81436d29ba874f3590eeeee25d666f8a5
github.com/imdario/mergo 0.2.1
golang.org/x/sync/syncmap de49d9dcd27d4f764488181bea099dfe6179bcf0
#get libnetwork packages
github.com/docker/libnetwork cace103704768d39bd88a23d0df76df125a0e39a

27
vendor/golang.org/x/sync/LICENSE generated vendored Normal file
View file

@ -0,0 +1,27 @@
Copyright (c) 2009 The Go Authors. All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
* Neither the name of Google Inc. nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

22
vendor/golang.org/x/sync/PATENTS generated vendored Normal file
View file

@ -0,0 +1,22 @@
Additional IP Rights Grant (Patents)
"This implementation" means the copyrightable works distributed by
Google as part of the Go project.
Google hereby grants to You a perpetual, worldwide, non-exclusive,
no-charge, royalty-free, irrevocable (except as stated in this section)
patent license to make, have made, use, offer to sell, sell, import,
transfer and otherwise run, modify and propagate the contents of this
implementation of Go, where such license applies only to those patent
claims, both currently owned or controlled by Google and acquired in
the future, licensable by Google that are necessarily infringed by this
implementation of Go. This grant does not include claims that would be
infringed only as a consequence of further modification of this
implementation. If you or your agent or exclusive licensee institute or
order or agree to the institution of patent litigation against any
entity (including a cross-claim or counterclaim in a lawsuit) alleging
that this implementation of Go or any code incorporated within this
implementation of Go constitutes direct or contributory patent
infringement, or inducement of patent infringement, then any patent
rights granted to you under this License for this implementation of Go
shall terminate as of the date such litigation is filed.

2
vendor/golang.org/x/sync/README generated vendored Normal file
View file

@ -0,0 +1,2 @@
This repository provides Go concurrency primitives in addition to the
ones provided by the language and "sync" and "sync/atomic" packages.

372
vendor/golang.org/x/sync/syncmap/map.go generated vendored Normal file
View file

@ -0,0 +1,372 @@
// Copyright 2016 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Package syncmap provides a concurrent map implementation.
// It is a prototype for a proposed addition to the sync package
// in the standard library.
// (https://golang.org/issue/18177)
package syncmap
import (
"sync"
"sync/atomic"
"unsafe"
)
// Map is a concurrent map with amortized-constant-time loads, stores, and deletes.
// It is safe for multiple goroutines to call a Map's methods concurrently.
//
// The zero Map is valid and empty.
//
// A Map must not be copied after first use.
type Map struct {
mu sync.Mutex
// read contains the portion of the map's contents that are safe for
// concurrent access (with or without mu held).
//
// The read field itself is always safe to load, but must only be stored with
// mu held.
//
// Entries stored in read may be updated concurrently without mu, but updating
// a previously-expunged entry requires that the entry be copied to the dirty
// map and unexpunged with mu held.
read atomic.Value // readOnly
// dirty contains the portion of the map's contents that require mu to be
// held. To ensure that the dirty map can be promoted to the read map quickly,
// it also includes all of the non-expunged entries in the read map.
//
// Expunged entries are not stored in the dirty map. An expunged entry in the
// clean map must be unexpunged and added to the dirty map before a new value
// can be stored to it.
//
// If the dirty map is nil, the next write to the map will initialize it by
// making a shallow copy of the clean map, omitting stale entries.
dirty map[interface{}]*entry
// misses counts the number of loads since the read map was last updated that
// needed to lock mu to determine whether the key was present.
//
// Once enough misses have occurred to cover the cost of copying the dirty
// map, the dirty map will be promoted to the read map (in the unamended
// state) and the next store to the map will make a new dirty copy.
misses int
}
// readOnly is an immutable struct stored atomically in the Map.read field.
type readOnly struct {
m map[interface{}]*entry
amended bool // true if the dirty map contains some key not in m.
}
// expunged is an arbitrary pointer that marks entries which have been deleted
// from the dirty map.
var expunged = unsafe.Pointer(new(interface{}))
// An entry is a slot in the map corresponding to a particular key.
type entry struct {
// p points to the interface{} value stored for the entry.
//
// If p == nil, the entry has been deleted and m.dirty == nil.
//
// If p == expunged, the entry has been deleted, m.dirty != nil, and the entry
// is missing from m.dirty.
//
// Otherwise, the entry is valid and recorded in m.read.m[key] and, if m.dirty
// != nil, in m.dirty[key].
//
// An entry can be deleted by atomic replacement with nil: when m.dirty is
// next created, it will atomically replace nil with expunged and leave
// m.dirty[key] unset.
//
// An entry's associated value can be updated by atomic replacement, provided
// p != expunged. If p == expunged, an entry's associated value can be updated
// only after first setting m.dirty[key] = e so that lookups using the dirty
// map find the entry.
p unsafe.Pointer // *interface{}
}
func newEntry(i interface{}) *entry {
return &entry{p: unsafe.Pointer(&i)}
}
// Load returns the value stored in the map for a key, or nil if no
// value is present.
// The ok result indicates whether value was found in the map.
func (m *Map) Load(key interface{}) (value interface{}, ok bool) {
read, _ := m.read.Load().(readOnly)
e, ok := read.m[key]
if !ok && read.amended {
m.mu.Lock()
// Avoid reporting a spurious miss if m.dirty got promoted while we were
// blocked on m.mu. (If further loads of the same key will not miss, it's
// not worth copying the dirty map for this key.)
read, _ = m.read.Load().(readOnly)
e, ok = read.m[key]
if !ok && read.amended {
e, ok = m.dirty[key]
// Regardless of whether the entry was present, record a miss: this key
// will take the slow path until the dirty map is promoted to the read
// map.
m.missLocked()
}
m.mu.Unlock()
}
if !ok {
return nil, false
}
return e.load()
}
func (e *entry) load() (value interface{}, ok bool) {
p := atomic.LoadPointer(&e.p)
if p == nil || p == expunged {
return nil, false
}
return *(*interface{})(p), true
}
// Store sets the value for a key.
func (m *Map) Store(key, value interface{}) {
read, _ := m.read.Load().(readOnly)
if e, ok := read.m[key]; ok && e.tryStore(&value) {
return
}
m.mu.Lock()
read, _ = m.read.Load().(readOnly)
if e, ok := read.m[key]; ok {
if e.unexpungeLocked() {
// The entry was previously expunged, which implies that there is a
// non-nil dirty map and this entry is not in it.
m.dirty[key] = e
}
e.storeLocked(&value)
} else if e, ok := m.dirty[key]; ok {
e.storeLocked(&value)
} else {
if !read.amended {
// We're adding the first new key to the dirty map.
// Make sure it is allocated and mark the read-only map as incomplete.
m.dirtyLocked()
m.read.Store(readOnly{m: read.m, amended: true})
}
m.dirty[key] = newEntry(value)
}
m.mu.Unlock()
}
// tryStore stores a value if the entry has not been expunged.
//
// If the entry is expunged, tryStore returns false and leaves the entry
// unchanged.
func (e *entry) tryStore(i *interface{}) bool {
p := atomic.LoadPointer(&e.p)
if p == expunged {
return false
}
for {
if atomic.CompareAndSwapPointer(&e.p, p, unsafe.Pointer(i)) {
return true
}
p = atomic.LoadPointer(&e.p)
if p == expunged {
return false
}
}
}
// unexpungeLocked ensures that the entry is not marked as expunged.
//
// If the entry was previously expunged, it must be added to the dirty map
// before m.mu is unlocked.
func (e *entry) unexpungeLocked() (wasExpunged bool) {
return atomic.CompareAndSwapPointer(&e.p, expunged, nil)
}
// storeLocked unconditionally stores a value to the entry.
//
// The entry must be known not to be expunged.
func (e *entry) storeLocked(i *interface{}) {
atomic.StorePointer(&e.p, unsafe.Pointer(i))
}
// LoadOrStore returns the existing value for the key if present.
// Otherwise, it stores and returns the given value.
// The loaded result is true if the value was loaded, false if stored.
func (m *Map) LoadOrStore(key, value interface{}) (actual interface{}, loaded bool) {
// Avoid locking if it's a clean hit.
read, _ := m.read.Load().(readOnly)
if e, ok := read.m[key]; ok {
actual, loaded, ok := e.tryLoadOrStore(value)
if ok {
return actual, loaded
}
}
m.mu.Lock()
read, _ = m.read.Load().(readOnly)
if e, ok := read.m[key]; ok {
if e.unexpungeLocked() {
m.dirty[key] = e
}
actual, loaded, _ = e.tryLoadOrStore(value)
} else if e, ok := m.dirty[key]; ok {
actual, loaded, _ = e.tryLoadOrStore(value)
m.missLocked()
} else {
if !read.amended {
// We're adding the first new key to the dirty map.
// Make sure it is allocated and mark the read-only map as incomplete.
m.dirtyLocked()
m.read.Store(readOnly{m: read.m, amended: true})
}
m.dirty[key] = newEntry(value)
actual, loaded = value, false
}
m.mu.Unlock()
return actual, loaded
}
// tryLoadOrStore atomically loads or stores a value if the entry is not
// expunged.
//
// If the entry is expunged, tryLoadOrStore leaves the entry unchanged and
// returns with ok==false.
func (e *entry) tryLoadOrStore(i interface{}) (actual interface{}, loaded, ok bool) {
p := atomic.LoadPointer(&e.p)
if p == expunged {
return nil, false, false
}
if p != nil {
return *(*interface{})(p), true, true
}
// Copy the interface after the first load to make this method more amenable
// to escape analysis: if we hit the "load" path or the entry is expunged, we
// shouldn't bother heap-allocating.
ic := i
for {
if atomic.CompareAndSwapPointer(&e.p, nil, unsafe.Pointer(&ic)) {
return i, false, true
}
p = atomic.LoadPointer(&e.p)
if p == expunged {
return nil, false, false
}
if p != nil {
return *(*interface{})(p), true, true
}
}
}
// Delete deletes the value for a key.
func (m *Map) Delete(key interface{}) {
read, _ := m.read.Load().(readOnly)
e, ok := read.m[key]
if !ok && read.amended {
m.mu.Lock()
read, _ = m.read.Load().(readOnly)
e, ok = read.m[key]
if !ok && read.amended {
delete(m.dirty, key)
}
m.mu.Unlock()
}
if ok {
e.delete()
}
}
func (e *entry) delete() (hadValue bool) {
for {
p := atomic.LoadPointer(&e.p)
if p == nil || p == expunged {
return false
}
if atomic.CompareAndSwapPointer(&e.p, p, nil) {
return true
}
}
}
// Range calls f sequentially for each key and value present in the map.
// If f returns false, range stops the iteration.
//
// Range does not necessarily correspond to any consistent snapshot of the Map's
// contents: no key will be visited more than once, but if the value for any key
// is stored or deleted concurrently, Range may reflect any mapping for that key
// from any point during the Range call.
//
// Range may be O(N) with the number of elements in the map even if f returns
// false after a constant number of calls.
func (m *Map) Range(f func(key, value interface{}) bool) {
// We need to be able to iterate over all of the keys that were already
// present at the start of the call to Range.
// If read.amended is false, then read.m satisfies that property without
// requiring us to hold m.mu for a long time.
read, _ := m.read.Load().(readOnly)
if read.amended {
// m.dirty contains keys not in read.m. Fortunately, Range is already O(N)
// (assuming the caller does not break out early), so a call to Range
// amortizes an entire copy of the map: we can promote the dirty copy
// immediately!
m.mu.Lock()
read, _ = m.read.Load().(readOnly)
if read.amended {
read = readOnly{m: m.dirty}
m.read.Store(read)
m.dirty = nil
m.misses = 0
}
m.mu.Unlock()
}
for k, e := range read.m {
v, ok := e.load()
if !ok {
continue
}
if !f(k, v) {
break
}
}
}
func (m *Map) missLocked() {
m.misses++
if m.misses < len(m.dirty) {
return
}
m.read.Store(readOnly{m: m.dirty})
m.dirty = nil
m.misses = 0
}
func (m *Map) dirtyLocked() {
if m.dirty != nil {
return
}
read, _ := m.read.Load().(readOnly)
m.dirty = make(map[interface{}]*entry, len(read.m))
for k, e := range read.m {
if !e.tryExpungeLocked() {
m.dirty[k] = e
}
}
}
func (e *entry) tryExpungeLocked() (isExpunged bool) {
p := atomic.LoadPointer(&e.p)
for p == nil {
if atomic.CompareAndSwapPointer(&e.p, nil, expunged) {
return true
}
p = atomic.LoadPointer(&e.p)
}
return p == expunged
}