beenull/moby
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

LCOW: Builder plumbing

Browse source 
Signed-off-by: John Howard <jhoward@microsoft.com>
This commit is contained in:
John Howard 2017-05-17 17:08:01 -07:00
parent a1fe1dc791
commit 08252bc963
13 changed files with 153 additions and 33 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
api/types/backend/build.go
View file

@ -28,4 +28,5 @@ type GetImageAndLayerOptions struct {
ForcePull bool
AuthConfig map[string]types.AuthConfig
Output io.Writer
Platform string
}

4
api/types/client.go
View file

@ -178,6 +178,10 @@ type ImageBuildOptions struct {
SecurityOpt []string
ExtraHosts []string // List of extra hosts
Target string
// TODO @jhowardmsft LCOW Support: This will require extending to include
// `Platform string`, but is ommited for now as it's hard-coded temporarily
// to avoid API changes.
}
// ImageBuildResponse holds information

1
api/types/configs.go
View file

@ -16,6 +16,7 @@ type ContainerCreateConfig struct {
HostConfig *container.HostConfig
NetworkingConfig *network.NetworkingConfig
AdjustCPUShares bool
Platform string
}
// ContainerRmConfig holds arguments for the container remove

45
builder/dockerfile/builder.go
View file

@ -5,6 +5,7 @@ import (
"fmt"
"io"
"io/ioutil"
"runtime"
"strings"
"github.com/Sirupsen/logrus"
@ -20,6 +21,7 @@ import (
"github.com/docker/docker/pkg/idtools"
"github.com/docker/docker/pkg/streamformatter"
"github.com/docker/docker/pkg/stringid"
"github.com/docker/docker/pkg/system"
"github.com/pkg/errors"
"golang.org/x/net/context"
"golang.org/x/sync/syncmap"
@ -73,13 +75,24 @@ func (bm *BuildManager) Build(ctx context.Context, config backend.BuildConfig) (
}()
}
// TODO @jhowardmsft LCOW support - this will require rework to allow both linux and Windows simultaneously.
// This is an interim solution to hardcode to linux if LCOW is turned on.
if dockerfile.Platform == "" {
dockerfile.Platform = runtime.GOOS
if dockerfile.Platform == "windows" && system.LCOWSupported() {
dockerfile.Platform = "linux"
}
}
builderOptions := builderOptions{
Options: config.Options,
ProgressWriter: config.ProgressWriter,
Backend: bm.backend,
PathCache: bm.pathCache,
Archiver: bm.archiver,
Platform: dockerfile.Platform,
}
return newBuilder(ctx, builderOptions).build(source, dockerfile)
}
@ -90,6 +103,7 @@ type builderOptions struct {
ProgressWriter backend.ProgressWriter
PathCache pathCache
Archiver *archive.Archiver
Platform string
}
// Builder is a Dockerfile builder
@ -113,14 +127,32 @@ type Builder struct {
pathCache pathCache
containerManager *containerManager
imageProber ImageProber
// TODO @jhowardmft LCOW Support. This will be moved to options at a later
// stage, however that cannot be done now as it affects the public API
// if it were.
platform string
}
// newBuilder creates a new Dockerfile builder from an optional dockerfile and a Options.
// TODO @jhowardmsft LCOW support: Eventually platform can be moved into the builder
// options, however, that would be an API change as it shares types.ImageBuildOptions.
func newBuilder(clientCtx context.Context, options builderOptions) *Builder {
config := options.Options
if config == nil {
config = new(types.ImageBuildOptions)
}
// @jhowardmsft LCOW Support. For the time being, this is interim. Eventually
// will be moved to types.ImageBuildOptions, but it can't for now as that would
// be an API change.
if options.Platform == "" {
options.Platform = runtime.GOOS
}
if options.Platform == "windows" && system.LCOWSupported() {
options.Platform = "linux"
}
b := &Builder{
clientCtx: clientCtx,
options: config,
@ -136,7 +168,9 @@ func newBuilder(clientCtx context.Context, options builderOptions) *Builder {
pathCache: options.PathCache,
imageProber: newImageProber(options.Backend, config.CacheFrom, config.NoCache),
containerManager: newContainerManager(options.Backend),
platform: options.Platform,
}
return b
}
@ -267,6 +301,17 @@ func BuildFromConfig(config *container.Config, changes []string) (*container.Con
return nil, err
}
// TODO @jhowardmsft LCOW support. For now, if LCOW enabled, switch to linux.
// Also explicitly set the platform. Ultimately this will be in the builder
// options, but we can't do that yet as it would change the API.
if dockerfile.Platform == "" {
dockerfile.Platform = runtime.GOOS
}
if dockerfile.Platform == "windows" && system.LCOWSupported() {
dockerfile.Platform = "linux"
}
b.platform = dockerfile.Platform
// ensure that the commands are valid
for _, n := range dockerfile.AST.Children {
if !validCommitCommands[n.Value] {

3
builder/dockerfile/containerbackend.go
View file

@ -28,10 +28,11 @@ func newContainerManager(docker builder.ExecBackend) *containerManager {
}
// Create a container
func (c *containerManager) Create(runConfig *container.Config, hostConfig *container.HostConfig) (container.ContainerCreateCreatedBody, error) {
func (c *containerManager) Create(runConfig *container.Config, hostConfig *container.HostConfig, platform string) (container.ContainerCreateCreatedBody, error) {
container, err := c.backend.ContainerCreate(types.ContainerCreateConfig{
Config: runConfig,
HostConfig: hostConfig,
Platform: platform,
})
if err != nil {
return container, err

7
builder/dockerfile/dispatchers.go
View file

@ -26,6 +26,7 @@ import (
"github.com/docker/docker/image"
"github.com/docker/docker/pkg/jsonmessage"
"github.com/docker/docker/pkg/signal"
"github.com/docker/docker/pkg/system"
"github.com/docker/go-connections/nat"
"github.com/pkg/errors"
)
@ -270,10 +271,12 @@ func (b *Builder) getFromImage(shlex *ShellLex, name string) (builder.Image, err
name = stage.ImageID()
}
// Windows cannot support a container with no base image.
// Windows cannot support a container with no base image unless it is LCOW.
if name == api.NoBaseImageSpecifier {
if runtime.GOOS == "windows" {
return nil, errors.New("Windows does not support FROM scratch")
if b.platform == "windows" || (b.platform != "windows" && !system.LCOWSupported()) {
return nil, errors.New("Windows does not support FROM scratch")
}
}
return scratchImage, nil
}

9
builder/dockerfile/dispatchers_test.go
View file

@ -194,7 +194,7 @@ func TestFromScratch(t *testing.T) {
req := defaultDispatchReq(b, "scratch")
err := from(req)
if runtime.GOOS == "windows" {
if runtime.GOOS == "windows" && !system.LCOWSupported() {
assert.EqualError(t, err, "Windows does not support FROM scratch")
return
}
@ -202,7 +202,12 @@ func TestFromScratch(t *testing.T) {
require.NoError(t, err)
assert.True(t, req.state.hasFromImage())
assert.Equal(t, "", req.state.imageID)
assert.Equal(t, []string{"PATH=" + system.DefaultPathEnv}, req.state.runConfig.Env)
// Windows does not set the default path. TODO @jhowardmsft LCOW support. This will need revisiting as we get further into the implementation
expected := "PATH=" + system.DefaultPathEnv
if runtime.GOOS == "windows" {
expected = ""
}
assert.Equal(t, []string{expected}, req.state.runConfig.Env)
}
func TestFromWithArg(t *testing.T) {

3
builder/dockerfile/imagecontext.go
View file

@ -97,12 +97,15 @@ type imageSources struct {
cache pathCache // TODO: remove
}
// TODO @jhowardmsft LCOW Support: Eventually, platform can be moved to options.Options.Platform,
// and removed from builderOptions, but that can't be done yet as it would affect the API.
func newImageSources(ctx context.Context, options builderOptions) *imageSources {
getAndMount := func(idOrRef string) (builder.Image, builder.ReleaseableLayer, error) {
return options.Backend.GetImageAndReleasableLayer(ctx, idOrRef, backend.GetImageAndLayerOptions{
ForcePull: options.Options.PullParent,
AuthConfig: options.Options.AuthConfigs,
Output: options.ProgressWriter.Output,
Platform: options.Platform,
})
}

4
builder/dockerfile/internals.go
View file

@ -256,13 +256,13 @@ func (b *Builder) probeAndCreate(dispatchState *dispatchState, runConfig *contai
}
// Set a log config to override any default value set on the daemon
hostConfig := &container.HostConfig{LogConfig: defaultLogConfig}
container, err := b.containerManager.Create(runConfig, hostConfig)
container, err := b.containerManager.Create(runConfig, hostConfig, b.platform)
return container.ID, err
}
func (b *Builder) create(runConfig *container.Config) (string, error) {
hostConfig := hostConfigFromOptions(b.options)
container, err := b.containerManager.Create(runConfig, hostConfig)
container, err := b.containerManager.Create(runConfig, hostConfig, b.platform)
if err != nil {
return "", err
}

53
builder/dockerfile/parser/parser.go
View file

@ -7,11 +7,13 @@ import (
"fmt"
"io"
"regexp"
"runtime"
"strconv"
"strings"
"unicode"
"github.com/docker/docker/builder/dockerfile/command"
"github.com/docker/docker/pkg/system"
"github.com/pkg/errors"
)
@ -79,22 +81,28 @@ func (node *Node) AddChild(child *Node, startLine, endLine int) {
}
var (
dispatch map[string]func(string, *Directive) (*Node, map[string]bool, error)
tokenWhitespace = regexp.MustCompile(`[\t\v\f\r ]+`)
tokenEscapeCommand = regexp.MustCompile(`^#[ \t]*escape[ \t]*=[ \t]*(?P<escapechar>.).*$`)
tokenComment = regexp.MustCompile(`^#.*$`)
dispatch map[string]func(string, *Directive) (*Node, map[string]bool, error)
tokenWhitespace = regexp.MustCompile(`[\t\v\f\r ]+`)
tokenEscapeCommand = regexp.MustCompile(`^#[ \t]*escape[ \t]*=[ \t]*(?P<escapechar>.).*$`)
tokenPlatformCommand = regexp.MustCompile(`^#[ \t]*platform[ \t]*=[ \t]*(?P<platform>.*)$`)
tokenComment = regexp.MustCompile(`^#.*$`)
)
// DefaultEscapeToken is the default escape token
const DefaultEscapeToken = '\\'
// DefaultPlatformToken is the platform assumed for the build if not explicitly provided
var DefaultPlatformToken = runtime.GOOS
// Directive is the structure used during a build run to hold the state of
// parsing directives.
type Directive struct {
escapeToken rune // Current escape token
platformToken string // Current platform token
lineContinuationRegex *regexp.Regexp // Current line continuation regex
processingComplete bool // Whether we are done looking for directives
escapeSeen bool // Whether the escape directive has been seen
platformSeen bool // Whether the platform directive has been seen
}
// setEscapeToken sets the default token for escaping characters in a Dockerfile.
@ -107,6 +115,22 @@ func (d *Directive) setEscapeToken(s string) error {
return nil
}
// setPlatformToken sets the default platform for pulling images in a Dockerfile.
func (d *Directive) setPlatformToken(s string) error {
s = strings.ToLower(s)
valid := []string{runtime.GOOS}
if runtime.GOOS == "windows" && system.LCOWSupported() {
valid = append(valid, "linux")
}
for _, item := range valid {
if s == item {
d.platformToken = s
return nil
}
}
return fmt.Errorf("invalid PLATFORM '%s'. Must be one of %v", s, valid)
}
// possibleParserDirective looks for one or more parser directives '# escapeToken=<char>' and
// '# platform=<string>'. Parser directives must precede any builder instruction
// or other comments, and cannot be repeated.
@ -128,6 +152,23 @@ func (d *Directive) possibleParserDirective(line string) error {
}
}
// TODO @jhowardmsft LCOW Support: Eventually this check can be removed,
// but only recognise a platform token if running in LCOW mode.
if runtime.GOOS == "windows" && system.LCOWSupported() {
tpcMatch := tokenPlatformCommand.FindStringSubmatch(strings.ToLower(line))
if len(tpcMatch) != 0 {
for i, n := range tokenPlatformCommand.SubexpNames() {
if n == "platform" {
if d.platformSeen == true {
return errors.New("only one platform parser directive can be used")
}
d.platformSeen = true
return d.setPlatformToken(tpcMatch[i])
}
}
}
}
d.processingComplete = true
return nil
}
@ -136,6 +177,7 @@ func (d *Directive) possibleParserDirective(line string) error {
func NewDefaultDirective() *Directive {
directive := Directive{}
directive.setEscapeToken(string(DefaultEscapeToken))
directive.setPlatformToken(runtime.GOOS)
return &directive
}
@ -200,6 +242,7 @@ func newNodeFromLine(line string, directive *Directive) (*Node, error) {
type Result struct {
AST *Node
EscapeToken rune
Platform string
}
// Parse reads lines from a Reader, parses the lines into an AST and returns
@ -252,7 +295,7 @@ func Parse(rwc io.Reader) (*Result, error) {
}
root.AddChild(child, startLine, currentLine)
}
return &Result{AST: root, EscapeToken: d.escapeToken}, nil
return &Result{AST: root, EscapeToken: d.escapeToken, Platform: d.platformToken}, nil
}
func trimComments(src []byte) []byte {

18
daemon/build.go
View file

@ -2,7 +2,6 @@ package daemon
import (
"io"
"runtime"
"github.com/Sirupsen/logrus"
"github.com/docker/distribution/reference"
@ -110,7 +109,7 @@ func newReleasableLayerForImage(img *image.Image, layerStore layer.Store) (build
}
// TODO: could this use the regular daemon PullImage ?
func (daemon *Daemon) pullForBuilder(ctx context.Context, name string, authConfigs map[string]types.AuthConfig, output io.Writer) (*image.Image, error) {
func (daemon *Daemon) pullForBuilder(ctx context.Context, name string, authConfigs map[string]types.AuthConfig, output io.Writer, platform string) (*image.Image, error) {
ref, err := reference.ParseNormalizedNamed(name)
if err != nil {
return nil, err
@ -129,12 +128,7 @@ func (daemon *Daemon) pullForBuilder(ctx context.Context, name string, authConfi
pullRegistryAuth = &resolvedConfig
}
// TODO @jhowardmsft LCOW Support: For now, use the runtime operating system of the host.
// When it gets to the builder part, this will need revisiting. There would have to be
// some indication from the user either through CLI flag to build, or through an explicit
// mechanism in a dockerfile such as a parser directive extension or an addition to
// the FROM statement syntax.
if err := daemon.pullImageWithReference(ctx, ref, runtime.GOOS, nil, pullRegistryAuth, output); err != nil {
if err := daemon.pullImageWithReference(ctx, ref, platform, nil, pullRegistryAuth, output); err != nil {
return nil, err
}
return daemon.GetImage(name)
@ -153,18 +147,16 @@ func (daemon *Daemon) GetImageAndReleasableLayer(ctx context.Context, refOrID st
image, _ := daemon.GetImage(refOrID)
// TODO: shouldn't we error out if error is different from "not found" ?
if image != nil {
// TODO LCOW @jhowardmsft. For now using runtime.GOOS for this, will need enhancing for platform when porting the builder
layer, err := newReleasableLayerForImage(image, daemon.stores[runtime.GOOS].layerStore)
layer, err := newReleasableLayerForImage(image, daemon.stores[opts.Platform].layerStore)
return image, layer, err
}
}
image, err := daemon.pullForBuilder(ctx, refOrID, opts.AuthConfig, opts.Output)
image, err := daemon.pullForBuilder(ctx, refOrID, opts.AuthConfig, opts.Output, opts.Platform)
if err != nil {
return nil, nil, err
}
// TODO LCOW @jhowardmsft. For now using runtime.GOOS for this, will need enhancing for platform when porting the builder
layer, err := newReleasableLayerForImage(image, daemon.stores[runtime.GOOS].layerStore)
layer, err := newReleasableLayerForImage(image, daemon.stores[opts.Platform].layerStore)
return image, layer, err
}

9
daemon/container.go
View file

@ -4,7 +4,6 @@ import (
"fmt"
"os"
"path/filepath"
"runtime"
"time"
"github.com/docker/docker/api/errors"
@ -112,7 +111,7 @@ func (daemon *Daemon) Register(c *container.Container) {
daemon.idIndex.Add(c.ID)
}
func (daemon *Daemon) newContainer(name string, config *containertypes.Config, hostConfig *containertypes.HostConfig, imgID image.ID, managed bool) (*container.Container, error) {
func (daemon *Daemon) newContainer(name string, platform string, config *containertypes.Config, hostConfig *containertypes.HostConfig, imgID image.ID, managed bool) (*container.Container, error) {
var (
id string
err error
@ -145,10 +144,8 @@ func (daemon *Daemon) newContainer(name string, config *containertypes.Config, h
base.ImageID = imgID
base.NetworkSettings = &network.Settings{IsAnonymousEndpoint: noExplicitName}
base.Name = name
// TODO @jhowardmsft LCOW - Get it from the platform of the container. For now, assume it is the OS of the host
base.Driver = daemon.GraphDriverName(runtime.GOOS)
// TODO @jhowardmsft LCOW - Similarly on this field. To solve this it will need a CLI/REST change in a subsequent PR during LCOW development
base.Platform = runtime.GOOS
base.Driver = daemon.GraphDriverName(platform)
base.Platform = platform
return base, err
}

29
daemon/create.go
View file

@ -19,6 +19,7 @@ import (
"github.com/docker/docker/layer"
"github.com/docker/docker/pkg/idtools"
"github.com/docker/docker/pkg/stringid"
"github.com/docker/docker/pkg/system"
"github.com/docker/docker/runconfig"
"github.com/opencontainers/selinux/go-selinux/label"
)
@ -75,6 +76,16 @@ func (daemon *Daemon) create(params types.ContainerCreateConfig, managed bool) (
err error
)
// TODO: @jhowardmsft LCOW support - at a later point, can remove the hard-coding
// to force the platform to be linux.
// Default the platform if not supplied
if params.Platform == "" {
params.Platform = runtime.GOOS
}
if params.Platform == "windows" && system.LCOWSupported() {
params.Platform = "linux"
}
if params.Config.Image != "" {
img, err = daemon.GetImage(params.Config.Image)
if err != nil {
@ -82,9 +93,23 @@ func (daemon *Daemon) create(params types.ContainerCreateConfig, managed bool) (
}
if runtime.GOOS == "solaris" && img.OS != "solaris " {
return nil, errors.New("Platform on which parent image was created is not Solaris")
return nil, errors.New("platform on which parent image was created is not Solaris")
}
imgID = img.ID()
if runtime.GOOS == "windows" && img.OS == "linux" && !system.LCOWSupported() {
return nil, errors.New("platform on which parent image was created is not Windows")
}
}
// Make sure the platform requested matches the image
if img != nil {
if params.Platform != img.Platform() {
// Ignore this in LCOW mode. @jhowardmsft TODO - This will need revisiting later.
if !(runtime.GOOS == "windows" && system.LCOWSupported()) {
return nil, fmt.Errorf("cannot create a %s container from a %s image", params.Platform, img.Platform())
}
}
}
if err := daemon.mergeAndVerifyConfig(params.Config, img); err != nil {
@ -95,7 +120,7 @@ func (daemon *Daemon) create(params types.ContainerCreateConfig, managed bool) (
return nil, err
}
if container, err = daemon.newContainer(params.Name, params.Config, params.HostConfig, imgID, managed); err != nil {
if container, err = daemon.newContainer(params.Name, params.Platform, params.Config, params.HostConfig, imgID, managed); err != nil {
return nil, err
}
defer func() {