From 27319da0d260b42acdbc3fd2a639ad3986d1ba64 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Wed, 24 Apr 2013 11:03:01 -0700 Subject: [PATCH 01/27] Add build command --- builder.go | 111 ++++++++++++++++++++++++++++++++++++++++++++++++++++ commands.go | 28 +++++++++++++ 2 files changed, 139 insertions(+) create mode 100644 builder.go diff --git a/builder.go b/builder.go new file mode 100644 index 0000000000..240170ce9b --- /dev/null +++ b/builder.go @@ -0,0 +1,111 @@ +package docker + +import ( + "bufio" + "fmt" + "io" + "strings" +) + +type Builder struct { + runtime *Runtime +} + +func NewBuilder(runtime *Runtime) *Builder { + return &Builder{ + runtime: runtime, + } +} + +func (builder *Builder) run(image *Image, cmd string) (*Container, error) { + // FIXME: pass a NopWriter instead of nil + config, err := ParseRun([]string{"-d", image.Id, "/bin/sh", "-c", cmd}, nil, builder.runtime.capabilities) + if config.Image == "" { + return nil, fmt.Errorf("Image not specified") + } + if len(config.Cmd) == 0 { + return nil, fmt.Errorf("Command not specified") + } + if config.Tty { + return nil, fmt.Errorf("The tty mode is not supported within the builder") + } + + // Create new container + container, err := builder.runtime.Create(config) + if err != nil { + return nil, err + } + if err := container.Start(); err != nil { + return nil, err + } + return container, nil +} + +func (builder *Builder) runCommit(image *Image, cmd string) (*Image, error) { + c, err := builder.run(image, cmd) + if err != nil { + return nil, err + } + if result := c.Wait(); result != 0 { + return nil, fmt.Errorf("!!! '%s' return non-zero exit code '%d'. Aborting.", cmd, result) + } + img, err := builder.runtime.Commit(c.Id, "", "", "", "") + if err != nil { + return nil, err + } + return img, nil +} + +func (builder *Builder) Build(dockerfile io.Reader, stdout io.Writer) error { + var image, base *Image + + file := bufio.NewReader(dockerfile) + for { + line, err := file.ReadString('\n') + if err != nil { + if err == io.EOF { + break + } + return err + } + line = strings.TrimSpace(line) + // Skip comments and empty line + if len(line) == 0 || line[0] == '#' { + continue + } + tmp := strings.SplitN(line, " ", 2) + if len(tmp) != 2 { + return fmt.Errorf("Invalid Dockerfile format") + } + switch tmp[0] { + case "from": + fmt.Fprintf(stdout, "FROM %s\n", tmp[1]) + image, err = builder.runtime.repositories.LookupImage(tmp[1]) + if err != nil { + return err + } + break + case "run": + fmt.Fprintf(stdout, "RUN %s\n", tmp[1]) + if image == nil { + return fmt.Errorf("Please provide a source image with `from` prior to run") + } + base, err = builder.runCommit(image, tmp[1]) + if err != nil { + return err + } + fmt.Fprintf(stdout, "===> %s\n", base.Id) + break + case "copy": + return fmt.Errorf("The copy operator has not yet been implemented") + default: + fmt.Fprintf(stdout, "Skipping unknown op %s\n", tmp[0]) + } + } + if base != nil { + fmt.Fprintf(stdout, "Build finished. image id: %s\n", base.Id) + } else { + fmt.Fprintf(stdout, "An error occured during the build\n") + } + return nil +} diff --git a/commands.go b/commands.go index cdc948e785..7116d919f3 100644 --- a/commands.go +++ b/commands.go @@ -11,6 +11,7 @@ import ( "net/http" "net/url" "path/filepath" + "os" "runtime" "strconv" "strings" @@ -34,6 +35,7 @@ func (srv *Server) Help() string { help := "Usage: docker COMMAND [arg...]\n\nA self-sufficient runtime for linux containers.\n\nCommands:\n" for _, cmd := range [][]string{ {"attach", "Attach to a running container"}, + {"build", "Build a container from Dockerfile"}, {"commit", "Create a new image from a container's changes"}, {"diff", "Inspect changes on a container's filesystem"}, {"export", "Stream the contents of a container as a tar archive"}, @@ -64,6 +66,32 @@ func (srv *Server) Help() string { return help } +func (srv *Server) CmdBuild(stdin io.ReadCloser, stdout rcli.DockerConn, args ...string) error { + stdout.Flush() + cmd := rcli.Subcmd(stdout, "build", "[Dockerfile|-]", "Build a container from Dockerfile") + if err := cmd.Parse(args); err != nil { + return nil + } + dockerfile := cmd.Arg(0) + if dockerfile == "" { + dockerfile = "Dockerfile" + } + + var file io.Reader + + if dockerfile != "-" { + f, err := os.Open(dockerfile) + if err != nil { + return err + } + defer f.Close() + file = f + } else { + file = stdin + } + return NewBuilder(srv.runtime).Build(file, stdout) +} + // 'docker login': login / register a user to registry service. func (srv *Server) CmdLogin(stdin io.ReadCloser, stdout rcli.DockerConn, args ...string) error { // Read a line on raw terminal with support for simple backspace From b8f66c0d14780bd8cb3b8916acc0287fcaf8cafa Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Wed, 24 Apr 2013 12:02:00 -0700 Subject: [PATCH 02/27] Clear the containers/images upon failure --- builder.go | 51 ++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 38 insertions(+), 13 deletions(-) diff --git a/builder.go b/builder.go index 240170ce9b..1b54df81ba 100644 --- a/builder.go +++ b/builder.go @@ -41,23 +41,25 @@ func (builder *Builder) run(image *Image, cmd string) (*Container, error) { return container, nil } -func (builder *Builder) runCommit(image *Image, cmd string) (*Image, error) { - c, err := builder.run(image, cmd) - if err != nil { - return nil, err +func (builder *Builder) clearTmp(containers, images map[string]struct{}) { + for c := range containers { + tmp := builder.runtime.Get(c) + builder.runtime.Destroy(tmp) + Debugf("Removing container %s", c) } - if result := c.Wait(); result != 0 { - return nil, fmt.Errorf("!!! '%s' return non-zero exit code '%d'. Aborting.", cmd, result) + for i := range images { + builder.runtime.graph.Delete(i) + Debugf("Removing image %s", i) } - img, err := builder.runtime.Commit(c.Id, "", "", "", "") - if err != nil { - return nil, err - } - return img, nil } func (builder *Builder) Build(dockerfile io.Reader, stdout io.Writer) error { - var image, base *Image + var ( + image, base *Image + tmpContainers map[string]struct{} = make(map[string]struct{}) + tmpImages map[string]struct{} = make(map[string]struct{}) + ) + defer builder.clearTmp(tmpContainers, tmpImages) file := bufio.NewReader(dockerfile) for { @@ -90,10 +92,26 @@ func (builder *Builder) Build(dockerfile io.Reader, stdout io.Writer) error { if image == nil { return fmt.Errorf("Please provide a source image with `from` prior to run") } - base, err = builder.runCommit(image, tmp[1]) + + // Create the container and start it + c, err := builder.run(image, tmp[1]) if err != nil { return err } + tmpContainers[c.Id] = struct{}{} + + // Wait for it to finish + if result := c.Wait(); result != 0 { + return fmt.Errorf("!!! '%s' return non-zero exit code '%d'. Aborting.", tmp[1], result) + } + + // Commit the container + base, err := builder.runtime.Commit(c.Id, "", "", "", "") + if err != nil { + return err + } + tmpImages[base.Id] = struct{}{} + fmt.Fprintf(stdout, "===> %s\n", base.Id) break case "copy": @@ -103,6 +121,13 @@ func (builder *Builder) Build(dockerfile io.Reader, stdout io.Writer) error { } } if base != nil { + // The build is successful, keep the temporary containers and images + for i := range tmpImages { + delete(tmpImages, i) + } + for i := range tmpContainers { + delete(tmpContainers, i) + } fmt.Fprintf(stdout, "Build finished. image id: %s\n", base.Id) } else { fmt.Fprintf(stdout, "An error occured during the build\n") From 97215ca384373bfa65c08dc05454c14cef493d30 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Wed, 24 Apr 2013 12:31:20 -0700 Subject: [PATCH 03/27] make builder.Run public it now runs only given arguments without sh -c --- builder.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/builder.go b/builder.go index 1b54df81ba..cbc8711aa7 100644 --- a/builder.go +++ b/builder.go @@ -17,9 +17,9 @@ func NewBuilder(runtime *Runtime) *Builder { } } -func (builder *Builder) run(image *Image, cmd string) (*Container, error) { +func (builder *Builder) Run(image *Image, cmd ...string) (*Container, error) { // FIXME: pass a NopWriter instead of nil - config, err := ParseRun([]string{"-d", image.Id, "/bin/sh", "-c", cmd}, nil, builder.runtime.capabilities) + config, err := ParseRun(append([]string{"-d", image.Id}, cmd...), nil, builder.runtime.capabilities) if config.Image == "" { return nil, fmt.Errorf("Image not specified") } @@ -94,7 +94,7 @@ func (builder *Builder) Build(dockerfile io.Reader, stdout io.Writer) error { } // Create the container and start it - c, err := builder.run(image, tmp[1]) + c, err := builder.Run(image, "/bin/sh", "-c", tmp[1]) if err != nil { return err } From 7bccdc0d33d0589792b9b867b13c9e50dcf48358 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Wed, 24 Apr 2013 13:35:57 -0700 Subject: [PATCH 04/27] Add a Builder.Commit method --- builder.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/builder.go b/builder.go index cbc8711aa7..b15a71d621 100644 --- a/builder.go +++ b/builder.go @@ -41,6 +41,10 @@ func (builder *Builder) Run(image *Image, cmd ...string) (*Container, error) { return container, nil } +func (builder *Builder) Commit(container *Container, repository, tag, comment, author string) (*Image, error) { + return builder.runtime.Commit(container.Id, repository, tag, comment, author) +} + func (builder *Builder) clearTmp(containers, images map[string]struct{}) { for c := range containers { tmp := builder.runtime.Get(c) @@ -106,7 +110,7 @@ func (builder *Builder) Build(dockerfile io.Reader, stdout io.Writer) error { } // Commit the container - base, err := builder.runtime.Commit(c.Id, "", "", "", "") + base, err := builder.Commit(c, "", "", "", "") if err != nil { return err } From 97514831128a7bc7bc6a2f740275bf6336ede999 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Wed, 24 Apr 2013 13:37:00 -0700 Subject: [PATCH 05/27] Add insert command in order to insert external files within an image --- commands.go | 43 +++++++++++++++++++++++++++++++++++++++++++ container.go | 12 ++++++++++++ 2 files changed, 55 insertions(+) diff --git a/commands.go b/commands.go index 7116d919f3..21c8b68abc 100644 --- a/commands.go +++ b/commands.go @@ -43,6 +43,7 @@ func (srv *Server) Help() string { {"images", "List images"}, {"import", "Create a new filesystem image from the contents of a tarball"}, {"info", "Display system-wide information"}, + {"insert", "Insert a file in an image"}, {"inspect", "Return low-level information on a container"}, {"kill", "Kill a running container"}, {"login", "Register or Login to the docker registry server"}, @@ -66,6 +67,48 @@ func (srv *Server) Help() string { return help } +func (srv *Server) CmdInsert(stdin io.ReadCloser, stdout rcli.DockerConn, args ...string) error { + stdout.Flush() + cmd := rcli.Subcmd(stdout, "insert", "IMAGE URL PATH", "Insert a file from URL in the IMAGE at PATH") + if err := cmd.Parse(args); err != nil { + return nil + } + if cmd.NArg() != 3 { + cmd.Usage() + return nil + } + imageId := cmd.Arg(0) + url := cmd.Arg(1) + path := cmd.Arg(2) + + img, err := srv.runtime.repositories.LookupImage(imageId) + if err != nil { + return err + } + file, err := Download(url, stdout) + if err != nil { + return err + } + defer file.Body.Close() + + b := NewBuilder(srv.runtime) + c, err := b.Run(img, "echo", "insert", url, path) + if err != nil { + return err + } + + if err := c.Inject(ProgressReader(file.Body, int(file.ContentLength), stdout, "Downloading %v/%v (%v)"), path); err != nil { + return err + } + // FIXME: Handle custom repo, tag comment, author + img, err = b.Commit(c, "", "", img.Comment, img.Author) + if err != nil { + return err + } + fmt.Fprintf(stdout, "%s\n", img) + return nil +} + func (srv *Server) CmdBuild(stdin io.ReadCloser, stdout rcli.DockerConn, args ...string) error { stdout.Flush() cmd := rcli.Subcmd(stdout, "build", "[Dockerfile|-]", "Build a container from Dockerfile") diff --git a/container.go b/container.go index 311e475ac1..ed3bdce94a 100644 --- a/container.go +++ b/container.go @@ -178,6 +178,18 @@ func (settings *NetworkSettings) PortMappingHuman() string { return strings.Join(mapping, ", ") } +// Inject the io.Reader at the given path. Note: do not close the reader +func (container *Container) Inject(file io.Reader, pth string) error { + dest, err := os.Open(path.Join(container.rwPath(), pth)) + if err != nil { + return err + } + if _, err := io.Copy(dest, file); err != nil { + return err + } + return nil +} + func (container *Container) Cmd() *exec.Cmd { return container.cmd } From 9db4972a70c2fb0921f984d67f816b81ea687a11 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Wed, 24 Apr 2013 13:51:28 -0700 Subject: [PATCH 06/27] Make sure the destination directory exists when using docker insert --- builder.go | 2 +- commands.go | 2 +- container.go | 7 ++++++- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/builder.go b/builder.go index b15a71d621..717525831b 100644 --- a/builder.go +++ b/builder.go @@ -110,7 +110,7 @@ func (builder *Builder) Build(dockerfile io.Reader, stdout io.Writer) error { } // Commit the container - base, err := builder.Commit(c, "", "", "", "") + base, err = builder.Commit(c, "", "", "", "") if err != nil { return err } diff --git a/commands.go b/commands.go index 21c8b68abc..9d848b95db 100644 --- a/commands.go +++ b/commands.go @@ -105,7 +105,7 @@ func (srv *Server) CmdInsert(stdin io.ReadCloser, stdout rcli.DockerConn, args . if err != nil { return err } - fmt.Fprintf(stdout, "%s\n", img) + fmt.Fprintf(stdout, "%s\n", img.Id) return nil } diff --git a/container.go b/container.go index ed3bdce94a..a4bee6a6fe 100644 --- a/container.go +++ b/container.go @@ -180,7 +180,12 @@ func (settings *NetworkSettings) PortMappingHuman() string { // Inject the io.Reader at the given path. Note: do not close the reader func (container *Container) Inject(file io.Reader, pth string) error { - dest, err := os.Open(path.Join(container.rwPath(), pth)) + // Make sure the directory exists + if err := os.MkdirAll(path.Join(container.rwPath(), path.Dir(pth)), 0755); err != nil { + return err + } + // FIXME: Handle permissions/already existing dest + dest, err := os.Create(path.Join(container.rwPath(), pth)) if err != nil { return err } From 0aebb254106be3086838cac145404c5d21c6bcd0 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Wed, 24 Apr 2013 14:28:51 -0700 Subject: [PATCH 07/27] Implement the COPY operator within the builder --- builder.go | 35 ++++++++++++++++++++++++++++++++--- 1 file changed, 32 insertions(+), 3 deletions(-) diff --git a/builder.go b/builder.go index 717525831b..105cb91e33 100644 --- a/builder.go +++ b/builder.go @@ -116,10 +116,39 @@ func (builder *Builder) Build(dockerfile io.Reader, stdout io.Writer) error { } tmpImages[base.Id] = struct{}{} - fmt.Fprintf(stdout, "===> %s\n", base.Id) + fmt.Fprintf(stdout, "===> %s\n", base.ShortId()) break case "copy": - return fmt.Errorf("The copy operator has not yet been implemented") + if image == nil { + return fmt.Errorf("Please provide a source image with `from` prior to copy") + } + tmp2 := strings.SplitN(tmp[1], " ", 2) + if len(tmp) != 2 { + return fmt.Errorf("Invalid COPY format") + } + fmt.Fprintf(stdout, "COPY %s to %s in %s\n", tmp2[0], tmp2[1], base.ShortId()) + + file, err := Download(tmp2[0], stdout) + if err != nil { + return err + } + defer file.Body.Close() + + c, err := builder.Run(base, "echo", "insert", tmp2[0], tmp2[1]) + if err != nil { + return err + } + + if err := c.Inject(file.Body, tmp2[1]); err != nil { + return err + } + + base, err = builder.Commit(c, "", "", "", "") + if err != nil { + return err + } + fmt.Fprintf(stdout, "===> %s\n", base.ShortId()) + break default: fmt.Fprintf(stdout, "Skipping unknown op %s\n", tmp[0]) } @@ -132,7 +161,7 @@ func (builder *Builder) Build(dockerfile io.Reader, stdout io.Writer) error { for i := range tmpContainers { delete(tmpContainers, i) } - fmt.Fprintf(stdout, "Build finished. image id: %s\n", base.Id) + fmt.Fprintf(stdout, "Build finished. image id: %s\n", base.ShortId()) } else { fmt.Fprintf(stdout, "An error occured during the build\n") } From 6f2125386a07a01eccce3bb4602aeecb8be3458b Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Wed, 24 Apr 2013 15:14:10 -0700 Subject: [PATCH 08/27] Moving runtime.Create to builder.Create --- builder.go | 102 +++++++++++++++++++++++++++++++++++++++++----------- commands.go | 13 +++++-- runtime.go | 4 +-- 3 files changed, 93 insertions(+), 26 deletions(-) diff --git a/builder.go b/builder.go index 105cb91e33..66118a9879 100644 --- a/builder.go +++ b/builder.go @@ -4,38 +4,80 @@ import ( "bufio" "fmt" "io" + "os" + "path" "strings" + "time" ) type Builder struct { - runtime *Runtime + runtime *Runtime + repositories *TagStore } func NewBuilder(runtime *Runtime) *Builder { return &Builder{ - runtime: runtime, + runtime: runtime, + repositories: runtime.repositories, } } -func (builder *Builder) Run(image *Image, cmd ...string) (*Container, error) { - // FIXME: pass a NopWriter instead of nil - config, err := ParseRun(append([]string{"-d", image.Id}, cmd...), nil, builder.runtime.capabilities) - if config.Image == "" { - return nil, fmt.Errorf("Image not specified") - } - if len(config.Cmd) == 0 { - return nil, fmt.Errorf("Command not specified") - } - if config.Tty { - return nil, fmt.Errorf("The tty mode is not supported within the builder") - } - - // Create new container - container, err := builder.runtime.Create(config) +func (builder *Builder) Create(config *Config) (*Container, error) { + // Lookup image + img, err := builder.repositories.LookupImage(config.Image) if err != nil { return nil, err } - if err := container.Start(); err != nil { + // Generate id + id := GenerateId() + // Generate default hostname + // FIXME: the lxc template no longer needs to set a default hostname + if config.Hostname == "" { + config.Hostname = id[:12] + } + + container := &Container{ + // FIXME: we should generate the ID here instead of receiving it as an argument + Id: id, + Created: time.Now(), + Path: config.Cmd[0], + Args: config.Cmd[1:], //FIXME: de-duplicate from config + Config: config, + Image: img.Id, // Always use the resolved image id + NetworkSettings: &NetworkSettings{}, + // FIXME: do we need to store this in the container? + SysInitPath: sysInitPath, + } + container.root = builder.runtime.containerRoot(container.Id) + // Step 1: create the container directory. + // This doubles as a barrier to avoid race conditions. + if err := os.Mkdir(container.root, 0700); err != nil { + return nil, err + } + + // If custom dns exists, then create a resolv.conf for the container + if len(config.Dns) > 0 { + container.ResolvConfPath = path.Join(container.root, "resolv.conf") + f, err := os.Create(container.ResolvConfPath) + if err != nil { + return nil, err + } + defer f.Close() + for _, dns := range config.Dns { + if _, err := f.Write([]byte("nameserver " + dns + "\n")); err != nil { + return nil, err + } + } + } else { + container.ResolvConfPath = "/etc/resolv.conf" + } + + // Step 2: save the container json + if err := container.ToDisk(); err != nil { + return nil, err + } + // Step 3: register the container + if err := builder.runtime.Register(container); err != nil { return nil, err } return container, nil @@ -96,12 +138,19 @@ func (builder *Builder) Build(dockerfile io.Reader, stdout io.Writer) error { if image == nil { return fmt.Errorf("Please provide a source image with `from` prior to run") } + config, err := ParseRun([]string{image.Id, "/bin/sh", "-c", tmp[1]}, nil, builder.runtime.capabilities) + if err != nil { + return err + } // Create the container and start it - c, err := builder.Run(image, "/bin/sh", "-c", tmp[1]) + c, err := builder.Create(config) if err != nil { return err } + if err := c.Start(); err != nil { + return err + } tmpContainers[c.Id] = struct{}{} // Wait for it to finish @@ -134,10 +183,23 @@ func (builder *Builder) Build(dockerfile io.Reader, stdout io.Writer) error { } defer file.Body.Close() - c, err := builder.Run(base, "echo", "insert", tmp2[0], tmp2[1]) + config, err := ParseRun([]string{base.Id, "echo", "insert", tmp2[0], tmp2[1]}, nil, builder.runtime.capabilities) if err != nil { return err } + c, err := builder.Create(config) + if err != nil { + return err + } + + if err := c.Start(); err != nil { + return err + } + + // Wait for echo to finish + if result := c.Wait(); result != 0 { + return fmt.Errorf("!!! '%s' return non-zero exit code '%d'. Aborting.", tmp[1], result) + } if err := c.Inject(file.Body, tmp2[1]); err != nil { return err diff --git a/commands.go b/commands.go index 9d848b95db..aaa4dab3e5 100644 --- a/commands.go +++ b/commands.go @@ -91,8 +91,13 @@ func (srv *Server) CmdInsert(stdin io.ReadCloser, stdout rcli.DockerConn, args . } defer file.Body.Close() + config, err := ParseRun([]string{img.Id, "echo", "insert", url, path}, nil, srv.runtime.capabilities) + if err != nil { + return err + } + b := NewBuilder(srv.runtime) - c, err := b.Run(img, "echo", "insert", url, path) + c, err := b.Create(config) if err != nil { return err } @@ -1065,8 +1070,10 @@ func (srv *Server) CmdRun(stdin io.ReadCloser, stdout rcli.DockerConn, args ...s // or tell the client there is no options stdout.Flush() + b := NewBuilder(srv.runtime) + // Create new container - container, err := srv.runtime.Create(config) + container, err := b.Create(config) if err != nil { // If container not found, try to pull it if srv.runtime.graph.IsNotExist(err) { @@ -1074,7 +1081,7 @@ func (srv *Server) CmdRun(stdin io.ReadCloser, stdout rcli.DockerConn, args ...s if err = srv.CmdPull(stdin, stdout, config.Image); err != nil { return err } - if container, err = srv.runtime.Create(config); err != nil { + if container, err = b.Create(config); err != nil { return err } } else { diff --git a/runtime.go b/runtime.go index 79a7170c7d..a298d9b714 100644 --- a/runtime.go +++ b/runtime.go @@ -12,7 +12,6 @@ import ( "path" "sort" "strings" - "time" ) type Capabilities struct { @@ -116,7 +115,6 @@ func (runtime *Runtime) mergeConfig(userConf, imageConf *Config) { } func (runtime *Runtime) Create(config *Config) (*Container, error) { - // Lookup image img, err := runtime.repositories.LookupImage(config.Image) if err != nil { @@ -151,7 +149,6 @@ func (runtime *Runtime) Create(config *Config) (*Container, error) { // FIXME: do we need to store this in the container? SysInitPath: sysInitPath, } - container.root = runtime.containerRoot(container.Id) // Step 1: create the container directory. // This doubles as a barrier to avoid race conditions. @@ -187,6 +184,7 @@ func (runtime *Runtime) Create(config *Config) (*Container, error) { return container, nil } +======= end func (runtime *Runtime) Load(id string) (*Container, error) { container := &Container{root: runtime.containerRoot(id)} if err := container.FromDisk(); err != nil { From f7c5e92a2e1ec30f50b0affe952a0496c62195f5 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Wed, 24 Apr 2013 15:24:14 -0700 Subject: [PATCH 09/27] Move runtime.Commit to builder.Commit --- builder.go | 23 ++++++++++++++++++++++- commands.go | 7 ++++++- runtime.go | 27 --------------------------- 3 files changed, 28 insertions(+), 29 deletions(-) diff --git a/builder.go b/builder.go index 66118a9879..1c354ea8a9 100644 --- a/builder.go +++ b/builder.go @@ -13,11 +13,13 @@ import ( type Builder struct { runtime *Runtime repositories *TagStore + graph *Graph } func NewBuilder(runtime *Runtime) *Builder { return &Builder{ runtime: runtime, + graph: runtime.graph, repositories: runtime.repositories, } } @@ -83,8 +85,27 @@ func (builder *Builder) Create(config *Config) (*Container, error) { return container, nil } +// Commit creates a new filesystem image from the current state of a container. +// The image can optionally be tagged into a repository func (builder *Builder) Commit(container *Container, repository, tag, comment, author string) (*Image, error) { - return builder.runtime.Commit(container.Id, repository, tag, comment, author) + // FIXME: freeze the container before copying it to avoid data corruption? + // FIXME: this shouldn't be in commands. + rwTar, err := container.ExportRw() + if err != nil { + return nil, err + } + // Create a new image from the container's base layers + a new layer from container changes + img, err := builder.graph.Create(rwTar, container, comment, author) + if err != nil { + return nil, err + } + // Register the image if needed + if repository != "" { + if err := builder.repositories.Set(repository, tag, img.Id, true); err != nil { + return img, err + } + } + return img, nil } func (builder *Builder) clearTmp(containers, images map[string]struct{}) { diff --git a/commands.go b/commands.go index aaa4dab3e5..39730afe56 100644 --- a/commands.go +++ b/commands.go @@ -852,7 +852,12 @@ func (srv *Server) CmdCommit(stdin io.ReadCloser, stdout io.Writer, args ...stri } } - img, err := srv.runtime.Commit(containerName, repository, tag, *flComment, *flAuthor, config) + container := srv.runtime.Get(containerName) + if container == nil { + return fmt.Errorf("No such container: %s", containerName) + } + + img, err := NewBuilder(srv.runtime).Commit(container, repository, tag, *flComment, *flAuthor, config) if err != nil { return err } diff --git a/runtime.go b/runtime.go index a298d9b714..ca27b7a7bf 100644 --- a/runtime.go +++ b/runtime.go @@ -309,33 +309,6 @@ func (runtime *Runtime) Destroy(container *Container) error { return nil } -// Commit creates a new filesystem image from the current state of a container. -// The image can optionally be tagged into a repository -func (runtime *Runtime) Commit(id, repository, tag, comment, author string, config *Config) (*Image, error) { - container := runtime.Get(id) - if container == nil { - return nil, fmt.Errorf("No such container: %s", id) - } - // FIXME: freeze the container before copying it to avoid data corruption? - // FIXME: this shouldn't be in commands. - rwTar, err := container.ExportRw() - if err != nil { - return nil, err - } - // Create a new image from the container's base layers + a new layer from container changes - img, err := runtime.graph.Create(rwTar, container, comment, author, config) - if err != nil { - return nil, err - } - // Register the image if needed - if repository != "" { - if err := runtime.repositories.Set(repository, tag, img.Id, true); err != nil { - return img, err - } - } - return img, nil -} - func (runtime *Runtime) restore() error { dir, err := ioutil.ReadDir(runtime.repository) if err != nil { From ff95f2b0ecf54fdfeffa1aa008e22cb5c7ec59d6 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Wed, 24 Apr 2013 15:35:28 -0700 Subject: [PATCH 10/27] Update the unit tests to reflect the new API --- commands_test.go | 2 +- container_test.go | 67 +++++++++++++++++++++++++++-------------------- runtime_test.go | 21 +++++++++------ 3 files changed, 53 insertions(+), 37 deletions(-) diff --git a/commands_test.go b/commands_test.go index 83b480d52a..469364b4a2 100644 --- a/commands_test.go +++ b/commands_test.go @@ -339,7 +339,7 @@ func TestAttachDisconnect(t *testing.T) { srv := &Server{runtime: runtime} - container, err := runtime.Create( + container, err := NewBuilder(runtime).Create( &Config{ Image: GetTestImage(runtime).Id, Memory: 33554432, diff --git a/container_test.go b/container_test.go index c3a891cf4c..c06b8de876 100644 --- a/container_test.go +++ b/container_test.go @@ -20,7 +20,7 @@ func TestIdFormat(t *testing.T) { t.Fatal(err) } defer nuke(runtime) - container1, err := runtime.Create( + container1, err := NewBuilder(runtime).Create( &Config{ Image: GetTestImage(runtime).Id, Cmd: []string{"/bin/sh", "-c", "echo hello world"}, @@ -44,7 +44,7 @@ func TestMultipleAttachRestart(t *testing.T) { t.Fatal(err) } defer nuke(runtime) - container, err := runtime.Create( + container, err := NewBuilder(runtime).Create( &Config{ Image: GetTestImage(runtime).Id, Cmd: []string{"/bin/sh", "-c", @@ -148,8 +148,10 @@ func TestDiff(t *testing.T) { } defer nuke(runtime) + builder := NewBuilder(runtime) + // Create a container and remove a file - container1, err := runtime.Create( + container1, err := builder.Create( &Config{ Image: GetTestImage(runtime).Id, Cmd: []string{"/bin/rm", "/etc/passwd"}, @@ -190,7 +192,7 @@ func TestDiff(t *testing.T) { } // Create a new container from the commited image - container2, err := runtime.Create( + container2, err := builder.Create( &Config{ Image: img.Id, Cmd: []string{"cat", "/etc/passwd"}, @@ -301,7 +303,10 @@ func TestCommitRun(t *testing.T) { t.Fatal(err) } defer nuke(runtime) - container1, err := runtime.Create( + + builder := NewBuilder(runtime) + + container1, err := builder.Create( &Config{ Image: GetTestImage(runtime).Id, Cmd: []string{"/bin/sh", "-c", "echo hello > /world"}, @@ -333,7 +338,7 @@ func TestCommitRun(t *testing.T) { // FIXME: Make a TestCommit that stops here and check docker.root/layers/img.id/world - container2, err := runtime.Create( + container2, err := builder.Create( &Config{ Image: img.Id, Cmd: []string{"cat", "/world"}, @@ -380,7 +385,7 @@ func TestStart(t *testing.T) { t.Fatal(err) } defer nuke(runtime) - container, err := runtime.Create( + container, err := NewBuilder(runtime).Create( &Config{ Image: GetTestImage(runtime).Id, Memory: 33554432, @@ -419,7 +424,7 @@ func TestRun(t *testing.T) { t.Fatal(err) } defer nuke(runtime) - container, err := runtime.Create( + container, err := NewBuilder(runtime).Create( &Config{ Image: GetTestImage(runtime).Id, Cmd: []string{"ls", "-al"}, @@ -447,7 +452,7 @@ func TestOutput(t *testing.T) { t.Fatal(err) } defer nuke(runtime) - container, err := runtime.Create( + container, err := NewBuilder(runtime).Create( &Config{ Image: GetTestImage(runtime).Id, Cmd: []string{"echo", "-n", "foobar"}, @@ -472,7 +477,7 @@ func TestKillDifferentUser(t *testing.T) { t.Fatal(err) } defer nuke(runtime) - container, err := runtime.Create(&Config{ + container, err := NewBuilder(runtime).Create(&Config{ Image: GetTestImage(runtime).Id, Cmd: []string{"tail", "-f", "/etc/resolv.conf"}, User: "daemon", @@ -520,7 +525,7 @@ func TestKill(t *testing.T) { t.Fatal(err) } defer nuke(runtime) - container, err := runtime.Create(&Config{ + container, err := NewBuilder(runtime).Create(&Config{ Image: GetTestImage(runtime).Id, Cmd: []string{"cat", "/dev/zero"}, }, @@ -566,7 +571,9 @@ func TestExitCode(t *testing.T) { } defer nuke(runtime) - trueContainer, err := runtime.Create(&Config{ + builder := NewBuilder(runtime) + + trueContainer, err := builder.Create(&Config{ Image: GetTestImage(runtime).Id, Cmd: []string{"/bin/true", ""}, }) @@ -581,7 +588,7 @@ func TestExitCode(t *testing.T) { t.Errorf("Unexpected exit code %d (expected 0)", trueContainer.State.ExitCode) } - falseContainer, err := runtime.Create(&Config{ + falseContainer, err := builder.Create(&Config{ Image: GetTestImage(runtime).Id, Cmd: []string{"/bin/false", ""}, }) @@ -603,7 +610,7 @@ func TestRestart(t *testing.T) { t.Fatal(err) } defer nuke(runtime) - container, err := runtime.Create(&Config{ + container, err := NewBuilder(runtime).Create(&Config{ Image: GetTestImage(runtime).Id, Cmd: []string{"echo", "-n", "foobar"}, }, @@ -636,7 +643,7 @@ func TestRestartStdin(t *testing.T) { t.Fatal(err) } defer nuke(runtime) - container, err := runtime.Create(&Config{ + container, err := NewBuilder(runtime).Create(&Config{ Image: GetTestImage(runtime).Id, Cmd: []string{"cat"}, @@ -715,8 +722,10 @@ func TestUser(t *testing.T) { } defer nuke(runtime) + builder := NewBuilder(runtime) + // Default user must be root - container, err := runtime.Create(&Config{ + container, err := builder.Create(&Config{ Image: GetTestImage(runtime).Id, Cmd: []string{"id"}, }, @@ -734,7 +743,7 @@ func TestUser(t *testing.T) { } // Set a username - container, err = runtime.Create(&Config{ + container, err = builder.Create(&Config{ Image: GetTestImage(runtime).Id, Cmd: []string{"id"}, @@ -754,7 +763,7 @@ func TestUser(t *testing.T) { } // Set a UID - container, err = runtime.Create(&Config{ + container, err = builder.Create(&Config{ Image: GetTestImage(runtime).Id, Cmd: []string{"id"}, @@ -774,7 +783,7 @@ func TestUser(t *testing.T) { } // Set a different user by uid - container, err = runtime.Create(&Config{ + container, err = builder.Create(&Config{ Image: GetTestImage(runtime).Id, Cmd: []string{"id"}, @@ -796,7 +805,7 @@ func TestUser(t *testing.T) { } // Set a different user by username - container, err = runtime.Create(&Config{ + container, err = builder.Create(&Config{ Image: GetTestImage(runtime).Id, Cmd: []string{"id"}, @@ -823,7 +832,9 @@ func TestMultipleContainers(t *testing.T) { } defer nuke(runtime) - container1, err := runtime.Create(&Config{ + builder := NewBuilder(runtime) + + container1, err := builder.Create(&Config{ Image: GetTestImage(runtime).Id, Cmd: []string{"cat", "/dev/zero"}, }, @@ -833,7 +844,7 @@ func TestMultipleContainers(t *testing.T) { } defer runtime.Destroy(container1) - container2, err := runtime.Create(&Config{ + container2, err := builder.Create(&Config{ Image: GetTestImage(runtime).Id, Cmd: []string{"cat", "/dev/zero"}, }, @@ -879,7 +890,7 @@ func TestStdin(t *testing.T) { t.Fatal(err) } defer nuke(runtime) - container, err := runtime.Create(&Config{ + container, err := NewBuilder(runtime).Create(&Config{ Image: GetTestImage(runtime).Id, Cmd: []string{"cat"}, @@ -926,7 +937,7 @@ func TestTty(t *testing.T) { t.Fatal(err) } defer nuke(runtime) - container, err := runtime.Create(&Config{ + container, err := NewBuilder(runtime).Create(&Config{ Image: GetTestImage(runtime).Id, Cmd: []string{"cat"}, @@ -973,7 +984,7 @@ func TestEnv(t *testing.T) { t.Fatal(err) } defer nuke(runtime) - container, err := runtime.Create(&Config{ + container, err := NewBuilder(runtime).Create(&Config{ Image: GetTestImage(runtime).Id, Cmd: []string{"/usr/bin/env"}, }, @@ -1047,7 +1058,7 @@ func TestLXCConfig(t *testing.T) { memMin := 33554432 memMax := 536870912 mem := memMin + rand.Intn(memMax-memMin) - container, err := runtime.Create(&Config{ + container, err := NewBuilder(runtime).Create(&Config{ Image: GetTestImage(runtime).Id, Cmd: []string{"/bin/true"}, @@ -1074,7 +1085,7 @@ func BenchmarkRunSequencial(b *testing.B) { } defer nuke(runtime) for i := 0; i < b.N; i++ { - container, err := runtime.Create(&Config{ + container, err := NewBuilder(runtime).Create(&Config{ Image: GetTestImage(runtime).Id, Cmd: []string{"echo", "-n", "foo"}, }, @@ -1109,7 +1120,7 @@ func BenchmarkRunParallel(b *testing.B) { complete := make(chan error) tasks = append(tasks, complete) go func(i int, complete chan error) { - container, err := runtime.Create(&Config{ + container, err := NewBuilder(runtime).Create(&Config{ Image: GetTestImage(runtime).Id, Cmd: []string{"echo", "-n", "foo"}, }, diff --git a/runtime_test.go b/runtime_test.go index e9be838c0e..8e21f57bc5 100644 --- a/runtime_test.go +++ b/runtime_test.go @@ -118,7 +118,7 @@ func TestRuntimeCreate(t *testing.T) { if len(runtime.List()) != 0 { t.Errorf("Expected 0 containers, %v found", len(runtime.List())) } - container, err := runtime.Create(&Config{ + container, err := NewBuilder(runtime).Create(&Config{ Image: GetTestImage(runtime).Id, Cmd: []string{"ls", "-al"}, }, @@ -165,7 +165,7 @@ func TestDestroy(t *testing.T) { t.Fatal(err) } defer nuke(runtime) - container, err := runtime.Create(&Config{ + container, err := NewBuilder(runtime).Create(&Config{ Image: GetTestImage(runtime).Id, Cmd: []string{"ls", "-al"}, }, @@ -212,7 +212,10 @@ func TestGet(t *testing.T) { t.Fatal(err) } defer nuke(runtime) - container1, err := runtime.Create(&Config{ + + builder := NewBuilder(runtime) + + container1, err := builder.Create(&Config{ Image: GetTestImage(runtime).Id, Cmd: []string{"ls", "-al"}, }, @@ -222,7 +225,7 @@ func TestGet(t *testing.T) { } defer runtime.Destroy(container1) - container2, err := runtime.Create(&Config{ + container2, err := builder.Create(&Config{ Image: GetTestImage(runtime).Id, Cmd: []string{"ls", "-al"}, }, @@ -232,7 +235,7 @@ func TestGet(t *testing.T) { } defer runtime.Destroy(container2) - container3, err := runtime.Create(&Config{ + container3, err := builder.Create(&Config{ Image: GetTestImage(runtime).Id, Cmd: []string{"ls", "-al"}, }, @@ -262,7 +265,7 @@ func TestAllocatePortLocalhost(t *testing.T) { if err != nil { t.Fatal(err) } - container, err := runtime.Create(&Config{ + container, err := NewBuilder(runtime).Create(&Config{ Image: GetTestImage(runtime).Id, Cmd: []string{"sh", "-c", "echo well hello there | nc -l -p 5555"}, PortSpecs: []string{"5555"}, @@ -325,8 +328,10 @@ func TestRestore(t *testing.T) { t.Fatal(err) } + builder := NewBuilder(runtime1) + // Create a container with one instance of docker - container1, err := runtime1.Create(&Config{ + container1, err := builder.Create(&Config{ Image: GetTestImage(runtime1).Id, Cmd: []string{"ls", "-al"}, }, @@ -337,7 +342,7 @@ func TestRestore(t *testing.T) { defer runtime1.Destroy(container1) // Create a second container meant to be killed - container2, err := runtime1.Create(&Config{ + container2, err := builder.Create(&Config{ Image: GetTestImage(runtime1).Id, Cmd: []string{"/bin/cat"}, OpenStdin: true, From 74b9e851f6f407ede9d46ab4960e5e134ff666c7 Mon Sep 17 00:00:00 2001 From: Nate Jones Date: Thu, 25 Apr 2013 08:08:05 -0700 Subject: [PATCH 11/27] use new image as base of next command --- builder.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/builder.go b/builder.go index 1c354ea8a9..faed9c7794 100644 --- a/builder.go +++ b/builder.go @@ -187,6 +187,10 @@ func (builder *Builder) Build(dockerfile io.Reader, stdout io.Writer) error { tmpImages[base.Id] = struct{}{} fmt.Fprintf(stdout, "===> %s\n", base.ShortId()) + + // use the base as the new image + image = base + break case "copy": if image == nil { From dade95844feaa2446e472767e1e2763996241dee Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Thu, 25 Apr 2013 11:20:45 -0700 Subject: [PATCH 12/27] Make Builder.Build return the builded image --- builder.go | 40 ++++++++++++++++++++-------------------- commands.go | 7 ++++++- utils.go | 7 +++++++ 3 files changed, 33 insertions(+), 21 deletions(-) diff --git a/builder.go b/builder.go index faed9c7794..6c389b34b4 100644 --- a/builder.go +++ b/builder.go @@ -120,7 +120,7 @@ func (builder *Builder) clearTmp(containers, images map[string]struct{}) { } } -func (builder *Builder) Build(dockerfile io.Reader, stdout io.Writer) error { +func (builder *Builder) Build(dockerfile io.Reader, stdout io.Writer) (*Image, error) { var ( image, base *Image tmpContainers map[string]struct{} = make(map[string]struct{}) @@ -135,7 +135,7 @@ func (builder *Builder) Build(dockerfile io.Reader, stdout io.Writer) error { if err == io.EOF { break } - return err + return nil, err } line = strings.TrimSpace(line) // Skip comments and empty line @@ -144,45 +144,45 @@ func (builder *Builder) Build(dockerfile io.Reader, stdout io.Writer) error { } tmp := strings.SplitN(line, " ", 2) if len(tmp) != 2 { - return fmt.Errorf("Invalid Dockerfile format") + return nil, fmt.Errorf("Invalid Dockerfile format") } switch tmp[0] { case "from": fmt.Fprintf(stdout, "FROM %s\n", tmp[1]) image, err = builder.runtime.repositories.LookupImage(tmp[1]) if err != nil { - return err + return nil, err } break case "run": fmt.Fprintf(stdout, "RUN %s\n", tmp[1]) if image == nil { - return fmt.Errorf("Please provide a source image with `from` prior to run") + return nil, fmt.Errorf("Please provide a source image with `from` prior to run") } config, err := ParseRun([]string{image.Id, "/bin/sh", "-c", tmp[1]}, nil, builder.runtime.capabilities) if err != nil { - return err + return nil, err } // Create the container and start it c, err := builder.Create(config) if err != nil { - return err + return nil, err } if err := c.Start(); err != nil { - return err + return nil, err } tmpContainers[c.Id] = struct{}{} // Wait for it to finish if result := c.Wait(); result != 0 { - return fmt.Errorf("!!! '%s' return non-zero exit code '%d'. Aborting.", tmp[1], result) + return nil, fmt.Errorf("!!! '%s' return non-zero exit code '%d'. Aborting.", tmp[1], result) } // Commit the container base, err = builder.Commit(c, "", "", "", "") if err != nil { - return err + return nil, err } tmpImages[base.Id] = struct{}{} @@ -194,45 +194,45 @@ func (builder *Builder) Build(dockerfile io.Reader, stdout io.Writer) error { break case "copy": if image == nil { - return fmt.Errorf("Please provide a source image with `from` prior to copy") + return nil, fmt.Errorf("Please provide a source image with `from` prior to copy") } tmp2 := strings.SplitN(tmp[1], " ", 2) if len(tmp) != 2 { - return fmt.Errorf("Invalid COPY format") + return nil, fmt.Errorf("Invalid COPY format") } fmt.Fprintf(stdout, "COPY %s to %s in %s\n", tmp2[0], tmp2[1], base.ShortId()) file, err := Download(tmp2[0], stdout) if err != nil { - return err + return nil, err } defer file.Body.Close() config, err := ParseRun([]string{base.Id, "echo", "insert", tmp2[0], tmp2[1]}, nil, builder.runtime.capabilities) if err != nil { - return err + return nil, err } c, err := builder.Create(config) if err != nil { - return err + return nil, err } if err := c.Start(); err != nil { - return err + return nil, err } // Wait for echo to finish if result := c.Wait(); result != 0 { - return fmt.Errorf("!!! '%s' return non-zero exit code '%d'. Aborting.", tmp[1], result) + return nil, fmt.Errorf("!!! '%s' return non-zero exit code '%d'. Aborting.", tmp[1], result) } if err := c.Inject(file.Body, tmp2[1]); err != nil { - return err + return nil, err } base, err = builder.Commit(c, "", "", "", "") if err != nil { - return err + return nil, err } fmt.Fprintf(stdout, "===> %s\n", base.ShortId()) break @@ -252,5 +252,5 @@ func (builder *Builder) Build(dockerfile io.Reader, stdout io.Writer) error { } else { fmt.Fprintf(stdout, "An error occured during the build\n") } - return nil + return base, nil } diff --git a/commands.go b/commands.go index 39730afe56..3975d67671 100644 --- a/commands.go +++ b/commands.go @@ -137,7 +137,12 @@ func (srv *Server) CmdBuild(stdin io.ReadCloser, stdout rcli.DockerConn, args .. } else { file = stdin } - return NewBuilder(srv.runtime).Build(file, stdout) + img, err := NewBuilder(srv.runtime).Build(file, stdout) + if err != nil { + return err + } + fmt.Fprintf(stdout, "%s\n", img.ShortId()) + return nil } // 'docker login': login / register a user to registry service. diff --git a/utils.go b/utils.go index 229b938830..047a29abef 100644 --- a/utils.go +++ b/utils.go @@ -155,6 +155,13 @@ func SelfPath() string { return path } +type nopWriter struct { +} + +func (w *nopWriter) Write(buf []byte) (int, error) { + return len(buf), nil +} + type nopWriteCloser struct { io.Writer } From e337949cb0ce8eda0673e005e670c12e997a3336 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Thu, 25 Apr 2013 11:20:56 -0700 Subject: [PATCH 13/27] Add builder_test.go --- builder_test.go | 88 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 88 insertions(+) create mode 100644 builder_test.go diff --git a/builder_test.go b/builder_test.go new file mode 100644 index 0000000000..345fd2c594 --- /dev/null +++ b/builder_test.go @@ -0,0 +1,88 @@ +package docker + +import ( + "strings" + "testing" +) + +const Dockerfile = ` +# VERSION 0.1 +# DOCKER-VERSION 0.1.6 + +from docker-ut +run sh -c 'echo root:testpass > /tmp/passwd' +run mkdir -p /var/run/sshd +copy https://raw.github.com/dotcloud/docker/master/CHANGELOG.md /tmp/CHANGELOG.md +` + +func TestBuild(t *testing.T) { + runtime, err := newTestRuntime() + if err != nil { + t.Fatal(err) + } + defer nuke(runtime) + + builder := NewBuilder(runtime) + + img, err := builder.Build(strings.NewReader(Dockerfile), &nopWriter{}) + if err != nil { + t.Fatal(err) + } + + container, err := builder.Create( + &Config{ + Image: img.Id, + Cmd: []string{"cat", "/tmp/passwd"}, + }, + ) + if err != nil { + t.Fatal(err) + } + defer runtime.Destroy(container) + + output, err := container.Output() + if err != nil { + t.Fatal(err) + } + if string(output) != "root:testpass\n" { + t.Fatalf("Unexpected output. Read '%s', expected '%s'", output, "root:testpass\n") + } + + container2, err := builder.Create( + &Config{ + Image: img.Id, + Cmd: []string{"ls", "-d", "/var/run/sshd"}, + }, + ) + if err != nil { + t.Fatal(err) + } + defer runtime.Destroy(container2) + + output, err = container2.Output() + if err != nil { + t.Fatal(err) + } + if string(output) != "/var/run/sshd\n" { + t.Fatal("/var/run/sshd has not been created") + } + + container3, err := builder.Create( + &Config{ + Image: img.Id, + Cmd: []string{"cat", "/tmp/CHANGELOG.md"}, + }, + ) + if err != nil { + t.Fatal(err) + } + defer runtime.Destroy(container3) + + output, err = container3.Output() + if err != nil { + t.Fatal(err) + } + if len(output) == 0 { + t.Fatal("/tmp/CHANGELOG.md has not been copied") + } +} From 4390a3182f9ea4fc1e215917c2c6494d6fc453ac Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Sat, 27 Apr 2013 21:45:05 -0700 Subject: [PATCH 14/27] Fix image pipe with Builder COPY --- builder.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/builder.go b/builder.go index 6c389b34b4..5ede4ae5b2 100644 --- a/builder.go +++ b/builder.go @@ -235,6 +235,9 @@ func (builder *Builder) Build(dockerfile io.Reader, stdout io.Writer) (*Image, e return nil, err } fmt.Fprintf(stdout, "===> %s\n", base.ShortId()) + + image = base + break default: fmt.Fprintf(stdout, "Skipping unknown op %s\n", tmp[0]) From 034c7a7a5e3838efb1fd06f544ae9a8b21e1e1b9 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Tue, 30 Apr 2013 18:03:15 -0700 Subject: [PATCH 15/27] Remove the open from CmdBuild --- commands.go | 23 +++-------------------- 1 file changed, 3 insertions(+), 20 deletions(-) diff --git a/commands.go b/commands.go index 3975d67671..08f9f7e18e 100644 --- a/commands.go +++ b/commands.go @@ -35,7 +35,7 @@ func (srv *Server) Help() string { help := "Usage: docker COMMAND [arg...]\n\nA self-sufficient runtime for linux containers.\n\nCommands:\n" for _, cmd := range [][]string{ {"attach", "Attach to a running container"}, - {"build", "Build a container from Dockerfile"}, + {"build", "Build a container from Dockerfile via stdin"}, {"commit", "Create a new image from a container's changes"}, {"diff", "Inspect changes on a container's filesystem"}, {"export", "Stream the contents of a container as a tar archive"}, @@ -116,28 +116,11 @@ func (srv *Server) CmdInsert(stdin io.ReadCloser, stdout rcli.DockerConn, args . func (srv *Server) CmdBuild(stdin io.ReadCloser, stdout rcli.DockerConn, args ...string) error { stdout.Flush() - cmd := rcli.Subcmd(stdout, "build", "[Dockerfile|-]", "Build a container from Dockerfile") + cmd := rcli.Subcmd(stdout, "build", "-", "Build a container from Dockerfile via stdin") if err := cmd.Parse(args); err != nil { return nil } - dockerfile := cmd.Arg(0) - if dockerfile == "" { - dockerfile = "Dockerfile" - } - - var file io.Reader - - if dockerfile != "-" { - f, err := os.Open(dockerfile) - if err != nil { - return err - } - defer f.Close() - file = f - } else { - file = stdin - } - img, err := NewBuilder(srv.runtime).Build(file, stdout) + img, err := NewBuilder(srv.runtime).Build(stdin, stdout) if err != nil { return err } From bbb634a98068171b6b6f4bbb9c7160284ef252e4 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Wed, 1 May 2013 13:37:32 -0700 Subject: [PATCH 16/27] Add doc for the builder --- docs/sources/builder/basics.rst | 91 ++++++++++++++++++++++ docs/sources/builder/index.rst | 14 ++++ docs/sources/commandline/cli.rst | 1 + docs/sources/commandline/command/build.rst | 9 +++ docs/sources/index.rst | 3 +- 5 files changed, 117 insertions(+), 1 deletion(-) create mode 100644 docs/sources/builder/basics.rst create mode 100644 docs/sources/builder/index.rst create mode 100644 docs/sources/commandline/command/build.rst diff --git a/docs/sources/builder/basics.rst b/docs/sources/builder/basics.rst new file mode 100644 index 0000000000..a233515ef2 --- /dev/null +++ b/docs/sources/builder/basics.rst @@ -0,0 +1,91 @@ +============== +Docker Builder +============== + +.. contents:: Table of Contents + +1. Format +========= + +The Docker builder format is quite simple: + + ``instruction arguments`` + +The first instruction must be `FROM` + +All instruction are to be placed in a file named `Dockerfile` + +In order to place comments within a Dockerfile, simply prefix the line with "`#`" + +2. Instructions +=============== + +Docker builder comes with a set of instructions: + +1. FROM: Set from what image to build +2. RUN: Execute a command +3. INSERT: Insert a remote file (http) into the image + +2.1 FROM +-------- + ``FROM `` + +The `FROM` instruction must be the first one in order for Builder to know from where to run commands. + +`FROM` can also be used in order to build multiple images within a single Dockerfile + +2.2 RUN +------- + ``RUN `` + +The `RUN` instruction is the main one, it allows you to execute any commands on the `FROM` image and to save the results. +You can use as many `RUN` as you want within a Dockerfile, the commands will be executed on the result of the previous command. + +2.3 INSERT +---------- + + ``INSERT `` + +The `INSERT` instruction will download the file at the given url and place it within the image at the given path. + +.. note:: + The path must include the file name. + +3. Dockerfile Examples +====================== + +:: + + # Nginx + # + # VERSION 0.0.1 + # DOCKER-VERSION 0.2 + + from ubuntu + + # make sure the package repository is up to date + run echo "deb http://archive.ubuntu.com/ubuntu precise main universe" > /etc/apt/sources.list + run apt-get update + + run apt-get install -y inotify-tools nginx apache openssh-server + insert https://raw.github.com/creack/docker-vps/master/nginx-wrapper.sh /usr/sbin/nginx-wrapper + +:: + + # Firefox over VNC + # + # VERSION 0.3 + # DOCKER-VERSION 0.2 + + from ubuntu + # make sure the package repository is up to date + run echo "deb http://archive.ubuntu.com/ubuntu precise main universe" > /etc/apt/sources.list + run apt-get update + + # Install vnc, xvfb in order to create a 'fake' display and firefox + run apt-get install -y x11vnc xvfb firefox + run mkdir /.vnc + # Setup a password + run x11vnc -storepasswd 1234 ~/.vnc/passwd + # Autostart firefox (might not be the best way to do it, but it does the trick) + run bash -c 'echo "firefox" >> /.bashrc' diff --git a/docs/sources/builder/index.rst b/docs/sources/builder/index.rst new file mode 100644 index 0000000000..170be1a5ab --- /dev/null +++ b/docs/sources/builder/index.rst @@ -0,0 +1,14 @@ +:title: docker documentation +:description: Documentation for docker builder +:keywords: docker, builder, dockerfile + + +Builder +======= + +Contents: + +.. toctree:: + :maxdepth: 2 + + basics diff --git a/docs/sources/commandline/cli.rst b/docs/sources/commandline/cli.rst index 2657b91777..46ea3e4a7f 100644 --- a/docs/sources/commandline/cli.rst +++ b/docs/sources/commandline/cli.rst @@ -27,6 +27,7 @@ Available Commands :maxdepth: 1 command/attach + command/build command/commit command/diff command/export diff --git a/docs/sources/commandline/command/build.rst b/docs/sources/commandline/command/build.rst new file mode 100644 index 0000000000..6415f11f7b --- /dev/null +++ b/docs/sources/commandline/command/build.rst @@ -0,0 +1,9 @@ +=========================================== +``build`` -- Build a container from Dockerfile via stdin +=========================================== + +:: + + Usage: docker build - + Example: cat Dockerfile | docker build - + Build a new image from the Dockerfile passed via stdin diff --git a/docs/sources/index.rst b/docs/sources/index.rst index e6a1482ccd..9a272d2a34 100644 --- a/docs/sources/index.rst +++ b/docs/sources/index.rst @@ -17,7 +17,8 @@ This documentation has the following resources: commandline/index registry/index index/index + builder/index faq -.. image:: http://www.docker.io/_static/lego_docker.jpg \ No newline at end of file +.. image:: http://www.docker.io/_static/lego_docker.jpg From 6bfb652f5b1bce989868f8269c3d736f3faf259a Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Wed, 1 May 2013 13:45:35 -0700 Subject: [PATCH 17/27] Change dockerbulder format, no more tabs and COPY becomes INSERT to avoid conflict with contrib script --- builder.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/builder.go b/builder.go index 5ede4ae5b2..e252d3d8c0 100644 --- a/builder.go +++ b/builder.go @@ -142,7 +142,7 @@ func (builder *Builder) Build(dockerfile io.Reader, stdout io.Writer) (*Image, e if len(line) == 0 || line[0] == '#' { continue } - tmp := strings.SplitN(line, " ", 2) + tmp := strings.SplitN(line, " ", 2) if len(tmp) != 2 { return nil, fmt.Errorf("Invalid Dockerfile format") } @@ -192,13 +192,13 @@ func (builder *Builder) Build(dockerfile io.Reader, stdout io.Writer) (*Image, e image = base break - case "copy": + case "insert": if image == nil { return nil, fmt.Errorf("Please provide a source image with `from` prior to copy") } tmp2 := strings.SplitN(tmp[1], " ", 2) if len(tmp) != 2 { - return nil, fmt.Errorf("Invalid COPY format") + return nil, fmt.Errorf("Invalid INSERT format") } fmt.Fprintf(stdout, "COPY %s to %s in %s\n", tmp2[0], tmp2[1], base.ShortId()) @@ -240,7 +240,7 @@ func (builder *Builder) Build(dockerfile io.Reader, stdout io.Writer) (*Image, e break default: - fmt.Fprintf(stdout, "Skipping unknown op %s\n", tmp[0]) + fmt.Fprintf(stdout, "Skipping unknown instruction %s\n", instruction) } } if base != nil { From 4386edff0bc940543c61aa91d1f2144802ce1a08 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Wed, 1 May 2013 13:45:50 -0700 Subject: [PATCH 18/27] Better varibale names --- builder.go | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/builder.go b/builder.go index e252d3d8c0..62a7731b4b 100644 --- a/builder.go +++ b/builder.go @@ -146,20 +146,22 @@ func (builder *Builder) Build(dockerfile io.Reader, stdout io.Writer) (*Image, e if len(tmp) != 2 { return nil, fmt.Errorf("Invalid Dockerfile format") } - switch tmp[0] { + instruction := tmp[0] + arguments := tmp[1] + switch strings.ToLower(instruction) { case "from": - fmt.Fprintf(stdout, "FROM %s\n", tmp[1]) - image, err = builder.runtime.repositories.LookupImage(tmp[1]) + fmt.Fprintf(stdout, "FROM %s\n", arguments) + image, err = builder.runtime.repositories.LookupImage(arguments) if err != nil { return nil, err } break case "run": - fmt.Fprintf(stdout, "RUN %s\n", tmp[1]) + fmt.Fprintf(stdout, "RUN %s\n", arguments) if image == nil { return nil, fmt.Errorf("Please provide a source image with `from` prior to run") } - config, err := ParseRun([]string{image.Id, "/bin/sh", "-c", tmp[1]}, nil, builder.runtime.capabilities) + config, err := ParseRun([]string{image.Id, "/bin/sh", "-c", arguments}, nil, builder.runtime.capabilities) if err != nil { return nil, err } @@ -176,7 +178,7 @@ func (builder *Builder) Build(dockerfile io.Reader, stdout io.Writer) (*Image, e // Wait for it to finish if result := c.Wait(); result != 0 { - return nil, fmt.Errorf("!!! '%s' return non-zero exit code '%d'. Aborting.", tmp[1], result) + return nil, fmt.Errorf("!!! '%s' return non-zero exit code '%d'. Aborting.", arguments, result) } // Commit the container @@ -196,19 +198,21 @@ func (builder *Builder) Build(dockerfile io.Reader, stdout io.Writer) (*Image, e if image == nil { return nil, fmt.Errorf("Please provide a source image with `from` prior to copy") } - tmp2 := strings.SplitN(tmp[1], " ", 2) + tmp = strings.SplitN(arguments, " ", 2) if len(tmp) != 2 { return nil, fmt.Errorf("Invalid INSERT format") } - fmt.Fprintf(stdout, "COPY %s to %s in %s\n", tmp2[0], tmp2[1], base.ShortId()) + sourceUrl := tmp[0] + destPath := tmp[1] + fmt.Fprintf(stdout, "COPY %s to %s in %s\n", sourceUrl, destPath, base.ShortId()) - file, err := Download(tmp2[0], stdout) + file, err := Download(sourceUrl, stdout) if err != nil { return nil, err } defer file.Body.Close() - config, err := ParseRun([]string{base.Id, "echo", "insert", tmp2[0], tmp2[1]}, nil, builder.runtime.capabilities) + config, err := ParseRun([]string{base.Id, "echo", "insert", sourceUrl, destPath}, nil, builder.runtime.capabilities) if err != nil { return nil, err } @@ -223,10 +227,10 @@ func (builder *Builder) Build(dockerfile io.Reader, stdout io.Writer) (*Image, e // Wait for echo to finish if result := c.Wait(); result != 0 { - return nil, fmt.Errorf("!!! '%s' return non-zero exit code '%d'. Aborting.", tmp[1], result) + return nil, fmt.Errorf("!!! '%s' return non-zero exit code '%d'. Aborting.", arguments, result) } - if err := c.Inject(file.Body, tmp2[1]); err != nil { + if err := c.Inject(file.Body, destPath); err != nil { return nil, err } From 6c168a8986261c95ae5e036f008375f77afbe78e Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Wed, 1 May 2013 14:36:45 -0700 Subject: [PATCH 19/27] Rebase master (autorun) --- builder.go | 53 +++++++++++++++++++++-- commands.go | 2 +- container_test.go | 7 +-- runtime.go | 107 ---------------------------------------------- 4 files changed, 54 insertions(+), 115 deletions(-) diff --git a/builder.go b/builder.go index 62a7731b4b..9ecf5f4e8e 100644 --- a/builder.go +++ b/builder.go @@ -24,12 +24,57 @@ func NewBuilder(runtime *Runtime) *Builder { } } +func (builder *Builder) mergeConfig(userConf, imageConf *Config) { + if userConf.Hostname != "" { + userConf.Hostname = imageConf.Hostname + } + if userConf.User != "" { + userConf.User = imageConf.User + } + if userConf.Memory == 0 { + userConf.Memory = imageConf.Memory + } + if userConf.MemorySwap == 0 { + userConf.MemorySwap = imageConf.MemorySwap + } + if userConf.PortSpecs == nil || len(userConf.PortSpecs) == 0 { + userConf.PortSpecs = imageConf.PortSpecs + } + if !userConf.Tty { + userConf.Tty = userConf.Tty + } + if !userConf.OpenStdin { + userConf.OpenStdin = imageConf.OpenStdin + } + if !userConf.StdinOnce { + userConf.StdinOnce = imageConf.StdinOnce + } + if userConf.Env == nil || len(userConf.Env) == 0 { + userConf.Env = imageConf.Env + } + if userConf.Cmd == nil || len(userConf.Cmd) == 0 { + userConf.Cmd = imageConf.Cmd + } + if userConf.Dns == nil || len(userConf.Dns) == 0 { + userConf.Dns = imageConf.Dns + } +} + func (builder *Builder) Create(config *Config) (*Container, error) { // Lookup image img, err := builder.repositories.LookupImage(config.Image) if err != nil { return nil, err } + + if img.Config != nil { + builder.mergeConfig(config, img.Config) + } + + if config.Cmd == nil { + return nil, fmt.Errorf("No command specified") + } + // Generate id id := GenerateId() // Generate default hostname @@ -87,7 +132,7 @@ func (builder *Builder) Create(config *Config) (*Container, error) { // Commit creates a new filesystem image from the current state of a container. // The image can optionally be tagged into a repository -func (builder *Builder) Commit(container *Container, repository, tag, comment, author string) (*Image, error) { +func (builder *Builder) Commit(container *Container, repository, tag, comment, author string, config *Config) (*Image, error) { // FIXME: freeze the container before copying it to avoid data corruption? // FIXME: this shouldn't be in commands. rwTar, err := container.ExportRw() @@ -95,7 +140,7 @@ func (builder *Builder) Commit(container *Container, repository, tag, comment, a return nil, err } // Create a new image from the container's base layers + a new layer from container changes - img, err := builder.graph.Create(rwTar, container, comment, author) + img, err := builder.graph.Create(rwTar, container, comment, author, config) if err != nil { return nil, err } @@ -182,7 +227,7 @@ func (builder *Builder) Build(dockerfile io.Reader, stdout io.Writer) (*Image, e } // Commit the container - base, err = builder.Commit(c, "", "", "", "") + base, err = builder.Commit(c, "", "", "", "", nil) if err != nil { return nil, err } @@ -234,7 +279,7 @@ func (builder *Builder) Build(dockerfile io.Reader, stdout io.Writer) (*Image, e return nil, err } - base, err = builder.Commit(c, "", "", "", "") + base, err = builder.Commit(c, "", "", "", "", nil) if err != nil { return nil, err } diff --git a/commands.go b/commands.go index 08f9f7e18e..c72e24aa61 100644 --- a/commands.go +++ b/commands.go @@ -106,7 +106,7 @@ func (srv *Server) CmdInsert(stdin io.ReadCloser, stdout rcli.DockerConn, args . return err } // FIXME: Handle custom repo, tag comment, author - img, err = b.Commit(c, "", "", img.Comment, img.Author) + img, err = b.Commit(c, "", "", img.Comment, img.Author, nil) if err != nil { return err } diff --git a/container_test.go b/container_test.go index c06b8de876..5b63b2a0e7 100644 --- a/container_test.go +++ b/container_test.go @@ -225,7 +225,9 @@ func TestCommitAutoRun(t *testing.T) { t.Fatal(err) } defer nuke(runtime) - container1, err := runtime.Create( + + builder := NewBuilder(runtime) + container1, err := builder.Create( &Config{ Image: GetTestImage(runtime).Id, Cmd: []string{"/bin/sh", "-c", "echo hello > /world"}, @@ -256,8 +258,7 @@ func TestCommitAutoRun(t *testing.T) { } // FIXME: Make a TestCommit that stops here and check docker.root/layers/img.id/world - - container2, err := runtime.Create( + container2, err := builder.Create( &Config{ Image: img.Id, }, diff --git a/runtime.go b/runtime.go index ca27b7a7bf..5958aa1811 100644 --- a/runtime.go +++ b/runtime.go @@ -78,113 +78,6 @@ func (runtime *Runtime) containerRoot(id string) string { return path.Join(runtime.repository, id) } -func (runtime *Runtime) mergeConfig(userConf, imageConf *Config) { - if userConf.Hostname == "" { - userConf.Hostname = imageConf.Hostname - } - if userConf.User == "" { - userConf.User = imageConf.User - } - if userConf.Memory == 0 { - userConf.Memory = imageConf.Memory - } - if userConf.MemorySwap == 0 { - userConf.MemorySwap = imageConf.MemorySwap - } - if userConf.PortSpecs == nil || len(userConf.PortSpecs) == 0 { - userConf.PortSpecs = imageConf.PortSpecs - } - if !userConf.Tty { - userConf.Tty = userConf.Tty - } - if !userConf.OpenStdin { - userConf.OpenStdin = imageConf.OpenStdin - } - if !userConf.StdinOnce { - userConf.StdinOnce = imageConf.StdinOnce - } - if userConf.Env == nil || len(userConf.Env) == 0 { - userConf.Env = imageConf.Env - } - if userConf.Cmd == nil || len(userConf.Cmd) == 0 { - userConf.Cmd = imageConf.Cmd - } - if userConf.Dns == nil || len(userConf.Dns) == 0 { - userConf.Dns = imageConf.Dns - } -} - -func (runtime *Runtime) Create(config *Config) (*Container, error) { - // Lookup image - img, err := runtime.repositories.LookupImage(config.Image) - if err != nil { - return nil, err - } - - if img.Config != nil { - runtime.mergeConfig(config, img.Config) - } - - if config.Cmd == nil || len(config.Cmd) == 0 { - return nil, fmt.Errorf("No command specified") - } - - // Generate id - id := GenerateId() - // Generate default hostname - // FIXME: the lxc template no longer needs to set a default hostname - if config.Hostname == "" { - config.Hostname = id[:12] - } - - container := &Container{ - // FIXME: we should generate the ID here instead of receiving it as an argument - Id: id, - Created: time.Now(), - Path: config.Cmd[0], - Args: config.Cmd[1:], //FIXME: de-duplicate from config - Config: config, - Image: img.Id, // Always use the resolved image id - NetworkSettings: &NetworkSettings{}, - // FIXME: do we need to store this in the container? - SysInitPath: sysInitPath, - } - container.root = runtime.containerRoot(container.Id) - // Step 1: create the container directory. - // This doubles as a barrier to avoid race conditions. - if err := os.Mkdir(container.root, 0700); err != nil { - return nil, err - } - - // If custom dns exists, then create a resolv.conf for the container - if len(config.Dns) > 0 { - container.ResolvConfPath = path.Join(container.root, "resolv.conf") - f, err := os.Create(container.ResolvConfPath) - if err != nil { - return nil, err - } - defer f.Close() - for _, dns := range config.Dns { - if _, err := f.Write([]byte("nameserver " + dns + "\n")); err != nil { - return nil, err - } - } - } else { - container.ResolvConfPath = "/etc/resolv.conf" - } - - // Step 2: save the container json - if err := container.ToDisk(); err != nil { - return nil, err - } - // Step 3: register the container - if err := runtime.Register(container); err != nil { - return nil, err - } - return container, nil -} - -======= end func (runtime *Runtime) Load(id string) (*Container, error) { container := &Container{root: runtime.containerRoot(id)} if err := container.FromDisk(); err != nil { From 15ea5a479a78d1011cda9bd55fd442d810234eb6 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Wed, 1 May 2013 14:37:26 -0700 Subject: [PATCH 20/27] Update the TestBuild with new format --- builder_test.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/builder_test.go b/builder_test.go index 345fd2c594..d7fa921e3d 100644 --- a/builder_test.go +++ b/builder_test.go @@ -7,12 +7,12 @@ import ( const Dockerfile = ` # VERSION 0.1 -# DOCKER-VERSION 0.1.6 +# DOCKER-VERSION 0.2 -from docker-ut -run sh -c 'echo root:testpass > /tmp/passwd' -run mkdir -p /var/run/sshd -copy https://raw.github.com/dotcloud/docker/master/CHANGELOG.md /tmp/CHANGELOG.md +from docker-ut +run sh -c 'echo root:testpass > /tmp/passwd' +run mkdir -p /var/run/sshd +insert https://raw.github.com/dotcloud/docker/master/CHANGELOG.md /tmp/CHANGELOG.md ` func TestBuild(t *testing.T) { From 4ebec08add53bd9b3e10c331168daa6579ebd6db Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Wed, 1 May 2013 15:05:52 -0700 Subject: [PATCH 21/27] Trim the splited builder lines --- builder.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/builder.go b/builder.go index 9ecf5f4e8e..feaa81256c 100644 --- a/builder.go +++ b/builder.go @@ -191,8 +191,8 @@ func (builder *Builder) Build(dockerfile io.Reader, stdout io.Writer) (*Image, e if len(tmp) != 2 { return nil, fmt.Errorf("Invalid Dockerfile format") } - instruction := tmp[0] - arguments := tmp[1] + instruction := strings.Trim(tmp[0], " ") + arguments := strings.Trim(tmp[1], " ") switch strings.ToLower(instruction) { case "from": fmt.Fprintf(stdout, "FROM %s\n", arguments) @@ -247,8 +247,8 @@ func (builder *Builder) Build(dockerfile io.Reader, stdout io.Writer) (*Image, e if len(tmp) != 2 { return nil, fmt.Errorf("Invalid INSERT format") } - sourceUrl := tmp[0] - destPath := tmp[1] + sourceUrl := strings.Trim(tmp[0], "") + destPath := strings.Trim(tmp[1], " ") fmt.Fprintf(stdout, "COPY %s to %s in %s\n", sourceUrl, destPath, base.ShortId()) file, err := Download(sourceUrl, stdout) From 924b61328cd63fc8b1da77d9cb4f1e6b85eb1592 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Wed, 1 May 2013 15:06:23 -0700 Subject: [PATCH 22/27] Make the FROM instruction pull the image if not existing --- builder.go | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/builder.go b/builder.go index feaa81256c..8d94c40f95 100644 --- a/builder.go +++ b/builder.go @@ -198,8 +198,27 @@ func (builder *Builder) Build(dockerfile io.Reader, stdout io.Writer) (*Image, e fmt.Fprintf(stdout, "FROM %s\n", arguments) image, err = builder.runtime.repositories.LookupImage(arguments) if err != nil { - return nil, err + if builder.runtime.graph.IsNotExist(err) { + if builder.runtime.graph.LookupRemoteImage(arguments, builder.runtime.authConfig) { + if err := builder.runtime.graph.PullImage(stdout, arguments, builder.runtime.authConfig); err != nil { + return nil, err + } + } else { + // FIXME: Allow pull repo:tag + if err := builder.runtime.graph.PullRepository(stdout, arguments, "", builder.runtime.repositories, builder.runtime.authConfig); err != nil { + return nil, err + } + } + + image, err = builder.runtime.repositories.LookupImage(arguments) + if err != nil { + return nil, err + } + } else { + return nil, err + } } + break case "run": fmt.Fprintf(stdout, "RUN %s\n", arguments) From 6d6a03dfba37a8f1bd7304467db86ebe248ed2b5 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Wed, 1 May 2013 15:06:40 -0700 Subject: [PATCH 23/27] More consistent docker build test --- builder_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/builder_test.go b/builder_test.go index d7fa921e3d..08b7dd58cc 100644 --- a/builder_test.go +++ b/builder_test.go @@ -9,7 +9,7 @@ const Dockerfile = ` # VERSION 0.1 # DOCKER-VERSION 0.2 -from docker-ut +from ` + unitTestImageName + ` run sh -c 'echo root:testpass > /tmp/passwd' run mkdir -p /var/run/sshd insert https://raw.github.com/dotcloud/docker/master/CHANGELOG.md /tmp/CHANGELOG.md From 92e98c66af7543f9a9d6c1a223333201b39aa6dc Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Wed, 1 May 2013 16:26:26 -0700 Subject: [PATCH 24/27] Implement MAINTAINER to builder --- builder.go | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/builder.go b/builder.go index 8d94c40f95..6ab3a6f84e 100644 --- a/builder.go +++ b/builder.go @@ -168,6 +168,7 @@ func (builder *Builder) clearTmp(containers, images map[string]struct{}) { func (builder *Builder) Build(dockerfile io.Reader, stdout io.Writer) (*Image, error) { var ( image, base *Image + maintainer string tmpContainers map[string]struct{} = make(map[string]struct{}) tmpImages map[string]struct{} = make(map[string]struct{}) ) @@ -219,6 +220,10 @@ func (builder *Builder) Build(dockerfile io.Reader, stdout io.Writer) (*Image, e } } + break + case "mainainer": + fmt.Fprintf(stdout, "MAINTAINER %s\n", arguments) + maintainer = arguments break case "run": fmt.Fprintf(stdout, "RUN %s\n", arguments) @@ -246,7 +251,7 @@ func (builder *Builder) Build(dockerfile io.Reader, stdout io.Writer) (*Image, e } // Commit the container - base, err = builder.Commit(c, "", "", "", "", nil) + base, err = builder.Commit(c, "", "", "", maintainer, nil) if err != nil { return nil, err } @@ -266,7 +271,7 @@ func (builder *Builder) Build(dockerfile io.Reader, stdout io.Writer) (*Image, e if len(tmp) != 2 { return nil, fmt.Errorf("Invalid INSERT format") } - sourceUrl := strings.Trim(tmp[0], "") + sourceUrl := strings.Trim(tmp[0], " ") destPath := strings.Trim(tmp[1], " ") fmt.Fprintf(stdout, "COPY %s to %s in %s\n", sourceUrl, destPath, base.ShortId()) @@ -298,7 +303,7 @@ func (builder *Builder) Build(dockerfile io.Reader, stdout io.Writer) (*Image, e return nil, err } - base, err = builder.Commit(c, "", "", "", "", nil) + base, err = builder.Commit(c, "", "", "", maintainer, nil) if err != nil { return nil, err } From ae1e655fb1d95b5828a2b0cd67d19d090d04ff74 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Wed, 1 May 2013 16:26:46 -0700 Subject: [PATCH 25/27] Implement EXPOSE to builder --- builder.go | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/builder.go b/builder.go index 6ab3a6f84e..e49f7d4527 100644 --- a/builder.go +++ b/builder.go @@ -262,6 +262,35 @@ func (builder *Builder) Build(dockerfile io.Reader, stdout io.Writer) (*Image, e // use the base as the new image image = base + break + case "expose": + ports := strings.Split(arguments, " ") + + fmt.Fprintf(stdout, "EXPOSE %v\n", ports) + if image == nil { + return nil, fmt.Errorf("Please provide a source image with `from` prior to copy") + } + + // Create the container and start it + c, err := builder.Create(&Config{Image: image.Id, Cmd: []string{"", ""}}) + if err != nil { + return nil, err + } + if err := c.Start(); err != nil { + return nil, err + } + tmpContainers[c.Id] = struct{}{} + + // Commit the container + base, err = builder.Commit(c, "", "", "", maintainer, &Config{PortSpecs: ports}) + if err != nil { + return nil, err + } + tmpImages[base.Id] = struct{}{} + + fmt.Fprintf(stdout, "===> %s\n", base.ShortId()) + + image = base break case "insert": if image == nil { From 2bc4ad940259bd46f0ff4d3c9462893004188416 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Sun, 5 May 2013 06:27:15 -0700 Subject: [PATCH 26/27] Rebase fix --- commands.go | 1 - 1 file changed, 1 deletion(-) diff --git a/commands.go b/commands.go index c72e24aa61..442d7f55ee 100644 --- a/commands.go +++ b/commands.go @@ -11,7 +11,6 @@ import ( "net/http" "net/url" "path/filepath" - "os" "runtime" "strconv" "strings" From 62a1850c16180ac83418e044e315ba8776f6a651 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Mon, 6 May 2013 16:40:45 -0700 Subject: [PATCH 27/27] Make the autopull compatible with new registry --- builder.go | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/builder.go b/builder.go index e49f7d4527..72989a8a0a 100644 --- a/builder.go +++ b/builder.go @@ -200,21 +200,23 @@ func (builder *Builder) Build(dockerfile io.Reader, stdout io.Writer) (*Image, e image, err = builder.runtime.repositories.LookupImage(arguments) if err != nil { if builder.runtime.graph.IsNotExist(err) { - if builder.runtime.graph.LookupRemoteImage(arguments, builder.runtime.authConfig) { - if err := builder.runtime.graph.PullImage(stdout, arguments, builder.runtime.authConfig); err != nil { - return nil, err - } - } else { - // FIXME: Allow pull repo:tag - if err := builder.runtime.graph.PullRepository(stdout, arguments, "", builder.runtime.repositories, builder.runtime.authConfig); err != nil { - return nil, err - } + + var tag, remote string + if strings.Contains(remote, ":") { + remoteParts := strings.Split(remote, ":") + tag = remoteParts[1] + remote = remoteParts[0] + } + + if err := builder.runtime.graph.PullRepository(stdout, remote, tag, builder.runtime.repositories, builder.runtime.authConfig); err != nil { + return nil, err } image, err = builder.runtime.repositories.LookupImage(arguments) if err != nil { return nil, err } + } else { return nil, err }