Merge pull request #472 from dotcloud/builder

+ Builder: Add support for docker builder with native API as top level command

builder.go

@@ -0,0 +1,363 @@
+package docker
+
+import (
+	"bufio"
+	"fmt"
+	"io"
+	"os"
+	"path"
+	"strings"
+	"time"
+)
+
+type Builder struct {
+	runtime      *Runtime
+	repositories *TagStore
+	graph        *Graph
+}
+
+func NewBuilder(runtime *Runtime) *Builder {
+	return &Builder{
+		runtime:      runtime,
+		graph:        runtime.graph,
+		repositories: runtime.repositories,
+	}
+}
+
+func (builder *Builder) mergeConfig(userConf, imageConf *Config) {
+	if userConf.Hostname != "" {
+		userConf.Hostname = imageConf.Hostname
+	}
+	if userConf.User != "" {
+		userConf.User = imageConf.User
+	}
+	if userConf.Memory == 0 {
+		userConf.Memory = imageConf.Memory
+	}
+	if userConf.MemorySwap == 0 {
+		userConf.MemorySwap = imageConf.MemorySwap
+	}
+	if userConf.PortSpecs == nil || len(userConf.PortSpecs) == 0 {
+		userConf.PortSpecs = imageConf.PortSpecs
+	}
+	if !userConf.Tty {
+		userConf.Tty = userConf.Tty
+	}
+	if !userConf.OpenStdin {
+		userConf.OpenStdin = imageConf.OpenStdin
+	}
+	if !userConf.StdinOnce {
+		userConf.StdinOnce = imageConf.StdinOnce
+	}
+	if userConf.Env == nil || len(userConf.Env) == 0 {
+		userConf.Env = imageConf.Env
+	}
+	if userConf.Cmd == nil || len(userConf.Cmd) == 0 {
+		userConf.Cmd = imageConf.Cmd
+	}
+	if userConf.Dns == nil || len(userConf.Dns) == 0 {
+		userConf.Dns = imageConf.Dns
+	}
+}
+
+func (builder *Builder) Create(config *Config) (*Container, error) {
+	// Lookup image
+	img, err := builder.repositories.LookupImage(config.Image)
+	if err != nil {
+		return nil, err
+	}
+
+	if img.Config != nil {
+		builder.mergeConfig(config, img.Config)
+	}
+
+	if config.Cmd == nil {
+		return nil, fmt.Errorf("No command specified")
+	}
+
+	// Generate id
+	id := GenerateId()
+	// Generate default hostname
+	// FIXME: the lxc template no longer needs to set a default hostname
+	if config.Hostname == "" {
+		config.Hostname = id[:12]
+	}
+
+	container := &Container{
+		// FIXME: we should generate the ID here instead of receiving it as an argument
+		Id:              id,
+		Created:         time.Now(),
+		Path:            config.Cmd[0],
+		Args:            config.Cmd[1:], //FIXME: de-duplicate from config
+		Config:          config,
+		Image:           img.Id, // Always use the resolved image id
+		NetworkSettings: &NetworkSettings{},
+		// FIXME: do we need to store this in the container?
+		SysInitPath: sysInitPath,
+	}
+	container.root = builder.runtime.containerRoot(container.Id)
+	// Step 1: create the container directory.
+	// This doubles as a barrier to avoid race conditions.
+	if err := os.Mkdir(container.root, 0700); err != nil {
+		return nil, err
+	}
+
+	// If custom dns exists, then create a resolv.conf for the container
+	if len(config.Dns) > 0 {
+		container.ResolvConfPath = path.Join(container.root, "resolv.conf")
+		f, err := os.Create(container.ResolvConfPath)
+		if err != nil {
+			return nil, err
+		}
+		defer f.Close()
+		for _, dns := range config.Dns {
+			if _, err := f.Write([]byte("nameserver " + dns + "\n")); err != nil {
+				return nil, err
+			}
+		}
+	} else {
+		container.ResolvConfPath = "/etc/resolv.conf"
+	}
+
+	// Step 2: save the container json
+	if err := container.ToDisk(); err != nil {
+		return nil, err
+	}
+	// Step 3: register the container
+	if err := builder.runtime.Register(container); err != nil {
+		return nil, err
+	}
+	return container, nil
+}
+
+// Commit creates a new filesystem image from the current state of a container.
+// The image can optionally be tagged into a repository
+func (builder *Builder) Commit(container *Container, repository, tag, comment, author string, config *Config) (*Image, error) {
+	// FIXME: freeze the container before copying it to avoid data corruption?
+	// FIXME: this shouldn't be in commands.
+	rwTar, err := container.ExportRw()
+	if err != nil {
+		return nil, err
+	}
+	// Create a new image from the container's base layers + a new layer from container changes
+	img, err := builder.graph.Create(rwTar, container, comment, author, config)
+	if err != nil {
+		return nil, err
+	}
+	// Register the image if needed
+	if repository != "" {
+		if err := builder.repositories.Set(repository, tag, img.Id, true); err != nil {
+			return img, err
+		}
+	}
+	return img, nil
+}
+
+func (builder *Builder) clearTmp(containers, images map[string]struct{}) {
+	for c := range containers {
+		tmp := builder.runtime.Get(c)
+		builder.runtime.Destroy(tmp)
+		Debugf("Removing container %s", c)
+	}
+	for i := range images {
+		builder.runtime.graph.Delete(i)
+		Debugf("Removing image %s", i)
+	}
+}
+
+func (builder *Builder) Build(dockerfile io.Reader, stdout io.Writer) (*Image, error) {
+	var (
+		image, base   *Image
+		maintainer    string
+		tmpContainers map[string]struct{} = make(map[string]struct{})
+		tmpImages     map[string]struct{} = make(map[string]struct{})
+	)
+	defer builder.clearTmp(tmpContainers, tmpImages)
+
+	file := bufio.NewReader(dockerfile)
+	for {
+		line, err := file.ReadString('\n')
+		if err != nil {
+			if err == io.EOF {
+				break
+			}
+			return nil, err
+		}
+		line = strings.TrimSpace(line)
+		// Skip comments and empty line
+		if len(line) == 0 || line[0] == '#' {
+			continue
+		}
+		tmp := strings.SplitN(line, " ", 2)
+		if len(tmp) != 2 {
+			return nil, fmt.Errorf("Invalid Dockerfile format")
+		}
+		instruction := strings.Trim(tmp[0], " ")
+		arguments := strings.Trim(tmp[1], " ")
+		switch strings.ToLower(instruction) {
+		case "from":
+			fmt.Fprintf(stdout, "FROM %s\n", arguments)
+			image, err = builder.runtime.repositories.LookupImage(arguments)
+			if err != nil {
+				if builder.runtime.graph.IsNotExist(err) {
+
+					var tag, remote string
+					if strings.Contains(remote, ":") {
+						remoteParts := strings.Split(remote, ":")
+						tag = remoteParts[1]
+						remote = remoteParts[0]
+					}
+
+					if err := builder.runtime.graph.PullRepository(stdout, remote, tag, builder.runtime.repositories, builder.runtime.authConfig); err != nil {
+						return nil, err
+					}
+
+					image, err = builder.runtime.repositories.LookupImage(arguments)
+					if err != nil {
+						return nil, err
+					}
+
+				} else {
+					return nil, err
+				}
+			}
+
+			break
+		case "mainainer":
+			fmt.Fprintf(stdout, "MAINTAINER %s\n", arguments)
+			maintainer = arguments
+			break
+		case "run":
+			fmt.Fprintf(stdout, "RUN %s\n", arguments)
+			if image == nil {
+				return nil, fmt.Errorf("Please provide a source image with `from` prior to run")
+			}
+			config, err := ParseRun([]string{image.Id, "/bin/sh", "-c", arguments}, nil, builder.runtime.capabilities)
+			if err != nil {
+				return nil, err
+			}
+
+			// Create the container and start it
+			c, err := builder.Create(config)
+			if err != nil {
+				return nil, err
+			}
+			if err := c.Start(); err != nil {
+				return nil, err
+			}
+			tmpContainers[c.Id] = struct{}{}
+
+			// Wait for it to finish
+			if result := c.Wait(); result != 0 {
+				return nil, fmt.Errorf("!!! '%s' return non-zero exit code '%d'. Aborting.", arguments, result)
+			}
+
+			// Commit the container
+			base, err = builder.Commit(c, "", "", "", maintainer, nil)
+			if err != nil {
+				return nil, err
+			}
+			tmpImages[base.Id] = struct{}{}
+
+			fmt.Fprintf(stdout, "===> %s\n", base.ShortId())
+
+			// use the base as the new image
+			image = base
+
+			break
+		case "expose":
+			ports := strings.Split(arguments, " ")
+
+			fmt.Fprintf(stdout, "EXPOSE %v\n", ports)
+			if image == nil {
+				return nil, fmt.Errorf("Please provide a source image with `from` prior to copy")
+			}
+
+			// Create the container and start it
+			c, err := builder.Create(&Config{Image: image.Id, Cmd: []string{"", ""}})
+			if err != nil {
+				return nil, err
+			}
+			if err := c.Start(); err != nil {
+				return nil, err
+			}
+			tmpContainers[c.Id] = struct{}{}
+
+			// Commit the container
+			base, err = builder.Commit(c, "", "", "", maintainer, &Config{PortSpecs: ports})
+			if err != nil {
+				return nil, err
+			}
+			tmpImages[base.Id] = struct{}{}
+
+			fmt.Fprintf(stdout, "===> %s\n", base.ShortId())
+
+			image = base
+			break
+		case "insert":
+			if image == nil {
+				return nil, fmt.Errorf("Please provide a source image with `from` prior to copy")
+			}
+			tmp = strings.SplitN(arguments, " ", 2)
+			if len(tmp) != 2 {
+				return nil, fmt.Errorf("Invalid INSERT format")
+			}
+			sourceUrl := strings.Trim(tmp[0], " ")
+			destPath := strings.Trim(tmp[1], " ")
+			fmt.Fprintf(stdout, "COPY %s to %s in %s\n", sourceUrl, destPath, base.ShortId())
+
+			file, err := Download(sourceUrl, stdout)
+			if err != nil {
+				return nil, err
+			}
+			defer file.Body.Close()
+
+			config, err := ParseRun([]string{base.Id, "echo", "insert", sourceUrl, destPath}, nil, builder.runtime.capabilities)
+			if err != nil {
+				return nil, err
+			}
+			c, err := builder.Create(config)
+			if err != nil {
+				return nil, err
+			}
+
+			if err := c.Start(); err != nil {
+				return nil, err
+			}
+
+			// Wait for echo to finish
+			if result := c.Wait(); result != 0 {
+				return nil, fmt.Errorf("!!! '%s' return non-zero exit code '%d'. Aborting.", arguments, result)
+			}
+
+			if err := c.Inject(file.Body, destPath); err != nil {
+				return nil, err
+			}
+
+			base, err = builder.Commit(c, "", "", "", maintainer, nil)
+			if err != nil {
+				return nil, err
+			}
+			fmt.Fprintf(stdout, "===> %s\n", base.ShortId())
+
+			image = base
+
+			break
+		default:
+			fmt.Fprintf(stdout, "Skipping unknown instruction %s\n", instruction)
+		}
+	}
+	if base != nil {
+		// The build is successful, keep the temporary containers and images
+		for i := range tmpImages {
+			delete(tmpImages, i)
+		}
+		for i := range tmpContainers {
+			delete(tmpContainers, i)
+		}
+		fmt.Fprintf(stdout, "Build finished. image id: %s\n", base.ShortId())
+	} else {
+		fmt.Fprintf(stdout, "An error occured during the build\n")
+	}
+	return base, nil
+}

builder_test.go

@@ -0,0 +1,88 @@
+package docker
+
+import (
+	"strings"
+	"testing"
+)
+
+const Dockerfile = `
+# VERSION		0.1
+# DOCKER-VERSION	0.2
+
+from   ` + unitTestImageName + `
+run    sh -c 'echo root:testpass > /tmp/passwd'
+run    mkdir -p /var/run/sshd
+insert https://raw.github.com/dotcloud/docker/master/CHANGELOG.md /tmp/CHANGELOG.md
+`
+
+func TestBuild(t *testing.T) {
+	runtime, err := newTestRuntime()
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer nuke(runtime)
+
+	builder := NewBuilder(runtime)
+
+	img, err := builder.Build(strings.NewReader(Dockerfile), &nopWriter{})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	container, err := builder.Create(
+		&Config{
+			Image: img.Id,
+			Cmd:   []string{"cat", "/tmp/passwd"},
+		},
+	)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer runtime.Destroy(container)
+
+	output, err := container.Output()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if string(output) != "root:testpass\n" {
+		t.Fatalf("Unexpected output. Read '%s', expected '%s'", output, "root:testpass\n")
+	}
+
+	container2, err := builder.Create(
+		&Config{
+			Image: img.Id,
+			Cmd:   []string{"ls", "-d", "/var/run/sshd"},
+		},
+	)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer runtime.Destroy(container2)
+
+	output, err = container2.Output()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if string(output) != "/var/run/sshd\n" {
+		t.Fatal("/var/run/sshd has not been created")
+	}
+
+	container3, err := builder.Create(
+		&Config{
+			Image: img.Id,
+			Cmd:   []string{"cat", "/tmp/CHANGELOG.md"},
+		},
+	)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer runtime.Destroy(container3)
+
+	output, err = container3.Output()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(output) == 0 {
+		t.Fatal("/tmp/CHANGELOG.md has not been copied")
+	}
+}

commands.go

@@ -34,6 +34,7 @@ func (srv *Server) Help() string {
 	help := "Usage: docker COMMAND [arg...]\n\nA self-sufficient runtime for linux containers.\n\nCommands:\n"
 	for _, cmd := range [][]string{
 		{"attach", "Attach to a running container"},
+		{"build", "Build a container from Dockerfile via stdin"},
 		{"commit", "Create a new image from a container's changes"},
 		{"diff", "Inspect changes on a container's filesystem"},
 		{"export", "Stream the contents of a container as a tar archive"},
@@ -41,6 +42,7 @@ func (srv *Server) Help() string {
 		{"images", "List images"},
 		{"import", "Create a new filesystem image from the contents of a tarball"},
 		{"info", "Display system-wide information"},
+		{"insert", "Insert a file in an image"},
 		{"inspect", "Return low-level information on a container"},
 		{"kill", "Kill a running container"},
 		{"login", "Register or Login to the docker registry server"},
@@ -64,6 +66,67 @@ func (srv *Server) Help() string {
 	return help
 }
 
+func (srv *Server) CmdInsert(stdin io.ReadCloser, stdout rcli.DockerConn, args ...string) error {
+	stdout.Flush()
+	cmd := rcli.Subcmd(stdout, "insert", "IMAGE URL PATH", "Insert a file from URL in the IMAGE at PATH")
+	if err := cmd.Parse(args); err != nil {
+		return nil
+	}
+	if cmd.NArg() != 3 {
+		cmd.Usage()
+		return nil
+	}
+	imageId := cmd.Arg(0)
+	url := cmd.Arg(1)
+	path := cmd.Arg(2)
+
+	img, err := srv.runtime.repositories.LookupImage(imageId)
+	if err != nil {
+		return err
+	}
+	file, err := Download(url, stdout)
+	if err != nil {
+		return err
+	}
+	defer file.Body.Close()
+
+	config, err := ParseRun([]string{img.Id, "echo", "insert", url, path}, nil, srv.runtime.capabilities)
+	if err != nil {
+		return err
+	}
+
+	b := NewBuilder(srv.runtime)
+	c, err := b.Create(config)
+	if err != nil {
+		return err
+	}
+
+	if err := c.Inject(ProgressReader(file.Body, int(file.ContentLength), stdout, "Downloading %v/%v (%v)"), path); err != nil {
+		return err
+	}
+	// FIXME: Handle custom repo, tag comment, author
+	img, err = b.Commit(c, "", "", img.Comment, img.Author, nil)
+	if err != nil {
+		return err
+	}
+	fmt.Fprintf(stdout, "%s\n", img.Id)
+	return nil
+}
+
+func (srv *Server) CmdBuild(stdin io.ReadCloser, stdout rcli.DockerConn, args ...string) error {
+	stdout.Flush()
+	cmd := rcli.Subcmd(stdout, "build", "-", "Build a container from Dockerfile via stdin")
+	if err := cmd.Parse(args); err != nil {
+		return nil
+	}
+	img, err := NewBuilder(srv.runtime).Build(stdin, stdout)
+	if err != nil {
+		return err
+	}
+	fmt.Fprintf(stdout, "%s\n", img.ShortId())
+	return nil
+}
+
 // 'docker login': login / register a user to registry service.
 func (srv *Server) CmdLogin(stdin io.ReadCloser, stdout rcli.DockerConn, args ...string) error {
 	// Read a line on raw terminal with support for simple backspace
@@ -776,7 +839,12 @@ func (srv *Server) CmdCommit(stdin io.ReadCloser, stdout io.Writer, args ...stri
 		}
 	}
 
-	img, err := srv.runtime.Commit(containerName, repository, tag, *flComment, *flAuthor, config)
+	container := srv.runtime.Get(containerName)
+	if container == nil {
+		return fmt.Errorf("No such container: %s", containerName)
+	}
+
+	img, err := NewBuilder(srv.runtime).Commit(container, repository, tag, *flComment, *flAuthor, config)
 	if err != nil {
 		return err
 	}
@@ -994,8 +1062,10 @@ func (srv *Server) CmdRun(stdin io.ReadCloser, stdout rcli.DockerConn, args ...s
 	// or tell the client there is no options
 	stdout.Flush()
 
+	b := NewBuilder(srv.runtime)
+
 	// Create new container
-	container, err := srv.runtime.Create(config)
+	container, err := b.Create(config)
 	if err != nil {
 		// If container not found, try to pull it
 		if srv.runtime.graph.IsNotExist(err) {
@@ -1003,7 +1073,7 @@ func (srv *Server) CmdRun(stdin io.ReadCloser, stdout rcli.DockerConn, args ...s
 			if err = srv.CmdPull(stdin, stdout, config.Image); err != nil {
 				return err
 			}
-			if container, err = srv.runtime.Create(config); err != nil {
+			if container, err = b.Create(config); err != nil {
 				return err
 			}
 		} else {

commands_test.go

@@ -339,7 +339,7 @@ func TestAttachDisconnect(t *testing.T) {
 
 	srv := &Server{runtime: runtime}
 
-	container, err := runtime.Create(
+	container, err := NewBuilder(runtime).Create(
 		&Config{
 			Image:     GetTestImage(runtime).Id,
 			Memory:    33554432,

container.go

@@ -178,6 +178,23 @@ func (settings *NetworkSettings) PortMappingHuman() string {
 	return strings.Join(mapping, ", ")
 }
 
+// Inject the io.Reader at the given path. Note: do not close the reader
+func (container *Container) Inject(file io.Reader, pth string) error {
+	// Make sure the directory exists
+	if err := os.MkdirAll(path.Join(container.rwPath(), path.Dir(pth)), 0755); err != nil {
+		return err
+	}
+	// FIXME: Handle permissions/already existing dest
+	dest, err := os.Create(path.Join(container.rwPath(), pth))
+	if err != nil {
+		return err
+	}
+	if _, err := io.Copy(dest, file); err != nil {
+		return err
+	}
+	return nil
+}
+
 func (container *Container) Cmd() *exec.Cmd {
 	return container.cmd
 }

container_test.go

@@ -20,7 +20,7 @@ func TestIdFormat(t *testing.T) {
 		t.Fatal(err)
 	}
 	defer nuke(runtime)
-	container1, err := runtime.Create(
+	container1, err := NewBuilder(runtime).Create(
 		&Config{
 			Image: GetTestImage(runtime).Id,
 			Cmd:   []string{"/bin/sh", "-c", "echo hello world"},
@@ -44,7 +44,7 @@ func TestMultipleAttachRestart(t *testing.T) {
 		t.Fatal(err)
 	}
 	defer nuke(runtime)
-	container, err := runtime.Create(
+	container, err := NewBuilder(runtime).Create(
 		&Config{
 			Image: GetTestImage(runtime).Id,
 			Cmd: []string{"/bin/sh", "-c",
@@ -148,8 +148,10 @@ func TestDiff(t *testing.T) {
 	}
 	defer nuke(runtime)
 
+	builder := NewBuilder(runtime)
+
 	// Create a container and remove a file
-	container1, err := runtime.Create(
+	container1, err := builder.Create(
 		&Config{
 			Image: GetTestImage(runtime).Id,
 			Cmd:   []string{"/bin/rm", "/etc/passwd"},
@@ -190,7 +192,7 @@ func TestDiff(t *testing.T) {
 	}
 
 	// Create a new container from the commited image
-	container2, err := runtime.Create(
+	container2, err := builder.Create(
 		&Config{
 			Image: img.Id,
 			Cmd:   []string{"cat", "/etc/passwd"},
@@ -223,7 +225,9 @@ func TestCommitAutoRun(t *testing.T) {
 		t.Fatal(err)
 	}
 	defer nuke(runtime)
-	container1, err := runtime.Create(
+
+	builder := NewBuilder(runtime)
+	container1, err := builder.Create(
 		&Config{
 			Image: GetTestImage(runtime).Id,
 			Cmd:   []string{"/bin/sh", "-c", "echo hello > /world"},
@@ -254,8 +258,7 @@ func TestCommitAutoRun(t *testing.T) {
 	}
 
 	// FIXME: Make a TestCommit that stops here and check docker.root/layers/img.id/world
-
-	container2, err := runtime.Create(
+	container2, err := builder.Create(
 		&Config{
 			Image: img.Id,
 		},
@@ -301,7 +304,10 @@ func TestCommitRun(t *testing.T) {
 		t.Fatal(err)
 	}
 	defer nuke(runtime)
-	container1, err := runtime.Create(
+
+	builder := NewBuilder(runtime)
+
+	container1, err := builder.Create(
 		&Config{
 			Image: GetTestImage(runtime).Id,
 			Cmd:   []string{"/bin/sh", "-c", "echo hello > /world"},
@@ -333,7 +339,7 @@ func TestCommitRun(t *testing.T) {
 
 	// FIXME: Make a TestCommit that stops here and check docker.root/layers/img.id/world
 
-	container2, err := runtime.Create(
+	container2, err := builder.Create(
 		&Config{
 			Image: img.Id,
 			Cmd:   []string{"cat", "/world"},
@@ -380,7 +386,7 @@ func TestStart(t *testing.T) {
 		t.Fatal(err)
 	}
 	defer nuke(runtime)
-	container, err := runtime.Create(
+	container, err := NewBuilder(runtime).Create(
 		&Config{
 			Image:     GetTestImage(runtime).Id,
 			Memory:    33554432,
@@ -419,7 +425,7 @@ func TestRun(t *testing.T) {
 		t.Fatal(err)
 	}
 	defer nuke(runtime)
-	container, err := runtime.Create(
+	container, err := NewBuilder(runtime).Create(
 		&Config{
 			Image: GetTestImage(runtime).Id,
 			Cmd:   []string{"ls", "-al"},
@@ -447,7 +453,7 @@ func TestOutput(t *testing.T) {
 		t.Fatal(err)
 	}
 	defer nuke(runtime)
-	container, err := runtime.Create(
+	container, err := NewBuilder(runtime).Create(
 		&Config{
 			Image: GetTestImage(runtime).Id,
 			Cmd:   []string{"echo", "-n", "foobar"},
@@ -472,7 +478,7 @@ func TestKillDifferentUser(t *testing.T) {
 		t.Fatal(err)
 	}
 	defer nuke(runtime)
-	container, err := runtime.Create(&Config{
+	container, err := NewBuilder(runtime).Create(&Config{
 		Image: GetTestImage(runtime).Id,
 		Cmd:   []string{"tail", "-f", "/etc/resolv.conf"},
 		User:  "daemon",
@@ -520,7 +526,7 @@ func TestKill(t *testing.T) {
 		t.Fatal(err)
 	}
 	defer nuke(runtime)
-	container, err := runtime.Create(&Config{
+	container, err := NewBuilder(runtime).Create(&Config{
 		Image: GetTestImage(runtime).Id,
 		Cmd:   []string{"cat", "/dev/zero"},
 	},
@@ -566,7 +572,9 @@ func TestExitCode(t *testing.T) {
 	}
 	defer nuke(runtime)
 
-	trueContainer, err := runtime.Create(&Config{
+	builder := NewBuilder(runtime)
+
+	trueContainer, err := builder.Create(&Config{
 		Image: GetTestImage(runtime).Id,
 		Cmd:   []string{"/bin/true", ""},
 	})
@@ -581,7 +589,7 @@ func TestExitCode(t *testing.T) {
 		t.Errorf("Unexpected exit code %d (expected 0)", trueContainer.State.ExitCode)
 	}
 
-	falseContainer, err := runtime.Create(&Config{
+	falseContainer, err := builder.Create(&Config{
 		Image: GetTestImage(runtime).Id,
 		Cmd:   []string{"/bin/false", ""},
 	})
@@ -603,7 +611,7 @@ func TestRestart(t *testing.T) {
 		t.Fatal(err)
 	}
 	defer nuke(runtime)
-	container, err := runtime.Create(&Config{
+	container, err := NewBuilder(runtime).Create(&Config{
 		Image: GetTestImage(runtime).Id,
 		Cmd:   []string{"echo", "-n", "foobar"},
 	},
@@ -636,7 +644,7 @@ func TestRestartStdin(t *testing.T) {
 		t.Fatal(err)
 	}
 	defer nuke(runtime)
-	container, err := runtime.Create(&Config{
+	container, err := NewBuilder(runtime).Create(&Config{
 		Image: GetTestImage(runtime).Id,
 		Cmd:   []string{"cat"},
 
@@ -715,8 +723,10 @@ func TestUser(t *testing.T) {
 	}
 	defer nuke(runtime)
 
+	builder := NewBuilder(runtime)
+
 	// Default user must be root
-	container, err := runtime.Create(&Config{
+	container, err := builder.Create(&Config{
 		Image: GetTestImage(runtime).Id,
 		Cmd:   []string{"id"},
 	},
@@ -734,7 +744,7 @@ func TestUser(t *testing.T) {
 	}
 
 	// Set a username
-	container, err = runtime.Create(&Config{
+	container, err = builder.Create(&Config{
 		Image: GetTestImage(runtime).Id,
 		Cmd:   []string{"id"},
 
@@ -754,7 +764,7 @@ func TestUser(t *testing.T) {
 	}
 
 	// Set a UID
-	container, err = runtime.Create(&Config{
+	container, err = builder.Create(&Config{
 		Image: GetTestImage(runtime).Id,
 		Cmd:   []string{"id"},
 
@@ -774,7 +784,7 @@ func TestUser(t *testing.T) {
 	}
 
 	// Set a different user by uid
-	container, err = runtime.Create(&Config{
+	container, err = builder.Create(&Config{
 		Image: GetTestImage(runtime).Id,
 		Cmd:   []string{"id"},
 
@@ -796,7 +806,7 @@ func TestUser(t *testing.T) {
 	}
 
 	// Set a different user by username
-	container, err = runtime.Create(&Config{
+	container, err = builder.Create(&Config{
 		Image: GetTestImage(runtime).Id,
 		Cmd:   []string{"id"},
 
@@ -823,7 +833,9 @@ func TestMultipleContainers(t *testing.T) {
 	}
 	defer nuke(runtime)
 
-	container1, err := runtime.Create(&Config{
+	builder := NewBuilder(runtime)
+
+	container1, err := builder.Create(&Config{
 		Image: GetTestImage(runtime).Id,
 		Cmd:   []string{"cat", "/dev/zero"},
 	},
@@ -833,7 +845,7 @@ func TestMultipleContainers(t *testing.T) {
 	}
 	defer runtime.Destroy(container1)
 
-	container2, err := runtime.Create(&Config{
+	container2, err := builder.Create(&Config{
 		Image: GetTestImage(runtime).Id,
 		Cmd:   []string{"cat", "/dev/zero"},
 	},
@@ -879,7 +891,7 @@ func TestStdin(t *testing.T) {
 		t.Fatal(err)
 	}
 	defer nuke(runtime)
-	container, err := runtime.Create(&Config{
+	container, err := NewBuilder(runtime).Create(&Config{
 		Image: GetTestImage(runtime).Id,
 		Cmd:   []string{"cat"},
 
@@ -926,7 +938,7 @@ func TestTty(t *testing.T) {
 		t.Fatal(err)
 	}
 	defer nuke(runtime)
-	container, err := runtime.Create(&Config{
+	container, err := NewBuilder(runtime).Create(&Config{
 		Image: GetTestImage(runtime).Id,
 		Cmd:   []string{"cat"},
 
@@ -973,7 +985,7 @@ func TestEnv(t *testing.T) {
 		t.Fatal(err)
 	}
 	defer nuke(runtime)
-	container, err := runtime.Create(&Config{
+	container, err := NewBuilder(runtime).Create(&Config{
 		Image: GetTestImage(runtime).Id,
 		Cmd:   []string{"/usr/bin/env"},
 	},
@@ -1047,7 +1059,7 @@ func TestLXCConfig(t *testing.T) {
 	memMin := 33554432
 	memMax := 536870912
 	mem := memMin + rand.Intn(memMax-memMin)
-	container, err := runtime.Create(&Config{
+	container, err := NewBuilder(runtime).Create(&Config{
 		Image: GetTestImage(runtime).Id,
 		Cmd:   []string{"/bin/true"},
 
@@ -1074,7 +1086,7 @@ func BenchmarkRunSequencial(b *testing.B) {
 	}
 	defer nuke(runtime)
 	for i := 0; i < b.N; i++ {
-		container, err := runtime.Create(&Config{
+		container, err := NewBuilder(runtime).Create(&Config{
 			Image: GetTestImage(runtime).Id,
 			Cmd:   []string{"echo", "-n", "foo"},
 		},
@@ -1109,7 +1121,7 @@ func BenchmarkRunParallel(b *testing.B) {
 		complete := make(chan error)
 		tasks = append(tasks, complete)
 		go func(i int, complete chan error) {
-			container, err := runtime.Create(&Config{
+			container, err := NewBuilder(runtime).Create(&Config{
 				Image: GetTestImage(runtime).Id,
 				Cmd:   []string{"echo", "-n", "foo"},
 			},

docs/sources/builder/basics.rst

@@ -0,0 +1,91 @@
+==============
+Docker Builder
+==============
+
+.. contents:: Table of Contents
+
+1. Format
+=========
+
+The Docker builder format is quite simple:
+
+    ``instruction arguments``
+
+The first instruction must be `FROM`
+
+All instruction are to be placed in a file named `Dockerfile`
+
+In order to place comments within a Dockerfile, simply prefix the line with "`#`"
+
+2. Instructions
+===============
+
+Docker builder comes with a set of instructions:
+
+1. FROM: Set from what image to build
+2. RUN: Execute a command
+3. INSERT: Insert a remote file (http) into the image
+
+2.1 FROM
+--------
+    ``FROM <image>``
+
+The `FROM` instruction must be the first one in order for Builder to know from where to run commands.
+
+`FROM` can also be used in order to build multiple images within a single Dockerfile
+
+2.2 RUN
+-------
+    ``RUN <command>``
+
+The `RUN` instruction is the main one, it allows you to execute any commands on the `FROM` image and to save the results.
+You can use as many `RUN` as you want within a Dockerfile, the commands will be executed on the result of the previous command.
+
+2.3 INSERT
+----------
+
+    ``INSERT <file url> <path>``
+
+The `INSERT` instruction will download the file at the given url and place it within the image at the given path.
+
+.. note::
+    The path must include the file name.
+
+3. Dockerfile Examples
+======================
+
+::
+
+    # Nginx
+    #
+    # VERSION               0.0.1
+    # DOCKER-VERSION        0.2
+    
+    from ubuntu
+
+    # make sure the package repository is up to date
+    run echo "deb http://archive.ubuntu.com/ubuntu precise main universe" > /etc/apt/sources.list
+    run apt-get update
+    
+    run apt-get install -y inotify-tools nginx apache openssh-server
+    insert https://raw.github.com/creack/docker-vps/master/nginx-wrapper.sh /usr/sbin/nginx-wrapper
+
+::
+
+    # Firefox over VNC
+    #
+    # VERSION               0.3
+    # DOCKER-VERSION        0.2
+    
+    from ubuntu
+    # make sure the package repository is up to date
+    run echo "deb http://archive.ubuntu.com/ubuntu precise main universe" > /etc/apt/sources.list
+    run apt-get update
+    
+    # Install vnc, xvfb in order to create a 'fake' display and firefox
+    run apt-get install -y x11vnc xvfb firefox
+    run mkdir /.vnc
+    # Setup a password
+    run x11vnc -storepasswd 1234 ~/.vnc/passwd
+    # Autostart firefox (might not be the best way to do it, but it does the trick)
+    run bash -c 'echo "firefox" >> /.bashrc'

docs/sources/builder/index.rst

@@ -0,0 +1,14 @@
+:title: docker documentation
+:description: Documentation for docker builder
+:keywords: docker, builder, dockerfile
+
+
+Builder
+=======
+
+Contents:
+
+.. toctree::
+  :maxdepth: 2
+
+  basics

docs/sources/commandline/cli.rst

@@ -27,6 +27,7 @@ Available Commands
    :maxdepth: 1
 
    command/attach
+   command/build
    command/commit
    command/diff
    command/export

docs/sources/commandline/command/build.rst

@@ -0,0 +1,9 @@
+===========================================
+``build`` -- Build a container from Dockerfile via stdin
+===========================================
+
+::
+
+    Usage: docker build -
+    Example: cat Dockerfile | docker build -
+    Build a new image from the Dockerfile passed via stdin

docs/sources/index.rst

@@ -17,7 +17,8 @@ This documentation has the following resources:
    commandline/index
    registry/index
    index/index
+   builder/index
    faq
 
 
-.. image:: http://www.docker.io/_static/lego_docker.jpg
+.. image:: http://www.docker.io/_static/lego_docker.jpg

runtime.go

@@ -12,7 +12,6 @@ import (
 	"path"
 	"sort"
 	"strings"
-	"time"
 )
 
 type Capabilities struct {
@@ -79,114 +78,6 @@ func (runtime *Runtime) containerRoot(id string) string {
 	return path.Join(runtime.repository, id)
 }
 
-func (runtime *Runtime) mergeConfig(userConf, imageConf *Config) {
-	if userConf.Hostname == "" {
-		userConf.Hostname = imageConf.Hostname
-	}
-	if userConf.User == "" {
-		userConf.User = imageConf.User
-	}
-	if userConf.Memory == 0 {
-		userConf.Memory = imageConf.Memory
-	}
-	if userConf.MemorySwap == 0 {
-		userConf.MemorySwap = imageConf.MemorySwap
-	}
-	if userConf.PortSpecs == nil || len(userConf.PortSpecs) == 0 {
-		userConf.PortSpecs = imageConf.PortSpecs
-	}
-	if !userConf.Tty {
-		userConf.Tty = userConf.Tty
-	}
-	if !userConf.OpenStdin {
-		userConf.OpenStdin = imageConf.OpenStdin
-	}
-	if !userConf.StdinOnce {
-		userConf.StdinOnce = imageConf.StdinOnce
-	}
-	if userConf.Env == nil || len(userConf.Env) == 0 {
-		userConf.Env = imageConf.Env
-	}
-	if userConf.Cmd == nil || len(userConf.Cmd) == 0 {
-		userConf.Cmd = imageConf.Cmd
-	}
-	if userConf.Dns == nil || len(userConf.Dns) == 0 {
-		userConf.Dns = imageConf.Dns
-	}
-}
-
-func (runtime *Runtime) Create(config *Config) (*Container, error) {
-
-	// Lookup image
-	img, err := runtime.repositories.LookupImage(config.Image)
-	if err != nil {
-		return nil, err
-	}
-
-	if img.Config != nil {
-		runtime.mergeConfig(config, img.Config)
-	}
-
-	if config.Cmd == nil || len(config.Cmd) == 0 {
-		return nil, fmt.Errorf("No command specified")
-	}
-
-	// Generate id
-	id := GenerateId()
-	// Generate default hostname
-	// FIXME: the lxc template no longer needs to set a default hostname
-	if config.Hostname == "" {
-		config.Hostname = id[:12]
-	}
-
-	container := &Container{
-		// FIXME: we should generate the ID here instead of receiving it as an argument
-		Id:              id,
-		Created:         time.Now(),
-		Path:            config.Cmd[0],
-		Args:            config.Cmd[1:], //FIXME: de-duplicate from config
-		Config:          config,
-		Image:           img.Id, // Always use the resolved image id
-		NetworkSettings: &NetworkSettings{},
-		// FIXME: do we need to store this in the container?
-		SysInitPath: sysInitPath,
-	}
-
-	container.root = runtime.containerRoot(container.Id)
-	// Step 1: create the container directory.
-	// This doubles as a barrier to avoid race conditions.
-	if err := os.Mkdir(container.root, 0700); err != nil {
-		return nil, err
-	}
-
-	// If custom dns exists, then create a resolv.conf for the container
-	if len(config.Dns) > 0 {
-		container.ResolvConfPath = path.Join(container.root, "resolv.conf")
-		f, err := os.Create(container.ResolvConfPath)
-		if err != nil {
-			return nil, err
-		}
-		defer f.Close()
-		for _, dns := range config.Dns {
-			if _, err := f.Write([]byte("nameserver " + dns + "\n")); err != nil {
-				return nil, err
-			}
-		}
-	} else {
-		container.ResolvConfPath = "/etc/resolv.conf"
-	}
-
-	// Step 2: save the container json
-	if err := container.ToDisk(); err != nil {
-		return nil, err
-	}
-	// Step 3: register the container
-	if err := runtime.Register(container); err != nil {
-		return nil, err
-	}
-	return container, nil
-}
-
 func (runtime *Runtime) Load(id string) (*Container, error) {
 	container := &Container{root: runtime.containerRoot(id)}
 	if err := container.FromDisk(); err != nil {
@@ -311,33 +202,6 @@ func (runtime *Runtime) Destroy(container *Container) error {
 	return nil
 }
 
-// Commit creates a new filesystem image from the current state of a container.
-// The image can optionally be tagged into a repository
-func (runtime *Runtime) Commit(id, repository, tag, comment, author string, config *Config) (*Image, error) {
-	container := runtime.Get(id)
-	if container == nil {
-		return nil, fmt.Errorf("No such container: %s", id)
-	}
-	// FIXME: freeze the container before copying it to avoid data corruption?
-	// FIXME: this shouldn't be in commands.
-	rwTar, err := container.ExportRw()
-	if err != nil {
-		return nil, err
-	}
-	// Create a new image from the container's base layers + a new layer from container changes
-	img, err := runtime.graph.Create(rwTar, container, comment, author, config)
-	if err != nil {
-		return nil, err
-	}
-	// Register the image if needed
-	if repository != "" {
-		if err := runtime.repositories.Set(repository, tag, img.Id, true); err != nil {
-			return img, err
-		}
-	}
-	return img, nil
-}
-
 func (runtime *Runtime) restore() error {
 	dir, err := ioutil.ReadDir(runtime.repository)
 	if err != nil {

runtime_test.go

@@ -118,7 +118,7 @@ func TestRuntimeCreate(t *testing.T) {
 	if len(runtime.List()) != 0 {
 		t.Errorf("Expected 0 containers, %v found", len(runtime.List()))
 	}
-	container, err := runtime.Create(&Config{
+	container, err := NewBuilder(runtime).Create(&Config{
 		Image: GetTestImage(runtime).Id,
 		Cmd:   []string{"ls", "-al"},
 	},
@@ -165,7 +165,7 @@ func TestDestroy(t *testing.T) {
 		t.Fatal(err)
 	}
 	defer nuke(runtime)
-	container, err := runtime.Create(&Config{
+	container, err := NewBuilder(runtime).Create(&Config{
 		Image: GetTestImage(runtime).Id,
 		Cmd:   []string{"ls", "-al"},
 	},
@@ -212,7 +212,10 @@ func TestGet(t *testing.T) {
 		t.Fatal(err)
 	}
 	defer nuke(runtime)
-	container1, err := runtime.Create(&Config{
+
+	builder := NewBuilder(runtime)
+
+	container1, err := builder.Create(&Config{
 		Image: GetTestImage(runtime).Id,
 		Cmd:   []string{"ls", "-al"},
 	},
@@ -222,7 +225,7 @@ func TestGet(t *testing.T) {
 	}
 	defer runtime.Destroy(container1)
 
-	container2, err := runtime.Create(&Config{
+	container2, err := builder.Create(&Config{
 		Image: GetTestImage(runtime).Id,
 		Cmd:   []string{"ls", "-al"},
 	},
@@ -232,7 +235,7 @@ func TestGet(t *testing.T) {
 	}
 	defer runtime.Destroy(container2)
 
-	container3, err := runtime.Create(&Config{
+	container3, err := builder.Create(&Config{
 		Image: GetTestImage(runtime).Id,
 		Cmd:   []string{"ls", "-al"},
 	},
@@ -262,7 +265,7 @@ func TestAllocatePortLocalhost(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	container, err := runtime.Create(&Config{
+	container, err := NewBuilder(runtime).Create(&Config{
 		Image:     GetTestImage(runtime).Id,
 		Cmd:       []string{"sh", "-c", "echo well hello there | nc -l -p 5555"},
 		PortSpecs: []string{"5555"},
@@ -325,8 +328,10 @@ func TestRestore(t *testing.T) {
 		t.Fatal(err)
 	}
 
+	builder := NewBuilder(runtime1)
+
 	// Create a container with one instance of docker
-	container1, err := runtime1.Create(&Config{
+	container1, err := builder.Create(&Config{
 		Image: GetTestImage(runtime1).Id,
 		Cmd:   []string{"ls", "-al"},
 	},
@@ -337,7 +342,7 @@ func TestRestore(t *testing.T) {
 	defer runtime1.Destroy(container1)
 
 	// Create a second container meant to be killed
-	container2, err := runtime1.Create(&Config{
+	container2, err := builder.Create(&Config{
 		Image:     GetTestImage(runtime1).Id,
 		Cmd:       []string{"/bin/cat"},
 		OpenStdin: true,

utils.go

@@ -155,6 +155,13 @@ func SelfPath() string {
 	return path
 }
 
+type nopWriter struct {
+}
+
+func (w *nopWriter) Write(buf []byte) (int, error) {
+	return len(buf), nil
+}
+
 type nopWriteCloser struct {
 	io.Writer
 }