beenull/moby
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Merge pull request #848 from dotcloud/builder_server-3

Browse source 
Improve Docker build
This commit is contained in:
Guillaume J. Charmes 2013-06-21 14:55:08 -07:00
commit de1a5a75cc
15 changed files with 1387 additions and 558 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
FIXME
View file

@ -33,3 +33,4 @@ to put them - so we put them here :)
* Caching after an ADD
* entry point config
* bring back git revision info, looks like it was lost
* Clean up the ProgressReader api, it's a PITA to use

69
api.go
View file

@ -7,15 +7,17 @@ import (
"github.com/dotcloud/docker/utils"
"github.com/gorilla/mux"
"io"
"io/ioutil"
"log"
"net"
"net/http"
"os"
"os/exec"
"strconv"
"strings"
)
const APIVERSION = 1.2
const APIVERSION = 1.3
const DEFAULTHTTPHOST string = "127.0.0.1"
const DEFAULTHTTPPORT int = 4243
@ -723,34 +725,65 @@ func postImagesGetCache(srv *Server, version float64, w http.ResponseWriter, r *
}
func postBuild(srv *Server, version float64, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
if err := r.ParseMultipartForm(4096); err != nil {
return err
if version < 1.3 {
return fmt.Errorf("Multipart upload for build is no longer supported. Please upgrade your docker client.")
}
remote := r.FormValue("t")
remoteURL := r.FormValue("remote")
repoName := r.FormValue("t")
tag := ""
if strings.Contains(remote, ":") {
remoteParts := strings.Split(remote, ":")
if strings.Contains(repoName, ":") {
remoteParts := strings.Split(repoName, ":")
tag = remoteParts[1]
remote = remoteParts[0]
repoName = remoteParts[0]
}
dockerfile, _, err := r.FormFile("Dockerfile")
if err != nil {
return err
}
var context io.Reader
context, _, err := r.FormFile("Context")
if err != nil {
if err != http.ErrMissingFile {
if remoteURL == "" {
context = r.Body
} else if utils.IsGIT(remoteURL) {
if !strings.HasPrefix(remoteURL, "git://") {
remoteURL = "https://" + remoteURL
}
root, err := ioutil.TempDir("", "docker-build-git")
if err != nil {
return err
}
}
defer os.RemoveAll(root)
if output, err := exec.Command("git", "clone", remoteURL, root).CombinedOutput(); err != nil {
return fmt.Errorf("Error trying to use git: %s (%s)", err, output)
}
c, err := Tar(root, Bzip2)
if err != nil {
return err
}
context = c
} else if utils.IsURL(remoteURL) {
f, err := utils.Download(remoteURL, ioutil.Discard)
if err != nil {
return err
}
defer f.Body.Close()
dockerFile, err := ioutil.ReadAll(f.Body)
if err != nil {
return err
}
c, err := mkBuildContext(string(dockerFile), nil)
if err != nil {
return err
}
context = c
}
b := NewBuildFile(srv, utils.NewWriteFlusher(w))
if id, err := b.Build(dockerfile, context); err != nil {
id, err := b.Build(context)
if err != nil {
fmt.Fprintf(w, "Error build: %s\n", err)
} else if remote != "" {
srv.runtime.repositories.Set(remote, tag, id, false)
return err
}
if repoName != "" {
srv.runtime.repositories.Set(repoName, tag, id, false)
}
return nil
}

98
archive.go
View file

@ -1,7 +1,9 @@
package docker
import (
"archive/tar"
"bufio"
"bytes"
"errors"
"fmt"
"github.com/dotcloud/docker/utils"
@ -10,6 +12,7 @@ import (
"os"
"os/exec"
"path"
"path/filepath"
)
type Archive io.Reader
@ -160,51 +163,60 @@ func CopyWithTar(src, dst string) error {
if err != nil {
return err
}
var dstExists bool
dstSt, err := os.Stat(dst)
if err != nil {
if !os.IsNotExist(err) {
return err
}
} else {
dstExists = true
}
// Things that can go wrong if the source is a directory
if srcSt.IsDir() {
// The destination exists and is a regular file
if dstExists && !dstSt.IsDir() {
return fmt.Errorf("Can't copy a directory over a regular file")
}
// Things that can go wrong if the source is a regular file
} else {
utils.Debugf("The destination exists, it's a directory, and doesn't end in /")
// The destination exists, it's a directory, and doesn't end in /
if dstExists && dstSt.IsDir() && dst[len(dst)-1] != '/' {
return fmt.Errorf("Can't copy a regular file over a directory %s |%s|", dst, dst[len(dst)-1])
}
}
// Create the destination
var dstDir string
if srcSt.IsDir() || dst[len(dst)-1] == '/' {
// The destination ends in /, or the source is a directory
// --> dst is the holding directory and needs to be created for -C
dstDir = dst
} else {
// The destination doesn't end in /
// --> dst is the file
dstDir = path.Dir(dst)
}
if !dstExists {
// Create the holding directory if necessary
utils.Debugf("Creating the holding directory %s", dstDir)
if err := os.MkdirAll(dstDir, 0700); err != nil && !os.IsExist(err) {
return err
}
}
if !srcSt.IsDir() {
return TarUntar(path.Dir(src), []string{path.Base(src)}, dstDir)
return CopyFileWithTar(src, dst)
}
return TarUntar(src, nil, dstDir)
// Create dst, copy src's content into it
utils.Debugf("Creating dest directory: %s", dst)
if err := os.MkdirAll(dst, 0700); err != nil && !os.IsExist(err) {
return err
}
utils.Debugf("Calling TarUntar(%s, %s)", src, dst)
return TarUntar(src, nil, dst)
}
// CopyFileWithTar emulates the behavior of the 'cp' command-line
// for a single file. It copies a regular file from path `src` to
// path `dst`, and preserves all its metadata.
//
// If `dst` ends with a trailing slash '/', the final destination path
// will be `dst/base(src)`.
func CopyFileWithTar(src, dst string) error {
utils.Debugf("CopyFileWithTar(%s, %s)", src, dst)
srcSt, err := os.Stat(src)
if err != nil {
return err
}
if srcSt.IsDir() {
return fmt.Errorf("Can't copy a directory")
}
// Clean up the trailing /
if dst[len(dst)-1] == '/' {
dst = path.Join(dst, filepath.Base(src))
}
// Create the holding directory if necessary
if err := os.MkdirAll(filepath.Dir(dst), 0700); err != nil && !os.IsExist(err) {
return err
}
buf := new(bytes.Buffer)
tw := tar.NewWriter(buf)
hdr, err := tar.FileInfoHeader(srcSt, "")
if err != nil {
return err
}
hdr.Name = filepath.Base(dst)
if err := tw.WriteHeader(hdr); err != nil {
return err
}
srcF, err := os.Open(src)
if err != nil {
return err
}
if _, err := io.Copy(tw, srcF); err != nil {
return err
}
tw.Close()
return Untar(buf, filepath.Dir(dst))
}
// CmdStream executes a command, and returns its stdout as a stream.

314
builder_client.go
View file

@ -1,314 +0,0 @@
package docker
import (
"bufio"
"encoding/json"
"fmt"
"github.com/dotcloud/docker/utils"
"io"
"net/url"
"os"
"reflect"
"strings"
)
type builderClient struct {
cli *DockerCli
image string
maintainer string
config *Config
tmpContainers map[string]struct{}
tmpImages map[string]struct{}
needCommit bool
}
func (b *builderClient) clearTmp(containers, images map[string]struct{}) {
for i := range images {
if _, _, err := b.cli.call("DELETE", "/images/"+i, nil); err != nil {
utils.Debugf("%s", err)
}
utils.Debugf("Removing image %s", i)
}
}
func (b *builderClient) CmdFrom(name string) error {
obj, statusCode, err := b.cli.call("GET", "/images/"+name+"/json", nil)
if statusCode == 404 {
remote := name
var tag string
if strings.Contains(remote, ":") {
remoteParts := strings.Split(remote, ":")
tag = remoteParts[1]
remote = remoteParts[0]
}
var out io.Writer
if os.Getenv("DEBUG") != "" {
out = os.Stdout
} else {
out = &utils.NopWriter{}
}
if err := b.cli.stream("POST", "/images/create?fromImage="+remote+"&tag="+tag, nil, out); err != nil {
return err
}
obj, _, err = b.cli.call("GET", "/images/"+name+"/json", nil)
if err != nil {
return err
}
}
if err != nil {
return err
}
img := &APIID{}
if err := json.Unmarshal(obj, img); err != nil {
return err
}
b.image = img.ID
utils.Debugf("Using image %s", b.image)
return nil
}
func (b *builderClient) CmdMaintainer(name string) error {
b.needCommit = true
b.maintainer = name
return nil
}
func (b *builderClient) CmdRun(args string) error {
if b.image == "" {
return fmt.Errorf("Please provide a source image with `from` prior to run")
}
config, _, err := ParseRun([]string{b.image, "/bin/sh", "-c", args}, nil)
if err != nil {
return err
}
cmd, env := b.config.Cmd, b.config.Env
b.config.Cmd = nil
MergeConfig(b.config, config)
body, statusCode, err := b.cli.call("POST", "/images/getCache", &APIImageConfig{ID: b.image, Config: b.config})
if err != nil {
if statusCode != 404 {
return err
}
}
if statusCode != 404 {
apiID := &APIID{}
if err := json.Unmarshal(body, apiID); err != nil {
return err
}
utils.Debugf("Use cached version")
b.image = apiID.ID
return nil
}
cid, err := b.run()
if err != nil {
return err
}
b.config.Cmd, b.config.Env = cmd, env
return b.commit(cid)
}
func (b *builderClient) CmdEnv(args string) error {
b.needCommit = true
tmp := strings.SplitN(args, " ", 2)
if len(tmp) != 2 {
return fmt.Errorf("Invalid ENV format")
}
key := strings.Trim(tmp[0], " ")
value := strings.Trim(tmp[1], " ")
for i, elem := range b.config.Env {
if strings.HasPrefix(elem, key+"=") {
b.config.Env[i] = key + "=" + value
return nil
}
}
b.config.Env = append(b.config.Env, key+"="+value)
return nil
}
func (b *builderClient) CmdCmd(args string) error {
b.needCommit = true
var cmd []string
if err := json.Unmarshal([]byte(args), &cmd); err != nil {
utils.Debugf("Error unmarshalling: %s, using /bin/sh -c", err)
b.config.Cmd = []string{"/bin/sh", "-c", args}
} else {
b.config.Cmd = cmd
}
return nil
}
func (b *builderClient) CmdExpose(args string) error {
ports := strings.Split(args, " ")
b.config.PortSpecs = append(ports, b.config.PortSpecs...)
return nil
}
func (b *builderClient) CmdInsert(args string) error {
// tmp := strings.SplitN(args, "\t ", 2)
// sourceUrl, destPath := tmp[0], tmp[1]
// v := url.Values{}
// v.Set("url", sourceUrl)
// v.Set("path", destPath)
// body, _, err := b.cli.call("POST", "/images/insert?"+v.Encode(), nil)
// if err != nil {
// return err
// }
// apiId := &APIId{}
// if err := json.Unmarshal(body, apiId); err != nil {
// return err
// }
// FIXME: Reimplement this, we need to retrieve the resulting Id
return fmt.Errorf("INSERT not implemented")
}
func (b *builderClient) run() (string, error) {
if b.image == "" {
return "", fmt.Errorf("Please provide a source image with `from` prior to run")
}
b.config.Image = b.image
body, _, err := b.cli.call("POST", "/containers/create", b.config)
if err != nil {
return "", err
}
apiRun := &APIRun{}
if err := json.Unmarshal(body, apiRun); err != nil {
return "", err
}
for _, warning := range apiRun.Warnings {
fmt.Fprintln(os.Stderr, "WARNING: ", warning)
}
//start the container
_, _, err = b.cli.call("POST", "/containers/"+apiRun.ID+"/start", nil)
if err != nil {
return "", err
}
b.tmpContainers[apiRun.ID] = struct{}{}
// Wait for it to finish
body, _, err = b.cli.call("POST", "/containers/"+apiRun.ID+"/wait", nil)
if err != nil {
return "", err
}
apiWait := &APIWait{}
if err := json.Unmarshal(body, apiWait); err != nil {
return "", err
}
if apiWait.StatusCode != 0 {
return "", fmt.Errorf("The command %v returned a non-zero code: %d", b.config.Cmd, apiWait.StatusCode)
}
return apiRun.ID, nil
}
func (b *builderClient) commit(id string) error {
if b.image == "" {
return fmt.Errorf("Please provide a source image with `from` prior to run")
}
b.config.Image = b.image
if id == "" {
cmd := b.config.Cmd
b.config.Cmd = []string{"true"}
cid, err := b.run()
if err != nil {
return err
}
id = cid
b.config.Cmd = cmd
}
// Commit the container
v := url.Values{}
v.Set("container", id)
v.Set("author", b.maintainer)
body, _, err := b.cli.call("POST", "/commit?"+v.Encode(), b.config)
if err != nil {
return err
}
apiID := &APIID{}
if err := json.Unmarshal(body, apiID); err != nil {
return err
}
b.tmpImages[apiID.ID] = struct{}{}
b.image = apiID.ID
b.needCommit = false
return nil
}
func (b *builderClient) Build(dockerfile, context io.Reader) (string, error) {
defer b.clearTmp(b.tmpContainers, b.tmpImages)
file := bufio.NewReader(dockerfile)
for {
line, err := file.ReadString('\n')
if err != nil {
if err == io.EOF {
break
}
return "", err
}
line = strings.Replace(strings.TrimSpace(line), " ", " ", 1)
// Skip comments and empty line
if len(line) == 0 || line[0] == '#' {
continue
}
tmp := strings.SplitN(line, " ", 2)
if len(tmp) != 2 {
return "", fmt.Errorf("Invalid Dockerfile format")
}
instruction := strings.ToLower(strings.Trim(tmp[0], " "))
arguments := strings.Trim(tmp[1], " ")
fmt.Fprintf(os.Stderr, "%s %s (%s)\n", strings.ToUpper(instruction), arguments, b.image)
method, exists := reflect.TypeOf(b).MethodByName("Cmd" + strings.ToUpper(instruction[:1]) + strings.ToLower(instruction[1:]))
if !exists {
fmt.Fprintf(os.Stderr, "Skipping unknown instruction %s\n", strings.ToUpper(instruction))
}
ret := method.Func.Call([]reflect.Value{reflect.ValueOf(b), reflect.ValueOf(arguments)})[0].Interface()
if ret != nil {
return "", ret.(error)
}
fmt.Fprintf(os.Stderr, "===> %v\n", b.image)
}
if b.needCommit {
if err := b.commit(""); err != nil {
return "", err
}
}
if b.image != "" {
// The build is successful, keep the temporary containers and images
for i := range b.tmpImages {
delete(b.tmpImages, i)
}
for i := range b.tmpContainers {
delete(b.tmpContainers, i)
}
fmt.Fprintf(os.Stderr, "Build finished. image id: %s\n", b.image)
return b.image, nil
}
return "", fmt.Errorf("An error occured during the build\n")
}
func NewBuilderClient(proto, addr string) BuildFile {
return &builderClient{
cli: NewDockerCli(proto, addr),
config: &Config{},
tmpContainers: make(map[string]struct{}),
tmpImages: make(map[string]struct{}),
}
}

112
buildfile.go
View file

@ -14,7 +14,7 @@ import (
)
type BuildFile interface {
Build(io.Reader, io.Reader) (string, error)
Build(io.Reader) (string, error)
CmdFrom(string) error
CmdRun(string) error
}
@ -125,8 +125,8 @@ func (b *buildFile) CmdEnv(args string) error {
if len(tmp) != 2 {
return fmt.Errorf("Invalid ENV format")
}
key := strings.Trim(tmp[0], " ")
value := strings.Trim(tmp[1], " ")
key := strings.Trim(tmp[0], " \t")
value := strings.Trim(tmp[1], " \t")
for i, elem := range b.config.Env {
if strings.HasPrefix(elem, key+"=") {
@ -165,34 +165,17 @@ func (b *buildFile) CmdCopy(args string) error {
return fmt.Errorf("COPY has been deprecated. Please use ADD instead")
}
func (b *buildFile) CmdAdd(args string) error {
if b.context == "" {
return fmt.Errorf("No context given. Impossible to use ADD")
}
tmp := strings.SplitN(args, " ", 2)
if len(tmp) != 2 {
return fmt.Errorf("Invalid ADD format")
}
orig := strings.Trim(tmp[0], " ")
dest := strings.Trim(tmp[1], " ")
cmd := b.config.Cmd
// Create the container and start it
b.config.Cmd = []string{"/bin/sh", "-c", fmt.Sprintf("#(nop) ADD %s in %s", orig, dest)}
b.config.Image = b.image
container, err := b.builder.Create(b.config)
func (b *buildFile) addRemote(container *Container, orig, dest string) error {
file, err := utils.Download(orig, ioutil.Discard)
if err != nil {
return err
}
b.tmpContainers[container.ID] = struct{}{}
fmt.Fprintf(b.out, " ---> Running in %s\n", utils.TruncateID(container.ID))
defer file.Body.Close()
if err := container.EnsureMounted(); err != nil {
return err
}
defer container.Unmount()
return container.Inject(file.Body, dest)
}
func (b *buildFile) addContext(container *Container, orig, dest string) error {
origPath := path.Join(b.context, orig)
destPath := path.Join(container.RootfsPath(), dest)
// Preserve the trailing '/'
@ -218,6 +201,46 @@ func (b *buildFile) CmdAdd(args string) error {
return err
}
}
return nil
}
func (b *buildFile) CmdAdd(args string) error {
if b.context == "" {
return fmt.Errorf("No context given. Impossible to use ADD")
}
tmp := strings.SplitN(args, " ", 2)
if len(tmp) != 2 {
return fmt.Errorf("Invalid ADD format")
}
orig := strings.Trim(tmp[0], " \t")
dest := strings.Trim(tmp[1], " \t")
cmd := b.config.Cmd
b.config.Cmd = []string{"/bin/sh", "-c", fmt.Sprintf("#(nop) ADD %s in %s", orig, dest)}
b.config.Image = b.image
// Create the container and start it
container, err := b.builder.Create(b.config)
if err != nil {
return err
}
b.tmpContainers[container.ID] = struct{}{}
if err := container.EnsureMounted(); err != nil {
return err
}
defer container.Unmount()
if utils.IsURL(orig) {
if err := b.addRemote(container, orig, dest); err != nil {
return err
}
} else {
if err := b.addContext(container, orig, dest); err != nil {
return err
}
}
if err := b.commit(container.ID, cmd, fmt.Sprintf("ADD %s in %s", orig, dest)); err != nil {
return err
}
@ -259,7 +282,9 @@ func (b *buildFile) commit(id string, autoCmd []string, comment string) error {
}
b.config.Image = b.image
if id == "" {
cmd := b.config.Cmd
b.config.Cmd = []string{"/bin/sh", "-c", "#(nop) " + comment}
defer func(cmd []string) { b.config.Cmd = cmd }(cmd)
if cache, err := b.srv.ImageGetCached(b.image, b.config); err != nil {
return err
@ -271,21 +296,17 @@ func (b *buildFile) commit(id string, autoCmd []string, comment string) error {
} else {
utils.Debugf("[BUILDER] Cache miss")
}
// Create the container and start it
container, err := b.builder.Create(b.config)
if err != nil {
return err
}
b.tmpContainers[container.ID] = struct{}{}
fmt.Fprintf(b.out, " ---> Running in %s\n", utils.TruncateID(container.ID))
id = container.ID
if err := container.EnsureMounted(); err != nil {
return err
}
defer container.Unmount()
id = container.ID
}
container := b.runtime.Get(id)
@ -306,18 +327,23 @@ func (b *buildFile) commit(id string, autoCmd []string, comment string) error {
return nil
}
func (b *buildFile) Build(dockerfile, context io.Reader) (string, error) {
if context != nil {
name, err := ioutil.TempDir("/tmp", "docker-build")
if err != nil {
return "", err
}
if err := Untar(context, name); err != nil {
return "", err
}
defer os.RemoveAll(name)
b.context = name
func (b *buildFile) Build(context io.Reader) (string, error) {
// FIXME: @creack any reason for using /tmp instead of ""?
// FIXME: @creack "name" is a terrible variable name
name, err := ioutil.TempDir("/tmp", "docker-build")
if err != nil {
return "", err
}
if err := Untar(context, name); err != nil {
return "", err
}
defer os.RemoveAll(name)
b.context = name
dockerfile, err := os.Open(path.Join(name, "Dockerfile"))
if err != nil {
return "", fmt.Errorf("Can't build a directory with no Dockerfile")
}
// FIXME: "file" is also a terrible variable name ;)
file := bufio.NewReader(dockerfile)
stepN := 0
for {
@ -329,7 +355,7 @@ func (b *buildFile) Build(dockerfile, context io.Reader) (string, error) {
return "", err
}
}
line = strings.Replace(strings.TrimSpace(line), " ", " ", 1)
line = strings.Trim(strings.Replace(line, "\t", " ", -1), " \t\r\n")
// Skip comments and empty line
if len(line) == 0 || line[0] == '#' {
continue

135
buildfile_test.go
View file

@ -1,37 +1,91 @@
package docker
import (
"github.com/dotcloud/docker/utils"
"strings"
"io/ioutil"
"testing"
)
const Dockerfile = `
# VERSION 0.1
# DOCKER-VERSION 0.2
// mkTestContext generates a build context from the contents of the provided dockerfile.
// This context is suitable for use as an argument to BuildFile.Build()
func mkTestContext(dockerfile string, files [][2]string, t *testing.T) Archive {
context, err := mkBuildContext(dockerfile, files)
if err != nil {
t.Fatal(err)
}
return context
}
from ` + unitTestImageName + `
// A testContextTemplate describes a build context and how to test it
type testContextTemplate struct {
// Contents of the Dockerfile
dockerfile string
// Additional files in the context, eg [][2]string{"./passwd", "gordon"}
files [][2]string
}
// A table of all the contexts to build and test.
// A new docker runtime will be created and torn down for each context.
var testContexts []testContextTemplate = []testContextTemplate{
{
`
from docker-ut
run sh -c 'echo root:testpass > /tmp/passwd'
run mkdir -p /var/run/sshd
`
run [ "$(cat /tmp/passwd)" = "root:testpass" ]
run [ "$(ls -d /var/run/sshd)" = "/var/run/sshd" ]
`,
nil,
},
const DockerfileNoNewLine = `
# VERSION 0.1
# DOCKER-VERSION 0.2
{
`
from docker-ut
add foo /usr/lib/bla/bar
run [ "$(cat /usr/lib/bla/bar)" = 'hello world!' ]
`,
[][2]string{{"foo", "hello world!"}},
},
from ` + unitTestImageName + `
run sh -c 'echo root:testpass > /tmp/passwd'
run mkdir -p /var/run/sshd`
{
`
from docker-ut
add f /
run [ "$(cat /f)" = "hello" ]
add f /abc
run [ "$(cat /abc)" = "hello" ]
add f /x/y/z
run [ "$(cat /x/y/z)" = "hello" ]
add f /x/y/d/
run [ "$(cat /x/y/d/f)" = "hello" ]
add d /
run [ "$(cat /ga)" = "bu" ]
add d /somewhere
run [ "$(cat /somewhere/ga)" = "bu" ]
add d /anotherplace/
run [ "$(cat /anotherplace/ga)" = "bu" ]
add d /somewheeeere/over/the/rainbooow
run [ "$(cat /somewheeeere/over/the/rainbooow/ga)" = "bu" ]
`,
[][2]string{
{"f", "hello"},
{"d/ga", "bu"},
},
},
// FIXME: test building with a context
// FIXME: test building with a local ADD as first command
{
`
from docker-ut
env FOO BAR
run [ "$FOO" = "BAR" ]
`,
nil,
},
}
// FIXME: test building with 2 successive overlapping ADD commands
func TestBuild(t *testing.T) {
dockerfiles := []string{Dockerfile, DockerfileNoNewLine}
for _, Dockerfile := range dockerfiles {
for _, ctx := range testContexts {
runtime, err := newTestRuntime()
if err != nil {
t.Fatal(err)
@ -40,50 +94,9 @@ func TestBuild(t *testing.T) {
srv := &Server{runtime: runtime}
buildfile := NewBuildFile(srv, &utils.NopWriter{})
imgID, err := buildfile.Build(strings.NewReader(Dockerfile), nil)
if err != nil {
buildfile := NewBuildFile(srv, ioutil.Discard)
if _, err := buildfile.Build(mkTestContext(ctx.dockerfile, ctx.files, t)); err != nil {
t.Fatal(err)
}
builder := NewBuilder(runtime)
container, err := builder.Create(
&Config{
Image: imgID,
Cmd: []string{"cat", "/tmp/passwd"},
},
)
if err != nil {
t.Fatal(err)
}
defer runtime.Destroy(container)
output, err := container.Output()
if err != nil {
t.Fatal(err)
}
if string(output) != "root:testpass\n" {
t.Fatalf("Unexpected output. Read '%s', expected '%s'", output, "root:testpass\n")
}
container2, err := builder.Create(
&Config{
Image: imgID,
Cmd: []string{"ls", "-d", "/var/run/sshd"},
},
)
if err != nil {
t.Fatal(err)
}
defer runtime.Destroy(container2)
output, err = container2.Output()
if err != nil {
t.Fatal(err)
}
if string(output) != "/var/run/sshd\n" {
t.Fatal("/var/run/sshd has not been created")
}
}
}

108
commands.go
View file

@ -1,6 +1,7 @@
package docker
import (
"archive/tar"
"bytes"
"encoding/json"
"flag"
@ -10,14 +11,12 @@ import (
"github.com/dotcloud/docker/utils"
"io"
"io/ioutil"
"mime/multipart"
"net"
"net/http"
"net/http/httputil"
"net/url"
"os"
"os/signal"
"path"
"path/filepath"
"reflect"
"regexp"
@ -131,8 +130,33 @@ func (cli *DockerCli) CmdInsert(args ...string) error {
return nil
}
// mkBuildContext returns an archive of an empty context with the contents
// of `dockerfile` at the path ./Dockerfile
func mkBuildContext(dockerfile string, files [][2]string) (Archive, error) {
buf := new(bytes.Buffer)
tw := tar.NewWriter(buf)
files = append(files, [2]string{"Dockerfile", dockerfile})
for _, file := range files {
name, content := file[0], file[1]
hdr := &tar.Header{
Name: name,
Size: int64(len(content)),
}
if err := tw.WriteHeader(hdr); err != nil {
return nil, err
}
if _, err := tw.Write([]byte(content)); err != nil {
return nil, err
}
}
if err := tw.Close(); err != nil {
return nil, err
}
return buf, nil
}
func (cli *DockerCli) CmdBuild(args ...string) error {
cmd := Subcmd("build", "[OPTIONS] PATH | -", "Build a new container image from the source code at PATH")
cmd := Subcmd("build", "[OPTIONS] PATH | URL | -", "Build a new container image from the source code at PATH")
tag := cmd.String("t", "", "Tag to be applied to the resulting image in case of success")
if err := cmd.Parse(args); err != nil {
return nil
@ -143,68 +167,43 @@ func (cli *DockerCli) CmdBuild(args ...string) error {
}
var (
multipartBody io.Reader
file io.ReadCloser
contextPath string
context Archive
isRemote bool
err error
)
// Init the needed component for the Multipart
buff := bytes.NewBuffer([]byte{})
multipartBody = buff
w := multipart.NewWriter(buff)
boundary := strings.NewReader("\r\n--" + w.Boundary() + "--\r\n")
compression := Bzip2
if cmd.Arg(0) == "-" {
file = os.Stdin
// As a special case, 'docker build -' will build from an empty context with the
// contents of stdin as a Dockerfile
dockerfile, err := ioutil.ReadAll(os.Stdin)
if err != nil {
return err
}
context, err = mkBuildContext(string(dockerfile), nil)
} else if utils.IsURL(cmd.Arg(0)) || utils.IsGIT(cmd.Arg(0)) {
isRemote = true
} else {
// Send Dockerfile from arg/Dockerfile (deprecate later)
f, err := os.Open(path.Join(cmd.Arg(0), "Dockerfile"))
if err != nil {
return err
}
file = f
// Send context from arg
// Create a FormFile multipart for the context if needed
// FIXME: Use NewTempArchive in order to have the size and avoid too much memory usage?
context, err := Tar(cmd.Arg(0), compression)
if err != nil {
return err
}
// NOTE: Do this in case '.' or '..' is input
absPath, err := filepath.Abs(cmd.Arg(0))
if err != nil {
return err
}
wField, err := w.CreateFormFile("Context", filepath.Base(absPath)+"."+compression.Extension())
if err != nil {
return err
}
// FIXME: Find a way to have a progressbar for the upload too
context, err = Tar(cmd.Arg(0), Uncompressed)
}
var body io.Reader
// Setup an upload progress bar
// FIXME: ProgressReader shouldn't be this annoyning to use
if context != nil {
sf := utils.NewStreamFormatter(false)
io.Copy(wField, utils.ProgressReader(ioutil.NopCloser(context), -1, os.Stdout, sf.FormatProgress("Caching Context", "%v/%v (%v)"), sf))
multipartBody = io.MultiReader(multipartBody, boundary)
body = utils.ProgressReader(ioutil.NopCloser(context), 0, os.Stderr, sf.FormatProgress("Uploading context", "%v bytes%0.0s%0.0s"), sf)
}
// Create a FormFile multipart for the Dockerfile
wField, err := w.CreateFormFile("Dockerfile", "Dockerfile")
if err != nil {
return err
}
io.Copy(wField, file)
multipartBody = io.MultiReader(multipartBody, boundary)
// Upload the build context
v := &url.Values{}
v.Set("t", *tag)
// Send the multipart request with correct content-type
req, err := http.NewRequest("POST", fmt.Sprintf("/v%g/build?%s", APIVERSION, v.Encode()), multipartBody)
if isRemote {
v.Set("remote", cmd.Arg(0))
}
req, err := http.NewRequest("POST", fmt.Sprintf("/v%g/build?%s", APIVERSION, v.Encode()), body)
if err != nil {
return err
}
req.Header.Set("Content-Type", w.FormDataContentType())
if contextPath != "" {
req.Header.Set("X-Docker-Context-Compression", compression.Flag())
fmt.Println("Uploading Context...")
if context != nil {
req.Header.Set("Content-Type", "application/tar")
}
dial, err := net.Dial(cli.proto, cli.addr)
if err != nil {
@ -217,7 +216,6 @@ func (cli *DockerCli) CmdBuild(args ...string) error {
return err
}
defer resp.Body.Close()
// Check for errors
if resp.StatusCode < 200 || resp.StatusCode >= 400 {
body, err := ioutil.ReadAll(resp.Body)

5
docker/docker.go
View file

@ -126,7 +126,7 @@ func daemon(pidfile string, protoAddrs []string, autoRestart, enableCors bool, f
for _, protoAddr := range protoAddrs {
protoAddrParts := strings.SplitN(protoAddr, "://", 2)
if protoAddrParts[0] == "unix" {
syscall.Unlink(protoAddrParts[1]);
syscall.Unlink(protoAddrParts[1])
} else if protoAddrParts[0] == "tcp" {
if !strings.HasPrefix(protoAddrParts[1], "127.0.0.1") {
log.Println("/!\\ DON'T BIND ON ANOTHER IP ADDRESS THAN 127.0.0.1 IF YOU DON'T KNOW WHAT YOU'RE DOING /!\\")
@ -139,7 +139,7 @@ func daemon(pidfile string, protoAddrs []string, autoRestart, enableCors bool, f
chErrors <- docker.ListenAndServe(protoAddrParts[0], protoAddrParts[1], server, true)
}()
}
for i :=0 ; i < len(protoAddrs); i+=1 {
for i := 0; i < len(protoAddrs); i += 1 {
err := <-chErrors
if err != nil {
return err
@ -147,4 +147,3 @@ func daemon(pidfile string, protoAddrs []string, autoRestart, enableCors bool, f
}
return nil
}

19
docs/sources/api/docker_remote_api.rst
View file

@ -19,10 +19,25 @@ Docker Remote API
2. Versions
===========
The current verson of the API is 1.2
Calling /images/<name>/insert is the same as calling /v1.2/images/<name>/insert
The current verson of the API is 1.3
Calling /images/<name>/insert is the same as calling /v1.3/images/<name>/insert
You can still call an old version of the api using /v1.0/images/<name>/insert
:doc:`docker_remote_api_v1.3`
*****************************
What's new
----------
Builder (/build):
- Simplify the upload of the build context
- Simply stream a tarball instead of multipart upload with 4 intermediary buffers
- Simpler, less memory usage, less disk usage and faster
.. Note::
The /build improvements are not reverse-compatible. Pre 1.3 clients will break on /build.
:doc:`docker_remote_api_v1.2`
*****************************

5
docs/sources/api/docker_remote_api_v1.2.rst
View file

@ -847,7 +847,7 @@ Build an image from Dockerfile via stdin
.. http:post:: /build
Build an image from Dockerfile via stdin
Build an image from Dockerfile
**Example request**:
@ -866,9 +866,12 @@ Build an image from Dockerfile via stdin
{{ STREAM }}
:query t: tag to be applied to the resulting image in case of success
:query remote: resource to fetch, as URI
:statuscode 200: no error
:statuscode 500: server error
{{ STREAM }} is the raw text output of the build command. It uses the HTTP Hijack method in order to stream.
Check auth configuration
************************

1038
docs/sources/api/docker_remote_api_v1.3.rst Normal file
View file

File diff suppressed because it is too large Load diff

14
docs/sources/commandline/command/build.rst
View file

@ -8,9 +8,11 @@
::
Usage: docker build [OPTIONS] PATH | -
Usage: docker build [OPTIONS] PATH | URL | -
Build a new container image from the source code at PATH
-t="": Tag to be applied to the resulting image in case of success.
When a single Dockerfile is given as URL, then no context is set. When a git repository is set as URL, the repository is used as context
Examples
--------
@ -27,7 +29,15 @@ Examples
.. code-block:: bash
docker build -
docker build - < Dockerfile
| This will read a Dockerfile from Stdin without context. Due to the lack of a context, no contents of any local directory will be sent to the docker daemon.
| ADD doesn't work when running in this mode due to the absence of the context, thus having no source files to copy to the container.
.. code-block:: bash
docker build github.com/creack/docker-firefox
| This will clone the github repository and use it as context. The Dockerfile at the root of the repository is used as Dockerfile.
| Note that you can specify an arbitrary git repository by using the 'git://' schema.

17
docs/sources/use/builder.rst
View file

@ -121,19 +121,7 @@ functionally equivalent to prefixing the command with `<key>=<value>`
.. note::
The environment variables will persist when a container is run from the resulting image.
2.7 INSERT
----------
``INSERT <file url> <path>``
The `INSERT` instruction will download the file from the given url to the given
path within the image. It is similar to `RUN curl -o <path> <url>`, assuming
curl was installed within the image.
.. note::
The path must include the file name.
2.8 ADD
2.7 ADD
-------
``ADD <src> <dest>``
@ -141,7 +129,7 @@ curl was installed within the image.
The `ADD` instruction will copy new files from <src> and add them to the container's filesystem at path `<dest>`.
`<src>` must be the path to a file or directory relative to the source directory being built (also called the
context of the build).
context of the build) or a remote file URL.
`<dest>` is the path at which the source will be copied in the destination container.
@ -182,7 +170,6 @@ files and directories are created with mode 0700, uid and gid 0.
RUN apt-get update
RUN apt-get install -y inotify-tools nginx apache2 openssh-server
INSERT https://raw.github.com/creack/docker-vps/master/nginx-wrapper.sh /usr/sbin/nginx-wrapper
.. code-block:: bash

2
registry/registry.go
View file

@ -314,7 +314,7 @@ func (r *Registry) opaqueRequest(method, urlStr string, body io.Reader) (*http.R
if err != nil {
return nil, err
}
req.URL.Opaque = strings.Replace(urlStr, req.URL.Scheme + ":", "", 1)
req.URL.Opaque = strings.Replace(urlStr, req.URL.Scheme+":", "", 1)
return req, err
}

8
utils/utils.go
View file

@ -636,6 +636,14 @@ func (sf *StreamFormatter) Used() bool {
return sf.used
}
func IsURL(str string) bool {
return strings.HasPrefix(str, "http://") || strings.HasPrefix(str, "https://")
}
func IsGIT(str string) bool {
return strings.HasPrefix(str, "git://") || strings.HasPrefix(str, "github.com/")
}
func CheckLocalDns() bool {
resolv, err := ioutil.ReadFile("/etc/resolv.conf")
if err != nil {