diff --git a/AUTHORS b/AUTHORS index 7c7ba52477..7506cd885c 100644 --- a/AUTHORS +++ b/AUTHORS @@ -29,6 +29,7 @@ Dr Nic Williams Elias Probst Eric Hanchrow Evan Wies +Eric Myhre ezbercih Flavio Castelli Francisco Souza diff --git a/CHANGELOG.md b/CHANGELOG.md index 1144800150..4d6a7a00a7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,27 @@ # Changelog +## 0.4.7 (2013-06-28) + * Registry: easier push/pull to a custom registry + * Remote API: the progress bar updates faster when downloading and uploading large files + - Remote API: fix a bug in the optional unix socket transport + * Runtime: improve detection of kernel version + + Runtime: host directories can be mounted as volumes with 'docker run -b' + - Runtime: fix an issue when only attaching to stdin + * Runtime: use 'tar --numeric-owner' to avoid uid mismatch across multiple hosts + * Hack: improve test suite and dev environment + * Hack: remove dependency on unit tests on 'os/user' + + Documentation: add terminology section + +## 0.4.6 (2013-06-22) + - Runtime: fix a bug which caused creation of empty images (and volumes) to crash. + +## 0.4.5 (2013-06-21) + + Builder: 'docker build git://URL' fetches and builds a remote git repository + * Runtime: 'docker ps -s' optionally prints container size + * Tests: Improved and simplified + - Runtime: fix a regression introduced in 0.4.3 which caused the logs command to fail. + - Builder: fix a regression when using ADD with single regular file. + ## 0.4.4 (2013-06-19) - Builder: fix a regression introduced in 0.4.3 which caused builds to fail on new clients. diff --git a/FIXME b/FIXME index e182d38d30..97a0e0ebb1 100644 --- a/FIXME +++ b/FIXME @@ -33,3 +33,4 @@ to put them - so we put them here :) * Caching after an ADD * entry point config * bring back git revision info, looks like it was lost +* Clean up the ProgressReader api, it's a PITA to use diff --git a/Makefile b/Makefile index af483a03a6..9b06df3d64 100644 --- a/Makefile +++ b/Makefile @@ -2,6 +2,8 @@ DOCKER_PACKAGE := github.com/dotcloud/docker RELEASE_VERSION := $(shell git tag | grep -E "v[0-9\.]+$$" | sort -nr | head -n 1) SRCRELEASE := docker-$(RELEASE_VERSION) BINRELEASE := docker-$(RELEASE_VERSION).tgz +BUILD_SRC := build_src +BUILD_PATH := ${BUILD_SRC}/src/${DOCKER_PACKAGE} GIT_ROOT := $(shell git rev-parse --show-toplevel) BUILD_DIR := $(CURDIR)/.gopath @@ -71,8 +73,13 @@ else ifneq ($(DOCKER_DIR), $(realpath $(DOCKER_DIR))) @rm -f $(DOCKER_DIR) endif -test: all - @(cd $(DOCKER_DIR); sudo -E go test $(GO_OPTIONS)) +test: + # Copy docker source and dependencies for testing + rm -rf ${BUILD_SRC}; mkdir -p ${BUILD_PATH} + tar --exclude=${BUILD_SRC} -cz . | tar -xz -C ${BUILD_PATH} + GOPATH=${CURDIR}/${BUILD_SRC} go get -d + # Do the test + sudo -E GOPATH=${CURDIR}/${BUILD_SRC} go test ${GO_OPTIONS} testall: all @(cd $(DOCKER_DIR); sudo -E go test ./... $(GO_OPTIONS)) diff --git a/api.go b/api.go index 18c5e5c67a..4a8605856f 100644 --- a/api.go +++ b/api.go @@ -7,15 +7,17 @@ import ( "github.com/dotcloud/docker/utils" "github.com/gorilla/mux" "io" + "io/ioutil" "log" "net" "net/http" "os" + "os/exec" "strconv" "strings" ) -const APIVERSION = 1.2 +const APIVERSION = 1.3 const DEFAULTHTTPHOST string = "127.0.0.1" const DEFAULTHTTPPORT int = 4243 @@ -67,15 +69,15 @@ func writeJSON(w http.ResponseWriter, b []byte) { w.Write(b) } -// FIXME: Use stvconv.ParseBool() instead? func getBoolParam(value string) (bool, error) { - if value == "1" || strings.ToLower(value) == "true" { - return true, nil - } - if value == "" || value == "0" || strings.ToLower(value) == "false" { + if value == "" { return false, nil } - return false, fmt.Errorf("Bad parameter") + ret, err := strconv.ParseBool(value) + if err != nil { + return false, fmt.Errorf("Bad parameter") + } + return ret, nil } func getAuth(srv *Server, version float64, w http.ResponseWriter, r *http.Request, vars map[string]string) error { @@ -256,6 +258,10 @@ func getContainersJSON(srv *Server, version float64, w http.ResponseWriter, r *h if err != nil { return err } + size, err := getBoolParam(r.Form.Get("size")) + if err != nil { + return err + } since := r.Form.Get("since") before := r.Form.Get("before") n, err := strconv.Atoi(r.Form.Get("limit")) @@ -263,7 +269,7 @@ func getContainersJSON(srv *Server, version float64, w http.ResponseWriter, r *h n = -1 } - outs := srv.Containers(all, n, since, before) + outs := srv.Containers(all, size, n, since, before) b, err := json.Marshal(outs) if err != nil { return err @@ -529,7 +535,7 @@ func deleteImages(srv *Server, version float64, w http.ResponseWriter, r *http.R return err } if imgs != nil { - if len(*imgs) != 0 { + if len(imgs) != 0 { b, err := json.Marshal(imgs) if err != nil { return err @@ -545,11 +551,20 @@ func deleteImages(srv *Server, version float64, w http.ResponseWriter, r *http.R } func postContainersStart(srv *Server, version float64, w http.ResponseWriter, r *http.Request, vars map[string]string) error { + hostConfig := &HostConfig{} + + // allow a nil body for backwards compatibility + if r.Body != nil { + if err := json.NewDecoder(r.Body).Decode(hostConfig); err != nil { + return err + } + } + if vars == nil { return fmt.Errorf("Missing parameter") } name := vars["name"] - if err := srv.ContainerStart(name); err != nil { + if err := srv.ContainerStart(name, hostConfig); err != nil { return err } w.WriteHeader(http.StatusNoContent) @@ -654,7 +669,20 @@ func postContainersAttach(srv *Server, version float64, w http.ResponseWriter, r if err != nil { return err } - defer in.Close() + defer func() { + if tcpc, ok := in.(*net.TCPConn); ok { + tcpc.CloseWrite() + } else { + in.Close() + } + }() + defer func() { + if tcpc, ok := out.(*net.TCPConn); ok { + tcpc.CloseWrite() + } else if closer, ok := out.(io.Closer); ok { + closer.Close() + } + }() fmt.Fprintf(out, "HTTP/1.1 200 OK\r\nContent-Type: application/vnd.docker.raw-stream\r\n\r\n") if err := srv.ContainerAttach(name, logs, stream, stdin, stdout, stderr, in, out); err != nil { @@ -723,34 +751,65 @@ func postImagesGetCache(srv *Server, version float64, w http.ResponseWriter, r * } func postBuild(srv *Server, version float64, w http.ResponseWriter, r *http.Request, vars map[string]string) error { - if err := r.ParseMultipartForm(4096); err != nil { - return err + if version < 1.3 { + return fmt.Errorf("Multipart upload for build is no longer supported. Please upgrade your docker client.") } - remote := r.FormValue("t") + remoteURL := r.FormValue("remote") + repoName := r.FormValue("t") tag := "" - if strings.Contains(remote, ":") { - remoteParts := strings.Split(remote, ":") + if strings.Contains(repoName, ":") { + remoteParts := strings.Split(repoName, ":") tag = remoteParts[1] - remote = remoteParts[0] + repoName = remoteParts[0] } - dockerfile, _, err := r.FormFile("Dockerfile") - if err != nil { - return err - } + var context io.Reader - context, _, err := r.FormFile("Context") - if err != nil { - if err != http.ErrMissingFile { + if remoteURL == "" { + context = r.Body + } else if utils.IsGIT(remoteURL) { + if !strings.HasPrefix(remoteURL, "git://") { + remoteURL = "https://" + remoteURL + } + root, err := ioutil.TempDir("", "docker-build-git") + if err != nil { return err } - } + defer os.RemoveAll(root) + if output, err := exec.Command("git", "clone", remoteURL, root).CombinedOutput(); err != nil { + return fmt.Errorf("Error trying to use git: %s (%s)", err, output) + } + + c, err := Tar(root, Bzip2) + if err != nil { + return err + } + context = c + } else if utils.IsURL(remoteURL) { + f, err := utils.Download(remoteURL, ioutil.Discard) + if err != nil { + return err + } + defer f.Body.Close() + dockerFile, err := ioutil.ReadAll(f.Body) + if err != nil { + return err + } + c, err := mkBuildContext(string(dockerFile), nil) + if err != nil { + return err + } + context = c + } b := NewBuildFile(srv, utils.NewWriteFlusher(w)) - if id, err := b.Build(dockerfile, context); err != nil { + id, err := b.Build(context) + if err != nil { fmt.Fprintf(w, "Error build: %s\n", err) - } else if remote != "" { - srv.runtime.repositories.Set(remote, tag, id, false) + return err + } + if repoName != "" { + srv.runtime.repositories.Set(repoName, tag, id, false) } return nil } diff --git a/api_test.go b/api_test.go index 40b31d4961..4306f74100 100644 --- a/api_test.go +++ b/api_test.go @@ -17,6 +17,30 @@ import ( "time" ) +func TestGetBoolParam(t *testing.T) { + if ret, err := getBoolParam("true"); err != nil || !ret { + t.Fatalf("true -> true, nil | got %t %s", ret, err) + } + if ret, err := getBoolParam("True"); err != nil || !ret { + t.Fatalf("True -> true, nil | got %t %s", ret, err) + } + if ret, err := getBoolParam("1"); err != nil || !ret { + t.Fatalf("1 -> true, nil | got %t %s", ret, err) + } + if ret, err := getBoolParam(""); err != nil || ret { + t.Fatalf("\"\" -> false, nil | got %t %s", ret, err) + } + if ret, err := getBoolParam("false"); err != nil || ret { + t.Fatalf("false -> false, nil | got %t %s", ret, err) + } + if ret, err := getBoolParam("0"); err != nil || ret { + t.Fatalf("0 -> false, nil | got %t %s", ret, err) + } + if ret, err := getBoolParam("faux"); err == nil || ret { + t.Fatalf("faux -> false, err | got %t %s", ret, err) + } +} + func TestPostAuth(t *testing.T) { runtime, err := newTestRuntime() if err != nil { @@ -849,7 +873,8 @@ func TestPostContainersKill(t *testing.T) { } defer runtime.Destroy(container) - if err := container.Start(); err != nil { + hostConfig := &HostConfig{} + if err := container.Start(hostConfig); err != nil { t.Fatal(err) } @@ -893,7 +918,8 @@ func TestPostContainersRestart(t *testing.T) { } defer runtime.Destroy(container) - if err := container.Start(); err != nil { + hostConfig := &HostConfig{} + if err := container.Start(hostConfig); err != nil { t.Fatal(err) } @@ -949,8 +975,15 @@ func TestPostContainersStart(t *testing.T) { } defer runtime.Destroy(container) + hostConfigJSON, err := json.Marshal(&HostConfig{}) + + req, err := http.NewRequest("POST", "/containers/"+container.ID+"/start", bytes.NewReader(hostConfigJSON)) + if err != nil { + t.Fatal(err) + } + r := httptest.NewRecorder() - if err := postContainersStart(srv, APIVERSION, r, nil, map[string]string{"name": container.ID}); err != nil { + if err := postContainersStart(srv, APIVERSION, r, req, map[string]string{"name": container.ID}); err != nil { t.Fatal(err) } if r.Code != http.StatusNoContent { @@ -965,7 +998,7 @@ func TestPostContainersStart(t *testing.T) { } r = httptest.NewRecorder() - if err = postContainersStart(srv, APIVERSION, r, nil, map[string]string{"name": container.ID}); err == nil { + if err = postContainersStart(srv, APIVERSION, r, req, map[string]string{"name": container.ID}); err == nil { t.Fatalf("A running containter should be able to be started") } @@ -995,7 +1028,8 @@ func TestPostContainersStop(t *testing.T) { } defer runtime.Destroy(container) - if err := container.Start(); err != nil { + hostConfig := &HostConfig{} + if err := container.Start(hostConfig); err != nil { t.Fatal(err) } @@ -1044,7 +1078,8 @@ func TestPostContainersWait(t *testing.T) { } defer runtime.Destroy(container) - if err := container.Start(); err != nil { + hostConfig := &HostConfig{} + if err := container.Start(hostConfig); err != nil { t.Fatal(err) } @@ -1089,7 +1124,8 @@ func TestPostContainersAttach(t *testing.T) { defer runtime.Destroy(container) // Start the process - if err := container.Start(); err != nil { + hostConfig := &HostConfig{} + if err := container.Start(hostConfig); err != nil { t.Fatal(err) } diff --git a/archive.go b/archive.go index 16401e29fb..357fdd9a71 100644 --- a/archive.go +++ b/archive.go @@ -1,7 +1,9 @@ package docker import ( + "archive/tar" "bufio" + "bytes" "errors" "fmt" "github.com/dotcloud/docker/utils" @@ -10,6 +12,7 @@ import ( "os" "os/exec" "path" + "path/filepath" ) type Archive io.Reader @@ -89,7 +92,7 @@ func Tar(path string, compression Compression) (io.Reader, error) { // Tar creates an archive from the directory at `path`, only including files whose relative // paths are included in `filter`. If `filter` is nil, then all files are included. func TarFilter(path string, compression Compression, filter []string) (io.Reader, error) { - args := []string{"tar", "-f", "-", "-C", path} + args := []string{"tar", "--numeric-owner", "-f", "-", "-C", path} if filter == nil { filter = []string{"."} } @@ -105,7 +108,9 @@ func TarFilter(path string, compression Compression, filter []string) (io.Reader // identity (uncompressed), gzip, bzip2, xz. // FIXME: specify behavior when target path exists vs. doesn't exist. func Untar(archive io.Reader, path string) error { - + if archive == nil { + return fmt.Errorf("Empty archive") + } bufferedArchive := bufio.NewReaderSize(archive, 10) buf, err := bufferedArchive.Peek(10) if err != nil { @@ -115,7 +120,7 @@ func Untar(archive io.Reader, path string) error { utils.Debugf("Archive compression detected: %s", compression.Extension()) - cmd := exec.Command("tar", "-f", "-", "-C", path, "-x"+compression.Flag()) + cmd := exec.Command("tar", "--numeric-owner", "-f", "-", "-C", path, "-x"+compression.Flag()) cmd.Stdin = bufferedArchive // Hardcode locale environment for predictable outcome regardless of host configuration. // (see https://github.com/dotcloud/docker/issues/355) @@ -160,51 +165,60 @@ func CopyWithTar(src, dst string) error { if err != nil { return err } - var dstExists bool - dstSt, err := os.Stat(dst) - if err != nil { - if !os.IsNotExist(err) { - return err - } - } else { - dstExists = true - } - // Things that can go wrong if the source is a directory - if srcSt.IsDir() { - // The destination exists and is a regular file - if dstExists && !dstSt.IsDir() { - return fmt.Errorf("Can't copy a directory over a regular file") - } - // Things that can go wrong if the source is a regular file - } else { - utils.Debugf("The destination exists, it's a directory, and doesn't end in /") - // The destination exists, it's a directory, and doesn't end in / - if dstExists && dstSt.IsDir() && dst[len(dst)-1] != '/' { - return fmt.Errorf("Can't copy a regular file over a directory %s |%s|", dst, dst[len(dst)-1]) - } - } - // Create the destination - var dstDir string - if srcSt.IsDir() || dst[len(dst)-1] == '/' { - // The destination ends in /, or the source is a directory - // --> dst is the holding directory and needs to be created for -C - dstDir = dst - } else { - // The destination doesn't end in / - // --> dst is the file - dstDir = path.Dir(dst) - } - if !dstExists { - // Create the holding directory if necessary - utils.Debugf("Creating the holding directory %s", dstDir) - if err := os.MkdirAll(dstDir, 0700); err != nil && !os.IsExist(err) { - return err - } - } if !srcSt.IsDir() { - return TarUntar(path.Dir(src), []string{path.Base(src)}, dstDir) + return CopyFileWithTar(src, dst) } - return TarUntar(src, nil, dstDir) + // Create dst, copy src's content into it + utils.Debugf("Creating dest directory: %s", dst) + if err := os.MkdirAll(dst, 0700); err != nil && !os.IsExist(err) { + return err + } + utils.Debugf("Calling TarUntar(%s, %s)", src, dst) + return TarUntar(src, nil, dst) +} + +// CopyFileWithTar emulates the behavior of the 'cp' command-line +// for a single file. It copies a regular file from path `src` to +// path `dst`, and preserves all its metadata. +// +// If `dst` ends with a trailing slash '/', the final destination path +// will be `dst/base(src)`. +func CopyFileWithTar(src, dst string) error { + utils.Debugf("CopyFileWithTar(%s, %s)", src, dst) + srcSt, err := os.Stat(src) + if err != nil { + return err + } + if srcSt.IsDir() { + return fmt.Errorf("Can't copy a directory") + } + // Clean up the trailing / + if dst[len(dst)-1] == '/' { + dst = path.Join(dst, filepath.Base(src)) + } + // Create the holding directory if necessary + if err := os.MkdirAll(filepath.Dir(dst), 0700); err != nil && !os.IsExist(err) { + return err + } + buf := new(bytes.Buffer) + tw := tar.NewWriter(buf) + hdr, err := tar.FileInfoHeader(srcSt, "") + if err != nil { + return err + } + hdr.Name = filepath.Base(dst) + if err := tw.WriteHeader(hdr); err != nil { + return err + } + srcF, err := os.Open(src) + if err != nil { + return err + } + if _, err := io.Copy(tw, srcF); err != nil { + return err + } + tw.Close() + return Untar(buf, filepath.Dir(dst)) } // CmdStream executes a command, and returns its stdout as a stream. diff --git a/auth/auth.go b/auth/auth.go index 12c9471699..205b9479f5 100644 --- a/auth/auth.go +++ b/auth/auth.go @@ -74,7 +74,6 @@ func decodeAuth(authStr string) (*AuthConfig, error) { } password := strings.Trim(arr[1], "\x00") return &AuthConfig{Username: arr[0], Password: password}, nil - } // load up the auth config information and return values diff --git a/builder_client.go b/builder_client.go deleted file mode 100644 index d11e7fc995..0000000000 --- a/builder_client.go +++ /dev/null @@ -1,314 +0,0 @@ -package docker - -import ( - "bufio" - "encoding/json" - "fmt" - "github.com/dotcloud/docker/utils" - "io" - "net/url" - "os" - "reflect" - "strings" -) - -type builderClient struct { - cli *DockerCli - - image string - maintainer string - config *Config - - tmpContainers map[string]struct{} - tmpImages map[string]struct{} - - needCommit bool -} - -func (b *builderClient) clearTmp(containers, images map[string]struct{}) { - for i := range images { - if _, _, err := b.cli.call("DELETE", "/images/"+i, nil); err != nil { - utils.Debugf("%s", err) - } - utils.Debugf("Removing image %s", i) - } -} - -func (b *builderClient) CmdFrom(name string) error { - obj, statusCode, err := b.cli.call("GET", "/images/"+name+"/json", nil) - if statusCode == 404 { - - remote := name - var tag string - if strings.Contains(remote, ":") { - remoteParts := strings.Split(remote, ":") - tag = remoteParts[1] - remote = remoteParts[0] - } - var out io.Writer - if os.Getenv("DEBUG") != "" { - out = os.Stdout - } else { - out = &utils.NopWriter{} - } - if err := b.cli.stream("POST", "/images/create?fromImage="+remote+"&tag="+tag, nil, out); err != nil { - return err - } - obj, _, err = b.cli.call("GET", "/images/"+name+"/json", nil) - if err != nil { - return err - } - } - if err != nil { - return err - } - - img := &APIID{} - if err := json.Unmarshal(obj, img); err != nil { - return err - } - b.image = img.ID - utils.Debugf("Using image %s", b.image) - return nil -} - -func (b *builderClient) CmdMaintainer(name string) error { - b.needCommit = true - b.maintainer = name - return nil -} - -func (b *builderClient) CmdRun(args string) error { - if b.image == "" { - return fmt.Errorf("Please provide a source image with `from` prior to run") - } - config, _, err := ParseRun([]string{b.image, "/bin/sh", "-c", args}, nil) - if err != nil { - return err - } - - cmd, env := b.config.Cmd, b.config.Env - b.config.Cmd = nil - MergeConfig(b.config, config) - - body, statusCode, err := b.cli.call("POST", "/images/getCache", &APIImageConfig{ID: b.image, Config: b.config}) - if err != nil { - if statusCode != 404 { - return err - } - } - if statusCode != 404 { - apiID := &APIID{} - if err := json.Unmarshal(body, apiID); err != nil { - return err - } - utils.Debugf("Use cached version") - b.image = apiID.ID - return nil - } - cid, err := b.run() - if err != nil { - return err - } - b.config.Cmd, b.config.Env = cmd, env - return b.commit(cid) -} - -func (b *builderClient) CmdEnv(args string) error { - b.needCommit = true - tmp := strings.SplitN(args, " ", 2) - if len(tmp) != 2 { - return fmt.Errorf("Invalid ENV format") - } - key := strings.Trim(tmp[0], " ") - value := strings.Trim(tmp[1], " ") - - for i, elem := range b.config.Env { - if strings.HasPrefix(elem, key+"=") { - b.config.Env[i] = key + "=" + value - return nil - } - } - b.config.Env = append(b.config.Env, key+"="+value) - return nil -} - -func (b *builderClient) CmdCmd(args string) error { - b.needCommit = true - var cmd []string - if err := json.Unmarshal([]byte(args), &cmd); err != nil { - utils.Debugf("Error unmarshalling: %s, using /bin/sh -c", err) - b.config.Cmd = []string{"/bin/sh", "-c", args} - } else { - b.config.Cmd = cmd - } - return nil -} - -func (b *builderClient) CmdExpose(args string) error { - ports := strings.Split(args, " ") - b.config.PortSpecs = append(ports, b.config.PortSpecs...) - return nil -} - -func (b *builderClient) CmdInsert(args string) error { - // tmp := strings.SplitN(args, "\t ", 2) - // sourceUrl, destPath := tmp[0], tmp[1] - - // v := url.Values{} - // v.Set("url", sourceUrl) - // v.Set("path", destPath) - // body, _, err := b.cli.call("POST", "/images/insert?"+v.Encode(), nil) - // if err != nil { - // return err - // } - - // apiId := &APIId{} - // if err := json.Unmarshal(body, apiId); err != nil { - // return err - // } - - // FIXME: Reimplement this, we need to retrieve the resulting Id - return fmt.Errorf("INSERT not implemented") -} - -func (b *builderClient) run() (string, error) { - if b.image == "" { - return "", fmt.Errorf("Please provide a source image with `from` prior to run") - } - b.config.Image = b.image - body, _, err := b.cli.call("POST", "/containers/create", b.config) - if err != nil { - return "", err - } - - apiRun := &APIRun{} - if err := json.Unmarshal(body, apiRun); err != nil { - return "", err - } - for _, warning := range apiRun.Warnings { - fmt.Fprintln(os.Stderr, "WARNING: ", warning) - } - - //start the container - _, _, err = b.cli.call("POST", "/containers/"+apiRun.ID+"/start", nil) - if err != nil { - return "", err - } - b.tmpContainers[apiRun.ID] = struct{}{} - - // Wait for it to finish - body, _, err = b.cli.call("POST", "/containers/"+apiRun.ID+"/wait", nil) - if err != nil { - return "", err - } - apiWait := &APIWait{} - if err := json.Unmarshal(body, apiWait); err != nil { - return "", err - } - if apiWait.StatusCode != 0 { - return "", fmt.Errorf("The command %v returned a non-zero code: %d", b.config.Cmd, apiWait.StatusCode) - } - - return apiRun.ID, nil -} - -func (b *builderClient) commit(id string) error { - if b.image == "" { - return fmt.Errorf("Please provide a source image with `from` prior to run") - } - b.config.Image = b.image - - if id == "" { - cmd := b.config.Cmd - b.config.Cmd = []string{"true"} - cid, err := b.run() - if err != nil { - return err - } - id = cid - b.config.Cmd = cmd - } - - // Commit the container - v := url.Values{} - v.Set("container", id) - v.Set("author", b.maintainer) - - body, _, err := b.cli.call("POST", "/commit?"+v.Encode(), b.config) - if err != nil { - return err - } - apiID := &APIID{} - if err := json.Unmarshal(body, apiID); err != nil { - return err - } - b.tmpImages[apiID.ID] = struct{}{} - b.image = apiID.ID - b.needCommit = false - return nil -} - -func (b *builderClient) Build(dockerfile, context io.Reader) (string, error) { - defer b.clearTmp(b.tmpContainers, b.tmpImages) - file := bufio.NewReader(dockerfile) - for { - line, err := file.ReadString('\n') - if err != nil { - if err == io.EOF { - break - } - return "", err - } - line = strings.Replace(strings.TrimSpace(line), " ", " ", 1) - // Skip comments and empty line - if len(line) == 0 || line[0] == '#' { - continue - } - tmp := strings.SplitN(line, " ", 2) - if len(tmp) != 2 { - return "", fmt.Errorf("Invalid Dockerfile format") - } - instruction := strings.ToLower(strings.Trim(tmp[0], " ")) - arguments := strings.Trim(tmp[1], " ") - - fmt.Fprintf(os.Stderr, "%s %s (%s)\n", strings.ToUpper(instruction), arguments, b.image) - - method, exists := reflect.TypeOf(b).MethodByName("Cmd" + strings.ToUpper(instruction[:1]) + strings.ToLower(instruction[1:])) - if !exists { - fmt.Fprintf(os.Stderr, "Skipping unknown instruction %s\n", strings.ToUpper(instruction)) - } - ret := method.Func.Call([]reflect.Value{reflect.ValueOf(b), reflect.ValueOf(arguments)})[0].Interface() - if ret != nil { - return "", ret.(error) - } - - fmt.Fprintf(os.Stderr, "===> %v\n", b.image) - } - if b.needCommit { - if err := b.commit(""); err != nil { - return "", err - } - } - if b.image != "" { - // The build is successful, keep the temporary containers and images - for i := range b.tmpImages { - delete(b.tmpImages, i) - } - for i := range b.tmpContainers { - delete(b.tmpContainers, i) - } - fmt.Fprintf(os.Stderr, "Build finished. image id: %s\n", b.image) - return b.image, nil - } - return "", fmt.Errorf("An error occured during the build\n") -} - -func NewBuilderClient(proto, addr string) BuildFile { - return &builderClient{ - cli: NewDockerCli(proto, addr), - config: &Config{}, - tmpContainers: make(map[string]struct{}), - tmpImages: make(map[string]struct{}), - } -} diff --git a/buildfile.go b/buildfile.go index b8ac55640e..63625e1a40 100644 --- a/buildfile.go +++ b/buildfile.go @@ -14,7 +14,7 @@ import ( ) type BuildFile interface { - Build(io.Reader, io.Reader) (string, error) + Build(io.Reader) (string, error) CmdFrom(string) error CmdRun(string) error } @@ -87,7 +87,7 @@ func (b *buildFile) CmdRun(args string) error { if b.image == "" { return fmt.Errorf("Please provide a source image with `from` prior to run") } - config, _, err := ParseRun([]string{b.image, "/bin/sh", "-c", args}, nil) + config, _, _, err := ParseRun([]string{b.image, "/bin/sh", "-c", args}, nil) if err != nil { return err } @@ -125,8 +125,8 @@ func (b *buildFile) CmdEnv(args string) error { if len(tmp) != 2 { return fmt.Errorf("Invalid ENV format") } - key := strings.Trim(tmp[0], " ") - value := strings.Trim(tmp[1], " ") + key := strings.Trim(tmp[0], " \t") + value := strings.Trim(tmp[1], " \t") for i, elem := range b.config.Env { if strings.HasPrefix(elem, key+"=") { @@ -165,34 +165,17 @@ func (b *buildFile) CmdCopy(args string) error { return fmt.Errorf("COPY has been deprecated. Please use ADD instead") } -func (b *buildFile) CmdAdd(args string) error { - if b.context == "" { - return fmt.Errorf("No context given. Impossible to use ADD") - } - tmp := strings.SplitN(args, " ", 2) - if len(tmp) != 2 { - return fmt.Errorf("Invalid ADD format") - } - orig := strings.Trim(tmp[0], " ") - dest := strings.Trim(tmp[1], " ") - - cmd := b.config.Cmd - - // Create the container and start it - b.config.Cmd = []string{"/bin/sh", "-c", fmt.Sprintf("#(nop) ADD %s in %s", orig, dest)} - b.config.Image = b.image - container, err := b.builder.Create(b.config) +func (b *buildFile) addRemote(container *Container, orig, dest string) error { + file, err := utils.Download(orig, ioutil.Discard) if err != nil { return err } - b.tmpContainers[container.ID] = struct{}{} - fmt.Fprintf(b.out, " ---> Running in %s\n", utils.TruncateID(container.ID)) + defer file.Body.Close() - if err := container.EnsureMounted(); err != nil { - return err - } - defer container.Unmount() + return container.Inject(file.Body, dest) +} +func (b *buildFile) addContext(container *Container, orig, dest string) error { origPath := path.Join(b.context, orig) destPath := path.Join(container.RootfsPath(), dest) // Preserve the trailing '/' @@ -218,6 +201,46 @@ func (b *buildFile) CmdAdd(args string) error { return err } } + return nil +} + +func (b *buildFile) CmdAdd(args string) error { + if b.context == "" { + return fmt.Errorf("No context given. Impossible to use ADD") + } + tmp := strings.SplitN(args, " ", 2) + if len(tmp) != 2 { + return fmt.Errorf("Invalid ADD format") + } + orig := strings.Trim(tmp[0], " \t") + dest := strings.Trim(tmp[1], " \t") + + cmd := b.config.Cmd + b.config.Cmd = []string{"/bin/sh", "-c", fmt.Sprintf("#(nop) ADD %s in %s", orig, dest)} + + b.config.Image = b.image + // Create the container and start it + container, err := b.builder.Create(b.config) + if err != nil { + return err + } + b.tmpContainers[container.ID] = struct{}{} + + if err := container.EnsureMounted(); err != nil { + return err + } + defer container.Unmount() + + if utils.IsURL(orig) { + if err := b.addRemote(container, orig, dest); err != nil { + return err + } + } else { + if err := b.addContext(container, orig, dest); err != nil { + return err + } + } + if err := b.commit(container.ID, cmd, fmt.Sprintf("ADD %s in %s", orig, dest)); err != nil { return err } @@ -240,7 +263,8 @@ func (b *buildFile) run() (string, error) { fmt.Fprintf(b.out, " ---> Running in %s\n", utils.TruncateID(c.ID)) //start the container - if err := c.Start(); err != nil { + hostConfig := &HostConfig{} + if err := c.Start(hostConfig); err != nil { return "", err } @@ -259,7 +283,9 @@ func (b *buildFile) commit(id string, autoCmd []string, comment string) error { } b.config.Image = b.image if id == "" { + cmd := b.config.Cmd b.config.Cmd = []string{"/bin/sh", "-c", "#(nop) " + comment} + defer func(cmd []string) { b.config.Cmd = cmd }(cmd) if cache, err := b.srv.ImageGetCached(b.image, b.config); err != nil { return err @@ -271,21 +297,17 @@ func (b *buildFile) commit(id string, autoCmd []string, comment string) error { } else { utils.Debugf("[BUILDER] Cache miss") } - - // Create the container and start it container, err := b.builder.Create(b.config) if err != nil { return err } b.tmpContainers[container.ID] = struct{}{} fmt.Fprintf(b.out, " ---> Running in %s\n", utils.TruncateID(container.ID)) - + id = container.ID if err := container.EnsureMounted(); err != nil { return err } defer container.Unmount() - - id = container.ID } container := b.runtime.Get(id) @@ -306,18 +328,23 @@ func (b *buildFile) commit(id string, autoCmd []string, comment string) error { return nil } -func (b *buildFile) Build(dockerfile, context io.Reader) (string, error) { - if context != nil { - name, err := ioutil.TempDir("/tmp", "docker-build") - if err != nil { - return "", err - } - if err := Untar(context, name); err != nil { - return "", err - } - defer os.RemoveAll(name) - b.context = name +func (b *buildFile) Build(context io.Reader) (string, error) { + // FIXME: @creack any reason for using /tmp instead of ""? + // FIXME: @creack "name" is a terrible variable name + name, err := ioutil.TempDir("/tmp", "docker-build") + if err != nil { + return "", err } + if err := Untar(context, name); err != nil { + return "", err + } + defer os.RemoveAll(name) + b.context = name + dockerfile, err := os.Open(path.Join(name, "Dockerfile")) + if err != nil { + return "", fmt.Errorf("Can't build a directory with no Dockerfile") + } + // FIXME: "file" is also a terrible variable name ;) file := bufio.NewReader(dockerfile) stepN := 0 for { @@ -329,7 +356,7 @@ func (b *buildFile) Build(dockerfile, context io.Reader) (string, error) { return "", err } } - line = strings.Replace(strings.TrimSpace(line), " ", " ", 1) + line = strings.Trim(strings.Replace(line, "\t", " ", -1), " \t\r\n") // Skip comments and empty line if len(line) == 0 || line[0] == '#' { continue diff --git a/buildfile_test.go b/buildfile_test.go index 33e6a3146b..c2ae79f199 100644 --- a/buildfile_test.go +++ b/buildfile_test.go @@ -1,89 +1,109 @@ package docker import ( - "github.com/dotcloud/docker/utils" - "strings" + "io/ioutil" + "sync" "testing" + "fmt" ) -const Dockerfile = ` -# VERSION 0.1 -# DOCKER-VERSION 0.2 +// mkTestContext generates a build context from the contents of the provided dockerfile. +// This context is suitable for use as an argument to BuildFile.Build() +func mkTestContext(dockerfile string, files [][2]string, t *testing.T) Archive { + context, err := mkBuildContext(fmt.Sprintf(dockerfile, unitTestImageId), files) + if err != nil { + t.Fatal(err) + } + return context +} -from ` + unitTestImageName + ` +// A testContextTemplate describes a build context and how to test it +type testContextTemplate struct { + // Contents of the Dockerfile + dockerfile string + // Additional files in the context, eg [][2]string{"./passwd", "gordon"} + files [][2]string +} + +// A table of all the contexts to build and test. +// A new docker runtime will be created and torn down for each context. +var testContexts []testContextTemplate = []testContextTemplate{ + { + ` +from %s run sh -c 'echo root:testpass > /tmp/passwd' run mkdir -p /var/run/sshd -` +run [ "$(cat /tmp/passwd)" = "root:testpass" ] +run [ "$(ls -d /var/run/sshd)" = "/var/run/sshd" ] +`, + nil, + }, -const DockerfileNoNewLine = ` -# VERSION 0.1 -# DOCKER-VERSION 0.2 + { + ` +from %s +add foo /usr/lib/bla/bar +run [ "$(cat /usr/lib/bla/bar)" = 'hello world!' ] +`, + [][2]string{{"foo", "hello world!"}}, + }, -from ` + unitTestImageName + ` -run sh -c 'echo root:testpass > /tmp/passwd' -run mkdir -p /var/run/sshd` + { + ` +from %s +add f / +run [ "$(cat /f)" = "hello" ] +add f /abc +run [ "$(cat /abc)" = "hello" ] +add f /x/y/z +run [ "$(cat /x/y/z)" = "hello" ] +add f /x/y/d/ +run [ "$(cat /x/y/d/f)" = "hello" ] +add d / +run [ "$(cat /ga)" = "bu" ] +add d /somewhere +run [ "$(cat /somewhere/ga)" = "bu" ] +add d /anotherplace/ +run [ "$(cat /anotherplace/ga)" = "bu" ] +add d /somewheeeere/over/the/rainbooow +run [ "$(cat /somewheeeere/over/the/rainbooow/ga)" = "bu" ] +`, + [][2]string{ + {"f", "hello"}, + {"d/ga", "bu"}, + }, + }, -// FIXME: test building with a context - -// FIXME: test building with a local ADD as first command + { + ` +from %s +env FOO BAR +run [ "$FOO" = "BAR" ] +`, + nil, + }, +} // FIXME: test building with 2 successive overlapping ADD commands func TestBuild(t *testing.T) { - dockerfiles := []string{Dockerfile, DockerfileNoNewLine} - for _, Dockerfile := range dockerfiles { + for _, ctx := range testContexts { runtime, err := newTestRuntime() if err != nil { t.Fatal(err) } defer nuke(runtime) - srv := &Server{runtime: runtime} + srv := &Server{ + runtime: runtime, + lock: &sync.Mutex{}, + pullingPool: make(map[string]struct{}), + pushingPool: make(map[string]struct{}), + } - buildfile := NewBuildFile(srv, &utils.NopWriter{}) - - imgID, err := buildfile.Build(strings.NewReader(Dockerfile), nil) - if err != nil { + buildfile := NewBuildFile(srv, ioutil.Discard) + if _, err := buildfile.Build(mkTestContext(ctx.dockerfile, ctx.files, t)); err != nil { t.Fatal(err) } - - builder := NewBuilder(runtime) - container, err := builder.Create( - &Config{ - Image: imgID, - Cmd: []string{"cat", "/tmp/passwd"}, - }, - ) - if err != nil { - t.Fatal(err) - } - defer runtime.Destroy(container) - - output, err := container.Output() - if err != nil { - t.Fatal(err) - } - if string(output) != "root:testpass\n" { - t.Fatalf("Unexpected output. Read '%s', expected '%s'", output, "root:testpass\n") - } - - container2, err := builder.Create( - &Config{ - Image: imgID, - Cmd: []string{"ls", "-d", "/var/run/sshd"}, - }, - ) - if err != nil { - t.Fatal(err) - } - defer runtime.Destroy(container2) - - output, err = container2.Output() - if err != nil { - t.Fatal(err) - } - if string(output) != "/var/run/sshd\n" { - t.Fatal("/var/run/sshd has not been created") - } } } diff --git a/commands.go b/commands.go index f21dfdfe5c..caa2999ef0 100644 --- a/commands.go +++ b/commands.go @@ -1,6 +1,7 @@ package docker import ( + "archive/tar" "bytes" "encoding/json" "flag" @@ -10,14 +11,12 @@ import ( "github.com/dotcloud/docker/utils" "io" "io/ioutil" - "mime/multipart" "net" "net/http" "net/http/httputil" "net/url" "os" "os/signal" - "path" "path/filepath" "reflect" "regexp" @@ -29,7 +28,7 @@ import ( "unicode" ) -const VERSION = "0.4.4" +const VERSION = "0.4.7" var ( GITCOMMIT string @@ -41,7 +40,7 @@ func (cli *DockerCli) getMethod(name string) (reflect.Method, bool) { } func ParseCommands(proto, addr string, args ...string) error { - cli := NewDockerCli(proto, addr) + cli := NewDockerCli(os.Stdin, os.Stdout, os.Stderr, proto, addr) if len(args) > 0 { method, exists := cli.getMethod(args[0]) @@ -65,7 +64,7 @@ func (cli *DockerCli) CmdHelp(args ...string) error { if len(args) > 0 { method, exists := cli.getMethod(args[0]) if !exists { - fmt.Println("Error: Command not found:", args[0]) + fmt.Fprintf(cli.err, "Error: Command not found: %s\n", args[0]) } else { method.Func.CallSlice([]reflect.Value{ reflect.ValueOf(cli), @@ -75,7 +74,7 @@ func (cli *DockerCli) CmdHelp(args ...string) error { } } help := fmt.Sprintf("Usage: docker [OPTIONS] COMMAND [arg...]\n -H=[tcp://%s:%d]: tcp://host:port to bind/connect to or unix://path/to/socker to use\n\nA self-sufficient runtime for linux containers.\n\nCommands:\n", DEFAULTHTTPHOST, DEFAULTHTTPPORT) - for _, command := range [][2]string{ + for _, command := range [][]string{ {"attach", "Attach to a running container"}, {"build", "Build a container from a Dockerfile"}, {"commit", "Create a new image from a container's changes"}, @@ -107,7 +106,7 @@ func (cli *DockerCli) CmdHelp(args ...string) error { } { help += fmt.Sprintf(" %-10.10s%s\n", command[0], command[1]) } - fmt.Println(help) + fmt.Fprintf(cli.err, "%s\n", help) return nil } @@ -125,14 +124,39 @@ func (cli *DockerCli) CmdInsert(args ...string) error { v.Set("url", cmd.Arg(1)) v.Set("path", cmd.Arg(2)) - if err := cli.stream("POST", "/images/"+cmd.Arg(0)+"/insert?"+v.Encode(), nil, os.Stdout); err != nil { + if err := cli.stream("POST", "/images/"+cmd.Arg(0)+"/insert?"+v.Encode(), nil, cli.out); err != nil { return err } return nil } +// mkBuildContext returns an archive of an empty context with the contents +// of `dockerfile` at the path ./Dockerfile +func mkBuildContext(dockerfile string, files [][2]string) (Archive, error) { + buf := new(bytes.Buffer) + tw := tar.NewWriter(buf) + files = append(files, [2]string{"Dockerfile", dockerfile}) + for _, file := range files { + name, content := file[0], file[1] + hdr := &tar.Header{ + Name: name, + Size: int64(len(content)), + } + if err := tw.WriteHeader(hdr); err != nil { + return nil, err + } + if _, err := tw.Write([]byte(content)); err != nil { + return nil, err + } + } + if err := tw.Close(); err != nil { + return nil, err + } + return buf, nil +} + func (cli *DockerCli) CmdBuild(args ...string) error { - cmd := Subcmd("build", "[OPTIONS] PATH | -", "Build a new container image from the source code at PATH") + cmd := Subcmd("build", "[OPTIONS] PATH | URL | -", "Build a new container image from the source code at PATH") tag := cmd.String("t", "", "Tag to be applied to the resulting image in case of success") if err := cmd.Parse(args); err != nil { return nil @@ -143,68 +167,43 @@ func (cli *DockerCli) CmdBuild(args ...string) error { } var ( - multipartBody io.Reader - file io.ReadCloser - contextPath string + context Archive + isRemote bool + err error ) - // Init the needed component for the Multipart - buff := bytes.NewBuffer([]byte{}) - multipartBody = buff - w := multipart.NewWriter(buff) - boundary := strings.NewReader("\r\n--" + w.Boundary() + "--\r\n") - - compression := Bzip2 - if cmd.Arg(0) == "-" { - file = os.Stdin + // As a special case, 'docker build -' will build from an empty context with the + // contents of stdin as a Dockerfile + dockerfile, err := ioutil.ReadAll(cli.in) + if err != nil { + return err + } + context, err = mkBuildContext(string(dockerfile), nil) + } else if utils.IsURL(cmd.Arg(0)) || utils.IsGIT(cmd.Arg(0)) { + isRemote = true } else { - // Send Dockerfile from arg/Dockerfile (deprecate later) - f, err := os.Open(path.Join(cmd.Arg(0), "Dockerfile")) - if err != nil { - return err - } - file = f - // Send context from arg - // Create a FormFile multipart for the context if needed - // FIXME: Use NewTempArchive in order to have the size and avoid too much memory usage? - context, err := Tar(cmd.Arg(0), compression) - if err != nil { - return err - } - // NOTE: Do this in case '.' or '..' is input - absPath, err := filepath.Abs(cmd.Arg(0)) - if err != nil { - return err - } - wField, err := w.CreateFormFile("Context", filepath.Base(absPath)+"."+compression.Extension()) - if err != nil { - return err - } - // FIXME: Find a way to have a progressbar for the upload too + context, err = Tar(cmd.Arg(0), Uncompressed) + } + var body io.Reader + // Setup an upload progress bar + // FIXME: ProgressReader shouldn't be this annoyning to use + if context != nil { sf := utils.NewStreamFormatter(false) - io.Copy(wField, utils.ProgressReader(ioutil.NopCloser(context), -1, os.Stdout, sf.FormatProgress("Caching Context", "%v/%v (%v)"), sf)) - multipartBody = io.MultiReader(multipartBody, boundary) + body = utils.ProgressReader(ioutil.NopCloser(context), 0, cli.err, sf.FormatProgress("Uploading context", "%v bytes%0.0s%0.0s"), sf) } - // Create a FormFile multipart for the Dockerfile - wField, err := w.CreateFormFile("Dockerfile", "Dockerfile") - if err != nil { - return err - } - io.Copy(wField, file) - multipartBody = io.MultiReader(multipartBody, boundary) - + // Upload the build context v := &url.Values{} v.Set("t", *tag) - // Send the multipart request with correct content-type - req, err := http.NewRequest("POST", fmt.Sprintf("/v%g/build?%s", APIVERSION, v.Encode()), multipartBody) + if isRemote { + v.Set("remote", cmd.Arg(0)) + } + req, err := http.NewRequest("POST", fmt.Sprintf("/v%g/build?%s", APIVERSION, v.Encode()), body) if err != nil { return err } - req.Header.Set("Content-Type", w.FormDataContentType()) - if contextPath != "" { - req.Header.Set("X-Docker-Context-Compression", compression.Flag()) - fmt.Println("Uploading Context...") + if context != nil { + req.Header.Set("Content-Type", "application/tar") } dial, err := net.Dial(cli.proto, cli.addr) if err != nil { @@ -217,7 +216,6 @@ func (cli *DockerCli) CmdBuild(args ...string) error { return err } defer resp.Body.Close() - // Check for errors if resp.StatusCode < 200 || resp.StatusCode >= 400 { body, err := ioutil.ReadAll(resp.Body) @@ -231,7 +229,7 @@ func (cli *DockerCli) CmdBuild(args ...string) error { } // Output the result - if _, err := io.Copy(os.Stdout, resp.Body); err != nil { + if _, err := io.Copy(cli.out, resp.Body); err != nil { return err } @@ -290,22 +288,25 @@ func (cli *DockerCli) CmdLogin(args ...string) error { if err != nil { return nil } + var oldState *term.State if *flUsername == "" || *flPassword == "" || *flEmail == "" { - oldState, err = term.SetRawTerminal() + oldState, err = term.SetRawTerminal(cli.terminalFd) if err != nil { return err } - defer term.RestoreTerminal(oldState) + defer term.RestoreTerminal(cli.terminalFd, oldState) } - var username string - var password string - var email string + var ( + username string + password string + email string + ) if *flUsername == "" { - fmt.Print("Username (", cli.authConfig.Username, "): ") - username = readAndEchoString(os.Stdin, os.Stdout) + fmt.Fprintf(cli.out, "Username (%s): ", cli.authConfig.Username) + username = readAndEchoString(cli.in, cli.out) if username == "" { username = cli.authConfig.Username } @@ -314,8 +315,8 @@ func (cli *DockerCli) CmdLogin(args ...string) error { } if username != cli.authConfig.Username { if *flPassword == "" { - fmt.Print("Password: ") - password = readString(os.Stdin, os.Stdout) + fmt.Fprintf(cli.out, "Password: ") + password = readString(cli.in, cli.out) if password == "" { return fmt.Errorf("Error : Password Required") } @@ -324,8 +325,8 @@ func (cli *DockerCli) CmdLogin(args ...string) error { } if *flEmail == "" { - fmt.Print("Email (", cli.authConfig.Email, "): ") - email = readAndEchoString(os.Stdin, os.Stdout) + fmt.Fprintf(cli.out, "Email (%s): ", cli.authConfig.Email) + email = readAndEchoString(cli.in, cli.out) if email == "" { email = cli.authConfig.Email } @@ -337,7 +338,7 @@ func (cli *DockerCli) CmdLogin(args ...string) error { email = cli.authConfig.Email } if oldState != nil { - term.RestoreTerminal(oldState) + term.RestoreTerminal(cli.terminalFd, oldState) } cli.authConfig.Username = username cli.authConfig.Password = password @@ -363,7 +364,7 @@ func (cli *DockerCli) CmdLogin(args ...string) error { } auth.SaveConfig(cli.authConfig) if out2.Status != "" { - fmt.Println(out2.Status) + fmt.Fprintf(cli.out, "%s\n", out2.Status) } return nil } @@ -381,14 +382,14 @@ func (cli *DockerCli) CmdWait(args ...string) error { for _, name := range cmd.Args() { body, _, err := cli.call("POST", "/containers/"+name+"/wait", nil) if err != nil { - fmt.Printf("%s", err) + fmt.Fprintf(cli.err, "%s", err) } else { var out APIWait err = json.Unmarshal(body, &out) if err != nil { return err } - fmt.Println(out.StatusCode) + fmt.Fprintf(cli.out, "%d\n", out.StatusCode) } } return nil @@ -417,13 +418,13 @@ func (cli *DockerCli) CmdVersion(args ...string) error { utils.Debugf("Error unmarshal: body: %s, err: %s\n", body, err) return err } - fmt.Println("Client version:", VERSION) - fmt.Println("Server version:", out.Version) + fmt.Fprintf(cli.out, "Client version: %s\n", VERSION) + fmt.Fprintf(cli.out, "Server version: %s\n", out.Version) if out.GitCommit != "" { - fmt.Println("Git commit:", out.GitCommit) + fmt.Fprintf(cli.out, "Git commit: %s\n", out.GitCommit) } if out.GoVersion != "" { - fmt.Println("Go version:", out.GoVersion) + fmt.Fprintf(cli.out, "Go version: %s\n", out.GoVersion) } return nil } @@ -449,19 +450,19 @@ func (cli *DockerCli) CmdInfo(args ...string) error { return err } - fmt.Printf("Containers: %d\n", out.Containers) - fmt.Printf("Images: %d\n", out.Images) + fmt.Fprintf(cli.out, "Containers: %d\n", out.Containers) + fmt.Fprintf(cli.out, "Images: %d\n", out.Images) if out.Debug || os.Getenv("DEBUG") != "" { - fmt.Printf("Debug mode (server): %v\n", out.Debug) - fmt.Printf("Debug mode (client): %v\n", os.Getenv("DEBUG") != "") - fmt.Printf("Fds: %d\n", out.NFd) - fmt.Printf("Goroutines: %d\n", out.NGoroutines) + fmt.Fprintf(cli.out, "Debug mode (server): %v\n", out.Debug) + fmt.Fprintf(cli.out, "Debug mode (client): %v\n", os.Getenv("DEBUG") != "") + fmt.Fprintf(cli.out, "Fds: %d\n", out.NFd) + fmt.Fprintf(cli.out, "Goroutines: %d\n", out.NGoroutines) } if !out.MemoryLimit { - fmt.Println("WARNING: No memory limit support") + fmt.Fprintf(cli.err, "WARNING: No memory limit support\n") } if !out.SwapLimit { - fmt.Println("WARNING: No swap limit support") + fmt.Fprintf(cli.err, "WARNING: No swap limit support\n") } return nil } @@ -483,9 +484,9 @@ func (cli *DockerCli) CmdStop(args ...string) error { for _, name := range cmd.Args() { _, _, err := cli.call("POST", "/containers/"+name+"/stop?"+v.Encode(), nil) if err != nil { - fmt.Fprintf(os.Stderr, "%s", err) + fmt.Fprintf(cli.err, "%s\n", err) } else { - fmt.Println(name) + fmt.Fprintf(cli.out, "%s\n", name) } } return nil @@ -508,9 +509,9 @@ func (cli *DockerCli) CmdRestart(args ...string) error { for _, name := range cmd.Args() { _, _, err := cli.call("POST", "/containers/"+name+"/restart?"+v.Encode(), nil) if err != nil { - fmt.Fprintf(os.Stderr, "%s", err) + fmt.Fprintf(cli.err, "%s\n", err) } else { - fmt.Println(name) + fmt.Fprintf(cli.out, "%s\n", name) } } return nil @@ -529,9 +530,9 @@ func (cli *DockerCli) CmdStart(args ...string) error { for _, name := range args { _, _, err := cli.call("POST", "/containers/"+name+"/start", nil) if err != nil { - fmt.Fprintf(os.Stderr, "%s", err) + fmt.Fprintf(cli.err, "%s\n", err) } else { - fmt.Println(name) + fmt.Fprintf(cli.out, "%s\n", name) } } return nil @@ -546,30 +547,30 @@ func (cli *DockerCli) CmdInspect(args ...string) error { cmd.Usage() return nil } - fmt.Printf("[") + fmt.Fprintf(cli.out, "[") for i, name := range args { if i > 0 { - fmt.Printf(",") + fmt.Fprintf(cli.out, ",") } obj, _, err := cli.call("GET", "/containers/"+name+"/json", nil) if err != nil { obj, _, err = cli.call("GET", "/images/"+name+"/json", nil) if err != nil { - fmt.Fprintf(os.Stderr, "%s", err) + fmt.Fprintf(cli.err, "%s\n", err) continue } } indented := new(bytes.Buffer) if err = json.Indent(indented, obj, "", " "); err != nil { - fmt.Fprintf(os.Stderr, "%s", err) + fmt.Fprintf(cli.err, "%s\n", err) continue } - if _, err := io.Copy(os.Stdout, indented); err != nil { - fmt.Fprintf(os.Stderr, "%s", err) + if _, err := io.Copy(cli.out, indented); err != nil { + fmt.Fprintf(cli.err, "%s\n", err) } } - fmt.Printf("]") + fmt.Fprintf(cli.out, "]") return nil } @@ -594,7 +595,7 @@ func (cli *DockerCli) CmdPort(args ...string) error { } if frontend, exists := out.NetworkSettings.PortMapping[cmd.Arg(1)]; exists { - fmt.Println(frontend) + fmt.Fprintf(cli.out, "%s\n", frontend) } else { return fmt.Errorf("Error: No private port '%s' allocated on %s", cmd.Arg(1), cmd.Arg(0)) } @@ -615,7 +616,7 @@ func (cli *DockerCli) CmdRmi(args ...string) error { for _, name := range cmd.Args() { body, _, err := cli.call("DELETE", "/images/"+name, nil) if err != nil { - fmt.Fprintf(os.Stderr, "%s", err) + fmt.Fprintf(cli.err, "%s", err) } else { var outs []APIRmi err = json.Unmarshal(body, &outs) @@ -624,9 +625,9 @@ func (cli *DockerCli) CmdRmi(args ...string) error { } for _, out := range outs { if out.Deleted != "" { - fmt.Println("Deleted:", out.Deleted) + fmt.Fprintf(cli.out, "Deleted: %s\n", out.Deleted) } else { - fmt.Println("Untagged:", out.Untagged) + fmt.Fprintf(cli.out, "Untagged: %s\n", out.Untagged) } } } @@ -654,7 +655,7 @@ func (cli *DockerCli) CmdHistory(args ...string) error { if err != nil { return err } - w := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0) + w := tabwriter.NewWriter(cli.out, 20, 1, 3, ' ', 0) fmt.Fprintln(w, "ID\tCREATED\tCREATED BY") for _, out := range outs { @@ -684,9 +685,9 @@ func (cli *DockerCli) CmdRm(args ...string) error { for _, name := range cmd.Args() { _, _, err := cli.call("DELETE", "/containers/"+name+"?"+val.Encode(), nil) if err != nil { - fmt.Printf("%s", err) + fmt.Fprintf(cli.err, "%s\n", err) } else { - fmt.Println(name) + fmt.Fprintf(cli.out, "%s\n", name) } } return nil @@ -706,9 +707,9 @@ func (cli *DockerCli) CmdKill(args ...string) error { for _, name := range args { _, _, err := cli.call("POST", "/containers/"+name+"/kill", nil) if err != nil { - fmt.Printf("%s", err) + fmt.Fprintf(cli.err, "%s\n", err) } else { - fmt.Println(name) + fmt.Fprintf(cli.out, "%s\n", name) } } return nil @@ -730,7 +731,7 @@ func (cli *DockerCli) CmdImport(args ...string) error { v.Set("tag", tag) v.Set("fromSrc", src) - err := cli.stream("POST", "/images/create?"+v.Encode(), os.Stdin, os.Stdout) + err := cli.stream("POST", "/images/create?"+v.Encode(), cli.in, cli.out) if err != nil { return err } @@ -754,27 +755,34 @@ func (cli *DockerCli) CmdPush(args ...string) error { return err } - if len(strings.SplitN(name, "/", 2)) == 1 { - return fmt.Errorf("Impossible to push a \"root\" repository. Please rename your repository in / (ex: %s/%s)", cli.authConfig.Username, name) + if *registry == "" { + // If we're not using a custom registry, we know the restrictions + // applied to repository names and can warn the user in advance. + // Custom repositories can have different rules, and we must also + // allow pushing by image ID. + if len(strings.SplitN(name, "/", 2)) == 1 { + return fmt.Errorf("Impossible to push a \"root\" repository. Please rename your repository in / (ex: %s/%s)", cli.authConfig.Username, name) + } + + nameParts := strings.SplitN(name, "/", 2) + validNamespace := regexp.MustCompile(`^([a-z0-9_]{4,30})$`) + if !validNamespace.MatchString(nameParts[0]) { + return fmt.Errorf("Invalid namespace name (%s), only [a-z0-9_] are allowed, size between 4 and 30", nameParts[0]) + } + validRepo := regexp.MustCompile(`^([a-zA-Z0-9-_.]+)$`) + if !validRepo.MatchString(nameParts[1]) { + return fmt.Errorf("Invalid repository name (%s), only [a-zA-Z0-9-_.] are allowed", nameParts[1]) + } } buf, err := json.Marshal(cli.authConfig) if err != nil { return err } - nameParts := strings.SplitN(name, "/", 2) - validNamespace := regexp.MustCompile(`^([a-z0-9_]{4,30})$`) - if !validNamespace.MatchString(nameParts[0]) { - return fmt.Errorf("Invalid namespace name (%s), only [a-z0-9_] are allowed, size between 4 and 30", nameParts[0]) - } - validRepo := regexp.MustCompile(`^([a-zA-Z0-9-_.]+)$`) - if !validRepo.MatchString(nameParts[1]) { - return fmt.Errorf("Invalid repository name (%s), only [a-zA-Z0-9-_.] are allowed", nameParts[1]) - } v := url.Values{} v.Set("registry", *registry) - if err := cli.stream("POST", "/images/"+name+"/push?"+v.Encode(), bytes.NewBuffer(buf), os.Stdout); err != nil { + if err := cli.stream("POST", "/images/"+name+"/push?"+v.Encode(), bytes.NewBuffer(buf), cli.out); err != nil { return err } return nil @@ -805,7 +813,7 @@ func (cli *DockerCli) CmdPull(args ...string) error { v.Set("tag", *tag) v.Set("registry", *registry) - if err := cli.stream("POST", "/images/create?"+v.Encode(), nil, os.Stdout); err != nil { + if err := cli.stream("POST", "/images/create?"+v.Encode(), nil, cli.out); err != nil { return err } @@ -832,7 +840,7 @@ func (cli *DockerCli) CmdImages(args ...string) error { if err != nil { return err } - fmt.Printf("%s", body) + fmt.Fprintf(cli.out, "%s", body) } else { v := url.Values{} if cmd.NArg() == 1 { @@ -853,7 +861,7 @@ func (cli *DockerCli) CmdImages(args ...string) error { return err } - w := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0) + w := tabwriter.NewWriter(cli.out, 20, 1, 3, ' ', 0) if !*quiet { fmt.Fprintln(w, "REPOSITORY\tTAG\tID\tCREATED\tSIZE") } @@ -898,6 +906,7 @@ func (cli *DockerCli) CmdImages(args ...string) error { func (cli *DockerCli) CmdPs(args ...string) error { cmd := Subcmd("ps", "[OPTIONS]", "List containers") quiet := cmd.Bool("q", false, "Only display numeric IDs") + size := cmd.Bool("s", false, "Display sizes") all := cmd.Bool("a", false, "Show all containers. Only running containers are shown by default.") noTrunc := cmd.Bool("notrunc", false, "Don't truncate output") nLatest := cmd.Bool("l", false, "Show only the latest created container, include non-running ones.") @@ -924,6 +933,9 @@ func (cli *DockerCli) CmdPs(args ...string) error { if *before != "" { v.Set("before", *before) } + if *size { + v.Set("size", "1") + } body, _, err := cli.call("GET", "/containers/json?"+v.Encode(), nil) if err != nil { @@ -935,9 +947,14 @@ func (cli *DockerCli) CmdPs(args ...string) error { if err != nil { return err } - w := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0) + w := tabwriter.NewWriter(cli.out, 20, 1, 3, ' ', 0) if !*quiet { - fmt.Fprintln(w, "ID\tIMAGE\tCOMMAND\tCREATED\tSTATUS\tPORTS\tSIZE") + fmt.Fprint(w, "ID\tIMAGE\tCOMMAND\tCREATED\tSTATUS\tPORTS") + if *size { + fmt.Fprintln(w, "\tSIZE") + } else { + fmt.Fprint(w, "\n") + } } for _, out := range outs { @@ -947,10 +964,14 @@ func (cli *DockerCli) CmdPs(args ...string) error { } else { fmt.Fprintf(w, "%s\t%s\t%s\t%s ago\t%s\t%s\t", utils.TruncateID(out.ID), out.Image, utils.Trunc(out.Command, 20), utils.HumanDuration(time.Now().Sub(time.Unix(out.Created, 0))), out.Status, out.Ports) } - if out.SizeRootFs > 0 { - fmt.Fprintf(w, "%s (virtual %s)\n", utils.HumanSize(out.SizeRw), utils.HumanSize(out.SizeRootFs)) + if *size { + if out.SizeRootFs > 0 { + fmt.Fprintf(w, "%s (virtual %s)\n", utils.HumanSize(out.SizeRw), utils.HumanSize(out.SizeRootFs)) + } else { + fmt.Fprintf(w, "%s\n", utils.HumanSize(out.SizeRw)) + } } else { - fmt.Fprintf(w, "%s\n", utils.HumanSize(out.SizeRw)) + fmt.Fprint(w, "\n") } } else { if *noTrunc { @@ -1005,7 +1026,7 @@ func (cli *DockerCli) CmdCommit(args ...string) error { return err } - fmt.Println(apiID.ID) + fmt.Fprintf(cli.out, "%s\n", apiID.ID) return nil } @@ -1020,7 +1041,7 @@ func (cli *DockerCli) CmdExport(args ...string) error { return nil } - if err := cli.stream("GET", "/containers/"+cmd.Arg(0)+"/export", nil, os.Stdout); err != nil { + if err := cli.stream("GET", "/containers/"+cmd.Arg(0)+"/export", nil, cli.out); err != nil { return err } return nil @@ -1047,7 +1068,7 @@ func (cli *DockerCli) CmdDiff(args ...string) error { return err } for _, change := range changes { - fmt.Println(change.String()) + fmt.Fprintf(cli.out, "%s\n", change.String()) } return nil } @@ -1062,10 +1083,10 @@ func (cli *DockerCli) CmdLogs(args ...string) error { return nil } - if err := cli.hijack("POST", "/containers/"+cmd.Arg(0)+"/attach?logs=1&stdout=1", false, nil, os.Stdout); err != nil { + if err := cli.hijack("POST", "/containers/"+cmd.Arg(0)+"/attach?logs=1&stdout=1", false, nil, cli.out); err != nil { return err } - if err := cli.hijack("POST", "/containers/"+cmd.Arg(0)+"/attach?logs=1&stderr=1", false, nil, os.Stderr); err != nil { + if err := cli.hijack("POST", "/containers/"+cmd.Arg(0)+"/attach?logs=1&stderr=1", false, nil, cli.err); err != nil { return err } return nil @@ -1097,7 +1118,9 @@ func (cli *DockerCli) CmdAttach(args ...string) error { } if container.Config.Tty { - cli.monitorTtySize(cmd.Arg(0)) + if err := cli.monitorTtySize(cmd.Arg(0)); err != nil { + return err + } } v := url.Values{} @@ -1106,7 +1129,7 @@ func (cli *DockerCli) CmdAttach(args ...string) error { v.Set("stdout", "1") v.Set("stderr", "1") - if err := cli.hijack("POST", "/containers/"+cmd.Arg(0)+"/attach?"+v.Encode(), container.Config.Tty, os.Stdin, os.Stdout); err != nil { + if err := cli.hijack("POST", "/containers/"+cmd.Arg(0)+"/attach?"+v.Encode(), container.Config.Tty, cli.in, cli.out); err != nil { return err } return nil @@ -1134,8 +1157,8 @@ func (cli *DockerCli) CmdSearch(args ...string) error { if err != nil { return err } - fmt.Printf("Found %d results matching your query (\"%s\")\n", len(outs), cmd.Arg(0)) - w := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0) + fmt.Fprintf(cli.out, "Found %d results matching your query (\"%s\")\n", len(outs), cmd.Arg(0)) + w := tabwriter.NewWriter(cli.out, 20, 1, 3, ' ', 0) fmt.Fprintf(w, "NAME\tDESCRIPTION\n") for _, out := range outs { desc := strings.Replace(out.Description, "\n", " ", -1) @@ -1238,7 +1261,7 @@ func (cli *DockerCli) CmdTag(args ...string) error { } func (cli *DockerCli) CmdRun(args ...string) error { - config, cmd, err := ParseRun(args, nil) + config, hostConfig, cmd, err := ParseRun(args, nil) if err != nil { return err } @@ -1253,7 +1276,7 @@ func (cli *DockerCli) CmdRun(args ...string) error { if statusCode == 404 { v := url.Values{} v.Set("fromImage", config.Image) - err = cli.stream("POST", "/images/create?"+v.Encode(), nil, os.Stderr) + err = cli.stream("POST", "/images/create?"+v.Encode(), nil, cli.err) if err != nil { return err } @@ -1266,27 +1289,31 @@ func (cli *DockerCli) CmdRun(args ...string) error { return err } - out := &APIRun{} - err = json.Unmarshal(body, out) + runResult := &APIRun{} + err = json.Unmarshal(body, runResult) if err != nil { return err } - for _, warning := range out.Warnings { - fmt.Fprintln(os.Stderr, "WARNING: ", warning) + for _, warning := range runResult.Warnings { + fmt.Fprintln(cli.err, "WARNING: ", warning) } //start the container - _, _, err = cli.call("POST", "/containers/"+out.ID+"/start", nil) - if err != nil { + if _, _, err = cli.call("POST", "/containers/"+runResult.ID+"/start", hostConfig); err != nil { return err } if !config.AttachStdout && !config.AttachStderr { - fmt.Println(out.ID) - } else { + // Make this asynchrone in order to let the client write to stdin before having to read the ID + go fmt.Fprintf(cli.out, "%s\n", runResult.ID) + } + + if config.AttachStdin || config.AttachStdout || config.AttachStderr { if config.Tty { - cli.monitorTtySize(out.ID) + if err := cli.monitorTtySize(runResult.ID); err != nil { + return err + } } v := url.Values{} @@ -1302,7 +1329,8 @@ func (cli *DockerCli) CmdRun(args ...string) error { if config.AttachStderr { v.Set("stderr", "1") } - if err := cli.hijack("POST", "/containers/"+out.ID+"/attach?"+v.Encode(), config.Tty, os.Stdin, os.Stdout); err != nil { + + if err := cli.hijack("POST", "/containers/"+runResult.ID+"/attach?"+v.Encode(), config.Tty, cli.in, cli.out); err != nil { utils.Debugf("Error hijack: %s", err) return err } @@ -1433,7 +1461,7 @@ func (cli *DockerCli) stream(method, path string, in io.Reader, out io.Writer) e return nil } -func (cli *DockerCli) hijack(method, path string, setRawTerminal bool, in *os.File, out io.Writer) error { +func (cli *DockerCli) hijack(method, path string, setRawTerminal bool, in io.ReadCloser, out io.Writer) error { req, err := http.NewRequest(method, fmt.Sprintf("/v%g%s", APIVERSION, path), nil) if err != nil { @@ -1460,17 +1488,26 @@ func (cli *DockerCli) hijack(method, path string, setRawTerminal bool, in *os.Fi return err }) - if in != nil && setRawTerminal && term.IsTerminal(in.Fd()) && os.Getenv("NORAW") == "" { - oldState, err := term.SetRawTerminal() + if in != nil && setRawTerminal && cli.isTerminal && os.Getenv("NORAW") == "" { + oldState, err := term.SetRawTerminal(cli.terminalFd) if err != nil { return err } - defer term.RestoreTerminal(oldState) + defer term.RestoreTerminal(cli.terminalFd, oldState) } + sendStdin := utils.Go(func() error { - io.Copy(rwc, in) - if err := rwc.(*net.TCPConn).CloseWrite(); err != nil { - utils.Debugf("Couldn't send EOF: %s\n", err) + if in != nil { + io.Copy(rwc, in) + } + if tcpc, ok := rwc.(*net.TCPConn); ok { + if err := tcpc.CloseWrite(); err != nil { + utils.Debugf("Couldn't send EOF: %s\n", err) + } + } else if unixc, ok := rwc.(*net.UnixConn); ok { + if err := unixc.CloseWrite(); err != nil { + utils.Debugf("Couldn't send EOF: %s\n", err) + } } // Discard errors due to pipe interruption return nil @@ -1481,7 +1518,7 @@ func (cli *DockerCli) hijack(method, path string, setRawTerminal bool, in *os.Fi return err } - if !term.IsTerminal(in.Fd()) { + if !cli.isTerminal { if err := <-sendStdin; err != nil { utils.Debugf("Error sendStdin: %s", err) return err @@ -1492,7 +1529,10 @@ func (cli *DockerCli) hijack(method, path string, setRawTerminal bool, in *os.Fi } func (cli *DockerCli) resizeTty(id string) { - ws, err := term.GetWinsize(os.Stdin.Fd()) + if !cli.isTerminal { + return + } + ws, err := term.GetWinsize(cli.terminalFd) if err != nil { utils.Debugf("Error getting size: %s", err) } @@ -1504,7 +1544,10 @@ func (cli *DockerCli) resizeTty(id string) { } } -func (cli *DockerCli) monitorTtySize(id string) { +func (cli *DockerCli) monitorTtySize(id string) error { + if !cli.isTerminal { + return fmt.Errorf("Impossible to monitor size on non-tty") + } cli.resizeTty(id) c := make(chan os.Signal, 1) @@ -1516,24 +1559,56 @@ func (cli *DockerCli) monitorTtySize(id string) { } } }() + return nil } func Subcmd(name, signature, description string) *flag.FlagSet { flags := flag.NewFlagSet(name, flag.ContinueOnError) flags.Usage = func() { - fmt.Printf("\nUsage: docker %s %s\n\n%s\n\n", name, signature, description) + // FIXME: use custom stdout or return error + fmt.Fprintf(os.Stdout, "\nUsage: docker %s %s\n\n%s\n\n", name, signature, description) flags.PrintDefaults() } return flags } -func NewDockerCli(proto, addr string) *DockerCli { +func NewDockerCli(in io.ReadCloser, out, err io.Writer, proto, addr string) *DockerCli { + var ( + isTerminal bool = false + terminalFd uintptr + ) + + if in != nil { + if file, ok := in.(*os.File); ok { + terminalFd = file.Fd() + isTerminal = term.IsTerminal(terminalFd) + } + } + + if err == nil { + err = out + } + authConfig, _ := auth.LoadConfig(os.Getenv("HOME")) - return &DockerCli{proto, addr, authConfig} + return &DockerCli{ + proto: proto, + addr: addr, + authConfig: authConfig, + in: in, + out: out, + err: err, + isTerminal: isTerminal, + terminalFd: terminalFd, + } } type DockerCli struct { proto string addr string authConfig *auth.AuthConfig + in io.ReadCloser + out io.Writer + err io.Writer + isTerminal bool + terminalFd uintptr } diff --git a/commands_test.go b/commands_test.go index 05ece80dac..87c4c02a52 100644 --- a/commands_test.go +++ b/commands_test.go @@ -3,8 +3,9 @@ package docker import ( "bufio" "fmt" + "github.com/dotcloud/docker/utils" "io" - _ "io/ioutil" + "io/ioutil" "strings" "testing" "time" @@ -59,20 +60,6 @@ func assertPipe(input, output string, r io.Reader, w io.Writer, count int) error } /*TODO -func cmdWait(srv *Server, container *Container) error { - stdout, stdoutPipe := io.Pipe() - - go func() { - srv.CmdWait(nil, stdoutPipe, container.Id) - }() - - if _, err := bufio.NewReader(stdout).ReadString('\n'); err != nil { - return err - } - // Cleanup pipes - return closeWrap(stdout, stdoutPipe) -} - func cmdImages(srv *Server, args ...string) (string, error) { stdout, stdoutPipe := io.Pipe() @@ -144,41 +131,39 @@ func TestImages(t *testing.T) { // todo: add checks for -a } +*/ // TestRunHostname checks that 'docker run -h' correctly sets a custom hostname func TestRunHostname(t *testing.T) { - runtime, err := newTestRuntime() - if err != nil { - t.Fatal(err) - } - defer nuke(runtime) - - srv := &Server{runtime: runtime} - - stdin, _ := io.Pipe() stdout, stdoutPipe := io.Pipe() + cli := NewDockerCli(nil, stdoutPipe, nil, testDaemonProto, testDaemonAddr) + defer cleanup(globalRuntime) + c := make(chan struct{}) go func() { - if err := srv.CmdRun(stdin, rcli.NewDockerLocalConn(stdoutPipe), "-h", "foobar", GetTestImage(runtime).Id, "hostname"); err != nil { + defer close(c) + if err := cli.CmdRun("-h", "foobar", unitTestImageId, "hostname"); err != nil { t.Fatal(err) } - close(c) }() - cmdOutput, err := bufio.NewReader(stdout).ReadString('\n') - if err != nil { - t.Fatal(err) - } - if cmdOutput != "foobar\n" { - t.Fatalf("'hostname' should display '%s', not '%s'", "foobar\n", cmdOutput) - } + utils.Debugf("--") + setTimeout(t, "Reading command output time out", 2*time.Second, func() { + cmdOutput, err := bufio.NewReader(stdout).ReadString('\n') + if err != nil { + t.Fatal(err) + } + if cmdOutput != "foobar\n" { + t.Fatalf("'hostname' should display '%s', not '%s'", "foobar\n", cmdOutput) + } + }) - setTimeout(t, "CmdRun timed out", 2*time.Second, func() { + setTimeout(t, "CmdRun timed out", 5*time.Second, func() { <-c - cmdWait(srv, srv.runtime.List()[0]) }) } +/* func TestRunExit(t *testing.T) { runtime, err := newTestRuntime() if err != nil { @@ -334,29 +319,27 @@ func TestRunDisconnectTty(t *testing.T) { t.Fatalf("/bin/cat should still be running after closing stdin (tty mode)") } } +*/ // TestAttachStdin checks attaching to stdin without stdout and stderr. // 'docker run -i -a stdin' should sends the client's stdin to the command, // then detach from it and print the container id. func TestRunAttachStdin(t *testing.T) { - runtime, err := newTestRuntime() - if err != nil { - t.Fatal(err) - } - defer nuke(runtime) - srv := &Server{runtime: runtime} stdin, stdinPipe := io.Pipe() stdout, stdoutPipe := io.Pipe() + cli := NewDockerCli(stdin, stdoutPipe, nil, testDaemonProto, testDaemonAddr) + defer cleanup(globalRuntime) + ch := make(chan struct{}) go func() { - srv.CmdRun(stdin, rcli.NewDockerLocalConn(stdoutPipe), "-i", "-a", "stdin", GetTestImage(runtime).Id, "sh", "-c", "echo hello; cat") - close(ch) + defer close(ch) + cli.CmdRun("-i", "-a", "stdin", unitTestImageId, "sh", "-c", "echo hello && cat") }() // Send input to the command, close stdin - setTimeout(t, "Write timed out", 2*time.Second, func() { + setTimeout(t, "Write timed out", 10*time.Second, func() { if _, err := stdinPipe.Write([]byte("hi there\n")); err != nil { t.Fatal(err) } @@ -365,23 +348,27 @@ func TestRunAttachStdin(t *testing.T) { } }) - container := runtime.List()[0] + container := globalRuntime.List()[0] // Check output - cmdOutput, err := bufio.NewReader(stdout).ReadString('\n') - if err != nil { - t.Fatal(err) - } - if cmdOutput != container.ShortId()+"\n" { - t.Fatalf("Wrong output: should be '%s', not '%s'\n", container.ShortId()+"\n", cmdOutput) - } + setTimeout(t, "Reading command output time out", 10*time.Second, func() { + cmdOutput, err := bufio.NewReader(stdout).ReadString('\n') + if err != nil { + t.Fatal(err) + } + if cmdOutput != container.ShortID()+"\n" { + t.Fatalf("Wrong output: should be '%s', not '%s'\n", container.ShortID()+"\n", cmdOutput) + } + }) // wait for CmdRun to return - setTimeout(t, "Waiting for CmdRun timed out", 2*time.Second, func() { + setTimeout(t, "Waiting for CmdRun timed out", 5*time.Second, func() { + // Unblock hijack end + stdout.Read([]byte{}) <-ch }) - setTimeout(t, "Waiting for command to exit timed out", 2*time.Second, func() { + setTimeout(t, "Waiting for command to exit timed out", 5*time.Second, func() { container.Wait() }) @@ -400,6 +387,7 @@ func TestRunAttachStdin(t *testing.T) { } } +/* // Expected behaviour, the process stays alive when the client disconnects func TestAttachDisconnect(t *testing.T) { runtime, err := newTestRuntime() diff --git a/container.go b/container.go index f60de21bdc..45bb728b96 100644 --- a/container.go +++ b/container.go @@ -52,6 +52,9 @@ type Container struct { waitLock chan struct{} Volumes map[string]string + // Store rw/ro in a separate structure to preserve reserve-compatibility on-disk. + // Easier than migrating older container configs :) + VolumesRW map[string]bool } type Config struct { @@ -75,8 +78,18 @@ type Config struct { VolumesFrom string } -func ParseRun(args []string, capabilities *Capabilities) (*Config, *flag.FlagSet, error) { - cmd := Subcmd("run", "[OPTIONS] IMAGE COMMAND [ARG...]", "Run a command in a new container") +type HostConfig struct { + Binds []string +} + +type BindMap struct { + SrcPath string + DstPath string + Mode string +} + +func ParseRun(args []string, capabilities *Capabilities) (*Config, *HostConfig, *flag.FlagSet, error) { + cmd := Subcmd("run", "[OPTIONS] IMAGE [COMMAND] [ARG...]", "Run a command in a new container") if len(args) > 0 && args[0] != "--help" { cmd.SetOutput(ioutil.Discard) } @@ -111,11 +124,14 @@ func ParseRun(args []string, capabilities *Capabilities) (*Config, *flag.FlagSet flVolumesFrom := cmd.String("volumes-from", "", "Mount volumes from the specified container") + var flBinds ListOpts + cmd.Var(&flBinds, "b", "Bind mount a volume from the host (e.g. -b /host:/container)") + if err := cmd.Parse(args); err != nil { - return nil, cmd, err + return nil, nil, cmd, err } if *flDetach && len(flAttach) > 0 { - return nil, cmd, fmt.Errorf("Conflicting options: -a and -d") + return nil, nil, cmd, fmt.Errorf("Conflicting options: -a and -d") } // If neither -d or -a are set, attach to everything by default if len(flAttach) == 0 && !*flDetach { @@ -127,6 +143,14 @@ func ParseRun(args []string, capabilities *Capabilities) (*Config, *flag.FlagSet } } } + + // add any bind targets to the list of container volumes + for _, bind := range flBinds { + arr := strings.Split(bind, ":") + dstDir := arr[1] + flVolumes[dstDir] = struct{}{} + } + parsedArgs := cmd.Args() runCmd := []string{} image := "" @@ -154,6 +178,9 @@ func ParseRun(args []string, capabilities *Capabilities) (*Config, *flag.FlagSet Volumes: flVolumes, VolumesFrom: *flVolumesFrom, } + hostConfig := &HostConfig{ + Binds: flBinds, + } if capabilities != nil && *flMemory > 0 && !capabilities.SwapLimit { //fmt.Fprintf(stdout, "WARNING: Your kernel does not support swap limit capabilities. Limitation discarded.\n") @@ -164,7 +191,7 @@ func ParseRun(args []string, capabilities *Capabilities) (*Config, *flag.FlagSet if config.OpenStdin && config.AttachStdin { config.StdinOnce = true } - return config, cmd, nil + return config, hostConfig, cmd, nil } type NetworkSettings struct { @@ -430,7 +457,7 @@ func (container *Container) Attach(stdin io.ReadCloser, stdinCloser io.Closer, s }) } -func (container *Container) Start() error { +func (container *Container) Start(hostConfig *HostConfig) error { container.State.lock() defer container.State.unlock() @@ -454,17 +481,71 @@ func (container *Container) Start() error { container.Config.MemorySwap = -1 } container.Volumes = make(map[string]string) + container.VolumesRW = make(map[string]bool) + // Create the requested bind mounts + binds := make(map[string]BindMap) + // Define illegal container destinations + illegal_dsts := []string{"/", "."} + + for _, bind := range hostConfig.Binds { + // FIXME: factorize bind parsing in parseBind + var src, dst, mode string + arr := strings.Split(bind, ":") + if len(arr) == 2 { + src = arr[0] + dst = arr[1] + mode = "rw" + } else if len(arr) == 3 { + src = arr[0] + dst = arr[1] + mode = arr[2] + } else { + return fmt.Errorf("Invalid bind specification: %s", bind) + } + + // Bail if trying to mount to an illegal destination + for _, illegal := range illegal_dsts { + if dst == illegal { + return fmt.Errorf("Illegal bind destination: %s", dst) + } + } + + bindMap := BindMap{ + SrcPath: src, + DstPath: dst, + Mode: mode, + } + binds[path.Clean(dst)] = bindMap + } + + // FIXME: evaluate volumes-from before individual volumes, so that the latter can override the former. // Create the requested volumes volumes for volPath := range container.Config.Volumes { - c, err := container.runtime.volumes.Create(nil, container, "", "", nil) - if err != nil { - return err + volPath = path.Clean(volPath) + // If an external bind is defined for this volume, use that as a source + if bindMap, exists := binds[volPath]; exists { + container.Volumes[volPath] = bindMap.SrcPath + if strings.ToLower(bindMap.Mode) == "rw" { + container.VolumesRW[volPath] = true + } + // Otherwise create an directory in $ROOT/volumes/ and use that + } else { + c, err := container.runtime.volumes.Create(nil, container, "", "", nil) + if err != nil { + return err + } + srcPath, err := c.layer() + if err != nil { + return err + } + container.Volumes[volPath] = srcPath + container.VolumesRW[volPath] = true // RW by default } + // Create the mountpoint if err := os.MkdirAll(path.Join(container.RootfsPath(), volPath), 0755); err != nil { return nil } - container.Volumes[volPath] = c.ID } if container.Config.VolumesFrom != "" { @@ -552,7 +633,8 @@ func (container *Container) Start() error { } func (container *Container) Run() error { - if err := container.Start(); err != nil { + hostConfig := &HostConfig{} + if err := container.Start(hostConfig); err != nil { return err } container.Wait() @@ -565,7 +647,8 @@ func (container *Container) Output() (output []byte, err error) { return nil, err } defer pipe.Close() - if err := container.Start(); err != nil { + hostConfig := &HostConfig{} + if err := container.Start(hostConfig); err != nil { return nil, err } output, err = ioutil.ReadAll(pipe) @@ -632,7 +715,6 @@ func (container *Container) waitLxc() error { } time.Sleep(500 * time.Millisecond) } - panic("Unreachable") } func (container *Container) monitor() { @@ -769,7 +851,8 @@ func (container *Container) Restart(seconds int) error { if err := container.Stop(seconds); err != nil { return err } - if err := container.Start(); err != nil { + hostConfig := &HostConfig{} + if err := container.Start(hostConfig); err != nil { return err } return nil @@ -821,8 +904,6 @@ func (container *Container) WaitTimeout(timeout time.Duration) error { case <-done: return nil } - - panic("Unreachable") } func (container *Container) EnsureMounted() error { @@ -894,22 +975,6 @@ func (container *Container) RootfsPath() string { return path.Join(container.root, "rootfs") } -func (container *Container) GetVolumes() (map[string]string, error) { - ret := make(map[string]string) - for volPath, id := range container.Volumes { - volume, err := container.runtime.volumes.Get(id) - if err != nil { - return nil, err - } - root, err := volume.root() - if err != nil { - return nil, err - } - ret[volPath] = path.Join(root, "layer") - } - return ret, nil -} - func (container *Container) rwPath() string { return path.Join(container.root, "rw") } diff --git a/container_test.go b/container_test.go index 8ec1fa40ee..cb33dcf983 100644 --- a/container_test.go +++ b/container_test.go @@ -7,6 +7,7 @@ import ( "io/ioutil" "math/rand" "os" + "path" "regexp" "sort" "strings" @@ -15,10 +16,7 @@ import ( ) func TestIDFormat(t *testing.T) { - runtime, err := newTestRuntime() - if err != nil { - t.Fatal(err) - } + runtime := mkRuntime(t) defer nuke(runtime) container1, err := NewBuilder(runtime).Create( &Config{ @@ -39,10 +37,7 @@ func TestIDFormat(t *testing.T) { } func TestMultipleAttachRestart(t *testing.T) { - runtime, err := newTestRuntime() - if err != nil { - t.Fatal(err) - } + runtime := mkRuntime(t) defer nuke(runtime) container, err := NewBuilder(runtime).Create( &Config{ @@ -70,7 +65,8 @@ func TestMultipleAttachRestart(t *testing.T) { if err != nil { t.Fatal(err) } - if err := container.Start(); err != nil { + hostConfig := &HostConfig{} + if err := container.Start(hostConfig); err != nil { t.Fatal(err) } l1, err := bufio.NewReader(stdout1).ReadString('\n') @@ -111,7 +107,7 @@ func TestMultipleAttachRestart(t *testing.T) { if err != nil { t.Fatal(err) } - if err := container.Start(); err != nil { + if err := container.Start(hostConfig); err != nil { t.Fatal(err) } @@ -142,10 +138,7 @@ func TestMultipleAttachRestart(t *testing.T) { } func TestDiff(t *testing.T) { - runtime, err := newTestRuntime() - if err != nil { - t.Fatal(err) - } + runtime := mkRuntime(t) defer nuke(runtime) builder := NewBuilder(runtime) @@ -251,10 +244,7 @@ func TestDiff(t *testing.T) { } func TestCommitAutoRun(t *testing.T) { - runtime, err := newTestRuntime() - if err != nil { - t.Fatal(err) - } + runtime := mkRuntime(t) defer nuke(runtime) builder := NewBuilder(runtime) @@ -306,7 +296,8 @@ func TestCommitAutoRun(t *testing.T) { if err != nil { t.Fatal(err) } - if err := container2.Start(); err != nil { + hostConfig := &HostConfig{} + if err := container2.Start(hostConfig); err != nil { t.Fatal(err) } container2.Wait() @@ -330,10 +321,7 @@ func TestCommitAutoRun(t *testing.T) { } func TestCommitRun(t *testing.T) { - runtime, err := newTestRuntime() - if err != nil { - t.Fatal(err) - } + runtime := mkRuntime(t) defer nuke(runtime) builder := NewBuilder(runtime) @@ -388,7 +376,8 @@ func TestCommitRun(t *testing.T) { if err != nil { t.Fatal(err) } - if err := container2.Start(); err != nil { + hostConfig := &HostConfig{} + if err := container2.Start(hostConfig); err != nil { t.Fatal(err) } container2.Wait() @@ -412,10 +401,7 @@ func TestCommitRun(t *testing.T) { } func TestStart(t *testing.T) { - runtime, err := newTestRuntime() - if err != nil { - t.Fatal(err) - } + runtime := mkRuntime(t) defer nuke(runtime) container, err := NewBuilder(runtime).Create( &Config{ @@ -436,7 +422,8 @@ func TestStart(t *testing.T) { t.Fatal(err) } - if err := container.Start(); err != nil { + hostConfig := &HostConfig{} + if err := container.Start(hostConfig); err != nil { t.Fatal(err) } @@ -446,7 +433,7 @@ func TestStart(t *testing.T) { if !container.State.Running { t.Errorf("Container should be running") } - if err := container.Start(); err == nil { + if err := container.Start(hostConfig); err == nil { t.Fatalf("A running containter should be able to be started") } @@ -456,10 +443,7 @@ func TestStart(t *testing.T) { } func TestRun(t *testing.T) { - runtime, err := newTestRuntime() - if err != nil { - t.Fatal(err) - } + runtime := mkRuntime(t) defer nuke(runtime) container, err := NewBuilder(runtime).Create( &Config{ @@ -484,10 +468,7 @@ func TestRun(t *testing.T) { } func TestOutput(t *testing.T) { - runtime, err := newTestRuntime() - if err != nil { - t.Fatal(err) - } + runtime := mkRuntime(t) defer nuke(runtime) container, err := NewBuilder(runtime).Create( &Config{ @@ -509,10 +490,7 @@ func TestOutput(t *testing.T) { } func TestKillDifferentUser(t *testing.T) { - runtime, err := newTestRuntime() - if err != nil { - t.Fatal(err) - } + runtime := mkRuntime(t) defer nuke(runtime) container, err := NewBuilder(runtime).Create(&Config{ Image: GetTestImage(runtime).ID, @@ -528,7 +506,8 @@ func TestKillDifferentUser(t *testing.T) { if container.State.Running { t.Errorf("Container shouldn't be running") } - if err := container.Start(); err != nil { + hostConfig := &HostConfig{} + if err := container.Start(hostConfig); err != nil { t.Fatal(err) } @@ -556,15 +535,33 @@ func TestKillDifferentUser(t *testing.T) { } } -func TestKill(t *testing.T) { - runtime, err := newTestRuntime() +// Test that creating a container with a volume doesn't crash. Regression test for #995. +func TestCreateVolume(t *testing.T) { + runtime := mkRuntime(t) + defer nuke(runtime) + + config, hc, _, err := ParseRun([]string{"-v", "/var/lib/data", GetTestImage(runtime).ID, "echo", "hello", "world"}, nil) if err != nil { t.Fatal(err) } + c, err := NewBuilder(runtime).Create(config) + if err != nil { + t.Fatal(err) + } + defer runtime.Destroy(c) + if err := c.Start(hc); err != nil { + t.Fatal(err) + } + c.WaitTimeout(500 * time.Millisecond) + c.Wait() +} + +func TestKill(t *testing.T) { + runtime := mkRuntime(t) defer nuke(runtime) container, err := NewBuilder(runtime).Create(&Config{ Image: GetTestImage(runtime).ID, - Cmd: []string{"cat", "/dev/zero"}, + Cmd: []string{"sleep", "2"}, }, ) if err != nil { @@ -575,7 +572,8 @@ func TestKill(t *testing.T) { if container.State.Running { t.Errorf("Container shouldn't be running") } - if err := container.Start(); err != nil { + hostConfig := &HostConfig{} + if err := container.Start(hostConfig); err != nil { t.Fatal(err) } @@ -602,10 +600,7 @@ func TestKill(t *testing.T) { } func TestExitCode(t *testing.T) { - runtime, err := newTestRuntime() - if err != nil { - t.Fatal(err) - } + runtime := mkRuntime(t) defer nuke(runtime) builder := NewBuilder(runtime) @@ -642,10 +637,7 @@ func TestExitCode(t *testing.T) { } func TestRestart(t *testing.T) { - runtime, err := newTestRuntime() - if err != nil { - t.Fatal(err) - } + runtime := mkRuntime(t) defer nuke(runtime) container, err := NewBuilder(runtime).Create(&Config{ Image: GetTestImage(runtime).ID, @@ -675,10 +667,7 @@ func TestRestart(t *testing.T) { } func TestRestartStdin(t *testing.T) { - runtime, err := newTestRuntime() - if err != nil { - t.Fatal(err) - } + runtime := mkRuntime(t) defer nuke(runtime) container, err := NewBuilder(runtime).Create(&Config{ Image: GetTestImage(runtime).ID, @@ -700,7 +689,8 @@ func TestRestartStdin(t *testing.T) { if err != nil { t.Fatal(err) } - if err := container.Start(); err != nil { + hostConfig := &HostConfig{} + if err := container.Start(hostConfig); err != nil { t.Fatal(err) } if _, err := io.WriteString(stdin, "hello world"); err != nil { @@ -730,7 +720,7 @@ func TestRestartStdin(t *testing.T) { if err != nil { t.Fatal(err) } - if err := container.Start(); err != nil { + if err := container.Start(hostConfig); err != nil { t.Fatal(err) } if _, err := io.WriteString(stdin, "hello world #2"); err != nil { @@ -753,10 +743,7 @@ func TestRestartStdin(t *testing.T) { } func TestUser(t *testing.T) { - runtime, err := newTestRuntime() - if err != nil { - t.Fatal(err) - } + runtime := mkRuntime(t) defer nuke(runtime) builder := NewBuilder(runtime) @@ -863,17 +850,14 @@ func TestUser(t *testing.T) { } func TestMultipleContainers(t *testing.T) { - runtime, err := newTestRuntime() - if err != nil { - t.Fatal(err) - } + runtime := mkRuntime(t) defer nuke(runtime) builder := NewBuilder(runtime) container1, err := builder.Create(&Config{ Image: GetTestImage(runtime).ID, - Cmd: []string{"cat", "/dev/zero"}, + Cmd: []string{"sleep", "2"}, }, ) if err != nil { @@ -883,7 +867,7 @@ func TestMultipleContainers(t *testing.T) { container2, err := builder.Create(&Config{ Image: GetTestImage(runtime).ID, - Cmd: []string{"cat", "/dev/zero"}, + Cmd: []string{"sleep", "2"}, }, ) if err != nil { @@ -892,10 +876,11 @@ func TestMultipleContainers(t *testing.T) { defer runtime.Destroy(container2) // Start both containers - if err := container1.Start(); err != nil { + hostConfig := &HostConfig{} + if err := container1.Start(hostConfig); err != nil { t.Fatal(err) } - if err := container2.Start(); err != nil { + if err := container2.Start(hostConfig); err != nil { t.Fatal(err) } @@ -922,10 +907,7 @@ func TestMultipleContainers(t *testing.T) { } func TestStdin(t *testing.T) { - runtime, err := newTestRuntime() - if err != nil { - t.Fatal(err) - } + runtime := mkRuntime(t) defer nuke(runtime) container, err := NewBuilder(runtime).Create(&Config{ Image: GetTestImage(runtime).ID, @@ -947,7 +929,8 @@ func TestStdin(t *testing.T) { if err != nil { t.Fatal(err) } - if err := container.Start(); err != nil { + hostConfig := &HostConfig{} + if err := container.Start(hostConfig); err != nil { t.Fatal(err) } defer stdin.Close() @@ -969,10 +952,7 @@ func TestStdin(t *testing.T) { } func TestTty(t *testing.T) { - runtime, err := newTestRuntime() - if err != nil { - t.Fatal(err) - } + runtime := mkRuntime(t) defer nuke(runtime) container, err := NewBuilder(runtime).Create(&Config{ Image: GetTestImage(runtime).ID, @@ -994,7 +974,8 @@ func TestTty(t *testing.T) { if err != nil { t.Fatal(err) } - if err := container.Start(); err != nil { + hostConfig := &HostConfig{} + if err := container.Start(hostConfig); err != nil { t.Fatal(err) } defer stdin.Close() @@ -1016,10 +997,7 @@ func TestTty(t *testing.T) { } func TestEnv(t *testing.T) { - runtime, err := newTestRuntime() - if err != nil { - t.Fatal(err) - } + runtime := mkRuntime(t) defer nuke(runtime) container, err := NewBuilder(runtime).Create(&Config{ Image: GetTestImage(runtime).ID, @@ -1036,7 +1014,8 @@ func TestEnv(t *testing.T) { t.Fatal(err) } defer stdout.Close() - if err := container.Start(); err != nil { + hostConfig := &HostConfig{} + if err := container.Start(hostConfig); err != nil { t.Fatal(err) } container.Wait() @@ -1085,10 +1064,7 @@ func grepFile(t *testing.T, path string, pattern string) { } func TestLXCConfig(t *testing.T) { - runtime, err := newTestRuntime() - if err != nil { - t.Fatal(err) - } + runtime := mkRuntime(t) defer nuke(runtime) // Memory is allocated randomly for testing rand.Seed(time.Now().UTC().UnixNano()) @@ -1172,7 +1148,8 @@ func BenchmarkRunParallel(b *testing.B) { return } defer runtime.Destroy(container) - if err := container.Start(); err != nil { + hostConfig := &HostConfig{} + if err := container.Start(hostConfig); err != nil { complete <- err return } @@ -1201,3 +1178,35 @@ func BenchmarkRunParallel(b *testing.B) { b.Fatal(errors) } } + +func tempDir(t *testing.T) string { + tmpDir, err := ioutil.TempDir("", "docker-test") + if err != nil { + t.Fatal(err) + } + return tmpDir +} + +func TestBindMounts(t *testing.T) { + r := mkRuntime(t) + defer nuke(r) + tmpDir := tempDir(t) + defer os.RemoveAll(tmpDir) + writeFile(path.Join(tmpDir, "touch-me"), "", t) + + // Test reading from a read-only bind mount + stdout, _ := runContainer(r, []string{"-b", fmt.Sprintf("%s:/tmp:ro", tmpDir), "_", "ls", "/tmp"}, t) + if !strings.Contains(stdout, "touch-me") { + t.Fatal("Container failed to read from bind mount") + } + + // test writing to bind mount + runContainer(r, []string{"-b", fmt.Sprintf("%s:/tmp:rw", tmpDir), "_", "touch", "/tmp/holla"}, t) + readFile(path.Join(tmpDir, "holla"), t) // Will fail if the file doesn't exist + + // test mounting to an illegal destination directory + if _, err := runContainer(r, []string{"-b", fmt.Sprintf("%s:.", tmpDir), "ls", "."}, nil); err == nil { + t.Fatal("Container bind mounted illegal directory") + + } +} diff --git a/contrib/crashTest.go b/contrib/crashTest.go index b3dbacaf03..d3ba80698c 100644 --- a/contrib/crashTest.go +++ b/contrib/crashTest.go @@ -116,7 +116,6 @@ func crashTest() error { return err } } - return nil } func main() { diff --git a/contrib/mkimage-unittest.sh b/contrib/mkimage-unittest.sh new file mode 100755 index 0000000000..40494c642f --- /dev/null +++ b/contrib/mkimage-unittest.sh @@ -0,0 +1,46 @@ +#!/bin/bash +# Generate a very minimal filesystem based on busybox-static, +# and load it into the local docker under the name "docker-ut". + +missing_pkg() { + echo "Sorry, I could not locate $1" + echo "Try 'apt-get install ${2:-$1}'?" + exit 1 +} + +BUSYBOX=$(which busybox) +[ "$BUSYBOX" ] || missing_pkg busybox busybox-static +SOCAT=$(which socat) +[ "$SOCAT" ] || missing_pkg socat + +shopt -s extglob +set -ex +ROOTFS=`mktemp -d /tmp/rootfs-busybox.XXXXXXXXXX` +trap "rm -rf $ROOTFS" INT QUIT TERM +cd $ROOTFS + +mkdir bin etc dev dev/pts lib proc sys tmp +touch etc/resolv.conf +cp /etc/nsswitch.conf etc/nsswitch.conf +echo root:x:0:0:root:/:/bin/sh > etc/passwd +echo root:x:0: > etc/group +ln -s lib lib64 +ln -s bin sbin +cp $BUSYBOX $SOCAT bin +for X in $(busybox --list) +do + ln -s busybox bin/$X +done +rm bin/init +ln bin/busybox bin/init +cp -P /lib/x86_64-linux-gnu/lib{pthread*,c*(-*),dl*(-*),nsl*(-*),nss_*,util*(-*),wrap,z}.so* lib +cp /lib/x86_64-linux-gnu/ld-linux-x86-64.so.2 lib +cp -P /usr/lib/x86_64-linux-gnu/lib{crypto,ssl}.so* lib +for X in console null ptmx random stdin stdout stderr tty urandom zero +do + cp -a /dev/$X dev +done + +tar -cf- . | docker import - docker-ut +docker run -i -u root docker-ut /bin/echo Success. +rm -rf $ROOTFS diff --git a/docker/docker.go b/docker/docker.go index 6d79972bd6..c508d8905e 100644 --- a/docker/docker.go +++ b/docker/docker.go @@ -30,6 +30,7 @@ func main() { flAutoRestart := flag.Bool("r", false, "Restart previously running containers") bridgeName := flag.String("b", "", "Attach containers to a pre-existing network bridge") pidfile := flag.String("p", "/var/run/docker.pid", "File containing process PID") + flGraphPath := flag.String("g", "/var/lib/docker", "Path to graph storage base dir.") flEnableCors := flag.Bool("api-enable-cors", false, "Enable CORS requests in the remote api.") flDns := flag.String("dns", "", "Set custom dns servers") flHosts := docker.ListOpts{fmt.Sprintf("tcp://%s:%d", docker.DEFAULTHTTPHOST, docker.DEFAULTHTTPPORT)} @@ -56,7 +57,7 @@ func main() { flag.Usage() return } - if err := daemon(*pidfile, flHosts, *flAutoRestart, *flEnableCors, *flDns); err != nil { + if err := daemon(*pidfile, *flGraphPath, flHosts, *flAutoRestart, *flEnableCors, *flDns); err != nil { log.Fatal(err) os.Exit(-1) } @@ -100,7 +101,7 @@ func removePidFile(pidfile string) { } } -func daemon(pidfile string, protoAddrs []string, autoRestart, enableCors bool, flDns string) error { +func daemon(pidfile string, flGraphPath string, protoAddrs []string, autoRestart, enableCors bool, flDns string) error { if err := createPidFile(pidfile); err != nil { log.Fatal(err) } @@ -118,7 +119,7 @@ func daemon(pidfile string, protoAddrs []string, autoRestart, enableCors bool, f if flDns != "" { dns = []string{flDns} } - server, err := docker.NewServer(autoRestart, enableCors, dns) + server, err := docker.NewServer(flGraphPath, autoRestart, enableCors, dns) if err != nil { return err } @@ -126,7 +127,7 @@ func daemon(pidfile string, protoAddrs []string, autoRestart, enableCors bool, f for _, protoAddr := range protoAddrs { protoAddrParts := strings.SplitN(protoAddr, "://", 2) if protoAddrParts[0] == "unix" { - syscall.Unlink(protoAddrParts[1]); + syscall.Unlink(protoAddrParts[1]) } else if protoAddrParts[0] == "tcp" { if !strings.HasPrefix(protoAddrParts[1], "127.0.0.1") { log.Println("/!\\ DON'T BIND ON ANOTHER IP ADDRESS THAN 127.0.0.1 IF YOU DON'T KNOW WHAT YOU'RE DOING /!\\") @@ -139,7 +140,7 @@ func daemon(pidfile string, protoAddrs []string, autoRestart, enableCors bool, f chErrors <- docker.ListenAndServe(protoAddrParts[0], protoAddrParts[1], server, true) }() } - for i :=0 ; i < len(protoAddrs); i+=1 { + for i := 0; i < len(protoAddrs); i += 1 { err := <-chErrors if err != nil { return err @@ -147,4 +148,3 @@ func daemon(pidfile string, protoAddrs []string, autoRestart, enableCors bool, f } return nil } - diff --git a/docs/sources/api/docker_remote_api.rst b/docs/sources/api/docker_remote_api.rst index 99e819655d..c6e50a3f06 100644 --- a/docs/sources/api/docker_remote_api.rst +++ b/docs/sources/api/docker_remote_api.rst @@ -19,13 +19,38 @@ Docker Remote API 2. Versions =========== -The current verson of the API is 1.2 -Calling /images//insert is the same as calling /v1.2/images//insert +The current verson of the API is 1.3 +Calling /images//insert is the same as calling /v1.3/images//insert You can still call an old version of the api using /v1.0/images//insert +:doc:`docker_remote_api_v1.3` +***************************** + +What's new +---------- + +Builder (/build): + +- Simplify the upload of the build context +- Simply stream a tarball instead of multipart upload with 4 intermediary buffers +- Simpler, less memory usage, less disk usage and faster + +.. Note:: +The /build improvements are not reverse-compatible. Pre 1.3 clients will break on /build. + +List containers (/containers/json): + +- You can use size=1 to get the size of the containers + +Start containers (/containers//start): + +- You can now pass host-specific configuration (e.g. bind mounts) in the POST body for start calls + :doc:`docker_remote_api_v1.2` ***************************** +docker v0.4.2 2e7649b_ + What's new ---------- @@ -65,6 +90,9 @@ Uses json stream instead of HTML hijack, it looks like this: ... +:doc:`docker_remote_api_v1.0` +***************************** + docker v0.3.4 8d73740_ What's new @@ -75,6 +103,7 @@ Initial version .. _a8ae398: https://github.com/dotcloud/docker/commit/a8ae398bf52e97148ee7bd0d5868de2e15bd297f .. _8d73740: https://github.com/dotcloud/docker/commit/8d73740343778651c09160cde9661f5f387b36f4 +.. _2e7649b: https://github.com/dotcloud/docker/commit/2e7649beda7c820793bd46766cbc2cfeace7b168 ================================== Docker Remote API Client Libraries @@ -94,6 +123,8 @@ and we will add the libraries here. +----------------------+----------------+--------------------------------------------+ | Ruby | docker-client | https://github.com/geku/docker-client | +----------------------+----------------+--------------------------------------------+ +| Ruby | docker-api | https://github.com/swipely/docker-api | ++----------------------+----------------+--------------------------------------------+ | Javascript | docker-js | https://github.com/dgoujard/docker-js | +----------------------+----------------+--------------------------------------------+ | Javascript (Angular) | dockerui | https://github.com/crosbymichael/dockerui | diff --git a/docs/sources/api/docker_remote_api_v1.2.rst b/docs/sources/api/docker_remote_api_v1.2.rst index 5448436d75..a6c2c31920 100644 --- a/docs/sources/api/docker_remote_api_v1.2.rst +++ b/docs/sources/api/docker_remote_api_v1.2.rst @@ -847,7 +847,7 @@ Build an image from Dockerfile via stdin .. http:post:: /build - Build an image from Dockerfile via stdin + Build an image from Dockerfile **Example request**: @@ -866,9 +866,12 @@ Build an image from Dockerfile via stdin {{ STREAM }} :query t: tag to be applied to the resulting image in case of success + :query remote: resource to fetch, as URI :statuscode 200: no error :statuscode 500: server error +{{ STREAM }} is the raw text output of the build command. It uses the HTTP Hijack method in order to stream. + Check auth configuration ************************ diff --git a/docs/sources/api/docker_remote_api_v1.3.rst b/docs/sources/api/docker_remote_api_v1.3.rst new file mode 100644 index 0000000000..8eeb010d98 --- /dev/null +++ b/docs/sources/api/docker_remote_api_v1.3.rst @@ -0,0 +1,1046 @@ +:title: Remote API v1.3 +:description: API Documentation for Docker +:keywords: API, Docker, rcli, REST, documentation + +====================== +Docker Remote API v1.3 +====================== + +.. contents:: Table of Contents + +1. Brief introduction +===================== + +- The Remote API is replacing rcli +- Default port in the docker deamon is 4243 +- The API tends to be REST, but for some complex commands, like attach or pull, the HTTP connection is hijacked to transport stdout stdin and stderr + +2. Endpoints +============ + +2.1 Containers +-------------- + +List containers +*************** + +.. http:get:: /containers/json + + List containers + + **Example request**: + + .. sourcecode:: http + + GET /containers/json?all=1&before=8dfafdbc3a40&size=1 HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + [ + { + "Id": "8dfafdbc3a40", + "Image": "base:latest", + "Command": "echo 1", + "Created": 1367854155, + "Status": "Exit 0", + "Ports":"", + "SizeRw":12288, + "SizeRootFs":0 + }, + { + "Id": "9cd87474be90", + "Image": "base:latest", + "Command": "echo 222222", + "Created": 1367854155, + "Status": "Exit 0", + "Ports":"", + "SizeRw":12288, + "SizeRootFs":0 + }, + { + "Id": "3176a2479c92", + "Image": "base:latest", + "Command": "echo 3333333333333333", + "Created": 1367854154, + "Status": "Exit 0", + "Ports":"", + "SizeRw":12288, + "SizeRootFs":0 + }, + { + "Id": "4cb07b47f9fb", + "Image": "base:latest", + "Command": "echo 444444444444444444444444444444444", + "Created": 1367854152, + "Status": "Exit 0", + "Ports":"", + "SizeRw":12288, + "SizeRootFs":0 + } + ] + + :query all: 1/True/true or 0/False/false, Show all containers. Only running containers are shown by default + :query limit: Show ``limit`` last created containers, include non-running ones. + :query since: Show only containers created since Id, include non-running ones. + :query before: Show only containers created before Id, include non-running ones. + :query size: 1/True/true or 0/False/false, Show the containers sizes + :statuscode 200: no error + :statuscode 400: bad parameter + :statuscode 500: server error + + +Create a container +****************** + +.. http:post:: /containers/create + + Create a container + + **Example request**: + + .. sourcecode:: http + + POST /containers/create HTTP/1.1 + Content-Type: application/json + + { + "Hostname":"", + "User":"", + "Memory":0, + "MemorySwap":0, + "AttachStdin":false, + "AttachStdout":true, + "AttachStderr":true, + "PortSpecs":null, + "Tty":false, + "OpenStdin":false, + "StdinOnce":false, + "Env":null, + "Cmd":[ + "date" + ], + "Dns":null, + "Image":"base", + "Volumes":{}, + "VolumesFrom":"" + } + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 201 OK + Content-Type: application/json + + { + "Id":"e90e34656806" + "Warnings":[] + } + + :jsonparam config: the container's configuration + :statuscode 201: no error + :statuscode 404: no such container + :statuscode 406: impossible to attach (container not running) + :statuscode 500: server error + + +Inspect a container +******************* + +.. http:get:: /containers/(id)/json + + Return low-level information on the container ``id`` + + **Example request**: + + .. sourcecode:: http + + GET /containers/4fa6e0f0c678/json HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + { + "Id": "4fa6e0f0c6786287e131c3852c58a2e01cc697a68231826813597e4994f1d6e2", + "Created": "2013-05-07T14:51:42.041847+02:00", + "Path": "date", + "Args": [], + "Config": { + "Hostname": "4fa6e0f0c678", + "User": "", + "Memory": 0, + "MemorySwap": 0, + "AttachStdin": false, + "AttachStdout": true, + "AttachStderr": true, + "PortSpecs": null, + "Tty": false, + "OpenStdin": false, + "StdinOnce": false, + "Env": null, + "Cmd": [ + "date" + ], + "Dns": null, + "Image": "base", + "Volumes": {}, + "VolumesFrom": "" + }, + "State": { + "Running": false, + "Pid": 0, + "ExitCode": 0, + "StartedAt": "2013-05-07T14:51:42.087658+02:01360", + "Ghost": false + }, + "Image": "b750fe79269d2ec9a3c593ef05b4332b1d1a02a62b4accb2c21d589ff2f5f2dc", + "NetworkSettings": { + "IpAddress": "", + "IpPrefixLen": 0, + "Gateway": "", + "Bridge": "", + "PortMapping": null + }, + "SysInitPath": "/home/kitty/go/src/github.com/dotcloud/docker/bin/docker", + "ResolvConfPath": "/etc/resolv.conf", + "Volumes": {} + } + + :statuscode 200: no error + :statuscode 404: no such container + :statuscode 500: server error + + +Inspect changes on a container's filesystem +******************************************* + +.. http:get:: /containers/(id)/changes + + Inspect changes on container ``id`` 's filesystem + + **Example request**: + + .. sourcecode:: http + + GET /containers/4fa6e0f0c678/changes HTTP/1.1 + + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + [ + { + "Path":"/dev", + "Kind":0 + }, + { + "Path":"/dev/kmsg", + "Kind":1 + }, + { + "Path":"/test", + "Kind":1 + } + ] + + :statuscode 200: no error + :statuscode 404: no such container + :statuscode 500: server error + + +Export a container +****************** + +.. http:get:: /containers/(id)/export + + Export the contents of container ``id`` + + **Example request**: + + .. sourcecode:: http + + GET /containers/4fa6e0f0c678/export HTTP/1.1 + + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/octet-stream + + {{ STREAM }} + + :statuscode 200: no error + :statuscode 404: no such container + :statuscode 500: server error + + +Start a container +***************** + +.. http:post:: /containers/(id)/start + + Start the container ``id`` + + **Example request**: + + .. sourcecode:: http + + POST /containers/(id)/start HTTP/1.1 + Content-Type: application/json + + { + "Binds":["/tmp:/tmp"] + } + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 204 No Content + Content-Type: text/plain + + :jsonparam hostConfig: the container's host configuration (optional) + :statuscode 200: no error + :statuscode 404: no such container + :statuscode 500: server error + + +Stop a contaier +*************** + +.. http:post:: /containers/(id)/stop + + Stop the container ``id`` + + **Example request**: + + .. sourcecode:: http + + POST /containers/e90e34656806/stop?t=5 HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 204 OK + + :query t: number of seconds to wait before killing the container + :statuscode 204: no error + :statuscode 404: no such container + :statuscode 500: server error + + +Restart a container +******************* + +.. http:post:: /containers/(id)/restart + + Restart the container ``id`` + + **Example request**: + + .. sourcecode:: http + + POST /containers/e90e34656806/restart?t=5 HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 204 OK + + :query t: number of seconds to wait before killing the container + :statuscode 204: no error + :statuscode 404: no such container + :statuscode 500: server error + + +Kill a container +**************** + +.. http:post:: /containers/(id)/kill + + Kill the container ``id`` + + **Example request**: + + .. sourcecode:: http + + POST /containers/e90e34656806/kill HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 204 OK + + :statuscode 204: no error + :statuscode 404: no such container + :statuscode 500: server error + + +Attach to a container +********************* + +.. http:post:: /containers/(id)/attach + + Attach to the container ``id`` + + **Example request**: + + .. sourcecode:: http + + POST /containers/16253994b7c4/attach?logs=1&stream=0&stdout=1 HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/vnd.docker.raw-stream + + {{ STREAM }} + + :query logs: 1/True/true or 0/False/false, return logs. Default false + :query stream: 1/True/true or 0/False/false, return stream. Default false + :query stdin: 1/True/true or 0/False/false, if stream=true, attach to stdin. Default false + :query stdout: 1/True/true or 0/False/false, if logs=true, return stdout log, if stream=true, attach to stdout. Default false + :query stderr: 1/True/true or 0/False/false, if logs=true, return stderr log, if stream=true, attach to stderr. Default false + :statuscode 200: no error + :statuscode 400: bad parameter + :statuscode 404: no such container + :statuscode 500: server error + + +Wait a container +**************** + +.. http:post:: /containers/(id)/wait + + Block until container ``id`` stops, then returns the exit code + + **Example request**: + + .. sourcecode:: http + + POST /containers/16253994b7c4/wait HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + {"StatusCode":0} + + :statuscode 200: no error + :statuscode 404: no such container + :statuscode 500: server error + + +Remove a container +******************* + +.. http:delete:: /containers/(id) + + Remove the container ``id`` from the filesystem + + **Example request**: + + .. sourcecode:: http + + DELETE /containers/16253994b7c4?v=1 HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 204 OK + + :query v: 1/True/true or 0/False/false, Remove the volumes associated to the container. Default false + :statuscode 204: no error + :statuscode 400: bad parameter + :statuscode 404: no such container + :statuscode 500: server error + + +2.2 Images +---------- + +List Images +*********** + +.. http:get:: /images/(format) + + List images ``format`` could be json or viz (json default) + + **Example request**: + + .. sourcecode:: http + + GET /images/json?all=0 HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + [ + { + "Repository":"base", + "Tag":"ubuntu-12.10", + "Id":"b750fe79269d", + "Created":1364102658, + "Size":24653, + "VirtualSize":180116135 + }, + { + "Repository":"base", + "Tag":"ubuntu-quantal", + "Id":"b750fe79269d", + "Created":1364102658, + "Size":24653, + "VirtualSize":180116135 + } + ] + + + **Example request**: + + .. sourcecode:: http + + GET /images/viz HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: text/plain + + digraph docker { + "d82cbacda43a" -> "074be284591f" + "1496068ca813" -> "08306dc45919" + "08306dc45919" -> "0e7893146ac2" + "b750fe79269d" -> "1496068ca813" + base -> "27cf78414709" [style=invis] + "f71189fff3de" -> "9a33b36209ed" + "27cf78414709" -> "b750fe79269d" + "0e7893146ac2" -> "d6434d954665" + "d6434d954665" -> "d82cbacda43a" + base -> "e9aa60c60128" [style=invis] + "074be284591f" -> "f71189fff3de" + "b750fe79269d" [label="b750fe79269d\nbase",shape=box,fillcolor="paleturquoise",style="filled,rounded"]; + "e9aa60c60128" [label="e9aa60c60128\nbase2",shape=box,fillcolor="paleturquoise",style="filled,rounded"]; + "9a33b36209ed" [label="9a33b36209ed\ntest",shape=box,fillcolor="paleturquoise",style="filled,rounded"]; + base [style=invisible] + } + + :query all: 1/True/true or 0/False/false, Show all containers. Only running containers are shown by default + :statuscode 200: no error + :statuscode 400: bad parameter + :statuscode 500: server error + + +Create an image +*************** + +.. http:post:: /images/create + + Create an image, either by pull it from the registry or by importing it + + **Example request**: + + .. sourcecode:: http + + POST /images/create?fromImage=base HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + {"status":"Pulling..."} + {"status":"Pulling", "progress":"1/? (n/a)"} + {"error":"Invalid..."} + ... + + :query fromImage: name of the image to pull + :query fromSrc: source to import, - means stdin + :query repo: repository + :query tag: tag + :query registry: the registry to pull from + :statuscode 200: no error + :statuscode 500: server error + + +Insert a file in a image +************************ + +.. http:post:: /images/(name)/insert + + Insert a file from ``url`` in the image ``name`` at ``path`` + + **Example request**: + + .. sourcecode:: http + + POST /images/test/insert?path=/usr&url=myurl HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + {"status":"Inserting..."} + {"status":"Inserting", "progress":"1/? (n/a)"} + {"error":"Invalid..."} + ... + + :statuscode 200: no error + :statuscode 500: server error + + +Inspect an image +**************** + +.. http:get:: /images/(name)/json + + Return low-level information on the image ``name`` + + **Example request**: + + .. sourcecode:: http + + GET /images/base/json HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + { + "id":"b750fe79269d2ec9a3c593ef05b4332b1d1a02a62b4accb2c21d589ff2f5f2dc", + "parent":"27cf784147099545", + "created":"2013-03-23T22:24:18.818426-07:00", + "container":"3d67245a8d72ecf13f33dffac9f79dcdf70f75acb84d308770391510e0c23ad0", + "container_config": + { + "Hostname":"", + "User":"", + "Memory":0, + "MemorySwap":0, + "AttachStdin":false, + "AttachStdout":false, + "AttachStderr":false, + "PortSpecs":null, + "Tty":true, + "OpenStdin":true, + "StdinOnce":false, + "Env":null, + "Cmd": ["/bin/bash"] + ,"Dns":null, + "Image":"base", + "Volumes":null, + "VolumesFrom":"" + }, + "Size": 6824592 + } + + :statuscode 200: no error + :statuscode 404: no such image + :statuscode 500: server error + + +Get the history of an image +*************************** + +.. http:get:: /images/(name)/history + + Return the history of the image ``name`` + + **Example request**: + + .. sourcecode:: http + + GET /images/base/history HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + [ + { + "Id":"b750fe79269d", + "Created":1364102658, + "CreatedBy":"/bin/bash" + }, + { + "Id":"27cf78414709", + "Created":1364068391, + "CreatedBy":"" + } + ] + + :statuscode 200: no error + :statuscode 404: no such image + :statuscode 500: server error + + +Push an image on the registry +***************************** + +.. http:post:: /images/(name)/push + + Push the image ``name`` on the registry + + **Example request**: + + .. sourcecode:: http + + POST /images/test/push HTTP/1.1 + {{ authConfig }} + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + {"status":"Pushing..."} + {"status":"Pushing", "progress":"1/? (n/a)"} + {"error":"Invalid..."} + ... + + :query registry: the registry you wan to push, optional + :statuscode 200: no error + :statuscode 404: no such image + :statuscode 500: server error + + +Tag an image into a repository +****************************** + +.. http:post:: /images/(name)/tag + + Tag the image ``name`` into a repository + + **Example request**: + + .. sourcecode:: http + + POST /images/test/tag?repo=myrepo&force=0 HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + + :query repo: The repository to tag in + :query force: 1/True/true or 0/False/false, default false + :statuscode 200: no error + :statuscode 400: bad parameter + :statuscode 404: no such image + :statuscode 409: conflict + :statuscode 500: server error + + +Remove an image +*************** + +.. http:delete:: /images/(name) + + Remove the image ``name`` from the filesystem + + **Example request**: + + .. sourcecode:: http + + DELETE /images/test HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-type: application/json + + [ + {"Untagged":"3e2f21a89f"}, + {"Deleted":"3e2f21a89f"}, + {"Deleted":"53b4f83ac9"} + ] + + :statuscode 204: no error + :statuscode 404: no such image + :statuscode 409: conflict + :statuscode 500: server error + + +Search images +************* + +.. http:get:: /images/search + + Search for an image in the docker index + + **Example request**: + + .. sourcecode:: http + + GET /images/search?term=sshd HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + [ + { + "Name":"cespare/sshd", + "Description":"" + }, + { + "Name":"johnfuller/sshd", + "Description":"" + }, + { + "Name":"dhrp/mongodb-sshd", + "Description":"" + } + ] + + :query term: term to search + :statuscode 200: no error + :statuscode 500: server error + + +2.3 Misc +-------- + +Build an image from Dockerfile via stdin +**************************************** + +.. http:post:: /build + + Build an image from Dockerfile via stdin + + **Example request**: + + .. sourcecode:: http + + POST /build HTTP/1.1 + + {{ STREAM }} + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + + {{ STREAM }} + + + The stream must be a tar archive compressed with one of the following algorithms: + identity (no compression), gzip, bzip2, xz. The archive must include a file called + `Dockerfile` at its root. It may include any number of other files, which will be + accessible in the build context (See the ADD build command). + + The Content-type header should be set to "application/tar". + + :query t: tag to be applied to the resulting image in case of success + :statuscode 200: no error + :statuscode 500: server error + + +Check auth configuration +************************ + +.. http:post:: /auth + + Get the default username and email + + **Example request**: + + .. sourcecode:: http + + POST /auth HTTP/1.1 + Content-Type: application/json + + { + "username":"hannibal", + "password:"xxxx", + "email":"hannibal@a-team.com" + } + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + + :statuscode 200: no error + :statuscode 204: no error + :statuscode 500: server error + + +Display system-wide information +******************************* + +.. http:get:: /info + + Display system-wide information + + **Example request**: + + .. sourcecode:: http + + GET /info HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + { + "Containers":11, + "Images":16, + "Debug":false, + "NFd": 11, + "NGoroutines":21, + "MemoryLimit":true, + "SwapLimit":false + } + + :statuscode 200: no error + :statuscode 500: server error + + +Show the docker version information +*********************************** + +.. http:get:: /version + + Show the docker version information + + **Example request**: + + .. sourcecode:: http + + GET /version HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + { + "Version":"0.2.2", + "GitCommit":"5a2a5cc+CHANGES", + "GoVersion":"go1.0.3" + } + + :statuscode 200: no error + :statuscode 500: server error + + +Create a new image from a container's changes +********************************************* + +.. http:post:: /commit + + Create a new image from a container's changes + + **Example request**: + + .. sourcecode:: http + + POST /commit?container=44c004db4b17&m=message&repo=myrepo HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 201 OK + Content-Type: application/vnd.docker.raw-stream + + {"Id":"596069db4bf5"} + + :query container: source container + :query repo: repository + :query tag: tag + :query m: commit message + :query author: author (eg. "John Hannibal Smith ") + :query run: config automatically applied when the image is run. (ex: {"Cmd": ["cat", "/world"], "PortSpecs":["22"]}) + :statuscode 201: no error + :statuscode 404: no such container + :statuscode 500: server error + + +3. Going further +================ + +3.1 Inside 'docker run' +----------------------- + +Here are the steps of 'docker run' : + +* Create the container +* If the status code is 404, it means the image doesn't exists: + * Try to pull it + * Then retry to create the container +* Start the container +* If you are not in detached mode: + * Attach to the container, using logs=1 (to have stdout and stderr from the container's start) and stream=1 +* If in detached mode or only stdin is attached: + * Display the container's id + + +3.2 Hijacking +------------- + +In this version of the API, /attach, uses hijacking to transport stdin, stdout and stderr on the same socket. This might change in the future. + +3.3 CORS Requests +----------------- + +To enable cross origin requests to the remote api add the flag "-api-enable-cors" when running docker in daemon mode. + + docker -d -H="192.168.1.9:4243" -api-enable-cors + diff --git a/docs/sources/commandline/command/build.rst b/docs/sources/commandline/command/build.rst index 254b0371a9..1645002ba2 100644 --- a/docs/sources/commandline/command/build.rst +++ b/docs/sources/commandline/command/build.rst @@ -8,9 +8,11 @@ :: - Usage: docker build [OPTIONS] PATH | - + Usage: docker build [OPTIONS] PATH | URL | - Build a new container image from the source code at PATH -t="": Tag to be applied to the resulting image in case of success. + When a single Dockerfile is given as URL, then no context is set. When a git repository is set as URL, the repository is used as context + Examples -------- @@ -27,7 +29,15 @@ Examples .. code-block:: bash - docker build - + docker build - < Dockerfile | This will read a Dockerfile from Stdin without context. Due to the lack of a context, no contents of any local directory will be sent to the docker daemon. | ADD doesn't work when running in this mode due to the absence of the context, thus having no source files to copy to the container. + + +.. code-block:: bash + + docker build github.com/creack/docker-firefox + +| This will clone the github repository and use it as context. The Dockerfile at the root of the repository is used as Dockerfile. +| Note that you can specify an arbitrary git repository by using the 'git://' schema. diff --git a/docs/sources/commandline/command/run.rst b/docs/sources/commandline/command/run.rst index d6c9aef315..c119ce11e2 100644 --- a/docs/sources/commandline/command/run.rst +++ b/docs/sources/commandline/command/run.rst @@ -8,7 +8,7 @@ :: - Usage: docker run [OPTIONS] IMAGE COMMAND [ARG...] + Usage: docker run [OPTIONS] IMAGE [COMMAND] [ARG...] Run a command in a new container @@ -25,3 +25,4 @@ -d=[]: Set custom dns servers for the container -v=[]: Creates a new volume and mounts it at the specified path. -volumes-from="": Mount all volumes from the given container. + -b=[]: Create a bind mount with: [host-dir]:[container-dir]:[rw|ro] diff --git a/docs/sources/conf.py b/docs/sources/conf.py index 41dba70201..ea8e1f43f0 100644 --- a/docs/sources/conf.py +++ b/docs/sources/conf.py @@ -30,6 +30,7 @@ import sys, os html_additional_pages = { 'concepts/containers': 'redirect_home.html', 'concepts/introduction': 'redirect_home.html', + 'builder/basics': 'redirect_build.html', } diff --git a/docs/sources/terms/fundamentals.rst b/docs/sources/terms/fundamentals.rst new file mode 100644 index 0000000000..fed3decb08 --- /dev/null +++ b/docs/sources/terms/fundamentals.rst @@ -0,0 +1,97 @@ +:title: Image & Container +:description: Definitions of an image and container +:keywords: containers, lxc, concepts, explanation, image, container + +File Systems +============ + +.. image:: images/docker-filesystems-generic.png + +In order for a Linux system to run, it typically needs two `file +systems `_: + +1. boot file system (bootfs) +2. root file system (rootfs) + +The **boot file system** contains the bootloader and the kernel. The +user never makes any changes to the boot file system. In fact, soon +after the boot process is complete, the entire kernel is in memory, +and the boot file system is unmounted to free up the RAM associated +with the initrd disk image. + +The **root file system** includes the typical directory structure we +associate with Unix-like operating systems: ``/dev, /proc, /bin, /etc, +/lib, /usr,`` and ``/tmp`` plus all the configuration files, binaries +and libraries required to run user applications (like bash, ls, and so +forth). + +While there can be important kernal differences between different +Linux distributions, the contents and organization of the root file +system are usually what make your software packages dependent on one +distribution versus another. Docker can help solve this problem by +running multiple distributions at the same time. + +.. image:: images/docker-filesystems-multiroot.png + +Layers and Union Mounts +======================= + +In a traditional Linux boot, the kernel first mounts the root file +system as read-only, checks its integrity, and then switches the whole +rootfs volume to read-write mode. Docker does something similar, +*except* that instead of changing the file system to read-write mode, +it takes advantage of a `union mount +`_ to add a read-write file +system *over* the read-only file system. In fact there may be multiple +read-only file systems stacked on top of each other. + +.. image:: images/docker-filesystems-multilayer.png + +At first, the top layer has nothing in it, but any time a process +creates a file, this happens in the top layer. And if something needs +to update an existing file in a lower layer, then the file gets copied +to the upper layer and changes go into the copy. The version of the +file on the lower layer cannot be seen by the applications anymore, +but it is there, unchanged. + +We call the union of the read-write layer and all the read-only layers +a **union file system**. + +Image +===== + +In Docker terminology, a read-only layer is called an **image**. An +image never changes. Because Docker uses a union file system, the +applications think the whole file system is mounted read-write, +because any file can be changed. But all the changes go to the +top-most layer, and underneath, the image is unchanged. Since they +don't change, images do not have state. + +Each image may depend on one more image which forms the layer beneath +it. We sometimes say that the lower image is the **parent** of the +upper image. + +Base Image +========== + +An image that has no parent is a **base image**. + +Container +========= + +Once you start a process in Docker from an image, Docker fetches the +image and its parent, and repeats the process until it reaches the +base image. Then the union file system adds a read-write layer on +top. That read-write layer, plus the information about its parent and +some additional information like its unique id, is called a +**container**. + +Containers can change, and so they have state. A container may be +running or exited. In either case, the state of the file system and +its exit value is preserved. You can start, stop, and restart a +container. The processes restart from scratch (their memory state is +**not** preserved in a container), but the file system is just as it +was when the container was stopped. + +You can promote a container to an image with ``docker commit``. Once a +container is an image, you can use it as a parent for new containers. diff --git a/docs/sources/terms/images/docker-filesystems-busyboxrw.png b/docs/sources/terms/images/docker-filesystems-busyboxrw.png new file mode 100644 index 0000000000..b99a58242a Binary files /dev/null and b/docs/sources/terms/images/docker-filesystems-busyboxrw.png differ diff --git a/docs/sources/terms/images/docker-filesystems-debian.png b/docs/sources/terms/images/docker-filesystems-debian.png new file mode 100644 index 0000000000..0a6468a472 Binary files /dev/null and b/docs/sources/terms/images/docker-filesystems-debian.png differ diff --git a/docs/sources/terms/images/docker-filesystems-debianrw.png b/docs/sources/terms/images/docker-filesystems-debianrw.png new file mode 100644 index 0000000000..68537f2f51 Binary files /dev/null and b/docs/sources/terms/images/docker-filesystems-debianrw.png differ diff --git a/docs/sources/terms/images/docker-filesystems-generic.png b/docs/sources/terms/images/docker-filesystems-generic.png new file mode 100644 index 0000000000..3866b95609 Binary files /dev/null and b/docs/sources/terms/images/docker-filesystems-generic.png differ diff --git a/docs/sources/terms/images/docker-filesystems-multilayer.png b/docs/sources/terms/images/docker-filesystems-multilayer.png new file mode 100644 index 0000000000..2fdb236551 Binary files /dev/null and b/docs/sources/terms/images/docker-filesystems-multilayer.png differ diff --git a/docs/sources/terms/images/docker-filesystems-multiroot.png b/docs/sources/terms/images/docker-filesystems-multiroot.png new file mode 100644 index 0000000000..d575b3a4c1 Binary files /dev/null and b/docs/sources/terms/images/docker-filesystems-multiroot.png differ diff --git a/docs/sources/terms/images/docker-filesystems.svg b/docs/sources/terms/images/docker-filesystems.svg new file mode 100644 index 0000000000..c0e7b5ba12 --- /dev/null +++ b/docs/sources/terms/images/docker-filesystems.svg @@ -0,0 +1,1345 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + image/svg+xml + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docs/sources/terms/index.rst b/docs/sources/terms/index.rst new file mode 100644 index 0000000000..e46ba5b71a --- /dev/null +++ b/docs/sources/terms/index.rst @@ -0,0 +1,18 @@ +:title: Terms +:description: Definitions of terms used in Docker documentation +:keywords: concepts, documentation, docker, containers + + + +Terms +===== + +Definitions of terms used in Docker documentation. + +Contents: + +.. toctree:: + :maxdepth: 1 + + fundamentals + diff --git a/docs/sources/toctree.rst b/docs/sources/toctree.rst index ae6d5f010c..6226e155f3 100644 --- a/docs/sources/toctree.rst +++ b/docs/sources/toctree.rst @@ -18,5 +18,6 @@ This documentation has the following resources: contributing/index api/index faq + terms/index .. image:: concepts/images/lego_docker.jpg diff --git a/docs/sources/use/builder.rst b/docs/sources/use/builder.rst index 5ceba4b210..0978bd7d4c 100644 --- a/docs/sources/use/builder.rst +++ b/docs/sources/use/builder.rst @@ -121,19 +121,7 @@ functionally equivalent to prefixing the command with `=` .. note:: The environment variables will persist when a container is run from the resulting image. -2.7 INSERT ----------- - - ``INSERT `` - -The `INSERT` instruction will download the file from the given url to the given -path within the image. It is similar to `RUN curl -o `, assuming -curl was installed within the image. - -.. note:: - The path must include the file name. - -2.8 ADD +2.7 ADD ------- ``ADD `` @@ -141,7 +129,7 @@ curl was installed within the image. The `ADD` instruction will copy new files from and add them to the container's filesystem at path ``. `` must be the path to a file or directory relative to the source directory being built (also called the -context of the build). +context of the build) or a remote file URL. `` is the path at which the source will be copied in the destination container. @@ -182,7 +170,6 @@ files and directories are created with mode 0700, uid and gid 0. RUN apt-get update RUN apt-get install -y inotify-tools nginx apache2 openssh-server - INSERT https://raw.github.com/creack/docker-vps/master/nginx-wrapper.sh /usr/sbin/nginx-wrapper .. code-block:: bash diff --git a/docs/theme/docker/layout.html b/docs/theme/docker/layout.html index 707888a927..f4f17f2dba 100755 --- a/docs/theme/docker/layout.html +++ b/docs/theme/docker/layout.html @@ -40,8 +40,11 @@ {%- set script_files = script_files + ['_static/js/docs.js'] %} + {%- if pagename == 'index' %} + + {% else %} - + {% endif %} {%- for cssfile in css_files %} {%- endfor %} diff --git a/docs/theme/docker/redirect_build.html b/docs/theme/docker/redirect_build.html new file mode 100644 index 0000000000..1f26fc3aaa --- /dev/null +++ b/docs/theme/docker/redirect_build.html @@ -0,0 +1,12 @@ + + + + Page Moved + + + + +This page has moved. Perhaps you should visit the Builder page + + + diff --git a/docs/theme/docker/redirect_home.html b/docs/theme/docker/redirect_home.html index 41f8da244d..109239f819 100644 --- a/docs/theme/docker/redirect_home.html +++ b/docs/theme/docker/redirect_home.html @@ -2,7 +2,7 @@ Page Moved - + diff --git a/graph.go b/graph.go index f2b2ccec8e..0bf7eccdbe 100644 --- a/graph.go +++ b/graph.go @@ -189,7 +189,7 @@ func (graph *Graph) Mktemp(id string) (string, error) { return "", fmt.Errorf("Couldn't create temp: %s", err) } if tmp.Exists(id) { - return "", fmt.Errorf("Image %d already exists", id) + return "", fmt.Errorf("Image %s already exists", id) } return tmp.imageRoot(id), nil } diff --git a/hack/README.rst b/hack/README.rst index 4607b6a4a9..c693e1ffa8 100644 --- a/hack/README.rst +++ b/hack/README.rst @@ -3,8 +3,8 @@ This directory contains material helpful for hacking on docker. make hack ========= -Set up an Ubuntu 13.04 virtual machine for developers including kernel 3.8 -and buildbot. The environment is setup in a way that can be used through +Set up an Ubuntu 12.04 virtual machine for developers including kernel 3.8 +go1.1 and buildbot. The environment is setup in a way that can be used through the usual go workflow and/or the root Makefile. You can either edit on your host, or inside the VM (using make ssh-dev) and run and test docker inside the VM. @@ -22,6 +22,7 @@ developers are inconvenienced by the failure. When running 'make hack' at the docker root directory, it spawns a virtual machine in the background running a buildbot instance and adds a git -post-commit hook that automatically run docker tests for you. +post-commit hook that automatically run docker tests for you each time you +commit in your local docker repository. You can check your buildbot instance at http://192.168.33.21:8010/waterfall diff --git a/hack/RELEASE.md b/hack/RELEASE.md new file mode 100644 index 0000000000..dd5a67623c --- /dev/null +++ b/hack/RELEASE.md @@ -0,0 +1,119 @@ +## A maintainer's guide to releasing Docker + +So you're in charge of a docker release? Cool. Here's what to do. + +If your experience deviates from this document, please document the changes to keep it +up-to-date. + + +### 1. Pull from master and create a release branch + + ```bash + $ git checkout master + $ git pull + $ git checkout -b bump_$VERSION + ``` + +### 2. Update CHANGELOG.md + + You can run this command for reference: + + ```bash + LAST_VERSION=$(git tag | grep -E "v[0-9\.]+$" | sort -nr | head -n 1) + git log $LAST_VERSION..HEAD + ``` + + Each change should be formatted as ```BULLET CATEGORY: DESCRIPTION``` + + * BULLET is either ```-```, ```+``` or ```*```, to indicate a bugfix, + new feature or upgrade, respectively. + + * CATEGORY should describe which part of the project is affected. + Valid categories are: + * Runtime + * Remote API + * Builder + * Documentation + * Hack + + * DESCRIPTION: a concise description of the change that is relevant to the end-user, + using the present tense. + Changes should be described in terms of how they affect the user, for example "new feature + X which allows Y", "fixed bug which caused X", "increased performance of Y". + + EXAMPLES: + + ``` + + Builder: 'docker build -t FOO' applies the tag FOO to the newly built container. + * Runtime: improve detection of kernel version + - Remote API: fix a bug in the optional unix socket transport + ``` + +### 3. Change VERSION in commands.go + +### 4. Run all tests + +### 5. Commit and create a pull request + + ```bash + $ git add commands.go CHANGELOG.md + $ git commit -m "Bump version to $VERSION" + $ git push origin bump_$VERSION + ``` + +### 6. Get 2 other maintainers to validate the pull request + +### 7. Merge the pull request and apply tags + + ```bash + $ git checkout master + $ git merge bump_$VERSION + $ git tag -a v$VERSION # Don't forget the v! + $ git tag -f -a latest + $ git push + $ git push --tags + ``` + +### 8. Publish binaries + + To run this you will need access to the release credentials. + Get them from [the infrastructure maintainers](https://github.com/dotcloud/docker/blob/master/hack/infrastructure/MAINTAINERS). + + ```bash + $ RELEASE_IMAGE=image_provided_by_infrastructure_maintainers + $ BUILD=$(docker run -d -e RELEASE_PPA=0 $RELEASE_IMAGE) + ``` + + This will do 2 things: + + * It will build and upload the binaries on http://get.docker.io + * It will *test* the release on our Ubuntu PPA (a PPA is a community repository for ubuntu packages) + + Wait for the build to complete. + + ```bash + $ docker wait $BUILD # This should print 0. If it doesn't, your build failed. + ``` + + Check that the output looks OK. Here's an example of a correct output: + + ```bash + $ docker logs 2>&1 b4e7c8299d73 | grep -e 'Public URL' -e 'Successfully uploaded' + Public URL of the object is: http://get.docker.io.s3.amazonaws.com/builds/Linux/x86_64/docker-v0.4.7.tgz + Public URL of the object is: http://get.docker.io.s3.amazonaws.com/builds/Linux/x86_64/docker-latest.tgz + Successfully uploaded packages. + ``` + + If you don't see 3 lines similar to this, something might be wrong. Check the full logs and try again. + + +### 9. Publish Ubuntu packages + + If everything went well in the previous step, you can finalize the release by submitting the Ubuntu packages. + + ```bash + $ RELEASE_IMAGE=image_provided_by_infrastructure_maintainers + $ docker run -e RELEASE_PPA=1 $RELEASE_IMAGE + ``` + + If that goes well, congratulations! You're done. diff --git a/hack/Vagrantfile b/hack/Vagrantfile index e02dfe06db..0ac07363b8 100644 --- a/hack/Vagrantfile +++ b/hack/Vagrantfile @@ -2,7 +2,7 @@ # vi: set ft=ruby : BOX_NAME = "ubuntu-dev" -BOX_URI = "http://cloud-images.ubuntu.com/raring/current/raring-server-cloudimg-vagrant-amd64-disk1.box" +BOX_URI = "http://files.vagrantup.com/precise64.box" VM_IP = "192.168.33.21" USER = "vagrant" GOPATH = "/data/docker" @@ -21,14 +21,15 @@ Vagrant::Config.run do |config| # Touch for makefile pkg_cmd = "touch #{DOCKER_PATH}; " # Install docker dependencies - pkg_cmd << "export DEBIAN_FRONTEND=noninteractive; apt-get -qq update; " \ - "apt-get install -q -y lxc git aufs-tools golang make linux-image-extra-3.8.0-19-generic; " \ + pkg_cmd << "apt-get update -qq; apt-get install -y python-software-properties; " \ + "add-apt-repository -y ppa:dotcloud/docker-golang/ubuntu; apt-get update -qq; " \ + "apt-get install -y linux-image-generic-lts-raring lxc git aufs-tools golang-stable make; " \ "chown -R #{USER}.#{USER} #{GOPATH}; " \ "install -m 0664 #{CFG_PATH}/bash_profile /home/#{USER}/.bash_profile" config.vm.provision :shell, :inline => pkg_cmd # Deploy buildbot CI pkg_cmd = "apt-get install -q -y python-dev python-pip supervisor; " \ - "pip install -r #{CFG_PATH}/requirements.txt; " \ + "pip install -q -r #{CFG_PATH}/requirements.txt; " \ "chown #{USER}.#{USER} /data; cd /data; " \ "#{CFG_PATH}/setup.sh #{USER} #{GOPATH} #{DOCKER_PATH} #{CFG_PATH} #{BUILDBOT_PATH}" config.vm.provision :shell, :inline => pkg_cmd diff --git a/image.go b/image.go index cd76b8c432..bb6598b262 100644 --- a/image.go +++ b/image.go @@ -92,9 +92,11 @@ func StoreImage(img *Image, layerData Archive, root string, store bool) error { defer file.Close() layerData = file } - - if err := Untar(layerData, layer); err != nil { - return err + // If layerData is not nil, unpack it into the new layer + if layerData != nil { + if err := Untar(layerData, layer); err != nil { + return err + } } return StoreSize(img, root) diff --git a/lxc_template.go b/lxc_template.go index 45408d4bfb..93b795e901 100644 --- a/lxc_template.go +++ b/lxc_template.go @@ -84,8 +84,9 @@ lxc.mount.entry = {{.SysInitPath}} {{$ROOTFS}}/sbin/init none bind,ro 0 0 # In order to get a working DNS environment, mount bind (ro) the host's /etc/resolv.conf into the container lxc.mount.entry = {{.ResolvConfPath}} {{$ROOTFS}}/etc/resolv.conf none bind,ro 0 0 {{if .Volumes}} -{{range $virtualPath, $realPath := .GetVolumes}} -lxc.mount.entry = {{$realPath}} {{$ROOTFS}}/{{$virtualPath}} none bind,rw 0 0 +{{ $rw := .VolumesRW }} +{{range $virtualPath, $realPath := .Volumes}} +lxc.mount.entry = {{$realPath}} {{$ROOTFS}}/{{$virtualPath}} none bind,{{ if index $rw $virtualPath }}rw{{else}}ro{{end}} 0 0 {{end}} {{end}} diff --git a/network.go b/network.go index ea5e5c8586..37037dd14a 100644 --- a/network.go +++ b/network.go @@ -257,7 +257,6 @@ func proxy(listener net.Listener, proto, address string) error { utils.Debugf("Connected to backend, splicing") splice(src, dst) } - panic("Unreachable") } func halfSplice(dst, src net.Conn) error { diff --git a/packaging/ubuntu/Makefile b/packaging/ubuntu/Makefile index f9e034c7b2..7bddf2376f 100644 --- a/packaging/ubuntu/Makefile +++ b/packaging/ubuntu/Makefile @@ -23,7 +23,7 @@ install: mkdir -p ${DESTDIR}/etc/init mkdir -p ${DESTDIR}/DEBIAN install -m 0755 src/${GITHUB_PATH}/docker/docker ${DESTDIR}/usr/bin - install -o root -m 0755 debian/docker.upstart ${DESTDIR}/etc/init/docker.conf + install -o root -m 0644 debian/docker.upstart ${DESTDIR}/etc/init/docker.conf install debian/lxc-docker.prerm ${DESTDIR}/DEBIAN/prerm install debian/lxc-docker.postinst ${DESTDIR}/DEBIAN/postinst diff --git a/registry/registry.go b/registry/registry.go index f03a67ab61..622c09b3f3 100644 --- a/registry/registry.go +++ b/registry/registry.go @@ -18,6 +18,14 @@ import ( var ErrAlreadyExists = errors.New("Image already exists") +func UrlScheme() string { + u, err := url.Parse(auth.IndexServerAddress()) + if err != nil { + return "https" + } + return u.Scheme +} + func doWithCookies(c *http.Client, req *http.Request) (*http.Response, error) { for _, cookie := range c.Jar.Cookies(req.URL) { req.AddCookie(cookie) @@ -56,20 +64,19 @@ func (r *Registry) GetRemoteHistory(imgId, registry string, token []string) ([]s } // Check if an image exists in the Registry -func (r *Registry) LookupRemoteImage(imgId, registry string, authConfig *auth.AuthConfig) bool { +func (r *Registry) LookupRemoteImage(imgId, registry string, token []string) bool { rt := &http.Transport{Proxy: http.ProxyFromEnvironment} req, err := http.NewRequest("GET", registry+"/images/"+imgId+"/json", nil) if err != nil { return false } - req.SetBasicAuth(authConfig.Username, authConfig.Password) res, err := rt.RoundTrip(req) if err != nil { return false } res.Body.Close() - return res.StatusCode == 307 + return res.StatusCode == 200 } func (r *Registry) getImagesInRepository(repository string, authConfig *auth.AuthConfig) ([]map[string]string, error) { @@ -155,7 +162,10 @@ func (r *Registry) GetRemoteTags(registries []string, repository string, token [ repository = "library/" + repository } for _, host := range registries { - endpoint := fmt.Sprintf("https://%s/v1/repositories/%s/tags", host, repository) + endpoint := fmt.Sprintf("%s/v1/repositories/%s/tags", host, repository) + if !(strings.HasPrefix(endpoint, "http://") || strings.HasPrefix(endpoint, "https://")) { + endpoint = fmt.Sprintf("%s://%s", UrlScheme(), endpoint) + } req, err := r.opaqueRequest("GET", endpoint, nil) if err != nil { return nil, err @@ -165,6 +175,7 @@ func (r *Registry) GetRemoteTags(registries []string, repository string, token [ if err != nil { return nil, err } + utils.Debugf("Got status code %d from %s", res.StatusCode, endpoint) defer res.Body.Close() @@ -249,7 +260,7 @@ func (r *Registry) GetRepositoryData(remote string) (*RepositoryData, error) { // Push a local image to the registry func (r *Registry) PushImageJSONRegistry(imgData *ImgData, jsonRaw []byte, registry string, token []string) error { - registry = "https://" + registry + "/v1" + registry = registry + "/v1" // FIXME: try json with UTF8 req, err := http.NewRequest("PUT", registry+"/images/"+imgData.ID+"/json", strings.NewReader(string(jsonRaw))) if err != nil { @@ -285,7 +296,7 @@ func (r *Registry) PushImageJSONRegistry(imgData *ImgData, jsonRaw []byte, regis } func (r *Registry) PushImageLayerRegistry(imgId string, layer io.Reader, registry string, token []string) error { - registry = "https://" + registry + "/v1" + registry = registry + "/v1" req, err := http.NewRequest("PUT", registry+"/images/"+imgId+"/layer", layer) if err != nil { return err @@ -314,7 +325,7 @@ func (r *Registry) opaqueRequest(method, urlStr string, body io.Reader) (*http.R if err != nil { return nil, err } - req.URL.Opaque = strings.Replace(urlStr, req.URL.Scheme + ":", "", 1) + req.URL.Opaque = strings.Replace(urlStr, req.URL.Scheme+":", "", 1) return req, err } @@ -323,7 +334,7 @@ func (r *Registry) opaqueRequest(method, urlStr string, body io.Reader) (*http.R func (r *Registry) PushRegistryTag(remote, revision, tag, registry string, token []string) error { // "jsonify" the string revision = "\"" + revision + "\"" - registry = "https://" + registry + "/v1" + registry = registry + "/v1" req, err := r.opaqueRequest("PUT", registry+"/repositories/"+remote+"/tags/"+tag, strings.NewReader(revision)) if err != nil { diff --git a/runtime.go b/runtime.go index c37e292d22..06b1f8e1b9 100644 --- a/runtime.go +++ b/runtime.go @@ -144,7 +144,9 @@ func (runtime *Runtime) Register(container *Container) error { utils.Debugf("Restarting") container.State.Ghost = false container.State.setStopped(0) - if err := container.Start(); err != nil { + // assume empty host config + hostConfig := &HostConfig{} + if err := container.Start(hostConfig); err != nil { return err } nomonitor = true @@ -246,8 +248,8 @@ func (runtime *Runtime) UpdateCapabilities(quiet bool) { } // FIXME: harmonize with NewGraph() -func NewRuntime(autoRestart bool, dns []string) (*Runtime, error) { - runtime, err := NewRuntimeFromDirectory("/var/lib/docker", autoRestart) +func NewRuntime(flGraphPath string, autoRestart bool, dns []string) (*Runtime, error) { + runtime, err := NewRuntimeFromDirectory(flGraphPath, autoRestart) if err != nil { return nil, err } diff --git a/runtime_test.go b/runtime_test.go index db6367dfaf..d8cc5169a2 100644 --- a/runtime_test.go +++ b/runtime_test.go @@ -8,17 +8,23 @@ import ( "log" "net" "os" - "os/user" "strconv" "strings" "sync" + "syscall" "testing" "time" ) -const unitTestImageName string = "docker-ut" -const unitTestImageId string = "e9aa60c60128cad1" -const unitTestStoreBase string = "/var/lib/docker/unit-tests" +const ( + unitTestImageName = "docker-unit-tests" + unitTestImageId = "e9aa60c60128cad1" + unitTestStoreBase = "/var/lib/docker/unit-tests" + testDaemonAddr = "127.0.0.1:4270" + testDaemonProto = "tcp" +) + +var globalRuntime *Runtime func nuke(runtime *Runtime) error { var wg sync.WaitGroup @@ -33,6 +39,23 @@ func nuke(runtime *Runtime) error { return os.RemoveAll(runtime.root) } +func cleanup(runtime *Runtime) error { + for _, container := range runtime.List() { + container.Kill() + runtime.Destroy(container) + } + images, err := runtime.graph.All() + if err != nil { + return err + } + for _, image := range images { + if image.ID != unitTestImageId { + runtime.graph.Delete(image.ID) + } + } + return nil +} + func layerArchive(tarfile string) (io.Reader, error) { // FIXME: need to close f somewhere f, err := os.Open(tarfile) @@ -49,10 +72,8 @@ func init() { return } - if usr, err := user.Current(); err != nil { - panic(err) - } else if usr.Uid != "0" { - panic("docker tests needs to be run as root") + if uid := syscall.Geteuid(); uid != 0 { + log.Fatal("docker tests needs to be run as root") } NetworkBridgeIface = "testdockbr0" @@ -62,6 +83,7 @@ func init() { if err != nil { panic(err) } + globalRuntime = runtime // Create the "Server" srv := &Server{ @@ -75,6 +97,16 @@ func init() { if err := srv.ImagePull(unitTestImageName, "", "", os.Stdout, utils.NewStreamFormatter(false), nil); err != nil { panic(err) } + + // Spawn a Daemon + go func() { + if err := ListenAndServe(testDaemonProto, testDaemonAddr, srv, os.Getenv("DEBUG") != ""); err != nil { + panic(err) + } + }() + + // Give some time to ListenAndServer to actually start + time.Sleep(time.Second) } // FIXME: test that ImagePull(json=true) send correct json output @@ -103,10 +135,13 @@ func GetTestImage(runtime *Runtime) *Image { imgs, err := runtime.graph.All() if err != nil { panic(err) - } else if len(imgs) < 1 { - panic("GASP") } - return imgs[0] + for i := range imgs { + if imgs[i].ID == unitTestImageId { + return imgs[i] + } + } + panic(fmt.Errorf("Test image %v not found", unitTestImageId)) } func TestRuntimeCreate(t *testing.T) { @@ -295,7 +330,8 @@ func findAvailalblePort(runtime *Runtime, port int) (*Container, error) { if err != nil { return nil, err } - if err := container.Start(); err != nil { + hostConfig := &HostConfig{} + if err := container.Start(hostConfig); err != nil { if strings.Contains(err.Error(), "address already in use") { return nil, nil } @@ -405,7 +441,8 @@ func TestRestore(t *testing.T) { defer runtime1.Destroy(container2) // Start the container non blocking - if err := container2.Start(); err != nil { + hostConfig := &HostConfig{} + if err := container2.Start(hostConfig); err != nil { t.Fatal(err) } diff --git a/server.go b/server.go index ad4f092a7f..78915585af 100644 --- a/server.go +++ b/server.go @@ -87,7 +87,7 @@ func (srv *Server) ImageInsert(name, url, path string, out io.Writer, sf *utils. } defer file.Body.Close() - config, _, err := ParseRun([]string{img.ID, "echo", "insert", url, path}, srv.runtime.capabilities) + config, _, _, err := ParseRun([]string{img.ID, "echo", "insert", url, path}, srv.runtime.capabilities) if err != nil { return "", err } @@ -254,7 +254,7 @@ func (srv *Server) ContainerChanges(name string) ([]Change, error) { return nil, fmt.Errorf("No such container: %s", name) } -func (srv *Server) Containers(all bool, n int, since, before string) []APIContainers { +func (srv *Server) Containers(all, size bool, n int, since, before string) []APIContainers { var foundBefore bool var displayed int retContainers := []APIContainers{} @@ -288,8 +288,9 @@ func (srv *Server) Containers(all bool, n int, since, before string) []APIContai c.Created = container.Created.Unix() c.Status = container.State.String() c.Ports = container.NetworkSettings.PortMappingHuman() - c.SizeRw, c.SizeRootFs = container.GetSize() - + if size { + c.SizeRw, c.SizeRootFs = container.GetSize() + } retContainers = append(retContainers, c) } return retContainers @@ -350,26 +351,49 @@ func (srv *Server) pullImage(r *registry.Registry, out io.Writer, imgId, endpoin return nil } -func (srv *Server) pullRepository(r *registry.Registry, out io.Writer, local, remote, askedTag string, sf *utils.StreamFormatter) error { +func (srv *Server) pullRepository(r *registry.Registry, out io.Writer, local, remote, askedTag, registryEp string, sf *utils.StreamFormatter) error { out.Write(sf.FormatStatus("Pulling repository %s from %s", local, auth.IndexServerAddress())) - repoData, err := r.GetRepositoryData(remote) - if err != nil { - return err - } - utils.Debugf("Updating checksums") - // Reload the json file to make sure not to overwrite faster sums - if err := srv.runtime.graph.UpdateChecksums(repoData.ImgList); err != nil { - return err + var repoData *registry.RepositoryData + var err error + if registryEp == "" { + repoData, err = r.GetRepositoryData(remote) + if err != nil { + return err + } + + utils.Debugf("Updating checksums") + // Reload the json file to make sure not to overwrite faster sums + if err := srv.runtime.graph.UpdateChecksums(repoData.ImgList); err != nil { + return err + } + } else { + repoData = ®istry.RepositoryData{ + Tokens: []string{}, + ImgList: make(map[string]*registry.ImgData), + Endpoints: []string{registryEp}, + } } utils.Debugf("Retrieving the tag list") tagsList, err := r.GetRemoteTags(repoData.Endpoints, remote, repoData.Tokens) if err != nil { + utils.Debugf("%v", err) return err } + + if registryEp != "" { + for tag, id := range tagsList { + repoData.ImgList[id] = ®istry.ImgData{ + ID: id, + Tag: tag, + Checksum: "", + } + } + } + utils.Debugf("Registering tags") - // If not specific tag have been asked, take all + // If no tag has been specified, pull them all if askedTag == "" { for tag, id := range tagsList { repoData.ImgList[id].Tag = tag @@ -391,7 +415,10 @@ func (srv *Server) pullRepository(r *registry.Registry, out io.Writer, local, re out.Write(sf.FormatStatus("Pulling image %s (%s) from %s", img.ID, img.Tag, remote)) success := false for _, ep := range repoData.Endpoints { - if err := srv.pullImage(r, out, img.ID, "https://"+ep+"/v1", repoData.Tokens, sf); err != nil { + if !(strings.HasPrefix(ep, "http://") || strings.HasPrefix(ep, "https://")) { + ep = fmt.Sprintf("%s://%s", registry.UrlScheme(), ep) + } + if err := srv.pullImage(r, out, img.ID, ep+"/v1", repoData.Tokens, sf); err != nil { out.Write(sf.FormatStatus("Error while retrieving image for tag: %s (%s); checking next endpoint", askedTag, err)) continue } @@ -451,7 +478,6 @@ func (srv *Server) poolRemove(kind, key string) error { } return nil } - func (srv *Server) ImagePull(name, tag, endpoint string, out io.Writer, sf *utils.StreamFormatter, authConfig *auth.AuthConfig) error { r, err := registry.NewRegistry(srv.runtime.root, authConfig) if err != nil { @@ -462,21 +488,20 @@ func (srv *Server) ImagePull(name, tag, endpoint string, out io.Writer, sf *util } defer srv.poolRemove("pull", name+":"+tag) - out = utils.NewWriteFlusher(out) - if endpoint != "" { - if err := srv.pullImage(r, out, name, endpoint, nil, sf); err != nil { - return err - } - return nil - } remote := name parts := strings.Split(name, "/") if len(parts) > 2 { remote = fmt.Sprintf("src/%s", url.QueryEscape(strings.Join(parts, "/"))) } - if err := srv.pullRepository(r, out, name, remote, tag, sf); err != nil { - return err + out = utils.NewWriteFlusher(out) + err = srv.pullRepository(r, out, name, remote, tag, endpoint, sf) + if err != nil && endpoint != "" { + if err := srv.pullImage(r, out, name, endpoint, nil, sf); err != nil { + return err + } + return nil } + return nil } @@ -546,7 +571,7 @@ func (srv *Server) getImageList(localRepo map[string]string) ([]*registry.ImgDat return imgList, nil } -func (srv *Server) pushRepository(r *registry.Registry, out io.Writer, name string, localRepo map[string]string, sf *utils.StreamFormatter) error { +func (srv *Server) pushRepository(r *registry.Registry, out io.Writer, name, registryEp string, localRepo map[string]string, sf *utils.StreamFormatter) error { out = utils.NewWriteFlusher(out) out.Write(sf.FormatStatus("Processing checksums")) imgList, err := srv.getImageList(localRepo) @@ -554,25 +579,51 @@ func (srv *Server) pushRepository(r *registry.Registry, out io.Writer, name stri return err } out.Write(sf.FormatStatus("Sending image list")) - srvName := name parts := strings.Split(name, "/") if len(parts) > 2 { srvName = fmt.Sprintf("src/%s", url.QueryEscape(strings.Join(parts, "/"))) } - repoData, err := r.PushImageJSONIndex(srvName, imgList, false, nil) - if err != nil { - return err + var repoData *registry.RepositoryData + if registryEp == "" { + repoData, err = r.PushImageJSONIndex(name, imgList, false, nil) + if err != nil { + return err + } + } else { + repoData = ®istry.RepositoryData{ + ImgList: make(map[string]*registry.ImgData), + Tokens: []string{}, + Endpoints: []string{registryEp}, + } + tagsList, err := r.GetRemoteTags(repoData.Endpoints, name, repoData.Tokens) + if err != nil && err.Error() != "Repository not found" { + return err + } else if err == nil { + for tag, id := range tagsList { + repoData.ImgList[id] = ®istry.ImgData{ + ID: id, + Tag: tag, + Checksum: "", + } + } + } } for _, ep := range repoData.Endpoints { + if !(strings.HasPrefix(ep, "http://") || strings.HasPrefix(ep, "https://")) { + ep = fmt.Sprintf("%s://%s", registry.UrlScheme(), ep) + } out.Write(sf.FormatStatus("Pushing repository %s to %s (%d tags)", name, ep, len(localRepo))) // For each image within the repo, push them for _, elem := range imgList { if _, exists := repoData.ImgList[elem.ID]; exists { out.Write(sf.FormatStatus("Image %s already on registry, skipping", name)) continue + } else if registryEp != "" && r.LookupRemoteImage(elem.ID, registryEp, repoData.Tokens) { + fmt.Fprintf(out, "Image %s already on registry, skipping\n", name) + continue } if err := srv.pushImage(r, out, name, elem.ID, ep, repoData.Tokens, sf); err != nil { // FIXME: Continue on error? @@ -585,9 +636,12 @@ func (srv *Server) pushRepository(r *registry.Registry, out io.Writer, name stri } } - if _, err := r.PushImageJSONIndex(srvName, imgList, true, repoData.Endpoints); err != nil { - return err + if registryEp == "" { + if _, err := r.PushImageJSONIndex(name, imgList, true, repoData.Endpoints); err != nil { + return err + } } + return nil } @@ -664,11 +718,12 @@ func (srv *Server) ImagePush(name, endpoint string, out io.Writer, sf *utils.Str if err2 != nil { return err2 } + if err != nil { out.Write(sf.FormatStatus("The push refers to a repository [%s] (len: %d)", name, len(srv.runtime.repositories.Repositories[name]))) // If it fails, try to get the repository if localRepo, exists := srv.runtime.repositories.Repositories[name]; exists { - if err := srv.pushRepository(r, out, name, localRepo, sf); err != nil { + if err := srv.pushRepository(r, out, name, endpoint, localRepo, sf); err != nil { return err } return nil @@ -857,7 +912,7 @@ func (srv *Server) deleteImageParents(img *Image, imgs *[]APIRmi) error { return nil } -func (srv *Server) deleteImage(img *Image, repoName, tag string) (*[]APIRmi, error) { +func (srv *Server) deleteImage(img *Image, repoName, tag string) ([]APIRmi, error) { //Untag the current image var imgs []APIRmi tagDeleted, err := srv.runtime.repositories.Delete(repoName, tag) @@ -870,18 +925,18 @@ func (srv *Server) deleteImage(img *Image, repoName, tag string) (*[]APIRmi, err if len(srv.runtime.repositories.ByID()[img.ID]) == 0 { if err := srv.deleteImageAndChildren(img.ID, &imgs); err != nil { if err != ErrImageReferenced { - return &imgs, err + return imgs, err } } else if err := srv.deleteImageParents(img, &imgs); err != nil { if err != ErrImageReferenced { - return &imgs, err + return imgs, err } } } - return &imgs, nil + return imgs, nil } -func (srv *Server) ImageDelete(name string, autoPrune bool) (*[]APIRmi, error) { +func (srv *Server) ImageDelete(name string, autoPrune bool) ([]APIRmi, error) { img, err := srv.runtime.repositories.LookupImage(name) if err != nil { return nil, fmt.Errorf("No such image: %s", name) @@ -933,9 +988,9 @@ func (srv *Server) ImageGetCached(imgId string, config *Config) (*Image, error) return nil, nil } -func (srv *Server) ContainerStart(name string) error { +func (srv *Server) ContainerStart(name string, hostConfig *HostConfig) error { if container := srv.runtime.Get(name); container != nil { - if err := container.Start(); err != nil { + if err := container.Start(hostConfig); err != nil { return fmt.Errorf("Error starting container %s: %s", name, err.Error()) } } else { @@ -1048,11 +1103,11 @@ func (srv *Server) ImageInspect(name string) (*Image, error) { return nil, fmt.Errorf("No such image: %s", name) } -func NewServer(autoRestart, enableCors bool, dns ListOpts) (*Server, error) { +func NewServer(flGraphPath string, autoRestart, enableCors bool, dns ListOpts) (*Server, error) { if runtime.GOARCH != "amd64" { log.Fatalf("The docker runtime currently only supports amd64 (not %s). This will change in the future. Aborting.", runtime.GOARCH) } - runtime, err := NewRuntime(autoRestart, dns) + runtime, err := NewRuntime(flGraphPath, autoRestart, dns) if err != nil { return nil, err } diff --git a/server_test.go b/server_test.go index 7fdec18f61..8d1ea8be94 100644 --- a/server_test.go +++ b/server_test.go @@ -65,7 +65,7 @@ func TestCreateRm(t *testing.T) { srv := &Server{runtime: runtime} - config, _, err := ParseRun([]string{GetTestImage(runtime).ID, "echo test"}, nil) + config, _, _, err := ParseRun([]string{GetTestImage(runtime).ID, "echo test"}, nil) if err != nil { t.Fatal(err) } @@ -98,7 +98,7 @@ func TestCreateStartRestartStopStartKillRm(t *testing.T) { srv := &Server{runtime: runtime} - config, _, err := ParseRun([]string{GetTestImage(runtime).ID, "/bin/cat"}, nil) + config, hostConfig, _, err := ParseRun([]string{GetTestImage(runtime).ID, "/bin/cat"}, nil) if err != nil { t.Fatal(err) } @@ -112,7 +112,7 @@ func TestCreateStartRestartStopStartKillRm(t *testing.T) { t.Errorf("Expected 1 container, %v found", len(runtime.List())) } - err = srv.ContainerStart(id) + err = srv.ContainerStart(id, hostConfig) if err != nil { t.Fatal(err) } @@ -127,7 +127,7 @@ func TestCreateStartRestartStopStartKillRm(t *testing.T) { t.Fatal(err) } - err = srv.ContainerStart(id) + err = srv.ContainerStart(id, hostConfig) if err != nil { t.Fatal(err) } diff --git a/term/term.go b/term/term.go index 0cc91ea1b6..3f743d227f 100644 --- a/term/term.go +++ b/term/term.go @@ -38,13 +38,13 @@ func IsTerminal(fd uintptr) bool { // Restore restores the terminal connected to the given file descriptor to a // previous state. -func Restore(fd uintptr, state *State) error { +func RestoreTerminal(fd uintptr, state *State) error { _, _, err := syscall.Syscall(syscall.SYS_IOCTL, fd, uintptr(setTermios), uintptr(unsafe.Pointer(&state.termios))) return err } -func SetRawTerminal() (*State, error) { - oldState, err := MakeRaw(os.Stdin.Fd()) +func SetRawTerminal(fd uintptr) (*State, error) { + oldState, err := MakeRaw(fd) if err != nil { return nil, err } @@ -52,12 +52,8 @@ func SetRawTerminal() (*State, error) { signal.Notify(c, os.Interrupt) go func() { _ = <-c - Restore(os.Stdin.Fd(), oldState) + RestoreTerminal(fd, oldState) os.Exit(0) }() return oldState, err } - -func RestoreTerminal(state *State) { - Restore(os.Stdin.Fd(), state) -} diff --git a/testing/Vagrantfile b/testing/Vagrantfile index f2f6ca8248..e3b25a6f9d 100644 --- a/testing/Vagrantfile +++ b/testing/Vagrantfile @@ -19,9 +19,10 @@ Vagrant::Config.run do |config| config.vm.share_folder "v-data", DOCKER_PATH, "#{File.dirname(__FILE__)}/.." config.vm.network :hostonly, BUILDBOT_IP + # Deploy buildbot and its dependencies if it was not done if Dir.glob("#{File.dirname(__FILE__)}/.vagrant/machines/default/*/id").empty? - pkg_cmd = "apt-get update -qq; apt-get install -q -y linux-image-3.8.0-19-generic; " + pkg_cmd = "apt-get update -qq; apt-get install -q -y linux-image-generic-lts-raring; " # Deploy buildbot CI pkg_cmd << "apt-get install -q -y python-dev python-pip supervisor; " \ "pip install -r #{CFG_PATH}/requirements.txt; " \ @@ -29,7 +30,7 @@ Vagrant::Config.run do |config| "#{CFG_PATH}/setup.sh #{USER} #{CFG_PATH}; " # Install docker dependencies pkg_cmd << "apt-get install -q -y python-software-properties; " \ - "add-apt-repository -y ppa:gophers/go/ubuntu; apt-get update -qq; " \ + "add-apt-repository -y ppa:dotcloud/docker-golang/ubuntu; apt-get update -qq; " \ "DEBIAN_FRONTEND=noninteractive apt-get install -q -y lxc git golang-stable aufs-tools make; " # Activate new kernel pkg_cmd << "shutdown -r +1; " @@ -40,6 +41,7 @@ end # Providers were added on Vagrant >= 1.1.0 Vagrant::VERSION >= "1.1.0" and Vagrant.configure("2") do |config| config.vm.provider :aws do |aws, override| + aws.tags = { 'Name' => 'docker-ci' } aws.access_key_id = ENV["AWS_ACCESS_KEY_ID"] aws.secret_access_key = ENV["AWS_SECRET_ACCESS_KEY"] aws.keypair_name = ENV["AWS_KEYPAIR_NAME"] diff --git a/utils/utils.go b/utils/utils.go index 3cd2f4e7ff..2f2a52867e 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -78,16 +78,16 @@ func (r *progressReader) Read(p []byte) (n int, err error) { read, err := io.ReadCloser(r.reader).Read(p) r.readProgress += read - updateEvery := 4096 + updateEvery := 1024 * 512 //512kB if r.readTotal > 0 { - // Only update progress for every 1% read - if increment := int(0.01 * float64(r.readTotal)); increment > updateEvery { + // Update progress for every 1% read if 1% < 512kB + if increment := int(0.01 * float64(r.readTotal)); increment < updateEvery { updateEvery = increment } } if r.readProgress-r.lastUpdate > updateEvery || err != nil { if r.readTotal > 0 { - fmt.Fprintf(r.output, r.template, HumanSize(int64(r.readProgress)), HumanSize(int64(r.readTotal)), fmt.Sprintf("%.0f%%", float64(r.readProgress)/float64(r.readTotal)*100)) + fmt.Fprintf(r.output, r.template, HumanSize(int64(r.readProgress)), HumanSize(int64(r.readTotal)), fmt.Sprintf("%2.0f%%", float64(r.readProgress)/float64(r.readTotal)*100)) } else { fmt.Fprintf(r.output, r.template, r.readProgress, "?", "n/a") } @@ -133,7 +133,7 @@ func HumanDuration(d time.Duration) string { } else if hours < 24*365*2 { return fmt.Sprintf("%d months", hours/24/30) } - return fmt.Sprintf("%d years", d.Hours()/24/365) + return fmt.Sprintf("%f years", d.Hours()/24/365) } // HumanSize returns a human-readable approximation of a size @@ -147,7 +147,7 @@ func HumanSize(size int64) string { sizef = sizef / 1000.0 i++ } - return fmt.Sprintf("%.4g %s", sizef, units[i]) + return fmt.Sprintf("%5.4g %s", sizef, units[i]) } func Trunc(s string, maxlen int) string { @@ -236,7 +236,6 @@ func (r *bufReader) Read(p []byte) (n int, err error) { } r.wait.Wait() } - panic("unreachable") } func (r *bufReader) Close() error { @@ -529,7 +528,9 @@ func GetKernelVersion() (*KernelVersionInfo, error) { } if len(tmp2) > 2 { - minor, err = strconv.Atoi(tmp2[2]) + // Removes "+" because git kernels might set it + minorUnparsed := strings.Trim(tmp2[2], "+") + minor, err = strconv.Atoi(minorUnparsed) if err != nil { return nil, err } @@ -637,6 +638,14 @@ func (sf *StreamFormatter) Used() bool { return sf.used } +func IsURL(str string) bool { + return strings.HasPrefix(str, "http://") || strings.HasPrefix(str, "https://") +} + +func IsGIT(str string) bool { + return strings.HasPrefix(str, "git://") || strings.HasPrefix(str, "github.com/") +} + func CheckLocalDns() bool { resolv, err := ioutil.ReadFile("/etc/resolv.conf") if err != nil { @@ -678,5 +687,3 @@ func ParseHost(host string, port int, addr string) string { } return fmt.Sprintf("tcp://%s:%d", host, port) } - - diff --git a/utils/utils_test.go b/utils/utils_test.go index 623f08e383..1a7639d8d2 100644 --- a/utils/utils_test.go +++ b/utils/utils_test.go @@ -265,8 +265,8 @@ func TestCompareKernelVersion(t *testing.T) { func TestHumanSize(t *testing.T) { size1000 := HumanSize(1000) - if size1000 != "1 kB" { - t.Errorf("1000 -> expected 1 kB, got %s", size1000) + if size1000 != " 1 kB" { + t.Errorf("1000 -> expected 1 kB, got %s", size1000) } size1024 := HumanSize(1024) diff --git a/utils_test.go b/utils_test.go new file mode 100644 index 0000000000..5958aa70fe --- /dev/null +++ b/utils_test.go @@ -0,0 +1,103 @@ +package docker + +import ( + "io" + "io/ioutil" + "os" + "path" + "strings" + "testing" +) + +// This file contains utility functions for docker's unit test suite. +// It has to be named XXX_test.go, apparently, in other to access private functions +// from other XXX_test.go functions. + +// Create a temporary runtime suitable for unit testing. +// Call t.Fatal() at the first error. +func mkRuntime(t *testing.T) *Runtime { + runtime, err := newTestRuntime() + if err != nil { + t.Fatal(err) + } + return runtime +} + +// Write `content` to the file at path `dst`, creating it if necessary, +// as well as any missing directories. +// The file is truncated if it already exists. +// Call t.Fatal() at the first error. +func writeFile(dst, content string, t *testing.T) { + // Create subdirectories if necessary + if err := os.MkdirAll(path.Dir(dst), 0700); err != nil && !os.IsExist(err) { + t.Fatal(err) + } + f, err := os.OpenFile(dst, os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0700) + if err != nil { + t.Fatal(err) + } + // Write content (truncate if it exists) + if _, err := io.Copy(f, strings.NewReader(content)); err != nil { + t.Fatal(err) + } +} + +// Return the contents of file at path `src`. +// Call t.Fatal() at the first error (including if the file doesn't exist) +func readFile(src string, t *testing.T) (content string) { + f, err := os.Open(src) + if err != nil { + t.Fatal(err) + } + data, err := ioutil.ReadAll(f) + if err != nil { + t.Fatal(err) + } + return string(data) +} + +// Create a test container from the given runtime `r` and run arguments `args`. +// The image name (eg. the XXX in []string{"-i", "-t", "XXX", "bash"}, is dynamically replaced by the current test image. +// The caller is responsible for destroying the container. +// Call t.Fatal() at the first error. +func mkContainer(r *Runtime, args []string, t *testing.T) (*Container, *HostConfig) { + config, hostConfig, _, err := ParseRun(args, nil) + if err != nil { + t.Fatal(err) + } + config.Image = GetTestImage(r).ID + c, err := NewBuilder(r).Create(config) + if err != nil { + t.Fatal(err) + } + return c, hostConfig +} + +// Create a test container, start it, wait for it to complete, destroy it, +// and return its standard output as a string. +// The image name (eg. the XXX in []string{"-i", "-t", "XXX", "bash"}, is dynamically replaced by the current test image. +// If t is not nil, call t.Fatal() at the first error. Otherwise return errors normally. +func runContainer(r *Runtime, args []string, t *testing.T) (output string, err error) { + defer func() { + if err != nil && t != nil { + t.Fatal(err) + } + }() + container, hostConfig := mkContainer(r, args, t) + defer r.Destroy(container) + stdout, err := container.StdoutPipe() + if err != nil { + return "", err + } + defer stdout.Close() + if err := container.Start(hostConfig); err != nil { + return "", err + } + container.Wait() + data, err := ioutil.ReadAll(stdout) + if err != nil { + return "", err + } + output = string(data) + return +} diff --git a/z_final_test.go b/z_final_test.go new file mode 100644 index 0000000000..78a7acf6e7 --- /dev/null +++ b/z_final_test.go @@ -0,0 +1,12 @@ +package docker + +import ( + "github.com/dotcloud/docker/utils" + "runtime" + "testing" +) + +func TestFinal(t *testing.T) { + cleanup(globalRuntime) + t.Logf("Fds: %d, Goroutines: %d", utils.GetTotalUsedFds(), runtime.NumGoroutine()) +}