Merge pull request #578 from dotcloud/registry-update-tests

+ Registry: Allow INDEX override

api.go

@@ -4,8 +4,8 @@ import (
 	"encoding/json"
 	"fmt"
 	"github.com/dotcloud/docker/auth"
+	"github.com/dotcloud/docker/utils"
 	"github.com/gorilla/mux"
-	"github.com/shin-/cookiejar"
 	"io"
 	"log"
 	"net/http"
@@ -45,11 +45,7 @@ func writeJson(w http.ResponseWriter, b []byte) {
 }
 
 func getAuth(srv *Server, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
-	config := &auth.AuthConfig{
-		Username: srv.runtime.authConfig.Username,
-		Email:    srv.runtime.authConfig.Email,
-	}
-	b, err := json.Marshal(config)
+	b, err := json.Marshal(srv.registry.GetAuthConfig())
 	if err != nil {
 		return err
 	}
@@ -63,18 +59,17 @@ func postAuth(srv *Server, w http.ResponseWriter, r *http.Request, vars map[stri
 		return err
 	}
 
-	if config.Username == srv.runtime.authConfig.Username {
-		config.Password = srv.runtime.authConfig.Password
+	if config.Username == srv.registry.GetAuthConfig().Username {
+		config.Password = srv.registry.GetAuthConfig().Password
 	}
 
 	newAuthConfig := auth.NewAuthConfig(config.Username, config.Password, config.Email, srv.runtime.root)
 	status, err := auth.Login(newAuthConfig)
 	if err != nil {
 		return err
-	} else {
-		srv.runtime.graph.getHttpClient().Jar = cookiejar.NewCookieJar()
-		srv.runtime.authConfig = newAuthConfig
 	}
+	srv.registry.ResetClient(newAuthConfig)
+
 	if status != "" {
 		b, err := json.Marshal(&ApiAuth{Status: status})
 		if err != nil {
@@ -116,7 +111,7 @@ func getContainersExport(srv *Server, w http.ResponseWriter, r *http.Request, va
 	name := vars["name"]
 
 	if err := srv.ContainerExport(name, w); err != nil {
-		Debugf("%s", err.Error())
+		utils.Debugf("%s", err.Error())
 		return err
 	}
 	return nil
@@ -239,7 +234,7 @@ func postCommit(srv *Server, w http.ResponseWriter, r *http.Request, vars map[st
 	}
 	config := &Config{}
 	if err := json.NewDecoder(r.Body).Decode(config); err != nil {
-		Debugf("%s", err.Error())
+		utils.Debugf("%s", err.Error())
 	}
 	repo := r.Form.Get("repo")
 	tag := r.Form.Get("tag")
@@ -335,7 +330,6 @@ func postImagesPush(srv *Server, w http.ResponseWriter, r *http.Request, vars ma
 	if err := parseForm(r); err != nil {
 		return err
 	}
-
 	registry := r.Form.Get("registry")
 
 	if vars == nil {
@@ -602,20 +596,20 @@ func ListenAndServe(addr string, srv *Server, logging bool) error {
 
 	for method, routes := range m {
 		for route, fct := range routes {
-			Debugf("Registering %s, %s", method, route)
+			utils.Debugf("Registering %s, %s", method, route)
 			// NOTE: scope issue, make sure the variables are local and won't be changed
 			localRoute := route
 			localMethod := method
 			localFct := fct
 			r.Path(localRoute).Methods(localMethod).HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-				Debugf("Calling %s %s", localMethod, localRoute)
+				utils.Debugf("Calling %s %s", localMethod, localRoute)
 				if logging {
 					log.Println(r.Method, r.RequestURI)
 				}
 				if strings.Contains(r.Header.Get("User-Agent"), "Docker-Client/") {
 					userAgent := strings.Split(r.Header.Get("User-Agent"), "/")
 					if len(userAgent) == 2 && userAgent[1] != VERSION {
-						Debugf("Warning: client and server don't have the same version (client: %s, server: %s)", userAgent[1], VERSION)
+						utils.Debugf("Warning: client and server don't have the same version (client: %s, server: %s)", userAgent[1], VERSION)
 					}
 				}
 				if err := localFct(srv, w, r, mux.Vars(r)); err != nil {

api_test.go

@@ -6,6 +6,8 @@ import (
 	"bytes"
 	"encoding/json"
 	"github.com/dotcloud/docker/auth"
+	"github.com/dotcloud/docker/registry"
+	"github.com/dotcloud/docker/utils"
 	"io"
 	"net"
 	"net/http"
@@ -23,7 +25,10 @@ func TestGetAuth(t *testing.T) {
 	}
 	defer nuke(runtime)
 
-	srv := &Server{runtime: runtime}
+	srv := &Server{
+		runtime:  runtime,
+		registry: registry.NewRegistry(runtime.root),
+	}
 
 	r := httptest.NewRecorder()
 
@@ -46,13 +51,14 @@ func TestGetAuth(t *testing.T) {
 	if err := postAuth(srv, r, req, nil); err != nil {
 		t.Fatal(err)
 	}
+
 	if r.Code != http.StatusOK && r.Code != 0 {
 		t.Fatalf("%d OK or 0 expected, received %d\n", http.StatusOK, r.Code)
 	}
 
-	if runtime.authConfig.Username != authConfig.Username ||
-		runtime.authConfig.Password != authConfig.Password ||
-		runtime.authConfig.Email != authConfig.Email {
+	newAuthConfig := srv.registry.GetAuthConfig()
+	if newAuthConfig.Username != authConfig.Username ||
+		newAuthConfig.Email != authConfig.Email {
 		t.Fatalf("The auth configuration hasn't been set correctly")
 	}
 }
@@ -222,7 +228,10 @@ func TestGetImagesSearch(t *testing.T) {
 	}
 	defer nuke(runtime)
 
-	srv := &Server{runtime: runtime}
+	srv := &Server{
+		runtime:  runtime,
+		registry: registry.NewRegistry(runtime.root),
+	}
 
 	r := httptest.NewRecorder()
 
@@ -476,13 +485,16 @@ func TestPostAuth(t *testing.T) {
 	}
 	defer nuke(runtime)
 
-	srv := &Server{runtime: runtime}
+	srv := &Server{
+		runtime:  runtime,
+		registry: registry.NewRegistry(runtime.root),
+	}
 
 	authConfigOrig := &auth.AuthConfig{
 		Username: "utest",
 		Email:    "utest@yopmail.com",
 	}
-	runtime.authConfig = authConfigOrig
+	srv.registry.ResetClient(authConfigOrig)
 
 	r := httptest.NewRecorder()
 	if err := getAuth(srv, r, nil, nil); err != nil {
@@ -811,7 +823,7 @@ func TestPostContainersCreate(t *testing.T) {
 
 	if _, err := os.Stat(path.Join(container.rwPath(), "test")); err != nil {
 		if os.IsNotExist(err) {
-			Debugf("Err: %s", err)
+			utils.Debugf("Err: %s", err)
 			t.Fatalf("The test file has not been created")
 		}
 		t.Fatal(err)

auth/auth.go

@@ -15,7 +15,7 @@ import (
 const CONFIGFILE = ".dockercfg"
 
 // the registry server we want to login against
-const INDEX_SERVER = "https://index.docker.io"
+const INDEX_SERVER = "https://index.docker.io/v1"
 
 type AuthConfig struct {
 	Username string `json:"username"`
@@ -33,6 +33,13 @@ func NewAuthConfig(username, password, email, rootPath string) *AuthConfig {
 	}
 }
 
+func IndexServerAddress() string {
+	if os.Getenv("DOCKER_INDEX_URL") != "" {
+		return os.Getenv("DOCKER_INDEX_URL") + "/v1"
+	}
+	return INDEX_SERVER
+}
+
 // create a base64 encoded auth string to store in config
 func EncodeAuth(authConfig *AuthConfig) string {
 	authStr := authConfig.Username + ":" + authConfig.Password
@@ -119,7 +126,7 @@ func Login(authConfig *AuthConfig) (string, error) {
 
 	// using `bytes.NewReader(jsonBody)` here causes the server to respond with a 411 status.
 	b := strings.NewReader(string(jsonBody))
-	req1, err := http.Post(INDEX_SERVER+"/v1/users/", "application/json; charset=utf-8", b)
+	req1, err := http.Post(IndexServerAddress()+"/users/", "application/json; charset=utf-8", b)
 	if err != nil {
 		return "", fmt.Errorf("Server Error: %s", err)
 	}
@@ -139,7 +146,7 @@ func Login(authConfig *AuthConfig) (string, error) {
 			"Please check your e-mail for a confirmation link.")
 	} else if reqStatusCode == 400 {
 		if string(reqBody) == "\"Username or email already exists\"" {
-			req, err := http.NewRequest("GET", INDEX_SERVER+"/v1/users/", nil)
+			req, err := http.NewRequest("GET", IndexServerAddress()+"/users/", nil)
 			req.SetBasicAuth(authConfig.Username, authConfig.Password)
 			resp, err := client.Do(req)
 			if err != nil {

auth/auth_test.go

@@ -1,6 +1,10 @@
 package auth
 
 import (
+	"crypto/rand"
+	"encoding/hex"
+	"os"
+	"strings"
 	"testing"
 )
 
@@ -21,3 +25,49 @@ func TestEncodeAuth(t *testing.T) {
 		t.Fatal("AuthString encoding isn't correct.")
 	}
 }
+
+func TestLogin(t *testing.T) {
+	os.Setenv("DOCKER_INDEX_URL", "https://indexstaging-docker.dotcloud.com")
+	defer os.Setenv("DOCKER_INDEX_URL", "")
+	authConfig := NewAuthConfig("unittester", "surlautrerivejetattendrai", "noise+unittester@dotcloud.com", "/tmp")
+	status, err := Login(authConfig)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if status != "Login Succeeded\n" {
+		t.Fatalf("Expected status \"Login Succeeded\", found \"%s\" instead", status)
+	}
+}
+
+func TestCreateAccount(t *testing.T) {
+	os.Setenv("DOCKER_INDEX_URL", "https://indexstaging-docker.dotcloud.com")
+	defer os.Setenv("DOCKER_INDEX_URL", "")
+	tokenBuffer := make([]byte, 16)
+	_, err := rand.Read(tokenBuffer)
+	if err != nil {
+		t.Fatal(err)
+	}
+	token := hex.EncodeToString(tokenBuffer)[:12]
+	username := "ut" + token
+	authConfig := NewAuthConfig(username, "test42", "docker-ut+"+token+"@example.com", "/tmp")
+	status, err := Login(authConfig)
+	if err != nil {
+		t.Fatal(err)
+	}
+	expectedStatus := "Account created. Please use the confirmation link we sent" +
+		" to your e-mail to activate it.\n"
+	if status != expectedStatus {
+		t.Fatalf("Expected status: \"%s\", found \"%s\" instead.", expectedStatus, status)
+	}
+
+	status, err = Login(authConfig)
+	if err == nil {
+		t.Fatalf("Expected error but found nil instead")
+	}
+
+	expectedError := "Login: Account is not Active"
+
+	if !strings.Contains(err.Error(), expectedError) {
+		t.Fatalf("Expected message \"%s\" but found \"%s\" instead", expectedError, err.Error())
+	}
+}

builder.go

@@ -4,6 +4,7 @@ import (
 	"bufio"
 	"encoding/json"
 	"fmt"
+	"github.com/dotcloud/docker/utils"
 	"io"
 	"os"
 	"path"
@@ -161,11 +162,11 @@ func (builder *Builder) clearTmp(containers, images map[string]struct{}) {
 	for c := range containers {
 		tmp := builder.runtime.Get(c)
 		builder.runtime.Destroy(tmp)
-		Debugf("Removing container %s", c)
+		utils.Debugf("Removing container %s", c)
 	}
 	for i := range images {
 		builder.runtime.graph.Delete(i)
-		Debugf("Removing image %s", i)
+		utils.Debugf("Removing image %s", i)
 	}
 }
 
@@ -234,28 +235,29 @@ func (builder *Builder) Build(dockerfile io.Reader, stdout io.Writer) (*Image, e
 			fmt.Fprintf(stdout, "FROM %s\n", arguments)
 			image, err = builder.runtime.repositories.LookupImage(arguments)
 			if err != nil {
-				if builder.runtime.graph.IsNotExist(err) {
-
-					var tag, remote string
-					if strings.Contains(arguments, ":") {
-						remoteParts := strings.Split(arguments, ":")
-						tag = remoteParts[1]
-						remote = remoteParts[0]
-					} else {
-						remote = arguments
-					}
-
-					if err := builder.runtime.graph.PullRepository(stdout, remote, tag, builder.runtime.repositories, builder.runtime.authConfig); err != nil {
-						return nil, err
-					}
-
-					image, err = builder.runtime.repositories.LookupImage(arguments)
-					if err != nil {
-						return nil, err
-					}
-				} else {
-					return nil, err
-				}
+				// if builder.runtime.graph.IsNotExist(err) {
+
+				// 	var tag, remote string
+				// 	if strings.Contains(arguments, ":") {
+				// 		remoteParts := strings.Split(arguments, ":")
+				// 		tag = remoteParts[1]
+				// 		remote = remoteParts[0]
+				// 	} else {
+				// 		remote = arguments
+				// 	}
+
+				// 	panic("TODO: reimplement this")
+				// 	// if err := builder.runtime.graph.PullRepository(stdout, remote, tag, builder.runtime.repositories, builder.runtime.authConfig); err != nil {
+				// 	// 	return nil, err
+				// 	// }
+
+				// 	image, err = builder.runtime.repositories.LookupImage(arguments)
+				// 	if err != nil {
+				// 		return nil, err
+				// 	}
+				// } else {
+				return nil, err
+				// }
 			}
 			config = &Config{}
 
@@ -286,7 +288,7 @@ func (builder *Builder) Build(dockerfile io.Reader, stdout io.Writer) (*Image, e
 				break
 			}
 
-			Debugf("Env -----> %v ------ %v\n", config.Env, env)
+			utils.Debugf("Env -----> %v ------ %v\n", config.Env, env)
 
 			// Create the container and start it
 			c, err := builder.Create(config)
@@ -410,7 +412,7 @@ func (builder *Builder) Build(dockerfile io.Reader, stdout io.Writer) (*Image, e
 			destPath := strings.Trim(tmp[1], " ")
 			fmt.Fprintf(stdout, "COPY %s to %s in %s\n", sourceUrl, destPath, base.ShortId())
 
-			file, err := Download(sourceUrl, stdout)
+			file, err := utils.Download(sourceUrl, stdout)
 			if err != nil {
 				return nil, err
 			}

builder_test.go

@@ -1,6 +1,7 @@
 package docker
 
 import (
+	"github.com/dotcloud/docker/utils"
 	"strings"
 	"testing"
 )
@@ -24,7 +25,7 @@ func TestBuild(t *testing.T) {
 
 	builder := NewBuilder(runtime)
 
-	img, err := builder.Build(strings.NewReader(Dockerfile), &nopWriter{})
+	img, err := builder.Build(strings.NewReader(Dockerfile), &utils.NopWriter{})
 	if err != nil {
 		t.Fatal(err)
 	}

commands.go

@@ -7,6 +7,7 @@ import (
 	"fmt"
 	"github.com/dotcloud/docker/auth"
 	"github.com/dotcloud/docker/term"
+	"github.com/dotcloud/docker/utils"
 	"io"
 	"io/ioutil"
 	"net"
@@ -188,11 +189,11 @@ func CmdLogin(args ...string) error {
 		return readStringOnRawTerminal(stdin, stdout, false)
 	}
 
-	oldState, err := SetRawTerminal()
+	oldState, err := term.SetRawTerminal()
 	if err != nil {
 		return err
 	} else {
-		defer RestoreTerminal(oldState)
+		defer term.RestoreTerminal(oldState)
 	}
 
 	cmd := Subcmd("login", "", "Register or Login to the docker registry server")
@@ -252,7 +253,7 @@ func CmdLogin(args ...string) error {
 		return err
 	}
 	if out2.Status != "" {
-		RestoreTerminal(oldState)
+		term.RestoreTerminal(oldState)
 		fmt.Print(out2.Status)
 	}
 	return nil
@@ -303,7 +304,7 @@ func CmdVersion(args ...string) error {
 	var out ApiVersion
 	err = json.Unmarshal(body, &out)
 	if err != nil {
-		Debugf("Error unmarshal: body: %s, err: %s\n", body, err)
+		utils.Debugf("Error unmarshal: body: %s, err: %s\n", body, err)
 		return err
 	}
 	fmt.Println("Version:", out.Version)
@@ -519,7 +520,7 @@ func CmdHistory(args ...string) error {
 	fmt.Fprintln(w, "ID\tCREATED\tCREATED BY")
 
 	for _, out := range outs {
-		fmt.Fprintf(w, "%s\t%s ago\t%s\n", out.Id, HumanDuration(time.Now().Sub(time.Unix(out.Created, 0))), out.CreatedBy)
+		fmt.Fprintf(w, "%s\t%s ago\t%s\n", out.Id, utils.HumanDuration(time.Now().Sub(time.Unix(out.Created, 0))), out.CreatedBy)
 	}
 	w.Flush()
 	return nil
@@ -742,14 +743,14 @@ func CmdImages(args ...string) error {
 				if *noTrunc {
 					fmt.Fprintf(w, "%s\t", out.Id)
 				} else {
-					fmt.Fprintf(w, "%s\t", TruncateId(out.Id))
+					fmt.Fprintf(w, "%s\t", utils.TruncateId(out.Id))
 				}
-				fmt.Fprintf(w, "%s ago\n", HumanDuration(time.Now().Sub(time.Unix(out.Created, 0))))
+				fmt.Fprintf(w, "%s ago\n", utils.HumanDuration(time.Now().Sub(time.Unix(out.Created, 0))))
 			} else {
 				if *noTrunc {
 					fmt.Fprintln(w, out.Id)
 				} else {
-					fmt.Fprintln(w, TruncateId(out.Id))
+					fmt.Fprintln(w, utils.TruncateId(out.Id))
 				}
 			}
 		}
@@ -809,15 +810,15 @@ func CmdPs(args ...string) error {
 	for _, out := range outs {
 		if !*quiet {
 			if *noTrunc {
-				fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s ago\t%s\n", out.Id, out.Image, out.Command, out.Status, HumanDuration(time.Now().Sub(time.Unix(out.Created, 0))), out.Ports)
+				fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s ago\t%s\n", out.Id, out.Image, out.Command, out.Status, utils.HumanDuration(time.Now().Sub(time.Unix(out.Created, 0))), out.Ports)
 			} else {
-				fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s ago\t%s\n", TruncateId(out.Id), out.Image, Trunc(out.Command, 20), out.Status, HumanDuration(time.Now().Sub(time.Unix(out.Created, 0))), out.Ports)
+				fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s ago\t%s\n", utils.TruncateId(out.Id), out.Image, utils.Trunc(out.Command, 20), out.Status, utils.HumanDuration(time.Now().Sub(time.Unix(out.Created, 0))), out.Ports)
 			}
 		} else {
 			if *noTrunc {
 				fmt.Fprintln(w, out.Id)
 			} else {
-				fmt.Fprintln(w, TruncateId(out.Id))
+				fmt.Fprintln(w, utils.TruncateId(out.Id))
 			}
 		}
 	}
@@ -1244,20 +1245,20 @@ func hijack(method, path string, setRawTerminal bool) error {
 	rwc, br := clientconn.Hijack()
 	defer rwc.Close()
 
-	receiveStdout := Go(func() error {
+	receiveStdout := utils.Go(func() error {
 		_, err := io.Copy(os.Stdout, br)
 		return err
 	})
 
 	if setRawTerminal && term.IsTerminal(int(os.Stdin.Fd())) && os.Getenv("NORAW") == "" {
-		if oldState, err := SetRawTerminal(); err != nil {
+		if oldState, err := term.SetRawTerminal(); err != nil {
 			return err
 		} else {
-			defer RestoreTerminal(oldState)
+			defer term.RestoreTerminal(oldState)
 		}
 	}
 
-	sendStdin := Go(func() error {
+	sendStdin := utils.Go(func() error {
 		_, err := io.Copy(rwc, os.Stdin)
 		if err := rwc.(*net.TCPConn).CloseWrite(); err != nil {
 			fmt.Fprintf(os.Stderr, "Couldn't send EOF: %s\n", err)

container.go

@@ -4,6 +4,7 @@ import (
 	"encoding/json"
 	"flag"
 	"fmt"
+	"github.com/dotcloud/docker/utils"
 	"github.com/kr/pty"
 	"io"
 	"io/ioutil"
@@ -39,8 +40,8 @@ type Container struct {
 	ResolvConfPath string
 
 	cmd       *exec.Cmd
-	stdout    *writeBroadcaster
-	stderr    *writeBroadcaster
+	stdout    *utils.WriteBroadcaster
+	stderr    *utils.WriteBroadcaster
 	stdin     io.ReadCloser
 	stdinPipe io.WriteCloser
 	ptyMaster io.Closer
@@ -251,9 +252,9 @@ func (container *Container) startPty() error {
 	// Copy the PTYs to our broadcasters
 	go func() {
 		defer container.stdout.CloseWriters()
-		Debugf("[startPty] Begin of stdout pipe")
+		utils.Debugf("[startPty] Begin of stdout pipe")
 		io.Copy(container.stdout, ptyMaster)
-		Debugf("[startPty] End of stdout pipe")
+		utils.Debugf("[startPty] End of stdout pipe")
 	}()
 
 	// stdin
@@ -262,9 +263,9 @@ func (container *Container) startPty() error {
 		container.cmd.SysProcAttr = &syscall.SysProcAttr{Setctty: true, Setsid: true}
 		go func() {
 			defer container.stdin.Close()
-			Debugf("[startPty] Begin of stdin pipe")
+			utils.Debugf("[startPty] Begin of stdin pipe")
 			io.Copy(ptyMaster, container.stdin)
-			Debugf("[startPty] End of stdin pipe")
+			utils.Debugf("[startPty] End of stdin pipe")
 		}()
 	}
 	if err := container.cmd.Start(); err != nil {
@@ -284,9 +285,9 @@ func (container *Container) start() error {
 		}
 		go func() {
 			defer stdin.Close()
-			Debugf("Begin of stdin pipe [start]")
+			utils.Debugf("Begin of stdin pipe [start]")
 			io.Copy(stdin, container.stdin)
-			Debugf("End of stdin pipe [start]")
+			utils.Debugf("End of stdin pipe [start]")
 		}()
 	}
 	return container.cmd.Start()
@@ -303,8 +304,8 @@ func (container *Container) Attach(stdin io.ReadCloser, stdinCloser io.Closer, s
 			errors <- err
 		} else {
 			go func() {
-				Debugf("[start] attach stdin\n")
-				defer Debugf("[end] attach stdin\n")
+				utils.Debugf("[start] attach stdin\n")
+				defer utils.Debugf("[end] attach stdin\n")
 				// No matter what, when stdin is closed (io.Copy unblock), close stdout and stderr
 				if cStdout != nil {
 					defer cStdout.Close()
@@ -316,12 +317,12 @@ func (container *Container) Attach(stdin io.ReadCloser, stdinCloser io.Closer, s
 					defer cStdin.Close()
 				}
 				if container.Config.Tty {
-					_, err = CopyEscapable(cStdin, stdin)
+					_, err = utils.CopyEscapable(cStdin, stdin)
 				} else {
 					_, err = io.Copy(cStdin, stdin)
 				}
 				if err != nil {
-					Debugf("[error] attach stdin: %s\n", err)
+					utils.Debugf("[error] attach stdin: %s\n", err)
 				}
 				// Discard error, expecting pipe error
 				errors <- nil
@@ -335,8 +336,8 @@ func (container *Container) Attach(stdin io.ReadCloser, stdinCloser io.Closer, s
 		} else {
 			cStdout = p
 			go func() {
-				Debugf("[start] attach stdout\n")
-				defer Debugf("[end]  attach stdout\n")
+				utils.Debugf("[start] attach stdout\n")
+				defer utils.Debugf("[end]  attach stdout\n")
 				// If we are in StdinOnce mode, then close stdin
 				if container.Config.StdinOnce {
 					if stdin != nil {
@@ -348,7 +349,7 @@ func (container *Container) Attach(stdin io.ReadCloser, stdinCloser io.Closer, s
 				}
 				_, err := io.Copy(stdout, cStdout)
 				if err != nil {
-					Debugf("[error] attach stdout: %s\n", err)
+					utils.Debugf("[error] attach stdout: %s\n", err)
 				}
 				errors <- err
 			}()
@@ -361,8 +362,8 @@ func (container *Container) Attach(stdin io.ReadCloser, stdinCloser io.Closer, s
 		} else {
 			cStderr = p
 			go func() {
-				Debugf("[start] attach stderr\n")
-				defer Debugf("[end]  attach stderr\n")
+				utils.Debugf("[start] attach stderr\n")
+				defer utils.Debugf("[end]  attach stderr\n")
 				// If we are in StdinOnce mode, then close stdin
 				if container.Config.StdinOnce {
 					if stdin != nil {
@@ -374,13 +375,13 @@ func (container *Container) Attach(stdin io.ReadCloser, stdinCloser io.Closer, s
 				}
 				_, err := io.Copy(stderr, cStderr)
 				if err != nil {
-					Debugf("[error] attach stderr: %s\n", err)
+					utils.Debugf("[error] attach stderr: %s\n", err)
 				}
 				errors <- err
 			}()
 		}
 	}
-	return Go(func() error {
+	return utils.Go(func() error {
 		if cStdout != nil {
 			defer cStdout.Close()
 		}
@@ -390,14 +391,14 @@ func (container *Container) Attach(stdin io.ReadCloser, stdinCloser io.Closer, s
 		// FIXME: how do clean up the stdin goroutine without the unwanted side effect
 		// of closing the passed stdin? Add an intermediary io.Pipe?
 		for i := 0; i < nJobs; i += 1 {
-			Debugf("Waiting for job %d/%d\n", i+1, nJobs)
+			utils.Debugf("Waiting for job %d/%d\n", i+1, nJobs)
 			if err := <-errors; err != nil {
-				Debugf("Job %d returned error %s. Aborting all jobs\n", i+1, err)
+				utils.Debugf("Job %d returned error %s. Aborting all jobs\n", i+1, err)
 				return err
 			}
-			Debugf("Job %d completed successfully\n", i+1)
+			utils.Debugf("Job %d completed successfully\n", i+1)
 		}
-		Debugf("All jobs completed successfully\n")
+		utils.Debugf("All jobs completed successfully\n")
 		return nil
 	})
 }
@@ -555,13 +556,13 @@ func (container *Container) StdinPipe() (io.WriteCloser, error) {
 func (container *Container) StdoutPipe() (io.ReadCloser, error) {
 	reader, writer := io.Pipe()
 	container.stdout.AddWriter(writer)
-	return newBufReader(reader), nil
+	return utils.NewBufReader(reader), nil
 }
 
 func (container *Container) StderrPipe() (io.ReadCloser, error) {
 	reader, writer := io.Pipe()
 	container.stderr.AddWriter(writer)
-	return newBufReader(reader), nil
+	return utils.NewBufReader(reader), nil
 }
 
 func (container *Container) allocateNetwork() error {
@@ -609,20 +610,20 @@ func (container *Container) waitLxc() error {
 
 func (container *Container) monitor() {
 	// Wait for the program to exit
-	Debugf("Waiting for process")
+	utils.Debugf("Waiting for process")
 
 	// If the command does not exists, try to wait via lxc
 	if container.cmd == nil {
 		if err := container.waitLxc(); err != nil {
-			Debugf("%s: Process: %s", container.Id, err)
+			utils.Debugf("%s: Process: %s", container.Id, err)
 		}
 	} else {
 		if err := container.cmd.Wait(); err != nil {
 			// Discard the error as any signals or non 0 returns will generate an error
-			Debugf("%s: Process: %s", container.Id, err)
+			utils.Debugf("%s: Process: %s", container.Id, err)
 		}
 	}
-	Debugf("Process finished")
+	utils.Debugf("Process finished")
 
 	var exitCode int = -1
 	if container.cmd != nil {
@@ -633,19 +634,19 @@ func (container *Container) monitor() {
 	container.releaseNetwork()
 	if container.Config.OpenStdin {
 		if err := container.stdin.Close(); err != nil {
-			Debugf("%s: Error close stdin: %s", container.Id, err)
+			utils.Debugf("%s: Error close stdin: %s", container.Id, err)
 		}
 	}
 	if err := container.stdout.CloseWriters(); err != nil {
-		Debugf("%s: Error close stdout: %s", container.Id, err)
+		utils.Debugf("%s: Error close stdout: %s", container.Id, err)
 	}
 	if err := container.stderr.CloseWriters(); err != nil {
-		Debugf("%s: Error close stderr: %s", container.Id, err)
+		utils.Debugf("%s: Error close stderr: %s", container.Id, err)
 	}
 
 	if container.ptyMaster != nil {
 		if err := container.ptyMaster.Close(); err != nil {
-			Debugf("%s: Error closing Pty master: %s", container.Id, err)
+			utils.Debugf("%s: Error closing Pty master: %s", container.Id, err)
 		}
 	}
 
@@ -762,7 +763,7 @@ func (container *Container) RwChecksum() (string, error) {
 	if err != nil {
 		return "", err
 	}
-	return HashData(rwData)
+	return utils.HashData(rwData)
 }
 
 func (container *Container) Export() (Archive, error) {
@@ -833,7 +834,7 @@ func (container *Container) Unmount() error {
 // In case of a collision a lookup with Runtime.Get() will fail, and the caller
 // will need to use a langer prefix, or the full-length container Id.
 func (container *Container) ShortId() string {
-	return TruncateId(container.Id)
+	return utils.TruncateId(container.Id)
 }
 
 func (container *Container) logPath(name string) string {

docker/docker.go

@@ -4,6 +4,7 @@ import (
 	"flag"
 	"fmt"
 	"github.com/dotcloud/docker"
+	"github.com/dotcloud/docker/utils"
 	"io/ioutil"
 	"log"
 	"os"
@@ -17,7 +18,7 @@ var (
 )
 
 func main() {
-	if docker.SelfPath() == "/sbin/init" {
+	if utils.SelfPath() == "/sbin/init" {
 		// Running in init mode
 		docker.SysInit()
 		return

docs/sources/index/variable.rst

@@ -0,0 +1,23 @@
+=================================
+Docker Index Environment Variable
+=================================
+
+Variable
+--------
+
+.. code-block:: sh
+
+    DOCKER_INDEX_URL
+
+Setting this environment variable on the docker server will change the URL docker index.
+This address is used in commands such as ``docker login``, ``docker push`` and ``docker pull``.
+The docker daemon doesn't need to be restarted for this parameter to take effect.
+
+Example
+-------
+
+.. code-block:: sh
+
+    docker -d &
+    export DOCKER_INDEX_URL="https://index.docker.io"
+

getKernelVersion_darwin.go

@@ -2,8 +2,9 @@ package docker
 
 import (
 	"fmt"
+	"github.com/dotcloud/docker/utils"
 )
 
-func getKernelVersion() (*KernelVersionInfo, error) {
+func getKernelVersion() (*utils.KernelVersionInfo, error) {
 	return nil, fmt.Errorf("Kernel version detection is not available on darwin")
 }

getKernelVersion_linux.go

@@ -2,12 +2,14 @@ package docker
 
 import (
 	"bytes"
+	"github.com/dotcloud/docker/utils"
 	"strconv"
 	"strings"
 	"syscall"
 )
 
-func getKernelVersion() (*KernelVersionInfo, error) {
+// FIXME: Move this to utils package
+func getKernelVersion() (*utils.KernelVersionInfo, error) {
 	var (
 		uts                  syscall.Utsname
 		flavor               string
@@ -60,7 +62,7 @@ func getKernelVersion() (*KernelVersionInfo, error) {
 		flavor = ""
 	}
 
-	return &KernelVersionInfo{
+	return &utils.KernelVersionInfo{
 		Kernel: kernel,
 		Major:  major,
 		Minor:  minor,

graph.go

@@ -3,9 +3,10 @@ package docker
 import (
 	"encoding/json"
 	"fmt"
+	"github.com/dotcloud/docker/registry"
+	"github.com/dotcloud/docker/utils"
 	"io"
 	"io/ioutil"
-	"net/http"
 	"os"
 	"path"
 	"path/filepath"
@@ -17,8 +18,7 @@ import (
 // A Graph is a store for versioned filesystem images and the relationship between them.
 type Graph struct {
 	Root         string
-	idIndex      *TruncIndex
-	httpClient   *http.Client
+	idIndex      *utils.TruncIndex
 	checksumLock map[string]*sync.Mutex
 	lockSumFile  *sync.Mutex
 	lockSumMap   *sync.Mutex
@@ -37,7 +37,7 @@ func NewGraph(root string) (*Graph, error) {
 	}
 	graph := &Graph{
 		Root:         abspath,
-		idIndex:      NewTruncIndex(),
+		idIndex:      utils.NewTruncIndex(),
 		checksumLock: make(map[string]*sync.Mutex),
 		lockSumFile:  &sync.Mutex{},
 		lockSumMap:   &sync.Mutex{},
@@ -165,7 +165,7 @@ func (graph *Graph) TempLayerArchive(id string, compression Compression, output
 	if err != nil {
 		return nil, err
 	}
-	return NewTempArchive(ProgressReader(ioutil.NopCloser(archive), 0, output, "Buffering to disk %v/%v (%v)"), tmp.Root)
+	return NewTempArchive(utils.ProgressReader(ioutil.NopCloser(archive), 0, output, "Buffering to disk %v/%v (%v)"), tmp.Root)
 }
 
 // Mktemp creates a temporary sub-directory inside the graph's filesystem.
@@ -324,3 +324,17 @@ func (graph *Graph) storeChecksums(checksums map[string]string) error {
 	}
 	return nil
 }
+
+func (graph *Graph) UpdateChecksums(newChecksums map[string]*registry.ImgData) error {
+	graph.lockSumFile.Lock()
+	defer graph.lockSumFile.Unlock()
+
+	localChecksums, err := graph.getStoredChecksums()
+	if err != nil {
+		return err
+	}
+	for id, elem := range newChecksums {
+		localChecksums[id] = elem.Checksum
+	}
+	return graph.storeChecksums(localChecksums)
+}

graph_test.go

@@ -4,6 +4,7 @@ import (
 	"archive/tar"
 	"bytes"
 	"errors"
+	"github.com/dotcloud/docker/utils"
 	"io"
 	"io/ioutil"
 	"os"
@@ -155,7 +156,7 @@ func TestDeletePrefix(t *testing.T) {
 	graph := tempGraph(t)
 	defer os.RemoveAll(graph.Root)
 	img := createTestImage(graph, t)
-	if err := graph.Delete(TruncateId(img.Id)); err != nil {
+	if err := graph.Delete(utils.TruncateId(img.Id)); err != nil {
 		t.Fatal(err)
 	}
 	assertNImages(graph, t, 0)

image.go

@@ -6,6 +6,7 @@ import (
 	"encoding/hex"
 	"encoding/json"
 	"fmt"
+	"github.com/dotcloud/docker/utils"
 	"io"
 	"io/ioutil"
 	"log"
@@ -180,7 +181,7 @@ func (image *Image) Changes(rw string) ([]Change, error) {
 }
 
 func (image *Image) ShortId() string {
-	return TruncateId(image.Id)
+	return utils.TruncateId(image.Id)
 }
 
 func ValidateId(id string) error {
@@ -359,3 +360,15 @@ func (img *Image) Checksum() (string, error) {
 
 	return hash, nil
 }
+
+// Build an Image object from raw json data
+func NewImgJson(src []byte) (*Image, error) {
+	ret := &Image{}
+
+	utils.Debugf("Json string: {%s}\n", src)
+	// FIXME: Is there a cleaner way to "purify" the input json?
+	if err := json.Unmarshal(src, ret); err != nil {
+		return nil, err
+	}
+	return ret, nil
+}

network.go

@@ -4,6 +4,7 @@ import (
 	"encoding/binary"
 	"errors"
 	"fmt"
+	"github.com/dotcloud/docker/utils"
 	"io"
 	"log"
 	"net"
@@ -97,7 +98,7 @@ func checkRouteOverlaps(dockerNetwork *net.IPNet) error {
 	if err != nil {
 		return err
 	}
-	Debugf("Routes:\n\n%s", output)
+	utils.Debugf("Routes:\n\n%s", output)
 	for _, line := range strings.Split(output, "\n") {
 		if strings.Trim(line, "\r\n\t ") == "" || strings.Contains(line, "default") {
 			continue
@@ -126,13 +127,13 @@ func CreateBridgeIface(ifaceName string) error {
 			ifaceAddr = addr
 			break
 		} else {
-			Debugf("%s: %s", addr, err)
+			utils.Debugf("%s: %s", addr, err)
 		}
 	}
 	if ifaceAddr == "" {
 		return fmt.Errorf("Could not find a free IP address range for interface '%s'. Please configure its address manually and run 'docker -b %s'", ifaceName, ifaceName)
 	} else {
-		Debugf("Creating bridge %s with network %s", ifaceName, ifaceAddr)
+		utils.Debugf("Creating bridge %s with network %s", ifaceName, ifaceAddr)
 	}
 
 	if output, err := ip("link", "add", ifaceName, "type", "bridge"); err != nil {
@@ -239,22 +240,22 @@ func (mapper *PortMapper) Map(port int, dest net.TCPAddr) error {
 // proxy listens for socket connections on `listener`, and forwards them unmodified
 // to `proto:address`
 func proxy(listener net.Listener, proto, address string) error {
-	Debugf("proxying to %s:%s", proto, address)
-	defer Debugf("Done proxying to %s:%s", proto, address)
+	utils.Debugf("proxying to %s:%s", proto, address)
+	defer utils.Debugf("Done proxying to %s:%s", proto, address)
 	for {
-		Debugf("Listening on %s", listener)
+		utils.Debugf("Listening on %s", listener)
 		src, err := listener.Accept()
 		if err != nil {
 			return err
 		}
-		Debugf("Connecting to %s:%s", proto, address)
+		utils.Debugf("Connecting to %s:%s", proto, address)
 		dst, err := net.Dial(proto, address)
 		if err != nil {
 			log.Printf("Error connecting to %s:%s: %s", proto, address, err)
 			src.Close()
 			continue
 		}
-		Debugf("Connected to backend, splicing")
+		utils.Debugf("Connected to backend, splicing")
 		splice(src, dst)
 	}
 	return nil
@@ -317,7 +318,7 @@ func (alloc *PortAllocator) runFountain() {
 
 // FIXME: Release can no longer fail, change its prototype to reflect that.
 func (alloc *PortAllocator) Release(port int) error {
-	Debugf("Releasing %d", port)
+	utils.Debugf("Releasing %d", port)
 	alloc.lock.Lock()
 	delete(alloc.inUse, port)
 	alloc.lock.Unlock()
@@ -325,7 +326,7 @@ func (alloc *PortAllocator) Release(port int) error {
 }
 
 func (alloc *PortAllocator) Acquire(port int) (int, error) {
-	Debugf("Acquiring %d", port)
+	utils.Debugf("Acquiring %d", port)
 	if port == 0 {
 		// Allocate a port from the fountain
 		for port := range alloc.fountain {

registry.go

@@ -1,748 +0,0 @@
-package docker
-
-import (
-	"bytes"
-	"encoding/json"
-	"fmt"
-	"github.com/dotcloud/docker/auth"
-	"github.com/shin-/cookiejar"
-	"io"
-	"io/ioutil"
-	"net/http"
-	"net/url"
-	"os"
-	"path"
-	"strings"
-)
-
-//FIXME: Set the endpoint in a conf file or via commandline
-const INDEX_ENDPOINT = auth.INDEX_SERVER + "/v1"
-
-// Build an Image object from raw json data
-func NewImgJson(src []byte) (*Image, error) {
-	ret := &Image{}
-
-	Debugf("Json string: {%s}\n", src)
-	// FIXME: Is there a cleaner way to "purify" the input json?
-	if err := json.Unmarshal(src, ret); err != nil {
-		return nil, err
-	}
-	return ret, nil
-}
-
-func doWithCookies(c *http.Client, req *http.Request) (*http.Response, error) {
-	for _, cookie := range c.Jar.Cookies(req.URL) {
-		req.AddCookie(cookie)
-	}
-	return c.Do(req)
-}
-
-// Retrieve the history of a given image from the Registry.
-// Return a list of the parent's json (requested image included)
-func (graph *Graph) getRemoteHistory(imgId, registry string, token []string) ([]string, error) {
-	client := graph.getHttpClient()
-
-	req, err := http.NewRequest("GET", registry+"/images/"+imgId+"/ancestry", nil)
-	if err != nil {
-		return nil, err
-	}
-	req.Header.Set("Authorization", "Token "+strings.Join(token, ", "))
-	res, err := client.Do(req)
-	if err != nil || res.StatusCode != 200 {
-		if res != nil {
-			return nil, fmt.Errorf("Internal server error: %d trying to fetch remote history for %s", res.StatusCode, imgId)
-		}
-		return nil, err
-	}
-	defer res.Body.Close()
-
-	jsonString, err := ioutil.ReadAll(res.Body)
-	if err != nil {
-		return nil, fmt.Errorf("Error while reading the http response: %s\n", err)
-	}
-
-	Debugf("Ancestry: %s", jsonString)
-	history := new([]string)
-	if err := json.Unmarshal(jsonString, history); err != nil {
-		return nil, err
-	}
-	return *history, nil
-}
-
-func (graph *Graph) getHttpClient() *http.Client {
-	if graph.httpClient == nil {
-		graph.httpClient = &http.Client{}
-		graph.httpClient.Jar = cookiejar.NewCookieJar()
-	}
-	return graph.httpClient
-}
-
-// Check if an image exists in the Registry
-func (graph *Graph) LookupRemoteImage(imgId, registry string, authConfig *auth.AuthConfig) bool {
-	rt := &http.Transport{Proxy: http.ProxyFromEnvironment}
-
-	req, err := http.NewRequest("GET", registry+"/images/"+imgId+"/json", nil)
-	if err != nil {
-		return false
-	}
-	req.SetBasicAuth(authConfig.Username, authConfig.Password)
-	res, err := rt.RoundTrip(req)
-	return err == nil && res.StatusCode == 307
-}
-
-func (graph *Graph) getImagesInRepository(repository string, authConfig *auth.AuthConfig) ([]map[string]string, error) {
-	u := INDEX_ENDPOINT + "/repositories/" + repository + "/images"
-	req, err := http.NewRequest("GET", u, nil)
-	if err != nil {
-		return nil, err
-	}
-	if authConfig != nil && len(authConfig.Username) > 0 {
-		req.SetBasicAuth(authConfig.Username, authConfig.Password)
-	}
-	res, err := graph.getHttpClient().Do(req)
-	if err != nil {
-		return nil, err
-	}
-	defer res.Body.Close()
-
-	// Repository doesn't exist yet
-	if res.StatusCode == 404 {
-		return nil, nil
-	}
-
-	jsonData, err := ioutil.ReadAll(res.Body)
-	if err != nil {
-		return nil, err
-	}
-
-	imageList := []map[string]string{}
-
-	err = json.Unmarshal(jsonData, &imageList)
-	if err != nil {
-		Debugf("Body: %s (%s)\n", res.Body, u)
-		return nil, err
-	}
-
-	return imageList, nil
-}
-
-// Retrieve an image from the Registry.
-// Returns the Image object as well as the layer as an Archive (io.Reader)
-func (graph *Graph) getRemoteImage(stdout io.Writer, imgId, registry string, token []string) (*Image, Archive, error) {
-	client := graph.getHttpClient()
-
-	fmt.Fprintf(stdout, "Pulling %s metadata\r\n", imgId)
-	// Get the Json
-	req, err := http.NewRequest("GET", registry+"/images/"+imgId+"/json", nil)
-	if err != nil {
-		return nil, nil, fmt.Errorf("Failed to download json: %s", err)
-	}
-	req.Header.Set("Authorization", "Token "+strings.Join(token, ", "))
-	res, err := client.Do(req)
-	if err != nil {
-		return nil, nil, fmt.Errorf("Failed to download json: %s", err)
-	}
-	if res.StatusCode != 200 {
-		return nil, nil, fmt.Errorf("HTTP code %d", res.StatusCode)
-	}
-	defer res.Body.Close()
-
-	jsonString, err := ioutil.ReadAll(res.Body)
-	if err != nil {
-		return nil, nil, fmt.Errorf("Failed to download json: %s", err)
-	}
-
-	img, err := NewImgJson(jsonString)
-	if err != nil {
-		return nil, nil, fmt.Errorf("Failed to parse json: %s", err)
-	}
-	img.Id = imgId
-
-	// Get the layer
-	fmt.Fprintf(stdout, "Pulling %s fs layer\r\n", imgId)
-	req, err = http.NewRequest("GET", registry+"/images/"+imgId+"/layer", nil)
-	if err != nil {
-		return nil, nil, fmt.Errorf("Error while getting from the server: %s\n", err)
-	}
-	req.Header.Set("Authorization", "Token "+strings.Join(token, ", "))
-	res, err = client.Do(req)
-	if err != nil {
-		return nil, nil, err
-	}
-	return img, ProgressReader(res.Body, int(res.ContentLength), stdout, "Downloading %v/%v (%v)"), nil
-}
-
-func (graph *Graph) getRemoteTags(stdout io.Writer, registries []string, repository string, token []string) (map[string]string, error) {
-	client := graph.getHttpClient()
-	if strings.Count(repository, "/") == 0 {
-		// This will be removed once the Registry supports auto-resolution on
-		// the "library" namespace
-		repository = "library/" + repository
-	}
-	for _, host := range registries {
-		endpoint := fmt.Sprintf("https://%s/v1/repositories/%s/tags", host, repository)
-		req, err := http.NewRequest("GET", endpoint, nil)
-		if err != nil {
-			return nil, err
-		}
-		req.Header.Set("Authorization", "Token "+strings.Join(token, ", "))
-		res, err := client.Do(req)
-		defer res.Body.Close()
-		Debugf("Got status code %d from %s", res.StatusCode, endpoint)
-		if err != nil || (res.StatusCode != 200 && res.StatusCode != 404) {
-			continue
-		} else if res.StatusCode == 404 {
-			return nil, fmt.Errorf("Repository not found")
-		}
-
-		result := make(map[string]string)
-
-		rawJson, err := ioutil.ReadAll(res.Body)
-		if err != nil {
-			return nil, err
-		}
-		if err = json.Unmarshal(rawJson, &result); err != nil {
-			return nil, err
-		}
-		return result, nil
-	}
-	return nil, fmt.Errorf("Could not reach any registry endpoint")
-}
-
-func (graph *Graph) getImageForTag(stdout io.Writer, tag, remote, registry string, token []string) (string, error) {
-	client := graph.getHttpClient()
-
-	if !strings.Contains(remote, "/") {
-		remote = "library/" + remote
-	}
-
-	registryEndpoint := "https://" + registry + "/v1"
-	repositoryTarget := registryEndpoint + "/repositories/" + remote + "/tags/" + tag
-
-	req, err := http.NewRequest("GET", repositoryTarget, nil)
-	if err != nil {
-		return "", err
-	}
-	req.Header.Set("Authorization", "Token "+strings.Join(token, ", "))
-	res, err := client.Do(req)
-	if err != nil {
-		return "", fmt.Errorf("Error while retrieving repository info: %v", err)
-	}
-	defer res.Body.Close()
-	if res.StatusCode == 403 {
-		return "", fmt.Errorf("You aren't authorized to access this resource")
-	} else if res.StatusCode != 200 {
-		return "", fmt.Errorf("HTTP code: %d", res.StatusCode)
-	}
-
-	var imgId string
-	rawJson, err := ioutil.ReadAll(res.Body)
-	if err != nil {
-		return "", err
-	}
-	if err = json.Unmarshal(rawJson, &imgId); err != nil {
-		return "", err
-	}
-	return imgId, nil
-}
-
-func (graph *Graph) PullImage(stdout io.Writer, imgId, registry string, token []string) error {
-	history, err := graph.getRemoteHistory(imgId, registry, token)
-	if err != nil {
-		return err
-	}
-	// FIXME: Try to stream the images?
-	// FIXME: Launch the getRemoteImage() in goroutines
-	for _, id := range history {
-		if !graph.Exists(id) {
-			img, layer, err := graph.getRemoteImage(stdout, id, registry, token)
-			if err != nil {
-				// FIXME: Keep goging in case of error?
-				return err
-			}
-			if err = graph.Register(layer, false, img); err != nil {
-				return err
-			}
-		}
-	}
-	return nil
-}
-
-func (graph *Graph) PullRepository(stdout io.Writer, remote, askedTag string, repositories *TagStore, authConfig *auth.AuthConfig) error {
-	client := graph.getHttpClient()
-
-	fmt.Fprintf(stdout, "Pulling repository %s from %s\r\n", remote, INDEX_ENDPOINT)
-	repositoryTarget := INDEX_ENDPOINT + "/repositories/" + remote + "/images"
-
-	req, err := http.NewRequest("GET", repositoryTarget, nil)
-	if err != nil {
-		return err
-	}
-	if authConfig != nil && len(authConfig.Username) > 0 {
-		req.SetBasicAuth(authConfig.Username, authConfig.Password)
-	}
-	req.Header.Set("X-Docker-Token", "true")
-
-	res, err := client.Do(req)
-	if err != nil {
-		return err
-	}
-	defer res.Body.Close()
-	if res.StatusCode == 401 {
-		return fmt.Errorf("Please login first (HTTP code %d)", res.StatusCode)
-	}
-	// TODO: Right now we're ignoring checksums in the response body.
-	// In the future, we need to use them to check image validity.
-	if res.StatusCode != 200 {
-		return fmt.Errorf("HTTP code: %d", res.StatusCode)
-	}
-
-	var token, endpoints []string
-	if res.Header.Get("X-Docker-Token") != "" {
-		token = res.Header["X-Docker-Token"]
-	}
-	if res.Header.Get("X-Docker-Endpoints") != "" {
-		endpoints = res.Header["X-Docker-Endpoints"]
-	} else {
-		return fmt.Errorf("Index response didn't contain any endpoints")
-	}
-
-	checksumsJson, err := ioutil.ReadAll(res.Body)
-	if err != nil {
-		return err
-	}
-
-	// Reload the json file to make sure not to overwrite faster sums
-	err = func() error {
-		localChecksums := make(map[string]string)
-		remoteChecksums := []ImgListJson{}
-		checksumDictPth := path.Join(graph.Root, "checksums")
-
-		if err := json.Unmarshal(checksumsJson, &remoteChecksums); err != nil {
-			return err
-		}
-
-		graph.lockSumFile.Lock()
-		defer graph.lockSumFile.Unlock()
-
-		if checksumDict, err := ioutil.ReadFile(checksumDictPth); err == nil {
-			if err := json.Unmarshal(checksumDict, &localChecksums); err != nil {
-				return err
-			}
-		}
-
-		for _, elem := range remoteChecksums {
-			localChecksums[elem.Id] = elem.Checksum
-		}
-
-		checksumsJson, err = json.Marshal(localChecksums)
-		if err != nil {
-			return err
-		}
-		if err := ioutil.WriteFile(checksumDictPth, checksumsJson, 0600); err != nil {
-			return err
-		}
-		return nil
-	}()
-	if err != nil {
-		return err
-	}
-
-	var tagsList map[string]string
-	if askedTag == "" {
-		tagsList, err = graph.getRemoteTags(stdout, endpoints, remote, token)
-		if err != nil {
-			return err
-		}
-	} else {
-		tagsList = map[string]string{askedTag: ""}
-	}
-
-	for askedTag, imgId := range tagsList {
-		fmt.Fprintf(stdout, "Resolving tag \"%s:%s\" from %s\n", remote, askedTag, endpoints)
-		success := false
-		for _, registry := range endpoints {
-			if imgId == "" {
-				imgId, err = graph.getImageForTag(stdout, askedTag, remote, registry, token)
-				if err != nil {
-					fmt.Fprintf(stdout, "Error while retrieving image for tag: %v (%v) ; "+
-						"checking next endpoint", askedTag, err)
-					continue
-				}
-			}
-
-			if err := graph.PullImage(stdout, imgId, "https://"+registry+"/v1", token); err != nil {
-				return err
-			}
-
-			if err = repositories.Set(remote, askedTag, imgId, true); err != nil {
-				return err
-			}
-			success = true
-		}
-
-		if !success {
-			return fmt.Errorf("Could not find repository on any of the indexed registries.")
-		}
-	}
-
-	if err = repositories.Save(); err != nil {
-		return err
-	}
-
-	return nil
-}
-
-// Push a local image to the registry
-func (graph *Graph) PushImage(stdout io.Writer, img *Image, registry string, token []string) error {
-	registry = "https://" + registry + "/v1"
-
-	client := graph.getHttpClient()
-	jsonRaw, err := ioutil.ReadFile(path.Join(graph.Root, img.Id, "json"))
-	if err != nil {
-		return fmt.Errorf("Error while retreiving the path for {%s}: %s", img.Id, err)
-	}
-
-	fmt.Fprintf(stdout, "Pushing %s metadata\r\n", img.Id)
-
-	// FIXME: try json with UTF8
-	jsonData := strings.NewReader(string(jsonRaw))
-	req, err := http.NewRequest("PUT", registry+"/images/"+img.Id+"/json", jsonData)
-	if err != nil {
-		return err
-	}
-	req.Header.Add("Content-type", "application/json")
-	req.Header.Set("Authorization", "Token "+strings.Join(token, ","))
-
-	checksum, err := img.Checksum()
-	if err != nil {
-		return fmt.Errorf("Error while retrieving checksum for %s: %v", img.Id, err)
-	}
-	req.Header.Set("X-Docker-Checksum", checksum)
-	Debugf("Setting checksum for %s: %s", img.ShortId(), checksum)
-	res, err := doWithCookies(client, req)
-	if err != nil {
-		return fmt.Errorf("Failed to upload metadata: %s", err)
-	}
-	defer res.Body.Close()
-	if len(res.Cookies()) > 0 {
-		client.Jar.SetCookies(req.URL, res.Cookies())
-	}
-	if res.StatusCode != 200 {
-		errBody, err := ioutil.ReadAll(res.Body)
-		if err != nil {
-			return fmt.Errorf("HTTP code %d while uploading metadata and error when"+
-				" trying to parse response body: %v", res.StatusCode, err)
-		}
-		var jsonBody map[string]string
-		if err := json.Unmarshal(errBody, &jsonBody); err != nil {
-			errBody = []byte(err.Error())
-		} else if jsonBody["error"] == "Image already exists" {
-			fmt.Fprintf(stdout, "Image %v already uploaded ; skipping\n", img.Id)
-			return nil
-		}
-		return fmt.Errorf("HTTP code %d while uploading metadata: %s", res.StatusCode, errBody)
-	}
-
-	fmt.Fprintf(stdout, "Pushing %s fs layer\r\n", img.Id)
-	root, err := img.root()
-	if err != nil {
-		return err
-	}
-
-	var layerData *TempArchive
-	// If the archive exists, use it
-	file, err := os.Open(layerArchivePath(root))
-	if err != nil {
-		if os.IsNotExist(err) {
-			// If the archive does not exist, create one from the layer
-			layerData, err = graph.TempLayerArchive(img.Id, Xz, stdout)
-			if err != nil {
-				return fmt.Errorf("Failed to generate layer archive: %s", err)
-			}
-		} else {
-			return err
-		}
-	} else {
-		defer file.Close()
-		st, err := file.Stat()
-		if err != nil {
-			return err
-		}
-		layerData = &TempArchive{file, st.Size()}
-	}
-
-	req3, err := http.NewRequest("PUT", registry+"/images/"+img.Id+"/layer",
-		ProgressReader(layerData, int(layerData.Size), stdout, ""))
-	if err != nil {
-		return err
-	}
-
-	req3.ContentLength = -1
-	req3.TransferEncoding = []string{"chunked"}
-	req3.Header.Set("Authorization", "Token "+strings.Join(token, ","))
-	res3, err := doWithCookies(client, req3)
-	if err != nil {
-		return fmt.Errorf("Failed to upload layer: %s", err)
-	}
-	defer res3.Body.Close()
-
-	if res3.StatusCode != 200 {
-		errBody, err := ioutil.ReadAll(res3.Body)
-		if err != nil {
-			return fmt.Errorf("HTTP code %d while uploading metadata and error when"+
-				" trying to parse response body: %v", res.StatusCode, err)
-		}
-		return fmt.Errorf("Received HTTP code %d while uploading layer: %s", res3.StatusCode, errBody)
-	}
-	return nil
-}
-
-// push a tag on the registry.
-// Remote has the format '<user>/<repo>
-func (graph *Graph) pushTag(remote, revision, tag, registry string, token []string) error {
-	// "jsonify" the string
-	revision = "\"" + revision + "\""
-	registry = "https://" + registry + "/v1"
-
-	Debugf("Pushing tags for rev [%s] on {%s}\n", revision, registry+"/users/"+remote+"/"+tag)
-
-	client := graph.getHttpClient()
-	req, err := http.NewRequest("PUT", registry+"/repositories/"+remote+"/tags/"+tag, strings.NewReader(revision))
-	if err != nil {
-		return err
-	}
-	req.Header.Add("Content-type", "application/json")
-	req.Header.Set("Authorization", "Token "+strings.Join(token, ","))
-	req.ContentLength = int64(len(revision))
-	res, err := doWithCookies(client, req)
-	if err != nil {
-		return err
-	}
-	res.Body.Close()
-	if res.StatusCode != 200 && res.StatusCode != 201 {
-		return fmt.Errorf("Internal server error: %d trying to push tag %s on %s", res.StatusCode, tag, remote)
-	}
-	return nil
-}
-
-// FIXME: this should really be PushTag
-func (graph *Graph) pushPrimitive(stdout io.Writer, remote, tag, imgId, registry string, token []string) error {
-	// Check if the local impage exists
-	img, err := graph.Get(imgId)
-	if err != nil {
-		fmt.Fprintf(stdout, "Skipping tag %s:%s: %s does not exist\r\n", remote, tag, imgId)
-		return nil
-	}
-	fmt.Fprintf(stdout, "Pushing image %s:%s\r\n", remote, tag)
-	// Push the image
-	if err = graph.PushImage(stdout, img, registry, token); err != nil {
-		return err
-	}
-	fmt.Fprintf(stdout, "Registering tag %s:%s\r\n", remote, tag)
-	// And then the tag
-	if err = graph.pushTag(remote, imgId, tag, registry, token); err != nil {
-		return err
-	}
-	return nil
-}
-
-// Retrieve the checksum of an image
-// Priority:
-// - Check on the stored checksums
-// - Check if the archive exists, if it does not, ask the registry
-// - If the archive does exists, process the checksum from it
-// - If the archive does not exists and not found on registry, process checksum from layer
-func (graph *Graph) getChecksum(imageId string) (string, error) {
-	// FIXME: Use in-memory map instead of reading the file each time
-	if sums, err := graph.getStoredChecksums(); err != nil {
-		return "", err
-	} else if checksum, exists := sums[imageId]; exists {
-		return checksum, nil
-	}
-
-	img, err := graph.Get(imageId)
-	if err != nil {
-		return "", err
-	}
-
-	if _, err := os.Stat(layerArchivePath(graph.imageRoot(imageId))); err != nil {
-		if os.IsNotExist(err) {
-			// TODO: Ask the registry for the checksum
-			//       As the archive is not there, it is supposed to come from a pull.
-		} else {
-			return "", err
-		}
-	}
-
-	checksum, err := img.Checksum()
-	if err != nil {
-		return "", err
-	}
-	return checksum, nil
-}
-
-type ImgListJson struct {
-	Id       string `json:"id"`
-	Checksum string `json:"checksum,omitempty"`
-	tag      string
-}
-
-// Push a repository to the registry.
-// Remote has the format '<user>/<repo>
-func (graph *Graph) PushRepository(stdout io.Writer, remote string, localRepo Repository, authConfig *auth.AuthConfig) error {
-	client := graph.getHttpClient()
-	// FIXME: Do not reset the cookie each time? (need to reset it in case updating latest of a repo and repushing)
-	client.Jar = cookiejar.NewCookieJar()
-	var imgList []*ImgListJson
-
-	fmt.Fprintf(stdout, "Processing checksums\n")
-	imageSet := make(map[string]struct{})
-
-	for tag, id := range localRepo {
-		img, err := graph.Get(id)
-		if err != nil {
-			return err
-		}
-		img.WalkHistory(func(img *Image) error {
-			if _, exists := imageSet[img.Id]; exists {
-				return nil
-			}
-			imageSet[img.Id] = struct{}{}
-			checksum, err := graph.getChecksum(img.Id)
-			if err != nil {
-				return err
-			}
-			imgList = append([]*ImgListJson{{
-				Id:       img.Id,
-				Checksum: checksum,
-				tag:      tag,
-			}}, imgList...)
-			return nil
-		})
-	}
-
-	imgListJson, err := json.Marshal(imgList)
-	if err != nil {
-		return err
-	}
-
-	Debugf("json sent: %s\n", imgListJson)
-
-	fmt.Fprintf(stdout, "Sending image list\n")
-	req, err := http.NewRequest("PUT", INDEX_ENDPOINT+"/repositories/"+remote+"/", bytes.NewReader(imgListJson))
-	if err != nil {
-		return err
-	}
-	req.SetBasicAuth(authConfig.Username, authConfig.Password)
-	req.ContentLength = int64(len(imgListJson))
-	req.Header.Set("X-Docker-Token", "true")
-
-	res, err := client.Do(req)
-	if err != nil {
-		return err
-	}
-	defer res.Body.Close()
-
-	for res.StatusCode >= 300 && res.StatusCode < 400 {
-		Debugf("Redirected to %s\n", res.Header.Get("Location"))
-		req, err = http.NewRequest("PUT", res.Header.Get("Location"), bytes.NewReader(imgListJson))
-		if err != nil {
-			return err
-		}
-		req.SetBasicAuth(authConfig.Username, authConfig.Password)
-		req.ContentLength = int64(len(imgListJson))
-		req.Header.Set("X-Docker-Token", "true")
-
-		res, err = client.Do(req)
-		if err != nil {
-			return err
-		}
-		defer res.Body.Close()
-	}
-
-	if res.StatusCode != 200 && res.StatusCode != 201 {
-		errBody, err := ioutil.ReadAll(res.Body)
-		if err != nil {
-			return err
-		}
-		return fmt.Errorf("Error: Status %d trying to push repository %s: %s", res.StatusCode, remote, errBody)
-	}
-
-	var token, endpoints []string
-	if res.Header.Get("X-Docker-Token") != "" {
-		token = res.Header["X-Docker-Token"]
-		Debugf("Auth token: %v", token)
-	} else {
-		return fmt.Errorf("Index response didn't contain an access token")
-	}
-	if res.Header.Get("X-Docker-Endpoints") != "" {
-		endpoints = res.Header["X-Docker-Endpoints"]
-	} else {
-		return fmt.Errorf("Index response didn't contain any endpoints")
-	}
-
-	// FIXME: Send only needed images
-	for _, registry := range endpoints {
-		fmt.Fprintf(stdout, "Pushing repository %s to %s (%d tags)\r\n", remote, registry, len(localRepo))
-		// For each image within the repo, push them
-		for _, elem := range imgList {
-			if err := graph.pushPrimitive(stdout, remote, elem.tag, elem.Id, registry, token); err != nil {
-				// FIXME: Continue on error?
-				return err
-			}
-		}
-	}
-
-	req2, err := http.NewRequest("PUT", INDEX_ENDPOINT+"/repositories/"+remote+"/images", bytes.NewReader(imgListJson))
-	if err != nil {
-		return err
-	}
-	req2.SetBasicAuth(authConfig.Username, authConfig.Password)
-	req2.Header["X-Docker-Endpoints"] = endpoints
-	req2.ContentLength = int64(len(imgListJson))
-	res2, err := client.Do(req2)
-	if err != nil {
-		return err
-	}
-	defer res2.Body.Close()
-	if res2.StatusCode != 204 {
-		if errBody, err := ioutil.ReadAll(res2.Body); err != nil {
-			return err
-		} else {
-			return fmt.Errorf("Error: Status %d trying to push checksums %s: %s", res2.StatusCode, remote, errBody)
-		}
-	}
-
-	return nil
-}
-
-type SearchResults struct {
-	Query      string              `json:"query"`
-	NumResults int                 `json:"num_results"`
-	Results    []map[string]string `json:"results"`
-}
-
-func (graph *Graph) SearchRepositories(stdout io.Writer, term string) (*SearchResults, error) {
-	client := graph.getHttpClient()
-	u := INDEX_ENDPOINT + "/search?q=" + url.QueryEscape(term)
-	req, err := http.NewRequest("GET", u, nil)
-	if err != nil {
-		return nil, err
-	}
-	res, err := client.Do(req)
-	if err != nil {
-		return nil, err
-	}
-	defer res.Body.Close()
-	if res.StatusCode != 200 {
-		return nil, fmt.Errorf("Unexepected status code %d", res.StatusCode)
-	}
-	rawData, err := ioutil.ReadAll(res.Body)
-	if err != nil {
-		return nil, err
-	}
-	result := new(SearchResults)
-	err = json.Unmarshal(rawData, result)
-	return result, err
-}

registry/registry.go

@@ -0,0 +1,472 @@
+package registry
+
+import (
+	"bytes"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"github.com/dotcloud/docker/auth"
+	"github.com/dotcloud/docker/utils"
+	"github.com/shin-/cookiejar"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"net/url"
+	"strings"
+)
+
+var ErrAlreadyExists error = errors.New("Image already exists")
+
+func doWithCookies(c *http.Client, req *http.Request) (*http.Response, error) {
+	for _, cookie := range c.Jar.Cookies(req.URL) {
+		req.AddCookie(cookie)
+	}
+	return c.Do(req)
+}
+
+// Retrieve the history of a given image from the Registry.
+// Return a list of the parent's json (requested image included)
+func (r *Registry) GetRemoteHistory(imgId, registry string, token []string) ([]string, error) {
+	req, err := http.NewRequest("GET", registry+"/images/"+imgId+"/ancestry", nil)
+	if err != nil {
+		return nil, err
+	}
+	req.Header.Set("Authorization", "Token "+strings.Join(token, ", "))
+	res, err := r.client.Do(req)
+	if err != nil || res.StatusCode != 200 {
+		if res != nil {
+			return nil, fmt.Errorf("Internal server error: %d trying to fetch remote history for %s", res.StatusCode, imgId)
+		}
+		return nil, err
+	}
+	defer res.Body.Close()
+
+	jsonString, err := ioutil.ReadAll(res.Body)
+	if err != nil {
+		return nil, fmt.Errorf("Error while reading the http response: %s", err)
+	}
+
+	utils.Debugf("Ancestry: %s", jsonString)
+	history := new([]string)
+	if err := json.Unmarshal(jsonString, history); err != nil {
+		return nil, err
+	}
+	return *history, nil
+}
+
+// Check if an image exists in the Registry
+func (r *Registry) LookupRemoteImage(imgId, registry string, authConfig *auth.AuthConfig) bool {
+	rt := &http.Transport{Proxy: http.ProxyFromEnvironment}
+
+	req, err := http.NewRequest("GET", registry+"/images/"+imgId+"/json", nil)
+	if err != nil {
+		return false
+	}
+	req.SetBasicAuth(authConfig.Username, authConfig.Password)
+	res, err := rt.RoundTrip(req)
+	return err == nil && res.StatusCode == 307
+}
+
+func (r *Registry) getImagesInRepository(repository string, authConfig *auth.AuthConfig) ([]map[string]string, error) {
+	u := auth.IndexServerAddress() + "/repositories/" + repository + "/images"
+	req, err := http.NewRequest("GET", u, nil)
+	if err != nil {
+		return nil, err
+	}
+	if authConfig != nil && len(authConfig.Username) > 0 {
+		req.SetBasicAuth(authConfig.Username, authConfig.Password)
+	}
+	res, err := r.client.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	defer res.Body.Close()
+
+	// Repository doesn't exist yet
+	if res.StatusCode == 404 {
+		return nil, nil
+	}
+
+	jsonData, err := ioutil.ReadAll(res.Body)
+	if err != nil {
+		return nil, err
+	}
+
+	imageList := []map[string]string{}
+	if err := json.Unmarshal(jsonData, &imageList); err != nil {
+		utils.Debugf("Body: %s (%s)\n", res.Body, u)
+		return nil, err
+	}
+
+	return imageList, nil
+}
+
+// Retrieve an image from the Registry.
+// Returns the Image object as well as the layer as an Archive (io.Reader)
+func (r *Registry) GetRemoteImageJson(imgId, registry string, token []string) ([]byte, error) {
+	// Get the Json
+	req, err := http.NewRequest("GET", registry+"/images/"+imgId+"/json", nil)
+	if err != nil {
+		return nil, fmt.Errorf("Failed to download json: %s", err)
+	}
+	req.Header.Set("Authorization", "Token "+strings.Join(token, ", "))
+	res, err := r.client.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("Failed to download json: %s", err)
+	}
+	defer res.Body.Close()
+	if res.StatusCode != 200 {
+		return nil, fmt.Errorf("HTTP code %d", res.StatusCode)
+	}
+	jsonString, err := ioutil.ReadAll(res.Body)
+	if err != nil {
+		return nil, fmt.Errorf("Failed to parse downloaded json: %s (%s)", err, jsonString)
+	}
+	return jsonString, nil
+}
+
+func (r *Registry) GetRemoteImageLayer(imgId, registry string, token []string) (io.ReadCloser, int, error) {
+	req, err := http.NewRequest("GET", registry+"/images/"+imgId+"/layer", nil)
+	if err != nil {
+		return nil, -1, fmt.Errorf("Error while getting from the server: %s\n", err)
+	}
+	req.Header.Set("Authorization", "Token "+strings.Join(token, ", "))
+	res, err := r.client.Do(req)
+	if err != nil {
+		return nil, -1, err
+	}
+	return res.Body, int(res.ContentLength), nil
+}
+
+func (r *Registry) GetRemoteTags(registries []string, repository string, token []string) (map[string]string, error) {
+	if strings.Count(repository, "/") == 0 {
+		// This will be removed once the Registry supports auto-resolution on
+		// the "library" namespace
+		repository = "library/" + repository
+	}
+	for _, host := range registries {
+		endpoint := fmt.Sprintf("https://%s/v1/repositories/%s/tags", host, repository)
+		req, err := http.NewRequest("GET", endpoint, nil)
+		if err != nil {
+			return nil, err
+		}
+		req.Header.Set("Authorization", "Token "+strings.Join(token, ", "))
+		res, err := r.client.Do(req)
+		defer res.Body.Close()
+		utils.Debugf("Got status code %d from %s", res.StatusCode, endpoint)
+		if err != nil || (res.StatusCode != 200 && res.StatusCode != 404) {
+			continue
+		} else if res.StatusCode == 404 {
+			return nil, fmt.Errorf("Repository not found")
+		}
+
+		result := make(map[string]string)
+
+		rawJson, err := ioutil.ReadAll(res.Body)
+		if err != nil {
+			return nil, err
+		}
+		if err := json.Unmarshal(rawJson, &result); err != nil {
+			return nil, err
+		}
+		return result, nil
+	}
+	return nil, fmt.Errorf("Could not reach any registry endpoint")
+}
+
+func (r *Registry) GetRepositoryData(remote string) (*RepositoryData, error) {
+	utils.Debugf("Pulling repository %s from %s\r\n", remote, auth.IndexServerAddress())
+	repositoryTarget := auth.IndexServerAddress() + "/repositories/" + remote + "/images"
+
+	req, err := http.NewRequest("GET", repositoryTarget, nil)
+	if err != nil {
+		return nil, err
+	}
+	if r.authConfig != nil && len(r.authConfig.Username) > 0 {
+		req.SetBasicAuth(r.authConfig.Username, r.authConfig.Password)
+	}
+	req.Header.Set("X-Docker-Token", "true")
+
+	res, err := r.client.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	defer res.Body.Close()
+	if res.StatusCode == 401 {
+		return nil, fmt.Errorf("Please login first (HTTP code %d)", res.StatusCode)
+	}
+	// TODO: Right now we're ignoring checksums in the response body.
+	// In the future, we need to use them to check image validity.
+	if res.StatusCode != 200 {
+		return nil, fmt.Errorf("HTTP code: %d", res.StatusCode)
+	}
+
+	var tokens []string
+	if res.Header.Get("X-Docker-Token") != "" {
+		tokens = res.Header["X-Docker-Token"]
+	}
+
+	var endpoints []string
+	if res.Header.Get("X-Docker-Endpoints") != "" {
+		endpoints = res.Header["X-Docker-Endpoints"]
+	} else {
+		return nil, fmt.Errorf("Index response didn't contain any endpoints")
+	}
+
+	checksumsJson, err := ioutil.ReadAll(res.Body)
+	if err != nil {
+		return nil, err
+	}
+	remoteChecksums := []*ImgData{}
+	if err := json.Unmarshal(checksumsJson, &remoteChecksums); err != nil {
+		return nil, err
+	}
+
+	// Forge a better object from the retrieved data
+	imgsData := make(map[string]*ImgData)
+	for _, elem := range remoteChecksums {
+		imgsData[elem.Id] = elem
+	}
+
+	return &RepositoryData{
+		ImgList:   imgsData,
+		Endpoints: endpoints,
+		Tokens:    tokens,
+	}, nil
+}
+
+// Push a local image to the registry
+func (r *Registry) PushImageJsonRegistry(imgData *ImgData, jsonRaw []byte, registry string, token []string) error {
+	registry = "https://" + registry + "/v1"
+	// FIXME: try json with UTF8
+	req, err := http.NewRequest("PUT", registry+"/images/"+imgData.Id+"/json", strings.NewReader(string(jsonRaw)))
+	if err != nil {
+		return err
+	}
+	req.Header.Add("Content-type", "application/json")
+	req.Header.Set("Authorization", "Token "+strings.Join(token, ","))
+	req.Header.Set("X-Docker-Checksum", imgData.Checksum)
+
+	utils.Debugf("Setting checksum for %s: %s", imgData.Id, imgData.Checksum)
+	res, err := doWithCookies(r.client, req)
+	if err != nil {
+		return fmt.Errorf("Failed to upload metadata: %s", err)
+	}
+	defer res.Body.Close()
+	if len(res.Cookies()) > 0 {
+		r.client.Jar.SetCookies(req.URL, res.Cookies())
+	}
+	if res.StatusCode != 200 {
+		errBody, err := ioutil.ReadAll(res.Body)
+		if err != nil {
+			return fmt.Errorf("HTTP code %d while uploading metadata and error when trying to parse response body: %s", res.StatusCode, err)
+		}
+		var jsonBody map[string]string
+		if err := json.Unmarshal(errBody, &jsonBody); err != nil {
+			errBody = []byte(err.Error())
+		} else if jsonBody["error"] == "Image already exists" {
+			return ErrAlreadyExists
+		}
+		return fmt.Errorf("HTTP code %d while uploading metadata: %s", res.StatusCode, errBody)
+	}
+	return nil
+}
+
+func (r *Registry) PushImageLayerRegistry(imgId string, layer io.Reader, registry string, token []string) error {
+	registry = "https://" + registry + "/v1"
+	req, err := http.NewRequest("PUT", registry+"/images/"+imgId+"/layer", layer)
+	if err != nil {
+		return err
+	}
+	req.ContentLength = -1
+	req.TransferEncoding = []string{"chunked"}
+	req.Header.Set("Authorization", "Token "+strings.Join(token, ","))
+	res, err := doWithCookies(r.client, req)
+	if err != nil {
+		return fmt.Errorf("Failed to upload layer: %s", err)
+	}
+	defer res.Body.Close()
+
+	if res.StatusCode != 200 {
+		errBody, err := ioutil.ReadAll(res.Body)
+		if err != nil {
+			return fmt.Errorf("HTTP code %d while uploading metadata and error when trying to parse response body: %s", res.StatusCode, err)
+		}
+		return fmt.Errorf("Received HTTP code %d while uploading layer: %s", res.StatusCode, errBody)
+	}
+	return nil
+}
+
+// push a tag on the registry.
+// Remote has the format '<user>/<repo>
+func (r *Registry) PushRegistryTag(remote, revision, tag, registry string, token []string) error {
+	// "jsonify" the string
+	revision = "\"" + revision + "\""
+	registry = "https://" + registry + "/v1"
+
+	req, err := http.NewRequest("PUT", registry+"/repositories/"+remote+"/tags/"+tag, strings.NewReader(revision))
+	if err != nil {
+		return err
+	}
+	req.Header.Add("Content-type", "application/json")
+	req.Header.Set("Authorization", "Token "+strings.Join(token, ","))
+	req.ContentLength = int64(len(revision))
+	res, err := doWithCookies(r.client, req)
+	if err != nil {
+		return err
+	}
+	res.Body.Close()
+	if res.StatusCode != 200 && res.StatusCode != 201 {
+		return fmt.Errorf("Internal server error: %d trying to push tag %s on %s", res.StatusCode, tag, remote)
+	}
+	return nil
+}
+
+func (r *Registry) PushImageJsonIndex(remote string, imgList []*ImgData, validate bool) (*RepositoryData, error) {
+	imgListJson, err := json.Marshal(imgList)
+	if err != nil {
+		return nil, err
+	}
+
+	utils.Debugf("json sent: %s\n", imgListJson)
+
+	req, err := http.NewRequest("PUT", auth.IndexServerAddress()+"/repositories/"+remote+"/", bytes.NewReader(imgListJson))
+	if err != nil {
+		return nil, err
+	}
+	req.SetBasicAuth(r.authConfig.Username, r.authConfig.Password)
+	req.ContentLength = int64(len(imgListJson))
+	req.Header.Set("X-Docker-Token", "true")
+
+	res, err := r.client.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	defer res.Body.Close()
+
+	// Redirect if necessary
+	for res.StatusCode >= 300 && res.StatusCode < 400 {
+		utils.Debugf("Redirected to %s\n", res.Header.Get("Location"))
+		req, err = http.NewRequest("PUT", res.Header.Get("Location"), bytes.NewReader(imgListJson))
+		if err != nil {
+			return nil, err
+		}
+		req.SetBasicAuth(r.authConfig.Username, r.authConfig.Password)
+		req.ContentLength = int64(len(imgListJson))
+		req.Header.Set("X-Docker-Token", "true")
+
+		res, err = r.client.Do(req)
+		if err != nil {
+			return nil, err
+		}
+		defer res.Body.Close()
+	}
+
+	if res.StatusCode != 200 && res.StatusCode != 201 {
+		errBody, err := ioutil.ReadAll(res.Body)
+		if err != nil {
+			return nil, err
+		}
+		return nil, fmt.Errorf("Error: Status %d trying to push repository %s: %s", res.StatusCode, remote, errBody)
+	}
+
+	var tokens []string
+	if res.Header.Get("X-Docker-Token") != "" {
+		tokens = res.Header["X-Docker-Token"]
+		utils.Debugf("Auth token: %v", tokens)
+	} else {
+		return nil, fmt.Errorf("Index response didn't contain an access token")
+	}
+
+	var endpoints []string
+	if res.Header.Get("X-Docker-Endpoints") != "" {
+		endpoints = res.Header["X-Docker-Endpoints"]
+	} else {
+		return nil, fmt.Errorf("Index response didn't contain any endpoints")
+	}
+
+	if validate {
+		if res.StatusCode != 204 {
+			if errBody, err := ioutil.ReadAll(res.Body); err != nil {
+				return nil, err
+			} else {
+				return nil, fmt.Errorf("Error: Status %d trying to push checksums %s: %s", res.StatusCode, remote, errBody)
+			}
+		}
+	}
+
+	return &RepositoryData{
+		Tokens:    tokens,
+		Endpoints: endpoints,
+	}, nil
+}
+
+func (r *Registry) SearchRepositories(term string) (*SearchResults, error) {
+	u := auth.IndexServerAddress() + "/search?q=" + url.QueryEscape(term)
+	req, err := http.NewRequest("GET", u, nil)
+	if err != nil {
+		return nil, err
+	}
+	res, err := r.client.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	defer res.Body.Close()
+	if res.StatusCode != 200 {
+		return nil, fmt.Errorf("Unexepected status code %d", res.StatusCode)
+	}
+	rawData, err := ioutil.ReadAll(res.Body)
+	if err != nil {
+		return nil, err
+	}
+	result := new(SearchResults)
+	err = json.Unmarshal(rawData, result)
+	return result, err
+}
+
+func (r *Registry) ResetClient(authConfig *auth.AuthConfig) {
+	r.authConfig = authConfig
+	r.client.Jar = cookiejar.NewCookieJar()
+}
+
+func (r *Registry) GetAuthConfig() *auth.AuthConfig {
+	return &auth.AuthConfig{
+		Username: r.authConfig.Username,
+		Email:    r.authConfig.Email,
+	}
+}
+
+type SearchResults struct {
+	Query      string              `json:"query"`
+	NumResults int                 `json:"num_results"`
+	Results    []map[string]string `json:"results"`
+}
+
+type RepositoryData struct {
+	ImgList   map[string]*ImgData
+	Endpoints []string
+	Tokens    []string
+}
+
+type ImgData struct {
+	Id       string `json:"id"`
+	Checksum string `json:"checksum,omitempty"`
+	Tag      string `json:",omitempty"`
+}
+
+type Registry struct {
+	client     *http.Client
+	authConfig *auth.AuthConfig
+}
+
+func NewRegistry(root string) *Registry {
+	// If the auth file does not exist, keep going
+	authConfig, _ := auth.LoadConfig(root)
+
+	r := &Registry{
+		authConfig: authConfig,
+		client:     &http.Client{},
+	}
+	r.client.Jar = cookiejar.NewCookieJar()
+	return r
+}

registry/registry_test.go

@@ -0,0 +1,168 @@
+package registry
+
+// import (
+// 	"crypto/rand"
+// 	"encoding/hex"
+// 	"github.com/dotcloud/docker"
+// 	"github.com/dotcloud/docker/auth"
+// 	"io/ioutil"
+// 	"os"
+// 	"path"
+// 	"testing"
+// )
+
+// func newTestRuntime() (*Runtime, error) {
+// 	root, err := ioutil.TempDir("", "docker-test")
+// 	if err != nil {
+// 		return nil, err
+// 	}
+// 	if err := os.Remove(root); err != nil {
+// 		return nil, err
+// 	}
+
+// 	if err := os.MkdirAll(root, 0700); err != nil && !os.IsExist(err) {
+// 		return nil, err
+// 	}
+
+// 	return runtime, nil
+// }
+
+// func TestPull(t *testing.T) {
+// 	os.Setenv("DOCKER_INDEX_URL", "")
+// 	runtime, err := newTestRuntime()
+// 	if err != nil {
+// 		t.Fatal(err)
+// 	}
+// 	defer nuke(runtime)
+
+// 	err = runtime.graph.PullRepository(ioutil.Discard, "busybox", "", runtime.repositories, nil)
+// 	if err != nil {
+// 		t.Fatal(err)
+// 	}
+// 	img, err := runtime.repositories.LookupImage("busybox")
+// 	if err != nil {
+// 		t.Fatal(err)
+// 	}
+
+// 	// Try to run something on this image to make sure the layer's been downloaded properly.
+// 	config, _, err := docker.ParseRun([]string{img.Id, "echo", "Hello World"}, runtime.capabilities)
+// 	if err != nil {
+// 		t.Fatal(err)
+// 	}
+
+// 	b := NewBuilder(runtime)
+// 	container, err := b.Create(config)
+// 	if err != nil {
+// 		t.Fatal(err)
+// 	}
+// 	if err := container.Start(); err != nil {
+// 		t.Fatal(err)
+// 	}
+
+// 	if status := container.Wait(); status != 0 {
+// 		t.Fatalf("Expected status code 0, found %d instead", status)
+// 	}
+// }
+
+// func TestPullTag(t *testing.T) {
+// 	os.Setenv("DOCKER_INDEX_URL", "")
+// 	runtime, err := newTestRuntime()
+// 	if err != nil {
+// 		t.Fatal(err)
+// 	}
+// 	defer nuke(runtime)
+
+// 	err = runtime.graph.PullRepository(ioutil.Discard, "ubuntu", "12.04", runtime.repositories, nil)
+// 	if err != nil {
+// 		t.Fatal(err)
+// 	}
+// 	_, err = runtime.repositories.LookupImage("ubuntu:12.04")
+// 	if err != nil {
+// 		t.Fatal(err)
+// 	}
+
+// 	img2, err := runtime.repositories.LookupImage("ubuntu:12.10")
+// 	if img2 != nil {
+// 		t.Fatalf("Expected nil image but found %v instead", img2.Id)
+// 	}
+// }
+
+// func login(runtime *Runtime) error {
+// 	authConfig := auth.NewAuthConfig("unittester", "surlautrerivejetattendrai", "noise+unittester@dotcloud.com", runtime.root)
+// 	runtime.authConfig = authConfig
+// 	_, err := auth.Login(authConfig)
+// 	return err
+// }
+
+// func TestPush(t *testing.T) {
+// 	os.Setenv("DOCKER_INDEX_URL", "https://indexstaging-docker.dotcloud.com")
+// 	defer os.Setenv("DOCKER_INDEX_URL", "")
+// 	runtime, err := newTestRuntime()
+// 	if err != nil {
+// 		t.Fatal(err)
+// 	}
+// 	defer nuke(runtime)
+
+// 	err = login(runtime)
+// 	if err != nil {
+// 		t.Fatal(err)
+// 	}
+
+// 	err = runtime.graph.PullRepository(ioutil.Discard, "joffrey/busybox", "", runtime.repositories, nil)
+// 	if err != nil {
+// 		t.Fatal(err)
+// 	}
+// 	tokenBuffer := make([]byte, 16)
+// 	_, err = rand.Read(tokenBuffer)
+// 	if err != nil {
+// 		t.Fatal(err)
+// 	}
+// 	token := hex.EncodeToString(tokenBuffer)[:29]
+// 	config, _, err := ParseRun([]string{"joffrey/busybox", "touch", "/" + token}, runtime.capabilities)
+// 	if err != nil {
+// 		t.Fatal(err)
+// 	}
+
+// 	b := NewBuilder(runtime)
+// 	container, err := b.Create(config)
+// 	if err != nil {
+// 		t.Fatal(err)
+// 	}
+// 	if err := container.Start(); err != nil {
+// 		t.Fatal(err)
+// 	}
+
+// 	if status := container.Wait(); status != 0 {
+// 		t.Fatalf("Expected status code 0, found %d instead", status)
+// 	}
+
+// 	img, err := b.Commit(container, "unittester/"+token, "", "", "", nil)
+// 	if err != nil {
+// 		t.Fatal(err)
+// 	}
+
+// 	repo := runtime.repositories.Repositories["unittester/"+token]
+// 	err = runtime.graph.PushRepository(ioutil.Discard, "unittester/"+token, repo, runtime.authConfig)
+// 	if err != nil {
+// 		t.Fatal(err)
+// 	}
+
+// 	// Remove image so we can pull it again
+// 	if err := runtime.graph.Delete(img.Id); err != nil {
+// 		t.Fatal(err)
+// 	}
+
+// 	err = runtime.graph.PullRepository(ioutil.Discard, "unittester/"+token, "", runtime.repositories, runtime.authConfig)
+// 	if err != nil {
+// 		t.Fatal(err)
+// 	}
+
+// 	layerPath, err := img.layer()
+// 	if err != nil {
+// 		t.Fatal(err)
+// 	}
+
+// 	if _, err := os.Stat(path.Join(layerPath, token)); err != nil {
+// 		t.Fatalf("Error while trying to retrieve token file: %v", err)
+// 	}
+// }

runtime.go

@@ -3,7 +3,7 @@ package docker
 import (
 	"container/list"
 	"fmt"
-	"github.com/dotcloud/docker/auth"
+	"github.com/dotcloud/docker/utils"
 	"io"
 	"io/ioutil"
 	"log"
@@ -26,18 +26,18 @@ type Runtime struct {
 	networkManager *NetworkManager
 	graph          *Graph
 	repositories   *TagStore
-	authConfig     *auth.AuthConfig
-	idIndex        *TruncIndex
+	idIndex        *utils.TruncIndex
 	capabilities   *Capabilities
-	kernelVersion  *KernelVersionInfo
+	kernelVersion  *utils.KernelVersionInfo
 	autoRestart    bool
 	volumes        *Graph
+	srv            *Server
 }
 
 var sysInitPath string
 
 func init() {
-	sysInitPath = SelfPath()
+	sysInitPath = utils.SelfPath()
 }
 
 func (runtime *Runtime) List() []*Container {
@@ -113,13 +113,13 @@ func (runtime *Runtime) Register(container *Container) error {
 	container.runtime = runtime
 
 	// Attach to stdout and stderr
-	container.stderr = newWriteBroadcaster()
-	container.stdout = newWriteBroadcaster()
+	container.stderr = utils.NewWriteBroadcaster()
+	container.stdout = utils.NewWriteBroadcaster()
 	// Attach to stdin
 	if container.Config.OpenStdin {
 		container.stdin, container.stdinPipe = io.Pipe()
 	} else {
-		container.stdinPipe = NopWriteCloser(ioutil.Discard) // Silently drop stdin
+		container.stdinPipe = utils.NopWriteCloser(ioutil.Discard) // Silently drop stdin
 	}
 	// done
 	runtime.containers.PushBack(container)
@@ -137,9 +137,9 @@ func (runtime *Runtime) Register(container *Container) error {
 			return err
 		} else {
 			if !strings.Contains(string(output), "RUNNING") {
-				Debugf("Container %s was supposed to be running be is not.", container.Id)
+				utils.Debugf("Container %s was supposed to be running be is not.", container.Id)
 				if runtime.autoRestart {
-					Debugf("Restarting")
+					utils.Debugf("Restarting")
 					container.State.Ghost = false
 					container.State.setStopped(0)
 					if err := container.Start(); err != nil {
@@ -147,7 +147,7 @@ func (runtime *Runtime) Register(container *Container) error {
 					}
 					nomonitor = true
 				} else {
-					Debugf("Marking as stopped")
+					utils.Debugf("Marking as stopped")
 					container.State.setStopped(-127)
 					if err := container.ToDisk(); err != nil {
 						return err
@@ -168,7 +168,7 @@ func (runtime *Runtime) Register(container *Container) error {
 	return nil
 }
 
-func (runtime *Runtime) LogToDisk(src *writeBroadcaster, dst string) error {
+func (runtime *Runtime) LogToDisk(src *utils.WriteBroadcaster, dst string) error {
 	log, err := os.OpenFile(dst, os.O_RDWR|os.O_APPEND|os.O_CREATE, 0600)
 	if err != nil {
 		return err
@@ -215,16 +215,16 @@ func (runtime *Runtime) restore() error {
 		id := v.Name()
 		container, err := runtime.Load(id)
 		if err != nil {
-			Debugf("Failed to load container %v: %v", id, err)
+			utils.Debugf("Failed to load container %v: %v", id, err)
 			continue
 		}
-		Debugf("Loaded container %v", container.Id)
+		utils.Debugf("Loaded container %v", container.Id)
 	}
 	return nil
 }
 
 func (runtime *Runtime) UpdateCapabilities(quiet bool) {
-	if cgroupMemoryMountpoint, err := FindCgroupMountpoint("memory"); err != nil {
+	if cgroupMemoryMountpoint, err := utils.FindCgroupMountpoint("memory"); err != nil {
 		if !quiet {
 			log.Printf("WARNING: %s\n", err)
 		}
@@ -251,11 +251,11 @@ func NewRuntime(autoRestart bool) (*Runtime, error) {
 		return nil, err
 	}
 
-	if k, err := GetKernelVersion(); err != nil {
+	if k, err := utils.GetKernelVersion(); err != nil {
 		log.Printf("WARNING: %s\n", err)
 	} else {
 		runtime.kernelVersion = k
-		if CompareKernelVersion(k, &KernelVersionInfo{Kernel: 3, Major: 8, Minor: 0}) < 0 {
+		if utils.CompareKernelVersion(k, &utils.KernelVersionInfo{Kernel: 3, Major: 8, Minor: 0}) < 0 {
 			log.Printf("WARNING: You are running linux kernel version %s, which might be unstable running docker. Please upgrade your kernel to 3.8.0.", k.String())
 		}
 	}
@@ -289,11 +289,6 @@ func NewRuntimeFromDirectory(root string, autoRestart bool) (*Runtime, error) {
 	if err != nil {
 		return nil, err
 	}
-	authConfig, err := auth.LoadConfig(root)
-	if err != nil && authConfig == nil {
-		// If the auth file does not exist, keep going
-		return nil, err
-	}
 	runtime := &Runtime{
 		root:           root,
 		repository:     runtimeRepo,
@@ -301,8 +296,7 @@ func NewRuntimeFromDirectory(root string, autoRestart bool) (*Runtime, error) {
 		networkManager: netManager,
 		graph:          g,
 		repositories:   repositories,
-		authConfig:     authConfig,
-		idIndex:        NewTruncIndex(),
+		idIndex:        utils.NewTruncIndex(),
 		capabilities:   &Capabilities{},
 		autoRestart:    autoRestart,
 		volumes:        volumes,

runtime_test.go

@@ -2,6 +2,8 @@ package docker
 
 import (
 	"fmt"
+	"github.com/dotcloud/docker/registry"
+	"github.com/dotcloud/docker/utils"
 	"io"
 	"io/ioutil"
 	"net"
@@ -48,7 +50,7 @@ func layerArchive(tarfile string) (io.Reader, error) {
 
 func init() {
 	// Hack to run sys init during unit testing
-	if SelfPath() == "/sbin/init" {
+	if utils.SelfPath() == "/sbin/init" {
 		SysInit()
 		return
 	}
@@ -69,7 +71,8 @@ func init() {
 
 	// Create the "Server"
 	srv := &Server{
-		runtime: runtime,
+		runtime:  runtime,
+		registry: registry.NewRegistry(runtime.root),
 	}
 	// Retrieve the Image
 	if err := srv.ImagePull(unitTestImageName, "", "", os.Stdout); err != nil {

server.go

@@ -2,11 +2,15 @@ package docker
 
 import (
 	"fmt"
+	"github.com/dotcloud/docker/registry"
+	"github.com/dotcloud/docker/utils"
 	"io"
+	"io/ioutil"
 	"log"
 	"net/http"
 	"net/url"
 	"os"
+	"path"
 	"runtime"
 	"strings"
 )
@@ -44,7 +48,7 @@ func (srv *Server) ContainerExport(name string, out io.Writer) error {
 }
 
 func (srv *Server) ImagesSearch(term string) ([]ApiSearch, error) {
-	results, err := srv.runtime.graph.SearchRepositories(nil, term)
+	results, err := srv.registry.SearchRepositories(term)
 	if err != nil {
 		return nil, err
 	}
@@ -54,7 +58,7 @@ func (srv *Server) ImagesSearch(term string) ([]ApiSearch, error) {
 		var out ApiSearch
 		out.Description = repo["description"]
 		if len(out.Description) > 45 {
-			out.Description = Trunc(out.Description, 42) + "..."
+			out.Description = utils.Trunc(out.Description, 42) + "..."
 		}
 		out.Name = repo["name"]
 		outs = append(outs, out)
@@ -68,7 +72,7 @@ func (srv *Server) ImageInsert(name, url, path string, out io.Writer) error {
 		return err
 	}
 
-	file, err := Download(url, out)
+	file, err := utils.Download(url, out)
 	if err != nil {
 		return err
 	}
@@ -85,7 +89,7 @@ func (srv *Server) ImageInsert(name, url, path string, out io.Writer) error {
 		return err
 	}
 
-	if err := c.Inject(ProgressReader(file.Body, int(file.ContentLength), out, "Downloading %v/%v (%v)"), path); err != nil {
+	if err := c.Inject(utils.ProgressReader(file.Body, int(file.ContentLength), out, "Downloading %v/%v (%v)"), path); err != nil {
 		return err
 	}
 	// FIXME: Handle custom repo, tag comment, author
@@ -124,7 +128,7 @@ func (srv *Server) ImagesViz(out io.Writer) error {
 
 	for name, repository := range srv.runtime.repositories.Repositories {
 		for tag, id := range repository {
-			reporefs[TruncateId(id)] = append(reporefs[TruncateId(id)], fmt.Sprintf("%s:%s", name, tag))
+			reporefs[utils.TruncateId(id)] = append(reporefs[utils.TruncateId(id)], fmt.Sprintf("%s:%s", name, tag))
 		}
 	}
 
@@ -193,7 +197,7 @@ func (srv *Server) DockerInfo() ApiInfo {
 	out.GoVersion = runtime.Version()
 	if os.Getenv("DEBUG") != "" {
 		out.Debug = true
-		out.NFd = getTotalUsedFds()
+		out.NFd = utils.GetTotalUsedFds()
 		out.NGoroutines = runtime.NumGoroutine()
 	}
 	return out
@@ -283,14 +287,272 @@ func (srv *Server) ContainerTag(name, repo, tag string, force bool) error {
 	return nil
 }
 
+func (srv *Server) pullImage(out io.Writer, imgId, registry string, token []string) error {
+	history, err := srv.registry.GetRemoteHistory(imgId, registry, token)
+	if err != nil {
+		return err
+	}
+
+	// FIXME: Try to stream the images?
+	// FIXME: Launch the getRemoteImage() in goroutines
+	for _, id := range history {
+		if !srv.runtime.graph.Exists(id) {
+			fmt.Fprintf(out, "Pulling %s metadata\r\n", id)
+			imgJson, err := srv.registry.GetRemoteImageJson(id, registry, token)
+			if err != nil {
+				// FIXME: Keep goging in case of error?
+				return err
+			}
+			img, err := NewImgJson(imgJson)
+			if err != nil {
+				return fmt.Errorf("Failed to parse json: %s", err)
+			}
+
+			// Get the layer
+			fmt.Fprintf(out, "Pulling %s fs layer\r\n", img.Id)
+			layer, contentLength, err := srv.registry.GetRemoteImageLayer(img.Id, registry, token)
+			if err != nil {
+				return err
+			}
+			if err := srv.runtime.graph.Register(utils.ProgressReader(layer, contentLength, out, "Downloading %v/%v (%v)"), false, img); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+func (srv *Server) pullRepository(stdout io.Writer, remote, askedTag string) error {
+	utils.Debugf("Retrieving repository data")
+	repoData, err := srv.registry.GetRepositoryData(remote)
+	if err != nil {
+		return err
+	}
+
+	utils.Debugf("Updating checksums")
+	// Reload the json file to make sure not to overwrite faster sums
+	if err := srv.runtime.graph.UpdateChecksums(repoData.ImgList); err != nil {
+		return err
+	}
+
+	utils.Debugf("Retrieving the tag list")
+	tagsList, err := srv.registry.GetRemoteTags(repoData.Endpoints, remote, repoData.Tokens)
+	if err != nil {
+		return err
+	}
+	for tag, id := range tagsList {
+		repoData.ImgList[id].Tag = tag
+	}
+
+	for _, img := range repoData.ImgList {
+		// If we asked for a specific tag, skip all tags expect the wanted one
+		if askedTag != "" && askedTag != img.Tag {
+			continue
+		}
+		fmt.Fprintf(stdout, "Pulling image %s (%s) from %s\n", img.Id, img.Tag, remote)
+		success := false
+		for _, ep := range repoData.Endpoints {
+			if err := srv.pullImage(stdout, img.Id, "https://"+ep+"/v1", repoData.Tokens); err != nil {
+				fmt.Fprintf(stdout, "Error while retrieving image for tag: %s (%s); checking next endpoint\n", askedTag, err)
+				continue
+			}
+			if err := srv.runtime.repositories.Set(remote, img.Tag, img.Id, true); err != nil {
+				return err
+			}
+			success = true
+			delete(tagsList, img.Tag)
+			break
+		}
+		if !success {
+			return fmt.Errorf("Could not find repository on any of the indexed registries.")
+		}
+	}
+	for tag, id := range tagsList {
+		if err := srv.runtime.repositories.Set(remote, tag, id, true); err != nil {
+			return err
+		}
+	}
+	if err := srv.runtime.repositories.Save(); err != nil {
+		return err
+	}
+
+	return nil
+}
+
 func (srv *Server) ImagePull(name, tag, registry string, out io.Writer) error {
 	if registry != "" {
-		if err := srv.runtime.graph.PullImage(out, name, registry, nil); err != nil {
+		if err := srv.pullImage(out, name, registry, nil); err != nil {
 			return err
 		}
 		return nil
 	}
-	if err := srv.runtime.graph.PullRepository(out, name, tag, srv.runtime.repositories, srv.runtime.authConfig); err != nil {
+
+	if err := srv.pullRepository(out, name, tag); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// Retrieve the checksum of an image
+// Priority:
+// - Check on the stored checksums
+// - Check if the archive exists, if it does not, ask the registry
+// - If the archive does exists, process the checksum from it
+// - If the archive does not exists and not found on registry, process checksum from layer
+func (srv *Server) getChecksum(imageId string) (string, error) {
+	// FIXME: Use in-memory map instead of reading the file each time
+	if sums, err := srv.runtime.graph.getStoredChecksums(); err != nil {
+		return "", err
+	} else if checksum, exists := sums[imageId]; exists {
+		return checksum, nil
+	}
+
+	img, err := srv.runtime.graph.Get(imageId)
+	if err != nil {
+		return "", err
+	}
+
+	if _, err := os.Stat(layerArchivePath(srv.runtime.graph.imageRoot(imageId))); err != nil {
+		if os.IsNotExist(err) {
+			// TODO: Ask the registry for the checksum
+			//       As the archive is not there, it is supposed to come from a pull.
+		} else {
+			return "", err
+		}
+	}
+
+	checksum, err := img.Checksum()
+	if err != nil {
+		return "", err
+	}
+	return checksum, nil
+}
+
+// Retrieve the all the images to be uploaded in the correct order
+// Note: we can't use a map as it is not ordered
+func (srv *Server) getImageList(localRepo map[string]string) ([]*registry.ImgData, error) {
+	var imgList []*registry.ImgData
+
+	imageSet := make(map[string]struct{})
+	for tag, id := range localRepo {
+		img, err := srv.runtime.graph.Get(id)
+		if err != nil {
+			return nil, err
+		}
+		img.WalkHistory(func(img *Image) error {
+			if _, exists := imageSet[img.Id]; exists {
+				return nil
+			}
+			imageSet[img.Id] = struct{}{}
+			checksum, err := srv.getChecksum(img.Id)
+			if err != nil {
+				return err
+			}
+			imgList = append([]*registry.ImgData{{
+				Id:       img.Id,
+				Checksum: checksum,
+				Tag:      tag,
+			}}, imgList...)
+			return nil
+		})
+	}
+	return imgList, nil
+}
+
+func (srv *Server) pushRepository(out io.Writer, name string, localRepo map[string]string) error {
+	fmt.Fprintf(out, "Processing checksums\n")
+	imgList, err := srv.getImageList(localRepo)
+	if err != nil {
+		return err
+	}
+	fmt.Fprintf(out, "Sending image list\n")
+
+	repoData, err := srv.registry.PushImageJsonIndex(name, imgList, false)
+	if err != nil {
+		return err
+	}
+
+	// FIXME: Send only needed images
+	for _, ep := range repoData.Endpoints {
+		fmt.Fprintf(out, "Pushing repository %s to %s (%d tags)\r\n", name, ep, len(localRepo))
+		// For each image within the repo, push them
+		for _, elem := range imgList {
+			if _, exists := repoData.ImgList[elem.Id]; exists {
+				fmt.Fprintf(out, "Image %s already on registry, skipping\n", name)
+				continue
+			}
+			if err := srv.pushImage(out, name, elem.Id, ep, repoData.Tokens); err != nil {
+				// FIXME: Continue on error?
+				return err
+			}
+			fmt.Fprintf(out, "Pushing tags for rev [%s] on {%s}\n", elem.Id, ep+"/users/"+name+"/"+elem.Tag)
+			if err := srv.registry.PushRegistryTag(name, elem.Id, elem.Tag, ep, repoData.Tokens); err != nil {
+				return err
+			}
+		}
+	}
+
+	if _, err := srv.registry.PushImageJsonIndex(name, imgList, true); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (srv *Server) pushImage(out io.Writer, remote, imgId, ep string, token []string) error {
+	jsonRaw, err := ioutil.ReadFile(path.Join(srv.runtime.graph.Root, imgId, "json"))
+	if err != nil {
+		return fmt.Errorf("Error while retreiving the path for {%s}: %s", imgId, err)
+	}
+	fmt.Fprintf(out, "Pushing %s\r\n", imgId)
+
+	// Make sure we have the image's checksum
+	checksum, err := srv.getChecksum(imgId)
+	if err != nil {
+		return err
+	}
+	imgData := &registry.ImgData{
+		Id:       imgId,
+		Checksum: checksum,
+	}
+
+	// Send the json
+	if err := srv.registry.PushImageJsonRegistry(imgData, jsonRaw, ep, token); err != nil {
+		if err == registry.ErrAlreadyExists {
+			fmt.Fprintf(out, "Image %s already uploaded ; skipping\n", imgData.Id)
+			return nil
+		}
+		return err
+	}
+
+	// Retrieve the tarball to be sent
+	var layerData *TempArchive
+	// If the archive exists, use it
+	file, err := os.Open(layerArchivePath(srv.runtime.graph.imageRoot(imgId)))
+	if err != nil {
+		if os.IsNotExist(err) {
+			// If the archive does not exist, create one from the layer
+			layerData, err = srv.runtime.graph.TempLayerArchive(imgId, Xz, out)
+			if err != nil {
+				return fmt.Errorf("Failed to generate layer archive: %s", err)
+			}
+		} else {
+			return err
+		}
+	} else {
+		defer file.Close()
+		st, err := file.Stat()
+		if err != nil {
+			return err
+		}
+		layerData = &TempArchive{
+			File: file,
+			Size: st.Size(),
+		}
+	}
+
+	// Send the layer
+	if err := srv.registry.PushImageLayerRegistry(imgData.Id, utils.ProgressReader(layerData, int(layerData.Size), out, ""), ep, token); err != nil {
 		return err
 	}
 	return nil
@@ -299,10 +561,10 @@ func (srv *Server) ImagePull(name, tag, registry string, out io.Writer) error {
 func (srv *Server) ImagePush(name, registry string, out io.Writer) error {
 	img, err := srv.runtime.graph.Get(name)
 	if err != nil {
-		Debugf("The push refers to a repository [%s] (len: %d)\n", name, len(srv.runtime.repositories.Repositories[name]))
+		fmt.Fprintf(out, "The push refers to a repository [%s] (len: %d)\n", name, len(srv.runtime.repositories.Repositories[name]))
 		// If it fails, try to get the repository
 		if localRepo, exists := srv.runtime.repositories.Repositories[name]; exists {
-			if err := srv.runtime.graph.PushRepository(out, name, localRepo, srv.runtime.authConfig); err != nil {
+			if err := srv.pushRepository(out, name, localRepo); err != nil {
 				return err
 			}
 			return nil
@@ -310,8 +572,8 @@ func (srv *Server) ImagePush(name, registry string, out io.Writer) error {
 
 		return err
 	}
-	err = srv.runtime.graph.PushImage(out, img, registry, nil)
-	if err != nil {
+	fmt.Fprintf(out, "The push refers to an image: [%s]\n", name)
+	if err := srv.pushImage(out, name, img.Id, registry, nil); err != nil {
 		return err
 	}
 	return nil
@@ -336,11 +598,11 @@ func (srv *Server) ImageImport(src, repo, tag string, in io.Reader, out io.Write
 		fmt.Fprintln(out, "Downloading from", u)
 		// Download with curl (pretty progress bar)
 		// If curl is not available, fallback to http.Get()
-		resp, err = Download(u.String(), out)
+		resp, err = utils.Download(u.String(), out)
 		if err != nil {
 			return err
 		}
-		archive = ProgressReader(resp.Body, int(resp.ContentLength), out, "Importing %v/%v (%v)")
+		archive = utils.ProgressReader(resp.Body, int(resp.ContentLength), out, "Importing %v/%v (%v)")
 	}
 	img, err := srv.runtime.graph.Create(archive, nil, "Imported from "+src, "", nil)
 	if err != nil {
@@ -397,7 +659,6 @@ func (srv *Server) ContainerRestart(name string, t int) error {
 }
 
 func (srv *Server) ContainerDestroy(name string, removeVolume bool) error {
-
 	if container := srv.runtime.Get(name); container != nil {
 		volumes := make(map[string]struct{})
 		// Store all the deleted containers volumes
@@ -486,17 +747,17 @@ func (srv *Server) ContainerAttach(name string, logs, stream, stdin, stdout, std
 		if stdout {
 			cLog, err := container.ReadLog("stdout")
 			if err != nil {
-				Debugf(err.Error())
+				utils.Debugf(err.Error())
 			} else if _, err := io.Copy(out, cLog); err != nil {
-				Debugf(err.Error())
+				utils.Debugf(err.Error())
 			}
 		}
 		if stderr {
 			cLog, err := container.ReadLog("stderr")
 			if err != nil {
-				Debugf(err.Error())
+				utils.Debugf(err.Error())
 			} else if _, err := io.Copy(out, cLog); err != nil {
-				Debugf(err.Error())
+				utils.Debugf(err.Error())
 			}
 		}
 	}
@@ -517,7 +778,7 @@ func (srv *Server) ContainerAttach(name string, logs, stream, stdin, stdout, std
 			r, w := io.Pipe()
 			go func() {
 				defer w.Close()
-				defer Debugf("Closing buffered stdin pipe")
+				defer utils.Debugf("Closing buffered stdin pipe")
 				io.Copy(w, in)
 			}()
 			cStdin = r
@@ -564,11 +825,14 @@ func NewServer(autoRestart bool) (*Server, error) {
 		return nil, err
 	}
 	srv := &Server{
-		runtime: runtime,
+		runtime:  runtime,
+		registry: registry.NewRegistry(runtime.root),
 	}
+	runtime.srv = srv
 	return srv, nil
 }
 
 type Server struct {
-	runtime *Runtime
+	runtime  *Runtime
+	registry *registry.Registry
 }

state.go

@@ -2,6 +2,7 @@ package docker
 
 import (
 	"fmt"
+	"github.com/dotcloud/docker/utils"
 	"sync"
 	"time"
 )
@@ -21,7 +22,7 @@ func (s *State) String() string {
 		if s.Ghost {
 			return fmt.Sprintf("Ghost")
 		}
-		return fmt.Sprintf("Up %s", HumanDuration(time.Now().Sub(s.StartedAt)))
+		return fmt.Sprintf("Up %s", utils.HumanDuration(time.Now().Sub(s.StartedAt)))
 	}
 	return fmt.Sprintf("Exit %d", s.ExitCode)
 }

tags.go

@@ -3,6 +3,7 @@ package docker
 import (
 	"encoding/json"
 	"fmt"
+	"github.com/dotcloud/docker/utils"
 	"io/ioutil"
 	"os"
 	"path/filepath"
@@ -106,7 +107,7 @@ func (store *TagStore) ImageName(id string) string {
 	if names, exists := store.ById()[id]; exists && len(names) > 0 {
 		return names[0]
 	}
-	return TruncateId(id)
+	return utils.TruncateId(id)
 }
 
 func (store *TagStore) Set(repoName, tag, imageName string, force bool) error {

term/term.go

@@ -1,6 +1,8 @@
 package term
 
 import (
+	"os"
+	"os/signal"
 	"syscall"
 	"unsafe"
 )
@@ -120,3 +122,22 @@ func Restore(fd int, state *State) error {
 	_, _, err := syscall.Syscall6(syscall.SYS_IOCTL, uintptr(fd), uintptr(setTermios), uintptr(unsafe.Pointer(&state.termios)), 0, 0, 0)
 	return err
 }
+
+func SetRawTerminal() (*State, error) {
+	oldState, err := MakeRaw(int(os.Stdin.Fd()))
+	if err != nil {
+		return nil, err
+	}
+	c := make(chan os.Signal, 1)
+	signal.Notify(c, os.Interrupt)
+	go func() {
+		_ = <-c
+		Restore(int(os.Stdin.Fd()), oldState)
+		os.Exit(0)
+	}()
+	return oldState, err
+}
+
+func RestoreTerminal(state *State) {
+	Restore(int(os.Stdin.Fd()), state)
+}

utils.go

@@ -1,500 +1,5 @@
 package docker
 
-import (
-	"bytes"
-	"crypto/sha256"
-	"encoding/hex"
-	"errors"
-	"fmt"
-	"github.com/dotcloud/docker/term"
-	"index/suffixarray"
-	"io"
-	"io/ioutil"
-	"net/http"
-	"os"
-	"os/exec"
-	"os/signal"
-	"path/filepath"
-	"runtime"
-	"strings"
-	"sync"
-	"time"
-)
-
-// Go is a basic promise implementation: it wraps calls a function in a goroutine,
-// and returns a channel which will later return the function's return value.
-func Go(f func() error) chan error {
-	ch := make(chan error)
-	go func() {
-		ch <- f()
-	}()
-	return ch
-}
-
-// Request a given URL and return an io.Reader
-func Download(url string, stderr io.Writer) (*http.Response, error) {
-	var resp *http.Response
-	var err error = nil
-	if resp, err = http.Get(url); err != nil {
-		return nil, err
-	}
-	if resp.StatusCode >= 400 {
-		return nil, errors.New("Got HTTP status code >= 400: " + resp.Status)
-	}
-	return resp, nil
-}
-
-// Debug function, if the debug flag is set, then display. Do nothing otherwise
-// If Docker is in damon mode, also send the debug info on the socket
-func Debugf(format string, a ...interface{}) {
-	if os.Getenv("DEBUG") != "" {
-
-		// Retrieve the stack infos
-		_, file, line, ok := runtime.Caller(1)
-		if !ok {
-			file = "<unknown>"
-			line = -1
-		} else {
-			file = file[strings.LastIndex(file, "/")+1:]
-		}
-
-		fmt.Fprintf(os.Stderr, fmt.Sprintf("[debug] %s:%d %s\n", file, line, format), a...)
-	}
-}
-
-// Reader with progress bar
-type progressReader struct {
-	reader       io.ReadCloser // Stream to read from
-	output       io.Writer     // Where to send progress bar to
-	readTotal    int           // Expected stream length (bytes)
-	readProgress int           // How much has been read so far (bytes)
-	lastUpdate   int           // How many bytes read at least update
-	template     string        // Template to print. Default "%v/%v (%v)"
-}
-
-func (r *progressReader) Read(p []byte) (n int, err error) {
-	read, err := io.ReadCloser(r.reader).Read(p)
-	r.readProgress += read
-
-	updateEvery := 4096
-	if r.readTotal > 0 {
-		// Only update progress for every 1% read
-		if increment := int(0.01 * float64(r.readTotal)); increment > updateEvery {
-			updateEvery = increment
-		}
-	}
-	if r.readProgress-r.lastUpdate > updateEvery || err != nil {
-		if r.readTotal > 0 {
-			fmt.Fprintf(r.output, r.template+"\r", r.readProgress, r.readTotal, fmt.Sprintf("%.0f%%", float64(r.readProgress)/float64(r.readTotal)*100))
-		} else {
-			fmt.Fprintf(r.output, r.template+"\r", r.readProgress, "?", "n/a")
-		}
-		r.lastUpdate = r.readProgress
-	}
-	// Send newline when complete
-	if err != nil {
-		fmt.Fprintf(r.output, "\n")
-	}
-
-	return read, err
-}
-func (r *progressReader) Close() error {
-	return io.ReadCloser(r.reader).Close()
-}
-func ProgressReader(r io.ReadCloser, size int, output io.Writer, template string) *progressReader {
-	if template == "" {
-		template = "%v/%v (%v)"
-	}
-	return &progressReader{r, output, size, 0, 0, template}
-}
-
-// HumanDuration returns a human-readable approximation of a duration
-// (eg. "About a minute", "4 hours ago", etc.)
-func HumanDuration(d time.Duration) string {
-	if seconds := int(d.Seconds()); seconds < 1 {
-		return "Less than a second"
-	} else if seconds < 60 {
-		return fmt.Sprintf("%d seconds", seconds)
-	} else if minutes := int(d.Minutes()); minutes == 1 {
-		return "About a minute"
-	} else if minutes < 60 {
-		return fmt.Sprintf("%d minutes", minutes)
-	} else if hours := int(d.Hours()); hours == 1 {
-		return "About an hour"
-	} else if hours < 48 {
-		return fmt.Sprintf("%d hours", hours)
-	} else if hours < 24*7*2 {
-		return fmt.Sprintf("%d days", hours/24)
-	} else if hours < 24*30*3 {
-		return fmt.Sprintf("%d weeks", hours/24/7)
-	} else if hours < 24*365*2 {
-		return fmt.Sprintf("%d months", hours/24/30)
-	}
-	return fmt.Sprintf("%d years", d.Hours()/24/365)
-}
-
-func Trunc(s string, maxlen int) string {
-	if len(s) <= maxlen {
-		return s
-	}
-	return s[:maxlen]
-}
-
-// Figure out the absolute path of our own binary
-func SelfPath() string {
-	path, err := exec.LookPath(os.Args[0])
-	if err != nil {
-		panic(err)
-	}
-	path, err = filepath.Abs(path)
-	if err != nil {
-		panic(err)
-	}
-	return path
-}
-
-type nopWriter struct {
-}
-
-func (w *nopWriter) Write(buf []byte) (int, error) {
-	return len(buf), nil
-}
-
-type nopWriteCloser struct {
-	io.Writer
-}
-
-func (w *nopWriteCloser) Close() error { return nil }
-
-func NopWriteCloser(w io.Writer) io.WriteCloser {
-	return &nopWriteCloser{w}
-}
-
-type bufReader struct {
-	buf    *bytes.Buffer
-	reader io.Reader
-	err    error
-	l      sync.Mutex
-	wait   sync.Cond
-}
-
-func newBufReader(r io.Reader) *bufReader {
-	reader := &bufReader{
-		buf:    &bytes.Buffer{},
-		reader: r,
-	}
-	reader.wait.L = &reader.l
-	go reader.drain()
-	return reader
-}
-
-func (r *bufReader) drain() {
-	buf := make([]byte, 1024)
-	for {
-		n, err := r.reader.Read(buf)
-		r.l.Lock()
-		if err != nil {
-			r.err = err
-		} else {
-			r.buf.Write(buf[0:n])
-		}
-		r.wait.Signal()
-		r.l.Unlock()
-		if err != nil {
-			break
-		}
-	}
-}
-
-func (r *bufReader) Read(p []byte) (n int, err error) {
-	r.l.Lock()
-	defer r.l.Unlock()
-	for {
-		n, err = r.buf.Read(p)
-		if n > 0 {
-			return n, err
-		}
-		if r.err != nil {
-			return 0, r.err
-		}
-		r.wait.Wait()
-	}
-	panic("unreachable")
-}
-
-func (r *bufReader) Close() error {
-	closer, ok := r.reader.(io.ReadCloser)
-	if !ok {
-		return nil
-	}
-	return closer.Close()
-}
-
-type writeBroadcaster struct {
-	mu      sync.Mutex
-	writers map[io.WriteCloser]struct{}
-}
-
-func (w *writeBroadcaster) AddWriter(writer io.WriteCloser) {
-	w.mu.Lock()
-	w.writers[writer] = struct{}{}
-	w.mu.Unlock()
-}
-
-// FIXME: Is that function used?
-// FIXME: This relies on the concrete writer type used having equality operator
-func (w *writeBroadcaster) RemoveWriter(writer io.WriteCloser) {
-	w.mu.Lock()
-	delete(w.writers, writer)
-	w.mu.Unlock()
-}
-
-func (w *writeBroadcaster) Write(p []byte) (n int, err error) {
-	w.mu.Lock()
-	defer w.mu.Unlock()
-	for writer := range w.writers {
-		if n, err := writer.Write(p); err != nil || n != len(p) {
-			// On error, evict the writer
-			delete(w.writers, writer)
-		}
-	}
-	return len(p), nil
-}
-
-func (w *writeBroadcaster) CloseWriters() error {
-	w.mu.Lock()
-	defer w.mu.Unlock()
-	for writer := range w.writers {
-		writer.Close()
-	}
-	w.writers = make(map[io.WriteCloser]struct{})
-	return nil
-}
-
-func newWriteBroadcaster() *writeBroadcaster {
-	return &writeBroadcaster{writers: make(map[io.WriteCloser]struct{})}
-}
-
-func getTotalUsedFds() int {
-	if fds, err := ioutil.ReadDir(fmt.Sprintf("/proc/%d/fd", os.Getpid())); err != nil {
-		Debugf("Error opening /proc/%d/fd: %s", os.Getpid(), err)
-	} else {
-		return len(fds)
-	}
-	return -1
-}
-
-// TruncIndex allows the retrieval of string identifiers by any of their unique prefixes.
-// This is used to retrieve image and container IDs by more convenient shorthand prefixes.
-type TruncIndex struct {
-	index *suffixarray.Index
-	ids   map[string]bool
-	bytes []byte
-}
-
-func NewTruncIndex() *TruncIndex {
-	return &TruncIndex{
-		index: suffixarray.New([]byte{' '}),
-		ids:   make(map[string]bool),
-		bytes: []byte{' '},
-	}
-}
-
-func (idx *TruncIndex) Add(id string) error {
-	if strings.Contains(id, " ") {
-		return fmt.Errorf("Illegal character: ' '")
-	}
-	if _, exists := idx.ids[id]; exists {
-		return fmt.Errorf("Id already exists: %s", id)
-	}
-	idx.ids[id] = true
-	idx.bytes = append(idx.bytes, []byte(id+" ")...)
-	idx.index = suffixarray.New(idx.bytes)
-	return nil
-}
-
-func (idx *TruncIndex) Delete(id string) error {
-	if _, exists := idx.ids[id]; !exists {
-		return fmt.Errorf("No such id: %s", id)
-	}
-	before, after, err := idx.lookup(id)
-	if err != nil {
-		return err
-	}
-	delete(idx.ids, id)
-	idx.bytes = append(idx.bytes[:before], idx.bytes[after:]...)
-	idx.index = suffixarray.New(idx.bytes)
-	return nil
-}
-
-func (idx *TruncIndex) lookup(s string) (int, int, error) {
-	offsets := idx.index.Lookup([]byte(" "+s), -1)
-	//log.Printf("lookup(%s): %v (index bytes: '%s')\n", s, offsets, idx.index.Bytes())
-	if offsets == nil || len(offsets) == 0 || len(offsets) > 1 {
-		return -1, -1, fmt.Errorf("No such id: %s", s)
-	}
-	offsetBefore := offsets[0] + 1
-	offsetAfter := offsetBefore + strings.Index(string(idx.bytes[offsetBefore:]), " ")
-	return offsetBefore, offsetAfter, nil
-}
-
-func (idx *TruncIndex) Get(s string) (string, error) {
-	before, after, err := idx.lookup(s)
-	//log.Printf("Get(%s) bytes=|%s| before=|%d| after=|%d|\n", s, idx.bytes, before, after)
-	if err != nil {
-		return "", err
-	}
-	return string(idx.bytes[before:after]), err
-}
-
-// TruncateId returns a shorthand version of a string identifier for convenience.
-// A collision with other shorthands is very unlikely, but possible.
-// In case of a collision a lookup with TruncIndex.Get() will fail, and the caller
-// will need to use a langer prefix, or the full-length Id.
-func TruncateId(id string) string {
-	shortLen := 12
-	if len(id) < shortLen {
-		shortLen = len(id)
-	}
-	return id[:shortLen]
-}
-
-// Code c/c from io.Copy() modified to handle escape sequence
-func CopyEscapable(dst io.Writer, src io.ReadCloser) (written int64, err error) {
-	buf := make([]byte, 32*1024)
-	for {
-		nr, er := src.Read(buf)
-		if nr > 0 {
-			// ---- Docker addition
-			// char 16 is C-p
-			if nr == 1 && buf[0] == 16 {
-				nr, er = src.Read(buf)
-				// char 17 is C-q
-				if nr == 1 && buf[0] == 17 {
-					if err := src.Close(); err != nil {
-						return 0, err
-					}
-					return 0, io.EOF
-				}
-			}
-			// ---- End of docker
-			nw, ew := dst.Write(buf[0:nr])
-			if nw > 0 {
-				written += int64(nw)
-			}
-			if ew != nil {
-				err = ew
-				break
-			}
-			if nr != nw {
-				err = io.ErrShortWrite
-				break
-			}
-		}
-		if er == io.EOF {
-			break
-		}
-		if er != nil {
-			err = er
-			break
-		}
-	}
-	return written, err
-}
-
-func SetRawTerminal() (*term.State, error) {
-	oldState, err := term.MakeRaw(int(os.Stdin.Fd()))
-	if err != nil {
-		return nil, err
-	}
-	c := make(chan os.Signal, 1)
-	signal.Notify(c, os.Interrupt)
-	go func() {
-		_ = <-c
-		term.Restore(int(os.Stdin.Fd()), oldState)
-		os.Exit(0)
-	}()
-	return oldState, err
-}
-
-func RestoreTerminal(state *term.State) {
-	term.Restore(int(os.Stdin.Fd()), state)
-}
-
-func HashData(src io.Reader) (string, error) {
-	h := sha256.New()
-	if _, err := io.Copy(h, src); err != nil {
-		return "", err
-	}
-	return "sha256:" + hex.EncodeToString(h.Sum(nil)), nil
-}
-
-type KernelVersionInfo struct {
-	Kernel int
-	Major  int
-	Minor  int
-	Flavor string
-}
-
-// FIXME: this doens't build on Darwin
-func GetKernelVersion() (*KernelVersionInfo, error) {
-	return getKernelVersion()
-}
-
-func (k *KernelVersionInfo) String() string {
-	flavor := ""
-	if len(k.Flavor) > 0 {
-		flavor = fmt.Sprintf("-%s", k.Flavor)
-	}
-	return fmt.Sprintf("%d.%d.%d%s", k.Kernel, k.Major, k.Minor, flavor)
-}
-
-// Compare two KernelVersionInfo struct.
-// Returns -1 if a < b, = if a == b, 1 it a > b
-func CompareKernelVersion(a, b *KernelVersionInfo) int {
-	if a.Kernel < b.Kernel {
-		return -1
-	} else if a.Kernel > b.Kernel {
-		return 1
-	}
-
-	if a.Major < b.Major {
-		return -1
-	} else if a.Major > b.Major {
-		return 1
-	}
-
-	if a.Minor < b.Minor {
-		return -1
-	} else if a.Minor > b.Minor {
-		return 1
-	}
-
-	return 0
-}
-
-func FindCgroupMountpoint(cgroupType string) (string, error) {
-	output, err := ioutil.ReadFile("/proc/mounts")
-	if err != nil {
-		return "", err
-	}
-
-	// /proc/mounts has 6 fields per line, one mount per line, e.g.
-	// cgroup /sys/fs/cgroup/devices cgroup rw,relatime,devices 0 0
-	for _, line := range strings.Split(string(output), "\n") {
-		parts := strings.Split(line, " ")
-		if len(parts) == 6 && parts[2] == "cgroup" {
-			for _, opt := range strings.Split(parts[3], ",") {
-				if opt == cgroupType {
-					return parts[1], nil
-				}
-			}
-		}
-	}
-
-	return "", fmt.Errorf("cgroup mountpoint not found for %s", cgroupType)
-}
-
 // Compare two Config struct. Do not compare the "Image" nor "Hostname" fields
 // If OpenStdin is set, then it differs
 func CompareConfig(a, b *Config) bool {

utils/uname_darwin.go

@@ -0,0 +1,10 @@
+package utils
+
+import (
+	"errors"
+	"syscall"
+)
+
+func uname() (*syscall.Utsname, error) {
+	return nil, errors.New("Kernel version detection is not available on darwin")
+}

utils/uname_linux.go

@@ -0,0 +1,15 @@
+package utils
+
+import (
+	"syscall"
+)
+
+// FIXME: Move this to utils package
+func uname() (*syscall.Utsname, error) {
+	uts := &syscall.Utsname{}
+
+	if err := syscall.Uname(uts); err != nil {
+		return nil, err
+	}
+	return uts, nil
+}

utils/utils.go

@@ -0,0 +1,532 @@
+package utils
+
+import (
+	"bytes"
+	"crypto/sha256"
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"index/suffixarray"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"runtime"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+)
+
+// Go is a basic promise implementation: it wraps calls a function in a goroutine,
+// and returns a channel which will later return the function's return value.
+func Go(f func() error) chan error {
+	ch := make(chan error)
+	go func() {
+		ch <- f()
+	}()
+	return ch
+}
+
+// Request a given URL and return an io.Reader
+func Download(url string, stderr io.Writer) (*http.Response, error) {
+	var resp *http.Response
+	var err error = nil
+	if resp, err = http.Get(url); err != nil {
+		return nil, err
+	}
+	if resp.StatusCode >= 400 {
+		return nil, errors.New("Got HTTP status code >= 400: " + resp.Status)
+	}
+	return resp, nil
+}
+
+// Debug function, if the debug flag is set, then display. Do nothing otherwise
+// If Docker is in damon mode, also send the debug info on the socket
+func Debugf(format string, a ...interface{}) {
+	if os.Getenv("DEBUG") != "" {
+
+		// Retrieve the stack infos
+		_, file, line, ok := runtime.Caller(1)
+		if !ok {
+			file = "<unknown>"
+			line = -1
+		} else {
+			file = file[strings.LastIndex(file, "/")+1:]
+		}
+
+		fmt.Fprintf(os.Stderr, fmt.Sprintf("[debug] %s:%d %s\n", file, line, format), a...)
+	}
+}
+
+// Reader with progress bar
+type progressReader struct {
+	reader       io.ReadCloser // Stream to read from
+	output       io.Writer     // Where to send progress bar to
+	readTotal    int           // Expected stream length (bytes)
+	readProgress int           // How much has been read so far (bytes)
+	lastUpdate   int           // How many bytes read at least update
+	template     string        // Template to print. Default "%v/%v (%v)"
+}
+
+func (r *progressReader) Read(p []byte) (n int, err error) {
+	read, err := io.ReadCloser(r.reader).Read(p)
+	r.readProgress += read
+
+	updateEvery := 4096
+	if r.readTotal > 0 {
+		// Only update progress for every 1% read
+		if increment := int(0.01 * float64(r.readTotal)); increment > updateEvery {
+			updateEvery = increment
+		}
+	}
+	if r.readProgress-r.lastUpdate > updateEvery || err != nil {
+		if r.readTotal > 0 {
+			fmt.Fprintf(r.output, r.template+"\r", r.readProgress, r.readTotal, fmt.Sprintf("%.0f%%", float64(r.readProgress)/float64(r.readTotal)*100))
+		} else {
+			fmt.Fprintf(r.output, r.template+"\r", r.readProgress, "?", "n/a")
+		}
+		r.lastUpdate = r.readProgress
+	}
+	// Send newline when complete
+	if err != nil {
+		fmt.Fprintf(r.output, "\n")
+	}
+
+	return read, err
+}
+func (r *progressReader) Close() error {
+	return io.ReadCloser(r.reader).Close()
+}
+func ProgressReader(r io.ReadCloser, size int, output io.Writer, template string) *progressReader {
+	if template == "" {
+		template = "%v/%v (%v)"
+	}
+	return &progressReader{r, output, size, 0, 0, template}
+}
+
+// HumanDuration returns a human-readable approximation of a duration
+// (eg. "About a minute", "4 hours ago", etc.)
+func HumanDuration(d time.Duration) string {
+	if seconds := int(d.Seconds()); seconds < 1 {
+		return "Less than a second"
+	} else if seconds < 60 {
+		return fmt.Sprintf("%d seconds", seconds)
+	} else if minutes := int(d.Minutes()); minutes == 1 {
+		return "About a minute"
+	} else if minutes < 60 {
+		return fmt.Sprintf("%d minutes", minutes)
+	} else if hours := int(d.Hours()); hours == 1 {
+		return "About an hour"
+	} else if hours < 48 {
+		return fmt.Sprintf("%d hours", hours)
+	} else if hours < 24*7*2 {
+		return fmt.Sprintf("%d days", hours/24)
+	} else if hours < 24*30*3 {
+		return fmt.Sprintf("%d weeks", hours/24/7)
+	} else if hours < 24*365*2 {
+		return fmt.Sprintf("%d months", hours/24/30)
+	}
+	return fmt.Sprintf("%d years", d.Hours()/24/365)
+}
+
+func Trunc(s string, maxlen int) string {
+	if len(s) <= maxlen {
+		return s
+	}
+	return s[:maxlen]
+}
+
+// Figure out the absolute path of our own binary
+func SelfPath() string {
+	path, err := exec.LookPath(os.Args[0])
+	if err != nil {
+		panic(err)
+	}
+	path, err = filepath.Abs(path)
+	if err != nil {
+		panic(err)
+	}
+	return path
+}
+
+type NopWriter struct {
+}
+
+func (w *NopWriter) Write(buf []byte) (int, error) {
+	return len(buf), nil
+}
+
+type nopWriteCloser struct {
+	io.Writer
+}
+
+func (w *nopWriteCloser) Close() error { return nil }
+
+func NopWriteCloser(w io.Writer) io.WriteCloser {
+	return &nopWriteCloser{w}
+}
+
+type bufReader struct {
+	buf    *bytes.Buffer
+	reader io.Reader
+	err    error
+	l      sync.Mutex
+	wait   sync.Cond
+}
+
+func NewBufReader(r io.Reader) *bufReader {
+	reader := &bufReader{
+		buf:    &bytes.Buffer{},
+		reader: r,
+	}
+	reader.wait.L = &reader.l
+	go reader.drain()
+	return reader
+}
+
+func (r *bufReader) drain() {
+	buf := make([]byte, 1024)
+	for {
+		n, err := r.reader.Read(buf)
+		r.l.Lock()
+		if err != nil {
+			r.err = err
+		} else {
+			r.buf.Write(buf[0:n])
+		}
+		r.wait.Signal()
+		r.l.Unlock()
+		if err != nil {
+			break
+		}
+	}
+}
+
+func (r *bufReader) Read(p []byte) (n int, err error) {
+	r.l.Lock()
+	defer r.l.Unlock()
+	for {
+		n, err = r.buf.Read(p)
+		if n > 0 {
+			return n, err
+		}
+		if r.err != nil {
+			return 0, r.err
+		}
+		r.wait.Wait()
+	}
+	panic("unreachable")
+}
+
+func (r *bufReader) Close() error {
+	closer, ok := r.reader.(io.ReadCloser)
+	if !ok {
+		return nil
+	}
+	return closer.Close()
+}
+
+type WriteBroadcaster struct {
+	mu      sync.Mutex
+	writers map[io.WriteCloser]struct{}
+}
+
+func (w *WriteBroadcaster) AddWriter(writer io.WriteCloser) {
+	w.mu.Lock()
+	w.writers[writer] = struct{}{}
+	w.mu.Unlock()
+}
+
+// FIXME: Is that function used?
+// FIXME: This relies on the concrete writer type used having equality operator
+func (w *WriteBroadcaster) RemoveWriter(writer io.WriteCloser) {
+	w.mu.Lock()
+	delete(w.writers, writer)
+	w.mu.Unlock()
+}
+
+func (w *WriteBroadcaster) Write(p []byte) (n int, err error) {
+	w.mu.Lock()
+	defer w.mu.Unlock()
+	for writer := range w.writers {
+		if n, err := writer.Write(p); err != nil || n != len(p) {
+			// On error, evict the writer
+			delete(w.writers, writer)
+		}
+	}
+	return len(p), nil
+}
+
+func (w *WriteBroadcaster) CloseWriters() error {
+	w.mu.Lock()
+	defer w.mu.Unlock()
+	for writer := range w.writers {
+		writer.Close()
+	}
+	w.writers = make(map[io.WriteCloser]struct{})
+	return nil
+}
+
+func NewWriteBroadcaster() *WriteBroadcaster {
+	return &WriteBroadcaster{writers: make(map[io.WriteCloser]struct{})}
+}
+
+func GetTotalUsedFds() int {
+	if fds, err := ioutil.ReadDir(fmt.Sprintf("/proc/%d/fd", os.Getpid())); err != nil {
+		Debugf("Error opening /proc/%d/fd: %s", os.Getpid(), err)
+	} else {
+		return len(fds)
+	}
+	return -1
+}
+
+// TruncIndex allows the retrieval of string identifiers by any of their unique prefixes.
+// This is used to retrieve image and container IDs by more convenient shorthand prefixes.
+type TruncIndex struct {
+	index *suffixarray.Index
+	ids   map[string]bool
+	bytes []byte
+}
+
+func NewTruncIndex() *TruncIndex {
+	return &TruncIndex{
+		index: suffixarray.New([]byte{' '}),
+		ids:   make(map[string]bool),
+		bytes: []byte{' '},
+	}
+}
+
+func (idx *TruncIndex) Add(id string) error {
+	if strings.Contains(id, " ") {
+		return fmt.Errorf("Illegal character: ' '")
+	}
+	if _, exists := idx.ids[id]; exists {
+		return fmt.Errorf("Id already exists: %s", id)
+	}
+	idx.ids[id] = true
+	idx.bytes = append(idx.bytes, []byte(id+" ")...)
+	idx.index = suffixarray.New(idx.bytes)
+	return nil
+}
+
+func (idx *TruncIndex) Delete(id string) error {
+	if _, exists := idx.ids[id]; !exists {
+		return fmt.Errorf("No such id: %s", id)
+	}
+	before, after, err := idx.lookup(id)
+	if err != nil {
+		return err
+	}
+	delete(idx.ids, id)
+	idx.bytes = append(idx.bytes[:before], idx.bytes[after:]...)
+	idx.index = suffixarray.New(idx.bytes)
+	return nil
+}
+
+func (idx *TruncIndex) lookup(s string) (int, int, error) {
+	offsets := idx.index.Lookup([]byte(" "+s), -1)
+	//log.Printf("lookup(%s): %v (index bytes: '%s')\n", s, offsets, idx.index.Bytes())
+	if offsets == nil || len(offsets) == 0 || len(offsets) > 1 {
+		return -1, -1, fmt.Errorf("No such id: %s", s)
+	}
+	offsetBefore := offsets[0] + 1
+	offsetAfter := offsetBefore + strings.Index(string(idx.bytes[offsetBefore:]), " ")
+	return offsetBefore, offsetAfter, nil
+}
+
+func (idx *TruncIndex) Get(s string) (string, error) {
+	before, after, err := idx.lookup(s)
+	//log.Printf("Get(%s) bytes=|%s| before=|%d| after=|%d|\n", s, idx.bytes, before, after)
+	if err != nil {
+		return "", err
+	}
+	return string(idx.bytes[before:after]), err
+}
+
+// TruncateId returns a shorthand version of a string identifier for convenience.
+// A collision with other shorthands is very unlikely, but possible.
+// In case of a collision a lookup with TruncIndex.Get() will fail, and the caller
+// will need to use a langer prefix, or the full-length Id.
+func TruncateId(id string) string {
+	shortLen := 12
+	if len(id) < shortLen {
+		shortLen = len(id)
+	}
+	return id[:shortLen]
+}
+
+// Code c/c from io.Copy() modified to handle escape sequence
+func CopyEscapable(dst io.Writer, src io.ReadCloser) (written int64, err error) {
+	buf := make([]byte, 32*1024)
+	for {
+		nr, er := src.Read(buf)
+		if nr > 0 {
+			// ---- Docker addition
+			// char 16 is C-p
+			if nr == 1 && buf[0] == 16 {
+				nr, er = src.Read(buf)
+				// char 17 is C-q
+				if nr == 1 && buf[0] == 17 {
+					if err := src.Close(); err != nil {
+						return 0, err
+					}
+					return 0, io.EOF
+				}
+			}
+			// ---- End of docker
+			nw, ew := dst.Write(buf[0:nr])
+			if nw > 0 {
+				written += int64(nw)
+			}
+			if ew != nil {
+				err = ew
+				break
+			}
+			if nr != nw {
+				err = io.ErrShortWrite
+				break
+			}
+		}
+		if er == io.EOF {
+			break
+		}
+		if er != nil {
+			err = er
+			break
+		}
+	}
+	return written, err
+}
+
+func HashData(src io.Reader) (string, error) {
+	h := sha256.New()
+	if _, err := io.Copy(h, src); err != nil {
+		return "", err
+	}
+	return "sha256:" + hex.EncodeToString(h.Sum(nil)), nil
+}
+
+type KernelVersionInfo struct {
+	Kernel int
+	Major  int
+	Minor  int
+	Flavor string
+}
+
+func (k *KernelVersionInfo) String() string {
+	flavor := ""
+	if len(k.Flavor) > 0 {
+		flavor = fmt.Sprintf("-%s", k.Flavor)
+	}
+	return fmt.Sprintf("%d.%d.%d%s", k.Kernel, k.Major, k.Minor, flavor)
+}
+
+// Compare two KernelVersionInfo struct.
+// Returns -1 if a < b, = if a == b, 1 it a > b
+func CompareKernelVersion(a, b *KernelVersionInfo) int {
+	if a.Kernel < b.Kernel {
+		return -1
+	} else if a.Kernel > b.Kernel {
+		return 1
+	}
+
+	if a.Major < b.Major {
+		return -1
+	} else if a.Major > b.Major {
+		return 1
+	}
+
+	if a.Minor < b.Minor {
+		return -1
+	} else if a.Minor > b.Minor {
+		return 1
+	}
+
+	return 0
+}
+
+func FindCgroupMountpoint(cgroupType string) (string, error) {
+	output, err := ioutil.ReadFile("/proc/mounts")
+	if err != nil {
+		return "", err
+	}
+
+	// /proc/mounts has 6 fields per line, one mount per line, e.g.
+	// cgroup /sys/fs/cgroup/devices cgroup rw,relatime,devices 0 0
+	for _, line := range strings.Split(string(output), "\n") {
+		parts := strings.Split(line, " ")
+		if len(parts) == 6 && parts[2] == "cgroup" {
+			for _, opt := range strings.Split(parts[3], ",") {
+				if opt == cgroupType {
+					return parts[1], nil
+				}
+			}
+		}
+	}
+
+	return "", fmt.Errorf("cgroup mountpoint not found for %s", cgroupType)
+}
+
+func GetKernelVersion() (*KernelVersionInfo, error) {
+	var (
+		flavor               string
+		kernel, major, minor int
+		err                  error
+	)
+
+	uts, err := uname()
+	if err != nil {
+		return nil, err
+	}
+
+	release := make([]byte, len(uts.Release))
+
+	i := 0
+	for _, c := range uts.Release {
+		release[i] = byte(c)
+		i++
+	}
+
+	// Remove the \x00 from the release for Atoi to parse correctly
+	release = release[:bytes.IndexByte(release, 0)]
+
+	tmp := strings.SplitN(string(release), "-", 2)
+	tmp2 := strings.SplitN(tmp[0], ".", 3)
+
+	if len(tmp2) > 0 {
+		kernel, err = strconv.Atoi(tmp2[0])
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	if len(tmp2) > 1 {
+		major, err = strconv.Atoi(tmp2[1])
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	if len(tmp2) > 2 {
+		minor, err = strconv.Atoi(tmp2[2])
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	if len(tmp) == 2 {
+		flavor = tmp[1]
+	} else {
+		flavor = ""
+	}
+
+	return &KernelVersionInfo{
+		Kernel: kernel,
+		Major:  major,
+		Minor:  minor,
+		Flavor: flavor,
+	}, nil
+}

utils_test.go → utils/utils_test.go

@@ -1,4 +1,4 @@
-package docker
+package utils
 
 import (
 	"bytes"
@@ -10,7 +10,7 @@ import (
 
 func TestBufReader(t *testing.T) {
 	reader, writer := io.Pipe()
-	bufreader := newBufReader(reader)
+	bufreader := NewBufReader(reader)
 
 	// Write everything down to a Pipe
 	// Usually, a pipe should block but because of the buffered reader,
@@ -55,7 +55,7 @@ func (dw *dummyWriter) Close() error {
 }
 
 func TestWriteBroadcaster(t *testing.T) {
-	writer := newWriteBroadcaster()
+	writer := NewWriteBroadcaster()
 
 	// Test 1: Both bufferA and bufferB should contain "foo"
 	bufferA := &dummyWriter{}
@@ -137,7 +137,7 @@ func (d devNullCloser) Write(buf []byte) (int, error) {
 
 // This test checks for races. It is only useful when run with the race detector.
 func TestRaceWriteBroadcaster(t *testing.T) {
-	writer := newWriteBroadcaster()
+	writer := NewWriteBroadcaster()
 	c := make(chan bool)
 	go func() {
 		writer.AddWriter(devNullCloser(0))