Factorize the pull/push commands and create a registry.go

commands.go

@@ -8,12 +8,10 @@ import (
 	"github.com/dotcloud/docker/auth"
 	"github.com/dotcloud/docker/rcli"
 	"io"
-	"io/ioutil"
 	"log"
 	"math/rand"
 	"net/http"
 	"net/url"
-	"path"
 	"runtime"
 	"strconv"
 	"strings"
@@ -23,7 +21,6 @@ import (
 )
 
 const VERSION = "0.0.3"
-const REGISTRY_ENDPOINT = "http://registry-creack.dotcloud.com/v1"
 
 func (srv *Server) Name() string {
 	return "docker"
@@ -434,168 +431,16 @@ func (srv *Server) CmdPush(stdin io.ReadCloser, stdout io.Writer, args ...string
 		return nil
 	}
 
-	client := &http.Client{}
-	if img, err := srv.runtime.graph.Get(cmd.Arg(0)); err != nil {
-		return nil
-	} else {
-		img.WalkHistory(func(img *Image) {
-			fmt.Fprintf(stdout, "Pushing %s\n", img.Id)
-
-			jsonRaw, err := ioutil.ReadFile(path.Join(srv.runtime.graph.Root, img.Id, "json"))
-			if err != nil {
-				fmt.Fprintf(stdout, "Error while retreiving the path for {%s}: %s\n", img.Id, err)
-				return
-			}
-			jsonData := strings.NewReader(string(jsonRaw))
-			req, err := http.NewRequest("PUT", REGISTRY_ENDPOINT+"/images/"+img.Id+"/json", jsonData)
-			res, err := client.Do(req)
-			if err != nil || res.StatusCode != 200 {
-				if res == nil {
-					fmt.Fprintf(stdout,
-						"Error: Internal server error trying to push image {%s} (json): %s\n",
-						img.Id, err)
-					return
-				}
-				switch res.StatusCode {
-				case 204:
-					fmt.Fprintf(stdout, "Image already on the repository\n")
-					return
-				case 400:
-					fmt.Fprintf(stdout, "Error: Invalid Json\n")
-					return
-				default:
-					fmt.Fprintf(stdout,
-						"Error: Internal server error trying to push image {%s} (json): %s (%d)\n",
-						img.Id, err, res.StatusCode)
-					return
-				}
-			}
-
-			req2, err := http.NewRequest("PUT", REGISTRY_ENDPOINT+"/images/"+img.Id+"/layer", nil)
-			res2, err := client.Do(req2)
-			if err != nil || res2.StatusCode != 307 {
-				fmt.Fprintf(stdout,
-					"Error trying to push image {%s} (layer 1): %s\n",
-					img.Id, err)
-				return
-			}
-			url, err := res2.Location()
-			if err != nil || url == nil {
-				fmt.Fprintf(stdout,
-					"Fail to retrieve layer storage URL for image {%s}: %s\n",
-					img.Id, err)
-				return
-			}
-			// FIXME: Don't do this :D. Check the S3 requierement and implement chunks of 5MB
-			layerData2, err := Tar(path.Join(srv.runtime.graph.Root, img.Id, "layer"), Gzip)
-			layerData, err := Tar(path.Join(srv.runtime.graph.Root, img.Id, "layer"), Gzip)
-			if err != nil {
-				fmt.Fprintf(stdout,
-					"Error while retrieving layer for {%s}: %s\n",
-					img.Id, err)
-				return
-			}
-			req3, err := http.NewRequest("PUT", url.String(), layerData)
-			tmp, _ := ioutil.ReadAll(layerData2)
-			req3.ContentLength = int64(len(tmp))
-
-			req3.TransferEncoding = []string{"none"}
-			res3, err := client.Do(req3)
-			if err != nil || res3.StatusCode != 200 {
-				if res3 == nil {
-					fmt.Fprintf(stdout,
-						"Error trying to push image {%s} (layer 2): %s\n",
-						img.Id, err)
-				} else {
-					fmt.Fprintf(stdout,
-						"Error trying to push image {%s} (layer 2): %s (%d)\n",
-						img.Id, err, res3.StatusCode)
-				}
-				return
-			}
-		})
-	}
-	return nil
-}
-
-func newImgJson(src []byte) (*Image, error) {
-	ret := &Image{}
-
-	fmt.Printf("Json string: {%s}\n", src)
-	// FIXME: Is there a cleaner way to "puryfy" the input json?
-	src = []byte(strings.Replace(string(src), "null", "\"\"", -1))
-
-	if err := json.Unmarshal(src, ret); err != nil {
-		return nil, err
-	}
-	return ret, nil
-}
-
-func newMultipleImgJson(src []byte) (map[*Image]Archive, error) {
-	ret := map[*Image]Archive{}
-
-	fmt.Printf("Json string2: {%s}\n", src)
-	dec := json.NewDecoder(strings.NewReader(strings.Replace(string(src), "null", "\"\"", -1)))
-	for {
-		m := &Image{}
-		if err := dec.Decode(m); err == io.EOF {
-			break
-		} else if err != nil {
-			return nil, err
-		}
-		ret[m] = nil
-	}
-	return ret, nil
-}
-
-func getHistory(base_uri, id string) (map[*Image]Archive, error) {
-	res, err := http.Get(base_uri + id + "/history")
+	img, err := srv.runtime.graph.Get(cmd.Arg(0))
 	if err != nil {
-		return nil, fmt.Errorf("Error while getting from the server: %s\n", err)
-	}
-	defer res.Body.Close()
-
-	jsonString, err := ioutil.ReadAll(res.Body)
-	if err != nil {
-		return nil, fmt.Errorf("Error while reading the http response: %s\n", err)
-	}
-
-	history, err := newMultipleImgJson(jsonString)
-	if err != nil {
-		return nil, fmt.Errorf("Error while parsing the json: %s\n", err)
-	}
-	return history, nil
-}
-
-func getRemoteImage(base_uri, id string) (*Image, Archive, error) {
-	// Get the Json
-	res, err := http.Get(base_uri + id + "/json")
-	if err != nil {
-		return nil, nil, fmt.Errorf("Error while getting from the server: %s\n", err)
-	}
-	defer res.Body.Close()
-
-	jsonString, err := ioutil.ReadAll(res.Body)
-	if err != nil {
-		return nil, nil, fmt.Errorf("Error while reading the http response: %s\n", err)
-	}
-
-	img, err := newImgJson(jsonString)
-	if err != nil {
-		return nil, nil, fmt.Errorf("Error while parsing the json: %s\n", err)
-	}
-	img.Id = id
-
-	// Get the layer
-	res, err = http.Get(base_uri + id + "/layer")
-	if err != nil {
-		return nil, nil, fmt.Errorf("Error while getting from the server: %s\n", err)
+		return err
 	}
-	return img, res.Body, nil
+	// FIXME: Handle repositories, etc. Not jist images
+	return srv.runtime.graph.PushImage(img)
 }
 
-func (srv *Server) CmdPulli(stdin io.ReadCloser, stdout io.Writer, args ...string) error {
-	cmd := rcli.Subcmd(stdout, "pulli", "[OPTIONS] IMAGE", "Pull an image from the registry")
+func (srv *Server) CmdPull(stdin io.ReadCloser, stdout io.Writer, args ...string) error {
+	cmd := rcli.Subcmd(stdout, "pull", "[OPTIONS] IMAGE", "Pull an image from the registry")
 	if err := cmd.Parse(args); err != nil {
 		return nil
 	}
@@ -603,28 +448,8 @@ func (srv *Server) CmdPulli(stdin io.ReadCloser, stdout io.Writer, args ...strin
 		cmd.Usage()
 		return nil
 	}
-
-	// First, retrieve the history
-	base_uri := REGISTRY_ENDPOINT + "/images/"
-
-	// Now we have the history, remove the images we already have
-	history, err := getHistory(base_uri, cmd.Arg(0))
-	if err != nil {
-		return err
-	}
-	for j := range history {
-		if !srv.runtime.graph.Exists(j.Id) {
-			img, layer, err := getRemoteImage(base_uri, j.Id)
-			if err != nil {
-				// FIXME: Keep goging in case of error?
-				return err
-			}
-			if err = srv.runtime.graph.Register(layer, img); err != nil {
-				return err
-			}
-		}
-	}
-	return nil
+	// FIXME: Handle repositories, etc. Not jist images
+	return srv.runtime.graph.PullImage(cmd.Arg(0))
 }
 
 func (srv *Server) CmdImages(stdin io.ReadCloser, stdout io.Writer, args ...string) error {

image.go

@@ -181,12 +181,12 @@ func ComputeId(content io.Reader) (string, error) {
 // Image includes convenience proxy functions to its graph
 // These functions will return an error if the image is not registered
 // (ie. if image.graph == nil)
-
 func (img *Image) History() ([]*Image, error) {
 	var parents []*Image
 	if err := img.WalkHistory(
-		func(img *Image) {
+		func(img *Image) error {
 			parents = append(parents, img)
+			return nil
 		},
 	); err != nil {
 		return nil, err
@@ -195,16 +195,19 @@ func (img *Image) History() ([]*Image, error) {
 }
 
 // layers returns all the filesystem layers needed to mount an image
+// FIXME: @shykes refactor this function with the new error handling
+//        (I'll do it if I have time tonight, I focus on the rest)
 func (img *Image) layers() ([]string, error) {
 	var list []string
 	var e error
 	if err := img.WalkHistory(
-		func(img *Image) {
+		func(img *Image) (err error) {
 			if layer, err := img.layer(); err != nil {
 				e = err
 			} else if layer != "" {
 				list = append(list, layer)
 			}
+			return err
 		},
 	); err != nil {
 		return nil, err
@@ -217,12 +220,13 @@ func (img *Image) layers() ([]string, error) {
 	return list, nil
 }
 
-func (img *Image) WalkHistory(handler func(*Image)) error {
-	var err error
+func (img *Image) WalkHistory(handler func(*Image) error) (err error) {
 	currentImg := img
 	for currentImg != nil {
 		if handler != nil {
-			handler(currentImg)
+			if err := handler(currentImg); err != nil {
+				return err
+			}
 		}
 		currentImg, err = currentImg.GetParent()
 		if err != nil {

registry.go

@@ -0,0 +1,201 @@
+package docker
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"path"
+	"strings"
+)
+
+//FIXME: Set the endpoint in a conf file or via commandline
+//const REGISTRY_ENDPOINT = "http://registry-creack.dotcloud.com/v1"
+const REGISTRY_ENDPOINT = "http://192.168.56.1:5000/v1"
+
+// Build an Image object from raw json data
+func NewImgJson(src []byte) (*Image, error) {
+	ret := &Image{}
+
+	fmt.Printf("Json string: {%s}\n", src)
+	// FIXME: Is there a cleaner way to "puryfy" the input json?
+	src = []byte(strings.Replace(string(src), "null", "\"\"", -1))
+
+	if err := json.Unmarshal(src, ret); err != nil {
+		return nil, err
+	}
+	return ret, nil
+}
+
+// Build an Image object list from a raw json data
+// FIXME: Do this in "stream" mode
+func NewMultipleImgJson(src []byte) ([]*Image, error) {
+	ret := []*Image{}
+
+	dec := json.NewDecoder(strings.NewReader(strings.Replace(string(src), "null", "\"\"", -1)))
+	for {
+		m := &Image{}
+		if err := dec.Decode(m); err == io.EOF {
+			break
+		} else if err != nil {
+			return nil, err
+		}
+		ret = append(ret, m)
+	}
+	return ret, nil
+}
+
+// Retrieve the history of a given image from the Registry.
+// Return a list of the parent's json (requested image included)
+func (graph *Graph) getRemoteHistory(imgId string) ([]*Image, error) {
+	res, err := http.Get(REGISTRY_ENDPOINT + "/images/" + imgId + "/history")
+	if err != nil {
+		return nil, fmt.Errorf("Error while getting from the server: %s\n", err)
+	}
+	defer res.Body.Close()
+
+	jsonString, err := ioutil.ReadAll(res.Body)
+	if err != nil {
+		return nil, fmt.Errorf("Error while reading the http response: %s\n", err)
+	}
+
+	history, err := NewMultipleImgJson(jsonString)
+	if err != nil {
+		return nil, fmt.Errorf("Error while parsing the json: %s\n", err)
+	}
+	return history, nil
+}
+
+// Retrieve an image from the Registry.
+// Returns the Image object as well as the layer as an Archive (io.Reader)
+func (graph *Graph) getRemoteImage(imgId string) (*Image, Archive, error) {
+	// Get the Json
+	res, err := http.Get(REGISTRY_ENDPOINT + "/images/" + imgId + "/json")
+	if err != nil {
+		return nil, nil, fmt.Errorf("Error while getting from the server: %s\n", err)
+	}
+	defer res.Body.Close()
+
+	jsonString, err := ioutil.ReadAll(res.Body)
+	if err != nil {
+		return nil, nil, fmt.Errorf("Error while reading the http response: %s\n", err)
+	}
+
+	img, err := NewImgJson(jsonString)
+	if err != nil {
+		return nil, nil, fmt.Errorf("Error while parsing the json: %s\n", err)
+	}
+	img.Id = imgId
+
+	// Get the layer
+	res, err = http.Get(REGISTRY_ENDPOINT + "/images/" + imgId + "/layer")
+	if err != nil {
+		return nil, nil, fmt.Errorf("Error while getting from the server: %s\n", err)
+	}
+	return img, res.Body, nil
+}
+
+func (graph *Graph) PullImage(imgId string) error {
+	history, err := graph.getRemoteHistory(imgId)
+	if err != nil {
+		return err
+	}
+	// FIXME: Try to stream the images?
+	// FIXME: Lunch the getRemoteImage() in goroutines
+	for _, j := range history {
+		if !graph.Exists(j.Id) {
+			img, layer, err := graph.getRemoteImage(j.Id)
+			if err != nil {
+				// FIXME: Keep goging in case of error?
+				return err
+			}
+			if err = graph.Register(layer, img); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+// Push a local image to the registry with its history if needed
+func (graph *Graph) PushImage(imgOrig *Image) error {
+	client := &http.Client{}
+
+	// FIXME: Factorize the code
+	// FIXME: Do the puts in goroutines
+	if err := imgOrig.WalkHistory(func(img *Image) error {
+
+		jsonRaw, err := ioutil.ReadFile(path.Join(graph.Root, img.Id, "json"))
+		if err != nil {
+			return fmt.Errorf("Error while retreiving the path for {%s}: %s", img.Id, err)
+		}
+		// FIXME: try json with URF8
+		jsonData := strings.NewReader(string(jsonRaw))
+		req, err := http.NewRequest("PUT", REGISTRY_ENDPOINT+"/images/"+img.Id+"/json", jsonData)
+		res, err := client.Do(req)
+		if err != nil || res.StatusCode != 200 {
+			if res == nil {
+				return fmt.Errorf(
+					"Error: Internal server error trying to push image {%s} (json): %s",
+					img.Id, err)
+			}
+			switch res.StatusCode {
+			case 204:
+				// Case where the image is already on the Registry
+				// FIXME: Do not be silent?
+				return nil
+			case 400:
+				return fmt.Errorf("Error: Invalid Json")
+			default:
+				return fmt.Errorf(
+					"Error: Internal server error trying to push image {%s} (json): %s (%d)\n",
+					img.Id, err, res.StatusCode)
+			}
+		}
+
+		req2, err := http.NewRequest("PUT", REGISTRY_ENDPOINT+"/images/"+img.Id+"/layer", nil)
+		res2, err := client.Do(req2)
+		if err != nil || res2.StatusCode != 307 {
+			return fmt.Errorf(
+				"Error trying to push image {%s} (layer 1): %s\n",
+				img.Id, err)
+		}
+		url, err := res2.Location()
+		if err != nil || url == nil {
+			return fmt.Errorf(
+				"Fail to retrieve layer storage URL for image {%s}: %s\n",
+				img.Id, err)
+		}
+		// FIXME: Don't do this :D. Check the S3 requierement and implement chunks of 5MB
+		// FIXME2: I won't stress it enough, DON'T DO THIS! very high priority
+		layerData2, err := Tar(path.Join(graph.Root, img.Id, "layer"), Gzip)
+		layerData, err := Tar(path.Join(graph.Root, img.Id, "layer"), Gzip)
+		if err != nil {
+			return fmt.Errorf(
+				"Error while retrieving layer for {%s}: %s\n",
+				img.Id, err)
+		}
+		req3, err := http.NewRequest("PUT", url.String(), layerData)
+		tmp, _ := ioutil.ReadAll(layerData2)
+		req3.ContentLength = int64(len(tmp))
+
+		req3.TransferEncoding = []string{"none"}
+		res3, err := client.Do(req3)
+		if err != nil || res3.StatusCode != 200 {
+			if res3 == nil {
+				return fmt.Errorf(
+					"Error trying to push image {%s} (layer 2): %s\n",
+					img.Id, err)
+			} else {
+				return fmt.Errorf(
+					"Error trying to push image {%s} (layer 2): %s (%d)\n",
+					img.Id, err, res3.StatusCode)
+			}
+		}
+		return nil
+	}); err != nil {
+		return err
+	}
+	return nil
+}