Merge pull request #700 from dotcloud/615-pushbyid

Allow to push/pull on independent registries (by repository or image ID)

commands.go

@@ -736,23 +736,30 @@ func (cli *DockerCli) CmdPush(args ...string) error {
 		return err
 	}
 
-	if len(strings.SplitN(name, "/", 2)) == 1 {
-		return fmt.Errorf("Impossible to push a \"root\" repository. Please rename your repository in <user>/<repo> (ex: %s/%s)", cli.authConfig.Username, name)
+	if *registry == "" {
+		// If we're not using a custom registry, we know the restrictions
+		// applied to repository names and can warn the user in advance.
+		// Custom repositories can have different rules, and we must also
+		// allow pushing by image ID.
+		if len(strings.SplitN(name, "/", 2)) == 1 {
+			return fmt.Errorf("Impossible to push a \"root\" repository. Please rename your repository in <user>/<repo> (ex: %s/%s)", cli.authConfig.Username, name)
+		}
+
+		nameParts := strings.SplitN(name, "/", 2)
+		validNamespace := regexp.MustCompile(`^([a-z0-9_]{4,30})$`)
+		if !validNamespace.MatchString(nameParts[0]) {
+			return fmt.Errorf("Invalid namespace name (%s), only [a-z0-9_] are allowed, size between 4 and 30", nameParts[0])
+		}
+		validRepo := regexp.MustCompile(`^([a-zA-Z0-9-_.]+)$`)
+		if !validRepo.MatchString(nameParts[1]) {
+			return fmt.Errorf("Invalid repository name (%s), only [a-zA-Z0-9-_.] are allowed", nameParts[1])
+		}
 	}
 
 	buf, err := json.Marshal(cli.authConfig)
 	if err != nil {
 		return err
 	}
-	nameParts := strings.SplitN(name, "/", 2)
-	validNamespace := regexp.MustCompile(`^([a-z0-9_]{4,30})$`)
-	if !validNamespace.MatchString(nameParts[0]) {
-		return fmt.Errorf("Invalid namespace name (%s), only [a-z0-9_] are allowed, size between 4 and 30", nameParts[0])
-	}
-	validRepo := regexp.MustCompile(`^([a-zA-Z0-9-_.]+)$`)
-	if !validRepo.MatchString(nameParts[1]) {
-		return fmt.Errorf("Invalid repository name (%s), only [a-zA-Z0-9-_.] are allowed", nameParts[1])
-	}
 
 	v := url.Values{}
 	v.Set("registry", *registry)

registry/registry.go

@@ -56,20 +56,19 @@ func (r *Registry) GetRemoteHistory(imgId, registry string, token []string) ([]s
 }
 
 // Check if an image exists in the Registry
-func (r *Registry) LookupRemoteImage(imgId, registry string, authConfig *auth.AuthConfig) bool {
+func (r *Registry) LookupRemoteImage(imgId, registry string, token []string) bool {
 	rt := &http.Transport{Proxy: http.ProxyFromEnvironment}
 
 	req, err := http.NewRequest("GET", registry+"/images/"+imgId+"/json", nil)
 	if err != nil {
 		return false
 	}
-	req.SetBasicAuth(authConfig.Username, authConfig.Password)
 	res, err := rt.RoundTrip(req)
 	if err != nil {
 		return false
 	}
 	res.Body.Close()
-	return res.StatusCode == 307
+	return res.StatusCode == 200
 }
 
 func (r *Registry) getImagesInRepository(repository string, authConfig *auth.AuthConfig) ([]map[string]string, error) {
@@ -155,7 +154,10 @@ func (r *Registry) GetRemoteTags(registries []string, repository string, token [
 		repository = "library/" + repository
 	}
 	for _, host := range registries {
-		endpoint := fmt.Sprintf("https://%s/v1/repositories/%s/tags", host, repository)
+		endpoint := fmt.Sprintf("%s/v1/repositories/%s/tags", host, repository)
+		if !(strings.HasPrefix(endpoint, "http://") || strings.HasPrefix(endpoint, "https://")) {
+				endpoint = "https://" + endpoint
+			}
 		req, err := r.opaqueRequest("GET", endpoint, nil)
 		if err != nil {
 			return nil, err
@@ -165,6 +167,7 @@ func (r *Registry) GetRemoteTags(registries []string, repository string, token [
 		if err != nil {
 			return nil, err
 		}
+
 		utils.Debugf("Got status code %d from %s", res.StatusCode, endpoint)
 		defer res.Body.Close()
 
@@ -249,7 +252,7 @@ func (r *Registry) GetRepositoryData(remote string) (*RepositoryData, error) {
 
 // Push a local image to the registry
 func (r *Registry) PushImageJSONRegistry(imgData *ImgData, jsonRaw []byte, registry string, token []string) error {
-	registry = "https://" + registry + "/v1"
+	registry = registry + "/v1"
 	// FIXME: try json with UTF8
 	req, err := http.NewRequest("PUT", registry+"/images/"+imgData.ID+"/json", strings.NewReader(string(jsonRaw)))
 	if err != nil {
@@ -285,7 +288,7 @@ func (r *Registry) PushImageJSONRegistry(imgData *ImgData, jsonRaw []byte, regis
 }
 
 func (r *Registry) PushImageLayerRegistry(imgId string, layer io.Reader, registry string, token []string) error {
-	registry = "https://" + registry + "/v1"
+	registry = registry + "/v1"
 	req, err := http.NewRequest("PUT", registry+"/images/"+imgId+"/layer", layer)
 	if err != nil {
 		return err
@@ -323,7 +326,7 @@ func (r *Registry) opaqueRequest(method, urlStr string, body io.Reader) (*http.R
 func (r *Registry) PushRegistryTag(remote, revision, tag, registry string, token []string) error {
 	// "jsonify" the string
 	revision = "\"" + revision + "\""
-	registry = "https://" + registry + "/v1"
+	registry = registry + "/v1"
 
 	req, err := r.opaqueRequest("PUT", registry+"/repositories/"+remote+"/tags/"+tag, strings.NewReader(revision))
 	if err != nil {

server.go

@@ -351,26 +351,49 @@ func (srv *Server) pullImage(r *registry.Registry, out io.Writer, imgId, endpoin
 	return nil
 }
 
-func (srv *Server) pullRepository(r *registry.Registry, out io.Writer, local, remote, askedTag string, sf *utils.StreamFormatter) error {
+func (srv *Server) pullRepository(r *registry.Registry, out io.Writer, local, remote, askedTag, registryEp string, sf *utils.StreamFormatter) error {
 	out.Write(sf.FormatStatus("Pulling repository %s from %s", local, auth.IndexServerAddress()))
-	repoData, err := r.GetRepositoryData(remote)
-	if err != nil {
-		return err
-	}
 
-	utils.Debugf("Updating checksums")
-	// Reload the json file to make sure not to overwrite faster sums
-	if err := srv.runtime.graph.UpdateChecksums(repoData.ImgList); err != nil {
-		return err
+	var repoData *registry.RepositoryData
+	var err error
+	if registryEp == "" {
+		repoData, err = r.GetRepositoryData(remote)
+		if err != nil {
+			return err
+		}
+
+		utils.Debugf("Updating checksums")
+		// Reload the json file to make sure not to overwrite faster sums
+		if err := srv.runtime.graph.UpdateChecksums(repoData.ImgList); err != nil {
+			return err
+		}
+	} else {
+		repoData = &registry.RepositoryData{
+			Tokens:    []string{},
+			ImgList:   make(map[string]*registry.ImgData),
+			Endpoints: []string{registryEp},
+		}
 	}
 
 	utils.Debugf("Retrieving the tag list")
 	tagsList, err := r.GetRemoteTags(repoData.Endpoints, remote, repoData.Tokens)
 	if err != nil {
+		utils.Debugf("%v", err)
 		return err
 	}
+
+	if registryEp != "" {
+		for tag, id := range tagsList {
+			repoData.ImgList[id] = &registry.ImgData{
+				ID:       id,
+				Tag:      tag,
+				Checksum: "",
+			}
+		}
+	}
+
 	utils.Debugf("Registering tags")
-	// If not specific tag have been asked, take all
+	// If no tag has been specified, pull them all
 	if askedTag == "" {
 		for tag, id := range tagsList {
 			repoData.ImgList[id].Tag = tag
@@ -392,7 +415,10 @@ func (srv *Server) pullRepository(r *registry.Registry, out io.Writer, local, re
 		out.Write(sf.FormatStatus("Pulling image %s (%s) from %s", img.ID, img.Tag, remote))
 		success := false
 		for _, ep := range repoData.Endpoints {
-			if err := srv.pullImage(r, out, img.ID, "https://"+ep+"/v1", repoData.Tokens, sf); err != nil {
+			if !(strings.HasPrefix(ep, "http://") || strings.HasPrefix(ep, "https://")) {
+				ep = "https://" + ep
+			}
+			if err := srv.pullImage(r, out, img.ID, ep+"/v1", repoData.Tokens, sf); err != nil {
 				out.Write(sf.FormatStatus("Error while retrieving image for tag: %s (%s); checking next endpoint", askedTag, err))
 				continue
 			}
@@ -452,7 +478,6 @@ func (srv *Server) poolRemove(kind, key string) error {
 	}
 	return nil
 }
-
 func (srv *Server) ImagePull(name, tag, endpoint string, out io.Writer, sf *utils.StreamFormatter, authConfig *auth.AuthConfig) error {
 	r, err := registry.NewRegistry(srv.runtime.root, authConfig)
 	if err != nil {
@@ -463,21 +488,20 @@ func (srv *Server) ImagePull(name, tag, endpoint string, out io.Writer, sf *util
 	}
 	defer srv.poolRemove("pull", name+":"+tag)
 
-	out = utils.NewWriteFlusher(out)
-	if endpoint != "" {
-		if err := srv.pullImage(r, out, name, endpoint, nil, sf); err != nil {
-			return err
-		}
-		return nil
-	}
 	remote := name
 	parts := strings.Split(name, "/")
 	if len(parts) > 2 {
 		remote = fmt.Sprintf("src/%s", url.QueryEscape(strings.Join(parts, "/")))
 	}
-	if err := srv.pullRepository(r, out, name, remote, tag, sf); err != nil {
-		return err
+	out = utils.NewWriteFlusher(out)
+	err = srv.pullRepository(r, out, name, remote, tag, endpoint, sf)
+	if err != nil && endpoint != "" {
+		if err := srv.pullImage(r, out, name, endpoint, nil, sf); err != nil {
+			return err
+		}
+		return nil
 	}
+
 	return nil
 }
 
@@ -547,7 +571,7 @@ func (srv *Server) getImageList(localRepo map[string]string) ([]*registry.ImgDat
 	return imgList, nil
 }
 
-func (srv *Server) pushRepository(r *registry.Registry, out io.Writer, name string, localRepo map[string]string, sf *utils.StreamFormatter) error {
+func (srv *Server) pushRepository(r *registry.Registry, out io.Writer, name, registryEp string, localRepo map[string]string, sf *utils.StreamFormatter) error {
 	out = utils.NewWriteFlusher(out)
 	out.Write(sf.FormatStatus("Processing checksums"))
 	imgList, err := srv.getImageList(localRepo)
@@ -555,25 +579,51 @@ func (srv *Server) pushRepository(r *registry.Registry, out io.Writer, name stri
 		return err
 	}
 	out.Write(sf.FormatStatus("Sending image list"))
-
 	srvName := name
 	parts := strings.Split(name, "/")
 	if len(parts) > 2 {
 		srvName = fmt.Sprintf("src/%s", url.QueryEscape(strings.Join(parts, "/")))
 	}
 
-	repoData, err := r.PushImageJSONIndex(srvName, imgList, false, nil)
-	if err != nil {
-		return err
+	var repoData *registry.RepositoryData
+	if registryEp == "" {
+		repoData, err = r.PushImageJSONIndex(name, imgList, false, nil)
+		if err != nil {
+			return err
+		}
+	} else {
+		repoData = &registry.RepositoryData{
+			ImgList:   make(map[string]*registry.ImgData),
+			Tokens:    []string{},
+			Endpoints: []string{registryEp},
+		}
+		tagsList, err := r.GetRemoteTags(repoData.Endpoints, name, repoData.Tokens)
+		if err != nil && err.Error() != "Repository not found" {
+			return err
+		} else if err == nil {
+			for tag, id := range tagsList {
+				repoData.ImgList[id] = &registry.ImgData{
+					ID:       id,
+					Tag:      tag,
+					Checksum: "",
+				}
+			}
+		}
 	}
 
 	for _, ep := range repoData.Endpoints {
+		if !(strings.HasPrefix(ep, "http://") || strings.HasPrefix(ep, "https://")) {
+			ep = "https://" + ep
+		}
 		out.Write(sf.FormatStatus("Pushing repository %s to %s (%d tags)", name, ep, len(localRepo)))
 		// For each image within the repo, push them
 		for _, elem := range imgList {
 			if _, exists := repoData.ImgList[elem.ID]; exists {
 				out.Write(sf.FormatStatus("Image %s already on registry, skipping", name))
 				continue
+			} else if registryEp != "" && r.LookupRemoteImage(elem.ID, registryEp, repoData.Tokens) {
+				fmt.Fprintf(out, "Image %s already on registry, skipping\n", name)
+				continue
 			}
 			if err := srv.pushImage(r, out, name, elem.ID, ep, repoData.Tokens, sf); err != nil {
 				// FIXME: Continue on error?
@@ -586,9 +636,12 @@ func (srv *Server) pushRepository(r *registry.Registry, out io.Writer, name stri
 		}
 	}
 
-	if _, err := r.PushImageJSONIndex(srvName, imgList, true, repoData.Endpoints); err != nil {
-		return err
+	if registryEp == "" {
+		if _, err := r.PushImageJSONIndex(name, imgList, true, repoData.Endpoints); err != nil {
+			return err
+		}
 	}
+
 	return nil
 }
 
@@ -665,11 +718,12 @@ func (srv *Server) ImagePush(name, endpoint string, out io.Writer, sf *utils.Str
 	if err2 != nil {
 		return err2
 	}
+
 	if err != nil {
 		out.Write(sf.FormatStatus("The push refers to a repository [%s] (len: %d)", name, len(srv.runtime.repositories.Repositories[name])))
 		// If it fails, try to get the repository
 		if localRepo, exists := srv.runtime.repositories.Repositories[name]; exists {
-			if err := srv.pushRepository(r, out, name, localRepo, sf); err != nil {
+			if err := srv.pushRepository(r, out, name, endpoint, localRepo, sf); err != nil {
 				return err
 			}
 			return nil