Merge pull request #17924 from aaronlehmann/content-addressability

Content addressability

api/client/build.go

@@ -18,16 +18,15 @@ import (
 	"strconv"
 	"strings"
 
+	"github.com/docker/distribution/reference"
 	"github.com/docker/docker/api"
 	Cli "github.com/docker/docker/cli"
-	"github.com/docker/docker/graph/tags"
 	"github.com/docker/docker/opts"
 	"github.com/docker/docker/pkg/archive"
 	"github.com/docker/docker/pkg/fileutils"
 	"github.com/docker/docker/pkg/httputils"
 	"github.com/docker/docker/pkg/jsonmessage"
 	flag "github.com/docker/docker/pkg/mflag"
-	"github.com/docker/docker/pkg/parsers"
 	"github.com/docker/docker/pkg/progressreader"
 	"github.com/docker/docker/pkg/streamformatter"
 	"github.com/docker/docker/pkg/ulimit"
@@ -35,6 +34,7 @@ import (
 	"github.com/docker/docker/pkg/urlutil"
 	"github.com/docker/docker/registry"
 	"github.com/docker/docker/runconfig"
+	tagpkg "github.com/docker/docker/tag"
 	"github.com/docker/docker/utils"
 )
 
@@ -323,7 +323,7 @@ func (cli *DockerCli) CmdBuild(args ...string) error {
 	// Since the build was successful, now we must tag any of the resolved
 	// images from the above Dockerfile rewrite.
 	for _, resolved := range resolvedTags {
-		if err := cli.tagTrusted(resolved.repoInfo, resolved.digestRef, resolved.tagRef); err != nil {
+		if err := cli.tagTrusted(resolved.digestRef, resolved.tagRef); err != nil {
 			return err
 		}
 	}
@@ -333,16 +333,12 @@ func (cli *DockerCli) CmdBuild(args ...string) error {
 
 // validateTag checks if the given image name can be resolved.
 func validateTag(rawRepo string) (string, error) {
-	repository, tag := parsers.ParseRepositoryTag(rawRepo)
-	if err := registry.ValidateRepositoryName(repository); err != nil {
+	ref, err := reference.ParseNamed(rawRepo)
+	if err != nil {
 		return "", err
 	}
 
-	if len(tag) == 0 {
-		return rawRepo, nil
-	}
-
-	if err := tags.ValidateTagName(tag); err != nil {
+	if err := registry.ValidateRepositoryName(ref); err != nil {
 		return "", err
 	}
 
@@ -565,15 +561,16 @@ func (td *trustedDockerfile) Close() error {
 // resolvedTag records the repository, tag, and resolved digest reference
 // from a Dockerfile rewrite.
 type resolvedTag struct {
-	repoInfo          *registry.RepositoryInfo
-	digestRef, tagRef registry.Reference
+	repoInfo  *registry.RepositoryInfo
+	digestRef reference.Canonical
+	tagRef    reference.NamedTagged
 }
 
 // rewriteDockerfileFrom rewrites the given Dockerfile by resolving images in
 // "FROM <image>" instructions to a digest reference. `translator` is a
 // function that takes a repository name and tag reference and returns a
 // trusted digest reference.
-func rewriteDockerfileFrom(dockerfileName string, translator func(string, registry.Reference) (registry.Reference, error)) (newDockerfile *trustedDockerfile, resolvedTags []*resolvedTag, err error) {
+func rewriteDockerfileFrom(dockerfileName string, translator func(reference.NamedTagged) (reference.Canonical, error)) (newDockerfile *trustedDockerfile, resolvedTags []*resolvedTag, err error) {
 	dockerfile, err := os.Open(dockerfileName)
 	if err != nil {
 		return nil, nil, fmt.Errorf("unable to open Dockerfile: %v", err)
@@ -607,29 +604,39 @@ func rewriteDockerfileFrom(dockerfileName string, translator func(string, regist
 		matches := dockerfileFromLinePattern.FindStringSubmatch(line)
 		if matches != nil && matches[1] != "scratch" {
 			// Replace the line with a resolved "FROM repo@digest"
-			repo, tag := parsers.ParseRepositoryTag(matches[1])
-			if tag == "" {
-				tag = tags.DefaultTag
+			ref, err := reference.ParseNamed(matches[1])
+			if err != nil {
+				return nil, nil, err
 			}
 
-			repoInfo, err := registry.ParseRepositoryInfo(repo)
-			if err != nil {
-				return nil, nil, fmt.Errorf("unable to parse repository info %q: %v", repo, err)
+			digested := false
+			switch ref.(type) {
+			case reference.Tagged:
+			case reference.Digested:
+				digested = true
+			default:
+				ref, err = reference.WithTag(ref, tagpkg.DefaultTag)
+				if err != nil {
+					return nil, nil, err
+				}
 			}
 
-			ref := registry.ParseReference(tag)
+			repoInfo, err := registry.ParseRepositoryInfo(ref)
+			if err != nil {
+				return nil, nil, fmt.Errorf("unable to parse repository info %q: %v", ref.String(), err)
+			}
 
-			if !ref.HasDigest() && isTrusted() {
-				trustedRef, err := translator(repo, ref)
+			if !digested && isTrusted() {
+				trustedRef, err := translator(ref.(reference.NamedTagged))
 				if err != nil {
 					return nil, nil, err
 				}
 
-				line = dockerfileFromLinePattern.ReplaceAllLiteralString(line, fmt.Sprintf("FROM %s", trustedRef.ImageName(repo)))
+				line = dockerfileFromLinePattern.ReplaceAllLiteralString(line, fmt.Sprintf("FROM %s", trustedRef.String()))
 				resolvedTags = append(resolvedTags, &resolvedTag{
 					repoInfo:  repoInfo,
 					digestRef: trustedRef,
-					tagRef:    ref,
+					tagRef:    ref.(reference.NamedTagged),
 				})
 			}
 		}

api/client/commit.go

@@ -2,14 +2,15 @@ package client
 
 import (
 	"encoding/json"
+	"errors"
 	"fmt"
 	"net/url"
 
+	"github.com/docker/distribution/reference"
 	"github.com/docker/docker/api/types"
 	Cli "github.com/docker/docker/cli"
 	"github.com/docker/docker/opts"
 	flag "github.com/docker/docker/pkg/mflag"
-	"github.com/docker/docker/pkg/parsers"
 	"github.com/docker/docker/registry"
 	"github.com/docker/docker/runconfig"
 )
@@ -32,20 +33,35 @@ func (cli *DockerCli) CmdCommit(args ...string) error {
 	cmd.ParseFlags(args, true)
 
 	var (
-		name            = cmd.Arg(0)
-		repository, tag = parsers.ParseRepositoryTag(cmd.Arg(1))
+		name             = cmd.Arg(0)
+		repositoryAndTag = cmd.Arg(1)
+		repositoryName   string
+		tag              string
 	)
 
 	//Check if the given image name can be resolved
-	if repository != "" {
-		if err := registry.ValidateRepositoryName(repository); err != nil {
+	if repositoryAndTag != "" {
+		ref, err := reference.ParseNamed(repositoryAndTag)
+		if err != nil {
 			return err
 		}
+		if err := registry.ValidateRepositoryName(ref); err != nil {
+			return err
+		}
+
+		repositoryName = ref.Name()
+
+		switch x := ref.(type) {
+		case reference.Digested:
+			return errors.New("cannot commit to digest reference")
+		case reference.Tagged:
+			tag = x.Tag()
+		}
 	}
 
 	v := url.Values{}
 	v.Set("container", name)
-	v.Set("repo", repository)
+	v.Set("repo", repositoryName)
 	v.Set("tag", tag)
 	v.Set("comment", *flComment)
 	v.Set("author", *flAuthor)

api/client/create.go

@@ -9,12 +9,12 @@ import (
 	"os"
 	"strings"
 
+	"github.com/docker/distribution/reference"
 	"github.com/docker/docker/api/types"
 	Cli "github.com/docker/docker/cli"
-	"github.com/docker/docker/graph/tags"
-	"github.com/docker/docker/pkg/parsers"
 	"github.com/docker/docker/registry"
 	"github.com/docker/docker/runconfig"
+	tagpkg "github.com/docker/docker/tag"
 )
 
 func (cli *DockerCli) pullImage(image string) error {
@@ -23,16 +23,28 @@ func (cli *DockerCli) pullImage(image string) error {
 
 func (cli *DockerCli) pullImageCustomOut(image string, out io.Writer) error {
 	v := url.Values{}
-	repos, tag := parsers.ParseRepositoryTag(image)
-	// pull only the image tagged 'latest' if no tag was specified
-	if tag == "" {
-		tag = tags.DefaultTag
+
+	ref, err := reference.ParseNamed(image)
+	if err != nil {
+		return err
+	}
+
+	var tag string
+	switch x := ref.(type) {
+	case reference.Digested:
+		tag = x.Digest().String()
+	case reference.Tagged:
+		tag = x.Tag()
+	default:
+		// pull only the image tagged 'latest' if no tag was specified
+		tag = tagpkg.DefaultTag
 	}
-	v.Set("fromImage", repos)
+
+	v.Set("fromImage", ref.Name())
 	v.Set("tag", tag)
 
 	// Resolve the Repository name from fqn to RepositoryInfo
-	repoInfo, err := registry.ParseRepositoryInfo(repos)
+	repoInfo, err := registry.ParseRepositoryInfo(ref)
 	if err != nil {
 		return err
 	}
@@ -94,39 +106,46 @@ func (cli *DockerCli) createContainer(config *runconfig.Config, hostConfig *runc
 		defer containerIDFile.Close()
 	}
 
-	repo, tag := parsers.ParseRepositoryTag(config.Image)
-	if tag == "" {
-		tag = tags.DefaultTag
+	ref, err := reference.ParseNamed(config.Image)
+	if err != nil {
+		return nil, err
 	}
 
-	ref := registry.ParseReference(tag)
-	var trustedRef registry.Reference
+	isDigested := false
+	switch ref.(type) {
+	case reference.Tagged:
+	case reference.Digested:
+		isDigested = true
+	default:
+		ref, err = reference.WithTag(ref, tagpkg.DefaultTag)
+		if err != nil {
+			return nil, err
+		}
+	}
 
-	if isTrusted() && !ref.HasDigest() {
+	var trustedRef reference.Canonical
+
+	if isTrusted() && !isDigested {
 		var err error
-		trustedRef, err = cli.trustedReference(repo, ref)
+		trustedRef, err = cli.trustedReference(ref.(reference.NamedTagged))
 		if err != nil {
 			return nil, err
 		}
-		config.Image = trustedRef.ImageName(repo)
+		config.Image = trustedRef.String()
 	}
 
 	//create the container
 	serverResp, err := cli.call("POST", "/containers/create?"+containerValues.Encode(), mergedConfig, nil)
 	//if image not found try to pull it
 	if serverResp.statusCode == 404 && strings.Contains(err.Error(), config.Image) {
-		fmt.Fprintf(cli.err, "Unable to find image '%s' locally\n", ref.ImageName(repo))
+		fmt.Fprintf(cli.err, "Unable to find image '%s' locally\n", ref.String())
 
 		// we don't want to write to stdout anything apart from container.ID
 		if err = cli.pullImageCustomOut(config.Image, cli.err); err != nil {
 			return nil, err
 		}
-		if trustedRef != nil && !ref.HasDigest() {
-			repoInfo, err := registry.ParseRepositoryInfo(repo)
-			if err != nil {
-				return nil, err
-			}
-			if err := cli.tagTrusted(repoInfo, trustedRef, ref); err != nil {
+		if trustedRef != nil && !isDigested {
+			if err := cli.tagTrusted(trustedRef, ref.(reference.NamedTagged)); err != nil {
 				return nil, err
 			}
 		}

api/client/images.go

@@ -4,18 +4,18 @@ import (
 	"encoding/json"
 	"fmt"
 	"net/url"
+	"strings"
 	"text/tabwriter"
 	"time"
 
+	"github.com/docker/distribution/reference"
 	"github.com/docker/docker/api/types"
 	Cli "github.com/docker/docker/cli"
 	"github.com/docker/docker/opts"
 	flag "github.com/docker/docker/pkg/mflag"
-	"github.com/docker/docker/pkg/parsers"
 	"github.com/docker/docker/pkg/parsers/filters"
 	"github.com/docker/docker/pkg/stringid"
 	"github.com/docker/docker/pkg/units"
-	"github.com/docker/docker/utils"
 )
 
 // CmdImages lists the images in a specified repository, or all top-level images if no repository is specified.
@@ -78,9 +78,9 @@ func (cli *DockerCli) CmdImages(args ...string) error {
 	w := tabwriter.NewWriter(cli.out, 20, 1, 3, ' ', 0)
 	if !*quiet {
 		if *showDigests {
-			fmt.Fprintln(w, "REPOSITORY\tTAG\tDIGEST\tIMAGE ID\tCREATED\tVIRTUAL SIZE")
+			fmt.Fprintln(w, "REPOSITORY\tTAG\tDIGEST\tIMAGE ID\tCREATED\tSIZE")
 		} else {
-			fmt.Fprintln(w, "REPOSITORY\tTAG\tIMAGE ID\tCREATED\tVIRTUAL SIZE")
+			fmt.Fprintln(w, "REPOSITORY\tTAG\tIMAGE ID\tCREATED\tSIZE")
 		}
 	}
 
@@ -101,21 +101,31 @@ func (cli *DockerCli) CmdImages(args ...string) error {
 		// combine the tags and digests lists
 		tagsAndDigests := append(repoTags, repoDigests...)
 		for _, repoAndRef := range tagsAndDigests {
-			repo, ref := parsers.ParseRepositoryTag(repoAndRef)
-			// default tag and digest to none - if there's a value, it'll be set below
+			// default repo, tag, and digest to none - if there's a value, it'll be set below
+			repo := "<none>"
 			tag := "<none>"
 			digest := "<none>"
-			if utils.DigestReference(ref) {
-				digest = ref
-			} else {
-				tag = ref
+
+			if !strings.HasPrefix(repoAndRef, "<none>") {
+				ref, err := reference.ParseNamed(repoAndRef)
+				if err != nil {
+					return err
+				}
+				repo = ref.Name()
+
+				switch x := ref.(type) {
+				case reference.Digested:
+					digest = x.Digest().String()
+				case reference.Tagged:
+					tag = x.Tag()
+				}
 			}
 
 			if !*quiet {
 				if *showDigests {
-					fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s ago\t%s\n", repo, tag, digest, ID, units.HumanDuration(time.Now().UTC().Sub(time.Unix(int64(image.Created), 0))), units.HumanSize(float64(image.VirtualSize)))
+					fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s ago\t%s\n", repo, tag, digest, ID, units.HumanDuration(time.Now().UTC().Sub(time.Unix(int64(image.Created), 0))), units.HumanSize(float64(image.Size)))
 				} else {
-					fmt.Fprintf(w, "%s\t%s\t%s\t%s ago\t%s\n", repo, tag, ID, units.HumanDuration(time.Now().UTC().Sub(time.Unix(int64(image.Created), 0))), units.HumanSize(float64(image.VirtualSize)))
+					fmt.Fprintf(w, "%s\t%s\t%s\t%s ago\t%s\n", repo, tag, ID, units.HumanDuration(time.Now().UTC().Sub(time.Unix(int64(image.Created), 0))), units.HumanSize(float64(image.Size)))
 				}
 			} else {
 				fmt.Fprintln(w, ID)

api/client/import.go

@@ -6,10 +6,10 @@ import (
 	"net/url"
 	"os"
 
+	"github.com/docker/distribution/reference"
 	Cli "github.com/docker/docker/cli"
 	"github.com/docker/docker/opts"
 	flag "github.com/docker/docker/pkg/mflag"
-	"github.com/docker/docker/pkg/parsers"
 	"github.com/docker/docker/pkg/urlutil"
 	"github.com/docker/docker/registry"
 )
@@ -47,8 +47,11 @@ func (cli *DockerCli) CmdImport(args ...string) error {
 
 	if repository != "" {
 		//Check if the given image name can be resolved
-		repo, _ := parsers.ParseRepositoryTag(repository)
-		if err := registry.ValidateRepositoryName(repo); err != nil {
+		ref, err := reference.ParseNamed(repository)
+		if err != nil {
+			return err
+		}
+		if err := registry.ValidateRepositoryName(ref); err != nil {
 			return err
 		}
 	}

api/client/ps/custom.go

@@ -62,8 +62,8 @@ func (c *containerContext) Image() string {
 		return "<no image>"
 	}
 	if c.trunc {
-		if stringid.TruncateID(c.c.ImageID) == stringid.TruncateID(c.c.Image) {
-			return stringutils.Truncate(c.c.Image, 12)
+		if trunc := stringid.TruncateID(c.c.ImageID); trunc == stringid.TruncateID(c.c.Image) {
+			return trunc
 		}
 	}
 	return c.c.Image

api/client/pull.go

@@ -1,16 +1,19 @@
 package client
 
 import (
+	"errors"
 	"fmt"
 	"net/url"
 
+	"github.com/docker/distribution/reference"
 	Cli "github.com/docker/docker/cli"
-	"github.com/docker/docker/graph/tags"
 	flag "github.com/docker/docker/pkg/mflag"
-	"github.com/docker/docker/pkg/parsers"
 	"github.com/docker/docker/registry"
+	tagpkg "github.com/docker/docker/tag"
 )
 
+var errTagCantBeUsed = errors.New("tag can't be used with --all-tags/-a")
+
 // CmdPull pulls an image or a repository from the registry.
 //
 // Usage: docker pull [OPTIONS] IMAGENAME[:TAG|@DIGEST]
@@ -23,18 +26,38 @@ func (cli *DockerCli) CmdPull(args ...string) error {
 	cmd.ParseFlags(args, true)
 	remote := cmd.Arg(0)
 
-	taglessRemote, tag := parsers.ParseRepositoryTag(remote)
-	if tag == "" && !*allTags {
-		tag = tags.DefaultTag
-		fmt.Fprintf(cli.out, "Using default tag: %s\n", tag)
-	} else if tag != "" && *allTags {
-		return fmt.Errorf("tag can't be used with --all-tags/-a")
+	distributionRef, err := reference.ParseNamed(remote)
+	if err != nil {
+		return err
+	}
+
+	var tag string
+	switch x := distributionRef.(type) {
+	case reference.Digested:
+		if *allTags {
+			return errTagCantBeUsed
+		}
+		tag = x.Digest().String()
+	case reference.Tagged:
+		if *allTags {
+			return errTagCantBeUsed
+		}
+		tag = x.Tag()
+	default:
+		if !*allTags {
+			tag = tagpkg.DefaultTag
+			distributionRef, err = reference.WithTag(distributionRef, tag)
+			if err != nil {
+				return err
+			}
+			fmt.Fprintf(cli.out, "Using default tag: %s\n", tag)
+		}
 	}
 
 	ref := registry.ParseReference(tag)
 
 	// Resolve the Repository name from fqn to RepositoryInfo
-	repoInfo, err := registry.ParseRepositoryInfo(taglessRemote)
+	repoInfo, err := registry.ParseRepositoryInfo(distributionRef)
 	if err != nil {
 		return err
 	}
@@ -46,7 +69,7 @@ func (cli *DockerCli) CmdPull(args ...string) error {
 	}
 
 	v := url.Values{}
-	v.Set("fromImage", ref.ImageName(taglessRemote))
+	v.Set("fromImage", distributionRef.String())
 
 	_, _, err = cli.clientRequestAttemptLogin("POST", "/images/create?"+v.Encode(), nil, cli.out, repoInfo.Index, "pull")
 	return err

api/client/push.go

@@ -1,12 +1,13 @@
 package client
 
 import (
+	"errors"
 	"fmt"
 	"net/url"
 
+	"github.com/docker/distribution/reference"
 	Cli "github.com/docker/docker/cli"
 	flag "github.com/docker/docker/pkg/mflag"
-	"github.com/docker/docker/pkg/parsers"
 	"github.com/docker/docker/registry"
 )
 
@@ -20,10 +21,21 @@ func (cli *DockerCli) CmdPush(args ...string) error {
 
 	cmd.ParseFlags(args, true)
 
-	remote, tag := parsers.ParseRepositoryTag(cmd.Arg(0))
+	ref, err := reference.ParseNamed(cmd.Arg(0))
+	if err != nil {
+		return err
+	}
+
+	var tag string
+	switch x := ref.(type) {
+	case reference.Digested:
+		return errors.New("cannot push a digest reference")
+	case reference.Tagged:
+		tag = x.Tag()
+	}
 
 	// Resolve the Repository name from fqn to RepositoryInfo
-	repoInfo, err := registry.ParseRepositoryInfo(remote)
+	repoInfo, err := registry.ParseRepositoryInfo(ref)
 	if err != nil {
 		return err
 	}
@@ -48,6 +60,6 @@ func (cli *DockerCli) CmdPush(args ...string) error {
 	v := url.Values{}
 	v.Set("tag", tag)
 
-	_, _, err = cli.clientRequestAttemptLogin("POST", "/images/"+remote+"/push?"+v.Encode(), nil, cli.out, repoInfo.Index, "push")
+	_, _, err = cli.clientRequestAttemptLogin("POST", "/images/"+ref.Name()+"/push?"+v.Encode(), nil, cli.out, repoInfo.Index, "push")
 	return err
 }

api/client/search.go

@@ -10,7 +10,6 @@ import (
 
 	Cli "github.com/docker/docker/cli"
 	flag "github.com/docker/docker/pkg/mflag"
-	"github.com/docker/docker/pkg/parsers"
 	"github.com/docker/docker/pkg/stringutils"
 	"github.com/docker/docker/registry"
 )
@@ -38,10 +37,7 @@ func (cli *DockerCli) CmdSearch(args ...string) error {
 	v := url.Values{}
 	v.Set("term", name)
 
-	// Resolve the Repository name from fqn to hostname + name
-	taglessRemote, _ := parsers.ParseRepositoryTag(name)
-
-	indexInfo, err := registry.ParseIndexInfo(taglessRemote)
+	indexInfo, err := registry.ParseSearchIndexInfo(name)
 	if err != nil {
 		return err
 	}

api/client/tag.go

@@ -1,11 +1,12 @@
 package client
 
 import (
+	"errors"
 	"net/url"
 
+	"github.com/docker/distribution/reference"
 	Cli "github.com/docker/docker/cli"
 	flag "github.com/docker/docker/pkg/mflag"
-	"github.com/docker/docker/pkg/parsers"
 	"github.com/docker/docker/registry"
 )
 
@@ -19,16 +20,28 @@ func (cli *DockerCli) CmdTag(args ...string) error {
 
 	cmd.ParseFlags(args, true)
 
-	var (
-		repository, tag = parsers.ParseRepositoryTag(cmd.Arg(1))
-		v               = url.Values{}
-	)
+	v := url.Values{}
+	ref, err := reference.ParseNamed(cmd.Arg(1))
+	if err != nil {
+		return err
+	}
+
+	_, isDigested := ref.(reference.Digested)
+	if isDigested {
+		return errors.New("refusing to create a tag with a digest reference")
+	}
+
+	tag := ""
+	tagged, isTagged := ref.(reference.Tagged)
+	if isTagged {
+		tag = tagged.Tag()
+	}
 
 	//Check if the given image name can be resolved
-	if err := registry.ValidateRepositoryName(repository); err != nil {
+	if err := registry.ValidateRepositoryName(ref); err != nil {
 		return err
 	}
-	v.Set("repo", repository)
+	v.Set("repo", ref.Name())
 	v.Set("tag", tag)
 
 	if *force {

api/client/trust.go

@@ -19,6 +19,7 @@ import (
 
 	"github.com/Sirupsen/logrus"
 	"github.com/docker/distribution/digest"
+	"github.com/docker/distribution/reference"
 	"github.com/docker/distribution/registry/client/auth"
 	"github.com/docker/distribution/registry/client/transport"
 	"github.com/docker/docker/cliconfig"
@@ -163,12 +164,12 @@ func (cli *DockerCli) getNotaryRepository(repoInfo *registry.RepositoryInfo, aut
 	}
 
 	creds := simpleCredentialStore{auth: authConfig}
-	tokenHandler := auth.NewTokenHandler(authTransport, creds, repoInfo.CanonicalName, "push", "pull")
+	tokenHandler := auth.NewTokenHandler(authTransport, creds, repoInfo.CanonicalName.Name(), "push", "pull")
 	basicHandler := auth.NewBasicHandler(creds)
 	modifiers = append(modifiers, transport.RequestModifier(auth.NewAuthorizer(challengeManager, tokenHandler, basicHandler)))
 	tr := transport.NewTransport(base, modifiers...)
 
-	return client.NewNotaryRepository(cli.trustDirectory(), repoInfo.CanonicalName, server, tr, cli.getPassphraseRetriever())
+	return client.NewNotaryRepository(cli.trustDirectory(), repoInfo.CanonicalName.Name(), server, tr, cli.getPassphraseRetriever())
 }
 
 func convertTarget(t client.Target) (target, error) {
@@ -219,8 +220,8 @@ func (cli *DockerCli) getPassphraseRetriever() passphrase.Retriever {
 	}
 }
 
-func (cli *DockerCli) trustedReference(repo string, ref registry.Reference) (registry.Reference, error) {
-	repoInfo, err := registry.ParseRepositoryInfo(repo)
+func (cli *DockerCli) trustedReference(ref reference.NamedTagged) (reference.Canonical, error) {
+	repoInfo, err := registry.ParseRepositoryInfo(ref)
 	if err != nil {
 		return nil, err
 	}
@@ -234,7 +235,7 @@ func (cli *DockerCli) trustedReference(repo string, ref registry.Reference) (reg
 		return nil, err
 	}
 
-	t, err := notaryRepo.GetTargetByName(ref.String())
+	t, err := notaryRepo.GetTargetByName(ref.Tag())
 	if err != nil {
 		return nil, err
 	}
@@ -244,18 +245,17 @@ func (cli *DockerCli) trustedReference(repo string, ref registry.Reference) (reg
 
 	}
 
-	return registry.DigestReference(r.digest), nil
+	return reference.WithDigest(ref, r.digest)
 }
 
-func (cli *DockerCli) tagTrusted(repoInfo *registry.RepositoryInfo, trustedRef, ref registry.Reference) error {
-	fullName := trustedRef.ImageName(repoInfo.LocalName)
-	fmt.Fprintf(cli.out, "Tagging %s as %s\n", fullName, ref.ImageName(repoInfo.LocalName))
+func (cli *DockerCli) tagTrusted(trustedRef reference.Canonical, ref reference.NamedTagged) error {
+	fmt.Fprintf(cli.out, "Tagging %s as %s\n", trustedRef.String(), ref.String())
 	tv := url.Values{}
-	tv.Set("repo", repoInfo.LocalName)
-	tv.Set("tag", ref.String())
+	tv.Set("repo", trustedRef.Name())
+	tv.Set("tag", ref.Tag())
 	tv.Set("force", "1")
 
-	if _, _, err := readBody(cli.call("POST", "/images/"+fullName+"/tag?"+tv.Encode(), nil, nil)); err != nil {
+	if _, _, err := readBody(cli.call("POST", "/images/"+trustedRef.String()+"/tag?"+tv.Encode(), nil, nil)); err != nil {
 		return err
 	}
 
@@ -317,7 +317,7 @@ func (cli *DockerCli) trustedPull(repoInfo *registry.RepositoryInfo, ref registr
 		refs = append(refs, r)
 	}
 
-	v.Set("fromImage", repoInfo.LocalName)
+	v.Set("fromImage", repoInfo.LocalName.Name())
 	for i, r := range refs {
 		displayTag := r.reference.String()
 		if displayTag != "" {
@@ -333,7 +333,12 @@ func (cli *DockerCli) trustedPull(repoInfo *registry.RepositoryInfo, ref registr
 
 		// If reference is not trusted, tag by trusted reference
 		if !r.reference.HasDigest() {
-			if err := cli.tagTrusted(repoInfo, registry.DigestReference(r.digest), r.reference); err != nil {
+			tagged, err := reference.WithTag(repoInfo.LocalName, r.reference.String())
+			if err != nil {
+				return err
+			}
+			trustedRef, err := reference.WithDigest(repoInfo.LocalName, r.digest)
+			if err := cli.tagTrusted(trustedRef, tagged); err != nil {
 				return err
 
 			}
@@ -386,7 +391,7 @@ func (cli *DockerCli) trustedPush(repoInfo *registry.RepositoryInfo, tag string,
 	v := url.Values{}
 	v.Set("tag", tag)
 
-	_, _, err := cli.clientRequestAttemptLogin("POST", "/images/"+repoInfo.LocalName+"/push?"+v.Encode(), nil, streamOut, repoInfo.Index, "push")
+	_, _, err := cli.clientRequestAttemptLogin("POST", "/images/"+repoInfo.LocalName.Name()+"/push?"+v.Encode(), nil, streamOut, repoInfo.Index, "push")
 	// Close stream channel to finish target parsing
 	if err := streamOut.Close(); err != nil {
 		return err

api/server/router/local/image.go

@@ -10,6 +10,8 @@ import (
 	"strings"
 
 	"github.com/Sirupsen/logrus"
+	"github.com/docker/distribution/digest"
+	"github.com/docker/distribution/reference"
 	"github.com/docker/docker/api/server/httputils"
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/builder"
@@ -17,17 +19,14 @@ import (
 	"github.com/docker/docker/cliconfig"
 	"github.com/docker/docker/daemon/daemonbuilder"
 	derr "github.com/docker/docker/errors"
-	"github.com/docker/docker/graph"
-	"github.com/docker/docker/graph/tags"
 	"github.com/docker/docker/pkg/archive"
 	"github.com/docker/docker/pkg/chrootarchive"
 	"github.com/docker/docker/pkg/ioutils"
-	"github.com/docker/docker/pkg/parsers"
 	"github.com/docker/docker/pkg/progressreader"
 	"github.com/docker/docker/pkg/streamformatter"
 	"github.com/docker/docker/pkg/ulimit"
-	"github.com/docker/docker/registry"
 	"github.com/docker/docker/runconfig"
+	tagpkg "github.com/docker/docker/tag"
 	"github.com/docker/docker/utils"
 	"golang.org/x/net/context"
 )
@@ -110,26 +109,55 @@ func (s *router) postImagesCreate(ctx context.Context, w http.ResponseWriter, r
 	w.Header().Set("Content-Type", "application/json")
 
 	if image != "" { //pull
-		if tag == "" {
-			image, tag = parsers.ParseRepositoryTag(image)
-		}
-		metaHeaders := map[string][]string{}
-		for k, v := range r.Header {
-			if strings.HasPrefix(k, "X-Meta-") {
-				metaHeaders[k] = v
+		// Special case: "pull -a" may send an image name with a
+		// trailing :. This is ugly, but let's not break API
+		// compatibility.
+		image = strings.TrimSuffix(image, ":")
+
+		var ref reference.Named
+		ref, err = reference.ParseNamed(image)
+		if err == nil {
+			if tag != "" {
+				// The "tag" could actually be a digest.
+				var dgst digest.Digest
+				dgst, err = digest.ParseDigest(tag)
+				if err == nil {
+					ref, err = reference.WithDigest(ref, dgst)
+				} else {
+					ref, err = reference.WithTag(ref, tag)
+				}
+			}
+			if err == nil {
+				metaHeaders := map[string][]string{}
+				for k, v := range r.Header {
+					if strings.HasPrefix(k, "X-Meta-") {
+						metaHeaders[k] = v
+					}
+				}
+
+				err = s.daemon.PullImage(ref, metaHeaders, authConfig, output)
 			}
 		}
+	} else { //import
+		var newRef reference.Named
+		if repo != "" {
+			var err error
+			newRef, err = reference.ParseNamed(repo)
+			if err != nil {
+				return err
+			}
 
-		imagePullConfig := &graph.ImagePullConfig{
-			MetaHeaders: metaHeaders,
-			AuthConfig:  authConfig,
-			OutStream:   output,
-		}
+			switch newRef.(type) {
+			case reference.Digested:
+				return errors.New("cannot import digest reference")
+			}
 
-		err = s.daemon.PullImage(image, tag, imagePullConfig)
-	} else { //import
-		if tag == "" {
-			repo, tag = parsers.ParseRepositoryTag(repo)
+			if tag != "" {
+				newRef, err = reference.WithTag(newRef, tag)
+				if err != nil {
+					return err
+				}
+			}
 		}
 
 		src := r.Form.Get("fromSrc")
@@ -143,7 +171,7 @@ func (s *router) postImagesCreate(ctx context.Context, w http.ResponseWriter, r
 			return err
 		}
 
-		err = s.daemon.ImportImage(src, repo, tag, message, r.Body, output, newConfig)
+		err = s.daemon.ImportImage(src, newRef, message, r.Body, output, newConfig)
 	}
 	if err != nil {
 		if !output.Flushed() {
@@ -183,19 +211,25 @@ func (s *router) postImagesPush(ctx context.Context, w http.ResponseWriter, r *h
 		}
 	}
 
-	name := vars["name"]
+	ref, err := reference.ParseNamed(vars["name"])
+	if err != nil {
+		return err
+	}
+	tag := r.Form.Get("tag")
+	if tag != "" {
+		// Push by digest is not supported, so only tags are supported.
+		ref, err = reference.WithTag(ref, tag)
+		if err != nil {
+			return err
+		}
+	}
+
 	output := ioutils.NewWriteFlusher(w)
 	defer output.Close()
-	imagePushConfig := &graph.ImagePushConfig{
-		MetaHeaders: metaHeaders,
-		AuthConfig:  authConfig,
-		Tag:         r.Form.Get("tag"),
-		OutStream:   output,
-	}
 
 	w.Header().Set("Content-Type", "application/json")
 
-	if err := s.daemon.PushImage(name, imagePushConfig); err != nil {
+	if err := s.daemon.PushImage(ref, metaHeaders, authConfig, output); err != nil {
 		if !output.Flushed() {
 			return err
 		}
@@ -428,7 +462,7 @@ func (s *router) postBuild(ctx context.Context, w http.ResponseWriter, r *http.R
 	}
 
 	for _, rt := range repoAndTags {
-		if err := s.daemon.TagImage(rt.repo, rt.tag, string(imgID), true); err != nil {
+		if err := s.daemon.TagImage(rt, imgID, true); err != nil {
 			return errf(err)
 		}
 	}
@@ -436,43 +470,38 @@ func (s *router) postBuild(ctx context.Context, w http.ResponseWriter, r *http.R
 	return nil
 }
 
-// repoAndTag is a helper struct for holding the parsed repositories and tags of
-// the input "t" argument.
-type repoAndTag struct {
-	repo, tag string
-}
-
 // sanitizeRepoAndTags parses the raw "t" parameter received from the client
 // to a slice of repoAndTag.
 // It also validates each repoName and tag.
-func sanitizeRepoAndTags(names []string) ([]repoAndTag, error) {
+func sanitizeRepoAndTags(names []string) ([]reference.Named, error) {
 	var (
-		repoAndTags []repoAndTag
+		repoAndTags []reference.Named
 		// This map is used for deduplicating the "-t" paramter.
 		uniqNames = make(map[string]struct{})
 	)
 	for _, repo := range names {
-		name, tag := parsers.ParseRepositoryTag(repo)
-		if name == "" {
+		if repo == "" {
 			continue
 		}
 
-		if err := registry.ValidateRepositoryName(name); err != nil {
+		ref, err := reference.ParseNamed(repo)
+		if err != nil {
 			return nil, err
 		}
 
-		nameWithTag := name
-		if len(tag) > 0 {
-			if err := tags.ValidateTagName(tag); err != nil {
-				return nil, err
-			}
-			nameWithTag += ":" + tag
-		} else {
-			nameWithTag += ":" + tags.DefaultTag
+		if _, isDigested := ref.(reference.Digested); isDigested {
+			return nil, errors.New("build tag cannot be a digest")
+		}
+
+		if _, isTagged := ref.(reference.Tagged); !isTagged {
+			ref, err = reference.WithTag(ref, tagpkg.DefaultTag)
 		}
+
+		nameWithTag := ref.String()
+
 		if _, exists := uniqNames[nameWithTag]; !exists {
 			uniqNames[nameWithTag] = struct{}{}
-			repoAndTags = append(repoAndTags, repoAndTag{repo: name, tag: tag})
+			repoAndTags = append(repoAndTags, ref)
 		}
 	}
 	return repoAndTags, nil
@@ -484,7 +513,7 @@ func (s *router) getImagesJSON(ctx context.Context, w http.ResponseWriter, r *ht
 	}
 
 	// FIXME: The filter parameter could just be a match filter
-	images, err := s.daemon.ListImages(r.Form.Get("filters"), r.Form.Get("filter"), httputils.BoolValue(r, "all"))
+	images, err := s.daemon.Images(r.Form.Get("filters"), r.Form.Get("filter"), httputils.BoolValue(r, "all"))
 	if err != nil {
 		return err
 	}
@@ -508,9 +537,17 @@ func (s *router) postImagesTag(ctx context.Context, w http.ResponseWriter, r *ht
 	}
 	repo := r.Form.Get("repo")
 	tag := r.Form.Get("tag")
-	name := vars["name"]
+	newTag, err := reference.WithName(repo)
+	if err != nil {
+		return err
+	}
+	if tag != "" {
+		if newTag, err = reference.WithTag(newTag, tag); err != nil {
+			return err
+		}
+	}
 	force := httputils.BoolValue(r, "force")
-	if err := s.daemon.TagImage(repo, tag, name, force); err != nil {
+	if err := s.daemon.TagImage(newTag, vars["name"], force); err != nil {
 		return err
 	}
 	w.WriteHeader(http.StatusCreated)

builder/builder.go

@@ -125,7 +125,7 @@ type Docker interface {
 	// Remove removes a container specified by `id`.
 	Remove(id string, cfg *daemon.ContainerRmConfig) error
 	// Commit creates a new Docker image from an existing Docker container.
-	Commit(string, *daemon.ContainerCommitConfig) (*image.Image, error)
+	Commit(string, *daemon.ContainerCommitConfig) (string, error)
 	// Copy copies/extracts a source FileInfo to a destination path inside a container
 	// specified by a container object.
 	// TODO: make an Extract method instead of passing `decompress`

builder/dockerfile/builder.go

@@ -277,9 +277,9 @@ func Commit(containerName string, d *daemon.Daemon, c *CommitConfig) (string, er
 		MergeConfigs: true,
 	}
 
-	img, err := d.Commit(containerName, commitCfg)
+	imgID, err := d.Commit(containerName, commitCfg)
 	if err != nil {
 		return "", err
 	}
-	return img.ID, nil
+	return imgID, nil
 }

builder/dockerfile/internals.go

@@ -83,13 +83,13 @@ func (b *Builder) commit(id string, autoCmd *stringutils.StrSlice, comment strin
 	}
 
 	// Commit the container
-	image, err := b.docker.Commit(id, commitCfg)
+	imageID, err := b.docker.Commit(id, commitCfg)
 	if err != nil {
 		return err
 	}
-	b.docker.Retain(b.id, image.ID)
-	b.activeImages = append(b.activeImages, image.ID)
-	b.image = image.ID
+	b.docker.Retain(b.id, imageID)
+	b.activeImages = append(b.activeImages, imageID)
+	b.image = imageID
 	return nil
 }
 
@@ -412,7 +412,7 @@ func containsWildcards(name string) bool {
 }
 
 func (b *Builder) processImageFrom(img *image.Image) error {
-	b.image = img.ID
+	b.image = img.ID().String()
 
 	if img.Config != nil {
 		b.runConfig = img.Config

daemon/commit.go

@@ -1,10 +1,16 @@
 package daemon
 
 import (
+	"encoding/json"
 	"fmt"
 	"runtime"
+	"strings"
+	"time"
 
+	"github.com/docker/distribution/reference"
+	"github.com/docker/docker/dockerversion"
 	"github.com/docker/docker/image"
+	"github.com/docker/docker/layer"
 	"github.com/docker/docker/pkg/archive"
 	"github.com/docker/docker/pkg/ioutils"
 	"github.com/docker/docker/runconfig"
@@ -25,15 +31,15 @@ type ContainerCommitConfig struct {
 
 // Commit creates a new filesystem image from the current state of a container.
 // The image can optionally be tagged into a repository.
-func (daemon *Daemon) Commit(name string, c *ContainerCommitConfig) (*image.Image, error) {
+func (daemon *Daemon) Commit(name string, c *ContainerCommitConfig) (string, error) {
 	container, err := daemon.Get(name)
 	if err != nil {
-		return nil, err
+		return "", err
 	}
 
 	// It is not possible to commit a running container on Windows
 	if runtime.GOOS == "windows" && container.IsRunning() {
-		return nil, fmt.Errorf("Windows does not support commit of a running container")
+		return "", fmt.Errorf("Windows does not support commit of a running container")
 	}
 
 	if c.Pause && !container.isPaused() {
@@ -43,13 +49,13 @@ func (daemon *Daemon) Commit(name string, c *ContainerCommitConfig) (*image.Imag
 
 	if c.MergeConfigs {
 		if err := runconfig.Merge(c.Config, container.Config); err != nil {
-			return nil, err
+			return "", err
 		}
 	}
 
 	rwTar, err := daemon.exportContainerRw(container)
 	if err != nil {
-		return nil, err
+		return "", err
 	}
 	defer func() {
 		if rwTar != nil {
@@ -57,31 +63,99 @@ func (daemon *Daemon) Commit(name string, c *ContainerCommitConfig) (*image.Imag
 		}
 	}()
 
-	// Create a new image from the container's base layers + a new layer from container changes
-	img, err := daemon.graph.Create(rwTar, container.ID, container.ImageID, c.Comment, c.Author, container.Config, c.Config)
+	var history []image.History
+	rootFS := image.NewRootFS()
+
+	if container.ImageID != "" {
+		img, err := daemon.imageStore.Get(container.ImageID)
+		if err != nil {
+			return "", err
+		}
+		history = img.History
+		rootFS = img.RootFS
+	}
+
+	l, err := daemon.layerStore.Register(rwTar, rootFS.ChainID())
 	if err != nil {
-		return nil, err
+		return "", err
+	}
+	defer layer.ReleaseAndLog(daemon.layerStore, l)
+
+	h := image.History{
+		Author:     c.Author,
+		Created:    time.Now().UTC(),
+		CreatedBy:  strings.Join(container.Config.Cmd.Slice(), " "),
+		Comment:    c.Comment,
+		EmptyLayer: true,
+	}
+
+	if diffID := l.DiffID(); layer.DigestSHA256EmptyTar != diffID {
+		h.EmptyLayer = false
+		rootFS.Append(diffID)
+	}
+
+	history = append(history, h)
+
+	config, err := json.Marshal(&image.Image{
+		V1Image: image.V1Image{
+			DockerVersion:   dockerversion.Version,
+			Config:          c.Config,
+			Architecture:    runtime.GOARCH,
+			OS:              runtime.GOOS,
+			Container:       container.ID,
+			ContainerConfig: *container.Config,
+			Author:          c.Author,
+			Created:         h.Created,
+		},
+		RootFS:  rootFS,
+		History: history,
+	})
+
+	if err != nil {
+		return "", err
+	}
+
+	id, err := daemon.imageStore.Create(config)
+	if err != nil {
+		return "", err
+	}
+
+	if container.ImageID != "" {
+		if err := daemon.imageStore.SetParent(id, container.ImageID); err != nil {
+			return "", err
+		}
 	}
 
-	// Register the image if needed
 	if c.Repo != "" {
-		if err := daemon.repositories.Tag(c.Repo, c.Tag, img.ID, true); err != nil {
-			return img, err
+		newTag, err := reference.WithName(c.Repo) // todo: should move this to API layer
+		if err != nil {
+			return "", err
+		}
+		if c.Tag != "" {
+			if newTag, err = reference.WithTag(newTag, c.Tag); err != nil {
+				return "", err
+			}
+		}
+		if err := daemon.TagImage(newTag, id.String(), true); err != nil {
+			return "", err
 		}
 	}
 
 	daemon.LogContainerEvent(container, "commit")
-	return img, nil
+	return id.String(), nil
 }
 
 func (daemon *Daemon) exportContainerRw(container *Container) (archive.Archive, error) {
-	archive, err := daemon.diff(container)
+	if err := daemon.Mount(container); err != nil {
+		return nil, err
+	}
+
+	archive, err := container.rwlayer.TarStream()
 	if err != nil {
 		return nil, err
 	}
 	return ioutils.NewReadCloserWrapper(archive, func() error {
-			err := archive.Close()
-			return err
+			return daemon.layerStore.Unmount(container.ID)
 		}),
 		nil
 }

daemon/container.go

@@ -20,6 +20,8 @@ import (
 	"github.com/docker/docker/daemon/logger/jsonfilelog"
 	"github.com/docker/docker/daemon/network"
 	derr "github.com/docker/docker/errors"
+	"github.com/docker/docker/image"
+	"github.com/docker/docker/layer"
 	"github.com/docker/docker/pkg/nat"
 	"github.com/docker/docker/pkg/promise"
 	"github.com/docker/docker/pkg/signal"
@@ -29,6 +31,8 @@ import (
 	"github.com/docker/docker/volume"
 )
 
+const configFileName = "config.v2.json"
+
 var (
 	// ErrRootFSReadOnly is returned when a container
 	// rootfs is marked readonly.
@@ -43,12 +47,13 @@ type CommonContainer struct {
 	*State          `json:"State"` // Needed for remote api version <= 1.11
 	root            string         // Path to the "home" of the container, including metadata.
 	basefs          string         // Path to the graphdriver mountpoint
+	rwlayer         layer.RWLayer
 	ID              string
 	Created         time.Time
 	Path            string
 	Args            []string
 	Config          *runconfig.Config
-	ImageID         string `json:"Image"`
+	ImageID         image.ID `json:"Image"`
 	NetworkSettings *network.Settings
 	LogPath         string
 	Name            string
@@ -256,7 +261,7 @@ func (container *Container) hostConfigPath() (string, error) {
 }
 
 func (container *Container) jsonPath() (string, error) {
-	return container.getRootResourcePath("config.json")
+	return container.getRootResourcePath(configFileName)
 }
 
 // This directory is only usable when the container is running
@@ -301,7 +306,7 @@ func (container *Container) StartLogger(cfg runconfig.LogConfig) (logger.Logger,
 		ContainerName:       container.Name,
 		ContainerEntrypoint: container.Path,
 		ContainerArgs:       container.Args,
-		ContainerImageID:    container.ImageID,
+		ContainerImageID:    container.ImageID.String(),
 		ContainerImageName:  container.Config.Image,
 		ContainerCreated:    container.Created,
 		ContainerEnv:        container.Config.Env,

daemon/container_unit_test.go

@@ -99,7 +99,7 @@ func TestContainerInitDNS(t *testing.T) {
 "Name":"/ubuntu","Driver":"aufs","MountLabel":"","ProcessLabel":"","AppArmorProfile":"","RestartCount":0,
 "UpdateDns":false,"Volumes":{},"VolumesRW":{},"AppliedVolumesFrom":null}`
 
-	if err = ioutil.WriteFile(filepath.Join(containerPath, "config.json"), []byte(config), 0644); err != nil {
+	if err = ioutil.WriteFile(filepath.Join(containerPath, configFileName), []byte(config), 0644); err != nil {
 		t.Fatal(err)
 	}
 

daemon/container_unix.go

@@ -19,7 +19,6 @@ import (
 	"github.com/docker/docker/daemon/links"
 	"github.com/docker/docker/daemon/network"
 	derr "github.com/docker/docker/errors"
-	"github.com/docker/docker/pkg/directory"
 	"github.com/docker/docker/pkg/fileutils"
 	"github.com/docker/docker/pkg/idtools"
 	"github.com/docker/docker/pkg/mount"
@@ -388,8 +387,7 @@ func (daemon *Daemon) getSize(container *Container) (int64, int64) {
 	}
 	defer daemon.Unmount(container)
 
-	initID := fmt.Sprintf("%s-init", container.ID)
-	sizeRw, err = daemon.driver.DiffSize(container.ID, initID)
+	sizeRw, err = container.rwlayer.Size()
 	if err != nil {
 		logrus.Errorf("Driver %s couldn't return diff size of container %s: %s", daemon.driver, container.ID, err)
 		// FIXME: GetSize should return an error. Not changing it now in case
@@ -397,9 +395,12 @@ func (daemon *Daemon) getSize(container *Container) (int64, int64) {
 		sizeRw = -1
 	}
 
-	if _, err = os.Stat(container.basefs); err == nil {
-		if sizeRootfs, err = directory.Size(container.basefs); err != nil {
+	if parent := container.rwlayer.Parent(); parent != nil {
+		sizeRootfs, err = parent.Size()
+		if err != nil {
 			sizeRootfs = -1
+		} else if sizeRw != -1 {
+			sizeRootfs += sizeRw
 		}
 	}
 	return sizeRw, sizeRootfs

daemon/container_windows.go

@@ -7,6 +7,7 @@ import (
 
 	"github.com/docker/docker/daemon/execdriver"
 	derr "github.com/docker/docker/errors"
+	"github.com/docker/docker/layer"
 	"github.com/docker/docker/volume"
 	"github.com/docker/libnetwork"
 )
@@ -98,22 +99,25 @@ func (daemon *Daemon) populateCommand(c *Container, env []string) error {
 	processConfig.Env = env
 
 	var layerPaths []string
-	img, err := daemon.graph.Get(c.ImageID)
+	img, err := daemon.imageStore.Get(c.ImageID)
 	if err != nil {
 		return derr.ErrorCodeGetGraph.WithArgs(c.ImageID, err)
 	}
-	for i := img; i != nil && err == nil; i, err = daemon.graph.GetParent(i) {
-		lp, err := daemon.driver.Get(i.ID, "")
-		if err != nil {
-			return derr.ErrorCodeGetLayer.WithArgs(daemon.driver.String(), i.ID, err)
-		}
-		layerPaths = append(layerPaths, lp)
-		err = daemon.driver.Put(i.ID)
-		if err != nil {
-			return derr.ErrorCodePutLayer.WithArgs(daemon.driver.String(), i.ID, err)
+
+	if img.RootFS != nil && img.RootFS.Type == "layers+base" {
+		max := len(img.RootFS.DiffIDs)
+		for i := 0; i <= max; i++ {
+			img.RootFS.DiffIDs = img.RootFS.DiffIDs[:i]
+			path, err := layer.GetLayerPath(daemon.layerStore, img.RootFS.ChainID())
+			if err != nil {
+				return derr.ErrorCodeGetLayer.WithArgs(err)
+			}
+			// Reverse order, expecting parent most first
+			layerPaths = append([]string{path}, layerPaths...)
 		}
 	}
-	m, err := daemon.driver.GetMetadata(c.ID)
+
+	m, err := layer.RWLayerMetadata(daemon.layerStore, c.ID)
 	if err != nil {
 		return derr.ErrorCodeGetLayerMetadata.WithArgs(err)
 	}

daemon/create.go

@@ -5,6 +5,7 @@ import (
 	"github.com/docker/docker/api/types"
 	derr "github.com/docker/docker/errors"
 	"github.com/docker/docker/image"
+	"github.com/docker/docker/pkg/idtools"
 	"github.com/docker/docker/pkg/stringid"
 	"github.com/docker/docker/runconfig"
 	"github.com/docker/docker/volume"
@@ -34,7 +35,7 @@ func (daemon *Daemon) ContainerCreate(params *ContainerCreateConfig) (types.Cont
 
 	container, err := daemon.create(params)
 	if err != nil {
-		return types.ContainerCreateResponse{ID: "", Warnings: warnings}, daemon.graphNotExistToErrcode(params.Config.Image, err)
+		return types.ContainerCreateResponse{ID: "", Warnings: warnings}, daemon.imageNotExistToErrcode(err)
 	}
 
 	return types.ContainerCreateResponse{ID: container.ID, Warnings: warnings}, nil
@@ -45,19 +46,16 @@ func (daemon *Daemon) create(params *ContainerCreateConfig) (retC *Container, re
 	var (
 		container *Container
 		img       *image.Image
-		imgID     string
+		imgID     image.ID
 		err       error
 	)
 
 	if params.Config.Image != "" {
-		img, err = daemon.repositories.LookupImage(params.Config.Image)
+		img, err = daemon.GetImage(params.Config.Image)
 		if err != nil {
 			return nil, err
 		}
-		if err = daemon.graph.CheckDepth(img); err != nil {
-			return nil, err
-		}
-		imgID = img.ID
+		imgID = img.ID()
 	}
 
 	if err := daemon.mergeAndVerifyConfig(params.Config, img); err != nil {
@@ -87,15 +85,14 @@ func (daemon *Daemon) create(params *ContainerCreateConfig) (retC *Container, re
 	if err := daemon.Register(container); err != nil {
 		return nil, err
 	}
-	container.Lock()
-	if err := parseSecurityOpt(container, params.HostConfig); err != nil {
-		container.Unlock()
+	rootUID, rootGID, err := idtools.GetRootUIDGID(daemon.uidMaps, daemon.gidMaps)
+	if err != nil {
 		return nil, err
 	}
-	container.Unlock()
-	if err := daemon.createRootfs(container); err != nil {
+	if err := idtools.MkdirAs(container.root, 0700, rootUID, rootGID); err != nil {
 		return nil, err
 	}
+
 	if err := daemon.setHostConfig(container, params.HostConfig); err != nil {
 		return nil, err
 	}

daemon/daemon.go

@@ -18,6 +18,8 @@ import (
 	"time"
 
 	"github.com/Sirupsen/logrus"
+	"github.com/docker/distribution/digest"
+	"github.com/docker/distribution/reference"
 	"github.com/docker/docker/api"
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/cliconfig"
@@ -29,9 +31,13 @@ import (
 	_ "github.com/docker/docker/daemon/graphdriver/vfs" // register vfs
 	"github.com/docker/docker/daemon/logger"
 	"github.com/docker/docker/daemon/network"
+	"github.com/docker/docker/distribution"
+	dmetadata "github.com/docker/docker/distribution/metadata"
 	derr "github.com/docker/docker/errors"
-	"github.com/docker/docker/graph"
 	"github.com/docker/docker/image"
+	"github.com/docker/docker/image/tarexport"
+	"github.com/docker/docker/layer"
+	"github.com/docker/docker/migrate/v1"
 	"github.com/docker/docker/pkg/archive"
 	"github.com/docker/docker/pkg/discovery"
 	"github.com/docker/docker/pkg/fileutils"
@@ -50,12 +56,14 @@ import (
 	"github.com/docker/docker/pkg/truncindex"
 	"github.com/docker/docker/registry"
 	"github.com/docker/docker/runconfig"
+	"github.com/docker/docker/tag"
 	"github.com/docker/docker/utils"
 	volumedrivers "github.com/docker/docker/volume/drivers"
 	"github.com/docker/docker/volume/local"
 	"github.com/docker/docker/volume/store"
 	"github.com/docker/libnetwork"
 	lntypes "github.com/docker/libnetwork/types"
+	"github.com/docker/libtrust"
 	"github.com/opencontainers/runc/libcontainer"
 )
 
@@ -66,6 +74,15 @@ var (
 	errSystemNotSupported = errors.New("The Docker daemon is not supported on this platform.")
 )
 
+// ErrImageDoesNotExist is error returned when no image can be found for a reference.
+type ErrImageDoesNotExist struct {
+	RefOrID string
+}
+
+func (e ErrImageDoesNotExist) Error() string {
+	return fmt.Sprintf("no such id: %s", e.RefOrID)
+}
+
 type contStore struct {
 	s map[string]*Container
 	sync.Mutex
@@ -103,29 +120,33 @@ func (c *contStore) List() []*Container {
 
 // Daemon holds information about the Docker daemon.
 type Daemon struct {
-	ID               string
-	repository       string
-	sysInitPath      string
-	containers       *contStore
-	execCommands     *exec.Store
-	graph            *graph.Graph
-	repositories     *graph.TagStore
-	idIndex          *truncindex.TruncIndex
-	configStore      *Config
-	containerGraphDB *graphdb.Database
-	driver           graphdriver.Driver
-	execDriver       execdriver.Driver
-	statsCollector   *statsCollector
-	defaultLogConfig runconfig.LogConfig
-	RegistryService  *registry.Service
-	EventsService    *events.Events
-	netController    libnetwork.NetworkController
-	volumes          *store.VolumeStore
-	discoveryWatcher discovery.Watcher
-	root             string
-	shutdown         bool
-	uidMaps          []idtools.IDMap
-	gidMaps          []idtools.IDMap
+	ID                        string
+	repository                string
+	sysInitPath               string
+	containers                *contStore
+	execCommands              *exec.Store
+	tagStore                  tag.Store
+	distributionPool          *distribution.Pool
+	distributionMetadataStore dmetadata.Store
+	trustKey                  libtrust.PrivateKey
+	idIndex                   *truncindex.TruncIndex
+	configStore               *Config
+	containerGraphDB          *graphdb.Database
+	driver                    graphdriver.Driver
+	execDriver                execdriver.Driver
+	statsCollector            *statsCollector
+	defaultLogConfig          runconfig.LogConfig
+	RegistryService           *registry.Service
+	EventsService             *events.Events
+	netController             libnetwork.NetworkController
+	volumes                   *store.VolumeStore
+	discoveryWatcher          discovery.Watcher
+	root                      string
+	shutdown                  bool
+	uidMaps                   []idtools.IDMap
+	gidMaps                   []idtools.IDMap
+	layerStore                layer.Store
+	imageStore                image.Store
 }
 
 // Get looks for a container using the provided information, which could be
@@ -229,9 +250,7 @@ func (daemon *Daemon) Register(container *Container) error {
 
 		container.unmountIpcMounts(mount.Unmount)
 
-		if err := daemon.Unmount(container); err != nil {
-			logrus.Debugf("unmount error %s", err)
-		}
+		daemon.Unmount(container)
 		if err := container.toDiskLocking(); err != nil {
 			logrus.Errorf("Error saving stopped state to disk: %v", err)
 		}
@@ -456,7 +475,7 @@ func (daemon *Daemon) getEntrypointAndArgs(configEntrypoint *stringutils.StrSlic
 	return cmdSlice[0], cmdSlice[1:]
 }
 
-func (daemon *Daemon) newContainer(name string, config *runconfig.Config, imgID string) (*Container, error) {
+func (daemon *Daemon) newContainer(name string, config *runconfig.Config, imgID image.ID) (*Container, error) {
 	var (
 		id             string
 		err            error
@@ -542,7 +561,7 @@ func (daemon *Daemon) GetLabels(id string) map[string]string {
 		return container.Config.Labels
 	}
 
-	img, err := daemon.repositories.LookupImage(id)
+	img, err := daemon.GetImage(id)
 	if err == nil {
 		return img.ContainerConfig.Labels
 	}
@@ -702,8 +721,25 @@ func NewDaemon(config *Config, registryService *registry.Service) (daemon *Daemo
 		return nil, err
 	}
 
-	logrus.Debug("Creating images graph")
-	g, err := graph.NewGraph(filepath.Join(config.Root, "graph"), d.driver, uidMaps, gidMaps)
+	imageRoot := filepath.Join(config.Root, "image", d.driver.String())
+	fms, err := layer.NewFSMetadataStore(filepath.Join(imageRoot, "layerdb"))
+	if err != nil {
+		return nil, err
+	}
+
+	d.layerStore, err = layer.NewStore(fms, d.driver)
+	if err != nil {
+		return nil, err
+	}
+
+	distributionPool := distribution.NewPool()
+
+	ifs, err := image.NewFSStoreBackend(filepath.Join(imageRoot, "imagedb"))
+	if err != nil {
+		return nil, err
+	}
+
+	d.imageStore, err = image.NewImageStore(ifs, d.layerStore)
 	if err != nil {
 		return nil, err
 	}
@@ -725,23 +761,24 @@ func NewDaemon(config *Config, registryService *registry.Service) (daemon *Daemo
 		return nil, err
 	}
 
+	distributionMetadataStore, err := dmetadata.NewFSMetadataStore(filepath.Join(imageRoot, "distribution"))
+	if err != nil {
+		return nil, err
+	}
+
 	eventsService := events.New()
-	logrus.Debug("Creating repository list")
-	tagCfg := &graph.TagStoreConfig{
-		Graph:    g,
-		Key:      trustKey,
-		Registry: registryService,
-		Events:   eventsService,
-	}
-	repositories, err := graph.NewTagStore(filepath.Join(config.Root, "repositories-"+d.driver.String()), tagCfg)
+
+	tagStore, err := tag.NewTagStore(filepath.Join(imageRoot, "repositories.json"))
 	if err != nil {
-		return nil, fmt.Errorf("Couldn't create Tag store repositories-%s: %s", d.driver.String(), err)
+		return nil, fmt.Errorf("Couldn't create Tag store repositories: %s", err)
 	}
 
-	if restorer, ok := d.driver.(graphdriver.ImageRestorer); ok {
-		if _, err := restorer.RestoreCustomImages(repositories, g); err != nil {
-			return nil, fmt.Errorf("Couldn't restore custom images: %s", err)
-		}
+	if err := restoreCustomImage(d.driver, d.imageStore, d.layerStore, tagStore); err != nil {
+		return nil, fmt.Errorf("Couldn't restore custom images: %s", err)
+	}
+
+	if err := v1.Migrate(config.Root, d.driver.String(), d.layerStore, d.imageStore, tagStore, distributionMetadataStore); err != nil {
+		return nil, err
 	}
 
 	// Discovery is only enabled when the daemon is launched with an address to advertise.  When
@@ -792,8 +829,10 @@ func NewDaemon(config *Config, registryService *registry.Service) (daemon *Daemo
 	d.repository = daemonRepo
 	d.containers = &contStore{s: make(map[string]*Container)}
 	d.execCommands = exec.NewStore()
-	d.graph = g
-	d.repositories = repositories
+	d.tagStore = tagStore
+	d.distributionPool = distributionPool
+	d.distributionMetadataStore = distributionMetadataStore
+	d.trustKey = trustKey
 	d.idIndex = truncindex.NewTruncIndex([]string{})
 	d.configStore = config
 	d.sysInitPath = sysInitPath
@@ -910,28 +949,44 @@ func (daemon *Daemon) Shutdown() error {
 // Mount sets container.basefs
 // (is it not set coming in? why is it unset?)
 func (daemon *Daemon) Mount(container *Container) error {
-	dir, err := daemon.driver.Get(container.ID, container.getMountLabel())
+	var layerID layer.ChainID
+	if container.ImageID != "" {
+		img, err := daemon.imageStore.Get(container.ImageID)
+		if err != nil {
+			return err
+		}
+		layerID = img.RootFS.ChainID()
+	}
+	rwlayer, err := daemon.layerStore.Mount(container.ID, layerID, container.getMountLabel(), daemon.setupInitLayer)
 	if err != nil {
-		return fmt.Errorf("Error getting container %s from driver %s: %s", container.ID, daemon.driver, err)
+		return err
+	}
+	dir, err := rwlayer.Path()
+	if err != nil {
+		return err
 	}
+	logrus.Debugf("container mounted via layerStore: %v", dir)
 
 	if container.basefs != dir {
 		// The mount path reported by the graph driver should always be trusted on Windows, since the
 		// volume path for a given mounted layer may change over time.  This should only be an error
 		// on non-Windows operating systems.
 		if container.basefs != "" && runtime.GOOS != "windows" {
-			daemon.driver.Put(container.ID)
+			daemon.Unmount(container)
 			return fmt.Errorf("Error: driver %s is returning inconsistent paths for container %s ('%s' then '%s')",
 				daemon.driver, container.ID, container.basefs, dir)
 		}
 	}
-	container.basefs = dir
+	container.basefs = dir // TODO: combine these fields
+	container.rwlayer = rwlayer
 	return nil
 }
 
 // Unmount unsets the container base filesystem
-func (daemon *Daemon) Unmount(container *Container) error {
-	return daemon.driver.Put(container.ID)
+func (daemon *Daemon) Unmount(container *Container) {
+	if err := daemon.layerStore.Unmount(container.ID); err != nil {
+		logrus.Errorf("Error unmounting container %s: %s", container.ID, err)
+	}
 }
 
 // Run uses the execution driver to run a given container
@@ -962,82 +1017,46 @@ func (daemon *Daemon) unsubscribeToContainerStats(c *Container, ch chan interfac
 }
 
 func (daemon *Daemon) changes(container *Container) ([]archive.Change, error) {
-	initID := fmt.Sprintf("%s-init", container.ID)
-	return daemon.driver.Changes(container.ID, initID)
-}
-
-func (daemon *Daemon) diff(container *Container) (archive.Archive, error) {
-	initID := fmt.Sprintf("%s-init", container.ID)
-	return daemon.driver.Diff(container.ID, initID)
+	return daemon.layerStore.Changes(container.ID)
 }
 
-func (daemon *Daemon) createRootfs(container *Container) error {
-	// Step 1: create the container directory.
-	// This doubles as a barrier to avoid race conditions.
-	rootUID, rootGID, err := idtools.GetRootUIDGID(daemon.uidMaps, daemon.gidMaps)
-	if err != nil {
-		return err
+// TagImage creates a tag in the repository reponame, pointing to the image named
+// imageName. If force is true, an existing tag with the same name may be
+// overwritten.
+func (daemon *Daemon) TagImage(newTag reference.Named, imageName string, force bool) error {
+	if _, isDigested := newTag.(reference.Digested); isDigested {
+		return errors.New("refusing to create a tag with a digest reference")
 	}
-	if err := idtools.MkdirAs(container.root, 0700, rootUID, rootGID); err != nil {
-		return err
+	if newTag.Name() == string(digest.Canonical) {
+		return errors.New("refusing to create an ambiguous tag using digest algorithm as name")
 	}
-	initID := fmt.Sprintf("%s-init", container.ID)
 
-	if err := daemon.driver.Create(initID, container.ImageID, container.getMountLabel()); err != nil {
-		return err
-	}
-	initPath, err := daemon.driver.Get(initID, "")
+	newTag = registry.NormalizeLocalReference(newTag)
+	imageID, err := daemon.GetImageID(imageName)
 	if err != nil {
 		return err
 	}
-
-	if err := setupInitLayer(initPath, rootUID, rootGID); err != nil {
-		if err := daemon.driver.Put(initID); err != nil {
-			logrus.Errorf("Failed to Put init layer: %v", err)
-		}
-		return err
-	}
-
-	// We want to unmount init layer before we take snapshot of it
-	// for the actual container.
-	if err := daemon.driver.Put(initID); err != nil {
-		return err
-	}
-
-	if err := daemon.driver.Create(container.ID, initID, ""); err != nil {
-		return err
-	}
-	return nil
-}
-
-// Graph returns *graph.Graph which can be using for layers graph operations.
-func (daemon *Daemon) Graph() *graph.Graph {
-	return daemon.graph
-}
-
-// TagImage creates a tag in the repository reponame, pointing to the image named
-// imageName. If force is true, an existing tag with the same name may be
-// overwritten.
-func (daemon *Daemon) TagImage(repoName, tag, imageName string, force bool) error {
-	if err := daemon.repositories.Tag(repoName, tag, imageName, force); err != nil {
-		return err
-	}
-	daemon.EventsService.Log("tag", utils.ImageReference(repoName, tag), "")
-	return nil
+	daemon.EventsService.Log("tag", newTag.String(), "")
+	return daemon.tagStore.Add(newTag, imageID, force)
 }
 
 // PullImage initiates a pull operation. image is the repository name to pull, and
 // tag may be either empty, or indicate a specific tag to pull.
-func (daemon *Daemon) PullImage(image string, tag string, imagePullConfig *graph.ImagePullConfig) error {
-	return daemon.repositories.Pull(image, tag, imagePullConfig)
-}
+func (daemon *Daemon) PullImage(ref reference.Named, metaHeaders map[string][]string, authConfig *cliconfig.AuthConfig, outStream io.Writer) error {
+	imagePullConfig := &distribution.ImagePullConfig{
+		MetaHeaders:     metaHeaders,
+		AuthConfig:      authConfig,
+		OutStream:       outStream,
+		RegistryService: daemon.RegistryService,
+		EventsService:   daemon.EventsService,
+		MetadataStore:   daemon.distributionMetadataStore,
+		LayerStore:      daemon.layerStore,
+		ImageStore:      daemon.imageStore,
+		TagStore:        daemon.tagStore,
+		Pool:            daemon.distributionPool,
+	}
 
-// ImportImage imports an image, getting the archived layer data either from
-// inConfig (if src is "-"), or from a URI specified in src. Progress output is
-// written to outStream. Repository and tag names can optionally be given in
-// the repo and tag arguments, respectively.
-func (daemon *Daemon) ImportImage(src, repo, tag, msg string, inConfig io.ReadCloser, outStream io.Writer, containerConfig *runconfig.Config) error {
-	return daemon.repositories.Import(src, repo, tag, msg, inConfig, outStream, containerConfig)
+	return distribution.Pull(ref, imagePullConfig)
 }
 
 // ExportImage exports a list of images to the given output stream. The
@@ -1046,47 +1065,214 @@ func (daemon *Daemon) ImportImage(src, repo, tag, msg string, inConfig io.ReadCl
 // the same tag are exported. names is the set of tags to export, and
 // outStream is the writer which the images are written to.
 func (daemon *Daemon) ExportImage(names []string, outStream io.Writer) error {
-	return daemon.repositories.ImageExport(names, outStream)
+	imageExporter := tarexport.NewTarExporter(daemon.imageStore, daemon.layerStore, daemon.tagStore)
+	return imageExporter.Save(names, outStream)
 }
 
 // PushImage initiates a push operation on the repository named localName.
-func (daemon *Daemon) PushImage(localName string, imagePushConfig *graph.ImagePushConfig) error {
-	return daemon.repositories.Push(localName, imagePushConfig)
+func (daemon *Daemon) PushImage(ref reference.Named, metaHeaders map[string][]string, authConfig *cliconfig.AuthConfig, outStream io.Writer) error {
+	imagePushConfig := &distribution.ImagePushConfig{
+		MetaHeaders:     metaHeaders,
+		AuthConfig:      authConfig,
+		OutStream:       outStream,
+		RegistryService: daemon.RegistryService,
+		EventsService:   daemon.EventsService,
+		MetadataStore:   daemon.distributionMetadataStore,
+		LayerStore:      daemon.layerStore,
+		ImageStore:      daemon.imageStore,
+		TagStore:        daemon.tagStore,
+		TrustKey:        daemon.trustKey,
+	}
+
+	return distribution.Push(ref, imagePushConfig)
 }
 
 // LookupImage looks up an image by name and returns it as an ImageInspect
 // structure.
 func (daemon *Daemon) LookupImage(name string) (*types.ImageInspect, error) {
-	return daemon.repositories.Lookup(name)
+	img, err := daemon.GetImage(name)
+	if err != nil {
+		return nil, fmt.Errorf("No such image: %s", name)
+	}
+
+	refs := daemon.tagStore.References(img.ID())
+	repoTags := []string{}
+	repoDigests := []string{}
+	for _, ref := range refs {
+		switch ref.(type) {
+		case reference.Tagged:
+			repoTags = append(repoTags, ref.String())
+		case reference.Digested:
+			repoDigests = append(repoDigests, ref.String())
+		}
+	}
+
+	var size int64
+	var layerMetadata map[string]string
+	layerID := img.RootFS.ChainID()
+	if layerID != "" {
+		l, err := daemon.layerStore.Get(layerID)
+		if err != nil {
+			return nil, err
+		}
+		defer layer.ReleaseAndLog(daemon.layerStore, l)
+		size, err = l.Size()
+		if err != nil {
+			return nil, err
+		}
+
+		layerMetadata, err = l.Metadata()
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	imageInspect := &types.ImageInspect{
+		ID:              img.ID().String(),
+		RepoTags:        repoTags,
+		RepoDigests:     repoDigests,
+		Parent:          img.Parent.String(),
+		Comment:         img.Comment,
+		Created:         img.Created.Format(time.RFC3339Nano),
+		Container:       img.Container,
+		ContainerConfig: &img.ContainerConfig,
+		DockerVersion:   img.DockerVersion,
+		Author:          img.Author,
+		Config:          img.Config,
+		Architecture:    img.Architecture,
+		Os:              img.OS,
+		Size:            size,
+		VirtualSize:     size, // TODO: field unused, deprecate
+	}
+
+	imageInspect.GraphDriver.Name = daemon.driver.String()
+
+	imageInspect.GraphDriver.Data = layerMetadata
+
+	return imageInspect, nil
 }
 
 // LoadImage uploads a set of images into the repository. This is the
 // complement of ImageExport.  The input stream is an uncompressed tar
 // ball containing images and metadata.
 func (daemon *Daemon) LoadImage(inTar io.ReadCloser, outStream io.Writer) error {
-	return daemon.repositories.Load(inTar, outStream)
-}
-
-// ListImages returns a filtered list of images. filterArgs is a JSON-encoded set
-// of filter arguments which will be interpreted by pkg/parsers/filters.
-// filter is a shell glob string applied to repository names. The argument
-// named all controls whether all images in the graph are filtered, or just
-// the heads.
-func (daemon *Daemon) ListImages(filterArgs, filter string, all bool) ([]*types.Image, error) {
-	return daemon.repositories.Images(filterArgs, filter, all)
+	imageExporter := tarexport.NewTarExporter(daemon.imageStore, daemon.layerStore, daemon.tagStore)
+	return imageExporter.Load(inTar, outStream)
 }
 
 // ImageHistory returns a slice of ImageHistory structures for the specified image
 // name by walking the image lineage.
 func (daemon *Daemon) ImageHistory(name string) ([]*types.ImageHistory, error) {
-	return daemon.repositories.History(name)
+	img, err := daemon.GetImage(name)
+	if err != nil {
+		return nil, err
+	}
+
+	history := []*types.ImageHistory{}
+
+	layerCounter := 0
+	rootFS := *img.RootFS
+	rootFS.DiffIDs = nil
+
+	for _, h := range img.History {
+		var layerSize int64
+
+		if !h.EmptyLayer {
+			if len(img.RootFS.DiffIDs) <= layerCounter {
+				return nil, errors.New("too many non-empty layers in History section")
+			}
+
+			rootFS.Append(img.RootFS.DiffIDs[layerCounter])
+			l, err := daemon.layerStore.Get(rootFS.ChainID())
+			if err != nil {
+				return nil, err
+			}
+			layerSize, err = l.DiffSize()
+			layer.ReleaseAndLog(daemon.layerStore, l)
+			if err != nil {
+				return nil, err
+			}
+
+			layerCounter++
+		}
+
+		history = append([]*types.ImageHistory{{
+			ID:        "<missing>",
+			Created:   h.Created.Unix(),
+			CreatedBy: h.CreatedBy,
+			Comment:   h.Comment,
+			Size:      layerSize,
+		}}, history...)
+	}
+
+	// Fill in image IDs and tags
+	histImg := img
+	id := img.ID()
+	for _, h := range history {
+		h.ID = id.String()
+
+		var tags []string
+		for _, r := range daemon.tagStore.References(id) {
+			if _, ok := r.(reference.NamedTagged); ok {
+				tags = append(tags, r.String())
+			}
+		}
+
+		h.Tags = tags
+
+		id = histImg.Parent
+		if id == "" {
+			break
+		}
+		histImg, err = daemon.GetImage(id.String())
+		if err != nil {
+			break
+		}
+	}
+
+	return history, nil
 }
 
-// GetImage returns pointer to an Image struct corresponding to the given
-// name. The name can include an optional tag; otherwise the default tag will
-// be used.
-func (daemon *Daemon) GetImage(name string) (*image.Image, error) {
-	return daemon.repositories.LookupImage(name)
+// GetImageID returns an image ID corresponding to the image referred to by
+// refOrID.
+func (daemon *Daemon) GetImageID(refOrID string) (image.ID, error) {
+	// Treat as an ID
+	if id, err := digest.ParseDigest(refOrID); err == nil {
+		return image.ID(id), nil
+	}
+
+	// Treat it as a possible tag or digest reference
+	if ref, err := reference.ParseNamed(refOrID); err == nil {
+		ref = registry.NormalizeLocalReference(ref)
+		if id, err := daemon.tagStore.Get(ref); err == nil {
+			return id, nil
+		}
+		if tagged, ok := ref.(reference.Tagged); ok {
+			if id, err := daemon.imageStore.Search(tagged.Tag()); err == nil {
+				for _, namedRef := range daemon.tagStore.References(id) {
+					if namedRef.Name() == ref.Name() {
+						return id, nil
+					}
+				}
+			}
+		}
+	}
+
+	// Search based on ID
+	if id, err := daemon.imageStore.Search(refOrID); err == nil {
+		return id, nil
+	}
+
+	return "", ErrImageDoesNotExist{refOrID}
+}
+
+// GetImage returns an image corresponding to the image referred to by refOrID.
+func (daemon *Daemon) GetImage(refOrID string) (*image.Image, error) {
+	imgID, err := daemon.GetImageID(refOrID)
+	if err != nil {
+		return nil, err
+	}
+	return daemon.imageStore.Get(imgID)
 }
 
 func (daemon *Daemon) config() *Config {
@@ -1132,33 +1318,23 @@ func (daemon *Daemon) GetRemappedUIDGID() (int, int) {
 // of the image with imgID, that had the same config when it was
 // created. nil is returned if a child cannot be found. An error is
 // returned if the parent image cannot be found.
-func (daemon *Daemon) ImageGetCached(imgID string, config *runconfig.Config) (*image.Image, error) {
-	// for now just exit if imgID has no children.
-	// maybe parentRefs in graph could be used to store
-	// the Image obj children for faster lookup below but this can
-	// be quite memory hungry.
-	if !daemon.Graph().HasChildren(imgID) {
-		return nil, nil
-	}
-
+func (daemon *Daemon) ImageGetCached(imgID image.ID, config *runconfig.Config) (*image.Image, error) {
 	// Retrieve all images
-	images := daemon.Graph().Map()
+	imgs := daemon.Map()
 
-	// Store the tree in a map of map (map[parentId][childId])
-	imageMap := make(map[string]map[string]struct{})
-	for _, img := range images {
-		if _, exists := imageMap[img.Parent]; !exists {
-			imageMap[img.Parent] = make(map[string]struct{})
+	var siblings []image.ID
+	for id, img := range imgs {
+		if img.Parent == imgID {
+			siblings = append(siblings, id)
 		}
-		imageMap[img.Parent][img.ID] = struct{}{}
 	}
 
 	// Loop on the children of the given image and check the config
 	var match *image.Image
-	for elem := range imageMap[imgID] {
-		img, ok := images[elem]
+	for _, id := range siblings {
+		img, ok := imgs[id]
 		if !ok {
-			return nil, fmt.Errorf("unable to find image %q", elem)
+			return nil, fmt.Errorf("unable to find image %q", id)
 		}
 		if runconfig.Compare(&img.ContainerConfig, config) {
 			if match == nil || match.Created.Before(img.Created) {
@@ -1179,6 +1355,12 @@ func tempDir(rootDir string, rootUID, rootGID int) (string, error) {
 }
 
 func (daemon *Daemon) setHostConfig(container *Container, hostConfig *runconfig.HostConfig) error {
+	container.Lock()
+	if err := parseSecurityOpt(container, hostConfig); err != nil {
+		container.Unlock()
+		return err
+	}
+	container.Unlock()
 
 	// Do not lock while creating volumes since this could be calling out to external plugins
 	// Don't want to block other actions, like `docker ps` because we're waiting on an external plugin
@@ -1199,6 +1381,11 @@ func (daemon *Daemon) setHostConfig(container *Container, hostConfig *runconfig.
 	return nil
 }
 
+func (daemon *Daemon) setupInitLayer(initPath string) error {
+	rootUID, rootGID := daemon.GetRemappedUIDGID()
+	return setupInitLayer(initPath, rootUID, rootGID)
+}
+
 func setDefaultMtu(config *Config) {
 	// do nothing if the config does not have the default 0 value.
 	if config.Mtu != 0 {

daemon/daemon_unix.go

@@ -14,12 +14,15 @@ import (
 	"github.com/Sirupsen/logrus"
 	"github.com/docker/docker/daemon/graphdriver"
 	derr "github.com/docker/docker/errors"
+	"github.com/docker/docker/image"
+	"github.com/docker/docker/layer"
 	pblkiodev "github.com/docker/docker/pkg/blkiodev"
 	"github.com/docker/docker/pkg/idtools"
 	"github.com/docker/docker/pkg/parsers"
 	"github.com/docker/docker/pkg/parsers/kernel"
 	"github.com/docker/docker/pkg/sysinfo"
 	"github.com/docker/docker/runconfig"
+	"github.com/docker/docker/tag"
 	"github.com/docker/libnetwork"
 	nwconfig "github.com/docker/libnetwork/config"
 	"github.com/docker/libnetwork/drivers/bridge"
@@ -601,9 +604,7 @@ func (daemon *Daemon) conditionalMountOnStart(container *Container) error {
 // conditionalUnmountOnCleanup is a platform specific helper function called
 // during the cleanup of a container to unmount.
 func (daemon *Daemon) conditionalUnmountOnCleanup(container *Container) {
-	if err := daemon.Unmount(container); err != nil {
-		logrus.Errorf("%v: Failed to umount filesystem: %v", container.ID, err)
-	}
+	daemon.Unmount(container)
 }
 
 // getDefaultRouteMtu returns the MTU for the default route's interface.
@@ -624,3 +625,8 @@ func getDefaultRouteMtu() (int, error) {
 	}
 	return 0, errNoDefaultRoute
 }
+
+func restoreCustomImage(driver graphdriver.Driver, is image.Store, ls layer.Store, ts tag.Store) error {
+	// Unix has no custom images to register
+	return nil
+}

daemon/daemon_windows.go

@@ -1,12 +1,22 @@
 package daemon
 
 import (
+	"encoding/json"
+	"errors"
 	"fmt"
+	"path/filepath"
+	"runtime"
+	"strings"
 
 	"github.com/Sirupsen/logrus"
+	"github.com/docker/distribution/reference"
 	"github.com/docker/docker/daemon/graphdriver"
+	"github.com/docker/docker/dockerversion"
+	"github.com/docker/docker/image"
+	"github.com/docker/docker/layer"
+	"github.com/docker/docker/tag"
 	// register the windows graph driver
-	_ "github.com/docker/docker/daemon/graphdriver/windows"
+	"github.com/docker/docker/daemon/graphdriver/windows"
 	"github.com/docker/docker/pkg/system"
 	"github.com/docker/docker/runconfig"
 	"github.com/docker/libnetwork"
@@ -128,8 +138,71 @@ func (daemon *Daemon) conditionalMountOnStart(container *Container) error {
 func (daemon *Daemon) conditionalUnmountOnCleanup(container *Container) {
 	// We do not unmount if a Hyper-V container
 	if !container.hostConfig.Isolation.IsHyperV() {
-		if err := daemon.Unmount(container); err != nil {
-			logrus.Errorf("%v: Failed to umount filesystem: %v", container.ID, err)
+		daemon.Unmount(container)
+	}
+}
+
+func restoreCustomImage(driver graphdriver.Driver, is image.Store, ls layer.Store, ts tag.Store) error {
+	if wd, ok := driver.(*windows.Driver); ok {
+		imageInfos, err := wd.GetCustomImageInfos()
+		if err != nil {
+			return err
 		}
+
+		// Convert imageData to valid image configuration
+		for i := range imageInfos {
+			name := strings.ToLower(imageInfos[i].Name)
+
+			type registrar interface {
+				RegisterDiffID(graphID string, size int64) (layer.Layer, error)
+			}
+			r, ok := ls.(registrar)
+			if !ok {
+				return errors.New("Layerstore doesn't support RegisterDiffID")
+			}
+			if _, err := r.RegisterDiffID(imageInfos[i].ID, imageInfos[i].Size); err != nil {
+				return err
+			}
+			// layer is intentionally not released
+
+			rootFS := image.NewRootFS()
+			rootFS.BaseLayer = filepath.Base(imageInfos[i].Path)
+
+			// Create history for base layer
+			config, err := json.Marshal(&image.Image{
+				V1Image: image.V1Image{
+					DockerVersion: dockerversion.Version,
+					Architecture:  runtime.GOARCH,
+					OS:            runtime.GOOS,
+					Created:       imageInfos[i].CreatedTime,
+				},
+				RootFS:  rootFS,
+				History: []image.History{},
+			})
+
+			named, err := reference.ParseNamed(name)
+			if err != nil {
+				return err
+			}
+
+			ref, err := reference.WithTag(named, imageInfos[i].Version)
+			if err != nil {
+				return err
+			}
+
+			id, err := is.Create(config)
+			if err != nil {
+				return err
+			}
+
+			if err := ts.Add(ref, id, true); err != nil {
+				return err
+			}
+
+			logrus.Debugf("Registered base layer %s as %s", ref, id)
+		}
+
 	}
+
+	return nil
 }

daemon/daemonbuilder/builder.go

@@ -9,17 +9,16 @@ import (
 	"strings"
 
 	"github.com/Sirupsen/logrus"
+	"github.com/docker/distribution/reference"
 	"github.com/docker/docker/api"
 	"github.com/docker/docker/builder"
 	"github.com/docker/docker/cliconfig"
 	"github.com/docker/docker/daemon"
-	"github.com/docker/docker/graph"
 	"github.com/docker/docker/image"
 	"github.com/docker/docker/pkg/archive"
 	"github.com/docker/docker/pkg/httputils"
 	"github.com/docker/docker/pkg/idtools"
 	"github.com/docker/docker/pkg/ioutils"
-	"github.com/docker/docker/pkg/parsers"
 	"github.com/docker/docker/pkg/progressreader"
 	"github.com/docker/docker/pkg/urlutil"
 	"github.com/docker/docker/registry"
@@ -44,15 +43,24 @@ func (d Docker) LookupImage(name string) (*image.Image, error) {
 
 // Pull tells Docker to pull image referenced by `name`.
 func (d Docker) Pull(name string) (*image.Image, error) {
-	remote, tag := parsers.ParseRepositoryTag(name)
-	if tag == "" {
-		tag = "latest"
+	ref, err := reference.ParseNamed(name)
+	if err != nil {
+		return nil, err
+	}
+	switch ref.(type) {
+	case reference.Tagged:
+	case reference.Digested:
+	default:
+		ref, err = reference.WithTag(ref, "latest")
+		if err != nil {
+			return nil, err
+		}
 	}
 
 	pullRegistryAuth := &cliconfig.AuthConfig{}
 	if len(d.AuthConfigs) > 0 {
 		// The request came with a full auth config file, we prefer to use that
-		repoInfo, err := d.Daemon.RegistryService.ResolveRepository(remote)
+		repoInfo, err := d.Daemon.RegistryService.ResolveRepository(ref)
 		if err != nil {
 			return nil, err
 		}
@@ -64,12 +72,7 @@ func (d Docker) Pull(name string) (*image.Image, error) {
 		pullRegistryAuth = &resolvedConfig
 	}
 
-	imagePullConfig := &graph.ImagePullConfig{
-		AuthConfig: pullRegistryAuth,
-		OutStream:  ioutils.NopWriteCloser(d.OutOld),
-	}
-
-	if err := d.Daemon.PullImage(remote, tag, imagePullConfig); err != nil {
+	if err := d.Daemon.PullImage(ref, nil, pullRegistryAuth, ioutils.NopWriteCloser(d.OutOld)); err != nil {
 		return nil, err
 	}
 
@@ -106,18 +109,20 @@ func (d Docker) Remove(id string, cfg *daemon.ContainerRmConfig) error {
 }
 
 // Commit creates a new Docker image from an existing Docker container.
-func (d Docker) Commit(name string, cfg *daemon.ContainerCommitConfig) (*image.Image, error) {
+func (d Docker) Commit(name string, cfg *daemon.ContainerCommitConfig) (string, error) {
 	return d.Daemon.Commit(name, cfg)
 }
 
 // Retain retains an image avoiding it to be removed or overwritten until a corresponding Release() call.
 func (d Docker) Retain(sessionID, imgID string) {
-	d.Daemon.Graph().Retain(sessionID, imgID)
+	// FIXME: This will be solved with tags in client-side builder
+	//d.Daemon.Graph().Retain(sessionID, imgID)
 }
 
 // Release releases a list of images that were retained for the time of a build.
 func (d Docker) Release(sessionID string, activeImages []string) {
-	d.Daemon.Graph().Release(sessionID, activeImages...)
+	// FIXME: This will be solved with tags in client-side builder
+	//d.Daemon.Graph().Release(sessionID, activeImages...)
 }
 
 // Copy copies/extracts a source FileInfo to a destination path inside a container
@@ -199,11 +204,11 @@ func (d Docker) Copy(c *daemon.Container, destPath string, src builder.FileInfo,
 // GetCachedImage returns a reference to a cached image whose parent equals `parent`
 // and runconfig equals `cfg`. A cache miss is expected to return an empty ID and a nil error.
 func (d Docker) GetCachedImage(imgID string, cfg *runconfig.Config) (string, error) {
-	cache, err := d.Daemon.ImageGetCached(imgID, cfg)
+	cache, err := d.Daemon.ImageGetCached(image.ID(imgID), cfg)
 	if cache == nil || err != nil {
 		return "", err
 	}
-	return cache.ID, nil
+	return cache.ID().String(), nil
 }
 
 // Kill stops the container execution abruptly.
@@ -218,7 +223,8 @@ func (d Docker) Mount(c *daemon.Container) error {
 
 // Unmount unmounts the root filesystem for the container.
 func (d Docker) Unmount(c *daemon.Container) error {
-	return d.Daemon.Unmount(c)
+	d.Daemon.Unmount(c)
+	return nil
 }
 
 // Start starts a container

daemon/delete.go

@@ -1,12 +1,12 @@
 package daemon
 
 import (
-	"fmt"
 	"os"
 	"path"
 
 	"github.com/Sirupsen/logrus"
 	derr "github.com/docker/docker/errors"
+	"github.com/docker/docker/layer"
 	volumestore "github.com/docker/docker/volume/store"
 )
 
@@ -119,15 +119,12 @@ func (daemon *Daemon) rm(container *Container, forceRemove bool) (err error) {
 		logrus.Debugf("Unable to remove container from link graph: %s", err)
 	}
 
-	if err = daemon.driver.Remove(container.ID); err != nil {
+	metadata, err := daemon.layerStore.DeleteMount(container.ID)
+	layer.LogReleaseMetadata(metadata)
+	if err != nil {
 		return derr.ErrorCodeRmDriverFS.WithArgs(daemon.driver, container.ID, err)
 	}
 
-	initID := fmt.Sprintf("%s-init", container.ID)
-	if err := daemon.driver.Remove(initID); err != nil {
-		return derr.ErrorCodeRmInit.WithArgs(daemon.driver, initID, err)
-	}
-
 	if err = os.RemoveAll(container.root); err != nil {
 		return derr.ErrorCodeRmFS.WithArgs(container.ID, err)
 	}

daemon/errors.go

@@ -3,21 +3,25 @@ package daemon
 import (
 	"strings"
 
+	"github.com/docker/distribution/reference"
 	derr "github.com/docker/docker/errors"
-	"github.com/docker/docker/graph/tags"
-	"github.com/docker/docker/pkg/parsers"
+	tagpkg "github.com/docker/docker/tag"
 )
 
-func (d *Daemon) graphNotExistToErrcode(imageName string, err error) error {
-	if d.Graph().IsNotExist(err, imageName) {
-		if strings.Contains(imageName, "@") {
-			return derr.ErrorCodeNoSuchImageHash.WithArgs(imageName)
+func (d *Daemon) imageNotExistToErrcode(err error) error {
+	if dne, isDNE := err.(ErrImageDoesNotExist); isDNE {
+		if strings.Contains(dne.RefOrID, "@") {
+			return derr.ErrorCodeNoSuchImageHash.WithArgs(dne.RefOrID)
 		}
-		img, tag := parsers.ParseRepositoryTag(imageName)
-		if tag == "" {
-			tag = tags.DefaultTag
+		tag := tagpkg.DefaultTag
+		ref, err := reference.ParseNamed(dne.RefOrID)
+		if err != nil {
+			return derr.ErrorCodeNoSuchImageTag.WithArgs(dne.RefOrID, tag)
 		}
-		return derr.ErrorCodeNoSuchImageTag.WithArgs(img, tag)
+		if tagged, isTagged := ref.(reference.Tagged); isTagged {
+			tag = tagged.Tag()
+		}
+		return derr.ErrorCodeNoSuchImageTag.WithArgs(ref.Name(), tag)
 	}
 	return err
 }

daemon/events/filter.go

@@ -1,8 +1,8 @@
 package events
 
 import (
+	"github.com/docker/distribution/reference"
 	"github.com/docker/docker/pkg/jsonmessage"
-	"github.com/docker/docker/pkg/parsers"
 	"github.com/docker/docker/pkg/parsers/filters"
 )
 
@@ -38,8 +38,11 @@ func (ef *Filter) isLabelFieldIncluded(id string) bool {
 // against the stripped repo name without any tags.
 func (ef *Filter) isImageIncluded(eventID string, eventFrom string) bool {
 	stripTag := func(image string) string {
-		repo, _ := parsers.ParseRepositoryTag(image)
-		return repo
+		ref, err := reference.ParseNamed(image)
+		if err != nil {
+			return image
+		}
+		return ref.Name()
 	}
 
 	return isFieldIncluded(eventID, ef.filter["image"]) ||

daemon/graphdriver/fsdiff.go

@@ -1,5 +1,3 @@
-// +build daemon
-
 package graphdriver
 
 import (
@@ -13,6 +11,12 @@ import (
 	"github.com/docker/docker/pkg/ioutils"
 )
 
+var (
+	// ApplyUncompressedLayer defines the unpack method used by the graph
+	// driver.
+	ApplyUncompressedLayer = chrootarchive.ApplyUncompressedLayer
+)
+
 // NaiveDiffDriver takes a ProtoDriver and adds the
 // capability of the Diffing methods which it may or may not
 // support on its own. See the comment on the exported
@@ -129,7 +133,7 @@ func (gdw *NaiveDiffDriver) ApplyDiff(id, parent string, diff archive.Reader) (s
 		GIDMaps: gdw.gidMaps}
 	start := time.Now().UTC()
 	logrus.Debugf("Start untar layer")
-	if size, err = chrootarchive.ApplyUncompressedLayer(layerFs, diff, options); err != nil {
+	if size, err = ApplyUncompressedLayer(layerFs, diff, options); err != nil {
 		return
 	}
 	logrus.Debugf("Untar time: %vs", time.Now().UTC().Sub(start).Seconds())

daemon/graphdriver/imagerestorer.go

@@ -1,31 +0,0 @@
-package graphdriver
-
-import (
-	"io"
-
-	"github.com/docker/docker/image"
-)
-
-// NOTE: These interfaces are used for implementing specific features of the Windows
-// graphdriver implementation.  The current versions are a short-term solution and
-// likely to change or possibly be eliminated, so avoid using them outside of the Windows
-// graphdriver code.
-
-// ImageRestorer interface allows the implementer to add a custom image to
-// the graph and tagstore.
-type ImageRestorer interface {
-	RestoreCustomImages(tagger Tagger, recorder Recorder) ([]string, error)
-}
-
-// Tagger is an interface that exposes the TagStore.Tag function without needing
-// to import graph.
-type Tagger interface {
-	Tag(repoName, tag, imageName string, force bool) error
-}
-
-// Recorder is an interface that exposes the Graph.Register and Graph.Exists
-// functions without needing to import graph.
-type Recorder interface {
-	Exists(id string) bool
-	Register(img image.Descriptor, layerData io.Reader) error
-}

daemon/graphdriver/plugin.go

@@ -1,5 +1,4 @@
 // +build experimental
-// +build daemon
 
 package graphdriver
 

daemon/graphdriver/proxy.go

@@ -1,5 +1,4 @@
 // +build experimental
-// +build daemon
 
 package graphdriver
 

daemon/graphdriver/vfs/driver.go

@@ -1,5 +1,3 @@
-// +build daemon
-
 package vfs
 
 import (
@@ -14,6 +12,11 @@ import (
 	"github.com/opencontainers/runc/libcontainer/label"
 )
 
+var (
+	// CopyWithTar defines the copy method to use.
+	CopyWithTar = chrootarchive.CopyWithTar
+)
+
 func init() {
 	graphdriver.Register("vfs", Init)
 }
@@ -89,7 +92,7 @@ func (d *Driver) Create(id, parent, mountLabel string) error {
 	if err != nil {
 		return fmt.Errorf("%s: %s", parent, err)
 	}
-	if err := chrootarchive.CopyWithTar(parentDir, dir); err != nil {
+	if err := CopyWithTar(parentDir, dir); err != nil {
 		return err
 	}
 	return nil

daemon/graphdriver/vfs/driver_unsupported.go

@@ -1,3 +0,0 @@
-// +build !daemon
-
-package vfs

daemon/graphdriver/windows/windows.go

@@ -6,10 +6,10 @@ import (
 	"crypto/sha512"
 	"encoding/json"
 	"fmt"
+	"io"
 	"io/ioutil"
 	"os"
 	"path/filepath"
-	"runtime"
 	"strconv"
 	"strings"
 	"sync"
@@ -17,8 +17,6 @@ import (
 
 	"github.com/Sirupsen/logrus"
 	"github.com/docker/docker/daemon/graphdriver"
-	"github.com/docker/docker/dockerversion"
-	"github.com/docker/docker/image"
 	"github.com/docker/docker/pkg/archive"
 	"github.com/docker/docker/pkg/chrootarchive"
 	"github.com/docker/docker/pkg/idtools"
@@ -40,26 +38,6 @@ const (
 	filterDriver
 )
 
-// CustomImageDescriptor is an image descriptor for use by RestoreCustomImages
-type customImageDescriptor struct {
-	img *image.Image
-}
-
-// ID returns the image ID specified in the image structure.
-func (img customImageDescriptor) ID() string {
-	return img.img.ID
-}
-
-// Parent returns the parent ID - in this case, none
-func (img customImageDescriptor) Parent() string {
-	return ""
-}
-
-// MarshalConfig renders the image structure into JSON.
-func (img customImageDescriptor) MarshalConfig() ([]byte, error) {
-	return json.Marshal(img.img)
-}
-
 // Driver represents a windows graph driver.
 type Driver struct {
 	// info stores the shim driver information
@@ -195,7 +173,7 @@ func (d *Driver) Remove(id string) error {
 	if err != nil {
 		return err
 	}
-
+	os.RemoveAll(filepath.Join(d.info.HomeDir, "sysfile-backups", rID)) // ok to fail
 	return hcsshim.DestroyLayer(d.info, rID)
 }
 
@@ -402,22 +380,27 @@ func (d *Driver) DiffSize(id, parent string) (size int64, err error) {
 	return archive.ChangesSize(layerFs, changes), nil
 }
 
-// RestoreCustomImages adds any auto-detected OS specific images to the tag and graph store.
-func (d *Driver) RestoreCustomImages(tagger graphdriver.Tagger, recorder graphdriver.Recorder) (imageIDs []string, err error) {
+// CustomImageInfo is the object returned by the driver describing the base
+// image.
+type CustomImageInfo struct {
+	ID          string
+	Name        string
+	Version     string
+	Path        string
+	Size        int64
+	CreatedTime time.Time
+}
+
+// GetCustomImageInfos returns the image infos for window specific
+// base images which should always be present.
+func (d *Driver) GetCustomImageInfos() ([]CustomImageInfo, error) {
 	strData, err := hcsshim.GetSharedBaseImages()
 	if err != nil {
 		return nil, fmt.Errorf("Failed to restore base images: %s", err)
 	}
 
-	type customImageInfo struct {
-		Name        string
-		Version     string
-		Path        string
-		Size        int64
-		CreatedTime time.Time
-	}
 	type customImageInfoList struct {
-		Images []customImageInfo
+		Images []CustomImageInfo
 	}
 
 	var infoData customImageInfoList
@@ -428,43 +411,28 @@ func (d *Driver) RestoreCustomImages(tagger graphdriver.Tagger, recorder graphdr
 		return nil, err
 	}
 
+	var images []CustomImageInfo
+
 	for _, imageData := range infoData.Images {
-		_, folderName := filepath.Split(imageData.Path)
+		folderName := filepath.Base(imageData.Path)
 
 		// Use crypto hash of the foldername to generate a docker style id.
 		h := sha512.Sum384([]byte(folderName))
 		id := fmt.Sprintf("%x", h[:32])
 
-		if !recorder.Exists(id) {
-			// Register the image.
-			img := &image.Image{
-				ID:            id,
-				Created:       imageData.CreatedTime,
-				DockerVersion: dockerversion.Version,
-				Architecture:  runtime.GOARCH,
-				OS:            runtime.GOOS,
-				Size:          imageData.Size,
-			}
-
-			if err := recorder.Register(customImageDescriptor{img}, nil); err != nil {
-				return nil, err
-			}
-
-			// Create tags for the new image.
-			if err := tagger.Tag(strings.ToLower(imageData.Name), imageData.Version, img.ID, true); err != nil {
-				return nil, err
-			}
-
-			// Create the alternate ID file.
-			if err := d.setID(img.ID, folderName); err != nil {
-				return nil, err
-			}
-
-			imageIDs = append(imageIDs, img.ID)
+		if err := d.Create(id, "", ""); err != nil {
+			return nil, err
 		}
+		// Create the alternate ID file.
+		if err := d.setID(id, folderName); err != nil {
+			return nil, err
+		}
+
+		imageData.ID = id
+		images = append(images, imageData)
 	}
 
-	return imageIDs, nil
+	return images, nil
 }
 
 // GetMetadata returns custom driver information.
@@ -533,6 +501,10 @@ func (d *Driver) importLayer(id string, layerData archive.Reader, parentLayerPat
 	if size, err = chrootarchive.ApplyLayer(tempFolder, layerData); err != nil {
 		return
 	}
+	err = copySysFiles(tempFolder, filepath.Join(d.info.HomeDir, "sysfile-backups", id))
+	if err != nil {
+		return
+	}
 	logrus.Debugf("Untar time: %vs", time.Now().UTC().Sub(start).Seconds())
 
 	if err = hcsshim.ImportLayer(d.info, id, tempFolder, parentLayerPaths); err != nil {
@@ -596,3 +568,103 @@ func (d *Driver) setLayerChain(id string, chain []string) error {
 
 	return nil
 }
+
+// DiffPath returns a directory that contains files needed to construct layer diff.
+func (d *Driver) DiffPath(id string) (path string, release func() error, err error) {
+	id, err = d.resolveID(id)
+	if err != nil {
+		return
+	}
+
+	// Getting the layer paths must be done outside of the lock.
+	layerChain, err := d.getLayerChain(id)
+	if err != nil {
+		return
+	}
+
+	layerFolder := d.dir(id)
+	tempFolder := layerFolder + "-" + strconv.FormatUint(uint64(random.Rand.Uint32()), 10)
+	if err = os.MkdirAll(tempFolder, 0755); err != nil {
+		logrus.Errorf("Could not create %s %s", tempFolder, err)
+		return
+	}
+
+	defer func() {
+		if err != nil {
+			_, folderName := filepath.Split(tempFolder)
+			if err2 := hcsshim.DestroyLayer(d.info, folderName); err2 != nil {
+				logrus.Warnf("Couldn't clean-up tempFolder: %s %s", tempFolder, err2)
+			}
+		}
+	}()
+
+	if err = hcsshim.ExportLayer(d.info, id, tempFolder, layerChain); err != nil {
+		return
+	}
+
+	err = copySysFiles(filepath.Join(d.info.HomeDir, "sysfile-backups", id), tempFolder)
+	if err != nil {
+		return
+	}
+
+	return tempFolder, func() error {
+		// TODO: activate layers and release here?
+		_, folderName := filepath.Split(tempFolder)
+		return hcsshim.DestroyLayer(d.info, folderName)
+	}, nil
+}
+
+var sysFileWhiteList = []string{
+	"Hives\\*",
+	"Files\\BOOTNXT",
+	"tombstones.txt",
+}
+
+// note this only handles files
+func copySysFiles(src string, dest string) error {
+	if err := os.MkdirAll(dest, 0700); err != nil {
+		return err
+	}
+	return filepath.Walk(src, func(path string, info os.FileInfo, err error) error {
+		rel, err := filepath.Rel(src, path)
+		if err != nil {
+			return err
+		}
+		for _, sysfile := range sysFileWhiteList {
+			if matches, err := filepath.Match(sysfile, rel); err != nil || !matches {
+				continue
+			}
+
+			fi, err := os.Lstat(path)
+			if err != nil {
+				return err
+			}
+
+			if !fi.Mode().IsRegular() {
+				continue
+			}
+
+			targetPath := filepath.Join(dest, rel)
+			if err = os.MkdirAll(filepath.Dir(targetPath), 0700); err != nil {
+				return err
+			}
+
+			in, err := os.Open(path)
+			if err != nil {
+				return err
+			}
+			out, err := os.Create(targetPath)
+			if err != nil {
+				in.Close()
+				return err
+			}
+			_, err = io.Copy(out, in)
+			in.Close()
+			out.Close()
+			if err != nil {
+				return err
+			}
+		}
+		return nil
+	})
+}

daemon/image_delete.go

@@ -4,13 +4,12 @@ import (
 	"fmt"
 	"strings"
 
+	"github.com/docker/distribution/reference"
 	"github.com/docker/docker/api/types"
 	derr "github.com/docker/docker/errors"
-	"github.com/docker/docker/graph/tags"
 	"github.com/docker/docker/image"
-	"github.com/docker/docker/pkg/parsers"
 	"github.com/docker/docker/pkg/stringid"
-	"github.com/docker/docker/utils"
+	tagpkg "github.com/docker/docker/tag"
 )
 
 // ImageDelete deletes the image referenced by the given imageRef from this
@@ -53,39 +52,46 @@ import (
 func (daemon *Daemon) ImageDelete(imageRef string, force, prune bool) ([]types.ImageDelete, error) {
 	records := []types.ImageDelete{}
 
-	img, err := daemon.repositories.LookupImage(imageRef)
+	imgID, err := daemon.GetImageID(imageRef)
 	if err != nil {
-		return nil, daemon.graphNotExistToErrcode(imageRef, err)
+		return nil, daemon.imageNotExistToErrcode(err)
 	}
 
+	repoRefs := daemon.tagStore.References(imgID)
+
 	var removedRepositoryRef bool
-	if !isImageIDPrefix(img.ID, imageRef) {
+	if !isImageIDPrefix(imgID.String(), imageRef) {
 		// A repository reference was given and should be removed
 		// first. We can only remove this reference if either force is
 		// true, there are multiple repository references to this
 		// image, or there are no containers using the given reference.
-		if !(force || daemon.imageHasMultipleRepositoryReferences(img.ID)) {
-			if container := daemon.getContainerUsingImage(img.ID); container != nil {
+		if !(force || len(repoRefs) > 1) {
+			if container := daemon.getContainerUsingImage(imgID); container != nil {
 				// If we removed the repository reference then
 				// this image would remain "dangling" and since
 				// we really want to avoid that the client must
 				// explicitly force its removal.
-				return nil, derr.ErrorCodeImgDelUsed.WithArgs(imageRef, stringid.TruncateID(container.ID), stringid.TruncateID(img.ID))
+				return nil, derr.ErrorCodeImgDelUsed.WithArgs(imageRef, stringid.TruncateID(container.ID), stringid.TruncateID(imgID.String()))
 			}
 		}
 
-		parsedRef, err := daemon.removeImageRef(imageRef)
+		parsedRef, err := reference.ParseNamed(imageRef)
 		if err != nil {
 			return nil, err
 		}
 
-		untaggedRecord := types.ImageDelete{Untagged: parsedRef}
+		parsedRef, err = daemon.removeImageRef(parsedRef)
+		if err != nil {
+			return nil, err
+		}
+
+		untaggedRecord := types.ImageDelete{Untagged: parsedRef.String()}
 
-		daemon.EventsService.Log("untag", img.ID, "")
+		daemon.EventsService.Log("untag", imgID.String(), "")
 		records = append(records, untaggedRecord)
 
 		// If has remaining references then untag finishes the remove
-		if daemon.repositories.HasReferences(img) {
+		if len(repoRefs) > 1 {
 			return records, nil
 		}
 
@@ -95,38 +101,39 @@ func (daemon *Daemon) ImageDelete(imageRef string, force, prune bool) ([]types.I
 		// repository reference to the image then we will want to
 		// remove that reference.
 		// FIXME: Is this the behavior we want?
-		repoRefs := daemon.repositories.ByID()[img.ID]
 		if len(repoRefs) == 1 {
 			parsedRef, err := daemon.removeImageRef(repoRefs[0])
 			if err != nil {
 				return nil, err
 			}
 
-			untaggedRecord := types.ImageDelete{Untagged: parsedRef}
+			untaggedRecord := types.ImageDelete{Untagged: parsedRef.String()}
 
-			daemon.EventsService.Log("untag", img.ID, "")
+			daemon.EventsService.Log("untag", imgID.String(), "")
 			records = append(records, untaggedRecord)
 		}
 	}
 
-	return records, daemon.imageDeleteHelper(img, &records, force, prune, removedRepositoryRef)
+	return records, daemon.imageDeleteHelper(imgID, &records, force, prune, removedRepositoryRef)
 }
 
 // isImageIDPrefix returns whether the given possiblePrefix is a prefix of the
 // given imageID.
 func isImageIDPrefix(imageID, possiblePrefix string) bool {
-	return strings.HasPrefix(imageID, possiblePrefix)
-}
+	if strings.HasPrefix(imageID, possiblePrefix) {
+		return true
+	}
+
+	if i := strings.IndexRune(imageID, ':'); i >= 0 {
+		return strings.HasPrefix(imageID[i+1:], possiblePrefix)
+	}
 
-// imageHasMultipleRepositoryReferences returns whether there are multiple
-// repository references to the given imageID.
-func (daemon *Daemon) imageHasMultipleRepositoryReferences(imageID string) bool {
-	return len(daemon.repositories.ByID()[imageID]) > 1
+	return false
 }
 
 // getContainerUsingImage returns a container that was created using the given
 // imageID. Returns nil if there is no such container.
-func (daemon *Daemon) getContainerUsingImage(imageID string) *Container {
+func (daemon *Daemon) getContainerUsingImage(imageID image.ID) *Container {
 	for _, container := range daemon.List() {
 		if container.ImageID == imageID {
 			return container
@@ -141,18 +148,24 @@ func (daemon *Daemon) getContainerUsingImage(imageID string) *Container {
 // repositoryRef must not be an image ID but a repository name followed by an
 // optional tag or digest reference. If tag or digest is omitted, the default
 // tag is used. Returns the resolved image reference and an error.
-func (daemon *Daemon) removeImageRef(repositoryRef string) (string, error) {
-	repository, ref := parsers.ParseRepositoryTag(repositoryRef)
-	if ref == "" {
-		ref = tags.DefaultTag
+func (daemon *Daemon) removeImageRef(ref reference.Named) (reference.Named, error) {
+	switch ref.(type) {
+	case reference.Tagged:
+	case reference.Digested:
+	default:
+		var err error
+		ref, err = reference.WithTag(ref, tagpkg.DefaultTag)
+		if err != nil {
+			return nil, err
+		}
 	}
 
 	// Ignore the boolean value returned, as far as we're concerned, this
 	// is an idempotent operation and it's okay if the reference didn't
 	// exist in the first place.
-	_, err := daemon.repositories.Delete(repository, ref)
+	_, err := daemon.tagStore.Delete(ref)
 
-	return utils.ImageReference(repository, ref), err
+	return ref, err
 }
 
 // removeAllReferencesToImageID attempts to remove every reference to the given
@@ -160,8 +173,8 @@ func (daemon *Daemon) removeImageRef(repositoryRef string) (string, error) {
 // on the first encountered error. Removed references are logged to this
 // daemon's event service. An "Untagged" types.ImageDelete is added to the
 // given list of records.
-func (daemon *Daemon) removeAllReferencesToImageID(imgID string, records *[]types.ImageDelete) error {
-	imageRefs := daemon.repositories.ByID()[imgID]
+func (daemon *Daemon) removeAllReferencesToImageID(imgID image.ID, records *[]types.ImageDelete) error {
+	imageRefs := daemon.tagStore.References(imgID)
 
 	for _, imageRef := range imageRefs {
 		parsedRef, err := daemon.removeImageRef(imageRef)
@@ -169,9 +182,9 @@ func (daemon *Daemon) removeAllReferencesToImageID(imgID string, records *[]type
 			return err
 		}
 
-		untaggedRecord := types.ImageDelete{Untagged: parsedRef}
+		untaggedRecord := types.ImageDelete{Untagged: parsedRef.String()}
 
-		daemon.EventsService.Log("untag", imgID, "")
+		daemon.EventsService.Log("untag", imgID.String(), "")
 		*records = append(*records, untaggedRecord)
 	}
 
@@ -182,7 +195,7 @@ func (daemon *Daemon) removeAllReferencesToImageID(imgID string, records *[]type
 // Implements the error interface.
 type imageDeleteConflict struct {
 	hard    bool
-	imgID   string
+	imgID   image.ID
 	message string
 }
 
@@ -194,7 +207,7 @@ func (idc *imageDeleteConflict) Error() string {
 		forceMsg = "must be forced"
 	}
 
-	return fmt.Sprintf("conflict: unable to delete %s (%s) - %s", stringid.TruncateID(idc.imgID), forceMsg, idc.message)
+	return fmt.Sprintf("conflict: unable to delete %s (%s) - %s", stringid.TruncateID(idc.imgID.String()), forceMsg, idc.message)
 }
 
 // imageDeleteHelper attempts to delete the given image from this daemon. If
@@ -208,11 +221,11 @@ func (idc *imageDeleteConflict) Error() string {
 // conflict is encountered, it will be returned immediately without deleting
 // the image. If quiet is true, any encountered conflicts will be ignored and
 // the function will return nil immediately without deleting the image.
-func (daemon *Daemon) imageDeleteHelper(img *image.Image, records *[]types.ImageDelete, force, prune, quiet bool) error {
+func (daemon *Daemon) imageDeleteHelper(imgID image.ID, records *[]types.ImageDelete, force, prune, quiet bool) error {
 	// First, determine if this image has any conflicts. Ignore soft conflicts
 	// if force is true.
-	if conflict := daemon.checkImageDeleteConflict(img, force); conflict != nil {
-		if quiet && !daemon.imageIsDangling(img) {
+	if conflict := daemon.checkImageDeleteConflict(imgID, force); conflict != nil {
+		if quiet && !daemon.imageIsDangling(imgID) {
 			// Ignore conflicts UNLESS the image is "dangling" in
 			// which case we want the user to know.
 			return nil
@@ -223,33 +236,38 @@ func (daemon *Daemon) imageDeleteHelper(img *image.Image, records *[]types.Image
 		return conflict
 	}
 
+	parent, err := daemon.imageStore.GetParent(imgID)
+	if err != nil {
+		// There may be no parent
+		parent = ""
+	}
+
 	// Delete all repository tag/digest references to this image.
-	if err := daemon.removeAllReferencesToImageID(img.ID, records); err != nil {
+	if err := daemon.removeAllReferencesToImageID(imgID, records); err != nil {
 		return err
 	}
 
-	if err := daemon.Graph().Delete(img.ID); err != nil {
+	removedLayers, err := daemon.imageStore.Delete(imgID)
+	if err != nil {
 		return err
 	}
 
-	daemon.EventsService.Log("delete", img.ID, "")
-	*records = append(*records, types.ImageDelete{Deleted: img.ID})
+	daemon.EventsService.Log("delete", imgID.String(), "")
+	*records = append(*records, types.ImageDelete{Deleted: imgID.String()})
+	for _, removedLayer := range removedLayers {
+		*records = append(*records, types.ImageDelete{Deleted: removedLayer.ChainID.String()})
+	}
 
-	if !prune || img.Parent == "" {
+	if !prune || parent == "" {
 		return nil
 	}
 
 	// We need to prune the parent image. This means delete it if there are
 	// no tags/digests referencing it and there are no containers using it (
 	// either running or stopped).
-	parentImg, err := daemon.Graph().Get(img.Parent)
-	if err != nil {
-		return derr.ErrorCodeImgNoParent.WithArgs(err)
-	}
-
 	// Do not force prunings, but do so quietly (stopping on any encountered
 	// conflicts).
-	return daemon.imageDeleteHelper(parentImg, records, false, true, true)
+	return daemon.imageDeleteHelper(parent, records, false, true, true)
 }
 
 // checkImageDeleteConflict determines whether there are any conflicts
@@ -258,9 +276,9 @@ func (daemon *Daemon) imageDeleteHelper(img *image.Image, records *[]types.Image
 // using the image. A soft conflict is any tags/digest referencing the given
 // image or any stopped container using the image. If ignoreSoftConflicts is
 // true, this function will not check for soft conflict conditions.
-func (daemon *Daemon) checkImageDeleteConflict(img *image.Image, ignoreSoftConflicts bool) *imageDeleteConflict {
+func (daemon *Daemon) checkImageDeleteConflict(imgID image.ID, ignoreSoftConflicts bool) *imageDeleteConflict {
 	// Check for hard conflicts first.
-	if conflict := daemon.checkImageDeleteHardConflict(img); conflict != nil {
+	if conflict := daemon.checkImageDeleteHardConflict(imgID); conflict != nil {
 		return conflict
 	}
 
@@ -270,24 +288,15 @@ func (daemon *Daemon) checkImageDeleteConflict(img *image.Image, ignoreSoftConfl
 		return nil
 	}
 
-	return daemon.checkImageDeleteSoftConflict(img)
+	return daemon.checkImageDeleteSoftConflict(imgID)
 }
 
-func (daemon *Daemon) checkImageDeleteHardConflict(img *image.Image) *imageDeleteConflict {
-	// Check if the image ID is being used by a pull or build.
-	if daemon.Graph().IsHeld(img.ID) {
-		return &imageDeleteConflict{
-			hard:    true,
-			imgID:   img.ID,
-			message: "image is held by an ongoing pull or build",
-		}
-	}
-
+func (daemon *Daemon) checkImageDeleteHardConflict(imgID image.ID) *imageDeleteConflict {
 	// Check if the image has any descendent images.
-	if daemon.Graph().HasChildren(img.ID) {
+	if len(daemon.imageStore.Children(imgID)) > 0 {
 		return &imageDeleteConflict{
 			hard:    true,
-			imgID:   img.ID,
+			imgID:   imgID,
 			message: "image has dependent child images",
 		}
 	}
@@ -299,9 +308,9 @@ func (daemon *Daemon) checkImageDeleteHardConflict(img *image.Image) *imageDelet
 			continue
 		}
 
-		if container.ImageID == img.ID {
+		if container.ImageID == imgID {
 			return &imageDeleteConflict{
-				imgID:   img.ID,
+				imgID:   imgID,
 				hard:    true,
 				message: fmt.Sprintf("image is being used by running container %s", stringid.TruncateID(container.ID)),
 			}
@@ -311,11 +320,11 @@ func (daemon *Daemon) checkImageDeleteHardConflict(img *image.Image) *imageDelet
 	return nil
 }
 
-func (daemon *Daemon) checkImageDeleteSoftConflict(img *image.Image) *imageDeleteConflict {
+func (daemon *Daemon) checkImageDeleteSoftConflict(imgID image.ID) *imageDeleteConflict {
 	// Check if any repository tags/digest reference this image.
-	if daemon.repositories.HasReferences(img) {
+	if len(daemon.tagStore.References(imgID)) > 0 {
 		return &imageDeleteConflict{
-			imgID:   img.ID,
+			imgID:   imgID,
 			message: "image is referenced in one or more repositories",
 		}
 	}
@@ -327,9 +336,9 @@ func (daemon *Daemon) checkImageDeleteSoftConflict(img *image.Image) *imageDelet
 			continue
 		}
 
-		if container.ImageID == img.ID {
+		if container.ImageID == imgID {
 			return &imageDeleteConflict{
-				imgID:   img.ID,
+				imgID:   imgID,
 				message: fmt.Sprintf("image is being used by stopped container %s", stringid.TruncateID(container.ID)),
 			}
 		}
@@ -341,6 +350,6 @@ func (daemon *Daemon) checkImageDeleteSoftConflict(img *image.Image) *imageDelet
 // imageIsDangling returns whether the given image is "dangling" which means
 // that there are no repository references to the given image and it has no
 // child images.
-func (daemon *Daemon) imageIsDangling(img *image.Image) bool {
-	return !(daemon.repositories.HasReferences(img) || daemon.Graph().HasChildren(img.ID))
+func (daemon *Daemon) imageIsDangling(imgID image.ID) bool {
+	return !(len(daemon.tagStore.References(imgID)) > 0 || len(daemon.imageStore.Children(imgID)) > 0)
 }

daemon/images.go

@@ -0,0 +1,163 @@
+package daemon
+
+import (
+	"fmt"
+	"path"
+	"sort"
+	"strings"
+
+	"github.com/docker/distribution/reference"
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/image"
+	"github.com/docker/docker/layer"
+	"github.com/docker/docker/pkg/parsers/filters"
+)
+
+var acceptedImageFilterTags = map[string]struct{}{
+	"dangling": {},
+	"label":    {},
+}
+
+// byCreated is a temporary type used to sort a list of images by creation
+// time.
+type byCreated []*types.Image
+
+func (r byCreated) Len() int           { return len(r) }
+func (r byCreated) Swap(i, j int)      { r[i], r[j] = r[j], r[i] }
+func (r byCreated) Less(i, j int) bool { return r[i].Created < r[j].Created }
+
+// Map returns a map of all images in the ImageStore
+func (daemon *Daemon) Map() map[image.ID]*image.Image {
+	return daemon.imageStore.Map()
+}
+
+// Images returns a filtered list of images. filterArgs is a JSON-encoded set
+// of filter arguments which will be interpreted by pkg/parsers/filters.
+// filter is a shell glob string applied to repository names. The argument
+// named all controls whether all images in the graph are filtered, or just
+// the heads.
+func (daemon *Daemon) Images(filterArgs, filter string, all bool) ([]*types.Image, error) {
+	var (
+		allImages    map[image.ID]*image.Image
+		err          error
+		danglingOnly = false
+	)
+
+	imageFilters, err := filters.FromParam(filterArgs)
+	if err != nil {
+		return nil, err
+	}
+	for name := range imageFilters {
+		if _, ok := acceptedImageFilterTags[name]; !ok {
+			return nil, fmt.Errorf("Invalid filter '%s'", name)
+		}
+	}
+
+	if i, ok := imageFilters["dangling"]; ok {
+		for _, value := range i {
+			if v := strings.ToLower(value); v == "true" {
+				danglingOnly = true
+			} else if v != "false" {
+				return nil, fmt.Errorf("Invalid filter 'dangling=%s'", v)
+			}
+		}
+	}
+
+	if danglingOnly {
+		allImages = daemon.imageStore.Heads()
+	} else {
+		allImages = daemon.imageStore.Map()
+	}
+
+	images := []*types.Image{}
+
+	var filterTagged bool
+	if filter != "" {
+		filterRef, err := reference.Parse(filter)
+		if err == nil { // parse error means wildcard repo
+			if _, ok := filterRef.(reference.Tagged); ok {
+				filterTagged = true
+			}
+		}
+	}
+
+	for id, img := range allImages {
+		if _, ok := imageFilters["label"]; ok {
+			if img.Config == nil {
+				// Very old image that do not have image.Config (or even labels)
+				continue
+			}
+			// We are now sure image.Config is not nil
+			if !imageFilters.MatchKVList("label", img.Config.Labels) {
+				continue
+			}
+		}
+
+		layerID := img.RootFS.ChainID()
+		var size int64
+		if layerID != "" {
+			l, err := daemon.layerStore.Get(layerID)
+			if err != nil {
+				return nil, err
+			}
+
+			size, err = l.Size()
+			layer.ReleaseAndLog(daemon.layerStore, l)
+			if err != nil {
+				return nil, err
+			}
+		}
+
+		newImage := newImage(img, size)
+
+		for _, ref := range daemon.tagStore.References(id) {
+			if filter != "" { // filter by tag/repo name
+				if filterTagged { // filter by tag, require full ref match
+					if ref.String() != filter {
+						continue
+					}
+				} else if matched, err := path.Match(filter, ref.Name()); !matched || err != nil { // name only match, FIXME: docs say exact
+					continue
+				}
+			}
+			if _, ok := ref.(reference.Digested); ok {
+				newImage.RepoDigests = append(newImage.RepoDigests, ref.String())
+			}
+			if _, ok := ref.(reference.Tagged); ok {
+				newImage.RepoTags = append(newImage.RepoTags, ref.String())
+			}
+		}
+		if newImage.RepoDigests == nil && newImage.RepoTags == nil {
+			if all || len(daemon.imageStore.Children(id)) == 0 {
+				if filter != "" { // skip images with no references if filtering by tag
+					continue
+				}
+				newImage.RepoDigests = []string{"<none>@<none>"}
+				newImage.RepoTags = []string{"<none>:<none>"}
+			} else {
+				continue
+			}
+		} else if danglingOnly {
+			continue
+		}
+
+		images = append(images, newImage)
+	}
+
+	sort.Sort(sort.Reverse(byCreated(images)))
+
+	return images, nil
+}
+
+func newImage(image *image.Image, size int64) *types.Image {
+	newImage := new(types.Image)
+	newImage.ParentID = image.Parent.String()
+	newImage.ID = image.ID().String()
+	newImage.Created = image.Created.Unix()
+	newImage.Size = size
+	newImage.VirtualSize = size
+	if image.Config != nil {
+		newImage.Labels = image.Config.Labels
+	}
+	return newImage
+}

daemon/import.go

@@ -0,0 +1,111 @@
+package daemon
+
+import (
+	"encoding/json"
+	"io"
+	"net/http"
+	"net/url"
+	"runtime"
+	"time"
+
+	"github.com/docker/distribution/reference"
+	"github.com/docker/docker/dockerversion"
+	"github.com/docker/docker/image"
+	"github.com/docker/docker/layer"
+	"github.com/docker/docker/pkg/httputils"
+	"github.com/docker/docker/pkg/progressreader"
+	"github.com/docker/docker/pkg/streamformatter"
+	"github.com/docker/docker/runconfig"
+)
+
+// ImportImage imports an image, getting the archived layer data either from
+// inConfig (if src is "-"), or from a URI specified in src. Progress output is
+// written to outStream. Repository and tag names can optionally be given in
+// the repo and tag arguments, respectively.
+func (daemon *Daemon) ImportImage(src string, newRef reference.Named, msg string, inConfig io.ReadCloser, outStream io.Writer, config *runconfig.Config) error {
+	var (
+		sf      = streamformatter.NewJSONStreamFormatter()
+		archive io.ReadCloser
+		resp    *http.Response
+	)
+
+	if src == "-" {
+		archive = inConfig
+	} else {
+		inConfig.Close()
+		u, err := url.Parse(src)
+		if err != nil {
+			return err
+		}
+		if u.Scheme == "" {
+			u.Scheme = "http"
+			u.Host = src
+			u.Path = ""
+		}
+		outStream.Write(sf.FormatStatus("", "Downloading from %s", u))
+		resp, err = httputils.Download(u.String())
+		if err != nil {
+			return err
+		}
+		progressReader := progressreader.New(progressreader.Config{
+			In:        resp.Body,
+			Out:       outStream,
+			Formatter: sf,
+			Size:      resp.ContentLength,
+			NewLines:  true,
+			ID:        "",
+			Action:    "Importing",
+		})
+		archive = progressReader
+	}
+
+	defer archive.Close()
+	if len(msg) == 0 {
+		msg = "Imported from " + src
+	}
+	// TODO: support windows baselayer?
+	l, err := daemon.layerStore.Register(archive, "")
+	if err != nil {
+		return err
+	}
+	defer layer.ReleaseAndLog(daemon.layerStore, l)
+
+	created := time.Now().UTC()
+	imgConfig, err := json.Marshal(&image.Image{
+		V1Image: image.V1Image{
+			DockerVersion: dockerversion.Version,
+			Config:        config,
+			Architecture:  runtime.GOARCH,
+			OS:            runtime.GOOS,
+			Created:       created,
+			Comment:       msg,
+		},
+		RootFS: &image.RootFS{
+			Type:    "layers",
+			DiffIDs: []layer.DiffID{l.DiffID()},
+		},
+		History: []image.History{{
+			Created: created,
+			Comment: msg,
+		}},
+	})
+	if err != nil {
+		return err
+	}
+
+	id, err := daemon.imageStore.Create(imgConfig)
+	if err != nil {
+		return err
+	}
+
+	// FIXME: connect with commit code and call tagstore directly
+	if newRef != nil {
+		if err := daemon.TagImage(newRef, id.String(), true); err != nil {
+			return err
+		}
+	}
+
+	outStream.Write(sf.FormatStatus("", id.String()))
+	daemon.EventsService.Log("import", id.String(), "")
+	return nil
+}

daemon/info.go

@@ -62,7 +62,7 @@ func (daemon *Daemon) SystemInfo() (*types.Info, error) {
 	v := &types.Info{
 		ID:                 daemon.ID,
 		Containers:         len(daemon.List()),
-		Images:             len(daemon.Graph().Map()),
+		Images:             len(daemon.imageStore.Map()),
 		Driver:             daemon.GraphDriver().String(),
 		DriverStatus:       daemon.GraphDriver().Status(),
 		Plugins:            daemon.showPluginsInfo(),

daemon/inspect.go

@@ -8,6 +8,7 @@ import (
 	"github.com/docker/docker/api/types/versions/v1p20"
 	"github.com/docker/docker/daemon/exec"
 	"github.com/docker/docker/daemon/network"
+	"github.com/docker/docker/layer"
 )
 
 // ContainerInspect returns low-level information about a
@@ -124,7 +125,7 @@ func (daemon *Daemon) getInspectData(container *Container, size bool) (*types.Co
 		Path:         container.Path,
 		Args:         container.Args,
 		State:        containerState,
-		Image:        container.ImageID,
+		Image:        container.ImageID.String(),
 		LogPath:      container.LogPath,
 		Name:         container.Name,
 		RestartCount: container.RestartCount,
@@ -149,7 +150,18 @@ func (daemon *Daemon) getInspectData(container *Container, size bool) (*types.Co
 	contJSONBase = setPlatformSpecificContainerFields(container, contJSONBase)
 
 	contJSONBase.GraphDriver.Name = container.Driver
-	graphDriverData, err := daemon.driver.GetMetadata(container.ID)
+
+	image, err := daemon.imageStore.Get(container.ImageID)
+	if err != nil {
+		return nil, err
+	}
+	l, err := daemon.layerStore.Get(image.RootFS.ChainID())
+	if err != nil {
+		return nil, err
+	}
+	defer layer.ReleaseAndLog(daemon.layerStore, l)
+
+	graphDriverData, err := l.Metadata()
 	if err != nil {
 		return nil, err
 	}

daemon/list.go

@@ -9,7 +9,6 @@ import (
 	"github.com/Sirupsen/logrus"
 	"github.com/docker/docker/api/types"
 	derr "github.com/docker/docker/errors"
-	"github.com/docker/docker/graph"
 	"github.com/docker/docker/image"
 	"github.com/docker/docker/pkg/graphdb"
 	"github.com/docker/docker/pkg/nat"
@@ -66,7 +65,7 @@ type listContext struct {
 	// names is a list of container names to filter with
 	names map[string][]string
 	// images is a list of images to filter with
-	images map[string]bool
+	images map[image.ID]bool
 	// filters is a collection of arguments to filter with, specified by the user
 	filters filters.Args
 	// exitAllowed is a list of exit codes allowed to filter with
@@ -176,25 +175,24 @@ func (daemon *Daemon) foldFilter(config *ContainersConfig) (*listContext, error)
 		}
 	}
 
-	imagesFilter := map[string]bool{}
+	imagesFilter := map[image.ID]bool{}
 	var ancestorFilter bool
 	if ancestors, ok := psFilters["ancestor"]; ok {
 		ancestorFilter = true
-		byParents := daemon.Graph().ByParent()
 		// The idea is to walk the graph down the most "efficient" way.
 		for _, ancestor := range ancestors {
 			// First, get the imageId of the ancestor filter (yay)
-			image, err := daemon.repositories.LookupImage(ancestor)
+			id, err := daemon.GetImageID(ancestor)
 			if err != nil {
 				logrus.Warnf("Error while looking up for image %v", ancestor)
 				continue
 			}
-			if imagesFilter[ancestor] {
+			if imagesFilter[id] {
 				// Already seen this ancestor, skip it
 				continue
 			}
 			// Then walk down the graph and put the imageIds in imagesFilter
-			populateImageFilterByParents(imagesFilter, image.ID, byParents)
+			populateImageFilterByParents(imagesFilter, id, daemon.imageStore.Children)
 		}
 	}
 
@@ -310,41 +308,29 @@ func includeContainerInList(container *Container, ctx *listContext) iterationAct
 	return includeContainer
 }
 
-func getImage(s *graph.TagStore, img, imgID string) (string, error) {
-	// both Image and ImageID is actually ids, nothing to guess
-	if strings.HasPrefix(imgID, img) {
-		return img, nil
-	}
-	id, err := s.GetID(img)
-	if err != nil {
-		if err == graph.ErrNameIsNotExist {
-			return imgID, nil
-		}
-		return "", err
-	}
-	if id != imgID {
-		return imgID, nil
-	}
-	return img, nil
-}
-
 // transformContainer generates the container type expected by the docker ps command.
 func (daemon *Daemon) transformContainer(container *Container, ctx *listContext) (*types.Container, error) {
 	newC := &types.Container{
 		ID:      container.ID,
 		Names:   ctx.names[container.ID],
-		ImageID: container.ImageID,
+		ImageID: container.ImageID.String(),
 	}
 	if newC.Names == nil {
 		// Dead containers will often have no name, so make sure the response isn't  null
 		newC.Names = []string{}
 	}
 
-	showImg, err := getImage(daemon.repositories, container.Config.Image, container.ImageID)
-	if err != nil {
-		return nil, err
+	image := container.Config.Image // if possible keep the original ref
+	if image != container.ImageID.String() {
+		id, err := daemon.GetImageID(image)
+		if _, isDNE := err.(ErrImageDoesNotExist); err != nil && !isDNE {
+			return nil, err
+		}
+		if err != nil || id != container.ImageID {
+			image = container.ImageID.String()
+		}
 	}
-	newC.Image = showImg
+	newC.Image = image
 
 	if len(container.Args) > 0 {
 		args := []string{}
@@ -433,12 +419,10 @@ func (daemon *Daemon) Volumes(filter string) ([]*types.Volume, error) {
 	return volumesOut, nil
 }
 
-func populateImageFilterByParents(ancestorMap map[string]bool, imageID string, byParents map[string][]*image.Image) {
+func populateImageFilterByParents(ancestorMap map[image.ID]bool, imageID image.ID, getChildren func(image.ID) []image.ID) {
 	if !ancestorMap[imageID] {
-		if images, ok := byParents[imageID]; ok {
-			for _, image := range images {
-				populateImageFilterByParents(ancestorMap, image.ID, byParents)
-			}
+		for _, id := range getChildren(imageID) {
+			populateImageFilterByParents(ancestorMap, id, getChildren)
 		}
 		ancestorMap[imageID] = true
 	}

distribution/metadata/blobsum_service.go

@@ -0,0 +1,100 @@
+package metadata
+
+import (
+	"encoding/json"
+
+	"github.com/docker/distribution/digest"
+	"github.com/docker/docker/layer"
+)
+
+// BlobSumService maps layer IDs to a set of known blobsums for
+// the layer.
+type BlobSumService struct {
+	store Store
+}
+
+// maxBlobSums is the number of blobsums to keep per layer DiffID.
+const maxBlobSums = 5
+
+// NewBlobSumService creates a new blobsum mapping service.
+func NewBlobSumService(store Store) *BlobSumService {
+	return &BlobSumService{
+		store: store,
+	}
+}
+
+func (blobserv *BlobSumService) diffIDNamespace() string {
+	return "blobsum-storage"
+}
+
+func (blobserv *BlobSumService) blobSumNamespace() string {
+	return "blobsum-lookup"
+}
+
+func (blobserv *BlobSumService) diffIDKey(diffID layer.DiffID) string {
+	return string(digest.Digest(diffID).Algorithm()) + "/" + digest.Digest(diffID).Hex()
+}
+
+func (blobserv *BlobSumService) blobSumKey(blobsum digest.Digest) string {
+	return string(blobsum.Algorithm()) + "/" + blobsum.Hex()
+}
+
+// GetBlobSums finds the blobsums associated with a layer DiffID.
+func (blobserv *BlobSumService) GetBlobSums(diffID layer.DiffID) ([]digest.Digest, error) {
+	jsonBytes, err := blobserv.store.Get(blobserv.diffIDNamespace(), blobserv.diffIDKey(diffID))
+	if err != nil {
+		return nil, err
+	}
+
+	var blobsums []digest.Digest
+	if err := json.Unmarshal(jsonBytes, &blobsums); err != nil {
+		return nil, err
+	}
+
+	return blobsums, nil
+}
+
+// GetDiffID finds a layer DiffID from a blobsum hash.
+func (blobserv *BlobSumService) GetDiffID(blobsum digest.Digest) (layer.DiffID, error) {
+	diffIDBytes, err := blobserv.store.Get(blobserv.blobSumNamespace(), blobserv.blobSumKey(blobsum))
+	if err != nil {
+		return layer.DiffID(""), err
+	}
+
+	return layer.DiffID(diffIDBytes), nil
+}
+
+// Add associates a blobsum with a layer DiffID. If too many blobsums are
+// present, the oldest one is dropped.
+func (blobserv *BlobSumService) Add(diffID layer.DiffID, blobsum digest.Digest) error {
+	oldBlobSums, err := blobserv.GetBlobSums(diffID)
+	if err != nil {
+		oldBlobSums = nil
+	}
+	newBlobSums := make([]digest.Digest, 0, len(oldBlobSums)+1)
+
+	// Copy all other blobsums to new slice
+	for _, oldSum := range oldBlobSums {
+		if oldSum != blobsum {
+			newBlobSums = append(newBlobSums, oldSum)
+		}
+	}
+
+	newBlobSums = append(newBlobSums, blobsum)
+
+	if len(newBlobSums) > maxBlobSums {
+		newBlobSums = newBlobSums[len(newBlobSums)-maxBlobSums:]
+	}
+
+	jsonBytes, err := json.Marshal(newBlobSums)
+	if err != nil {
+		return err
+	}
+
+	err = blobserv.store.Set(blobserv.diffIDNamespace(), blobserv.diffIDKey(diffID), jsonBytes)
+	if err != nil {
+		return err
+	}
+
+	return blobserv.store.Set(blobserv.blobSumNamespace(), blobserv.blobSumKey(blobsum), []byte(diffID))
+}

distribution/metadata/blobsum_service_test.go

@@ -0,0 +1,105 @@
+package metadata
+
+import (
+	"io/ioutil"
+	"os"
+	"reflect"
+	"testing"
+
+	"github.com/docker/distribution/digest"
+	"github.com/docker/docker/layer"
+)
+
+func TestBlobSumService(t *testing.T) {
+	tmpDir, err := ioutil.TempDir("", "blobsum-storage-service-test")
+	if err != nil {
+		t.Fatalf("could not create temp dir: %v", err)
+	}
+	defer os.RemoveAll(tmpDir)
+
+	metadataStore, err := NewFSMetadataStore(tmpDir)
+	if err != nil {
+		t.Fatalf("could not create metadata store: %v", err)
+	}
+	blobSumService := NewBlobSumService(metadataStore)
+
+	testVectors := []struct {
+		diffID   layer.DiffID
+		blobsums []digest.Digest
+	}{
+		{
+			diffID: layer.DiffID("sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4"),
+			blobsums: []digest.Digest{
+				digest.Digest("sha256:f0cd5ca10b07f35512fc2f1cbf9a6cefbdb5cba70ac6b0c9e5988f4497f71937"),
+			},
+		},
+		{
+			diffID: layer.DiffID("sha256:86e0e091d0da6bde2456dbb48306f3956bbeb2eae1b5b9a43045843f69fe4aaa"),
+			blobsums: []digest.Digest{
+				digest.Digest("sha256:f0cd5ca10b07f35512fc2f1cbf9a6cefbdb5cba70ac6b0c9e5988f4497f71937"),
+				digest.Digest("sha256:9e3447ca24cb96d86ebd5960cb34d1299b07e0a0e03801d90b9969a2c187dd6e"),
+			},
+		},
+		{
+			diffID: layer.DiffID("sha256:03f4658f8b782e12230c1783426bd3bacce651ce582a4ffb6fbbfa2079428ecb"),
+			blobsums: []digest.Digest{
+				digest.Digest("sha256:f0cd5ca10b07f35512fc2f1cbf9a6cefbdb5cba70ac6b0c9e5988f4497f71937"),
+				digest.Digest("sha256:9e3447ca24cb96d86ebd5960cb34d1299b07e0a0e03801d90b9969a2c187dd6e"),
+				digest.Digest("sha256:cbbf2f9a99b47fc460d422812b6a5adff7dfee951d8fa2e4a98caa0382cfbdbf"),
+				digest.Digest("sha256:8902a7ca89aabbb868835260912159026637634090dd8899eee969523252236e"),
+				digest.Digest("sha256:c84364306344ccc48532c52ff5209236273525231dddaaab53262322352883aa"),
+				digest.Digest("sha256:aa7583bbc87532a8352bbb72520a821b3623523523a8352523a52352aaa888fe"),
+			},
+		},
+	}
+
+	// Set some associations
+	for _, vec := range testVectors {
+		for _, blobsum := range vec.blobsums {
+			err := blobSumService.Add(vec.diffID, blobsum)
+			if err != nil {
+				t.Fatalf("error calling Set: %v", err)
+			}
+		}
+	}
+
+	// Check the correct values are read back
+	for _, vec := range testVectors {
+		blobsums, err := blobSumService.GetBlobSums(vec.diffID)
+		if err != nil {
+			t.Fatalf("error calling Get: %v", err)
+		}
+		expectedBlobsums := len(vec.blobsums)
+		if expectedBlobsums > 5 {
+			expectedBlobsums = 5
+		}
+		if !reflect.DeepEqual(blobsums, vec.blobsums[len(vec.blobsums)-expectedBlobsums:len(vec.blobsums)]) {
+			t.Fatal("Get returned incorrect layer ID")
+		}
+	}
+
+	// Test GetBlobSums on a nonexistent entry
+	_, err = blobSumService.GetBlobSums(layer.DiffID("sha256:82379823067823853223359023576437723560923756b03560378f4497753917"))
+	if err == nil {
+		t.Fatal("expected error looking up nonexistent entry")
+	}
+
+	// Test GetDiffID on a nonexistent entry
+	_, err = blobSumService.GetDiffID(digest.Digest("sha256:82379823067823853223359023576437723560923756b03560378f4497753917"))
+	if err == nil {
+		t.Fatal("expected error looking up nonexistent entry")
+	}
+
+	// Overwrite one of the entries and read it back
+	err = blobSumService.Add(testVectors[1].diffID, testVectors[0].blobsums[0])
+	if err != nil {
+		t.Fatalf("error calling Add: %v", err)
+	}
+	diffID, err := blobSumService.GetDiffID(testVectors[0].blobsums[0])
+	if err != nil {
+		t.Fatalf("error calling GetDiffID: %v", err)
+	}
+	if diffID != testVectors[1].diffID {
+		t.Fatal("GetDiffID returned incorrect diffID")
+	}
+}

distribution/metadata/metadata.go

@@ -0,0 +1,65 @@
+package metadata
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"sync"
+)
+
+// Store implements a K/V store for mapping distribution-related IDs
+// to on-disk layer IDs and image IDs. The namespace identifies the type of
+// mapping (i.e. "v1ids" or "artifacts"). MetadataStore is goroutine-safe.
+type Store interface {
+	// Get retrieves data by namespace and key.
+	Get(namespace string, key string) ([]byte, error)
+	// Set writes data indexed by namespace and key.
+	Set(namespace, key string, value []byte) error
+}
+
+// FSMetadataStore uses the filesystem to associate metadata with layer and
+// image IDs.
+type FSMetadataStore struct {
+	sync.RWMutex
+	basePath string
+}
+
+// NewFSMetadataStore creates a new filesystem-based metadata store.
+func NewFSMetadataStore(basePath string) (*FSMetadataStore, error) {
+	if err := os.MkdirAll(basePath, 0700); err != nil {
+		return nil, err
+	}
+	return &FSMetadataStore{
+		basePath: basePath,
+	}, nil
+}
+
+func (store *FSMetadataStore) path(namespace, key string) string {
+	return filepath.Join(store.basePath, namespace, key)
+}
+
+// Get retrieves data by namespace and key. The data is read from a file named
+// after the key, stored in the namespace's directory.
+func (store *FSMetadataStore) Get(namespace string, key string) ([]byte, error) {
+	store.RLock()
+	defer store.RUnlock()
+
+	return ioutil.ReadFile(store.path(namespace, key))
+}
+
+// Set writes data indexed by namespace and key. The data is written to a file
+// named after the key, stored in the namespace's directory.
+func (store *FSMetadataStore) Set(namespace, key string, value []byte) error {
+	store.Lock()
+	defer store.Unlock()
+
+	path := store.path(namespace, key)
+	tempFilePath := path + ".tmp"
+	if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil {
+		return err
+	}
+	if err := ioutil.WriteFile(tempFilePath, value, 0644); err != nil {
+		return err
+	}
+	return os.Rename(tempFilePath, path)
+}

distribution/metadata/v1_id_service.go

@@ -0,0 +1,44 @@
+package metadata
+
+import (
+	"github.com/docker/docker/image/v1"
+	"github.com/docker/docker/layer"
+)
+
+// V1IDService maps v1 IDs to layers on disk.
+type V1IDService struct {
+	store Store
+}
+
+// NewV1IDService creates a new V1 ID mapping service.
+func NewV1IDService(store Store) *V1IDService {
+	return &V1IDService{
+		store: store,
+	}
+}
+
+// namespace returns the namespace used by this service.
+func (idserv *V1IDService) namespace() string {
+	return "v1id"
+}
+
+// Get finds a layer by its V1 ID.
+func (idserv *V1IDService) Get(v1ID, registry string) (layer.ChainID, error) {
+	if err := v1.ValidateID(v1ID); err != nil {
+		return layer.ChainID(""), err
+	}
+
+	idBytes, err := idserv.store.Get(idserv.namespace(), registry+","+v1ID)
+	if err != nil {
+		return layer.ChainID(""), err
+	}
+	return layer.ChainID(idBytes), nil
+}
+
+// Set associates an image with a V1 ID.
+func (idserv *V1IDService) Set(v1ID, registry string, id layer.ChainID) error {
+	if err := v1.ValidateID(v1ID); err != nil {
+		return err
+	}
+	return idserv.store.Set(idserv.namespace(), registry+","+v1ID, []byte(id))
+}

distribution/metadata/v1_id_service_test.go

@@ -0,0 +1,83 @@
+package metadata
+
+import (
+	"io/ioutil"
+	"os"
+	"testing"
+
+	"github.com/docker/docker/layer"
+)
+
+func TestV1IDService(t *testing.T) {
+	tmpDir, err := ioutil.TempDir("", "v1-id-service-test")
+	if err != nil {
+		t.Fatalf("could not create temp dir: %v", err)
+	}
+	defer os.RemoveAll(tmpDir)
+
+	metadataStore, err := NewFSMetadataStore(tmpDir)
+	if err != nil {
+		t.Fatalf("could not create metadata store: %v", err)
+	}
+	v1IDService := NewV1IDService(metadataStore)
+
+	testVectors := []struct {
+		registry string
+		v1ID     string
+		layerID  layer.ChainID
+	}{
+		{
+			registry: "registry1",
+			v1ID:     "f0cd5ca10b07f35512fc2f1cbf9a6cefbdb5cba70ac6b0c9e5988f4497f71937",
+			layerID:  layer.ChainID("sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4"),
+		},
+		{
+			registry: "registry2",
+			v1ID:     "9e3447ca24cb96d86ebd5960cb34d1299b07e0a0e03801d90b9969a2c187dd6e",
+			layerID:  layer.ChainID("sha256:86e0e091d0da6bde2456dbb48306f3956bbeb2eae1b5b9a43045843f69fe4aaa"),
+		},
+		{
+			registry: "registry1",
+			v1ID:     "9e3447ca24cb96d86ebd5960cb34d1299b07e0a0e03801d90b9969a2c187dd6e",
+			layerID:  layer.ChainID("sha256:03f4658f8b782e12230c1783426bd3bacce651ce582a4ffb6fbbfa2079428ecb"),
+		},
+	}
+
+	// Set some associations
+	for _, vec := range testVectors {
+		err := v1IDService.Set(vec.v1ID, vec.registry, vec.layerID)
+		if err != nil {
+			t.Fatalf("error calling Set: %v", err)
+		}
+	}
+
+	// Check the correct values are read back
+	for _, vec := range testVectors {
+		layerID, err := v1IDService.Get(vec.v1ID, vec.registry)
+		if err != nil {
+			t.Fatalf("error calling Get: %v", err)
+		}
+		if layerID != vec.layerID {
+			t.Fatal("Get returned incorrect layer ID")
+		}
+	}
+
+	// Test Get on a nonexistent entry
+	_, err = v1IDService.Get("82379823067823853223359023576437723560923756b03560378f4497753917", "registry1")
+	if err == nil {
+		t.Fatal("expected error looking up nonexistent entry")
+	}
+
+	// Overwrite one of the entries and read it back
+	err = v1IDService.Set(testVectors[0].v1ID, testVectors[0].registry, testVectors[1].layerID)
+	if err != nil {
+		t.Fatalf("error calling Set: %v", err)
+	}
+	layerID, err := v1IDService.Get(testVectors[0].v1ID, testVectors[0].registry)
+	if err != nil {
+		t.Fatalf("error calling Get: %v", err)
+	}
+	if layerID != testVectors[1].layerID {
+		t.Fatal("Get returned incorrect layer ID")
+	}
+}

distribution/pool.go

@@ -0,0 +1,51 @@
+package distribution
+
+import (
+	"sync"
+
+	"github.com/docker/docker/pkg/broadcaster"
+)
+
+// A Pool manages concurrent pulls. It deduplicates in-progress downloads.
+type Pool struct {
+	sync.Mutex
+	pullingPool map[string]*broadcaster.Buffered
+}
+
+// NewPool creates a new Pool.
+func NewPool() *Pool {
+	return &Pool{
+		pullingPool: make(map[string]*broadcaster.Buffered),
+	}
+}
+
+// add checks if a pull is already running, and returns (broadcaster, true)
+// if a running operation is found. Otherwise, it creates a new one and returns
+// (broadcaster, false).
+func (pool *Pool) add(key string) (*broadcaster.Buffered, bool) {
+	pool.Lock()
+	defer pool.Unlock()
+
+	if p, exists := pool.pullingPool[key]; exists {
+		return p, true
+	}
+
+	broadcaster := broadcaster.NewBuffered()
+	pool.pullingPool[key] = broadcaster
+
+	return broadcaster, false
+}
+
+func (pool *Pool) removeWithError(key string, broadcasterResult error) error {
+	pool.Lock()
+	defer pool.Unlock()
+	if broadcaster, exists := pool.pullingPool[key]; exists {
+		broadcaster.CloseWithError(broadcasterResult)
+		delete(pool.pullingPool, key)
+	}
+	return nil
+}
+
+func (pool *Pool) remove(key string) error {
+	return pool.removeWithError(key, nil)
+}

distribution/pool_test.go

@@ -0,0 +1,28 @@
+package distribution
+
+import (
+	"testing"
+)
+
+func TestPools(t *testing.T) {
+	p := NewPool()
+
+	if _, found := p.add("test1"); found {
+		t.Fatal("Expected pull test1 not to be in progress")
+	}
+	if _, found := p.add("test2"); found {
+		t.Fatal("Expected pull test2 not to be in progress")
+	}
+	if _, found := p.add("test1"); !found {
+		t.Fatalf("Expected pull test1 to be in progress`")
+	}
+	if err := p.remove("test2"); err != nil {
+		t.Fatal(err)
+	}
+	if err := p.remove("test2"); err != nil {
+		t.Fatal(err)
+	}
+	if err := p.remove("test1"); err != nil {
+		t.Fatal(err)
+	}
+}

graph/pull.go → distribution/pull.go

@@ -1,4 +1,4 @@
-package graph
+package distribution
 
 import (
 	"fmt"
@@ -6,10 +6,15 @@ import (
 	"strings"
 
 	"github.com/Sirupsen/logrus"
+	"github.com/docker/distribution/reference"
 	"github.com/docker/docker/cliconfig"
+	"github.com/docker/docker/daemon/events"
+	"github.com/docker/docker/distribution/metadata"
+	"github.com/docker/docker/image"
+	"github.com/docker/docker/layer"
 	"github.com/docker/docker/pkg/streamformatter"
 	"github.com/docker/docker/registry"
-	"github.com/docker/docker/utils"
+	"github.com/docker/docker/tag"
 )
 
 // ImagePullConfig stores pull configuration.
@@ -23,15 +28,30 @@ type ImagePullConfig struct {
 	// OutStream is the output writer for showing the status of the pull
 	// operation.
 	OutStream io.Writer
+	// RegistryService is the registry service to use for TLS configuration
+	// and endpoint lookup.
+	RegistryService *registry.Service
+	// EventsService is the events service to use for logging.
+	EventsService *events.Events
+	// MetadataStore is the storage backend for distribution-specific
+	// metadata.
+	MetadataStore metadata.Store
+	// LayerStore manages layers.
+	LayerStore layer.Store
+	// ImageStore manages images.
+	ImageStore image.Store
+	// TagStore manages tags.
+	TagStore tag.Store
+	// Pool manages concurrent pulls.
+	Pool *Pool
 }
 
-// puller is an interface that abstracts pulling for different API versions.
-type puller interface {
+// Puller is an interface that abstracts pulling for different API versions.
+type Puller interface {
 	// Pull tries to pull the image referenced by `tag`
 	// Pull returns an error if any, as well as a boolean that determines whether to retry Pull on the next configured endpoint.
 	//
-	// TODO(tiborvass): have Pull() take a reference to repository + tag, so that the puller itself is repository-agnostic.
-	Pull(tag string) (fallback bool, err error)
+	Pull(ref reference.Named) (fallback bool, err error)
 }
 
 // newPuller returns a Puller interface that will pull from either a v1 or v2
@@ -39,23 +59,23 @@ type puller interface {
 // whether a v1 or v2 puller will be created. The other parameters are passed
 // through to the underlying puller implementation for use during the actual
 // pull operation.
-func newPuller(s *TagStore, endpoint registry.APIEndpoint, repoInfo *registry.RepositoryInfo, imagePullConfig *ImagePullConfig, sf *streamformatter.StreamFormatter) (puller, error) {
+func newPuller(endpoint registry.APIEndpoint, repoInfo *registry.RepositoryInfo, imagePullConfig *ImagePullConfig, sf *streamformatter.StreamFormatter) (Puller, error) {
 	switch endpoint.Version {
 	case registry.APIVersion2:
 		return &v2Puller{
-			TagStore: s,
-			endpoint: endpoint,
-			config:   imagePullConfig,
-			sf:       sf,
-			repoInfo: repoInfo,
+			blobSumService: metadata.NewBlobSumService(imagePullConfig.MetadataStore),
+			endpoint:       endpoint,
+			config:         imagePullConfig,
+			sf:             sf,
+			repoInfo:       repoInfo,
 		}, nil
 	case registry.APIVersion1:
 		return &v1Puller{
-			TagStore: s,
-			endpoint: endpoint,
-			config:   imagePullConfig,
-			sf:       sf,
-			repoInfo: repoInfo,
+			v1IDService: metadata.NewV1IDService(imagePullConfig.MetadataStore),
+			endpoint:    endpoint,
+			config:      imagePullConfig,
+			sf:          sf,
+			repoInfo:    repoInfo,
 		}, nil
 	}
 	return nil, fmt.Errorf("unknown version %d for registry %s", endpoint.Version, endpoint.URL)
@@ -63,29 +83,26 @@ func newPuller(s *TagStore, endpoint registry.APIEndpoint, repoInfo *registry.Re
 
 // Pull initiates a pull operation. image is the repository name to pull, and
 // tag may be either empty, or indicate a specific tag to pull.
-func (s *TagStore) Pull(image string, tag string, imagePullConfig *ImagePullConfig) error {
+func Pull(ref reference.Named, imagePullConfig *ImagePullConfig) error {
 	var sf = streamformatter.NewJSONStreamFormatter()
 
 	// Resolve the Repository name from fqn to RepositoryInfo
-	repoInfo, err := s.registryService.ResolveRepository(image)
+	repoInfo, err := imagePullConfig.RegistryService.ResolveRepository(ref)
 	if err != nil {
 		return err
 	}
 
 	// makes sure name is not empty or `scratch`
-	if err := validateRepoName(repoInfo.LocalName); err != nil {
+	if err := validateRepoName(repoInfo.LocalName.Name()); err != nil {
 		return err
 	}
 
-	endpoints, err := s.registryService.LookupPullEndpoints(repoInfo.CanonicalName)
+	endpoints, err := imagePullConfig.RegistryService.LookupPullEndpoints(repoInfo.CanonicalName)
 	if err != nil {
 		return err
 	}
 
-	logName := repoInfo.LocalName
-	if tag != "" {
-		logName = utils.ImageReference(logName, tag)
-	}
+	logName := registry.NormalizeLocalReference(ref)
 
 	var (
 		// use a slice to append the error strings and return a joined string to caller
@@ -103,12 +120,12 @@ func (s *TagStore) Pull(image string, tag string, imagePullConfig *ImagePullConf
 	for _, endpoint := range endpoints {
 		logrus.Debugf("Trying to pull %s from %s %s", repoInfo.LocalName, endpoint.URL, endpoint.Version)
 
-		puller, err := newPuller(s, endpoint, repoInfo, imagePullConfig, sf)
+		puller, err := newPuller(endpoint, repoInfo, imagePullConfig, sf)
 		if err != nil {
 			errors = append(errors, err.Error())
 			continue
 		}
-		if fallback, err := puller.Pull(tag); err != nil {
+		if fallback, err := puller.Pull(ref); err != nil {
 			if fallback {
 				if _, ok := err.(registry.ErrNoSupport); !ok {
 					// Because we found an error that's not ErrNoSupport, discard all subsequent ErrNoSupport errors.
@@ -130,12 +147,12 @@ func (s *TagStore) Pull(image string, tag string, imagePullConfig *ImagePullConf
 			}
 		}
 
-		s.eventsService.Log("pull", logName, "")
+		imagePullConfig.EventsService.Log("pull", logName.String(), "")
 		return nil
 	}
 
 	if len(errors) == 0 {
-		return fmt.Errorf("no endpoints found for %s", image)
+		return fmt.Errorf("no endpoints found for %s", ref.String())
 	}
 
 	if len(errors) > 0 {
@@ -155,3 +172,14 @@ func writeStatus(requestedTag string, out io.Writer, sf *streamformatter.StreamF
 		out.Write(sf.FormatStatus("", "Status: Image is up to date for %s", requestedTag))
 	}
 }
+
+// validateRepoName validates the name of a repository.
+func validateRepoName(name string) error {
+	if name == "" {
+		return fmt.Errorf("Repository name can't be empty")
+	}
+	if name == "scratch" {
+		return fmt.Errorf("'scratch' is a reserved name")
+	}
+	return nil
+}

distribution/pull_v1.go

@@ -0,0 +1,454 @@
+package distribution
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"net"
+	"net/url"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/Sirupsen/logrus"
+	"github.com/docker/distribution/reference"
+	"github.com/docker/distribution/registry/client/transport"
+	"github.com/docker/docker/distribution/metadata"
+	"github.com/docker/docker/image"
+	"github.com/docker/docker/image/v1"
+	"github.com/docker/docker/layer"
+	"github.com/docker/docker/pkg/archive"
+	"github.com/docker/docker/pkg/progressreader"
+	"github.com/docker/docker/pkg/streamformatter"
+	"github.com/docker/docker/pkg/stringid"
+	"github.com/docker/docker/registry"
+)
+
+type v1Puller struct {
+	v1IDService *metadata.V1IDService
+	endpoint    registry.APIEndpoint
+	config      *ImagePullConfig
+	sf          *streamformatter.StreamFormatter
+	repoInfo    *registry.RepositoryInfo
+	session     *registry.Session
+}
+
+func (p *v1Puller) Pull(ref reference.Named) (fallback bool, err error) {
+	if _, isDigested := ref.(reference.Digested); isDigested {
+		// Allowing fallback, because HTTPS v1 is before HTTP v2
+		return true, registry.ErrNoSupport{errors.New("Cannot pull by digest with v1 registry")}
+	}
+
+	tlsConfig, err := p.config.RegistryService.TLSConfig(p.repoInfo.Index.Name)
+	if err != nil {
+		return false, err
+	}
+	// Adds Docker-specific headers as well as user-specified headers (metaHeaders)
+	tr := transport.NewTransport(
+		// TODO(tiborvass): was ReceiveTimeout
+		registry.NewTransport(tlsConfig),
+		registry.DockerHeaders(p.config.MetaHeaders)...,
+	)
+	client := registry.HTTPClient(tr)
+	v1Endpoint, err := p.endpoint.ToV1Endpoint(p.config.MetaHeaders)
+	if err != nil {
+		logrus.Debugf("Could not get v1 endpoint: %v", err)
+		return true, err
+	}
+	p.session, err = registry.NewSession(client, p.config.AuthConfig, v1Endpoint)
+	if err != nil {
+		// TODO(dmcgowan): Check if should fallback
+		logrus.Debugf("Fallback from error: %s", err)
+		return true, err
+	}
+	if err := p.pullRepository(ref); err != nil {
+		// TODO(dmcgowan): Check if should fallback
+		return false, err
+	}
+	out := p.config.OutStream
+	out.Write(p.sf.FormatStatus("", "%s: this image was pulled from a legacy registry.  Important: This registry version will not be supported in future versions of docker.", p.repoInfo.CanonicalName.Name()))
+
+	return false, nil
+}
+
+func (p *v1Puller) pullRepository(ref reference.Named) error {
+	out := p.config.OutStream
+	out.Write(p.sf.FormatStatus("", "Pulling repository %s", p.repoInfo.CanonicalName.Name()))
+
+	repoData, err := p.session.GetRepositoryData(p.repoInfo.RemoteName)
+	if err != nil {
+		if strings.Contains(err.Error(), "HTTP code: 404") {
+			return fmt.Errorf("Error: image %s not found", p.repoInfo.RemoteName.Name())
+		}
+		// Unexpected HTTP error
+		return err
+	}
+
+	logrus.Debugf("Retrieving the tag list")
+	var tagsList map[string]string
+	tagged, isTagged := ref.(reference.Tagged)
+	if !isTagged {
+		tagsList, err = p.session.GetRemoteTags(repoData.Endpoints, p.repoInfo.RemoteName)
+	} else {
+		var tagID string
+		tagsList = make(map[string]string)
+		tagID, err = p.session.GetRemoteTag(repoData.Endpoints, p.repoInfo.RemoteName, tagged.Tag())
+		if err == registry.ErrRepoNotFound {
+			return fmt.Errorf("Tag %s not found in repository %s", tagged.Tag(), p.repoInfo.CanonicalName.Name())
+		}
+		tagsList[tagged.Tag()] = tagID
+	}
+	if err != nil {
+		logrus.Errorf("unable to get remote tags: %s", err)
+		return err
+	}
+
+	for tag, id := range tagsList {
+		repoData.ImgList[id] = &registry.ImgData{
+			ID:       id,
+			Tag:      tag,
+			Checksum: "",
+		}
+	}
+
+	errors := make(chan error)
+	layerDownloaded := make(chan struct{})
+
+	layersDownloaded := false
+	var wg sync.WaitGroup
+	for _, imgData := range repoData.ImgList {
+		if isTagged && imgData.Tag != tagged.Tag() {
+			continue
+		}
+
+		wg.Add(1)
+		go func(img *registry.ImgData) {
+			p.downloadImage(out, repoData, img, layerDownloaded, errors)
+			wg.Done()
+		}(imgData)
+	}
+
+	go func() {
+		wg.Wait()
+		close(errors)
+	}()
+
+	var lastError error
+selectLoop:
+	for {
+		select {
+		case err, ok := <-errors:
+			if !ok {
+				break selectLoop
+			}
+			lastError = err
+		case <-layerDownloaded:
+			layersDownloaded = true
+		}
+	}
+
+	if lastError != nil {
+		return lastError
+	}
+
+	localNameRef := p.repoInfo.LocalName
+	if isTagged {
+		localNameRef, err = reference.WithTag(localNameRef, tagged.Tag())
+		if err != nil {
+			localNameRef = p.repoInfo.LocalName
+		}
+	}
+	writeStatus(localNameRef.String(), out, p.sf, layersDownloaded)
+	return nil
+}
+
+func (p *v1Puller) downloadImage(out io.Writer, repoData *registry.RepositoryData, img *registry.ImgData, layerDownloaded chan struct{}, errors chan error) {
+	if img.Tag == "" {
+		logrus.Debugf("Image (id: %s) present in this repository but untagged, skipping", img.ID)
+		return
+	}
+
+	localNameRef, err := reference.WithTag(p.repoInfo.LocalName, img.Tag)
+	if err != nil {
+		retErr := fmt.Errorf("Image (id: %s) has invalid tag: %s", img.ID, img.Tag)
+		logrus.Debug(retErr.Error())
+		errors <- retErr
+	}
+
+	if err := v1.ValidateID(img.ID); err != nil {
+		errors <- err
+		return
+	}
+
+	out.Write(p.sf.FormatProgress(stringid.TruncateID(img.ID), fmt.Sprintf("Pulling image (%s) from %s", img.Tag, p.repoInfo.CanonicalName.Name()), nil))
+	success := false
+	var lastErr error
+	var isDownloaded bool
+	for _, ep := range p.repoInfo.Index.Mirrors {
+		ep += "v1/"
+		out.Write(p.sf.FormatProgress(stringid.TruncateID(img.ID), fmt.Sprintf("Pulling image (%s) from %s, mirror: %s", img.Tag, p.repoInfo.CanonicalName.Name(), ep), nil))
+		if isDownloaded, err = p.pullImage(out, img.ID, ep, localNameRef); err != nil {
+			// Don't report errors when pulling from mirrors.
+			logrus.Debugf("Error pulling image (%s) from %s, mirror: %s, %s", img.Tag, p.repoInfo.CanonicalName.Name(), ep, err)
+			continue
+		}
+		if isDownloaded {
+			layerDownloaded <- struct{}{}
+		}
+		success = true
+		break
+	}
+	if !success {
+		for _, ep := range repoData.Endpoints {
+			out.Write(p.sf.FormatProgress(stringid.TruncateID(img.ID), fmt.Sprintf("Pulling image (%s) from %s, endpoint: %s", img.Tag, p.repoInfo.CanonicalName.Name(), ep), nil))
+			if isDownloaded, err = p.pullImage(out, img.ID, ep, localNameRef); err != nil {
+				// It's not ideal that only the last error is returned, it would be better to concatenate the errors.
+				// As the error is also given to the output stream the user will see the error.
+				lastErr = err
+				out.Write(p.sf.FormatProgress(stringid.TruncateID(img.ID), fmt.Sprintf("Error pulling image (%s) from %s, endpoint: %s, %s", img.Tag, p.repoInfo.CanonicalName.Name(), ep, err), nil))
+				continue
+			}
+			if isDownloaded {
+				layerDownloaded <- struct{}{}
+			}
+			success = true
+			break
+		}
+	}
+	if !success {
+		err := fmt.Errorf("Error pulling image (%s) from %s, %v", img.Tag, p.repoInfo.CanonicalName.Name(), lastErr)
+		out.Write(p.sf.FormatProgress(stringid.TruncateID(img.ID), err.Error(), nil))
+		errors <- err
+		return
+	}
+	out.Write(p.sf.FormatProgress(stringid.TruncateID(img.ID), "Download complete", nil))
+}
+
+func (p *v1Puller) pullImage(out io.Writer, v1ID, endpoint string, localNameRef reference.Named) (layersDownloaded bool, err error) {
+	var history []string
+	history, err = p.session.GetRemoteHistory(v1ID, endpoint)
+	if err != nil {
+		return false, err
+	}
+	if len(history) < 1 {
+		return false, fmt.Errorf("empty history for image %s", v1ID)
+	}
+	out.Write(p.sf.FormatProgress(stringid.TruncateID(v1ID), "Pulling dependent layers", nil))
+	// FIXME: Try to stream the images?
+	// FIXME: Launch the getRemoteImage() in goroutines
+
+	var (
+		referencedLayers []layer.Layer
+		parentID         layer.ChainID
+		newHistory       []image.History
+		img              *image.V1Image
+		imgJSON          []byte
+		imgSize          int64
+	)
+
+	defer func() {
+		for _, l := range referencedLayers {
+			layer.ReleaseAndLog(p.config.LayerStore, l)
+		}
+	}()
+
+	layersDownloaded = false
+
+	// Iterate over layers from top-most to bottom-most, checking if any
+	// already exist on disk.
+	var i int
+	for i = 0; i != len(history); i++ {
+		v1LayerID := history[i]
+		// Do we have a mapping for this particular v1 ID on this
+		// registry?
+		if layerID, err := p.v1IDService.Get(v1LayerID, p.repoInfo.Index.Name); err == nil {
+			// Does the layer actually exist
+			if l, err := p.config.LayerStore.Get(layerID); err == nil {
+				for j := i; j >= 0; j-- {
+					logrus.Debugf("Layer already exists: %s", history[j])
+					out.Write(p.sf.FormatProgress(stringid.TruncateID(history[j]), "Already exists", nil))
+				}
+				referencedLayers = append(referencedLayers, l)
+				parentID = layerID
+				break
+			}
+		}
+	}
+
+	needsDownload := i
+
+	// Iterate over layers, in order from bottom-most to top-most. Download
+	// config for all layers, and download actual layer data if needed.
+	for i = len(history) - 1; i >= 0; i-- {
+		v1LayerID := history[i]
+		imgJSON, imgSize, err = p.downloadLayerConfig(out, v1LayerID, endpoint)
+		if err != nil {
+			return layersDownloaded, err
+		}
+
+		img = &image.V1Image{}
+		if err := json.Unmarshal(imgJSON, img); err != nil {
+			return layersDownloaded, err
+		}
+
+		if i < needsDownload {
+			l, err := p.downloadLayer(out, v1LayerID, endpoint, parentID, imgSize, &layersDownloaded)
+
+			// Note: This needs to be done even in the error case to avoid
+			// stale references to the layer.
+			if l != nil {
+				referencedLayers = append(referencedLayers, l)
+			}
+			if err != nil {
+				return layersDownloaded, err
+			}
+
+			parentID = l.ChainID()
+		}
+
+		// Create a new-style config from the legacy configs
+		h, err := v1.HistoryFromConfig(imgJSON, false)
+		if err != nil {
+			return layersDownloaded, err
+		}
+		newHistory = append(newHistory, h)
+	}
+
+	rootFS := image.NewRootFS()
+	l := referencedLayers[len(referencedLayers)-1]
+	for l != nil {
+		rootFS.DiffIDs = append([]layer.DiffID{l.DiffID()}, rootFS.DiffIDs...)
+		l = l.Parent()
+	}
+
+	config, err := v1.MakeConfigFromV1Config(imgJSON, rootFS, newHistory)
+	if err != nil {
+		return layersDownloaded, err
+	}
+
+	imageID, err := p.config.ImageStore.Create(config)
+	if err != nil {
+		return layersDownloaded, err
+	}
+
+	if err := p.config.TagStore.Add(localNameRef, imageID, true); err != nil {
+		return layersDownloaded, err
+	}
+
+	return layersDownloaded, nil
+}
+
+func (p *v1Puller) downloadLayerConfig(out io.Writer, v1LayerID, endpoint string) (imgJSON []byte, imgSize int64, err error) {
+	out.Write(p.sf.FormatProgress(stringid.TruncateID(v1LayerID), "Pulling metadata", nil))
+
+	retries := 5
+	for j := 1; j <= retries; j++ {
+		imgJSON, imgSize, err := p.session.GetRemoteImageJSON(v1LayerID, endpoint)
+		if err != nil && j == retries {
+			out.Write(p.sf.FormatProgress(stringid.TruncateID(v1LayerID), "Error pulling layer metadata", nil))
+			return nil, 0, err
+		} else if err != nil {
+			time.Sleep(time.Duration(j) * 500 * time.Millisecond)
+			continue
+		}
+
+		return imgJSON, imgSize, nil
+	}
+
+	// not reached
+	return nil, 0, nil
+}
+
+func (p *v1Puller) downloadLayer(out io.Writer, v1LayerID, endpoint string, parentID layer.ChainID, layerSize int64, layersDownloaded *bool) (l layer.Layer, err error) {
+	// ensure no two downloads of the same layer happen at the same time
+	poolKey := "layer:" + v1LayerID
+	broadcaster, found := p.config.Pool.add(poolKey)
+	broadcaster.Add(out)
+	if found {
+		logrus.Debugf("Image (id: %s) pull is already running, skipping", v1LayerID)
+		if err = broadcaster.Wait(); err != nil {
+			return nil, err
+		}
+		layerID, err := p.v1IDService.Get(v1LayerID, p.repoInfo.Index.Name)
+		if err != nil {
+			return nil, err
+		}
+		// Does the layer actually exist
+		l, err := p.config.LayerStore.Get(layerID)
+		if err != nil {
+			return nil, err
+		}
+		return l, nil
+	}
+
+	// This must use a closure so it captures the value of err when
+	// the function returns, not when the 'defer' is evaluated.
+	defer func() {
+		p.config.Pool.removeWithError(poolKey, err)
+	}()
+
+	retries := 5
+	for j := 1; j <= retries; j++ {
+		// Get the layer
+		status := "Pulling fs layer"
+		if j > 1 {
+			status = fmt.Sprintf("Pulling fs layer [retries: %d]", j)
+		}
+		broadcaster.Write(p.sf.FormatProgress(stringid.TruncateID(v1LayerID), status, nil))
+		layerReader, err := p.session.GetRemoteImageLayer(v1LayerID, endpoint, layerSize)
+		if uerr, ok := err.(*url.Error); ok {
+			err = uerr.Err
+		}
+		if terr, ok := err.(net.Error); ok && terr.Timeout() && j < retries {
+			time.Sleep(time.Duration(j) * 500 * time.Millisecond)
+			continue
+		} else if err != nil {
+			broadcaster.Write(p.sf.FormatProgress(stringid.TruncateID(v1LayerID), "Error pulling dependent layers", nil))
+			return nil, err
+		}
+		*layersDownloaded = true
+		defer layerReader.Close()
+
+		reader := progressreader.New(progressreader.Config{
+			In:        layerReader,
+			Out:       broadcaster,
+			Formatter: p.sf,
+			Size:      layerSize,
+			NewLines:  false,
+			ID:        stringid.TruncateID(v1LayerID),
+			Action:    "Downloading",
+		})
+
+		inflatedLayerData, err := archive.DecompressStream(reader)
+		if err != nil {
+			return nil, fmt.Errorf("could not get decompression stream: %v", err)
+		}
+
+		l, err := p.config.LayerStore.Register(inflatedLayerData, parentID)
+		if err != nil {
+			return nil, fmt.Errorf("failed to register layer: %v", err)
+		}
+		logrus.Debugf("layer %s registered successfully", l.DiffID())
+
+		if terr, ok := err.(net.Error); ok && terr.Timeout() && j < retries {
+			time.Sleep(time.Duration(j) * 500 * time.Millisecond)
+			continue
+		} else if err != nil {
+			broadcaster.Write(p.sf.FormatProgress(stringid.TruncateID(v1LayerID), "Error downloading dependent layers", nil))
+			return nil, err
+		}
+
+		// Cache mapping from this v1 ID to content-addressable layer ID
+		if err := p.v1IDService.Set(v1LayerID, p.repoInfo.Index.Name, l.ChainID()); err != nil {
+			return nil, err
+		}
+
+		broadcaster.Write(p.sf.FormatProgress(stringid.TruncateID(v1LayerID), "Download complete", nil))
+		broadcaster.Close()
+		return l, nil
+	}
+
+	// not reached
+	return nil, nil
+}

distribution/pull_v2.go

@@ -0,0 +1,512 @@
+package distribution
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"runtime"
+
+	"github.com/Sirupsen/logrus"
+	"github.com/docker/distribution"
+	"github.com/docker/distribution/digest"
+	"github.com/docker/distribution/manifest/schema1"
+	"github.com/docker/distribution/reference"
+	"github.com/docker/docker/distribution/metadata"
+	"github.com/docker/docker/image"
+	"github.com/docker/docker/image/v1"
+	"github.com/docker/docker/layer"
+	"github.com/docker/docker/pkg/archive"
+	"github.com/docker/docker/pkg/broadcaster"
+	"github.com/docker/docker/pkg/progressreader"
+	"github.com/docker/docker/pkg/streamformatter"
+	"github.com/docker/docker/pkg/stringid"
+	"github.com/docker/docker/registry"
+	"golang.org/x/net/context"
+)
+
+type v2Puller struct {
+	blobSumService *metadata.BlobSumService
+	endpoint       registry.APIEndpoint
+	config         *ImagePullConfig
+	sf             *streamformatter.StreamFormatter
+	repoInfo       *registry.RepositoryInfo
+	repo           distribution.Repository
+	sessionID      string
+}
+
+func (p *v2Puller) Pull(ref reference.Named) (fallback bool, err error) {
+	// TODO(tiborvass): was ReceiveTimeout
+	p.repo, err = NewV2Repository(p.repoInfo, p.endpoint, p.config.MetaHeaders, p.config.AuthConfig, "pull")
+	if err != nil {
+		logrus.Debugf("Error getting v2 registry: %v", err)
+		return true, err
+	}
+
+	p.sessionID = stringid.GenerateRandomID()
+
+	if err := p.pullV2Repository(ref); err != nil {
+		if registry.ContinueOnError(err) {
+			logrus.Debugf("Error trying v2 registry: %v", err)
+			return true, err
+		}
+		return false, err
+	}
+	return false, nil
+}
+
+func (p *v2Puller) pullV2Repository(ref reference.Named) (err error) {
+	var refs []reference.Named
+	taggedName := p.repoInfo.LocalName
+	if tagged, isTagged := ref.(reference.Tagged); isTagged {
+		taggedName, err = reference.WithTag(p.repoInfo.LocalName, tagged.Tag())
+		if err != nil {
+			return err
+		}
+		refs = []reference.Named{taggedName}
+	} else if digested, isDigested := ref.(reference.Digested); isDigested {
+		taggedName, err = reference.WithDigest(p.repoInfo.LocalName, digested.Digest())
+		if err != nil {
+			return err
+		}
+		refs = []reference.Named{taggedName}
+	} else {
+		manSvc, err := p.repo.Manifests(context.Background())
+		if err != nil {
+			return err
+		}
+
+		tags, err := manSvc.Tags()
+		if err != nil {
+			return err
+		}
+
+		// This probably becomes a lot nicer after the manifest
+		// refactor...
+		for _, tag := range tags {
+			tagRef, err := reference.WithTag(p.repoInfo.LocalName, tag)
+			if err != nil {
+				return err
+			}
+			refs = append(refs, tagRef)
+		}
+	}
+
+	var layersDownloaded bool
+	for _, pullRef := range refs {
+		// pulledNew is true if either new layers were downloaded OR if existing images were newly tagged
+		// TODO(tiborvass): should we change the name of `layersDownload`? What about message in WriteStatus?
+		pulledNew, err := p.pullV2Tag(p.config.OutStream, pullRef)
+		if err != nil {
+			return err
+		}
+		layersDownloaded = layersDownloaded || pulledNew
+	}
+
+	writeStatus(taggedName.String(), p.config.OutStream, p.sf, layersDownloaded)
+
+	return nil
+}
+
+// downloadInfo is used to pass information from download to extractor
+type downloadInfo struct {
+	tmpFile     *os.File
+	digest      digest.Digest
+	layer       distribution.ReadSeekCloser
+	size        int64
+	err         chan error
+	poolKey     string
+	broadcaster *broadcaster.Buffered
+}
+
+type errVerification struct{}
+
+func (errVerification) Error() string { return "verification failed" }
+
+func (p *v2Puller) download(di *downloadInfo) {
+	logrus.Debugf("pulling blob %q", di.digest)
+
+	blobs := p.repo.Blobs(context.Background())
+
+	desc, err := blobs.Stat(context.Background(), di.digest)
+	if err != nil {
+		logrus.Debugf("Error statting layer: %v", err)
+		di.err <- err
+		return
+	}
+	di.size = desc.Size
+
+	layerDownload, err := blobs.Open(context.Background(), di.digest)
+	if err != nil {
+		logrus.Debugf("Error fetching layer: %v", err)
+		di.err <- err
+		return
+	}
+	defer layerDownload.Close()
+
+	verifier, err := digest.NewDigestVerifier(di.digest)
+	if err != nil {
+		di.err <- err
+		return
+	}
+
+	digestStr := di.digest.String()
+
+	reader := progressreader.New(progressreader.Config{
+		In:        ioutil.NopCloser(io.TeeReader(layerDownload, verifier)),
+		Out:       di.broadcaster,
+		Formatter: p.sf,
+		Size:      di.size,
+		NewLines:  false,
+		ID:        stringid.TruncateID(digestStr),
+		Action:    "Downloading",
+	})
+	io.Copy(di.tmpFile, reader)
+
+	di.broadcaster.Write(p.sf.FormatProgress(stringid.TruncateID(digestStr), "Verifying Checksum", nil))
+
+	if !verifier.Verified() {
+		err = fmt.Errorf("filesystem layer verification failed for digest %s", di.digest)
+		logrus.Error(err)
+		di.err <- err
+		return
+	}
+
+	di.broadcaster.Write(p.sf.FormatProgress(stringid.TruncateID(digestStr), "Download complete", nil))
+
+	logrus.Debugf("Downloaded %s to tempfile %s", digestStr, di.tmpFile.Name())
+	di.layer = layerDownload
+
+	di.err <- nil
+}
+
+func (p *v2Puller) pullV2Tag(out io.Writer, ref reference.Named) (tagUpdated bool, err error) {
+	tagOrDigest := ""
+	if tagged, isTagged := ref.(reference.Tagged); isTagged {
+		tagOrDigest = tagged.Tag()
+	} else if digested, isDigested := ref.(reference.Digested); isDigested {
+		tagOrDigest = digested.Digest().String()
+	} else {
+		return false, fmt.Errorf("internal error: reference has neither a tag nor a digest: %s", ref.String())
+	}
+
+	logrus.Debugf("Pulling ref from V2 registry: %q", tagOrDigest)
+
+	manSvc, err := p.repo.Manifests(context.Background())
+	if err != nil {
+		return false, err
+	}
+
+	unverifiedManifest, err := manSvc.GetByTag(tagOrDigest)
+	if err != nil {
+		return false, err
+	}
+	if unverifiedManifest == nil {
+		return false, fmt.Errorf("image manifest does not exist for tag or digest %q", tagOrDigest)
+	}
+	var verifiedManifest *schema1.Manifest
+	verifiedManifest, err = verifyManifest(unverifiedManifest, ref)
+	if err != nil {
+		return false, err
+	}
+
+	rootFS := image.NewRootFS()
+
+	if err := detectBaseLayer(p.config.ImageStore, verifiedManifest, rootFS); err != nil {
+		return false, err
+	}
+
+	// remove duplicate layers and check parent chain validity
+	err = fixManifestLayers(verifiedManifest)
+	if err != nil {
+		return false, err
+	}
+
+	out.Write(p.sf.FormatStatus(tagOrDigest, "Pulling from %s", p.repo.Name()))
+
+	var downloads []*downloadInfo
+
+	defer func() {
+		for _, d := range downloads {
+			p.config.Pool.removeWithError(d.poolKey, err)
+			if d.tmpFile != nil {
+				d.tmpFile.Close()
+				if err := os.RemoveAll(d.tmpFile.Name()); err != nil {
+					logrus.Errorf("Failed to remove temp file: %s", d.tmpFile.Name())
+				}
+			}
+		}
+	}()
+
+	// Image history converted to the new format
+	var history []image.History
+
+	poolKey := "v2layer:"
+	notFoundLocally := false
+
+	// Note that the order of this loop is in the direction of bottom-most
+	// to top-most, so that the downloads slice gets ordered correctly.
+	for i := len(verifiedManifest.FSLayers) - 1; i >= 0; i-- {
+		blobSum := verifiedManifest.FSLayers[i].BlobSum
+		poolKey += blobSum.String()
+
+		var throwAway struct {
+			ThrowAway bool `json:"throwaway,omitempty"`
+		}
+		if err := json.Unmarshal([]byte(verifiedManifest.History[i].V1Compatibility), &throwAway); err != nil {
+			return false, err
+		}
+
+		h, err := v1.HistoryFromConfig([]byte(verifiedManifest.History[i].V1Compatibility), throwAway.ThrowAway)
+		if err != nil {
+			return false, err
+		}
+		history = append(history, h)
+
+		if throwAway.ThrowAway {
+			continue
+		}
+
+		// Do we have a layer on disk corresponding to the set of
+		// blobsums up to this point?
+		if !notFoundLocally {
+			notFoundLocally = true
+			diffID, err := p.blobSumService.GetDiffID(blobSum)
+			if err == nil {
+				rootFS.Append(diffID)
+				if l, err := p.config.LayerStore.Get(rootFS.ChainID()); err == nil {
+					notFoundLocally = false
+					logrus.Debugf("Layer already exists: %s", blobSum.String())
+					out.Write(p.sf.FormatProgress(stringid.TruncateID(blobSum.String()), "Already exists", nil))
+					defer layer.ReleaseAndLog(p.config.LayerStore, l)
+					continue
+				} else {
+					rootFS.DiffIDs = rootFS.DiffIDs[:len(rootFS.DiffIDs)-1]
+				}
+			}
+		}
+
+		out.Write(p.sf.FormatProgress(stringid.TruncateID(blobSum.String()), "Pulling fs layer", nil))
+
+		tmpFile, err := ioutil.TempFile("", "GetImageBlob")
+		if err != nil {
+			return false, err
+		}
+
+		d := &downloadInfo{
+			poolKey: poolKey,
+			digest:  blobSum,
+			tmpFile: tmpFile,
+			// TODO: seems like this chan buffer solved hanging problem in go1.5,
+			// this can indicate some deeper problem that somehow we never take
+			// error from channel in loop below
+			err: make(chan error, 1),
+		}
+
+		downloads = append(downloads, d)
+
+		broadcaster, found := p.config.Pool.add(d.poolKey)
+		broadcaster.Add(out)
+		d.broadcaster = broadcaster
+		if found {
+			d.err <- nil
+		} else {
+			go p.download(d)
+		}
+	}
+
+	for _, d := range downloads {
+		if err := <-d.err; err != nil {
+			return false, err
+		}
+
+		if d.layer == nil {
+			// Wait for a different pull to download and extract
+			// this layer.
+			err = d.broadcaster.Wait()
+			if err != nil {
+				return false, err
+			}
+
+			diffID, err := p.blobSumService.GetDiffID(d.digest)
+			if err != nil {
+				return false, err
+			}
+			rootFS.Append(diffID)
+
+			l, err := p.config.LayerStore.Get(rootFS.ChainID())
+			if err != nil {
+				return false, err
+			}
+
+			defer layer.ReleaseAndLog(p.config.LayerStore, l)
+
+			continue
+		}
+
+		d.tmpFile.Seek(0, 0)
+		reader := progressreader.New(progressreader.Config{
+			In:        d.tmpFile,
+			Out:       d.broadcaster,
+			Formatter: p.sf,
+			Size:      d.size,
+			NewLines:  false,
+			ID:        stringid.TruncateID(d.digest.String()),
+			Action:    "Extracting",
+		})
+
+		inflatedLayerData, err := archive.DecompressStream(reader)
+		if err != nil {
+			return false, fmt.Errorf("could not get decompression stream: %v", err)
+		}
+
+		l, err := p.config.LayerStore.Register(inflatedLayerData, rootFS.ChainID())
+		if err != nil {
+			return false, fmt.Errorf("failed to register layer: %v", err)
+		}
+		logrus.Debugf("layer %s registered successfully", l.DiffID())
+		rootFS.Append(l.DiffID())
+
+		// Cache mapping from this layer's DiffID to the blobsum
+		if err := p.blobSumService.Add(l.DiffID(), d.digest); err != nil {
+			return false, err
+		}
+
+		defer layer.ReleaseAndLog(p.config.LayerStore, l)
+
+		d.broadcaster.Write(p.sf.FormatProgress(stringid.TruncateID(d.digest.String()), "Pull complete", nil))
+		d.broadcaster.Close()
+		tagUpdated = true
+	}
+
+	config, err := v1.MakeConfigFromV1Config([]byte(verifiedManifest.History[0].V1Compatibility), rootFS, history)
+	if err != nil {
+		return false, err
+	}
+
+	imageID, err := p.config.ImageStore.Create(config)
+	if err != nil {
+		return false, err
+	}
+
+	manifestDigest, _, err := digestFromManifest(unverifiedManifest, p.repoInfo.LocalName.Name())
+	if err != nil {
+		return false, err
+	}
+
+	// Check for new tag if no layers downloaded
+	var oldTagImageID image.ID
+	if !tagUpdated {
+		oldTagImageID, err = p.config.TagStore.Get(ref)
+		if err != nil || oldTagImageID != imageID {
+			tagUpdated = true
+		}
+	}
+
+	if tagUpdated {
+		if err = p.config.TagStore.Add(ref, imageID, true); err != nil {
+			return false, err
+		}
+	}
+
+	if manifestDigest != "" {
+		out.Write(p.sf.FormatStatus("", "Digest: %s", manifestDigest))
+	}
+
+	return tagUpdated, nil
+}
+
+func verifyManifest(signedManifest *schema1.SignedManifest, ref reference.Reference) (m *schema1.Manifest, err error) {
+	// If pull by digest, then verify the manifest digest. NOTE: It is
+	// important to do this first, before any other content validation. If the
+	// digest cannot be verified, don't even bother with those other things.
+	if digested, isDigested := ref.(reference.Digested); isDigested {
+		verifier, err := digest.NewDigestVerifier(digested.Digest())
+		if err != nil {
+			return nil, err
+		}
+		payload, err := signedManifest.Payload()
+		if err != nil {
+			// If this failed, the signatures section was corrupted
+			// or missing. Treat the entire manifest as the payload.
+			payload = signedManifest.Raw
+		}
+		if _, err := verifier.Write(payload); err != nil {
+			return nil, err
+		}
+		if !verifier.Verified() {
+			err := fmt.Errorf("image verification failed for digest %s", digested.Digest())
+			logrus.Error(err)
+			return nil, err
+		}
+
+		var verifiedManifest schema1.Manifest
+		if err = json.Unmarshal(payload, &verifiedManifest); err != nil {
+			return nil, err
+		}
+		m = &verifiedManifest
+	} else {
+		m = &signedManifest.Manifest
+	}
+
+	if m.SchemaVersion != 1 {
+		return nil, fmt.Errorf("unsupported schema version %d for %q", m.SchemaVersion, ref.String())
+	}
+	if len(m.FSLayers) != len(m.History) {
+		return nil, fmt.Errorf("length of history not equal to number of layers for %q", ref.String())
+	}
+	if len(m.FSLayers) == 0 {
+		return nil, fmt.Errorf("no FSLayers in manifest for %q", ref.String())
+	}
+	return m, nil
+}
+
+// fixManifestLayers removes repeated layers from the manifest and checks the
+// correctness of the parent chain.
+func fixManifestLayers(m *schema1.Manifest) error {
+	imgs := make([]*image.V1Image, len(m.FSLayers))
+	for i := range m.FSLayers {
+		img := &image.V1Image{}
+
+		if err := json.Unmarshal([]byte(m.History[i].V1Compatibility), img); err != nil {
+			return err
+		}
+
+		imgs[i] = img
+		if err := v1.ValidateID(img.ID); err != nil {
+			return err
+		}
+	}
+
+	if imgs[len(imgs)-1].Parent != "" && runtime.GOOS != "windows" {
+		// Windows base layer can point to a base layer parent that is not in manifest.
+		return errors.New("Invalid parent ID in the base layer of the image.")
+	}
+
+	// check general duplicates to error instead of a deadlock
+	idmap := make(map[string]struct{})
+
+	var lastID string
+	for _, img := range imgs {
+		// skip IDs that appear after each other, we handle those later
+		if _, exists := idmap[img.ID]; img.ID != lastID && exists {
+			return fmt.Errorf("ID %+v appears multiple times in manifest", img.ID)
+		}
+		lastID = img.ID
+		idmap[lastID] = struct{}{}
+	}
+
+	// backwards loop so that we keep the remaining indexes after removing items
+	for i := len(imgs) - 2; i >= 0; i-- {
+		if imgs[i].ID == imgs[i+1].ID { // repeated ID. remove and continue
+			m.FSLayers = append(m.FSLayers[:i], m.FSLayers[i+1:]...)
+			m.History = append(m.History[:i], m.History[i+1:]...)
+		} else if imgs[i].Parent != imgs[i+1].ID {
+			return fmt.Errorf("Invalid parent ID. Expected %v, got %v.", imgs[i+1].ID, imgs[i].Parent)
+		}
+	}
+
+	return nil
+}

graph/pull_v2_test.go → distribution/pull_v2_test.go

@@ -1,4 +1,4 @@
-package graph
+package distribution
 
 import (
 	"encoding/json"
@@ -9,6 +9,7 @@ import (
 
 	"github.com/docker/distribution/digest"
 	"github.com/docker/distribution/manifest/schema1"
+	"github.com/docker/distribution/reference"
 )
 
 // TestFixManifestLayers checks that fixManifestLayers removes a duplicate
@@ -103,7 +104,10 @@ func TestFixManifestLayersBadParent(t *testing.T) {
 
 // TestValidateManifest verifies the validateManifest function
 func TestValidateManifest(t *testing.T) {
-	expectedDigest := "sha256:02fee8c3220ba806531f606525eceb83f4feb654f62b207191b1c9209188dedd"
+	expectedDigest, err := reference.Parse("repo@sha256:02fee8c3220ba806531f606525eceb83f4feb654f62b207191b1c9209188dedd")
+	if err != nil {
+		t.Fatal("could not parse reference")
+	}
 	expectedFSLayer0 := digest.Digest("sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4")
 
 	// Good manifest
@@ -167,29 +171,4 @@ func TestValidateManifest(t *testing.T) {
 	if err == nil || !strings.HasPrefix(err.Error(), "image verification failed for digest") {
 		t.Fatal("expected validateManifest to fail with digest error")
 	}
-
-	// Manifest with no signature
-
-	expectedWholeFileDigest := "7ec3615a120efcdfc270e9c7ea4183330775a3e52a09e2efb194b9a7c18e5ff7"
-
-	noSignatureManifestBytes, err := ioutil.ReadFile("fixtures/validate_manifest/no_signature_manifest")
-	if err != nil {
-		t.Fatal("error reading fixture:", err)
-	}
-
-	var noSignatureSignedManifest schema1.SignedManifest
-	noSignatureSignedManifest.Raw = noSignatureManifestBytes
-	err = json.Unmarshal(noSignatureManifestBytes, &noSignatureSignedManifest.Manifest)
-	if err != nil {
-		t.Fatal("error unmarshaling manifest:", err)
-	}
-
-	verifiedManifest, err = verifyManifest(&noSignatureSignedManifest, expectedWholeFileDigest)
-	if err != nil {
-		t.Fatal("validateManifest failed:", err)
-	}
-
-	if verifiedManifest.FSLayers[0].BlobSum != expectedFSLayer0 {
-		t.Fatal("unexpected FSLayer in no-signature manifest")
-	}
 }

distribution/pull_v2_unix.go

@@ -0,0 +1,12 @@
+// +build !windows
+
+package distribution
+
+import (
+	"github.com/docker/distribution/manifest/schema1"
+	"github.com/docker/docker/image"
+)
+
+func detectBaseLayer(is image.Store, m *schema1.Manifest, rootFS *image.RootFS) error {
+	return nil
+}

distribution/pull_v2_windows.go

@@ -0,0 +1,29 @@
+// +build windows
+
+package distribution
+
+import (
+	"encoding/json"
+	"fmt"
+
+	"github.com/docker/distribution/manifest/schema1"
+	"github.com/docker/docker/image"
+)
+
+func detectBaseLayer(is image.Store, m *schema1.Manifest, rootFS *image.RootFS) error {
+	v1img := &image.V1Image{}
+	if err := json.Unmarshal([]byte(m.History[len(m.History)-1].V1Compatibility), v1img); err != nil {
+		return err
+	}
+	if v1img.Parent == "" {
+		return fmt.Errorf("Last layer %q does not have a base layer reference", v1img.ID)
+	}
+	// There must be an image that already references the baselayer.
+	for _, img := range is.Map() {
+		if img.RootFS.BaseLayerID() == v1img.Parent {
+			rootFS.BaseLayer = img.RootFS.BaseLayer
+			return nil
+		}
+	}
+	return fmt.Errorf("Invalid base layer %q", v1img.Parent)
+}

distribution/push.go

@@ -0,0 +1,179 @@
+package distribution
+
+import (
+	"bufio"
+	"compress/gzip"
+	"fmt"
+	"io"
+
+	"github.com/Sirupsen/logrus"
+	"github.com/docker/distribution/digest"
+	"github.com/docker/distribution/reference"
+	"github.com/docker/docker/cliconfig"
+	"github.com/docker/docker/daemon/events"
+	"github.com/docker/docker/distribution/metadata"
+	"github.com/docker/docker/image"
+	"github.com/docker/docker/layer"
+	"github.com/docker/docker/pkg/streamformatter"
+	"github.com/docker/docker/registry"
+	"github.com/docker/docker/tag"
+	"github.com/docker/libtrust"
+)
+
+// ImagePushConfig stores push configuration.
+type ImagePushConfig struct {
+	// MetaHeaders store HTTP headers with metadata about the image
+	// (DockerHeaders with prefix X-Meta- in the request).
+	MetaHeaders map[string][]string
+	// AuthConfig holds authentication credentials for authenticating with
+	// the registry.
+	AuthConfig *cliconfig.AuthConfig
+	// OutStream is the output writer for showing the status of the push
+	// operation.
+	OutStream io.Writer
+	// RegistryService is the registry service to use for TLS configuration
+	// and endpoint lookup.
+	RegistryService *registry.Service
+	// EventsService is the events service to use for logging.
+	EventsService *events.Events
+	// MetadataStore is the storage backend for distribution-specific
+	// metadata.
+	MetadataStore metadata.Store
+	// LayerStore manges layers.
+	LayerStore layer.Store
+	// ImageStore manages images.
+	ImageStore image.Store
+	// TagStore manages tags.
+	TagStore tag.Store
+	// TrustKey is the private key for legacy signatures. This is typically
+	// an ephemeral key, since these signatures are no longer verified.
+	TrustKey libtrust.PrivateKey
+}
+
+// Pusher is an interface that abstracts pushing for different API versions.
+type Pusher interface {
+	// Push tries to push the image configured at the creation of Pusher.
+	// Push returns an error if any, as well as a boolean that determines whether to retry Push on the next configured endpoint.
+	//
+	// TODO(tiborvass): have Push() take a reference to repository + tag, so that the pusher itself is repository-agnostic.
+	Push() (fallback bool, err error)
+}
+
+const compressionBufSize = 32768
+
+// NewPusher creates a new Pusher interface that will push to either a v1 or v2
+// registry. The endpoint argument contains a Version field that determines
+// whether a v1 or v2 pusher will be created. The other parameters are passed
+// through to the underlying pusher implementation for use during the actual
+// push operation.
+func NewPusher(ref reference.Named, endpoint registry.APIEndpoint, repoInfo *registry.RepositoryInfo, imagePushConfig *ImagePushConfig, sf *streamformatter.StreamFormatter) (Pusher, error) {
+	switch endpoint.Version {
+	case registry.APIVersion2:
+		return &v2Pusher{
+			blobSumService: metadata.NewBlobSumService(imagePushConfig.MetadataStore),
+			ref:            ref,
+			endpoint:       endpoint,
+			repoInfo:       repoInfo,
+			config:         imagePushConfig,
+			sf:             sf,
+			layersPushed:   make(map[digest.Digest]bool),
+		}, nil
+	case registry.APIVersion1:
+		return &v1Pusher{
+			v1IDService: metadata.NewV1IDService(imagePushConfig.MetadataStore),
+			ref:         ref,
+			endpoint:    endpoint,
+			repoInfo:    repoInfo,
+			config:      imagePushConfig,
+			sf:          sf,
+		}, nil
+	}
+	return nil, fmt.Errorf("unknown version %d for registry %s", endpoint.Version, endpoint.URL)
+}
+
+// Push initiates a push operation on the repository named localName.
+// ref is the specific variant of the image to be pushed.
+// If no tag is provided, all tags will be pushed.
+func Push(ref reference.Named, imagePushConfig *ImagePushConfig) error {
+	// FIXME: Allow to interrupt current push when new push of same image is done.
+
+	var sf = streamformatter.NewJSONStreamFormatter()
+
+	// Resolve the Repository name from fqn to RepositoryInfo
+	repoInfo, err := imagePushConfig.RegistryService.ResolveRepository(ref)
+	if err != nil {
+		return err
+	}
+
+	endpoints, err := imagePushConfig.RegistryService.LookupPushEndpoints(repoInfo.CanonicalName)
+	if err != nil {
+		return err
+	}
+
+	imagePushConfig.OutStream.Write(sf.FormatStatus("", "The push refers to a repository [%s]", repoInfo.CanonicalName))
+
+	associations := imagePushConfig.TagStore.ReferencesByName(repoInfo.LocalName)
+	if len(associations) == 0 {
+		return fmt.Errorf("Repository does not exist: %s", repoInfo.LocalName)
+	}
+
+	var lastErr error
+	for _, endpoint := range endpoints {
+		logrus.Debugf("Trying to push %s to %s %s", repoInfo.CanonicalName, endpoint.URL, endpoint.Version)
+
+		pusher, err := NewPusher(ref, endpoint, repoInfo, imagePushConfig, sf)
+		if err != nil {
+			lastErr = err
+			continue
+		}
+		if fallback, err := pusher.Push(); err != nil {
+			if fallback {
+				lastErr = err
+				continue
+			}
+			logrus.Debugf("Not continuing with error: %v", err)
+			return err
+
+		}
+
+		imagePushConfig.EventsService.Log("push", repoInfo.LocalName.Name(), "")
+		return nil
+	}
+
+	if lastErr == nil {
+		lastErr = fmt.Errorf("no endpoints found for %s", repoInfo.CanonicalName)
+	}
+	return lastErr
+}
+
+// compress returns an io.ReadCloser which will supply a compressed version of
+// the provided Reader. The caller must close the ReadCloser after reading the
+// compressed data.
+//
+// Note that this function returns a reader instead of taking a writer as an
+// argument so that it can be used with httpBlobWriter's ReadFrom method.
+// Using httpBlobWriter's Write method would send a PATCH request for every
+// Write call.
+func compress(in io.Reader) io.ReadCloser {
+	pipeReader, pipeWriter := io.Pipe()
+	// Use a bufio.Writer to avoid excessive chunking in HTTP request.
+	bufWriter := bufio.NewWriterSize(pipeWriter, compressionBufSize)
+	compressor := gzip.NewWriter(bufWriter)
+
+	go func() {
+		_, err := io.Copy(compressor, in)
+		if err == nil {
+			err = compressor.Close()
+		}
+		if err == nil {
+			err = bufWriter.Flush()
+		}
+		if err != nil {
+			pipeWriter.CloseWithError(err)
+		} else {
+			pipeWriter.Close()
+		}
+	}()
+
+	return pipeReader
+}

distribution/push_v1.go

@@ -0,0 +1,466 @@
+package distribution
+
+import (
+	"fmt"
+	"io"
+	"io/ioutil"
+	"sync"
+
+	"github.com/Sirupsen/logrus"
+	"github.com/docker/distribution/digest"
+	"github.com/docker/distribution/reference"
+	"github.com/docker/distribution/registry/client/transport"
+	"github.com/docker/docker/distribution/metadata"
+	"github.com/docker/docker/image"
+	"github.com/docker/docker/image/v1"
+	"github.com/docker/docker/layer"
+	"github.com/docker/docker/pkg/ioutils"
+	"github.com/docker/docker/pkg/progressreader"
+	"github.com/docker/docker/pkg/streamformatter"
+	"github.com/docker/docker/pkg/stringid"
+	"github.com/docker/docker/registry"
+)
+
+type v1Pusher struct {
+	v1IDService *metadata.V1IDService
+	endpoint    registry.APIEndpoint
+	ref         reference.Named
+	repoInfo    *registry.RepositoryInfo
+	config      *ImagePushConfig
+	sf          *streamformatter.StreamFormatter
+	session     *registry.Session
+
+	out io.Writer
+}
+
+func (p *v1Pusher) Push() (fallback bool, err error) {
+	tlsConfig, err := p.config.RegistryService.TLSConfig(p.repoInfo.Index.Name)
+	if err != nil {
+		return false, err
+	}
+	// Adds Docker-specific headers as well as user-specified headers (metaHeaders)
+	tr := transport.NewTransport(
+		// TODO(tiborvass): was NoTimeout
+		registry.NewTransport(tlsConfig),
+		registry.DockerHeaders(p.config.MetaHeaders)...,
+	)
+	client := registry.HTTPClient(tr)
+	v1Endpoint, err := p.endpoint.ToV1Endpoint(p.config.MetaHeaders)
+	if err != nil {
+		logrus.Debugf("Could not get v1 endpoint: %v", err)
+		return true, err
+	}
+	p.session, err = registry.NewSession(client, p.config.AuthConfig, v1Endpoint)
+	if err != nil {
+		// TODO(dmcgowan): Check if should fallback
+		return true, err
+	}
+	if err := p.pushRepository(); err != nil {
+		// TODO(dmcgowan): Check if should fallback
+		return false, err
+	}
+	return false, nil
+}
+
+// v1Image exposes the configuration, filesystem layer ID, and a v1 ID for an
+// image being pushed to a v1 registry.
+type v1Image interface {
+	Config() []byte
+	Layer() layer.Layer
+	V1ID() string
+}
+
+type v1ImageCommon struct {
+	layer  layer.Layer
+	config []byte
+	v1ID   string
+}
+
+func (common *v1ImageCommon) Config() []byte {
+	return common.config
+}
+
+func (common *v1ImageCommon) V1ID() string {
+	return common.v1ID
+}
+
+func (common *v1ImageCommon) Layer() layer.Layer {
+	return common.layer
+}
+
+// v1TopImage defines a runnable (top layer) image being pushed to a v1
+// registry.
+type v1TopImage struct {
+	v1ImageCommon
+	imageID image.ID
+}
+
+func newV1TopImage(imageID image.ID, img *image.Image, l layer.Layer, parent *v1DependencyImage) (*v1TopImage, error) {
+	v1ID := digest.Digest(imageID).Hex()
+	parentV1ID := ""
+	if parent != nil {
+		parentV1ID = parent.V1ID()
+	}
+
+	config, err := v1.MakeV1ConfigFromConfig(img, v1ID, parentV1ID, false)
+	if err != nil {
+		return nil, err
+	}
+
+	return &v1TopImage{
+		v1ImageCommon: v1ImageCommon{
+			v1ID:   v1ID,
+			config: config,
+			layer:  l,
+		},
+		imageID: imageID,
+	}, nil
+}
+
+// v1DependencyImage defines a dependency layer being pushed to a v1 registry.
+type v1DependencyImage struct {
+	v1ImageCommon
+}
+
+func newV1DependencyImage(l layer.Layer, parent *v1DependencyImage) (*v1DependencyImage, error) {
+	v1ID := digest.Digest(l.ChainID()).Hex()
+
+	config := ""
+	if parent != nil {
+		config = fmt.Sprintf(`{"id":"%s","parent":"%s"}`, v1ID, parent.V1ID())
+	} else {
+		config = fmt.Sprintf(`{"id":"%s"}`, v1ID)
+	}
+	return &v1DependencyImage{
+		v1ImageCommon: v1ImageCommon{
+			v1ID:   v1ID,
+			config: []byte(config),
+			layer:  l,
+		},
+	}, nil
+}
+
+// Retrieve the all the images to be uploaded in the correct order
+func (p *v1Pusher) getImageList() (imageList []v1Image, tagsByImage map[image.ID][]string, referencedLayers []layer.Layer, err error) {
+	tagsByImage = make(map[image.ID][]string)
+
+	// Ignore digest references
+	_, isDigested := p.ref.(reference.Digested)
+	if isDigested {
+		return
+	}
+
+	tagged, isTagged := p.ref.(reference.Tagged)
+	if isTagged {
+		// Push a specific tag
+		var imgID image.ID
+		imgID, err = p.config.TagStore.Get(p.ref)
+		if err != nil {
+			return
+		}
+
+		imageList, err = p.imageListForTag(imgID, nil, &referencedLayers)
+		if err != nil {
+			return
+		}
+
+		tagsByImage[imgID] = []string{tagged.Tag()}
+
+		return
+	}
+
+	imagesSeen := make(map[image.ID]struct{})
+	dependenciesSeen := make(map[layer.ChainID]*v1DependencyImage)
+
+	associations := p.config.TagStore.ReferencesByName(p.ref)
+	for _, association := range associations {
+		if tagged, isTagged = association.Ref.(reference.Tagged); !isTagged {
+			// Ignore digest references.
+			continue
+		}
+
+		tagsByImage[association.ImageID] = append(tagsByImage[association.ImageID], tagged.Tag())
+
+		if _, present := imagesSeen[association.ImageID]; present {
+			// Skip generating image list for already-seen image
+			continue
+		}
+		imagesSeen[association.ImageID] = struct{}{}
+
+		imageListForThisTag, err := p.imageListForTag(association.ImageID, dependenciesSeen, &referencedLayers)
+		if err != nil {
+			return nil, nil, nil, err
+		}
+
+		// append to main image list
+		imageList = append(imageList, imageListForThisTag...)
+	}
+	if len(imageList) == 0 {
+		return nil, nil, nil, fmt.Errorf("No images found for the requested repository / tag")
+	}
+	logrus.Debugf("Image list: %v", imageList)
+	logrus.Debugf("Tags by image: %v", tagsByImage)
+
+	return
+}
+
+func (p *v1Pusher) imageListForTag(imgID image.ID, dependenciesSeen map[layer.ChainID]*v1DependencyImage, referencedLayers *[]layer.Layer) (imageListForThisTag []v1Image, err error) {
+	img, err := p.config.ImageStore.Get(imgID)
+	if err != nil {
+		return nil, err
+	}
+
+	topLayerID := img.RootFS.ChainID()
+
+	var l layer.Layer
+	if topLayerID == "" {
+		l = layer.EmptyLayer
+	} else {
+		l, err = p.config.LayerStore.Get(topLayerID)
+		*referencedLayers = append(*referencedLayers, l)
+		if err != nil {
+			return nil, fmt.Errorf("failed to get top layer from image: %v", err)
+		}
+	}
+
+	dependencyImages, parent, err := generateDependencyImages(l.Parent(), dependenciesSeen)
+	if err != nil {
+		return nil, err
+	}
+
+	topImage, err := newV1TopImage(imgID, img, l, parent)
+	if err != nil {
+		return nil, err
+	}
+
+	imageListForThisTag = append(dependencyImages, topImage)
+
+	return
+}
+
+func generateDependencyImages(l layer.Layer, dependenciesSeen map[layer.ChainID]*v1DependencyImage) (imageListForThisTag []v1Image, parent *v1DependencyImage, err error) {
+	if l == nil {
+		return nil, nil, nil
+	}
+
+	imageListForThisTag, parent, err = generateDependencyImages(l.Parent(), dependenciesSeen)
+
+	if dependenciesSeen != nil {
+		if dependencyImage, present := dependenciesSeen[l.ChainID()]; present {
+			// This layer is already on the list, we can ignore it
+			// and all its parents.
+			return imageListForThisTag, dependencyImage, nil
+		}
+	}
+
+	dependencyImage, err := newV1DependencyImage(l, parent)
+	if err != nil {
+		return nil, nil, err
+	}
+	imageListForThisTag = append(imageListForThisTag, dependencyImage)
+
+	if dependenciesSeen != nil {
+		dependenciesSeen[l.ChainID()] = dependencyImage
+	}
+
+	return imageListForThisTag, dependencyImage, nil
+}
+
+// createImageIndex returns an index of an image's layer IDs and tags.
+func createImageIndex(images []v1Image, tags map[image.ID][]string) []*registry.ImgData {
+	var imageIndex []*registry.ImgData
+	for _, img := range images {
+		v1ID := img.V1ID()
+
+		if topImage, isTopImage := img.(*v1TopImage); isTopImage {
+			if tags, hasTags := tags[topImage.imageID]; hasTags {
+				// If an image has tags you must add an entry in the image index
+				// for each tag
+				for _, tag := range tags {
+					imageIndex = append(imageIndex, &registry.ImgData{
+						ID:  v1ID,
+						Tag: tag,
+					})
+				}
+				continue
+			}
+		}
+
+		// If the image does not have a tag it still needs to be sent to the
+		// registry with an empty tag so that it is associated with the repository
+		imageIndex = append(imageIndex, &registry.ImgData{
+			ID:  v1ID,
+			Tag: "",
+		})
+	}
+	return imageIndex
+}
+
+// lookupImageOnEndpoint checks the specified endpoint to see if an image exists
+// and if it is absent then it sends the image id to the channel to be pushed.
+func (p *v1Pusher) lookupImageOnEndpoint(wg *sync.WaitGroup, endpoint string, images chan v1Image, imagesToPush chan string) {
+	defer wg.Done()
+	for image := range images {
+		v1ID := image.V1ID()
+		if err := p.session.LookupRemoteImage(v1ID, endpoint); err != nil {
+			logrus.Errorf("Error in LookupRemoteImage: %s", err)
+			imagesToPush <- v1ID
+		} else {
+			p.out.Write(p.sf.FormatStatus("", "Image %s already pushed, skipping", stringid.TruncateID(v1ID)))
+		}
+	}
+}
+
+func (p *v1Pusher) pushImageToEndpoint(endpoint string, imageList []v1Image, tags map[image.ID][]string, repo *registry.RepositoryData) error {
+	workerCount := len(imageList)
+	// start a maximum of 5 workers to check if images exist on the specified endpoint.
+	if workerCount > 5 {
+		workerCount = 5
+	}
+	var (
+		wg           = &sync.WaitGroup{}
+		imageData    = make(chan v1Image, workerCount*2)
+		imagesToPush = make(chan string, workerCount*2)
+		pushes       = make(chan map[string]struct{}, 1)
+	)
+	for i := 0; i < workerCount; i++ {
+		wg.Add(1)
+		go p.lookupImageOnEndpoint(wg, endpoint, imageData, imagesToPush)
+	}
+	// start a go routine that consumes the images to push
+	go func() {
+		shouldPush := make(map[string]struct{})
+		for id := range imagesToPush {
+			shouldPush[id] = struct{}{}
+		}
+		pushes <- shouldPush
+	}()
+	for _, v1Image := range imageList {
+		imageData <- v1Image
+	}
+	// close the channel to notify the workers that there will be no more images to check.
+	close(imageData)
+	wg.Wait()
+	close(imagesToPush)
+	// wait for all the images that require pushes to be collected into a consumable map.
+	shouldPush := <-pushes
+	// finish by pushing any images and tags to the endpoint.  The order that the images are pushed
+	// is very important that is why we are still iterating over the ordered list of imageIDs.
+	for _, img := range imageList {
+		v1ID := img.V1ID()
+		if _, push := shouldPush[v1ID]; push {
+			if _, err := p.pushImage(img, endpoint); err != nil {
+				// FIXME: Continue on error?
+				return err
+			}
+		}
+		if topImage, isTopImage := img.(*v1TopImage); isTopImage {
+			for _, tag := range tags[topImage.imageID] {
+				p.out.Write(p.sf.FormatStatus("", "Pushing tag for rev [%s] on {%s}", stringid.TruncateID(v1ID), endpoint+"repositories/"+p.repoInfo.RemoteName.Name()+"/tags/"+tag))
+				if err := p.session.PushRegistryTag(p.repoInfo.RemoteName, v1ID, tag, endpoint); err != nil {
+					return err
+				}
+			}
+		}
+	}
+	return nil
+}
+
+// pushRepository pushes layers that do not already exist on the registry.
+func (p *v1Pusher) pushRepository() error {
+	p.out = ioutils.NewWriteFlusher(p.config.OutStream)
+	imgList, tags, referencedLayers, err := p.getImageList()
+	defer func() {
+		for _, l := range referencedLayers {
+			p.config.LayerStore.Release(l)
+		}
+	}()
+	if err != nil {
+		return err
+	}
+	p.out.Write(p.sf.FormatStatus("", "Sending image list"))
+
+	imageIndex := createImageIndex(imgList, tags)
+	for _, data := range imageIndex {
+		logrus.Debugf("Pushing ID: %s with Tag: %s", data.ID, data.Tag)
+	}
+
+	// Register all the images in a repository with the registry
+	// If an image is not in this list it will not be associated with the repository
+	repoData, err := p.session.PushImageJSONIndex(p.repoInfo.RemoteName, imageIndex, false, nil)
+	if err != nil {
+		return err
+	}
+	p.out.Write(p.sf.FormatStatus("", "Pushing repository %s", p.repoInfo.CanonicalName))
+	// push the repository to each of the endpoints only if it does not exist.
+	for _, endpoint := range repoData.Endpoints {
+		if err := p.pushImageToEndpoint(endpoint, imgList, tags, repoData); err != nil {
+			return err
+		}
+	}
+	_, err = p.session.PushImageJSONIndex(p.repoInfo.RemoteName, imageIndex, true, repoData.Endpoints)
+	return err
+}
+
+func (p *v1Pusher) pushImage(v1Image v1Image, ep string) (checksum string, err error) {
+	v1ID := v1Image.V1ID()
+
+	jsonRaw := v1Image.Config()
+	p.out.Write(p.sf.FormatProgress(stringid.TruncateID(v1ID), "Pushing", nil))
+
+	// General rule is to use ID for graph accesses and compatibilityID for
+	// calls to session.registry()
+	imgData := &registry.ImgData{
+		ID: v1ID,
+	}
+
+	// Send the json
+	if err := p.session.PushImageJSONRegistry(imgData, jsonRaw, ep); err != nil {
+		if err == registry.ErrAlreadyExists {
+			p.out.Write(p.sf.FormatProgress(stringid.TruncateID(v1ID), "Image already pushed, skipping", nil))
+			return "", nil
+		}
+		return "", err
+	}
+
+	l := v1Image.Layer()
+
+	arch, err := l.TarStream()
+	if err != nil {
+		return "", err
+	}
+
+	// don't care if this fails; best effort
+	size, _ := l.Size()
+
+	// Send the layer
+	logrus.Debugf("rendered layer for %s of [%d] size", v1ID, size)
+
+	reader := progressreader.New(progressreader.Config{
+		In:        ioutil.NopCloser(arch),
+		Out:       p.out,
+		Formatter: p.sf,
+		Size:      size,
+		NewLines:  false,
+		ID:        stringid.TruncateID(v1ID),
+		Action:    "Pushing",
+	})
+
+	checksum, checksumPayload, err := p.session.PushImageLayerRegistry(v1ID, reader, ep, jsonRaw)
+	if err != nil {
+		return "", err
+	}
+	imgData.Checksum = checksum
+	imgData.ChecksumPayload = checksumPayload
+	// Send the checksum
+	if err := p.session.PushImageChecksumRegistry(imgData, ep); err != nil {
+		return "", err
+	}
+
+	if err := p.v1IDService.Set(v1ID, p.repoInfo.Index.Name, l.ChainID()); err != nil {
+		logrus.Warnf("Could not set v1 ID mapping: %v", err)
+	}
+
+	p.out.Write(p.sf.FormatProgress(stringid.TruncateID(v1ID), "Image successfully pushed", nil))
+	return imgData.Checksum, nil
+}

distribution/push_v2.go

@@ -0,0 +1,410 @@
+package distribution
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"time"
+
+	"github.com/Sirupsen/logrus"
+	"github.com/docker/distribution"
+	"github.com/docker/distribution/digest"
+	"github.com/docker/distribution/manifest"
+	"github.com/docker/distribution/manifest/schema1"
+	"github.com/docker/distribution/reference"
+	"github.com/docker/docker/distribution/metadata"
+	"github.com/docker/docker/image"
+	"github.com/docker/docker/image/v1"
+	"github.com/docker/docker/layer"
+	"github.com/docker/docker/pkg/progressreader"
+	"github.com/docker/docker/pkg/streamformatter"
+	"github.com/docker/docker/pkg/stringid"
+	"github.com/docker/docker/registry"
+	"github.com/docker/docker/tag"
+	"golang.org/x/net/context"
+)
+
+type v2Pusher struct {
+	blobSumService *metadata.BlobSumService
+	ref            reference.Named
+	endpoint       registry.APIEndpoint
+	repoInfo       *registry.RepositoryInfo
+	config         *ImagePushConfig
+	sf             *streamformatter.StreamFormatter
+	repo           distribution.Repository
+
+	// layersPushed is the set of layers known to exist on the remote side.
+	// This avoids redundant queries when pushing multiple tags that
+	// involve the same layers.
+	layersPushed map[digest.Digest]bool
+}
+
+func (p *v2Pusher) Push() (fallback bool, err error) {
+	p.repo, err = NewV2Repository(p.repoInfo, p.endpoint, p.config.MetaHeaders, p.config.AuthConfig, "push", "pull")
+	if err != nil {
+		logrus.Debugf("Error getting v2 registry: %v", err)
+		return true, err
+	}
+
+	localName := p.repoInfo.LocalName.Name()
+
+	var associations []tag.Association
+	if _, isTagged := p.ref.(reference.Tagged); isTagged {
+		imageID, err := p.config.TagStore.Get(p.ref)
+		if err != nil {
+			return false, fmt.Errorf("tag does not exist: %s", p.ref.String())
+		}
+
+		associations = []tag.Association{
+			{
+				Ref:     p.ref,
+				ImageID: imageID,
+			},
+		}
+	} else {
+		// Pull all tags
+		associations = p.config.TagStore.ReferencesByName(p.ref)
+	}
+	if err != nil {
+		return false, fmt.Errorf("error getting tags for %s: %s", localName, err)
+	}
+	if len(associations) == 0 {
+		return false, fmt.Errorf("no tags to push for %s", localName)
+	}
+
+	for _, association := range associations {
+		if err := p.pushV2Tag(association); err != nil {
+			return false, err
+		}
+	}
+
+	return false, nil
+}
+
+func (p *v2Pusher) pushV2Tag(association tag.Association) error {
+	ref := association.Ref
+	logrus.Debugf("Pushing repository: %s", ref.String())
+
+	img, err := p.config.ImageStore.Get(association.ImageID)
+	if err != nil {
+		return fmt.Errorf("could not find image from tag %s: %v", ref.String(), err)
+	}
+
+	out := p.config.OutStream
+
+	var l layer.Layer
+
+	topLayerID := img.RootFS.ChainID()
+	if topLayerID == "" {
+		l = layer.EmptyLayer
+	} else {
+		l, err = p.config.LayerStore.Get(topLayerID)
+		if err != nil {
+			return fmt.Errorf("failed to get top layer from image: %v", err)
+		}
+		defer layer.ReleaseAndLog(p.config.LayerStore, l)
+	}
+
+	fsLayers := make(map[layer.DiffID]schema1.FSLayer)
+
+	// Push empty layer if necessary
+	for _, h := range img.History {
+		if h.EmptyLayer {
+			dgst, err := p.pushLayerIfNecessary(out, layer.EmptyLayer)
+			if err != nil {
+				return err
+			}
+			p.layersPushed[dgst] = true
+			fsLayers[layer.EmptyLayer.DiffID()] = schema1.FSLayer{BlobSum: dgst}
+			break
+		}
+	}
+
+	for i := 0; i < len(img.RootFS.DiffIDs); i++ {
+		dgst, err := p.pushLayerIfNecessary(out, l)
+		if err != nil {
+			return err
+		}
+
+		p.layersPushed[dgst] = true
+		fsLayers[l.DiffID()] = schema1.FSLayer{BlobSum: dgst}
+
+		l = l.Parent()
+	}
+
+	var tag string
+	if tagged, isTagged := ref.(reference.Tagged); isTagged {
+		tag = tagged.Tag()
+	}
+	m, err := CreateV2Manifest(p.repo.Name(), tag, img, fsLayers)
+	if err != nil {
+		return err
+	}
+
+	logrus.Infof("Signed manifest for %s using daemon's key: %s", ref.String(), p.config.TrustKey.KeyID())
+	signed, err := schema1.Sign(m, p.config.TrustKey)
+	if err != nil {
+		return err
+	}
+
+	manifestDigest, manifestSize, err := digestFromManifest(signed, p.repo.Name())
+	if err != nil {
+		return err
+	}
+	if manifestDigest != "" {
+		if tagged, isTagged := ref.(reference.Tagged); isTagged {
+			// NOTE: do not change this format without first changing the trust client
+			// code. This information is used to determine what was pushed and should be signed.
+			out.Write(p.sf.FormatStatus("", "%s: digest: %s size: %d", tagged.Tag(), manifestDigest, manifestSize))
+		}
+	}
+
+	manSvc, err := p.repo.Manifests(context.Background())
+	if err != nil {
+		return err
+	}
+	return manSvc.Put(signed)
+}
+
+func (p *v2Pusher) pushLayerIfNecessary(out io.Writer, l layer.Layer) (digest.Digest, error) {
+	logrus.Debugf("Pushing layer: %s", l.DiffID())
+
+	// Do we have any blobsums associated with this layer's DiffID?
+	possibleBlobsums, err := p.blobSumService.GetBlobSums(l.DiffID())
+	if err == nil {
+		dgst, exists, err := p.blobSumAlreadyExists(possibleBlobsums)
+		if err != nil {
+			out.Write(p.sf.FormatProgress(stringid.TruncateID(string(l.DiffID())), "Image push failed", nil))
+			return "", err
+		}
+		if exists {
+			out.Write(p.sf.FormatProgress(stringid.TruncateID(string(l.DiffID())), "Layer already exists", nil))
+			return dgst, nil
+		}
+	}
+
+	// if digest was empty or not saved, or if blob does not exist on the remote repository,
+	// then push the blob.
+	pushDigest, err := p.pushV2Layer(p.repo.Blobs(context.Background()), l)
+	if err != nil {
+		return "", err
+	}
+	// Cache mapping from this layer's DiffID to the blobsum
+	if err := p.blobSumService.Add(l.DiffID(), pushDigest); err != nil {
+		return "", err
+	}
+
+	return pushDigest, nil
+}
+
+// blobSumAlreadyExists checks if the registry already know about any of the
+// blobsums passed in the "blobsums" slice. If it finds one that the registry
+// knows about, it returns the known digest and "true".
+func (p *v2Pusher) blobSumAlreadyExists(blobsums []digest.Digest) (digest.Digest, bool, error) {
+	for _, dgst := range blobsums {
+		if p.layersPushed[dgst] {
+			// it is already known that the push is not needed and
+			// therefore doing a stat is unnecessary
+			return dgst, true, nil
+		}
+		_, err := p.repo.Blobs(context.Background()).Stat(context.Background(), dgst)
+		switch err {
+		case nil:
+			return dgst, true, nil
+		case distribution.ErrBlobUnknown:
+			// nop
+		default:
+			return "", false, err
+		}
+	}
+	return "", false, nil
+}
+
+// CreateV2Manifest creates a V2 manifest from an image config and set of
+// FSLayer digests.
+// FIXME: This should be moved to the distribution repo, since it will also
+// be useful for converting new manifests to the old format.
+func CreateV2Manifest(name, tag string, img *image.Image, fsLayers map[layer.DiffID]schema1.FSLayer) (*schema1.Manifest, error) {
+	if len(img.History) == 0 {
+		return nil, errors.New("empty history when trying to create V2 manifest")
+	}
+
+	// Generate IDs for each layer
+	// For non-top-level layers, create fake V1Compatibility strings that
+	// fit the format and don't collide with anything else, but don't
+	// result in runnable images on their own.
+	type v1Compatibility struct {
+		ID              string    `json:"id"`
+		Parent          string    `json:"parent,omitempty"`
+		Comment         string    `json:"comment,omitempty"`
+		Created         time.Time `json:"created"`
+		ContainerConfig struct {
+			Cmd []string
+		} `json:"container_config,omitempty"`
+		ThrowAway bool `json:"throwaway,omitempty"`
+	}
+
+	fsLayerList := make([]schema1.FSLayer, len(img.History))
+	history := make([]schema1.History, len(img.History))
+
+	parent := ""
+	layerCounter := 0
+	for i, h := range img.History {
+		if i == len(img.History)-1 {
+			break
+		}
+
+		var diffID layer.DiffID
+		if h.EmptyLayer {
+			diffID = layer.EmptyLayer.DiffID()
+		} else {
+			if len(img.RootFS.DiffIDs) <= layerCounter {
+				return nil, errors.New("too many non-empty layers in History section")
+			}
+			diffID = img.RootFS.DiffIDs[layerCounter]
+			layerCounter++
+		}
+
+		fsLayer, present := fsLayers[diffID]
+		if !present {
+			return nil, fmt.Errorf("missing layer in CreateV2Manifest: %s", diffID.String())
+		}
+		dgst, err := digest.FromBytes([]byte(fsLayer.BlobSum.Hex() + " " + parent))
+		if err != nil {
+			return nil, err
+		}
+		v1ID := dgst.Hex()
+
+		v1Compatibility := v1Compatibility{
+			ID:      v1ID,
+			Parent:  parent,
+			Comment: h.Comment,
+			Created: h.Created,
+		}
+		v1Compatibility.ContainerConfig.Cmd = []string{img.History[i].CreatedBy}
+		if h.EmptyLayer {
+			v1Compatibility.ThrowAway = true
+		}
+		jsonBytes, err := json.Marshal(&v1Compatibility)
+		if err != nil {
+			return nil, err
+		}
+
+		reversedIndex := len(img.History) - i - 1
+		history[reversedIndex].V1Compatibility = string(jsonBytes)
+		fsLayerList[reversedIndex] = fsLayer
+
+		parent = v1ID
+	}
+
+	latestHistory := img.History[len(img.History)-1]
+
+	var diffID layer.DiffID
+	if latestHistory.EmptyLayer {
+		diffID = layer.EmptyLayer.DiffID()
+	} else {
+		if len(img.RootFS.DiffIDs) <= layerCounter {
+			return nil, errors.New("too many non-empty layers in History section")
+		}
+		diffID = img.RootFS.DiffIDs[layerCounter]
+	}
+	fsLayer, present := fsLayers[diffID]
+	if !present {
+		return nil, fmt.Errorf("missing layer in CreateV2Manifest: %s", diffID.String())
+	}
+
+	dgst, err := digest.FromBytes([]byte(fsLayer.BlobSum.Hex() + " " + parent + " " + string(img.RawJSON())))
+	if err != nil {
+		return nil, err
+	}
+	fsLayerList[0] = fsLayer
+
+	// Top-level v1compatibility string should be a modified version of the
+	// image config.
+	transformedConfig, err := v1.MakeV1ConfigFromConfig(img, dgst.Hex(), parent, latestHistory.EmptyLayer)
+	if err != nil {
+		return nil, err
+	}
+
+	history[0].V1Compatibility = string(transformedConfig)
+
+	// windows-only baselayer setup
+	if err := setupBaseLayer(history, *img.RootFS); err != nil {
+		return nil, err
+	}
+
+	return &schema1.Manifest{
+		Versioned: manifest.Versioned{
+			SchemaVersion: 1,
+		},
+		Name:         name,
+		Tag:          tag,
+		Architecture: img.Architecture,
+		FSLayers:     fsLayerList,
+		History:      history,
+	}, nil
+}
+
+func rawJSON(value interface{}) *json.RawMessage {
+	jsonval, err := json.Marshal(value)
+	if err != nil {
+		return nil
+	}
+	return (*json.RawMessage)(&jsonval)
+}
+
+func (p *v2Pusher) pushV2Layer(bs distribution.BlobService, l layer.Layer) (digest.Digest, error) {
+	out := p.config.OutStream
+	displayID := stringid.TruncateID(string(l.DiffID()))
+
+	out.Write(p.sf.FormatProgress(displayID, "Preparing", nil))
+
+	arch, err := l.TarStream()
+	if err != nil {
+		return "", err
+	}
+
+	// Send the layer
+	layerUpload, err := bs.Create(context.Background())
+	if err != nil {
+		return "", err
+	}
+	defer layerUpload.Close()
+
+	// don't care if this fails; best effort
+	size, _ := l.DiffSize()
+
+	reader := progressreader.New(progressreader.Config{
+		In:        ioutil.NopCloser(arch), // we'll take care of close here.
+		Out:       out,
+		Formatter: p.sf,
+		Size:      size,
+		NewLines:  false,
+		ID:        displayID,
+		Action:    "Pushing",
+	})
+
+	compressedReader := compress(reader)
+
+	digester := digest.Canonical.New()
+	tee := io.TeeReader(compressedReader, digester.Hash())
+
+	out.Write(p.sf.FormatProgress(displayID, "Pushing", nil))
+	nn, err := layerUpload.ReadFrom(tee)
+	compressedReader.Close()
+	if err != nil {
+		return "", err
+	}
+
+	dgst := digester.Digest()
+	if _, err := layerUpload.Commit(context.Background(), distribution.Descriptor{Digest: dgst}); err != nil {
+		return "", err
+	}
+
+	logrus.Debugf("uploaded layer %s (%s), %d bytes", l.DiffID(), dgst, nn)
+	out.Write(p.sf.FormatProgress(displayID, "Pushed", nil))
+
+	return dgst, nil
+}

distribution/push_v2_test.go

@@ -0,0 +1,176 @@
+package distribution
+
+import (
+	"reflect"
+	"testing"
+
+	"github.com/docker/distribution/digest"
+	"github.com/docker/distribution/manifest/schema1"
+	"github.com/docker/docker/image"
+	"github.com/docker/docker/layer"
+)
+
+func TestCreateV2Manifest(t *testing.T) {
+	imgJSON := `{
+    "architecture": "amd64",
+    "config": {
+        "AttachStderr": false,
+        "AttachStdin": false,
+        "AttachStdout": false,
+        "Cmd": [
+            "/bin/sh",
+            "-c",
+            "echo hi"
+        ],
+        "Domainname": "",
+        "Entrypoint": null,
+        "Env": [
+            "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
+            "derived=true",
+            "asdf=true"
+        ],
+        "Hostname": "23304fc829f9",
+        "Image": "sha256:4ab15c48b859c2920dd5224f92aabcd39a52794c5b3cf088fb3bbb438756c246",
+        "Labels": {},
+        "OnBuild": [],
+        "OpenStdin": false,
+        "StdinOnce": false,
+        "Tty": false,
+        "User": "",
+        "Volumes": null,
+        "WorkingDir": ""
+    },
+    "container": "e91032eb0403a61bfe085ff5a5a48e3659e5a6deae9f4d678daa2ae399d5a001",
+    "container_config": {
+        "AttachStderr": false,
+        "AttachStdin": false,
+        "AttachStdout": false,
+        "Cmd": [
+            "/bin/sh",
+            "-c",
+            "#(nop) CMD [\"/bin/sh\" \"-c\" \"echo hi\"]"
+        ],
+        "Domainname": "",
+        "Entrypoint": null,
+        "Env": [
+            "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
+            "derived=true",
+            "asdf=true"
+        ],
+        "Hostname": "23304fc829f9",
+        "Image": "sha256:4ab15c48b859c2920dd5224f92aabcd39a52794c5b3cf088fb3bbb438756c246",
+        "Labels": {},
+        "OnBuild": [],
+        "OpenStdin": false,
+        "StdinOnce": false,
+        "Tty": false,
+        "User": "",
+        "Volumes": null,
+        "WorkingDir": ""
+    },
+    "created": "2015-11-04T23:06:32.365666163Z",
+    "docker_version": "1.9.0-dev",
+    "history": [
+        {
+            "created": "2015-10-31T22:22:54.690851953Z",
+            "created_by": "/bin/sh -c #(nop) ADD file:a3bc1e842b69636f9df5256c49c5374fb4eef1e281fe3f282c65fb853ee171c5 in /"
+        },
+        {
+            "created": "2015-10-31T22:22:55.613815829Z",
+            "created_by": "/bin/sh -c #(nop) CMD [\"sh\"]"
+        },
+        {
+            "created": "2015-11-04T23:06:30.934316144Z",
+            "created_by": "/bin/sh -c #(nop) ENV derived=true",
+            "empty_layer": true
+        },
+        {
+            "created": "2015-11-04T23:06:31.192097572Z",
+            "created_by": "/bin/sh -c #(nop) ENV asdf=true",
+            "empty_layer": true
+        },
+        {
+            "created": "2015-11-04T23:06:32.083868454Z",
+            "created_by": "/bin/sh -c dd if=/dev/zero of=/file bs=1024 count=1024"
+        },
+        {
+            "created": "2015-11-04T23:06:32.365666163Z",
+            "created_by": "/bin/sh -c #(nop) CMD [\"/bin/sh\" \"-c\" \"echo hi\"]",
+            "empty_layer": true
+        }
+    ],
+    "os": "linux",
+    "rootfs": {
+        "diff_ids": [
+            "sha256:c6f988f4874bb0add23a778f753c65efe992244e148a1d2ec2a8b664fb66bbd1",
+            "sha256:5f70bf18a086007016e948b04aed3b82103a36bea41755b6cddfaf10ace3c6ef",
+            "sha256:13f53e08df5a220ab6d13c58b2bf83a59cbdc2e04d0a3f041ddf4b0ba4112d49"
+        ],
+        "type": "layers"
+    }
+}`
+
+	// To fill in rawJSON
+	img, err := image.NewFromJSON([]byte(imgJSON))
+	if err != nil {
+		t.Fatalf("json decoding failed: %v", err)
+	}
+
+	fsLayers := map[layer.DiffID]schema1.FSLayer{
+		layer.DiffID("sha256:c6f988f4874bb0add23a778f753c65efe992244e148a1d2ec2a8b664fb66bbd1"): {BlobSum: digest.Digest("sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4")},
+		layer.DiffID("sha256:5f70bf18a086007016e948b04aed3b82103a36bea41755b6cddfaf10ace3c6ef"): {BlobSum: digest.Digest("sha256:86e0e091d0da6bde2456dbb48306f3956bbeb2eae1b5b9a43045843f69fe4aaa")},
+		layer.DiffID("sha256:13f53e08df5a220ab6d13c58b2bf83a59cbdc2e04d0a3f041ddf4b0ba4112d49"): {BlobSum: digest.Digest("sha256:b4ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4")},
+	}
+
+	manifest, err := CreateV2Manifest("testrepo", "testtag", img, fsLayers)
+	if err != nil {
+		t.Fatalf("CreateV2Manifest returned error: %v", err)
+	}
+
+	if manifest.Versioned.SchemaVersion != 1 {
+		t.Fatal("SchemaVersion != 1")
+	}
+	if manifest.Name != "testrepo" {
+		t.Fatal("incorrect name in manifest")
+	}
+	if manifest.Tag != "testtag" {
+		t.Fatal("incorrect tag in manifest")
+	}
+	if manifest.Architecture != "amd64" {
+		t.Fatal("incorrect arch in manifest")
+	}
+
+	expectedFSLayers := []schema1.FSLayer{
+		{BlobSum: digest.Digest("sha256:86e0e091d0da6bde2456dbb48306f3956bbeb2eae1b5b9a43045843f69fe4aaa")},
+		{BlobSum: digest.Digest("sha256:b4ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4")},
+		{BlobSum: digest.Digest("sha256:86e0e091d0da6bde2456dbb48306f3956bbeb2eae1b5b9a43045843f69fe4aaa")},
+		{BlobSum: digest.Digest("sha256:86e0e091d0da6bde2456dbb48306f3956bbeb2eae1b5b9a43045843f69fe4aaa")},
+		{BlobSum: digest.Digest("sha256:86e0e091d0da6bde2456dbb48306f3956bbeb2eae1b5b9a43045843f69fe4aaa")},
+		{BlobSum: digest.Digest("sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4")},
+	}
+
+	if len(manifest.FSLayers) != len(expectedFSLayers) {
+		t.Fatalf("wrong number of FSLayers: %d", len(manifest.FSLayers))
+	}
+	if !reflect.DeepEqual(manifest.FSLayers, expectedFSLayers) {
+		t.Fatal("wrong FSLayers list")
+	}
+
+	expectedV1Compatibility := []string{
+		`{"architecture":"amd64","config":{"AttachStderr":false,"AttachStdin":false,"AttachStdout":false,"Cmd":["/bin/sh","-c","echo hi"],"Domainname":"","Entrypoint":null,"Env":["PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin","derived=true","asdf=true"],"Hostname":"23304fc829f9","Image":"sha256:4ab15c48b859c2920dd5224f92aabcd39a52794c5b3cf088fb3bbb438756c246","Labels":{},"OnBuild":[],"OpenStdin":false,"StdinOnce":false,"Tty":false,"User":"","Volumes":null,"WorkingDir":""},"container":"e91032eb0403a61bfe085ff5a5a48e3659e5a6deae9f4d678daa2ae399d5a001","container_config":{"AttachStderr":false,"AttachStdin":false,"AttachStdout":false,"Cmd":["/bin/sh","-c","#(nop) CMD [\"/bin/sh\" \"-c\" \"echo hi\"]"],"Domainname":"","Entrypoint":null,"Env":["PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin","derived=true","asdf=true"],"Hostname":"23304fc829f9","Image":"sha256:4ab15c48b859c2920dd5224f92aabcd39a52794c5b3cf088fb3bbb438756c246","Labels":{},"OnBuild":[],"OpenStdin":false,"StdinOnce":false,"Tty":false,"User":"","Volumes":null,"WorkingDir":""},"created":"2015-11-04T23:06:32.365666163Z","docker_version":"1.9.0-dev","id":"d728140d3fd23dfcac505954af0b2224b3579b177029eded62916579eb19ac64","os":"linux","parent":"0594e66a9830fa5ba73b66349eb221ea4beb6bac8d2148b90a0f371f8d67bcd5","throwaway":true}`,
+		`{"id":"0594e66a9830fa5ba73b66349eb221ea4beb6bac8d2148b90a0f371f8d67bcd5","parent":"39bc0dbed47060dd8952b048e73744ae471fe50354d2c267d308292c53b83ce1","created":"2015-11-04T23:06:32.083868454Z","container_config":{"Cmd":["/bin/sh -c dd if=/dev/zero of=/file bs=1024 count=1024"]}}`,
+		`{"id":"39bc0dbed47060dd8952b048e73744ae471fe50354d2c267d308292c53b83ce1","parent":"875d7f206c023dc979e1677567a01364074f82b61e220c9b83a4610170490381","created":"2015-11-04T23:06:31.192097572Z","container_config":{"Cmd":["/bin/sh -c #(nop) ENV asdf=true"]},"throwaway":true}`,
+		`{"id":"875d7f206c023dc979e1677567a01364074f82b61e220c9b83a4610170490381","parent":"9e3447ca24cb96d86ebd5960cb34d1299b07e0a0e03801d90b9969a2c187dd6e","created":"2015-11-04T23:06:30.934316144Z","container_config":{"Cmd":["/bin/sh -c #(nop) ENV derived=true"]},"throwaway":true}`,
+		`{"id":"9e3447ca24cb96d86ebd5960cb34d1299b07e0a0e03801d90b9969a2c187dd6e","parent":"3690474eb5b4b26fdfbd89c6e159e8cc376ca76ef48032a30fa6aafd56337880","created":"2015-10-31T22:22:55.613815829Z","container_config":{"Cmd":["/bin/sh -c #(nop) CMD [\"sh\"]"]}}`,
+		`{"id":"3690474eb5b4b26fdfbd89c6e159e8cc376ca76ef48032a30fa6aafd56337880","created":"2015-10-31T22:22:54.690851953Z","container_config":{"Cmd":["/bin/sh -c #(nop) ADD file:a3bc1e842b69636f9df5256c49c5374fb4eef1e281fe3f282c65fb853ee171c5 in /"]}}`,
+	}
+
+	if len(manifest.History) != len(expectedV1Compatibility) {
+		t.Fatalf("wrong number of history entries: %d", len(manifest.History))
+	}
+	for i := range expectedV1Compatibility {
+		if manifest.History[i].V1Compatibility != expectedV1Compatibility[i] {
+			t.Fatalf("wrong V1Compatibility %d. expected:\n%s\ngot:\n%s", i, expectedV1Compatibility[i], manifest.History[i].V1Compatibility)
+		}
+	}
+}

distribution/push_v2_unix.go

@@ -0,0 +1,12 @@
+// +build !windows
+
+package distribution
+
+import (
+	"github.com/docker/distribution/manifest/schema1"
+	"github.com/docker/docker/image"
+)
+
+func setupBaseLayer(history []schema1.History, rootFS image.RootFS) error {
+	return nil
+}

distribution/push_v2_windows.go

@@ -0,0 +1,28 @@
+// +build windows
+
+package distribution
+
+import (
+	"encoding/json"
+
+	"github.com/docker/distribution/manifest/schema1"
+	"github.com/docker/docker/image"
+)
+
+func setupBaseLayer(history []schema1.History, rootFS image.RootFS) error {
+	var v1Config map[string]*json.RawMessage
+	if err := json.Unmarshal([]byte(history[len(history)-1].V1Compatibility), &v1Config); err != nil {
+		return err
+	}
+	baseID, err := json.Marshal(rootFS.BaseLayerID())
+	if err != nil {
+		return err
+	}
+	v1Config["parent"] = (*json.RawMessage)(&baseID)
+	configJSON, err := json.Marshal(v1Config)
+	if err != nil {
+		return err
+	}
+	history[len(history)-1].V1Compatibility = string(configJSON)
+	return nil
+}

graph/registry.go → distribution/registry.go

@@ -1,13 +1,12 @@
-package graph
+package distribution
 
 import (
 	"errors"
 	"net"
 	"net/http"
 	"net/url"
-	"time"
-
 	"strings"
+	"time"
 
 	"github.com/Sirupsen/logrus"
 	"github.com/docker/distribution"
@@ -29,10 +28,10 @@ func (dcs dumbCredentialStore) Basic(*url.URL) (string, string) {
 	return dcs.auth.Username, dcs.auth.Password
 }
 
-// newV2Repository returns a repository (v2 only). It creates a HTTP transport
+// NewV2Repository returns a repository (v2 only). It creates a HTTP transport
 // providing timeout settings and authentication support, and also verifies the
 // remote API version.
-func newV2Repository(repoInfo *registry.RepositoryInfo, endpoint registry.APIEndpoint, metaHeaders http.Header, authConfig *cliconfig.AuthConfig, actions ...string) (distribution.Repository, error) {
+func NewV2Repository(repoInfo *registry.RepositoryInfo, endpoint registry.APIEndpoint, metaHeaders http.Header, authConfig *cliconfig.AuthConfig, actions ...string) (distribution.Repository, error) {
 	ctx := context.Background()
 
 	repoName := repoInfo.CanonicalName
@@ -59,7 +58,7 @@ func newV2Repository(repoInfo *registry.RepositoryInfo, endpoint registry.APIEnd
 	authTransport := transport.NewTransport(base, modifiers...)
 	pingClient := &http.Client{
 		Transport: authTransport,
-		Timeout:   15 * time.Second,
+		Timeout:   5 * time.Second,
 	}
 	endpointStr := strings.TrimRight(endpoint.URL, "/") + "/v2/"
 	req, err := http.NewRequest("GET", endpointStr, nil)
@@ -93,12 +92,12 @@ func newV2Repository(repoInfo *registry.RepositoryInfo, endpoint registry.APIEnd
 	}
 
 	creds := dumbCredentialStore{auth: authConfig}
-	tokenHandler := auth.NewTokenHandler(authTransport, creds, repoName, actions...)
+	tokenHandler := auth.NewTokenHandler(authTransport, creds, repoName.Name(), actions...)
 	basicHandler := auth.NewBasicHandler(creds)
 	modifiers = append(modifiers, auth.NewAuthorizer(challengeManager, tokenHandler, basicHandler))
 	tr := transport.NewTransport(base, modifiers...)
 
-	return client.NewRepository(ctx, repoName, endpoint.URL, tr)
+	return client.NewRepository(ctx, repoName.Name(), endpoint.URL, tr)
 }
 
 func digestFromManifest(m *schema1.SignedManifest, localName string) (digest.Digest, int, error) {

errors/daemon.go

@@ -832,15 +832,6 @@ var (
 		HTTPStatusCode: http.StatusInternalServerError,
 	})
 
-	// ErrorCodeRmInit is generated when we try to delete a container
-	// but failed deleting its init filesystem.
-	ErrorCodeRmInit = errcode.Register(errGroup, errcode.ErrorDescriptor{
-		Value:          "RMINIT",
-		Message:        "Driver %s failed to remove init filesystem %s: %s",
-		Description:    "While trying to delete a container, the driver failed to remove the init filesystem",
-		HTTPStatusCode: http.StatusInternalServerError,
-	})
-
 	// ErrorCodeRmFS is generated when we try to delete a container
 	// but failed deleting its filesystem.
 	ErrorCodeRmFS = errcode.Register(errGroup, errcode.ErrorDescriptor{

graph/export.go

@@ -1,180 +0,0 @@
-package graph
-
-import (
-	"encoding/json"
-	"fmt"
-	"io"
-	"io/ioutil"
-	"os"
-	"path/filepath"
-	"time"
-
-	"github.com/Sirupsen/logrus"
-	"github.com/docker/distribution/digest"
-	"github.com/docker/docker/pkg/archive"
-	"github.com/docker/docker/pkg/parsers"
-	"github.com/docker/docker/registry"
-)
-
-// ImageExport exports list of images to a output stream specified in the
-// config. The exported images are archived into a tar when written to the
-// output stream. All images with the given tag and all versions containing the
-// same tag are exported. names is the set of tags to export, and outStream
-// is the writer which the images are written to.
-func (s *TagStore) ImageExport(names []string, outStream io.Writer) error {
-	// get image json
-	tempdir, err := ioutil.TempDir("", "docker-export-")
-	if err != nil {
-		return err
-	}
-	defer os.RemoveAll(tempdir)
-
-	rootRepoMap := map[string]repository{}
-	addKey := func(name string, tag string, id string) {
-		logrus.Debugf("add key [%s:%s]", name, tag)
-		if repo, ok := rootRepoMap[name]; !ok {
-			rootRepoMap[name] = repository{tag: id}
-		} else {
-			repo[tag] = id
-		}
-	}
-	for _, name := range names {
-		name = registry.NormalizeLocalName(name)
-		logrus.Debugf("Serializing %s", name)
-		rootRepo := s.Repositories[name]
-		if rootRepo != nil {
-			// this is a base repo name, like 'busybox'
-			for tag, id := range rootRepo {
-				addKey(name, tag, id)
-				if err := s.exportImage(id, tempdir); err != nil {
-					return err
-				}
-			}
-		} else {
-			img, err := s.LookupImage(name)
-			if err != nil {
-				return err
-			}
-
-			if img != nil {
-				// This is a named image like 'busybox:latest'
-				repoName, repoTag := parsers.ParseRepositoryTag(name)
-
-				// Skip digests on save
-				if _, err := digest.ParseDigest(repoTag); err == nil {
-					repoTag = ""
-				}
-
-				// check this length, because a lookup of a truncated has will not have a tag
-				// and will not need to be added to this map
-				if len(repoTag) > 0 {
-					addKey(repoName, repoTag, img.ID)
-				}
-				if err := s.exportImage(img.ID, tempdir); err != nil {
-					return err
-				}
-
-			} else {
-				// this must be an ID that didn't get looked up just right?
-				if err := s.exportImage(name, tempdir); err != nil {
-					return err
-				}
-			}
-		}
-		logrus.Debugf("End Serializing %s", name)
-	}
-	// write repositories, if there is something to write
-	if len(rootRepoMap) > 0 {
-		f, err := os.OpenFile(filepath.Join(tempdir, "repositories"), os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644)
-		if err != nil {
-			f.Close()
-			return err
-		}
-		if err := json.NewEncoder(f).Encode(rootRepoMap); err != nil {
-			return err
-		}
-		if err := f.Close(); err != nil {
-			return err
-		}
-		if err := os.Chtimes(filepath.Join(tempdir, "repositories"), time.Unix(0, 0), time.Unix(0, 0)); err != nil {
-			return err
-		}
-	} else {
-		logrus.Debugf("There were no repositories to write")
-	}
-
-	fs, err := archive.Tar(tempdir, archive.Uncompressed)
-	if err != nil {
-		return err
-	}
-	defer fs.Close()
-
-	if _, err := io.Copy(outStream, fs); err != nil {
-		return err
-	}
-	logrus.Debugf("End export image")
-	return nil
-}
-
-func (s *TagStore) exportImage(name, tempdir string) error {
-	for n := name; n != ""; {
-		img, err := s.LookupImage(n)
-		if err != nil || img == nil {
-			return fmt.Errorf("No such image %s", n)
-		}
-
-		// temporary directory
-		tmpImageDir := filepath.Join(tempdir, n)
-		if err := os.Mkdir(tmpImageDir, os.FileMode(0755)); err != nil {
-			if os.IsExist(err) {
-				return nil
-			}
-			return err
-		}
-
-		var version = "1.0"
-		var versionBuf = []byte(version)
-
-		if err := ioutil.WriteFile(filepath.Join(tmpImageDir, "VERSION"), versionBuf, os.FileMode(0644)); err != nil {
-			return err
-		}
-
-		imageInspectRaw, err := json.Marshal(img)
-		if err != nil {
-			return err
-		}
-
-		// serialize json
-		json, err := os.Create(filepath.Join(tmpImageDir, "json"))
-		if err != nil {
-			return err
-		}
-
-		written, err := json.Write(imageInspectRaw)
-		if err != nil {
-			return err
-		}
-		if written != len(imageInspectRaw) {
-			logrus.Warnf("%d byes should have been written instead %d have been written", written, len(imageInspectRaw))
-		}
-
-		// serialize filesystem
-		fsTar, err := os.Create(filepath.Join(tmpImageDir, "layer.tar"))
-		if err != nil {
-			return err
-		}
-		if err := s.imageTarLayer(n, fsTar); err != nil {
-			return err
-		}
-
-		for _, fname := range []string{"", "VERSION", "json", "layer.tar"} {
-			if err := os.Chtimes(filepath.Join(tmpImageDir, fname), img.Created, img.Created); err != nil {
-				return err
-			}
-		}
-
-		// try again with parent
-		n = img.Parent
-	}
-	return nil
-}

graph/fixtures/validate_manifest/no_signature_manifest

@@ -1,22 +0,0 @@
-{
-   "schemaVersion": 1,
-   "name": "library/hello-world",
-   "tag": "latest",
-   "architecture": "amd64",
-   "fsLayers": [
-      {
-         "blobSum": "sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4"
-      },
-      {
-         "blobSum": "sha256:03f4658f8b782e12230c1783426bd3bacce651ce582a4ffb6fbbfa2079428ecb"
-      }
-   ],
-   "history": [
-      {
-         "v1Compatibility": "{\"id\":\"af340544ed62de0680f441c71fa1a80cb084678fed42bae393e543faea3a572c\",\"parent\":\"535020c3e8add9d6bb06e5ac15a261e73d9b213d62fb2c14d752b8e189b2b912\",\"created\":\"2015-08-06T23:53:22.608577814Z\",\"container\":\"c2b715156f640c7ac7d98472ea24335aba5432a1323a3bb722697e6d37ef794f\",\"container_config\":{\"Hostname\":\"9aeb0006ffa7\",\"Domainname\":\"\",\"User\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":null,\"Cmd\":[\"/bin/sh\",\"-c\",\"#(nop) CMD [\\\"/hello\\\"]\"],\"Image\":\"535020c3e8add9d6bb06e5ac15a261e73d9b213d62fb2c14d752b8e189b2b912\",\"Volumes\":null,\"VolumeDriver\":\"\",\"WorkingDir\":\"\",\"Entrypoint\":null,\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":null,\"Labels\":{}},\"docker_version\":\"1.7.1\",\"config\":{\"Hostname\":\"9aeb0006ffa7\",\"Domainname\":\"\",\"User\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":null,\"Cmd\":[\"/hello\"],\"Image\":\"535020c3e8add9d6bb06e5ac15a261e73d9b213d62fb2c14d752b8e189b2b912\",\"Volumes\":null,\"VolumeDriver\":\"\",\"WorkingDir\":\"\",\"Entrypoint\":null,\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":null,\"Labels\":{}},\"architecture\":\"amd64\",\"os\":\"linux\",\"Size\":0}\n"
-      },
-      {
-         "v1Compatibility": "{\"id\":\"535020c3e8add9d6bb06e5ac15a261e73d9b213d62fb2c14d752b8e189b2b912\",\"created\":\"2015-08-06T23:53:22.241352727Z\",\"container\":\"9aeb0006ffa72a8287564caaea87625896853701459261d3b569e320c0c9d5dc\",\"container_config\":{\"Hostname\":\"9aeb0006ffa7\",\"Domainname\":\"\",\"User\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":null,\"Cmd\":[\"/bin/sh\",\"-c\",\"#(nop) COPY file:4abd3bff60458ca3b079d7b131ce26b2719055a030dfa96ff827da2b7c7038a7 in /\"],\"Image\":\"\",\"Volumes\":null,\"VolumeDriver\":\"\",\"WorkingDir\":\"\",\"Entrypoint\":null,\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":null,\"Labels\":null},\"docker_version\":\"1.7.1\",\"config\":{\"Hostname\":\"9aeb0006ffa7\",\"Domainname\":\"\",\"User\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":null,\"Cmd\":null,\"Image\":\"\",\"Volumes\":null,\"VolumeDriver\":\"\",\"WorkingDir\":\"\",\"Entrypoint\":null,\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":null,\"Labels\":null},\"architecture\":\"amd64\",\"os\":\"linux\",\"Size\":960}\n"
-      }
-   ]
-}

graph/graph.go

@@ -1,813 +0,0 @@
-package graph
-
-import (
-	"compress/gzip"
-	"encoding/json"
-	"errors"
-	"fmt"
-	"io"
-	"io/ioutil"
-	"os"
-	"path/filepath"
-	"runtime"
-	"strconv"
-	"strings"
-	"sync"
-	"time"
-
-	"github.com/Sirupsen/logrus"
-	"github.com/docker/distribution/digest"
-	"github.com/docker/docker/daemon/graphdriver"
-	"github.com/docker/docker/dockerversion"
-	"github.com/docker/docker/image"
-	"github.com/docker/docker/pkg/archive"
-	"github.com/docker/docker/pkg/idtools"
-	"github.com/docker/docker/pkg/locker"
-	"github.com/docker/docker/pkg/progressreader"
-	"github.com/docker/docker/pkg/streamformatter"
-	"github.com/docker/docker/pkg/stringid"
-	"github.com/docker/docker/pkg/truncindex"
-	"github.com/docker/docker/runconfig"
-	"github.com/vbatts/tar-split/tar/asm"
-	"github.com/vbatts/tar-split/tar/storage"
-)
-
-// v1Descriptor is a non-content-addressable image descriptor
-type v1Descriptor struct {
-	img *image.Image
-}
-
-// ID returns the image ID specified in the image structure.
-func (img v1Descriptor) ID() string {
-	return img.img.ID
-}
-
-// Parent returns the parent ID specified in the image structure.
-func (img v1Descriptor) Parent() string {
-	return img.img.Parent
-}
-
-// MarshalConfig renders the image structure into JSON.
-func (img v1Descriptor) MarshalConfig() ([]byte, error) {
-	return json.Marshal(img.img)
-}
-
-// The type is used to protect pulling or building related image
-// layers from deleteing when filtered by dangling=true
-// The key of layers is the images ID which is pulling or building
-// The value of layers is a slice which hold layer IDs referenced to
-// pulling or building images
-type retainedLayers struct {
-	layerHolders map[string]map[string]struct{} // map[layerID]map[sessionID]
-	sync.Mutex
-}
-
-func (r *retainedLayers) Add(sessionID string, layerIDs []string) {
-	r.Lock()
-	defer r.Unlock()
-	for _, layerID := range layerIDs {
-		if r.layerHolders[layerID] == nil {
-			r.layerHolders[layerID] = map[string]struct{}{}
-		}
-		r.layerHolders[layerID][sessionID] = struct{}{}
-	}
-}
-
-func (r *retainedLayers) Delete(sessionID string, layerIDs []string) {
-	r.Lock()
-	defer r.Unlock()
-	for _, layerID := range layerIDs {
-		holders, ok := r.layerHolders[layerID]
-		if !ok {
-			continue
-		}
-		delete(holders, sessionID)
-		if len(holders) == 0 {
-			delete(r.layerHolders, layerID) // Delete any empty reference set.
-		}
-	}
-}
-
-func (r *retainedLayers) Exists(layerID string) bool {
-	r.Lock()
-	_, exists := r.layerHolders[layerID]
-	r.Unlock()
-	return exists
-}
-
-// A Graph is a store for versioned filesystem images and the relationship between them.
-type Graph struct {
-	root             string
-	idIndex          *truncindex.TruncIndex
-	driver           graphdriver.Driver
-	imagesMutex      sync.Mutex
-	imageMutex       locker.Locker // protect images in driver.
-	retained         *retainedLayers
-	tarSplitDisabled bool
-	uidMaps          []idtools.IDMap
-	gidMaps          []idtools.IDMap
-
-	// access to parentRefs must be protected with imageMutex locking the image id
-	// on the key of the map (e.g. imageMutex.Lock(img.ID), parentRefs[img.ID]...)
-	parentRefs map[string]int
-}
-
-// file names for ./graph/<ID>/
-const (
-	jsonFileName            = "json"
-	layersizeFileName       = "layersize"
-	digestFileName          = "checksum"
-	tarDataFileName         = "tar-data.json.gz"
-	v1CompatibilityFileName = "v1Compatibility"
-	parentFileName          = "parent"
-)
-
-var (
-	// errDigestNotSet is used when request the digest for a layer
-	// but the layer has no digest value or content to compute the
-	// the digest.
-	errDigestNotSet = errors.New("digest is not set for layer")
-)
-
-// NewGraph instantiates a new graph at the given root path in the filesystem.
-// `root` will be created if it doesn't exist.
-func NewGraph(root string, driver graphdriver.Driver, uidMaps, gidMaps []idtools.IDMap) (*Graph, error) {
-	abspath, err := filepath.Abs(root)
-	if err != nil {
-		return nil, err
-	}
-
-	rootUID, rootGID, err := idtools.GetRootUIDGID(uidMaps, gidMaps)
-	if err != nil {
-		return nil, err
-	}
-	// Create the root directory if it doesn't exists
-	if err := idtools.MkdirAllAs(root, 0700, rootUID, rootGID); err != nil && !os.IsExist(err) {
-		return nil, err
-	}
-
-	graph := &Graph{
-		root:       abspath,
-		idIndex:    truncindex.NewTruncIndex([]string{}),
-		driver:     driver,
-		retained:   &retainedLayers{layerHolders: make(map[string]map[string]struct{})},
-		uidMaps:    uidMaps,
-		gidMaps:    gidMaps,
-		parentRefs: make(map[string]int),
-	}
-
-	// Windows does not currently support tarsplit functionality.
-	if runtime.GOOS == "windows" {
-		graph.tarSplitDisabled = true
-	}
-
-	if err := graph.restore(); err != nil {
-		return nil, err
-	}
-	return graph, nil
-}
-
-// IsHeld returns whether the given layerID is being used by an ongoing pull or build.
-func (graph *Graph) IsHeld(layerID string) bool {
-	return graph.retained.Exists(layerID)
-}
-
-func (graph *Graph) restore() error {
-	dir, err := ioutil.ReadDir(graph.root)
-	if err != nil {
-		return err
-	}
-	var ids = []string{}
-	for _, v := range dir {
-		id := v.Name()
-		if graph.driver.Exists(id) {
-			img, err := graph.loadImage(id)
-			if err != nil {
-				logrus.Warnf("ignoring image %s, it could not be restored: %v", id, err)
-				continue
-			}
-			graph.imageMutex.Lock(img.Parent)
-			graph.parentRefs[img.Parent]++
-			graph.imageMutex.Unlock(img.Parent)
-			ids = append(ids, id)
-		}
-	}
-
-	graph.idIndex = truncindex.NewTruncIndex(ids)
-	logrus.Debugf("Restored %d elements", len(ids))
-	return nil
-}
-
-// IsNotExist detects whether an image exists by parsing the incoming error
-// message.
-func (graph *Graph) IsNotExist(err error, id string) bool {
-	// FIXME: Implement error subclass instead of looking at the error text
-	// Note: This is the way golang implements os.IsNotExists on Plan9
-	return err != nil && (strings.Contains(strings.ToLower(err.Error()), "does not exist") || strings.Contains(strings.ToLower(err.Error()), "no such")) && strings.Contains(err.Error(), id)
-}
-
-// Exists returns true if an image is registered at the given id.
-// If the image doesn't exist or if an error is encountered, false is returned.
-func (graph *Graph) Exists(id string) bool {
-	if _, err := graph.Get(id); err != nil {
-		return false
-	}
-	return true
-}
-
-// Get returns the image with the given id, or an error if the image doesn't exist.
-func (graph *Graph) Get(name string) (*image.Image, error) {
-	id, err := graph.idIndex.Get(name)
-	if err != nil {
-		if err == truncindex.ErrNotExist {
-			return nil, fmt.Errorf("image %s does not exist", name)
-		}
-		return nil, err
-	}
-	img, err := graph.loadImage(id)
-	if err != nil {
-		return nil, err
-	}
-	if img.ID != id {
-		return nil, fmt.Errorf("Image stored at '%s' has wrong id '%s'", id, img.ID)
-	}
-
-	if img.Size < 0 {
-		size, err := graph.driver.DiffSize(img.ID, img.Parent)
-		if err != nil {
-			return nil, fmt.Errorf("unable to calculate size of image id %q: %s", img.ID, err)
-		}
-
-		img.Size = size
-		if err := graph.saveSize(graph.imageRoot(id), img.Size); err != nil {
-			return nil, err
-		}
-	}
-	return img, nil
-}
-
-// Create creates a new image and registers it in the graph.
-func (graph *Graph) Create(layerData io.Reader, containerID, containerImage, comment, author string, containerConfig, config *runconfig.Config) (*image.Image, error) {
-	img := &image.Image{
-		ID:            stringid.GenerateRandomID(),
-		Comment:       comment,
-		Created:       time.Now().UTC(),
-		DockerVersion: dockerversion.Version,
-		Author:        author,
-		Config:        config,
-		Architecture:  runtime.GOARCH,
-		OS:            runtime.GOOS,
-	}
-
-	if containerID != "" {
-		img.Parent = containerImage
-		img.Container = containerID
-		img.ContainerConfig = *containerConfig
-	}
-
-	if err := graph.Register(v1Descriptor{img}, layerData); err != nil {
-		return nil, err
-	}
-	return img, nil
-}
-
-// Register imports a pre-existing image into the graph.
-// Returns nil if the image is already registered.
-func (graph *Graph) Register(im image.Descriptor, layerData io.Reader) (err error) {
-	imgID := im.ID()
-
-	if err := image.ValidateID(imgID); err != nil {
-		return err
-	}
-
-	// this is needed cause pull_v2 attemptIDReuse could deadlock
-	graph.imagesMutex.Lock()
-	defer graph.imagesMutex.Unlock()
-
-	// We need this entire operation to be atomic within the engine. Note that
-	// this doesn't mean Register is fully safe yet.
-	graph.imageMutex.Lock(imgID)
-	defer graph.imageMutex.Unlock(imgID)
-
-	return graph.register(im, layerData)
-}
-
-func (graph *Graph) register(im image.Descriptor, layerData io.Reader) (err error) {
-	imgID := im.ID()
-
-	// Skip register if image is already registered
-	if graph.Exists(imgID) {
-		return nil
-	}
-
-	// The returned `error` must be named in this function's signature so that
-	// `err` is not shadowed in this deferred cleanup.
-	defer func() {
-		// If any error occurs, remove the new dir from the driver.
-		// Don't check for errors since the dir might not have been created.
-		if err != nil {
-			graph.driver.Remove(imgID)
-		}
-	}()
-
-	// Ensure that the image root does not exist on the filesystem
-	// when it is not registered in the graph.
-	// This is common when you switch from one graph driver to another
-	if err := os.RemoveAll(graph.imageRoot(imgID)); err != nil && !os.IsNotExist(err) {
-		return err
-	}
-
-	// If the driver has this ID but the graph doesn't, remove it from the driver to start fresh.
-	// (the graph is the source of truth).
-	// Ignore errors, since we don't know if the driver correctly returns ErrNotExist.
-	// (FIXME: make that mandatory for drivers).
-	graph.driver.Remove(imgID)
-
-	tmp, err := graph.mktemp()
-	if err != nil {
-		return err
-	}
-	defer os.RemoveAll(tmp)
-
-	parent := im.Parent()
-
-	// Create root filesystem in the driver
-	if err := createRootFilesystemInDriver(graph, imgID, parent); err != nil {
-		return err
-	}
-
-	// Apply the diff/layer
-	config, err := im.MarshalConfig()
-	if err != nil {
-		return err
-	}
-	if err := graph.storeImage(imgID, parent, config, layerData, tmp); err != nil {
-		return err
-	}
-	// Commit
-	if err := os.Rename(tmp, graph.imageRoot(imgID)); err != nil {
-		return err
-	}
-
-	graph.idIndex.Add(imgID)
-
-	graph.imageMutex.Lock(parent)
-	graph.parentRefs[parent]++
-	graph.imageMutex.Unlock(parent)
-
-	return nil
-}
-
-func createRootFilesystemInDriver(graph *Graph, id, parent string) error {
-	if err := graph.driver.Create(id, parent, ""); err != nil {
-		return fmt.Errorf("Driver %s failed to create image rootfs %s: %s", graph.driver, id, err)
-	}
-	return nil
-}
-
-// TempLayerArchive creates a temporary archive of the given image's filesystem layer.
-//   The archive is stored on disk and will be automatically deleted as soon as has been read.
-//   If output is not nil, a human-readable progress bar will be written to it.
-func (graph *Graph) tempLayerArchive(id string, sf *streamformatter.StreamFormatter, output io.Writer) (*archive.TempArchive, error) {
-	image, err := graph.Get(id)
-	if err != nil {
-		return nil, err
-	}
-	tmp, err := graph.mktemp()
-	if err != nil {
-		return nil, err
-	}
-	defer os.RemoveAll(tmp)
-	a, err := graph.tarLayer(image)
-	if err != nil {
-		return nil, err
-	}
-	progressReader := progressreader.New(progressreader.Config{
-		In:        a,
-		Out:       output,
-		Formatter: sf,
-		Size:      0,
-		NewLines:  false,
-		ID:        stringid.TruncateID(id),
-		Action:    "Buffering to disk",
-	})
-	defer progressReader.Close()
-	return archive.NewTempArchive(progressReader, tmp)
-}
-
-// mktemp creates a temporary sub-directory inside the graph's filesystem.
-func (graph *Graph) mktemp() (string, error) {
-	dir := filepath.Join(graph.root, "_tmp", stringid.GenerateNonCryptoID())
-	rootUID, rootGID, err := idtools.GetRootUIDGID(graph.uidMaps, graph.gidMaps)
-	if err != nil {
-		return "", err
-	}
-	if err := idtools.MkdirAllAs(dir, 0700, rootUID, rootGID); err != nil {
-		return "", err
-	}
-	return dir, nil
-}
-
-// Delete atomically removes an image from the graph.
-func (graph *Graph) Delete(name string) error {
-	id, err := graph.idIndex.Get(name)
-	if err != nil {
-		return err
-	}
-	img, err := graph.Get(id)
-	if err != nil {
-		return err
-	}
-	graph.idIndex.Delete(id)
-	tmp, err := graph.mktemp()
-	if err != nil {
-		tmp = graph.imageRoot(id)
-	} else {
-		if err := os.Rename(graph.imageRoot(id), tmp); err != nil {
-			// On err make tmp point to old dir and cleanup unused tmp dir
-			os.RemoveAll(tmp)
-			tmp = graph.imageRoot(id)
-		}
-	}
-	// Remove rootfs data from the driver
-	graph.driver.Remove(id)
-
-	graph.imageMutex.Lock(img.Parent)
-	graph.parentRefs[img.Parent]--
-	if graph.parentRefs[img.Parent] == 0 {
-		delete(graph.parentRefs, img.Parent)
-	}
-	graph.imageMutex.Unlock(img.Parent)
-
-	// Remove the trashed image directory
-	return os.RemoveAll(tmp)
-}
-
-// Map returns a list of all images in the graph, addressable by ID.
-func (graph *Graph) Map() map[string]*image.Image {
-	images := make(map[string]*image.Image)
-	graph.walkAll(func(image *image.Image) {
-		images[image.ID] = image
-	})
-	return images
-}
-
-// walkAll iterates over each image in the graph, and passes it to a handler.
-// The walking order is undetermined.
-func (graph *Graph) walkAll(handler func(*image.Image)) {
-	graph.idIndex.Iterate(func(id string) {
-		img, err := graph.Get(id)
-		if err != nil {
-			return
-		}
-		if handler != nil {
-			handler(img)
-		}
-	})
-}
-
-// ByParent returns a lookup table of images by their parent.
-// If an image of key ID has 3 children images, then the value for key ID
-// will be a list of 3 images.
-// If an image has no children, it will not have an entry in the table.
-func (graph *Graph) ByParent() map[string][]*image.Image {
-	byParent := make(map[string][]*image.Image)
-	graph.walkAll(func(img *image.Image) {
-		parent, err := graph.Get(img.Parent)
-		if err != nil {
-			return
-		}
-		byParent[parent.ID] = append(byParent[parent.ID], img)
-	})
-	return byParent
-}
-
-// HasChildren returns whether the given image has any child images.
-func (graph *Graph) HasChildren(imgID string) bool {
-	graph.imageMutex.Lock(imgID)
-	count := graph.parentRefs[imgID]
-	graph.imageMutex.Unlock(imgID)
-	return count > 0
-}
-
-// Retain keeps the images and layers that are in the pulling chain so that
-// they are not deleted. If not retained, they may be deleted by rmi.
-func (graph *Graph) Retain(sessionID string, layerIDs ...string) {
-	graph.retained.Add(sessionID, layerIDs)
-}
-
-// Release removes the referenced image ID from the provided set of layers.
-func (graph *Graph) Release(sessionID string, layerIDs ...string) {
-	graph.retained.Delete(sessionID, layerIDs)
-}
-
-// heads returns all heads in the graph, keyed by id.
-// A head is an image which is not the parent of another image in the graph.
-func (graph *Graph) heads() map[string]*image.Image {
-	heads := make(map[string]*image.Image)
-	graph.walkAll(func(image *image.Image) {
-		// if it has no children, then it's not a parent, so it's an head
-		if !graph.HasChildren(image.ID) {
-			heads[image.ID] = image
-		}
-	})
-	return heads
-}
-
-// tarLayer returns a tar archive of the image's filesystem layer.
-func (graph *Graph) tarLayer(img *image.Image) (arch io.ReadCloser, err error) {
-	rdr, err := graph.assembleTarLayer(img)
-	if err != nil {
-		logrus.Debugf("[graph] tarLayer with traditional differ: %s", img.ID)
-		return graph.driver.Diff(img.ID, img.Parent)
-	}
-	return rdr, nil
-}
-
-func (graph *Graph) imageRoot(id string) string {
-	return filepath.Join(graph.root, id)
-}
-
-// loadImage fetches the image with the given id from the graph.
-func (graph *Graph) loadImage(id string) (*image.Image, error) {
-	root := graph.imageRoot(id)
-
-	// Open the JSON file to decode by streaming
-	jsonSource, err := os.Open(jsonPath(root))
-	if err != nil {
-		return nil, err
-	}
-	defer jsonSource.Close()
-
-	img := &image.Image{}
-	dec := json.NewDecoder(jsonSource)
-
-	// Decode the JSON data
-	if err := dec.Decode(img); err != nil {
-		return nil, err
-	}
-
-	if img.ID == "" {
-		img.ID = id
-	}
-
-	if img.Parent == "" && img.ParentID != "" && img.ParentID.Validate() == nil {
-		img.Parent = img.ParentID.Hex()
-	}
-
-	// compatibilityID for parent
-	parent, err := ioutil.ReadFile(filepath.Join(root, parentFileName))
-	if err == nil && len(parent) > 0 {
-		img.Parent = string(parent)
-	}
-
-	if err := image.ValidateID(img.ID); err != nil {
-		return nil, err
-	}
-
-	if buf, err := ioutil.ReadFile(filepath.Join(root, layersizeFileName)); err != nil {
-		if !os.IsNotExist(err) {
-			return nil, err
-		}
-		// If the layersize file does not exist then set the size to a negative number
-		// because a layer size of 0 (zero) is valid
-		img.Size = -1
-	} else {
-		// Using Atoi here instead would temporarily convert the size to a machine
-		// dependent integer type, which causes images larger than 2^31 bytes to
-		// display negative sizes on 32-bit machines:
-		size, err := strconv.ParseInt(string(buf), 10, 64)
-		if err != nil {
-			return nil, err
-		}
-		img.Size = int64(size)
-	}
-
-	return img, nil
-}
-
-// saveSize stores the `size` in the provided graph `img` directory `root`.
-func (graph *Graph) saveSize(root string, size int64) error {
-	if err := ioutil.WriteFile(filepath.Join(root, layersizeFileName), []byte(strconv.FormatInt(size, 10)), 0600); err != nil {
-		return fmt.Errorf("Error storing image size in %s/%s: %s", root, layersizeFileName, err)
-	}
-	return nil
-}
-
-// setLayerDigestWithLock sets the digest for the image layer to the provided value.
-func (graph *Graph) setLayerDigestWithLock(id string, dgst digest.Digest) error {
-	graph.imageMutex.Lock(id)
-	defer graph.imageMutex.Unlock(id)
-
-	return graph.setLayerDigest(id, dgst)
-}
-func (graph *Graph) setLayerDigest(id string, dgst digest.Digest) error {
-	root := graph.imageRoot(id)
-	if err := ioutil.WriteFile(filepath.Join(root, digestFileName), []byte(dgst.String()), 0600); err != nil {
-		return fmt.Errorf("Error storing digest in %s/%s: %s", root, digestFileName, err)
-	}
-	return nil
-}
-
-// getLayerDigestWithLock gets the digest for the provide image layer id.
-func (graph *Graph) getLayerDigestWithLock(id string) (digest.Digest, error) {
-	graph.imageMutex.Lock(id)
-	defer graph.imageMutex.Unlock(id)
-
-	return graph.getLayerDigest(id)
-}
-
-func (graph *Graph) getLayerDigest(id string) (digest.Digest, error) {
-	root := graph.imageRoot(id)
-	cs, err := ioutil.ReadFile(filepath.Join(root, digestFileName))
-	if err != nil {
-		if os.IsNotExist(err) {
-			return "", errDigestNotSet
-		}
-		return "", err
-	}
-	return digest.ParseDigest(string(cs))
-}
-
-// setV1CompatibilityConfig stores the v1Compatibility JSON data associated
-// with the image in the manifest to the disk
-func (graph *Graph) setV1CompatibilityConfig(id string, data []byte) error {
-	root := graph.imageRoot(id)
-	return ioutil.WriteFile(filepath.Join(root, v1CompatibilityFileName), data, 0600)
-}
-
-// getV1CompatibilityConfig reads the v1Compatibility JSON data for the image
-// from the disk
-func (graph *Graph) getV1CompatibilityConfig(id string) ([]byte, error) {
-	root := graph.imageRoot(id)
-	return ioutil.ReadFile(filepath.Join(root, v1CompatibilityFileName))
-}
-
-// generateV1CompatibilityChain makes sure v1Compatibility JSON data exists
-// for the image. If it doesn't it generates and stores it for the image and
-// all of it's parents based on the image config JSON.
-func (graph *Graph) generateV1CompatibilityChain(id string) ([]byte, error) {
-	graph.imageMutex.Lock(id)
-	defer graph.imageMutex.Unlock(id)
-
-	if v1config, err := graph.getV1CompatibilityConfig(id); err == nil {
-		return v1config, nil
-	}
-
-	// generate new, store it to disk
-	img, err := graph.Get(id)
-	if err != nil {
-		return nil, err
-	}
-
-	digestPrefix := string(digest.Canonical) + ":"
-	img.ID = strings.TrimPrefix(img.ID, digestPrefix)
-
-	if img.Parent != "" {
-		parentConfig, err := graph.generateV1CompatibilityChain(img.Parent)
-		if err != nil {
-			return nil, err
-		}
-		var parent struct{ ID string }
-		err = json.Unmarshal(parentConfig, &parent)
-		if err != nil {
-			return nil, err
-		}
-		img.Parent = parent.ID
-	}
-
-	json, err := json.Marshal(img)
-	if err != nil {
-		return nil, err
-	}
-	if err := graph.setV1CompatibilityConfig(id, json); err != nil {
-		return nil, err
-	}
-	return json, nil
-}
-
-func jsonPath(root string) string {
-	return filepath.Join(root, jsonFileName)
-}
-
-// storeImage stores file system layer data for the given image to the
-// graph's storage driver. Image metadata is stored in a file
-// at the specified root directory.
-func (graph *Graph) storeImage(id, parent string, config []byte, layerData io.Reader, root string) (err error) {
-	var size int64
-	// Store the layer. If layerData is not nil, unpack it into the new layer
-	if layerData != nil {
-		if size, err = graph.disassembleAndApplyTarLayer(id, parent, layerData, root); err != nil {
-			return err
-		}
-	}
-
-	if err := graph.saveSize(root, size); err != nil {
-		return err
-	}
-
-	if err := ioutil.WriteFile(jsonPath(root), config, 0600); err != nil {
-		return err
-	}
-
-	// If image is pointing to a parent via CompatibilityID write the reference to disk
-	img, err := image.NewImgJSON(config)
-	if err != nil {
-		return err
-	}
-
-	if (img.ParentID.Validate() == nil && parent != img.ParentID.Hex()) || (allowBaseParentImage && img.ParentID == "" && parent != "") {
-		// save compatibilityID parent if it doesn't match parentID
-		// on windows always save a parent file pointing to the base layer
-		if err := ioutil.WriteFile(filepath.Join(root, parentFileName), []byte(parent), 0600); err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
-func (graph *Graph) disassembleAndApplyTarLayer(id, parent string, layerData io.Reader, root string) (size int64, err error) {
-	var ar io.Reader
-
-	if graph.tarSplitDisabled {
-		ar = layerData
-	} else {
-		// this is saving the tar-split metadata
-		mf, err := os.OpenFile(filepath.Join(root, tarDataFileName), os.O_CREATE|os.O_WRONLY|os.O_TRUNC, os.FileMode(0600))
-		if err != nil {
-			return 0, err
-		}
-
-		mfz := gzip.NewWriter(mf)
-		metaPacker := storage.NewJSONPacker(mfz)
-		defer mf.Close()
-		defer mfz.Close()
-
-		inflatedLayerData, err := archive.DecompressStream(layerData)
-		if err != nil {
-			return 0, err
-		}
-
-		// we're passing nil here for the file putter, because the ApplyDiff will
-		// handle the extraction of the archive
-		rdr, err := asm.NewInputTarStream(inflatedLayerData, metaPacker, nil)
-		if err != nil {
-			return 0, err
-		}
-
-		ar = archive.Reader(rdr)
-	}
-
-	if size, err = graph.driver.ApplyDiff(id, parent, ar); err != nil {
-		return 0, err
-	}
-
-	return
-}
-
-func (graph *Graph) assembleTarLayer(img *image.Image) (io.ReadCloser, error) {
-	root := graph.imageRoot(img.ID)
-	mFileName := filepath.Join(root, tarDataFileName)
-	mf, err := os.Open(mFileName)
-	if err != nil {
-		if !os.IsNotExist(err) {
-			logrus.Errorf("failed to open %q: %s", mFileName, err)
-		}
-		return nil, err
-	}
-	pR, pW := io.Pipe()
-	// this will need to be in a goroutine, as we are returning the stream of a
-	// tar archive, but can not close the metadata reader early (when this
-	// function returns)...
-	go func() {
-		defer mf.Close()
-		// let's reassemble!
-		logrus.Debugf("[graph] TarLayer with reassembly: %s", img.ID)
-		mfz, err := gzip.NewReader(mf)
-		if err != nil {
-			pW.CloseWithError(fmt.Errorf("[graph] error with %s:  %s", mFileName, err))
-			return
-		}
-		defer mfz.Close()
-
-		// get our relative path to the container
-		fsLayer, err := graph.driver.Get(img.ID, "")
-		if err != nil {
-			pW.CloseWithError(err)
-			return
-		}
-		defer graph.driver.Put(img.ID)
-
-		metaUnpacker := storage.NewJSONUnpacker(mfz)
-		fileGetter := storage.NewPathFileGetter(fsLayer)
-		logrus.Debugf("[graph] %s is at %q", img.ID, fsLayer)
-		ots := asm.NewOutputTarStream(fileGetter, metaUnpacker)
-		defer ots.Close()
-		if _, err := io.Copy(pW, ots); err != nil {
-			pW.CloseWithError(err)
-			return
-		}
-		pW.Close()
-	}()
-	return pR, nil
-}

graph/graph_test.go

@@ -1,308 +0,0 @@
-package graph
-
-import (
-	"errors"
-	"io"
-	"io/ioutil"
-	"os"
-	"path"
-	"testing"
-	"time"
-
-	"github.com/docker/docker/daemon/graphdriver"
-	"github.com/docker/docker/dockerversion"
-	"github.com/docker/docker/image"
-	"github.com/docker/docker/pkg/stringid"
-)
-
-func TestMount(t *testing.T) {
-	graph, driver := tempGraph(t)
-	defer os.RemoveAll(graph.root)
-	defer driver.Cleanup()
-
-	archive, err := fakeTar()
-	if err != nil {
-		t.Fatal(err)
-	}
-	image, err := graph.Create(archive, "", "", "Testing", "", nil, nil)
-	if err != nil {
-		t.Fatal(err)
-	}
-	tmp, err := ioutil.TempDir("", "docker-test-graph-mount-")
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer os.RemoveAll(tmp)
-	rootfs := path.Join(tmp, "rootfs")
-	if err := os.MkdirAll(rootfs, 0700); err != nil {
-		t.Fatal(err)
-	}
-	rw := path.Join(tmp, "rw")
-	if err := os.MkdirAll(rw, 0700); err != nil {
-		t.Fatal(err)
-	}
-
-	if _, err := driver.Get(image.ID, ""); err != nil {
-		t.Fatal(err)
-	}
-
-}
-
-func TestInit(t *testing.T) {
-	graph, _ := tempGraph(t)
-	defer nukeGraph(graph)
-	// Root should exist
-	if _, err := os.Stat(graph.root); err != nil {
-		t.Fatal(err)
-	}
-	// Map() should be empty
-	l := graph.Map()
-	if len(l) != 0 {
-		t.Fatalf("len(Map()) should return %d, not %d", 0, len(l))
-	}
-}
-
-// Test that Register can be interrupted cleanly without side effects
-func TestInterruptedRegister(t *testing.T) {
-	graph, _ := tempGraph(t)
-	defer nukeGraph(graph)
-	badArchive, w := io.Pipe() // Use a pipe reader as a fake archive which never yields data
-	image := &image.Image{
-		ID:      stringid.GenerateNonCryptoID(),
-		Comment: "testing",
-		Created: time.Now(),
-	}
-	w.CloseWithError(errors.New("But I'm not a tarball!")) // (Nobody's perfect, darling)
-	graph.Register(v1Descriptor{image}, badArchive)
-	if _, err := graph.Get(image.ID); err == nil {
-		t.Fatal("Image should not exist after Register is interrupted")
-	}
-	// Registering the same image again should succeed if the first register was interrupted
-	goodArchive, err := fakeTar()
-	if err != nil {
-		t.Fatal(err)
-	}
-	if err := graph.Register(v1Descriptor{image}, goodArchive); err != nil {
-		t.Fatal(err)
-	}
-}
-
-// FIXME: Do more extensive tests (ex: create multiple, delete, recreate;
-//       create multiple, check the amount of images and paths, etc..)
-func TestGraphCreate(t *testing.T) {
-	graph, _ := tempGraph(t)
-	defer nukeGraph(graph)
-	archive, err := fakeTar()
-	if err != nil {
-		t.Fatal(err)
-	}
-	img, err := graph.Create(archive, "", "", "Testing", "", nil, nil)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if err := image.ValidateID(img.ID); err != nil {
-		t.Fatal(err)
-	}
-	if img.Comment != "Testing" {
-		t.Fatalf("Wrong comment: should be '%s', not '%s'", "Testing", img.Comment)
-	}
-	if img.DockerVersion != dockerversion.Version {
-		t.Fatalf("Wrong docker_version: should be '%s', not '%s'", dockerversion.Version, img.DockerVersion)
-	}
-	images := graph.Map()
-	if l := len(images); l != 1 {
-		t.Fatalf("Wrong number of images. Should be %d, not %d", 1, l)
-	}
-	if images[img.ID] == nil {
-		t.Fatalf("Could not find image with id %s", img.ID)
-	}
-}
-
-func TestRegister(t *testing.T) {
-	graph, _ := tempGraph(t)
-	defer nukeGraph(graph)
-	archive, err := fakeTar()
-	if err != nil {
-		t.Fatal(err)
-	}
-	image := &image.Image{
-		ID:      stringid.GenerateNonCryptoID(),
-		Comment: "testing",
-		Created: time.Now(),
-	}
-	err = graph.Register(v1Descriptor{image}, archive)
-	if err != nil {
-		t.Fatal(err)
-	}
-	images := graph.Map()
-	if l := len(images); l != 1 {
-		t.Fatalf("Wrong number of images. Should be %d, not %d", 1, l)
-	}
-	if resultImg, err := graph.Get(image.ID); err != nil {
-		t.Fatal(err)
-	} else {
-		if resultImg.ID != image.ID {
-			t.Fatalf("Wrong image ID. Should be '%s', not '%s'", image.ID, resultImg.ID)
-		}
-		if resultImg.Comment != image.Comment {
-			t.Fatalf("Wrong image comment. Should be '%s', not '%s'", image.Comment, resultImg.Comment)
-		}
-	}
-}
-
-// Test that an image can be deleted by its shorthand prefix
-func TestDeletePrefix(t *testing.T) {
-	graph, _ := tempGraph(t)
-	defer nukeGraph(graph)
-	img := createTestImage(graph, t)
-	if err := graph.Delete(stringid.TruncateID(img.ID)); err != nil {
-		t.Fatal(err)
-	}
-	assertNImages(graph, t, 0)
-}
-
-func TestDelete(t *testing.T) {
-	graph, _ := tempGraph(t)
-	defer nukeGraph(graph)
-	archive, err := fakeTar()
-	if err != nil {
-		t.Fatal(err)
-	}
-	assertNImages(graph, t, 0)
-	img, err := graph.Create(archive, "", "", "Bla bla", "", nil, nil)
-	if err != nil {
-		t.Fatal(err)
-	}
-	assertNImages(graph, t, 1)
-	if err := graph.Delete(img.ID); err != nil {
-		t.Fatal(err)
-	}
-	assertNImages(graph, t, 0)
-
-	archive, err = fakeTar()
-	if err != nil {
-		t.Fatal(err)
-	}
-	// Test 2 create (same name) / 1 delete
-	img1, err := graph.Create(archive, "", "", "Testing", "", nil, nil)
-	if err != nil {
-		t.Fatal(err)
-	}
-	archive, err = fakeTar()
-	if err != nil {
-		t.Fatal(err)
-	}
-	if _, err = graph.Create(archive, "", "", "Testing", "", nil, nil); err != nil {
-		t.Fatal(err)
-	}
-	assertNImages(graph, t, 2)
-	if err := graph.Delete(img1.ID); err != nil {
-		t.Fatal(err)
-	}
-	assertNImages(graph, t, 1)
-
-	// Test delete wrong name
-	if err := graph.Delete("Not_foo"); err == nil {
-		t.Fatalf("Deleting wrong ID should return an error")
-	}
-	assertNImages(graph, t, 1)
-
-	archive, err = fakeTar()
-	if err != nil {
-		t.Fatal(err)
-	}
-	// Test delete twice (pull -> rm -> pull -> rm)
-	if err := graph.Register(v1Descriptor{img1}, archive); err != nil {
-		t.Fatal(err)
-	}
-	if err := graph.Delete(img1.ID); err != nil {
-		t.Fatal(err)
-	}
-	assertNImages(graph, t, 1)
-}
-
-func TestByParent(t *testing.T) {
-	archive1, _ := fakeTar()
-	archive2, _ := fakeTar()
-	archive3, _ := fakeTar()
-
-	graph, _ := tempGraph(t)
-	defer nukeGraph(graph)
-	parentImage := &image.Image{
-		ID:      stringid.GenerateNonCryptoID(),
-		Comment: "parent",
-		Created: time.Now(),
-		Parent:  "",
-	}
-	childImage1 := &image.Image{
-		ID:      stringid.GenerateNonCryptoID(),
-		Comment: "child1",
-		Created: time.Now(),
-		Parent:  parentImage.ID,
-	}
-	childImage2 := &image.Image{
-		ID:      stringid.GenerateNonCryptoID(),
-		Comment: "child2",
-		Created: time.Now(),
-		Parent:  parentImage.ID,
-	}
-
-	err := graph.Register(v1Descriptor{parentImage}, archive1)
-	if err != nil {
-		t.Fatal(err)
-	}
-	err = graph.Register(v1Descriptor{childImage1}, archive2)
-	if err != nil {
-		t.Fatal(err)
-	}
-	err = graph.Register(v1Descriptor{childImage2}, archive3)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	byParent := graph.ByParent()
-	numChildren := len(byParent[parentImage.ID])
-	if numChildren != 2 {
-		t.Fatalf("Expected 2 children, found %d", numChildren)
-	}
-}
-
-func createTestImage(graph *Graph, t *testing.T) *image.Image {
-	archive, err := fakeTar()
-	if err != nil {
-		t.Fatal(err)
-	}
-	img, err := graph.Create(archive, "", "", "Test image", "", nil, nil)
-	if err != nil {
-		t.Fatal(err)
-	}
-	return img
-}
-
-func assertNImages(graph *Graph, t *testing.T, n int) {
-	images := graph.Map()
-	if actualN := len(images); actualN != n {
-		t.Fatalf("Expected %d images, found %d", n, actualN)
-	}
-}
-
-func tempGraph(t *testing.T) (*Graph, graphdriver.Driver) {
-	tmp, err := ioutil.TempDir("", "docker-graph-")
-	if err != nil {
-		t.Fatal(err)
-	}
-	driver, err := graphdriver.New(tmp, nil, nil, nil)
-	if err != nil {
-		t.Fatal(err)
-	}
-	graph, err := NewGraph(tmp, driver, nil, nil)
-	if err != nil {
-		t.Fatal(err)
-	}
-	return graph, driver
-}
-
-func nukeGraph(graph *Graph) {
-	graph.driver.Cleanup()
-	os.RemoveAll(graph.root)
-}

graph/graph_unix.go

@@ -1,8 +0,0 @@
-// +build !windows
-
-package graph
-
-// allowBaseParentImage allows images to define a custom parent that is not
-// transported with push/pull but already included with the installation.
-// Only used in Windows.
-const allowBaseParentImage = false

graph/graph_windows.go

@@ -1,8 +0,0 @@
-// +build windows
-
-package graph
-
-// allowBaseParentImage allows images to define a custom parent that is not
-// transported with push/pull but already included with the installation.
-// Only used in Windows.
-const allowBaseParentImage = true

graph/history.go

@@ -1,119 +0,0 @@
-package graph
-
-import (
-	"fmt"
-	"strings"
-
-	"github.com/docker/docker/api/types"
-	"github.com/docker/docker/image"
-	"github.com/docker/docker/utils"
-)
-
-// walkHistory calls the handler function for each image in the
-// provided images lineage starting from immediate parent.
-func (graph *Graph) walkHistory(img *image.Image, handler func(image.Image) error) (err error) {
-	currentImg := img
-	for currentImg != nil {
-		if handler != nil {
-			if err := handler(*currentImg); err != nil {
-				return err
-			}
-		}
-		currentImg, err = graph.GetParent(currentImg)
-		if err != nil {
-			return fmt.Errorf("Error while getting parent image: %v", err)
-		}
-	}
-	return nil
-}
-
-// depth returns the number of parents for the current image
-func (graph *Graph) depth(img *image.Image) (int, error) {
-	var (
-		count  = 0
-		parent = img
-		err    error
-	)
-
-	for parent != nil {
-		count++
-		if parent, err = graph.GetParent(parent); err != nil {
-			return -1, err
-		}
-	}
-	return count, nil
-}
-
-// Set the max depth to the aufs default that most kernels are compiled with.
-// For more information see: http://sourceforge.net/p/aufs/aufs3-standalone/ci/aufs3.12/tree/config.mk
-const maxImageDepth = 127
-
-// CheckDepth returns an error if the depth of an image, as returned
-// by ImageDepth, is too large to support creating a container from it
-// on this daemon.
-func (graph *Graph) CheckDepth(img *image.Image) error {
-	// We add 2 layers to the depth because the container's rw and
-	// init layer add to the restriction
-	depth, err := graph.depth(img)
-	if err != nil {
-		return err
-	}
-	if depth+2 >= maxImageDepth {
-		return fmt.Errorf("Cannot create container with more than %d parents", maxImageDepth)
-	}
-	return nil
-}
-
-// History returns a slice of ImageHistory structures for the specified image
-// name by walking the image lineage.
-func (s *TagStore) History(name string) ([]*types.ImageHistory, error) {
-	foundImage, err := s.LookupImage(name)
-	if err != nil {
-		return nil, err
-	}
-
-	lookupMap := make(map[string][]string)
-	for name, repository := range s.Repositories {
-		for tag, id := range repository {
-			// If the ID already has a reverse lookup, do not update it unless for "latest"
-			if _, exists := lookupMap[id]; !exists {
-				lookupMap[id] = []string{}
-			}
-			lookupMap[id] = append(lookupMap[id], utils.ImageReference(name, tag))
-		}
-	}
-
-	history := []*types.ImageHistory{}
-
-	err = s.graph.walkHistory(foundImage, func(img image.Image) error {
-		history = append(history, &types.ImageHistory{
-			ID:        img.ID,
-			Created:   img.Created.Unix(),
-			CreatedBy: strings.Join(img.ContainerConfig.Cmd.Slice(), " "),
-			Tags:      lookupMap[img.ID],
-			Size:      img.Size,
-			Comment:   img.Comment,
-		})
-		return nil
-	})
-
-	return history, err
-}
-
-// GetParent returns the parent image for the specified image.
-func (graph *Graph) GetParent(img *image.Image) (*image.Image, error) {
-	if img.Parent == "" {
-		return nil, nil
-	}
-	return graph.Get(img.Parent)
-}
-
-// getParentsSize returns the combined size of all parent images. If there is
-// no parent image or it's unavailable, it returns 0.
-func (graph *Graph) getParentsSize(img *image.Image) int64 {
-	parentImage, err := graph.GetParent(img)
-	if err != nil || parentImage == nil {
-		return 0
-	}
-	return parentImage.Size + graph.getParentsSize(parentImage)
-}

graph/import.go

@@ -1,73 +0,0 @@
-package graph
-
-import (
-	"io"
-	"net/http"
-	"net/url"
-
-	"github.com/docker/docker/pkg/httputils"
-	"github.com/docker/docker/pkg/progressreader"
-	"github.com/docker/docker/pkg/streamformatter"
-	"github.com/docker/docker/runconfig"
-)
-
-// Import imports an image, getting the archived layer data either from
-// inConfig (if src is "-"), or from a URI specified in src. Progress output is
-// written to outStream. Repository and tag names can optionally be given in
-// the repo and tag arguments, respectively.
-func (s *TagStore) Import(src string, repo string, tag string, msg string, inConfig io.ReadCloser, outStream io.Writer, containerConfig *runconfig.Config) error {
-	var (
-		sf      = streamformatter.NewJSONStreamFormatter()
-		archive io.ReadCloser
-		resp    *http.Response
-	)
-
-	if src == "-" {
-		archive = inConfig
-	} else {
-		inConfig.Close()
-		u, err := url.Parse(src)
-		if err != nil {
-			return err
-		}
-		if u.Scheme == "" {
-			u.Scheme = "http"
-			u.Host = src
-			u.Path = ""
-		}
-		outStream.Write(sf.FormatStatus("", "Downloading from %s", u))
-		resp, err = httputils.Download(u.String())
-		if err != nil {
-			return err
-		}
-		progressReader := progressreader.New(progressreader.Config{
-			In:        resp.Body,
-			Out:       outStream,
-			Formatter: sf,
-			Size:      resp.ContentLength,
-			NewLines:  true,
-			ID:        "",
-			Action:    "Importing",
-		})
-		archive = progressReader
-	}
-
-	defer archive.Close()
-	if len(msg) == 0 {
-		msg = "Imported from " + src
-	}
-
-	img, err := s.graph.Create(archive, "", "", msg, "", nil, containerConfig)
-	if err != nil {
-		return err
-	}
-	// Optionally register the image at REPO/TAG
-	if repo != "" {
-		if err := s.Tag(repo, tag, img.ID, true); err != nil {
-			return err
-		}
-	}
-	outStream.Write(sf.FormatStatus("", img.ID))
-	s.eventsService.Log("import", img.ID, "")
-	return nil
-}

graph/list.go

@@ -1,185 +0,0 @@
-package graph
-
-import (
-	"fmt"
-	"path"
-	"sort"
-	"strings"
-
-	"github.com/Sirupsen/logrus"
-	"github.com/docker/docker/api/types"
-	"github.com/docker/docker/image"
-	"github.com/docker/docker/pkg/parsers/filters"
-	"github.com/docker/docker/utils"
-)
-
-var acceptedImageFilterTags = map[string]struct{}{
-	"dangling": {},
-	"label":    {},
-}
-
-// byCreated is a temporary type used to sort a list of images by creation
-// time.
-type byCreated []*types.Image
-
-func (r byCreated) Len() int           { return len(r) }
-func (r byCreated) Swap(i, j int)      { r[i], r[j] = r[j], r[i] }
-func (r byCreated) Less(i, j int) bool { return r[i].Created < r[j].Created }
-
-// Images returns a filtered list of images. filterArgs is a JSON-encoded set
-// of filter arguments which will be interpreted by pkg/parsers/filters.
-// filter is a shell glob string applied to repository names. The argument
-// named all controls whether all images in the graph are filtered, or just
-// the heads.
-func (s *TagStore) Images(filterArgs, filter string, all bool) ([]*types.Image, error) {
-	var (
-		allImages  map[string]*image.Image
-		err        error
-		filtTagged = true
-		filtLabel  = false
-	)
-
-	imageFilters, err := filters.FromParam(filterArgs)
-	if err != nil {
-		return nil, err
-	}
-	for name := range imageFilters {
-		if _, ok := acceptedImageFilterTags[name]; !ok {
-			return nil, fmt.Errorf("Invalid filter '%s'", name)
-		}
-	}
-
-	if i, ok := imageFilters["dangling"]; ok {
-		for _, value := range i {
-			if v := strings.ToLower(value); v == "true" {
-				filtTagged = false
-			} else if v != "false" {
-				return nil, fmt.Errorf("Invalid filter 'dangling=%s'", v)
-			}
-		}
-	}
-
-	_, filtLabel = imageFilters["label"]
-
-	if all && filtTagged {
-		allImages = s.graph.Map()
-	} else {
-		allImages = s.graph.heads()
-	}
-
-	lookup := make(map[string]*types.Image)
-	s.Lock()
-	for repoName, repository := range s.Repositories {
-		filterTagName := ""
-		if filter != "" {
-			filterName := filter
-			// Test if the tag was in there, if yes, get the name
-			if strings.Contains(filterName, ":") {
-				filterWithTag := strings.Split(filter, ":")
-				filterName = filterWithTag[0]
-				filterTagName = filterWithTag[1]
-			}
-			if match, _ := path.Match(filterName, repoName); !match {
-				continue
-			}
-			if filterTagName != "" {
-				if _, ok := repository[filterTagName]; !ok {
-					continue
-				}
-			}
-		}
-		for ref, id := range repository {
-			imgRef := utils.ImageReference(repoName, ref)
-			if !strings.Contains(imgRef, filterTagName) {
-				continue
-			}
-			image, err := s.graph.Get(id)
-			if err != nil {
-				logrus.Warnf("couldn't load %s from %s: %s", id, imgRef, err)
-				continue
-			}
-
-			if lImage, exists := lookup[id]; exists {
-				if filtTagged {
-					if utils.DigestReference(ref) {
-						lImage.RepoDigests = append(lImage.RepoDigests, imgRef)
-					} else { // Tag Ref.
-						lImage.RepoTags = append(lImage.RepoTags, imgRef)
-					}
-				}
-			} else {
-				// get the boolean list for if only the untagged images are requested
-				delete(allImages, id)
-
-				if len(imageFilters["label"]) > 0 {
-					if image.Config == nil {
-						// Very old image that do not have image.Config (or even labels)
-						continue
-					}
-					// We are now sure image.Config is not nil
-					if !imageFilters.MatchKVList("label", image.Config.Labels) {
-						continue
-					}
-				}
-				if filtTagged {
-					newImage := newImage(image, s.graph.getParentsSize(image))
-
-					if utils.DigestReference(ref) {
-						newImage.RepoTags = []string{}
-						newImage.RepoDigests = []string{imgRef}
-					} else {
-						newImage.RepoTags = []string{imgRef}
-						newImage.RepoDigests = []string{}
-					}
-
-					lookup[id] = newImage
-				}
-			}
-
-		}
-	}
-	s.Unlock()
-
-	images := []*types.Image{}
-	for _, value := range lookup {
-		images = append(images, value)
-	}
-
-	// Display images which aren't part of a repository/tag
-	if filter == "" || filtLabel {
-		for _, image := range allImages {
-			if len(imageFilters["label"]) > 0 {
-				if image.Config == nil {
-					// Very old image that do not have image.Config (or even labels)
-					continue
-				}
-				// We are now sure image.Config is not nil
-				if !imageFilters.MatchKVList("label", image.Config.Labels) {
-					continue
-				}
-			}
-			newImage := newImage(image, s.graph.getParentsSize(image))
-			newImage.RepoTags = []string{"<none>:<none>"}
-			newImage.RepoDigests = []string{"<none>@<none>"}
-
-			images = append(images, newImage)
-		}
-	}
-
-	sort.Sort(sort.Reverse(byCreated(images)))
-
-	return images, nil
-}
-
-func newImage(image *image.Image, parentSize int64) *types.Image {
-	newImage := new(types.Image)
-	newImage.ParentID = image.Parent
-	newImage.ID = image.ID
-	newImage.Created = image.Created.Unix()
-	newImage.Size = image.Size
-	newImage.VirtualSize = parentSize + image.Size
-	if image.Config != nil {
-		newImage.Labels = image.Config.Labels
-	}
-	return newImage
-}

graph/load.go

@@ -1,134 +0,0 @@
-// +build linux windows
-
-package graph
-
-import (
-	"encoding/json"
-	"io"
-	"io/ioutil"
-	"os"
-	"path/filepath"
-
-	"github.com/Sirupsen/logrus"
-	"github.com/docker/docker/image"
-	"github.com/docker/docker/pkg/archive"
-	"github.com/docker/docker/pkg/chrootarchive"
-)
-
-// Load uploads a set of images into the repository. This is the complementary of ImageExport.
-// The input stream is an uncompressed tar ball containing images and metadata.
-func (s *TagStore) Load(inTar io.ReadCloser, outStream io.Writer) error {
-	tmpImageDir, err := ioutil.TempDir("", "docker-import-")
-	if err != nil {
-		return err
-	}
-	defer os.RemoveAll(tmpImageDir)
-
-	var (
-		repoDir = filepath.Join(tmpImageDir, "repo")
-	)
-
-	if err := os.Mkdir(repoDir, os.ModeDir); err != nil {
-		return err
-	}
-	images := s.graph.Map()
-	excludes := make([]string, len(images))
-	i := 0
-	for k := range images {
-		excludes[i] = k
-		i++
-	}
-	if err := chrootarchive.Untar(inTar, repoDir, &archive.TarOptions{ExcludePatterns: excludes}); err != nil {
-		return err
-	}
-
-	dirs, err := ioutil.ReadDir(repoDir)
-	if err != nil {
-		return err
-	}
-
-	for _, d := range dirs {
-		if d.IsDir() {
-			if err := s.recursiveLoad(d.Name(), tmpImageDir); err != nil {
-				return err
-			}
-		}
-	}
-
-	reposJSONFile, err := os.Open(filepath.Join(tmpImageDir, "repo", "repositories"))
-	if err != nil {
-		if !os.IsNotExist(err) {
-			return err
-		}
-		return nil
-	}
-	defer reposJSONFile.Close()
-
-	repositories := map[string]repository{}
-	if err := json.NewDecoder(reposJSONFile).Decode(&repositories); err != nil {
-		return err
-	}
-
-	for imageName, tagMap := range repositories {
-		for tag, address := range tagMap {
-			if err := s.setLoad(imageName, tag, address, true, outStream); err != nil {
-				return err
-			}
-		}
-	}
-
-	return nil
-}
-
-func (s *TagStore) recursiveLoad(address, tmpImageDir string) error {
-	if _, err := s.LookupImage(address); err != nil {
-		logrus.Debugf("Loading %s", address)
-
-		imageJSON, err := ioutil.ReadFile(filepath.Join(tmpImageDir, "repo", address, "json"))
-		if err != nil {
-			logrus.Debugf("Error reading json: %v", err)
-			return err
-		}
-
-		layer, err := os.Open(filepath.Join(tmpImageDir, "repo", address, "layer.tar"))
-		if err != nil {
-			logrus.Debugf("Error reading embedded tar: %v", err)
-			return err
-		}
-		img, err := image.NewImgJSON(imageJSON)
-		if err != nil {
-			logrus.Debugf("Error unmarshalling json: %v", err)
-			return err
-		}
-		if err := image.ValidateID(img.ID); err != nil {
-			logrus.Debugf("Error validating ID: %v", err)
-			return err
-		}
-
-		// ensure no two downloads of the same layer happen at the same time
-		poolKey := "layer:" + img.ID
-		broadcaster, found := s.poolAdd("pull", poolKey)
-		if found {
-			logrus.Debugf("Image (id: %s) load is already running, waiting", img.ID)
-			return broadcaster.Wait()
-		}
-
-		defer s.poolRemove("pull", poolKey)
-
-		if img.Parent != "" {
-			if !s.graph.Exists(img.Parent) {
-				if err := s.recursiveLoad(img.Parent, tmpImageDir); err != nil {
-					return err
-				}
-			}
-		}
-		if err := s.graph.Register(v1Descriptor{img}, layer); err != nil {
-			return err
-		}
-		logrus.Debugf("Completed processing %s", address)
-		return nil
-	}
-	logrus.Debugf("already loaded %s", address)
-
-	return nil
-}

graph/load_unsupported.go

@@ -1,14 +0,0 @@
-// +build !linux,!windows
-
-package graph
-
-import (
-	"fmt"
-	"io"
-)
-
-// Load method is implemented here for non-linux and non-windows platforms and
-// may return an error indicating that image load is not supported on other platforms.
-func (s *TagStore) Load(inTar io.ReadCloser, outStream io.Writer) error {
-	return fmt.Errorf("Load is not supported on this platform")
-}

graph/pools_test.go

@@ -1,44 +0,0 @@
-package graph
-
-import (
-	"testing"
-
-	"github.com/docker/docker/pkg/broadcaster"
-	"github.com/docker/docker/pkg/reexec"
-)
-
-func init() {
-	reexec.Init()
-}
-
-func TestPools(t *testing.T) {
-	s := &TagStore{
-		pullingPool: make(map[string]*broadcaster.Buffered),
-		pushingPool: make(map[string]*broadcaster.Buffered),
-	}
-
-	if _, found := s.poolAdd("pull", "test1"); found {
-		t.Fatal("Expected pull test1 not to be in progress")
-	}
-	if _, found := s.poolAdd("pull", "test2"); found {
-		t.Fatal("Expected pull test2 not to be in progress")
-	}
-	if _, found := s.poolAdd("push", "test1"); !found {
-		t.Fatalf("Expected pull test1 to be in progress`")
-	}
-	if _, found := s.poolAdd("pull", "test1"); !found {
-		t.Fatalf("Expected pull test1 to be in progress`")
-	}
-	if err := s.poolRemove("pull", "test2"); err != nil {
-		t.Fatal(err)
-	}
-	if err := s.poolRemove("pull", "test2"); err != nil {
-		t.Fatal(err)
-	}
-	if err := s.poolRemove("pull", "test1"); err != nil {
-		t.Fatal(err)
-	}
-	if err := s.poolRemove("push", "test1"); err != nil {
-		t.Fatal(err)
-	}
-}

graph/pull_v1.go

@@ -1,350 +0,0 @@
-package graph
-
-import (
-	"errors"
-	"fmt"
-	"io"
-	"net"
-	"net/url"
-	"strings"
-	"time"
-
-	"github.com/Sirupsen/logrus"
-	"github.com/docker/distribution/registry/client/transport"
-	"github.com/docker/docker/image"
-	"github.com/docker/docker/pkg/progressreader"
-	"github.com/docker/docker/pkg/streamformatter"
-	"github.com/docker/docker/pkg/stringid"
-	"github.com/docker/docker/registry"
-	"github.com/docker/docker/utils"
-)
-
-type v1Puller struct {
-	*TagStore
-	endpoint registry.APIEndpoint
-	config   *ImagePullConfig
-	sf       *streamformatter.StreamFormatter
-	repoInfo *registry.RepositoryInfo
-	session  *registry.Session
-}
-
-func (p *v1Puller) Pull(tag string) (fallback bool, err error) {
-	if utils.DigestReference(tag) {
-		// Allowing fallback, because HTTPS v1 is before HTTP v2
-		return true, registry.ErrNoSupport{Err: errors.New("Cannot pull by digest with v1 registry")}
-	}
-
-	tlsConfig, err := p.registryService.TLSConfig(p.repoInfo.Index.Name)
-	if err != nil {
-		return false, err
-	}
-	// Adds Docker-specific headers as well as user-specified headers (metaHeaders)
-	tr := transport.NewTransport(
-		// TODO(tiborvass): was ReceiveTimeout
-		registry.NewTransport(tlsConfig),
-		registry.DockerHeaders(p.config.MetaHeaders)...,
-	)
-	client := registry.HTTPClient(tr)
-	v1Endpoint, err := p.endpoint.ToV1Endpoint(p.config.MetaHeaders)
-	if err != nil {
-		logrus.Debugf("Could not get v1 endpoint: %v", err)
-		return true, err
-	}
-	p.session, err = registry.NewSession(client, p.config.AuthConfig, v1Endpoint)
-	if err != nil {
-		// TODO(dmcgowan): Check if should fallback
-		logrus.Debugf("Fallback from error: %s", err)
-		return true, err
-	}
-	if err := p.pullRepository(tag); err != nil {
-		// TODO(dmcgowan): Check if should fallback
-		return false, err
-	}
-	out := p.config.OutStream
-	out.Write(p.sf.FormatStatus("", "%s: this image was pulled from a legacy registry.  Important: This registry version will not be supported in future versions of docker.", p.repoInfo.CanonicalName))
-
-	return false, nil
-}
-
-func (p *v1Puller) pullRepository(askedTag string) error {
-	out := p.config.OutStream
-	out.Write(p.sf.FormatStatus("", "Pulling repository %s", p.repoInfo.CanonicalName))
-
-	repoData, err := p.session.GetRepositoryData(p.repoInfo.RemoteName)
-	if err != nil {
-		if strings.Contains(err.Error(), "HTTP code: 404") {
-			return fmt.Errorf("Error: image %s not found", utils.ImageReference(p.repoInfo.RemoteName, askedTag))
-		}
-		// Unexpected HTTP error
-		return err
-	}
-
-	logrus.Debugf("Retrieving the tag list")
-	tagsList := make(map[string]string)
-	if askedTag == "" {
-		tagsList, err = p.session.GetRemoteTags(repoData.Endpoints, p.repoInfo.RemoteName)
-	} else {
-		var tagID string
-		tagID, err = p.session.GetRemoteTag(repoData.Endpoints, p.repoInfo.RemoteName, askedTag)
-		tagsList[askedTag] = tagID
-	}
-	if err != nil {
-		if err == registry.ErrRepoNotFound && askedTag != "" {
-			return fmt.Errorf("Tag %s not found in repository %s", askedTag, p.repoInfo.CanonicalName)
-		}
-		logrus.Errorf("unable to get remote tags: %s", err)
-		return err
-	}
-
-	for tag, id := range tagsList {
-		repoData.ImgList[id] = &registry.ImgData{
-			ID:       id,
-			Tag:      tag,
-			Checksum: "",
-		}
-	}
-
-	logrus.Debugf("Registering tags")
-	// If no tag has been specified, pull them all
-	if askedTag == "" {
-		for tag, id := range tagsList {
-			repoData.ImgList[id].Tag = tag
-		}
-	} else {
-		// Otherwise, check that the tag exists and use only that one
-		id, exists := tagsList[askedTag]
-		if !exists {
-			return fmt.Errorf("Tag %s not found in repository %s", askedTag, p.repoInfo.CanonicalName)
-		}
-		repoData.ImgList[id].Tag = askedTag
-	}
-
-	errors := make(chan error)
-
-	layersDownloaded := false
-	imgIDs := []string{}
-	sessionID := p.session.ID()
-	defer func() {
-		p.graph.Release(sessionID, imgIDs...)
-	}()
-	for _, imgData := range repoData.ImgList {
-		downloadImage := func(img *registry.ImgData) {
-			if askedTag != "" && img.Tag != askedTag {
-				errors <- nil
-				return
-			}
-
-			if img.Tag == "" {
-				logrus.Debugf("Image (id: %s) present in this repository but untagged, skipping", img.ID)
-				errors <- nil
-				return
-			}
-
-			if err := image.ValidateID(img.ID); err != nil {
-				errors <- err
-				return
-			}
-
-			// ensure no two downloads of the same image happen at the same time
-			poolKey := "img:" + img.ID
-			broadcaster, found := p.poolAdd("pull", poolKey)
-			broadcaster.Add(out)
-			if found {
-				errors <- broadcaster.Wait()
-				return
-			}
-			defer p.poolRemove("pull", poolKey)
-
-			// we need to retain it until tagging
-			p.graph.Retain(sessionID, img.ID)
-			imgIDs = append(imgIDs, img.ID)
-
-			broadcaster.Write(p.sf.FormatProgress(stringid.TruncateID(img.ID), fmt.Sprintf("Pulling image (%s) from %s", img.Tag, p.repoInfo.CanonicalName), nil))
-			success := false
-			var lastErr, err error
-			var isDownloaded bool
-			for _, ep := range p.repoInfo.Index.Mirrors {
-				ep += "v1/"
-				broadcaster.Write(p.sf.FormatProgress(stringid.TruncateID(img.ID), fmt.Sprintf("Pulling image (%s) from %s, mirror: %s", img.Tag, p.repoInfo.CanonicalName, ep), nil))
-				if isDownloaded, err = p.pullImage(broadcaster, img.ID, ep); err != nil {
-					// Don't report errors when pulling from mirrors.
-					logrus.Debugf("Error pulling image (%s) from %s, mirror: %s, %s", img.Tag, p.repoInfo.CanonicalName, ep, err)
-					continue
-				}
-				layersDownloaded = layersDownloaded || isDownloaded
-				success = true
-				break
-			}
-			if !success {
-				for _, ep := range repoData.Endpoints {
-					broadcaster.Write(p.sf.FormatProgress(stringid.TruncateID(img.ID), fmt.Sprintf("Pulling image (%s) from %s, endpoint: %s", img.Tag, p.repoInfo.CanonicalName, ep), nil))
-					if isDownloaded, err = p.pullImage(broadcaster, img.ID, ep); err != nil {
-						// It's not ideal that only the last error is returned, it would be better to concatenate the errors.
-						// As the error is also given to the output stream the user will see the error.
-						lastErr = err
-						broadcaster.Write(p.sf.FormatProgress(stringid.TruncateID(img.ID), fmt.Sprintf("Error pulling image (%s) from %s, endpoint: %s, %s", img.Tag, p.repoInfo.CanonicalName, ep, err), nil))
-						continue
-					}
-					layersDownloaded = layersDownloaded || isDownloaded
-					success = true
-					break
-				}
-			}
-			if !success {
-				err := fmt.Errorf("Error pulling image (%s) from %s, %v", img.Tag, p.repoInfo.CanonicalName, lastErr)
-				broadcaster.Write(p.sf.FormatProgress(stringid.TruncateID(img.ID), err.Error(), nil))
-				errors <- err
-				broadcaster.CloseWithError(err)
-				return
-			}
-			broadcaster.Write(p.sf.FormatProgress(stringid.TruncateID(img.ID), "Download complete", nil))
-
-			errors <- nil
-		}
-
-		go downloadImage(imgData)
-	}
-
-	var lastError error
-	for i := 0; i < len(repoData.ImgList); i++ {
-		if err := <-errors; err != nil {
-			lastError = err
-		}
-	}
-	if lastError != nil {
-		return lastError
-	}
-
-	for tag, id := range tagsList {
-		if askedTag != "" && tag != askedTag {
-			continue
-		}
-		if err := p.Tag(p.repoInfo.LocalName, tag, id, true); err != nil {
-			return err
-		}
-	}
-
-	requestedTag := p.repoInfo.LocalName
-	if len(askedTag) > 0 {
-		requestedTag = utils.ImageReference(p.repoInfo.LocalName, askedTag)
-	}
-	writeStatus(requestedTag, out, p.sf, layersDownloaded)
-	return nil
-}
-
-func (p *v1Puller) pullImage(out io.Writer, imgID, endpoint string) (layersDownloaded bool, err error) {
-	var history []string
-	history, err = p.session.GetRemoteHistory(imgID, endpoint)
-	if err != nil {
-		return false, err
-	}
-	out.Write(p.sf.FormatProgress(stringid.TruncateID(imgID), "Pulling dependent layers", nil))
-	// FIXME: Try to stream the images?
-	// FIXME: Launch the getRemoteImage() in goroutines
-
-	sessionID := p.session.ID()
-	// As imgID has been retained in pullRepository, no need to retain again
-	p.graph.Retain(sessionID, history[1:]...)
-	defer p.graph.Release(sessionID, history[1:]...)
-
-	layersDownloaded = false
-	for i := len(history) - 1; i >= 0; i-- {
-		id := history[i]
-
-		// ensure no two downloads of the same layer happen at the same time
-		poolKey := "layer:" + id
-		broadcaster, found := p.poolAdd("pull", poolKey)
-		broadcaster.Add(out)
-		if found {
-			logrus.Debugf("Image (id: %s) pull is already running, skipping", id)
-			err = broadcaster.Wait()
-			if err != nil {
-				return layersDownloaded, err
-			}
-			continue
-		}
-
-		// This must use a closure so it captures the value of err when
-		// the function returns, not when the 'defer' is evaluated.
-		defer func() {
-			p.poolRemoveWithError("pull", poolKey, err)
-		}()
-
-		if !p.graph.Exists(id) {
-			broadcaster.Write(p.sf.FormatProgress(stringid.TruncateID(id), "Pulling metadata", nil))
-			var (
-				imgJSON []byte
-				imgSize int64
-				err     error
-				img     *image.Image
-			)
-			retries := 5
-			for j := 1; j <= retries; j++ {
-				imgJSON, imgSize, err = p.session.GetRemoteImageJSON(id, endpoint)
-				if err != nil && j == retries {
-					broadcaster.Write(p.sf.FormatProgress(stringid.TruncateID(id), "Error pulling dependent layers", nil))
-					return layersDownloaded, err
-				} else if err != nil {
-					time.Sleep(time.Duration(j) * 500 * time.Millisecond)
-					continue
-				}
-				img, err = image.NewImgJSON(imgJSON)
-				layersDownloaded = true
-				if err != nil && j == retries {
-					broadcaster.Write(p.sf.FormatProgress(stringid.TruncateID(id), "Error pulling dependent layers", nil))
-					return layersDownloaded, fmt.Errorf("Failed to parse json: %s", err)
-				} else if err != nil {
-					time.Sleep(time.Duration(j) * 500 * time.Millisecond)
-					continue
-				} else {
-					break
-				}
-			}
-
-			for j := 1; j <= retries; j++ {
-				// Get the layer
-				status := "Pulling fs layer"
-				if j > 1 {
-					status = fmt.Sprintf("Pulling fs layer [retries: %d]", j)
-				}
-				broadcaster.Write(p.sf.FormatProgress(stringid.TruncateID(id), status, nil))
-				layer, err := p.session.GetRemoteImageLayer(img.ID, endpoint, imgSize)
-				if uerr, ok := err.(*url.Error); ok {
-					err = uerr.Err
-				}
-				if terr, ok := err.(net.Error); ok && terr.Timeout() && j < retries {
-					time.Sleep(time.Duration(j) * 500 * time.Millisecond)
-					continue
-				} else if err != nil {
-					broadcaster.Write(p.sf.FormatProgress(stringid.TruncateID(id), "Error pulling dependent layers", nil))
-					return layersDownloaded, err
-				}
-				layersDownloaded = true
-				defer layer.Close()
-
-				err = p.graph.Register(v1Descriptor{img},
-					progressreader.New(progressreader.Config{
-						In:        layer,
-						Out:       broadcaster,
-						Formatter: p.sf,
-						Size:      imgSize,
-						NewLines:  false,
-						ID:        stringid.TruncateID(id),
-						Action:    "Downloading",
-					}))
-				if terr, ok := err.(net.Error); ok && terr.Timeout() && j < retries {
-					time.Sleep(time.Duration(j) * 500 * time.Millisecond)
-					continue
-				} else if err != nil {
-					broadcaster.Write(p.sf.FormatProgress(stringid.TruncateID(id), "Error downloading dependent layers", nil))
-					return layersDownloaded, err
-				} else {
-					break
-				}
-			}
-		}
-		broadcaster.Write(p.sf.FormatProgress(stringid.TruncateID(id), "Download complete", nil))
-		broadcaster.Close()
-	}
-	return layersDownloaded, nil
-}

graph/pull_v2.go

@@ -1,728 +0,0 @@
-package graph
-
-import (
-	"encoding/json"
-	"errors"
-	"fmt"
-	"io"
-	"io/ioutil"
-	"os"
-	"runtime"
-
-	"github.com/Sirupsen/logrus"
-	"github.com/docker/distribution"
-	"github.com/docker/distribution/digest"
-	"github.com/docker/distribution/manifest/schema1"
-	"github.com/docker/docker/image"
-	"github.com/docker/docker/pkg/broadcaster"
-	"github.com/docker/docker/pkg/progressreader"
-	"github.com/docker/docker/pkg/streamformatter"
-	"github.com/docker/docker/pkg/stringid"
-	"github.com/docker/docker/registry"
-	"github.com/docker/docker/utils"
-	"golang.org/x/net/context"
-)
-
-type v2Puller struct {
-	*TagStore
-	endpoint  registry.APIEndpoint
-	config    *ImagePullConfig
-	sf        *streamformatter.StreamFormatter
-	repoInfo  *registry.RepositoryInfo
-	repo      distribution.Repository
-	sessionID string
-}
-
-func (p *v2Puller) Pull(tag string) (fallback bool, err error) {
-	// TODO(tiborvass): was ReceiveTimeout
-	p.repo, err = newV2Repository(p.repoInfo, p.endpoint, p.config.MetaHeaders, p.config.AuthConfig, "pull")
-	if err != nil {
-		logrus.Warnf("Error getting v2 registry: %v", err)
-		return true, err
-	}
-
-	p.sessionID = stringid.GenerateRandomID()
-
-	if err := p.pullV2Repository(tag); err != nil {
-		if registry.ContinueOnError(err) {
-			logrus.Debugf("Error trying v2 registry: %v", err)
-			return true, err
-		}
-		return false, err
-	}
-	return false, nil
-}
-
-func (p *v2Puller) pullV2Repository(tag string) (err error) {
-	var tags []string
-	taggedName := p.repoInfo.LocalName
-	if len(tag) > 0 {
-		tags = []string{tag}
-		taggedName = utils.ImageReference(p.repoInfo.LocalName, tag)
-	} else {
-		var err error
-
-		manSvc, err := p.repo.Manifests(context.Background())
-		if err != nil {
-			return err
-		}
-
-		tags, err = manSvc.Tags()
-		if err != nil {
-			return err
-		}
-
-	}
-
-	poolKey := "v2:" + taggedName
-	broadcaster, found := p.poolAdd("pull", poolKey)
-	broadcaster.Add(p.config.OutStream)
-	if found {
-		// Another pull of the same repository is already taking place; just wait for it to finish
-		return broadcaster.Wait()
-	}
-
-	// This must use a closure so it captures the value of err when the
-	// function returns, not when the 'defer' is evaluated.
-	defer func() {
-		p.poolRemoveWithError("pull", poolKey, err)
-	}()
-
-	var layersDownloaded bool
-	for _, tag := range tags {
-		// pulledNew is true if either new layers were downloaded OR if existing images were newly tagged
-		// TODO(tiborvass): should we change the name of `layersDownload`? What about message in WriteStatus?
-		pulledNew, err := p.pullV2Tag(broadcaster, tag, taggedName)
-		if err != nil {
-			return err
-		}
-		layersDownloaded = layersDownloaded || pulledNew
-	}
-
-	writeStatus(taggedName, broadcaster, p.sf, layersDownloaded)
-
-	return nil
-}
-
-// downloadInfo is used to pass information from download to extractor
-type downloadInfo struct {
-	img         contentAddressableDescriptor
-	imgIndex    int
-	tmpFile     *os.File
-	digest      digest.Digest
-	layer       distribution.ReadSeekCloser
-	size        int64
-	err         chan error
-	poolKey     string
-	broadcaster *broadcaster.Buffered
-}
-
-// contentAddressableDescriptor is used to pass image data from a manifest to the
-// graph.
-type contentAddressableDescriptor struct {
-	id              string
-	parent          string
-	strongID        digest.Digest
-	compatibilityID string
-	config          []byte
-	v1Compatibility []byte
-}
-
-func newContentAddressableImage(v1Compatibility []byte, blobSum digest.Digest, parent digest.Digest) (contentAddressableDescriptor, error) {
-	img := contentAddressableDescriptor{
-		v1Compatibility: v1Compatibility,
-	}
-
-	var err error
-	img.config, err = image.MakeImageConfig(v1Compatibility, blobSum, parent)
-	if err != nil {
-		return img, err
-	}
-	img.strongID, err = image.StrongID(img.config)
-	if err != nil {
-		return img, err
-	}
-
-	unmarshalledConfig, err := image.NewImgJSON(v1Compatibility)
-	if err != nil {
-		return img, err
-	}
-
-	img.compatibilityID = unmarshalledConfig.ID
-	img.id = img.strongID.Hex()
-
-	return img, nil
-}
-
-// ID returns the actual ID to be used for the downloaded image. This may be
-// a computed ID.
-func (img contentAddressableDescriptor) ID() string {
-	return img.id
-}
-
-// Parent returns the parent ID to be used for the image. This may be a
-// computed ID.
-func (img contentAddressableDescriptor) Parent() string {
-	return img.parent
-}
-
-// MarshalConfig renders the image structure into JSON.
-func (img contentAddressableDescriptor) MarshalConfig() ([]byte, error) {
-	return img.config, nil
-}
-
-type errVerification struct{}
-
-func (errVerification) Error() string { return "verification failed" }
-
-func (p *v2Puller) download(di *downloadInfo) {
-	logrus.Debugf("pulling blob %q to %s", di.digest, di.img.id)
-
-	blobs := p.repo.Blobs(context.Background())
-
-	desc, err := blobs.Stat(context.Background(), di.digest)
-	if err != nil {
-		logrus.Debugf("Error statting layer: %v", err)
-		di.err <- err
-		return
-	}
-	di.size = desc.Size
-
-	layerDownload, err := blobs.Open(context.Background(), di.digest)
-	if err != nil {
-		logrus.Debugf("Error fetching layer: %v", err)
-		di.err <- err
-		return
-	}
-	defer layerDownload.Close()
-
-	verifier, err := digest.NewDigestVerifier(di.digest)
-	if err != nil {
-		di.err <- err
-		return
-	}
-
-	reader := progressreader.New(progressreader.Config{
-		In:        ioutil.NopCloser(io.TeeReader(layerDownload, verifier)),
-		Out:       di.broadcaster,
-		Formatter: p.sf,
-		Size:      di.size,
-		NewLines:  false,
-		ID:        stringid.TruncateID(di.img.id),
-		Action:    "Downloading",
-	})
-	io.Copy(di.tmpFile, reader)
-
-	di.broadcaster.Write(p.sf.FormatProgress(stringid.TruncateID(di.img.id), "Verifying Checksum", nil))
-
-	if !verifier.Verified() {
-		err = fmt.Errorf("filesystem layer verification failed for digest %s", di.digest)
-		logrus.Error(err)
-		di.err <- err
-		return
-	}
-
-	di.broadcaster.Write(p.sf.FormatProgress(stringid.TruncateID(di.img.id), "Download complete", nil))
-
-	logrus.Debugf("Downloaded %s to tempfile %s", di.img.id, di.tmpFile.Name())
-	di.layer = layerDownload
-
-	di.err <- nil
-}
-
-func (p *v2Puller) pullV2Tag(out io.Writer, tag, taggedName string) (tagUpdated bool, err error) {
-	logrus.Debugf("Pulling tag from V2 registry: %q", tag)
-
-	manSvc, err := p.repo.Manifests(context.Background())
-	if err != nil {
-		return false, err
-	}
-
-	unverifiedManifest, err := manSvc.GetByTag(tag)
-	if err != nil {
-		return false, err
-	}
-	if unverifiedManifest == nil {
-		return false, fmt.Errorf("image manifest does not exist for tag %q", tag)
-	}
-	var verifiedManifest *schema1.Manifest
-	verifiedManifest, err = verifyManifest(unverifiedManifest, tag)
-	if err != nil {
-		return false, err
-	}
-
-	// remove duplicate layers and check parent chain validity
-	err = fixManifestLayers(verifiedManifest)
-	if err != nil {
-		return false, err
-	}
-
-	imgs, err := p.getImageInfos(verifiedManifest)
-	if err != nil {
-		return false, err
-	}
-
-	out.Write(p.sf.FormatStatus(tag, "Pulling from %s", p.repo.Name()))
-
-	var downloads []*downloadInfo
-
-	var layerIDs []string
-	defer func() {
-		p.graph.Release(p.sessionID, layerIDs...)
-
-		for _, d := range downloads {
-			p.poolRemoveWithError("pull", d.poolKey, err)
-			if d.tmpFile != nil {
-				d.tmpFile.Close()
-				if err := os.RemoveAll(d.tmpFile.Name()); err != nil {
-					logrus.Errorf("Failed to remove temp file: %s", d.tmpFile.Name())
-				}
-			}
-		}
-	}()
-
-	for i := len(verifiedManifest.FSLayers) - 1; i >= 0; i-- {
-		img := imgs[i]
-
-		p.graph.Retain(p.sessionID, img.id)
-		layerIDs = append(layerIDs, img.id)
-
-		p.graph.imageMutex.Lock(img.id)
-
-		// Check if exists
-		if p.graph.Exists(img.id) {
-			if err := p.validateImageInGraph(img.id, imgs, i); err != nil {
-				p.graph.imageMutex.Unlock(img.id)
-				return false, fmt.Errorf("image validation failed: %v", err)
-			}
-			logrus.Debugf("Image already exists: %s", img.id)
-			p.graph.imageMutex.Unlock(img.id)
-			continue
-		}
-		p.graph.imageMutex.Unlock(img.id)
-
-		out.Write(p.sf.FormatProgress(stringid.TruncateID(img.id), "Pulling fs layer", nil))
-
-		d := &downloadInfo{
-			img:      img,
-			imgIndex: i,
-			poolKey:  "v2layer:" + img.id,
-			digest:   verifiedManifest.FSLayers[i].BlobSum,
-			// TODO: seems like this chan buffer solved hanging problem in go1.5,
-			// this can indicate some deeper problem that somehow we never take
-			// error from channel in loop below
-			err: make(chan error, 1),
-		}
-
-		tmpFile, err := ioutil.TempFile("", "GetImageBlob")
-		if err != nil {
-			return false, err
-		}
-		d.tmpFile = tmpFile
-
-		downloads = append(downloads, d)
-
-		broadcaster, found := p.poolAdd("pull", d.poolKey)
-		broadcaster.Add(out)
-		d.broadcaster = broadcaster
-		if found {
-			d.err <- nil
-		} else {
-			go p.download(d)
-		}
-	}
-
-	for _, d := range downloads {
-		if err := <-d.err; err != nil {
-			return false, err
-		}
-
-		if d.layer == nil {
-			// Wait for a different pull to download and extract
-			// this layer.
-			err = d.broadcaster.Wait()
-			if err != nil {
-				return false, err
-			}
-			continue
-		}
-
-		d.tmpFile.Seek(0, 0)
-		err := func() error {
-			reader := progressreader.New(progressreader.Config{
-				In:        d.tmpFile,
-				Out:       d.broadcaster,
-				Formatter: p.sf,
-				Size:      d.size,
-				NewLines:  false,
-				ID:        stringid.TruncateID(d.img.id),
-				Action:    "Extracting",
-			})
-
-			p.graph.imagesMutex.Lock()
-			defer p.graph.imagesMutex.Unlock()
-
-			p.graph.imageMutex.Lock(d.img.id)
-			defer p.graph.imageMutex.Unlock(d.img.id)
-
-			// Must recheck the data on disk if any exists.
-			// This protects against races where something
-			// else is written to the graph under this ID
-			// after attemptIDReuse.
-			if p.graph.Exists(d.img.id) {
-				if err := p.validateImageInGraph(d.img.id, imgs, d.imgIndex); err != nil {
-					return fmt.Errorf("image validation failed: %v", err)
-				}
-			}
-
-			if err := p.graph.register(d.img, reader); err != nil {
-				return err
-			}
-
-			if err := p.graph.setLayerDigest(d.img.id, d.digest); err != nil {
-				return err
-			}
-
-			if err := p.graph.setV1CompatibilityConfig(d.img.id, d.img.v1Compatibility); err != nil {
-				return err
-			}
-
-			return nil
-		}()
-		if err != nil {
-			return false, err
-		}
-
-		d.broadcaster.Write(p.sf.FormatProgress(stringid.TruncateID(d.img.id), "Pull complete", nil))
-		d.broadcaster.Close()
-		tagUpdated = true
-	}
-
-	manifestDigest, _, err := digestFromManifest(unverifiedManifest, p.repoInfo.LocalName)
-	if err != nil {
-		return false, err
-	}
-
-	// Check for new tag if no layers downloaded
-	if !tagUpdated {
-		repo, err := p.get(p.repoInfo.LocalName)
-		if err != nil {
-			return false, err
-		}
-		if repo != nil {
-			if _, exists := repo[tag]; !exists {
-				tagUpdated = true
-			}
-		} else {
-			tagUpdated = true
-		}
-	}
-
-	firstID := layerIDs[len(layerIDs)-1]
-	if utils.DigestReference(tag) {
-		// TODO(stevvooe): Ideally, we should always set the digest so we can
-		// use the digest whether we pull by it or not. Unfortunately, the tag
-		// store treats the digest as a separate tag, meaning there may be an
-		// untagged digest image that would seem to be dangling by a user.
-		if err = p.setDigest(p.repoInfo.LocalName, tag, firstID); err != nil {
-			return false, err
-		}
-	} else {
-		// only set the repository/tag -> image ID mapping when pulling by tag (i.e. not by digest)
-		if err = p.Tag(p.repoInfo.LocalName, tag, firstID, true); err != nil {
-			return false, err
-		}
-	}
-
-	if manifestDigest != "" {
-		out.Write(p.sf.FormatStatus("", "Digest: %s", manifestDigest))
-	}
-
-	return tagUpdated, nil
-}
-
-func verifyManifest(signedManifest *schema1.SignedManifest, tag string) (m *schema1.Manifest, err error) {
-	// If pull by digest, then verify the manifest digest. NOTE: It is
-	// important to do this first, before any other content validation. If the
-	// digest cannot be verified, don't even bother with those other things.
-	if manifestDigest, err := digest.ParseDigest(tag); err == nil {
-		verifier, err := digest.NewDigestVerifier(manifestDigest)
-		if err != nil {
-			return nil, err
-		}
-		payload, err := signedManifest.Payload()
-		if err != nil {
-			// If this failed, the signatures section was corrupted
-			// or missing. Treat the entire manifest as the payload.
-			payload = signedManifest.Raw
-		}
-		if _, err := verifier.Write(payload); err != nil {
-			return nil, err
-		}
-		if !verifier.Verified() {
-			err := fmt.Errorf("image verification failed for digest %s", manifestDigest)
-			logrus.Error(err)
-			return nil, err
-		}
-
-		var verifiedManifest schema1.Manifest
-		if err = json.Unmarshal(payload, &verifiedManifest); err != nil {
-			return nil, err
-		}
-		m = &verifiedManifest
-	} else {
-		m = &signedManifest.Manifest
-	}
-
-	if m.SchemaVersion != 1 {
-		return nil, fmt.Errorf("unsupported schema version %d for tag %q", m.SchemaVersion, tag)
-	}
-	if len(m.FSLayers) != len(m.History) {
-		return nil, fmt.Errorf("length of history not equal to number of layers for tag %q", tag)
-	}
-	if len(m.FSLayers) == 0 {
-		return nil, fmt.Errorf("no FSLayers in manifest for tag %q", tag)
-	}
-	return m, nil
-}
-
-// fixManifestLayers removes repeated layers from the manifest and checks the
-// correctness of the parent chain.
-func fixManifestLayers(m *schema1.Manifest) error {
-	images := make([]*image.Image, len(m.FSLayers))
-	for i := range m.FSLayers {
-		img, err := image.NewImgJSON([]byte(m.History[i].V1Compatibility))
-		if err != nil {
-			return err
-		}
-		images[i] = img
-		if err := image.ValidateID(img.ID); err != nil {
-			return err
-		}
-	}
-
-	if images[len(images)-1].Parent != "" && !allowBaseParentImage {
-		// Windows base layer can point to a base layer parent that is not in manifest.
-		return errors.New("Invalid parent ID in the base layer of the image.")
-	}
-
-	// check general duplicates to error instead of a deadlock
-	idmap := make(map[string]struct{})
-
-	var lastID string
-	for _, img := range images {
-		// skip IDs that appear after each other, we handle those later
-		if _, exists := idmap[img.ID]; img.ID != lastID && exists {
-			return fmt.Errorf("ID %+v appears multiple times in manifest", img.ID)
-		}
-		lastID = img.ID
-		idmap[lastID] = struct{}{}
-	}
-
-	// backwards loop so that we keep the remaining indexes after removing items
-	for i := len(images) - 2; i >= 0; i-- {
-		if images[i].ID == images[i+1].ID { // repeated ID. remove and continue
-			m.FSLayers = append(m.FSLayers[:i], m.FSLayers[i+1:]...)
-			m.History = append(m.History[:i], m.History[i+1:]...)
-		} else if images[i].Parent != images[i+1].ID {
-			return fmt.Errorf("Invalid parent ID. Expected %v, got %v.", images[i+1].ID, images[i].Parent)
-		}
-	}
-
-	return nil
-}
-
-// getImageInfos returns an imageinfo struct for every image in the manifest.
-// These objects contain both calculated strongIDs and compatibilityIDs found
-// in v1Compatibility object.
-func (p *v2Puller) getImageInfos(m *schema1.Manifest) ([]contentAddressableDescriptor, error) {
-	imgs := make([]contentAddressableDescriptor, len(m.FSLayers))
-
-	var parent digest.Digest
-	for i := len(imgs) - 1; i >= 0; i-- {
-		var err error
-		imgs[i], err = newContentAddressableImage([]byte(m.History[i].V1Compatibility), m.FSLayers[i].BlobSum, parent)
-		if err != nil {
-			return nil, err
-		}
-		parent = imgs[i].strongID
-	}
-
-	p.attemptIDReuse(imgs)
-
-	// reset the base layer parent for windows
-	if allowBaseParentImage {
-		var base struct{ Parent string }
-		if err := json.Unmarshal(imgs[len(imgs)-1].v1Compatibility, &base); err != nil {
-			return nil, err
-		}
-		if base.Parent != "" {
-			imgs[len(imgs)-1].parent = base.Parent
-		}
-	}
-
-	return imgs, nil
-}
-
-// attemptIDReuse does a best attempt to match verified compatibilityIDs
-// already in the graph with the computed strongIDs so we can keep using them.
-// This process will never fail but may just return the strongIDs if none of
-// the compatibilityIDs exists or can be verified. If the strongIDs themselves
-// fail verification, we deterministically generate alternate IDs to use until
-// we find one that's available or already exists with the correct data.
-func (p *v2Puller) attemptIDReuse(imgs []contentAddressableDescriptor) {
-	// This function needs to be protected with a global lock, because it
-	// locks multiple IDs at once, and there's no good way to make sure
-	// the locking happens a deterministic order.
-	p.graph.imagesMutex.Lock()
-	defer p.graph.imagesMutex.Unlock()
-
-	idMap := make(map[string]struct{})
-	for _, img := range imgs {
-		idMap[img.id] = struct{}{}
-		idMap[img.compatibilityID] = struct{}{}
-
-		if p.graph.Exists(img.compatibilityID) {
-			if _, err := p.graph.generateV1CompatibilityChain(img.compatibilityID); err != nil {
-				logrus.Debugf("Migration v1Compatibility generation error: %v", err)
-				return
-			}
-		}
-	}
-	for id := range idMap {
-		p.graph.imageMutex.Lock(id)
-		defer p.graph.imageMutex.Unlock(id)
-	}
-
-	// continueReuse controls whether the function will try to find
-	// existing layers on disk under the old v1 IDs, to avoid repulling
-	// them. The hashes are checked to ensure these layers are okay to
-	// use. continueReuse starts out as true, but is set to false if
-	// the code encounters something that doesn't match the expected hash.
-	continueReuse := true
-
-	for i := len(imgs) - 1; i >= 0; i-- {
-		// TODO - (swernli:11-16-2015) Skipping content addressable IDs on
-		// Windows as a hack for TP4 compat. The correct fix is to ensure that
-		// Windows layers do not have anything in them that takes a dependency
-		// on the ID of the layer in the management client. This will be fixed
-		// in Windows post-TP4.
-		if runtime.GOOS == "windows" {
-			imgs[i].id = imgs[i].compatibilityID
-		}
-
-		if p.graph.Exists(imgs[i].id) {
-			// Found an image in the graph under the strongID. Validate the
-			// image before using it.
-			if err := p.validateImageInGraph(imgs[i].id, imgs, i); err != nil {
-				continueReuse = false
-				logrus.Debugf("not using existing strongID: %v", err)
-
-				// The strong ID existed in the graph but didn't
-				// validate successfully. We can't use the strong ID
-				// because it didn't validate successfully. Treat the
-				// graph like a hash table with probing... compute
-				// SHA256(id) until we find an ID that either doesn't
-				// already exist in the graph, or has existing content
-				// that validates successfully.
-				for {
-					if err := p.tryNextID(imgs, i, idMap); err != nil {
-						logrus.Debug(err.Error())
-					} else {
-						break
-					}
-				}
-			}
-			continue
-		}
-
-		if continueReuse {
-			compatibilityID := imgs[i].compatibilityID
-			if err := p.validateImageInGraph(compatibilityID, imgs, i); err != nil {
-				logrus.Debugf("stopping ID reuse: %v", err)
-				continueReuse = false
-			} else {
-				// The compatibility ID exists in the graph and was
-				// validated. Use it.
-				imgs[i].id = compatibilityID
-			}
-		}
-	}
-
-	// fix up the parents of the images
-	for i := 0; i < len(imgs); i++ {
-		if i == len(imgs)-1 { // Base layer
-			imgs[i].parent = ""
-		} else {
-			imgs[i].parent = imgs[i+1].id
-		}
-	}
-}
-
-// validateImageInGraph checks that an image in the graph has the expected
-// strongID. id is the entry in the graph to check, imgs is the slice of
-// images being processed (for access to the parent), and i is the index
-// into this slice which the graph entry should be checked against.
-func (p *v2Puller) validateImageInGraph(id string, imgs []contentAddressableDescriptor, i int) error {
-	img, err := p.graph.Get(id)
-	if err != nil {
-		return fmt.Errorf("missing: %v", err)
-	}
-	if runtime.GOOS == "windows" {
-		// TODO - (swernli:11-16-2015) Skipping content addressable IDs on
-		// Windows as a hack for TP4 compat. The correct fix is to ensure that
-		// Windows layers do not have anything in them that takes a dependency
-		// on the ID of the layer in the management client. This will be fixed
-		// in Windows post-TP4.
-		return nil
-	}
-	layerID, err := p.graph.getLayerDigest(id)
-	if err != nil {
-		return fmt.Errorf("digest: %v", err)
-	}
-	var parentID digest.Digest
-	if i != len(imgs)-1 {
-		if img.Parent != imgs[i+1].id { // comparing that graph points to validated ID
-			return fmt.Errorf("parent: %v %v", img.Parent, imgs[i+1].id)
-		}
-		parentID = imgs[i+1].strongID
-	} else if img.Parent != "" {
-		return fmt.Errorf("unexpected parent: %v", img.Parent)
-	}
-
-	v1Config, err := p.graph.getV1CompatibilityConfig(img.ID)
-	if err != nil {
-		return fmt.Errorf("v1Compatibility: %v %v", img.ID, err)
-	}
-
-	json, err := image.MakeImageConfig(v1Config, layerID, parentID)
-	if err != nil {
-		return fmt.Errorf("make config: %v", err)
-	}
-
-	if dgst, err := image.StrongID(json); err == nil && dgst == imgs[i].strongID {
-		logrus.Debugf("Validated %v as %v", dgst, id)
-	} else {
-		return fmt.Errorf("digest mismatch: %v %v, error: %v", dgst, imgs[i].strongID, err)
-	}
-
-	// All clear
-	return nil
-}
-
-func (p *v2Puller) tryNextID(imgs []contentAddressableDescriptor, i int, idMap map[string]struct{}) error {
-	nextID, _ := digest.FromBytes([]byte(imgs[i].id))
-	imgs[i].id = nextID.Hex()
-
-	if _, exists := idMap[imgs[i].id]; !exists {
-		p.graph.imageMutex.Lock(imgs[i].id)
-		defer p.graph.imageMutex.Unlock(imgs[i].id)
-	}
-
-	if p.graph.Exists(imgs[i].id) {
-		if err := p.validateImageInGraph(imgs[i].id, imgs, i); err != nil {
-			return fmt.Errorf("not using existing strongID permutation %s: %v", imgs[i].id, err)
-		}
-	}
-	return nil
-}

graph/push.go

@@ -1,126 +0,0 @@
-package graph
-
-import (
-	"fmt"
-	"io"
-
-	"github.com/Sirupsen/logrus"
-	"github.com/docker/distribution/digest"
-	"github.com/docker/docker/cliconfig"
-	"github.com/docker/docker/pkg/streamformatter"
-	"github.com/docker/docker/registry"
-)
-
-// ImagePushConfig stores push configuration.
-type ImagePushConfig struct {
-	// MetaHeaders store HTTP headers with metadata about the image
-	// (DockerHeaders with prefix X-Meta- in the request).
-	MetaHeaders map[string][]string
-	// AuthConfig holds authentication credentials for authenticating with
-	// the registry.
-	AuthConfig *cliconfig.AuthConfig
-	// Tag is the specific variant of the image to be pushed.
-	// If no tag is provided, all tags will be pushed.
-	Tag string
-	// OutStream is the output writer for showing the status of the push
-	// operation.
-	OutStream io.Writer
-}
-
-// pusher is an interface that abstracts pushing for different API versions.
-type pusher interface {
-	// Push tries to push the image configured at the creation of Pusher.
-	// Push returns an error if any, as well as a boolean that determines whether to retry Push on the next configured endpoint.
-	//
-	// TODO(tiborvass): have Push() take a reference to repository + tag, so that the pusher itself is repository-agnostic.
-	Push() (fallback bool, err error)
-}
-
-// newPusher creates a new Pusher interface that will push to either a v1 or v2
-// registry. The endpoint argument contains a Version field that determines
-// whether a v1 or v2 pusher will be created. The other parameters are passed
-// through to the underlying pusher implementation for use during the actual
-// push operation.
-func (s *TagStore) newPusher(endpoint registry.APIEndpoint, localRepo repository, repoInfo *registry.RepositoryInfo, imagePushConfig *ImagePushConfig, sf *streamformatter.StreamFormatter) (pusher, error) {
-	switch endpoint.Version {
-	case registry.APIVersion2:
-		return &v2Pusher{
-			TagStore:     s,
-			endpoint:     endpoint,
-			localRepo:    localRepo,
-			repoInfo:     repoInfo,
-			config:       imagePushConfig,
-			sf:           sf,
-			layersPushed: make(map[digest.Digest]bool),
-		}, nil
-	case registry.APIVersion1:
-		return &v1Pusher{
-			TagStore:  s,
-			endpoint:  endpoint,
-			localRepo: localRepo,
-			repoInfo:  repoInfo,
-			config:    imagePushConfig,
-			sf:        sf,
-		}, nil
-	}
-	return nil, fmt.Errorf("unknown version %d for registry %s", endpoint.Version, endpoint.URL)
-}
-
-// Push initiates a push operation on the repository named localName.
-func (s *TagStore) Push(localName string, imagePushConfig *ImagePushConfig) error {
-	// FIXME: Allow to interrupt current push when new push of same image is done.
-
-	var sf = streamformatter.NewJSONStreamFormatter()
-
-	// Resolve the Repository name from fqn to RepositoryInfo
-	repoInfo, err := s.registryService.ResolveRepository(localName)
-	if err != nil {
-		return err
-	}
-
-	endpoints, err := s.registryService.LookupPushEndpoints(repoInfo.CanonicalName)
-	if err != nil {
-		return err
-	}
-
-	reposLen := 1
-	if imagePushConfig.Tag == "" {
-		reposLen = len(s.Repositories[repoInfo.LocalName])
-	}
-
-	imagePushConfig.OutStream.Write(sf.FormatStatus("", "The push refers to a repository [%s] (len: %d)", repoInfo.CanonicalName, reposLen))
-
-	// If it fails, try to get the repository
-	localRepo, exists := s.Repositories[repoInfo.LocalName]
-	if !exists {
-		return fmt.Errorf("Repository does not exist: %s", repoInfo.LocalName)
-	}
-
-	var lastErr error
-	for _, endpoint := range endpoints {
-		logrus.Debugf("Trying to push %s to %s %s", repoInfo.CanonicalName, endpoint.URL, endpoint.Version)
-
-		pusher, err := s.newPusher(endpoint, localRepo, repoInfo, imagePushConfig, sf)
-		if err != nil {
-			lastErr = err
-			continue
-		}
-		if fallback, err := pusher.Push(); err != nil {
-			if fallback {
-				lastErr = err
-				continue
-			}
-			logrus.Debugf("Not continuing with error: %v", err)
-			return err
-
-		}
-
-		s.eventsService.Log("push", repoInfo.LocalName, "")
-		return nil
-	}
-
-	if lastErr == nil {
-		lastErr = fmt.Errorf("no endpoints found for %s", repoInfo.CanonicalName)
-	}
-	return lastErr
-}

graph/push_v1.go

@@ -1,354 +0,0 @@
-package graph
-
-import (
-	"fmt"
-	"io"
-	"os"
-	"sync"
-
-	"github.com/Sirupsen/logrus"
-	"github.com/docker/distribution/registry/client/transport"
-	"github.com/docker/docker/image"
-	"github.com/docker/docker/pkg/ioutils"
-	"github.com/docker/docker/pkg/progressreader"
-	"github.com/docker/docker/pkg/streamformatter"
-	"github.com/docker/docker/pkg/stringid"
-	"github.com/docker/docker/registry"
-	"github.com/docker/docker/utils"
-)
-
-type v1Pusher struct {
-	*TagStore
-	endpoint  registry.APIEndpoint
-	localRepo repository
-	repoInfo  *registry.RepositoryInfo
-	config    *ImagePushConfig
-	sf        *streamformatter.StreamFormatter
-	session   *registry.Session
-
-	out io.Writer
-}
-
-func (p *v1Pusher) Push() (fallback bool, err error) {
-	tlsConfig, err := p.registryService.TLSConfig(p.repoInfo.Index.Name)
-	if err != nil {
-		return false, err
-	}
-	// Adds Docker-specific headers as well as user-specified headers (metaHeaders)
-	tr := transport.NewTransport(
-		// TODO(tiborvass): was NoTimeout
-		registry.NewTransport(tlsConfig),
-		registry.DockerHeaders(p.config.MetaHeaders)...,
-	)
-	client := registry.HTTPClient(tr)
-	v1Endpoint, err := p.endpoint.ToV1Endpoint(p.config.MetaHeaders)
-	if err != nil {
-		logrus.Debugf("Could not get v1 endpoint: %v", err)
-		return true, err
-	}
-	p.session, err = registry.NewSession(client, p.config.AuthConfig, v1Endpoint)
-	if err != nil {
-		// TODO(dmcgowan): Check if should fallback
-		return true, err
-	}
-	if err := p.pushRepository(p.config.Tag); err != nil {
-		// TODO(dmcgowan): Check if should fallback
-		return false, err
-	}
-	return false, nil
-}
-
-// Retrieve the all the images to be uploaded in the correct order
-func (p *v1Pusher) getImageList(requestedTag string) ([]string, map[string][]string, error) {
-	var (
-		imageList   []string
-		imagesSeen  = make(map[string]bool)
-		tagsByImage = make(map[string][]string)
-	)
-
-	for tag, id := range p.localRepo {
-		if requestedTag != "" && requestedTag != tag {
-			// Include only the requested tag.
-			continue
-		}
-
-		if utils.DigestReference(tag) {
-			// Ignore digest references.
-			continue
-		}
-
-		var imageListForThisTag []string
-
-		tagsByImage[id] = append(tagsByImage[id], tag)
-
-		for img, err := p.graph.Get(id); img != nil; img, err = p.graph.GetParent(img) {
-			if err != nil {
-				return nil, nil, err
-			}
-
-			if imagesSeen[img.ID] {
-				// This image is already on the list, we can ignore it and all its parents
-				break
-			}
-
-			imagesSeen[img.ID] = true
-			imageListForThisTag = append(imageListForThisTag, img.ID)
-		}
-
-		// reverse the image list for this tag (so the "most"-parent image is first)
-		for i, j := 0, len(imageListForThisTag)-1; i < j; i, j = i+1, j-1 {
-			imageListForThisTag[i], imageListForThisTag[j] = imageListForThisTag[j], imageListForThisTag[i]
-		}
-
-		// append to main image list
-		imageList = append(imageList, imageListForThisTag...)
-	}
-	if len(imageList) == 0 {
-		return nil, nil, fmt.Errorf("No images found for the requested repository / tag")
-	}
-	logrus.Debugf("Image list: %v", imageList)
-	logrus.Debugf("Tags by image: %v", tagsByImage)
-
-	return imageList, tagsByImage, nil
-}
-
-// createImageIndex returns an index of an image's layer IDs and tags.
-func (s *TagStore) createImageIndex(images []string, tags map[string][]string) []*registry.ImgData {
-	var imageIndex []*registry.ImgData
-	for _, id := range images {
-		if tags, hasTags := tags[id]; hasTags {
-			// If an image has tags you must add an entry in the image index
-			// for each tag
-			for _, tag := range tags {
-				imageIndex = append(imageIndex, &registry.ImgData{
-					ID:  id,
-					Tag: tag,
-				})
-			}
-			continue
-		}
-		// If the image does not have a tag it still needs to be sent to the
-		// registry with an empty tag so that it is associated with the repository
-		imageIndex = append(imageIndex, &registry.ImgData{
-			ID:  id,
-			Tag: "",
-		})
-	}
-	return imageIndex
-}
-
-type imagePushData struct {
-	id              string
-	compatibilityID string
-	endpoint        string
-}
-
-// lookupImageOnEndpoint checks the specified endpoint to see if an image exists
-// and if it is absent then it sends the image id to the channel to be pushed.
-func (p *v1Pusher) lookupImageOnEndpoint(wg *sync.WaitGroup, images chan imagePushData, imagesToPush chan string) {
-	defer wg.Done()
-	for image := range images {
-		if err := p.session.LookupRemoteImage(image.compatibilityID, image.endpoint); err != nil {
-			logrus.Errorf("Error in LookupRemoteImage: %s", err)
-			imagesToPush <- image.id
-			continue
-		}
-		p.out.Write(p.sf.FormatStatus("", "Image %s already pushed, skipping", stringid.TruncateID(image.id)))
-	}
-}
-
-func (p *v1Pusher) pushImageToEndpoint(endpoint string, imageIDs []string, tags map[string][]string, repo *registry.RepositoryData) error {
-	workerCount := len(imageIDs)
-	// start a maximum of 5 workers to check if images exist on the specified endpoint.
-	if workerCount > 5 {
-		workerCount = 5
-	}
-	var (
-		wg           = &sync.WaitGroup{}
-		imageData    = make(chan imagePushData, workerCount*2)
-		imagesToPush = make(chan string, workerCount*2)
-		pushes       = make(chan map[string]struct{}, 1)
-	)
-	for i := 0; i < workerCount; i++ {
-		wg.Add(1)
-		go p.lookupImageOnEndpoint(wg, imageData, imagesToPush)
-	}
-	// start a go routine that consumes the images to push
-	go func() {
-		shouldPush := make(map[string]struct{})
-		for id := range imagesToPush {
-			shouldPush[id] = struct{}{}
-		}
-		pushes <- shouldPush
-	}()
-	for _, id := range imageIDs {
-		compatibilityID, err := p.getV1ID(id)
-		if err != nil {
-			return err
-		}
-		imageData <- imagePushData{
-			id:              id,
-			compatibilityID: compatibilityID,
-			endpoint:        endpoint,
-		}
-	}
-	// close the channel to notify the workers that there will be no more images to check.
-	close(imageData)
-	wg.Wait()
-	close(imagesToPush)
-	// wait for all the images that require pushes to be collected into a consumable map.
-	shouldPush := <-pushes
-	// finish by pushing any images and tags to the endpoint.  The order that the images are pushed
-	// is very important that is why we are still iterating over the ordered list of imageIDs.
-	for _, id := range imageIDs {
-		if _, push := shouldPush[id]; push {
-			if _, err := p.pushImage(id, endpoint); err != nil {
-				// FIXME: Continue on error?
-				return err
-			}
-		}
-		for _, tag := range tags[id] {
-			p.out.Write(p.sf.FormatStatus("", "Pushing tag for rev [%s] on {%s}", stringid.TruncateID(id), endpoint+"repositories/"+p.repoInfo.RemoteName+"/tags/"+tag))
-			compatibilityID, err := p.getV1ID(id)
-			if err != nil {
-				return err
-			}
-			if err := p.session.PushRegistryTag(p.repoInfo.RemoteName, compatibilityID, tag, endpoint); err != nil {
-				return err
-			}
-		}
-	}
-	return nil
-}
-
-// pushRepository pushes layers that do not already exist on the registry.
-func (p *v1Pusher) pushRepository(tag string) error {
-	logrus.Debugf("Local repo: %s", p.localRepo)
-	p.out = ioutils.NewWriteFlusher(p.config.OutStream)
-	imgList, tags, err := p.getImageList(tag)
-	if err != nil {
-		return err
-	}
-	p.out.Write(p.sf.FormatStatus("", "Sending image list"))
-
-	imageIndex := p.createImageIndex(imgList, tags)
-	logrus.Debugf("Preparing to push %s with the following images and tags", p.localRepo)
-	for _, data := range imageIndex {
-		logrus.Debugf("Pushing ID: %s with Tag: %s", data.ID, data.Tag)
-
-		// convert IDs to compatibilityIDs, imageIndex only used in registry calls
-		data.ID, err = p.getV1ID(data.ID)
-		if err != nil {
-			return err
-		}
-	}
-
-	if _, found := p.poolAdd("push", p.repoInfo.LocalName); found {
-		return fmt.Errorf("push or pull %s is already in progress", p.repoInfo.LocalName)
-	}
-	defer p.poolRemove("push", p.repoInfo.LocalName)
-
-	// Register all the images in a repository with the registry
-	// If an image is not in this list it will not be associated with the repository
-	repoData, err := p.session.PushImageJSONIndex(p.repoInfo.RemoteName, imageIndex, false, nil)
-	if err != nil {
-		return err
-	}
-	nTag := 1
-	if tag == "" {
-		nTag = len(p.localRepo)
-	}
-	p.out.Write(p.sf.FormatStatus("", "Pushing repository %s (%d tags)", p.repoInfo.CanonicalName, nTag))
-	// push the repository to each of the endpoints only if it does not exist.
-	for _, endpoint := range repoData.Endpoints {
-		if err := p.pushImageToEndpoint(endpoint, imgList, tags, repoData); err != nil {
-			return err
-		}
-	}
-	_, err = p.session.PushImageJSONIndex(p.repoInfo.RemoteName, imageIndex, true, repoData.Endpoints)
-	return err
-}
-
-func (p *v1Pusher) pushImage(imgID, ep string) (checksum string, err error) {
-	jsonRaw, err := p.getV1Config(imgID)
-	if err != nil {
-		return "", fmt.Errorf("Cannot retrieve the path for {%s}: %s", imgID, err)
-	}
-	p.out.Write(p.sf.FormatProgress(stringid.TruncateID(imgID), "Pushing", nil))
-
-	compatibilityID, err := p.getV1ID(imgID)
-	if err != nil {
-		return "", err
-	}
-
-	// General rule is to use ID for graph accesses and compatibilityID for
-	// calls to session.registry()
-	imgData := &registry.ImgData{
-		ID: compatibilityID,
-	}
-
-	// Send the json
-	if err := p.session.PushImageJSONRegistry(imgData, jsonRaw, ep); err != nil {
-		if err == registry.ErrAlreadyExists {
-			p.out.Write(p.sf.FormatProgress(stringid.TruncateID(imgID), "Image already pushed, skipping", nil))
-			return "", nil
-		}
-		return "", err
-	}
-
-	layerData, err := p.graph.tempLayerArchive(imgID, p.sf, p.out)
-	if err != nil {
-		return "", fmt.Errorf("Failed to generate layer archive: %s", err)
-	}
-	defer os.RemoveAll(layerData.Name())
-
-	// Send the layer
-	logrus.Debugf("rendered layer for %s of [%d] size", imgID, layerData.Size)
-
-	checksum, checksumPayload, err := p.session.PushImageLayerRegistry(imgData.ID,
-		progressreader.New(progressreader.Config{
-			In:        layerData,
-			Out:       p.out,
-			Formatter: p.sf,
-			Size:      layerData.Size,
-			NewLines:  false,
-			ID:        stringid.TruncateID(imgID),
-			Action:    "Pushing",
-		}), ep, jsonRaw)
-	if err != nil {
-		return "", err
-	}
-	imgData.Checksum = checksum
-	imgData.ChecksumPayload = checksumPayload
-	// Send the checksum
-	if err := p.session.PushImageChecksumRegistry(imgData, ep); err != nil {
-		return "", err
-	}
-
-	p.out.Write(p.sf.FormatProgress(stringid.TruncateID(imgID), "Image successfully pushed", nil))
-	return imgData.Checksum, nil
-}
-
-// getV1ID returns the compatibilityID for the ID in the graph. compatibilityID
-// is read from from the v1Compatibility config file in the disk.
-func (p *v1Pusher) getV1ID(id string) (string, error) {
-	jsonData, err := p.getV1Config(id)
-	if err != nil {
-		return "", err
-	}
-	img, err := image.NewImgJSON(jsonData)
-	if err != nil {
-		return "", err
-	}
-	return img.ID, nil
-}
-
-// getV1Config returns v1Compatibility config for the image in the graph. If
-// there is no v1Compatibility file on disk for the image
-func (p *v1Pusher) getV1Config(id string) ([]byte, error) {
-	jsonData, err := p.graph.generateV1CompatibilityChain(id)
-	if err != nil {
-		return nil, err
-	}
-	return jsonData, nil
-}

graph/push_v2.go

@@ -1,397 +0,0 @@
-package graph
-
-import (
-	"bufio"
-	"compress/gzip"
-	"encoding/json"
-	"errors"
-	"fmt"
-	"io"
-	"io/ioutil"
-
-	"github.com/Sirupsen/logrus"
-	"github.com/docker/distribution"
-	"github.com/docker/distribution/digest"
-	"github.com/docker/distribution/manifest"
-	"github.com/docker/distribution/manifest/schema1"
-	"github.com/docker/docker/image"
-	"github.com/docker/docker/pkg/progressreader"
-	"github.com/docker/docker/pkg/streamformatter"
-	"github.com/docker/docker/pkg/stringid"
-	"github.com/docker/docker/registry"
-	"github.com/docker/docker/runconfig"
-	"github.com/docker/docker/utils"
-	"golang.org/x/net/context"
-)
-
-const compressionBufSize = 32768
-
-type v2Pusher struct {
-	*TagStore
-	endpoint  registry.APIEndpoint
-	localRepo repository
-	repoInfo  *registry.RepositoryInfo
-	config    *ImagePushConfig
-	sf        *streamformatter.StreamFormatter
-	repo      distribution.Repository
-
-	// layersPushed is the set of layers known to exist on the remote side.
-	// This avoids redundant queries when pushing multiple tags that
-	// involve the same layers.
-	layersPushed map[digest.Digest]bool
-}
-
-func (p *v2Pusher) Push() (fallback bool, err error) {
-	p.repo, err = newV2Repository(p.repoInfo, p.endpoint, p.config.MetaHeaders, p.config.AuthConfig, "push", "pull")
-	if err != nil {
-		logrus.Debugf("Error getting v2 registry: %v", err)
-		return true, err
-	}
-	return false, p.pushV2Repository(p.config.Tag)
-}
-
-func (p *v2Pusher) getImageTags(askedTag string) ([]string, error) {
-	logrus.Debugf("Checking %q against %#v", askedTag, p.localRepo)
-	if len(askedTag) > 0 {
-		if _, ok := p.localRepo[askedTag]; !ok || utils.DigestReference(askedTag) {
-			return nil, fmt.Errorf("Tag does not exist for %s", askedTag)
-		}
-		return []string{askedTag}, nil
-	}
-	var tags []string
-	for tag := range p.localRepo {
-		if !utils.DigestReference(tag) {
-			tags = append(tags, tag)
-		}
-	}
-	return tags, nil
-}
-
-func (p *v2Pusher) pushV2Repository(tag string) error {
-	localName := p.repoInfo.LocalName
-	if _, found := p.poolAdd("push", localName); found {
-		return fmt.Errorf("push or pull %s is already in progress", localName)
-	}
-	defer p.poolRemove("push", localName)
-
-	tags, err := p.getImageTags(tag)
-	if err != nil {
-		return fmt.Errorf("error getting tags for %s: %s", localName, err)
-	}
-	if len(tags) == 0 {
-		return fmt.Errorf("no tags to push for %s", localName)
-	}
-
-	for _, tag := range tags {
-		if err := p.pushV2Tag(tag); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func (p *v2Pusher) pushV2Tag(tag string) error {
-	logrus.Debugf("Pushing repository: %s:%s", p.repo.Name(), tag)
-
-	layerID, exists := p.localRepo[tag]
-	if !exists {
-		return fmt.Errorf("tag does not exist: %s", tag)
-	}
-
-	layersSeen := make(map[string]bool)
-
-	layer, err := p.graph.Get(layerID)
-	if err != nil {
-		return err
-	}
-
-	m := &schema1.Manifest{
-		Versioned: manifest.Versioned{
-			SchemaVersion: 1,
-		},
-		Name:         p.repo.Name(),
-		Tag:          tag,
-		Architecture: layer.Architecture,
-		FSLayers:     []schema1.FSLayer{},
-		History:      []schema1.History{},
-	}
-
-	var metadata runconfig.Config
-	if layer != nil && layer.Config != nil {
-		metadata = *layer.Config
-	}
-
-	out := p.config.OutStream
-
-	for ; layer != nil; layer, err = p.graph.GetParent(layer) {
-		if err != nil {
-			return err
-		}
-
-		// break early if layer has already been seen in this image,
-		// this prevents infinite loops on layers which loopback, this
-		// cannot be prevented since layer IDs are not merkle hashes
-		// TODO(dmcgowan): throw error if no valid use case is found
-		if layersSeen[layer.ID] {
-			break
-		}
-
-		// Skip the base layer on Windows. This cannot be pushed.
-		if allowBaseParentImage && layer.Parent == "" {
-			break
-		}
-
-		logrus.Debugf("Pushing layer: %s", layer.ID)
-
-		if layer.Config != nil && metadata.Image != layer.ID {
-			if err := runconfig.Merge(&metadata, layer.Config); err != nil {
-				return err
-			}
-		}
-
-		var exists bool
-		dgst, err := p.graph.getLayerDigestWithLock(layer.ID)
-		switch err {
-		case nil:
-			if p.layersPushed[dgst] {
-				exists = true
-				// break out of switch, it is already known that
-				// the push is not needed and therefore doing a
-				// stat is unnecessary
-				break
-			}
-			_, err := p.repo.Blobs(context.Background()).Stat(context.Background(), dgst)
-			switch err {
-			case nil:
-				exists = true
-				out.Write(p.sf.FormatProgress(stringid.TruncateID(layer.ID), "Image already exists", nil))
-			case distribution.ErrBlobUnknown:
-				// nop
-			default:
-				out.Write(p.sf.FormatProgress(stringid.TruncateID(layer.ID), "Image push failed", nil))
-				return err
-			}
-		case errDigestNotSet:
-			// nop
-		case digest.ErrDigestInvalidFormat, digest.ErrDigestUnsupported:
-			return fmt.Errorf("error getting image checksum: %v", err)
-		}
-
-		// if digest was empty or not saved, or if blob does not exist on the remote repository,
-		// then fetch it.
-		if !exists {
-			var pushDigest digest.Digest
-			if pushDigest, err = p.pushV2Image(p.repo.Blobs(context.Background()), layer); err != nil {
-				return err
-			}
-			if dgst == "" {
-				// Cache new checksum
-				if err := p.graph.setLayerDigestWithLock(layer.ID, pushDigest); err != nil {
-					return err
-				}
-			}
-			dgst = pushDigest
-		}
-
-		// read v1Compatibility config, generate new if needed
-		jsonData, err := p.graph.generateV1CompatibilityChain(layer.ID)
-		if err != nil {
-			return err
-		}
-
-		m.FSLayers = append(m.FSLayers, schema1.FSLayer{BlobSum: dgst})
-		m.History = append(m.History, schema1.History{V1Compatibility: string(jsonData)})
-
-		layersSeen[layer.ID] = true
-		p.layersPushed[dgst] = true
-	}
-
-	// Fix parent chain if necessary
-	if err = fixHistory(m); err != nil {
-		return err
-	}
-
-	logrus.Infof("Signed manifest for %s:%s using daemon's key: %s", p.repo.Name(), tag, p.trustKey.KeyID())
-	signed, err := schema1.Sign(m, p.trustKey)
-	if err != nil {
-		return err
-	}
-
-	manifestDigest, manifestSize, err := digestFromManifest(signed, p.repo.Name())
-	if err != nil {
-		return err
-	}
-	if manifestDigest != "" {
-		out.Write(p.sf.FormatStatus("", "%s: digest: %s size: %d", tag, manifestDigest, manifestSize))
-	}
-
-	manSvc, err := p.repo.Manifests(context.Background())
-	if err != nil {
-		return err
-	}
-	return manSvc.Put(signed)
-}
-
-// fixHistory makes sure that the manifest has parent IDs that are consistent
-// with its image IDs. Because local image IDs are generated from the
-// configuration and filesystem contents, but IDs in the manifest are preserved
-// from the original pull, it's possible to have inconsistencies where parent
-// IDs don't match up with the other IDs in the manifest. This happens in the
-// case where an engine pulls images where are identical except the IDs from the
-// manifest - the local ID will be the same, and one of the v1Compatibility
-// files gets discarded.
-func fixHistory(m *schema1.Manifest) error {
-	var lastID string
-
-	for i := len(m.History) - 1; i >= 0; i-- {
-		var historyEntry map[string]*json.RawMessage
-		if err := json.Unmarshal([]byte(m.History[i].V1Compatibility), &historyEntry); err != nil {
-			return err
-		}
-
-		idJSON, present := historyEntry["id"]
-		if !present || idJSON == nil {
-			return errors.New("missing id key in v1compatibility file")
-		}
-		var id string
-		if err := json.Unmarshal(*idJSON, &id); err != nil {
-			return err
-		}
-
-		parentJSON, present := historyEntry["parent"]
-
-		if i == len(m.History)-1 {
-			// The base layer must not reference a parent layer,
-			// otherwise the manifest is incomplete. There is an
-			// exception for Windows to handle base layers.
-			if !allowBaseParentImage && present && parentJSON != nil {
-				var parent string
-				if err := json.Unmarshal(*parentJSON, &parent); err != nil {
-					return err
-				}
-				if parent != "" {
-					logrus.Debugf("parent id mismatch detected; fixing. parent reference: %s", parent)
-					delete(historyEntry, "parent")
-					fixedHistory, err := json.Marshal(historyEntry)
-					if err != nil {
-						return err
-					}
-					m.History[i].V1Compatibility = string(fixedHistory)
-				}
-			}
-		} else {
-			// For all other layers, the parent ID should equal the
-			// ID of the next item in the history list. If it
-			// doesn't, fix it up (but preserve all other fields,
-			// possibly including fields that aren't known to this
-			// engine version).
-			if !present || parentJSON == nil {
-				return errors.New("missing parent key in v1compatibility file")
-			}
-			var parent string
-			if err := json.Unmarshal(*parentJSON, &parent); err != nil {
-				return err
-			}
-			if parent != lastID {
-				logrus.Debugf("parent id mismatch detected; fixing. parent reference: %s actual id: %s", parent, id)
-				historyEntry["parent"] = rawJSON(lastID)
-				fixedHistory, err := json.Marshal(historyEntry)
-				if err != nil {
-					return err
-				}
-				m.History[i].V1Compatibility = string(fixedHistory)
-			}
-		}
-		lastID = id
-	}
-
-	return nil
-}
-
-func rawJSON(value interface{}) *json.RawMessage {
-	jsonval, err := json.Marshal(value)
-	if err != nil {
-		return nil
-	}
-	return (*json.RawMessage)(&jsonval)
-}
-
-func (p *v2Pusher) pushV2Image(bs distribution.BlobService, img *image.Image) (digest.Digest, error) {
-	out := p.config.OutStream
-
-	out.Write(p.sf.FormatProgress(stringid.TruncateID(img.ID), "Preparing", nil))
-
-	image, err := p.graph.Get(img.ID)
-	if err != nil {
-		return "", err
-	}
-	arch, err := p.graph.tarLayer(image)
-	if err != nil {
-		return "", err
-	}
-	defer arch.Close()
-
-	// Send the layer
-	layerUpload, err := bs.Create(context.Background())
-	if err != nil {
-		return "", err
-	}
-	defer layerUpload.Close()
-
-	reader := progressreader.New(progressreader.Config{
-		In:        ioutil.NopCloser(arch), // we'll take care of close here.
-		Out:       out,
-		Formatter: p.sf,
-
-		// TODO(stevvooe): This may cause a size reporting error. Try to get
-		// this from tar-split or elsewhere. The main issue here is that we
-		// don't want to buffer to disk *just* to calculate the size.
-		Size: img.Size,
-
-		NewLines: false,
-		ID:       stringid.TruncateID(img.ID),
-		Action:   "Pushing",
-	})
-
-	digester := digest.Canonical.New()
-	// HACK: The MultiWriter doesn't write directly to layerUpload because
-	// we must make sure the ReadFrom is used, not Write. Using Write would
-	// send a PATCH request for every Write call.
-	pipeReader, pipeWriter := io.Pipe()
-	// Use a bufio.Writer to avoid excessive chunking in HTTP request.
-	bufWriter := bufio.NewWriterSize(io.MultiWriter(pipeWriter, digester.Hash()), compressionBufSize)
-	compressor := gzip.NewWriter(bufWriter)
-
-	go func() {
-		_, err := io.Copy(compressor, reader)
-		if err == nil {
-			err = compressor.Close()
-		}
-		if err == nil {
-			err = bufWriter.Flush()
-		}
-		if err != nil {
-			pipeWriter.CloseWithError(err)
-		} else {
-			pipeWriter.Close()
-		}
-	}()
-
-	out.Write(p.sf.FormatProgress(stringid.TruncateID(img.ID), "Pushing", nil))
-	nn, err := layerUpload.ReadFrom(pipeReader)
-	pipeReader.Close()
-	if err != nil {
-		return "", err
-	}
-
-	dgst := digester.Digest()
-	if _, err := layerUpload.Commit(context.Background(), distribution.Descriptor{Digest: dgst}); err != nil {
-		return "", err
-	}
-
-	logrus.Debugf("uploaded layer %s (%s), %d bytes", img.ID, dgst, nn)
-	out.Write(p.sf.FormatProgress(stringid.TruncateID(img.ID), "Pushed", nil))
-
-	return dgst, nil
-}

graph/service.go

@@ -1,89 +0,0 @@
-package graph
-
-import (
-	"fmt"
-	"io"
-	"runtime"
-	"time"
-
-	"github.com/Sirupsen/logrus"
-	"github.com/docker/docker/api/types"
-	"github.com/docker/docker/utils"
-)
-
-// Lookup looks up an image by name in a TagStore and returns it as an
-// ImageInspect structure.
-func (s *TagStore) Lookup(name string) (*types.ImageInspect, error) {
-	image, err := s.LookupImage(name)
-	if err != nil || image == nil {
-		return nil, fmt.Errorf("No such image: %s", name)
-	}
-
-	var repoTags = make([]string, 0)
-	var repoDigests = make([]string, 0)
-
-	s.Lock()
-	for repoName, repository := range s.Repositories {
-		for ref, id := range repository {
-			if id == image.ID {
-				imgRef := utils.ImageReference(repoName, ref)
-				if utils.DigestReference(ref) {
-					repoDigests = append(repoDigests, imgRef)
-				} else {
-					repoTags = append(repoTags, imgRef)
-				}
-			}
-		}
-	}
-	s.Unlock()
-
-	imageInspect := &types.ImageInspect{
-		ID:              image.ID,
-		RepoTags:        repoTags,
-		RepoDigests:     repoDigests,
-		Parent:          image.Parent,
-		Comment:         image.Comment,
-		Created:         image.Created.Format(time.RFC3339Nano),
-		Container:       image.Container,
-		ContainerConfig: &image.ContainerConfig,
-		DockerVersion:   image.DockerVersion,
-		Author:          image.Author,
-		Config:          image.Config,
-		Architecture:    image.Architecture,
-		Os:              image.OS,
-		Size:            image.Size,
-		VirtualSize:     s.graph.getParentsSize(image) + image.Size,
-	}
-
-	imageInspect.GraphDriver.Name = s.graph.driver.String()
-
-	graphDriverData, err := s.graph.driver.GetMetadata(image.ID)
-	if err != nil {
-		return nil, err
-	}
-	imageInspect.GraphDriver.Data = graphDriverData
-	return imageInspect, nil
-}
-
-// imageTarLayer return the tarLayer of the image
-func (s *TagStore) imageTarLayer(name string, dest io.Writer) error {
-	if image, err := s.LookupImage(name); err == nil && image != nil {
-		// On Windows, the base layer cannot be exported
-		if runtime.GOOS != "windows" || image.Parent != "" {
-
-			fs, err := s.graph.tarLayer(image)
-			if err != nil {
-				return err
-			}
-			defer fs.Close()
-
-			written, err := io.Copy(dest, fs)
-			if err != nil {
-				return err
-			}
-			logrus.Debugf("rendered layer for %s of [%d] size", image.ID, written)
-		}
-		return nil
-	}
-	return fmt.Errorf("No such image: %s", name)
-}

graph/tags.go

@@ -1,431 +0,0 @@
-package graph
-
-import (
-	"encoding/json"
-	"errors"
-	"fmt"
-	"io"
-	"io/ioutil"
-	"os"
-	"path/filepath"
-	"sort"
-	"strings"
-	"sync"
-
-	"github.com/docker/distribution/digest"
-	"github.com/docker/docker/daemon/events"
-	"github.com/docker/docker/graph/tags"
-	"github.com/docker/docker/image"
-	"github.com/docker/docker/pkg/broadcaster"
-	"github.com/docker/docker/pkg/parsers"
-	"github.com/docker/docker/pkg/stringid"
-	"github.com/docker/docker/registry"
-	"github.com/docker/docker/utils"
-	"github.com/docker/libtrust"
-)
-
-// ErrNameIsNotExist returned when there is no image with requested name.
-var ErrNameIsNotExist = errors.New("image with specified name does not exist")
-
-// TagStore manages repositories. It encompasses the Graph used for versioned
-// storage, as well as various services involved in pushing and pulling
-// repositories.
-type TagStore struct {
-	path  string
-	graph *Graph
-	// Repositories is a map of repositories, indexed by name.
-	Repositories map[string]repository
-	trustKey     libtrust.PrivateKey
-	sync.Mutex
-	// FIXME: move push/pull-related fields
-	// to a helper type
-	pullingPool     map[string]*broadcaster.Buffered
-	pushingPool     map[string]*broadcaster.Buffered
-	registryService *registry.Service
-	eventsService   *events.Events
-}
-
-// repository maps tags to image IDs.
-type repository map[string]string
-
-// TagStoreConfig provides parameters for a new TagStore.
-type TagStoreConfig struct {
-	// Graph is the versioned image store
-	Graph *Graph
-	// Key is the private key to use for signing manifests.
-	Key libtrust.PrivateKey
-	// Registry is the registry service to use for TLS configuration and
-	// endpoint lookup.
-	Registry *registry.Service
-	// Events is the events service to use for logging.
-	Events *events.Events
-}
-
-// NewTagStore creates a new TagStore at specified path, using the parameters
-// and services provided in cfg.
-func NewTagStore(path string, cfg *TagStoreConfig) (*TagStore, error) {
-	abspath, err := filepath.Abs(path)
-	if err != nil {
-		return nil, err
-	}
-
-	store := &TagStore{
-		path:            abspath,
-		graph:           cfg.Graph,
-		trustKey:        cfg.Key,
-		Repositories:    make(map[string]repository),
-		pullingPool:     make(map[string]*broadcaster.Buffered),
-		pushingPool:     make(map[string]*broadcaster.Buffered),
-		registryService: cfg.Registry,
-		eventsService:   cfg.Events,
-	}
-	// Load the json file if it exists, otherwise create it.
-	if err := store.reload(); os.IsNotExist(err) {
-		if err := store.save(); err != nil {
-			return nil, err
-		}
-	} else if err != nil {
-		return nil, err
-	}
-	return store, nil
-}
-
-func (store *TagStore) save() error {
-	// Store the json ball
-	jsonData, err := json.Marshal(store)
-	if err != nil {
-		return err
-	}
-	if err := ioutil.WriteFile(store.path, jsonData, 0600); err != nil {
-		return err
-	}
-	return nil
-}
-
-func (store *TagStore) reload() error {
-	f, err := os.Open(store.path)
-	if err != nil {
-		return err
-	}
-	defer f.Close()
-	if err := json.NewDecoder(f).Decode(&store); err != nil {
-		return err
-	}
-	return nil
-}
-
-// LookupImage returns pointer to an Image struct corresponding to the given
-// name. The name can include an optional tag; otherwise the default tag will
-// be used.
-func (store *TagStore) LookupImage(name string) (*image.Image, error) {
-	repoName, ref := parsers.ParseRepositoryTag(name)
-	if ref == "" {
-		ref = tags.DefaultTag
-	}
-	var (
-		err error
-		img *image.Image
-	)
-
-	img, err = store.getImage(repoName, ref)
-	if err != nil {
-		return nil, err
-	}
-
-	if img != nil {
-		return img, nil
-	}
-
-	// name must be an image ID.
-	store.Lock()
-	defer store.Unlock()
-	if img, err = store.graph.Get(name); err != nil {
-		return nil, err
-	}
-
-	return img, nil
-}
-
-// GetID returns ID for image name.
-func (store *TagStore) GetID(name string) (string, error) {
-	repoName, ref := parsers.ParseRepositoryTag(name)
-	if ref == "" {
-		ref = tags.DefaultTag
-	}
-	store.Lock()
-	defer store.Unlock()
-	repoName = registry.NormalizeLocalName(repoName)
-	repo, ok := store.Repositories[repoName]
-	if !ok {
-		return "", ErrNameIsNotExist
-	}
-	id, ok := repo[ref]
-	if !ok {
-		return "", ErrNameIsNotExist
-	}
-	return id, nil
-}
-
-// ByID returns a reverse-lookup table of all the names which refer to each
-// image - e.g. {"43b5f19b10584": {"base:latest", "base:v1"}}
-func (store *TagStore) ByID() map[string][]string {
-	store.Lock()
-	defer store.Unlock()
-	byID := make(map[string][]string)
-	for repoName, repository := range store.Repositories {
-		for tag, id := range repository {
-			name := utils.ImageReference(repoName, tag)
-			if _, exists := byID[id]; !exists {
-				byID[id] = []string{name}
-			} else {
-				byID[id] = append(byID[id], name)
-				sort.Strings(byID[id])
-			}
-		}
-	}
-	return byID
-}
-
-// HasReferences returns whether or not the given image is referenced in one or
-// more repositories.
-func (store *TagStore) HasReferences(img *image.Image) bool {
-	return len(store.ByID()[img.ID]) > 0
-}
-
-// Delete deletes a repository or a specific tag. If ref is empty, the entire
-// repository named repoName will be deleted; otherwise only the tag named by
-// ref will be deleted.
-func (store *TagStore) Delete(repoName, ref string) (bool, error) {
-	store.Lock()
-	defer store.Unlock()
-	deleted := false
-	if err := store.reload(); err != nil {
-		return false, err
-	}
-
-	if ref == "" {
-		// Delete the whole repository.
-		delete(store.Repositories, repoName)
-		return true, store.save()
-	}
-
-	repoRefs, exists := store.Repositories[repoName]
-
-	if !exists {
-		return false, fmt.Errorf("No such repository: %s", repoName)
-	}
-
-	if _, exists := repoRefs[ref]; exists {
-		delete(repoRefs, ref)
-		if len(repoRefs) == 0 {
-			delete(store.Repositories, repoName)
-		}
-		deleted = true
-	}
-
-	return deleted, store.save()
-}
-
-// Tag creates a tag in the repository reponame, pointing to the image named
-// imageName. If force is true, an existing tag with the same name may be
-// overwritten.
-func (store *TagStore) Tag(repoName, tag, imageName string, force bool) error {
-	return store.setLoad(repoName, tag, imageName, force, nil)
-}
-
-// setLoad stores the image to the store.
-// If the imageName is already in the repo then a '-f' flag should be used to replace existing image.
-func (store *TagStore) setLoad(repoName, tag, imageName string, force bool, out io.Writer) error {
-	img, err := store.LookupImage(imageName)
-	store.Lock()
-	defer store.Unlock()
-	if err != nil {
-		return err
-	}
-	if tag == "" {
-		tag = tags.DefaultTag
-	}
-	if err := validateRepoName(repoName); err != nil {
-		return err
-	}
-	if err := tags.ValidateTagName(tag); err != nil {
-		return err
-	}
-	if err := store.reload(); err != nil {
-		return err
-	}
-	var repo repository
-	repoName = registry.NormalizeLocalName(repoName)
-	if r, exists := store.Repositories[repoName]; exists {
-		repo = r
-		if old, exists := store.Repositories[repoName][tag]; exists {
-
-			if !force {
-				return fmt.Errorf("Conflict: Tag %s:%s is already set to image %s, if you want to replace it, please use -f option", repoName, tag, old[:12])
-			}
-
-			if old != img.ID && out != nil {
-
-				fmt.Fprintf(out, "The image %s:%s already exists, renaming the old one with ID %s to empty string\n", repoName, tag, old[:12])
-
-			}
-		}
-	} else {
-		repo = make(map[string]string)
-		store.Repositories[repoName] = repo
-	}
-	repo[tag] = img.ID
-	return store.save()
-}
-
-// setDigest creates a digest reference to an image ID.
-func (store *TagStore) setDigest(repoName, digest, imageName string) error {
-	img, err := store.LookupImage(imageName)
-	if err != nil {
-		return err
-	}
-
-	if err := validateRepoName(repoName); err != nil {
-		return err
-	}
-
-	if err := validateDigest(digest); err != nil {
-		return err
-	}
-
-	store.Lock()
-	defer store.Unlock()
-	if err := store.reload(); err != nil {
-		return err
-	}
-
-	repoName = registry.NormalizeLocalName(repoName)
-	repoRefs, exists := store.Repositories[repoName]
-	if !exists {
-		repoRefs = repository{}
-		store.Repositories[repoName] = repoRefs
-	} else if oldID, exists := repoRefs[digest]; exists && oldID != img.ID {
-		return fmt.Errorf("Conflict: Digest %s is already set to image %s", digest, oldID)
-	}
-
-	repoRefs[digest] = img.ID
-	return store.save()
-}
-
-// get returns the repository tag/image map for a given repository.
-func (store *TagStore) get(repoName string) (repository, error) {
-	store.Lock()
-	defer store.Unlock()
-	if err := store.reload(); err != nil {
-		return nil, err
-	}
-	repoName = registry.NormalizeLocalName(repoName)
-	if r, exists := store.Repositories[repoName]; exists {
-		return r, nil
-	}
-	return nil, nil
-}
-
-// getImage returns a pointer to an Image structure describing the image
-// referred to by refOrID inside repository repoName.
-func (store *TagStore) getImage(repoName, refOrID string) (*image.Image, error) {
-	repo, err := store.get(repoName)
-
-	if err != nil {
-		return nil, err
-	}
-	if repo == nil {
-		return nil, nil
-	}
-
-	store.Lock()
-	defer store.Unlock()
-	if imgID, exists := repo[refOrID]; exists {
-		return store.graph.Get(imgID)
-	}
-
-	// If no matching tag is found, search through images for a matching image id
-	// iff it looks like a short ID or would look like a short ID
-	if stringid.IsShortID(stringid.TruncateID(refOrID)) {
-		for _, revision := range repo {
-			if strings.HasPrefix(revision, refOrID) {
-				return store.graph.Get(revision)
-			}
-		}
-	}
-
-	return nil, nil
-}
-
-// validateRepoName validates the name of a repository.
-func validateRepoName(name string) error {
-	if name == "" {
-		return fmt.Errorf("Repository name can't be empty")
-	}
-	if name == "scratch" {
-		return fmt.Errorf("'scratch' is a reserved name")
-	}
-	return nil
-}
-
-func validateDigest(dgst string) error {
-	if dgst == "" {
-		return errors.New("digest can't be empty")
-	}
-	if _, err := digest.ParseDigest(dgst); err != nil {
-		return err
-	}
-	return nil
-}
-
-// poolAdd checks if a push or pull is already running, and returns
-// (broadcaster, true) if a running operation is found. Otherwise, it creates a
-// new one and returns (broadcaster, false).
-func (store *TagStore) poolAdd(kind, key string) (*broadcaster.Buffered, bool) {
-	store.Lock()
-	defer store.Unlock()
-
-	if p, exists := store.pullingPool[key]; exists {
-		return p, true
-	}
-	if p, exists := store.pushingPool[key]; exists {
-		return p, true
-	}
-
-	broadcaster := broadcaster.NewBuffered()
-
-	switch kind {
-	case "pull":
-		store.pullingPool[key] = broadcaster
-	case "push":
-		store.pushingPool[key] = broadcaster
-	default:
-		panic("Unknown pool type")
-	}
-
-	return broadcaster, false
-}
-
-func (store *TagStore) poolRemoveWithError(kind, key string, broadcasterResult error) error {
-	store.Lock()
-	defer store.Unlock()
-	switch kind {
-	case "pull":
-		if broadcaster, exists := store.pullingPool[key]; exists {
-			broadcaster.CloseWithError(broadcasterResult)
-			delete(store.pullingPool, key)
-		}
-	case "push":
-		if broadcaster, exists := store.pushingPool[key]; exists {
-			broadcaster.CloseWithError(broadcasterResult)
-			delete(store.pushingPool, key)
-		}
-	default:
-		return fmt.Errorf("Unknown pool type")
-	}
-	return nil
-}
-
-func (store *TagStore) poolRemove(kind, key string) error {
-	return store.poolRemoveWithError(kind, key, nil)
-}

graph/tags/tags.go

@@ -1,36 +0,0 @@
-package tags
-
-import (
-	"fmt"
-	"regexp"
-
-	"github.com/docker/distribution/reference"
-)
-
-// DefaultTag defines the default tag used when performing images related actions and no tag string is specified
-const DefaultTag = "latest"
-
-var anchoredTagRegexp = regexp.MustCompile(`^` + reference.TagRegexp.String() + `$`)
-
-// ErrTagInvalidFormat is returned if tag is invalid.
-type ErrTagInvalidFormat struct {
-	name string
-}
-
-func (e ErrTagInvalidFormat) Error() string {
-	return fmt.Sprintf("Illegal tag name (%s): only [A-Za-z0-9_.-] are allowed ('.' and '-' are NOT allowed in the initial), minimum 1, maximum 128 in length", e.name)
-}
-
-// ValidateTagName validates the name of a tag.
-// It returns an error if the given name is an emtpy string.
-// If name is not valid, it returns ErrTagInvalidFormat
-func ValidateTagName(name string) error {
-	if name == "" {
-		return fmt.Errorf("tag name can't be empty")
-	}
-
-	if !anchoredTagRegexp.MatchString(name) {
-		return ErrTagInvalidFormat{name}
-	}
-	return nil
-}

graph/tags/tags_unit_test.go

@@ -1,23 +0,0 @@
-package tags
-
-import (
-	"testing"
-)
-
-func TestValidTagName(t *testing.T) {
-	validTags := []string{"9", "foo", "foo-test", "bar.baz.boo"}
-	for _, tag := range validTags {
-		if err := ValidateTagName(tag); err != nil {
-			t.Errorf("'%s' should've been a valid tag", tag)
-		}
-	}
-}
-
-func TestInvalidTagName(t *testing.T) {
-	inValidTags := []string{"-9", ".foo", "-test", ".", "-"}
-	for _, tag := range inValidTags {
-		if err := ValidateTagName(tag); err == nil {
-			t.Errorf("'%s' should've been an invalid tag", tag)
-		}
-	}
-}

graph/tags_unit_test.go

@@ -1,205 +0,0 @@
-package graph
-
-import (
-	"archive/tar"
-	"bytes"
-	"io"
-	"os"
-	"path"
-	"testing"
-
-	"github.com/docker/docker/daemon/events"
-	"github.com/docker/docker/daemon/graphdriver"
-	_ "github.com/docker/docker/daemon/graphdriver/vfs" // import the vfs driver so it is used in the tests
-	"github.com/docker/docker/graph/tags"
-	"github.com/docker/docker/image"
-	"github.com/docker/docker/utils"
-)
-
-const (
-	testOfficialImageName    = "myapp"
-	testOfficialImageID      = "1a2d3c4d4e5fa2d2a21acea242a5e2345d3aefc3e7dfa2a2a2a21a2a2ad2d234"
-	testOfficialImageIDShort = "1a2d3c4d4e5f"
-	testPrivateImageName     = "127.0.0.1:8000/privateapp"
-	testPrivateImageID       = "5bc255f8699e4ee89ac4469266c3d11515da88fdcbde45d7b069b636ff4efd81"
-	testPrivateImageIDShort  = "5bc255f8699e"
-	testPrivateImageDigest   = "sha256:bc8813ea7b3603864987522f02a76101c17ad122e1c46d790efc0fca78ca7bfb"
-	testPrivateImageTag      = "sometag"
-)
-
-func fakeTar() (io.Reader, error) {
-	uid := os.Getuid()
-	gid := os.Getgid()
-
-	content := []byte("Hello world!\n")
-	buf := new(bytes.Buffer)
-	tw := tar.NewWriter(buf)
-	for _, name := range []string{"/etc/postgres/postgres.conf", "/etc/passwd", "/var/log/postgres/postgres.conf"} {
-		hdr := new(tar.Header)
-
-		// Leaving these fields blank requires root privileges
-		hdr.Uid = uid
-		hdr.Gid = gid
-
-		hdr.Size = int64(len(content))
-		hdr.Name = name
-		if err := tw.WriteHeader(hdr); err != nil {
-			return nil, err
-		}
-		tw.Write([]byte(content))
-	}
-	tw.Close()
-	return buf, nil
-}
-
-func mkTestTagStore(root string, t *testing.T) *TagStore {
-	driver, err := graphdriver.New(root, nil, nil, nil)
-	if err != nil {
-		t.Fatal(err)
-	}
-	graph, err := NewGraph(root, driver, nil, nil)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	tagCfg := &TagStoreConfig{
-		Graph:  graph,
-		Events: events.New(),
-	}
-	store, err := NewTagStore(path.Join(root, "tags"), tagCfg)
-	if err != nil {
-		t.Fatal(err)
-	}
-	officialArchive, err := fakeTar()
-	if err != nil {
-		t.Fatal(err)
-	}
-	img := &image.Image{ID: testOfficialImageID}
-	if err := graph.Register(v1Descriptor{img}, officialArchive); err != nil {
-		t.Fatal(err)
-	}
-	if err := store.Tag(testOfficialImageName, "", testOfficialImageID, false); err != nil {
-		t.Fatal(err)
-	}
-	privateArchive, err := fakeTar()
-	if err != nil {
-		t.Fatal(err)
-	}
-	img = &image.Image{ID: testPrivateImageID}
-	if err := graph.Register(v1Descriptor{img}, privateArchive); err != nil {
-		t.Fatal(err)
-	}
-	if err := store.Tag(testPrivateImageName, "", testPrivateImageID, false); err != nil {
-		t.Fatal(err)
-	}
-	if err := store.setDigest(testPrivateImageName, testPrivateImageDigest, testPrivateImageID); err != nil {
-		t.Fatal(err)
-	}
-	return store
-}
-
-func TestLookupImage(t *testing.T) {
-	tmp, err := utils.TestDirectory("")
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer os.RemoveAll(tmp)
-	store := mkTestTagStore(tmp, t)
-	defer store.graph.driver.Cleanup()
-
-	officialLookups := []string{
-		testOfficialImageID,
-		testOfficialImageIDShort,
-		testOfficialImageName + ":" + testOfficialImageID,
-		testOfficialImageName + ":" + testOfficialImageIDShort,
-		testOfficialImageName,
-		testOfficialImageName + ":" + tags.DefaultTag,
-		"docker.io/" + testOfficialImageName,
-		"docker.io/" + testOfficialImageName + ":" + tags.DefaultTag,
-		"index.docker.io/" + testOfficialImageName,
-		"index.docker.io/" + testOfficialImageName + ":" + tags.DefaultTag,
-		"library/" + testOfficialImageName,
-		"library/" + testOfficialImageName + ":" + tags.DefaultTag,
-		"docker.io/library/" + testOfficialImageName,
-		"docker.io/library/" + testOfficialImageName + ":" + tags.DefaultTag,
-		"index.docker.io/library/" + testOfficialImageName,
-		"index.docker.io/library/" + testOfficialImageName + ":" + tags.DefaultTag,
-	}
-
-	privateLookups := []string{
-		testPrivateImageID,
-		testPrivateImageIDShort,
-		testPrivateImageName + ":" + testPrivateImageID,
-		testPrivateImageName + ":" + testPrivateImageIDShort,
-		testPrivateImageName,
-		testPrivateImageName + ":" + tags.DefaultTag,
-	}
-
-	invalidLookups := []string{
-		testOfficialImageName + ":" + "fail",
-		"fail:fail",
-	}
-
-	digestLookups := []string{
-		testPrivateImageName + "@" + testPrivateImageDigest,
-	}
-
-	for _, name := range officialLookups {
-		if img, err := store.LookupImage(name); err != nil {
-			t.Errorf("Error looking up %s: %s", name, err)
-		} else if img == nil {
-			t.Errorf("Expected 1 image, none found: %s", name)
-		} else if img.ID != testOfficialImageID {
-			t.Errorf("Expected ID '%s' found '%s'", testOfficialImageID, img.ID)
-		}
-	}
-
-	for _, name := range privateLookups {
-		if img, err := store.LookupImage(name); err != nil {
-			t.Errorf("Error looking up %s: %s", name, err)
-		} else if img == nil {
-			t.Errorf("Expected 1 image, none found: %s", name)
-		} else if img.ID != testPrivateImageID {
-			t.Errorf("Expected ID '%s' found '%s'", testPrivateImageID, img.ID)
-		}
-	}
-
-	for _, name := range invalidLookups {
-		if img, err := store.LookupImage(name); err == nil {
-			t.Errorf("Expected error, none found: %s", name)
-		} else if img != nil {
-			t.Errorf("Expected 0 image, 1 found: %s", name)
-		}
-	}
-
-	for _, name := range digestLookups {
-		if img, err := store.LookupImage(name); err != nil {
-			t.Errorf("Error looking up %s: %s", name, err)
-		} else if img == nil {
-			t.Errorf("Expected 1 image, none found: %s", name)
-		} else if img.ID != testPrivateImageID {
-			t.Errorf("Expected ID '%s' found '%s'", testPrivateImageID, img.ID)
-		}
-	}
-}
-
-func TestValidateDigest(t *testing.T) {
-	tests := []struct {
-		input       string
-		expectError bool
-	}{
-		{"", true},
-		{"latest", true},
-		{"sha256:b", false},
-		{"tarsum+v1+sha256:bY852-_.+=", false},
-		{"#$%#$^:$%^#$%", true},
-	}
-
-	for i, test := range tests {
-		err := validateDigest(test.input)
-		gotError := err != nil
-		if e, a := test.expectError, gotError; e != a {
-			t.Errorf("%d: with input %s, expected error=%t, got %t: %s", i, test.input, test.expectError, gotError, err)
-		}
-	}
-}

image/fixtures/post1.9/expected_computed_id

@@ -1 +0,0 @@
-sha256:f2722a8ec6926e02fa9f2674072cbc2a25cf0f449f27350f613cd843b02c9105

image/fixtures/post1.9/expected_config

@@ -1 +0,0 @@
-{"architecture":"amd64","config":{"Hostname":"fb1f7270da95","Domainname":"","User":"","AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":["foo=bar"],"Cmd":null,"Image":"361a94d06b2b781b2f1ee6c72e1cbbfbbd032a103e26a3db75b431743829ae4f","Volumes":null,"VolumeDriver":"","WorkingDir":"","Entrypoint":null,"OnBuild":null,"Labels":null},"container":"fb1f7270da9519308361b99dc8e0d30f12c24dfd28537c2337ece995ac853a16","container_config":{"Hostname":"fb1f7270da95","Domainname":"","User":"","AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":["foo=bar"],"Cmd":["/bin/sh","-c","#(nop) ADD file:11998b2a4d664a75cd0c3f4e4cb1837434e0f997ba157a0ac1d3c68a07aa2f4f in /"],"Image":"361a94d06b2b781b2f1ee6c72e1cbbfbbd032a103e26a3db75b431743829ae4f","Volumes":null,"VolumeDriver":"","WorkingDir":"","Entrypoint":null,"OnBuild":null,"Labels":null},"created":"2015-09-08T21:30:30.807853054Z","docker_version":"1.9.0-dev","layer_id":"sha256:31176893850e05d308cdbfef88877e460d50c8063883fb13eb5753097da6422a","os":"linux","parent_id":"sha256:ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02"}

image/fixtures/post1.9/layer_id

@@ -1 +0,0 @@
-sha256:31176893850e05d308cdbfef88877e460d50c8063883fb13eb5753097da6422a

image/fixtures/post1.9/parent_id

@@ -1 +0,0 @@
-sha256:ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02

image/fixtures/post1.9/v1compatibility

@@ -1 +0,0 @@
-{"id":"8dfb96b5d09e6cf6f376d81f1e2770ee5ede309f9bd9e079688c9782649ab326","parent":"361a94d06b2b781b2f1ee6c72e1cbbfbbd032a103e26a3db75b431743829ae4f","created":"2015-09-08T21:30:30.807853054Z","container":"fb1f7270da9519308361b99dc8e0d30f12c24dfd28537c2337ece995ac853a16","container_config":{"Hostname":"fb1f7270da95","Domainname":"","User":"","AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":["foo=bar"],"Cmd":["/bin/sh","-c","#(nop) ADD file:11998b2a4d664a75cd0c3f4e4cb1837434e0f997ba157a0ac1d3c68a07aa2f4f in /"],"Image":"361a94d06b2b781b2f1ee6c72e1cbbfbbd032a103e26a3db75b431743829ae4f","Volumes":null,"VolumeDriver":"","WorkingDir":"","Entrypoint":null,"OnBuild":null,"Labels":null},"docker_version":"1.9.0-dev","config":{"Hostname":"fb1f7270da95","Domainname":"","User":"","AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":["foo=bar"],"Cmd":null,"Image":"361a94d06b2b781b2f1ee6c72e1cbbfbbd032a103e26a3db75b431743829ae4f","Volumes":null,"VolumeDriver":"","WorkingDir":"","Entrypoint":null,"OnBuild":null,"Labels":null},"architecture":"amd64","os":"linux"}

image/fixtures/pre1.9/expected_computed_id

@@ -1 +0,0 @@
-sha256:fd6ebfedda8ea140a9380767e15bd32c6e899303cfe34bc4580c931f2f816f89

image/fixtures/pre1.9/expected_config

@@ -1,2 +0,0 @@
-{"architecture":"amd64","config":{"Hostname":"03797203757d","Domainname":"","User":"","AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":["PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin","GOLANG_VERSION=1.4.1","GOPATH=/go"],"Cmd":null,"Image":"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02","Volumes":null,"WorkingDir":"/go","Entrypoint":["/go/bin/dnsdock"],"OnBuild":[],"Labels":{}},"container":"d91be3479d5b1e84b0c00d18eea9dc777ca0ad166d51174b24283e2e6f104253","container_config":{"Hostname":"03797203757d","Domainname":"","User":"","AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":["PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin","GOLANG_VERSION=1.4.1","GOPATH=/go"],"Cmd":["/bin/sh","-c","#(nop) ENTRYPOINT [\"/go/bin/dnsdock\"]"],"Image":"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02","Volumes":null,"WorkingDir":"/go","Entrypoint":["/go/bin/dnsdock"],"OnBuild":[],"Labels":{}},"created":"2015-08-19T16:49:11.368300679Z","docker_version":"1.6.2","layer_id":"sha256:31176893850e05d308cdbfef88877e460d50c8063883fb13eb5753097da6422a","os":"linux","parent_id":"sha256:ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02"}
-

image/fixtures/pre1.9/layer_id

@@ -1 +0,0 @@
-sha256:31176893850e05d308cdbfef88877e460d50c8063883fb13eb5753097da6422a

image/fixtures/pre1.9/parent_id

@@ -1 +0,0 @@
-sha256:ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02

image/fixtures/pre1.9/v1compatibility

@@ -1 +0,0 @@
-{"id":"3b38edc92eb7c074812e217b41a6ade66888531009d6286a6f5f36a06f9841b9","parent":"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02","created":"2015-08-19T16:49:11.368300679Z","container":"d91be3479d5b1e84b0c00d18eea9dc777ca0ad166d51174b24283e2e6f104253","container_config":{"Hostname":"03797203757d","Domainname":"","User":"","Memory":0,"MemorySwap":0,"CpuShares":0,"Cpuset":"","AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"PortSpecs":null,"ExposedPorts":null,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":["PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin","GOLANG_VERSION=1.4.1","GOPATH=/go"],"Cmd":["/bin/sh","-c","#(nop) ENTRYPOINT [\"/go/bin/dnsdock\"]"],"Image":"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02","Volumes":null,"WorkingDir":"/go","Entrypoint":["/go/bin/dnsdock"],"NetworkDisabled":false,"MacAddress":"","OnBuild":[],"Labels":{}},"docker_version":"1.6.2","config":{"Hostname":"03797203757d","Domainname":"","User":"","Memory":0,"MemorySwap":0,"CpuShares":0,"Cpuset":"","AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"PortSpecs":null,"ExposedPorts":null,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":["PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin","GOLANG_VERSION=1.4.1","GOPATH=/go"],"Cmd":null,"Image":"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02","Volumes":null,"WorkingDir":"/go","Entrypoint":["/go/bin/dnsdock"],"NetworkDisabled":false,"MacAddress":"","OnBuild":[],"Labels":{}},"architecture":"amd64","os":"linux","Size":0}

image/fs.go

@@ -0,0 +1,192 @@
+package image
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"sync"
+
+	"github.com/Sirupsen/logrus"
+	"github.com/docker/distribution/digest"
+)
+
+// IDWalkFunc is function called by StoreBackend.Walk
+type IDWalkFunc func(id ID) error
+
+// StoreBackend provides interface for image.Store persistence
+type StoreBackend interface {
+	Walk(f IDWalkFunc) error
+	Get(id ID) ([]byte, error)
+	Set(data []byte) (ID, error)
+	Delete(id ID) error
+	SetMetadata(id ID, key string, data []byte) error
+	GetMetadata(id ID, key string) ([]byte, error)
+	DeleteMetadata(id ID, key string) error
+}
+
+// fs implements StoreBackend using the filesystem.
+type fs struct {
+	sync.RWMutex
+	root string
+}
+
+const (
+	contentDirName  = "content"
+	metadataDirName = "metadata"
+)
+
+// NewFSStoreBackend returns new filesystem based backend for image.Store
+func NewFSStoreBackend(root string) (StoreBackend, error) {
+	return newFSStore(root)
+}
+
+func newFSStore(root string) (*fs, error) {
+	s := &fs{
+		root: root,
+	}
+	if err := os.MkdirAll(filepath.Join(root, contentDirName, string(digest.Canonical)), 0700); err != nil {
+		return nil, err
+	}
+	if err := os.MkdirAll(filepath.Join(root, metadataDirName, string(digest.Canonical)), 0700); err != nil {
+		return nil, err
+	}
+	return s, nil
+}
+
+func (s *fs) contentFile(id ID) string {
+	dgst := digest.Digest(id)
+	return filepath.Join(s.root, contentDirName, string(dgst.Algorithm()), dgst.Hex())
+}
+
+func (s *fs) metadataDir(id ID) string {
+	dgst := digest.Digest(id)
+	return filepath.Join(s.root, metadataDirName, string(dgst.Algorithm()), dgst.Hex())
+}
+
+// Walk calls the supplied callback for each image ID in the storage backend.
+func (s *fs) Walk(f IDWalkFunc) error {
+	// Only Canonical digest (sha256) is currently supported
+	s.RLock()
+	dir, err := ioutil.ReadDir(filepath.Join(s.root, contentDirName, string(digest.Canonical)))
+	s.RUnlock()
+	if err != nil {
+		return err
+	}
+	for _, v := range dir {
+		dgst := digest.NewDigestFromHex(string(digest.Canonical), v.Name())
+		if err := dgst.Validate(); err != nil {
+			logrus.Debugf("Skipping invalid digest %s: %s", dgst, err)
+			continue
+		}
+		if err := f(ID(dgst)); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// Get returns the content stored under a given ID.
+func (s *fs) Get(id ID) ([]byte, error) {
+	s.RLock()
+	defer s.RUnlock()
+
+	return s.get(id)
+}
+
+func (s *fs) get(id ID) ([]byte, error) {
+	content, err := ioutil.ReadFile(s.contentFile(id))
+	if err != nil {
+		return nil, err
+	}
+
+	// todo: maybe optional
+	validated, err := digest.FromBytes(content)
+	if err != nil {
+		return nil, err
+	}
+	if ID(validated) != id {
+		return nil, fmt.Errorf("failed to verify image: %v", id)
+	}
+
+	return content, nil
+}
+
+// Set stores content under a given ID.
+func (s *fs) Set(data []byte) (ID, error) {
+	s.Lock()
+	defer s.Unlock()
+
+	if len(data) == 0 {
+		return "", fmt.Errorf("Invalid empty data")
+	}
+
+	dgst, err := digest.FromBytes(data)
+	if err != nil {
+		return "", err
+	}
+	id := ID(dgst)
+	filePath := s.contentFile(id)
+	tempFilePath := s.contentFile(id) + ".tmp"
+	if err := ioutil.WriteFile(tempFilePath, data, 0600); err != nil {
+		return "", err
+	}
+	if err := os.Rename(tempFilePath, filePath); err != nil {
+		return "", err
+	}
+
+	return id, nil
+}
+
+// Delete removes content and metadata files associated with the ID.
+func (s *fs) Delete(id ID) error {
+	s.Lock()
+	defer s.Unlock()
+
+	if err := os.RemoveAll(s.metadataDir(id)); err != nil {
+		return err
+	}
+	if err := os.Remove(s.contentFile(id)); err != nil {
+		return err
+	}
+	return nil
+}
+
+// SetMetadata sets metadata for a given ID. It fails if there's no base file.
+func (s *fs) SetMetadata(id ID, key string, data []byte) error {
+	s.Lock()
+	defer s.Unlock()
+	if _, err := s.get(id); err != nil {
+		return err
+	}
+
+	baseDir := filepath.Join(s.metadataDir(id))
+	if err := os.MkdirAll(baseDir, 0700); err != nil {
+		return err
+	}
+	filePath := filepath.Join(s.metadataDir(id), key)
+	tempFilePath := filePath + ".tmp"
+	if err := ioutil.WriteFile(tempFilePath, data, 0600); err != nil {
+		return err
+	}
+	return os.Rename(tempFilePath, filePath)
+}
+
+// GetMetadata returns metadata for a given ID.
+func (s *fs) GetMetadata(id ID, key string) ([]byte, error) {
+	s.RLock()
+	defer s.RUnlock()
+
+	if _, err := s.get(id); err != nil {
+		return nil, err
+	}
+	return ioutil.ReadFile(filepath.Join(s.metadataDir(id), key))
+}
+
+// DeleteMetadata removes the metadata associated with an ID.
+func (s *fs) DeleteMetadata(id ID, key string) error {
+	s.Lock()
+	defer s.Unlock()
+
+	return os.RemoveAll(filepath.Join(s.metadataDir(id), key))
+}

image/fs_test.go

@@ -0,0 +1,391 @@
+package image
+
+import (
+	"bytes"
+	"crypto/rand"
+	"crypto/sha256"
+	"encoding/hex"
+	"errors"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/docker/distribution/digest"
+)
+
+func TestFSGetSet(t *testing.T) {
+	tmpdir, err := ioutil.TempDir("", "images-fs-store")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(tmpdir)
+	fs, err := NewFSStoreBackend(tmpdir)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	testGetSet(t, fs)
+}
+
+func TestFSGetInvalidData(t *testing.T) {
+	tmpdir, err := ioutil.TempDir("", "images-fs-store")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(tmpdir)
+	fs, err := NewFSStoreBackend(tmpdir)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	id, err := fs.Set([]byte("foobar"))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	dgst := digest.Digest(id)
+
+	if err := ioutil.WriteFile(filepath.Join(tmpdir, contentDirName, string(dgst.Algorithm()), dgst.Hex()), []byte("foobar2"), 0600); err != nil {
+		t.Fatal(err)
+	}
+
+	_, err = fs.Get(id)
+	if err == nil {
+		t.Fatal("Expected get to fail after data modification.")
+	}
+}
+
+func TestFSInvalidSet(t *testing.T) {
+	tmpdir, err := ioutil.TempDir("", "images-fs-store")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(tmpdir)
+	fs, err := NewFSStoreBackend(tmpdir)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	id, err := digest.FromBytes([]byte("foobar"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = os.Mkdir(filepath.Join(tmpdir, contentDirName, string(id.Algorithm()), id.Hex()), 0700)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	_, err = fs.Set([]byte("foobar"))
+	if err == nil {
+		t.Fatal("Expecting error from invalid filesystem data.")
+	}
+}
+
+func TestFSInvalidRoot(t *testing.T) {
+	tmpdir, err := ioutil.TempDir("", "images-fs-store")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(tmpdir)
+
+	tcases := []struct {
+		root, invalidFile string
+	}{
+		{"root", "root"},
+		{"root", "root/content"},
+		{"root", "root/metadata"},
+	}
+
+	for _, tc := range tcases {
+		root := filepath.Join(tmpdir, tc.root)
+		filePath := filepath.Join(tmpdir, tc.invalidFile)
+		err := os.MkdirAll(filepath.Dir(filePath), 0700)
+		if err != nil {
+			t.Fatal(err)
+		}
+		f, err := os.Create(filePath)
+		if err != nil {
+			t.Fatal(err)
+		}
+		f.Close()
+
+		_, err = NewFSStoreBackend(root)
+		if err == nil {
+			t.Fatalf("Expected error from root %q and invlid file %q", tc.root, tc.invalidFile)
+		}
+
+		os.RemoveAll(root)
+	}
+
+}
+
+func testMetadataGetSet(t *testing.T, store StoreBackend) {
+	id, err := store.Set([]byte("foo"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	id2, err := store.Set([]byte("bar"))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	tcases := []struct {
+		id    ID
+		key   string
+		value []byte
+	}{
+		{id, "tkey", []byte("tval1")},
+		{id, "tkey2", []byte("tval2")},
+		{id2, "tkey", []byte("tval3")},
+	}
+
+	for _, tc := range tcases {
+		err = store.SetMetadata(tc.id, tc.key, tc.value)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		actual, err := store.GetMetadata(tc.id, tc.key)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if bytes.Compare(actual, tc.value) != 0 {
+			t.Fatalf("Metadata expected %q, got %q", tc.value, actual)
+		}
+	}
+
+	_, err = store.GetMetadata(id2, "tkey2")
+	if err == nil {
+		t.Fatal("Expected error for getting metadata for unknown key")
+	}
+
+	id3, err := digest.FromBytes([]byte("baz"))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	err = store.SetMetadata(ID(id3), "tkey", []byte("tval"))
+	if err == nil {
+		t.Fatal("Expected error for setting metadata for unknown ID.")
+	}
+
+	_, err = store.GetMetadata(ID(id3), "tkey")
+	if err == nil {
+		t.Fatal("Expected error for getting metadata for unknown ID.")
+	}
+}
+
+func TestFSMetadataGetSet(t *testing.T) {
+	tmpdir, err := ioutil.TempDir("", "images-fs-store")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(tmpdir)
+	fs, err := NewFSStoreBackend(tmpdir)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	testMetadataGetSet(t, fs)
+}
+
+func TestFSDelete(t *testing.T) {
+	tmpdir, err := ioutil.TempDir("", "images-fs-store")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(tmpdir)
+	fs, err := NewFSStoreBackend(tmpdir)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	testDelete(t, fs)
+}
+
+func TestFSWalker(t *testing.T) {
+	tmpdir, err := ioutil.TempDir("", "images-fs-store")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(tmpdir)
+	fs, err := NewFSStoreBackend(tmpdir)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	testWalker(t, fs)
+}
+
+func TestFSInvalidWalker(t *testing.T) {
+	tmpdir, err := ioutil.TempDir("", "images-fs-store")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(tmpdir)
+	fs, err := NewFSStoreBackend(tmpdir)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	fooID, err := fs.Set([]byte("foo"))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if err := ioutil.WriteFile(filepath.Join(tmpdir, contentDirName, "sha256/foobar"), []byte("foobar"), 0600); err != nil {
+		t.Fatal(err)
+	}
+
+	n := 0
+	err = fs.Walk(func(id ID) error {
+		if id != fooID {
+			t.Fatalf("Invalid walker ID %q, expected %q", id, fooID)
+		}
+		n++
+		return nil
+	})
+	if err != nil {
+		t.Fatalf("Invalid data should not have caused walker error, got %v", err)
+	}
+	if n != 1 {
+		t.Fatalf("Expected 1 walk initialization, got %d", n)
+	}
+}
+
+func testGetSet(t *testing.T, store StoreBackend) {
+	type tcase struct {
+		input    []byte
+		expected ID
+	}
+	tcases := []tcase{
+		{[]byte("foobar"), ID("sha256:c3ab8ff13720e8ad9047dd39466b3c8974e592c2fa383d4a3960714caef0c4f2")},
+	}
+
+	randomInput := make([]byte, 8*1024)
+	_, err := rand.Read(randomInput)
+	if err != nil {
+		t.Fatal(err)
+	}
+	// skipping use of digest pkg because its used by the imlementation
+	h := sha256.New()
+	_, err = h.Write(randomInput)
+	if err != nil {
+		t.Fatal(err)
+	}
+	tcases = append(tcases, tcase{
+		input:    randomInput,
+		expected: ID("sha256:" + hex.EncodeToString(h.Sum(nil))),
+	})
+
+	for _, tc := range tcases {
+		id, err := store.Set([]byte(tc.input))
+		if err != nil {
+			t.Fatal(err)
+		}
+		if id != tc.expected {
+			t.Fatalf("Expected ID %q, got %q", tc.expected, id)
+		}
+	}
+
+	for _, emptyData := range [][]byte{nil, {}} {
+		_, err := store.Set(emptyData)
+		if err == nil {
+			t.Fatal("Expected error for nil input.")
+		}
+	}
+
+	for _, tc := range tcases {
+		data, err := store.Get(tc.expected)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if bytes.Compare(data, tc.input) != 0 {
+			t.Fatalf("Expected data %q, got %q", tc.input, data)
+		}
+	}
+
+	for _, key := range []ID{"foobar:abc", "sha256:abc", "sha256:c3ab8ff13720e8ad9047dd39466b3c8974e592c2fa383d4a3960714caef0c4f2a"} {
+		_, err := store.Get(key)
+		if err == nil {
+			t.Fatalf("Expected error for ID %q.", key)
+		}
+	}
+
+}
+
+func testDelete(t *testing.T, store StoreBackend) {
+	id, err := store.Set([]byte("foo"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	id2, err := store.Set([]byte("bar"))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	err = store.Delete(id)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	_, err = store.Get(id)
+	if err == nil {
+		t.Fatalf("Expected getting deleted item %q to fail", id)
+	}
+	_, err = store.Get(id2)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	err = store.Delete(id2)
+	if err != nil {
+		t.Fatal(err)
+	}
+	_, err = store.Get(id2)
+	if err == nil {
+		t.Fatalf("Expected getting deleted item %q to fail", id2)
+	}
+}
+
+func testWalker(t *testing.T, store StoreBackend) {
+	id, err := store.Set([]byte("foo"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	id2, err := store.Set([]byte("bar"))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	tcases := make(map[ID]struct{})
+	tcases[id] = struct{}{}
+	tcases[id2] = struct{}{}
+	n := 0
+	err = store.Walk(func(id ID) error {
+		delete(tcases, id)
+		n++
+		return nil
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if n != 2 {
+		t.Fatalf("Expected 2 walk initializations, got %d", n)
+	}
+	if len(tcases) != 0 {
+		t.Fatalf("Expected empty unwalked set, got %+v", tcases)
+	}
+
+	// stop on error
+	tcases = make(map[ID]struct{})
+	tcases[id] = struct{}{}
+	err = store.Walk(func(id ID) error {
+		return errors.New("")
+	})
+	if err == nil {
+		t.Fatalf("Exected error from walker.")
+	}
+}

image/image.go

@@ -2,36 +2,23 @@ package image
 
 import (
 	"encoding/json"
-	"fmt"
-	"regexp"
+	"errors"
+	"io"
 	"time"
 
-	"github.com/Sirupsen/logrus"
 	"github.com/docker/distribution/digest"
-	derr "github.com/docker/docker/errors"
-	"github.com/docker/docker/pkg/version"
 	"github.com/docker/docker/runconfig"
 )
 
-var validHex = regexp.MustCompile(`^([a-f0-9]{64})$`)
+// ID is the content-addressable ID of an image.
+type ID digest.Digest
 
-// noFallbackMinVersion is the minimum version for which v1compatibility
-// information will not be marshaled through the Image struct to remove
-// blank fields.
-var noFallbackMinVersion = version.Version("1.8.3")
-
-// Descriptor provides the information necessary to register an image in
-// the graph.
-type Descriptor interface {
-	ID() string
-	Parent() string
-	MarshalConfig() ([]byte, error)
+func (id ID) String() string {
+	return digest.Digest(id).String()
 }
 
-// Image stores the image configuration.
-// All fields in this struct must be marked `omitempty` to keep getting
-// predictable hashes from the old `v1Compatibility` configuration.
-type Image struct {
+// V1Image stores the V1 image configuration.
+type V1Image struct {
 	// ID a unique 64 character identifier of the image
 	ID string `json:"id,omitempty"`
 	// Parent id of the image
@@ -55,95 +42,87 @@ type Image struct {
 	// OS is the operating system used to build and run the image
 	OS string `json:"os,omitempty"`
 	// Size is the total size of the image including all layers it is composed of
-	Size int64 `json:",omitempty"` // capitalized for backwards compatibility
-	// ParentID specifies the strong, content address of the parent configuration.
-	ParentID digest.Digest `json:"parent_id,omitempty"`
-	// LayerID provides the content address of the associated layer.
-	LayerID digest.Digest `json:"layer_id,omitempty"`
+	Size int64 `json:",omitempty"`
 }
 
-// NewImgJSON creates an Image configuration from json.
-func NewImgJSON(src []byte) (*Image, error) {
-	ret := &Image{}
+// Image stores the image configuration
+type Image struct {
+	V1Image
+	Parent  ID        `json:"parent,omitempty"`
+	RootFS  *RootFS   `json:"rootfs,omitempty"`
+	History []History `json:"history,omitempty"`
 
-	// FIXME: Is there a cleaner way to "purify" the input json?
-	if err := json.Unmarshal(src, ret); err != nil {
-		return nil, err
-	}
-	return ret, nil
+	// rawJSON caches the immutable JSON associated with this image.
+	rawJSON []byte
+
+	// computedID is the ID computed from the hash of the image config.
+	// Not to be confused with the legacy V1 ID in V1Image.
+	computedID ID
 }
 
-// ValidateID checks whether an ID string is a valid image ID.
-func ValidateID(id string) error {
-	if ok := validHex.MatchString(id); !ok {
-		return derr.ErrorCodeInvalidImageID.WithArgs(id)
-	}
-	return nil
+// RawJSON returns the immutable JSON associated with the image.
+func (img *Image) RawJSON() []byte {
+	return img.rawJSON
+}
+
+// ID returns the image's content-addressable ID.
+func (img *Image) ID() ID {
+	return img.computedID
 }
 
-// MakeImageConfig returns immutable configuration JSON for image based on the
-// v1Compatibility object, layer digest and parent StrongID. SHA256() of this
-// config is the new image ID (strongID).
-func MakeImageConfig(v1Compatibility []byte, layerID, parentID digest.Digest) ([]byte, error) {
+// MarshalJSON serializes the image to JSON. It sorts the top-level keys so
+// that JSON that's been manipulated by a push/pull cycle with a legacy
+// registry won't end up with a different key order.
+func (img *Image) MarshalJSON() ([]byte, error) {
+	type MarshalImage Image
 
-	// Detect images created after 1.8.3
-	img, err := NewImgJSON(v1Compatibility)
+	pass1, err := json.Marshal(MarshalImage(*img))
 	if err != nil {
 		return nil, err
 	}
-	useFallback := version.Version(img.DockerVersion).LessThan(noFallbackMinVersion)
-
-	if useFallback {
-		// Fallback for pre-1.8.3. Calculate base config based on Image struct
-		// so that fields with default values added by Docker will use same ID
-		logrus.Debugf("Using fallback hash for %v", layerID)
-
-		v1Compatibility, err = json.Marshal(img)
-		if err != nil {
-			return nil, err
-		}
-	}
 
 	var c map[string]*json.RawMessage
-	if err := json.Unmarshal(v1Compatibility, &c); err != nil {
+	if err := json.Unmarshal(pass1, &c); err != nil {
 		return nil, err
 	}
-
-	if err := layerID.Validate(); err != nil {
-		return nil, fmt.Errorf("invalid layerID: %v", err)
-	}
-
-	c["layer_id"] = rawJSON(layerID)
-
-	if parentID != "" {
-		if err := parentID.Validate(); err != nil {
-			return nil, fmt.Errorf("invalid parentID %v", err)
-		}
-		c["parent_id"] = rawJSON(parentID)
-	}
-
-	delete(c, "id")
-	delete(c, "parent")
-	delete(c, "Size") // Size is calculated from data on disk and is inconsitent
-
 	return json.Marshal(c)
 }
 
-// StrongID returns image ID for the config JSON.
-func StrongID(configJSON []byte) (digest.Digest, error) {
-	digester := digest.Canonical.New()
-	if _, err := digester.Hash().Write(configJSON); err != nil {
-		return "", err
-	}
-	dgst := digester.Digest()
-	logrus.Debugf("H(%v) = %v", string(configJSON), dgst)
-	return dgst, nil
+// History stores build commands that were used to create an image
+type History struct {
+	// Created timestamp for build point
+	Created time.Time `json:"created"`
+	// Author of the build point
+	Author string `json:"author,omitempty"`
+	// CreatedBy keeps the Dockerfile command used while building image.
+	CreatedBy string `json:"created_by,omitempty"`
+	// Comment is custom mesage set by the user when creating the image.
+	Comment string `json:"comment,omitempty"`
+	// EmptyLayer is set to true if this history item did not generate a
+	// layer. Otherwise, the history item is associated with the next
+	// layer in the RootFS section.
+	EmptyLayer bool `json:"empty_layer,omitempty"`
 }
 
-func rawJSON(value interface{}) *json.RawMessage {
-	jsonval, err := json.Marshal(value)
-	if err != nil {
-		return nil
+// Exporter provides interface for exporting and importing images
+type Exporter interface {
+	Load(io.ReadCloser, io.Writer) error
+	// TODO: Load(net.Context, io.ReadCloser, <- chan StatusMessage) error
+	Save([]string, io.Writer) error
+}
+
+// NewFromJSON creates an Image configuration from json.
+func NewFromJSON(src []byte) (*Image, error) {
+	img := &Image{}
+
+	if err := json.Unmarshal(src, img); err != nil {
+		return nil, err
 	}
-	return (*json.RawMessage)(&jsonval)
+	if img.RootFS == nil {
+		return nil, errors.New("Invalid image JSON, no RootFS key.")
+	}
+
+	img.rawJSON = src
+
+	return img, nil
 }