0ct0pu5/moby
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Merge pull request #17924 from aaronlehmann/content-addressability

Browse source 
Content addressability
This commit is contained in:
David Calavera 2015-11-24 11:01:33 -08:00
commit a2ab05098d
155 changed files with 11531 additions and 6137 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

57
api/client/build.go
View file

@ -18,16 +18,15 @@ import (
"strconv"
"strings"
"github.com/docker/distribution/reference"
"github.com/docker/docker/api"
Cli "github.com/docker/docker/cli"
"github.com/docker/docker/graph/tags"
"github.com/docker/docker/opts"
"github.com/docker/docker/pkg/archive"
"github.com/docker/docker/pkg/fileutils"
"github.com/docker/docker/pkg/httputils"
"github.com/docker/docker/pkg/jsonmessage"
flag "github.com/docker/docker/pkg/mflag"
"github.com/docker/docker/pkg/parsers"
"github.com/docker/docker/pkg/progressreader"
"github.com/docker/docker/pkg/streamformatter"
"github.com/docker/docker/pkg/ulimit"
@ -35,6 +34,7 @@ import (
"github.com/docker/docker/pkg/urlutil"
"github.com/docker/docker/registry"
"github.com/docker/docker/runconfig"
tagpkg "github.com/docker/docker/tag"
"github.com/docker/docker/utils"
)
@ -323,7 +323,7 @@ func (cli *DockerCli) CmdBuild(args ...string) error {
// Since the build was successful, now we must tag any of the resolved
// images from the above Dockerfile rewrite.
for _, resolved := range resolvedTags {
if err := cli.tagTrusted(resolved.repoInfo, resolved.digestRef, resolved.tagRef); err != nil {
if err := cli.tagTrusted(resolved.digestRef, resolved.tagRef); err != nil {
return err
}
}
@ -333,16 +333,12 @@ func (cli *DockerCli) CmdBuild(args ...string) error {
// validateTag checks if the given image name can be resolved.
func validateTag(rawRepo string) (string, error) {
repository, tag := parsers.ParseRepositoryTag(rawRepo)
if err := registry.ValidateRepositoryName(repository); err != nil {
ref, err := reference.ParseNamed(rawRepo)
if err != nil {
return "", err
}
if len(tag) == 0 {
return rawRepo, nil
}
if err := tags.ValidateTagName(tag); err != nil {
if err := registry.ValidateRepositoryName(ref); err != nil {
return "", err
}
@ -565,15 +561,16 @@ func (td *trustedDockerfile) Close() error {
// resolvedTag records the repository, tag, and resolved digest reference
// from a Dockerfile rewrite.
type resolvedTag struct {
repoInfo *registry.RepositoryInfo
digestRef, tagRef registry.Reference
repoInfo *registry.RepositoryInfo
digestRef reference.Canonical
tagRef reference.NamedTagged
}
// rewriteDockerfileFrom rewrites the given Dockerfile by resolving images in
// "FROM <image>" instructions to a digest reference. `translator` is a
// function that takes a repository name and tag reference and returns a
// trusted digest reference.
func rewriteDockerfileFrom(dockerfileName string, translator func(string, registry.Reference) (registry.Reference, error)) (newDockerfile *trustedDockerfile, resolvedTags []*resolvedTag, err error) {
func rewriteDockerfileFrom(dockerfileName string, translator func(reference.NamedTagged) (reference.Canonical, error)) (newDockerfile *trustedDockerfile, resolvedTags []*resolvedTag, err error) {
dockerfile, err := os.Open(dockerfileName)
if err != nil {
return nil, nil, fmt.Errorf("unable to open Dockerfile: %v", err)
@ -607,29 +604,39 @@ func rewriteDockerfileFrom(dockerfileName string, translator func(string, regist
matches := dockerfileFromLinePattern.FindStringSubmatch(line)
if matches != nil && matches[1] != "scratch" {
// Replace the line with a resolved "FROM repo@digest"
repo, tag := parsers.ParseRepositoryTag(matches[1])
if tag == "" {
tag = tags.DefaultTag
}
repoInfo, err := registry.ParseRepositoryInfo(repo)
ref, err := reference.ParseNamed(matches[1])
if err != nil {
return nil, nil, fmt.Errorf("unable to parse repository info %q: %v", repo, err)
return nil, nil, err
}
ref := registry.ParseReference(tag)
digested := false
switch ref.(type) {
case reference.Tagged:
case reference.Digested:
digested = true
default:
ref, err = reference.WithTag(ref, tagpkg.DefaultTag)
if err != nil {
return nil, nil, err
}
}
if !ref.HasDigest() && isTrusted() {
trustedRef, err := translator(repo, ref)
repoInfo, err := registry.ParseRepositoryInfo(ref)
if err != nil {
return nil, nil, fmt.Errorf("unable to parse repository info %q: %v", ref.String(), err)
}
if !digested && isTrusted() {
trustedRef, err := translator(ref.(reference.NamedTagged))
if err != nil {
return nil, nil, err
}
line = dockerfileFromLinePattern.ReplaceAllLiteralString(line, fmt.Sprintf("FROM %s", trustedRef.ImageName(repo)))
line = dockerfileFromLinePattern.ReplaceAllLiteralString(line, fmt.Sprintf("FROM %s", trustedRef.String()))
resolvedTags = append(resolvedTags, &resolvedTag{
repoInfo: repoInfo,
digestRef: trustedRef,
tagRef: ref,
tagRef: ref.(reference.NamedTagged),
})
}
}

28
api/client/commit.go
View file

@ -2,14 +2,15 @@ package client
import (
"encoding/json"
"errors"
"fmt"
"net/url"
"github.com/docker/distribution/reference"
"github.com/docker/docker/api/types"
Cli "github.com/docker/docker/cli"
"github.com/docker/docker/opts"
flag "github.com/docker/docker/pkg/mflag"
"github.com/docker/docker/pkg/parsers"
"github.com/docker/docker/registry"
"github.com/docker/docker/runconfig"
)
@ -32,20 +33,35 @@ func (cli *DockerCli) CmdCommit(args ...string) error {
cmd.ParseFlags(args, true)
var (
name = cmd.Arg(0)
repository, tag = parsers.ParseRepositoryTag(cmd.Arg(1))
name = cmd.Arg(0)
repositoryAndTag = cmd.Arg(1)
repositoryName string
tag string
)
//Check if the given image name can be resolved
if repository != "" {
if err := registry.ValidateRepositoryName(repository); err != nil {
if repositoryAndTag != "" {
ref, err := reference.ParseNamed(repositoryAndTag)
if err != nil {
return err
}
if err := registry.ValidateRepositoryName(ref); err != nil {
return err
}
repositoryName = ref.Name()
switch x := ref.(type) {
case reference.Digested:
return errors.New("cannot commit to digest reference")
case reference.Tagged:
tag = x.Tag()
}
}
v := url.Values{}
v.Set("container", name)
v.Set("repo", repository)
v.Set("repo", repositoryName)
v.Set("tag", tag)
v.Set("comment", *flComment)
v.Set("author", *flAuthor)

69
api/client/create.go
View file

@ -9,12 +9,12 @@ import (
"os"
"strings"
"github.com/docker/distribution/reference"
"github.com/docker/docker/api/types"
Cli "github.com/docker/docker/cli"
"github.com/docker/docker/graph/tags"
"github.com/docker/docker/pkg/parsers"
"github.com/docker/docker/registry"
"github.com/docker/docker/runconfig"
tagpkg "github.com/docker/docker/tag"
)
func (cli *DockerCli) pullImage(image string) error {
@ -23,16 +23,28 @@ func (cli *DockerCli) pullImage(image string) error {
func (cli *DockerCli) pullImageCustomOut(image string, out io.Writer) error {
v := url.Values{}
repos, tag := parsers.ParseRepositoryTag(image)
// pull only the image tagged 'latest' if no tag was specified
if tag == "" {
tag = tags.DefaultTag
ref, err := reference.ParseNamed(image)
if err != nil {
return err
}
v.Set("fromImage", repos)
var tag string
switch x := ref.(type) {
case reference.Digested:
tag = x.Digest().String()
case reference.Tagged:
tag = x.Tag()
default:
// pull only the image tagged 'latest' if no tag was specified
tag = tagpkg.DefaultTag
}
v.Set("fromImage", ref.Name())
v.Set("tag", tag)
// Resolve the Repository name from fqn to RepositoryInfo
repoInfo, err := registry.ParseRepositoryInfo(repos)
repoInfo, err := registry.ParseRepositoryInfo(ref)
if err != nil {
return err
}
@ -94,39 +106,46 @@ func (cli *DockerCli) createContainer(config *runconfig.Config, hostConfig *runc
defer containerIDFile.Close()
}
repo, tag := parsers.ParseRepositoryTag(config.Image)
if tag == "" {
tag = tags.DefaultTag
ref, err := reference.ParseNamed(config.Image)
if err != nil {
return nil, err
}
ref := registry.ParseReference(tag)
var trustedRef registry.Reference
if isTrusted() && !ref.HasDigest() {
var err error
trustedRef, err = cli.trustedReference(repo, ref)
isDigested := false
switch ref.(type) {
case reference.Tagged:
case reference.Digested:
isDigested = true
default:
ref, err = reference.WithTag(ref, tagpkg.DefaultTag)
if err != nil {
return nil, err
}
config.Image = trustedRef.ImageName(repo)
}
var trustedRef reference.Canonical
if isTrusted() && !isDigested {
var err error
trustedRef, err = cli.trustedReference(ref.(reference.NamedTagged))
if err != nil {
return nil, err
}
config.Image = trustedRef.String()
}
//create the container
serverResp, err := cli.call("POST", "/containers/create?"+containerValues.Encode(), mergedConfig, nil)
//if image not found try to pull it
if serverResp.statusCode == 404 && strings.Contains(err.Error(), config.Image) {
fmt.Fprintf(cli.err, "Unable to find image '%s' locally\n", ref.ImageName(repo))
fmt.Fprintf(cli.err, "Unable to find image '%s' locally\n", ref.String())
// we don't want to write to stdout anything apart from container.ID
if err = cli.pullImageCustomOut(config.Image, cli.err); err != nil {
return nil, err
}
if trustedRef != nil && !ref.HasDigest() {
repoInfo, err := registry.ParseRepositoryInfo(repo)
if err != nil {
return nil, err
}
if err := cli.tagTrusted(repoInfo, trustedRef, ref); err != nil {
if trustedRef != nil && !isDigested {
if err := cli.tagTrusted(trustedRef, ref.(reference.NamedTagged)); err != nil {
return nil, err
}
}

34
api/client/images.go
View file

@ -4,18 +4,18 @@ import (
"encoding/json"
"fmt"
"net/url"
"strings"
"text/tabwriter"
"time"
"github.com/docker/distribution/reference"
"github.com/docker/docker/api/types"
Cli "github.com/docker/docker/cli"
"github.com/docker/docker/opts"
flag "github.com/docker/docker/pkg/mflag"
"github.com/docker/docker/pkg/parsers"
"github.com/docker/docker/pkg/parsers/filters"
"github.com/docker/docker/pkg/stringid"
"github.com/docker/docker/pkg/units"
"github.com/docker/docker/utils"
)
// CmdImages lists the images in a specified repository, or all top-level images if no repository is specified.
@ -78,9 +78,9 @@ func (cli *DockerCli) CmdImages(args ...string) error {
w := tabwriter.NewWriter(cli.out, 20, 1, 3, ' ', 0)
if !*quiet {
if *showDigests {
fmt.Fprintln(w, "REPOSITORY\tTAG\tDIGEST\tIMAGE ID\tCREATED\tVIRTUAL SIZE")
fmt.Fprintln(w, "REPOSITORY\tTAG\tDIGEST\tIMAGE ID\tCREATED\tSIZE")
} else {
fmt.Fprintln(w, "REPOSITORY\tTAG\tIMAGE ID\tCREATED\tVIRTUAL SIZE")
fmt.Fprintln(w, "REPOSITORY\tTAG\tIMAGE ID\tCREATED\tSIZE")
}
}
@ -101,21 +101,31 @@ func (cli *DockerCli) CmdImages(args ...string) error {
// combine the tags and digests lists
tagsAndDigests := append(repoTags, repoDigests...)
for _, repoAndRef := range tagsAndDigests {
repo, ref := parsers.ParseRepositoryTag(repoAndRef)
// default tag and digest to none - if there's a value, it'll be set below
// default repo, tag, and digest to none - if there's a value, it'll be set below
repo := "<none>"
tag := "<none>"
digest := "<none>"
if utils.DigestReference(ref) {
digest = ref
} else {
tag = ref
if !strings.HasPrefix(repoAndRef, "<none>") {
ref, err := reference.ParseNamed(repoAndRef)
if err != nil {
return err
}
repo = ref.Name()
switch x := ref.(type) {
case reference.Digested:
digest = x.Digest().String()
case reference.Tagged:
tag = x.Tag()
}
}
if !*quiet {
if *showDigests {
fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s ago\t%s\n", repo, tag, digest, ID, units.HumanDuration(time.Now().UTC().Sub(time.Unix(int64(image.Created), 0))), units.HumanSize(float64(image.VirtualSize)))
fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s ago\t%s\n", repo, tag, digest, ID, units.HumanDuration(time.Now().UTC().Sub(time.Unix(int64(image.Created), 0))), units.HumanSize(float64(image.Size)))
} else {
fmt.Fprintf(w, "%s\t%s\t%s\t%s ago\t%s\n", repo, tag, ID, units.HumanDuration(time.Now().UTC().Sub(time.Unix(int64(image.Created), 0))), units.HumanSize(float64(image.VirtualSize)))
fmt.Fprintf(w, "%s\t%s\t%s\t%s ago\t%s\n", repo, tag, ID, units.HumanDuration(time.Now().UTC().Sub(time.Unix(int64(image.Created), 0))), units.HumanSize(float64(image.Size)))
}
} else {
fmt.Fprintln(w, ID)

9
api/client/import.go
View file

@ -6,10 +6,10 @@ import (
"net/url"
"os"
"github.com/docker/distribution/reference"
Cli "github.com/docker/docker/cli"
"github.com/docker/docker/opts"
flag "github.com/docker/docker/pkg/mflag"
"github.com/docker/docker/pkg/parsers"
"github.com/docker/docker/pkg/urlutil"
"github.com/docker/docker/registry"
)
@ -47,8 +47,11 @@ func (cli *DockerCli) CmdImport(args ...string) error {
if repository != "" {
//Check if the given image name can be resolved
repo, _ := parsers.ParseRepositoryTag(repository)
if err := registry.ValidateRepositoryName(repo); err != nil {
ref, err := reference.ParseNamed(repository)
if err != nil {
return err
}
if err := registry.ValidateRepositoryName(ref); err != nil {
return err
}
}

4
api/client/ps/custom.go
View file

@ -62,8 +62,8 @@ func (c *containerContext) Image() string {
return "<no image>"
}
if c.trunc {
if stringid.TruncateID(c.c.ImageID) == stringid.TruncateID(c.c.Image) {
return stringutils.Truncate(c.c.Image, 12)
if trunc := stringid.TruncateID(c.c.ImageID); trunc == stringid.TruncateID(c.c.Image) {
return trunc
}
}
return c.c.Image

43
api/client/pull.go
View file

@ -1,16 +1,19 @@
package client
import (
"errors"
"fmt"
"net/url"
"github.com/docker/distribution/reference"
Cli "github.com/docker/docker/cli"
"github.com/docker/docker/graph/tags"
flag "github.com/docker/docker/pkg/mflag"
"github.com/docker/docker/pkg/parsers"
"github.com/docker/docker/registry"
tagpkg "github.com/docker/docker/tag"
)
var errTagCantBeUsed = errors.New("tag can't be used with --all-tags/-a")
// CmdPull pulls an image or a repository from the registry.
//
// Usage: docker pull [OPTIONS] IMAGENAME[:TAG|@DIGEST]
@ -23,18 +26,38 @@ func (cli *DockerCli) CmdPull(args ...string) error {
cmd.ParseFlags(args, true)
remote := cmd.Arg(0)
taglessRemote, tag := parsers.ParseRepositoryTag(remote)
if tag == "" && !*allTags {
tag = tags.DefaultTag
fmt.Fprintf(cli.out, "Using default tag: %s\n", tag)
} else if tag != "" && *allTags {
return fmt.Errorf("tag can't be used with --all-tags/-a")
distributionRef, err := reference.ParseNamed(remote)
if err != nil {
return err
}
var tag string
switch x := distributionRef.(type) {
case reference.Digested:
if *allTags {
return errTagCantBeUsed
}
tag = x.Digest().String()
case reference.Tagged:
if *allTags {
return errTagCantBeUsed
}
tag = x.Tag()
default:
if !*allTags {
tag = tagpkg.DefaultTag
distributionRef, err = reference.WithTag(distributionRef, tag)
if err != nil {
return err
}
fmt.Fprintf(cli.out, "Using default tag: %s\n", tag)
}
}
ref := registry.ParseReference(tag)
// Resolve the Repository name from fqn to RepositoryInfo
repoInfo, err := registry.ParseRepositoryInfo(taglessRemote)
repoInfo, err := registry.ParseRepositoryInfo(distributionRef)
if err != nil {
return err
}
@ -46,7 +69,7 @@ func (cli *DockerCli) CmdPull(args ...string) error {
}
v := url.Values{}
v.Set("fromImage", ref.ImageName(taglessRemote))
v.Set("fromImage", distributionRef.String())
_, _, err = cli.clientRequestAttemptLogin("POST", "/images/create?"+v.Encode(), nil, cli.out, repoInfo.Index, "pull")
return err

20
api/client/push.go
View file

@ -1,12 +1,13 @@
package client
import (
"errors"
"fmt"
"net/url"
"github.com/docker/distribution/reference"
Cli "github.com/docker/docker/cli"
flag "github.com/docker/docker/pkg/mflag"
"github.com/docker/docker/pkg/parsers"
"github.com/docker/docker/registry"
)
@ -20,10 +21,21 @@ func (cli *DockerCli) CmdPush(args ...string) error {
cmd.ParseFlags(args, true)
remote, tag := parsers.ParseRepositoryTag(cmd.Arg(0))
ref, err := reference.ParseNamed(cmd.Arg(0))
if err != nil {
return err
}
var tag string
switch x := ref.(type) {
case reference.Digested:
return errors.New("cannot push a digest reference")
case reference.Tagged:
tag = x.Tag()
}
// Resolve the Repository name from fqn to RepositoryInfo
repoInfo, err := registry.ParseRepositoryInfo(remote)
repoInfo, err := registry.ParseRepositoryInfo(ref)
if err != nil {
return err
}
@ -48,6 +60,6 @@ func (cli *DockerCli) CmdPush(args ...string) error {
v := url.Values{}
v.Set("tag", tag)
_, _, err = cli.clientRequestAttemptLogin("POST", "/images/"+remote+"/push?"+v.Encode(), nil, cli.out, repoInfo.Index, "push")
_, _, err = cli.clientRequestAttemptLogin("POST", "/images/"+ref.Name()+"/push?"+v.Encode(), nil, cli.out, repoInfo.Index, "push")
return err
}

6
api/client/search.go
View file

@ -10,7 +10,6 @@ import (
Cli "github.com/docker/docker/cli"
flag "github.com/docker/docker/pkg/mflag"
"github.com/docker/docker/pkg/parsers"
"github.com/docker/docker/pkg/stringutils"
"github.com/docker/docker/registry"
)
@ -38,10 +37,7 @@ func (cli *DockerCli) CmdSearch(args ...string) error {
v := url.Values{}
v.Set("term", name)
// Resolve the Repository name from fqn to hostname + name
taglessRemote, _ := parsers.ParseRepositoryTag(name)
indexInfo, err := registry.ParseIndexInfo(taglessRemote)
indexInfo, err := registry.ParseSearchIndexInfo(name)
if err != nil {
return err
}

31
api/client/tag.go
View file

@ -1,11 +1,12 @@
package client
import (
"errors"
"net/url"
"github.com/docker/distribution/reference"
Cli "github.com/docker/docker/cli"
flag "github.com/docker/docker/pkg/mflag"
"github.com/docker/docker/pkg/parsers"
"github.com/docker/docker/registry"
)
@ -19,16 +20,28 @@ func (cli *DockerCli) CmdTag(args ...string) error {
cmd.ParseFlags(args, true)
var (
repository, tag = parsers.ParseRepositoryTag(cmd.Arg(1))
v = url.Values{}
)
//Check if the given image name can be resolved
if err := registry.ValidateRepositoryName(repository); err != nil {
v := url.Values{}
ref, err := reference.ParseNamed(cmd.Arg(1))
if err != nil {
return err
}
v.Set("repo", repository)
_, isDigested := ref.(reference.Digested)
if isDigested {
return errors.New("refusing to create a tag with a digest reference")
}
tag := ""
tagged, isTagged := ref.(reference.Tagged)
if isTagged {
tag = tagged.Tag()
}
//Check if the given image name can be resolved
if err := registry.ValidateRepositoryName(ref); err != nil {
return err
}
v.Set("repo", ref.Name())
v.Set("tag", tag)
if *force {

35
api/client/trust.go
View file

@ -19,6 +19,7 @@ import (
"github.com/Sirupsen/logrus"
"github.com/docker/distribution/digest"
"github.com/docker/distribution/reference"
"github.com/docker/distribution/registry/client/auth"
"github.com/docker/distribution/registry/client/transport"
"github.com/docker/docker/cliconfig"
@ -163,12 +164,12 @@ func (cli *DockerCli) getNotaryRepository(repoInfo *registry.RepositoryInfo, aut
}
creds := simpleCredentialStore{auth: authConfig}
tokenHandler := auth.NewTokenHandler(authTransport, creds, repoInfo.CanonicalName, "push", "pull")
tokenHandler := auth.NewTokenHandler(authTransport, creds, repoInfo.CanonicalName.Name(), "push", "pull")
basicHandler := auth.NewBasicHandler(creds)
modifiers = append(modifiers, transport.RequestModifier(auth.NewAuthorizer(challengeManager, tokenHandler, basicHandler)))
tr := transport.NewTransport(base, modifiers...)
return client.NewNotaryRepository(cli.trustDirectory(), repoInfo.CanonicalName, server, tr, cli.getPassphraseRetriever())
return client.NewNotaryRepository(cli.trustDirectory(), repoInfo.CanonicalName.Name(), server, tr, cli.getPassphraseRetriever())
}
func convertTarget(t client.Target) (target, error) {
@ -219,8 +220,8 @@ func (cli *DockerCli) getPassphraseRetriever() passphrase.Retriever {
}
}
func (cli *DockerCli) trustedReference(repo string, ref registry.Reference) (registry.Reference, error) {
repoInfo, err := registry.ParseRepositoryInfo(repo)
func (cli *DockerCli) trustedReference(ref reference.NamedTagged) (reference.Canonical, error) {
repoInfo, err := registry.ParseRepositoryInfo(ref)
if err != nil {
return nil, err
}
@ -234,7 +235,7 @@ func (cli *DockerCli) trustedReference(repo string, ref registry.Reference) (reg
return nil, err
}
t, err := notaryRepo.GetTargetByName(ref.String())
t, err := notaryRepo.GetTargetByName(ref.Tag())
if err != nil {
return nil, err
}
@ -244,18 +245,17 @@ func (cli *DockerCli) trustedReference(repo string, ref registry.Reference) (reg
}
return registry.DigestReference(r.digest), nil
return reference.WithDigest(ref, r.digest)
}
func (cli *DockerCli) tagTrusted(repoInfo *registry.RepositoryInfo, trustedRef, ref registry.Reference) error {
fullName := trustedRef.ImageName(repoInfo.LocalName)
fmt.Fprintf(cli.out, "Tagging %s as %s\n", fullName, ref.ImageName(repoInfo.LocalName))
func (cli *DockerCli) tagTrusted(trustedRef reference.Canonical, ref reference.NamedTagged) error {
fmt.Fprintf(cli.out, "Tagging %s as %s\n", trustedRef.String(), ref.String())
tv := url.Values{}
tv.Set("repo", repoInfo.LocalName)
tv.Set("tag", ref.String())
tv.Set("repo", trustedRef.Name())
tv.Set("tag", ref.Tag())
tv.Set("force", "1")
if _, _, err := readBody(cli.call("POST", "/images/"+fullName+"/tag?"+tv.Encode(), nil, nil)); err != nil {
if _, _, err := readBody(cli.call("POST", "/images/"+trustedRef.String()+"/tag?"+tv.Encode(), nil, nil)); err != nil {
return err
}
@ -317,7 +317,7 @@ func (cli *DockerCli) trustedPull(repoInfo *registry.RepositoryInfo, ref registr
refs = append(refs, r)
}
v.Set("fromImage", repoInfo.LocalName)
v.Set("fromImage", repoInfo.LocalName.Name())
for i, r := range refs {
displayTag := r.reference.String()
if displayTag != "" {
@ -333,7 +333,12 @@ func (cli *DockerCli) trustedPull(repoInfo *registry.RepositoryInfo, ref registr
// If reference is not trusted, tag by trusted reference
if !r.reference.HasDigest() {
if err := cli.tagTrusted(repoInfo, registry.DigestReference(r.digest), r.reference); err != nil {
tagged, err := reference.WithTag(repoInfo.LocalName, r.reference.String())
if err != nil {
return err
}
trustedRef, err := reference.WithDigest(repoInfo.LocalName, r.digest)
if err := cli.tagTrusted(trustedRef, tagged); err != nil {
return err
}
@ -386,7 +391,7 @@ func (cli *DockerCli) trustedPush(repoInfo *registry.RepositoryInfo, tag string,
v := url.Values{}
v.Set("tag", tag)
_, _, err := cli.clientRequestAttemptLogin("POST", "/images/"+repoInfo.LocalName+"/push?"+v.Encode(), nil, streamOut, repoInfo.Index, "push")
_, _, err := cli.clientRequestAttemptLogin("POST", "/images/"+repoInfo.LocalName.Name()+"/push?"+v.Encode(), nil, streamOut, repoInfo.Index, "push")
// Close stream channel to finish target parsing
if err := streamOut.Close(); err != nil {
return err

145
api/server/router/local/image.go
View file

@ -10,6 +10,8 @@ import (
"strings"
"github.com/Sirupsen/logrus"
"github.com/docker/distribution/digest"
"github.com/docker/distribution/reference"
"github.com/docker/docker/api/server/httputils"
"github.com/docker/docker/api/types"
"github.com/docker/docker/builder"
@ -17,17 +19,14 @@ import (
"github.com/docker/docker/cliconfig"
"github.com/docker/docker/daemon/daemonbuilder"
derr "github.com/docker/docker/errors"
"github.com/docker/docker/graph"
"github.com/docker/docker/graph/tags"
"github.com/docker/docker/pkg/archive"
"github.com/docker/docker/pkg/chrootarchive"
"github.com/docker/docker/pkg/ioutils"
"github.com/docker/docker/pkg/parsers"
"github.com/docker/docker/pkg/progressreader"
"github.com/docker/docker/pkg/streamformatter"
"github.com/docker/docker/pkg/ulimit"
"github.com/docker/docker/registry"
"github.com/docker/docker/runconfig"
tagpkg "github.com/docker/docker/tag"
"github.com/docker/docker/utils"
"golang.org/x/net/context"
)
@ -110,26 +109,55 @@ func (s *router) postImagesCreate(ctx context.Context, w http.ResponseWriter, r
w.Header().Set("Content-Type", "application/json")
if image != "" { //pull
if tag == "" {
image, tag = parsers.ParseRepositoryTag(image)
}
metaHeaders := map[string][]string{}
for k, v := range r.Header {
if strings.HasPrefix(k, "X-Meta-") {
metaHeaders[k] = v
// Special case: "pull -a" may send an image name with a
// trailing :. This is ugly, but let's not break API
// compatibility.
image = strings.TrimSuffix(image, ":")
var ref reference.Named
ref, err = reference.ParseNamed(image)
if err == nil {
if tag != "" {
// The "tag" could actually be a digest.
var dgst digest.Digest
dgst, err = digest.ParseDigest(tag)
if err == nil {
ref, err = reference.WithDigest(ref, dgst)
} else {
ref, err = reference.WithTag(ref, tag)
}
}
if err == nil {
metaHeaders := map[string][]string{}
for k, v := range r.Header {
if strings.HasPrefix(k, "X-Meta-") {
metaHeaders[k] = v
}
}
err = s.daemon.PullImage(ref, metaHeaders, authConfig, output)
}
}
imagePullConfig := &graph.ImagePullConfig{
MetaHeaders: metaHeaders,
AuthConfig: authConfig,
OutStream: output,
}
err = s.daemon.PullImage(image, tag, imagePullConfig)
} else { //import
if tag == "" {
repo, tag = parsers.ParseRepositoryTag(repo)
var newRef reference.Named
if repo != "" {
var err error
newRef, err = reference.ParseNamed(repo)
if err != nil {
return err
}
switch newRef.(type) {
case reference.Digested:
return errors.New("cannot import digest reference")
}
if tag != "" {
newRef, err = reference.WithTag(newRef, tag)
if err != nil {
return err
}
}
}
src := r.Form.Get("fromSrc")
@ -143,7 +171,7 @@ func (s *router) postImagesCreate(ctx context.Context, w http.ResponseWriter, r
return err
}
err = s.daemon.ImportImage(src, repo, tag, message, r.Body, output, newConfig)
err = s.daemon.ImportImage(src, newRef, message, r.Body, output, newConfig)
}
if err != nil {
if !output.Flushed() {
@ -183,19 +211,25 @@ func (s *router) postImagesPush(ctx context.Context, w http.ResponseWriter, r *h
}
}
name := vars["name"]
ref, err := reference.ParseNamed(vars["name"])
if err != nil {
return err
}
tag := r.Form.Get("tag")
if tag != "" {
// Push by digest is not supported, so only tags are supported.
ref, err = reference.WithTag(ref, tag)
if err != nil {
return err
}
}
output := ioutils.NewWriteFlusher(w)
defer output.Close()
imagePushConfig := &graph.ImagePushConfig{
MetaHeaders: metaHeaders,
AuthConfig: authConfig,
Tag: r.Form.Get("tag"),
OutStream: output,
}
w.Header().Set("Content-Type", "application/json")
if err := s.daemon.PushImage(name, imagePushConfig); err != nil {
if err := s.daemon.PushImage(ref, metaHeaders, authConfig, output); err != nil {
if !output.Flushed() {
return err
}
@ -428,7 +462,7 @@ func (s *router) postBuild(ctx context.Context, w http.ResponseWriter, r *http.R
}
for _, rt := range repoAndTags {
if err := s.daemon.TagImage(rt.repo, rt.tag, string(imgID), true); err != nil {
if err := s.daemon.TagImage(rt, imgID, true); err != nil {
return errf(err)
}
}
@ -436,43 +470,38 @@ func (s *router) postBuild(ctx context.Context, w http.ResponseWriter, r *http.R
return nil
}
// repoAndTag is a helper struct for holding the parsed repositories and tags of
// the input "t" argument.
type repoAndTag struct {
repo, tag string
}
// sanitizeRepoAndTags parses the raw "t" parameter received from the client
// to a slice of repoAndTag.
// It also validates each repoName and tag.
func sanitizeRepoAndTags(names []string) ([]repoAndTag, error) {
func sanitizeRepoAndTags(names []string) ([]reference.Named, error) {
var (
repoAndTags []repoAndTag
repoAndTags []reference.Named
// This map is used for deduplicating the "-t" paramter.
uniqNames = make(map[string]struct{})
)
for _, repo := range names {
name, tag := parsers.ParseRepositoryTag(repo)
if name == "" {
if repo == "" {
continue
}
if err := registry.ValidateRepositoryName(name); err != nil {
ref, err := reference.ParseNamed(repo)
if err != nil {
return nil, err
}
nameWithTag := name
if len(tag) > 0 {
if err := tags.ValidateTagName(tag); err != nil {
return nil, err
}
nameWithTag += ":" + tag
} else {
nameWithTag += ":" + tags.DefaultTag
if _, isDigested := ref.(reference.Digested); isDigested {
return nil, errors.New("build tag cannot be a digest")
}
if _, isTagged := ref.(reference.Tagged); !isTagged {
ref, err = reference.WithTag(ref, tagpkg.DefaultTag)
}
nameWithTag := ref.String()
if _, exists := uniqNames[nameWithTag]; !exists {
uniqNames[nameWithTag] = struct{}{}
repoAndTags = append(repoAndTags, repoAndTag{repo: name, tag: tag})
repoAndTags = append(repoAndTags, ref)
}
}
return repoAndTags, nil
@ -484,7 +513,7 @@ func (s *router) getImagesJSON(ctx context.Context, w http.ResponseWriter, r *ht
}
// FIXME: The filter parameter could just be a match filter
images, err := s.daemon.ListImages(r.Form.Get("filters"), r.Form.Get("filter"), httputils.BoolValue(r, "all"))
images, err := s.daemon.Images(r.Form.Get("filters"), r.Form.Get("filter"), httputils.BoolValue(r, "all"))
if err != nil {
return err
}
@ -508,9 +537,17 @@ func (s *router) postImagesTag(ctx context.Context, w http.ResponseWriter, r *ht
}
repo := r.Form.Get("repo")
tag := r.Form.Get("tag")
name := vars["name"]
newTag, err := reference.WithName(repo)
if err != nil {
return err
}
if tag != "" {
if newTag, err = reference.WithTag(newTag, tag); err != nil {
return err
}
}
force := httputils.BoolValue(r, "force")
if err := s.daemon.TagImage(repo, tag, name, force); err != nil {
if err := s.daemon.TagImage(newTag, vars["name"], force); err != nil {
return err
}
w.WriteHeader(http.StatusCreated)

2
builder/builder.go
View file

@ -125,7 +125,7 @@ type Docker interface {
// Remove removes a container specified by `id`.
Remove(id string, cfg *daemon.ContainerRmConfig) error
// Commit creates a new Docker image from an existing Docker container.
Commit(string, *daemon.ContainerCommitConfig) (*image.Image, error)
Commit(string, *daemon.ContainerCommitConfig) (string, error)
// Copy copies/extracts a source FileInfo to a destination path inside a container
// specified by a container object.
// TODO: make an Extract method instead of passing `decompress`

4
builder/dockerfile/builder.go
View file

@ -277,9 +277,9 @@ func Commit(containerName string, d *daemon.Daemon, c *CommitConfig) (string, er
MergeConfigs: true,
}
img, err := d.Commit(containerName, commitCfg)
imgID, err := d.Commit(containerName, commitCfg)
if err != nil {
return "", err
}
return img.ID, nil
return imgID, nil
}

10
builder/dockerfile/internals.go
View file

@ -83,13 +83,13 @@ func (b *Builder) commit(id string, autoCmd *stringutils.StrSlice, comment strin
}
// Commit the container
image, err := b.docker.Commit(id, commitCfg)
imageID, err := b.docker.Commit(id, commitCfg)
if err != nil {
return err
}
b.docker.Retain(b.id, image.ID)
b.activeImages = append(b.activeImages, image.ID)
b.image = image.ID
b.docker.Retain(b.id, imageID)
b.activeImages = append(b.activeImages, imageID)
b.image = imageID
return nil
}
@ -412,7 +412,7 @@ func containsWildcards(name string) bool {
}
func (b *Builder) processImageFrom(img *image.Image) error {
b.image = img.ID
b.image = img.ID().String()
if img.Config != nil {
b.runConfig = img.Config

106
daemon/commit.go
View file

@ -1,10 +1,16 @@
package daemon
import (
"encoding/json"
"fmt"
"runtime"
"strings"
"time"
"github.com/docker/distribution/reference"
"github.com/docker/docker/dockerversion"
"github.com/docker/docker/image"
"github.com/docker/docker/layer"
"github.com/docker/docker/pkg/archive"
"github.com/docker/docker/pkg/ioutils"
"github.com/docker/docker/runconfig"
@ -25,15 +31,15 @@ type ContainerCommitConfig struct {
// Commit creates a new filesystem image from the current state of a container.
// The image can optionally be tagged into a repository.
func (daemon *Daemon) Commit(name string, c *ContainerCommitConfig) (*image.Image, error) {
func (daemon *Daemon) Commit(name string, c *ContainerCommitConfig) (string, error) {
container, err := daemon.Get(name)
if err != nil {
return nil, err
return "", err
}
// It is not possible to commit a running container on Windows
if runtime.GOOS == "windows" && container.IsRunning() {
return nil, fmt.Errorf("Windows does not support commit of a running container")
return "", fmt.Errorf("Windows does not support commit of a running container")
}
if c.Pause && !container.isPaused() {
@ -43,13 +49,13 @@ func (daemon *Daemon) Commit(name string, c *ContainerCommitConfig) (*image.Imag
if c.MergeConfigs {
if err := runconfig.Merge(c.Config, container.Config); err != nil {
return nil, err
return "", err
}
}
rwTar, err := daemon.exportContainerRw(container)
if err != nil {
return nil, err
return "", err
}
defer func() {
if rwTar != nil {
@ -57,31 +63,99 @@ func (daemon *Daemon) Commit(name string, c *ContainerCommitConfig) (*image.Imag
}
}()
// Create a new image from the container's base layers + a new layer from container changes
img, err := daemon.graph.Create(rwTar, container.ID, container.ImageID, c.Comment, c.Author, container.Config, c.Config)
if err != nil {
return nil, err
var history []image.History
rootFS := image.NewRootFS()
if container.ImageID != "" {
img, err := daemon.imageStore.Get(container.ImageID)
if err != nil {
return "", err
}
history = img.History
rootFS = img.RootFS
}
l, err := daemon.layerStore.Register(rwTar, rootFS.ChainID())
if err != nil {
return "", err
}
defer layer.ReleaseAndLog(daemon.layerStore, l)
h := image.History{
Author: c.Author,
Created: time.Now().UTC(),
CreatedBy: strings.Join(container.Config.Cmd.Slice(), " "),
Comment: c.Comment,
EmptyLayer: true,
}
if diffID := l.DiffID(); layer.DigestSHA256EmptyTar != diffID {
h.EmptyLayer = false
rootFS.Append(diffID)
}
history = append(history, h)
config, err := json.Marshal(&image.Image{
V1Image: image.V1Image{
DockerVersion: dockerversion.Version,
Config: c.Config,
Architecture: runtime.GOARCH,
OS: runtime.GOOS,
Container: container.ID,
ContainerConfig: *container.Config,
Author: c.Author,
Created: h.Created,
},
RootFS: rootFS,
History: history,
})
if err != nil {
return "", err
}
id, err := daemon.imageStore.Create(config)
if err != nil {
return "", err
}
if container.ImageID != "" {
if err := daemon.imageStore.SetParent(id, container.ImageID); err != nil {
return "", err
}
}
// Register the image if needed
if c.Repo != "" {
if err := daemon.repositories.Tag(c.Repo, c.Tag, img.ID, true); err != nil {
return img, err
newTag, err := reference.WithName(c.Repo) // todo: should move this to API layer
if err != nil {
return "", err
}
if c.Tag != "" {
if newTag, err = reference.WithTag(newTag, c.Tag); err != nil {
return "", err
}
}
if err := daemon.TagImage(newTag, id.String(), true); err != nil {
return "", err
}
}
daemon.LogContainerEvent(container, "commit")
return img, nil
return id.String(), nil
}
func (daemon *Daemon) exportContainerRw(container *Container) (archive.Archive, error) {
archive, err := daemon.diff(container)
if err := daemon.Mount(container); err != nil {
return nil, err
}
archive, err := container.rwlayer.TarStream()
if err != nil {
return nil, err
}
return ioutils.NewReadCloserWrapper(archive, func() error {
err := archive.Close()
return err
return daemon.layerStore.Unmount(container.ID)
}),
nil
}

11
daemon/container.go
View file

@ -20,6 +20,8 @@ import (
"github.com/docker/docker/daemon/logger/jsonfilelog"
"github.com/docker/docker/daemon/network"
derr "github.com/docker/docker/errors"
"github.com/docker/docker/image"
"github.com/docker/docker/layer"
"github.com/docker/docker/pkg/nat"
"github.com/docker/docker/pkg/promise"
"github.com/docker/docker/pkg/signal"
@ -29,6 +31,8 @@ import (
"github.com/docker/docker/volume"
)
const configFileName = "config.v2.json"
var (
// ErrRootFSReadOnly is returned when a container
// rootfs is marked readonly.
@ -43,12 +47,13 @@ type CommonContainer struct {
*State `json:"State"` // Needed for remote api version <= 1.11
root string // Path to the "home" of the container, including metadata.
basefs string // Path to the graphdriver mountpoint
rwlayer layer.RWLayer
ID string
Created time.Time
Path string
Args []string
Config *runconfig.Config
ImageID string `json:"Image"`
ImageID image.ID `json:"Image"`
NetworkSettings *network.Settings
LogPath string
Name string
@ -256,7 +261,7 @@ func (container *Container) hostConfigPath() (string, error) {
}
func (container *Container) jsonPath() (string, error) {
return container.getRootResourcePath("config.json")
return container.getRootResourcePath(configFileName)
}
// This directory is only usable when the container is running
@ -301,7 +306,7 @@ func (container *Container) StartLogger(cfg runconfig.LogConfig) (logger.Logger,
ContainerName: container.Name,
ContainerEntrypoint: container.Path,
ContainerArgs: container.Args,
ContainerImageID: container.ImageID,
ContainerImageID: container.ImageID.String(),
ContainerImageName: container.Config.Image,
ContainerCreated: container.Created,
ContainerEnv: container.Config.Env,

2
daemon/container_unit_test.go
View file

@ -99,7 +99,7 @@ func TestContainerInitDNS(t *testing.T) {
"Name":"/ubuntu","Driver":"aufs","MountLabel":"","ProcessLabel":"","AppArmorProfile":"","RestartCount":0,
"UpdateDns":false,"Volumes":{},"VolumesRW":{},"AppliedVolumesFrom":null}`
if err = ioutil.WriteFile(filepath.Join(containerPath, "config.json"), []byte(config), 0644); err != nil {
if err = ioutil.WriteFile(filepath.Join(containerPath, configFileName), []byte(config), 0644); err != nil {
t.Fatal(err)
}

11
daemon/container_unix.go
View file

@ -19,7 +19,6 @@ import (
"github.com/docker/docker/daemon/links"
"github.com/docker/docker/daemon/network"
derr "github.com/docker/docker/errors"
"github.com/docker/docker/pkg/directory"
"github.com/docker/docker/pkg/fileutils"
"github.com/docker/docker/pkg/idtools"
"github.com/docker/docker/pkg/mount"
@ -388,8 +387,7 @@ func (daemon *Daemon) getSize(container *Container) (int64, int64) {
}
defer daemon.Unmount(container)
initID := fmt.Sprintf("%s-init", container.ID)
sizeRw, err = daemon.driver.DiffSize(container.ID, initID)
sizeRw, err = container.rwlayer.Size()
if err != nil {
logrus.Errorf("Driver %s couldn't return diff size of container %s: %s", daemon.driver, container.ID, err)
// FIXME: GetSize should return an error. Not changing it now in case
@ -397,9 +395,12 @@ func (daemon *Daemon) getSize(container *Container) (int64, int64) {
sizeRw = -1
}
if _, err = os.Stat(container.basefs); err == nil {
if sizeRootfs, err = directory.Size(container.basefs); err != nil {
if parent := container.rwlayer.Parent(); parent != nil {
sizeRootfs, err = parent.Size()
if err != nil {
sizeRootfs = -1
} else if sizeRw != -1 {
sizeRootfs += sizeRw
}
}
return sizeRw, sizeRootfs

26
daemon/container_windows.go
View file

@ -7,6 +7,7 @@ import (
"github.com/docker/docker/daemon/execdriver"
derr "github.com/docker/docker/errors"
"github.com/docker/docker/layer"
"github.com/docker/docker/volume"
"github.com/docker/libnetwork"
)
@ -98,22 +99,25 @@ func (daemon *Daemon) populateCommand(c *Container, env []string) error {
processConfig.Env = env
var layerPaths []string
img, err := daemon.graph.Get(c.ImageID)
img, err := daemon.imageStore.Get(c.ImageID)
if err != nil {
return derr.ErrorCodeGetGraph.WithArgs(c.ImageID, err)
}
for i := img; i != nil && err == nil; i, err = daemon.graph.GetParent(i) {
lp, err := daemon.driver.Get(i.ID, "")
if err != nil {
return derr.ErrorCodeGetLayer.WithArgs(daemon.driver.String(), i.ID, err)
}
layerPaths = append(layerPaths, lp)
err = daemon.driver.Put(i.ID)
if err != nil {
return derr.ErrorCodePutLayer.WithArgs(daemon.driver.String(), i.ID, err)
if img.RootFS != nil && img.RootFS.Type == "layers+base" {
max := len(img.RootFS.DiffIDs)
for i := 0; i <= max; i++ {
img.RootFS.DiffIDs = img.RootFS.DiffIDs[:i]
path, err := layer.GetLayerPath(daemon.layerStore, img.RootFS.ChainID())
if err != nil {
return derr.ErrorCodeGetLayer.WithArgs(err)
}
// Reverse order, expecting parent most first
layerPaths = append([]string{path}, layerPaths...)
}
}
m, err := daemon.driver.GetMetadata(c.ID)
m, err := layer.RWLayerMetadata(daemon.layerStore, c.ID)
if err != nil {
return derr.ErrorCodeGetLayerMetadata.WithArgs(err)
}

21
daemon/create.go
View file

@ -5,6 +5,7 @@ import (
"github.com/docker/docker/api/types"
derr "github.com/docker/docker/errors"
"github.com/docker/docker/image"
"github.com/docker/docker/pkg/idtools"
"github.com/docker/docker/pkg/stringid"
"github.com/docker/docker/runconfig"
"github.com/docker/docker/volume"
@ -34,7 +35,7 @@ func (daemon *Daemon) ContainerCreate(params *ContainerCreateConfig) (types.Cont
container, err := daemon.create(params)
if err != nil {
return types.ContainerCreateResponse{ID: "", Warnings: warnings}, daemon.graphNotExistToErrcode(params.Config.Image, err)
return types.ContainerCreateResponse{ID: "", Warnings: warnings}, daemon.imageNotExistToErrcode(err)
}
return types.ContainerCreateResponse{ID: container.ID, Warnings: warnings}, nil
@ -45,19 +46,16 @@ func (daemon *Daemon) create(params *ContainerCreateConfig) (retC *Container, re
var (
container *Container
img *image.Image
imgID string
imgID image.ID
err error
)
if params.Config.Image != "" {
img, err = daemon.repositories.LookupImage(params.Config.Image)
img, err = daemon.GetImage(params.Config.Image)
if err != nil {
return nil, err
}
if err = daemon.graph.CheckDepth(img); err != nil {
return nil, err
}
imgID = img.ID
imgID = img.ID()
}
if err := daemon.mergeAndVerifyConfig(params.Config, img); err != nil {
@ -87,15 +85,14 @@ func (daemon *Daemon) create(params *ContainerCreateConfig) (retC *Container, re
if err := daemon.Register(container); err != nil {
return nil, err
}
container.Lock()
if err := parseSecurityOpt(container, params.HostConfig); err != nil {
container.Unlock()
rootUID, rootGID, err := idtools.GetRootUIDGID(daemon.uidMaps, daemon.gidMaps)
if err != nil {
return nil, err
}
container.Unlock()
if err := daemon.createRootfs(container); err != nil {
if err := idtools.MkdirAs(container.root, 0700, rootUID, rootGID); err != nil {
return nil, err
}
if err := daemon.setHostConfig(container, params.HostConfig); err != nil {
return nil, err
}

501
daemon/daemon.go
View file

@ -18,6 +18,8 @@ import (
"time"
"github.com/Sirupsen/logrus"
"github.com/docker/distribution/digest"
"github.com/docker/distribution/reference"
"github.com/docker/docker/api"
"github.com/docker/docker/api/types"
"github.com/docker/docker/cliconfig"
@ -29,9 +31,13 @@ import (
_ "github.com/docker/docker/daemon/graphdriver/vfs" // register vfs
"github.com/docker/docker/daemon/logger"
"github.com/docker/docker/daemon/network"
"github.com/docker/docker/distribution"
dmetadata "github.com/docker/docker/distribution/metadata"
derr "github.com/docker/docker/errors"
"github.com/docker/docker/graph"
"github.com/docker/docker/image"
"github.com/docker/docker/image/tarexport"
"github.com/docker/docker/layer"
"github.com/docker/docker/migrate/v1"
"github.com/docker/docker/pkg/archive"
"github.com/docker/docker/pkg/discovery"
"github.com/docker/docker/pkg/fileutils"
@ -50,12 +56,14 @@ import (
"github.com/docker/docker/pkg/truncindex"
"github.com/docker/docker/registry"
"github.com/docker/docker/runconfig"
"github.com/docker/docker/tag"
"github.com/docker/docker/utils"
volumedrivers "github.com/docker/docker/volume/drivers"
"github.com/docker/docker/volume/local"
"github.com/docker/docker/volume/store"
"github.com/docker/libnetwork"
lntypes "github.com/docker/libnetwork/types"
"github.com/docker/libtrust"
"github.com/opencontainers/runc/libcontainer"
)
@ -66,6 +74,15 @@ var (
errSystemNotSupported = errors.New("The Docker daemon is not supported on this platform.")
)
// ErrImageDoesNotExist is error returned when no image can be found for a reference.
type ErrImageDoesNotExist struct {
RefOrID string
}
func (e ErrImageDoesNotExist) Error() string {
return fmt.Sprintf("no such id: %s", e.RefOrID)
}
type contStore struct {
s map[string]*Container
sync.Mutex
@ -103,29 +120,33 @@ func (c *contStore) List() []*Container {
// Daemon holds information about the Docker daemon.
type Daemon struct {
ID string
repository string
sysInitPath string
containers *contStore
execCommands *exec.Store
graph *graph.Graph
repositories *graph.TagStore
idIndex *truncindex.TruncIndex
configStore *Config
containerGraphDB *graphdb.Database
driver graphdriver.Driver
execDriver execdriver.Driver
statsCollector *statsCollector
defaultLogConfig runconfig.LogConfig
RegistryService *registry.Service
EventsService *events.Events
netController libnetwork.NetworkController
volumes *store.VolumeStore
discoveryWatcher discovery.Watcher
root string
shutdown bool
uidMaps []idtools.IDMap
gidMaps []idtools.IDMap
ID string
repository string
sysInitPath string
containers *contStore
execCommands *exec.Store
tagStore tag.Store
distributionPool *distribution.Pool
distributionMetadataStore dmetadata.Store
trustKey libtrust.PrivateKey
idIndex *truncindex.TruncIndex
configStore *Config
containerGraphDB *graphdb.Database
driver graphdriver.Driver
execDriver execdriver.Driver
statsCollector *statsCollector
defaultLogConfig runconfig.LogConfig
RegistryService *registry.Service
EventsService *events.Events
netController libnetwork.NetworkController
volumes *store.VolumeStore
discoveryWatcher discovery.Watcher
root string
shutdown bool
uidMaps []idtools.IDMap
gidMaps []idtools.IDMap
layerStore layer.Store
imageStore image.Store
}
// Get looks for a container using the provided information, which could be
@ -229,9 +250,7 @@ func (daemon *Daemon) Register(container *Container) error {
container.unmountIpcMounts(mount.Unmount)
if err := daemon.Unmount(container); err != nil {
logrus.Debugf("unmount error %s", err)
}
daemon.Unmount(container)
if err := container.toDiskLocking(); err != nil {
logrus.Errorf("Error saving stopped state to disk: %v", err)
}
@ -456,7 +475,7 @@ func (daemon *Daemon) getEntrypointAndArgs(configEntrypoint *stringutils.StrSlic
return cmdSlice[0], cmdSlice[1:]
}
func (daemon *Daemon) newContainer(name string, config *runconfig.Config, imgID string) (*Container, error) {
func (daemon *Daemon) newContainer(name string, config *runconfig.Config, imgID image.ID) (*Container, error) {
var (
id string
err error
@ -542,7 +561,7 @@ func (daemon *Daemon) GetLabels(id string) map[string]string {
return container.Config.Labels
}
img, err := daemon.repositories.LookupImage(id)
img, err := daemon.GetImage(id)
if err == nil {
return img.ContainerConfig.Labels
}
@ -702,8 +721,25 @@ func NewDaemon(config *Config, registryService *registry.Service) (daemon *Daemo
return nil, err
}
logrus.Debug("Creating images graph")
g, err := graph.NewGraph(filepath.Join(config.Root, "graph"), d.driver, uidMaps, gidMaps)
imageRoot := filepath.Join(config.Root, "image", d.driver.String())
fms, err := layer.NewFSMetadataStore(filepath.Join(imageRoot, "layerdb"))
if err != nil {
return nil, err
}
d.layerStore, err = layer.NewStore(fms, d.driver)
if err != nil {
return nil, err
}
distributionPool := distribution.NewPool()
ifs, err := image.NewFSStoreBackend(filepath.Join(imageRoot, "imagedb"))
if err != nil {
return nil, err
}
d.imageStore, err = image.NewImageStore(ifs, d.layerStore)
if err != nil {
return nil, err
}
@ -725,23 +761,24 @@ func NewDaemon(config *Config, registryService *registry.Service) (daemon *Daemo
return nil, err
}
eventsService := events.New()
logrus.Debug("Creating repository list")
tagCfg := &graph.TagStoreConfig{
Graph: g,
Key: trustKey,
Registry: registryService,
Events: eventsService,
}
repositories, err := graph.NewTagStore(filepath.Join(config.Root, "repositories-"+d.driver.String()), tagCfg)
distributionMetadataStore, err := dmetadata.NewFSMetadataStore(filepath.Join(imageRoot, "distribution"))
if err != nil {
return nil, fmt.Errorf("Couldn't create Tag store repositories-%s: %s", d.driver.String(), err)
return nil, err
}
if restorer, ok := d.driver.(graphdriver.ImageRestorer); ok {
if _, err := restorer.RestoreCustomImages(repositories, g); err != nil {
return nil, fmt.Errorf("Couldn't restore custom images: %s", err)
}
eventsService := events.New()
tagStore, err := tag.NewTagStore(filepath.Join(imageRoot, "repositories.json"))
if err != nil {
return nil, fmt.Errorf("Couldn't create Tag store repositories: %s", err)
}
if err := restoreCustomImage(d.driver, d.imageStore, d.layerStore, tagStore); err != nil {
return nil, fmt.Errorf("Couldn't restore custom images: %s", err)
}
if err := v1.Migrate(config.Root, d.driver.String(), d.layerStore, d.imageStore, tagStore, distributionMetadataStore); err != nil {
return nil, err
}
// Discovery is only enabled when the daemon is launched with an address to advertise. When
@ -792,8 +829,10 @@ func NewDaemon(config *Config, registryService *registry.Service) (daemon *Daemo
d.repository = daemonRepo
d.containers = &contStore{s: make(map[string]*Container)}
d.execCommands = exec.NewStore()
d.graph = g
d.repositories = repositories
d.tagStore = tagStore
d.distributionPool = distributionPool
d.distributionMetadataStore = distributionMetadataStore
d.trustKey = trustKey
d.idIndex = truncindex.NewTruncIndex([]string{})
d.configStore = config
d.sysInitPath = sysInitPath
@ -910,28 +949,44 @@ func (daemon *Daemon) Shutdown() error {
// Mount sets container.basefs
// (is it not set coming in? why is it unset?)
func (daemon *Daemon) Mount(container *Container) error {
dir, err := daemon.driver.Get(container.ID, container.getMountLabel())
if err != nil {
return fmt.Errorf("Error getting container %s from driver %s: %s", container.ID, daemon.driver, err)
var layerID layer.ChainID
if container.ImageID != "" {
img, err := daemon.imageStore.Get(container.ImageID)
if err != nil {
return err
}
layerID = img.RootFS.ChainID()
}
rwlayer, err := daemon.layerStore.Mount(container.ID, layerID, container.getMountLabel(), daemon.setupInitLayer)
if err != nil {
return err
}
dir, err := rwlayer.Path()
if err != nil {
return err
}
logrus.Debugf("container mounted via layerStore: %v", dir)
if container.basefs != dir {
// The mount path reported by the graph driver should always be trusted on Windows, since the
// volume path for a given mounted layer may change over time. This should only be an error
// on non-Windows operating systems.
if container.basefs != "" && runtime.GOOS != "windows" {
daemon.driver.Put(container.ID)
daemon.Unmount(container)
return fmt.Errorf("Error: driver %s is returning inconsistent paths for container %s ('%s' then '%s')",
daemon.driver, container.ID, container.basefs, dir)
}
}
container.basefs = dir
container.basefs = dir // TODO: combine these fields
container.rwlayer = rwlayer
return nil
}
// Unmount unsets the container base filesystem
func (daemon *Daemon) Unmount(container *Container) error {
return daemon.driver.Put(container.ID)
func (daemon *Daemon) Unmount(container *Container) {
if err := daemon.layerStore.Unmount(container.ID); err != nil {
logrus.Errorf("Error unmounting container %s: %s", container.ID, err)
}
}
// Run uses the execution driver to run a given container
@ -962,82 +1017,46 @@ func (daemon *Daemon) unsubscribeToContainerStats(c *Container, ch chan interfac
}
func (daemon *Daemon) changes(container *Container) ([]archive.Change, error) {
initID := fmt.Sprintf("%s-init", container.ID)
return daemon.driver.Changes(container.ID, initID)
}
func (daemon *Daemon) diff(container *Container) (archive.Archive, error) {
initID := fmt.Sprintf("%s-init", container.ID)
return daemon.driver.Diff(container.ID, initID)
}
func (daemon *Daemon) createRootfs(container *Container) error {
// Step 1: create the container directory.
// This doubles as a barrier to avoid race conditions.
rootUID, rootGID, err := idtools.GetRootUIDGID(daemon.uidMaps, daemon.gidMaps)
if err != nil {
return err
}
if err := idtools.MkdirAs(container.root, 0700, rootUID, rootGID); err != nil {
return err
}
initID := fmt.Sprintf("%s-init", container.ID)
if err := daemon.driver.Create(initID, container.ImageID, container.getMountLabel()); err != nil {
return err
}
initPath, err := daemon.driver.Get(initID, "")
if err != nil {
return err
}
if err := setupInitLayer(initPath, rootUID, rootGID); err != nil {
if err := daemon.driver.Put(initID); err != nil {
logrus.Errorf("Failed to Put init layer: %v", err)
}
return err
}
// We want to unmount init layer before we take snapshot of it
// for the actual container.
if err := daemon.driver.Put(initID); err != nil {
return err
}
if err := daemon.driver.Create(container.ID, initID, ""); err != nil {
return err
}
return nil
}
// Graph returns *graph.Graph which can be using for layers graph operations.
func (daemon *Daemon) Graph() *graph.Graph {
return daemon.graph
return daemon.layerStore.Changes(container.ID)
}
// TagImage creates a tag in the repository reponame, pointing to the image named
// imageName. If force is true, an existing tag with the same name may be
// overwritten.
func (daemon *Daemon) TagImage(repoName, tag, imageName string, force bool) error {
if err := daemon.repositories.Tag(repoName, tag, imageName, force); err != nil {
func (daemon *Daemon) TagImage(newTag reference.Named, imageName string, force bool) error {
if _, isDigested := newTag.(reference.Digested); isDigested {
return errors.New("refusing to create a tag with a digest reference")
}
if newTag.Name() == string(digest.Canonical) {
return errors.New("refusing to create an ambiguous tag using digest algorithm as name")
}
newTag = registry.NormalizeLocalReference(newTag)
imageID, err := daemon.GetImageID(imageName)
if err != nil {
return err
}
daemon.EventsService.Log("tag", utils.ImageReference(repoName, tag), "")
return nil
daemon.EventsService.Log("tag", newTag.String(), "")
return daemon.tagStore.Add(newTag, imageID, force)
}
// PullImage initiates a pull operation. image is the repository name to pull, and
// tag may be either empty, or indicate a specific tag to pull.
func (daemon *Daemon) PullImage(image string, tag string, imagePullConfig *graph.ImagePullConfig) error {
return daemon.repositories.Pull(image, tag, imagePullConfig)
}
func (daemon *Daemon) PullImage(ref reference.Named, metaHeaders map[string][]string, authConfig *cliconfig.AuthConfig, outStream io.Writer) error {
imagePullConfig := &distribution.ImagePullConfig{
MetaHeaders: metaHeaders,
AuthConfig: authConfig,
OutStream: outStream,
RegistryService: daemon.RegistryService,
EventsService: daemon.EventsService,
MetadataStore: daemon.distributionMetadataStore,
LayerStore: daemon.layerStore,
ImageStore: daemon.imageStore,
TagStore: daemon.tagStore,
Pool: daemon.distributionPool,
}
// ImportImage imports an image, getting the archived layer data either from
// inConfig (if src is "-"), or from a URI specified in src. Progress output is
// written to outStream. Repository and tag names can optionally be given in
// the repo and tag arguments, respectively.
func (daemon *Daemon) ImportImage(src, repo, tag, msg string, inConfig io.ReadCloser, outStream io.Writer, containerConfig *runconfig.Config) error {
return daemon.repositories.Import(src, repo, tag, msg, inConfig, outStream, containerConfig)
return distribution.Pull(ref, imagePullConfig)
}
// ExportImage exports a list of images to the given output stream. The
@ -1046,47 +1065,214 @@ func (daemon *Daemon) ImportImage(src, repo, tag, msg string, inConfig io.ReadCl
// the same tag are exported. names is the set of tags to export, and
// outStream is the writer which the images are written to.
func (daemon *Daemon) ExportImage(names []string, outStream io.Writer) error {
return daemon.repositories.ImageExport(names, outStream)
imageExporter := tarexport.NewTarExporter(daemon.imageStore, daemon.layerStore, daemon.tagStore)
return imageExporter.Save(names, outStream)
}
// PushImage initiates a push operation on the repository named localName.
func (daemon *Daemon) PushImage(localName string, imagePushConfig *graph.ImagePushConfig) error {
return daemon.repositories.Push(localName, imagePushConfig)
func (daemon *Daemon) PushImage(ref reference.Named, metaHeaders map[string][]string, authConfig *cliconfig.AuthConfig, outStream io.Writer) error {
imagePushConfig := &distribution.ImagePushConfig{
MetaHeaders: metaHeaders,
AuthConfig: authConfig,
OutStream: outStream,
RegistryService: daemon.RegistryService,
EventsService: daemon.EventsService,
MetadataStore: daemon.distributionMetadataStore,
LayerStore: daemon.layerStore,
ImageStore: daemon.imageStore,
TagStore: daemon.tagStore,
TrustKey: daemon.trustKey,
}
return distribution.Push(ref, imagePushConfig)
}
// LookupImage looks up an image by name and returns it as an ImageInspect
// structure.
func (daemon *Daemon) LookupImage(name string) (*types.ImageInspect, error) {
return daemon.repositories.Lookup(name)
img, err := daemon.GetImage(name)
if err != nil {
return nil, fmt.Errorf("No such image: %s", name)
}
refs := daemon.tagStore.References(img.ID())
repoTags := []string{}
repoDigests := []string{}
for _, ref := range refs {
switch ref.(type) {
case reference.Tagged:
repoTags = append(repoTags, ref.String())
case reference.Digested:
repoDigests = append(repoDigests, ref.String())
}
}
var size int64
var layerMetadata map[string]string
layerID := img.RootFS.ChainID()
if layerID != "" {
l, err := daemon.layerStore.Get(layerID)
if err != nil {
return nil, err
}
defer layer.ReleaseAndLog(daemon.layerStore, l)
size, err = l.Size()
if err != nil {
return nil, err
}
layerMetadata, err = l.Metadata()
if err != nil {
return nil, err
}
}
imageInspect := &types.ImageInspect{
ID: img.ID().String(),
RepoTags: repoTags,
RepoDigests: repoDigests,
Parent: img.Parent.String(),
Comment: img.Comment,
Created: img.Created.Format(time.RFC3339Nano),
Container: img.Container,
ContainerConfig: &img.ContainerConfig,
DockerVersion: img.DockerVersion,
Author: img.Author,
Config: img.Config,
Architecture: img.Architecture,
Os: img.OS,
Size: size,
VirtualSize: size, // TODO: field unused, deprecate
}
imageInspect.GraphDriver.Name = daemon.driver.String()
imageInspect.GraphDriver.Data = layerMetadata
return imageInspect, nil
}
// LoadImage uploads a set of images into the repository. This is the
// complement of ImageExport. The input stream is an uncompressed tar
// ball containing images and metadata.
func (daemon *Daemon) LoadImage(inTar io.ReadCloser, outStream io.Writer) error {
return daemon.repositories.Load(inTar, outStream)
}
// ListImages returns a filtered list of images. filterArgs is a JSON-encoded set
// of filter arguments which will be interpreted by pkg/parsers/filters.
// filter is a shell glob string applied to repository names. The argument
// named all controls whether all images in the graph are filtered, or just
// the heads.
func (daemon *Daemon) ListImages(filterArgs, filter string, all bool) ([]*types.Image, error) {
return daemon.repositories.Images(filterArgs, filter, all)
imageExporter := tarexport.NewTarExporter(daemon.imageStore, daemon.layerStore, daemon.tagStore)
return imageExporter.Load(inTar, outStream)
}
// ImageHistory returns a slice of ImageHistory structures for the specified image
// name by walking the image lineage.
func (daemon *Daemon) ImageHistory(name string) ([]*types.ImageHistory, error) {
return daemon.repositories.History(name)
img, err := daemon.GetImage(name)
if err != nil {
return nil, err
}
history := []*types.ImageHistory{}
layerCounter := 0
rootFS := *img.RootFS
rootFS.DiffIDs = nil
for _, h := range img.History {
var layerSize int64
if !h.EmptyLayer {
if len(img.RootFS.DiffIDs) <= layerCounter {
return nil, errors.New("too many non-empty layers in History section")
}
rootFS.Append(img.RootFS.DiffIDs[layerCounter])
l, err := daemon.layerStore.Get(rootFS.ChainID())
if err != nil {
return nil, err
}
layerSize, err = l.DiffSize()
layer.ReleaseAndLog(daemon.layerStore, l)
if err != nil {
return nil, err
}
layerCounter++
}
history = append([]*types.ImageHistory{{
ID: "<missing>",
Created: h.Created.Unix(),
CreatedBy: h.CreatedBy,
Comment: h.Comment,
Size: layerSize,
}}, history...)
}
// Fill in image IDs and tags
histImg := img
id := img.ID()
for _, h := range history {
h.ID = id.String()
var tags []string
for _, r := range daemon.tagStore.References(id) {
if _, ok := r.(reference.NamedTagged); ok {
tags = append(tags, r.String())
}
}
h.Tags = tags
id = histImg.Parent
if id == "" {
break
}
histImg, err = daemon.GetImage(id.String())
if err != nil {
break
}
}
return history, nil
}
// GetImage returns pointer to an Image struct corresponding to the given
// name. The name can include an optional tag; otherwise the default tag will
// be used.
func (daemon *Daemon) GetImage(name string) (*image.Image, error) {
return daemon.repositories.LookupImage(name)
// GetImageID returns an image ID corresponding to the image referred to by
// refOrID.
func (daemon *Daemon) GetImageID(refOrID string) (image.ID, error) {
// Treat as an ID
if id, err := digest.ParseDigest(refOrID); err == nil {
return image.ID(id), nil
}
// Treat it as a possible tag or digest reference
if ref, err := reference.ParseNamed(refOrID); err == nil {
ref = registry.NormalizeLocalReference(ref)
if id, err := daemon.tagStore.Get(ref); err == nil {
return id, nil
}
if tagged, ok := ref.(reference.Tagged); ok {
if id, err := daemon.imageStore.Search(tagged.Tag()); err == nil {
for _, namedRef := range daemon.tagStore.References(id) {
if namedRef.Name() == ref.Name() {
return id, nil
}
}
}
}
}
// Search based on ID
if id, err := daemon.imageStore.Search(refOrID); err == nil {
return id, nil
}
return "", ErrImageDoesNotExist{refOrID}
}
// GetImage returns an image corresponding to the image referred to by refOrID.
func (daemon *Daemon) GetImage(refOrID string) (*image.Image, error) {
imgID, err := daemon.GetImageID(refOrID)
if err != nil {
return nil, err
}
return daemon.imageStore.Get(imgID)
}
func (daemon *Daemon) config() *Config {
@ -1132,33 +1318,23 @@ func (daemon *Daemon) GetRemappedUIDGID() (int, int) {
// of the image with imgID, that had the same config when it was
// created. nil is returned if a child cannot be found. An error is
// returned if the parent image cannot be found.
func (daemon *Daemon) ImageGetCached(imgID string, config *runconfig.Config) (*image.Image, error) {
// for now just exit if imgID has no children.
// maybe parentRefs in graph could be used to store
// the Image obj children for faster lookup below but this can
// be quite memory hungry.
if !daemon.Graph().HasChildren(imgID) {
return nil, nil
}
func (daemon *Daemon) ImageGetCached(imgID image.ID, config *runconfig.Config) (*image.Image, error) {
// Retrieve all images
images := daemon.Graph().Map()
imgs := daemon.Map()
// Store the tree in a map of map (map[parentId][childId])
imageMap := make(map[string]map[string]struct{})
for _, img := range images {
if _, exists := imageMap[img.Parent]; !exists {
imageMap[img.Parent] = make(map[string]struct{})
var siblings []image.ID
for id, img := range imgs {
if img.Parent == imgID {
siblings = append(siblings, id)
}
imageMap[img.Parent][img.ID] = struct{}{}
}
// Loop on the children of the given image and check the config
var match *image.Image
for elem := range imageMap[imgID] {
img, ok := images[elem]
for _, id := range siblings {
img, ok := imgs[id]
if !ok {
return nil, fmt.Errorf("unable to find image %q", elem)
return nil, fmt.Errorf("unable to find image %q", id)
}
if runconfig.Compare(&img.ContainerConfig, config) {
if match == nil || match.Created.Before(img.Created) {
@ -1179,6 +1355,12 @@ func tempDir(rootDir string, rootUID, rootGID int) (string, error) {
}
func (daemon *Daemon) setHostConfig(container *Container, hostConfig *runconfig.HostConfig) error {
container.Lock()
if err := parseSecurityOpt(container, hostConfig); err != nil {
container.Unlock()
return err
}
container.Unlock()
// Do not lock while creating volumes since this could be calling out to external plugins
// Don't want to block other actions, like `docker ps` because we're waiting on an external plugin
@ -1199,6 +1381,11 @@ func (daemon *Daemon) setHostConfig(container *Container, hostConfig *runconfig.
return nil
}
func (daemon *Daemon) setupInitLayer(initPath string) error {
rootUID, rootGID := daemon.GetRemappedUIDGID()
return setupInitLayer(initPath, rootUID, rootGID)
}
func setDefaultMtu(config *Config) {
// do nothing if the config does not have the default 0 value.
if config.Mtu != 0 {

12
daemon/daemon_unix.go
View file

@ -14,12 +14,15 @@ import (
"github.com/Sirupsen/logrus"
"github.com/docker/docker/daemon/graphdriver"
derr "github.com/docker/docker/errors"
"github.com/docker/docker/image"
"github.com/docker/docker/layer"
pblkiodev "github.com/docker/docker/pkg/blkiodev"
"github.com/docker/docker/pkg/idtools"
"github.com/docker/docker/pkg/parsers"
"github.com/docker/docker/pkg/parsers/kernel"
"github.com/docker/docker/pkg/sysinfo"
"github.com/docker/docker/runconfig"
"github.com/docker/docker/tag"
"github.com/docker/libnetwork"
nwconfig "github.com/docker/libnetwork/config"
"github.com/docker/libnetwork/drivers/bridge"
@ -601,9 +604,7 @@ func (daemon *Daemon) conditionalMountOnStart(container *Container) error {
// conditionalUnmountOnCleanup is a platform specific helper function called
// during the cleanup of a container to unmount.
func (daemon *Daemon) conditionalUnmountOnCleanup(container *Container) {
if err := daemon.Unmount(container); err != nil {
logrus.Errorf("%v: Failed to umount filesystem: %v", container.ID, err)
}
daemon.Unmount(container)
}
// getDefaultRouteMtu returns the MTU for the default route's interface.
@ -624,3 +625,8 @@ func getDefaultRouteMtu() (int, error) {
}
return 0, errNoDefaultRoute
}
func restoreCustomImage(driver graphdriver.Driver, is image.Store, ls layer.Store, ts tag.Store) error {
// Unix has no custom images to register
return nil
}

81
daemon/daemon_windows.go
View file

@ -1,12 +1,22 @@
package daemon
import (
"encoding/json"
"errors"
"fmt"
"path/filepath"
"runtime"
"strings"
"github.com/Sirupsen/logrus"
"github.com/docker/distribution/reference"
"github.com/docker/docker/daemon/graphdriver"
"github.com/docker/docker/dockerversion"
"github.com/docker/docker/image"
"github.com/docker/docker/layer"
"github.com/docker/docker/tag"
// register the windows graph driver
_ "github.com/docker/docker/daemon/graphdriver/windows"
"github.com/docker/docker/daemon/graphdriver/windows"
"github.com/docker/docker/pkg/system"
"github.com/docker/docker/runconfig"
"github.com/docker/libnetwork"
@ -128,8 +138,71 @@ func (daemon *Daemon) conditionalMountOnStart(container *Container) error {
func (daemon *Daemon) conditionalUnmountOnCleanup(container *Container) {
// We do not unmount if a Hyper-V container
if !container.hostConfig.Isolation.IsHyperV() {
if err := daemon.Unmount(container); err != nil {
logrus.Errorf("%v: Failed to umount filesystem: %v", container.ID, err)
}
daemon.Unmount(container)
}
}
func restoreCustomImage(driver graphdriver.Driver, is image.Store, ls layer.Store, ts tag.Store) error {
if wd, ok := driver.(*windows.Driver); ok {
imageInfos, err := wd.GetCustomImageInfos()
if err != nil {
return err
}
// Convert imageData to valid image configuration
for i := range imageInfos {
name := strings.ToLower(imageInfos[i].Name)
type registrar interface {
RegisterDiffID(graphID string, size int64) (layer.Layer, error)
}
r, ok := ls.(registrar)
if !ok {
return errors.New("Layerstore doesn't support RegisterDiffID")
}
if _, err := r.RegisterDiffID(imageInfos[i].ID, imageInfos[i].Size); err != nil {
return err
}
// layer is intentionally not released
rootFS := image.NewRootFS()
rootFS.BaseLayer = filepath.Base(imageInfos[i].Path)
// Create history for base layer
config, err := json.Marshal(&image.Image{
V1Image: image.V1Image{
DockerVersion: dockerversion.Version,
Architecture: runtime.GOARCH,
OS: runtime.GOOS,
Created: imageInfos[i].CreatedTime,
},
RootFS: rootFS,
History: []image.History{},
})
named, err := reference.ParseNamed(name)
if err != nil {
return err
}
ref, err := reference.WithTag(named, imageInfos[i].Version)
if err != nil {
return err
}
id, err := is.Create(config)
if err != nil {
return err
}
if err := ts.Add(ref, id, true); err != nil {
return err
}
logrus.Debugf("Registered base layer %s as %s", ref, id)
}
}
return nil
}

42
daemon/daemonbuilder/builder.go
View file

@ -9,17 +9,16 @@ import (
"strings"
"github.com/Sirupsen/logrus"
"github.com/docker/distribution/reference"
"github.com/docker/docker/api"
"github.com/docker/docker/builder"
"github.com/docker/docker/cliconfig"
"github.com/docker/docker/daemon"
"github.com/docker/docker/graph"
"github.com/docker/docker/image"
"github.com/docker/docker/pkg/archive"
"github.com/docker/docker/pkg/httputils"
"github.com/docker/docker/pkg/idtools"
"github.com/docker/docker/pkg/ioutils"
"github.com/docker/docker/pkg/parsers"
"github.com/docker/docker/pkg/progressreader"
"github.com/docker/docker/pkg/urlutil"
"github.com/docker/docker/registry"
@ -44,15 +43,24 @@ func (d Docker) LookupImage(name string) (*image.Image, error) {
// Pull tells Docker to pull image referenced by `name`.
func (d Docker) Pull(name string) (*image.Image, error) {
remote, tag := parsers.ParseRepositoryTag(name)
if tag == "" {
tag = "latest"
ref, err := reference.ParseNamed(name)
if err != nil {
return nil, err
}
switch ref.(type) {
case reference.Tagged:
case reference.Digested:
default:
ref, err = reference.WithTag(ref, "latest")
if err != nil {
return nil, err
}
}
pullRegistryAuth := &cliconfig.AuthConfig{}
if len(d.AuthConfigs) > 0 {
// The request came with a full auth config file, we prefer to use that
repoInfo, err := d.Daemon.RegistryService.ResolveRepository(remote)
repoInfo, err := d.Daemon.RegistryService.ResolveRepository(ref)
if err != nil {
return nil, err
}
@ -64,12 +72,7 @@ func (d Docker) Pull(name string) (*image.Image, error) {
pullRegistryAuth = &resolvedConfig
}
imagePullConfig := &graph.ImagePullConfig{
AuthConfig: pullRegistryAuth,
OutStream: ioutils.NopWriteCloser(d.OutOld),
}
if err := d.Daemon.PullImage(remote, tag, imagePullConfig); err != nil {
if err := d.Daemon.PullImage(ref, nil, pullRegistryAuth, ioutils.NopWriteCloser(d.OutOld)); err != nil {
return nil, err
}
@ -106,18 +109,20 @@ func (d Docker) Remove(id string, cfg *daemon.ContainerRmConfig) error {
}
// Commit creates a new Docker image from an existing Docker container.
func (d Docker) Commit(name string, cfg *daemon.ContainerCommitConfig) (*image.Image, error) {
func (d Docker) Commit(name string, cfg *daemon.ContainerCommitConfig) (string, error) {
return d.Daemon.Commit(name, cfg)
}
// Retain retains an image avoiding it to be removed or overwritten until a corresponding Release() call.
func (d Docker) Retain(sessionID, imgID string) {
d.Daemon.Graph().Retain(sessionID, imgID)
// FIXME: This will be solved with tags in client-side builder
//d.Daemon.Graph().Retain(sessionID, imgID)
}
// Release releases a list of images that were retained for the time of a build.
func (d Docker) Release(sessionID string, activeImages []string) {
d.Daemon.Graph().Release(sessionID, activeImages...)
// FIXME: This will be solved with tags in client-side builder
//d.Daemon.Graph().Release(sessionID, activeImages...)
}
// Copy copies/extracts a source FileInfo to a destination path inside a container
@ -199,11 +204,11 @@ func (d Docker) Copy(c *daemon.Container, destPath string, src builder.FileInfo,
// GetCachedImage returns a reference to a cached image whose parent equals `parent`
// and runconfig equals `cfg`. A cache miss is expected to return an empty ID and a nil error.
func (d Docker) GetCachedImage(imgID string, cfg *runconfig.Config) (string, error) {
cache, err := d.Daemon.ImageGetCached(imgID, cfg)
cache, err := d.Daemon.ImageGetCached(image.ID(imgID), cfg)
if cache == nil || err != nil {
return "", err
}
return cache.ID, nil
return cache.ID().String(), nil
}
// Kill stops the container execution abruptly.
@ -218,7 +223,8 @@ func (d Docker) Mount(c *daemon.Container) error {
// Unmount unmounts the root filesystem for the container.
func (d Docker) Unmount(c *daemon.Container) error {
return d.Daemon.Unmount(c)
d.Daemon.Unmount(c)
return nil
}
// Start starts a container

11
daemon/delete.go
View file

@ -1,12 +1,12 @@
package daemon
import (
"fmt"
"os"
"path"
"github.com/Sirupsen/logrus"
derr "github.com/docker/docker/errors"
"github.com/docker/docker/layer"
volumestore "github.com/docker/docker/volume/store"
)
@ -119,15 +119,12 @@ func (daemon *Daemon) rm(container *Container, forceRemove bool) (err error) {
logrus.Debugf("Unable to remove container from link graph: %s", err)
}
if err = daemon.driver.Remove(container.ID); err != nil {
metadata, err := daemon.layerStore.DeleteMount(container.ID)
layer.LogReleaseMetadata(metadata)
if err != nil {
return derr.ErrorCodeRmDriverFS.WithArgs(daemon.driver, container.ID, err)
}
initID := fmt.Sprintf("%s-init", container.ID)
if err := daemon.driver.Remove(initID); err != nil {
return derr.ErrorCodeRmInit.WithArgs(daemon.driver, initID, err)
}
if err = os.RemoveAll(container.root); err != nil {
return derr.ErrorCodeRmFS.WithArgs(container.ID, err)
}

24
daemon/errors.go
View file

@ -3,21 +3,25 @@ package daemon
import (
"strings"
"github.com/docker/distribution/reference"
derr "github.com/docker/docker/errors"
"github.com/docker/docker/graph/tags"
"github.com/docker/docker/pkg/parsers"
tagpkg "github.com/docker/docker/tag"
)
func (d *Daemon) graphNotExistToErrcode(imageName string, err error) error {
if d.Graph().IsNotExist(err, imageName) {
if strings.Contains(imageName, "@") {
return derr.ErrorCodeNoSuchImageHash.WithArgs(imageName)
func (d *Daemon) imageNotExistToErrcode(err error) error {
if dne, isDNE := err.(ErrImageDoesNotExist); isDNE {
if strings.Contains(dne.RefOrID, "@") {
return derr.ErrorCodeNoSuchImageHash.WithArgs(dne.RefOrID)
}
img, tag := parsers.ParseRepositoryTag(imageName)
if tag == "" {
tag = tags.DefaultTag
tag := tagpkg.DefaultTag
ref, err := reference.ParseNamed(dne.RefOrID)
if err != nil {
return derr.ErrorCodeNoSuchImageTag.WithArgs(dne.RefOrID, tag)
}
return derr.ErrorCodeNoSuchImageTag.WithArgs(img, tag)
if tagged, isTagged := ref.(reference.Tagged); isTagged {
tag = tagged.Tag()
}
return derr.ErrorCodeNoSuchImageTag.WithArgs(ref.Name(), tag)
}
return err
}

9
daemon/events/filter.go
View file

@ -1,8 +1,8 @@
package events
import (
"github.com/docker/distribution/reference"
"github.com/docker/docker/pkg/jsonmessage"
"github.com/docker/docker/pkg/parsers"
"github.com/docker/docker/pkg/parsers/filters"
)
@ -38,8 +38,11 @@ func (ef *Filter) isLabelFieldIncluded(id string) bool {
// against the stripped repo name without any tags.
func (ef *Filter) isImageIncluded(eventID string, eventFrom string) bool {
stripTag := func(image string) string {
repo, _ := parsers.ParseRepositoryTag(image)
return repo
ref, err := reference.ParseNamed(image)
if err != nil {
return image
}
return ref.Name()
}
return isFieldIncluded(eventID, ef.filter["image"]) ||

10
daemon/graphdriver/fsdiff.go
View file

@ -1,5 +1,3 @@
// +build daemon
package graphdriver
import (
@ -13,6 +11,12 @@ import (
"github.com/docker/docker/pkg/ioutils"
)
var (
// ApplyUncompressedLayer defines the unpack method used by the graph
// driver.
ApplyUncompressedLayer = chrootarchive.ApplyUncompressedLayer
)
// NaiveDiffDriver takes a ProtoDriver and adds the
// capability of the Diffing methods which it may or may not
// support on its own. See the comment on the exported
@ -129,7 +133,7 @@ func (gdw *NaiveDiffDriver) ApplyDiff(id, parent string, diff archive.Reader) (s
GIDMaps: gdw.gidMaps}
start := time.Now().UTC()
logrus.Debugf("Start untar layer")
if size, err = chrootarchive.ApplyUncompressedLayer(layerFs, diff, options); err != nil {
if size, err = ApplyUncompressedLayer(layerFs, diff, options); err != nil {
return
}
logrus.Debugf("Untar time: %vs", time.Now().UTC().Sub(start).Seconds())

31
daemon/graphdriver/imagerestorer.go
View file

@ -1,31 +0,0 @@
package graphdriver
import (
"io"
"github.com/docker/docker/image"
)
// NOTE: These interfaces are used for implementing specific features of the Windows
// graphdriver implementation. The current versions are a short-term solution and
// likely to change or possibly be eliminated, so avoid using them outside of the Windows
// graphdriver code.
// ImageRestorer interface allows the implementer to add a custom image to
// the graph and tagstore.
type ImageRestorer interface {
RestoreCustomImages(tagger Tagger, recorder Recorder) ([]string, error)
}
// Tagger is an interface that exposes the TagStore.Tag function without needing
// to import graph.
type Tagger interface {
Tag(repoName, tag, imageName string, force bool) error
}
// Recorder is an interface that exposes the Graph.Register and Graph.Exists
// functions without needing to import graph.
type Recorder interface {
Exists(id string) bool
Register(img image.Descriptor, layerData io.Reader) error
}

1
daemon/graphdriver/plugin.go
View file

@ -1,5 +1,4 @@
// +build experimental
// +build daemon
package graphdriver

1
daemon/graphdriver/proxy.go
View file

@ -1,5 +1,4 @@
// +build experimental
// +build daemon
package graphdriver

9
daemon/graphdriver/vfs/driver.go
View file

@ -1,5 +1,3 @@
// +build daemon
package vfs
import (
@ -14,6 +12,11 @@ import (
"github.com/opencontainers/runc/libcontainer/label"
)
var (
// CopyWithTar defines the copy method to use.
CopyWithTar = chrootarchive.CopyWithTar
)
func init() {
graphdriver.Register("vfs", Init)
}
@ -89,7 +92,7 @@ func (d *Driver) Create(id, parent, mountLabel string) error {
if err != nil {
return fmt.Errorf("%s: %s", parent, err)
}
if err := chrootarchive.CopyWithTar(parentDir, dir); err != nil {
if err := CopyWithTar(parentDir, dir); err != nil {
return err
}
return nil

3
daemon/graphdriver/vfs/driver_unsupported.go
View file

@ -1,3 +0,0 @@
// +build !daemon
package vfs

196
daemon/graphdriver/windows/windows.go
View file

@ -6,10 +6,10 @@ import (
"crypto/sha512"
"encoding/json"
"fmt"
"io"
"io/ioutil"
"os"
"path/filepath"
"runtime"
"strconv"
"strings"
"sync"
@ -17,8 +17,6 @@ import (
"github.com/Sirupsen/logrus"
"github.com/docker/docker/daemon/graphdriver"
"github.com/docker/docker/dockerversion"
"github.com/docker/docker/image"
"github.com/docker/docker/pkg/archive"
"github.com/docker/docker/pkg/chrootarchive"
"github.com/docker/docker/pkg/idtools"
@ -40,26 +38,6 @@ const (
filterDriver
)
// CustomImageDescriptor is an image descriptor for use by RestoreCustomImages
type customImageDescriptor struct {
img *image.Image
}
// ID returns the image ID specified in the image structure.
func (img customImageDescriptor) ID() string {
return img.img.ID
}
// Parent returns the parent ID - in this case, none
func (img customImageDescriptor) Parent() string {
return ""
}
// MarshalConfig renders the image structure into JSON.
func (img customImageDescriptor) MarshalConfig() ([]byte, error) {
return json.Marshal(img.img)
}
// Driver represents a windows graph driver.
type Driver struct {
// info stores the shim driver information
@ -195,7 +173,7 @@ func (d *Driver) Remove(id string) error {
if err != nil {
return err
}
os.RemoveAll(filepath.Join(d.info.HomeDir, "sysfile-backups", rID)) // ok to fail
return hcsshim.DestroyLayer(d.info, rID)
}
@ -402,22 +380,27 @@ func (d *Driver) DiffSize(id, parent string) (size int64, err error) {
return archive.ChangesSize(layerFs, changes), nil
}
// RestoreCustomImages adds any auto-detected OS specific images to the tag and graph store.
func (d *Driver) RestoreCustomImages(tagger graphdriver.Tagger, recorder graphdriver.Recorder) (imageIDs []string, err error) {
// CustomImageInfo is the object returned by the driver describing the base
// image.
type CustomImageInfo struct {
ID string
Name string
Version string
Path string
Size int64
CreatedTime time.Time
}
// GetCustomImageInfos returns the image infos for window specific
// base images which should always be present.
func (d *Driver) GetCustomImageInfos() ([]CustomImageInfo, error) {
strData, err := hcsshim.GetSharedBaseImages()
if err != nil {
return nil, fmt.Errorf("Failed to restore base images: %s", err)
}
type customImageInfo struct {
Name string
Version string
Path string
Size int64
CreatedTime time.Time
}
type customImageInfoList struct {
Images []customImageInfo
Images []CustomImageInfo
}
var infoData customImageInfoList
@ -428,43 +411,28 @@ func (d *Driver) RestoreCustomImages(tagger graphdriver.Tagger, recorder graphdr
return nil, err
}
var images []CustomImageInfo
for _, imageData := range infoData.Images {
_, folderName := filepath.Split(imageData.Path)
folderName := filepath.Base(imageData.Path)
// Use crypto hash of the foldername to generate a docker style id.
h := sha512.Sum384([]byte(folderName))
id := fmt.Sprintf("%x", h[:32])
if !recorder.Exists(id) {
// Register the image.
img := &image.Image{
ID: id,
Created: imageData.CreatedTime,
DockerVersion: dockerversion.Version,
Architecture: runtime.GOARCH,
OS: runtime.GOOS,
Size: imageData.Size,
}
if err := recorder.Register(customImageDescriptor{img}, nil); err != nil {
return nil, err
}
// Create tags for the new image.
if err := tagger.Tag(strings.ToLower(imageData.Name), imageData.Version, img.ID, true); err != nil {
return nil, err
}
// Create the alternate ID file.
if err := d.setID(img.ID, folderName); err != nil {
return nil, err
}
imageIDs = append(imageIDs, img.ID)
if err := d.Create(id, "", ""); err != nil {
return nil, err
}
// Create the alternate ID file.
if err := d.setID(id, folderName); err != nil {
return nil, err
}
imageData.ID = id
images = append(images, imageData)
}
return imageIDs, nil
return images, nil
}
// GetMetadata returns custom driver information.
@ -533,6 +501,10 @@ func (d *Driver) importLayer(id string, layerData archive.Reader, parentLayerPat
if size, err = chrootarchive.ApplyLayer(tempFolder, layerData); err != nil {
return
}
err = copySysFiles(tempFolder, filepath.Join(d.info.HomeDir, "sysfile-backups", id))
if err != nil {
return
}
logrus.Debugf("Untar time: %vs", time.Now().UTC().Sub(start).Seconds())
if err = hcsshim.ImportLayer(d.info, id, tempFolder, parentLayerPaths); err != nil {
@ -596,3 +568,103 @@ func (d *Driver) setLayerChain(id string, chain []string) error {
return nil
}
// DiffPath returns a directory that contains files needed to construct layer diff.
func (d *Driver) DiffPath(id string) (path string, release func() error, err error) {
id, err = d.resolveID(id)
if err != nil {
return
}
// Getting the layer paths must be done outside of the lock.
layerChain, err := d.getLayerChain(id)
if err != nil {
return
}
layerFolder := d.dir(id)
tempFolder := layerFolder + "-" + strconv.FormatUint(uint64(random.Rand.Uint32()), 10)
if err = os.MkdirAll(tempFolder, 0755); err != nil {
logrus.Errorf("Could not create %s %s", tempFolder, err)
return
}
defer func() {
if err != nil {
_, folderName := filepath.Split(tempFolder)
if err2 := hcsshim.DestroyLayer(d.info, folderName); err2 != nil {
logrus.Warnf("Couldn't clean-up tempFolder: %s %s", tempFolder, err2)
}
}
}()
if err = hcsshim.ExportLayer(d.info, id, tempFolder, layerChain); err != nil {
return
}
err = copySysFiles(filepath.Join(d.info.HomeDir, "sysfile-backups", id), tempFolder)
if err != nil {
return
}
return tempFolder, func() error {
// TODO: activate layers and release here?
_, folderName := filepath.Split(tempFolder)
return hcsshim.DestroyLayer(d.info, folderName)
}, nil
}
var sysFileWhiteList = []string{
"Hives\\*",
"Files\\BOOTNXT",
"tombstones.txt",
}
// note this only handles files
func copySysFiles(src string, dest string) error {
if err := os.MkdirAll(dest, 0700); err != nil {
return err
}
return filepath.Walk(src, func(path string, info os.FileInfo, err error) error {
rel, err := filepath.Rel(src, path)
if err != nil {
return err
}
for _, sysfile := range sysFileWhiteList {
if matches, err := filepath.Match(sysfile, rel); err != nil || !matches {
continue
}
fi, err := os.Lstat(path)
if err != nil {
return err
}
if !fi.Mode().IsRegular() {
continue
}
targetPath := filepath.Join(dest, rel)
if err = os.MkdirAll(filepath.Dir(targetPath), 0700); err != nil {
return err
}
in, err := os.Open(path)
if err != nil {
return err
}
out, err := os.Create(targetPath)
if err != nil {
in.Close()
return err
}
_, err = io.Copy(out, in)
in.Close()
out.Close()
if err != nil {
return err
}
}
return nil
})
}

157
daemon/image_delete.go
View file

@ -4,13 +4,12 @@ import (
"fmt"
"strings"
"github.com/docker/distribution/reference"
"github.com/docker/docker/api/types"
derr "github.com/docker/docker/errors"
"github.com/docker/docker/graph/tags"
"github.com/docker/docker/image"
"github.com/docker/docker/pkg/parsers"
"github.com/docker/docker/pkg/stringid"
"github.com/docker/docker/utils"
tagpkg "github.com/docker/docker/tag"
)
// ImageDelete deletes the image referenced by the given imageRef from this
@ -53,39 +52,46 @@ import (
func (daemon *Daemon) ImageDelete(imageRef string, force, prune bool) ([]types.ImageDelete, error) {
records := []types.ImageDelete{}
img, err := daemon.repositories.LookupImage(imageRef)
imgID, err := daemon.GetImageID(imageRef)
if err != nil {
return nil, daemon.graphNotExistToErrcode(imageRef, err)
return nil, daemon.imageNotExistToErrcode(err)
}
repoRefs := daemon.tagStore.References(imgID)
var removedRepositoryRef bool
if !isImageIDPrefix(img.ID, imageRef) {
if !isImageIDPrefix(imgID.String(), imageRef) {
// A repository reference was given and should be removed
// first. We can only remove this reference if either force is
// true, there are multiple repository references to this
// image, or there are no containers using the given reference.
if !(force || daemon.imageHasMultipleRepositoryReferences(img.ID)) {
if container := daemon.getContainerUsingImage(img.ID); container != nil {
if !(force || len(repoRefs) > 1) {
if container := daemon.getContainerUsingImage(imgID); container != nil {
// If we removed the repository reference then
// this image would remain "dangling" and since
// we really want to avoid that the client must
// explicitly force its removal.
return nil, derr.ErrorCodeImgDelUsed.WithArgs(imageRef, stringid.TruncateID(container.ID), stringid.TruncateID(img.ID))
return nil, derr.ErrorCodeImgDelUsed.WithArgs(imageRef, stringid.TruncateID(container.ID), stringid.TruncateID(imgID.String()))
}
}
parsedRef, err := daemon.removeImageRef(imageRef)
parsedRef, err := reference.ParseNamed(imageRef)
if err != nil {
return nil, err
}
untaggedRecord := types.ImageDelete{Untagged: parsedRef}
parsedRef, err = daemon.removeImageRef(parsedRef)
if err != nil {
return nil, err
}
daemon.EventsService.Log("untag", img.ID, "")
untaggedRecord := types.ImageDelete{Untagged: parsedRef.String()}
daemon.EventsService.Log("untag", imgID.String(), "")
records = append(records, untaggedRecord)
// If has remaining references then untag finishes the remove
if daemon.repositories.HasReferences(img) {
if len(repoRefs) > 1 {
return records, nil
}
@ -95,38 +101,39 @@ func (daemon *Daemon) ImageDelete(imageRef string, force, prune bool) ([]types.I
// repository reference to the image then we will want to
// remove that reference.
// FIXME: Is this the behavior we want?
repoRefs := daemon.repositories.ByID()[img.ID]
if len(repoRefs) == 1 {
parsedRef, err := daemon.removeImageRef(repoRefs[0])
if err != nil {
return nil, err
}
untaggedRecord := types.ImageDelete{Untagged: parsedRef}
untaggedRecord := types.ImageDelete{Untagged: parsedRef.String()}
daemon.EventsService.Log("untag", img.ID, "")
daemon.EventsService.Log("untag", imgID.String(), "")
records = append(records, untaggedRecord)
}
}
return records, daemon.imageDeleteHelper(img, &records, force, prune, removedRepositoryRef)
return records, daemon.imageDeleteHelper(imgID, &records, force, prune, removedRepositoryRef)
}
// isImageIDPrefix returns whether the given possiblePrefix is a prefix of the
// given imageID.
func isImageIDPrefix(imageID, possiblePrefix string) bool {
return strings.HasPrefix(imageID, possiblePrefix)
}
if strings.HasPrefix(imageID, possiblePrefix) {
return true
}
// imageHasMultipleRepositoryReferences returns whether there are multiple
// repository references to the given imageID.
func (daemon *Daemon) imageHasMultipleRepositoryReferences(imageID string) bool {
return len(daemon.repositories.ByID()[imageID]) > 1
if i := strings.IndexRune(imageID, ':'); i >= 0 {
return strings.HasPrefix(imageID[i+1:], possiblePrefix)
}
return false
}
// getContainerUsingImage returns a container that was created using the given
// imageID. Returns nil if there is no such container.
func (daemon *Daemon) getContainerUsingImage(imageID string) *Container {
func (daemon *Daemon) getContainerUsingImage(imageID image.ID) *Container {
for _, container := range daemon.List() {
if container.ImageID == imageID {
return container
@ -141,18 +148,24 @@ func (daemon *Daemon) getContainerUsingImage(imageID string) *Container {
// repositoryRef must not be an image ID but a repository name followed by an
// optional tag or digest reference. If tag or digest is omitted, the default
// tag is used. Returns the resolved image reference and an error.
func (daemon *Daemon) removeImageRef(repositoryRef string) (string, error) {
repository, ref := parsers.ParseRepositoryTag(repositoryRef)
if ref == "" {
ref = tags.DefaultTag
func (daemon *Daemon) removeImageRef(ref reference.Named) (reference.Named, error) {
switch ref.(type) {
case reference.Tagged:
case reference.Digested:
default:
var err error
ref, err = reference.WithTag(ref, tagpkg.DefaultTag)
if err != nil {
return nil, err
}
}
// Ignore the boolean value returned, as far as we're concerned, this
// is an idempotent operation and it's okay if the reference didn't
// exist in the first place.
_, err := daemon.repositories.Delete(repository, ref)
_, err := daemon.tagStore.Delete(ref)
return utils.ImageReference(repository, ref), err
return ref, err
}
// removeAllReferencesToImageID attempts to remove every reference to the given
@ -160,8 +173,8 @@ func (daemon *Daemon) removeImageRef(repositoryRef string) (string, error) {
// on the first encountered error. Removed references are logged to this
// daemon's event service. An "Untagged" types.ImageDelete is added to the
// given list of records.
func (daemon *Daemon) removeAllReferencesToImageID(imgID string, records *[]types.ImageDelete) error {
imageRefs := daemon.repositories.ByID()[imgID]
func (daemon *Daemon) removeAllReferencesToImageID(imgID image.ID, records *[]types.ImageDelete) error {
imageRefs := daemon.tagStore.References(imgID)
for _, imageRef := range imageRefs {
parsedRef, err := daemon.removeImageRef(imageRef)
@ -169,9 +182,9 @@ func (daemon *Daemon) removeAllReferencesToImageID(imgID string, records *[]type
return err
}
untaggedRecord := types.ImageDelete{Untagged: parsedRef}
untaggedRecord := types.ImageDelete{Untagged: parsedRef.String()}
daemon.EventsService.Log("untag", imgID, "")
daemon.EventsService.Log("untag", imgID.String(), "")
*records = append(*records, untaggedRecord)
}
@ -182,7 +195,7 @@ func (daemon *Daemon) removeAllReferencesToImageID(imgID string, records *[]type
// Implements the error interface.
type imageDeleteConflict struct {
hard bool
imgID string
imgID image.ID
message string
}
@ -194,7 +207,7 @@ func (idc *imageDeleteConflict) Error() string {
forceMsg = "must be forced"
}
return fmt.Sprintf("conflict: unable to delete %s (%s) - %s", stringid.TruncateID(idc.imgID), forceMsg, idc.message)
return fmt.Sprintf("conflict: unable to delete %s (%s) - %s", stringid.TruncateID(idc.imgID.String()), forceMsg, idc.message)
}
// imageDeleteHelper attempts to delete the given image from this daemon. If
@ -208,11 +221,11 @@ func (idc *imageDeleteConflict) Error() string {
// conflict is encountered, it will be returned immediately without deleting
// the image. If quiet is true, any encountered conflicts will be ignored and
// the function will return nil immediately without deleting the image.
func (daemon *Daemon) imageDeleteHelper(img *image.Image, records *[]types.ImageDelete, force, prune, quiet bool) error {
func (daemon *Daemon) imageDeleteHelper(imgID image.ID, records *[]types.ImageDelete, force, prune, quiet bool) error {
// First, determine if this image has any conflicts. Ignore soft conflicts
// if force is true.
if conflict := daemon.checkImageDeleteConflict(img, force); conflict != nil {
if quiet && !daemon.imageIsDangling(img) {
if conflict := daemon.checkImageDeleteConflict(imgID, force); conflict != nil {
if quiet && !daemon.imageIsDangling(imgID) {
// Ignore conflicts UNLESS the image is "dangling" in
// which case we want the user to know.
return nil
@ -223,33 +236,38 @@ func (daemon *Daemon) imageDeleteHelper(img *image.Image, records *[]types.Image
return conflict
}
parent, err := daemon.imageStore.GetParent(imgID)
if err != nil {
// There may be no parent
parent = ""
}
// Delete all repository tag/digest references to this image.
if err := daemon.removeAllReferencesToImageID(img.ID, records); err != nil {
if err := daemon.removeAllReferencesToImageID(imgID, records); err != nil {
return err
}
if err := daemon.Graph().Delete(img.ID); err != nil {
removedLayers, err := daemon.imageStore.Delete(imgID)
if err != nil {
return err
}
daemon.EventsService.Log("delete", img.ID, "")
*records = append(*records, types.ImageDelete{Deleted: img.ID})
daemon.EventsService.Log("delete", imgID.String(), "")
*records = append(*records, types.ImageDelete{Deleted: imgID.String()})
for _, removedLayer := range removedLayers {
*records = append(*records, types.ImageDelete{Deleted: removedLayer.ChainID.String()})
}
if !prune || img.Parent == "" {
if !prune || parent == "" {
return nil
}
// We need to prune the parent image. This means delete it if there are
// no tags/digests referencing it and there are no containers using it (
// either running or stopped).
parentImg, err := daemon.Graph().Get(img.Parent)
if err != nil {
return derr.ErrorCodeImgNoParent.WithArgs(err)
}
// Do not force prunings, but do so quietly (stopping on any encountered
// conflicts).
return daemon.imageDeleteHelper(parentImg, records, false, true, true)
return daemon.imageDeleteHelper(parent, records, false, true, true)
}
// checkImageDeleteConflict determines whether there are any conflicts
@ -258,9 +276,9 @@ func (daemon *Daemon) imageDeleteHelper(img *image.Image, records *[]types.Image
// using the image. A soft conflict is any tags/digest referencing the given
// image or any stopped container using the image. If ignoreSoftConflicts is
// true, this function will not check for soft conflict conditions.
func (daemon *Daemon) checkImageDeleteConflict(img *image.Image, ignoreSoftConflicts bool) *imageDeleteConflict {
func (daemon *Daemon) checkImageDeleteConflict(imgID image.ID, ignoreSoftConflicts bool) *imageDeleteConflict {
// Check for hard conflicts first.
if conflict := daemon.checkImageDeleteHardConflict(img); conflict != nil {
if conflict := daemon.checkImageDeleteHardConflict(imgID); conflict != nil {
return conflict
}
@ -270,24 +288,15 @@ func (daemon *Daemon) checkImageDeleteConflict(img *image.Image, ignoreSoftConfl
return nil
}
return daemon.checkImageDeleteSoftConflict(img)
return daemon.checkImageDeleteSoftConflict(imgID)
}
func (daemon *Daemon) checkImageDeleteHardConflict(img *image.Image) *imageDeleteConflict {
// Check if the image ID is being used by a pull or build.
if daemon.Graph().IsHeld(img.ID) {
return &imageDeleteConflict{
hard: true,
imgID: img.ID,
message: "image is held by an ongoing pull or build",
}
}
func (daemon *Daemon) checkImageDeleteHardConflict(imgID image.ID) *imageDeleteConflict {
// Check if the image has any descendent images.
if daemon.Graph().HasChildren(img.ID) {
if len(daemon.imageStore.Children(imgID)) > 0 {
return &imageDeleteConflict{
hard: true,
imgID: img.ID,
imgID: imgID,
message: "image has dependent child images",
}
}
@ -299,9 +308,9 @@ func (daemon *Daemon) checkImageDeleteHardConflict(img *image.Image) *imageDelet
continue
}
if container.ImageID == img.ID {
if container.ImageID == imgID {
return &imageDeleteConflict{
imgID: img.ID,
imgID: imgID,
hard: true,
message: fmt.Sprintf("image is being used by running container %s", stringid.TruncateID(container.ID)),
}
@ -311,11 +320,11 @@ func (daemon *Daemon) checkImageDeleteHardConflict(img *image.Image) *imageDelet
return nil
}
func (daemon *Daemon) checkImageDeleteSoftConflict(img *image.Image) *imageDeleteConflict {
func (daemon *Daemon) checkImageDeleteSoftConflict(imgID image.ID) *imageDeleteConflict {
// Check if any repository tags/digest reference this image.
if daemon.repositories.HasReferences(img) {
if len(daemon.tagStore.References(imgID)) > 0 {
return &imageDeleteConflict{
imgID: img.ID,
imgID: imgID,
message: "image is referenced in one or more repositories",
}
}
@ -327,9 +336,9 @@ func (daemon *Daemon) checkImageDeleteSoftConflict(img *image.Image) *imageDelet
continue
}
if container.ImageID == img.ID {
if container.ImageID == imgID {
return &imageDeleteConflict{
imgID: img.ID,
imgID: imgID,
message: fmt.Sprintf("image is being used by stopped container %s", stringid.TruncateID(container.ID)),
}
}
@ -341,6 +350,6 @@ func (daemon *Daemon) checkImageDeleteSoftConflict(img *image.Image) *imageDelet
// imageIsDangling returns whether the given image is "dangling" which means
// that there are no repository references to the given image and it has no
// child images.
func (daemon *Daemon) imageIsDangling(img *image.Image) bool {
return !(daemon.repositories.HasReferences(img) || daemon.Graph().HasChildren(img.ID))
func (daemon *Daemon) imageIsDangling(imgID image.ID) bool {
return !(len(daemon.tagStore.References(imgID)) > 0 || len(daemon.imageStore.Children(imgID)) > 0)
}

163
daemon/images.go Normal file
View file

@ -0,0 +1,163 @@
package daemon
import (
"fmt"
"path"
"sort"
"strings"
"github.com/docker/distribution/reference"
"github.com/docker/docker/api/types"
"github.com/docker/docker/image"
"github.com/docker/docker/layer"
"github.com/docker/docker/pkg/parsers/filters"
)
var acceptedImageFilterTags = map[string]struct{}{
"dangling": {},
"label": {},
}
// byCreated is a temporary type used to sort a list of images by creation
// time.
type byCreated []*types.Image
func (r byCreated) Len() int { return len(r) }
func (r byCreated) Swap(i, j int) { r[i], r[j] = r[j], r[i] }
func (r byCreated) Less(i, j int) bool { return r[i].Created < r[j].Created }
// Map returns a map of all images in the ImageStore
func (daemon *Daemon) Map() map[image.ID]*image.Image {
return daemon.imageStore.Map()
}
// Images returns a filtered list of images. filterArgs is a JSON-encoded set
// of filter arguments which will be interpreted by pkg/parsers/filters.
// filter is a shell glob string applied to repository names. The argument
// named all controls whether all images in the graph are filtered, or just
// the heads.
func (daemon *Daemon) Images(filterArgs, filter string, all bool) ([]*types.Image, error) {
var (
allImages map[image.ID]*image.Image
err error
danglingOnly = false
)
imageFilters, err := filters.FromParam(filterArgs)
if err != nil {
return nil, err
}
for name := range imageFilters {
if _, ok := acceptedImageFilterTags[name]; !ok {
return nil, fmt.Errorf("Invalid filter '%s'", name)
}
}
if i, ok := imageFilters["dangling"]; ok {
for _, value := range i {
if v := strings.ToLower(value); v == "true" {
danglingOnly = true
} else if v != "false" {
return nil, fmt.Errorf("Invalid filter 'dangling=%s'", v)
}
}
}
if danglingOnly {
allImages = daemon.imageStore.Heads()
} else {
allImages = daemon.imageStore.Map()
}
images := []*types.Image{}
var filterTagged bool
if filter != "" {
filterRef, err := reference.Parse(filter)
if err == nil { // parse error means wildcard repo
if _, ok := filterRef.(reference.Tagged); ok {
filterTagged = true
}
}
}
for id, img := range allImages {
if _, ok := imageFilters["label"]; ok {
if img.Config == nil {
// Very old image that do not have image.Config (or even labels)
continue
}
// We are now sure image.Config is not nil
if !imageFilters.MatchKVList("label", img.Config.Labels) {
continue
}
}
layerID := img.RootFS.ChainID()
var size int64
if layerID != "" {
l, err := daemon.layerStore.Get(layerID)
if err != nil {
return nil, err
}
size, err = l.Size()
layer.ReleaseAndLog(daemon.layerStore, l)
if err != nil {
return nil, err
}
}
newImage := newImage(img, size)
for _, ref := range daemon.tagStore.References(id) {
if filter != "" { // filter by tag/repo name
if filterTagged { // filter by tag, require full ref match
if ref.String() != filter {
continue
}
} else if matched, err := path.Match(filter, ref.Name()); !matched || err != nil { // name only match, FIXME: docs say exact
continue
}
}
if _, ok := ref.(reference.Digested); ok {
newImage.RepoDigests = append(newImage.RepoDigests, ref.String())
}
if _, ok := ref.(reference.Tagged); ok {
newImage.RepoTags = append(newImage.RepoTags, ref.String())
}
}
if newImage.RepoDigests == nil && newImage.RepoTags == nil {
if all || len(daemon.imageStore.Children(id)) == 0 {
if filter != "" { // skip images with no references if filtering by tag
continue
}
newImage.RepoDigests = []string{"<none>@<none>"}
newImage.RepoTags = []string{"<none>:<none>"}
} else {
continue
}
} else if danglingOnly {
continue
}
images = append(images, newImage)
}
sort.Sort(sort.Reverse(byCreated(images)))
return images, nil
}
func newImage(image *image.Image, size int64) *types.Image {
newImage := new(types.Image)
newImage.ParentID = image.Parent.String()
newImage.ID = image.ID().String()
newImage.Created = image.Created.Unix()
newImage.Size = size
newImage.VirtualSize = size
if image.Config != nil {
newImage.Labels = image.Config.Labels
}
return newImage
}

111
daemon/import.go Normal file
View file

@ -0,0 +1,111 @@
package daemon
import (
"encoding/json"
"io"
"net/http"
"net/url"
"runtime"
"time"
"github.com/docker/distribution/reference"
"github.com/docker/docker/dockerversion"
"github.com/docker/docker/image"
"github.com/docker/docker/layer"
"github.com/docker/docker/pkg/httputils"
"github.com/docker/docker/pkg/progressreader"
"github.com/docker/docker/pkg/streamformatter"
"github.com/docker/docker/runconfig"
)
// ImportImage imports an image, getting the archived layer data either from
// inConfig (if src is "-"), or from a URI specified in src. Progress output is
// written to outStream. Repository and tag names can optionally be given in
// the repo and tag arguments, respectively.
func (daemon *Daemon) ImportImage(src string, newRef reference.Named, msg string, inConfig io.ReadCloser, outStream io.Writer, config *runconfig.Config) error {
var (
sf = streamformatter.NewJSONStreamFormatter()
archive io.ReadCloser
resp *http.Response
)
if src == "-" {
archive = inConfig
} else {
inConfig.Close()
u, err := url.Parse(src)
if err != nil {
return err
}
if u.Scheme == "" {
u.Scheme = "http"
u.Host = src
u.Path = ""
}
outStream.Write(sf.FormatStatus("", "Downloading from %s", u))
resp, err = httputils.Download(u.String())
if err != nil {
return err
}
progressReader := progressreader.New(progressreader.Config{
In: resp.Body,
Out: outStream,
Formatter: sf,
Size: resp.ContentLength,
NewLines: true,
ID: "",
Action: "Importing",
})
archive = progressReader
}
defer archive.Close()
if len(msg) == 0 {
msg = "Imported from " + src
}
// TODO: support windows baselayer?
l, err := daemon.layerStore.Register(archive, "")
if err != nil {
return err
}
defer layer.ReleaseAndLog(daemon.layerStore, l)
created := time.Now().UTC()
imgConfig, err := json.Marshal(&image.Image{
V1Image: image.V1Image{
DockerVersion: dockerversion.Version,
Config: config,
Architecture: runtime.GOARCH,
OS: runtime.GOOS,
Created: created,
Comment: msg,
},
RootFS: &image.RootFS{
Type: "layers",
DiffIDs: []layer.DiffID{l.DiffID()},
},
History: []image.History{{
Created: created,
Comment: msg,
}},
})
if err != nil {
return err
}
id, err := daemon.imageStore.Create(imgConfig)
if err != nil {
return err
}
// FIXME: connect with commit code and call tagstore directly
if newRef != nil {
if err := daemon.TagImage(newRef, id.String(), true); err != nil {
return err
}
}
outStream.Write(sf.FormatStatus("", id.String()))
daemon.EventsService.Log("import", id.String(), "")
return nil
}

2
daemon/info.go
View file

@ -62,7 +62,7 @@ func (daemon *Daemon) SystemInfo() (*types.Info, error) {
v := &types.Info{
ID: daemon.ID,
Containers: len(daemon.List()),
Images: len(daemon.Graph().Map()),
Images: len(daemon.imageStore.Map()),
Driver: daemon.GraphDriver().String(),
DriverStatus: daemon.GraphDriver().Status(),
Plugins: daemon.showPluginsInfo(),

16
daemon/inspect.go
View file

@ -8,6 +8,7 @@ import (
"github.com/docker/docker/api/types/versions/v1p20"
"github.com/docker/docker/daemon/exec"
"github.com/docker/docker/daemon/network"
"github.com/docker/docker/layer"
)
// ContainerInspect returns low-level information about a
@ -124,7 +125,7 @@ func (daemon *Daemon) getInspectData(container *Container, size bool) (*types.Co
Path: container.Path,
Args: container.Args,
State: containerState,
Image: container.ImageID,
Image: container.ImageID.String(),
LogPath: container.LogPath,
Name: container.Name,
RestartCount: container.RestartCount,
@ -149,7 +150,18 @@ func (daemon *Daemon) getInspectData(container *Container, size bool) (*types.Co
contJSONBase = setPlatformSpecificContainerFields(container, contJSONBase)
contJSONBase.GraphDriver.Name = container.Driver
graphDriverData, err := daemon.driver.GetMetadata(container.ID)
image, err := daemon.imageStore.Get(container.ImageID)
if err != nil {
return nil, err
}
l, err := daemon.layerStore.Get(image.RootFS.ChainID())
if err != nil {
return nil, err
}
defer layer.ReleaseAndLog(daemon.layerStore, l)
graphDriverData, err := l.Metadata()
if err != nil {
return nil, err
}

54
daemon/list.go
View file

@ -9,7 +9,6 @@ import (
"github.com/Sirupsen/logrus"
"github.com/docker/docker/api/types"
derr "github.com/docker/docker/errors"
"github.com/docker/docker/graph"
"github.com/docker/docker/image"
"github.com/docker/docker/pkg/graphdb"
"github.com/docker/docker/pkg/nat"
@ -66,7 +65,7 @@ type listContext struct {
// names is a list of container names to filter with
names map[string][]string
// images is a list of images to filter with
images map[string]bool
images map[image.ID]bool
// filters is a collection of arguments to filter with, specified by the user
filters filters.Args
// exitAllowed is a list of exit codes allowed to filter with
@ -176,25 +175,24 @@ func (daemon *Daemon) foldFilter(config *ContainersConfig) (*listContext, error)
}
}
imagesFilter := map[string]bool{}
imagesFilter := map[image.ID]bool{}
var ancestorFilter bool
if ancestors, ok := psFilters["ancestor"]; ok {
ancestorFilter = true
byParents := daemon.Graph().ByParent()
// The idea is to walk the graph down the most "efficient" way.
for _, ancestor := range ancestors {
// First, get the imageId of the ancestor filter (yay)
image, err := daemon.repositories.LookupImage(ancestor)
id, err := daemon.GetImageID(ancestor)
if err != nil {
logrus.Warnf("Error while looking up for image %v", ancestor)
continue
}
if imagesFilter[ancestor] {
if imagesFilter[id] {
// Already seen this ancestor, skip it
continue
}
// Then walk down the graph and put the imageIds in imagesFilter
populateImageFilterByParents(imagesFilter, image.ID, byParents)
populateImageFilterByParents(imagesFilter, id, daemon.imageStore.Children)
}
}
@ -310,41 +308,29 @@ func includeContainerInList(container *Container, ctx *listContext) iterationAct
return includeContainer
}
func getImage(s *graph.TagStore, img, imgID string) (string, error) {
// both Image and ImageID is actually ids, nothing to guess
if strings.HasPrefix(imgID, img) {
return img, nil
}
id, err := s.GetID(img)
if err != nil {
if err == graph.ErrNameIsNotExist {
return imgID, nil
}
return "", err
}
if id != imgID {
return imgID, nil
}
return img, nil
}
// transformContainer generates the container type expected by the docker ps command.
func (daemon *Daemon) transformContainer(container *Container, ctx *listContext) (*types.Container, error) {
newC := &types.Container{
ID: container.ID,
Names: ctx.names[container.ID],
ImageID: container.ImageID,
ImageID: container.ImageID.String(),
}
if newC.Names == nil {
// Dead containers will often have no name, so make sure the response isn't null
newC.Names = []string{}
}
showImg, err := getImage(daemon.repositories, container.Config.Image, container.ImageID)
if err != nil {
return nil, err
image := container.Config.Image // if possible keep the original ref
if image != container.ImageID.String() {
id, err := daemon.GetImageID(image)
if _, isDNE := err.(ErrImageDoesNotExist); err != nil && !isDNE {
return nil, err
}
if err != nil || id != container.ImageID {
image = container.ImageID.String()
}
}
newC.Image = showImg
newC.Image = image
if len(container.Args) > 0 {
args := []string{}
@ -433,12 +419,10 @@ func (daemon *Daemon) Volumes(filter string) ([]*types.Volume, error) {
return volumesOut, nil
}
func populateImageFilterByParents(ancestorMap map[string]bool, imageID string, byParents map[string][]*image.Image) {
func populateImageFilterByParents(ancestorMap map[image.ID]bool, imageID image.ID, getChildren func(image.ID) []image.ID) {
if !ancestorMap[imageID] {
if images, ok := byParents[imageID]; ok {
for _, image := range images {
populateImageFilterByParents(ancestorMap, image.ID, byParents)
}
for _, id := range getChildren(imageID) {
populateImageFilterByParents(ancestorMap, id, getChildren)
}
ancestorMap[imageID] = true
}

0
graph/fixtures/validate_manifest/bad_manifest → distribution/fixtures/validate_manifest/bad_manifest
View file

0
graph/fixtures/validate_manifest/extra_data_manifest → distribution/fixtures/validate_manifest/extra_data_manifest
View file

0
graph/fixtures/validate_manifest/good_manifest → distribution/fixtures/validate_manifest/good_manifest
View file

100
distribution/metadata/blobsum_service.go Normal file
View file

@ -0,0 +1,100 @@
package metadata
import (
"encoding/json"
"github.com/docker/distribution/digest"
"github.com/docker/docker/layer"
)
// BlobSumService maps layer IDs to a set of known blobsums for
// the layer.
type BlobSumService struct {
store Store
}
// maxBlobSums is the number of blobsums to keep per layer DiffID.
const maxBlobSums = 5
// NewBlobSumService creates a new blobsum mapping service.
func NewBlobSumService(store Store) *BlobSumService {
return &BlobSumService{
store: store,
}
}
func (blobserv *BlobSumService) diffIDNamespace() string {
return "blobsum-storage"
}
func (blobserv *BlobSumService) blobSumNamespace() string {
return "blobsum-lookup"
}
func (blobserv *BlobSumService) diffIDKey(diffID layer.DiffID) string {
return string(digest.Digest(diffID).Algorithm()) + "/" + digest.Digest(diffID).Hex()
}
func (blobserv *BlobSumService) blobSumKey(blobsum digest.Digest) string {
return string(blobsum.Algorithm()) + "/" + blobsum.Hex()
}
// GetBlobSums finds the blobsums associated with a layer DiffID.
func (blobserv *BlobSumService) GetBlobSums(diffID layer.DiffID) ([]digest.Digest, error) {
jsonBytes, err := blobserv.store.Get(blobserv.diffIDNamespace(), blobserv.diffIDKey(diffID))
if err != nil {
return nil, err
}
var blobsums []digest.Digest
if err := json.Unmarshal(jsonBytes, &blobsums); err != nil {
return nil, err
}
return blobsums, nil
}
// GetDiffID finds a layer DiffID from a blobsum hash.
func (blobserv *BlobSumService) GetDiffID(blobsum digest.Digest) (layer.DiffID, error) {
diffIDBytes, err := blobserv.store.Get(blobserv.blobSumNamespace(), blobserv.blobSumKey(blobsum))
if err != nil {
return layer.DiffID(""), err
}
return layer.DiffID(diffIDBytes), nil
}
// Add associates a blobsum with a layer DiffID. If too many blobsums are
// present, the oldest one is dropped.
func (blobserv *BlobSumService) Add(diffID layer.DiffID, blobsum digest.Digest) error {
oldBlobSums, err := blobserv.GetBlobSums(diffID)
if err != nil {
oldBlobSums = nil
}
newBlobSums := make([]digest.Digest, 0, len(oldBlobSums)+1)
// Copy all other blobsums to new slice
for _, oldSum := range oldBlobSums {
if oldSum != blobsum {
newBlobSums = append(newBlobSums, oldSum)
}
}
newBlobSums = append(newBlobSums, blobsum)
if len(newBlobSums) > maxBlobSums {
newBlobSums = newBlobSums[len(newBlobSums)-maxBlobSums:]
}
jsonBytes, err := json.Marshal(newBlobSums)
if err != nil {
return err
}
err = blobserv.store.Set(blobserv.diffIDNamespace(), blobserv.diffIDKey(diffID), jsonBytes)
if err != nil {
return err
}
return blobserv.store.Set(blobserv.blobSumNamespace(), blobserv.blobSumKey(blobsum), []byte(diffID))
}

105
distribution/metadata/blobsum_service_test.go Normal file
View file

@ -0,0 +1,105 @@
package metadata
import (
"io/ioutil"
"os"
"reflect"
"testing"
"github.com/docker/distribution/digest"
"github.com/docker/docker/layer"
)
func TestBlobSumService(t *testing.T) {
tmpDir, err := ioutil.TempDir("", "blobsum-storage-service-test")
if err != nil {
t.Fatalf("could not create temp dir: %v", err)
}
defer os.RemoveAll(tmpDir)
metadataStore, err := NewFSMetadataStore(tmpDir)
if err != nil {
t.Fatalf("could not create metadata store: %v", err)
}
blobSumService := NewBlobSumService(metadataStore)
testVectors := []struct {
diffID layer.DiffID
blobsums []digest.Digest
}{
{
diffID: layer.DiffID("sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4"),
blobsums: []digest.Digest{
digest.Digest("sha256:f0cd5ca10b07f35512fc2f1cbf9a6cefbdb5cba70ac6b0c9e5988f4497f71937"),
},
},
{
diffID: layer.DiffID("sha256:86e0e091d0da6bde2456dbb48306f3956bbeb2eae1b5b9a43045843f69fe4aaa"),
blobsums: []digest.Digest{
digest.Digest("sha256:f0cd5ca10b07f35512fc2f1cbf9a6cefbdb5cba70ac6b0c9e5988f4497f71937"),
digest.Digest("sha256:9e3447ca24cb96d86ebd5960cb34d1299b07e0a0e03801d90b9969a2c187dd6e"),
},
},
{
diffID: layer.DiffID("sha256:03f4658f8b782e12230c1783426bd3bacce651ce582a4ffb6fbbfa2079428ecb"),
blobsums: []digest.Digest{
digest.Digest("sha256:f0cd5ca10b07f35512fc2f1cbf9a6cefbdb5cba70ac6b0c9e5988f4497f71937"),
digest.Digest("sha256:9e3447ca24cb96d86ebd5960cb34d1299b07e0a0e03801d90b9969a2c187dd6e"),
digest.Digest("sha256:cbbf2f9a99b47fc460d422812b6a5adff7dfee951d8fa2e4a98caa0382cfbdbf"),
digest.Digest("sha256:8902a7ca89aabbb868835260912159026637634090dd8899eee969523252236e"),
digest.Digest("sha256:c84364306344ccc48532c52ff5209236273525231dddaaab53262322352883aa"),
digest.Digest("sha256:aa7583bbc87532a8352bbb72520a821b3623523523a8352523a52352aaa888fe"),
},
},
}
// Set some associations
for _, vec := range testVectors {
for _, blobsum := range vec.blobsums {
err := blobSumService.Add(vec.diffID, blobsum)
if err != nil {
t.Fatalf("error calling Set: %v", err)
}
}
}
// Check the correct values are read back
for _, vec := range testVectors {
blobsums, err := blobSumService.GetBlobSums(vec.diffID)
if err != nil {
t.Fatalf("error calling Get: %v", err)
}
expectedBlobsums := len(vec.blobsums)
if expectedBlobsums > 5 {
expectedBlobsums = 5
}
if !reflect.DeepEqual(blobsums, vec.blobsums[len(vec.blobsums)-expectedBlobsums:len(vec.blobsums)]) {
t.Fatal("Get returned incorrect layer ID")
}
}
// Test GetBlobSums on a nonexistent entry
_, err = blobSumService.GetBlobSums(layer.DiffID("sha256:82379823067823853223359023576437723560923756b03560378f4497753917"))
if err == nil {
t.Fatal("expected error looking up nonexistent entry")
}
// Test GetDiffID on a nonexistent entry
_, err = blobSumService.GetDiffID(digest.Digest("sha256:82379823067823853223359023576437723560923756b03560378f4497753917"))
if err == nil {
t.Fatal("expected error looking up nonexistent entry")
}
// Overwrite one of the entries and read it back
err = blobSumService.Add(testVectors[1].diffID, testVectors[0].blobsums[0])
if err != nil {
t.Fatalf("error calling Add: %v", err)
}
diffID, err := blobSumService.GetDiffID(testVectors[0].blobsums[0])
if err != nil {
t.Fatalf("error calling GetDiffID: %v", err)
}
if diffID != testVectors[1].diffID {
t.Fatal("GetDiffID returned incorrect diffID")
}
}

65
distribution/metadata/metadata.go Normal file
View file

@ -0,0 +1,65 @@
package metadata
import (
"io/ioutil"
"os"
"path/filepath"
"sync"
)
// Store implements a K/V store for mapping distribution-related IDs
// to on-disk layer IDs and image IDs. The namespace identifies the type of
// mapping (i.e. "v1ids" or "artifacts"). MetadataStore is goroutine-safe.
type Store interface {
// Get retrieves data by namespace and key.
Get(namespace string, key string) ([]byte, error)
// Set writes data indexed by namespace and key.
Set(namespace, key string, value []byte) error
}
// FSMetadataStore uses the filesystem to associate metadata with layer and
// image IDs.
type FSMetadataStore struct {
sync.RWMutex
basePath string
}
// NewFSMetadataStore creates a new filesystem-based metadata store.
func NewFSMetadataStore(basePath string) (*FSMetadataStore, error) {
if err := os.MkdirAll(basePath, 0700); err != nil {
return nil, err
}
return &FSMetadataStore{
basePath: basePath,
}, nil
}
func (store *FSMetadataStore) path(namespace, key string) string {
return filepath.Join(store.basePath, namespace, key)
}
// Get retrieves data by namespace and key. The data is read from a file named
// after the key, stored in the namespace's directory.
func (store *FSMetadataStore) Get(namespace string, key string) ([]byte, error) {
store.RLock()
defer store.RUnlock()
return ioutil.ReadFile(store.path(namespace, key))
}
// Set writes data indexed by namespace and key. The data is written to a file
// named after the key, stored in the namespace's directory.
func (store *FSMetadataStore) Set(namespace, key string, value []byte) error {
store.Lock()
defer store.Unlock()
path := store.path(namespace, key)
tempFilePath := path + ".tmp"
if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil {
return err
}
if err := ioutil.WriteFile(tempFilePath, value, 0644); err != nil {
return err
}
return os.Rename(tempFilePath, path)
}

44
distribution/metadata/v1_id_service.go Normal file
View file

@ -0,0 +1,44 @@
package metadata
import (
"github.com/docker/docker/image/v1"
"github.com/docker/docker/layer"
)
// V1IDService maps v1 IDs to layers on disk.
type V1IDService struct {
store Store
}
// NewV1IDService creates a new V1 ID mapping service.
func NewV1IDService(store Store) *V1IDService {
return &V1IDService{
store: store,
}
}
// namespace returns the namespace used by this service.
func (idserv *V1IDService) namespace() string {
return "v1id"
}
// Get finds a layer by its V1 ID.
func (idserv *V1IDService) Get(v1ID, registry string) (layer.ChainID, error) {
if err := v1.ValidateID(v1ID); err != nil {
return layer.ChainID(""), err
}
idBytes, err := idserv.store.Get(idserv.namespace(), registry+","+v1ID)
if err != nil {
return layer.ChainID(""), err
}
return layer.ChainID(idBytes), nil
}
// Set associates an image with a V1 ID.
func (idserv *V1IDService) Set(v1ID, registry string, id layer.ChainID) error {
if err := v1.ValidateID(v1ID); err != nil {
return err
}
return idserv.store.Set(idserv.namespace(), registry+","+v1ID, []byte(id))
}

83
distribution/metadata/v1_id_service_test.go Normal file
View file

@ -0,0 +1,83 @@
package metadata
import (
"io/ioutil"
"os"
"testing"
"github.com/docker/docker/layer"
)
func TestV1IDService(t *testing.T) {
tmpDir, err := ioutil.TempDir("", "v1-id-service-test")
if err != nil {
t.Fatalf("could not create temp dir: %v", err)
}
defer os.RemoveAll(tmpDir)
metadataStore, err := NewFSMetadataStore(tmpDir)
if err != nil {
t.Fatalf("could not create metadata store: %v", err)
}
v1IDService := NewV1IDService(metadataStore)
testVectors := []struct {
registry string
v1ID string
layerID layer.ChainID
}{
{
registry: "registry1",
v1ID: "f0cd5ca10b07f35512fc2f1cbf9a6cefbdb5cba70ac6b0c9e5988f4497f71937",
layerID: layer.ChainID("sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4"),
},
{
registry: "registry2",
v1ID: "9e3447ca24cb96d86ebd5960cb34d1299b07e0a0e03801d90b9969a2c187dd6e",
layerID: layer.ChainID("sha256:86e0e091d0da6bde2456dbb48306f3956bbeb2eae1b5b9a43045843f69fe4aaa"),
},
{
registry: "registry1",
v1ID: "9e3447ca24cb96d86ebd5960cb34d1299b07e0a0e03801d90b9969a2c187dd6e",
layerID: layer.ChainID("sha256:03f4658f8b782e12230c1783426bd3bacce651ce582a4ffb6fbbfa2079428ecb"),
},
}
// Set some associations
for _, vec := range testVectors {
err := v1IDService.Set(vec.v1ID, vec.registry, vec.layerID)
if err != nil {
t.Fatalf("error calling Set: %v", err)
}
}
// Check the correct values are read back
for _, vec := range testVectors {
layerID, err := v1IDService.Get(vec.v1ID, vec.registry)
if err != nil {
t.Fatalf("error calling Get: %v", err)
}
if layerID != vec.layerID {
t.Fatal("Get returned incorrect layer ID")
}
}
// Test Get on a nonexistent entry
_, err = v1IDService.Get("82379823067823853223359023576437723560923756b03560378f4497753917", "registry1")
if err == nil {
t.Fatal("expected error looking up nonexistent entry")
}
// Overwrite one of the entries and read it back
err = v1IDService.Set(testVectors[0].v1ID, testVectors[0].registry, testVectors[1].layerID)
if err != nil {
t.Fatalf("error calling Set: %v", err)
}
layerID, err := v1IDService.Get(testVectors[0].v1ID, testVectors[0].registry)
if err != nil {
t.Fatalf("error calling Get: %v", err)
}
if layerID != testVectors[1].layerID {
t.Fatal("Get returned incorrect layer ID")
}
}

51
distribution/pool.go Normal file
View file

@ -0,0 +1,51 @@
package distribution
import (
"sync"
"github.com/docker/docker/pkg/broadcaster"
)
// A Pool manages concurrent pulls. It deduplicates in-progress downloads.
type Pool struct {
sync.Mutex
pullingPool map[string]*broadcaster.Buffered
}
// NewPool creates a new Pool.
func NewPool() *Pool {
return &Pool{
pullingPool: make(map[string]*broadcaster.Buffered),
}
}
// add checks if a pull is already running, and returns (broadcaster, true)
// if a running operation is found. Otherwise, it creates a new one and returns
// (broadcaster, false).
func (pool *Pool) add(key string) (*broadcaster.Buffered, bool) {
pool.Lock()
defer pool.Unlock()
if p, exists := pool.pullingPool[key]; exists {
return p, true
}
broadcaster := broadcaster.NewBuffered()
pool.pullingPool[key] = broadcaster
return broadcaster, false
}
func (pool *Pool) removeWithError(key string, broadcasterResult error) error {
pool.Lock()
defer pool.Unlock()
if broadcaster, exists := pool.pullingPool[key]; exists {
broadcaster.CloseWithError(broadcasterResult)
delete(pool.pullingPool, key)
}
return nil
}
func (pool *Pool) remove(key string) error {
return pool.removeWithError(key, nil)
}

28
distribution/pool_test.go Normal file
View file

@ -0,0 +1,28 @@
package distribution
import (
"testing"
)
func TestPools(t *testing.T) {
p := NewPool()
if _, found := p.add("test1"); found {
t.Fatal("Expected pull test1 not to be in progress")
}
if _, found := p.add("test2"); found {
t.Fatal("Expected pull test2 not to be in progress")
}
if _, found := p.add("test1"); !found {
t.Fatalf("Expected pull test1 to be in progress`")
}
if err := p.remove("test2"); err != nil {
t.Fatal(err)
}
if err := p.remove("test2"); err != nil {
t.Fatal(err)
}
if err := p.remove("test1"); err != nil {
t.Fatal(err)
}
}

86
graph/pull.go → distribution/pull.go
View file

@ -1,4 +1,4 @@
package graph
package distribution
import (
"fmt"
@ -6,10 +6,15 @@ import (
"strings"
"github.com/Sirupsen/logrus"
"github.com/docker/distribution/reference"
"github.com/docker/docker/cliconfig"
"github.com/docker/docker/daemon/events"
"github.com/docker/docker/distribution/metadata"
"github.com/docker/docker/image"
"github.com/docker/docker/layer"
"github.com/docker/docker/pkg/streamformatter"
"github.com/docker/docker/registry"
"github.com/docker/docker/utils"
"github.com/docker/docker/tag"
)
// ImagePullConfig stores pull configuration.
@ -23,15 +28,30 @@ type ImagePullConfig struct {
// OutStream is the output writer for showing the status of the pull
// operation.
OutStream io.Writer
// RegistryService is the registry service to use for TLS configuration
// and endpoint lookup.
RegistryService *registry.Service
// EventsService is the events service to use for logging.
EventsService *events.Events
// MetadataStore is the storage backend for distribution-specific
// metadata.
MetadataStore metadata.Store
// LayerStore manages layers.
LayerStore layer.Store
// ImageStore manages images.
ImageStore image.Store
// TagStore manages tags.
TagStore tag.Store
// Pool manages concurrent pulls.
Pool *Pool
}
// puller is an interface that abstracts pulling for different API versions.
type puller interface {
// Puller is an interface that abstracts pulling for different API versions.
type Puller interface {
// Pull tries to pull the image referenced by `tag`
// Pull returns an error if any, as well as a boolean that determines whether to retry Pull on the next configured endpoint.
//
// TODO(tiborvass): have Pull() take a reference to repository + tag, so that the puller itself is repository-agnostic.
Pull(tag string) (fallback bool, err error)
Pull(ref reference.Named) (fallback bool, err error)
}
// newPuller returns a Puller interface that will pull from either a v1 or v2
@ -39,23 +59,23 @@ type puller interface {
// whether a v1 or v2 puller will be created. The other parameters are passed
// through to the underlying puller implementation for use during the actual
// pull operation.
func newPuller(s *TagStore, endpoint registry.APIEndpoint, repoInfo *registry.RepositoryInfo, imagePullConfig *ImagePullConfig, sf *streamformatter.StreamFormatter) (puller, error) {
func newPuller(endpoint registry.APIEndpoint, repoInfo *registry.RepositoryInfo, imagePullConfig *ImagePullConfig, sf *streamformatter.StreamFormatter) (Puller, error) {
switch endpoint.Version {
case registry.APIVersion2:
return &v2Puller{
TagStore: s,
endpoint: endpoint,
config: imagePullConfig,
sf: sf,
repoInfo: repoInfo,
blobSumService: metadata.NewBlobSumService(imagePullConfig.MetadataStore),
endpoint: endpoint,
config: imagePullConfig,
sf: sf,
repoInfo: repoInfo,
}, nil
case registry.APIVersion1:
return &v1Puller{
TagStore: s,
endpoint: endpoint,
config: imagePullConfig,
sf: sf,
repoInfo: repoInfo,
v1IDService: metadata.NewV1IDService(imagePullConfig.MetadataStore),
endpoint: endpoint,
config: imagePullConfig,
sf: sf,
repoInfo: repoInfo,
}, nil
}
return nil, fmt.Errorf("unknown version %d for registry %s", endpoint.Version, endpoint.URL)
@ -63,29 +83,26 @@ func newPuller(s *TagStore, endpoint registry.APIEndpoint, repoInfo *registry.Re
// Pull initiates a pull operation. image is the repository name to pull, and
// tag may be either empty, or indicate a specific tag to pull.
func (s *TagStore) Pull(image string, tag string, imagePullConfig *ImagePullConfig) error {
func Pull(ref reference.Named, imagePullConfig *ImagePullConfig) error {
var sf = streamformatter.NewJSONStreamFormatter()
// Resolve the Repository name from fqn to RepositoryInfo
repoInfo, err := s.registryService.ResolveRepository(image)
repoInfo, err := imagePullConfig.RegistryService.ResolveRepository(ref)
if err != nil {
return err
}
// makes sure name is not empty or `scratch`
if err := validateRepoName(repoInfo.LocalName); err != nil {
if err := validateRepoName(repoInfo.LocalName.Name()); err != nil {
return err
}
endpoints, err := s.registryService.LookupPullEndpoints(repoInfo.CanonicalName)
endpoints, err := imagePullConfig.RegistryService.LookupPullEndpoints(repoInfo.CanonicalName)
if err != nil {
return err
}
logName := repoInfo.LocalName
if tag != "" {
logName = utils.ImageReference(logName, tag)
}
logName := registry.NormalizeLocalReference(ref)
var (
// use a slice to append the error strings and return a joined string to caller
@ -103,12 +120,12 @@ func (s *TagStore) Pull(image string, tag string, imagePullConfig *ImagePullConf
for _, endpoint := range endpoints {
logrus.Debugf("Trying to pull %s from %s %s", repoInfo.LocalName, endpoint.URL, endpoint.Version)
puller, err := newPuller(s, endpoint, repoInfo, imagePullConfig, sf)
puller, err := newPuller(endpoint, repoInfo, imagePullConfig, sf)
if err != nil {
errors = append(errors, err.Error())
continue
}
if fallback, err := puller.Pull(tag); err != nil {
if fallback, err := puller.Pull(ref); err != nil {
if fallback {
if _, ok := err.(registry.ErrNoSupport); !ok {
// Because we found an error that's not ErrNoSupport, discard all subsequent ErrNoSupport errors.
@ -130,12 +147,12 @@ func (s *TagStore) Pull(image string, tag string, imagePullConfig *ImagePullConf
}
}
s.eventsService.Log("pull", logName, "")
imagePullConfig.EventsService.Log("pull", logName.String(), "")
return nil
}
if len(errors) == 0 {
return fmt.Errorf("no endpoints found for %s", image)
return fmt.Errorf("no endpoints found for %s", ref.String())
}
if len(errors) > 0 {
@ -155,3 +172,14 @@ func writeStatus(requestedTag string, out io.Writer, sf *streamformatter.StreamF
out.Write(sf.FormatStatus("", "Status: Image is up to date for %s", requestedTag))
}
}
// validateRepoName validates the name of a repository.
func validateRepoName(name string) error {
if name == "" {
return fmt.Errorf("Repository name can't be empty")
}
if name == "scratch" {
return fmt.Errorf("'scratch' is a reserved name")
}
return nil
}

454
distribution/pull_v1.go Normal file
View file

@ -0,0 +1,454 @@
package distribution
import (
"encoding/json"
"errors"
"fmt"
"io"
"net"
"net/url"
"strings"
"sync"
"time"
"github.com/Sirupsen/logrus"
"github.com/docker/distribution/reference"
"github.com/docker/distribution/registry/client/transport"
"github.com/docker/docker/distribution/metadata"
"github.com/docker/docker/image"
"github.com/docker/docker/image/v1"
"github.com/docker/docker/layer"
"github.com/docker/docker/pkg/archive"
"github.com/docker/docker/pkg/progressreader"
"github.com/docker/docker/pkg/streamformatter"
"github.com/docker/docker/pkg/stringid"
"github.com/docker/docker/registry"
)
type v1Puller struct {
v1IDService *metadata.V1IDService
endpoint registry.APIEndpoint
config *ImagePullConfig
sf *streamformatter.StreamFormatter
repoInfo *registry.RepositoryInfo
session *registry.Session
}
func (p *v1Puller) Pull(ref reference.Named) (fallback bool, err error) {
if _, isDigested := ref.(reference.Digested); isDigested {
// Allowing fallback, because HTTPS v1 is before HTTP v2
return true, registry.ErrNoSupport{errors.New("Cannot pull by digest with v1 registry")}
}
tlsConfig, err := p.config.RegistryService.TLSConfig(p.repoInfo.Index.Name)
if err != nil {
return false, err
}
// Adds Docker-specific headers as well as user-specified headers (metaHeaders)
tr := transport.NewTransport(
// TODO(tiborvass): was ReceiveTimeout
registry.NewTransport(tlsConfig),
registry.DockerHeaders(p.config.MetaHeaders)...,
)
client := registry.HTTPClient(tr)
v1Endpoint, err := p.endpoint.ToV1Endpoint(p.config.MetaHeaders)
if err != nil {
logrus.Debugf("Could not get v1 endpoint: %v", err)
return true, err
}
p.session, err = registry.NewSession(client, p.config.AuthConfig, v1Endpoint)
if err != nil {
// TODO(dmcgowan): Check if should fallback
logrus.Debugf("Fallback from error: %s", err)
return true, err
}
if err := p.pullRepository(ref); err != nil {
// TODO(dmcgowan): Check if should fallback
return false, err
}
out := p.config.OutStream
out.Write(p.sf.FormatStatus("", "%s: this image was pulled from a legacy registry. Important: This registry version will not be supported in future versions of docker.", p.repoInfo.CanonicalName.Name()))
return false, nil
}
func (p *v1Puller) pullRepository(ref reference.Named) error {
out := p.config.OutStream
out.Write(p.sf.FormatStatus("", "Pulling repository %s", p.repoInfo.CanonicalName.Name()))
repoData, err := p.session.GetRepositoryData(p.repoInfo.RemoteName)
if err != nil {
if strings.Contains(err.Error(), "HTTP code: 404") {
return fmt.Errorf("Error: image %s not found", p.repoInfo.RemoteName.Name())
}
// Unexpected HTTP error
return err
}
logrus.Debugf("Retrieving the tag list")
var tagsList map[string]string
tagged, isTagged := ref.(reference.Tagged)
if !isTagged {
tagsList, err = p.session.GetRemoteTags(repoData.Endpoints, p.repoInfo.RemoteName)
} else {
var tagID string
tagsList = make(map[string]string)
tagID, err = p.session.GetRemoteTag(repoData.Endpoints, p.repoInfo.RemoteName, tagged.Tag())
if err == registry.ErrRepoNotFound {
return fmt.Errorf("Tag %s not found in repository %s", tagged.Tag(), p.repoInfo.CanonicalName.Name())
}
tagsList[tagged.Tag()] = tagID
}
if err != nil {
logrus.Errorf("unable to get remote tags: %s", err)
return err
}
for tag, id := range tagsList {
repoData.ImgList[id] = &registry.ImgData{
ID: id,
Tag: tag,
Checksum: "",
}
}
errors := make(chan error)
layerDownloaded := make(chan struct{})
layersDownloaded := false
var wg sync.WaitGroup
for _, imgData := range repoData.ImgList {
if isTagged && imgData.Tag != tagged.Tag() {
continue
}
wg.Add(1)
go func(img *registry.ImgData) {
p.downloadImage(out, repoData, img, layerDownloaded, errors)
wg.Done()
}(imgData)
}
go func() {
wg.Wait()
close(errors)
}()
var lastError error
selectLoop:
for {
select {
case err, ok := <-errors:
if !ok {
break selectLoop
}
lastError = err
case <-layerDownloaded:
layersDownloaded = true
}
}
if lastError != nil {
return lastError
}
localNameRef := p.repoInfo.LocalName
if isTagged {
localNameRef, err = reference.WithTag(localNameRef, tagged.Tag())
if err != nil {
localNameRef = p.repoInfo.LocalName
}
}
writeStatus(localNameRef.String(), out, p.sf, layersDownloaded)
return nil
}
func (p *v1Puller) downloadImage(out io.Writer, repoData *registry.RepositoryData, img *registry.ImgData, layerDownloaded chan struct{}, errors chan error) {
if img.Tag == "" {
logrus.Debugf("Image (id: %s) present in this repository but untagged, skipping", img.ID)
return
}
localNameRef, err := reference.WithTag(p.repoInfo.LocalName, img.Tag)
if err != nil {
retErr := fmt.Errorf("Image (id: %s) has invalid tag: %s", img.ID, img.Tag)
logrus.Debug(retErr.Error())
errors <- retErr
}
if err := v1.ValidateID(img.ID); err != nil {
errors <- err
return
}
out.Write(p.sf.FormatProgress(stringid.TruncateID(img.ID), fmt.Sprintf("Pulling image (%s) from %s", img.Tag, p.repoInfo.CanonicalName.Name()), nil))
success := false
var lastErr error
var isDownloaded bool
for _, ep := range p.repoInfo.Index.Mirrors {
ep += "v1/"
out.Write(p.sf.FormatProgress(stringid.TruncateID(img.ID), fmt.Sprintf("Pulling image (%s) from %s, mirror: %s", img.Tag, p.repoInfo.CanonicalName.Name(), ep), nil))
if isDownloaded, err = p.pullImage(out, img.ID, ep, localNameRef); err != nil {
// Don't report errors when pulling from mirrors.
logrus.Debugf("Error pulling image (%s) from %s, mirror: %s, %s", img.Tag, p.repoInfo.CanonicalName.Name(), ep, err)
continue
}
if isDownloaded {
layerDownloaded <- struct{}{}
}
success = true
break
}
if !success {
for _, ep := range repoData.Endpoints {
out.Write(p.sf.FormatProgress(stringid.TruncateID(img.ID), fmt.Sprintf("Pulling image (%s) from %s, endpoint: %s", img.Tag, p.repoInfo.CanonicalName.Name(), ep), nil))
if isDownloaded, err = p.pullImage(out, img.ID, ep, localNameRef); err != nil {
// It's not ideal that only the last error is returned, it would be better to concatenate the errors.
// As the error is also given to the output stream the user will see the error.
lastErr = err
out.Write(p.sf.FormatProgress(stringid.TruncateID(img.ID), fmt.Sprintf("Error pulling image (%s) from %s, endpoint: %s, %s", img.Tag, p.repoInfo.CanonicalName.Name(), ep, err), nil))
continue
}
if isDownloaded {
layerDownloaded <- struct{}{}
}
success = true
break
}
}
if !success {
err := fmt.Errorf("Error pulling image (%s) from %s, %v", img.Tag, p.repoInfo.CanonicalName.Name(), lastErr)
out.Write(p.sf.FormatProgress(stringid.TruncateID(img.ID), err.Error(), nil))
errors <- err
return
}
out.Write(p.sf.FormatProgress(stringid.TruncateID(img.ID), "Download complete", nil))
}
func (p *v1Puller) pullImage(out io.Writer, v1ID, endpoint string, localNameRef reference.Named) (layersDownloaded bool, err error) {
var history []string
history, err = p.session.GetRemoteHistory(v1ID, endpoint)
if err != nil {
return false, err
}
if len(history) < 1 {
return false, fmt.Errorf("empty history for image %s", v1ID)
}
out.Write(p.sf.FormatProgress(stringid.TruncateID(v1ID), "Pulling dependent layers", nil))
// FIXME: Try to stream the images?
// FIXME: Launch the getRemoteImage() in goroutines
var (
referencedLayers []layer.Layer
parentID layer.ChainID
newHistory []image.History
img *image.V1Image
imgJSON []byte
imgSize int64
)
defer func() {
for _, l := range referencedLayers {
layer.ReleaseAndLog(p.config.LayerStore, l)
}
}()
layersDownloaded = false
// Iterate over layers from top-most to bottom-most, checking if any
// already exist on disk.
var i int
for i = 0; i != len(history); i++ {
v1LayerID := history[i]
// Do we have a mapping for this particular v1 ID on this
// registry?
if layerID, err := p.v1IDService.Get(v1LayerID, p.repoInfo.Index.Name); err == nil {
// Does the layer actually exist
if l, err := p.config.LayerStore.Get(layerID); err == nil {
for j := i; j >= 0; j-- {
logrus.Debugf("Layer already exists: %s", history[j])
out.Write(p.sf.FormatProgress(stringid.TruncateID(history[j]), "Already exists", nil))
}
referencedLayers = append(referencedLayers, l)
parentID = layerID
break
}
}
}
needsDownload := i
// Iterate over layers, in order from bottom-most to top-most. Download
// config for all layers, and download actual layer data if needed.
for i = len(history) - 1; i >= 0; i-- {
v1LayerID := history[i]
imgJSON, imgSize, err = p.downloadLayerConfig(out, v1LayerID, endpoint)
if err != nil {
return layersDownloaded, err
}
img = &image.V1Image{}
if err := json.Unmarshal(imgJSON, img); err != nil {
return layersDownloaded, err
}
if i < needsDownload {
l, err := p.downloadLayer(out, v1LayerID, endpoint, parentID, imgSize, &layersDownloaded)
// Note: This needs to be done even in the error case to avoid
// stale references to the layer.
if l != nil {
referencedLayers = append(referencedLayers, l)
}
if err != nil {
return layersDownloaded, err
}
parentID = l.ChainID()
}
// Create a new-style config from the legacy configs
h, err := v1.HistoryFromConfig(imgJSON, false)
if err != nil {
return layersDownloaded, err
}
newHistory = append(newHistory, h)
}
rootFS := image.NewRootFS()
l := referencedLayers[len(referencedLayers)-1]
for l != nil {
rootFS.DiffIDs = append([]layer.DiffID{l.DiffID()}, rootFS.DiffIDs...)
l = l.Parent()
}
config, err := v1.MakeConfigFromV1Config(imgJSON, rootFS, newHistory)
if err != nil {
return layersDownloaded, err
}
imageID, err := p.config.ImageStore.Create(config)
if err != nil {
return layersDownloaded, err
}
if err := p.config.TagStore.Add(localNameRef, imageID, true); err != nil {
return layersDownloaded, err
}
return layersDownloaded, nil
}
func (p *v1Puller) downloadLayerConfig(out io.Writer, v1LayerID, endpoint string) (imgJSON []byte, imgSize int64, err error) {
out.Write(p.sf.FormatProgress(stringid.TruncateID(v1LayerID), "Pulling metadata", nil))
retries := 5
for j := 1; j <= retries; j++ {
imgJSON, imgSize, err := p.session.GetRemoteImageJSON(v1LayerID, endpoint)
if err != nil && j == retries {
out.Write(p.sf.FormatProgress(stringid.TruncateID(v1LayerID), "Error pulling layer metadata", nil))
return nil, 0, err
} else if err != nil {
time.Sleep(time.Duration(j) * 500 * time.Millisecond)
continue
}
return imgJSON, imgSize, nil
}
// not reached
return nil, 0, nil
}
func (p *v1Puller) downloadLayer(out io.Writer, v1LayerID, endpoint string, parentID layer.ChainID, layerSize int64, layersDownloaded *bool) (l layer.Layer, err error) {
// ensure no two downloads of the same layer happen at the same time
poolKey := "layer:" + v1LayerID
broadcaster, found := p.config.Pool.add(poolKey)
broadcaster.Add(out)
if found {
logrus.Debugf("Image (id: %s) pull is already running, skipping", v1LayerID)
if err = broadcaster.Wait(); err != nil {
return nil, err
}
layerID, err := p.v1IDService.Get(v1LayerID, p.repoInfo.Index.Name)
if err != nil {
return nil, err
}
// Does the layer actually exist
l, err := p.config.LayerStore.Get(layerID)
if err != nil {
return nil, err
}
return l, nil
}
// This must use a closure so it captures the value of err when
// the function returns, not when the 'defer' is evaluated.
defer func() {
p.config.Pool.removeWithError(poolKey, err)
}()
retries := 5
for j := 1; j <= retries; j++ {
// Get the layer
status := "Pulling fs layer"
if j > 1 {
status = fmt.Sprintf("Pulling fs layer [retries: %d]", j)
}
broadcaster.Write(p.sf.FormatProgress(stringid.TruncateID(v1LayerID), status, nil))
layerReader, err := p.session.GetRemoteImageLayer(v1LayerID, endpoint, layerSize)
if uerr, ok := err.(*url.Error); ok {
err = uerr.Err
}
if terr, ok := err.(net.Error); ok && terr.Timeout() && j < retries {
time.Sleep(time.Duration(j) * 500 * time.Millisecond)
continue
} else if err != nil {
broadcaster.Write(p.sf.FormatProgress(stringid.TruncateID(v1LayerID), "Error pulling dependent layers", nil))
return nil, err
}
*layersDownloaded = true
defer layerReader.Close()
reader := progressreader.New(progressreader.Config{
In: layerReader,
Out: broadcaster,
Formatter: p.sf,
Size: layerSize,
NewLines: false,
ID: stringid.TruncateID(v1LayerID),
Action: "Downloading",
})
inflatedLayerData, err := archive.DecompressStream(reader)
if err != nil {
return nil, fmt.Errorf("could not get decompression stream: %v", err)
}
l, err := p.config.LayerStore.Register(inflatedLayerData, parentID)
if err != nil {
return nil, fmt.Errorf("failed to register layer: %v", err)
}
logrus.Debugf("layer %s registered successfully", l.DiffID())
if terr, ok := err.(net.Error); ok && terr.Timeout() && j < retries {
time.Sleep(time.Duration(j) * 500 * time.Millisecond)
continue
} else if err != nil {
broadcaster.Write(p.sf.FormatProgress(stringid.TruncateID(v1LayerID), "Error downloading dependent layers", nil))
return nil, err
}
// Cache mapping from this v1 ID to content-addressable layer ID
if err := p.v1IDService.Set(v1LayerID, p.repoInfo.Index.Name, l.ChainID()); err != nil {
return nil, err
}
broadcaster.Write(p.sf.FormatProgress(stringid.TruncateID(v1LayerID), "Download complete", nil))
broadcaster.Close()
return l, nil
}
// not reached
return nil, nil
}

512
distribution/pull_v2.go Normal file
View file

@ -0,0 +1,512 @@
package distribution
import (
"encoding/json"
"errors"
"fmt"
"io"
"io/ioutil"
"os"
"runtime"
"github.com/Sirupsen/logrus"
"github.com/docker/distribution"
"github.com/docker/distribution/digest"
"github.com/docker/distribution/manifest/schema1"
"github.com/docker/distribution/reference"
"github.com/docker/docker/distribution/metadata"
"github.com/docker/docker/image"
"github.com/docker/docker/image/v1"
"github.com/docker/docker/layer"
"github.com/docker/docker/pkg/archive"
"github.com/docker/docker/pkg/broadcaster"
"github.com/docker/docker/pkg/progressreader"
"github.com/docker/docker/pkg/streamformatter"
"github.com/docker/docker/pkg/stringid"
"github.com/docker/docker/registry"
"golang.org/x/net/context"
)
type v2Puller struct {
blobSumService *metadata.BlobSumService
endpoint registry.APIEndpoint
config *ImagePullConfig
sf *streamformatter.StreamFormatter
repoInfo *registry.RepositoryInfo
repo distribution.Repository
sessionID string
}
func (p *v2Puller) Pull(ref reference.Named) (fallback bool, err error) {
// TODO(tiborvass): was ReceiveTimeout
p.repo, err = NewV2Repository(p.repoInfo, p.endpoint, p.config.MetaHeaders, p.config.AuthConfig, "pull")
if err != nil {
logrus.Debugf("Error getting v2 registry: %v", err)
return true, err
}
p.sessionID = stringid.GenerateRandomID()
if err := p.pullV2Repository(ref); err != nil {
if registry.ContinueOnError(err) {
logrus.Debugf("Error trying v2 registry: %v", err)
return true, err
}
return false, err
}
return false, nil
}
func (p *v2Puller) pullV2Repository(ref reference.Named) (err error) {
var refs []reference.Named
taggedName := p.repoInfo.LocalName
if tagged, isTagged := ref.(reference.Tagged); isTagged {
taggedName, err = reference.WithTag(p.repoInfo.LocalName, tagged.Tag())
if err != nil {
return err
}
refs = []reference.Named{taggedName}
} else if digested, isDigested := ref.(reference.Digested); isDigested {
taggedName, err = reference.WithDigest(p.repoInfo.LocalName, digested.Digest())
if err != nil {
return err
}
refs = []reference.Named{taggedName}
} else {
manSvc, err := p.repo.Manifests(context.Background())
if err != nil {
return err
}
tags, err := manSvc.Tags()
if err != nil {
return err
}
// This probably becomes a lot nicer after the manifest
// refactor...
for _, tag := range tags {
tagRef, err := reference.WithTag(p.repoInfo.LocalName, tag)
if err != nil {
return err
}
refs = append(refs, tagRef)
}
}
var layersDownloaded bool
for _, pullRef := range refs {
// pulledNew is true if either new layers were downloaded OR if existing images were newly tagged
// TODO(tiborvass): should we change the name of `layersDownload`? What about message in WriteStatus?
pulledNew, err := p.pullV2Tag(p.config.OutStream, pullRef)
if err != nil {
return err
}
layersDownloaded = layersDownloaded || pulledNew
}
writeStatus(taggedName.String(), p.config.OutStream, p.sf, layersDownloaded)
return nil
}
// downloadInfo is used to pass information from download to extractor
type downloadInfo struct {
tmpFile *os.File
digest digest.Digest
layer distribution.ReadSeekCloser
size int64
err chan error
poolKey string
broadcaster *broadcaster.Buffered
}
type errVerification struct{}
func (errVerification) Error() string { return "verification failed" }
func (p *v2Puller) download(di *downloadInfo) {
logrus.Debugf("pulling blob %q", di.digest)
blobs := p.repo.Blobs(context.Background())
desc, err := blobs.Stat(context.Background(), di.digest)
if err != nil {
logrus.Debugf("Error statting layer: %v", err)
di.err <- err
return
}
di.size = desc.Size
layerDownload, err := blobs.Open(context.Background(), di.digest)
if err != nil {
logrus.Debugf("Error fetching layer: %v", err)
di.err <- err
return
}
defer layerDownload.Close()
verifier, err := digest.NewDigestVerifier(di.digest)
if err != nil {
di.err <- err
return
}
digestStr := di.digest.String()
reader := progressreader.New(progressreader.Config{
In: ioutil.NopCloser(io.TeeReader(layerDownload, verifier)),
Out: di.broadcaster,
Formatter: p.sf,
Size: di.size,
NewLines: false,
ID: stringid.TruncateID(digestStr),
Action: "Downloading",
})
io.Copy(di.tmpFile, reader)
di.broadcaster.Write(p.sf.FormatProgress(stringid.TruncateID(digestStr), "Verifying Checksum", nil))
if !verifier.Verified() {
err = fmt.Errorf("filesystem layer verification failed for digest %s", di.digest)
logrus.Error(err)
di.err <- err
return
}
di.broadcaster.Write(p.sf.FormatProgress(stringid.TruncateID(digestStr), "Download complete", nil))
logrus.Debugf("Downloaded %s to tempfile %s", digestStr, di.tmpFile.Name())
di.layer = layerDownload
di.err <- nil
}
func (p *v2Puller) pullV2Tag(out io.Writer, ref reference.Named) (tagUpdated bool, err error) {
tagOrDigest := ""
if tagged, isTagged := ref.(reference.Tagged); isTagged {
tagOrDigest = tagged.Tag()
} else if digested, isDigested := ref.(reference.Digested); isDigested {
tagOrDigest = digested.Digest().String()
} else {
return false, fmt.Errorf("internal error: reference has neither a tag nor a digest: %s", ref.String())
}
logrus.Debugf("Pulling ref from V2 registry: %q", tagOrDigest)
manSvc, err := p.repo.Manifests(context.Background())
if err != nil {
return false, err
}
unverifiedManifest, err := manSvc.GetByTag(tagOrDigest)
if err != nil {
return false, err
}
if unverifiedManifest == nil {
return false, fmt.Errorf("image manifest does not exist for tag or digest %q", tagOrDigest)
}
var verifiedManifest *schema1.Manifest
verifiedManifest, err = verifyManifest(unverifiedManifest, ref)
if err != nil {
return false, err
}
rootFS := image.NewRootFS()
if err := detectBaseLayer(p.config.ImageStore, verifiedManifest, rootFS); err != nil {
return false, err
}
// remove duplicate layers and check parent chain validity
err = fixManifestLayers(verifiedManifest)
if err != nil {
return false, err
}
out.Write(p.sf.FormatStatus(tagOrDigest, "Pulling from %s", p.repo.Name()))
var downloads []*downloadInfo
defer func() {
for _, d := range downloads {
p.config.Pool.removeWithError(d.poolKey, err)
if d.tmpFile != nil {
d.tmpFile.Close()
if err := os.RemoveAll(d.tmpFile.Name()); err != nil {
logrus.Errorf("Failed to remove temp file: %s", d.tmpFile.Name())
}
}
}
}()
// Image history converted to the new format
var history []image.History
poolKey := "v2layer:"
notFoundLocally := false
// Note that the order of this loop is in the direction of bottom-most
// to top-most, so that the downloads slice gets ordered correctly.
for i := len(verifiedManifest.FSLayers) - 1; i >= 0; i-- {
blobSum := verifiedManifest.FSLayers[i].BlobSum
poolKey += blobSum.String()
var throwAway struct {
ThrowAway bool `json:"throwaway,omitempty"`
}
if err := json.Unmarshal([]byte(verifiedManifest.History[i].V1Compatibility), &throwAway); err != nil {
return false, err
}
h, err := v1.HistoryFromConfig([]byte(verifiedManifest.History[i].V1Compatibility), throwAway.ThrowAway)
if err != nil {
return false, err
}
history = append(history, h)
if throwAway.ThrowAway {
continue
}
// Do we have a layer on disk corresponding to the set of
// blobsums up to this point?
if !notFoundLocally {
notFoundLocally = true
diffID, err := p.blobSumService.GetDiffID(blobSum)
if err == nil {
rootFS.Append(diffID)
if l, err := p.config.LayerStore.Get(rootFS.ChainID()); err == nil {
notFoundLocally = false
logrus.Debugf("Layer already exists: %s", blobSum.String())
out.Write(p.sf.FormatProgress(stringid.TruncateID(blobSum.String()), "Already exists", nil))
defer layer.ReleaseAndLog(p.config.LayerStore, l)
continue
} else {
rootFS.DiffIDs = rootFS.DiffIDs[:len(rootFS.DiffIDs)-1]
}
}
}
out.Write(p.sf.FormatProgress(stringid.TruncateID(blobSum.String()), "Pulling fs layer", nil))
tmpFile, err := ioutil.TempFile("", "GetImageBlob")
if err != nil {
return false, err
}
d := &downloadInfo{
poolKey: poolKey,
digest: blobSum,
tmpFile: tmpFile,
// TODO: seems like this chan buffer solved hanging problem in go1.5,
// this can indicate some deeper problem that somehow we never take
// error from channel in loop below
err: make(chan error, 1),
}
downloads = append(downloads, d)
broadcaster, found := p.config.Pool.add(d.poolKey)
broadcaster.Add(out)
d.broadcaster = broadcaster
if found {
d.err <- nil
} else {
go p.download(d)
}
}
for _, d := range downloads {
if err := <-d.err; err != nil {
return false, err
}
if d.layer == nil {
// Wait for a different pull to download and extract
// this layer.
err = d.broadcaster.Wait()
if err != nil {
return false, err
}
diffID, err := p.blobSumService.GetDiffID(d.digest)
if err != nil {
return false, err
}
rootFS.Append(diffID)
l, err := p.config.LayerStore.Get(rootFS.ChainID())
if err != nil {
return false, err
}
defer layer.ReleaseAndLog(p.config.LayerStore, l)
continue
}
d.tmpFile.Seek(0, 0)
reader := progressreader.New(progressreader.Config{
In: d.tmpFile,
Out: d.broadcaster,
Formatter: p.sf,
Size: d.size,
NewLines: false,
ID: stringid.TruncateID(d.digest.String()),
Action: "Extracting",
})
inflatedLayerData, err := archive.DecompressStream(reader)
if err != nil {
return false, fmt.Errorf("could not get decompression stream: %v", err)
}
l, err := p.config.LayerStore.Register(inflatedLayerData, rootFS.ChainID())
if err != nil {
return false, fmt.Errorf("failed to register layer: %v", err)
}
logrus.Debugf("layer %s registered successfully", l.DiffID())
rootFS.Append(l.DiffID())
// Cache mapping from this layer's DiffID to the blobsum
if err := p.blobSumService.Add(l.DiffID(), d.digest); err != nil {
return false, err
}
defer layer.ReleaseAndLog(p.config.LayerStore, l)
d.broadcaster.Write(p.sf.FormatProgress(stringid.TruncateID(d.digest.String()), "Pull complete", nil))
d.broadcaster.Close()
tagUpdated = true
}
config, err := v1.MakeConfigFromV1Config([]byte(verifiedManifest.History[0].V1Compatibility), rootFS, history)
if err != nil {
return false, err
}
imageID, err := p.config.ImageStore.Create(config)
if err != nil {
return false, err
}
manifestDigest, _, err := digestFromManifest(unverifiedManifest, p.repoInfo.LocalName.Name())
if err != nil {
return false, err
}
// Check for new tag if no layers downloaded
var oldTagImageID image.ID
if !tagUpdated {
oldTagImageID, err = p.config.TagStore.Get(ref)
if err != nil || oldTagImageID != imageID {
tagUpdated = true
}
}
if tagUpdated {
if err = p.config.TagStore.Add(ref, imageID, true); err != nil {
return false, err
}
}
if manifestDigest != "" {
out.Write(p.sf.FormatStatus("", "Digest: %s", manifestDigest))
}
return tagUpdated, nil
}
func verifyManifest(signedManifest *schema1.SignedManifest, ref reference.Reference) (m *schema1.Manifest, err error) {
// If pull by digest, then verify the manifest digest. NOTE: It is
// important to do this first, before any other content validation. If the
// digest cannot be verified, don't even bother with those other things.
if digested, isDigested := ref.(reference.Digested); isDigested {
verifier, err := digest.NewDigestVerifier(digested.Digest())
if err != nil {
return nil, err
}
payload, err := signedManifest.Payload()
if err != nil {
// If this failed, the signatures section was corrupted
// or missing. Treat the entire manifest as the payload.
payload = signedManifest.Raw
}
if _, err := verifier.Write(payload); err != nil {
return nil, err
}
if !verifier.Verified() {
err := fmt.Errorf("image verification failed for digest %s", digested.Digest())
logrus.Error(err)
return nil, err
}
var verifiedManifest schema1.Manifest
if err = json.Unmarshal(payload, &verifiedManifest); err != nil {
return nil, err
}
m = &verifiedManifest
} else {
m = &signedManifest.Manifest
}
if m.SchemaVersion != 1 {
return nil, fmt.Errorf("unsupported schema version %d for %q", m.SchemaVersion, ref.String())
}
if len(m.FSLayers) != len(m.History) {
return nil, fmt.Errorf("length of history not equal to number of layers for %q", ref.String())
}
if len(m.FSLayers) == 0 {
return nil, fmt.Errorf("no FSLayers in manifest for %q", ref.String())
}
return m, nil
}
// fixManifestLayers removes repeated layers from the manifest and checks the
// correctness of the parent chain.
func fixManifestLayers(m *schema1.Manifest) error {
imgs := make([]*image.V1Image, len(m.FSLayers))
for i := range m.FSLayers {
img := &image.V1Image{}
if err := json.Unmarshal([]byte(m.History[i].V1Compatibility), img); err != nil {
return err
}
imgs[i] = img
if err := v1.ValidateID(img.ID); err != nil {
return err
}
}
if imgs[len(imgs)-1].Parent != "" && runtime.GOOS != "windows" {
// Windows base layer can point to a base layer parent that is not in manifest.
return errors.New("Invalid parent ID in the base layer of the image.")
}
// check general duplicates to error instead of a deadlock
idmap := make(map[string]struct{})
var lastID string
for _, img := range imgs {
// skip IDs that appear after each other, we handle those later
if _, exists := idmap[img.ID]; img.ID != lastID && exists {
return fmt.Errorf("ID %+v appears multiple times in manifest", img.ID)
}
lastID = img.ID
idmap[lastID] = struct{}{}
}
// backwards loop so that we keep the remaining indexes after removing items
for i := len(imgs) - 2; i >= 0; i-- {
if imgs[i].ID == imgs[i+1].ID { // repeated ID. remove and continue
m.FSLayers = append(m.FSLayers[:i], m.FSLayers[i+1:]...)
m.History = append(m.History[:i], m.History[i+1:]...)
} else if imgs[i].Parent != imgs[i+1].ID {
return fmt.Errorf("Invalid parent ID. Expected %v, got %v.", imgs[i+1].ID, imgs[i].Parent)
}
}
return nil
}

33
graph/pull_v2_test.go → distribution/pull_v2_test.go
View file

@ -1,4 +1,4 @@
package graph
package distribution
import (
"encoding/json"
@ -9,6 +9,7 @@ import (
"github.com/docker/distribution/digest"
"github.com/docker/distribution/manifest/schema1"
"github.com/docker/distribution/reference"
)
// TestFixManifestLayers checks that fixManifestLayers removes a duplicate
@ -103,7 +104,10 @@ func TestFixManifestLayersBadParent(t *testing.T) {
// TestValidateManifest verifies the validateManifest function
func TestValidateManifest(t *testing.T) {
expectedDigest := "sha256:02fee8c3220ba806531f606525eceb83f4feb654f62b207191b1c9209188dedd"
expectedDigest, err := reference.Parse("repo@sha256:02fee8c3220ba806531f606525eceb83f4feb654f62b207191b1c9209188dedd")
if err != nil {
t.Fatal("could not parse reference")
}
expectedFSLayer0 := digest.Digest("sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4")
// Good manifest
@ -167,29 +171,4 @@ func TestValidateManifest(t *testing.T) {
if err == nil || !strings.HasPrefix(err.Error(), "image verification failed for digest") {
t.Fatal("expected validateManifest to fail with digest error")
}
// Manifest with no signature
expectedWholeFileDigest := "7ec3615a120efcdfc270e9c7ea4183330775a3e52a09e2efb194b9a7c18e5ff7"
noSignatureManifestBytes, err := ioutil.ReadFile("fixtures/validate_manifest/no_signature_manifest")
if err != nil {
t.Fatal("error reading fixture:", err)
}
var noSignatureSignedManifest schema1.SignedManifest
noSignatureSignedManifest.Raw = noSignatureManifestBytes
err = json.Unmarshal(noSignatureManifestBytes, &noSignatureSignedManifest.Manifest)
if err != nil {
t.Fatal("error unmarshaling manifest:", err)
}
verifiedManifest, err = verifyManifest(&noSignatureSignedManifest, expectedWholeFileDigest)
if err != nil {
t.Fatal("validateManifest failed:", err)
}
if verifiedManifest.FSLayers[0].BlobSum != expectedFSLayer0 {
t.Fatal("unexpected FSLayer in no-signature manifest")
}
}

12
distribution/pull_v2_unix.go Normal file
View file

@ -0,0 +1,12 @@
// +build !windows
package distribution
import (
"github.com/docker/distribution/manifest/schema1"
"github.com/docker/docker/image"
)
func detectBaseLayer(is image.Store, m *schema1.Manifest, rootFS *image.RootFS) error {
return nil
}

29
distribution/pull_v2_windows.go Normal file
View file

@ -0,0 +1,29 @@
// +build windows
package distribution
import (
"encoding/json"
"fmt"
"github.com/docker/distribution/manifest/schema1"
"github.com/docker/docker/image"
)
func detectBaseLayer(is image.Store, m *schema1.Manifest, rootFS *image.RootFS) error {
v1img := &image.V1Image{}
if err := json.Unmarshal([]byte(m.History[len(m.History)-1].V1Compatibility), v1img); err != nil {
return err
}
if v1img.Parent == "" {
return fmt.Errorf("Last layer %q does not have a base layer reference", v1img.ID)
}
// There must be an image that already references the baselayer.
for _, img := range is.Map() {
if img.RootFS.BaseLayerID() == v1img.Parent {
rootFS.BaseLayer = img.RootFS.BaseLayer
return nil
}
}
return fmt.Errorf("Invalid base layer %q", v1img.Parent)
}

179
distribution/push.go Normal file
View file

@ -0,0 +1,179 @@
package distribution
import (
"bufio"
"compress/gzip"
"fmt"
"io"
"github.com/Sirupsen/logrus"
"github.com/docker/distribution/digest"
"github.com/docker/distribution/reference"
"github.com/docker/docker/cliconfig"
"github.com/docker/docker/daemon/events"
"github.com/docker/docker/distribution/metadata"
"github.com/docker/docker/image"
"github.com/docker/docker/layer"
"github.com/docker/docker/pkg/streamformatter"
"github.com/docker/docker/registry"
"github.com/docker/docker/tag"
"github.com/docker/libtrust"
)
// ImagePushConfig stores push configuration.
type ImagePushConfig struct {
// MetaHeaders store HTTP headers with metadata about the image
// (DockerHeaders with prefix X-Meta- in the request).
MetaHeaders map[string][]string
// AuthConfig holds authentication credentials for authenticating with
// the registry.
AuthConfig *cliconfig.AuthConfig
// OutStream is the output writer for showing the status of the push
// operation.
OutStream io.Writer
// RegistryService is the registry service to use for TLS configuration
// and endpoint lookup.
RegistryService *registry.Service
// EventsService is the events service to use for logging.
EventsService *events.Events
// MetadataStore is the storage backend for distribution-specific
// metadata.
MetadataStore metadata.Store
// LayerStore manges layers.
LayerStore layer.Store
// ImageStore manages images.
ImageStore image.Store
// TagStore manages tags.
TagStore tag.Store
// TrustKey is the private key for legacy signatures. This is typically
// an ephemeral key, since these signatures are no longer verified.
TrustKey libtrust.PrivateKey
}
// Pusher is an interface that abstracts pushing for different API versions.
type Pusher interface {
// Push tries to push the image configured at the creation of Pusher.
// Push returns an error if any, as well as a boolean that determines whether to retry Push on the next configured endpoint.
//
// TODO(tiborvass): have Push() take a reference to repository + tag, so that the pusher itself is repository-agnostic.
Push() (fallback bool, err error)
}
const compressionBufSize = 32768
// NewPusher creates a new Pusher interface that will push to either a v1 or v2
// registry. The endpoint argument contains a Version field that determines
// whether a v1 or v2 pusher will be created. The other parameters are passed
// through to the underlying pusher implementation for use during the actual
// push operation.
func NewPusher(ref reference.Named, endpoint registry.APIEndpoint, repoInfo *registry.RepositoryInfo, imagePushConfig *ImagePushConfig, sf *streamformatter.StreamFormatter) (Pusher, error) {
switch endpoint.Version {
case registry.APIVersion2:
return &v2Pusher{
blobSumService: metadata.NewBlobSumService(imagePushConfig.MetadataStore),
ref: ref,
endpoint: endpoint,
repoInfo: repoInfo,
config: imagePushConfig,
sf: sf,
layersPushed: make(map[digest.Digest]bool),
}, nil
case registry.APIVersion1:
return &v1Pusher{
v1IDService: metadata.NewV1IDService(imagePushConfig.MetadataStore),
ref: ref,
endpoint: endpoint,
repoInfo: repoInfo,
config: imagePushConfig,
sf: sf,
}, nil
}
return nil, fmt.Errorf("unknown version %d for registry %s", endpoint.Version, endpoint.URL)
}
// Push initiates a push operation on the repository named localName.
// ref is the specific variant of the image to be pushed.
// If no tag is provided, all tags will be pushed.
func Push(ref reference.Named, imagePushConfig *ImagePushConfig) error {
// FIXME: Allow to interrupt current push when new push of same image is done.
var sf = streamformatter.NewJSONStreamFormatter()
// Resolve the Repository name from fqn to RepositoryInfo
repoInfo, err := imagePushConfig.RegistryService.ResolveRepository(ref)
if err != nil {
return err
}
endpoints, err := imagePushConfig.RegistryService.LookupPushEndpoints(repoInfo.CanonicalName)
if err != nil {
return err
}
imagePushConfig.OutStream.Write(sf.FormatStatus("", "The push refers to a repository [%s]", repoInfo.CanonicalName))
associations := imagePushConfig.TagStore.ReferencesByName(repoInfo.LocalName)
if len(associations) == 0 {
return fmt.Errorf("Repository does not exist: %s", repoInfo.LocalName)
}
var lastErr error
for _, endpoint := range endpoints {
logrus.Debugf("Trying to push %s to %s %s", repoInfo.CanonicalName, endpoint.URL, endpoint.Version)
pusher, err := NewPusher(ref, endpoint, repoInfo, imagePushConfig, sf)
if err != nil {
lastErr = err
continue
}
if fallback, err := pusher.Push(); err != nil {
if fallback {
lastErr = err
continue
}
logrus.Debugf("Not continuing with error: %v", err)
return err
}
imagePushConfig.EventsService.Log("push", repoInfo.LocalName.Name(), "")
return nil
}
if lastErr == nil {
lastErr = fmt.Errorf("no endpoints found for %s", repoInfo.CanonicalName)
}
return lastErr
}
// compress returns an io.ReadCloser which will supply a compressed version of
// the provided Reader. The caller must close the ReadCloser after reading the
// compressed data.
//
// Note that this function returns a reader instead of taking a writer as an
// argument so that it can be used with httpBlobWriter's ReadFrom method.
// Using httpBlobWriter's Write method would send a PATCH request for every
// Write call.
func compress(in io.Reader) io.ReadCloser {
pipeReader, pipeWriter := io.Pipe()
// Use a bufio.Writer to avoid excessive chunking in HTTP request.
bufWriter := bufio.NewWriterSize(pipeWriter, compressionBufSize)
compressor := gzip.NewWriter(bufWriter)
go func() {
_, err := io.Copy(compressor, in)
if err == nil {
err = compressor.Close()
}
if err == nil {
err = bufWriter.Flush()
}
if err != nil {
pipeWriter.CloseWithError(err)
} else {
pipeWriter.Close()
}
}()
return pipeReader
}

466
distribution/push_v1.go Normal file
View file

@ -0,0 +1,466 @@
package distribution
import (
"fmt"
"io"
"io/ioutil"
"sync"
"github.com/Sirupsen/logrus"
"github.com/docker/distribution/digest"
"github.com/docker/distribution/reference"
"github.com/docker/distribution/registry/client/transport"
"github.com/docker/docker/distribution/metadata"
"github.com/docker/docker/image"
"github.com/docker/docker/image/v1"
"github.com/docker/docker/layer"
"github.com/docker/docker/pkg/ioutils"
"github.com/docker/docker/pkg/progressreader"
"github.com/docker/docker/pkg/streamformatter"
"github.com/docker/docker/pkg/stringid"
"github.com/docker/docker/registry"
)
type v1Pusher struct {
v1IDService *metadata.V1IDService
endpoint registry.APIEndpoint
ref reference.Named
repoInfo *registry.RepositoryInfo
config *ImagePushConfig
sf *streamformatter.StreamFormatter
session *registry.Session
out io.Writer
}
func (p *v1Pusher) Push() (fallback bool, err error) {
tlsConfig, err := p.config.RegistryService.TLSConfig(p.repoInfo.Index.Name)
if err != nil {
return false, err
}
// Adds Docker-specific headers as well as user-specified headers (metaHeaders)
tr := transport.NewTransport(
// TODO(tiborvass): was NoTimeout
registry.NewTransport(tlsConfig),
registry.DockerHeaders(p.config.MetaHeaders)...,
)
client := registry.HTTPClient(tr)
v1Endpoint, err := p.endpoint.ToV1Endpoint(p.config.MetaHeaders)
if err != nil {
logrus.Debugf("Could not get v1 endpoint: %v", err)
return true, err
}
p.session, err = registry.NewSession(client, p.config.AuthConfig, v1Endpoint)
if err != nil {
// TODO(dmcgowan): Check if should fallback
return true, err
}
if err := p.pushRepository(); err != nil {
// TODO(dmcgowan): Check if should fallback
return false, err
}
return false, nil
}
// v1Image exposes the configuration, filesystem layer ID, and a v1 ID for an
// image being pushed to a v1 registry.
type v1Image interface {
Config() []byte
Layer() layer.Layer
V1ID() string
}
type v1ImageCommon struct {
layer layer.Layer
config []byte
v1ID string
}
func (common *v1ImageCommon) Config() []byte {
return common.config
}
func (common *v1ImageCommon) V1ID() string {
return common.v1ID
}
func (common *v1ImageCommon) Layer() layer.Layer {
return common.layer
}
// v1TopImage defines a runnable (top layer) image being pushed to a v1
// registry.
type v1TopImage struct {
v1ImageCommon
imageID image.ID
}
func newV1TopImage(imageID image.ID, img *image.Image, l layer.Layer, parent *v1DependencyImage) (*v1TopImage, error) {
v1ID := digest.Digest(imageID).Hex()
parentV1ID := ""
if parent != nil {
parentV1ID = parent.V1ID()
}
config, err := v1.MakeV1ConfigFromConfig(img, v1ID, parentV1ID, false)
if err != nil {
return nil, err
}
return &v1TopImage{
v1ImageCommon: v1ImageCommon{
v1ID: v1ID,
config: config,
layer: l,
},
imageID: imageID,
}, nil
}
// v1DependencyImage defines a dependency layer being pushed to a v1 registry.
type v1DependencyImage struct {
v1ImageCommon
}
func newV1DependencyImage(l layer.Layer, parent *v1DependencyImage) (*v1DependencyImage, error) {
v1ID := digest.Digest(l.ChainID()).Hex()
config := ""
if parent != nil {
config = fmt.Sprintf(`{"id":"%s","parent":"%s"}`, v1ID, parent.V1ID())
} else {
config = fmt.Sprintf(`{"id":"%s"}`, v1ID)
}
return &v1DependencyImage{
v1ImageCommon: v1ImageCommon{
v1ID: v1ID,
config: []byte(config),
layer: l,
},
}, nil
}
// Retrieve the all the images to be uploaded in the correct order
func (p *v1Pusher) getImageList() (imageList []v1Image, tagsByImage map[image.ID][]string, referencedLayers []layer.Layer, err error) {
tagsByImage = make(map[image.ID][]string)
// Ignore digest references
_, isDigested := p.ref.(reference.Digested)
if isDigested {
return
}
tagged, isTagged := p.ref.(reference.Tagged)
if isTagged {
// Push a specific tag
var imgID image.ID
imgID, err = p.config.TagStore.Get(p.ref)
if err != nil {
return
}
imageList, err = p.imageListForTag(imgID, nil, &referencedLayers)
if err != nil {
return
}
tagsByImage[imgID] = []string{tagged.Tag()}
return
}
imagesSeen := make(map[image.ID]struct{})
dependenciesSeen := make(map[layer.ChainID]*v1DependencyImage)
associations := p.config.TagStore.ReferencesByName(p.ref)
for _, association := range associations {
if tagged, isTagged = association.Ref.(reference.Tagged); !isTagged {
// Ignore digest references.
continue
}
tagsByImage[association.ImageID] = append(tagsByImage[association.ImageID], tagged.Tag())
if _, present := imagesSeen[association.ImageID]; present {
// Skip generating image list for already-seen image
continue
}
imagesSeen[association.ImageID] = struct{}{}
imageListForThisTag, err := p.imageListForTag(association.ImageID, dependenciesSeen, &referencedLayers)
if err != nil {
return nil, nil, nil, err
}
// append to main image list
imageList = append(imageList, imageListForThisTag...)
}
if len(imageList) == 0 {
return nil, nil, nil, fmt.Errorf("No images found for the requested repository / tag")
}
logrus.Debugf("Image list: %v", imageList)
logrus.Debugf("Tags by image: %v", tagsByImage)
return
}
func (p *v1Pusher) imageListForTag(imgID image.ID, dependenciesSeen map[layer.ChainID]*v1DependencyImage, referencedLayers *[]layer.Layer) (imageListForThisTag []v1Image, err error) {
img, err := p.config.ImageStore.Get(imgID)
if err != nil {
return nil, err
}
topLayerID := img.RootFS.ChainID()
var l layer.Layer
if topLayerID == "" {
l = layer.EmptyLayer
} else {
l, err = p.config.LayerStore.Get(topLayerID)
*referencedLayers = append(*referencedLayers, l)
if err != nil {
return nil, fmt.Errorf("failed to get top layer from image: %v", err)
}
}
dependencyImages, parent, err := generateDependencyImages(l.Parent(), dependenciesSeen)
if err != nil {
return nil, err
}
topImage, err := newV1TopImage(imgID, img, l, parent)
if err != nil {
return nil, err
}
imageListForThisTag = append(dependencyImages, topImage)
return
}
func generateDependencyImages(l layer.Layer, dependenciesSeen map[layer.ChainID]*v1DependencyImage) (imageListForThisTag []v1Image, parent *v1DependencyImage, err error) {
if l == nil {
return nil, nil, nil
}
imageListForThisTag, parent, err = generateDependencyImages(l.Parent(), dependenciesSeen)
if dependenciesSeen != nil {
if dependencyImage, present := dependenciesSeen[l.ChainID()]; present {
// This layer is already on the list, we can ignore it
// and all its parents.
return imageListForThisTag, dependencyImage, nil
}
}
dependencyImage, err := newV1DependencyImage(l, parent)
if err != nil {
return nil, nil, err
}
imageListForThisTag = append(imageListForThisTag, dependencyImage)
if dependenciesSeen != nil {
dependenciesSeen[l.ChainID()] = dependencyImage
}
return imageListForThisTag, dependencyImage, nil
}
// createImageIndex returns an index of an image's layer IDs and tags.
func createImageIndex(images []v1Image, tags map[image.ID][]string) []*registry.ImgData {
var imageIndex []*registry.ImgData
for _, img := range images {
v1ID := img.V1ID()
if topImage, isTopImage := img.(*v1TopImage); isTopImage {
if tags, hasTags := tags[topImage.imageID]; hasTags {
// If an image has tags you must add an entry in the image index
// for each tag
for _, tag := range tags {
imageIndex = append(imageIndex, &registry.ImgData{
ID: v1ID,
Tag: tag,
})
}
continue
}
}
// If the image does not have a tag it still needs to be sent to the
// registry with an empty tag so that it is associated with the repository
imageIndex = append(imageIndex, &registry.ImgData{
ID: v1ID,
Tag: "",
})
}
return imageIndex
}
// lookupImageOnEndpoint checks the specified endpoint to see if an image exists
// and if it is absent then it sends the image id to the channel to be pushed.
func (p *v1Pusher) lookupImageOnEndpoint(wg *sync.WaitGroup, endpoint string, images chan v1Image, imagesToPush chan string) {
defer wg.Done()
for image := range images {
v1ID := image.V1ID()
if err := p.session.LookupRemoteImage(v1ID, endpoint); err != nil {
logrus.Errorf("Error in LookupRemoteImage: %s", err)
imagesToPush <- v1ID
} else {
p.out.Write(p.sf.FormatStatus("", "Image %s already pushed, skipping", stringid.TruncateID(v1ID)))
}
}
}
func (p *v1Pusher) pushImageToEndpoint(endpoint string, imageList []v1Image, tags map[image.ID][]string, repo *registry.RepositoryData) error {
workerCount := len(imageList)
// start a maximum of 5 workers to check if images exist on the specified endpoint.
if workerCount > 5 {
workerCount = 5
}
var (
wg = &sync.WaitGroup{}
imageData = make(chan v1Image, workerCount*2)
imagesToPush = make(chan string, workerCount*2)
pushes = make(chan map[string]struct{}, 1)
)
for i := 0; i < workerCount; i++ {
wg.Add(1)
go p.lookupImageOnEndpoint(wg, endpoint, imageData, imagesToPush)
}
// start a go routine that consumes the images to push
go func() {
shouldPush := make(map[string]struct{})
for id := range imagesToPush {
shouldPush[id] = struct{}{}
}
pushes <- shouldPush
}()
for _, v1Image := range imageList {
imageData <- v1Image
}
// close the channel to notify the workers that there will be no more images to check.
close(imageData)
wg.Wait()
close(imagesToPush)
// wait for all the images that require pushes to be collected into a consumable map.
shouldPush := <-pushes
// finish by pushing any images and tags to the endpoint. The order that the images are pushed
// is very important that is why we are still iterating over the ordered list of imageIDs.
for _, img := range imageList {
v1ID := img.V1ID()
if _, push := shouldPush[v1ID]; push {
if _, err := p.pushImage(img, endpoint); err != nil {
// FIXME: Continue on error?
return err
}
}
if topImage, isTopImage := img.(*v1TopImage); isTopImage {
for _, tag := range tags[topImage.imageID] {
p.out.Write(p.sf.FormatStatus("", "Pushing tag for rev [%s] on {%s}", stringid.TruncateID(v1ID), endpoint+"repositories/"+p.repoInfo.RemoteName.Name()+"/tags/"+tag))
if err := p.session.PushRegistryTag(p.repoInfo.RemoteName, v1ID, tag, endpoint); err != nil {
return err
}
}
}
}
return nil
}
// pushRepository pushes layers that do not already exist on the registry.
func (p *v1Pusher) pushRepository() error {
p.out = ioutils.NewWriteFlusher(p.config.OutStream)
imgList, tags, referencedLayers, err := p.getImageList()
defer func() {
for _, l := range referencedLayers {
p.config.LayerStore.Release(l)
}
}()
if err != nil {
return err
}
p.out.Write(p.sf.FormatStatus("", "Sending image list"))
imageIndex := createImageIndex(imgList, tags)
for _, data := range imageIndex {
logrus.Debugf("Pushing ID: %s with Tag: %s", data.ID, data.Tag)
}
// Register all the images in a repository with the registry
// If an image is not in this list it will not be associated with the repository
repoData, err := p.session.PushImageJSONIndex(p.repoInfo.RemoteName, imageIndex, false, nil)
if err != nil {
return err
}
p.out.Write(p.sf.FormatStatus("", "Pushing repository %s", p.repoInfo.CanonicalName))
// push the repository to each of the endpoints only if it does not exist.
for _, endpoint := range repoData.Endpoints {
if err := p.pushImageToEndpoint(endpoint, imgList, tags, repoData); err != nil {
return err
}
}
_, err = p.session.PushImageJSONIndex(p.repoInfo.RemoteName, imageIndex, true, repoData.Endpoints)
return err
}
func (p *v1Pusher) pushImage(v1Image v1Image, ep string) (checksum string, err error) {
v1ID := v1Image.V1ID()
jsonRaw := v1Image.Config()
p.out.Write(p.sf.FormatProgress(stringid.TruncateID(v1ID), "Pushing", nil))
// General rule is to use ID for graph accesses and compatibilityID for
// calls to session.registry()
imgData := &registry.ImgData{
ID: v1ID,
}
// Send the json
if err := p.session.PushImageJSONRegistry(imgData, jsonRaw, ep); err != nil {
if err == registry.ErrAlreadyExists {
p.out.Write(p.sf.FormatProgress(stringid.TruncateID(v1ID), "Image already pushed, skipping", nil))
return "", nil
}
return "", err
}
l := v1Image.Layer()
arch, err := l.TarStream()
if err != nil {
return "", err
}
// don't care if this fails; best effort
size, _ := l.Size()
// Send the layer
logrus.Debugf("rendered layer for %s of [%d] size", v1ID, size)
reader := progressreader.New(progressreader.Config{
In: ioutil.NopCloser(arch),
Out: p.out,
Formatter: p.sf,
Size: size,
NewLines: false,
ID: stringid.TruncateID(v1ID),
Action: "Pushing",
})
checksum, checksumPayload, err := p.session.PushImageLayerRegistry(v1ID, reader, ep, jsonRaw)
if err != nil {
return "", err
}
imgData.Checksum = checksum
imgData.ChecksumPayload = checksumPayload
// Send the checksum
if err := p.session.PushImageChecksumRegistry(imgData, ep); err != nil {
return "", err
}
if err := p.v1IDService.Set(v1ID, p.repoInfo.Index.Name, l.ChainID()); err != nil {
logrus.Warnf("Could not set v1 ID mapping: %v", err)
}
p.out.Write(p.sf.FormatProgress(stringid.TruncateID(v1ID), "Image successfully pushed", nil))
return imgData.Checksum, nil
}

410
distribution/push_v2.go Normal file
View file

@ -0,0 +1,410 @@
package distribution
import (
"encoding/json"
"errors"
"fmt"
"io"
"io/ioutil"
"time"
"github.com/Sirupsen/logrus"
"github.com/docker/distribution"
"github.com/docker/distribution/digest"
"github.com/docker/distribution/manifest"
"github.com/docker/distribution/manifest/schema1"
"github.com/docker/distribution/reference"
"github.com/docker/docker/distribution/metadata"
"github.com/docker/docker/image"
"github.com/docker/docker/image/v1"
"github.com/docker/docker/layer"
"github.com/docker/docker/pkg/progressreader"
"github.com/docker/docker/pkg/streamformatter"
"github.com/docker/docker/pkg/stringid"
"github.com/docker/docker/registry"
"github.com/docker/docker/tag"
"golang.org/x/net/context"
)
type v2Pusher struct {
blobSumService *metadata.BlobSumService
ref reference.Named
endpoint registry.APIEndpoint
repoInfo *registry.RepositoryInfo
config *ImagePushConfig
sf *streamformatter.StreamFormatter
repo distribution.Repository
// layersPushed is the set of layers known to exist on the remote side.
// This avoids redundant queries when pushing multiple tags that
// involve the same layers.
layersPushed map[digest.Digest]bool
}
func (p *v2Pusher) Push() (fallback bool, err error) {
p.repo, err = NewV2Repository(p.repoInfo, p.endpoint, p.config.MetaHeaders, p.config.AuthConfig, "push", "pull")
if err != nil {
logrus.Debugf("Error getting v2 registry: %v", err)
return true, err
}
localName := p.repoInfo.LocalName.Name()
var associations []tag.Association
if _, isTagged := p.ref.(reference.Tagged); isTagged {
imageID, err := p.config.TagStore.Get(p.ref)
if err != nil {
return false, fmt.Errorf("tag does not exist: %s", p.ref.String())
}
associations = []tag.Association{
{
Ref: p.ref,
ImageID: imageID,
},
}
} else {
// Pull all tags
associations = p.config.TagStore.ReferencesByName(p.ref)
}
if err != nil {
return false, fmt.Errorf("error getting tags for %s: %s", localName, err)
}
if len(associations) == 0 {
return false, fmt.Errorf("no tags to push for %s", localName)
}
for _, association := range associations {
if err := p.pushV2Tag(association); err != nil {
return false, err
}
}
return false, nil
}
func (p *v2Pusher) pushV2Tag(association tag.Association) error {
ref := association.Ref
logrus.Debugf("Pushing repository: %s", ref.String())
img, err := p.config.ImageStore.Get(association.ImageID)
if err != nil {
return fmt.Errorf("could not find image from tag %s: %v", ref.String(), err)
}
out := p.config.OutStream
var l layer.Layer
topLayerID := img.RootFS.ChainID()
if topLayerID == "" {
l = layer.EmptyLayer
} else {
l, err = p.config.LayerStore.Get(topLayerID)
if err != nil {
return fmt.Errorf("failed to get top layer from image: %v", err)
}
defer layer.ReleaseAndLog(p.config.LayerStore, l)
}
fsLayers := make(map[layer.DiffID]schema1.FSLayer)
// Push empty layer if necessary
for _, h := range img.History {
if h.EmptyLayer {
dgst, err := p.pushLayerIfNecessary(out, layer.EmptyLayer)
if err != nil {
return err
}
p.layersPushed[dgst] = true
fsLayers[layer.EmptyLayer.DiffID()] = schema1.FSLayer{BlobSum: dgst}
break
}
}
for i := 0; i < len(img.RootFS.DiffIDs); i++ {
dgst, err := p.pushLayerIfNecessary(out, l)
if err != nil {
return err
}
p.layersPushed[dgst] = true
fsLayers[l.DiffID()] = schema1.FSLayer{BlobSum: dgst}
l = l.Parent()
}
var tag string
if tagged, isTagged := ref.(reference.Tagged); isTagged {
tag = tagged.Tag()
}
m, err := CreateV2Manifest(p.repo.Name(), tag, img, fsLayers)
if err != nil {
return err
}
logrus.Infof("Signed manifest for %s using daemon's key: %s", ref.String(), p.config.TrustKey.KeyID())
signed, err := schema1.Sign(m, p.config.TrustKey)
if err != nil {
return err
}
manifestDigest, manifestSize, err := digestFromManifest(signed, p.repo.Name())
if err != nil {
return err
}
if manifestDigest != "" {
if tagged, isTagged := ref.(reference.Tagged); isTagged {
// NOTE: do not change this format without first changing the trust client
// code. This information is used to determine what was pushed and should be signed.
out.Write(p.sf.FormatStatus("", "%s: digest: %s size: %d", tagged.Tag(), manifestDigest, manifestSize))
}
}
manSvc, err := p.repo.Manifests(context.Background())
if err != nil {
return err
}
return manSvc.Put(signed)
}
func (p *v2Pusher) pushLayerIfNecessary(out io.Writer, l layer.Layer) (digest.Digest, error) {
logrus.Debugf("Pushing layer: %s", l.DiffID())
// Do we have any blobsums associated with this layer's DiffID?
possibleBlobsums, err := p.blobSumService.GetBlobSums(l.DiffID())
if err == nil {
dgst, exists, err := p.blobSumAlreadyExists(possibleBlobsums)
if err != nil {
out.Write(p.sf.FormatProgress(stringid.TruncateID(string(l.DiffID())), "Image push failed", nil))
return "", err
}
if exists {
out.Write(p.sf.FormatProgress(stringid.TruncateID(string(l.DiffID())), "Layer already exists", nil))
return dgst, nil
}
}
// if digest was empty or not saved, or if blob does not exist on the remote repository,
// then push the blob.
pushDigest, err := p.pushV2Layer(p.repo.Blobs(context.Background()), l)
if err != nil {
return "", err
}
// Cache mapping from this layer's DiffID to the blobsum
if err := p.blobSumService.Add(l.DiffID(), pushDigest); err != nil {
return "", err
}
return pushDigest, nil
}
// blobSumAlreadyExists checks if the registry already know about any of the
// blobsums passed in the "blobsums" slice. If it finds one that the registry
// knows about, it returns the known digest and "true".
func (p *v2Pusher) blobSumAlreadyExists(blobsums []digest.Digest) (digest.Digest, bool, error) {
for _, dgst := range blobsums {
if p.layersPushed[dgst] {
// it is already known that the push is not needed and
// therefore doing a stat is unnecessary
return dgst, true, nil
}
_, err := p.repo.Blobs(context.Background()).Stat(context.Background(), dgst)
switch err {
case nil:
return dgst, true, nil
case distribution.ErrBlobUnknown:
// nop
default:
return "", false, err
}
}
return "", false, nil
}
// CreateV2Manifest creates a V2 manifest from an image config and set of
// FSLayer digests.
// FIXME: This should be moved to the distribution repo, since it will also
// be useful for converting new manifests to the old format.
func CreateV2Manifest(name, tag string, img *image.Image, fsLayers map[layer.DiffID]schema1.FSLayer) (*schema1.Manifest, error) {
if len(img.History) == 0 {
return nil, errors.New("empty history when trying to create V2 manifest")
}
// Generate IDs for each layer
// For non-top-level layers, create fake V1Compatibility strings that
// fit the format and don't collide with anything else, but don't
// result in runnable images on their own.
type v1Compatibility struct {
ID string `json:"id"`
Parent string `json:"parent,omitempty"`
Comment string `json:"comment,omitempty"`
Created time.Time `json:"created"`
ContainerConfig struct {
Cmd []string
} `json:"container_config,omitempty"`
ThrowAway bool `json:"throwaway,omitempty"`
}
fsLayerList := make([]schema1.FSLayer, len(img.History))
history := make([]schema1.History, len(img.History))
parent := ""
layerCounter := 0
for i, h := range img.History {
if i == len(img.History)-1 {
break
}
var diffID layer.DiffID
if h.EmptyLayer {
diffID = layer.EmptyLayer.DiffID()
} else {
if len(img.RootFS.DiffIDs) <= layerCounter {
return nil, errors.New("too many non-empty layers in History section")
}
diffID = img.RootFS.DiffIDs[layerCounter]
layerCounter++
}
fsLayer, present := fsLayers[diffID]
if !present {
return nil, fmt.Errorf("missing layer in CreateV2Manifest: %s", diffID.String())
}
dgst, err := digest.FromBytes([]byte(fsLayer.BlobSum.Hex() + " " + parent))
if err != nil {
return nil, err
}
v1ID := dgst.Hex()
v1Compatibility := v1Compatibility{
ID: v1ID,
Parent: parent,
Comment: h.Comment,
Created: h.Created,
}
v1Compatibility.ContainerConfig.Cmd = []string{img.History[i].CreatedBy}
if h.EmptyLayer {
v1Compatibility.ThrowAway = true
}
jsonBytes, err := json.Marshal(&v1Compatibility)
if err != nil {
return nil, err
}
reversedIndex := len(img.History) - i - 1
history[reversedIndex].V1Compatibility = string(jsonBytes)
fsLayerList[reversedIndex] = fsLayer
parent = v1ID
}
latestHistory := img.History[len(img.History)-1]
var diffID layer.DiffID
if latestHistory.EmptyLayer {
diffID = layer.EmptyLayer.DiffID()
} else {
if len(img.RootFS.DiffIDs) <= layerCounter {
return nil, errors.New("too many non-empty layers in History section")
}
diffID = img.RootFS.DiffIDs[layerCounter]
}
fsLayer, present := fsLayers[diffID]
if !present {
return nil, fmt.Errorf("missing layer in CreateV2Manifest: %s", diffID.String())
}
dgst, err := digest.FromBytes([]byte(fsLayer.BlobSum.Hex() + " " + parent + " " + string(img.RawJSON())))
if err != nil {
return nil, err
}
fsLayerList[0] = fsLayer
// Top-level v1compatibility string should be a modified version of the
// image config.
transformedConfig, err := v1.MakeV1ConfigFromConfig(img, dgst.Hex(), parent, latestHistory.EmptyLayer)
if err != nil {
return nil, err
}
history[0].V1Compatibility = string(transformedConfig)
// windows-only baselayer setup
if err := setupBaseLayer(history, *img.RootFS); err != nil {
return nil, err
}
return &schema1.Manifest{
Versioned: manifest.Versioned{
SchemaVersion: 1,
},
Name: name,
Tag: tag,
Architecture: img.Architecture,
FSLayers: fsLayerList,
History: history,
}, nil
}
func rawJSON(value interface{}) *json.RawMessage {
jsonval, err := json.Marshal(value)
if err != nil {
return nil
}
return (*json.RawMessage)(&jsonval)
}
func (p *v2Pusher) pushV2Layer(bs distribution.BlobService, l layer.Layer) (digest.Digest, error) {
out := p.config.OutStream
displayID := stringid.TruncateID(string(l.DiffID()))
out.Write(p.sf.FormatProgress(displayID, "Preparing", nil))
arch, err := l.TarStream()
if err != nil {
return "", err
}
// Send the layer
layerUpload, err := bs.Create(context.Background())
if err != nil {
return "", err
}
defer layerUpload.Close()
// don't care if this fails; best effort
size, _ := l.DiffSize()
reader := progressreader.New(progressreader.Config{
In: ioutil.NopCloser(arch), // we'll take care of close here.
Out: out,
Formatter: p.sf,
Size: size,
NewLines: false,
ID: displayID,
Action: "Pushing",
})
compressedReader := compress(reader)
digester := digest.Canonical.New()
tee := io.TeeReader(compressedReader, digester.Hash())
out.Write(p.sf.FormatProgress(displayID, "Pushing", nil))
nn, err := layerUpload.ReadFrom(tee)
compressedReader.Close()
if err != nil {
return "", err
}
dgst := digester.Digest()
if _, err := layerUpload.Commit(context.Background(), distribution.Descriptor{Digest: dgst}); err != nil {
return "", err
}
logrus.Debugf("uploaded layer %s (%s), %d bytes", l.DiffID(), dgst, nn)
out.Write(p.sf.FormatProgress(displayID, "Pushed", nil))
return dgst, nil
}

176
distribution/push_v2_test.go Normal file
View file

@ -0,0 +1,176 @@
package distribution
import (
"reflect"
"testing"
"github.com/docker/distribution/digest"
"github.com/docker/distribution/manifest/schema1"
"github.com/docker/docker/image"
"github.com/docker/docker/layer"
)
func TestCreateV2Manifest(t *testing.T) {
imgJSON := `{
"architecture": "amd64",
"config": {
"AttachStderr": false,
"AttachStdin": false,
"AttachStdout": false,
"Cmd": [
"/bin/sh",
"-c",
"echo hi"
],
"Domainname": "",
"Entrypoint": null,
"Env": [
"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
"derived=true",
"asdf=true"
],
"Hostname": "23304fc829f9",
"Image": "sha256:4ab15c48b859c2920dd5224f92aabcd39a52794c5b3cf088fb3bbb438756c246",
"Labels": {},
"OnBuild": [],
"OpenStdin": false,
"StdinOnce": false,
"Tty": false,
"User": "",
"Volumes": null,
"WorkingDir": ""
},
"container": "e91032eb0403a61bfe085ff5a5a48e3659e5a6deae9f4d678daa2ae399d5a001",
"container_config": {
"AttachStderr": false,
"AttachStdin": false,
"AttachStdout": false,
"Cmd": [
"/bin/sh",
"-c",
"#(nop) CMD [\"/bin/sh\" \"-c\" \"echo hi\"]"
],
"Domainname": "",
"Entrypoint": null,
"Env": [
"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
"derived=true",
"asdf=true"
],
"Hostname": "23304fc829f9",
"Image": "sha256:4ab15c48b859c2920dd5224f92aabcd39a52794c5b3cf088fb3bbb438756c246",
"Labels": {},
"OnBuild": [],
"OpenStdin": false,
"StdinOnce": false,
"Tty": false,
"User": "",
"Volumes": null,
"WorkingDir": ""
},
"created": "2015-11-04T23:06:32.365666163Z",
"docker_version": "1.9.0-dev",
"history": [
{
"created": "2015-10-31T22:22:54.690851953Z",
"created_by": "/bin/sh -c #(nop) ADD file:a3bc1e842b69636f9df5256c49c5374fb4eef1e281fe3f282c65fb853ee171c5 in /"
},
{
"created": "2015-10-31T22:22:55.613815829Z",
"created_by": "/bin/sh -c #(nop) CMD [\"sh\"]"
},
{
"created": "2015-11-04T23:06:30.934316144Z",
"created_by": "/bin/sh -c #(nop) ENV derived=true",
"empty_layer": true
},
{
"created": "2015-11-04T23:06:31.192097572Z",
"created_by": "/bin/sh -c #(nop) ENV asdf=true",
"empty_layer": true
},
{
"created": "2015-11-04T23:06:32.083868454Z",
"created_by": "/bin/sh -c dd if=/dev/zero of=/file bs=1024 count=1024"
},
{
"created": "2015-11-04T23:06:32.365666163Z",
"created_by": "/bin/sh -c #(nop) CMD [\"/bin/sh\" \"-c\" \"echo hi\"]",
"empty_layer": true
}
],
"os": "linux",
"rootfs": {
"diff_ids": [
"sha256:c6f988f4874bb0add23a778f753c65efe992244e148a1d2ec2a8b664fb66bbd1",
"sha256:5f70bf18a086007016e948b04aed3b82103a36bea41755b6cddfaf10ace3c6ef",
"sha256:13f53e08df5a220ab6d13c58b2bf83a59cbdc2e04d0a3f041ddf4b0ba4112d49"
],
"type": "layers"
}
}`
// To fill in rawJSON
img, err := image.NewFromJSON([]byte(imgJSON))
if err != nil {
t.Fatalf("json decoding failed: %v", err)
}
fsLayers := map[layer.DiffID]schema1.FSLayer{
layer.DiffID("sha256:c6f988f4874bb0add23a778f753c65efe992244e148a1d2ec2a8b664fb66bbd1"): {BlobSum: digest.Digest("sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4")},
layer.DiffID("sha256:5f70bf18a086007016e948b04aed3b82103a36bea41755b6cddfaf10ace3c6ef"): {BlobSum: digest.Digest("sha256:86e0e091d0da6bde2456dbb48306f3956bbeb2eae1b5b9a43045843f69fe4aaa")},
layer.DiffID("sha256:13f53e08df5a220ab6d13c58b2bf83a59cbdc2e04d0a3f041ddf4b0ba4112d49"): {BlobSum: digest.Digest("sha256:b4ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4")},
}
manifest, err := CreateV2Manifest("testrepo", "testtag", img, fsLayers)
if err != nil {
t.Fatalf("CreateV2Manifest returned error: %v", err)
}
if manifest.Versioned.SchemaVersion != 1 {
t.Fatal("SchemaVersion != 1")
}
if manifest.Name != "testrepo" {
t.Fatal("incorrect name in manifest")
}
if manifest.Tag != "testtag" {
t.Fatal("incorrect tag in manifest")
}
if manifest.Architecture != "amd64" {
t.Fatal("incorrect arch in manifest")
}
expectedFSLayers := []schema1.FSLayer{
{BlobSum: digest.Digest("sha256:86e0e091d0da6bde2456dbb48306f3956bbeb2eae1b5b9a43045843f69fe4aaa")},
{BlobSum: digest.Digest("sha256:b4ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4")},
{BlobSum: digest.Digest("sha256:86e0e091d0da6bde2456dbb48306f3956bbeb2eae1b5b9a43045843f69fe4aaa")},
{BlobSum: digest.Digest("sha256:86e0e091d0da6bde2456dbb48306f3956bbeb2eae1b5b9a43045843f69fe4aaa")},
{BlobSum: digest.Digest("sha256:86e0e091d0da6bde2456dbb48306f3956bbeb2eae1b5b9a43045843f69fe4aaa")},
{BlobSum: digest.Digest("sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4")},
}
if len(manifest.FSLayers) != len(expectedFSLayers) {
t.Fatalf("wrong number of FSLayers: %d", len(manifest.FSLayers))
}
if !reflect.DeepEqual(manifest.FSLayers, expectedFSLayers) {
t.Fatal("wrong FSLayers list")
}
expectedV1Compatibility := []string{
`{"architecture":"amd64","config":{"AttachStderr":false,"AttachStdin":false,"AttachStdout":false,"Cmd":["/bin/sh","-c","echo hi"],"Domainname":"","Entrypoint":null,"Env":["PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin","derived=true","asdf=true"],"Hostname":"23304fc829f9","Image":"sha256:4ab15c48b859c2920dd5224f92aabcd39a52794c5b3cf088fb3bbb438756c246","Labels":{},"OnBuild":[],"OpenStdin":false,"StdinOnce":false,"Tty":false,"User":"","Volumes":null,"WorkingDir":""},"container":"e91032eb0403a61bfe085ff5a5a48e3659e5a6deae9f4d678daa2ae399d5a001","container_config":{"AttachStderr":false,"AttachStdin":false,"AttachStdout":false,"Cmd":["/bin/sh","-c","#(nop) CMD [\"/bin/sh\" \"-c\" \"echo hi\"]"],"Domainname":"","Entrypoint":null,"Env":["PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin","derived=true","asdf=true"],"Hostname":"23304fc829f9","Image":"sha256:4ab15c48b859c2920dd5224f92aabcd39a52794c5b3cf088fb3bbb438756c246","Labels":{},"OnBuild":[],"OpenStdin":false,"StdinOnce":false,"Tty":false,"User":"","Volumes":null,"WorkingDir":""},"created":"2015-11-04T23:06:32.365666163Z","docker_version":"1.9.0-dev","id":"d728140d3fd23dfcac505954af0b2224b3579b177029eded62916579eb19ac64","os":"linux","parent":"0594e66a9830fa5ba73b66349eb221ea4beb6bac8d2148b90a0f371f8d67bcd5","throwaway":true}`,
`{"id":"0594e66a9830fa5ba73b66349eb221ea4beb6bac8d2148b90a0f371f8d67bcd5","parent":"39bc0dbed47060dd8952b048e73744ae471fe50354d2c267d308292c53b83ce1","created":"2015-11-04T23:06:32.083868454Z","container_config":{"Cmd":["/bin/sh -c dd if=/dev/zero of=/file bs=1024 count=1024"]}}`,
`{"id":"39bc0dbed47060dd8952b048e73744ae471fe50354d2c267d308292c53b83ce1","parent":"875d7f206c023dc979e1677567a01364074f82b61e220c9b83a4610170490381","created":"2015-11-04T23:06:31.192097572Z","container_config":{"Cmd":["/bin/sh -c #(nop) ENV asdf=true"]},"throwaway":true}`,
`{"id":"875d7f206c023dc979e1677567a01364074f82b61e220c9b83a4610170490381","parent":"9e3447ca24cb96d86ebd5960cb34d1299b07e0a0e03801d90b9969a2c187dd6e","created":"2015-11-04T23:06:30.934316144Z","container_config":{"Cmd":["/bin/sh -c #(nop) ENV derived=true"]},"throwaway":true}`,
`{"id":"9e3447ca24cb96d86ebd5960cb34d1299b07e0a0e03801d90b9969a2c187dd6e","parent":"3690474eb5b4b26fdfbd89c6e159e8cc376ca76ef48032a30fa6aafd56337880","created":"2015-10-31T22:22:55.613815829Z","container_config":{"Cmd":["/bin/sh -c #(nop) CMD [\"sh\"]"]}}`,
`{"id":"3690474eb5b4b26fdfbd89c6e159e8cc376ca76ef48032a30fa6aafd56337880","created":"2015-10-31T22:22:54.690851953Z","container_config":{"Cmd":["/bin/sh -c #(nop) ADD file:a3bc1e842b69636f9df5256c49c5374fb4eef1e281fe3f282c65fb853ee171c5 in /"]}}`,
}
if len(manifest.History) != len(expectedV1Compatibility) {
t.Fatalf("wrong number of history entries: %d", len(manifest.History))
}
for i := range expectedV1Compatibility {
if manifest.History[i].V1Compatibility != expectedV1Compatibility[i] {
t.Fatalf("wrong V1Compatibility %d. expected:\n%s\ngot:\n%s", i, expectedV1Compatibility[i], manifest.History[i].V1Compatibility)
}
}
}

12
distribution/push_v2_unix.go Normal file
View file

@ -0,0 +1,12 @@
// +build !windows
package distribution
import (
"github.com/docker/distribution/manifest/schema1"
"github.com/docker/docker/image"
)
func setupBaseLayer(history []schema1.History, rootFS image.RootFS) error {
return nil
}

28
distribution/push_v2_windows.go Normal file
View file

@ -0,0 +1,28 @@
// +build windows
package distribution
import (
"encoding/json"
"github.com/docker/distribution/manifest/schema1"
"github.com/docker/docker/image"
)
func setupBaseLayer(history []schema1.History, rootFS image.RootFS) error {
var v1Config map[string]*json.RawMessage
if err := json.Unmarshal([]byte(history[len(history)-1].V1Compatibility), &v1Config); err != nil {
return err
}
baseID, err := json.Marshal(rootFS.BaseLayerID())
if err != nil {
return err
}
v1Config["parent"] = (*json.RawMessage)(&baseID)
configJSON, err := json.Marshal(v1Config)
if err != nil {
return err
}
history[len(history)-1].V1Compatibility = string(configJSON)
return nil
}

15
graph/registry.go → distribution/registry.go
View file

@ -1,13 +1,12 @@
package graph
package distribution
import (
"errors"
"net"
"net/http"
"net/url"
"time"
"strings"
"time"
"github.com/Sirupsen/logrus"
"github.com/docker/distribution"
@ -29,10 +28,10 @@ func (dcs dumbCredentialStore) Basic(*url.URL) (string, string) {
return dcs.auth.Username, dcs.auth.Password
}
// newV2Repository returns a repository (v2 only). It creates a HTTP transport
// NewV2Repository returns a repository (v2 only). It creates a HTTP transport
// providing timeout settings and authentication support, and also verifies the
// remote API version.
func newV2Repository(repoInfo *registry.RepositoryInfo, endpoint registry.APIEndpoint, metaHeaders http.Header, authConfig *cliconfig.AuthConfig, actions ...string) (distribution.Repository, error) {
func NewV2Repository(repoInfo *registry.RepositoryInfo, endpoint registry.APIEndpoint, metaHeaders http.Header, authConfig *cliconfig.AuthConfig, actions ...string) (distribution.Repository, error) {
ctx := context.Background()
repoName := repoInfo.CanonicalName
@ -59,7 +58,7 @@ func newV2Repository(repoInfo *registry.RepositoryInfo, endpoint registry.APIEnd
authTransport := transport.NewTransport(base, modifiers...)
pingClient := &http.Client{
Transport: authTransport,
Timeout: 15 * time.Second,
Timeout: 5 * time.Second,
}
endpointStr := strings.TrimRight(endpoint.URL, "/") + "/v2/"
req, err := http.NewRequest("GET", endpointStr, nil)
@ -93,12 +92,12 @@ func newV2Repository(repoInfo *registry.RepositoryInfo, endpoint registry.APIEnd
}
creds := dumbCredentialStore{auth: authConfig}
tokenHandler := auth.NewTokenHandler(authTransport, creds, repoName, actions...)
tokenHandler := auth.NewTokenHandler(authTransport, creds, repoName.Name(), actions...)
basicHandler := auth.NewBasicHandler(creds)
modifiers = append(modifiers, auth.NewAuthorizer(challengeManager, tokenHandler, basicHandler))
tr := transport.NewTransport(base, modifiers...)
return client.NewRepository(ctx, repoName, endpoint.URL, tr)
return client.NewRepository(ctx, repoName.Name(), endpoint.URL, tr)
}
func digestFromManifest(m *schema1.SignedManifest, localName string) (digest.Digest, int, error) {

9
errors/daemon.go
View file

@ -832,15 +832,6 @@ var (
HTTPStatusCode: http.StatusInternalServerError,
})
// ErrorCodeRmInit is generated when we try to delete a container
// but failed deleting its init filesystem.
ErrorCodeRmInit = errcode.Register(errGroup, errcode.ErrorDescriptor{
Value: "RMINIT",
Message: "Driver %s failed to remove init filesystem %s: %s",
Description: "While trying to delete a container, the driver failed to remove the init filesystem",
HTTPStatusCode: http.StatusInternalServerError,
})
// ErrorCodeRmFS is generated when we try to delete a container
// but failed deleting its filesystem.
ErrorCodeRmFS = errcode.Register(errGroup, errcode.ErrorDescriptor{

180
graph/export.go
View file

@ -1,180 +0,0 @@
package graph
import (
"encoding/json"
"fmt"
"io"
"io/ioutil"
"os"
"path/filepath"
"time"
"github.com/Sirupsen/logrus"
"github.com/docker/distribution/digest"
"github.com/docker/docker/pkg/archive"
"github.com/docker/docker/pkg/parsers"
"github.com/docker/docker/registry"
)
// ImageExport exports list of images to a output stream specified in the
// config. The exported images are archived into a tar when written to the
// output stream. All images with the given tag and all versions containing the
// same tag are exported. names is the set of tags to export, and outStream
// is the writer which the images are written to.
func (s *TagStore) ImageExport(names []string, outStream io.Writer) error {
// get image json
tempdir, err := ioutil.TempDir("", "docker-export-")
if err != nil {
return err
}
defer os.RemoveAll(tempdir)
rootRepoMap := map[string]repository{}
addKey := func(name string, tag string, id string) {
logrus.Debugf("add key [%s:%s]", name, tag)
if repo, ok := rootRepoMap[name]; !ok {
rootRepoMap[name] = repository{tag: id}
} else {
repo[tag] = id
}
}
for _, name := range names {
name = registry.NormalizeLocalName(name)
logrus.Debugf("Serializing %s", name)
rootRepo := s.Repositories[name]
if rootRepo != nil {
// this is a base repo name, like 'busybox'
for tag, id := range rootRepo {
addKey(name, tag, id)
if err := s.exportImage(id, tempdir); err != nil {
return err
}
}
} else {
img, err := s.LookupImage(name)
if err != nil {
return err
}
if img != nil {
// This is a named image like 'busybox:latest'
repoName, repoTag := parsers.ParseRepositoryTag(name)
// Skip digests on save
if _, err := digest.ParseDigest(repoTag); err == nil {
repoTag = ""
}
// check this length, because a lookup of a truncated has will not have a tag
// and will not need to be added to this map
if len(repoTag) > 0 {
addKey(repoName, repoTag, img.ID)
}
if err := s.exportImage(img.ID, tempdir); err != nil {
return err
}
} else {
// this must be an ID that didn't get looked up just right?
if err := s.exportImage(name, tempdir); err != nil {
return err
}
}
}
logrus.Debugf("End Serializing %s", name)
}
// write repositories, if there is something to write
if len(rootRepoMap) > 0 {
f, err := os.OpenFile(filepath.Join(tempdir, "repositories"), os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644)
if err != nil {
f.Close()
return err
}
if err := json.NewEncoder(f).Encode(rootRepoMap); err != nil {
return err
}
if err := f.Close(); err != nil {
return err
}
if err := os.Chtimes(filepath.Join(tempdir, "repositories"), time.Unix(0, 0), time.Unix(0, 0)); err != nil {
return err
}
} else {
logrus.Debugf("There were no repositories to write")
}
fs, err := archive.Tar(tempdir, archive.Uncompressed)
if err != nil {
return err
}
defer fs.Close()
if _, err := io.Copy(outStream, fs); err != nil {
return err
}
logrus.Debugf("End export image")
return nil
}
func (s *TagStore) exportImage(name, tempdir string) error {
for n := name; n != ""; {
img, err := s.LookupImage(n)
if err != nil || img == nil {
return fmt.Errorf("No such image %s", n)
}
// temporary directory
tmpImageDir := filepath.Join(tempdir, n)
if err := os.Mkdir(tmpImageDir, os.FileMode(0755)); err != nil {
if os.IsExist(err) {
return nil
}
return err
}
var version = "1.0"
var versionBuf = []byte(version)
if err := ioutil.WriteFile(filepath.Join(tmpImageDir, "VERSION"), versionBuf, os.FileMode(0644)); err != nil {
return err
}
imageInspectRaw, err := json.Marshal(img)
if err != nil {
return err
}
// serialize json
json, err := os.Create(filepath.Join(tmpImageDir, "json"))
if err != nil {
return err
}
written, err := json.Write(imageInspectRaw)
if err != nil {
return err
}
if written != len(imageInspectRaw) {
logrus.Warnf("%d byes should have been written instead %d have been written", written, len(imageInspectRaw))
}
// serialize filesystem
fsTar, err := os.Create(filepath.Join(tmpImageDir, "layer.tar"))
if err != nil {
return err
}
if err := s.imageTarLayer(n, fsTar); err != nil {
return err
}
for _, fname := range []string{"", "VERSION", "json", "layer.tar"} {
if err := os.Chtimes(filepath.Join(tmpImageDir, fname), img.Created, img.Created); err != nil {
return err
}
}
// try again with parent
n = img.Parent
}
return nil
}

22
graph/fixtures/validate_manifest/no_signature_manifest
View file

@ -1,22 +0,0 @@
{
"schemaVersion": 1,
"name": "library/hello-world",
"tag": "latest",
"architecture": "amd64",
"fsLayers": [
{
"blobSum": "sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4"
},
{
"blobSum": "sha256:03f4658f8b782e12230c1783426bd3bacce651ce582a4ffb6fbbfa2079428ecb"
}
],
"history": [
{
"v1Compatibility": "{\"id\":\"af340544ed62de0680f441c71fa1a80cb084678fed42bae393e543faea3a572c\",\"parent\":\"535020c3e8add9d6bb06e5ac15a261e73d9b213d62fb2c14d752b8e189b2b912\",\"created\":\"2015-08-06T23:53:22.608577814Z\",\"container\":\"c2b715156f640c7ac7d98472ea24335aba5432a1323a3bb722697e6d37ef794f\",\"container_config\":{\"Hostname\":\"9aeb0006ffa7\",\"Domainname\":\"\",\"User\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":null,\"Cmd\":[\"/bin/sh\",\"-c\",\"#(nop) CMD [\\\"/hello\\\"]\"],\"Image\":\"535020c3e8add9d6bb06e5ac15a261e73d9b213d62fb2c14d752b8e189b2b912\",\"Volumes\":null,\"VolumeDriver\":\"\",\"WorkingDir\":\"\",\"Entrypoint\":null,\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":null,\"Labels\":{}},\"docker_version\":\"1.7.1\",\"config\":{\"Hostname\":\"9aeb0006ffa7\",\"Domainname\":\"\",\"User\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":null,\"Cmd\":[\"/hello\"],\"Image\":\"535020c3e8add9d6bb06e5ac15a261e73d9b213d62fb2c14d752b8e189b2b912\",\"Volumes\":null,\"VolumeDriver\":\"\",\"WorkingDir\":\"\",\"Entrypoint\":null,\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":null,\"Labels\":{}},\"architecture\":\"amd64\",\"os\":\"linux\",\"Size\":0}\n"
},
{
"v1Compatibility": "{\"id\":\"535020c3e8add9d6bb06e5ac15a261e73d9b213d62fb2c14d752b8e189b2b912\",\"created\":\"2015-08-06T23:53:22.241352727Z\",\"container\":\"9aeb0006ffa72a8287564caaea87625896853701459261d3b569e320c0c9d5dc\",\"container_config\":{\"Hostname\":\"9aeb0006ffa7\",\"Domainname\":\"\",\"User\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":null,\"Cmd\":[\"/bin/sh\",\"-c\",\"#(nop) COPY file:4abd3bff60458ca3b079d7b131ce26b2719055a030dfa96ff827da2b7c7038a7 in /\"],\"Image\":\"\",\"Volumes\":null,\"VolumeDriver\":\"\",\"WorkingDir\":\"\",\"Entrypoint\":null,\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":null,\"Labels\":null},\"docker_version\":\"1.7.1\",\"config\":{\"Hostname\":\"9aeb0006ffa7\",\"Domainname\":\"\",\"User\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":null,\"Cmd\":null,\"Image\":\"\",\"Volumes\":null,\"VolumeDriver\":\"\",\"WorkingDir\":\"\",\"Entrypoint\":null,\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":null,\"Labels\":null},\"architecture\":\"amd64\",\"os\":\"linux\",\"Size\":960}\n"
}
]
}

813
graph/graph.go
View file

@ -1,813 +0,0 @@
package graph
import (
"compress/gzip"
"encoding/json"
"errors"
"fmt"
"io"
"io/ioutil"
"os"
"path/filepath"
"runtime"
"strconv"
"strings"
"sync"
"time"
"github.com/Sirupsen/logrus"
"github.com/docker/distribution/digest"
"github.com/docker/docker/daemon/graphdriver"
"github.com/docker/docker/dockerversion"
"github.com/docker/docker/image"
"github.com/docker/docker/pkg/archive"
"github.com/docker/docker/pkg/idtools"
"github.com/docker/docker/pkg/locker"
"github.com/docker/docker/pkg/progressreader"
"github.com/docker/docker/pkg/streamformatter"
"github.com/docker/docker/pkg/stringid"
"github.com/docker/docker/pkg/truncindex"
"github.com/docker/docker/runconfig"
"github.com/vbatts/tar-split/tar/asm"
"github.com/vbatts/tar-split/tar/storage"
)
// v1Descriptor is a non-content-addressable image descriptor
type v1Descriptor struct {
img *image.Image
}
// ID returns the image ID specified in the image structure.
func (img v1Descriptor) ID() string {
return img.img.ID
}
// Parent returns the parent ID specified in the image structure.
func (img v1Descriptor) Parent() string {
return img.img.Parent
}
// MarshalConfig renders the image structure into JSON.
func (img v1Descriptor) MarshalConfig() ([]byte, error) {
return json.Marshal(img.img)
}
// The type is used to protect pulling or building related image
// layers from deleteing when filtered by dangling=true
// The key of layers is the images ID which is pulling or building
// The value of layers is a slice which hold layer IDs referenced to
// pulling or building images
type retainedLayers struct {
layerHolders map[string]map[string]struct{} // map[layerID]map[sessionID]
sync.Mutex
}
func (r *retainedLayers) Add(sessionID string, layerIDs []string) {
r.Lock()
defer r.Unlock()
for _, layerID := range layerIDs {
if r.layerHolders[layerID] == nil {
r.layerHolders[layerID] = map[string]struct{}{}
}
r.layerHolders[layerID][sessionID] = struct{}{}
}
}
func (r *retainedLayers) Delete(sessionID string, layerIDs []string) {
r.Lock()
defer r.Unlock()
for _, layerID := range layerIDs {
holders, ok := r.layerHolders[layerID]
if !ok {
continue
}
delete(holders, sessionID)
if len(holders) == 0 {
delete(r.layerHolders, layerID) // Delete any empty reference set.
}
}
}
func (r *retainedLayers) Exists(layerID string) bool {
r.Lock()
_, exists := r.layerHolders[layerID]
r.Unlock()
return exists
}
// A Graph is a store for versioned filesystem images and the relationship between them.
type Graph struct {
root string
idIndex *truncindex.TruncIndex
driver graphdriver.Driver
imagesMutex sync.Mutex
imageMutex locker.Locker // protect images in driver.
retained *retainedLayers
tarSplitDisabled bool
uidMaps []idtools.IDMap
gidMaps []idtools.IDMap
// access to parentRefs must be protected with imageMutex locking the image id
// on the key of the map (e.g. imageMutex.Lock(img.ID), parentRefs[img.ID]...)
parentRefs map[string]int
}
// file names for ./graph/<ID>/
const (
jsonFileName = "json"
layersizeFileName = "layersize"
digestFileName = "checksum"
tarDataFileName = "tar-data.json.gz"
v1CompatibilityFileName = "v1Compatibility"
parentFileName = "parent"
)
var (
// errDigestNotSet is used when request the digest for a layer
// but the layer has no digest value or content to compute the
// the digest.
errDigestNotSet = errors.New("digest is not set for layer")
)
// NewGraph instantiates a new graph at the given root path in the filesystem.
// `root` will be created if it doesn't exist.
func NewGraph(root string, driver graphdriver.Driver, uidMaps, gidMaps []idtools.IDMap) (*Graph, error) {
abspath, err := filepath.Abs(root)
if err != nil {
return nil, err
}
rootUID, rootGID, err := idtools.GetRootUIDGID(uidMaps, gidMaps)
if err != nil {
return nil, err
}
// Create the root directory if it doesn't exists
if err := idtools.MkdirAllAs(root, 0700, rootUID, rootGID); err != nil && !os.IsExist(err) {
return nil, err
}
graph := &Graph{
root: abspath,
idIndex: truncindex.NewTruncIndex([]string{}),
driver: driver,
retained: &retainedLayers{layerHolders: make(map[string]map[string]struct{})},
uidMaps: uidMaps,
gidMaps: gidMaps,
parentRefs: make(map[string]int),
}
// Windows does not currently support tarsplit functionality.
if runtime.GOOS == "windows" {
graph.tarSplitDisabled = true
}
if err := graph.restore(); err != nil {
return nil, err
}
return graph, nil
}
// IsHeld returns whether the given layerID is being used by an ongoing pull or build.
func (graph *Graph) IsHeld(layerID string) bool {
return graph.retained.Exists(layerID)
}
func (graph *Graph) restore() error {
dir, err := ioutil.ReadDir(graph.root)
if err != nil {
return err
}
var ids = []string{}
for _, v := range dir {
id := v.Name()
if graph.driver.Exists(id) {
img, err := graph.loadImage(id)
if err != nil {
logrus.Warnf("ignoring image %s, it could not be restored: %v", id, err)
continue
}
graph.imageMutex.Lock(img.Parent)
graph.parentRefs[img.Parent]++
graph.imageMutex.Unlock(img.Parent)
ids = append(ids, id)
}
}
graph.idIndex = truncindex.NewTruncIndex(ids)
logrus.Debugf("Restored %d elements", len(ids))
return nil
}
// IsNotExist detects whether an image exists by parsing the incoming error
// message.
func (graph *Graph) IsNotExist(err error, id string) bool {
// FIXME: Implement error subclass instead of looking at the error text
// Note: This is the way golang implements os.IsNotExists on Plan9
return err != nil && (strings.Contains(strings.ToLower(err.Error()), "does not exist") || strings.Contains(strings.ToLower(err.Error()), "no such")) && strings.Contains(err.Error(), id)
}
// Exists returns true if an image is registered at the given id.
// If the image doesn't exist or if an error is encountered, false is returned.
func (graph *Graph) Exists(id string) bool {
if _, err := graph.Get(id); err != nil {
return false
}
return true
}
// Get returns the image with the given id, or an error if the image doesn't exist.
func (graph *Graph) Get(name string) (*image.Image, error) {
id, err := graph.idIndex.Get(name)
if err != nil {
if err == truncindex.ErrNotExist {
return nil, fmt.Errorf("image %s does not exist", name)
}
return nil, err
}
img, err := graph.loadImage(id)
if err != nil {
return nil, err
}
if img.ID != id {
return nil, fmt.Errorf("Image stored at '%s' has wrong id '%s'", id, img.ID)
}
if img.Size < 0 {
size, err := graph.driver.DiffSize(img.ID, img.Parent)
if err != nil {
return nil, fmt.Errorf("unable to calculate size of image id %q: %s", img.ID, err)
}
img.Size = size
if err := graph.saveSize(graph.imageRoot(id), img.Size); err != nil {
return nil, err
}
}
return img, nil
}
// Create creates a new image and registers it in the graph.
func (graph *Graph) Create(layerData io.Reader, containerID, containerImage, comment, author string, containerConfig, config *runconfig.Config) (*image.Image, error) {
img := &image.Image{
ID: stringid.GenerateRandomID(),
Comment: comment,
Created: time.Now().UTC(),
DockerVersion: dockerversion.Version,
Author: author,
Config: config,
Architecture: runtime.GOARCH,
OS: runtime.GOOS,
}
if containerID != "" {
img.Parent = containerImage
img.Container = containerID
img.ContainerConfig = *containerConfig
}
if err := graph.Register(v1Descriptor{img}, layerData); err != nil {
return nil, err
}
return img, nil
}
// Register imports a pre-existing image into the graph.
// Returns nil if the image is already registered.
func (graph *Graph) Register(im image.Descriptor, layerData io.Reader) (err error) {
imgID := im.ID()
if err := image.ValidateID(imgID); err != nil {
return err
}
// this is needed cause pull_v2 attemptIDReuse could deadlock
graph.imagesMutex.Lock()
defer graph.imagesMutex.Unlock()
// We need this entire operation to be atomic within the engine. Note that
// this doesn't mean Register is fully safe yet.
graph.imageMutex.Lock(imgID)
defer graph.imageMutex.Unlock(imgID)
return graph.register(im, layerData)
}
func (graph *Graph) register(im image.Descriptor, layerData io.Reader) (err error) {
imgID := im.ID()
// Skip register if image is already registered
if graph.Exists(imgID) {
return nil
}
// The returned `error` must be named in this function's signature so that
// `err` is not shadowed in this deferred cleanup.
defer func() {
// If any error occurs, remove the new dir from the driver.
// Don't check for errors since the dir might not have been created.
if err != nil {
graph.driver.Remove(imgID)
}
}()
// Ensure that the image root does not exist on the filesystem
// when it is not registered in the graph.
// This is common when you switch from one graph driver to another
if err := os.RemoveAll(graph.imageRoot(imgID)); err != nil && !os.IsNotExist(err) {
return err
}
// If the driver has this ID but the graph doesn't, remove it from the driver to start fresh.
// (the graph is the source of truth).
// Ignore errors, since we don't know if the driver correctly returns ErrNotExist.
// (FIXME: make that mandatory for drivers).
graph.driver.Remove(imgID)
tmp, err := graph.mktemp()
if err != nil {
return err
}
defer os.RemoveAll(tmp)
parent := im.Parent()
// Create root filesystem in the driver
if err := createRootFilesystemInDriver(graph, imgID, parent); err != nil {
return err
}
// Apply the diff/layer
config, err := im.MarshalConfig()
if err != nil {
return err
}
if err := graph.storeImage(imgID, parent, config, layerData, tmp); err != nil {
return err
}
// Commit
if err := os.Rename(tmp, graph.imageRoot(imgID)); err != nil {
return err
}
graph.idIndex.Add(imgID)
graph.imageMutex.Lock(parent)
graph.parentRefs[parent]++
graph.imageMutex.Unlock(parent)
return nil
}
func createRootFilesystemInDriver(graph *Graph, id, parent string) error {
if err := graph.driver.Create(id, parent, ""); err != nil {
return fmt.Errorf("Driver %s failed to create image rootfs %s: %s", graph.driver, id, err)
}
return nil
}
// TempLayerArchive creates a temporary archive of the given image's filesystem layer.
// The archive is stored on disk and will be automatically deleted as soon as has been read.
// If output is not nil, a human-readable progress bar will be written to it.
func (graph *Graph) tempLayerArchive(id string, sf *streamformatter.StreamFormatter, output io.Writer) (*archive.TempArchive, error) {
image, err := graph.Get(id)
if err != nil {
return nil, err
}
tmp, err := graph.mktemp()
if err != nil {
return nil, err
}
defer os.RemoveAll(tmp)
a, err := graph.tarLayer(image)
if err != nil {
return nil, err
}
progressReader := progressreader.New(progressreader.Config{
In: a,
Out: output,
Formatter: sf,
Size: 0,
NewLines: false,
ID: stringid.TruncateID(id),
Action: "Buffering to disk",
})
defer progressReader.Close()
return archive.NewTempArchive(progressReader, tmp)
}
// mktemp creates a temporary sub-directory inside the graph's filesystem.
func (graph *Graph) mktemp() (string, error) {
dir := filepath.Join(graph.root, "_tmp", stringid.GenerateNonCryptoID())
rootUID, rootGID, err := idtools.GetRootUIDGID(graph.uidMaps, graph.gidMaps)
if err != nil {
return "", err
}
if err := idtools.MkdirAllAs(dir, 0700, rootUID, rootGID); err != nil {
return "", err
}
return dir, nil
}
// Delete atomically removes an image from the graph.
func (graph *Graph) Delete(name string) error {
id, err := graph.idIndex.Get(name)
if err != nil {
return err
}
img, err := graph.Get(id)
if err != nil {
return err
}
graph.idIndex.Delete(id)
tmp, err := graph.mktemp()
if err != nil {
tmp = graph.imageRoot(id)
} else {
if err := os.Rename(graph.imageRoot(id), tmp); err != nil {
// On err make tmp point to old dir and cleanup unused tmp dir
os.RemoveAll(tmp)
tmp = graph.imageRoot(id)
}
}
// Remove rootfs data from the driver
graph.driver.Remove(id)
graph.imageMutex.Lock(img.Parent)
graph.parentRefs[img.Parent]--
if graph.parentRefs[img.Parent] == 0 {
delete(graph.parentRefs, img.Parent)
}
graph.imageMutex.Unlock(img.Parent)
// Remove the trashed image directory
return os.RemoveAll(tmp)
}
// Map returns a list of all images in the graph, addressable by ID.
func (graph *Graph) Map() map[string]*image.Image {
images := make(map[string]*image.Image)
graph.walkAll(func(image *image.Image) {
images[image.ID] = image
})
return images
}
// walkAll iterates over each image in the graph, and passes it to a handler.
// The walking order is undetermined.
func (graph *Graph) walkAll(handler func(*image.Image)) {
graph.idIndex.Iterate(func(id string) {
img, err := graph.Get(id)
if err != nil {
return
}
if handler != nil {
handler(img)
}
})
}
// ByParent returns a lookup table of images by their parent.
// If an image of key ID has 3 children images, then the value for key ID
// will be a list of 3 images.
// If an image has no children, it will not have an entry in the table.
func (graph *Graph) ByParent() map[string][]*image.Image {
byParent := make(map[string][]*image.Image)
graph.walkAll(func(img *image.Image) {
parent, err := graph.Get(img.Parent)
if err != nil {
return
}
byParent[parent.ID] = append(byParent[parent.ID], img)
})
return byParent
}
// HasChildren returns whether the given image has any child images.
func (graph *Graph) HasChildren(imgID string) bool {
graph.imageMutex.Lock(imgID)
count := graph.parentRefs[imgID]
graph.imageMutex.Unlock(imgID)
return count > 0
}
// Retain keeps the images and layers that are in the pulling chain so that
// they are not deleted. If not retained, they may be deleted by rmi.
func (graph *Graph) Retain(sessionID string, layerIDs ...string) {
graph.retained.Add(sessionID, layerIDs)
}
// Release removes the referenced image ID from the provided set of layers.
func (graph *Graph) Release(sessionID string, layerIDs ...string) {
graph.retained.Delete(sessionID, layerIDs)
}
// heads returns all heads in the graph, keyed by id.
// A head is an image which is not the parent of another image in the graph.
func (graph *Graph) heads() map[string]*image.Image {
heads := make(map[string]*image.Image)
graph.walkAll(func(image *image.Image) {
// if it has no children, then it's not a parent, so it's an head
if !graph.HasChildren(image.ID) {
heads[image.ID] = image
}
})
return heads
}
// tarLayer returns a tar archive of the image's filesystem layer.
func (graph *Graph) tarLayer(img *image.Image) (arch io.ReadCloser, err error) {
rdr, err := graph.assembleTarLayer(img)
if err != nil {
logrus.Debugf("[graph] tarLayer with traditional differ: %s", img.ID)
return graph.driver.Diff(img.ID, img.Parent)
}
return rdr, nil
}
func (graph *Graph) imageRoot(id string) string {
return filepath.Join(graph.root, id)
}
// loadImage fetches the image with the given id from the graph.
func (graph *Graph) loadImage(id string) (*image.Image, error) {
root := graph.imageRoot(id)
// Open the JSON file to decode by streaming
jsonSource, err := os.Open(jsonPath(root))
if err != nil {
return nil, err
}
defer jsonSource.Close()
img := &image.Image{}
dec := json.NewDecoder(jsonSource)
// Decode the JSON data
if err := dec.Decode(img); err != nil {
return nil, err
}
if img.ID == "" {
img.ID = id
}
if img.Parent == "" && img.ParentID != "" && img.ParentID.Validate() == nil {
img.Parent = img.ParentID.Hex()
}
// compatibilityID for parent
parent, err := ioutil.ReadFile(filepath.Join(root, parentFileName))
if err == nil && len(parent) > 0 {
img.Parent = string(parent)
}
if err := image.ValidateID(img.ID); err != nil {
return nil, err
}
if buf, err := ioutil.ReadFile(filepath.Join(root, layersizeFileName)); err != nil {
if !os.IsNotExist(err) {
return nil, err
}
// If the layersize file does not exist then set the size to a negative number
// because a layer size of 0 (zero) is valid
img.Size = -1
} else {
// Using Atoi here instead would temporarily convert the size to a machine
// dependent integer type, which causes images larger than 2^31 bytes to
// display negative sizes on 32-bit machines:
size, err := strconv.ParseInt(string(buf), 10, 64)
if err != nil {
return nil, err
}
img.Size = int64(size)
}
return img, nil
}
// saveSize stores the `size` in the provided graph `img` directory `root`.
func (graph *Graph) saveSize(root string, size int64) error {
if err := ioutil.WriteFile(filepath.Join(root, layersizeFileName), []byte(strconv.FormatInt(size, 10)), 0600); err != nil {
return fmt.Errorf("Error storing image size in %s/%s: %s", root, layersizeFileName, err)
}
return nil
}
// setLayerDigestWithLock sets the digest for the image layer to the provided value.
func (graph *Graph) setLayerDigestWithLock(id string, dgst digest.Digest) error {
graph.imageMutex.Lock(id)
defer graph.imageMutex.Unlock(id)
return graph.setLayerDigest(id, dgst)
}
func (graph *Graph) setLayerDigest(id string, dgst digest.Digest) error {
root := graph.imageRoot(id)
if err := ioutil.WriteFile(filepath.Join(root, digestFileName), []byte(dgst.String()), 0600); err != nil {
return fmt.Errorf("Error storing digest in %s/%s: %s", root, digestFileName, err)
}
return nil
}
// getLayerDigestWithLock gets the digest for the provide image layer id.
func (graph *Graph) getLayerDigestWithLock(id string) (digest.Digest, error) {
graph.imageMutex.Lock(id)
defer graph.imageMutex.Unlock(id)
return graph.getLayerDigest(id)
}
func (graph *Graph) getLayerDigest(id string) (digest.Digest, error) {
root := graph.imageRoot(id)
cs, err := ioutil.ReadFile(filepath.Join(root, digestFileName))
if err != nil {
if os.IsNotExist(err) {
return "", errDigestNotSet
}
return "", err
}
return digest.ParseDigest(string(cs))
}
// setV1CompatibilityConfig stores the v1Compatibility JSON data associated
// with the image in the manifest to the disk
func (graph *Graph) setV1CompatibilityConfig(id string, data []byte) error {
root := graph.imageRoot(id)
return ioutil.WriteFile(filepath.Join(root, v1CompatibilityFileName), data, 0600)
}
// getV1CompatibilityConfig reads the v1Compatibility JSON data for the image
// from the disk
func (graph *Graph) getV1CompatibilityConfig(id string) ([]byte, error) {
root := graph.imageRoot(id)
return ioutil.ReadFile(filepath.Join(root, v1CompatibilityFileName))
}
// generateV1CompatibilityChain makes sure v1Compatibility JSON data exists
// for the image. If it doesn't it generates and stores it for the image and
// all of it's parents based on the image config JSON.
func (graph *Graph) generateV1CompatibilityChain(id string) ([]byte, error) {
graph.imageMutex.Lock(id)
defer graph.imageMutex.Unlock(id)
if v1config, err := graph.getV1CompatibilityConfig(id); err == nil {
return v1config, nil
}
// generate new, store it to disk
img, err := graph.Get(id)
if err != nil {
return nil, err
}
digestPrefix := string(digest.Canonical) + ":"
img.ID = strings.TrimPrefix(img.ID, digestPrefix)
if img.Parent != "" {
parentConfig, err := graph.generateV1CompatibilityChain(img.Parent)
if err != nil {
return nil, err
}
var parent struct{ ID string }
err = json.Unmarshal(parentConfig, &parent)
if err != nil {
return nil, err
}
img.Parent = parent.ID
}
json, err := json.Marshal(img)
if err != nil {
return nil, err
}
if err := graph.setV1CompatibilityConfig(id, json); err != nil {
return nil, err
}
return json, nil
}
func jsonPath(root string) string {
return filepath.Join(root, jsonFileName)
}
// storeImage stores file system layer data for the given image to the
// graph's storage driver. Image metadata is stored in a file
// at the specified root directory.
func (graph *Graph) storeImage(id, parent string, config []byte, layerData io.Reader, root string) (err error) {
var size int64
// Store the layer. If layerData is not nil, unpack it into the new layer
if layerData != nil {
if size, err = graph.disassembleAndApplyTarLayer(id, parent, layerData, root); err != nil {
return err
}
}
if err := graph.saveSize(root, size); err != nil {
return err
}
if err := ioutil.WriteFile(jsonPath(root), config, 0600); err != nil {
return err
}
// If image is pointing to a parent via CompatibilityID write the reference to disk
img, err := image.NewImgJSON(config)
if err != nil {
return err
}
if (img.ParentID.Validate() == nil && parent != img.ParentID.Hex()) || (allowBaseParentImage && img.ParentID == "" && parent != "") {
// save compatibilityID parent if it doesn't match parentID
// on windows always save a parent file pointing to the base layer
if err := ioutil.WriteFile(filepath.Join(root, parentFileName), []byte(parent), 0600); err != nil {
return err
}
}
return nil
}
func (graph *Graph) disassembleAndApplyTarLayer(id, parent string, layerData io.Reader, root string) (size int64, err error) {
var ar io.Reader
if graph.tarSplitDisabled {
ar = layerData
} else {
// this is saving the tar-split metadata
mf, err := os.OpenFile(filepath.Join(root, tarDataFileName), os.O_CREATE|os.O_WRONLY|os.O_TRUNC, os.FileMode(0600))
if err != nil {
return 0, err
}
mfz := gzip.NewWriter(mf)
metaPacker := storage.NewJSONPacker(mfz)
defer mf.Close()
defer mfz.Close()
inflatedLayerData, err := archive.DecompressStream(layerData)
if err != nil {
return 0, err
}
// we're passing nil here for the file putter, because the ApplyDiff will
// handle the extraction of the archive
rdr, err := asm.NewInputTarStream(inflatedLayerData, metaPacker, nil)
if err != nil {
return 0, err
}
ar = archive.Reader(rdr)
}
if size, err = graph.driver.ApplyDiff(id, parent, ar); err != nil {
return 0, err
}
return
}
func (graph *Graph) assembleTarLayer(img *image.Image) (io.ReadCloser, error) {
root := graph.imageRoot(img.ID)
mFileName := filepath.Join(root, tarDataFileName)
mf, err := os.Open(mFileName)
if err != nil {
if !os.IsNotExist(err) {
logrus.Errorf("failed to open %q: %s", mFileName, err)
}
return nil, err
}
pR, pW := io.Pipe()
// this will need to be in a goroutine, as we are returning the stream of a
// tar archive, but can not close the metadata reader early (when this
// function returns)...
go func() {
defer mf.Close()
// let's reassemble!
logrus.Debugf("[graph] TarLayer with reassembly: %s", img.ID)
mfz, err := gzip.NewReader(mf)
if err != nil {
pW.CloseWithError(fmt.Errorf("[graph] error with %s: %s", mFileName, err))
return
}
defer mfz.Close()
// get our relative path to the container
fsLayer, err := graph.driver.Get(img.ID, "")
if err != nil {
pW.CloseWithError(err)
return
}
defer graph.driver.Put(img.ID)
metaUnpacker := storage.NewJSONUnpacker(mfz)
fileGetter := storage.NewPathFileGetter(fsLayer)
logrus.Debugf("[graph] %s is at %q", img.ID, fsLayer)
ots := asm.NewOutputTarStream(fileGetter, metaUnpacker)
defer ots.Close()
if _, err := io.Copy(pW, ots); err != nil {
pW.CloseWithError(err)
return
}
pW.Close()
}()
return pR, nil
}

308
graph/graph_test.go
View file

@ -1,308 +0,0 @@
package graph
import (
"errors"
"io"
"io/ioutil"
"os"
"path"
"testing"
"time"
"github.com/docker/docker/daemon/graphdriver"
"github.com/docker/docker/dockerversion"
"github.com/docker/docker/image"
"github.com/docker/docker/pkg/stringid"
)
func TestMount(t *testing.T) {
graph, driver := tempGraph(t)
defer os.RemoveAll(graph.root)
defer driver.Cleanup()
archive, err := fakeTar()
if err != nil {
t.Fatal(err)
}
image, err := graph.Create(archive, "", "", "Testing", "", nil, nil)
if err != nil {
t.Fatal(err)
}
tmp, err := ioutil.TempDir("", "docker-test-graph-mount-")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(tmp)
rootfs := path.Join(tmp, "rootfs")
if err := os.MkdirAll(rootfs, 0700); err != nil {
t.Fatal(err)
}
rw := path.Join(tmp, "rw")
if err := os.MkdirAll(rw, 0700); err != nil {
t.Fatal(err)
}
if _, err := driver.Get(image.ID, ""); err != nil {
t.Fatal(err)
}
}
func TestInit(t *testing.T) {
graph, _ := tempGraph(t)
defer nukeGraph(graph)
// Root should exist
if _, err := os.Stat(graph.root); err != nil {
t.Fatal(err)
}
// Map() should be empty
l := graph.Map()
if len(l) != 0 {
t.Fatalf("len(Map()) should return %d, not %d", 0, len(l))
}
}
// Test that Register can be interrupted cleanly without side effects
func TestInterruptedRegister(t *testing.T) {
graph, _ := tempGraph(t)
defer nukeGraph(graph)
badArchive, w := io.Pipe() // Use a pipe reader as a fake archive which never yields data
image := &image.Image{
ID: stringid.GenerateNonCryptoID(),
Comment: "testing",
Created: time.Now(),
}
w.CloseWithError(errors.New("But I'm not a tarball!")) // (Nobody's perfect, darling)
graph.Register(v1Descriptor{image}, badArchive)
if _, err := graph.Get(image.ID); err == nil {
t.Fatal("Image should not exist after Register is interrupted")
}
// Registering the same image again should succeed if the first register was interrupted
goodArchive, err := fakeTar()
if err != nil {
t.Fatal(err)
}
if err := graph.Register(v1Descriptor{image}, goodArchive); err != nil {
t.Fatal(err)
}
}
// FIXME: Do more extensive tests (ex: create multiple, delete, recreate;
// create multiple, check the amount of images and paths, etc..)
func TestGraphCreate(t *testing.T) {
graph, _ := tempGraph(t)
defer nukeGraph(graph)
archive, err := fakeTar()
if err != nil {
t.Fatal(err)
}
img, err := graph.Create(archive, "", "", "Testing", "", nil, nil)
if err != nil {
t.Fatal(err)
}
if err := image.ValidateID(img.ID); err != nil {
t.Fatal(err)
}
if img.Comment != "Testing" {
t.Fatalf("Wrong comment: should be '%s', not '%s'", "Testing", img.Comment)
}
if img.DockerVersion != dockerversion.Version {
t.Fatalf("Wrong docker_version: should be '%s', not '%s'", dockerversion.Version, img.DockerVersion)
}
images := graph.Map()
if l := len(images); l != 1 {
t.Fatalf("Wrong number of images. Should be %d, not %d", 1, l)
}
if images[img.ID] == nil {
t.Fatalf("Could not find image with id %s", img.ID)
}
}
func TestRegister(t *testing.T) {
graph, _ := tempGraph(t)
defer nukeGraph(graph)
archive, err := fakeTar()
if err != nil {
t.Fatal(err)
}
image := &image.Image{
ID: stringid.GenerateNonCryptoID(),
Comment: "testing",
Created: time.Now(),
}
err = graph.Register(v1Descriptor{image}, archive)
if err != nil {
t.Fatal(err)
}
images := graph.Map()
if l := len(images); l != 1 {
t.Fatalf("Wrong number of images. Should be %d, not %d", 1, l)
}
if resultImg, err := graph.Get(image.ID); err != nil {
t.Fatal(err)
} else {
if resultImg.ID != image.ID {
t.Fatalf("Wrong image ID. Should be '%s', not '%s'", image.ID, resultImg.ID)
}
if resultImg.Comment != image.Comment {
t.Fatalf("Wrong image comment. Should be '%s', not '%s'", image.Comment, resultImg.Comment)
}
}
}
// Test that an image can be deleted by its shorthand prefix
func TestDeletePrefix(t *testing.T) {
graph, _ := tempGraph(t)
defer nukeGraph(graph)
img := createTestImage(graph, t)
if err := graph.Delete(stringid.TruncateID(img.ID)); err != nil {
t.Fatal(err)
}
assertNImages(graph, t, 0)
}
func TestDelete(t *testing.T) {
graph, _ := tempGraph(t)
defer nukeGraph(graph)
archive, err := fakeTar()
if err != nil {
t.Fatal(err)
}
assertNImages(graph, t, 0)
img, err := graph.Create(archive, "", "", "Bla bla", "", nil, nil)
if err != nil {
t.Fatal(err)
}
assertNImages(graph, t, 1)
if err := graph.Delete(img.ID); err != nil {
t.Fatal(err)
}
assertNImages(graph, t, 0)
archive, err = fakeTar()
if err != nil {
t.Fatal(err)
}
// Test 2 create (same name) / 1 delete
img1, err := graph.Create(archive, "", "", "Testing", "", nil, nil)
if err != nil {
t.Fatal(err)
}
archive, err = fakeTar()
if err != nil {
t.Fatal(err)
}
if _, err = graph.Create(archive, "", "", "Testing", "", nil, nil); err != nil {
t.Fatal(err)
}
assertNImages(graph, t, 2)
if err := graph.Delete(img1.ID); err != nil {
t.Fatal(err)
}
assertNImages(graph, t, 1)
// Test delete wrong name
if err := graph.Delete("Not_foo"); err == nil {
t.Fatalf("Deleting wrong ID should return an error")
}
assertNImages(graph, t, 1)
archive, err = fakeTar()
if err != nil {
t.Fatal(err)
}
// Test delete twice (pull -> rm -> pull -> rm)
if err := graph.Register(v1Descriptor{img1}, archive); err != nil {
t.Fatal(err)
}
if err := graph.Delete(img1.ID); err != nil {
t.Fatal(err)
}
assertNImages(graph, t, 1)
}
func TestByParent(t *testing.T) {
archive1, _ := fakeTar()
archive2, _ := fakeTar()
archive3, _ := fakeTar()
graph, _ := tempGraph(t)
defer nukeGraph(graph)
parentImage := &image.Image{
ID: stringid.GenerateNonCryptoID(),
Comment: "parent",
Created: time.Now(),
Parent: "",
}
childImage1 := &image.Image{
ID: stringid.GenerateNonCryptoID(),
Comment: "child1",
Created: time.Now(),
Parent: parentImage.ID,
}
childImage2 := &image.Image{
ID: stringid.GenerateNonCryptoID(),
Comment: "child2",
Created: time.Now(),
Parent: parentImage.ID,
}
err := graph.Register(v1Descriptor{parentImage}, archive1)
if err != nil {
t.Fatal(err)
}
err = graph.Register(v1Descriptor{childImage1}, archive2)
if err != nil {
t.Fatal(err)
}
err = graph.Register(v1Descriptor{childImage2}, archive3)
if err != nil {
t.Fatal(err)
}
byParent := graph.ByParent()
numChildren := len(byParent[parentImage.ID])
if numChildren != 2 {
t.Fatalf("Expected 2 children, found %d", numChildren)
}
}
func createTestImage(graph *Graph, t *testing.T) *image.Image {
archive, err := fakeTar()
if err != nil {
t.Fatal(err)
}
img, err := graph.Create(archive, "", "", "Test image", "", nil, nil)
if err != nil {
t.Fatal(err)
}
return img
}
func assertNImages(graph *Graph, t *testing.T, n int) {
images := graph.Map()
if actualN := len(images); actualN != n {
t.Fatalf("Expected %d images, found %d", n, actualN)
}
}
func tempGraph(t *testing.T) (*Graph, graphdriver.Driver) {
tmp, err := ioutil.TempDir("", "docker-graph-")
if err != nil {
t.Fatal(err)
}
driver, err := graphdriver.New(tmp, nil, nil, nil)
if err != nil {
t.Fatal(err)
}
graph, err := NewGraph(tmp, driver, nil, nil)
if err != nil {
t.Fatal(err)
}
return graph, driver
}
func nukeGraph(graph *Graph) {
graph.driver.Cleanup()
os.RemoveAll(graph.root)
}

8
graph/graph_unix.go
View file

@ -1,8 +0,0 @@
// +build !windows
package graph
// allowBaseParentImage allows images to define a custom parent that is not
// transported with push/pull but already included with the installation.
// Only used in Windows.
const allowBaseParentImage = false

8
graph/graph_windows.go
View file

@ -1,8 +0,0 @@
// +build windows
package graph
// allowBaseParentImage allows images to define a custom parent that is not
// transported with push/pull but already included with the installation.
// Only used in Windows.
const allowBaseParentImage = true

119
graph/history.go
View file

@ -1,119 +0,0 @@
package graph
import (
"fmt"
"strings"
"github.com/docker/docker/api/types"
"github.com/docker/docker/image"
"github.com/docker/docker/utils"
)
// walkHistory calls the handler function for each image in the
// provided images lineage starting from immediate parent.
func (graph *Graph) walkHistory(img *image.Image, handler func(image.Image) error) (err error) {
currentImg := img
for currentImg != nil {
if handler != nil {
if err := handler(*currentImg); err != nil {
return err
}
}
currentImg, err = graph.GetParent(currentImg)
if err != nil {
return fmt.Errorf("Error while getting parent image: %v", err)
}
}
return nil
}
// depth returns the number of parents for the current image
func (graph *Graph) depth(img *image.Image) (int, error) {
var (
count = 0
parent = img
err error
)
for parent != nil {
count++
if parent, err = graph.GetParent(parent); err != nil {
return -1, err
}
}
return count, nil
}
// Set the max depth to the aufs default that most kernels are compiled with.
// For more information see: http://sourceforge.net/p/aufs/aufs3-standalone/ci/aufs3.12/tree/config.mk
const maxImageDepth = 127
// CheckDepth returns an error if the depth of an image, as returned
// by ImageDepth, is too large to support creating a container from it
// on this daemon.
func (graph *Graph) CheckDepth(img *image.Image) error {
// We add 2 layers to the depth because the container's rw and
// init layer add to the restriction
depth, err := graph.depth(img)
if err != nil {
return err
}
if depth+2 >= maxImageDepth {
return fmt.Errorf("Cannot create container with more than %d parents", maxImageDepth)
}
return nil
}
// History returns a slice of ImageHistory structures for the specified image
// name by walking the image lineage.
func (s *TagStore) History(name string) ([]*types.ImageHistory, error) {
foundImage, err := s.LookupImage(name)
if err != nil {
return nil, err
}
lookupMap := make(map[string][]string)
for name, repository := range s.Repositories {
for tag, id := range repository {
// If the ID already has a reverse lookup, do not update it unless for "latest"
if _, exists := lookupMap[id]; !exists {
lookupMap[id] = []string{}
}
lookupMap[id] = append(lookupMap[id], utils.ImageReference(name, tag))
}
}
history := []*types.ImageHistory{}
err = s.graph.walkHistory(foundImage, func(img image.Image) error {
history = append(history, &types.ImageHistory{
ID: img.ID,
Created: img.Created.Unix(),
CreatedBy: strings.Join(img.ContainerConfig.Cmd.Slice(), " "),
Tags: lookupMap[img.ID],
Size: img.Size,
Comment: img.Comment,
})
return nil
})
return history, err
}
// GetParent returns the parent image for the specified image.
func (graph *Graph) GetParent(img *image.Image) (*image.Image, error) {
if img.Parent == "" {
return nil, nil
}
return graph.Get(img.Parent)
}
// getParentsSize returns the combined size of all parent images. If there is
// no parent image or it's unavailable, it returns 0.
func (graph *Graph) getParentsSize(img *image.Image) int64 {
parentImage, err := graph.GetParent(img)
if err != nil || parentImage == nil {
return 0
}
return parentImage.Size + graph.getParentsSize(parentImage)
}

73
graph/import.go
View file

@ -1,73 +0,0 @@
package graph
import (
"io"
"net/http"
"net/url"
"github.com/docker/docker/pkg/httputils"
"github.com/docker/docker/pkg/progressreader"
"github.com/docker/docker/pkg/streamformatter"
"github.com/docker/docker/runconfig"
)
// Import imports an image, getting the archived layer data either from
// inConfig (if src is "-"), or from a URI specified in src. Progress output is
// written to outStream. Repository and tag names can optionally be given in
// the repo and tag arguments, respectively.
func (s *TagStore) Import(src string, repo string, tag string, msg string, inConfig io.ReadCloser, outStream io.Writer, containerConfig *runconfig.Config) error {
var (
sf = streamformatter.NewJSONStreamFormatter()
archive io.ReadCloser
resp *http.Response
)
if src == "-" {
archive = inConfig
} else {
inConfig.Close()
u, err := url.Parse(src)
if err != nil {
return err
}
if u.Scheme == "" {
u.Scheme = "http"
u.Host = src
u.Path = ""
}
outStream.Write(sf.FormatStatus("", "Downloading from %s", u))
resp, err = httputils.Download(u.String())
if err != nil {
return err
}
progressReader := progressreader.New(progressreader.Config{
In: resp.Body,
Out: outStream,
Formatter: sf,
Size: resp.ContentLength,
NewLines: true,
ID: "",
Action: "Importing",
})
archive = progressReader
}
defer archive.Close()
if len(msg) == 0 {
msg = "Imported from " + src
}
img, err := s.graph.Create(archive, "", "", msg, "", nil, containerConfig)
if err != nil {
return err
}
// Optionally register the image at REPO/TAG
if repo != "" {
if err := s.Tag(repo, tag, img.ID, true); err != nil {
return err
}
}
outStream.Write(sf.FormatStatus("", img.ID))
s.eventsService.Log("import", img.ID, "")
return nil
}

185
graph/list.go
View file

@ -1,185 +0,0 @@
package graph
import (
"fmt"
"path"
"sort"
"strings"
"github.com/Sirupsen/logrus"
"github.com/docker/docker/api/types"
"github.com/docker/docker/image"
"github.com/docker/docker/pkg/parsers/filters"
"github.com/docker/docker/utils"
)
var acceptedImageFilterTags = map[string]struct{}{
"dangling": {},
"label": {},
}
// byCreated is a temporary type used to sort a list of images by creation
// time.
type byCreated []*types.Image
func (r byCreated) Len() int { return len(r) }
func (r byCreated) Swap(i, j int) { r[i], r[j] = r[j], r[i] }
func (r byCreated) Less(i, j int) bool { return r[i].Created < r[j].Created }
// Images returns a filtered list of images. filterArgs is a JSON-encoded set
// of filter arguments which will be interpreted by pkg/parsers/filters.
// filter is a shell glob string applied to repository names. The argument
// named all controls whether all images in the graph are filtered, or just
// the heads.
func (s *TagStore) Images(filterArgs, filter string, all bool) ([]*types.Image, error) {
var (
allImages map[string]*image.Image
err error
filtTagged = true
filtLabel = false
)
imageFilters, err := filters.FromParam(filterArgs)
if err != nil {
return nil, err
}
for name := range imageFilters {
if _, ok := acceptedImageFilterTags[name]; !ok {
return nil, fmt.Errorf("Invalid filter '%s'", name)
}
}
if i, ok := imageFilters["dangling"]; ok {
for _, value := range i {
if v := strings.ToLower(value); v == "true" {
filtTagged = false
} else if v != "false" {
return nil, fmt.Errorf("Invalid filter 'dangling=%s'", v)
}
}
}
_, filtLabel = imageFilters["label"]
if all && filtTagged {
allImages = s.graph.Map()
} else {
allImages = s.graph.heads()
}
lookup := make(map[string]*types.Image)
s.Lock()
for repoName, repository := range s.Repositories {
filterTagName := ""
if filter != "" {
filterName := filter
// Test if the tag was in there, if yes, get the name
if strings.Contains(filterName, ":") {
filterWithTag := strings.Split(filter, ":")
filterName = filterWithTag[0]
filterTagName = filterWithTag[1]
}
if match, _ := path.Match(filterName, repoName); !match {
continue
}
if filterTagName != "" {
if _, ok := repository[filterTagName]; !ok {
continue
}
}
}
for ref, id := range repository {
imgRef := utils.ImageReference(repoName, ref)
if !strings.Contains(imgRef, filterTagName) {
continue
}
image, err := s.graph.Get(id)
if err != nil {
logrus.Warnf("couldn't load %s from %s: %s", id, imgRef, err)
continue
}
if lImage, exists := lookup[id]; exists {
if filtTagged {
if utils.DigestReference(ref) {
lImage.RepoDigests = append(lImage.RepoDigests, imgRef)
} else { // Tag Ref.
lImage.RepoTags = append(lImage.RepoTags, imgRef)
}
}
} else {
// get the boolean list for if only the untagged images are requested
delete(allImages, id)
if len(imageFilters["label"]) > 0 {
if image.Config == nil {
// Very old image that do not have image.Config (or even labels)
continue
}
// We are now sure image.Config is not nil
if !imageFilters.MatchKVList("label", image.Config.Labels) {
continue
}
}
if filtTagged {
newImage := newImage(image, s.graph.getParentsSize(image))
if utils.DigestReference(ref) {
newImage.RepoTags = []string{}
newImage.RepoDigests = []string{imgRef}
} else {
newImage.RepoTags = []string{imgRef}
newImage.RepoDigests = []string{}
}
lookup[id] = newImage
}
}
}
}
s.Unlock()
images := []*types.Image{}
for _, value := range lookup {
images = append(images, value)
}
// Display images which aren't part of a repository/tag
if filter == "" || filtLabel {
for _, image := range allImages {
if len(imageFilters["label"]) > 0 {
if image.Config == nil {
// Very old image that do not have image.Config (or even labels)
continue
}
// We are now sure image.Config is not nil
if !imageFilters.MatchKVList("label", image.Config.Labels) {
continue
}
}
newImage := newImage(image, s.graph.getParentsSize(image))
newImage.RepoTags = []string{"<none>:<none>"}
newImage.RepoDigests = []string{"<none>@<none>"}
images = append(images, newImage)
}
}
sort.Sort(sort.Reverse(byCreated(images)))
return images, nil
}
func newImage(image *image.Image, parentSize int64) *types.Image {
newImage := new(types.Image)
newImage.ParentID = image.Parent
newImage.ID = image.ID
newImage.Created = image.Created.Unix()
newImage.Size = image.Size
newImage.VirtualSize = parentSize + image.Size
if image.Config != nil {
newImage.Labels = image.Config.Labels
}
return newImage
}

134
graph/load.go
View file

@ -1,134 +0,0 @@
// +build linux windows
package graph
import (
"encoding/json"
"io"
"io/ioutil"
"os"
"path/filepath"
"github.com/Sirupsen/logrus"
"github.com/docker/docker/image"
"github.com/docker/docker/pkg/archive"
"github.com/docker/docker/pkg/chrootarchive"
)
// Load uploads a set of images into the repository. This is the complementary of ImageExport.
// The input stream is an uncompressed tar ball containing images and metadata.
func (s *TagStore) Load(inTar io.ReadCloser, outStream io.Writer) error {
tmpImageDir, err := ioutil.TempDir("", "docker-import-")
if err != nil {
return err
}
defer os.RemoveAll(tmpImageDir)
var (
repoDir = filepath.Join(tmpImageDir, "repo")
)
if err := os.Mkdir(repoDir, os.ModeDir); err != nil {
return err
}
images := s.graph.Map()
excludes := make([]string, len(images))
i := 0
for k := range images {
excludes[i] = k
i++
}
if err := chrootarchive.Untar(inTar, repoDir, &archive.TarOptions{ExcludePatterns: excludes}); err != nil {
return err
}
dirs, err := ioutil.ReadDir(repoDir)
if err != nil {
return err
}
for _, d := range dirs {
if d.IsDir() {
if err := s.recursiveLoad(d.Name(), tmpImageDir); err != nil {
return err
}
}
}
reposJSONFile, err := os.Open(filepath.Join(tmpImageDir, "repo", "repositories"))
if err != nil {
if !os.IsNotExist(err) {
return err
}
return nil
}
defer reposJSONFile.Close()
repositories := map[string]repository{}
if err := json.NewDecoder(reposJSONFile).Decode(&repositories); err != nil {
return err
}
for imageName, tagMap := range repositories {
for tag, address := range tagMap {
if err := s.setLoad(imageName, tag, address, true, outStream); err != nil {
return err
}
}
}
return nil
}
func (s *TagStore) recursiveLoad(address, tmpImageDir string) error {
if _, err := s.LookupImage(address); err != nil {
logrus.Debugf("Loading %s", address)
imageJSON, err := ioutil.ReadFile(filepath.Join(tmpImageDir, "repo", address, "json"))
if err != nil {
logrus.Debugf("Error reading json: %v", err)
return err
}
layer, err := os.Open(filepath.Join(tmpImageDir, "repo", address, "layer.tar"))
if err != nil {
logrus.Debugf("Error reading embedded tar: %v", err)
return err
}
img, err := image.NewImgJSON(imageJSON)
if err != nil {
logrus.Debugf("Error unmarshalling json: %v", err)
return err
}
if err := image.ValidateID(img.ID); err != nil {
logrus.Debugf("Error validating ID: %v", err)
return err
}
// ensure no two downloads of the same layer happen at the same time
poolKey := "layer:" + img.ID
broadcaster, found := s.poolAdd("pull", poolKey)
if found {
logrus.Debugf("Image (id: %s) load is already running, waiting", img.ID)
return broadcaster.Wait()
}
defer s.poolRemove("pull", poolKey)
if img.Parent != "" {
if !s.graph.Exists(img.Parent) {
if err := s.recursiveLoad(img.Parent, tmpImageDir); err != nil {
return err
}
}
}
if err := s.graph.Register(v1Descriptor{img}, layer); err != nil {
return err
}
logrus.Debugf("Completed processing %s", address)
return nil
}
logrus.Debugf("already loaded %s", address)
return nil
}

14
graph/load_unsupported.go
View file

@ -1,14 +0,0 @@
// +build !linux,!windows
package graph
import (
"fmt"
"io"
)
// Load method is implemented here for non-linux and non-windows platforms and
// may return an error indicating that image load is not supported on other platforms.
func (s *TagStore) Load(inTar io.ReadCloser, outStream io.Writer) error {
return fmt.Errorf("Load is not supported on this platform")
}

44
graph/pools_test.go
View file

@ -1,44 +0,0 @@
package graph
import (
"testing"
"github.com/docker/docker/pkg/broadcaster"
"github.com/docker/docker/pkg/reexec"
)
func init() {
reexec.Init()
}
func TestPools(t *testing.T) {
s := &TagStore{
pullingPool: make(map[string]*broadcaster.Buffered),
pushingPool: make(map[string]*broadcaster.Buffered),
}
if _, found := s.poolAdd("pull", "test1"); found {
t.Fatal("Expected pull test1 not to be in progress")
}
if _, found := s.poolAdd("pull", "test2"); found {
t.Fatal("Expected pull test2 not to be in progress")
}
if _, found := s.poolAdd("push", "test1"); !found {
t.Fatalf("Expected pull test1 to be in progress`")
}
if _, found := s.poolAdd("pull", "test1"); !found {
t.Fatalf("Expected pull test1 to be in progress`")
}
if err := s.poolRemove("pull", "test2"); err != nil {
t.Fatal(err)
}
if err := s.poolRemove("pull", "test2"); err != nil {
t.Fatal(err)
}
if err := s.poolRemove("pull", "test1"); err != nil {
t.Fatal(err)
}
if err := s.poolRemove("push", "test1"); err != nil {
t.Fatal(err)
}
}

350
graph/pull_v1.go
View file

@ -1,350 +0,0 @@
package graph
import (
"errors"
"fmt"
"io"
"net"
"net/url"
"strings"
"time"
"github.com/Sirupsen/logrus"
"github.com/docker/distribution/registry/client/transport"
"github.com/docker/docker/image"
"github.com/docker/docker/pkg/progressreader"
"github.com/docker/docker/pkg/streamformatter"
"github.com/docker/docker/pkg/stringid"
"github.com/docker/docker/registry"
"github.com/docker/docker/utils"
)
type v1Puller struct {
*TagStore
endpoint registry.APIEndpoint
config *ImagePullConfig
sf *streamformatter.StreamFormatter
repoInfo *registry.RepositoryInfo
session *registry.Session
}
func (p *v1Puller) Pull(tag string) (fallback bool, err error) {
if utils.DigestReference(tag) {
// Allowing fallback, because HTTPS v1 is before HTTP v2
return true, registry.ErrNoSupport{Err: errors.New("Cannot pull by digest with v1 registry")}
}
tlsConfig, err := p.registryService.TLSConfig(p.repoInfo.Index.Name)
if err != nil {
return false, err
}
// Adds Docker-specific headers as well as user-specified headers (metaHeaders)
tr := transport.NewTransport(
// TODO(tiborvass): was ReceiveTimeout
registry.NewTransport(tlsConfig),
registry.DockerHeaders(p.config.MetaHeaders)...,
)
client := registry.HTTPClient(tr)
v1Endpoint, err := p.endpoint.ToV1Endpoint(p.config.MetaHeaders)
if err != nil {
logrus.Debugf("Could not get v1 endpoint: %v", err)
return true, err
}
p.session, err = registry.NewSession(client, p.config.AuthConfig, v1Endpoint)
if err != nil {
// TODO(dmcgowan): Check if should fallback
logrus.Debugf("Fallback from error: %s", err)
return true, err
}
if err := p.pullRepository(tag); err != nil {
// TODO(dmcgowan): Check if should fallback
return false, err
}
out := p.config.OutStream
out.Write(p.sf.FormatStatus("", "%s: this image was pulled from a legacy registry. Important: This registry version will not be supported in future versions of docker.", p.repoInfo.CanonicalName))
return false, nil
}
func (p *v1Puller) pullRepository(askedTag string) error {
out := p.config.OutStream
out.Write(p.sf.FormatStatus("", "Pulling repository %s", p.repoInfo.CanonicalName))
repoData, err := p.session.GetRepositoryData(p.repoInfo.RemoteName)
if err != nil {
if strings.Contains(err.Error(), "HTTP code: 404") {
return fmt.Errorf("Error: image %s not found", utils.ImageReference(p.repoInfo.RemoteName, askedTag))
}
// Unexpected HTTP error
return err
}
logrus.Debugf("Retrieving the tag list")
tagsList := make(map[string]string)
if askedTag == "" {
tagsList, err = p.session.GetRemoteTags(repoData.Endpoints, p.repoInfo.RemoteName)
} else {
var tagID string
tagID, err = p.session.GetRemoteTag(repoData.Endpoints, p.repoInfo.RemoteName, askedTag)
tagsList[askedTag] = tagID
}
if err != nil {
if err == registry.ErrRepoNotFound && askedTag != "" {
return fmt.Errorf("Tag %s not found in repository %s", askedTag, p.repoInfo.CanonicalName)
}
logrus.Errorf("unable to get remote tags: %s", err)
return err
}
for tag, id := range tagsList {
repoData.ImgList[id] = &registry.ImgData{
ID: id,
Tag: tag,
Checksum: "",
}
}
logrus.Debugf("Registering tags")
// If no tag has been specified, pull them all
if askedTag == "" {
for tag, id := range tagsList {
repoData.ImgList[id].Tag = tag
}
} else {
// Otherwise, check that the tag exists and use only that one
id, exists := tagsList[askedTag]
if !exists {
return fmt.Errorf("Tag %s not found in repository %s", askedTag, p.repoInfo.CanonicalName)
}
repoData.ImgList[id].Tag = askedTag
}
errors := make(chan error)
layersDownloaded := false
imgIDs := []string{}
sessionID := p.session.ID()
defer func() {
p.graph.Release(sessionID, imgIDs...)
}()
for _, imgData := range repoData.ImgList {
downloadImage := func(img *registry.ImgData) {
if askedTag != "" && img.Tag != askedTag {
errors <- nil
return
}
if img.Tag == "" {
logrus.Debugf("Image (id: %s) present in this repository but untagged, skipping", img.ID)
errors <- nil
return
}
if err := image.ValidateID(img.ID); err != nil {
errors <- err
return
}
// ensure no two downloads of the same image happen at the same time
poolKey := "img:" + img.ID
broadcaster, found := p.poolAdd("pull", poolKey)
broadcaster.Add(out)
if found {
errors <- broadcaster.Wait()
return
}
defer p.poolRemove("pull", poolKey)
// we need to retain it until tagging
p.graph.Retain(sessionID, img.ID)
imgIDs = append(imgIDs, img.ID)
broadcaster.Write(p.sf.FormatProgress(stringid.TruncateID(img.ID), fmt.Sprintf("Pulling image (%s) from %s", img.Tag, p.repoInfo.CanonicalName), nil))
success := false
var lastErr, err error
var isDownloaded bool
for _, ep := range p.repoInfo.Index.Mirrors {
ep += "v1/"
broadcaster.Write(p.sf.FormatProgress(stringid.TruncateID(img.ID), fmt.Sprintf("Pulling image (%s) from %s, mirror: %s", img.Tag, p.repoInfo.CanonicalName, ep), nil))
if isDownloaded, err = p.pullImage(broadcaster, img.ID, ep); err != nil {
// Don't report errors when pulling from mirrors.
logrus.Debugf("Error pulling image (%s) from %s, mirror: %s, %s", img.Tag, p.repoInfo.CanonicalName, ep, err)
continue
}
layersDownloaded = layersDownloaded || isDownloaded
success = true
break
}
if !success {
for _, ep := range repoData.Endpoints {
broadcaster.Write(p.sf.FormatProgress(stringid.TruncateID(img.ID), fmt.Sprintf("Pulling image (%s) from %s, endpoint: %s", img.Tag, p.repoInfo.CanonicalName, ep), nil))
if isDownloaded, err = p.pullImage(broadcaster, img.ID, ep); err != nil {
// It's not ideal that only the last error is returned, it would be better to concatenate the errors.
// As the error is also given to the output stream the user will see the error.
lastErr = err
broadcaster.Write(p.sf.FormatProgress(stringid.TruncateID(img.ID), fmt.Sprintf("Error pulling image (%s) from %s, endpoint: %s, %s", img.Tag, p.repoInfo.CanonicalName, ep, err), nil))
continue
}
layersDownloaded = layersDownloaded || isDownloaded
success = true
break
}
}
if !success {
err := fmt.Errorf("Error pulling image (%s) from %s, %v", img.Tag, p.repoInfo.CanonicalName, lastErr)
broadcaster.Write(p.sf.FormatProgress(stringid.TruncateID(img.ID), err.Error(), nil))
errors <- err
broadcaster.CloseWithError(err)
return
}
broadcaster.Write(p.sf.FormatProgress(stringid.TruncateID(img.ID), "Download complete", nil))
errors <- nil
}
go downloadImage(imgData)
}
var lastError error
for i := 0; i < len(repoData.ImgList); i++ {
if err := <-errors; err != nil {
lastError = err
}
}
if lastError != nil {
return lastError
}
for tag, id := range tagsList {
if askedTag != "" && tag != askedTag {
continue
}
if err := p.Tag(p.repoInfo.LocalName, tag, id, true); err != nil {
return err
}
}
requestedTag := p.repoInfo.LocalName
if len(askedTag) > 0 {
requestedTag = utils.ImageReference(p.repoInfo.LocalName, askedTag)
}
writeStatus(requestedTag, out, p.sf, layersDownloaded)
return nil
}
func (p *v1Puller) pullImage(out io.Writer, imgID, endpoint string) (layersDownloaded bool, err error) {
var history []string
history, err = p.session.GetRemoteHistory(imgID, endpoint)
if err != nil {
return false, err
}
out.Write(p.sf.FormatProgress(stringid.TruncateID(imgID), "Pulling dependent layers", nil))
// FIXME: Try to stream the images?
// FIXME: Launch the getRemoteImage() in goroutines
sessionID := p.session.ID()
// As imgID has been retained in pullRepository, no need to retain again
p.graph.Retain(sessionID, history[1:]...)
defer p.graph.Release(sessionID, history[1:]...)
layersDownloaded = false
for i := len(history) - 1; i >= 0; i-- {
id := history[i]
// ensure no two downloads of the same layer happen at the same time
poolKey := "layer:" + id
broadcaster, found := p.poolAdd("pull", poolKey)
broadcaster.Add(out)
if found {
logrus.Debugf("Image (id: %s) pull is already running, skipping", id)
err = broadcaster.Wait()
if err != nil {
return layersDownloaded, err
}
continue
}
// This must use a closure so it captures the value of err when
// the function returns, not when the 'defer' is evaluated.
defer func() {
p.poolRemoveWithError("pull", poolKey, err)
}()
if !p.graph.Exists(id) {
broadcaster.Write(p.sf.FormatProgress(stringid.TruncateID(id), "Pulling metadata", nil))
var (
imgJSON []byte
imgSize int64
err error
img *image.Image
)
retries := 5
for j := 1; j <= retries; j++ {
imgJSON, imgSize, err = p.session.GetRemoteImageJSON(id, endpoint)
if err != nil && j == retries {
broadcaster.Write(p.sf.FormatProgress(stringid.TruncateID(id), "Error pulling dependent layers", nil))
return layersDownloaded, err
} else if err != nil {
time.Sleep(time.Duration(j) * 500 * time.Millisecond)
continue
}
img, err = image.NewImgJSON(imgJSON)
layersDownloaded = true
if err != nil && j == retries {
broadcaster.Write(p.sf.FormatProgress(stringid.TruncateID(id), "Error pulling dependent layers", nil))
return layersDownloaded, fmt.Errorf("Failed to parse json: %s", err)
} else if err != nil {
time.Sleep(time.Duration(j) * 500 * time.Millisecond)
continue
} else {
break
}
}
for j := 1; j <= retries; j++ {
// Get the layer
status := "Pulling fs layer"
if j > 1 {
status = fmt.Sprintf("Pulling fs layer [retries: %d]", j)
}
broadcaster.Write(p.sf.FormatProgress(stringid.TruncateID(id), status, nil))
layer, err := p.session.GetRemoteImageLayer(img.ID, endpoint, imgSize)
if uerr, ok := err.(*url.Error); ok {
err = uerr.Err
}
if terr, ok := err.(net.Error); ok && terr.Timeout() && j < retries {
time.Sleep(time.Duration(j) * 500 * time.Millisecond)
continue
} else if err != nil {
broadcaster.Write(p.sf.FormatProgress(stringid.TruncateID(id), "Error pulling dependent layers", nil))
return layersDownloaded, err
}
layersDownloaded = true
defer layer.Close()
err = p.graph.Register(v1Descriptor{img},
progressreader.New(progressreader.Config{
In: layer,
Out: broadcaster,
Formatter: p.sf,
Size: imgSize,
NewLines: false,
ID: stringid.TruncateID(id),
Action: "Downloading",
}))
if terr, ok := err.(net.Error); ok && terr.Timeout() && j < retries {
time.Sleep(time.Duration(j) * 500 * time.Millisecond)
continue
} else if err != nil {
broadcaster.Write(p.sf.FormatProgress(stringid.TruncateID(id), "Error downloading dependent layers", nil))
return layersDownloaded, err
} else {
break
}
}
}
broadcaster.Write(p.sf.FormatProgress(stringid.TruncateID(id), "Download complete", nil))
broadcaster.Close()
}
return layersDownloaded, nil
}

728
graph/pull_v2.go
View file

@ -1,728 +0,0 @@
package graph
import (
"encoding/json"
"errors"
"fmt"
"io"
"io/ioutil"
"os"
"runtime"
"github.com/Sirupsen/logrus"
"github.com/docker/distribution"
"github.com/docker/distribution/digest"
"github.com/docker/distribution/manifest/schema1"
"github.com/docker/docker/image"
"github.com/docker/docker/pkg/broadcaster"
"github.com/docker/docker/pkg/progressreader"
"github.com/docker/docker/pkg/streamformatter"
"github.com/docker/docker/pkg/stringid"
"github.com/docker/docker/registry"
"github.com/docker/docker/utils"
"golang.org/x/net/context"
)
type v2Puller struct {
*TagStore
endpoint registry.APIEndpoint
config *ImagePullConfig
sf *streamformatter.StreamFormatter
repoInfo *registry.RepositoryInfo
repo distribution.Repository
sessionID string
}
func (p *v2Puller) Pull(tag string) (fallback bool, err error) {
// TODO(tiborvass): was ReceiveTimeout
p.repo, err = newV2Repository(p.repoInfo, p.endpoint, p.config.MetaHeaders, p.config.AuthConfig, "pull")
if err != nil {
logrus.Warnf("Error getting v2 registry: %v", err)
return true, err
}
p.sessionID = stringid.GenerateRandomID()
if err := p.pullV2Repository(tag); err != nil {
if registry.ContinueOnError(err) {
logrus.Debugf("Error trying v2 registry: %v", err)
return true, err
}
return false, err
}
return false, nil
}
func (p *v2Puller) pullV2Repository(tag string) (err error) {
var tags []string
taggedName := p.repoInfo.LocalName
if len(tag) > 0 {
tags = []string{tag}
taggedName = utils.ImageReference(p.repoInfo.LocalName, tag)
} else {
var err error
manSvc, err := p.repo.Manifests(context.Background())
if err != nil {
return err
}
tags, err = manSvc.Tags()
if err != nil {
return err
}
}
poolKey := "v2:" + taggedName
broadcaster, found := p.poolAdd("pull", poolKey)
broadcaster.Add(p.config.OutStream)
if found {
// Another pull of the same repository is already taking place; just wait for it to finish
return broadcaster.Wait()
}
// This must use a closure so it captures the value of err when the
// function returns, not when the 'defer' is evaluated.
defer func() {
p.poolRemoveWithError("pull", poolKey, err)
}()
var layersDownloaded bool
for _, tag := range tags {
// pulledNew is true if either new layers were downloaded OR if existing images were newly tagged
// TODO(tiborvass): should we change the name of `layersDownload`? What about message in WriteStatus?
pulledNew, err := p.pullV2Tag(broadcaster, tag, taggedName)
if err != nil {
return err
}
layersDownloaded = layersDownloaded || pulledNew
}
writeStatus(taggedName, broadcaster, p.sf, layersDownloaded)
return nil
}
// downloadInfo is used to pass information from download to extractor
type downloadInfo struct {
img contentAddressableDescriptor
imgIndex int
tmpFile *os.File
digest digest.Digest
layer distribution.ReadSeekCloser
size int64
err chan error
poolKey string
broadcaster *broadcaster.Buffered
}
// contentAddressableDescriptor is used to pass image data from a manifest to the
// graph.
type contentAddressableDescriptor struct {
id string
parent string
strongID digest.Digest
compatibilityID string
config []byte
v1Compatibility []byte
}
func newContentAddressableImage(v1Compatibility []byte, blobSum digest.Digest, parent digest.Digest) (contentAddressableDescriptor, error) {
img := contentAddressableDescriptor{
v1Compatibility: v1Compatibility,
}
var err error
img.config, err = image.MakeImageConfig(v1Compatibility, blobSum, parent)
if err != nil {
return img, err
}
img.strongID, err = image.StrongID(img.config)
if err != nil {
return img, err
}
unmarshalledConfig, err := image.NewImgJSON(v1Compatibility)
if err != nil {
return img, err
}
img.compatibilityID = unmarshalledConfig.ID
img.id = img.strongID.Hex()
return img, nil
}
// ID returns the actual ID to be used for the downloaded image. This may be
// a computed ID.
func (img contentAddressableDescriptor) ID() string {
return img.id
}
// Parent returns the parent ID to be used for the image. This may be a
// computed ID.
func (img contentAddressableDescriptor) Parent() string {
return img.parent
}
// MarshalConfig renders the image structure into JSON.
func (img contentAddressableDescriptor) MarshalConfig() ([]byte, error) {
return img.config, nil
}
type errVerification struct{}
func (errVerification) Error() string { return "verification failed" }
func (p *v2Puller) download(di *downloadInfo) {
logrus.Debugf("pulling blob %q to %s", di.digest, di.img.id)
blobs := p.repo.Blobs(context.Background())
desc, err := blobs.Stat(context.Background(), di.digest)
if err != nil {
logrus.Debugf("Error statting layer: %v", err)
di.err <- err
return
}
di.size = desc.Size
layerDownload, err := blobs.Open(context.Background(), di.digest)
if err != nil {
logrus.Debugf("Error fetching layer: %v", err)
di.err <- err
return
}
defer layerDownload.Close()
verifier, err := digest.NewDigestVerifier(di.digest)
if err != nil {
di.err <- err
return
}
reader := progressreader.New(progressreader.Config{
In: ioutil.NopCloser(io.TeeReader(layerDownload, verifier)),
Out: di.broadcaster,
Formatter: p.sf,
Size: di.size,
NewLines: false,
ID: stringid.TruncateID(di.img.id),
Action: "Downloading",
})
io.Copy(di.tmpFile, reader)
di.broadcaster.Write(p.sf.FormatProgress(stringid.TruncateID(di.img.id), "Verifying Checksum", nil))
if !verifier.Verified() {
err = fmt.Errorf("filesystem layer verification failed for digest %s", di.digest)
logrus.Error(err)
di.err <- err
return
}
di.broadcaster.Write(p.sf.FormatProgress(stringid.TruncateID(di.img.id), "Download complete", nil))
logrus.Debugf("Downloaded %s to tempfile %s", di.img.id, di.tmpFile.Name())
di.layer = layerDownload
di.err <- nil
}
func (p *v2Puller) pullV2Tag(out io.Writer, tag, taggedName string) (tagUpdated bool, err error) {
logrus.Debugf("Pulling tag from V2 registry: %q", tag)
manSvc, err := p.repo.Manifests(context.Background())
if err != nil {
return false, err
}
unverifiedManifest, err := manSvc.GetByTag(tag)
if err != nil {
return false, err
}
if unverifiedManifest == nil {
return false, fmt.Errorf("image manifest does not exist for tag %q", tag)
}
var verifiedManifest *schema1.Manifest
verifiedManifest, err = verifyManifest(unverifiedManifest, tag)
if err != nil {
return false, err
}
// remove duplicate layers and check parent chain validity
err = fixManifestLayers(verifiedManifest)
if err != nil {
return false, err
}
imgs, err := p.getImageInfos(verifiedManifest)
if err != nil {
return false, err
}
out.Write(p.sf.FormatStatus(tag, "Pulling from %s", p.repo.Name()))
var downloads []*downloadInfo
var layerIDs []string
defer func() {
p.graph.Release(p.sessionID, layerIDs...)
for _, d := range downloads {
p.poolRemoveWithError("pull", d.poolKey, err)
if d.tmpFile != nil {
d.tmpFile.Close()
if err := os.RemoveAll(d.tmpFile.Name()); err != nil {
logrus.Errorf("Failed to remove temp file: %s", d.tmpFile.Name())
}
}
}
}()
for i := len(verifiedManifest.FSLayers) - 1; i >= 0; i-- {
img := imgs[i]
p.graph.Retain(p.sessionID, img.id)
layerIDs = append(layerIDs, img.id)
p.graph.imageMutex.Lock(img.id)
// Check if exists
if p.graph.Exists(img.id) {
if err := p.validateImageInGraph(img.id, imgs, i); err != nil {
p.graph.imageMutex.Unlock(img.id)
return false, fmt.Errorf("image validation failed: %v", err)
}
logrus.Debugf("Image already exists: %s", img.id)
p.graph.imageMutex.Unlock(img.id)
continue
}
p.graph.imageMutex.Unlock(img.id)
out.Write(p.sf.FormatProgress(stringid.TruncateID(img.id), "Pulling fs layer", nil))
d := &downloadInfo{
img: img,
imgIndex: i,
poolKey: "v2layer:" + img.id,
digest: verifiedManifest.FSLayers[i].BlobSum,
// TODO: seems like this chan buffer solved hanging problem in go1.5,
// this can indicate some deeper problem that somehow we never take
// error from channel in loop below
err: make(chan error, 1),
}
tmpFile, err := ioutil.TempFile("", "GetImageBlob")
if err != nil {
return false, err
}
d.tmpFile = tmpFile
downloads = append(downloads, d)
broadcaster, found := p.poolAdd("pull", d.poolKey)
broadcaster.Add(out)
d.broadcaster = broadcaster
if found {
d.err <- nil
} else {
go p.download(d)
}
}
for _, d := range downloads {
if err := <-d.err; err != nil {
return false, err
}
if d.layer == nil {
// Wait for a different pull to download and extract
// this layer.
err = d.broadcaster.Wait()
if err != nil {
return false, err
}
continue
}
d.tmpFile.Seek(0, 0)
err := func() error {
reader := progressreader.New(progressreader.Config{
In: d.tmpFile,
Out: d.broadcaster,
Formatter: p.sf,
Size: d.size,
NewLines: false,
ID: stringid.TruncateID(d.img.id),
Action: "Extracting",
})
p.graph.imagesMutex.Lock()
defer p.graph.imagesMutex.Unlock()
p.graph.imageMutex.Lock(d.img.id)
defer p.graph.imageMutex.Unlock(d.img.id)
// Must recheck the data on disk if any exists.
// This protects against races where something
// else is written to the graph under this ID
// after attemptIDReuse.
if p.graph.Exists(d.img.id) {
if err := p.validateImageInGraph(d.img.id, imgs, d.imgIndex); err != nil {
return fmt.Errorf("image validation failed: %v", err)
}
}
if err := p.graph.register(d.img, reader); err != nil {
return err
}
if err := p.graph.setLayerDigest(d.img.id, d.digest); err != nil {
return err
}
if err := p.graph.setV1CompatibilityConfig(d.img.id, d.img.v1Compatibility); err != nil {
return err
}
return nil
}()
if err != nil {
return false, err
}
d.broadcaster.Write(p.sf.FormatProgress(stringid.TruncateID(d.img.id), "Pull complete", nil))
d.broadcaster.Close()
tagUpdated = true
}
manifestDigest, _, err := digestFromManifest(unverifiedManifest, p.repoInfo.LocalName)
if err != nil {
return false, err
}
// Check for new tag if no layers downloaded
if !tagUpdated {
repo, err := p.get(p.repoInfo.LocalName)
if err != nil {
return false, err
}
if repo != nil {
if _, exists := repo[tag]; !exists {
tagUpdated = true
}
} else {
tagUpdated = true
}
}
firstID := layerIDs[len(layerIDs)-1]
if utils.DigestReference(tag) {
// TODO(stevvooe): Ideally, we should always set the digest so we can
// use the digest whether we pull by it or not. Unfortunately, the tag
// store treats the digest as a separate tag, meaning there may be an
// untagged digest image that would seem to be dangling by a user.
if err = p.setDigest(p.repoInfo.LocalName, tag, firstID); err != nil {
return false, err
}
} else {
// only set the repository/tag -> image ID mapping when pulling by tag (i.e. not by digest)
if err = p.Tag(p.repoInfo.LocalName, tag, firstID, true); err != nil {
return false, err
}
}
if manifestDigest != "" {
out.Write(p.sf.FormatStatus("", "Digest: %s", manifestDigest))
}
return tagUpdated, nil
}
func verifyManifest(signedManifest *schema1.SignedManifest, tag string) (m *schema1.Manifest, err error) {
// If pull by digest, then verify the manifest digest. NOTE: It is
// important to do this first, before any other content validation. If the
// digest cannot be verified, don't even bother with those other things.
if manifestDigest, err := digest.ParseDigest(tag); err == nil {
verifier, err := digest.NewDigestVerifier(manifestDigest)
if err != nil {
return nil, err
}
payload, err := signedManifest.Payload()
if err != nil {
// If this failed, the signatures section was corrupted
// or missing. Treat the entire manifest as the payload.
payload = signedManifest.Raw
}
if _, err := verifier.Write(payload); err != nil {
return nil, err
}
if !verifier.Verified() {
err := fmt.Errorf("image verification failed for digest %s", manifestDigest)
logrus.Error(err)
return nil, err
}
var verifiedManifest schema1.Manifest
if err = json.Unmarshal(payload, &verifiedManifest); err != nil {
return nil, err
}
m = &verifiedManifest
} else {
m = &signedManifest.Manifest
}
if m.SchemaVersion != 1 {
return nil, fmt.Errorf("unsupported schema version %d for tag %q", m.SchemaVersion, tag)
}
if len(m.FSLayers) != len(m.History) {
return nil, fmt.Errorf("length of history not equal to number of layers for tag %q", tag)
}
if len(m.FSLayers) == 0 {
return nil, fmt.Errorf("no FSLayers in manifest for tag %q", tag)
}
return m, nil
}
// fixManifestLayers removes repeated layers from the manifest and checks the
// correctness of the parent chain.
func fixManifestLayers(m *schema1.Manifest) error {
images := make([]*image.Image, len(m.FSLayers))
for i := range m.FSLayers {
img, err := image.NewImgJSON([]byte(m.History[i].V1Compatibility))
if err != nil {
return err
}
images[i] = img
if err := image.ValidateID(img.ID); err != nil {
return err
}
}
if images[len(images)-1].Parent != "" && !allowBaseParentImage {
// Windows base layer can point to a base layer parent that is not in manifest.
return errors.New("Invalid parent ID in the base layer of the image.")
}
// check general duplicates to error instead of a deadlock
idmap := make(map[string]struct{})
var lastID string
for _, img := range images {
// skip IDs that appear after each other, we handle those later
if _, exists := idmap[img.ID]; img.ID != lastID && exists {
return fmt.Errorf("ID %+v appears multiple times in manifest", img.ID)
}
lastID = img.ID
idmap[lastID] = struct{}{}
}
// backwards loop so that we keep the remaining indexes after removing items
for i := len(images) - 2; i >= 0; i-- {
if images[i].ID == images[i+1].ID { // repeated ID. remove and continue
m.FSLayers = append(m.FSLayers[:i], m.FSLayers[i+1:]...)
m.History = append(m.History[:i], m.History[i+1:]...)
} else if images[i].Parent != images[i+1].ID {
return fmt.Errorf("Invalid parent ID. Expected %v, got %v.", images[i+1].ID, images[i].Parent)
}
}
return nil
}
// getImageInfos returns an imageinfo struct for every image in the manifest.
// These objects contain both calculated strongIDs and compatibilityIDs found
// in v1Compatibility object.
func (p *v2Puller) getImageInfos(m *schema1.Manifest) ([]contentAddressableDescriptor, error) {
imgs := make([]contentAddressableDescriptor, len(m.FSLayers))
var parent digest.Digest
for i := len(imgs) - 1; i >= 0; i-- {
var err error
imgs[i], err = newContentAddressableImage([]byte(m.History[i].V1Compatibility), m.FSLayers[i].BlobSum, parent)
if err != nil {
return nil, err
}
parent = imgs[i].strongID
}
p.attemptIDReuse(imgs)
// reset the base layer parent for windows
if allowBaseParentImage {
var base struct{ Parent string }
if err := json.Unmarshal(imgs[len(imgs)-1].v1Compatibility, &base); err != nil {
return nil, err
}
if base.Parent != "" {
imgs[len(imgs)-1].parent = base.Parent
}
}
return imgs, nil
}
// attemptIDReuse does a best attempt to match verified compatibilityIDs
// already in the graph with the computed strongIDs so we can keep using them.
// This process will never fail but may just return the strongIDs if none of
// the compatibilityIDs exists or can be verified. If the strongIDs themselves
// fail verification, we deterministically generate alternate IDs to use until
// we find one that's available or already exists with the correct data.
func (p *v2Puller) attemptIDReuse(imgs []contentAddressableDescriptor) {
// This function needs to be protected with a global lock, because it
// locks multiple IDs at once, and there's no good way to make sure
// the locking happens a deterministic order.
p.graph.imagesMutex.Lock()
defer p.graph.imagesMutex.Unlock()
idMap := make(map[string]struct{})
for _, img := range imgs {
idMap[img.id] = struct{}{}
idMap[img.compatibilityID] = struct{}{}
if p.graph.Exists(img.compatibilityID) {
if _, err := p.graph.generateV1CompatibilityChain(img.compatibilityID); err != nil {
logrus.Debugf("Migration v1Compatibility generation error: %v", err)
return
}
}
}
for id := range idMap {
p.graph.imageMutex.Lock(id)
defer p.graph.imageMutex.Unlock(id)
}
// continueReuse controls whether the function will try to find
// existing layers on disk under the old v1 IDs, to avoid repulling
// them. The hashes are checked to ensure these layers are okay to
// use. continueReuse starts out as true, but is set to false if
// the code encounters something that doesn't match the expected hash.
continueReuse := true
for i := len(imgs) - 1; i >= 0; i-- {
// TODO - (swernli:11-16-2015) Skipping content addressable IDs on
// Windows as a hack for TP4 compat. The correct fix is to ensure that
// Windows layers do not have anything in them that takes a dependency
// on the ID of the layer in the management client. This will be fixed
// in Windows post-TP4.
if runtime.GOOS == "windows" {
imgs[i].id = imgs[i].compatibilityID
}
if p.graph.Exists(imgs[i].id) {
// Found an image in the graph under the strongID. Validate the
// image before using it.
if err := p.validateImageInGraph(imgs[i].id, imgs, i); err != nil {
continueReuse = false
logrus.Debugf("not using existing strongID: %v", err)
// The strong ID existed in the graph but didn't
// validate successfully. We can't use the strong ID
// because it didn't validate successfully. Treat the
// graph like a hash table with probing... compute
// SHA256(id) until we find an ID that either doesn't
// already exist in the graph, or has existing content
// that validates successfully.
for {
if err := p.tryNextID(imgs, i, idMap); err != nil {
logrus.Debug(err.Error())
} else {
break
}
}
}
continue
}
if continueReuse {
compatibilityID := imgs[i].compatibilityID
if err := p.validateImageInGraph(compatibilityID, imgs, i); err != nil {
logrus.Debugf("stopping ID reuse: %v", err)
continueReuse = false
} else {
// The compatibility ID exists in the graph and was
// validated. Use it.
imgs[i].id = compatibilityID
}
}
}
// fix up the parents of the images
for i := 0; i < len(imgs); i++ {
if i == len(imgs)-1 { // Base layer
imgs[i].parent = ""
} else {
imgs[i].parent = imgs[i+1].id
}
}
}
// validateImageInGraph checks that an image in the graph has the expected
// strongID. id is the entry in the graph to check, imgs is the slice of
// images being processed (for access to the parent), and i is the index
// into this slice which the graph entry should be checked against.
func (p *v2Puller) validateImageInGraph(id string, imgs []contentAddressableDescriptor, i int) error {
img, err := p.graph.Get(id)
if err != nil {
return fmt.Errorf("missing: %v", err)
}
if runtime.GOOS == "windows" {
// TODO - (swernli:11-16-2015) Skipping content addressable IDs on
// Windows as a hack for TP4 compat. The correct fix is to ensure that
// Windows layers do not have anything in them that takes a dependency
// on the ID of the layer in the management client. This will be fixed
// in Windows post-TP4.
return nil
}
layerID, err := p.graph.getLayerDigest(id)
if err != nil {
return fmt.Errorf("digest: %v", err)
}
var parentID digest.Digest
if i != len(imgs)-1 {
if img.Parent != imgs[i+1].id { // comparing that graph points to validated ID
return fmt.Errorf("parent: %v %v", img.Parent, imgs[i+1].id)
}
parentID = imgs[i+1].strongID
} else if img.Parent != "" {
return fmt.Errorf("unexpected parent: %v", img.Parent)
}
v1Config, err := p.graph.getV1CompatibilityConfig(img.ID)
if err != nil {
return fmt.Errorf("v1Compatibility: %v %v", img.ID, err)
}
json, err := image.MakeImageConfig(v1Config, layerID, parentID)
if err != nil {
return fmt.Errorf("make config: %v", err)
}
if dgst, err := image.StrongID(json); err == nil && dgst == imgs[i].strongID {
logrus.Debugf("Validated %v as %v", dgst, id)
} else {
return fmt.Errorf("digest mismatch: %v %v, error: %v", dgst, imgs[i].strongID, err)
}
// All clear
return nil
}
func (p *v2Puller) tryNextID(imgs []contentAddressableDescriptor, i int, idMap map[string]struct{}) error {
nextID, _ := digest.FromBytes([]byte(imgs[i].id))
imgs[i].id = nextID.Hex()
if _, exists := idMap[imgs[i].id]; !exists {
p.graph.imageMutex.Lock(imgs[i].id)
defer p.graph.imageMutex.Unlock(imgs[i].id)
}
if p.graph.Exists(imgs[i].id) {
if err := p.validateImageInGraph(imgs[i].id, imgs, i); err != nil {
return fmt.Errorf("not using existing strongID permutation %s: %v", imgs[i].id, err)
}
}
return nil
}

126
graph/push.go
View file

@ -1,126 +0,0 @@
package graph
import (
"fmt"
"io"
"github.com/Sirupsen/logrus"
"github.com/docker/distribution/digest"
"github.com/docker/docker/cliconfig"
"github.com/docker/docker/pkg/streamformatter"
"github.com/docker/docker/registry"
)
// ImagePushConfig stores push configuration.
type ImagePushConfig struct {
// MetaHeaders store HTTP headers with metadata about the image
// (DockerHeaders with prefix X-Meta- in the request).
MetaHeaders map[string][]string
// AuthConfig holds authentication credentials for authenticating with
// the registry.
AuthConfig *cliconfig.AuthConfig
// Tag is the specific variant of the image to be pushed.
// If no tag is provided, all tags will be pushed.
Tag string
// OutStream is the output writer for showing the status of the push
// operation.
OutStream io.Writer
}
// pusher is an interface that abstracts pushing for different API versions.
type pusher interface {
// Push tries to push the image configured at the creation of Pusher.
// Push returns an error if any, as well as a boolean that determines whether to retry Push on the next configured endpoint.
//
// TODO(tiborvass): have Push() take a reference to repository + tag, so that the pusher itself is repository-agnostic.
Push() (fallback bool, err error)
}
// newPusher creates a new Pusher interface that will push to either a v1 or v2
// registry. The endpoint argument contains a Version field that determines
// whether a v1 or v2 pusher will be created. The other parameters are passed
// through to the underlying pusher implementation for use during the actual
// push operation.
func (s *TagStore) newPusher(endpoint registry.APIEndpoint, localRepo repository, repoInfo *registry.RepositoryInfo, imagePushConfig *ImagePushConfig, sf *streamformatter.StreamFormatter) (pusher, error) {
switch endpoint.Version {
case registry.APIVersion2:
return &v2Pusher{
TagStore: s,
endpoint: endpoint,
localRepo: localRepo,
repoInfo: repoInfo,
config: imagePushConfig,
sf: sf,
layersPushed: make(map[digest.Digest]bool),
}, nil
case registry.APIVersion1:
return &v1Pusher{
TagStore: s,
endpoint: endpoint,
localRepo: localRepo,
repoInfo: repoInfo,
config: imagePushConfig,
sf: sf,
}, nil
}
return nil, fmt.Errorf("unknown version %d for registry %s", endpoint.Version, endpoint.URL)
}
// Push initiates a push operation on the repository named localName.
func (s *TagStore) Push(localName string, imagePushConfig *ImagePushConfig) error {
// FIXME: Allow to interrupt current push when new push of same image is done.
var sf = streamformatter.NewJSONStreamFormatter()
// Resolve the Repository name from fqn to RepositoryInfo
repoInfo, err := s.registryService.ResolveRepository(localName)
if err != nil {
return err
}
endpoints, err := s.registryService.LookupPushEndpoints(repoInfo.CanonicalName)
if err != nil {
return err
}
reposLen := 1
if imagePushConfig.Tag == "" {
reposLen = len(s.Repositories[repoInfo.LocalName])
}
imagePushConfig.OutStream.Write(sf.FormatStatus("", "The push refers to a repository [%s] (len: %d)", repoInfo.CanonicalName, reposLen))
// If it fails, try to get the repository
localRepo, exists := s.Repositories[repoInfo.LocalName]
if !exists {
return fmt.Errorf("Repository does not exist: %s", repoInfo.LocalName)
}
var lastErr error
for _, endpoint := range endpoints {
logrus.Debugf("Trying to push %s to %s %s", repoInfo.CanonicalName, endpoint.URL, endpoint.Version)
pusher, err := s.newPusher(endpoint, localRepo, repoInfo, imagePushConfig, sf)
if err != nil {
lastErr = err
continue
}
if fallback, err := pusher.Push(); err != nil {
if fallback {
lastErr = err
continue
}
logrus.Debugf("Not continuing with error: %v", err)
return err
}
s.eventsService.Log("push", repoInfo.LocalName, "")
return nil
}
if lastErr == nil {
lastErr = fmt.Errorf("no endpoints found for %s", repoInfo.CanonicalName)
}
return lastErr
}

354
graph/push_v1.go
View file

@ -1,354 +0,0 @@
package graph
import (
"fmt"
"io"
"os"
"sync"
"github.com/Sirupsen/logrus"
"github.com/docker/distribution/registry/client/transport"
"github.com/docker/docker/image"
"github.com/docker/docker/pkg/ioutils"
"github.com/docker/docker/pkg/progressreader"
"github.com/docker/docker/pkg/streamformatter"
"github.com/docker/docker/pkg/stringid"
"github.com/docker/docker/registry"
"github.com/docker/docker/utils"
)
type v1Pusher struct {
*TagStore
endpoint registry.APIEndpoint
localRepo repository
repoInfo *registry.RepositoryInfo
config *ImagePushConfig
sf *streamformatter.StreamFormatter
session *registry.Session
out io.Writer
}
func (p *v1Pusher) Push() (fallback bool, err error) {
tlsConfig, err := p.registryService.TLSConfig(p.repoInfo.Index.Name)
if err != nil {
return false, err
}
// Adds Docker-specific headers as well as user-specified headers (metaHeaders)
tr := transport.NewTransport(
// TODO(tiborvass): was NoTimeout
registry.NewTransport(tlsConfig),
registry.DockerHeaders(p.config.MetaHeaders)...,
)
client := registry.HTTPClient(tr)
v1Endpoint, err := p.endpoint.ToV1Endpoint(p.config.MetaHeaders)
if err != nil {
logrus.Debugf("Could not get v1 endpoint: %v", err)
return true, err
}
p.session, err = registry.NewSession(client, p.config.AuthConfig, v1Endpoint)
if err != nil {
// TODO(dmcgowan): Check if should fallback
return true, err
}
if err := p.pushRepository(p.config.Tag); err != nil {
// TODO(dmcgowan): Check if should fallback
return false, err
}
return false, nil
}
// Retrieve the all the images to be uploaded in the correct order
func (p *v1Pusher) getImageList(requestedTag string) ([]string, map[string][]string, error) {
var (
imageList []string
imagesSeen = make(map[string]bool)
tagsByImage = make(map[string][]string)
)
for tag, id := range p.localRepo {
if requestedTag != "" && requestedTag != tag {
// Include only the requested tag.
continue
}
if utils.DigestReference(tag) {
// Ignore digest references.
continue
}
var imageListForThisTag []string
tagsByImage[id] = append(tagsByImage[id], tag)
for img, err := p.graph.Get(id); img != nil; img, err = p.graph.GetParent(img) {
if err != nil {
return nil, nil, err
}
if imagesSeen[img.ID] {
// This image is already on the list, we can ignore it and all its parents
break
}
imagesSeen[img.ID] = true
imageListForThisTag = append(imageListForThisTag, img.ID)
}
// reverse the image list for this tag (so the "most"-parent image is first)
for i, j := 0, len(imageListForThisTag)-1; i < j; i, j = i+1, j-1 {
imageListForThisTag[i], imageListForThisTag[j] = imageListForThisTag[j], imageListForThisTag[i]
}
// append to main image list
imageList = append(imageList, imageListForThisTag...)
}
if len(imageList) == 0 {
return nil, nil, fmt.Errorf("No images found for the requested repository / tag")
}
logrus.Debugf("Image list: %v", imageList)
logrus.Debugf("Tags by image: %v", tagsByImage)
return imageList, tagsByImage, nil
}
// createImageIndex returns an index of an image's layer IDs and tags.
func (s *TagStore) createImageIndex(images []string, tags map[string][]string) []*registry.ImgData {
var imageIndex []*registry.ImgData
for _, id := range images {
if tags, hasTags := tags[id]; hasTags {
// If an image has tags you must add an entry in the image index
// for each tag
for _, tag := range tags {
imageIndex = append(imageIndex, &registry.ImgData{
ID: id,
Tag: tag,
})
}
continue
}
// If the image does not have a tag it still needs to be sent to the
// registry with an empty tag so that it is associated with the repository
imageIndex = append(imageIndex, &registry.ImgData{
ID: id,
Tag: "",
})
}
return imageIndex
}
type imagePushData struct {
id string
compatibilityID string
endpoint string
}
// lookupImageOnEndpoint checks the specified endpoint to see if an image exists
// and if it is absent then it sends the image id to the channel to be pushed.
func (p *v1Pusher) lookupImageOnEndpoint(wg *sync.WaitGroup, images chan imagePushData, imagesToPush chan string) {
defer wg.Done()
for image := range images {
if err := p.session.LookupRemoteImage(image.compatibilityID, image.endpoint); err != nil {
logrus.Errorf("Error in LookupRemoteImage: %s", err)
imagesToPush <- image.id
continue
}
p.out.Write(p.sf.FormatStatus("", "Image %s already pushed, skipping", stringid.TruncateID(image.id)))
}
}
func (p *v1Pusher) pushImageToEndpoint(endpoint string, imageIDs []string, tags map[string][]string, repo *registry.RepositoryData) error {
workerCount := len(imageIDs)
// start a maximum of 5 workers to check if images exist on the specified endpoint.
if workerCount > 5 {
workerCount = 5
}
var (
wg = &sync.WaitGroup{}
imageData = make(chan imagePushData, workerCount*2)
imagesToPush = make(chan string, workerCount*2)
pushes = make(chan map[string]struct{}, 1)
)
for i := 0; i < workerCount; i++ {
wg.Add(1)
go p.lookupImageOnEndpoint(wg, imageData, imagesToPush)
}
// start a go routine that consumes the images to push
go func() {
shouldPush := make(map[string]struct{})
for id := range imagesToPush {
shouldPush[id] = struct{}{}
}
pushes <- shouldPush
}()
for _, id := range imageIDs {
compatibilityID, err := p.getV1ID(id)
if err != nil {
return err
}
imageData <- imagePushData{
id: id,
compatibilityID: compatibilityID,
endpoint: endpoint,
}
}
// close the channel to notify the workers that there will be no more images to check.
close(imageData)
wg.Wait()
close(imagesToPush)
// wait for all the images that require pushes to be collected into a consumable map.
shouldPush := <-pushes
// finish by pushing any images and tags to the endpoint. The order that the images are pushed
// is very important that is why we are still iterating over the ordered list of imageIDs.
for _, id := range imageIDs {
if _, push := shouldPush[id]; push {
if _, err := p.pushImage(id, endpoint); err != nil {
// FIXME: Continue on error?
return err
}
}
for _, tag := range tags[id] {
p.out.Write(p.sf.FormatStatus("", "Pushing tag for rev [%s] on {%s}", stringid.TruncateID(id), endpoint+"repositories/"+p.repoInfo.RemoteName+"/tags/"+tag))
compatibilityID, err := p.getV1ID(id)
if err != nil {
return err
}
if err := p.session.PushRegistryTag(p.repoInfo.RemoteName, compatibilityID, tag, endpoint); err != nil {
return err
}
}
}
return nil
}
// pushRepository pushes layers that do not already exist on the registry.
func (p *v1Pusher) pushRepository(tag string) error {
logrus.Debugf("Local repo: %s", p.localRepo)
p.out = ioutils.NewWriteFlusher(p.config.OutStream)
imgList, tags, err := p.getImageList(tag)
if err != nil {
return err
}
p.out.Write(p.sf.FormatStatus("", "Sending image list"))
imageIndex := p.createImageIndex(imgList, tags)
logrus.Debugf("Preparing to push %s with the following images and tags", p.localRepo)
for _, data := range imageIndex {
logrus.Debugf("Pushing ID: %s with Tag: %s", data.ID, data.Tag)
// convert IDs to compatibilityIDs, imageIndex only used in registry calls
data.ID, err = p.getV1ID(data.ID)
if err != nil {
return err
}
}
if _, found := p.poolAdd("push", p.repoInfo.LocalName); found {
return fmt.Errorf("push or pull %s is already in progress", p.repoInfo.LocalName)
}
defer p.poolRemove("push", p.repoInfo.LocalName)
// Register all the images in a repository with the registry
// If an image is not in this list it will not be associated with the repository
repoData, err := p.session.PushImageJSONIndex(p.repoInfo.RemoteName, imageIndex, false, nil)
if err != nil {
return err
}
nTag := 1
if tag == "" {
nTag = len(p.localRepo)
}
p.out.Write(p.sf.FormatStatus("", "Pushing repository %s (%d tags)", p.repoInfo.CanonicalName, nTag))
// push the repository to each of the endpoints only if it does not exist.
for _, endpoint := range repoData.Endpoints {
if err := p.pushImageToEndpoint(endpoint, imgList, tags, repoData); err != nil {
return err
}
}
_, err = p.session.PushImageJSONIndex(p.repoInfo.RemoteName, imageIndex, true, repoData.Endpoints)
return err
}
func (p *v1Pusher) pushImage(imgID, ep string) (checksum string, err error) {
jsonRaw, err := p.getV1Config(imgID)
if err != nil {
return "", fmt.Errorf("Cannot retrieve the path for {%s}: %s", imgID, err)
}
p.out.Write(p.sf.FormatProgress(stringid.TruncateID(imgID), "Pushing", nil))
compatibilityID, err := p.getV1ID(imgID)
if err != nil {
return "", err
}
// General rule is to use ID for graph accesses and compatibilityID for
// calls to session.registry()
imgData := &registry.ImgData{
ID: compatibilityID,
}
// Send the json
if err := p.session.PushImageJSONRegistry(imgData, jsonRaw, ep); err != nil {
if err == registry.ErrAlreadyExists {
p.out.Write(p.sf.FormatProgress(stringid.TruncateID(imgID), "Image already pushed, skipping", nil))
return "", nil
}
return "", err
}
layerData, err := p.graph.tempLayerArchive(imgID, p.sf, p.out)
if err != nil {
return "", fmt.Errorf("Failed to generate layer archive: %s", err)
}
defer os.RemoveAll(layerData.Name())
// Send the layer
logrus.Debugf("rendered layer for %s of [%d] size", imgID, layerData.Size)
checksum, checksumPayload, err := p.session.PushImageLayerRegistry(imgData.ID,
progressreader.New(progressreader.Config{
In: layerData,
Out: p.out,
Formatter: p.sf,
Size: layerData.Size,
NewLines: false,
ID: stringid.TruncateID(imgID),
Action: "Pushing",
}), ep, jsonRaw)
if err != nil {
return "", err
}
imgData.Checksum = checksum
imgData.ChecksumPayload = checksumPayload
// Send the checksum
if err := p.session.PushImageChecksumRegistry(imgData, ep); err != nil {
return "", err
}
p.out.Write(p.sf.FormatProgress(stringid.TruncateID(imgID), "Image successfully pushed", nil))
return imgData.Checksum, nil
}
// getV1ID returns the compatibilityID for the ID in the graph. compatibilityID
// is read from from the v1Compatibility config file in the disk.
func (p *v1Pusher) getV1ID(id string) (string, error) {
jsonData, err := p.getV1Config(id)
if err != nil {
return "", err
}
img, err := image.NewImgJSON(jsonData)
if err != nil {
return "", err
}
return img.ID, nil
}
// getV1Config returns v1Compatibility config for the image in the graph. If
// there is no v1Compatibility file on disk for the image
func (p *v1Pusher) getV1Config(id string) ([]byte, error) {
jsonData, err := p.graph.generateV1CompatibilityChain(id)
if err != nil {
return nil, err
}
return jsonData, nil
}

397
graph/push_v2.go
View file

@ -1,397 +0,0 @@
package graph
import (
"bufio"
"compress/gzip"
"encoding/json"
"errors"
"fmt"
"io"
"io/ioutil"
"github.com/Sirupsen/logrus"
"github.com/docker/distribution"
"github.com/docker/distribution/digest"
"github.com/docker/distribution/manifest"
"github.com/docker/distribution/manifest/schema1"
"github.com/docker/docker/image"
"github.com/docker/docker/pkg/progressreader"
"github.com/docker/docker/pkg/streamformatter"
"github.com/docker/docker/pkg/stringid"
"github.com/docker/docker/registry"
"github.com/docker/docker/runconfig"
"github.com/docker/docker/utils"
"golang.org/x/net/context"
)
const compressionBufSize = 32768
type v2Pusher struct {
*TagStore
endpoint registry.APIEndpoint
localRepo repository
repoInfo *registry.RepositoryInfo
config *ImagePushConfig
sf *streamformatter.StreamFormatter
repo distribution.Repository
// layersPushed is the set of layers known to exist on the remote side.
// This avoids redundant queries when pushing multiple tags that
// involve the same layers.
layersPushed map[digest.Digest]bool
}
func (p *v2Pusher) Push() (fallback bool, err error) {
p.repo, err = newV2Repository(p.repoInfo, p.endpoint, p.config.MetaHeaders, p.config.AuthConfig, "push", "pull")
if err != nil {
logrus.Debugf("Error getting v2 registry: %v", err)
return true, err
}
return false, p.pushV2Repository(p.config.Tag)
}
func (p *v2Pusher) getImageTags(askedTag string) ([]string, error) {
logrus.Debugf("Checking %q against %#v", askedTag, p.localRepo)
if len(askedTag) > 0 {
if _, ok := p.localRepo[askedTag]; !ok || utils.DigestReference(askedTag) {
return nil, fmt.Errorf("Tag does not exist for %s", askedTag)
}
return []string{askedTag}, nil
}
var tags []string
for tag := range p.localRepo {
if !utils.DigestReference(tag) {
tags = append(tags, tag)
}
}
return tags, nil
}
func (p *v2Pusher) pushV2Repository(tag string) error {
localName := p.repoInfo.LocalName
if _, found := p.poolAdd("push", localName); found {
return fmt.Errorf("push or pull %s is already in progress", localName)
}
defer p.poolRemove("push", localName)
tags, err := p.getImageTags(tag)
if err != nil {
return fmt.Errorf("error getting tags for %s: %s", localName, err)
}
if len(tags) == 0 {
return fmt.Errorf("no tags to push for %s", localName)
}
for _, tag := range tags {
if err := p.pushV2Tag(tag); err != nil {
return err
}
}
return nil
}
func (p *v2Pusher) pushV2Tag(tag string) error {
logrus.Debugf("Pushing repository: %s:%s", p.repo.Name(), tag)
layerID, exists := p.localRepo[tag]
if !exists {
return fmt.Errorf("tag does not exist: %s", tag)
}
layersSeen := make(map[string]bool)
layer, err := p.graph.Get(layerID)
if err != nil {
return err
}
m := &schema1.Manifest{
Versioned: manifest.Versioned{
SchemaVersion: 1,
},
Name: p.repo.Name(),
Tag: tag,
Architecture: layer.Architecture,
FSLayers: []schema1.FSLayer{},
History: []schema1.History{},
}
var metadata runconfig.Config
if layer != nil && layer.Config != nil {
metadata = *layer.Config
}
out := p.config.OutStream
for ; layer != nil; layer, err = p.graph.GetParent(layer) {
if err != nil {
return err
}
// break early if layer has already been seen in this image,
// this prevents infinite loops on layers which loopback, this
// cannot be prevented since layer IDs are not merkle hashes
// TODO(dmcgowan): throw error if no valid use case is found
if layersSeen[layer.ID] {
break
}
// Skip the base layer on Windows. This cannot be pushed.
if allowBaseParentImage && layer.Parent == "" {
break
}
logrus.Debugf("Pushing layer: %s", layer.ID)
if layer.Config != nil && metadata.Image != layer.ID {
if err := runconfig.Merge(&metadata, layer.Config); err != nil {
return err
}
}
var exists bool
dgst, err := p.graph.getLayerDigestWithLock(layer.ID)
switch err {
case nil:
if p.layersPushed[dgst] {
exists = true
// break out of switch, it is already known that
// the push is not needed and therefore doing a
// stat is unnecessary
break
}
_, err := p.repo.Blobs(context.Background()).Stat(context.Background(), dgst)
switch err {
case nil:
exists = true
out.Write(p.sf.FormatProgress(stringid.TruncateID(layer.ID), "Image already exists", nil))
case distribution.ErrBlobUnknown:
// nop
default:
out.Write(p.sf.FormatProgress(stringid.TruncateID(layer.ID), "Image push failed", nil))
return err
}
case errDigestNotSet:
// nop
case digest.ErrDigestInvalidFormat, digest.ErrDigestUnsupported:
return fmt.Errorf("error getting image checksum: %v", err)
}
// if digest was empty or not saved, or if blob does not exist on the remote repository,
// then fetch it.
if !exists {
var pushDigest digest.Digest
if pushDigest, err = p.pushV2Image(p.repo.Blobs(context.Background()), layer); err != nil {
return err
}
if dgst == "" {
// Cache new checksum
if err := p.graph.setLayerDigestWithLock(layer.ID, pushDigest); err != nil {
return err
}
}
dgst = pushDigest
}
// read v1Compatibility config, generate new if needed
jsonData, err := p.graph.generateV1CompatibilityChain(layer.ID)
if err != nil {
return err
}
m.FSLayers = append(m.FSLayers, schema1.FSLayer{BlobSum: dgst})
m.History = append(m.History, schema1.History{V1Compatibility: string(jsonData)})
layersSeen[layer.ID] = true
p.layersPushed[dgst] = true
}
// Fix parent chain if necessary
if err = fixHistory(m); err != nil {
return err
}
logrus.Infof("Signed manifest for %s:%s using daemon's key: %s", p.repo.Name(), tag, p.trustKey.KeyID())
signed, err := schema1.Sign(m, p.trustKey)
if err != nil {
return err
}
manifestDigest, manifestSize, err := digestFromManifest(signed, p.repo.Name())
if err != nil {
return err
}
if manifestDigest != "" {
out.Write(p.sf.FormatStatus("", "%s: digest: %s size: %d", tag, manifestDigest, manifestSize))
}
manSvc, err := p.repo.Manifests(context.Background())
if err != nil {
return err
}
return manSvc.Put(signed)
}
// fixHistory makes sure that the manifest has parent IDs that are consistent
// with its image IDs. Because local image IDs are generated from the
// configuration and filesystem contents, but IDs in the manifest are preserved
// from the original pull, it's possible to have inconsistencies where parent
// IDs don't match up with the other IDs in the manifest. This happens in the
// case where an engine pulls images where are identical except the IDs from the
// manifest - the local ID will be the same, and one of the v1Compatibility
// files gets discarded.
func fixHistory(m *schema1.Manifest) error {
var lastID string
for i := len(m.History) - 1; i >= 0; i-- {
var historyEntry map[string]*json.RawMessage
if err := json.Unmarshal([]byte(m.History[i].V1Compatibility), &historyEntry); err != nil {
return err
}
idJSON, present := historyEntry["id"]
if !present || idJSON == nil {
return errors.New("missing id key in v1compatibility file")
}
var id string
if err := json.Unmarshal(*idJSON, &id); err != nil {
return err
}
parentJSON, present := historyEntry["parent"]
if i == len(m.History)-1 {
// The base layer must not reference a parent layer,
// otherwise the manifest is incomplete. There is an
// exception for Windows to handle base layers.
if !allowBaseParentImage && present && parentJSON != nil {
var parent string
if err := json.Unmarshal(*parentJSON, &parent); err != nil {
return err
}
if parent != "" {
logrus.Debugf("parent id mismatch detected; fixing. parent reference: %s", parent)
delete(historyEntry, "parent")
fixedHistory, err := json.Marshal(historyEntry)
if err != nil {
return err
}
m.History[i].V1Compatibility = string(fixedHistory)
}
}
} else {
// For all other layers, the parent ID should equal the
// ID of the next item in the history list. If it
// doesn't, fix it up (but preserve all other fields,
// possibly including fields that aren't known to this
// engine version).
if !present || parentJSON == nil {
return errors.New("missing parent key in v1compatibility file")
}
var parent string
if err := json.Unmarshal(*parentJSON, &parent); err != nil {
return err
}
if parent != lastID {
logrus.Debugf("parent id mismatch detected; fixing. parent reference: %s actual id: %s", parent, id)
historyEntry["parent"] = rawJSON(lastID)
fixedHistory, err := json.Marshal(historyEntry)
if err != nil {
return err
}
m.History[i].V1Compatibility = string(fixedHistory)
}
}
lastID = id
}
return nil
}
func rawJSON(value interface{}) *json.RawMessage {
jsonval, err := json.Marshal(value)
if err != nil {
return nil
}
return (*json.RawMessage)(&jsonval)
}
func (p *v2Pusher) pushV2Image(bs distribution.BlobService, img *image.Image) (digest.Digest, error) {
out := p.config.OutStream
out.Write(p.sf.FormatProgress(stringid.TruncateID(img.ID), "Preparing", nil))
image, err := p.graph.Get(img.ID)
if err != nil {
return "", err
}
arch, err := p.graph.tarLayer(image)
if err != nil {
return "", err
}
defer arch.Close()
// Send the layer
layerUpload, err := bs.Create(context.Background())
if err != nil {
return "", err
}
defer layerUpload.Close()
reader := progressreader.New(progressreader.Config{
In: ioutil.NopCloser(arch), // we'll take care of close here.
Out: out,
Formatter: p.sf,
// TODO(stevvooe): This may cause a size reporting error. Try to get
// this from tar-split or elsewhere. The main issue here is that we
// don't want to buffer to disk *just* to calculate the size.
Size: img.Size,
NewLines: false,
ID: stringid.TruncateID(img.ID),
Action: "Pushing",
})
digester := digest.Canonical.New()
// HACK: The MultiWriter doesn't write directly to layerUpload because
// we must make sure the ReadFrom is used, not Write. Using Write would
// send a PATCH request for every Write call.
pipeReader, pipeWriter := io.Pipe()
// Use a bufio.Writer to avoid excessive chunking in HTTP request.
bufWriter := bufio.NewWriterSize(io.MultiWriter(pipeWriter, digester.Hash()), compressionBufSize)
compressor := gzip.NewWriter(bufWriter)
go func() {
_, err := io.Copy(compressor, reader)
if err == nil {
err = compressor.Close()
}
if err == nil {
err = bufWriter.Flush()
}
if err != nil {
pipeWriter.CloseWithError(err)
} else {
pipeWriter.Close()
}
}()
out.Write(p.sf.FormatProgress(stringid.TruncateID(img.ID), "Pushing", nil))
nn, err := layerUpload.ReadFrom(pipeReader)
pipeReader.Close()
if err != nil {
return "", err
}
dgst := digester.Digest()
if _, err := layerUpload.Commit(context.Background(), distribution.Descriptor{Digest: dgst}); err != nil {
return "", err
}
logrus.Debugf("uploaded layer %s (%s), %d bytes", img.ID, dgst, nn)
out.Write(p.sf.FormatProgress(stringid.TruncateID(img.ID), "Pushed", nil))
return dgst, nil
}

89
graph/service.go
View file

@ -1,89 +0,0 @@
package graph
import (
"fmt"
"io"
"runtime"
"time"
"github.com/Sirupsen/logrus"
"github.com/docker/docker/api/types"
"github.com/docker/docker/utils"
)
// Lookup looks up an image by name in a TagStore and returns it as an
// ImageInspect structure.
func (s *TagStore) Lookup(name string) (*types.ImageInspect, error) {
image, err := s.LookupImage(name)
if err != nil || image == nil {
return nil, fmt.Errorf("No such image: %s", name)
}
var repoTags = make([]string, 0)
var repoDigests = make([]string, 0)
s.Lock()
for repoName, repository := range s.Repositories {
for ref, id := range repository {
if id == image.ID {
imgRef := utils.ImageReference(repoName, ref)
if utils.DigestReference(ref) {
repoDigests = append(repoDigests, imgRef)
} else {
repoTags = append(repoTags, imgRef)
}
}
}
}
s.Unlock()
imageInspect := &types.ImageInspect{
ID: image.ID,
RepoTags: repoTags,
RepoDigests: repoDigests,
Parent: image.Parent,
Comment: image.Comment,
Created: image.Created.Format(time.RFC3339Nano),
Container: image.Container,
ContainerConfig: &image.ContainerConfig,
DockerVersion: image.DockerVersion,
Author: image.Author,
Config: image.Config,
Architecture: image.Architecture,
Os: image.OS,
Size: image.Size,
VirtualSize: s.graph.getParentsSize(image) + image.Size,
}
imageInspect.GraphDriver.Name = s.graph.driver.String()
graphDriverData, err := s.graph.driver.GetMetadata(image.ID)
if err != nil {
return nil, err
}
imageInspect.GraphDriver.Data = graphDriverData
return imageInspect, nil
}
// imageTarLayer return the tarLayer of the image
func (s *TagStore) imageTarLayer(name string, dest io.Writer) error {
if image, err := s.LookupImage(name); err == nil && image != nil {
// On Windows, the base layer cannot be exported
if runtime.GOOS != "windows" || image.Parent != "" {
fs, err := s.graph.tarLayer(image)
if err != nil {
return err
}
defer fs.Close()
written, err := io.Copy(dest, fs)
if err != nil {
return err
}
logrus.Debugf("rendered layer for %s of [%d] size", image.ID, written)
}
return nil
}
return fmt.Errorf("No such image: %s", name)
}

431
graph/tags.go
View file

@ -1,431 +0,0 @@
package graph
import (
"encoding/json"
"errors"
"fmt"
"io"
"io/ioutil"
"os"
"path/filepath"
"sort"
"strings"
"sync"
"github.com/docker/distribution/digest"
"github.com/docker/docker/daemon/events"
"github.com/docker/docker/graph/tags"
"github.com/docker/docker/image"
"github.com/docker/docker/pkg/broadcaster"
"github.com/docker/docker/pkg/parsers"
"github.com/docker/docker/pkg/stringid"
"github.com/docker/docker/registry"
"github.com/docker/docker/utils"
"github.com/docker/libtrust"
)
// ErrNameIsNotExist returned when there is no image with requested name.
var ErrNameIsNotExist = errors.New("image with specified name does not exist")
// TagStore manages repositories. It encompasses the Graph used for versioned
// storage, as well as various services involved in pushing and pulling
// repositories.
type TagStore struct {
path string
graph *Graph
// Repositories is a map of repositories, indexed by name.
Repositories map[string]repository
trustKey libtrust.PrivateKey
sync.Mutex
// FIXME: move push/pull-related fields
// to a helper type
pullingPool map[string]*broadcaster.Buffered
pushingPool map[string]*broadcaster.Buffered
registryService *registry.Service
eventsService *events.Events
}
// repository maps tags to image IDs.
type repository map[string]string
// TagStoreConfig provides parameters for a new TagStore.
type TagStoreConfig struct {
// Graph is the versioned image store
Graph *Graph
// Key is the private key to use for signing manifests.
Key libtrust.PrivateKey
// Registry is the registry service to use for TLS configuration and
// endpoint lookup.
Registry *registry.Service
// Events is the events service to use for logging.
Events *events.Events
}
// NewTagStore creates a new TagStore at specified path, using the parameters
// and services provided in cfg.
func NewTagStore(path string, cfg *TagStoreConfig) (*TagStore, error) {
abspath, err := filepath.Abs(path)
if err != nil {
return nil, err
}
store := &TagStore{
path: abspath,
graph: cfg.Graph,
trustKey: cfg.Key,
Repositories: make(map[string]repository),
pullingPool: make(map[string]*broadcaster.Buffered),
pushingPool: make(map[string]*broadcaster.Buffered),
registryService: cfg.Registry,
eventsService: cfg.Events,
}
// Load the json file if it exists, otherwise create it.
if err := store.reload(); os.IsNotExist(err) {
if err := store.save(); err != nil {
return nil, err
}
} else if err != nil {
return nil, err
}
return store, nil
}
func (store *TagStore) save() error {
// Store the json ball
jsonData, err := json.Marshal(store)
if err != nil {
return err
}
if err := ioutil.WriteFile(store.path, jsonData, 0600); err != nil {
return err
}
return nil
}
func (store *TagStore) reload() error {
f, err := os.Open(store.path)
if err != nil {
return err
}
defer f.Close()
if err := json.NewDecoder(f).Decode(&store); err != nil {
return err
}
return nil
}
// LookupImage returns pointer to an Image struct corresponding to the given
// name. The name can include an optional tag; otherwise the default tag will
// be used.
func (store *TagStore) LookupImage(name string) (*image.Image, error) {
repoName, ref := parsers.ParseRepositoryTag(name)
if ref == "" {
ref = tags.DefaultTag
}
var (
err error
img *image.Image
)
img, err = store.getImage(repoName, ref)
if err != nil {
return nil, err
}
if img != nil {
return img, nil
}
// name must be an image ID.
store.Lock()
defer store.Unlock()
if img, err = store.graph.Get(name); err != nil {
return nil, err
}
return img, nil
}
// GetID returns ID for image name.
func (store *TagStore) GetID(name string) (string, error) {
repoName, ref := parsers.ParseRepositoryTag(name)
if ref == "" {
ref = tags.DefaultTag
}
store.Lock()
defer store.Unlock()
repoName = registry.NormalizeLocalName(repoName)
repo, ok := store.Repositories[repoName]
if !ok {
return "", ErrNameIsNotExist
}
id, ok := repo[ref]
if !ok {
return "", ErrNameIsNotExist
}
return id, nil
}
// ByID returns a reverse-lookup table of all the names which refer to each
// image - e.g. {"43b5f19b10584": {"base:latest", "base:v1"}}
func (store *TagStore) ByID() map[string][]string {
store.Lock()
defer store.Unlock()
byID := make(map[string][]string)
for repoName, repository := range store.Repositories {
for tag, id := range repository {
name := utils.ImageReference(repoName, tag)
if _, exists := byID[id]; !exists {
byID[id] = []string{name}
} else {
byID[id] = append(byID[id], name)
sort.Strings(byID[id])
}
}
}
return byID
}
// HasReferences returns whether or not the given image is referenced in one or
// more repositories.
func (store *TagStore) HasReferences(img *image.Image) bool {
return len(store.ByID()[img.ID]) > 0
}
// Delete deletes a repository or a specific tag. If ref is empty, the entire
// repository named repoName will be deleted; otherwise only the tag named by
// ref will be deleted.
func (store *TagStore) Delete(repoName, ref string) (bool, error) {
store.Lock()
defer store.Unlock()
deleted := false
if err := store.reload(); err != nil {
return false, err
}
if ref == "" {
// Delete the whole repository.
delete(store.Repositories, repoName)
return true, store.save()
}
repoRefs, exists := store.Repositories[repoName]
if !exists {
return false, fmt.Errorf("No such repository: %s", repoName)
}
if _, exists := repoRefs[ref]; exists {
delete(repoRefs, ref)
if len(repoRefs) == 0 {
delete(store.Repositories, repoName)
}
deleted = true
}
return deleted, store.save()
}
// Tag creates a tag in the repository reponame, pointing to the image named
// imageName. If force is true, an existing tag with the same name may be
// overwritten.
func (store *TagStore) Tag(repoName, tag, imageName string, force bool) error {
return store.setLoad(repoName, tag, imageName, force, nil)
}
// setLoad stores the image to the store.
// If the imageName is already in the repo then a '-f' flag should be used to replace existing image.
func (store *TagStore) setLoad(repoName, tag, imageName string, force bool, out io.Writer) error {
img, err := store.LookupImage(imageName)
store.Lock()
defer store.Unlock()
if err != nil {
return err
}
if tag == "" {
tag = tags.DefaultTag
}
if err := validateRepoName(repoName); err != nil {
return err
}
if err := tags.ValidateTagName(tag); err != nil {
return err
}
if err := store.reload(); err != nil {
return err
}
var repo repository
repoName = registry.NormalizeLocalName(repoName)
if r, exists := store.Repositories[repoName]; exists {
repo = r
if old, exists := store.Repositories[repoName][tag]; exists {
if !force {
return fmt.Errorf("Conflict: Tag %s:%s is already set to image %s, if you want to replace it, please use -f option", repoName, tag, old[:12])
}
if old != img.ID && out != nil {
fmt.Fprintf(out, "The image %s:%s already exists, renaming the old one with ID %s to empty string\n", repoName, tag, old[:12])
}
}
} else {
repo = make(map[string]string)
store.Repositories[repoName] = repo
}
repo[tag] = img.ID
return store.save()
}
// setDigest creates a digest reference to an image ID.
func (store *TagStore) setDigest(repoName, digest, imageName string) error {
img, err := store.LookupImage(imageName)
if err != nil {
return err
}
if err := validateRepoName(repoName); err != nil {
return err
}
if err := validateDigest(digest); err != nil {
return err
}
store.Lock()
defer store.Unlock()
if err := store.reload(); err != nil {
return err
}
repoName = registry.NormalizeLocalName(repoName)
repoRefs, exists := store.Repositories[repoName]
if !exists {
repoRefs = repository{}
store.Repositories[repoName] = repoRefs
} else if oldID, exists := repoRefs[digest]; exists && oldID != img.ID {
return fmt.Errorf("Conflict: Digest %s is already set to image %s", digest, oldID)
}
repoRefs[digest] = img.ID
return store.save()
}
// get returns the repository tag/image map for a given repository.
func (store *TagStore) get(repoName string) (repository, error) {
store.Lock()
defer store.Unlock()
if err := store.reload(); err != nil {
return nil, err
}
repoName = registry.NormalizeLocalName(repoName)
if r, exists := store.Repositories[repoName]; exists {
return r, nil
}
return nil, nil
}
// getImage returns a pointer to an Image structure describing the image
// referred to by refOrID inside repository repoName.
func (store *TagStore) getImage(repoName, refOrID string) (*image.Image, error) {
repo, err := store.get(repoName)
if err != nil {
return nil, err
}
if repo == nil {
return nil, nil
}
store.Lock()
defer store.Unlock()
if imgID, exists := repo[refOrID]; exists {
return store.graph.Get(imgID)
}
// If no matching tag is found, search through images for a matching image id
// iff it looks like a short ID or would look like a short ID
if stringid.IsShortID(stringid.TruncateID(refOrID)) {
for _, revision := range repo {
if strings.HasPrefix(revision, refOrID) {
return store.graph.Get(revision)
}
}
}
return nil, nil
}
// validateRepoName validates the name of a repository.
func validateRepoName(name string) error {
if name == "" {
return fmt.Errorf("Repository name can't be empty")
}
if name == "scratch" {
return fmt.Errorf("'scratch' is a reserved name")
}
return nil
}
func validateDigest(dgst string) error {
if dgst == "" {
return errors.New("digest can't be empty")
}
if _, err := digest.ParseDigest(dgst); err != nil {
return err
}
return nil
}
// poolAdd checks if a push or pull is already running, and returns
// (broadcaster, true) if a running operation is found. Otherwise, it creates a
// new one and returns (broadcaster, false).
func (store *TagStore) poolAdd(kind, key string) (*broadcaster.Buffered, bool) {
store.Lock()
defer store.Unlock()
if p, exists := store.pullingPool[key]; exists {
return p, true
}
if p, exists := store.pushingPool[key]; exists {
return p, true
}
broadcaster := broadcaster.NewBuffered()
switch kind {
case "pull":
store.pullingPool[key] = broadcaster
case "push":
store.pushingPool[key] = broadcaster
default:
panic("Unknown pool type")
}
return broadcaster, false
}
func (store *TagStore) poolRemoveWithError(kind, key string, broadcasterResult error) error {
store.Lock()
defer store.Unlock()
switch kind {
case "pull":
if broadcaster, exists := store.pullingPool[key]; exists {
broadcaster.CloseWithError(broadcasterResult)
delete(store.pullingPool, key)
}
case "push":
if broadcaster, exists := store.pushingPool[key]; exists {
broadcaster.CloseWithError(broadcasterResult)
delete(store.pushingPool, key)
}
default:
return fmt.Errorf("Unknown pool type")
}
return nil
}
func (store *TagStore) poolRemove(kind, key string) error {
return store.poolRemoveWithError(kind, key, nil)
}

36
graph/tags/tags.go
View file

@ -1,36 +0,0 @@
package tags
import (
"fmt"
"regexp"
"github.com/docker/distribution/reference"
)
// DefaultTag defines the default tag used when performing images related actions and no tag string is specified
const DefaultTag = "latest"
var anchoredTagRegexp = regexp.MustCompile(`^` + reference.TagRegexp.String() + `$`)
// ErrTagInvalidFormat is returned if tag is invalid.
type ErrTagInvalidFormat struct {
name string
}
func (e ErrTagInvalidFormat) Error() string {
return fmt.Sprintf("Illegal tag name (%s): only [A-Za-z0-9_.-] are allowed ('.' and '-' are NOT allowed in the initial), minimum 1, maximum 128 in length", e.name)
}
// ValidateTagName validates the name of a tag.
// It returns an error if the given name is an emtpy string.
// If name is not valid, it returns ErrTagInvalidFormat
func ValidateTagName(name string) error {
if name == "" {
return fmt.Errorf("tag name can't be empty")
}
if !anchoredTagRegexp.MatchString(name) {
return ErrTagInvalidFormat{name}
}
return nil
}

23
graph/tags/tags_unit_test.go
View file

@ -1,23 +0,0 @@
package tags
import (
"testing"
)
func TestValidTagName(t *testing.T) {
validTags := []string{"9", "foo", "foo-test", "bar.baz.boo"}
for _, tag := range validTags {
if err := ValidateTagName(tag); err != nil {
t.Errorf("'%s' should've been a valid tag", tag)
}
}
}
func TestInvalidTagName(t *testing.T) {
inValidTags := []string{"-9", ".foo", "-test", ".", "-"}
for _, tag := range inValidTags {
if err := ValidateTagName(tag); err == nil {
t.Errorf("'%s' should've been an invalid tag", tag)
}
}
}

205
graph/tags_unit_test.go
View file

@ -1,205 +0,0 @@
package graph
import (
"archive/tar"
"bytes"
"io"
"os"
"path"
"testing"
"github.com/docker/docker/daemon/events"
"github.com/docker/docker/daemon/graphdriver"
_ "github.com/docker/docker/daemon/graphdriver/vfs" // import the vfs driver so it is used in the tests
"github.com/docker/docker/graph/tags"
"github.com/docker/docker/image"
"github.com/docker/docker/utils"
)
const (
testOfficialImageName = "myapp"
testOfficialImageID = "1a2d3c4d4e5fa2d2a21acea242a5e2345d3aefc3e7dfa2a2a2a21a2a2ad2d234"
testOfficialImageIDShort = "1a2d3c4d4e5f"
testPrivateImageName = "127.0.0.1:8000/privateapp"
testPrivateImageID = "5bc255f8699e4ee89ac4469266c3d11515da88fdcbde45d7b069b636ff4efd81"
testPrivateImageIDShort = "5bc255f8699e"
testPrivateImageDigest = "sha256:bc8813ea7b3603864987522f02a76101c17ad122e1c46d790efc0fca78ca7bfb"
testPrivateImageTag = "sometag"
)
func fakeTar() (io.Reader, error) {
uid := os.Getuid()
gid := os.Getgid()
content := []byte("Hello world!\n")
buf := new(bytes.Buffer)
tw := tar.NewWriter(buf)
for _, name := range []string{"/etc/postgres/postgres.conf", "/etc/passwd", "/var/log/postgres/postgres.conf"} {
hdr := new(tar.Header)
// Leaving these fields blank requires root privileges
hdr.Uid = uid
hdr.Gid = gid
hdr.Size = int64(len(content))
hdr.Name = name
if err := tw.WriteHeader(hdr); err != nil {
return nil, err
}
tw.Write([]byte(content))
}
tw.Close()
return buf, nil
}
func mkTestTagStore(root string, t *testing.T) *TagStore {
driver, err := graphdriver.New(root, nil, nil, nil)
if err != nil {
t.Fatal(err)
}
graph, err := NewGraph(root, driver, nil, nil)
if err != nil {
t.Fatal(err)
}
tagCfg := &TagStoreConfig{
Graph: graph,
Events: events.New(),
}
store, err := NewTagStore(path.Join(root, "tags"), tagCfg)
if err != nil {
t.Fatal(err)
}
officialArchive, err := fakeTar()
if err != nil {
t.Fatal(err)
}
img := &image.Image{ID: testOfficialImageID}
if err := graph.Register(v1Descriptor{img}, officialArchive); err != nil {
t.Fatal(err)
}
if err := store.Tag(testOfficialImageName, "", testOfficialImageID, false); err != nil {
t.Fatal(err)
}
privateArchive, err := fakeTar()
if err != nil {
t.Fatal(err)
}
img = &image.Image{ID: testPrivateImageID}
if err := graph.Register(v1Descriptor{img}, privateArchive); err != nil {
t.Fatal(err)
}
if err := store.Tag(testPrivateImageName, "", testPrivateImageID, false); err != nil {
t.Fatal(err)
}
if err := store.setDigest(testPrivateImageName, testPrivateImageDigest, testPrivateImageID); err != nil {
t.Fatal(err)
}
return store
}
func TestLookupImage(t *testing.T) {
tmp, err := utils.TestDirectory("")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(tmp)
store := mkTestTagStore(tmp, t)
defer store.graph.driver.Cleanup()
officialLookups := []string{
testOfficialImageID,
testOfficialImageIDShort,
testOfficialImageName + ":" + testOfficialImageID,
testOfficialImageName + ":" + testOfficialImageIDShort,
testOfficialImageName,
testOfficialImageName + ":" + tags.DefaultTag,
"docker.io/" + testOfficialImageName,
"docker.io/" + testOfficialImageName + ":" + tags.DefaultTag,
"index.docker.io/" + testOfficialImageName,
"index.docker.io/" + testOfficialImageName + ":" + tags.DefaultTag,
"library/" + testOfficialImageName,
"library/" + testOfficialImageName + ":" + tags.DefaultTag,
"docker.io/library/" + testOfficialImageName,
"docker.io/library/" + testOfficialImageName + ":" + tags.DefaultTag,
"index.docker.io/library/" + testOfficialImageName,
"index.docker.io/library/" + testOfficialImageName + ":" + tags.DefaultTag,
}
privateLookups := []string{
testPrivateImageID,
testPrivateImageIDShort,
testPrivateImageName + ":" + testPrivateImageID,
testPrivateImageName + ":" + testPrivateImageIDShort,
testPrivateImageName,
testPrivateImageName + ":" + tags.DefaultTag,
}
invalidLookups := []string{
testOfficialImageName + ":" + "fail",
"fail:fail",
}
digestLookups := []string{
testPrivateImageName + "@" + testPrivateImageDigest,
}
for _, name := range officialLookups {
if img, err := store.LookupImage(name); err != nil {
t.Errorf("Error looking up %s: %s", name, err)
} else if img == nil {
t.Errorf("Expected 1 image, none found: %s", name)
} else if img.ID != testOfficialImageID {
t.Errorf("Expected ID '%s' found '%s'", testOfficialImageID, img.ID)
}
}
for _, name := range privateLookups {
if img, err := store.LookupImage(name); err != nil {
t.Errorf("Error looking up %s: %s", name, err)
} else if img == nil {
t.Errorf("Expected 1 image, none found: %s", name)
} else if img.ID != testPrivateImageID {
t.Errorf("Expected ID '%s' found '%s'", testPrivateImageID, img.ID)
}
}
for _, name := range invalidLookups {
if img, err := store.LookupImage(name); err == nil {
t.Errorf("Expected error, none found: %s", name)
} else if img != nil {
t.Errorf("Expected 0 image, 1 found: %s", name)
}
}
for _, name := range digestLookups {
if img, err := store.LookupImage(name); err != nil {
t.Errorf("Error looking up %s: %s", name, err)
} else if img == nil {
t.Errorf("Expected 1 image, none found: %s", name)
} else if img.ID != testPrivateImageID {
t.Errorf("Expected ID '%s' found '%s'", testPrivateImageID, img.ID)
}
}
}
func TestValidateDigest(t *testing.T) {
tests := []struct {
input string
expectError bool
}{
{"", true},
{"latest", true},
{"sha256:b", false},
{"tarsum+v1+sha256:bY852-_.+=", false},
{"#$%#$^:$%^#$%", true},
}
for i, test := range tests {
err := validateDigest(test.input)
gotError := err != nil
if e, a := test.expectError, gotError; e != a {
t.Errorf("%d: with input %s, expected error=%t, got %t: %s", i, test.input, test.expectError, gotError, err)
}
}
}

1
image/fixtures/post1.9/expected_computed_id
View file

@ -1 +0,0 @@
sha256:f2722a8ec6926e02fa9f2674072cbc2a25cf0f449f27350f613cd843b02c9105

1
image/fixtures/post1.9/expected_config
View file

@ -1 +0,0 @@
{"architecture":"amd64","config":{"Hostname":"fb1f7270da95","Domainname":"","User":"","AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":["foo=bar"],"Cmd":null,"Image":"361a94d06b2b781b2f1ee6c72e1cbbfbbd032a103e26a3db75b431743829ae4f","Volumes":null,"VolumeDriver":"","WorkingDir":"","Entrypoint":null,"OnBuild":null,"Labels":null},"container":"fb1f7270da9519308361b99dc8e0d30f12c24dfd28537c2337ece995ac853a16","container_config":{"Hostname":"fb1f7270da95","Domainname":"","User":"","AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":["foo=bar"],"Cmd":["/bin/sh","-c","#(nop) ADD file:11998b2a4d664a75cd0c3f4e4cb1837434e0f997ba157a0ac1d3c68a07aa2f4f in /"],"Image":"361a94d06b2b781b2f1ee6c72e1cbbfbbd032a103e26a3db75b431743829ae4f","Volumes":null,"VolumeDriver":"","WorkingDir":"","Entrypoint":null,"OnBuild":null,"Labels":null},"created":"2015-09-08T21:30:30.807853054Z","docker_version":"1.9.0-dev","layer_id":"sha256:31176893850e05d308cdbfef88877e460d50c8063883fb13eb5753097da6422a","os":"linux","parent_id":"sha256:ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02"}

1
image/fixtures/post1.9/layer_id
View file

@ -1 +0,0 @@
sha256:31176893850e05d308cdbfef88877e460d50c8063883fb13eb5753097da6422a

1
image/fixtures/post1.9/parent_id
View file

@ -1 +0,0 @@
sha256:ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02

1
image/fixtures/post1.9/v1compatibility
View file

@ -1 +0,0 @@
{"id":"8dfb96b5d09e6cf6f376d81f1e2770ee5ede309f9bd9e079688c9782649ab326","parent":"361a94d06b2b781b2f1ee6c72e1cbbfbbd032a103e26a3db75b431743829ae4f","created":"2015-09-08T21:30:30.807853054Z","container":"fb1f7270da9519308361b99dc8e0d30f12c24dfd28537c2337ece995ac853a16","container_config":{"Hostname":"fb1f7270da95","Domainname":"","User":"","AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":["foo=bar"],"Cmd":["/bin/sh","-c","#(nop) ADD file:11998b2a4d664a75cd0c3f4e4cb1837434e0f997ba157a0ac1d3c68a07aa2f4f in /"],"Image":"361a94d06b2b781b2f1ee6c72e1cbbfbbd032a103e26a3db75b431743829ae4f","Volumes":null,"VolumeDriver":"","WorkingDir":"","Entrypoint":null,"OnBuild":null,"Labels":null},"docker_version":"1.9.0-dev","config":{"Hostname":"fb1f7270da95","Domainname":"","User":"","AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":["foo=bar"],"Cmd":null,"Image":"361a94d06b2b781b2f1ee6c72e1cbbfbbd032a103e26a3db75b431743829ae4f","Volumes":null,"VolumeDriver":"","WorkingDir":"","Entrypoint":null,"OnBuild":null,"Labels":null},"architecture":"amd64","os":"linux"}

1
image/fixtures/pre1.9/expected_computed_id
View file

@ -1 +0,0 @@
sha256:fd6ebfedda8ea140a9380767e15bd32c6e899303cfe34bc4580c931f2f816f89

2
image/fixtures/pre1.9/expected_config
View file

@ -1,2 +0,0 @@
{"architecture":"amd64","config":{"Hostname":"03797203757d","Domainname":"","User":"","AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":["PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin","GOLANG_VERSION=1.4.1","GOPATH=/go"],"Cmd":null,"Image":"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02","Volumes":null,"WorkingDir":"/go","Entrypoint":["/go/bin/dnsdock"],"OnBuild":[],"Labels":{}},"container":"d91be3479d5b1e84b0c00d18eea9dc777ca0ad166d51174b24283e2e6f104253","container_config":{"Hostname":"03797203757d","Domainname":"","User":"","AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":["PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin","GOLANG_VERSION=1.4.1","GOPATH=/go"],"Cmd":["/bin/sh","-c","#(nop) ENTRYPOINT [\"/go/bin/dnsdock\"]"],"Image":"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02","Volumes":null,"WorkingDir":"/go","Entrypoint":["/go/bin/dnsdock"],"OnBuild":[],"Labels":{}},"created":"2015-08-19T16:49:11.368300679Z","docker_version":"1.6.2","layer_id":"sha256:31176893850e05d308cdbfef88877e460d50c8063883fb13eb5753097da6422a","os":"linux","parent_id":"sha256:ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02"}

1
image/fixtures/pre1.9/layer_id
View file

@ -1 +0,0 @@
sha256:31176893850e05d308cdbfef88877e460d50c8063883fb13eb5753097da6422a

1
image/fixtures/pre1.9/parent_id
View file

@ -1 +0,0 @@
sha256:ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02

1
image/fixtures/pre1.9/v1compatibility
View file

@ -1 +0,0 @@
{"id":"3b38edc92eb7c074812e217b41a6ade66888531009d6286a6f5f36a06f9841b9","parent":"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02","created":"2015-08-19T16:49:11.368300679Z","container":"d91be3479d5b1e84b0c00d18eea9dc777ca0ad166d51174b24283e2e6f104253","container_config":{"Hostname":"03797203757d","Domainname":"","User":"","Memory":0,"MemorySwap":0,"CpuShares":0,"Cpuset":"","AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"PortSpecs":null,"ExposedPorts":null,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":["PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin","GOLANG_VERSION=1.4.1","GOPATH=/go"],"Cmd":["/bin/sh","-c","#(nop) ENTRYPOINT [\"/go/bin/dnsdock\"]"],"Image":"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02","Volumes":null,"WorkingDir":"/go","Entrypoint":["/go/bin/dnsdock"],"NetworkDisabled":false,"MacAddress":"","OnBuild":[],"Labels":{}},"docker_version":"1.6.2","config":{"Hostname":"03797203757d","Domainname":"","User":"","Memory":0,"MemorySwap":0,"CpuShares":0,"Cpuset":"","AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"PortSpecs":null,"ExposedPorts":null,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":["PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin","GOLANG_VERSION=1.4.1","GOPATH=/go"],"Cmd":null,"Image":"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02","Volumes":null,"WorkingDir":"/go","Entrypoint":["/go/bin/dnsdock"],"NetworkDisabled":false,"MacAddress":"","OnBuild":[],"Labels":{}},"architecture":"amd64","os":"linux","Size":0}

192
image/fs.go Normal file
View file

@ -0,0 +1,192 @@
package image
import (
"fmt"
"io/ioutil"
"os"
"path/filepath"
"sync"
"github.com/Sirupsen/logrus"
"github.com/docker/distribution/digest"
)
// IDWalkFunc is function called by StoreBackend.Walk
type IDWalkFunc func(id ID) error
// StoreBackend provides interface for image.Store persistence
type StoreBackend interface {
Walk(f IDWalkFunc) error
Get(id ID) ([]byte, error)
Set(data []byte) (ID, error)
Delete(id ID) error
SetMetadata(id ID, key string, data []byte) error
GetMetadata(id ID, key string) ([]byte, error)
DeleteMetadata(id ID, key string) error
}
// fs implements StoreBackend using the filesystem.
type fs struct {
sync.RWMutex
root string
}
const (
contentDirName = "content"
metadataDirName = "metadata"
)
// NewFSStoreBackend returns new filesystem based backend for image.Store
func NewFSStoreBackend(root string) (StoreBackend, error) {
return newFSStore(root)
}
func newFSStore(root string) (*fs, error) {
s := &fs{
root: root,
}
if err := os.MkdirAll(filepath.Join(root, contentDirName, string(digest.Canonical)), 0700); err != nil {
return nil, err
}
if err := os.MkdirAll(filepath.Join(root, metadataDirName, string(digest.Canonical)), 0700); err != nil {
return nil, err
}
return s, nil
}
func (s *fs) contentFile(id ID) string {
dgst := digest.Digest(id)
return filepath.Join(s.root, contentDirName, string(dgst.Algorithm()), dgst.Hex())
}
func (s *fs) metadataDir(id ID) string {
dgst := digest.Digest(id)
return filepath.Join(s.root, metadataDirName, string(dgst.Algorithm()), dgst.Hex())
}
// Walk calls the supplied callback for each image ID in the storage backend.
func (s *fs) Walk(f IDWalkFunc) error {
// Only Canonical digest (sha256) is currently supported
s.RLock()
dir, err := ioutil.ReadDir(filepath.Join(s.root, contentDirName, string(digest.Canonical)))
s.RUnlock()
if err != nil {
return err
}
for _, v := range dir {
dgst := digest.NewDigestFromHex(string(digest.Canonical), v.Name())
if err := dgst.Validate(); err != nil {
logrus.Debugf("Skipping invalid digest %s: %s", dgst, err)
continue
}
if err := f(ID(dgst)); err != nil {
return err
}
}
return nil
}
// Get returns the content stored under a given ID.
func (s *fs) Get(id ID) ([]byte, error) {
s.RLock()
defer s.RUnlock()
return s.get(id)
}
func (s *fs) get(id ID) ([]byte, error) {
content, err := ioutil.ReadFile(s.contentFile(id))
if err != nil {
return nil, err
}
// todo: maybe optional
validated, err := digest.FromBytes(content)
if err != nil {
return nil, err
}
if ID(validated) != id {
return nil, fmt.Errorf("failed to verify image: %v", id)
}
return content, nil
}
// Set stores content under a given ID.
func (s *fs) Set(data []byte) (ID, error) {
s.Lock()
defer s.Unlock()
if len(data) == 0 {
return "", fmt.Errorf("Invalid empty data")
}
dgst, err := digest.FromBytes(data)
if err != nil {
return "", err
}
id := ID(dgst)
filePath := s.contentFile(id)
tempFilePath := s.contentFile(id) + ".tmp"
if err := ioutil.WriteFile(tempFilePath, data, 0600); err != nil {
return "", err
}
if err := os.Rename(tempFilePath, filePath); err != nil {
return "", err
}
return id, nil
}
// Delete removes content and metadata files associated with the ID.
func (s *fs) Delete(id ID) error {
s.Lock()
defer s.Unlock()
if err := os.RemoveAll(s.metadataDir(id)); err != nil {
return err
}
if err := os.Remove(s.contentFile(id)); err != nil {
return err
}
return nil
}
// SetMetadata sets metadata for a given ID. It fails if there's no base file.
func (s *fs) SetMetadata(id ID, key string, data []byte) error {
s.Lock()
defer s.Unlock()
if _, err := s.get(id); err != nil {
return err
}
baseDir := filepath.Join(s.metadataDir(id))
if err := os.MkdirAll(baseDir, 0700); err != nil {
return err
}
filePath := filepath.Join(s.metadataDir(id), key)
tempFilePath := filePath + ".tmp"
if err := ioutil.WriteFile(tempFilePath, data, 0600); err != nil {
return err
}
return os.Rename(tempFilePath, filePath)
}
// GetMetadata returns metadata for a given ID.
func (s *fs) GetMetadata(id ID, key string) ([]byte, error) {
s.RLock()
defer s.RUnlock()
if _, err := s.get(id); err != nil {
return nil, err
}
return ioutil.ReadFile(filepath.Join(s.metadataDir(id), key))
}
// DeleteMetadata removes the metadata associated with an ID.
func (s *fs) DeleteMetadata(id ID, key string) error {
s.Lock()
defer s.Unlock()
return os.RemoveAll(filepath.Join(s.metadataDir(id), key))
}

391
image/fs_test.go Normal file
View file

@ -0,0 +1,391 @@
package image
import (
"bytes"
"crypto/rand"
"crypto/sha256"
"encoding/hex"
"errors"
"io/ioutil"
"os"
"path/filepath"
"testing"
"github.com/docker/distribution/digest"
)
func TestFSGetSet(t *testing.T) {
tmpdir, err := ioutil.TempDir("", "images-fs-store")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(tmpdir)
fs, err := NewFSStoreBackend(tmpdir)
if err != nil {
t.Fatal(err)
}
testGetSet(t, fs)
}
func TestFSGetInvalidData(t *testing.T) {
tmpdir, err := ioutil.TempDir("", "images-fs-store")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(tmpdir)
fs, err := NewFSStoreBackend(tmpdir)
if err != nil {
t.Fatal(err)
}
id, err := fs.Set([]byte("foobar"))
if err != nil {
t.Fatal(err)
}
dgst := digest.Digest(id)
if err := ioutil.WriteFile(filepath.Join(tmpdir, contentDirName, string(dgst.Algorithm()), dgst.Hex()), []byte("foobar2"), 0600); err != nil {
t.Fatal(err)
}
_, err = fs.Get(id)
if err == nil {
t.Fatal("Expected get to fail after data modification.")
}
}
func TestFSInvalidSet(t *testing.T) {
tmpdir, err := ioutil.TempDir("", "images-fs-store")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(tmpdir)
fs, err := NewFSStoreBackend(tmpdir)
if err != nil {
t.Fatal(err)
}
id, err := digest.FromBytes([]byte("foobar"))
if err != nil {
t.Fatal(err)
}
err = os.Mkdir(filepath.Join(tmpdir, contentDirName, string(id.Algorithm()), id.Hex()), 0700)
if err != nil {
t.Fatal(err)
}
_, err = fs.Set([]byte("foobar"))
if err == nil {
t.Fatal("Expecting error from invalid filesystem data.")
}
}
func TestFSInvalidRoot(t *testing.T) {
tmpdir, err := ioutil.TempDir("", "images-fs-store")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(tmpdir)
tcases := []struct {
root, invalidFile string
}{
{"root", "root"},
{"root", "root/content"},
{"root", "root/metadata"},
}
for _, tc := range tcases {
root := filepath.Join(tmpdir, tc.root)
filePath := filepath.Join(tmpdir, tc.invalidFile)
err := os.MkdirAll(filepath.Dir(filePath), 0700)
if err != nil {
t.Fatal(err)
}
f, err := os.Create(filePath)
if err != nil {
t.Fatal(err)
}
f.Close()
_, err = NewFSStoreBackend(root)
if err == nil {
t.Fatalf("Expected error from root %q and invlid file %q", tc.root, tc.invalidFile)
}
os.RemoveAll(root)
}
}
func testMetadataGetSet(t *testing.T, store StoreBackend) {
id, err := store.Set([]byte("foo"))
if err != nil {
t.Fatal(err)
}
id2, err := store.Set([]byte("bar"))
if err != nil {
t.Fatal(err)
}
tcases := []struct {
id ID
key string
value []byte
}{
{id, "tkey", []byte("tval1")},
{id, "tkey2", []byte("tval2")},
{id2, "tkey", []byte("tval3")},
}
for _, tc := range tcases {
err = store.SetMetadata(tc.id, tc.key, tc.value)
if err != nil {
t.Fatal(err)
}
actual, err := store.GetMetadata(tc.id, tc.key)
if err != nil {
t.Fatal(err)
}
if bytes.Compare(actual, tc.value) != 0 {
t.Fatalf("Metadata expected %q, got %q", tc.value, actual)
}
}
_, err = store.GetMetadata(id2, "tkey2")
if err == nil {
t.Fatal("Expected error for getting metadata for unknown key")
}
id3, err := digest.FromBytes([]byte("baz"))
if err != nil {
t.Fatal(err)
}
err = store.SetMetadata(ID(id3), "tkey", []byte("tval"))
if err == nil {
t.Fatal("Expected error for setting metadata for unknown ID.")
}
_, err = store.GetMetadata(ID(id3), "tkey")
if err == nil {
t.Fatal("Expected error for getting metadata for unknown ID.")
}
}
func TestFSMetadataGetSet(t *testing.T) {
tmpdir, err := ioutil.TempDir("", "images-fs-store")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(tmpdir)
fs, err := NewFSStoreBackend(tmpdir)
if err != nil {
t.Fatal(err)
}
testMetadataGetSet(t, fs)
}
func TestFSDelete(t *testing.T) {
tmpdir, err := ioutil.TempDir("", "images-fs-store")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(tmpdir)
fs, err := NewFSStoreBackend(tmpdir)
if err != nil {
t.Fatal(err)
}
testDelete(t, fs)
}
func TestFSWalker(t *testing.T) {
tmpdir, err := ioutil.TempDir("", "images-fs-store")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(tmpdir)
fs, err := NewFSStoreBackend(tmpdir)
if err != nil {
t.Fatal(err)
}
testWalker(t, fs)
}
func TestFSInvalidWalker(t *testing.T) {
tmpdir, err := ioutil.TempDir("", "images-fs-store")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(tmpdir)
fs, err := NewFSStoreBackend(tmpdir)
if err != nil {
t.Fatal(err)
}
fooID, err := fs.Set([]byte("foo"))
if err != nil {
t.Fatal(err)
}
if err := ioutil.WriteFile(filepath.Join(tmpdir, contentDirName, "sha256/foobar"), []byte("foobar"), 0600); err != nil {
t.Fatal(err)
}
n := 0
err = fs.Walk(func(id ID) error {
if id != fooID {
t.Fatalf("Invalid walker ID %q, expected %q", id, fooID)
}
n++
return nil
})
if err != nil {
t.Fatalf("Invalid data should not have caused walker error, got %v", err)
}
if n != 1 {
t.Fatalf("Expected 1 walk initialization, got %d", n)
}
}
func testGetSet(t *testing.T, store StoreBackend) {
type tcase struct {
input []byte
expected ID
}
tcases := []tcase{
{[]byte("foobar"), ID("sha256:c3ab8ff13720e8ad9047dd39466b3c8974e592c2fa383d4a3960714caef0c4f2")},
}
randomInput := make([]byte, 8*1024)
_, err := rand.Read(randomInput)
if err != nil {
t.Fatal(err)
}
// skipping use of digest pkg because its used by the imlementation
h := sha256.New()
_, err = h.Write(randomInput)
if err != nil {
t.Fatal(err)
}
tcases = append(tcases, tcase{
input: randomInput,
expected: ID("sha256:" + hex.EncodeToString(h.Sum(nil))),
})
for _, tc := range tcases {
id, err := store.Set([]byte(tc.input))
if err != nil {
t.Fatal(err)
}
if id != tc.expected {
t.Fatalf("Expected ID %q, got %q", tc.expected, id)
}
}
for _, emptyData := range [][]byte{nil, {}} {
_, err := store.Set(emptyData)
if err == nil {
t.Fatal("Expected error for nil input.")
}
}
for _, tc := range tcases {
data, err := store.Get(tc.expected)
if err != nil {
t.Fatal(err)
}
if bytes.Compare(data, tc.input) != 0 {
t.Fatalf("Expected data %q, got %q", tc.input, data)
}
}
for _, key := range []ID{"foobar:abc", "sha256:abc", "sha256:c3ab8ff13720e8ad9047dd39466b3c8974e592c2fa383d4a3960714caef0c4f2a"} {
_, err := store.Get(key)
if err == nil {
t.Fatalf("Expected error for ID %q.", key)
}
}
}
func testDelete(t *testing.T, store StoreBackend) {
id, err := store.Set([]byte("foo"))
if err != nil {
t.Fatal(err)
}
id2, err := store.Set([]byte("bar"))
if err != nil {
t.Fatal(err)
}
err = store.Delete(id)
if err != nil {
t.Fatal(err)
}
_, err = store.Get(id)
if err == nil {
t.Fatalf("Expected getting deleted item %q to fail", id)
}
_, err = store.Get(id2)
if err != nil {
t.Fatal(err)
}
err = store.Delete(id2)
if err != nil {
t.Fatal(err)
}
_, err = store.Get(id2)
if err == nil {
t.Fatalf("Expected getting deleted item %q to fail", id2)
}
}
func testWalker(t *testing.T, store StoreBackend) {
id, err := store.Set([]byte("foo"))
if err != nil {
t.Fatal(err)
}
id2, err := store.Set([]byte("bar"))
if err != nil {
t.Fatal(err)
}
tcases := make(map[ID]struct{})
tcases[id] = struct{}{}
tcases[id2] = struct{}{}
n := 0
err = store.Walk(func(id ID) error {
delete(tcases, id)
n++
return nil
})
if err != nil {
t.Fatal(err)
}
if n != 2 {
t.Fatalf("Expected 2 walk initializations, got %d", n)
}
if len(tcases) != 0 {
t.Fatalf("Expected empty unwalked set, got %+v", tcases)
}
// stop on error
tcases = make(map[ID]struct{})
tcases[id] = struct{}{}
err = store.Walk(func(id ID) error {
return errors.New("")
})
if err == nil {
t.Fatalf("Exected error from walker.")
}
}

163
image/image.go
View file

@ -2,36 +2,23 @@ package image
import (
"encoding/json"
"fmt"
"regexp"
"errors"
"io"
"time"
"github.com/Sirupsen/logrus"
"github.com/docker/distribution/digest"
derr "github.com/docker/docker/errors"
"github.com/docker/docker/pkg/version"
"github.com/docker/docker/runconfig"
)
var validHex = regexp.MustCompile(`^([a-f0-9]{64})$`)
// ID is the content-addressable ID of an image.
type ID digest.Digest
// noFallbackMinVersion is the minimum version for which v1compatibility
// information will not be marshaled through the Image struct to remove
// blank fields.
var noFallbackMinVersion = version.Version("1.8.3")
// Descriptor provides the information necessary to register an image in
// the graph.
type Descriptor interface {
ID() string
Parent() string
MarshalConfig() ([]byte, error)
func (id ID) String() string {
return digest.Digest(id).String()
}
// Image stores the image configuration.
// All fields in this struct must be marked `omitempty` to keep getting
// predictable hashes from the old `v1Compatibility` configuration.
type Image struct {
// V1Image stores the V1 image configuration.
type V1Image struct {
// ID a unique 64 character identifier of the image
ID string `json:"id,omitempty"`
// Parent id of the image
@ -55,95 +42,87 @@ type Image struct {
// OS is the operating system used to build and run the image
OS string `json:"os,omitempty"`
// Size is the total size of the image including all layers it is composed of
Size int64 `json:",omitempty"` // capitalized for backwards compatibility
// ParentID specifies the strong, content address of the parent configuration.
ParentID digest.Digest `json:"parent_id,omitempty"`
// LayerID provides the content address of the associated layer.
LayerID digest.Digest `json:"layer_id,omitempty"`
Size int64 `json:",omitempty"`
}
// NewImgJSON creates an Image configuration from json.
func NewImgJSON(src []byte) (*Image, error) {
ret := &Image{}
// Image stores the image configuration
type Image struct {
V1Image
Parent ID `json:"parent,omitempty"`
RootFS *RootFS `json:"rootfs,omitempty"`
History []History `json:"history,omitempty"`
// FIXME: Is there a cleaner way to "purify" the input json?
if err := json.Unmarshal(src, ret); err != nil {
return nil, err
}
return ret, nil
// rawJSON caches the immutable JSON associated with this image.
rawJSON []byte
// computedID is the ID computed from the hash of the image config.
// Not to be confused with the legacy V1 ID in V1Image.
computedID ID
}
// ValidateID checks whether an ID string is a valid image ID.
func ValidateID(id string) error {
if ok := validHex.MatchString(id); !ok {
return derr.ErrorCodeInvalidImageID.WithArgs(id)
}
return nil
// RawJSON returns the immutable JSON associated with the image.
func (img *Image) RawJSON() []byte {
return img.rawJSON
}
// MakeImageConfig returns immutable configuration JSON for image based on the
// v1Compatibility object, layer digest and parent StrongID. SHA256() of this
// config is the new image ID (strongID).
func MakeImageConfig(v1Compatibility []byte, layerID, parentID digest.Digest) ([]byte, error) {
// ID returns the image's content-addressable ID.
func (img *Image) ID() ID {
return img.computedID
}
// Detect images created after 1.8.3
img, err := NewImgJSON(v1Compatibility)
// MarshalJSON serializes the image to JSON. It sorts the top-level keys so
// that JSON that's been manipulated by a push/pull cycle with a legacy
// registry won't end up with a different key order.
func (img *Image) MarshalJSON() ([]byte, error) {
type MarshalImage Image
pass1, err := json.Marshal(MarshalImage(*img))
if err != nil {
return nil, err
}
useFallback := version.Version(img.DockerVersion).LessThan(noFallbackMinVersion)
if useFallback {
// Fallback for pre-1.8.3. Calculate base config based on Image struct
// so that fields with default values added by Docker will use same ID
logrus.Debugf("Using fallback hash for %v", layerID)
v1Compatibility, err = json.Marshal(img)
if err != nil {
return nil, err
}
}
var c map[string]*json.RawMessage
if err := json.Unmarshal(v1Compatibility, &c); err != nil {
if err := json.Unmarshal(pass1, &c); err != nil {
return nil, err
}
if err := layerID.Validate(); err != nil {
return nil, fmt.Errorf("invalid layerID: %v", err)
}
c["layer_id"] = rawJSON(layerID)
if parentID != "" {
if err := parentID.Validate(); err != nil {
return nil, fmt.Errorf("invalid parentID %v", err)
}
c["parent_id"] = rawJSON(parentID)
}
delete(c, "id")
delete(c, "parent")
delete(c, "Size") // Size is calculated from data on disk and is inconsitent
return json.Marshal(c)
}
// StrongID returns image ID for the config JSON.
func StrongID(configJSON []byte) (digest.Digest, error) {
digester := digest.Canonical.New()
if _, err := digester.Hash().Write(configJSON); err != nil {
return "", err
}
dgst := digester.Digest()
logrus.Debugf("H(%v) = %v", string(configJSON), dgst)
return dgst, nil
// History stores build commands that were used to create an image
type History struct {
// Created timestamp for build point
Created time.Time `json:"created"`
// Author of the build point
Author string `json:"author,omitempty"`
// CreatedBy keeps the Dockerfile command used while building image.
CreatedBy string `json:"created_by,omitempty"`
// Comment is custom mesage set by the user when creating the image.
Comment string `json:"comment,omitempty"`
// EmptyLayer is set to true if this history item did not generate a
// layer. Otherwise, the history item is associated with the next
// layer in the RootFS section.
EmptyLayer bool `json:"empty_layer,omitempty"`
}
func rawJSON(value interface{}) *json.RawMessage {
jsonval, err := json.Marshal(value)
if err != nil {
return nil
}
return (*json.RawMessage)(&jsonval)
// Exporter provides interface for exporting and importing images
type Exporter interface {
Load(io.ReadCloser, io.Writer) error
// TODO: Load(net.Context, io.ReadCloser, <- chan StatusMessage) error
Save([]string, io.Writer) error
}
// NewFromJSON creates an Image configuration from json.
func NewFromJSON(src []byte) (*Image, error) {
img := &Image{}
if err := json.Unmarshal(src, img); err != nil {
return nil, err
}
if img.RootFS == nil {
return nil, errors.New("Invalid image JSON, no RootFS key.")
}
img.rawJSON = src
return img, nil
}

Some files were not shown because too many files have changed in this diff Show more