diff --git a/Dockerfile b/Dockerfile index 59d9ced906..5dd36d41a0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -148,6 +148,17 @@ RUN set -x \ && git clone -b v1.2 https://github.com/russross/blackfriday.git /go/src/github.com/russross/blackfriday \ && go install -v github.com/cpuguy83/go-md2man +# Install registry +COPY pkg/tarsum /go/src/github.com/docker/docker/pkg/tarsum +# REGISTRY_COMMIT gives us the repeatability guarantees we need +# (so that we're all testing the same version of the registry) +ENV REGISTRY_COMMIT 21a69f53b5c7986b831f33849d551cd59ec8cbd1 +RUN set -x \ + && git clone https://github.com/docker/distribution.git /go/src/github.com/docker/distribution \ + && (cd /go/src/github.com/docker/distribution && git checkout -q $REGISTRY_COMMIT) \ + && go get -d github.com/docker/distribution/cmd/registry \ + && go build -o /go/bin/registry-v2 github.com/docker/distribution/cmd/registry + # Wrap all commands in the "docker-in-docker" script to allow nested containers ENTRYPOINT ["hack/dind"] diff --git a/api/client/commands.go b/api/client/commands.go index 237caa6227..b50fad6853 100644 --- a/api/client/commands.go +++ b/api/client/commands.go @@ -43,6 +43,7 @@ import ( "github.com/docker/docker/registry" "github.com/docker/docker/runconfig" "github.com/docker/docker/utils" + "github.com/docker/libtrust" ) const ( @@ -1215,6 +1216,26 @@ func (cli *DockerCli) CmdPush(args ...string) error { v := url.Values{} v.Set("tag", tag) + + body, _, err := readBody(cli.call("GET", "/images/"+remote+"/manifest?"+v.Encode(), nil, false)) + if err != nil { + return err + } + + js, err := libtrust.NewJSONSignature(body) + if err != nil { + return err + } + err = js.Sign(cli.key) + if err != nil { + return err + } + + signedBody, err := js.PrettySignature("signatures") + if err != nil { + return err + } + push := func(authConfig registry.AuthConfig) error { buf, err := json.Marshal(authConfig) if err != nil { @@ -1224,7 +1245,7 @@ func (cli *DockerCli) CmdPush(args ...string) error { base64.URLEncoding.EncodeToString(buf), } - return cli.stream("POST", "/images/"+remote+"/push?"+v.Encode(), nil, cli.out, map[string][]string{ + return cli.stream("POST", "/images/"+remote+"/push?"+v.Encode(), bytes.NewReader(signedBody), cli.out, map[string][]string{ "X-Registry-Auth": registryAuthHeader, }) } diff --git a/api/server/server.go b/api/server/server.go index 4907ff0349..d2715f1bc6 100644 --- a/api/server/server.go +++ b/api/server/server.go @@ -608,6 +608,18 @@ func getImagesSearch(eng *engine.Engine, version version.Version, w http.Respons return job.Run() } +func getImageManifest(eng *engine.Engine, version version.Version, w http.ResponseWriter, r *http.Request, vars map[string]string) error { + if err := parseForm(r); err != nil { + return err + } + + job := eng.Job("image_manifest", vars["name"]) + job.Setenv("tag", r.Form.Get("tag")) + job.Stdout.Add(utils.NewWriteFlusher(w)) + + return job.Run() +} + func postImagesPush(eng *engine.Engine, version version.Version, w http.ResponseWriter, r *http.Request, vars map[string]string) error { if vars == nil { return fmt.Errorf("Missing parameter") @@ -639,9 +651,15 @@ func postImagesPush(eng *engine.Engine, version version.Version, w http.Response } } + manifest, err := ioutil.ReadAll(r.Body) + if err != nil { + return err + } + job := eng.Job("push", vars["name"]) job.SetenvJson("metaHeaders", metaHeaders) job.SetenvJson("authConfig", authConfig) + job.Setenv("manifest", string(manifest)) job.Setenv("tag", r.Form.Get("tag")) if version.GreaterThan("1.0") { job.SetenvBool("json", true) @@ -1294,6 +1312,7 @@ func createRouter(eng *engine.Engine, logging, enableCors bool, dockerVersion st "/images/viz": getImagesViz, "/images/search": getImagesSearch, "/images/get": getImagesGet, + "/images/{name:.*}/manifest": getImageManifest, "/images/{name:.*}/get": getImagesGet, "/images/{name:.*}/history": getImagesHistory, "/images/{name:.*}/json": getImagesByName, diff --git a/daemon/daemon.go b/daemon/daemon.go index 9f5df4c3c5..8a5db74a33 100644 --- a/daemon/daemon.go +++ b/daemon/daemon.go @@ -895,8 +895,13 @@ func NewDaemonFromDirectory(config *Config, eng *engine.Engine) (*Daemon, error) return nil, err } + trustKey, err := api.LoadOrCreateTrustKey(config.TrustKeyPath) + if err != nil { + return nil, err + } + log.Debugf("Creating repository list") - repositories, err := graph.NewTagStore(path.Join(config.Root, "repositories-"+driver.String()), g) + repositories, err := graph.NewTagStore(path.Join(config.Root, "repositories-"+driver.String()), g, trustKey) if err != nil { return nil, fmt.Errorf("Couldn't create Tag store: %s", err) } @@ -961,11 +966,6 @@ func NewDaemonFromDirectory(config *Config, eng *engine.Engine) (*Daemon, error) return nil, err } - trustKey, err := api.LoadOrCreateTrustKey(config.TrustKeyPath) - if err != nil { - return nil, err - } - daemon := &Daemon{ ID: trustKey.PublicKey().KeyID(), repository: daemonRepo, diff --git a/docker/docker.go b/docker/docker.go index 3137f5c99f..92f5f14603 100644 --- a/docker/docker.go +++ b/docker/docker.go @@ -77,6 +77,11 @@ func main() { } protoAddrParts := strings.SplitN(flHosts[0], "://", 2) + trustKey, err := api.LoadOrCreateTrustKey(*flTrustKey) + if err != nil { + log.Fatal(err) + } + var ( cli *client.DockerCli tlsConfig tls.Config @@ -118,9 +123,9 @@ func main() { } if *flTls || *flTlsVerify { - cli = client.NewDockerCli(os.Stdin, os.Stdout, os.Stderr, nil, protoAddrParts[0], protoAddrParts[1], &tlsConfig) + cli = client.NewDockerCli(os.Stdin, os.Stdout, os.Stderr, trustKey, protoAddrParts[0], protoAddrParts[1], &tlsConfig) } else { - cli = client.NewDockerCli(os.Stdin, os.Stdout, os.Stderr, nil, protoAddrParts[0], protoAddrParts[1], nil) + cli = client.NewDockerCli(os.Stdin, os.Stdout, os.Stderr, trustKey, protoAddrParts[0], protoAddrParts[1], nil) } if err := cli.Cmd(flag.Args()...); err != nil { diff --git a/graph/manifest.go b/graph/manifest.go new file mode 100644 index 0000000000..3d4ab1c5de --- /dev/null +++ b/graph/manifest.go @@ -0,0 +1,196 @@ +package graph + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "io" + "io/ioutil" + "path" + + log "github.com/Sirupsen/logrus" + "github.com/docker/docker/engine" + "github.com/docker/docker/pkg/tarsum" + "github.com/docker/docker/registry" + "github.com/docker/docker/runconfig" + "github.com/docker/libtrust" +) + +func (s *TagStore) CmdManifest(job *engine.Job) engine.Status { + if len(job.Args) != 1 { + return job.Errorf("usage: %s NAME", job.Name) + } + name := job.Args[0] + tag := job.Getenv("tag") + if tag == "" { + tag = "latest" + } + + // Resolve the Repository name from fqn to endpoint + name + repoInfo, err := registry.ParseRepositoryInfo(name) + if err != nil { + return job.Error(err) + } + + manifestBytes, err := s.newManifest(name, repoInfo.RemoteName, tag) + if err != nil { + return job.Error(err) + } + + _, err = job.Stdout.Write(manifestBytes) + if err != nil { + return job.Error(err) + } + + return engine.StatusOK +} + +func (s *TagStore) newManifest(localName, remoteName, tag string) ([]byte, error) { + manifest := ®istry.ManifestData{ + Name: remoteName, + Tag: tag, + SchemaVersion: 1, + } + localRepo, err := s.Get(localName) + if err != nil { + return nil, err + } + if localRepo == nil { + return nil, fmt.Errorf("Repo does not exist: %s", localName) + } + + // Get the top-most layer id which the tag points to + layerId, exists := localRepo[tag] + if !exists { + return nil, fmt.Errorf("Tag does not exist for %s: %s", localName, tag) + } + layersSeen := make(map[string]bool) + + layer, err := s.graph.Get(layerId) + if err != nil { + return nil, err + } + if layer.Config == nil { + return nil, errors.New("Missing layer configuration") + } + manifest.Architecture = layer.Architecture + manifest.FSLayers = make([]*registry.FSLayer, 0, 4) + manifest.History = make([]*registry.ManifestHistory, 0, 4) + var metadata runconfig.Config + metadata = *layer.Config + + for ; layer != nil; layer, err = layer.GetParent() { + if err != nil { + return nil, err + } + + if layersSeen[layer.ID] { + break + } + if layer.Config != nil && metadata.Image != layer.ID { + err = runconfig.Merge(&metadata, layer.Config) + if err != nil { + return nil, err + } + } + + archive, err := layer.TarLayer() + if err != nil { + return nil, err + } + + tarSum, err := tarsum.NewTarSum(archive, true, tarsum.Version1) + if err != nil { + return nil, err + } + if _, err := io.Copy(ioutil.Discard, tarSum); err != nil { + return nil, err + } + + tarId := tarSum.Sum(nil) + + manifest.FSLayers = append(manifest.FSLayers, ®istry.FSLayer{BlobSum: tarId}) + + layersSeen[layer.ID] = true + jsonData, err := ioutil.ReadFile(path.Join(s.graph.Root, layer.ID, "json")) + if err != nil { + return nil, fmt.Errorf("Cannot retrieve the path for {%s}: %s", layer.ID, err) + } + manifest.History = append(manifest.History, ®istry.ManifestHistory{V1Compatibility: string(jsonData)}) + } + + manifestBytes, err := json.MarshalIndent(manifest, "", " ") + if err != nil { + return nil, err + } + + return manifestBytes, nil +} + +func (s *TagStore) verifyManifest(eng *engine.Engine, manifestBytes []byte) (*registry.ManifestData, bool, error) { + sig, err := libtrust.ParsePrettySignature(manifestBytes, "signatures") + if err != nil { + return nil, false, fmt.Errorf("error parsing payload: %s", err) + } + + keys, err := sig.Verify() + if err != nil { + return nil, false, fmt.Errorf("error verifying payload: %s", err) + } + + payload, err := sig.Payload() + if err != nil { + return nil, false, fmt.Errorf("error retrieving payload: %s", err) + } + + var manifest registry.ManifestData + if err := json.Unmarshal(payload, &manifest); err != nil { + return nil, false, fmt.Errorf("error unmarshalling manifest: %s", err) + } + if manifest.SchemaVersion != 1 { + return nil, false, fmt.Errorf("unsupported schema version: %d", manifest.SchemaVersion) + } + + var verified bool + for _, key := range keys { + job := eng.Job("trust_key_check") + b, err := key.MarshalJSON() + if err != nil { + return nil, false, fmt.Errorf("error marshalling public key: %s", err) + } + namespace := manifest.Name + if namespace[0] != '/' { + namespace = "/" + namespace + } + stdoutBuffer := bytes.NewBuffer(nil) + + job.Args = append(job.Args, namespace) + job.Setenv("PublicKey", string(b)) + // Check key has read/write permission (0x03) + job.SetenvInt("Permission", 0x03) + job.Stdout.Add(stdoutBuffer) + if err = job.Run(); err != nil { + return nil, false, fmt.Errorf("error running key check: %s", err) + } + result := engine.Tail(stdoutBuffer, 1) + log.Debugf("Key check result: %q", result) + if result == "verified" { + verified = true + } + } + + return &manifest, verified, nil +} + +func checkValidManifest(manifest *registry.ManifestData) error { + if len(manifest.FSLayers) != len(manifest.History) { + return fmt.Errorf("length of history not equal to number of layers") + } + + if len(manifest.FSLayers) == 0 { + return fmt.Errorf("no FSLayers in manifest") + } + + return nil +} diff --git a/graph/pull.go b/graph/pull.go index 587eb5f500..6129ea39a1 100644 --- a/graph/pull.go +++ b/graph/pull.go @@ -1,8 +1,6 @@ package graph import ( - "bytes" - "encoding/json" "fmt" "io" "io/ioutil" @@ -15,65 +13,11 @@ import ( log "github.com/Sirupsen/logrus" "github.com/docker/docker/engine" "github.com/docker/docker/image" + "github.com/docker/docker/pkg/tarsum" "github.com/docker/docker/registry" "github.com/docker/docker/utils" - "github.com/docker/libtrust" ) -func (s *TagStore) verifyManifest(eng *engine.Engine, manifestBytes []byte) (*registry.ManifestData, bool, error) { - sig, err := libtrust.ParsePrettySignature(manifestBytes, "signatures") - if err != nil { - return nil, false, fmt.Errorf("error parsing payload: %s", err) - } - keys, err := sig.Verify() - if err != nil { - return nil, false, fmt.Errorf("error verifying payload: %s", err) - } - - payload, err := sig.Payload() - if err != nil { - return nil, false, fmt.Errorf("error retrieving payload: %s", err) - } - - var manifest registry.ManifestData - if err := json.Unmarshal(payload, &manifest); err != nil { - return nil, false, fmt.Errorf("error unmarshalling manifest: %s", err) - } - if manifest.SchemaVersion != 1 { - return nil, false, fmt.Errorf("unsupported schema version: %d", manifest.SchemaVersion) - } - - var verified bool - for _, key := range keys { - job := eng.Job("trust_key_check") - b, err := key.MarshalJSON() - if err != nil { - return nil, false, fmt.Errorf("error marshalling public key: %s", err) - } - namespace := manifest.Name - if namespace[0] != '/' { - namespace = "/" + namespace - } - stdoutBuffer := bytes.NewBuffer(nil) - - job.Args = append(job.Args, namespace) - job.Setenv("PublicKey", string(b)) - // Check key has read/write permission (0x03) - job.SetenvInt("Permission", 0x03) - job.Stdout.Add(stdoutBuffer) - if err = job.Run(); err != nil { - return nil, false, fmt.Errorf("error running key check: %s", err) - } - result := engine.Tail(stdoutBuffer, 1) - log.Debugf("Key check result: %q", result) - if result == "verified" { - verified = true - } - } - - return &manifest, verified, nil -} - func (s *TagStore) CmdPull(job *engine.Job) engine.Status { if n := len(job.Args); n != 1 && n != 2 { return job.Errorf("Usage: %s IMAGE [TAG]", job.Name) @@ -112,6 +56,7 @@ func (s *TagStore) CmdPull(job *engine.Job) engine.Status { } defer s.poolRemove("pull", repoInfo.LocalName+":"+tag) + log.Debugf("pulling image from host %q with remote name %q", repoInfo.Index.Name, repoInfo.RemoteName) endpoint, err := repoInfo.GetEndpoint() if err != nil { return job.Error(err) @@ -127,12 +72,13 @@ func (s *TagStore) CmdPull(job *engine.Job) engine.Status { logName += ":" + tag } - if len(repoInfo.Index.Mirrors) == 0 && (repoInfo.Official || endpoint.Version == registry.APIVersion2) { + if len(repoInfo.Index.Mirrors) == 0 && (repoInfo.Index.Official || endpoint.Version == registry.APIVersion2) { j := job.Eng.Job("trust_update_base") if err = j.Run(); err != nil { - return job.Errorf("error updating trust base graph: %s", err) + log.Errorf("error updating trust base graph: %s", err) } + log.Debugf("pulling v2 repository with local name %q", repoInfo.LocalName) if err := s.pullV2Repository(job.Eng, r, job.Stdout, repoInfo, tag, sf, job.GetenvBool("parallel")); err == nil { if err = job.Eng.Job("log", "pull", logName, "").Run(); err != nil { log.Errorf("Error logging event 'pull' for %s: %s", logName, err) @@ -141,8 +87,11 @@ func (s *TagStore) CmdPull(job *engine.Job) engine.Status { } else if err != registry.ErrDoesNotExist { log.Errorf("Error from V2 registry: %s", err) } + + log.Debug("image does not exist on v2 registry, falling back to v1") } + log.Debugf("pulling v1 repository with local name %q", repoInfo.LocalName) if err = s.pullRepository(r, job.Stdout, repoInfo, tag, sf, job.GetenvBool("parallel")); err != nil { return job.Error(err) } @@ -169,7 +118,7 @@ func (s *TagStore) pullRepository(r *registry.Session, out io.Writer, repoInfo * log.Debugf("Retrieving the tag list") tagsList, err := r.GetRemoteTags(repoData.Endpoints, repoInfo.RemoteName, repoData.Tokens) if err != nil { - log.Errorf("%v", err) + log.Errorf("unable to get remote tags: %s", err) return err } @@ -424,22 +373,30 @@ type downloadInfo struct { } func (s *TagStore) pullV2Repository(eng *engine.Engine, r *registry.Session, out io.Writer, repoInfo *registry.RepositoryInfo, tag string, sf *utils.StreamFormatter, parallel bool) error { + endpoint, err := r.V2RegistryEndpoint(repoInfo.Index) + if err != nil { + return fmt.Errorf("error getting registry endpoint: %s", err) + } + auth, err := r.GetV2Authorization(endpoint, repoInfo.RemoteName, true) + if err != nil { + return fmt.Errorf("error getting authorization: %s", err) + } var layersDownloaded bool if tag == "" { log.Debugf("Pulling tag list from V2 registry for %s", repoInfo.CanonicalName) - tags, err := r.GetV2RemoteTags(repoInfo.RemoteName, nil) + tags, err := r.GetV2RemoteTags(endpoint, repoInfo.RemoteName, auth) if err != nil { return err } for _, t := range tags { - if downloaded, err := s.pullV2Tag(eng, r, out, repoInfo, t, sf, parallel); err != nil { + if downloaded, err := s.pullV2Tag(eng, r, out, endpoint, repoInfo, t, sf, parallel, auth); err != nil { return err } else if downloaded { layersDownloaded = true } } } else { - if downloaded, err := s.pullV2Tag(eng, r, out, repoInfo, tag, sf, parallel); err != nil { + if downloaded, err := s.pullV2Tag(eng, r, out, endpoint, repoInfo, tag, sf, parallel, auth); err != nil { return err } else if downloaded { layersDownloaded = true @@ -454,9 +411,9 @@ func (s *TagStore) pullV2Repository(eng *engine.Engine, r *registry.Session, out return nil } -func (s *TagStore) pullV2Tag(eng *engine.Engine, r *registry.Session, out io.Writer, repoInfo *registry.RepositoryInfo, tag string, sf *utils.StreamFormatter, parallel bool) (bool, error) { +func (s *TagStore) pullV2Tag(eng *engine.Engine, r *registry.Session, out io.Writer, endpoint *registry.Endpoint, repoInfo *registry.RepositoryInfo, tag string, sf *utils.StreamFormatter, parallel bool, auth *registry.RequestAuthorization) (bool, error) { log.Debugf("Pulling tag from V2 registry: %q", tag) - manifestBytes, err := r.GetV2ImageManifest(repoInfo.RemoteName, tag, nil) + manifestBytes, err := r.GetV2ImageManifest(endpoint, repoInfo.RemoteName, tag, auth) if err != nil { return false, err } @@ -466,8 +423,8 @@ func (s *TagStore) pullV2Tag(eng *engine.Engine, r *registry.Session, out io.Wri return false, fmt.Errorf("error verifying manifest: %s", err) } - if len(manifest.FSLayers) != len(manifest.History) { - return false, fmt.Errorf("length of history not equal to number of layers") + if err := checkValidManifest(manifest); err != nil { + return false, err } if verified { @@ -475,11 +432,6 @@ func (s *TagStore) pullV2Tag(eng *engine.Engine, r *registry.Session, out io.Wri } else { out.Write(sf.FormatStatus(tag, "Pulling from %s", repoInfo.CanonicalName)) } - - if len(manifest.FSLayers) == 0 { - return false, fmt.Errorf("no blobSums in manifest") - } - downloads := make([]downloadInfo, len(manifest.FSLayers)) for i := len(manifest.FSLayers) - 1; i >= 0; i-- { @@ -525,12 +477,25 @@ func (s *TagStore) pullV2Tag(eng *engine.Engine, r *registry.Session, out io.Wri return err } - r, l, err := r.GetV2ImageBlobReader(repoInfo.RemoteName, sumType, checksum, nil) + r, l, err := r.GetV2ImageBlobReader(endpoint, repoInfo.RemoteName, sumType, checksum, auth) if err != nil { return err } defer r.Close() - io.Copy(tmpFile, utils.ProgressReader(r, int(l), out, sf, false, utils.TruncateID(img.ID), "Downloading")) + + // Wrap the reader with the appropriate TarSum reader. + tarSumReader, err := tarsum.NewTarSumForLabel(r, true, sumType) + if err != nil { + return fmt.Errorf("unable to wrap image blob reader with TarSum: %s", err) + } + + io.Copy(tmpFile, utils.ProgressReader(ioutil.NopCloser(tarSumReader), int(l), out, sf, false, utils.TruncateID(img.ID), "Downloading")) + + out.Write(sf.FormatProgress(utils.TruncateID(img.ID), "Verifying Checksum", nil)) + + if finalChecksum := tarSumReader.Sum(nil); !strings.EqualFold(finalChecksum, sumStr) { + return fmt.Errorf("image verification failed: checksum mismatch - expected %q but got %q", sumStr, finalChecksum) + } out.Write(sf.FormatProgress(utils.TruncateID(img.ID), "Download complete", nil)) diff --git a/graph/push.go b/graph/push.go index 0ec81a5152..46469daead 100644 --- a/graph/push.go +++ b/graph/push.go @@ -1,18 +1,22 @@ package graph import ( + "bytes" "fmt" "io" "io/ioutil" "os" "path" + "strings" "sync" log "github.com/Sirupsen/logrus" "github.com/docker/docker/engine" + "github.com/docker/docker/image" "github.com/docker/docker/pkg/archive" "github.com/docker/docker/registry" "github.com/docker/docker/utils" + "github.com/docker/libtrust" ) // Retrieve the all the images to be uploaded in the correct order @@ -248,6 +252,109 @@ func (s *TagStore) pushImage(r *registry.Session, out io.Writer, imgID, ep strin return imgData.Checksum, nil } +func (s *TagStore) pushV2Repository(r *registry.Session, eng *engine.Engine, out io.Writer, repoInfo *registry.RepositoryInfo, manifestBytes, tag string, sf *utils.StreamFormatter) error { + if repoInfo.Official { + j := eng.Job("trust_update_base") + if err := j.Run(); err != nil { + log.Errorf("error updating trust base graph: %s", err) + } + } + + endpoint, err := r.V2RegistryEndpoint(repoInfo.Index) + if err != nil { + return fmt.Errorf("error getting registry endpoint: %s", err) + } + auth, err := r.GetV2Authorization(endpoint, repoInfo.RemoteName, false) + if err != nil { + return fmt.Errorf("error getting authorization: %s", err) + } + + // if no manifest is given, generate and sign with the key associated with the local tag store + if len(manifestBytes) == 0 { + mBytes, err := s.newManifest(repoInfo.LocalName, repoInfo.RemoteName, tag) + if err != nil { + return err + } + js, err := libtrust.NewJSONSignature(mBytes) + if err != nil { + return err + } + + if err = js.Sign(s.trustKey); err != nil { + return err + } + + signedBody, err := js.PrettySignature("signatures") + if err != nil { + return err + } + log.Infof("Signed manifest using daemon's key: %s", s.trustKey.KeyID()) + + manifestBytes = string(signedBody) + } + + manifest, verified, err := s.verifyManifest(eng, []byte(manifestBytes)) + if err != nil { + return fmt.Errorf("error verifying manifest: %s", err) + } + + if err := checkValidManifest(manifest); err != nil { + return fmt.Errorf("invalid manifest: %s", err) + } + + if !verified { + log.Debugf("Pushing unverified image") + } + + for i := len(manifest.FSLayers) - 1; i >= 0; i-- { + var ( + sumStr = manifest.FSLayers[i].BlobSum + imgJSON = []byte(manifest.History[i].V1Compatibility) + ) + + sumParts := strings.SplitN(sumStr, ":", 2) + if len(sumParts) < 2 { + return fmt.Errorf("Invalid checksum: %s", sumStr) + } + manifestSum := sumParts[1] + + img, err := image.NewImgJSON(imgJSON) + if err != nil { + return fmt.Errorf("Failed to parse json: %s", err) + } + + img, err = s.graph.Get(img.ID) + if err != nil { + return err + } + + arch, err := img.TarLayer() + if err != nil { + return fmt.Errorf("Could not get tar layer: %s", err) + } + + // Call mount blob + exists, err := r.HeadV2ImageBlob(endpoint, repoInfo.RemoteName, sumParts[0], manifestSum, auth) + if err != nil { + out.Write(sf.FormatProgress(utils.TruncateID(img.ID), "Image push failed", nil)) + return err + } + if !exists { + err = r.PutV2ImageBlob(endpoint, repoInfo.RemoteName, sumParts[0], manifestSum, utils.ProgressReader(arch, int(img.Size), out, sf, false, utils.TruncateID(img.ID), "Pushing"), auth) + if err != nil { + out.Write(sf.FormatProgress(utils.TruncateID(img.ID), "Image push failed", nil)) + return err + } + out.Write(sf.FormatProgress(utils.TruncateID(img.ID), "Image successfully pushed", nil)) + } else { + out.Write(sf.FormatProgress(utils.TruncateID(img.ID), "Image already exists", nil)) + } + } + + // push the manifest + return r.PutV2ImageManifest(endpoint, repoInfo.RemoteName, tag, bytes.NewReader([]byte(manifestBytes)), auth) +} + // FIXME: Allow to interrupt current push when new push of same image is done. func (s *TagStore) CmdPush(job *engine.Job) engine.Status { if n := len(job.Args); n != 1 { @@ -267,6 +374,7 @@ func (s *TagStore) CmdPush(job *engine.Job) engine.Status { } tag := job.Getenv("tag") + manifestBytes := job.Getenv("manifest") job.GetenvJson("authConfig", authConfig) job.GetenvJson("metaHeaders", &metaHeaders) @@ -286,6 +394,20 @@ func (s *TagStore) CmdPush(job *engine.Job) engine.Status { return job.Error(err2) } + if len(tag) == 0 { + tag = DEFAULTTAG + } + + if repoInfo.Index.Official || endpoint.Version == registry.APIVersion2 { + err := s.pushV2Repository(r, job.Eng, job.Stdout, repoInfo, manifestBytes, tag, sf) + if err == nil { + return engine.StatusOK + } + + // error out, no fallback to V1 + return job.Error(err) + } + if err != nil { reposLen := 1 if tag == "" { diff --git a/graph/service.go b/graph/service.go index 2858d9b3e6..675e12a1a9 100644 --- a/graph/service.go +++ b/graph/service.go @@ -25,6 +25,7 @@ func (s *TagStore) Install(eng *engine.Engine) error { "import": s.CmdImport, "pull": s.CmdPull, "push": s.CmdPush, + "image_manifest": s.CmdManifest, } { if err := eng.Register(name, handler); err != nil { return fmt.Errorf("Could not register %q: %v", name, err) diff --git a/graph/tags.go b/graph/tags.go index 998b447e6c..6bdb296cd1 100644 --- a/graph/tags.go +++ b/graph/tags.go @@ -15,6 +15,7 @@ import ( "github.com/docker/docker/pkg/parsers" "github.com/docker/docker/registry" "github.com/docker/docker/utils" + "github.com/docker/libtrust" ) const DEFAULTTAG = "latest" @@ -27,6 +28,7 @@ type TagStore struct { path string graph *Graph Repositories map[string]Repository + trustKey libtrust.PrivateKey sync.Mutex // FIXME: move push/pull-related fields // to a helper type @@ -54,7 +56,7 @@ func (r Repository) Contains(u Repository) bool { return true } -func NewTagStore(path string, graph *Graph) (*TagStore, error) { +func NewTagStore(path string, graph *Graph, key libtrust.PrivateKey) (*TagStore, error) { abspath, err := filepath.Abs(path) if err != nil { return nil, err @@ -63,6 +65,7 @@ func NewTagStore(path string, graph *Graph) (*TagStore, error) { store := &TagStore{ path: abspath, graph: graph, + trustKey: key, Repositories: make(map[string]Repository), pullingPool: make(map[string]chan struct{}), pushingPool: make(map[string]chan struct{}), diff --git a/graph/tags_unit_test.go b/graph/tags_unit_test.go index 45dad62951..58ad8ed878 100644 --- a/graph/tags_unit_test.go +++ b/graph/tags_unit_test.go @@ -57,7 +57,7 @@ func mkTestTagStore(root string, t *testing.T) *TagStore { if err != nil { t.Fatal(err) } - store, err := NewTagStore(path.Join(root, "tags"), graph) + store, err := NewTagStore(path.Join(root, "tags"), graph, nil) if err != nil { t.Fatal(err) } diff --git a/image/image.go b/image/image.go index 8cd9aa3755..7664602cd8 100644 --- a/image/image.go +++ b/image/image.go @@ -94,28 +94,29 @@ func StoreImage(img *Image, layerData archive.ArchiveReader, root string) error // If layerData is not nil, unpack it into the new layer if layerData != nil { - layerDataDecompressed, err := archive.DecompressStream(layerData) - if err != nil { + // If the image doesn't have a checksum, we should add it. The layer + // checksums are verified when they are pulled from a remote, but when + // a container is committed it should be added here. + if img.Checksum == "" { + layerDataDecompressed, err := archive.DecompressStream(layerData) + if err != nil { + return err + } + defer layerDataDecompressed.Close() + + if layerTarSum, err = tarsum.NewTarSum(layerDataDecompressed, true, tarsum.VersionDev); err != nil { + return err + } + + if size, err = driver.ApplyDiff(img.ID, img.Parent, layerTarSum); err != nil { + return err + } + + img.Checksum = layerTarSum.Sum(nil) + } else if size, err = driver.ApplyDiff(img.ID, img.Parent, layerData); err != nil { return err } - defer layerDataDecompressed.Close() - - if layerTarSum, err = tarsum.NewTarSum(layerDataDecompressed, true, tarsum.VersionDev); err != nil { - return err - } - - if size, err = driver.ApplyDiff(img.ID, img.Parent, layerTarSum); err != nil { - return err - } - - checksum := layerTarSum.Sum(nil) - - if img.Checksum != "" && img.Checksum != checksum { - log.Warnf("image layer checksum mismatch: computed %q, expected %q", checksum, img.Checksum) - } - - img.Checksum = checksum } img.Size = size diff --git a/integration-cli/docker_cli_pull_test.go b/integration-cli/docker_cli_pull_test.go index bed015be0e..7649688583 100644 --- a/integration-cli/docker_cli_pull_test.go +++ b/integration-cli/docker_cli_pull_test.go @@ -1,12 +1,57 @@ package main import ( + "fmt" "os/exec" "strings" "testing" ) -// FIXME: we need a test for pulling all aliases for an image (issue #8141) +// See issue docker/docker#8141 +func TestPullImageWithAliases(t *testing.T) { + defer setupRegistry(t)() + + repoName := fmt.Sprintf("%v/dockercli/busybox", privateRegistryURL) + defer deleteImages(repoName) + + repos := []string{} + for _, tag := range []string{"recent", "fresh"} { + repos = append(repos, fmt.Sprintf("%v:%v", repoName, tag)) + } + + // Tag and push the same image multiple times. + for _, repo := range repos { + if out, _, err := runCommandWithOutput(exec.Command(dockerBinary, "tag", "busybox", repo)); err != nil { + t.Fatalf("Failed to tag image %v: error %v, output %q", repos, err, out) + } + if out, err := exec.Command(dockerBinary, "push", repo).CombinedOutput(); err != nil { + t.Fatalf("Failed to push image %v: error %v, output %q", err, string(out)) + } + } + + // Clear local images store. + args := append([]string{"rmi"}, repos...) + if out, err := exec.Command(dockerBinary, args...).CombinedOutput(); err != nil { + t.Fatalf("Failed to clean images: error %v, output %q", err, string(out)) + } + + // Pull a single tag and verify it doesn't bring down all aliases. + pullCmd := exec.Command(dockerBinary, "pull", repos[0]) + if out, _, err := runCommandWithOutput(pullCmd); err != nil { + t.Fatalf("Failed to pull %v: error %v, output %q", repoName, err, out) + } + defer deleteImages(repos[0]) + if err := exec.Command(dockerBinary, "inspect", repos[0]).Run(); err != nil { + t.Fatalf("Image %v was not pulled down", repos[0]) + } + for _, repo := range repos[1:] { + if err := exec.Command(dockerBinary, "inspect", repo).Run(); err == nil { + t.Fatalf("Image %v shouldn't have been pulled down", repo) + } + } + + logDone("pull - image with aliases") +} // pulling an image from the central registry should work func TestPullImageFromCentralRegistry(t *testing.T) { diff --git a/integration-cli/docker_cli_push_test.go b/integration-cli/docker_cli_push_test.go index 0dfd85a9d4..484e5db70b 100644 --- a/integration-cli/docker_cli_push_test.go +++ b/integration-cli/docker_cli_push_test.go @@ -3,39 +3,80 @@ package main import ( "fmt" "os/exec" + "strings" "testing" + "time" ) -// these tests need a freshly started empty private docker registry - // pulling an image from the central registry should work func TestPushBusyboxImage(t *testing.T) { - // skip this test until we're able to use a registry - t.Skip() + defer setupRegistry(t)() + + repoName := fmt.Sprintf("%v/dockercli/busybox", privateRegistryURL) // tag the image to upload it tot he private registry - repoName := fmt.Sprintf("%v/busybox", privateRegistryURL) tagCmd := exec.Command(dockerBinary, "tag", "busybox", repoName) if out, _, err := runCommandWithOutput(tagCmd); err != nil { t.Fatalf("image tagging failed: %s, %v", out, err) } + defer deleteImages(repoName) pushCmd := exec.Command(dockerBinary, "push", repoName) if out, _, err := runCommandWithOutput(pushCmd); err != nil { t.Fatalf("pushing the image to the private registry has failed: %s, %v", out, err) } - - deleteImages(repoName) - - logDone("push - push busybox to private registry") + logDone("push - busybox to private registry") } // pushing an image without a prefix should throw an error func TestPushUnprefixedRepo(t *testing.T) { - // skip this test until we're able to use a registry - t.Skip() pushCmd := exec.Command(dockerBinary, "push", "busybox") if out, _, err := runCommandWithOutput(pushCmd); err == nil { t.Fatalf("pushing an unprefixed repo didn't result in a non-zero exit status: %s", out) } - logDone("push - push unprefixed busybox repo --> must fail") + logDone("push - unprefixed busybox repo must fail") +} + +func TestPushUntagged(t *testing.T) { + defer setupRegistry(t)() + + repoName := fmt.Sprintf("%v/dockercli/busybox", privateRegistryURL) + + expected := "does not exist" + pushCmd := exec.Command(dockerBinary, "push", repoName) + if out, _, err := runCommandWithOutput(pushCmd); err == nil { + t.Fatalf("pushing the image to the private registry should have failed: outuput %q", out) + } else if !strings.Contains(out, expected) { + t.Fatalf("pushing the image failed with an unexpected message: expected %q, got %q", expected, out) + } + logDone("push - untagged image") +} + +func TestPushInterrupt(t *testing.T) { + defer setupRegistry(t)() + + repoName := fmt.Sprintf("%v/dockercli/busybox", privateRegistryURL) + // tag the image to upload it tot he private registry + tagCmd := exec.Command(dockerBinary, "tag", "busybox", repoName) + if out, _, err := runCommandWithOutput(tagCmd); err != nil { + t.Fatalf("image tagging failed: %s, %v", out, err) + } + defer deleteImages(repoName) + + pushCmd := exec.Command(dockerBinary, "push", repoName) + if err := pushCmd.Start(); err != nil { + t.Fatalf("Failed to start pushing to private registry: %v", err) + } + + // Interrupt push (yes, we have no idea at what point it will get killed). + time.Sleep(200 * time.Millisecond) + if err := pushCmd.Process.Kill(); err != nil { + t.Fatalf("Failed to kill push process: %v", err) + } + // Try agin + pushCmd = exec.Command(dockerBinary, "push", repoName) + if err := pushCmd.Start(); err != nil { + t.Fatalf("Failed to start pushing to private registry: %v", err) + } + + logDone("push - interrupted") } diff --git a/integration-cli/docker_utils.go b/integration-cli/docker_utils.go index c58bcfbf75..3af6d9a60e 100644 --- a/integration-cli/docker_utils.go +++ b/integration-cli/docker_utils.go @@ -864,3 +864,11 @@ func readContainerFile(containerId, filename string) ([]byte, error) { return content, nil } + +func setupRegistry(t *testing.T) func() { + reg, err := newTestRegistryV2(t) + if err != nil { + t.Fatal(err) + } + return func() { reg.Close() } +} diff --git a/integration-cli/registry.go b/integration-cli/registry.go new file mode 100644 index 0000000000..00ba3030a9 --- /dev/null +++ b/integration-cli/registry.go @@ -0,0 +1,58 @@ +package main + +import ( + "fmt" + "io/ioutil" + "os" + "os/exec" + "path/filepath" + "testing" +) + +const v2binary = "registry-v2" + +type testRegistryV2 struct { + cmd *exec.Cmd + dir string +} + +func newTestRegistryV2(t *testing.T) (*testRegistryV2, error) { + template := `version: 0.1 +loglevel: debug +storage: + filesystem: + rootdirectory: %s +http: + addr: %s` + tmp, err := ioutil.TempDir("", "registry-test-") + if err != nil { + return nil, err + } + confPath := filepath.Join(tmp, "config.yaml") + config, err := os.Create(confPath) + if err != nil { + return nil, err + } + if _, err := fmt.Fprintf(config, template, tmp, privateRegistryURL); err != nil { + os.RemoveAll(tmp) + return nil, err + } + + cmd := exec.Command(v2binary, confPath) + if err := cmd.Start(); err != nil { + os.RemoveAll(tmp) + if os.IsNotExist(err) { + t.Skip() + } + return nil, err + } + return &testRegistryV2{ + cmd: cmd, + dir: tmp, + }, nil +} + +func (r *testRegistryV2) Close() { + r.cmd.Process.Kill() + os.RemoveAll(r.dir) +} diff --git a/pkg/tarsum/tarsum.go b/pkg/tarsum/tarsum.go index c9f1315cf5..c6a7294e74 100644 --- a/pkg/tarsum/tarsum.go +++ b/pkg/tarsum/tarsum.go @@ -3,8 +3,11 @@ package tarsum import ( "bytes" "compress/gzip" + "crypto" "crypto/sha256" "encoding/hex" + "errors" + "fmt" "hash" "io" "strings" @@ -39,6 +42,30 @@ func NewTarSumHash(r io.Reader, dc bool, v Version, tHash THash) (TarSum, error) return ts, err } +// Create a new TarSum using the provided TarSum version+hash label. +func NewTarSumForLabel(r io.Reader, disableCompression bool, label string) (TarSum, error) { + parts := strings.SplitN(label, "+", 2) + if len(parts) != 2 { + return nil, errors.New("tarsum label string should be of the form: {tarsum_version}+{hash_name}") + } + + versionName, hashName := parts[0], parts[1] + + version, ok := tarSumVersionsByName[versionName] + if !ok { + return nil, fmt.Errorf("unknown TarSum version name: %q", versionName) + } + + hashConfig, ok := standardHashConfigs[hashName] + if !ok { + return nil, fmt.Errorf("unknown TarSum hash name: %q", hashName) + } + + tHash := NewTHash(hashConfig.name, hashConfig.hash.New) + + return NewTarSumHash(r, disableCompression, version, tHash) +} + // TarSum is the generic interface for calculating fixed time // checksums of a tar archive type TarSum interface { @@ -89,6 +116,18 @@ func NewTHash(name string, h func() hash.Hash) THash { return simpleTHash{n: name, h: h} } +type tHashConfig struct { + name string + hash crypto.Hash +} + +var ( + standardHashConfigs = map[string]tHashConfig{ + "sha256": {name: "sha256", hash: crypto.SHA256}, + "sha512": {name: "sha512", hash: crypto.SHA512}, + } +) + // TarSum default is "sha256" var DefaultTHash = NewTHash("sha256", sha256.New) diff --git a/pkg/tarsum/versioning.go b/pkg/tarsum/versioning.go index 3a656612ff..be1d07040f 100644 --- a/pkg/tarsum/versioning.go +++ b/pkg/tarsum/versioning.go @@ -31,11 +31,18 @@ func GetVersions() []Version { return v } -var tarSumVersions = map[Version]string{ - Version0: "tarsum", - Version1: "tarsum.v1", - VersionDev: "tarsum.dev", -} +var ( + tarSumVersions = map[Version]string{ + Version0: "tarsum", + Version1: "tarsum.v1", + VersionDev: "tarsum.dev", + } + tarSumVersionsByName = map[string]Version{ + "tarsum": Version0, + "tarsum.v1": Version1, + "tarsum.dev": VersionDev, + } +) func (tsv Version) String() string { return tarSumVersions[tsv] diff --git a/registry/auth.go b/registry/auth.go index 9d223f77ea..3207c87e82 100644 --- a/registry/auth.go +++ b/registry/auth.go @@ -10,7 +10,10 @@ import ( "os" "path" "strings" + "sync" + "time" + log "github.com/Sirupsen/logrus" "github.com/docker/docker/utils" ) @@ -36,6 +39,88 @@ type ConfigFile struct { rootPath string } +type RequestAuthorization struct { + authConfig *AuthConfig + registryEndpoint *Endpoint + resource string + scope string + actions []string + + tokenLock sync.Mutex + tokenCache string + tokenExpiration time.Time +} + +func NewRequestAuthorization(authConfig *AuthConfig, registryEndpoint *Endpoint, resource, scope string, actions []string) *RequestAuthorization { + return &RequestAuthorization{ + authConfig: authConfig, + registryEndpoint: registryEndpoint, + resource: resource, + scope: scope, + actions: actions, + } +} + +func (auth *RequestAuthorization) getToken() (string, error) { + auth.tokenLock.Lock() + defer auth.tokenLock.Unlock() + now := time.Now() + if now.Before(auth.tokenExpiration) { + log.Debugf("Using cached token for %s", auth.authConfig.Username) + return auth.tokenCache, nil + } + + client := &http.Client{ + Transport: &http.Transport{ + DisableKeepAlives: true, + Proxy: http.ProxyFromEnvironment}, + CheckRedirect: AddRequiredHeadersToRedirectedRequests, + } + factory := HTTPRequestFactory(nil) + + for _, challenge := range auth.registryEndpoint.AuthChallenges { + switch strings.ToLower(challenge.Scheme) { + case "basic": + // no token necessary + case "bearer": + log.Debugf("Getting bearer token with %s for %s", challenge.Parameters, auth.authConfig.Username) + params := map[string]string{} + for k, v := range challenge.Parameters { + params[k] = v + } + params["scope"] = fmt.Sprintf("%s:%s:%s", auth.resource, auth.scope, strings.Join(auth.actions, ",")) + token, err := getToken(auth.authConfig.Username, auth.authConfig.Password, params, auth.registryEndpoint, client, factory) + if err != nil { + return "", err + } + auth.tokenCache = token + auth.tokenExpiration = now.Add(time.Minute) + + return token, nil + default: + log.Infof("Unsupported auth scheme: %q", challenge.Scheme) + } + } + + // Do not expire cache since there are no challenges which use a token + auth.tokenExpiration = time.Now().Add(time.Hour * 24) + + return "", nil +} + +func (auth *RequestAuthorization) Authorize(req *http.Request) error { + token, err := auth.getToken() + if err != nil { + return err + } + if token != "" { + req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", token)) + } else if auth.authConfig.Username != "" && auth.authConfig.Password != "" { + req.SetBasicAuth(auth.authConfig.Username, auth.authConfig.Password) + } + return nil +} + // create a base64 encoded auth string to store in config func encodeAuth(authConfig *AuthConfig) string { authStr := authConfig.Username + ":" + authConfig.Password @@ -144,8 +229,18 @@ func SaveConfig(configFile *ConfigFile) error { return nil } -// try to register/login to the registry server -func Login(authConfig *AuthConfig, factory *utils.HTTPRequestFactory) (string, error) { +// Login tries to register/login to the registry server. +func Login(authConfig *AuthConfig, registryEndpoint *Endpoint, factory *utils.HTTPRequestFactory) (string, error) { + // Separates the v2 registry login logic from the v1 logic. + if registryEndpoint.Version == APIVersion2 { + return loginV2(authConfig, registryEndpoint, factory) + } + + return loginV1(authConfig, registryEndpoint, factory) +} + +// loginV1 tries to register/login to the v1 registry server. +func loginV1(authConfig *AuthConfig, registryEndpoint *Endpoint, factory *utils.HTTPRequestFactory) (string, error) { var ( status string reqBody []byte @@ -161,6 +256,8 @@ func Login(authConfig *AuthConfig, factory *utils.HTTPRequestFactory) (string, e serverAddress = authConfig.ServerAddress ) + log.Debugf("attempting v1 login to registry endpoint %s", registryEndpoint) + if serverAddress == "" { return "", fmt.Errorf("Server Error: Server Address not set.") } @@ -253,6 +350,103 @@ func Login(authConfig *AuthConfig, factory *utils.HTTPRequestFactory) (string, e return status, nil } +// loginV2 tries to login to the v2 registry server. The given registry endpoint has been +// pinged or setup with a list of authorization challenges. Each of these challenges are +// tried until one of them succeeds. Currently supported challenge schemes are: +// HTTP Basic Authorization +// Token Authorization with a separate token issuing server +// NOTE: the v2 logic does not attempt to create a user account if one doesn't exist. For +// now, users should create their account through other means like directly from a web page +// served by the v2 registry service provider. Whether this will be supported in the future +// is to be determined. +func loginV2(authConfig *AuthConfig, registryEndpoint *Endpoint, factory *utils.HTTPRequestFactory) (string, error) { + log.Debugf("attempting v2 login to registry endpoint %s", registryEndpoint) + + client := &http.Client{ + Transport: &http.Transport{ + DisableKeepAlives: true, + Proxy: http.ProxyFromEnvironment, + }, + CheckRedirect: AddRequiredHeadersToRedirectedRequests, + } + + var ( + err error + allErrors []error + ) + + for _, challenge := range registryEndpoint.AuthChallenges { + log.Debugf("trying %q auth challenge with params %s", challenge.Scheme, challenge.Parameters) + + switch strings.ToLower(challenge.Scheme) { + case "basic": + err = tryV2BasicAuthLogin(authConfig, challenge.Parameters, registryEndpoint, client, factory) + case "bearer": + err = tryV2TokenAuthLogin(authConfig, challenge.Parameters, registryEndpoint, client, factory) + default: + // Unsupported challenge types are explicitly skipped. + err = fmt.Errorf("unsupported auth scheme: %q", challenge.Scheme) + } + + if err == nil { + return "Login Succeeded", nil + } + + log.Debugf("error trying auth challenge %q: %s", challenge.Scheme, err) + + allErrors = append(allErrors, err) + } + + return "", fmt.Errorf("no successful auth challenge for %s - errors: %s", registryEndpoint, allErrors) +} + +func tryV2BasicAuthLogin(authConfig *AuthConfig, params map[string]string, registryEndpoint *Endpoint, client *http.Client, factory *utils.HTTPRequestFactory) error { + req, err := factory.NewRequest("GET", registryEndpoint.Path(""), nil) + if err != nil { + return err + } + + req.SetBasicAuth(authConfig.Username, authConfig.Password) + + resp, err := client.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("basic auth attempt to %s realm %q failed with status: %d %s", registryEndpoint, params["realm"], resp.StatusCode, http.StatusText(resp.StatusCode)) + } + + return nil +} + +func tryV2TokenAuthLogin(authConfig *AuthConfig, params map[string]string, registryEndpoint *Endpoint, client *http.Client, factory *utils.HTTPRequestFactory) error { + token, err := getToken(authConfig.Username, authConfig.Password, params, registryEndpoint, client, factory) + if err != nil { + return err + } + + req, err := factory.NewRequest("GET", registryEndpoint.Path(""), nil) + if err != nil { + return err + } + + req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", token)) + + resp, err := client.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("token auth attempt to %s realm %q failed with status: %d %s", registryEndpoint, params["realm"], resp.StatusCode, http.StatusText(resp.StatusCode)) + } + + return nil +} + // this method matches a auth configuration to a server address or a url func (config *ConfigFile) ResolveAuthConfig(index *IndexInfo) AuthConfig { configKey := index.GetAuthConfigKey() diff --git a/registry/authchallenge.go b/registry/authchallenge.go new file mode 100644 index 0000000000..e300d82a05 --- /dev/null +++ b/registry/authchallenge.go @@ -0,0 +1,150 @@ +package registry + +import ( + "net/http" + "strings" +) + +// Octet types from RFC 2616. +type octetType byte + +// AuthorizationChallenge carries information +// from a WWW-Authenticate response header. +type AuthorizationChallenge struct { + Scheme string + Parameters map[string]string +} + +var octetTypes [256]octetType + +const ( + isToken octetType = 1 << iota + isSpace +) + +func init() { + // OCTET = + // CHAR = + // CTL = + // CR = + // LF = + // SP = + // HT = + // <"> = + // CRLF = CR LF + // LWS = [CRLF] 1*( SP | HT ) + // TEXT = + // separators = "(" | ")" | "<" | ">" | "@" | "," | ";" | ":" | "\" | <"> + // | "/" | "[" | "]" | "?" | "=" | "{" | "}" | SP | HT + // token = 1* + // qdtext = > + + for c := 0; c < 256; c++ { + var t octetType + isCtl := c <= 31 || c == 127 + isChar := 0 <= c && c <= 127 + isSeparator := strings.IndexRune(" \t\"(),/:;<=>?@[]\\{}", rune(c)) >= 0 + if strings.IndexRune(" \t\r\n", rune(c)) >= 0 { + t |= isSpace + } + if isChar && !isCtl && !isSeparator { + t |= isToken + } + octetTypes[c] = t + } +} + +func parseAuthHeader(header http.Header) []*AuthorizationChallenge { + var challenges []*AuthorizationChallenge + for _, h := range header[http.CanonicalHeaderKey("WWW-Authenticate")] { + v, p := parseValueAndParams(h) + if v != "" { + challenges = append(challenges, &AuthorizationChallenge{Scheme: v, Parameters: p}) + } + } + return challenges +} + +func parseValueAndParams(header string) (value string, params map[string]string) { + params = make(map[string]string) + value, s := expectToken(header) + if value == "" { + return + } + value = strings.ToLower(value) + s = "," + skipSpace(s) + for strings.HasPrefix(s, ",") { + var pkey string + pkey, s = expectToken(skipSpace(s[1:])) + if pkey == "" { + return + } + if !strings.HasPrefix(s, "=") { + return + } + var pvalue string + pvalue, s = expectTokenOrQuoted(s[1:]) + if pvalue == "" { + return + } + pkey = strings.ToLower(pkey) + params[pkey] = pvalue + s = skipSpace(s) + } + return +} + +func skipSpace(s string) (rest string) { + i := 0 + for ; i < len(s); i++ { + if octetTypes[s[i]]&isSpace == 0 { + break + } + } + return s[i:] +} + +func expectToken(s string) (token, rest string) { + i := 0 + for ; i < len(s); i++ { + if octetTypes[s[i]]&isToken == 0 { + break + } + } + return s[:i], s[i:] +} + +func expectTokenOrQuoted(s string) (value string, rest string) { + if !strings.HasPrefix(s, "\"") { + return expectToken(s) + } + s = s[1:] + for i := 0; i < len(s); i++ { + switch s[i] { + case '"': + return s[:i], s[i+1:] + case '\\': + p := make([]byte, len(s)-1) + j := copy(p, s[:i]) + escape := true + for i = i + i; i < len(s); i++ { + b := s[i] + switch { + case escape: + escape = false + p[j] = b + j++ + case b == '\\': + escape = true + case b == '"': + return string(p[:j]), s[i+1:] + default: + p[j] = b + j++ + } + } + return "", "" + } + } + return "", "" +} diff --git a/registry/config.go b/registry/config.go index b5652b15d8..4d13aaea35 100644 --- a/registry/config.go +++ b/registry/config.go @@ -23,7 +23,7 @@ type Options struct { const ( // Only used for user auth + account creation INDEXSERVER = "https://index.docker.io/v1/" - REGISTRYSERVER = "https://registry-1.docker.io/v1/" + REGISTRYSERVER = "https://registry-1.docker.io/v2/" INDEXNAME = "docker.io" // INDEXSERVER = "https://registry-stage.hub.docker.com/v1/" diff --git a/registry/endpoint.go b/registry/endpoint.go index 95680c5efc..9ca9ed8b9a 100644 --- a/registry/endpoint.go +++ b/registry/endpoint.go @@ -10,115 +10,170 @@ import ( "strings" log "github.com/Sirupsen/logrus" + "github.com/docker/docker/registry/v2" ) // for mocking in unit tests var lookupIP = net.LookupIP -// scans string for api version in the URL path. returns the trimmed hostname, if version found, string and API version. -func scanForAPIVersion(hostname string) (string, APIVersion) { +// scans string for api version in the URL path. returns the trimmed address, if version found, string and API version. +func scanForAPIVersion(address string) (string, APIVersion) { var ( chunks []string apiVersionStr string ) - if strings.HasSuffix(hostname, "/") { - chunks = strings.Split(hostname[:len(hostname)-1], "/") - apiVersionStr = chunks[len(chunks)-1] - } else { - chunks = strings.Split(hostname, "/") - apiVersionStr = chunks[len(chunks)-1] + + if strings.HasSuffix(address, "/") { + address = address[:len(address)-1] } + + chunks = strings.Split(address, "/") + apiVersionStr = chunks[len(chunks)-1] + for k, v := range apiVersions { if apiVersionStr == v { - hostname = strings.Join(chunks[:len(chunks)-1], "/") - return hostname, k + address = strings.Join(chunks[:len(chunks)-1], "/") + return address, k } } - return hostname, DefaultAPIVersion + + return address, APIVersionUnknown } +// NewEndpoint parses the given address to return a registry endpoint. func NewEndpoint(index *IndexInfo) (*Endpoint, error) { // *TODO: Allow per-registry configuration of endpoints. endpoint, err := newEndpoint(index.GetAuthConfigKey(), index.Secure) if err != nil { return nil, err } + if err := validateEndpoint(endpoint); err != nil { + return nil, err + } + + return endpoint, nil +} + +func validateEndpoint(endpoint *Endpoint) error { + log.Debugf("pinging registry endpoint %s", endpoint) // Try HTTPS ping to registry endpoint.URL.Scheme = "https" if _, err := endpoint.Ping(); err != nil { - - //TODO: triggering highland build can be done there without "failing" - - if index.Secure { + if endpoint.IsSecure { // If registry is secure and HTTPS failed, show user the error and tell them about `--insecure-registry` // in case that's what they need. DO NOT accept unknown CA certificates, and DO NOT fallback to HTTP. - return nil, fmt.Errorf("Invalid registry endpoint %s: %v. If this private registry supports only HTTP or HTTPS with an unknown CA certificate, please add `--insecure-registry %s` to the daemon's arguments. In the case of HTTPS, if you have access to the registry's CA certificate, no need for the flag; simply place the CA certificate at /etc/docker/certs.d/%s/ca.crt", endpoint, err, endpoint.URL.Host, endpoint.URL.Host) + return fmt.Errorf("invalid registry endpoint %s: %v. If this private registry supports only HTTP or HTTPS with an unknown CA certificate, please add `--insecure-registry %s` to the daemon's arguments. In the case of HTTPS, if you have access to the registry's CA certificate, no need for the flag; simply place the CA certificate at /etc/docker/certs.d/%s/ca.crt", endpoint, err, endpoint.URL.Host, endpoint.URL.Host) } // If registry is insecure and HTTPS failed, fallback to HTTP. log.Debugf("Error from registry %q marked as insecure: %v. Insecurely falling back to HTTP", endpoint, err) endpoint.URL.Scheme = "http" - _, err2 := endpoint.Ping() - if err2 == nil { - return endpoint, nil + + var err2 error + if _, err2 = endpoint.Ping(); err2 == nil { + return nil } - return nil, fmt.Errorf("Invalid registry endpoint %q. HTTPS attempt: %v. HTTP attempt: %v", endpoint, err, err2) + return fmt.Errorf("invalid registry endpoint %q. HTTPS attempt: %v. HTTP attempt: %v", endpoint, err, err2) } - return endpoint, nil + return nil } -func newEndpoint(hostname string, secure bool) (*Endpoint, error) { + +func newEndpoint(address string, secure bool) (*Endpoint, error) { var ( - endpoint = Endpoint{} - trimmedHostname string - err error + endpoint = new(Endpoint) + trimmedAddress string + err error ) - if !strings.HasPrefix(hostname, "http") { - hostname = "https://" + hostname + + if !strings.HasPrefix(address, "http") { + address = "https://" + address } - trimmedHostname, endpoint.Version = scanForAPIVersion(hostname) - endpoint.URL, err = url.Parse(trimmedHostname) - if err != nil { + + trimmedAddress, endpoint.Version = scanForAPIVersion(address) + + if endpoint.URL, err = url.Parse(trimmedAddress); err != nil { return nil, err } - endpoint.secure = secure - return &endpoint, nil + endpoint.IsSecure = secure + return endpoint, nil } func (repoInfo *RepositoryInfo) GetEndpoint() (*Endpoint, error) { return NewEndpoint(repoInfo.Index) } +// Endpoint stores basic information about a registry endpoint. type Endpoint struct { - URL *url.URL - Version APIVersion - secure bool + URL *url.URL + Version APIVersion + IsSecure bool + AuthChallenges []*AuthorizationChallenge + URLBuilder *v2.URLBuilder } // Get the formated URL for the root of this registry Endpoint -func (e Endpoint) String() string { - return fmt.Sprintf("%s/v%d/", e.URL.String(), e.Version) +func (e *Endpoint) String() string { + return fmt.Sprintf("%s/v%d/", e.URL, e.Version) } -func (e Endpoint) VersionString(version APIVersion) string { - return fmt.Sprintf("%s/v%d/", e.URL.String(), version) +// VersionString returns a formatted string of this +// endpoint address using the given API Version. +func (e *Endpoint) VersionString(version APIVersion) string { + return fmt.Sprintf("%s/v%d/", e.URL, version) } -func (e Endpoint) Ping() (RegistryInfo, error) { +// Path returns a formatted string for the URL +// of this endpoint with the given path appended. +func (e *Endpoint) Path(path string) string { + return fmt.Sprintf("%s/v%d/%s", e.URL, e.Version, path) +} + +func (e *Endpoint) Ping() (RegistryInfo, error) { + // The ping logic to use is determined by the registry endpoint version. + switch e.Version { + case APIVersion1: + return e.pingV1() + case APIVersion2: + return e.pingV2() + } + + // APIVersionUnknown + // We should try v2 first... + e.Version = APIVersion2 + regInfo, errV2 := e.pingV2() + if errV2 == nil { + return regInfo, nil + } + + // ... then fallback to v1. + e.Version = APIVersion1 + regInfo, errV1 := e.pingV1() + if errV1 == nil { + return regInfo, nil + } + + e.Version = APIVersionUnknown + return RegistryInfo{}, fmt.Errorf("unable to ping registry endpoint %s\nv2 ping attempt failed with error: %s\n v1 ping attempt failed with error: %s", e, errV2, errV1) +} + +func (e *Endpoint) pingV1() (RegistryInfo, error) { + log.Debugf("attempting v1 ping for registry endpoint %s", e) + if e.String() == IndexServerAddress() { - // Skip the check, we now this one is valid + // Skip the check, we know this one is valid // (and we never want to fallback to http in case of error) return RegistryInfo{Standalone: false}, nil } - req, err := http.NewRequest("GET", e.String()+"_ping", nil) + req, err := http.NewRequest("GET", e.Path("_ping"), nil) if err != nil { return RegistryInfo{Standalone: false}, err } - resp, _, err := doRequest(req, nil, ConnectTimeout, e.secure) + resp, _, err := doRequest(req, nil, ConnectTimeout, e.IsSecure) if err != nil { return RegistryInfo{Standalone: false}, err } @@ -127,7 +182,7 @@ func (e Endpoint) Ping() (RegistryInfo, error) { jsonString, err := ioutil.ReadAll(resp.Body) if err != nil { - return RegistryInfo{Standalone: false}, fmt.Errorf("Error while reading the http response: %s", err) + return RegistryInfo{Standalone: false}, fmt.Errorf("error while reading the http response: %s", err) } // If the header is absent, we assume true for compatibility with earlier @@ -157,3 +212,33 @@ func (e Endpoint) Ping() (RegistryInfo, error) { log.Debugf("RegistryInfo.Standalone: %t", info.Standalone) return info, nil } + +func (e *Endpoint) pingV2() (RegistryInfo, error) { + log.Debugf("attempting v2 ping for registry endpoint %s", e) + + req, err := http.NewRequest("GET", e.Path(""), nil) + if err != nil { + return RegistryInfo{}, err + } + + resp, _, err := doRequest(req, nil, ConnectTimeout, e.IsSecure) + if err != nil { + return RegistryInfo{}, err + } + defer resp.Body.Close() + + if resp.StatusCode == http.StatusOK { + // It would seem that no authentication/authorization is required. + // So we don't need to parse/add any authorization schemes. + return RegistryInfo{Standalone: true}, nil + } + + if resp.StatusCode == http.StatusUnauthorized { + // Parse the WWW-Authenticate Header and store the challenges + // on this endpoint object. + e.AuthChallenges = parseAuthHeader(resp.Header) + return RegistryInfo{}, nil + } + + return RegistryInfo{}, fmt.Errorf("v2 registry endpoint returned status %d: %q", resp.StatusCode, http.StatusText(resp.StatusCode)) +} diff --git a/registry/endpoint_test.go b/registry/endpoint_test.go index b691a4fb98..f6489034fe 100644 --- a/registry/endpoint_test.go +++ b/registry/endpoint_test.go @@ -8,8 +8,10 @@ func TestEndpointParse(t *testing.T) { expected string }{ {IndexServerAddress(), IndexServerAddress()}, - {"http://0.0.0.0:5000", "http://0.0.0.0:5000/v1/"}, - {"0.0.0.0:5000", "https://0.0.0.0:5000/v1/"}, + {"http://0.0.0.0:5000/v1/", "http://0.0.0.0:5000/v1/"}, + {"http://0.0.0.0:5000/v2/", "http://0.0.0.0:5000/v2/"}, + {"http://0.0.0.0:5000", "http://0.0.0.0:5000/v0/"}, + {"0.0.0.0:5000", "https://0.0.0.0:5000/v0/"}, } for _, td := range testData { e, err := newEndpoint(td.str, false) diff --git a/registry/service.go b/registry/service.go index c34e384236..0483402248 100644 --- a/registry/service.go +++ b/registry/service.go @@ -1,6 +1,7 @@ package registry import ( + log "github.com/Sirupsen/logrus" "github.com/docker/docker/engine" ) @@ -38,28 +39,39 @@ func (s *Service) Install(eng *engine.Engine) error { // and returns OK if authentication was sucessful. // It can be used to verify the validity of a client's credentials. func (s *Service) Auth(job *engine.Job) engine.Status { - var authConfig = new(AuthConfig) + var ( + authConfig = new(AuthConfig) + endpoint *Endpoint + index *IndexInfo + status string + err error + ) job.GetenvJson("authConfig", authConfig) - if authConfig.ServerAddress != "" { - index, err := ResolveIndexInfo(job, authConfig.ServerAddress) - if err != nil { - return job.Error(err) - } - if !index.Official { - endpoint, err := NewEndpoint(index) - if err != nil { - return job.Error(err) - } - authConfig.ServerAddress = endpoint.String() - } + addr := authConfig.ServerAddress + if addr == "" { + // Use the official registry address if not specified. + addr = IndexServerAddress() } - status, err := Login(authConfig, HTTPRequestFactory(nil)) - if err != nil { + if index, err = ResolveIndexInfo(job, addr); err != nil { return job.Error(err) } + + if endpoint, err = NewEndpoint(index); err != nil { + log.Errorf("unable to get new registry endpoint: %s", err) + return job.Error(err) + } + + authConfig.ServerAddress = endpoint.String() + + if status, err = Login(authConfig, endpoint, HTTPRequestFactory(nil)); err != nil { + log.Errorf("unable to login against registry endpoint %s: %s", endpoint, err) + return job.Error(err) + } + + log.Infof("successful registry login for endpoint %s: %s", endpoint, status) job.Printf("%s\n", status) return engine.StatusOK diff --git a/registry/session.go b/registry/session.go index 781a91b157..b1980e1ae6 100644 --- a/registry/session.go +++ b/registry/session.go @@ -65,7 +65,7 @@ func NewSession(authConfig *AuthConfig, factory *utils.HTTPRequestFactory, endpo } func (r *Session) doRequest(req *http.Request) (*http.Response, *http.Client, error) { - return doRequest(req, r.jar, r.timeout, r.indexEndpoint.secure) + return doRequest(req, r.jar, r.timeout, r.indexEndpoint.IsSecure) } // Retrieve the history of a given image from the Registry. diff --git a/registry/session_v2.go b/registry/session_v2.go index 20e9e2ee9c..11b96bd65a 100644 --- a/registry/session_v2.go +++ b/registry/session_v2.go @@ -5,104 +5,55 @@ import ( "fmt" "io" "io/ioutil" - "net/url" "strconv" log "github.com/Sirupsen/logrus" + "github.com/docker/docker/registry/v2" "github.com/docker/docker/utils" - "github.com/gorilla/mux" ) -func newV2RegistryRouter() *mux.Router { - router := mux.NewRouter() - - v2Router := router.PathPrefix("/v2/").Subrouter() - - // Version Info - v2Router.Path("/version").Name("version") - - // Image Manifests - v2Router.Path("/manifest/{imagename:[a-z0-9-._/]+}/{tagname:[a-zA-Z0-9-._]+}").Name("manifests") - - // List Image Tags - v2Router.Path("/tags/{imagename:[a-z0-9-._/]+}").Name("tags") - - // Download a blob - v2Router.Path("/blob/{imagename:[a-z0-9-._/]+}/{sumtype:[a-z0-9._+-]+}/{sum:[a-fA-F0-9]{4,}}").Name("downloadBlob") - - // Upload a blob - v2Router.Path("/blob/{imagename:[a-z0-9-._/]+}/{sumtype:[a-z0-9._+-]+}").Name("uploadBlob") - - // Mounting a blob in an image - v2Router.Path("/mountblob/{imagename:[a-z0-9-._/]+}/{sumtype:[a-z0-9._+-]+}/{sum:[a-fA-F0-9]{4,}}").Name("mountBlob") - - return router +func getV2Builder(e *Endpoint) *v2.URLBuilder { + if e.URLBuilder == nil { + e.URLBuilder = v2.NewURLBuilder(e.URL) + } + return e.URLBuilder } -// APIVersion2 /v2/ -var v2HTTPRoutes = newV2RegistryRouter() - -func getV2URL(e *Endpoint, routeName string, vars map[string]string) (*url.URL, error) { - route := v2HTTPRoutes.Get(routeName) - if route == nil { - return nil, fmt.Errorf("unknown regisry v2 route name: %q", routeName) +func (r *Session) V2RegistryEndpoint(index *IndexInfo) (ep *Endpoint, err error) { + // TODO check if should use Mirror + if index.Official { + ep, err = newEndpoint(REGISTRYSERVER, true) + if err != nil { + return + } + err = validateEndpoint(ep) + if err != nil { + return + } + } else if r.indexEndpoint.String() == index.GetAuthConfigKey() { + ep = r.indexEndpoint + } else { + ep, err = NewEndpoint(index) + if err != nil { + return + } } - varReplace := make([]string, 0, len(vars)*2) - for key, val := range vars { - varReplace = append(varReplace, key, val) - } - - routePath, err := route.URLPath(varReplace...) - if err != nil { - return nil, fmt.Errorf("unable to make registry route %q with vars %v: %s", routeName, vars, err) - } - u, err := url.Parse(REGISTRYSERVER) - if err != nil { - return nil, fmt.Errorf("invalid registry url: %s", err) - } - - return &url.URL{ - Scheme: u.Scheme, - Host: u.Host, - Path: routePath.Path, - }, nil + ep.URLBuilder = v2.NewURLBuilder(ep.URL) + return } -// V2 Provenance POC - -func (r *Session) GetV2Version(token []string) (*RegistryInfo, error) { - routeURL, err := getV2URL(r.indexEndpoint, "version", nil) - if err != nil { - return nil, err +// GetV2Authorization gets the authorization needed to the given image +// If readonly access is requested, then only the authorization may +// only be used for Get operations. +func (r *Session) GetV2Authorization(ep *Endpoint, imageName string, readOnly bool) (auth *RequestAuthorization, err error) { + scopes := []string{"pull"} + if !readOnly { + scopes = append(scopes, "push") } - method := "GET" - log.Debugf("[registry] Calling %q %s", method, routeURL.String()) - - req, err := r.reqFactory.NewRequest(method, routeURL.String(), nil) - if err != nil { - return nil, err - } - setTokenAuth(req, token) - res, _, err := r.doRequest(req) - if err != nil { - return nil, err - } - defer res.Body.Close() - if res.StatusCode != 200 { - return nil, utils.NewHTTPRequestError(fmt.Sprintf("Server error: %d fetching Version", res.StatusCode), res) - } - - decoder := json.NewDecoder(res.Body) - versionInfo := new(RegistryInfo) - - err = decoder.Decode(versionInfo) - if err != nil { - return nil, fmt.Errorf("unable to decode GetV2Version JSON response: %s", err) - } - - return versionInfo, nil + log.Debugf("Getting authorization for %s %s", imageName, scopes) + return NewRequestAuthorization(r.GetAuthConfig(true), ep, "repository", imageName, scopes), nil } // @@ -112,25 +63,22 @@ func (r *Session) GetV2Version(token []string) (*RegistryInfo, error) { // 1.c) if anything else, err // 2) PUT the created/signed manifest // -func (r *Session) GetV2ImageManifest(imageName, tagName string, token []string) ([]byte, error) { - vars := map[string]string{ - "imagename": imageName, - "tagname": tagName, - } - - routeURL, err := getV2URL(r.indexEndpoint, "manifests", vars) +func (r *Session) GetV2ImageManifest(ep *Endpoint, imageName, tagName string, auth *RequestAuthorization) ([]byte, error) { + routeURL, err := getV2Builder(ep).BuildManifestURL(imageName, tagName) if err != nil { return nil, err } method := "GET" - log.Debugf("[registry] Calling %q %s", method, routeURL.String()) + log.Debugf("[registry] Calling %q %s", method, routeURL) - req, err := r.reqFactory.NewRequest(method, routeURL.String(), nil) + req, err := r.reqFactory.NewRequest(method, routeURL, nil) if err != nil { return nil, err } - setTokenAuth(req, token) + if err := auth.Authorize(req); err != nil { + return nil, err + } res, _, err := r.doRequest(req) if err != nil { return nil, err @@ -152,29 +100,25 @@ func (r *Session) GetV2ImageManifest(imageName, tagName string, token []string) return buf, nil } -// - Succeeded to mount for this image scope -// - Failed with no error (So continue to Push the Blob) +// - Succeeded to head image blob (already exists) +// - Failed with no error (continue to Push the Blob) // - Failed with error -func (r *Session) PostV2ImageMountBlob(imageName, sumType, sum string, token []string) (bool, error) { - vars := map[string]string{ - "imagename": imageName, - "sumtype": sumType, - "sum": sum, - } - - routeURL, err := getV2URL(r.indexEndpoint, "mountBlob", vars) +func (r *Session) HeadV2ImageBlob(ep *Endpoint, imageName, sumType, sum string, auth *RequestAuthorization) (bool, error) { + routeURL, err := getV2Builder(ep).BuildBlobURL(imageName, sumType+":"+sum) if err != nil { return false, err } - method := "POST" - log.Debugf("[registry] Calling %q %s", method, routeURL.String()) + method := "HEAD" + log.Debugf("[registry] Calling %q %s", method, routeURL) - req, err := r.reqFactory.NewRequest(method, routeURL.String(), nil) + req, err := r.reqFactory.NewRequest(method, routeURL, nil) if err != nil { return false, err } - setTokenAuth(req, token) + if err := auth.Authorize(req); err != nil { + return false, err + } res, _, err := r.doRequest(req) if err != nil { return false, err @@ -184,32 +128,28 @@ func (r *Session) PostV2ImageMountBlob(imageName, sumType, sum string, token []s case 200: // return something indicating no push needed return true, nil - case 300: + case 404: // return something indicating blob push needed return false, nil } return false, fmt.Errorf("Failed to mount %q - %s:%s : %d", imageName, sumType, sum, res.StatusCode) } -func (r *Session) GetV2ImageBlob(imageName, sumType, sum string, blobWrtr io.Writer, token []string) error { - vars := map[string]string{ - "imagename": imageName, - "sumtype": sumType, - "sum": sum, - } - - routeURL, err := getV2URL(r.indexEndpoint, "downloadBlob", vars) +func (r *Session) GetV2ImageBlob(ep *Endpoint, imageName, sumType, sum string, blobWrtr io.Writer, auth *RequestAuthorization) error { + routeURL, err := getV2Builder(ep).BuildBlobURL(imageName, sumType+":"+sum) if err != nil { return err } method := "GET" - log.Debugf("[registry] Calling %q %s", method, routeURL.String()) - req, err := r.reqFactory.NewRequest(method, routeURL.String(), nil) + log.Debugf("[registry] Calling %q %s", method, routeURL) + req, err := r.reqFactory.NewRequest(method, routeURL, nil) if err != nil { return err } - setTokenAuth(req, token) + if err := auth.Authorize(req); err != nil { + return err + } res, _, err := r.doRequest(req) if err != nil { return err @@ -226,25 +166,21 @@ func (r *Session) GetV2ImageBlob(imageName, sumType, sum string, blobWrtr io.Wri return err } -func (r *Session) GetV2ImageBlobReader(imageName, sumType, sum string, token []string) (io.ReadCloser, int64, error) { - vars := map[string]string{ - "imagename": imageName, - "sumtype": sumType, - "sum": sum, - } - - routeURL, err := getV2URL(r.indexEndpoint, "downloadBlob", vars) +func (r *Session) GetV2ImageBlobReader(ep *Endpoint, imageName, sumType, sum string, auth *RequestAuthorization) (io.ReadCloser, int64, error) { + routeURL, err := getV2Builder(ep).BuildBlobURL(imageName, sumType+":"+sum) if err != nil { return nil, 0, err } method := "GET" - log.Debugf("[registry] Calling %q %s", method, routeURL.String()) - req, err := r.reqFactory.NewRequest(method, routeURL.String(), nil) + log.Debugf("[registry] Calling %q %s", method, routeURL) + req, err := r.reqFactory.NewRequest(method, routeURL, nil) if err != nil { return nil, 0, err } - setTokenAuth(req, token) + if err := auth.Authorize(req); err != nil { + return nil, 0, err + } res, _, err := r.doRequest(req) if err != nil { return nil, 0, err @@ -267,105 +203,110 @@ func (r *Session) GetV2ImageBlobReader(imageName, sumType, sum string, token []s // Push the image to the server for storage. // 'layer' is an uncompressed reader of the blob to be pushed. // The server will generate it's own checksum calculation. -func (r *Session) PutV2ImageBlob(imageName, sumType string, blobRdr io.Reader, token []string) (serverChecksum string, err error) { - vars := map[string]string{ - "imagename": imageName, - "sumtype": sumType, +func (r *Session) PutV2ImageBlob(ep *Endpoint, imageName, sumType, sumStr string, blobRdr io.Reader, auth *RequestAuthorization) error { + routeURL, err := getV2Builder(ep).BuildBlobUploadURL(imageName) + if err != nil { + return err } - routeURL, err := getV2URL(r.indexEndpoint, "uploadBlob", vars) + log.Debugf("[registry] Calling %q %s", "POST", routeURL) + req, err := r.reqFactory.NewRequest("POST", routeURL, nil) if err != nil { - return "", err + return err } - method := "PUT" - log.Debugf("[registry] Calling %q %s", method, routeURL.String()) - req, err := r.reqFactory.NewRequest(method, routeURL.String(), blobRdr) - if err != nil { - return "", err + if err := auth.Authorize(req); err != nil { + return err } - setTokenAuth(req, token) res, _, err := r.doRequest(req) if err != nil { - return "", err + return err + } + location := res.Header.Get("Location") + + method := "PUT" + log.Debugf("[registry] Calling %q %s", method, location) + req, err = r.reqFactory.NewRequest(method, location, blobRdr) + if err != nil { + return err + } + queryParams := req.URL.Query() + queryParams.Add("digest", sumType+":"+sumStr) + req.URL.RawQuery = queryParams.Encode() + if err := auth.Authorize(req); err != nil { + return err + } + res, _, err = r.doRequest(req) + if err != nil { + return err } defer res.Body.Close() - if res.StatusCode != 201 { - if res.StatusCode == 401 { - return "", errLoginRequired - } - return "", utils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to push %s blob", res.StatusCode, imageName), res) - } - type sumReturn struct { - Checksum string `json:"checksum"` - } - - decoder := json.NewDecoder(res.Body) - var sumInfo sumReturn - - err = decoder.Decode(&sumInfo) - if err != nil { - return "", fmt.Errorf("unable to decode PutV2ImageBlob JSON response: %s", err) - } - - // XXX this is a json struct from the registry, with its checksum - return sumInfo.Checksum, nil -} - -// Finally Push the (signed) manifest of the blobs we've just pushed -func (r *Session) PutV2ImageManifest(imageName, tagName string, manifestRdr io.Reader, token []string) error { - vars := map[string]string{ - "imagename": imageName, - "tagname": tagName, - } - - routeURL, err := getV2URL(r.indexEndpoint, "manifests", vars) - if err != nil { - return err - } - - method := "PUT" - log.Debugf("[registry] Calling %q %s", method, routeURL.String()) - req, err := r.reqFactory.NewRequest(method, routeURL.String(), manifestRdr) - if err != nil { - return err - } - setTokenAuth(req, token) - res, _, err := r.doRequest(req) - if err != nil { - return err - } - res.Body.Close() if res.StatusCode != 201 { if res.StatusCode == 401 { return errLoginRequired } + return utils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to push %s blob", res.StatusCode, imageName), res) + } + + return nil +} + +// Finally Push the (signed) manifest of the blobs we've just pushed +func (r *Session) PutV2ImageManifest(ep *Endpoint, imageName, tagName string, manifestRdr io.Reader, auth *RequestAuthorization) error { + routeURL, err := getV2Builder(ep).BuildManifestURL(imageName, tagName) + if err != nil { + return err + } + + method := "PUT" + log.Debugf("[registry] Calling %q %s", method, routeURL) + req, err := r.reqFactory.NewRequest(method, routeURL, manifestRdr) + if err != nil { + return err + } + if err := auth.Authorize(req); err != nil { + return err + } + res, _, err := r.doRequest(req) + if err != nil { + return err + } + b, _ := ioutil.ReadAll(res.Body) + res.Body.Close() + if res.StatusCode != 200 { + if res.StatusCode == 401 { + return errLoginRequired + } + log.Debugf("Unexpected response from server: %q %#v", b, res.Header) return utils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to push %s:%s manifest", res.StatusCode, imageName, tagName), res) } return nil } -// Given a repository name, returns a json array of string tags -func (r *Session) GetV2RemoteTags(imageName string, token []string) ([]string, error) { - vars := map[string]string{ - "imagename": imageName, - } +type remoteTags struct { + name string + tags []string +} - routeURL, err := getV2URL(r.indexEndpoint, "tags", vars) +// Given a repository name, returns a json array of string tags +func (r *Session) GetV2RemoteTags(ep *Endpoint, imageName string, auth *RequestAuthorization) ([]string, error) { + routeURL, err := getV2Builder(ep).BuildTagsURL(imageName) if err != nil { return nil, err } method := "GET" - log.Debugf("[registry] Calling %q %s", method, routeURL.String()) + log.Debugf("[registry] Calling %q %s", method, routeURL) - req, err := r.reqFactory.NewRequest(method, routeURL.String(), nil) + req, err := r.reqFactory.NewRequest(method, routeURL, nil) if err != nil { return nil, err } - setTokenAuth(req, token) + if err := auth.Authorize(req); err != nil { + return nil, err + } res, _, err := r.doRequest(req) if err != nil { return nil, err @@ -381,10 +322,10 @@ func (r *Session) GetV2RemoteTags(imageName string, token []string) ([]string, e } decoder := json.NewDecoder(res.Body) - var tags []string - err = decoder.Decode(&tags) + var remote remoteTags + err = decoder.Decode(&remote) if err != nil { return nil, fmt.Errorf("Error while decoding the http response: %s", err) } - return tags, nil + return remote.tags, nil } diff --git a/registry/token.go b/registry/token.go new file mode 100644 index 0000000000..2504863048 --- /dev/null +++ b/registry/token.go @@ -0,0 +1,81 @@ +package registry + +import ( + "encoding/json" + "errors" + "fmt" + "net/http" + "net/url" + "strings" + + "github.com/docker/docker/utils" +) + +type tokenResponse struct { + Token string `json:"token"` +} + +func getToken(username, password string, params map[string]string, registryEndpoint *Endpoint, client *http.Client, factory *utils.HTTPRequestFactory) (token string, err error) { + realm, ok := params["realm"] + if !ok { + return "", errors.New("no realm specified for token auth challenge") + } + + realmURL, err := url.Parse(realm) + if err != nil { + return "", fmt.Errorf("invalid token auth challenge realm: %s", err) + } + + if realmURL.Scheme == "" { + if registryEndpoint.IsSecure { + realmURL.Scheme = "https" + } else { + realmURL.Scheme = "http" + } + } + + req, err := factory.NewRequest("GET", realmURL.String(), nil) + if err != nil { + return "", err + } + + reqParams := req.URL.Query() + service := params["service"] + scope := params["scope"] + + if service != "" { + reqParams.Add("service", service) + } + + for _, scopeField := range strings.Fields(scope) { + reqParams.Add("scope", scopeField) + } + + reqParams.Add("account", username) + + req.URL.RawQuery = reqParams.Encode() + req.SetBasicAuth(username, password) + + resp, err := client.Do(req) + if err != nil { + return "", err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("token auth attempt for registry %s: %s request failed with status: %d %s", registryEndpoint, req.URL, resp.StatusCode, http.StatusText(resp.StatusCode)) + } + + decoder := json.NewDecoder(resp.Body) + + tr := new(tokenResponse) + if err = decoder.Decode(tr); err != nil { + return "", fmt.Errorf("unable to decode token response: %s", err) + } + + if tr.Token == "" { + return "", errors.New("authorization server did not include a token in the response") + } + + return tr.Token, nil +} diff --git a/registry/types.go b/registry/types.go index fbbc0e7098..bd0bf8b75b 100644 --- a/registry/types.go +++ b/registry/types.go @@ -55,14 +55,15 @@ func (av APIVersion) String() string { return apiVersions[av] } -var DefaultAPIVersion APIVersion = APIVersion1 var apiVersions = map[APIVersion]string{ 1: "v1", 2: "v2", } +// API Version identifiers. const ( - APIVersion1 = iota + 1 + APIVersionUnknown = iota + APIVersion1 APIVersion2 ) diff --git a/registry/v2/descriptors.go b/registry/v2/descriptors.go new file mode 100644 index 0000000000..68d182411d --- /dev/null +++ b/registry/v2/descriptors.go @@ -0,0 +1,144 @@ +package v2 + +import "net/http" + +// TODO(stevvooe): Add route descriptors for each named route, along with +// accepted methods, parameters, returned status codes and error codes. + +// ErrorDescriptor provides relevant information about a given error code. +type ErrorDescriptor struct { + // Code is the error code that this descriptor describes. + Code ErrorCode + + // Value provides a unique, string key, often captilized with + // underscores, to identify the error code. This value is used as the + // keyed value when serializing api errors. + Value string + + // Message is a short, human readable decription of the error condition + // included in API responses. + Message string + + // Description provides a complete account of the errors purpose, suitable + // for use in documentation. + Description string + + // HTTPStatusCodes provides a list of status under which this error + // condition may arise. If it is empty, the error condition may be seen + // for any status code. + HTTPStatusCodes []int +} + +// ErrorDescriptors provides a list of HTTP API Error codes that may be +// encountered when interacting with the registry API. +var ErrorDescriptors = []ErrorDescriptor{ + { + Code: ErrorCodeUnknown, + Value: "UNKNOWN", + Message: "unknown error", + Description: `Generic error returned when the error does not have an + API classification.`, + }, + { + Code: ErrorCodeDigestInvalid, + Value: "DIGEST_INVALID", + Message: "provided digest did not match uploaded content", + Description: `When a blob is uploaded, the registry will check that + the content matches the digest provided by the client. The error may + include a detail structure with the key "digest", including the + invalid digest string. This error may also be returned when a manifest + includes an invalid layer digest.`, + HTTPStatusCodes: []int{http.StatusBadRequest, http.StatusNotFound}, + }, + { + Code: ErrorCodeSizeInvalid, + Value: "SIZE_INVALID", + Message: "provided length did not match content length", + Description: `When a layer is uploaded, the provided size will be + checked against the uploaded content. If they do not match, this error + will be returned.`, + HTTPStatusCodes: []int{http.StatusBadRequest}, + }, + { + Code: ErrorCodeNameInvalid, + Value: "NAME_INVALID", + Message: "manifest name did not match URI", + Description: `During a manifest upload, if the name in the manifest + does not match the uri name, this error will be returned.`, + HTTPStatusCodes: []int{http.StatusBadRequest, http.StatusNotFound}, + }, + { + Code: ErrorCodeTagInvalid, + Value: "TAG_INVALID", + Message: "manifest tag did not match URI", + Description: `During a manifest upload, if the tag in the manifest + does not match the uri tag, this error will be returned.`, + HTTPStatusCodes: []int{http.StatusBadRequest, http.StatusNotFound}, + }, + { + Code: ErrorCodeNameUnknown, + Value: "NAME_UNKNOWN", + Message: "repository name not known to registry", + Description: `This is returned if the name used during an operation is + unknown to the registry.`, + HTTPStatusCodes: []int{http.StatusNotFound}, + }, + { + Code: ErrorCodeManifestUnknown, + Value: "MANIFEST_UNKNOWN", + Message: "manifest unknown", + Description: `This error is returned when the manifest, identified by + name and tag is unknown to the repository.`, + HTTPStatusCodes: []int{http.StatusNotFound}, + }, + { + Code: ErrorCodeManifestInvalid, + Value: "MANIFEST_INVALID", + Message: "manifest invalid", + Description: `During upload, manifests undergo several checks ensuring + validity. If those checks fail, this error may be returned, unless a + more specific error is included. The detail will contain information + the failed validation.`, + HTTPStatusCodes: []int{http.StatusBadRequest}, + }, + { + Code: ErrorCodeManifestUnverified, + Value: "MANIFEST_UNVERIFIED", + Message: "manifest failed signature verification", + Description: `During manifest upload, if the manifest fails signature + verification, this error will be returned.`, + HTTPStatusCodes: []int{http.StatusBadRequest}, + }, + { + Code: ErrorCodeBlobUnknown, + Value: "BLOB_UNKNOWN", + Message: "blob unknown to registry", + Description: `This error may be returned when a blob is unknown to the + registry in a specified repository. This can be returned with a + standard get or if a manifest references an unknown layer during + upload.`, + HTTPStatusCodes: []int{http.StatusBadRequest, http.StatusNotFound}, + }, + + { + Code: ErrorCodeBlobUploadUnknown, + Value: "BLOB_UPLOAD_UNKNOWN", + Message: "blob upload unknown to registry", + Description: `If a blob upload has been cancelled or was never + started, this error code may be returned.`, + HTTPStatusCodes: []int{http.StatusNotFound}, + }, +} + +var errorCodeToDescriptors map[ErrorCode]ErrorDescriptor +var idToDescriptors map[string]ErrorDescriptor + +func init() { + errorCodeToDescriptors = make(map[ErrorCode]ErrorDescriptor, len(ErrorDescriptors)) + idToDescriptors = make(map[string]ErrorDescriptor, len(ErrorDescriptors)) + + for _, descriptor := range ErrorDescriptors { + errorCodeToDescriptors[descriptor.Code] = descriptor + idToDescriptors[descriptor.Value] = descriptor + } +} diff --git a/registry/v2/doc.go b/registry/v2/doc.go new file mode 100644 index 0000000000..30fe2271a1 --- /dev/null +++ b/registry/v2/doc.go @@ -0,0 +1,13 @@ +// Package v2 describes routes, urls and the error codes used in the Docker +// Registry JSON HTTP API V2. In addition to declarations, descriptors are +// provided for routes and error codes that can be used for implementation and +// automatically generating documentation. +// +// Definitions here are considered to be locked down for the V2 registry api. +// Any changes must be considered carefully and should not proceed without a +// change proposal. +// +// Currently, while the HTTP API definitions are considered stable, the Go API +// exports are considered unstable. Go API consumers should take care when +// relying on these definitions until this message is deleted. +package v2 diff --git a/registry/v2/errors.go b/registry/v2/errors.go new file mode 100644 index 0000000000..8c85d3a97f --- /dev/null +++ b/registry/v2/errors.go @@ -0,0 +1,185 @@ +package v2 + +import ( + "fmt" + "strings" +) + +// ErrorCode represents the error type. The errors are serialized via strings +// and the integer format may change and should *never* be exported. +type ErrorCode int + +const ( + // ErrorCodeUnknown is a catch-all for errors not defined below. + ErrorCodeUnknown ErrorCode = iota + + // ErrorCodeDigestInvalid is returned when uploading a blob if the + // provided digest does not match the blob contents. + ErrorCodeDigestInvalid + + // ErrorCodeSizeInvalid is returned when uploading a blob if the provided + // size does not match the content length. + ErrorCodeSizeInvalid + + // ErrorCodeNameInvalid is returned when the name in the manifest does not + // match the provided name. + ErrorCodeNameInvalid + + // ErrorCodeTagInvalid is returned when the tag in the manifest does not + // match the provided tag. + ErrorCodeTagInvalid + + // ErrorCodeNameUnknown when the repository name is not known. + ErrorCodeNameUnknown + + // ErrorCodeManifestUnknown returned when image manifest is unknown. + ErrorCodeManifestUnknown + + // ErrorCodeManifestInvalid returned when an image manifest is invalid, + // typically during a PUT operation. This error encompasses all errors + // encountered during manifest validation that aren't signature errors. + ErrorCodeManifestInvalid + + // ErrorCodeManifestUnverified is returned when the manifest fails + // signature verfication. + ErrorCodeManifestUnverified + + // ErrorCodeBlobUnknown is returned when a blob is unknown to the + // registry. This can happen when the manifest references a nonexistent + // layer or the result is not found by a blob fetch. + ErrorCodeBlobUnknown + + // ErrorCodeBlobUploadUnknown is returned when an upload is unknown. + ErrorCodeBlobUploadUnknown +) + +// ParseErrorCode attempts to parse the error code string, returning +// ErrorCodeUnknown if the error is not known. +func ParseErrorCode(s string) ErrorCode { + desc, ok := idToDescriptors[s] + + if !ok { + return ErrorCodeUnknown + } + + return desc.Code +} + +// Descriptor returns the descriptor for the error code. +func (ec ErrorCode) Descriptor() ErrorDescriptor { + d, ok := errorCodeToDescriptors[ec] + + if !ok { + return ErrorCodeUnknown.Descriptor() + } + + return d +} + +// String returns the canonical identifier for this error code. +func (ec ErrorCode) String() string { + return ec.Descriptor().Value +} + +// Message returned the human-readable error message for this error code. +func (ec ErrorCode) Message() string { + return ec.Descriptor().Message +} + +// MarshalText encodes the receiver into UTF-8-encoded text and returns the +// result. +func (ec ErrorCode) MarshalText() (text []byte, err error) { + return []byte(ec.String()), nil +} + +// UnmarshalText decodes the form generated by MarshalText. +func (ec *ErrorCode) UnmarshalText(text []byte) error { + desc, ok := idToDescriptors[string(text)] + + if !ok { + desc = ErrorCodeUnknown.Descriptor() + } + + *ec = desc.Code + + return nil +} + +// Error provides a wrapper around ErrorCode with extra Details provided. +type Error struct { + Code ErrorCode `json:"code"` + Message string `json:"message,omitempty"` + Detail interface{} `json:"detail,omitempty"` +} + +// Error returns a human readable representation of the error. +func (e Error) Error() string { + return fmt.Sprintf("%s: %s", + strings.ToLower(strings.Replace(e.Code.String(), "_", " ", -1)), + e.Message) +} + +// Errors provides the envelope for multiple errors and a few sugar methods +// for use within the application. +type Errors struct { + Errors []Error `json:"errors,omitempty"` +} + +// Push pushes an error on to the error stack, with the optional detail +// argument. It is a programming error (ie panic) to push more than one +// detail at a time. +func (errs *Errors) Push(code ErrorCode, details ...interface{}) { + if len(details) > 1 { + panic("please specify zero or one detail items for this error") + } + + var detail interface{} + if len(details) > 0 { + detail = details[0] + } + + if err, ok := detail.(error); ok { + detail = err.Error() + } + + errs.PushErr(Error{ + Code: code, + Message: code.Message(), + Detail: detail, + }) +} + +// PushErr pushes an error interface onto the error stack. +func (errs *Errors) PushErr(err error) { + switch err.(type) { + case Error: + errs.Errors = append(errs.Errors, err.(Error)) + default: + errs.Errors = append(errs.Errors, Error{Message: err.Error()}) + } +} + +func (errs *Errors) Error() string { + switch errs.Len() { + case 0: + return "" + case 1: + return errs.Errors[0].Error() + default: + msg := "errors:\n" + for _, err := range errs.Errors { + msg += err.Error() + "\n" + } + return msg + } +} + +// Clear clears the errors. +func (errs *Errors) Clear() { + errs.Errors = errs.Errors[:0] +} + +// Len returns the current number of errors. +func (errs *Errors) Len() int { + return len(errs.Errors) +} diff --git a/registry/v2/errors_test.go b/registry/v2/errors_test.go new file mode 100644 index 0000000000..4a80cdfe2d --- /dev/null +++ b/registry/v2/errors_test.go @@ -0,0 +1,163 @@ +package v2 + +import ( + "encoding/json" + "reflect" + "testing" +) + +// TestErrorCodes ensures that error code format, mappings and +// marshaling/unmarshaling. round trips are stable. +func TestErrorCodes(t *testing.T) { + for _, desc := range ErrorDescriptors { + if desc.Code.String() != desc.Value { + t.Fatalf("error code string incorrect: %q != %q", desc.Code.String(), desc.Value) + } + + if desc.Code.Message() != desc.Message { + t.Fatalf("incorrect message for error code %v: %q != %q", desc.Code, desc.Code.Message(), desc.Message) + } + + // Serialize the error code using the json library to ensure that we + // get a string and it works round trip. + p, err := json.Marshal(desc.Code) + + if err != nil { + t.Fatalf("error marshaling error code %v: %v", desc.Code, err) + } + + if len(p) <= 0 { + t.Fatalf("expected content in marshaled before for error code %v", desc.Code) + } + + // First, unmarshal to interface and ensure we have a string. + var ecUnspecified interface{} + if err := json.Unmarshal(p, &ecUnspecified); err != nil { + t.Fatalf("error unmarshaling error code %v: %v", desc.Code, err) + } + + if _, ok := ecUnspecified.(string); !ok { + t.Fatalf("expected a string for error code %v on unmarshal got a %T", desc.Code, ecUnspecified) + } + + // Now, unmarshal with the error code type and ensure they are equal + var ecUnmarshaled ErrorCode + if err := json.Unmarshal(p, &ecUnmarshaled); err != nil { + t.Fatalf("error unmarshaling error code %v: %v", desc.Code, err) + } + + if ecUnmarshaled != desc.Code { + t.Fatalf("unexpected error code during error code marshal/unmarshal: %v != %v", ecUnmarshaled, desc.Code) + } + } +} + +// TestErrorsManagement does a quick check of the Errors type to ensure that +// members are properly pushed and marshaled. +func TestErrorsManagement(t *testing.T) { + var errs Errors + + errs.Push(ErrorCodeDigestInvalid) + errs.Push(ErrorCodeBlobUnknown, + map[string]string{"digest": "sometestblobsumdoesntmatter"}) + + p, err := json.Marshal(errs) + + if err != nil { + t.Fatalf("error marashaling errors: %v", err) + } + + expectedJSON := "{\"errors\":[{\"code\":\"DIGEST_INVALID\",\"message\":\"provided digest did not match uploaded content\"},{\"code\":\"BLOB_UNKNOWN\",\"message\":\"blob unknown to registry\",\"detail\":{\"digest\":\"sometestblobsumdoesntmatter\"}}]}" + + if string(p) != expectedJSON { + t.Fatalf("unexpected json: %q != %q", string(p), expectedJSON) + } + + errs.Clear() + errs.Push(ErrorCodeUnknown) + expectedJSON = "{\"errors\":[{\"code\":\"UNKNOWN\",\"message\":\"unknown error\"}]}" + p, err = json.Marshal(errs) + + if err != nil { + t.Fatalf("error marashaling errors: %v", err) + } + + if string(p) != expectedJSON { + t.Fatalf("unexpected json: %q != %q", string(p), expectedJSON) + } +} + +// TestMarshalUnmarshal ensures that api errors can round trip through json +// without losing information. +func TestMarshalUnmarshal(t *testing.T) { + + var errors Errors + + for _, testcase := range []struct { + description string + err Error + }{ + { + description: "unknown error", + err: Error{ + + Code: ErrorCodeUnknown, + Message: ErrorCodeUnknown.Descriptor().Message, + }, + }, + { + description: "unknown manifest", + err: Error{ + Code: ErrorCodeManifestUnknown, + Message: ErrorCodeManifestUnknown.Descriptor().Message, + }, + }, + { + description: "unknown manifest", + err: Error{ + Code: ErrorCodeBlobUnknown, + Message: ErrorCodeBlobUnknown.Descriptor().Message, + Detail: map[string]interface{}{"digest": "asdfqwerqwerqwerqwer"}, + }, + }, + } { + fatalf := func(format string, args ...interface{}) { + t.Fatalf(testcase.description+": "+format, args...) + } + + unexpectedErr := func(err error) { + fatalf("unexpected error: %v", err) + } + + p, err := json.Marshal(testcase.err) + if err != nil { + unexpectedErr(err) + } + + var unmarshaled Error + if err := json.Unmarshal(p, &unmarshaled); err != nil { + unexpectedErr(err) + } + + if !reflect.DeepEqual(unmarshaled, testcase.err) { + fatalf("errors not equal after round trip: %#v != %#v", unmarshaled, testcase.err) + } + + // Roll everything up into an error response envelope. + errors.PushErr(testcase.err) + } + + p, err := json.Marshal(errors) + if err != nil { + t.Fatalf("unexpected error marshaling error envelope: %v", err) + } + + var unmarshaled Errors + if err := json.Unmarshal(p, &unmarshaled); err != nil { + t.Fatalf("unexpected error unmarshaling error envelope: %v", err) + } + + if !reflect.DeepEqual(unmarshaled, errors) { + t.Fatalf("errors not equal after round trip: %#v != %#v", unmarshaled, errors) + } +} diff --git a/registry/v2/regexp.go b/registry/v2/regexp.go new file mode 100644 index 0000000000..b7e95b9ff3 --- /dev/null +++ b/registry/v2/regexp.go @@ -0,0 +1,19 @@ +package v2 + +import "regexp" + +// This file defines regular expressions for use in route definition. These +// are also defined in the registry code base. Until they are in a common, +// shared location, and exported, they must be repeated here. + +// RepositoryNameComponentRegexp restricts registtry path components names to +// start with at least two letters or numbers, with following parts able to +// separated by one period, dash or underscore. +var RepositoryNameComponentRegexp = regexp.MustCompile(`[a-z0-9]+(?:[._-][a-z0-9]+)*`) + +// RepositoryNameRegexp builds on RepositoryNameComponentRegexp to allow 2 to +// 5 path components, separated by a forward slash. +var RepositoryNameRegexp = regexp.MustCompile(`(?:` + RepositoryNameComponentRegexp.String() + `/){1,4}` + RepositoryNameComponentRegexp.String()) + +// TagNameRegexp matches valid tag names. From docker/docker:graph/tags.go. +var TagNameRegexp = regexp.MustCompile(`[\w][\w.-]{0,127}`) diff --git a/registry/v2/routes.go b/registry/v2/routes.go new file mode 100644 index 0000000000..08f36e2f71 --- /dev/null +++ b/registry/v2/routes.go @@ -0,0 +1,66 @@ +package v2 + +import "github.com/gorilla/mux" + +// The following are definitions of the name under which all V2 routes are +// registered. These symbols can be used to look up a route based on the name. +const ( + RouteNameBase = "base" + RouteNameManifest = "manifest" + RouteNameTags = "tags" + RouteNameBlob = "blob" + RouteNameBlobUpload = "blob-upload" + RouteNameBlobUploadChunk = "blob-upload-chunk" +) + +var allEndpoints = []string{ + RouteNameManifest, + RouteNameTags, + RouteNameBlob, + RouteNameBlobUpload, + RouteNameBlobUploadChunk, +} + +// Router builds a gorilla router with named routes for the various API +// methods. This can be used directly by both server implementations and +// clients. +func Router() *mux.Router { + router := mux.NewRouter(). + StrictSlash(true) + + // GET /v2/ Check Check that the registry implements API version 2(.1) + router. + Path("/v2/"). + Name(RouteNameBase) + + // GET /v2//manifest/ Image Manifest Fetch the image manifest identified by name and tag. + // PUT /v2//manifest/ Image Manifest Upload the image manifest identified by name and tag. + // DELETE /v2//manifest/ Image Manifest Delete the image identified by name and tag. + router. + Path("/v2/{name:" + RepositoryNameRegexp.String() + "}/manifests/{tag:" + TagNameRegexp.String() + "}"). + Name(RouteNameManifest) + + // GET /v2//tags/list Tags Fetch the tags under the repository identified by name. + router. + Path("/v2/{name:" + RepositoryNameRegexp.String() + "}/tags/list"). + Name(RouteNameTags) + + // GET /v2//blob/ Layer Fetch the blob identified by digest. + router. + Path("/v2/{name:" + RepositoryNameRegexp.String() + "}/blobs/{digest:[a-zA-Z0-9-_+.]+:[a-zA-Z0-9-_+.=]+}"). + Name(RouteNameBlob) + + // POST /v2//blob/upload/ Layer Upload Initiate an upload of the layer identified by tarsum. + router. + Path("/v2/{name:" + RepositoryNameRegexp.String() + "}/blobs/uploads/"). + Name(RouteNameBlobUpload) + + // GET /v2//blob/upload/ Layer Upload Get the status of the upload identified by tarsum and uuid. + // PUT /v2//blob/upload/ Layer Upload Upload all or a chunk of the upload identified by tarsum and uuid. + // DELETE /v2//blob/upload/ Layer Upload Cancel the upload identified by layer and uuid + router. + Path("/v2/{name:" + RepositoryNameRegexp.String() + "}/blobs/uploads/{uuid}"). + Name(RouteNameBlobUploadChunk) + + return router +} diff --git a/registry/v2/routes_test.go b/registry/v2/routes_test.go new file mode 100644 index 0000000000..9969ebcc44 --- /dev/null +++ b/registry/v2/routes_test.go @@ -0,0 +1,184 @@ +package v2 + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "reflect" + "testing" + + "github.com/gorilla/mux" +) + +type routeTestCase struct { + RequestURI string + Vars map[string]string + RouteName string + StatusCode int +} + +// TestRouter registers a test handler with all the routes and ensures that +// each route returns the expected path variables. Not method verification is +// present. This not meant to be exhaustive but as check to ensure that the +// expected variables are extracted. +// +// This may go away as the application structure comes together. +func TestRouter(t *testing.T) { + + router := Router() + + testHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + testCase := routeTestCase{ + RequestURI: r.RequestURI, + Vars: mux.Vars(r), + RouteName: mux.CurrentRoute(r).GetName(), + } + + enc := json.NewEncoder(w) + + if err := enc.Encode(testCase); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + }) + + // Startup test server + server := httptest.NewServer(router) + + for _, testcase := range []routeTestCase{ + { + RouteName: RouteNameBase, + RequestURI: "/v2/", + Vars: map[string]string{}, + }, + { + RouteName: RouteNameManifest, + RequestURI: "/v2/foo/bar/manifests/tag", + Vars: map[string]string{ + "name": "foo/bar", + "tag": "tag", + }, + }, + { + RouteName: RouteNameTags, + RequestURI: "/v2/foo/bar/tags/list", + Vars: map[string]string{ + "name": "foo/bar", + }, + }, + { + RouteName: RouteNameBlob, + RequestURI: "/v2/foo/bar/blobs/tarsum.dev+foo:abcdef0919234", + Vars: map[string]string{ + "name": "foo/bar", + "digest": "tarsum.dev+foo:abcdef0919234", + }, + }, + { + RouteName: RouteNameBlob, + RequestURI: "/v2/foo/bar/blobs/sha256:abcdef0919234", + Vars: map[string]string{ + "name": "foo/bar", + "digest": "sha256:abcdef0919234", + }, + }, + { + RouteName: RouteNameBlobUpload, + RequestURI: "/v2/foo/bar/blobs/uploads/", + Vars: map[string]string{ + "name": "foo/bar", + }, + }, + { + RouteName: RouteNameBlobUploadChunk, + RequestURI: "/v2/foo/bar/blobs/uploads/uuid", + Vars: map[string]string{ + "name": "foo/bar", + "uuid": "uuid", + }, + }, + { + RouteName: RouteNameBlobUploadChunk, + RequestURI: "/v2/foo/bar/blobs/uploads/D95306FA-FAD3-4E36-8D41-CF1C93EF8286", + Vars: map[string]string{ + "name": "foo/bar", + "uuid": "D95306FA-FAD3-4E36-8D41-CF1C93EF8286", + }, + }, + { + RouteName: RouteNameBlobUploadChunk, + RequestURI: "/v2/foo/bar/blobs/uploads/RDk1MzA2RkEtRkFEMy00RTM2LThENDEtQ0YxQzkzRUY4Mjg2IA==", + Vars: map[string]string{ + "name": "foo/bar", + "uuid": "RDk1MzA2RkEtRkFEMy00RTM2LThENDEtQ0YxQzkzRUY4Mjg2IA==", + }, + }, + { + // Check ambiguity: ensure we can distinguish between tags for + // "foo/bar/image/image" and image for "foo/bar/image" with tag + // "tags" + RouteName: RouteNameManifest, + RequestURI: "/v2/foo/bar/manifests/manifests/tags", + Vars: map[string]string{ + "name": "foo/bar/manifests", + "tag": "tags", + }, + }, + { + // This case presents an ambiguity between foo/bar with tag="tags" + // and list tags for "foo/bar/manifest" + RouteName: RouteNameTags, + RequestURI: "/v2/foo/bar/manifests/tags/list", + Vars: map[string]string{ + "name": "foo/bar/manifests", + }, + }, + { + RouteName: RouteNameBlobUploadChunk, + RequestURI: "/v2/foo/../../blob/uploads/D95306FA-FAD3-4E36-8D41-CF1C93EF8286", + StatusCode: http.StatusNotFound, + }, + } { + // Register the endpoint + router.GetRoute(testcase.RouteName).Handler(testHandler) + u := server.URL + testcase.RequestURI + + resp, err := http.Get(u) + + if err != nil { + t.Fatalf("error issuing get request: %v", err) + } + + if testcase.StatusCode == 0 { + // Override default, zero-value + testcase.StatusCode = http.StatusOK + } + + if resp.StatusCode != testcase.StatusCode { + t.Fatalf("unexpected status for %s: %v %v", u, resp.Status, resp.StatusCode) + } + + if testcase.StatusCode != http.StatusOK { + // We don't care about json response. + continue + } + + dec := json.NewDecoder(resp.Body) + + var actualRouteInfo routeTestCase + if err := dec.Decode(&actualRouteInfo); err != nil { + t.Fatalf("error reading json response: %v", err) + } + // Needs to be set out of band + actualRouteInfo.StatusCode = resp.StatusCode + + if actualRouteInfo.RouteName != testcase.RouteName { + t.Fatalf("incorrect route %q matched, expected %q", actualRouteInfo.RouteName, testcase.RouteName) + } + + if !reflect.DeepEqual(actualRouteInfo, testcase) { + t.Fatalf("actual does not equal expected: %#v != %#v", actualRouteInfo, testcase) + } + } + +} diff --git a/registry/v2/urls.go b/registry/v2/urls.go new file mode 100644 index 0000000000..19ef06fa12 --- /dev/null +++ b/registry/v2/urls.go @@ -0,0 +1,164 @@ +package v2 + +import ( + "net/http" + "net/url" + + "github.com/gorilla/mux" +) + +// URLBuilder creates registry API urls from a single base endpoint. It can be +// used to create urls for use in a registry client or server. +// +// All urls will be created from the given base, including the api version. +// For example, if a root of "/foo/" is provided, urls generated will be fall +// under "/foo/v2/...". Most application will only provide a schema, host and +// port, such as "https://localhost:5000/". +type URLBuilder struct { + root *url.URL // url root (ie http://localhost/) + router *mux.Router +} + +// NewURLBuilder creates a URLBuilder with provided root url object. +func NewURLBuilder(root *url.URL) *URLBuilder { + return &URLBuilder{ + root: root, + router: Router(), + } +} + +// NewURLBuilderFromString workes identically to NewURLBuilder except it takes +// a string argument for the root, returning an error if it is not a valid +// url. +func NewURLBuilderFromString(root string) (*URLBuilder, error) { + u, err := url.Parse(root) + if err != nil { + return nil, err + } + + return NewURLBuilder(u), nil +} + +// NewURLBuilderFromRequest uses information from an *http.Request to +// construct the root url. +func NewURLBuilderFromRequest(r *http.Request) *URLBuilder { + u := &url.URL{ + Scheme: r.URL.Scheme, + Host: r.Host, + } + + return NewURLBuilder(u) +} + +// BuildBaseURL constructs a base url for the API, typically just "/v2/". +func (ub *URLBuilder) BuildBaseURL() (string, error) { + route := ub.cloneRoute(RouteNameBase) + + baseURL, err := route.URL() + if err != nil { + return "", err + } + + return baseURL.String(), nil +} + +// BuildTagsURL constructs a url to list the tags in the named repository. +func (ub *URLBuilder) BuildTagsURL(name string) (string, error) { + route := ub.cloneRoute(RouteNameTags) + + tagsURL, err := route.URL("name", name) + if err != nil { + return "", err + } + + return tagsURL.String(), nil +} + +// BuildManifestURL constructs a url for the manifest identified by name and tag. +func (ub *URLBuilder) BuildManifestURL(name, tag string) (string, error) { + route := ub.cloneRoute(RouteNameManifest) + + manifestURL, err := route.URL("name", name, "tag", tag) + if err != nil { + return "", err + } + + return manifestURL.String(), nil +} + +// BuildBlobURL constructs the url for the blob identified by name and dgst. +func (ub *URLBuilder) BuildBlobURL(name string, dgst string) (string, error) { + route := ub.cloneRoute(RouteNameBlob) + + layerURL, err := route.URL("name", name, "digest", dgst) + if err != nil { + return "", err + } + + return layerURL.String(), nil +} + +// BuildBlobUploadURL constructs a url to begin a blob upload in the +// repository identified by name. +func (ub *URLBuilder) BuildBlobUploadURL(name string, values ...url.Values) (string, error) { + route := ub.cloneRoute(RouteNameBlobUpload) + + uploadURL, err := route.URL("name", name) + if err != nil { + return "", err + } + + return appendValuesURL(uploadURL, values...).String(), nil +} + +// BuildBlobUploadChunkURL constructs a url for the upload identified by uuid, +// including any url values. This should generally not be used by clients, as +// this url is provided by server implementations during the blob upload +// process. +func (ub *URLBuilder) BuildBlobUploadChunkURL(name, uuid string, values ...url.Values) (string, error) { + route := ub.cloneRoute(RouteNameBlobUploadChunk) + + uploadURL, err := route.URL("name", name, "uuid", uuid) + if err != nil { + return "", err + } + + return appendValuesURL(uploadURL, values...).String(), nil +} + +// clondedRoute returns a clone of the named route from the router. Routes +// must be cloned to avoid modifying them during url generation. +func (ub *URLBuilder) cloneRoute(name string) *mux.Route { + route := new(mux.Route) + *route = *ub.router.GetRoute(name) // clone the route + + return route. + Schemes(ub.root.Scheme). + Host(ub.root.Host) +} + +// appendValuesURL appends the parameters to the url. +func appendValuesURL(u *url.URL, values ...url.Values) *url.URL { + merged := u.Query() + + for _, v := range values { + for k, vv := range v { + merged[k] = append(merged[k], vv...) + } + } + + u.RawQuery = merged.Encode() + return u +} + +// appendValues appends the parameters to the url. Panics if the string is not +// a url. +func appendValues(u string, values ...url.Values) string { + up, err := url.Parse(u) + + if err != nil { + panic(err) // should never happen + } + + return appendValuesURL(up, values...).String() +} diff --git a/registry/v2/urls_test.go b/registry/v2/urls_test.go new file mode 100644 index 0000000000..a9590dba90 --- /dev/null +++ b/registry/v2/urls_test.go @@ -0,0 +1,100 @@ +package v2 + +import ( + "net/url" + "testing" +) + +type urlBuilderTestCase struct { + description string + expected string + build func() (string, error) +} + +// TestURLBuilder tests the various url building functions, ensuring they are +// returning the expected values. +func TestURLBuilder(t *testing.T) { + + root := "http://localhost:5000/" + urlBuilder, err := NewURLBuilderFromString(root) + if err != nil { + t.Fatalf("unexpected error creating urlbuilder: %v", err) + } + + for _, testcase := range []struct { + description string + expected string + build func() (string, error) + }{ + { + description: "test base url", + expected: "http://localhost:5000/v2/", + build: urlBuilder.BuildBaseURL, + }, + { + description: "test tags url", + expected: "http://localhost:5000/v2/foo/bar/tags/list", + build: func() (string, error) { + return urlBuilder.BuildTagsURL("foo/bar") + }, + }, + { + description: "test manifest url", + expected: "http://localhost:5000/v2/foo/bar/manifests/tag", + build: func() (string, error) { + return urlBuilder.BuildManifestURL("foo/bar", "tag") + }, + }, + { + description: "build blob url", + expected: "http://localhost:5000/v2/foo/bar/blobs/tarsum.v1+sha256:abcdef0123456789", + build: func() (string, error) { + return urlBuilder.BuildBlobURL("foo/bar", "tarsum.v1+sha256:abcdef0123456789") + }, + }, + { + description: "build blob upload url", + expected: "http://localhost:5000/v2/foo/bar/blobs/uploads/", + build: func() (string, error) { + return urlBuilder.BuildBlobUploadURL("foo/bar") + }, + }, + { + description: "build blob upload url with digest and size", + expected: "http://localhost:5000/v2/foo/bar/blobs/uploads/?digest=tarsum.v1%2Bsha256%3Aabcdef0123456789&size=10000", + build: func() (string, error) { + return urlBuilder.BuildBlobUploadURL("foo/bar", url.Values{ + "size": []string{"10000"}, + "digest": []string{"tarsum.v1+sha256:abcdef0123456789"}, + }) + }, + }, + { + description: "build blob upload chunk url", + expected: "http://localhost:5000/v2/foo/bar/blobs/uploads/uuid-part", + build: func() (string, error) { + return urlBuilder.BuildBlobUploadChunkURL("foo/bar", "uuid-part") + }, + }, + { + description: "build blob upload chunk url with digest and size", + expected: "http://localhost:5000/v2/foo/bar/blobs/uploads/uuid-part?digest=tarsum.v1%2Bsha256%3Aabcdef0123456789&size=10000", + build: func() (string, error) { + return urlBuilder.BuildBlobUploadChunkURL("foo/bar", "uuid-part", url.Values{ + "size": []string{"10000"}, + "digest": []string{"tarsum.v1+sha256:abcdef0123456789"}, + }) + }, + }, + } { + u, err := testcase.build() + if err != nil { + t.Fatalf("%s: error building url: %v", testcase.description, err) + } + + if u != testcase.expected { + t.Fatalf("%s: %q != %q", testcase.description, u, testcase.expected) + } + } + +} diff --git a/utils/jsonmessage.go b/utils/jsonmessage.go index a2bbbcf4d4..74d3112719 100644 --- a/utils/jsonmessage.go +++ b/utils/jsonmessage.go @@ -50,6 +50,9 @@ func (p *JSONProgress) String() string { } total := units.HumanSize(float64(p.Total)) percentage := int(float64(p.Current)/float64(p.Total)*100) / 2 + if percentage > 50 { + percentage = 50 + } if width > 110 { // this number can't be negetive gh#7136 numSpaces := 0 diff --git a/utils/jsonmessage_test.go b/utils/jsonmessage_test.go index 0ce9492c98..b9103da1a4 100644 --- a/utils/jsonmessage_test.go +++ b/utils/jsonmessage_test.go @@ -30,7 +30,7 @@ func TestProgress(t *testing.T) { } // this number can't be negetive gh#7136 - expected = "[==============================================================>] 50 B/40 B" + expected = "[==================================================>] 50 B/40 B" jp4 := JSONProgress{Current: 50, Total: 40} if jp4.String() != expected { t.Fatalf("Expected %q, got %q", expected, jp4.String())