Vendor updates to distribution

Pull in changes for refresh token in the registry client

Signed-off-by: Derek McGowan <derek@mcgstyle.net> (github: dmcgowan)

hack/vendor.sh

@@ -48,7 +48,7 @@ clone git github.com/boltdb/bolt v1.1.0
 clone git github.com/miekg/dns 75e6e86cc601825c5dbcd4e0c209eab180997cd7
 
 # get graph and distribution packages
-clone git github.com/docker/distribution 7b66c50bb7e0e4b3b83f8fd134a9f6ea4be08b57
+clone git github.com/docker/distribution db17a23b961978730892e12a0c6051d43a31aab3
 clone git github.com/vbatts/tar-split v0.9.11
 
 # get desired notary commit, might also need to be updated in Dockerfile

vendor/src/github.com/docker/distribution/CONTRIBUTING.md

@@ -90,7 +90,7 @@ It's mandatory to:
 
 Complying to these simple rules will greatly accelerate the review process, and will ensure you have a pleasant experience in contributing code to the Registry.
 
-Have a look at a great, succesful contribution: the [Ceph driver PR](https://github.com/docker/distribution/pull/443)
+Have a look at a great, successful contribution: the [Ceph driver PR](https://github.com/docker/distribution/pull/443)
 
 ## Coding Style
 

vendor/src/github.com/docker/distribution/Dockerfile

@@ -16,4 +16,4 @@ RUN make PREFIX=/go clean binaries
 VOLUME ["/var/lib/registry"]
 EXPOSE 5000
 ENTRYPOINT ["registry"]
-CMD ["/etc/docker/registry/config.yml"]
+CMD ["serve", "/etc/docker/registry/config.yml"]

vendor/src/github.com/docker/distribution/Makefile

@@ -14,8 +14,8 @@ endif
 GO_LDFLAGS=-ldflags "-X `go list ./version`.Version=$(VERSION)"
 
 .PHONY: clean all fmt vet lint build test binaries
-.DEFAULT: default
-all: AUTHORS clean fmt vet fmt lint build test binaries
+.DEFAULT: all
+all: fmt vet fmt lint build test binaries
 
 AUTHORS: .mailmap .git/HEAD
 	 git log --format='%aN <%aE>' | sort -fu > $@

vendor/src/github.com/docker/distribution/README.md

@@ -128,4 +128,4 @@ avenues are available for support:
 
 ## License
 
-This project is distributed under [Apache License, Version 2.0](LICENSE.md).
+This project is distributed under [Apache License, Version 2.0](LICENSE).

vendor/src/github.com/docker/distribution/blobs.go

@@ -97,6 +97,11 @@ type BlobDeleter interface {
 	Delete(ctx context.Context, dgst digest.Digest) error
 }
 
+// BlobEnumerator enables iterating over blobs from storage
+type BlobEnumerator interface {
+	Enumerate(ctx context.Context, ingester func(dgst digest.Digest) error) error
+}
+
 // BlobDescriptorService manages metadata about a blob by digest. Most
 // implementations will not expose such an interface explicitly. Such mappings
 // should be maintained by interacting with the BlobIngester. Hence, this is

vendor/src/github.com/docker/distribution/errors.go

@@ -8,6 +8,10 @@ import (
 	"github.com/docker/distribution/digest"
 )
 
+// ErrAccessDenied is returned when an access to a requested resource is
+// denied.
+var ErrAccessDenied = errors.New("access denied")
+
 // ErrManifestNotModified is returned when a conditional manifest GetByTag
 // returns nil due to the client indicating it has the latest version
 var ErrManifestNotModified = errors.New("manifest not modified")

vendor/src/github.com/docker/distribution/manifests.go

@@ -53,12 +53,18 @@ type ManifestService interface {
 	// Delete removes the manifest specified by the given digest. Deleting
 	// a manifest that doesn't exist will return ErrManifestNotFound
 	Delete(ctx context.Context, dgst digest.Digest) error
+}
+
+// ManifestEnumerator enables iterating over manifests
+type ManifestEnumerator interface {
+	// Enumerate calls ingester for each manifest.
+	Enumerate(ctx context.Context, ingester func(digest.Digest) error) error
+}
 
-	// Enumerate fills 'manifests' with the manifests in this service up
-	// to the size of 'manifests' and returns 'n' for the number of entries
-	// which were filled.  'last' contains an offset in the manifest set
-	// and can be used to resume iteration.
-	//Enumerate(ctx context.Context, manifests []Manifest, last Manifest) (n int, err error)
+// SignaturesGetter provides an interface for getting the signatures of a schema1 manifest. If the digest
+// referred to is not a schema1 manifest, an error should be returned.
+type SignaturesGetter interface {
+	GetSignatures(ctx context.Context, manifestDigest digest.Digest) ([]digest.Digest, error)
 }
 
 // Describable is an interface for descriptors

vendor/src/github.com/docker/distribution/reference/reference.go

@@ -3,7 +3,7 @@
 //
 // Grammar
 //
-// 	reference                       := repository [ ":" tag ] [ "@" digest ]
+// 	reference                       := name [ ":" tag ] [ "@" digest ]
 //	name                            := [hostname '/'] component ['/' component]*
 //	hostname                        := hostcomponent ['.' hostcomponent]* [':' port-number]
 //	hostcomponent                   := /([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9-]*[a-zA-Z0-9])/

vendor/src/github.com/docker/distribution/registry.go

@@ -40,6 +40,17 @@ type Namespace interface {
 	// which were filled.  'last' contains an offset in the catalog, and 'err' will be
 	// set to io.EOF if there are no more entries to obtain.
 	Repositories(ctx context.Context, repos []string, last string) (n int, err error)
+
+	// Blobs returns a blob enumerator to access all blobs
+	Blobs() BlobEnumerator
+
+	// BlobStatter returns a BlobStatter to control
+	BlobStatter() BlobStatter
+}
+
+// RepositoryEnumerator describes an operation to enumerate repositories
+type RepositoryEnumerator interface {
+	Enumerate(ctx context.Context, ingester func(string) error) error
 }
 
 // ManifestServiceOption is a function argument for Manifest Service methods

vendor/src/github.com/docker/distribution/registry/api/v2/descriptors.go

@@ -514,7 +514,7 @@ var routeDescriptors = []RouteDescriptor{
 									digestHeader,
 								},
 								Body: BodyDescriptor{
-									ContentType: "application/json; charset=utf-8",
+									ContentType: "<media type of manifest>",
 									Format:      manifestBody,
 								},
 							},
@@ -553,7 +553,7 @@ var routeDescriptors = []RouteDescriptor{
 							referenceParameterDescriptor,
 						},
 						Body: BodyDescriptor{
-							ContentType: "application/json; charset=utf-8",
+							ContentType: "<media type of manifest>",
 							Format:      manifestBody,
 						},
 						Successes: []ResponseDescriptor{

vendor/src/github.com/docker/distribution/registry/client/auth/session.go

@@ -19,6 +19,8 @@ import (
 // basic auth due to lack of credentials.
 var ErrNoBasicAuthCredentials = errors.New("no basic auth credentials")
 
+const defaultClientID = "registry-client"
+
 // AuthenticationHandler is an interface for authorizing a request from
 // params from a "WWW-Authenicate" header for a single scheme.
 type AuthenticationHandler interface {
@@ -36,6 +38,14 @@ type AuthenticationHandler interface {
 type CredentialStore interface {
 	// Basic returns basic auth for the given URL
 	Basic(*url.URL) (string, string)
+
+	// RefreshToken returns a refresh token for the
+	// given URL and service
+	RefreshToken(*url.URL, string) string
+
+	// SetRefreshToken sets the refresh token if none
+	// is provided for the given url and service
+	SetRefreshToken(realm *url.URL, service, token string)
 }
 
 // NewAuthorizer creates an authorizer which can handle multiple authentication
@@ -105,27 +115,47 @@ type clock interface {
 type tokenHandler struct {
 	header    http.Header
 	creds     CredentialStore
-	scope     tokenScope
 	transport http.RoundTripper
 	clock     clock
 
+	offlineAccess bool
+	forceOAuth    bool
+	clientID      string
+	scopes        []Scope
+
 	tokenLock       sync.Mutex
 	tokenCache      string
 	tokenExpiration time.Time
+}
 
-	additionalScopes map[string]struct{}
+// Scope is a type which is serializable to a string
+// using the allow scope grammar.
+type Scope interface {
+	String() string
 }
 
-// tokenScope represents the scope at which a token will be requested.
-// This represents a specific action on a registry resource.
-type tokenScope struct {
-	Resource string
-	Scope    string
-	Actions  []string
+// RepositoryScope represents a token scope for access
+// to a repository.
+type RepositoryScope struct {
+	Repository string
+	Actions    []string
 }
 
-func (ts tokenScope) String() string {
-	return fmt.Sprintf("%s:%s:%s", ts.Resource, ts.Scope, strings.Join(ts.Actions, ","))
+// String returns the string representation of the repository
+// using the scope grammar
+func (rs RepositoryScope) String() string {
+	return fmt.Sprintf("repository:%s:%s", rs.Repository, strings.Join(rs.Actions, ","))
+}
+
+// TokenHandlerOptions is used to configure a new token handler
+type TokenHandlerOptions struct {
+	Transport   http.RoundTripper
+	Credentials CredentialStore
+
+	OfflineAccess bool
+	ForceOAuth    bool
+	ClientID      string
+	Scopes        []Scope
 }
 
 // An implementation of clock for providing real time data.
@@ -137,22 +167,33 @@ func (realClock) Now() time.Time { return time.Now() }
 // NewTokenHandler creates a new AuthenicationHandler which supports
 // fetching tokens from a remote token server.
 func NewTokenHandler(transport http.RoundTripper, creds CredentialStore, scope string, actions ...string) AuthenticationHandler {
-	return newTokenHandler(transport, creds, realClock{}, scope, actions...)
+	// Create options...
+	return NewTokenHandlerWithOptions(TokenHandlerOptions{
+		Transport:   transport,
+		Credentials: creds,
+		Scopes: []Scope{
+			RepositoryScope{
+				Repository: scope,
+				Actions:    actions,
+			},
+		},
+	})
 }
 
-// newTokenHandler exposes the option to provide a clock to manipulate time in unit testing.
-func newTokenHandler(transport http.RoundTripper, creds CredentialStore, c clock, scope string, actions ...string) AuthenticationHandler {
-	return &tokenHandler{
-		transport: transport,
-		creds:     creds,
-		clock:     c,
-		scope: tokenScope{
-			Resource: "repository",
-			Scope:    scope,
-			Actions:  actions,
-		},
-		additionalScopes: map[string]struct{}{},
+// NewTokenHandlerWithOptions creates a new token handler using the provided
+// options structure.
+func NewTokenHandlerWithOptions(options TokenHandlerOptions) AuthenticationHandler {
+	handler := &tokenHandler{
+		transport:     options.Transport,
+		creds:         options.Credentials,
+		offlineAccess: options.OfflineAccess,
+		forceOAuth:    options.ForceOAuth,
+		clientID:      options.ClientID,
+		scopes:        options.Scopes,
+		clock:         realClock{},
 	}
+
+	return handler
 }
 
 func (th *tokenHandler) client() *http.Client {
@@ -169,88 +210,162 @@ func (th *tokenHandler) Scheme() string {
 func (th *tokenHandler) AuthorizeRequest(req *http.Request, params map[string]string) error {
 	var additionalScopes []string
 	if fromParam := req.URL.Query().Get("from"); fromParam != "" {
-		additionalScopes = append(additionalScopes, tokenScope{
-			Resource: "repository",
-			Scope:    fromParam,
-			Actions:  []string{"pull"},
+		additionalScopes = append(additionalScopes, RepositoryScope{
+			Repository: fromParam,
+			Actions:    []string{"pull"},
 		}.String())
 	}
-	if err := th.refreshToken(params, additionalScopes...); err != nil {
+
+	token, err := th.getToken(params, additionalScopes...)
+	if err != nil {
 		return err
 	}
 
-	req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", th.tokenCache))
+	req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", token))
 
 	return nil
 }
 
-func (th *tokenHandler) refreshToken(params map[string]string, additionalScopes ...string) error {
+func (th *tokenHandler) getToken(params map[string]string, additionalScopes ...string) (string, error) {
 	th.tokenLock.Lock()
 	defer th.tokenLock.Unlock()
+	scopes := make([]string, 0, len(th.scopes)+len(additionalScopes))
+	for _, scope := range th.scopes {
+		scopes = append(scopes, scope.String())
+	}
 	var addedScopes bool
 	for _, scope := range additionalScopes {
-		if _, ok := th.additionalScopes[scope]; !ok {
-			th.additionalScopes[scope] = struct{}{}
-			addedScopes = true
-		}
+		scopes = append(scopes, scope)
+		addedScopes = true
 	}
+
 	now := th.clock.Now()
 	if now.After(th.tokenExpiration) || addedScopes {
-		tr, err := th.fetchToken(params)
+		token, expiration, err := th.fetchToken(params, scopes)
 		if err != nil {
-			return err
+			return "", err
+		}
+
+		// do not update cache for added scope tokens
+		if !addedScopes {
+			th.tokenCache = token
+			th.tokenExpiration = expiration
 		}
-		th.tokenCache = tr.Token
-		th.tokenExpiration = tr.IssuedAt.Add(time.Duration(tr.ExpiresIn) * time.Second)
+
+		return token, nil
 	}
 
-	return nil
+	return th.tokenCache, nil
 }
 
-type tokenResponse struct {
-	Token       string    `json:"token"`
-	AccessToken string    `json:"access_token"`
-	ExpiresIn   int       `json:"expires_in"`
-	IssuedAt    time.Time `json:"issued_at"`
+type postTokenResponse struct {
+	AccessToken  string    `json:"access_token"`
+	RefreshToken string    `json:"refresh_token"`
+	ExpiresIn    int       `json:"expires_in"`
+	IssuedAt     time.Time `json:"issued_at"`
+	Scope        string    `json:"scope"`
 }
 
-func (th *tokenHandler) fetchToken(params map[string]string) (token *tokenResponse, err error) {
-	//log.Debugf("Getting bearer token with %s for %s", challenge.Parameters, ta.auth.Username)
-	realm, ok := params["realm"]
-	if !ok {
-		return nil, errors.New("no realm specified for token auth challenge")
-	}
+func (th *tokenHandler) fetchTokenWithOAuth(realm *url.URL, refreshToken, service string, scopes []string) (token string, expiration time.Time, err error) {
+	form := url.Values{}
+	form.Set("scope", strings.Join(scopes, " "))
+	form.Set("service", service)
 
-	// TODO(dmcgowan): Handle empty scheme
+	clientID := th.clientID
+	if clientID == "" {
+		// Use default client, this is a required field
+		clientID = defaultClientID
+	}
+	form.Set("client_id", clientID)
+
+	if refreshToken != "" {
+		form.Set("grant_type", "refresh_token")
+		form.Set("refresh_token", refreshToken)
+	} else if th.creds != nil {
+		form.Set("grant_type", "password")
+		username, password := th.creds.Basic(realm)
+		form.Set("username", username)
+		form.Set("password", password)
+
+		// attempt to get a refresh token
+		form.Set("access_type", "offline")
+	} else {
+		// refuse to do oauth without a grant type
+		return "", time.Time{}, fmt.Errorf("no supported grant type")
+	}
 
-	realmURL, err := url.Parse(realm)
+	resp, err := th.client().PostForm(realm.String(), form)
 	if err != nil {
-		return nil, fmt.Errorf("invalid token auth challenge realm: %s", err)
+		return "", time.Time{}, err
+	}
+	defer resp.Body.Close()
+
+	if !client.SuccessStatus(resp.StatusCode) {
+		err := client.HandleErrorResponse(resp)
+		return "", time.Time{}, err
+	}
+
+	decoder := json.NewDecoder(resp.Body)
+
+	var tr postTokenResponse
+	if err = decoder.Decode(&tr); err != nil {
+		return "", time.Time{}, fmt.Errorf("unable to decode token response: %s", err)
+	}
+
+	if tr.RefreshToken != "" && tr.RefreshToken != refreshToken {
+		th.creds.SetRefreshToken(realm, service, tr.RefreshToken)
+	}
+
+	if tr.ExpiresIn < minimumTokenLifetimeSeconds {
+		// The default/minimum lifetime.
+		tr.ExpiresIn = minimumTokenLifetimeSeconds
+		logrus.Debugf("Increasing token expiration to: %d seconds", tr.ExpiresIn)
 	}
 
-	req, err := http.NewRequest("GET", realmURL.String(), nil)
+	if tr.IssuedAt.IsZero() {
+		// issued_at is optional in the token response.
+		tr.IssuedAt = th.clock.Now().UTC()
+	}
+
+	return tr.AccessToken, tr.IssuedAt.Add(time.Duration(tr.ExpiresIn) * time.Second), nil
+}
+
+type getTokenResponse struct {
+	Token        string    `json:"token"`
+	AccessToken  string    `json:"access_token"`
+	ExpiresIn    int       `json:"expires_in"`
+	IssuedAt     time.Time `json:"issued_at"`
+	RefreshToken string    `json:"refresh_token"`
+}
+
+func (th *tokenHandler) fetchTokenWithBasicAuth(realm *url.URL, service string, scopes []string) (token string, expiration time.Time, err error) {
+
+	req, err := http.NewRequest("GET", realm.String(), nil)
 	if err != nil {
-		return nil, err
+		return "", time.Time{}, err
 	}
 
 	reqParams := req.URL.Query()
-	service := params["service"]
-	scope := th.scope.String()
 
 	if service != "" {
 		reqParams.Add("service", service)
 	}
 
-	for _, scopeField := range strings.Fields(scope) {
-		reqParams.Add("scope", scopeField)
+	for _, scope := range scopes {
+		reqParams.Add("scope", scope)
 	}
 
-	for scope := range th.additionalScopes {
-		reqParams.Add("scope", scope)
+	if th.offlineAccess {
+		reqParams.Add("offline_token", "true")
+		clientID := th.clientID
+		if clientID == "" {
+			clientID = defaultClientID
+		}
+		reqParams.Add("client_id", clientID)
 	}
 
 	if th.creds != nil {
-		username, password := th.creds.Basic(realmURL)
+		username, password := th.creds.Basic(realm)
 		if username != "" && password != "" {
 			reqParams.Add("account", username)
 			req.SetBasicAuth(username, password)
@@ -261,20 +376,24 @@ func (th *tokenHandler) fetchToken(params map[string]string) (token *tokenRespon
 
 	resp, err := th.client().Do(req)
 	if err != nil {
-		return nil, err
+		return "", time.Time{}, err
 	}
 	defer resp.Body.Close()
 
 	if !client.SuccessStatus(resp.StatusCode) {
 		err := client.HandleErrorResponse(resp)
-		return nil, err
+		return "", time.Time{}, err
 	}
 
 	decoder := json.NewDecoder(resp.Body)
 
-	tr := new(tokenResponse)
-	if err = decoder.Decode(tr); err != nil {
-		return nil, fmt.Errorf("unable to decode token response: %s", err)
+	var tr getTokenResponse
+	if err = decoder.Decode(&tr); err != nil {
+		return "", time.Time{}, fmt.Errorf("unable to decode token response: %s", err)
+	}
+
+	if tr.RefreshToken != "" && th.creds != nil {
+		th.creds.SetRefreshToken(realm, service, tr.RefreshToken)
 	}
 
 	// `access_token` is equivalent to `token` and if both are specified
@@ -285,7 +404,7 @@ func (th *tokenHandler) fetchToken(params map[string]string) (token *tokenRespon
 	}
 
 	if tr.Token == "" {
-		return nil, errors.New("authorization server did not include a token in the response")
+		return "", time.Time{}, errors.New("authorization server did not include a token in the response")
 	}
 
 	if tr.ExpiresIn < minimumTokenLifetimeSeconds {
@@ -296,10 +415,37 @@ func (th *tokenHandler) fetchToken(params map[string]string) (token *tokenRespon
 
 	if tr.IssuedAt.IsZero() {
 		// issued_at is optional in the token response.
-		tr.IssuedAt = th.clock.Now()
+		tr.IssuedAt = th.clock.Now().UTC()
+	}
+
+	return tr.Token, tr.IssuedAt.Add(time.Duration(tr.ExpiresIn) * time.Second), nil
+}
+
+func (th *tokenHandler) fetchToken(params map[string]string, scopes []string) (token string, expiration time.Time, err error) {
+	realm, ok := params["realm"]
+	if !ok {
+		return "", time.Time{}, errors.New("no realm specified for token auth challenge")
+	}
+
+	// TODO(dmcgowan): Handle empty scheme and relative realm
+	realmURL, err := url.Parse(realm)
+	if err != nil {
+		return "", time.Time{}, fmt.Errorf("invalid token auth challenge realm: %s", err)
+	}
+
+	service := params["service"]
+
+	var refreshToken string
+
+	if th.creds != nil {
+		refreshToken = th.creds.RefreshToken(realmURL, service)
+	}
+
+	if refreshToken != "" || th.forceOAuth {
+		return th.fetchTokenWithOAuth(realmURL, refreshToken, service, scopes)
 	}
 
-	return tr, nil
+	return th.fetchTokenWithBasicAuth(realmURL, service, scopes)
 }
 
 type basicHandler struct {

vendor/src/github.com/docker/distribution/registry/client/repository.go

@@ -292,9 +292,18 @@ func (t *tags) Get(ctx context.Context, tag string) (distribution.Descriptor, er
 	if err != nil {
 		return distribution.Descriptor{}, err
 	}
-	var attempts int
-	resp, err := t.client.Head(u)
 
+	req, err := http.NewRequest("HEAD", u, nil)
+	if err != nil {
+		return distribution.Descriptor{}, err
+	}
+
+	for _, t := range distribution.ManifestMediaTypes() {
+		req.Header.Add("Accept", t)
+	}
+
+	var attempts int
+	resp, err := t.client.Do(req)
 check:
 	if err != nil {
 		return distribution.Descriptor{}, err
@@ -304,7 +313,16 @@ check:
 	case resp.StatusCode >= 200 && resp.StatusCode < 400:
 		return descriptorFromResponse(resp)
 	case resp.StatusCode == http.StatusMethodNotAllowed:
-		resp, err = t.client.Get(u)
+		req, err = http.NewRequest("GET", u, nil)
+		if err != nil {
+			return distribution.Descriptor{}, err
+		}
+
+		for _, t := range distribution.ManifestMediaTypes() {
+			req.Header.Add("Accept", t)
+		}
+
+		resp, err = t.client.Do(req)
 		attempts++
 		if attempts > 1 {
 			return distribution.Descriptor{}, err

vendor/src/github.com/docker/distribution/registry/client/transport/http_reader.go

@@ -66,7 +66,7 @@ func (hrs *httpReadSeeker) Read(p []byte) (n int, err error) {
 		return 0, hrs.err
 	}
 
-	// If we seeked to a different position, we need to reset the
+	// If we sought to a different position, we need to reset the
 	// connection. This logic is here instead of Seek so that if
 	// a seek is undone before the next read, the connection doesn't
 	// need to be closed and reopened. A common example of this is