Pass upstream client's user agent through to registry on image pulls

Changes how the Engine interacts with Registry servers on image pull.
Previously, Engine sent a User-Agent string to the Registry server
that included only the Engine's version information.  This commit
appends to that string the fields from the User-Agent sent by the
client (e.g., Compose) of the Engine.  This allows Registry server
operators to understand what tools are actually generating pulls on
their registries.

Signed-off-by: Mike Goelzer <mgoelzer@docker.com>

api/client/trust.go

@@ -152,7 +152,7 @@ func (cli *DockerCli) getNotaryRepository(repoInfo *registry.RepositoryInfo, aut
 	}
 
 	// Skip configuration headers since request is not going to Docker daemon
-	modifiers := registry.DockerHeaders(dockerversion.DockerUserAgent(), http.Header{})
+	modifiers := registry.DockerHeaders(dockerversion.DockerUserAgent(""), http.Header{})
 	authTransport := transport.NewTransport(base, modifiers...)
 	pingClient := &http.Client{
 		Transport: authTransport,

api/server/httputils/httputils.go

@@ -16,6 +16,9 @@ import (
 // APIVersionKey is the client's requested API version.
 const APIVersionKey = "api-version"
 
+// UAStringKey is used as key type for user-agent string in net/context struct
+const UAStringKey = "upstream-user-agent"
+
 // APIFunc is an adapter to allow the use of ordinary functions as Docker API endpoints.
 // Any function that has the appropriate signature can be registered as a API endpoint (e.g. getVersion).
 type APIFunc func(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error

api/server/middleware/user_agent.go

@@ -16,6 +16,8 @@ func NewUserAgentMiddleware(versionCheck string) Middleware {
 
 	return func(handler httputils.APIFunc) httputils.APIFunc {
 		return func(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
+			ctx = context.WithValue(ctx, httputils.UAStringKey, r.Header.Get("User-Agent"))
+
 			if strings.Contains(r.Header.Get("User-Agent"), "Docker-Client/") {
 				userAgent := strings.Split(r.Header.Get("User-Agent"), "/")
 

api/server/router/image/backend.go

@@ -7,6 +7,7 @@ import (
 	"github.com/docker/engine-api/types"
 	"github.com/docker/engine-api/types/container"
 	"github.com/docker/engine-api/types/registry"
+	"golang.org/x/net/context"
 )
 
 // Backend is all the methods that need to be implemented
@@ -37,7 +38,7 @@ type importExportBackend interface {
 }
 
 type registryBackend interface {
-	PullImage(ref reference.Named, metaHeaders map[string][]string, authConfig *types.AuthConfig, outStream io.Writer) error
+	PullImage(ctx context.Context, ref reference.Named, metaHeaders map[string][]string, authConfig *types.AuthConfig, outStream io.Writer) error
 	PushImage(ref reference.Named, metaHeaders map[string][]string, authConfig *types.AuthConfig, outStream io.Writer) error
 	SearchRegistryForImages(term string, authConfig *types.AuthConfig, metaHeaders map[string][]string) (*registry.SearchResults, error)
 }

api/server/router/image/image_routes.go

@@ -129,7 +129,7 @@ func (s *imageRouter) postImagesCreate(ctx context.Context, w http.ResponseWrite
 					}
 				}
 
-				err = s.backend.PullImage(ref, metaHeaders, authConfig, output)
+				err = s.backend.PullImage(ctx, ref, metaHeaders, authConfig, output)
 			}
 		}
 		// Check the error from pulling an image to make sure the request

daemon/daemon.go

@@ -1007,14 +1007,14 @@ func isBrokenPipe(e error) bool {
 
 // PullImage initiates a pull operation. image is the repository name to pull, and
 // tag may be either empty, or indicate a specific tag to pull.
-func (daemon *Daemon) PullImage(ref reference.Named, metaHeaders map[string][]string, authConfig *types.AuthConfig, outStream io.Writer) error {
+func (daemon *Daemon) PullImage(ctx context.Context, ref reference.Named, metaHeaders map[string][]string, authConfig *types.AuthConfig, outStream io.Writer) error {
 	// Include a buffer so that slow client connections don't affect
 	// transfer performance.
 	progressChan := make(chan progress.Progress, 100)
 
 	writesDone := make(chan struct{})
 
-	ctx, cancelFunc := context.WithCancel(context.Background())
+	ctx, cancelFunc := context.WithCancel(ctx)
 
 	go func() {
 		writeDistributionProgress(cancelFunc, outStream, progressChan)
@@ -1062,7 +1062,7 @@ func (daemon *Daemon) PullOnBuild(name string, authConfigs map[string]types.Auth
 		pullRegistryAuth = &resolvedConfig
 	}
 
-	if err := daemon.PullImage(ref, nil, pullRegistryAuth, output); err != nil {
+	if err := daemon.PullImage(context.Background(), ref, nil, pullRegistryAuth, output); err != nil {
 		return nil, err
 	}
 	return daemon.GetImage(name)
@@ -1519,7 +1519,7 @@ func configureVolumes(config *Config, rootUID, rootGID int) (*store.VolumeStore,
 
 // AuthenticateToRegistry checks the validity of credentials in authConfig
 func (daemon *Daemon) AuthenticateToRegistry(authConfig *types.AuthConfig) (string, string, error) {
-	return daemon.RegistryService.Auth(authConfig, dockerversion.DockerUserAgent())
+	return daemon.RegistryService.Auth(authConfig, dockerversion.DockerUserAgent(""))
 }
 
 // SearchRegistryForImages queries the registry for images matching
@@ -1527,7 +1527,7 @@ func (daemon *Daemon) AuthenticateToRegistry(authConfig *types.AuthConfig) (stri
 func (daemon *Daemon) SearchRegistryForImages(term string,
 	authConfig *types.AuthConfig,
 	headers map[string][]string) (*registrytypes.SearchResults, error) {
-	return daemon.RegistryService.Search(term, authConfig, dockerversion.DockerUserAgent(), headers)
+	return daemon.RegistryService.Search(term, authConfig, dockerversion.DockerUserAgent(""), headers)
 }
 
 // IsShuttingDown tells whether the daemon is shutting down or not

distribution/pull_v1.go

@@ -49,10 +49,10 @@ func (p *v1Puller) Pull(ctx context.Context, ref reference.Named) error {
 	tr := transport.NewTransport(
 		// TODO(tiborvass): was ReceiveTimeout
 		registry.NewTransport(tlsConfig),
-		registry.DockerHeaders(dockerversion.DockerUserAgent(), p.config.MetaHeaders)...,
+		registry.DockerHeaders(dockerversion.DockerUserAgent(""), p.config.MetaHeaders)...,
 	)
 	client := registry.HTTPClient(tr)
-	v1Endpoint, err := p.endpoint.ToV1Endpoint(dockerversion.DockerUserAgent(), p.config.MetaHeaders)
+	v1Endpoint, err := p.endpoint.ToV1Endpoint(dockerversion.DockerUserAgent(""), p.config.MetaHeaders)
 	if err != nil {
 		logrus.Debugf("Could not get v1 endpoint: %v", err)
 		return fallbackError{err: err}

distribution/push_v1.go

@@ -38,10 +38,10 @@ func (p *v1Pusher) Push(ctx context.Context) error {
 	tr := transport.NewTransport(
 		// TODO(tiborvass): was NoTimeout
 		registry.NewTransport(tlsConfig),
-		registry.DockerHeaders(dockerversion.DockerUserAgent(), p.config.MetaHeaders)...,
+		registry.DockerHeaders(dockerversion.DockerUserAgent(""), p.config.MetaHeaders)...,
 	)
 	client := registry.HTTPClient(tr)
-	v1Endpoint, err := p.endpoint.ToV1Endpoint(dockerversion.DockerUserAgent(), p.config.MetaHeaders)
+	v1Endpoint, err := p.endpoint.ToV1Endpoint(dockerversion.DockerUserAgent(""), p.config.MetaHeaders)
 	if err != nil {
 		logrus.Debugf("Could not get v1 endpoint: %v", err)
 		return fallbackError{err: err}

distribution/registry.go

@@ -37,6 +37,8 @@ func (dcs dumbCredentialStore) SetRefreshToken(*url.URL, string, string) {
 // providing timeout settings and authentication support, and also verifies the
 // remote API version.
 func NewV2Repository(ctx context.Context, repoInfo *registry.RepositoryInfo, endpoint registry.APIEndpoint, metaHeaders http.Header, authConfig *types.AuthConfig, actions ...string) (repo distribution.Repository, foundVersion bool, err error) {
+	upstreamUA := dockerversion.GetUserAgentFromContext(ctx)
+
 	repoName := repoInfo.FullName()
 	// If endpoint does not support CanonicalName, use the RemoteName instead
 	if endpoint.TrimHostname {
@@ -57,7 +59,7 @@ func NewV2Repository(ctx context.Context, repoInfo *registry.RepositoryInfo, end
 		DisableKeepAlives: true,
 	}
 
-	modifiers := registry.DockerHeaders(dockerversion.DockerUserAgent(), metaHeaders)
+	modifiers := registry.DockerHeaders(dockerversion.DockerUserAgent(upstreamUA), metaHeaders)
 	authTransport := transport.NewTransport(base, modifiers...)
 
 	challengeManager, foundVersion, err := registry.PingV2Registry(endpoint, authTransport)

dockerversion/useragent.go

@@ -1,15 +1,19 @@
 package dockerversion
 
 import (
+	"fmt"
 	"runtime"
 
+	"github.com/docker/docker/api/server/httputils"
 	"github.com/docker/docker/pkg/parsers/kernel"
 	"github.com/docker/docker/pkg/useragent"
+	"golang.org/x/net/context"
 )
 
 // DockerUserAgent is the User-Agent the Docker client uses to identify itself.
-// It is populated from version information of different components.
-func DockerUserAgent() string {
+// In accordance with RFC 7231 (5.5.3) is of the form:
+//    [docker client's UA] UpstreamClient([upstream client's UA])
+func DockerUserAgent(upstreamUA string) string {
 	httpVersion := make([]useragent.VersionInfo, 0, 6)
 	httpVersion = append(httpVersion, useragent.VersionInfo{Name: "docker", Version: Version})
 	httpVersion = append(httpVersion, useragent.VersionInfo{Name: "go", Version: runtime.Version()})
@@ -20,5 +24,50 @@ func DockerUserAgent() string {
 	httpVersion = append(httpVersion, useragent.VersionInfo{Name: "os", Version: runtime.GOOS})
 	httpVersion = append(httpVersion, useragent.VersionInfo{Name: "arch", Version: runtime.GOARCH})
 
-	return useragent.AppendVersions("", httpVersion...)
+	dockerUA := useragent.AppendVersions("", httpVersion...)
+	if len(upstreamUA) > 0 {
+		ret := insertUpstreamUserAgent(upstreamUA, dockerUA)
+		return ret
+	}
+	return dockerUA
+}
+
+// GetUserAgentFromContext returns the previously saved user-agent context stored in ctx, if one exists
+func GetUserAgentFromContext(ctx context.Context) string {
+	var upstreamUA string
+	if ctx != nil {
+		var ki interface{} = ctx.Value(httputils.UAStringKey)
+		if ki != nil {
+			upstreamUA = ctx.Value(httputils.UAStringKey).(string)
+		}
+	}
+	return upstreamUA
+}
+
+// escapeStr returns s with every rune in charsToEscape escaped by a backslash
+func escapeStr(s string, charsToEscape string) string {
+	var ret string
+	for _, currRune := range s {
+		appended := false
+		for _, escapeableRune := range charsToEscape {
+			if currRune == escapeableRune {
+				ret += "\\" + string(currRune)
+				appended = true
+				break
+			}
+		}
+		if !appended {
+			ret += string(currRune)
+		}
+	}
+	return ret
+}
+
+// insertUpstreamUserAgent adds the upstream client useragent to create a user-agent
+// string of the form:
+//   $dockerUA UpstreamClient($upstreamUA)
+func insertUpstreamUserAgent(upstreamUA string, dockerUA string) string {
+	charsToEscape := "();\\" //["\\", ";", "(", ")"]string
+	upstreamUAEscaped := escapeStr(upstreamUA, charsToEscape)
+	return fmt.Sprintf("%s UpstreamClient(%s)", dockerUA, upstreamUAEscaped)
 }

integration-cli/docker_cli_registry_user_agent_test.go

@@ -0,0 +1,90 @@
+package main
+
+import (
+	"fmt"
+	"net/http"
+	"regexp"
+
+	"github.com/go-check/check"
+)
+
+// unescapeBackslashSemicolonParens unescapes \;()
+func unescapeBackslashSemicolonParens(s string) string {
+	re := regexp.MustCompile("\\\\;")
+	ret := re.ReplaceAll([]byte(s), []byte(";"))
+
+	re = regexp.MustCompile("\\\\\\(")
+	ret = re.ReplaceAll([]byte(ret), []byte("("))
+
+	re = regexp.MustCompile("\\\\\\)")
+	ret = re.ReplaceAll([]byte(ret), []byte(")"))
+
+	re = regexp.MustCompile("\\\\\\\\")
+	ret = re.ReplaceAll([]byte(ret), []byte("\\"))
+
+	return string(ret)
+}
+
+func regexpCheckUA(c *check.C, ua string) {
+	re := regexp.MustCompile("(?P<dockerUA>.+) UpstreamClient(?P<upstreamUA>.+)")
+	substrArr := re.FindStringSubmatch(ua)
+
+	c.Assert(substrArr, check.HasLen, 3, check.Commentf("Expected 'UpstreamClient()' with upstream client UA"))
+	dockerUA := substrArr[1]
+	upstreamUAEscaped := substrArr[2]
+
+	// check dockerUA looks correct
+	reDockerUA := regexp.MustCompile("^docker/[0-9A-Za-z+]")
+	bMatchDockerUA := reDockerUA.MatchString(dockerUA)
+	c.Assert(bMatchDockerUA, check.Equals, true, check.Commentf("Docker Engine User-Agent malformed"))
+
+	// check upstreamUA looks correct
+	// Expecting something like:  Docker-Client/1.11.0-dev (linux)
+	upstreamUA := unescapeBackslashSemicolonParens(upstreamUAEscaped)
+	reUpstreamUA := regexp.MustCompile("^\\(Docker-Client/[0-9A-Za-z+]")
+	bMatchUpstreamUA := reUpstreamUA.MatchString(upstreamUA)
+	c.Assert(bMatchUpstreamUA, check.Equals, true, check.Commentf("(Upstream) Docker Client User-Agent malformed"))
+}
+
+// TestUserAgentPassThroughOnPull verifies that when an image is pulled from
+// a registry, the registry should see a User-Agent string of the form
+//   [docker engine UA] UptreamClientSTREAM-CLIENT([client UA])
+func (s *DockerRegistrySuite) TestUserAgentPassThroughOnPull(c *check.C) {
+	reg, err := newTestRegistry(c)
+	c.Assert(err, check.IsNil)
+	expectUpstreamUA := false
+
+	reg.registerHandler("/v2/", func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(404)
+		var ua string
+		for k, v := range r.Header {
+			if k == "User-Agent" {
+				ua = v[0]
+			}
+		}
+		c.Assert(ua, check.Not(check.Equals), "", check.Commentf("No User-Agent found in request"))
+		if r.URL.Path == "/v2/busybox/manifests/latest" {
+			if expectUpstreamUA {
+				regexpCheckUA(c, ua)
+			}
+		}
+	})
+
+	repoName := fmt.Sprintf("%s/busybox", reg.hostport)
+	err = s.d.Start("--insecure-registry", reg.hostport, "--disable-legacy-registry=true")
+	c.Assert(err, check.IsNil)
+
+	dockerfileName, cleanup, err := makefile(fmt.Sprintf("FROM %s/busybox", reg.hostport))
+	c.Assert(err, check.IsNil, check.Commentf("Unable to create test dockerfile"))
+	defer cleanup()
+
+	s.d.Cmd("build", "--file", dockerfileName, ".")
+
+	s.d.Cmd("run", repoName)
+	s.d.Cmd("login", "-u", "richard", "-p", "testtest", "-e", "testuser@testdomain.com", reg.hostport)
+	s.d.Cmd("tag", "busybox", repoName)
+	s.d.Cmd("push", repoName)
+
+	expectUpstreamUA = true
+	s.d.Cmd("pull", repoName)
+}