Add daemon flag to specify public registry mirrors

Adds support for a --registry-mirror=scheme://<host>[:port]
daemon flag. The flag may be present multiple times. If
provided, mirrors are prepended to the list of endpoints used
for image pull. Note that only mirrors of the public
index.docker.io registry are supported, and image/tag resolution
is still performed via the official index.

Docker-DCO-1.1-Signed-off-by: Tim Smith <timbot@google.com> (github: timbot)

daemon/config.go

@@ -23,6 +23,7 @@ type Config struct {
 	AutoRestart                 bool
 	Dns                         []string
 	DnsSearch                   []string
+	Mirrors                     []string
 	EnableIptables              bool
 	EnableIpForward             bool
 	DefaultIp                   net.IP
@@ -60,6 +61,7 @@ func (config *Config) InstallFlags() {
 	// FIXME: why the inconsistency between "hosts" and "sockets"?
 	opts.IPListVar(&config.Dns, []string{"#dns", "-dns"}, "Force Docker to use specific DNS servers")
 	opts.DnsSearchListVar(&config.DnsSearch, []string{"-dns-search"}, "Force Docker to use specific DNS search domains")
+	opts.MirrorListVar(&config.Mirrors, []string{"-registry-mirror"}, "Specify a preferred Docker registry mirror")
 }
 
 func GetDefaultNetworkMtu() int {

daemon/daemon.go

@@ -791,7 +791,7 @@ func NewDaemonFromDirectory(config *Config, eng *engine.Engine) (*Daemon, error)
 		return nil, err
 	}
 	log.Debugf("Creating repository list")
-	repositories, err := graph.NewTagStore(path.Join(config.Root, "repositories-"+driver.String()), g)
+	repositories, err := graph.NewTagStore(path.Join(config.Root, "repositories-"+driver.String()), g, config.Mirrors)
 	if err != nil {
 		return nil, fmt.Errorf("Couldn't create Tag store: %s", err)
 	}

docs/man/docker.1.md

@@ -64,6 +64,9 @@ unix://[/path/to/socket] to use.
 **-p**=""
   Path to use for daemon PID file. Default is `/var/run/docker.pid`
 
+**--registry-mirror=<scheme>://<host>
+  Prepend a registry mirror to be used for image pulls. May be specified multiple times.
+
 **-s**=""
   Force the Docker runtime to use a specific storage driver.
 

docs/sources/articles.md

@@ -12,3 +12,4 @@
  - [Automatically Start Containers](host_integration/)
  - [Link via an Ambassador Container](ambassador_pattern_linking/)
  - [Increase a Boot2Docker Volume](b2d_volume_resize/)
+ - [Run a Local Registry Mirror](registry_mirror/)

docs/sources/articles/registry_mirror.md

@@ -0,0 +1,83 @@
+page_title: Run a local registry mirror
+page_description: How to set up and run a local registry mirror
+page_keywords: docker, registry, mirror, examples
+
+# Run a local registry mirror
+
+## Why?
+
+If you have multiple instances of Docker running in your environment
+(e.g., multiple physical or virtual machines, all running the Docker
+daemon), each time one of them requires an image that it doesn't have
+it will go out to the internet and fetch it from the public Docker
+registry. By running a local registry mirror, you can keep most of the
+image fetch traffic on your local network.
+
+## How does it work?
+
+The first time you request an image from your local registry mirror,
+it pulls the image from the public Docker registry and stores it locally
+before handing it back to you. On subsequent requests, the local registry
+mirror is able to serve the image from its own storage.
+
+## How do I set up a local registry mirror?
+
+There are two steps to set up and use a local registry mirror.
+
+### Step 1: Configure your Docker daemons to use the local registry mirror
+
+You will need to pass the `--registry-mirror` option to your Docker daemon on
+startup:
+
+    docker --registry-mirror=http://<my-docker-mirror-host> -d
+
+For example, if your mirror is serving on `http://10.0.0.2:5000`, you would run:
+
+    docker --registry-mirror=http://10.0.0.2:5000 -d
+
+**NOTE:**
+Depending on your local host setup, you may be able to add the
+`--registry-mirror` options to the `DOCKER_OPTS` variable in
+`/etc/defaults/docker`.
+
+### Step 2: Run the local registry mirror
+
+You will need to start a local registry mirror service. The
+[`registry` image](https://registry.hub.docker.com/_/registry/) provides this
+functionality. For example, to run a local registry mirror that serves on
+port `5000` and mirrors the content at `registry-1.docker.io`:
+
+    docker run -p 5000:5000 \
+        -e STANDALONE=false \
+        -e MIRROR_SOURCE=https://registry-1.docker.io \
+        -e MIRROR_SOURCE_INDEX=https://index.docker.io registry
+
+## Test it out
+
+With your mirror running, pull an image that you haven't pulled before (using
+`time` to time it):
+
+    $ time docker pull node:latest
+    Pulling repository node
+    [...]
+    
+    real   1m14.078s
+    user   0m0.176s
+    sys    0m0.120s
+
+Now, remove the image from your local machine:
+
+    $ docker rmi node:latest
+
+Finally, re-pull the image:
+
+    $ time docker pull node:latest
+    Pulling repository node
+    [...]
+    
+    real   0m51.376s
+    user   0m0.120s
+    sys    0m0.116s
+
+The second time around, the local registry mirror served the image from storage,
+avoiding a trip out to the internet to refetch it.

docs/sources/reference/commandline/cli.md

@@ -71,6 +71,7 @@ expect an integer, and they can only be specified once.
       --mtu=0                                    Set the containers network MTU
                                                    if no value is provided: default to the default route MTU or 1500 if no default route is available
       -p, --pidfile="/var/run/docker.pid"        Path to use for daemon PID file
+      --registry-mirror=[]                       Specify a preferred Docker registry mirror
       -s, --storage-driver=""                    Force the Docker runtime to use a specific storage driver
       --selinux-enabled=false                    Enable selinux support. SELinux does not presently support the BTRFS storage driver
       --storage-opt=[]                           Set storage driver options

graph/pull.go

@@ -25,6 +25,7 @@ func (s *TagStore) CmdPull(job *engine.Job) engine.Status {
 		sf          = utils.NewStreamFormatter(job.GetenvBool("json"))
 		authConfig  = &registry.AuthConfig{}
 		metaHeaders map[string][]string
+		mirrors     []string
 	)
 	if len(job.Args) > 1 {
 		tag = job.Args[1]
@@ -64,16 +65,19 @@ func (s *TagStore) CmdPull(job *engine.Job) engine.Status {
 	if endpoint == registry.IndexServerAddress() {
 		// If pull "index.docker.io/foo/bar", it's stored locally under "foo/bar"
 		localName = remoteName
+
+		// Use provided mirrors, if any
+		mirrors = s.mirrors
 	}
 
-	if err = s.pullRepository(r, job.Stdout, localName, remoteName, tag, sf, job.GetenvBool("parallel")); err != nil {
+	if err = s.pullRepository(r, job.Stdout, localName, remoteName, tag, sf, job.GetenvBool("parallel"), mirrors); err != nil {
 		return job.Error(err)
 	}
 
 	return engine.StatusOK
 }
 
-func (s *TagStore) pullRepository(r *registry.Session, out io.Writer, localName, remoteName, askedTag string, sf *utils.StreamFormatter, parallel bool) error {
+func (s *TagStore) pullRepository(r *registry.Session, out io.Writer, localName, remoteName, askedTag string, sf *utils.StreamFormatter, parallel bool, mirrors []string) error {
 	out.Write(sf.FormatStatus("", "Pulling repository %s", localName))
 
 	repoData, err := r.GetRepositoryData(remoteName)
@@ -153,17 +157,31 @@ func (s *TagStore) pullRepository(r *registry.Session, out io.Writer, localName,
 			out.Write(sf.FormatProgress(utils.TruncateID(img.ID), fmt.Sprintf("Pulling image (%s) from %s", img.Tag, localName), nil))
 			success := false
 			var lastErr error
-			for _, ep := range repoData.Endpoints {
-				out.Write(sf.FormatProgress(utils.TruncateID(img.ID), fmt.Sprintf("Pulling image (%s) from %s, endpoint: %s", img.Tag, localName, ep), nil))
-				if err := s.pullImage(r, out, img.ID, ep, repoData.Tokens, sf); err != nil {
-					// It's not ideal that only the last error is returned, it would be better to concatenate the errors.
-					// As the error is also given to the output stream the user will see the error.
-					lastErr = err
-					out.Write(sf.FormatProgress(utils.TruncateID(img.ID), fmt.Sprintf("Error pulling image (%s) from %s, endpoint: %s, %s", img.Tag, localName, ep, err), nil))
-					continue
+			if mirrors != nil {
+				for _, ep := range mirrors {
+					out.Write(sf.FormatProgress(utils.TruncateID(img.ID), fmt.Sprintf("Pulling image (%s) from %s, mirror: %s", img.Tag, localName, ep), nil))
+					if err := s.pullImage(r, out, img.ID, ep, repoData.Tokens, sf); err != nil {
+						// Don't report errors when pulling from mirrors.
+						log.Debugf("Error pulling image (%s) from %s, mirror: %s, %s", img.Tag, localName, ep, err)
+						continue
+					}
+					success = true
+					break
+				}
+			}
+			if !success {
+				for _, ep := range repoData.Endpoints {
+					out.Write(sf.FormatProgress(utils.TruncateID(img.ID), fmt.Sprintf("Pulling image (%s) from %s, endpoint: %s", img.Tag, localName, ep), nil))
+					if err := s.pullImage(r, out, img.ID, ep, repoData.Tokens, sf); err != nil {
+						// It's not ideal that only the last error is returned, it would be better to concatenate the errors.
+						// As the error is also given to the output stream the user will see the error.
+						lastErr = err
+						out.Write(sf.FormatProgress(utils.TruncateID(img.ID), fmt.Sprintf("Error pulling image (%s) from %s, endpoint: %s, %s", img.Tag, localName, ep, err), nil))
+						continue
+					}
+					success = true
+					break
 				}
-				success = true
-				break
 			}
 			if !success {
 				err := fmt.Errorf("Error pulling image (%s) from %s, %v", img.Tag, localName, lastErr)

graph/tags.go

@@ -20,6 +20,7 @@ const DEFAULTTAG = "latest"
 type TagStore struct {
 	path         string
 	graph        *Graph
+	mirrors      []string
 	Repositories map[string]Repository
 	sync.Mutex
 	// FIXME: move push/pull-related fields
@@ -48,7 +49,7 @@ func (r Repository) Contains(u Repository) bool {
 	return true
 }
 
-func NewTagStore(path string, graph *Graph) (*TagStore, error) {
+func NewTagStore(path string, graph *Graph, mirrors []string) (*TagStore, error) {
 	abspath, err := filepath.Abs(path)
 	if err != nil {
 		return nil, err
@@ -56,6 +57,7 @@ func NewTagStore(path string, graph *Graph) (*TagStore, error) {
 	store := &TagStore{
 		path:         abspath,
 		graph:        graph,
+		mirrors:      mirrors,
 		Repositories: make(map[string]Repository),
 		pullingPool:  make(map[string]chan struct{}),
 		pushingPool:  make(map[string]chan struct{}),

graph/tags_unit_test.go

@@ -52,7 +52,7 @@ func mkTestTagStore(root string, t *testing.T) *TagStore {
 	if err != nil {
 		t.Fatal(err)
 	}
-	store, err := NewTagStore(path.Join(root, "tags"), graph)
+	store, err := NewTagStore(path.Join(root, "tags"), graph, nil)
 	if err != nil {
 		t.Fatal(err)
 	}

opts/opts.go

@@ -3,6 +3,7 @@ package opts
 import (
 	"fmt"
 	"net"
+	"net/url"
 	"os"
 	"path/filepath"
 	"regexp"
@@ -33,6 +34,10 @@ func IPVar(value *net.IP, names []string, defaultValue, usage string) {
 	flag.Var(NewIpOpt(value, defaultValue), names, usage)
 }
 
+func MirrorListVar(values *[]string, names []string, usage string) {
+	flag.Var(newListOptsRef(values, ValidateMirror), names, usage)
+}
+
 // ListOpts type
 type ListOpts struct {
 	values    *[]string
@@ -190,3 +195,21 @@ func validateDomain(val string) (string, error) {
 	}
 	return "", fmt.Errorf("%s is not a valid domain", val)
 }
+
+// Validates an HTTP(S) registry mirror
+func ValidateMirror(val string) (string, error) {
+	uri, err := url.Parse(val)
+	if err != nil {
+		return "", fmt.Errorf("%s is not a valid URI", val)
+	}
+
+	if uri.Scheme != "http" && uri.Scheme != "https" {
+		return "", fmt.Errorf("Unsupported scheme %s", uri.Scheme)
+	}
+
+	if uri.Path != "" || uri.RawQuery != "" || uri.Fragment != "" {
+		return "", fmt.Errorf("Unsupported path/query/fragment at end of the URI")
+	}
+
+	return fmt.Sprintf("%s://%s/v1/", uri.Scheme, uri.Host), nil
+}