Merge pull request #7202 from timbot/add-registry-mirror-flag

Add daemon flag to specify public registry mirrors

daemon/config.go

@@ -23,6 +23,7 @@ type Config struct {
 	AutoRestart                 bool
 	Dns                         []string
 	DnsSearch                   []string
+	Mirrors                     []string
 	EnableIptables              bool
 	EnableIpForward             bool
 	DefaultIp                   net.IP
@@ -60,6 +61,7 @@ func (config *Config) InstallFlags() {
 	// FIXME: why the inconsistency between "hosts" and "sockets"?
 	opts.IPListVar(&config.Dns, []string{"#dns", "-dns"}, "Force Docker to use specific DNS servers")
 	opts.DnsSearchListVar(&config.DnsSearch, []string{"-dns-search"}, "Force Docker to use specific DNS search domains")
+	opts.MirrorListVar(&config.Mirrors, []string{"-registry-mirror"}, "Specify a preferred Docker registry mirror")
 }
 
 func GetDefaultNetworkMtu() int {

daemon/daemon.go

@@ -791,7 +791,7 @@ func NewDaemonFromDirectory(config *Config, eng *engine.Engine) (*Daemon, error)
 		return nil, err
 	}
 	log.Debugf("Creating repository list")
-	repositories, err := graph.NewTagStore(path.Join(config.Root, "repositories-"+driver.String()), g)
+	repositories, err := graph.NewTagStore(path.Join(config.Root, "repositories-"+driver.String()), g, config.Mirrors)
 	if err != nil {
 		return nil, fmt.Errorf("Couldn't create Tag store: %s", err)
 	}

docs/man/docker.1.md

@@ -64,6 +64,9 @@ unix://[/path/to/socket] to use.
 **-p**=""
   Path to use for daemon PID file. Default is `/var/run/docker.pid`
 
+**--registry-mirror=<scheme>://<host>
+  Prepend a registry mirror to be used for image pulls. May be specified multiple times.
+
 **-s**=""
   Force the Docker runtime to use a specific storage driver.
 

docs/sources/articles.md

@@ -12,3 +12,4 @@
  - [Automatically Start Containers](host_integration/)
  - [Link via an Ambassador Container](ambassador_pattern_linking/)
  - [Increase a Boot2Docker Volume](b2d_volume_resize/)
+ - [Run a Local Registry Mirror](registry_mirror/)

docs/sources/articles/registry_mirror.md

@@ -0,0 +1,83 @@
+page_title: Run a local registry mirror
+page_description: How to set up and run a local registry mirror
+page_keywords: docker, registry, mirror, examples
+
+# Run a local registry mirror
+
+## Why?
+
+If you have multiple instances of Docker running in your environment
+(e.g., multiple physical or virtual machines, all running the Docker
+daemon), each time one of them requires an image that it doesn't have
+it will go out to the internet and fetch it from the public Docker
+registry. By running a local registry mirror, you can keep most of the
+image fetch traffic on your local network.
+
+## How does it work?
+
+The first time you request an image from your local registry mirror,
+it pulls the image from the public Docker registry and stores it locally
+before handing it back to you. On subsequent requests, the local registry
+mirror is able to serve the image from its own storage.
+
+## How do I set up a local registry mirror?
+
+There are two steps to set up and use a local registry mirror.
+
+### Step 1: Configure your Docker daemons to use the local registry mirror
+
+You will need to pass the `--registry-mirror` option to your Docker daemon on
+startup:
+
+    docker --registry-mirror=http://<my-docker-mirror-host> -d
+
+For example, if your mirror is serving on `http://10.0.0.2:5000`, you would run:
+
+    docker --registry-mirror=http://10.0.0.2:5000 -d
+
+**NOTE:**
+Depending on your local host setup, you may be able to add the
+`--registry-mirror` options to the `DOCKER_OPTS` variable in
+`/etc/defaults/docker`.
+
+### Step 2: Run the local registry mirror
+
+You will need to start a local registry mirror service. The
+[`registry` image](https://registry.hub.docker.com/_/registry/) provides this
+functionality. For example, to run a local registry mirror that serves on
+port `5000` and mirrors the content at `registry-1.docker.io`:
+
+    docker run -p 5000:5000 \
+        -e STANDALONE=false \
+        -e MIRROR_SOURCE=https://registry-1.docker.io \
+        -e MIRROR_SOURCE_INDEX=https://index.docker.io registry
+
+## Test it out
+
+With your mirror running, pull an image that you haven't pulled before (using
+`time` to time it):
+
+    $ time docker pull node:latest
+    Pulling repository node
+    [...]
+    
+    real   1m14.078s
+    user   0m0.176s
+    sys    0m0.120s
+
+Now, remove the image from your local machine:
+
+    $ docker rmi node:latest
+
+Finally, re-pull the image:
+
+    $ time docker pull node:latest
+    Pulling repository node
+    [...]
+    
+    real   0m51.376s
+    user   0m0.120s
+    sys    0m0.116s
+
+The second time around, the local registry mirror served the image from storage,
+avoiding a trip out to the internet to refetch it.

docs/sources/reference/commandline/cli.md

@@ -71,6 +71,7 @@ expect an integer, and they can only be specified once.
       --mtu=0                                    Set the containers network MTU
                                                    if no value is provided: default to the default route MTU or 1500 if no default route is available
       -p, --pidfile="/var/run/docker.pid"        Path to use for daemon PID file
+      --registry-mirror=[]                       Specify a preferred Docker registry mirror
       -s, --storage-driver=""                    Force the Docker runtime to use a specific storage driver
       --selinux-enabled=false                    Enable selinux support. SELinux does not presently support the BTRFS storage driver
       --storage-opt=[]                           Set storage driver options

graph/pull.go

@@ -25,6 +25,7 @@ func (s *TagStore) CmdPull(job *engine.Job) engine.Status {
 		sf          = utils.NewStreamFormatter(job.GetenvBool("json"))
 		authConfig  = &registry.AuthConfig{}
 		metaHeaders map[string][]string
+		mirrors     []string
 	)
 	if len(job.Args) > 1 {
 		tag = job.Args[1]
@@ -64,16 +65,19 @@ func (s *TagStore) CmdPull(job *engine.Job) engine.Status {
 	if endpoint == registry.IndexServerAddress() {
 		// If pull "index.docker.io/foo/bar", it's stored locally under "foo/bar"
 		localName = remoteName
+
+		// Use provided mirrors, if any
+		mirrors = s.mirrors
 	}
 
-	if err = s.pullRepository(r, job.Stdout, localName, remoteName, tag, sf, job.GetenvBool("parallel")); err != nil {
+	if err = s.pullRepository(r, job.Stdout, localName, remoteName, tag, sf, job.GetenvBool("parallel"), mirrors); err != nil {
 		return job.Error(err)
 	}
 
 	return engine.StatusOK
 }
 
-func (s *TagStore) pullRepository(r *registry.Session, out io.Writer, localName, remoteName, askedTag string, sf *utils.StreamFormatter, parallel bool) error {
+func (s *TagStore) pullRepository(r *registry.Session, out io.Writer, localName, remoteName, askedTag string, sf *utils.StreamFormatter, parallel bool, mirrors []string) error {
 	out.Write(sf.FormatStatus("", "Pulling repository %s", localName))
 
 	repoData, err := r.GetRepositoryData(remoteName)
@@ -153,17 +157,31 @@ func (s *TagStore) pullRepository(r *registry.Session, out io.Writer, localName,
 			out.Write(sf.FormatProgress(utils.TruncateID(img.ID), fmt.Sprintf("Pulling image (%s) from %s", img.Tag, localName), nil))
 			success := false
 			var lastErr error
-			for _, ep := range repoData.Endpoints {
-				out.Write(sf.FormatProgress(utils.TruncateID(img.ID), fmt.Sprintf("Pulling image (%s) from %s, endpoint: %s", img.Tag, localName, ep), nil))
-				if err := s.pullImage(r, out, img.ID, ep, repoData.Tokens, sf); err != nil {
-					// It's not ideal that only the last error is returned, it would be better to concatenate the errors.
-					// As the error is also given to the output stream the user will see the error.
-					lastErr = err
-					out.Write(sf.FormatProgress(utils.TruncateID(img.ID), fmt.Sprintf("Error pulling image (%s) from %s, endpoint: %s, %s", img.Tag, localName, ep, err), nil))
-					continue
+			if mirrors != nil {
+				for _, ep := range mirrors {
+					out.Write(sf.FormatProgress(utils.TruncateID(img.ID), fmt.Sprintf("Pulling image (%s) from %s, mirror: %s", img.Tag, localName, ep), nil))
+					if err := s.pullImage(r, out, img.ID, ep, repoData.Tokens, sf); err != nil {
+						// Don't report errors when pulling from mirrors.
+						log.Debugf("Error pulling image (%s) from %s, mirror: %s, %s", img.Tag, localName, ep, err)
+						continue
+					}
+					success = true
+					break
+				}
+			}
+			if !success {
+				for _, ep := range repoData.Endpoints {
+					out.Write(sf.FormatProgress(utils.TruncateID(img.ID), fmt.Sprintf("Pulling image (%s) from %s, endpoint: %s", img.Tag, localName, ep), nil))
+					if err := s.pullImage(r, out, img.ID, ep, repoData.Tokens, sf); err != nil {
+						// It's not ideal that only the last error is returned, it would be better to concatenate the errors.
+						// As the error is also given to the output stream the user will see the error.
+						lastErr = err
+						out.Write(sf.FormatProgress(utils.TruncateID(img.ID), fmt.Sprintf("Error pulling image (%s) from %s, endpoint: %s, %s", img.Tag, localName, ep, err), nil))
+						continue
+					}
+					success = true
+					break
 				}
-				success = true
-				break
 			}
 			if !success {
 				err := fmt.Errorf("Error pulling image (%s) from %s, %v", img.Tag, localName, lastErr)

graph/tags.go

@@ -20,6 +20,7 @@ const DEFAULTTAG = "latest"
 type TagStore struct {
 	path         string
 	graph        *Graph
+	mirrors      []string
 	Repositories map[string]Repository
 	sync.Mutex
 	// FIXME: move push/pull-related fields
@@ -48,7 +49,7 @@ func (r Repository) Contains(u Repository) bool {
 	return true
 }
 
-func NewTagStore(path string, graph *Graph) (*TagStore, error) {
+func NewTagStore(path string, graph *Graph, mirrors []string) (*TagStore, error) {
 	abspath, err := filepath.Abs(path)
 	if err != nil {
 		return nil, err
@@ -56,6 +57,7 @@ func NewTagStore(path string, graph *Graph) (*TagStore, error) {
 	store := &TagStore{
 		path:         abspath,
 		graph:        graph,
+		mirrors:      mirrors,
 		Repositories: make(map[string]Repository),
 		pullingPool:  make(map[string]chan struct{}),
 		pushingPool:  make(map[string]chan struct{}),

graph/tags_unit_test.go

@@ -52,7 +52,7 @@ func mkTestTagStore(root string, t *testing.T) *TagStore {
 	if err != nil {
 		t.Fatal(err)
 	}
-	store, err := NewTagStore(path.Join(root, "tags"), graph)
+	store, err := NewTagStore(path.Join(root, "tags"), graph, nil)
 	if err != nil {
 		t.Fatal(err)
 	}

opts/opts.go

@@ -3,6 +3,7 @@ package opts
 import (
 	"fmt"
 	"net"
+	"net/url"
 	"os"
 	"path/filepath"
 	"regexp"
@@ -33,6 +34,10 @@ func IPVar(value *net.IP, names []string, defaultValue, usage string) {
 	flag.Var(NewIpOpt(value, defaultValue), names, usage)
 }
 
+func MirrorListVar(values *[]string, names []string, usage string) {
+	flag.Var(newListOptsRef(values, ValidateMirror), names, usage)
+}
+
 // ListOpts type
 type ListOpts struct {
 	values    *[]string
@@ -190,3 +195,21 @@ func validateDomain(val string) (string, error) {
 	}
 	return "", fmt.Errorf("%s is not a valid domain", val)
 }
+
+// Validates an HTTP(S) registry mirror
+func ValidateMirror(val string) (string, error) {
+	uri, err := url.Parse(val)
+	if err != nil {
+		return "", fmt.Errorf("%s is not a valid URI", val)
+	}
+
+	if uri.Scheme != "http" && uri.Scheme != "https" {
+		return "", fmt.Errorf("Unsupported scheme %s", uri.Scheme)
+	}
+
+	if uri.Path != "" || uri.RawQuery != "" || uri.Fragment != "" {
+		return "", fmt.Errorf("Unsupported path/query/fragment at end of the URI")
+	}
+
+	return fmt.Sprintf("%s://%s/v1/", uri.Scheme, uri.Host), nil
+}