diff --git a/daemon/config.go b/daemon/config.go index 1ca8086933..07d41153a3 100644 --- a/daemon/config.go +++ b/daemon/config.go @@ -23,6 +23,7 @@ type Config struct { AutoRestart bool Dns []string DnsSearch []string + Mirrors []string EnableIptables bool EnableIpForward bool DefaultIp net.IP @@ -60,6 +61,7 @@ func (config *Config) InstallFlags() { // FIXME: why the inconsistency between "hosts" and "sockets"? opts.IPListVar(&config.Dns, []string{"#dns", "-dns"}, "Force Docker to use specific DNS servers") opts.DnsSearchListVar(&config.DnsSearch, []string{"-dns-search"}, "Force Docker to use specific DNS search domains") + opts.MirrorListVar(&config.Mirrors, []string{"-registry-mirror"}, "Specify a preferred Docker registry mirror") } func GetDefaultNetworkMtu() int { diff --git a/daemon/daemon.go b/daemon/daemon.go index fab28110c6..f24745901c 100644 --- a/daemon/daemon.go +++ b/daemon/daemon.go @@ -791,7 +791,7 @@ func NewDaemonFromDirectory(config *Config, eng *engine.Engine) (*Daemon, error) return nil, err } log.Debugf("Creating repository list") - repositories, err := graph.NewTagStore(path.Join(config.Root, "repositories-"+driver.String()), g) + repositories, err := graph.NewTagStore(path.Join(config.Root, "repositories-"+driver.String()), g, config.Mirrors) if err != nil { return nil, fmt.Errorf("Couldn't create Tag store: %s", err) } diff --git a/docs/man/docker.1.md b/docs/man/docker.1.md index 3932097255..db6b3e3051 100644 --- a/docs/man/docker.1.md +++ b/docs/man/docker.1.md @@ -64,6 +64,9 @@ unix://[/path/to/socket] to use. **-p**="" Path to use for daemon PID file. Default is `/var/run/docker.pid` +**--registry-mirror=:// + Prepend a registry mirror to be used for image pulls. May be specified multiple times. + **-s**="" Force the Docker runtime to use a specific storage driver. diff --git a/docs/sources/articles.md b/docs/sources/articles.md index 325037b4ce..37f2cd80f1 100644 --- a/docs/sources/articles.md +++ b/docs/sources/articles.md @@ -12,3 +12,4 @@ - [Automatically Start Containers](host_integration/) - [Link via an Ambassador Container](ambassador_pattern_linking/) - [Increase a Boot2Docker Volume](b2d_volume_resize/) + - [Run a Local Registry Mirror](registry_mirror/) diff --git a/docs/sources/articles/registry_mirror.md b/docs/sources/articles/registry_mirror.md new file mode 100644 index 0000000000..2e6bfd79c6 --- /dev/null +++ b/docs/sources/articles/registry_mirror.md @@ -0,0 +1,83 @@ +page_title: Run a local registry mirror +page_description: How to set up and run a local registry mirror +page_keywords: docker, registry, mirror, examples + +# Run a local registry mirror + +## Why? + +If you have multiple instances of Docker running in your environment +(e.g., multiple physical or virtual machines, all running the Docker +daemon), each time one of them requires an image that it doesn't have +it will go out to the internet and fetch it from the public Docker +registry. By running a local registry mirror, you can keep most of the +image fetch traffic on your local network. + +## How does it work? + +The first time you request an image from your local registry mirror, +it pulls the image from the public Docker registry and stores it locally +before handing it back to you. On subsequent requests, the local registry +mirror is able to serve the image from its own storage. + +## How do I set up a local registry mirror? + +There are two steps to set up and use a local registry mirror. + +### Step 1: Configure your Docker daemons to use the local registry mirror + +You will need to pass the `--registry-mirror` option to your Docker daemon on +startup: + + docker --registry-mirror=http:// -d + +For example, if your mirror is serving on `http://10.0.0.2:5000`, you would run: + + docker --registry-mirror=http://10.0.0.2:5000 -d + +**NOTE:** +Depending on your local host setup, you may be able to add the +`--registry-mirror` options to the `DOCKER_OPTS` variable in +`/etc/defaults/docker`. + +### Step 2: Run the local registry mirror + +You will need to start a local registry mirror service. The +[`registry` image](https://registry.hub.docker.com/_/registry/) provides this +functionality. For example, to run a local registry mirror that serves on +port `5000` and mirrors the content at `registry-1.docker.io`: + + docker run -p 5000:5000 \ + -e STANDALONE=false \ + -e MIRROR_SOURCE=https://registry-1.docker.io \ + -e MIRROR_SOURCE_INDEX=https://index.docker.io registry + +## Test it out + +With your mirror running, pull an image that you haven't pulled before (using +`time` to time it): + + $ time docker pull node:latest + Pulling repository node + [...] + + real 1m14.078s + user 0m0.176s + sys 0m0.120s + +Now, remove the image from your local machine: + + $ docker rmi node:latest + +Finally, re-pull the image: + + $ time docker pull node:latest + Pulling repository node + [...] + + real 0m51.376s + user 0m0.120s + sys 0m0.116s + +The second time around, the local registry mirror served the image from storage, +avoiding a trip out to the internet to refetch it. diff --git a/docs/sources/reference/commandline/cli.md b/docs/sources/reference/commandline/cli.md index 714116ddc5..286ba4a7a8 100644 --- a/docs/sources/reference/commandline/cli.md +++ b/docs/sources/reference/commandline/cli.md @@ -71,6 +71,7 @@ expect an integer, and they can only be specified once. --mtu=0 Set the containers network MTU if no value is provided: default to the default route MTU or 1500 if no default route is available -p, --pidfile="/var/run/docker.pid" Path to use for daemon PID file + --registry-mirror=[] Specify a preferred Docker registry mirror -s, --storage-driver="" Force the Docker runtime to use a specific storage driver --selinux-enabled=false Enable selinux support. SELinux does not presently support the BTRFS storage driver --storage-opt=[] Set storage driver options diff --git a/graph/pull.go b/graph/pull.go index 1a4e516207..600e524d81 100644 --- a/graph/pull.go +++ b/graph/pull.go @@ -25,6 +25,7 @@ func (s *TagStore) CmdPull(job *engine.Job) engine.Status { sf = utils.NewStreamFormatter(job.GetenvBool("json")) authConfig = ®istry.AuthConfig{} metaHeaders map[string][]string + mirrors []string ) if len(job.Args) > 1 { tag = job.Args[1] @@ -64,16 +65,19 @@ func (s *TagStore) CmdPull(job *engine.Job) engine.Status { if endpoint == registry.IndexServerAddress() { // If pull "index.docker.io/foo/bar", it's stored locally under "foo/bar" localName = remoteName + + // Use provided mirrors, if any + mirrors = s.mirrors } - if err = s.pullRepository(r, job.Stdout, localName, remoteName, tag, sf, job.GetenvBool("parallel")); err != nil { + if err = s.pullRepository(r, job.Stdout, localName, remoteName, tag, sf, job.GetenvBool("parallel"), mirrors); err != nil { return job.Error(err) } return engine.StatusOK } -func (s *TagStore) pullRepository(r *registry.Session, out io.Writer, localName, remoteName, askedTag string, sf *utils.StreamFormatter, parallel bool) error { +func (s *TagStore) pullRepository(r *registry.Session, out io.Writer, localName, remoteName, askedTag string, sf *utils.StreamFormatter, parallel bool, mirrors []string) error { out.Write(sf.FormatStatus("", "Pulling repository %s", localName)) repoData, err := r.GetRepositoryData(remoteName) @@ -153,17 +157,31 @@ func (s *TagStore) pullRepository(r *registry.Session, out io.Writer, localName, out.Write(sf.FormatProgress(utils.TruncateID(img.ID), fmt.Sprintf("Pulling image (%s) from %s", img.Tag, localName), nil)) success := false var lastErr error - for _, ep := range repoData.Endpoints { - out.Write(sf.FormatProgress(utils.TruncateID(img.ID), fmt.Sprintf("Pulling image (%s) from %s, endpoint: %s", img.Tag, localName, ep), nil)) - if err := s.pullImage(r, out, img.ID, ep, repoData.Tokens, sf); err != nil { - // It's not ideal that only the last error is returned, it would be better to concatenate the errors. - // As the error is also given to the output stream the user will see the error. - lastErr = err - out.Write(sf.FormatProgress(utils.TruncateID(img.ID), fmt.Sprintf("Error pulling image (%s) from %s, endpoint: %s, %s", img.Tag, localName, ep, err), nil)) - continue + if mirrors != nil { + for _, ep := range mirrors { + out.Write(sf.FormatProgress(utils.TruncateID(img.ID), fmt.Sprintf("Pulling image (%s) from %s, mirror: %s", img.Tag, localName, ep), nil)) + if err := s.pullImage(r, out, img.ID, ep, repoData.Tokens, sf); err != nil { + // Don't report errors when pulling from mirrors. + log.Debugf("Error pulling image (%s) from %s, mirror: %s, %s", img.Tag, localName, ep, err) + continue + } + success = true + break + } + } + if !success { + for _, ep := range repoData.Endpoints { + out.Write(sf.FormatProgress(utils.TruncateID(img.ID), fmt.Sprintf("Pulling image (%s) from %s, endpoint: %s", img.Tag, localName, ep), nil)) + if err := s.pullImage(r, out, img.ID, ep, repoData.Tokens, sf); err != nil { + // It's not ideal that only the last error is returned, it would be better to concatenate the errors. + // As the error is also given to the output stream the user will see the error. + lastErr = err + out.Write(sf.FormatProgress(utils.TruncateID(img.ID), fmt.Sprintf("Error pulling image (%s) from %s, endpoint: %s, %s", img.Tag, localName, ep, err), nil)) + continue + } + success = true + break } - success = true - break } if !success { err := fmt.Errorf("Error pulling image (%s) from %s, %v", img.Tag, localName, lastErr) diff --git a/graph/tags.go b/graph/tags.go index eee3be9f2f..573b89e286 100644 --- a/graph/tags.go +++ b/graph/tags.go @@ -20,6 +20,7 @@ const DEFAULTTAG = "latest" type TagStore struct { path string graph *Graph + mirrors []string Repositories map[string]Repository sync.Mutex // FIXME: move push/pull-related fields @@ -48,7 +49,7 @@ func (r Repository) Contains(u Repository) bool { return true } -func NewTagStore(path string, graph *Graph) (*TagStore, error) { +func NewTagStore(path string, graph *Graph, mirrors []string) (*TagStore, error) { abspath, err := filepath.Abs(path) if err != nil { return nil, err @@ -56,6 +57,7 @@ func NewTagStore(path string, graph *Graph) (*TagStore, error) { store := &TagStore{ path: abspath, graph: graph, + mirrors: mirrors, Repositories: make(map[string]Repository), pullingPool: make(map[string]chan struct{}), pushingPool: make(map[string]chan struct{}), diff --git a/graph/tags_unit_test.go b/graph/tags_unit_test.go index 36cda68a2b..e8cedb46c3 100644 --- a/graph/tags_unit_test.go +++ b/graph/tags_unit_test.go @@ -52,7 +52,7 @@ func mkTestTagStore(root string, t *testing.T) *TagStore { if err != nil { t.Fatal(err) } - store, err := NewTagStore(path.Join(root, "tags"), graph) + store, err := NewTagStore(path.Join(root, "tags"), graph, nil) if err != nil { t.Fatal(err) } diff --git a/opts/opts.go b/opts/opts.go index 65806f3698..0ed0f62bcd 100644 --- a/opts/opts.go +++ b/opts/opts.go @@ -3,6 +3,7 @@ package opts import ( "fmt" "net" + "net/url" "os" "path/filepath" "regexp" @@ -33,6 +34,10 @@ func IPVar(value *net.IP, names []string, defaultValue, usage string) { flag.Var(NewIpOpt(value, defaultValue), names, usage) } +func MirrorListVar(values *[]string, names []string, usage string) { + flag.Var(newListOptsRef(values, ValidateMirror), names, usage) +} + // ListOpts type type ListOpts struct { values *[]string @@ -190,3 +195,21 @@ func validateDomain(val string) (string, error) { } return "", fmt.Errorf("%s is not a valid domain", val) } + +// Validates an HTTP(S) registry mirror +func ValidateMirror(val string) (string, error) { + uri, err := url.Parse(val) + if err != nil { + return "", fmt.Errorf("%s is not a valid URI", val) + } + + if uri.Scheme != "http" && uri.Scheme != "https" { + return "", fmt.Errorf("Unsupported scheme %s", uri.Scheme) + } + + if uri.Path != "" || uri.RawQuery != "" || uri.Fragment != "" { + return "", fmt.Errorf("Unsupported path/query/fragment at end of the URI") + } + + return fmt.Sprintf("%s://%s/v1/", uri.Scheme, uri.Host), nil +}