beenull/moby
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Merge pull request #37852 from AntaresS/patch-buildkit

Browse source 
add support for "registry-mirrors" and "insecure-registries" to buildkit
This commit is contained in:
Tõnis Tiigi 2018-09-21 09:42:35 -07:00 committed by GitHub
commit b1116479b2
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
10 changed files with 131 additions and 18 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

17
builder/builder-next/adapters/containerimage/pull.go
View file

@ -34,6 +34,7 @@ import (
"github.com/moby/buildkit/util/flightcontrol"
"github.com/moby/buildkit/util/imageutil"
"github.com/moby/buildkit/util/progress"
"github.com/moby/buildkit/util/resolver"
"github.com/moby/buildkit/util/tracing"
digest "github.com/opencontainers/go-digest"
"github.com/opencontainers/image-spec/identity"
@ -51,6 +52,7 @@ type SourceOpt struct {
DownloadManager distribution.RootFSDownloadManager
MetadataStore metadata.V2MetadataService
ImageStore image.Store
ResolverOpt resolver.ResolveOptionsFunc
}
type imageSource struct {
@ -71,11 +73,16 @@ func (is *imageSource) ID() string {
return source.DockerImageScheme
}
func (is *imageSource) getResolver(ctx context.Context) remotes.Resolver {
return docker.NewResolver(docker.ResolverOptions{
func (is *imageSource) getResolver(ctx context.Context, rfn resolver.ResolveOptionsFunc, ref string) remotes.Resolver {
opt := docker.ResolverOptions{
Client: tracing.DefaultClient,
Credentials: is.getCredentialsFromSession(ctx),
})
}
if rfn != nil {
opt = rfn(ref)
}
r := docker.NewResolver(opt)
return r
}
func (is *imageSource) getCredentialsFromSession(ctx context.Context) func(string) (string, string, error) {
@ -118,7 +125,7 @@ func (is *imageSource) resolveRemote(ctx context.Context, ref string, platform *
dt []byte
}
res, err := is.g.Do(ctx, ref, func(ctx context.Context) (interface{}, error) {
dgst, dt, err := imageutil.Config(ctx, ref, is.getResolver(ctx), is.ContentStore, platform)
dgst, dt, err := imageutil.Config(ctx, ref, is.getResolver(ctx, is.ResolverOpt, ref), is.ContentStore, platform)
if err != nil {
return nil, err
}
@ -181,7 +188,7 @@ func (is *imageSource) Resolve(ctx context.Context, id source.Identifier) (sourc
p := &puller{
src: imageIdentifier,
is: is,
resolver: is.getResolver(ctx),
resolver: is.getResolver(ctx, is.ResolverOpt, imageIdentifier.Reference.String()),
platform: platform,
}
return p, nil

2
builder/builder-next/builder.go
View file

@ -23,6 +23,7 @@ import (
"github.com/moby/buildkit/session"
"github.com/moby/buildkit/solver/llbsolver"
"github.com/moby/buildkit/util/entitlements"
"github.com/moby/buildkit/util/resolver"
"github.com/moby/buildkit/util/tracing"
"github.com/pkg/errors"
"golang.org/x/sync/errgroup"
@ -55,6 +56,7 @@ type Opt struct {
Dist images.DistributionServices
NetworkController libnetwork.NetworkController
DefaultCgroupParent string
ResolverOpt resolver.ResolveOptionsFunc
}
// Builder can build using BuildKit backend

3
builder/builder-next/controller.go
View file

@ -97,6 +97,7 @@ func newController(rt http.RoundTripper, opt Opt) (*control.Controller, error) {
MetadataStore: dist.V2MetadataService,
ImageStore: dist.ImageStore,
ReferenceStore: dist.ReferenceStore,
ResolverOpt: opt.ResolverOpt,
})
if err != nil {
return nil, err
@ -160,7 +161,7 @@ func newController(rt http.RoundTripper, opt Opt) (*control.Controller, error) {
WorkerController: wc,
Frontends: frontends,
CacheKeyStorage: cacheStorage,
ResolveCacheImporterFunc: registryremotecache.ResolveCacheImporterFunc(opt.SessionManager),
ResolveCacheImporterFunc: registryremotecache.ResolveCacheImporterFunc(opt.SessionManager, opt.ResolverOpt),
// TODO: set ResolveCacheExporterFunc for exporting cache
})
}

1
cmd/dockerd/daemon.go
View file

@ -291,6 +291,7 @@ func newRouterOptions(config *config.Config, d *daemon.Daemon) (routerOptions, e
Dist: d.DistributionServices(),
NetworkController: d.NetworkController(),
DefaultCgroupParent: cgroupParent,
ResolverOpt: d.NewResolveOptionsFunc(),
})
if err != nil {
return opts, err

56
daemon/daemon.go
View file

@ -9,6 +9,7 @@ import (
"context"
"fmt"
"io/ioutil"
"math/rand"
"net"
"os"
"path"
@ -23,6 +24,8 @@ import (
"github.com/containerd/containerd"
"github.com/containerd/containerd/defaults"
"github.com/containerd/containerd/pkg/dialer"
"github.com/containerd/containerd/remotes/docker"
"github.com/docker/distribution/reference"
"github.com/docker/docker/api/types"
containertypes "github.com/docker/docker/api/types/container"
"github.com/docker/docker/api/types/swarm"
@ -36,6 +39,8 @@ import (
"github.com/docker/docker/daemon/logger"
"github.com/docker/docker/daemon/network"
"github.com/docker/docker/errdefs"
"github.com/moby/buildkit/util/resolver"
"github.com/moby/buildkit/util/tracing"
"github.com/sirupsen/logrus"
// register graph drivers
_ "github.com/docker/docker/daemon/graphdriver/register"
@ -141,6 +146,57 @@ func (daemon *Daemon) Features() *map[string]bool {
return &daemon.configStore.Features
}
// NewResolveOptionsFunc returns a call back function to resolve "registry-mirrors" and
// "insecure-registries" for buildkit
func (daemon *Daemon) NewResolveOptionsFunc() resolver.ResolveOptionsFunc {
return func(ref string) docker.ResolverOptions {
var (
registryKey = "docker.io"
mirrors = make([]string, len(daemon.configStore.Mirrors))
m = map[string]resolver.RegistryConf{}
)
// must trim "https://" or "http://" prefix
for i, v := range daemon.configStore.Mirrors {
v = strings.TrimPrefix(v, "https://")
v = strings.TrimPrefix(v, "http://")
mirrors[i] = v
}
// set "registry-mirrors"
m[registryKey] = resolver.RegistryConf{Mirrors: mirrors}
// set "insecure-registries"
for _, v := range daemon.configStore.InsecureRegistries {
v = strings.TrimPrefix(v, "http://")
m[v] = resolver.RegistryConf{
PlainHTTP: true,
}
}
def := docker.ResolverOptions{
Client: tracing.DefaultClient,
}
parsed, err := reference.ParseNormalizedNamed(ref)
if err != nil {
return def
}
host := reference.Domain(parsed)
c, ok := m[host]
if !ok {
return def
}
if len(c.Mirrors) > 0 {
def.Host = func(string) (string, error) {
return c.Mirrors[rand.Intn(len(c.Mirrors))], nil
}
}
def.PlainHTTP = c.PlainHTTP
return def
}
}
func (daemon *Daemon) restore() error {
containers := make(map[string]*container.Container)

2
vendor.conf
View file

@ -26,7 +26,7 @@ github.com/imdario/mergo v0.3.6
golang.org/x/sync 1d60e4601c6fd243af51cc01ddf169918a5407ca
# buildkit
github.com/moby/buildkit a9fe50acf16dd05d1f9877b27068884543ad7a1f
github.com/moby/buildkit d88354f7856a1fafef6f23bc9c5a538c246f4023
github.com/tonistiigi/fsutil b19464cd1b6a00773b4f2eb7acf9c30426f9df42
github.com/grpc-ecosystem/grpc-opentracing 8e809c8a86450a29b90dcc9efbf062d0fe6d9746
github.com/opentracing/opentracing-go 1361b9cd60be79c4c3a7fa9841b3c132e40066a7

19
vendor/github.com/moby/buildkit/cache/remotecache/registry/registry.go generated vendored
View file

@ -10,17 +10,17 @@ import (
"github.com/moby/buildkit/session"
"github.com/moby/buildkit/session/auth"
"github.com/moby/buildkit/util/contentutil"
"github.com/moby/buildkit/util/tracing"
"github.com/moby/buildkit/util/resolver"
specs "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/pkg/errors"
)
func ResolveCacheExporterFunc(sm *session.Manager) remotecache.ResolveCacheExporterFunc {
func ResolveCacheExporterFunc(sm *session.Manager, resolverOpt resolver.ResolveOptionsFunc) remotecache.ResolveCacheExporterFunc {
return func(ctx context.Context, typ, ref string) (remotecache.Exporter, error) {
if typ != "" {
return nil, errors.Errorf("unsupported cache exporter type: %s", typ)
}
remote := newRemoteResolver(ctx, sm)
remote := newRemoteResolver(ctx, resolverOpt, sm, ref)
pusher, err := remote.Pusher(ctx, ref)
if err != nil {
return nil, err
@ -29,12 +29,12 @@ func ResolveCacheExporterFunc(sm *session.Manager) remotecache.ResolveCacheExpor
}
}
func ResolveCacheImporterFunc(sm *session.Manager) remotecache.ResolveCacheImporterFunc {
func ResolveCacheImporterFunc(sm *session.Manager, resolverOpt resolver.ResolveOptionsFunc) remotecache.ResolveCacheImporterFunc {
return func(ctx context.Context, typ, ref string) (remotecache.Importer, specs.Descriptor, error) {
if typ != "" {
return nil, specs.Descriptor{}, errors.Errorf("unsupported cache importer type: %s", typ)
}
remote := newRemoteResolver(ctx, sm)
remote := newRemoteResolver(ctx, resolverOpt, sm, ref)
xref, desc, err := remote.Resolve(ctx, ref)
if err != nil {
return nil, specs.Descriptor{}, err
@ -47,11 +47,10 @@ func ResolveCacheImporterFunc(sm *session.Manager) remotecache.ResolveCacheImpor
}
}
func newRemoteResolver(ctx context.Context, sm *session.Manager) remotes.Resolver {
return docker.NewResolver(docker.ResolverOptions{
Client: tracing.DefaultClient,
Credentials: getCredentialsFunc(ctx, sm),
})
func newRemoteResolver(ctx context.Context, resolverOpt resolver.ResolveOptionsFunc, sm *session.Manager, ref string) remotes.Resolver {
opt := resolverOpt(ref)
opt.Credentials = getCredentialsFunc(ctx, sm)
return docker.NewResolver(opt)
}
func getCredentialsFunc(ctx context.Context, sm *session.Manager) func(string) (string, string, error) {

2
vendor/github.com/moby/buildkit/solver/jobs.go generated vendored
View file

@ -444,6 +444,7 @@ func (j *Job) Discard() error {
j.pw.Close()
for k, st := range j.list.actives {
st.mu.Lock()
if _, ok := st.jobs[j]; ok {
delete(st.jobs, j)
j.list.deleteIfUnreferenced(k, st)
@ -451,6 +452,7 @@ func (j *Job) Discard() error {
if _, ok := st.allPw[j.pw]; ok {
delete(st.allPw, j.pw)
}
st.mu.Unlock()
}
return nil
}

45
vendor/github.com/moby/buildkit/util/resolver/resolver.go generated vendored Normal file
View file

@ -0,0 +1,45 @@
package resolver
import (
"math/rand"
"github.com/containerd/containerd/remotes/docker"
"github.com/docker/distribution/reference"
"github.com/moby/buildkit/util/tracing"
)
type RegistryConf struct {
Mirrors []string
PlainHTTP bool
}
type ResolveOptionsFunc func(string) docker.ResolverOptions
func NewResolveOptionsFunc(m map[string]RegistryConf) ResolveOptionsFunc {
return func(ref string) docker.ResolverOptions {
def := docker.ResolverOptions{
Client: tracing.DefaultClient,
}
parsed, err := reference.ParseNormalizedNamed(ref)
if err != nil {
return def
}
host := reference.Domain(parsed)
c, ok := m[host]
if !ok {
return def
}
if len(c.Mirrors) > 0 {
def.Host = func(string) (string, error) {
return c.Mirrors[rand.Intn(len(c.Mirrors))], nil
}
}
def.PlainHTTP = c.PlainHTTP
return def
}
}

2
vendor/github.com/moby/buildkit/vendor.conf generated vendored
View file

@ -39,7 +39,7 @@ golang.org/x/time f51c12702a4d776e4c1fa9b0fabab841babae631
github.com/docker/docker 71cd53e4a197b303c6ba086bd584ffd67a884281
github.com/pkg/profile 5b67d428864e92711fcbd2f8629456121a56d91f
github.com/tonistiigi/fsutil b19464cd1b6a00773b4f2eb7acf9c30426f9df42
github.com/tonistiigi/fsutil 7e391b0e788f9b925f22bd3cf88e0210d1643673
github.com/hashicorp/go-immutable-radix 826af9ccf0feeee615d546d69b11f8e98da8c8f1 git://github.com/tonistiigi/go-immutable-radix.git
github.com/hashicorp/golang-lru a0d98a5f288019575c6d1f4bb1573fef2d1fcdc4
github.com/mitchellh/hashstructure 2bca23e0e452137f789efbc8610126fd8b94f73b