diff --git a/builder/builder-next/builder.go b/builder/builder-next/builder.go index d654cbca1b..44538c2b81 100644 --- a/builder/builder-next/builder.go +++ b/builder/builder-next/builder.go @@ -21,6 +21,7 @@ import ( "github.com/docker/docker/pkg/jsonmessage" controlapi "github.com/moby/buildkit/api/services/control" "github.com/moby/buildkit/cache" + "github.com/moby/buildkit/cache/cacheimport" "github.com/moby/buildkit/cache/metadata" "github.com/moby/buildkit/control" "github.com/moby/buildkit/executor/runcexecutor" @@ -118,6 +119,10 @@ func (b *Builder) Build(ctx context.Context, opt backend.BuildConfig) (*builder. frontendAttrs["context"] = opt.Options.RemoteContext } + if len(opt.Options.CacheFrom) > 0 { + frontendAttrs["cache-from"] = opt.Options.CacheFrom[0] + } + logrus.Debugf("frontend: %+v", frontendAttrs) for k, v := range opt.Options.BuildArgs { @@ -255,6 +260,8 @@ func newController(opt Opt, reporter chan containerimageexp.Result) (*control.Co ContentStore: store, DownloadManager: dist.DownloadManager, MetadataStore: dist.V2MetadataService, + ImageStore: dist.ImageStore, + ReferenceStore: dist.ReferenceStore, }) if err != nil { return nil, err @@ -311,14 +318,16 @@ func newController(opt Opt, reporter chan containerimageexp.Result) (*control.Co // } wopt := mobyworker.WorkerOpt{ - ID: "moby", - SessionManager: opt.SessionManager, - MetadataStore: md, - ContentStore: store, - CacheManager: cm, - Snapshotter: snapshotter, - Executor: exec, - ImageSource: src, + ID: "moby", + SessionManager: opt.SessionManager, + MetadataStore: md, + ContentStore: store, + CacheManager: cm, + Snapshotter: snapshotter, + Executor: exec, + ImageSource: src, + DownloadManager: dist.DownloadManager, + V2MetadataService: dist.V2MetadataService, Exporters: map[string]exporter.Exporter{ "image": exp, }, @@ -331,13 +340,18 @@ func newController(opt Opt, reporter chan containerimageexp.Result) (*control.Co } wc.Add(w) + ci := cacheimport.NewCacheImporter(cacheimport.ImportOpt{ + Worker: w, + SessionManager: opt.SessionManager, + }) + return control.NewController(control.Opt{ SessionManager: opt.SessionManager, WorkerController: wc, Frontends: frontends, CacheKeyStorage: cacheStorage, // CacheExporter: ce, - // CacheImporter: ci, + CacheImporter: ci, }) } diff --git a/builder/builder-next/containerimage/pull.go b/builder/builder-next/containerimage/pull.go index f785b4aa48..88dfc3c239 100644 --- a/builder/builder-next/containerimage/pull.go +++ b/builder/builder-next/containerimage/pull.go @@ -18,6 +18,7 @@ import ( "github.com/containerd/containerd/remotes" "github.com/containerd/containerd/remotes/docker" "github.com/containerd/containerd/remotes/docker/schema1" + distreference "github.com/docker/distribution/reference" "github.com/docker/docker/distribution" "github.com/docker/docker/distribution/metadata" "github.com/docker/docker/distribution/xfer" @@ -40,6 +41,8 @@ import ( "golang.org/x/time/rate" ) +const preferLocal = true // FIXME: make this optional from the op + type SourceOpt struct { SessionManager *session.Manager ContentStore content.Store @@ -47,6 +50,7 @@ type SourceOpt struct { ReferenceStore reference.Store DownloadManager distribution.RootFSDownloadManager MetadataStore metadata.V2MetadataService + ImageStore image.Store } type imageSource struct { @@ -91,7 +95,30 @@ func (is *imageSource) getCredentialsFromSession(ctx context.Context) func(strin } } +func (is *imageSource) resolveLocal(refStr string) ([]byte, error) { + ref, err := distreference.ParseNormalizedNamed(refStr) + if err != nil { + return nil, err + } + dgst, err := is.ReferenceStore.Get(ref) + if err != nil { + return nil, err + } + img, err := is.ImageStore.Get(image.ID(dgst)) + if err != nil { + return nil, err + } + return img.RawJSON(), nil +} + func (is *imageSource) ResolveImageConfig(ctx context.Context, ref string) (digest.Digest, []byte, error) { + if preferLocal { + dt, err := is.resolveLocal(ref) + if err == nil { + return "", dt, nil + } + } + type t struct { dgst digest.Digest dt []byte @@ -132,41 +159,84 @@ type puller struct { ref string resolveErr error resolver remotes.Resolver + imageID image.ID + cacheKey digest.Digest } func (p *puller) resolve(ctx context.Context) error { p.resolveOnce.Do(func() { resolveProgressDone := oneOffProgress(ctx, "resolve "+p.src.Reference.String()) - dgst := p.src.Reference.Digest() - if dgst != "" { - info, err := p.is.ContentStore.Info(ctx, dgst) + // dgst := p.src.Reference.Digest() + // if dgst != "" { + // info, err := p.is.ContentStore.Info(ctx, dgst) + // if err == nil { + // p.ref = p.src.Reference.String() + // ra, err := p.is.ContentStore.ReaderAt(ctx, dgst) + // if err == nil { + // mt, err := imageutil.DetectManifestMediaType(ra) + // if err == nil { + // p.desc = ocispec.Descriptor{ + // Size: info.Size, + // Digest: dgst, + // MediaType: mt, + // } + // resolveProgressDone(nil) + // return + // } + // } + // } + // } + + // ref, desc, err := p.resolver.Resolve(ctx, p.src.Reference.String()) + // if err != nil { + // p.resolveErr = err + // resolveProgressDone(err) + // return + // } + + if preferLocal { + dt, err := p.is.resolveLocal(p.src.Reference.String()) if err == nil { - p.ref = p.src.Reference.String() - ra, err := p.is.ContentStore.ReaderAt(ctx, dgst) - if err == nil { - mt, err := imageutil.DetectManifestMediaType(ra) - if err == nil { - p.desc = ocispec.Descriptor{ - Size: info.Size, - Digest: dgst, - MediaType: mt, - } - resolveProgressDone(nil) - return - } - } + dgst := digest.FromBytes(dt) + p.imageID = image.ID(dgst) + p.cacheKey = dgst + resolveProgressDone(nil) + return } + } - ref, desc, err := p.resolver.Resolve(ctx, p.src.Reference.String()) + ref, err := distreference.ParseNormalizedNamed(p.src.Reference.String()) + if err != nil { + p.resolveErr = err + resolveProgressDone(err) + return + } + + outRef, desc, err := p.resolver.Resolve(ctx, p.src.Reference.String()) + if err != nil { + p.resolveErr = err + resolveProgressDone(err) + return + } + + ref, err = distreference.WithDigest(ref, desc.Digest) + if err != nil { + p.resolveErr = err + resolveProgressDone(err) + return + } + + _, dt, err := p.is.ResolveImageConfig(ctx, ref.String()) if err != nil { p.resolveErr = err resolveProgressDone(err) return } p.desc = desc - p.ref = ref + p.cacheKey = digest.FromBytes(dt) + p.ref = outRef resolveProgressDone(nil) }) return p.resolveErr @@ -176,7 +246,7 @@ func (p *puller) CacheKey(ctx context.Context) (string, error) { if err := p.resolve(ctx); err != nil { return "", err } - return p.desc.Digest.String(), nil + return p.cacheKey.String(), nil } func (p *puller) Snapshot(ctx context.Context) (cache.ImmutableRef, error) { @@ -184,6 +254,18 @@ func (p *puller) Snapshot(ctx context.Context) (cache.ImmutableRef, error) { return nil, err } + if p.imageID != "" { + img, err := p.is.ImageStore.Get(p.imageID) + if err != nil { + return nil, err + } + ref, err := p.is.CacheAccessor.Get(ctx, string(img.RootFS.ChainID()), cache.WithDescription(fmt.Sprintf("from local %s", p.ref))) + if err != nil { + return nil, err + } + return ref, nil + } + ongoing := newJobs(p.ref) pctx, stopProgress := context.WithCancel(ctx) @@ -393,7 +475,7 @@ func (ld *layerDescriptor) Download(ctx netcontext.Context, progressOutput pkgpr } func (ld *layerDescriptor) Close() { - ld.is.ContentStore.Delete(context.TODO(), ld.desc.Digest) + // ld.is.ContentStore.Delete(context.TODO(), ld.desc.Digest)) } func (ld *layerDescriptor) Registered(diffID layer.DiffID) { diff --git a/builder/builder-next/worker/worker.go b/builder/builder-next/worker/worker.go index 761f99d821..bdf9d49876 100644 --- a/builder/builder-next/worker/worker.go +++ b/builder/builder-next/worker/worker.go @@ -2,11 +2,20 @@ package worker import ( "context" + "fmt" "io" + "io/ioutil" + "runtime" "time" "github.com/containerd/containerd/content" "github.com/containerd/containerd/rootfs" + "github.com/docker/docker/distribution" + distmetadata "github.com/docker/docker/distribution/metadata" + "github.com/docker/docker/distribution/xfer" + "github.com/docker/docker/image" + "github.com/docker/docker/layer" + pkgprogress "github.com/docker/docker/pkg/progress" "github.com/moby/buildkit/cache" "github.com/moby/buildkit/cache/metadata" "github.com/moby/buildkit/client" @@ -22,10 +31,12 @@ import ( "github.com/moby/buildkit/source/git" "github.com/moby/buildkit/source/http" "github.com/moby/buildkit/source/local" + "github.com/moby/buildkit/util/contentutil" "github.com/moby/buildkit/util/progress" digest "github.com/opencontainers/go-digest" ocispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" + netcontext "golang.org/x/net/context" ) // TODO: this file should be removed. containerd defines ContainerdWorker, oci defines OCIWorker. There is no base worker. @@ -33,16 +44,18 @@ import ( // WorkerOpt is specific to a worker. // See also CommonOpt. type WorkerOpt struct { - ID string - Labels map[string]string - SessionManager *session.Manager - MetadataStore *metadata.Store - Executor executor.Executor - Snapshotter snapshot.Snapshotter - ContentStore content.Store - CacheManager cache.Manager - ImageSource source.Source - Exporters map[string]exporter.Exporter + ID string + Labels map[string]string + SessionManager *session.Manager + MetadataStore *metadata.Store + Executor executor.Executor + Snapshotter snapshot.Snapshotter + ContentStore content.Store + CacheManager cache.Manager + ImageSource source.Source + Exporters map[string]exporter.Exporter + DownloadManager distribution.RootFSDownloadManager + V2MetadataService distmetadata.V2MetadataService // ImageStore images.Store // optional } @@ -198,6 +211,43 @@ func (w *Worker) GetRemote(ctx context.Context, ref cache.ImmutableRef) (*solver } func (w *Worker) FromRemote(ctx context.Context, remote *solver.Remote) (cache.ImmutableRef, error) { + rootfs, err := getLayers(ctx, remote.Descriptors) + if err != nil { + return nil, err + } + + layers := make([]xfer.DownloadDescriptor, 0, len(rootfs)) + + for _, l := range rootfs { + // ongoing.add(desc) + layers = append(layers, &layerDescriptor{ + desc: l.Blob, + diffID: layer.DiffID(l.Diff.Digest), + provider: remote.Provider, + w: w, + pctx: ctx, + // ref: l.Blob.Digest.String(), + }) + } + + defer func() { + for _, l := range rootfs { + w.ContentStore.Delete(context.TODO(), l.Blob.Digest) + } + }() + + r := image.NewRootFS() + rootFS, release, err := w.DownloadManager.Download(ctx, *r, runtime.GOOS, layers, &discardProgress{}) + if err != nil { + return nil, err + } + defer release() + + ref, err := w.CacheManager.Get(ctx, string(rootFS.ChainID()), cache.WithDescription(fmt.Sprintf("imported %s", remote.Descriptors[len(remote.Descriptors)-1].Digest))) + if err != nil { + return nil, err + } + // eg, gctx := errgroup.WithContext(ctx) // for _, desc := range remote.Descriptors { // func(desc ocispec.Descriptor) { @@ -223,7 +273,8 @@ func (w *Worker) FromRemote(ctx context.Context, remote *solver.Remote) (cache.I // unpackProgressDone(nil) // // return w.CacheManager.Get(ctx, chainID, cache.WithDescription(fmt.Sprintf("imported %s", remote.Descriptors[len(remote.Descriptors)-1].Digest))) - return nil, errors.Errorf("fromremote not implemented") + // return nil, errors.Errorf("fromremote not implemented") + return ref, nil } // utility function. could be moved to the constructor logic? @@ -259,6 +310,58 @@ func (w *Worker) FromRemote(ctx context.Context, remote *solver.Remote) (cache.I // return string(b), nil // } +type discardProgress struct{} + +func (_ *discardProgress) WriteProgress(_ pkgprogress.Progress) error { + return nil +} + +// Fetch(ctx context.Context, desc ocispec.Descriptor) (io.ReadCloser, error) +type layerDescriptor struct { + provider content.Provider + desc ocispec.Descriptor + diffID layer.DiffID + // ref ctdreference.Spec + w *Worker + pctx context.Context +} + +func (ld *layerDescriptor) Key() string { + return "v2:" + ld.desc.Digest.String() +} + +func (ld *layerDescriptor) ID() string { + return ld.desc.Digest.String() +} + +func (ld *layerDescriptor) DiffID() (layer.DiffID, error) { + return ld.diffID, nil +} + +func (ld *layerDescriptor) Download(ctx netcontext.Context, progressOutput pkgprogress.Output) (io.ReadCloser, int64, error) { + done := oneOffProgress(ld.pctx, fmt.Sprintf("pulling %s", ld.desc.Digest)) + if err := contentutil.Copy(ctx, ld.w.ContentStore, ld.provider, ld.desc); err != nil { + return nil, 0, done(err) + } + done(nil) + + ra, err := ld.w.ContentStore.ReaderAt(ctx, ld.desc.Digest) + if err != nil { + return nil, 0, err + } + + return ioutil.NopCloser(content.NewReader(ra)), ld.desc.Size, nil +} + +func (ld *layerDescriptor) Close() { + // ld.is.ContentStore.Delete(context.TODO(), ld.desc.Digest) +} + +func (ld *layerDescriptor) Registered(diffID layer.DiffID) { + // Cache mapping from this layer's DiffID to the blobsum + ld.w.V2MetadataService.Add(diffID, distmetadata.V2Metadata{Digest: ld.desc.Digest}) +} + func getLayers(ctx context.Context, descs []ocispec.Descriptor) ([]rootfs.Layer, error) { layers := make([]rootfs.Layer, len(descs)) for i, desc := range descs {