builder: add cache-from support to buildkit

Signed-off-by: Tonis Tiigi <tonistiigi@gmail.com>

builder/builder-next/builder.go

@@ -21,6 +21,7 @@ import (
 	"github.com/docker/docker/pkg/jsonmessage"
 	controlapi "github.com/moby/buildkit/api/services/control"
 	"github.com/moby/buildkit/cache"
+	"github.com/moby/buildkit/cache/cacheimport"
 	"github.com/moby/buildkit/cache/metadata"
 	"github.com/moby/buildkit/control"
 	"github.com/moby/buildkit/executor/runcexecutor"
@@ -118,6 +119,10 @@ func (b *Builder) Build(ctx context.Context, opt backend.BuildConfig) (*builder.
 		frontendAttrs["context"] = opt.Options.RemoteContext
 	}
 
+	if len(opt.Options.CacheFrom) > 0 {
+		frontendAttrs["cache-from"] = opt.Options.CacheFrom[0]
+	}
+
 	logrus.Debugf("frontend: %+v", frontendAttrs)
 
 	for k, v := range opt.Options.BuildArgs {
@@ -255,6 +260,8 @@ func newController(opt Opt, reporter chan containerimageexp.Result) (*control.Co
 		ContentStore:    store,
 		DownloadManager: dist.DownloadManager,
 		MetadataStore:   dist.V2MetadataService,
+		ImageStore:      dist.ImageStore,
+		ReferenceStore:  dist.ReferenceStore,
 	})
 	if err != nil {
 		return nil, err
@@ -311,14 +318,16 @@ func newController(opt Opt, reporter chan containerimageexp.Result) (*control.Co
 	// }
 
 	wopt := mobyworker.WorkerOpt{
-		ID:             "moby",
-		SessionManager: opt.SessionManager,
-		MetadataStore:  md,
-		ContentStore:   store,
-		CacheManager:   cm,
-		Snapshotter:    snapshotter,
-		Executor:       exec,
-		ImageSource:    src,
+		ID:                "moby",
+		SessionManager:    opt.SessionManager,
+		MetadataStore:     md,
+		ContentStore:      store,
+		CacheManager:      cm,
+		Snapshotter:       snapshotter,
+		Executor:          exec,
+		ImageSource:       src,
+		DownloadManager:   dist.DownloadManager,
+		V2MetadataService: dist.V2MetadataService,
 		Exporters: map[string]exporter.Exporter{
 			"image": exp,
 		},
@@ -331,13 +340,18 @@ func newController(opt Opt, reporter chan containerimageexp.Result) (*control.Co
 	}
 	wc.Add(w)
 
+	ci := cacheimport.NewCacheImporter(cacheimport.ImportOpt{
+		Worker:         w,
+		SessionManager: opt.SessionManager,
+	})
+
 	return control.NewController(control.Opt{
 		SessionManager:   opt.SessionManager,
 		WorkerController: wc,
 		Frontends:        frontends,
 		CacheKeyStorage:  cacheStorage,
 		// CacheExporter:    ce,
-		// CacheImporter:    ci,
+		CacheImporter: ci,
 	})
 }
 

builder/builder-next/containerimage/pull.go

@@ -18,6 +18,7 @@ import (
 	"github.com/containerd/containerd/remotes"
 	"github.com/containerd/containerd/remotes/docker"
 	"github.com/containerd/containerd/remotes/docker/schema1"
+	distreference "github.com/docker/distribution/reference"
 	"github.com/docker/docker/distribution"
 	"github.com/docker/docker/distribution/metadata"
 	"github.com/docker/docker/distribution/xfer"
@@ -40,6 +41,8 @@ import (
 	"golang.org/x/time/rate"
 )
 
+const preferLocal = true // FIXME: make this optional from the op
+
 type SourceOpt struct {
 	SessionManager  *session.Manager
 	ContentStore    content.Store
@@ -47,6 +50,7 @@ type SourceOpt struct {
 	ReferenceStore  reference.Store
 	DownloadManager distribution.RootFSDownloadManager
 	MetadataStore   metadata.V2MetadataService
+	ImageStore      image.Store
 }
 
 type imageSource struct {
@@ -91,7 +95,30 @@ func (is *imageSource) getCredentialsFromSession(ctx context.Context) func(strin
 	}
 }
 
+func (is *imageSource) resolveLocal(refStr string) ([]byte, error) {
+	ref, err := distreference.ParseNormalizedNamed(refStr)
+	if err != nil {
+		return nil, err
+	}
+	dgst, err := is.ReferenceStore.Get(ref)
+	if err != nil {
+		return nil, err
+	}
+	img, err := is.ImageStore.Get(image.ID(dgst))
+	if err != nil {
+		return nil, err
+	}
+	return img.RawJSON(), nil
+}
+
 func (is *imageSource) ResolveImageConfig(ctx context.Context, ref string) (digest.Digest, []byte, error) {
+	if preferLocal {
+		dt, err := is.resolveLocal(ref)
+		if err == nil {
+			return "", dt, nil
+		}
+	}
+
 	type t struct {
 		dgst digest.Digest
 		dt   []byte
@@ -132,41 +159,84 @@ type puller struct {
 	ref         string
 	resolveErr  error
 	resolver    remotes.Resolver
+	imageID     image.ID
+	cacheKey    digest.Digest
 }
 
 func (p *puller) resolve(ctx context.Context) error {
 	p.resolveOnce.Do(func() {
 		resolveProgressDone := oneOffProgress(ctx, "resolve "+p.src.Reference.String())
 
-		dgst := p.src.Reference.Digest()
-		if dgst != "" {
-			info, err := p.is.ContentStore.Info(ctx, dgst)
+		// dgst := p.src.Reference.Digest()
+		// if dgst != "" {
+		// 	info, err := p.is.ContentStore.Info(ctx, dgst)
+		// 	if err == nil {
+		// 		p.ref = p.src.Reference.String()
+		// 		ra, err := p.is.ContentStore.ReaderAt(ctx, dgst)
+		// 		if err == nil {
+		// 			mt, err := imageutil.DetectManifestMediaType(ra)
+		// 			if err == nil {
+		// 				p.desc = ocispec.Descriptor{
+		// 					Size:      info.Size,
+		// 					Digest:    dgst,
+		// 					MediaType: mt,
+		// 				}
+		// 				resolveProgressDone(nil)
+		// 				return
+		// 			}
+		// 		}
+		// 	}
+		// }
+
+		// ref, desc, err := p.resolver.Resolve(ctx, p.src.Reference.String())
+		// if err != nil {
+		// 	p.resolveErr = err
+		// 	resolveProgressDone(err)
+		// 	return
+		// }
+
+		if preferLocal {
+			dt, err := p.is.resolveLocal(p.src.Reference.String())
 			if err == nil {
-				p.ref = p.src.Reference.String()
-				ra, err := p.is.ContentStore.ReaderAt(ctx, dgst)
-				if err == nil {
-					mt, err := imageutil.DetectManifestMediaType(ra)
-					if err == nil {
-						p.desc = ocispec.Descriptor{
-							Size:      info.Size,
-							Digest:    dgst,
-							MediaType: mt,
-						}
-						resolveProgressDone(nil)
-						return
-					}
-				}
+				dgst := digest.FromBytes(dt)
+				p.imageID = image.ID(dgst)
+				p.cacheKey = dgst
+				resolveProgressDone(nil)
+				return
 			}
+
+		}
+
+		ref, err := distreference.ParseNormalizedNamed(p.src.Reference.String())
+		if err != nil {
+			p.resolveErr = err
+			resolveProgressDone(err)
+			return
+		}
+
+		outRef, desc, err := p.resolver.Resolve(ctx, p.src.Reference.String())
+		if err != nil {
+			p.resolveErr = err
+			resolveProgressDone(err)
+			return
+		}
+
+		ref, err = distreference.WithDigest(ref, desc.Digest)
+		if err != nil {
+			p.resolveErr = err
+			resolveProgressDone(err)
+			return
 		}
 
-		ref, desc, err := p.resolver.Resolve(ctx, p.src.Reference.String())
+		_, dt, err := p.is.ResolveImageConfig(ctx, ref.String())
 		if err != nil {
 			p.resolveErr = err
 			resolveProgressDone(err)
 			return
 		}
 		p.desc = desc
-		p.ref = ref
+		p.cacheKey = digest.FromBytes(dt)
+		p.ref = outRef
 		resolveProgressDone(nil)
 	})
 	return p.resolveErr
@@ -176,7 +246,7 @@ func (p *puller) CacheKey(ctx context.Context) (string, error) {
 	if err := p.resolve(ctx); err != nil {
 		return "", err
 	}
-	return p.desc.Digest.String(), nil
+	return p.cacheKey.String(), nil
 }
 
 func (p *puller) Snapshot(ctx context.Context) (cache.ImmutableRef, error) {
@@ -184,6 +254,18 @@ func (p *puller) Snapshot(ctx context.Context) (cache.ImmutableRef, error) {
 		return nil, err
 	}
 
+	if p.imageID != "" {
+		img, err := p.is.ImageStore.Get(p.imageID)
+		if err != nil {
+			return nil, err
+		}
+		ref, err := p.is.CacheAccessor.Get(ctx, string(img.RootFS.ChainID()), cache.WithDescription(fmt.Sprintf("from local %s", p.ref)))
+		if err != nil {
+			return nil, err
+		}
+		return ref, nil
+	}
+
 	ongoing := newJobs(p.ref)
 
 	pctx, stopProgress := context.WithCancel(ctx)
@@ -393,7 +475,7 @@ func (ld *layerDescriptor) Download(ctx netcontext.Context, progressOutput pkgpr
 }
 
 func (ld *layerDescriptor) Close() {
-	ld.is.ContentStore.Delete(context.TODO(), ld.desc.Digest)
+	// ld.is.ContentStore.Delete(context.TODO(), ld.desc.Digest))
 }
 
 func (ld *layerDescriptor) Registered(diffID layer.DiffID) {

builder/builder-next/worker/worker.go

@@ -2,11 +2,20 @@ package worker
 
 import (
 	"context"
+	"fmt"
 	"io"
+	"io/ioutil"
+	"runtime"
 	"time"
 
 	"github.com/containerd/containerd/content"
 	"github.com/containerd/containerd/rootfs"
+	"github.com/docker/docker/distribution"
+	distmetadata "github.com/docker/docker/distribution/metadata"
+	"github.com/docker/docker/distribution/xfer"
+	"github.com/docker/docker/image"
+	"github.com/docker/docker/layer"
+	pkgprogress "github.com/docker/docker/pkg/progress"
 	"github.com/moby/buildkit/cache"
 	"github.com/moby/buildkit/cache/metadata"
 	"github.com/moby/buildkit/client"
@@ -22,10 +31,12 @@ import (
 	"github.com/moby/buildkit/source/git"
 	"github.com/moby/buildkit/source/http"
 	"github.com/moby/buildkit/source/local"
+	"github.com/moby/buildkit/util/contentutil"
 	"github.com/moby/buildkit/util/progress"
 	digest "github.com/opencontainers/go-digest"
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/pkg/errors"
+	netcontext "golang.org/x/net/context"
 )
 
 // TODO: this file should be removed. containerd defines ContainerdWorker, oci defines OCIWorker. There is no base worker.
@@ -33,16 +44,18 @@ import (
 // WorkerOpt is specific to a worker.
 // See also CommonOpt.
 type WorkerOpt struct {
-	ID             string
-	Labels         map[string]string
-	SessionManager *session.Manager
-	MetadataStore  *metadata.Store
-	Executor       executor.Executor
-	Snapshotter    snapshot.Snapshotter
-	ContentStore   content.Store
-	CacheManager   cache.Manager
-	ImageSource    source.Source
-	Exporters      map[string]exporter.Exporter
+	ID                string
+	Labels            map[string]string
+	SessionManager    *session.Manager
+	MetadataStore     *metadata.Store
+	Executor          executor.Executor
+	Snapshotter       snapshot.Snapshotter
+	ContentStore      content.Store
+	CacheManager      cache.Manager
+	ImageSource       source.Source
+	Exporters         map[string]exporter.Exporter
+	DownloadManager   distribution.RootFSDownloadManager
+	V2MetadataService distmetadata.V2MetadataService
 	// ImageStore     images.Store // optional
 }
 
@@ -198,6 +211,43 @@ func (w *Worker) GetRemote(ctx context.Context, ref cache.ImmutableRef) (*solver
 }
 
 func (w *Worker) FromRemote(ctx context.Context, remote *solver.Remote) (cache.ImmutableRef, error) {
+	rootfs, err := getLayers(ctx, remote.Descriptors)
+	if err != nil {
+		return nil, err
+	}
+
+	layers := make([]xfer.DownloadDescriptor, 0, len(rootfs))
+
+	for _, l := range rootfs {
+		// ongoing.add(desc)
+		layers = append(layers, &layerDescriptor{
+			desc:     l.Blob,
+			diffID:   layer.DiffID(l.Diff.Digest),
+			provider: remote.Provider,
+			w:        w,
+			pctx:     ctx,
+			// ref:      l.Blob.Digest.String(),
+		})
+	}
+
+	defer func() {
+		for _, l := range rootfs {
+			w.ContentStore.Delete(context.TODO(), l.Blob.Digest)
+		}
+	}()
+
+	r := image.NewRootFS()
+	rootFS, release, err := w.DownloadManager.Download(ctx, *r, runtime.GOOS, layers, &discardProgress{})
+	if err != nil {
+		return nil, err
+	}
+	defer release()
+
+	ref, err := w.CacheManager.Get(ctx, string(rootFS.ChainID()), cache.WithDescription(fmt.Sprintf("imported %s", remote.Descriptors[len(remote.Descriptors)-1].Digest)))
+	if err != nil {
+		return nil, err
+	}
+
 	// eg, gctx := errgroup.WithContext(ctx)
 	// for _, desc := range remote.Descriptors {
 	// 	func(desc ocispec.Descriptor) {
@@ -223,7 +273,8 @@ func (w *Worker) FromRemote(ctx context.Context, remote *solver.Remote) (cache.I
 	// unpackProgressDone(nil)
 	//
 	// return w.CacheManager.Get(ctx, chainID, cache.WithDescription(fmt.Sprintf("imported %s", remote.Descriptors[len(remote.Descriptors)-1].Digest)))
-	return nil, errors.Errorf("fromremote not implemented")
+	// return nil, errors.Errorf("fromremote not implemented")
+	return ref, nil
 }
 
 // utility function. could be moved to the constructor logic?
@@ -259,6 +310,58 @@ func (w *Worker) FromRemote(ctx context.Context, remote *solver.Remote) (cache.I
 // 	return string(b), nil
 // }
 
+type discardProgress struct{}
+
+func (_ *discardProgress) WriteProgress(_ pkgprogress.Progress) error {
+	return nil
+}
+
+// Fetch(ctx context.Context, desc ocispec.Descriptor) (io.ReadCloser, error)
+type layerDescriptor struct {
+	provider content.Provider
+	desc     ocispec.Descriptor
+	diffID   layer.DiffID
+	// ref      ctdreference.Spec
+	w    *Worker
+	pctx context.Context
+}
+
+func (ld *layerDescriptor) Key() string {
+	return "v2:" + ld.desc.Digest.String()
+}
+
+func (ld *layerDescriptor) ID() string {
+	return ld.desc.Digest.String()
+}
+
+func (ld *layerDescriptor) DiffID() (layer.DiffID, error) {
+	return ld.diffID, nil
+}
+
+func (ld *layerDescriptor) Download(ctx netcontext.Context, progressOutput pkgprogress.Output) (io.ReadCloser, int64, error) {
+	done := oneOffProgress(ld.pctx, fmt.Sprintf("pulling %s", ld.desc.Digest))
+	if err := contentutil.Copy(ctx, ld.w.ContentStore, ld.provider, ld.desc); err != nil {
+		return nil, 0, done(err)
+	}
+	done(nil)
+
+	ra, err := ld.w.ContentStore.ReaderAt(ctx, ld.desc.Digest)
+	if err != nil {
+		return nil, 0, err
+	}
+
+	return ioutil.NopCloser(content.NewReader(ra)), ld.desc.Size, nil
+}
+
+func (ld *layerDescriptor) Close() {
+	// ld.is.ContentStore.Delete(context.TODO(), ld.desc.Digest)
+}
+
+func (ld *layerDescriptor) Registered(diffID layer.DiffID) {
+	// Cache mapping from this layer's DiffID to the blobsum
+	ld.w.V2MetadataService.Add(diffID, distmetadata.V2Metadata{Digest: ld.desc.Digest})
+}
+
 func getLayers(ctx context.Context, descs []ocispec.Descriptor) ([]rootfs.Layer, error) {
 	layers := make([]rootfs.Layer, len(descs))
 	for i, desc := range descs {