beenull/moby
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

builder: add cache-from support to buildkit

Browse source 
Signed-off-by: Tonis Tiigi <tonistiigi@gmail.com>
This commit is contained in:
Tonis Tiigi 2018-04-20 11:52:10 -07:00
parent 0bddd4ccfe
commit 46bd229b51
3 changed files with 240 additions and 41 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

32
builder/builder-next/builder.go
View file

@ -21,6 +21,7 @@ import (
"github.com/docker/docker/pkg/jsonmessage"
controlapi "github.com/moby/buildkit/api/services/control"
"github.com/moby/buildkit/cache"
"github.com/moby/buildkit/cache/cacheimport"
"github.com/moby/buildkit/cache/metadata"
"github.com/moby/buildkit/control"
"github.com/moby/buildkit/executor/runcexecutor"
@ -118,6 +119,10 @@ func (b *Builder) Build(ctx context.Context, opt backend.BuildConfig) (*builder.
frontendAttrs["context"] = opt.Options.RemoteContext
}
if len(opt.Options.CacheFrom) > 0 {
frontendAttrs["cache-from"] = opt.Options.CacheFrom[0]
}
logrus.Debugf("frontend: %+v", frontendAttrs)
for k, v := range opt.Options.BuildArgs {
@ -255,6 +260,8 @@ func newController(opt Opt, reporter chan containerimageexp.Result) (*control.Co
ContentStore: store,
DownloadManager: dist.DownloadManager,
MetadataStore: dist.V2MetadataService,
ImageStore: dist.ImageStore,
ReferenceStore: dist.ReferenceStore,
})
if err != nil {
return nil, err
@ -311,14 +318,16 @@ func newController(opt Opt, reporter chan containerimageexp.Result) (*control.Co
// }
wopt := mobyworker.WorkerOpt{
ID: "moby",
SessionManager: opt.SessionManager,
MetadataStore: md,
ContentStore: store,
CacheManager: cm,
Snapshotter: snapshotter,
Executor: exec,
ImageSource: src,
ID: "moby",
SessionManager: opt.SessionManager,
MetadataStore: md,
ContentStore: store,
CacheManager: cm,
Snapshotter: snapshotter,
Executor: exec,
ImageSource: src,
DownloadManager: dist.DownloadManager,
V2MetadataService: dist.V2MetadataService,
Exporters: map[string]exporter.Exporter{
"image": exp,
},
@ -331,13 +340,18 @@ func newController(opt Opt, reporter chan containerimageexp.Result) (*control.Co
}
wc.Add(w)
ci := cacheimport.NewCacheImporter(cacheimport.ImportOpt{
Worker: w,
SessionManager: opt.SessionManager,
})
return control.NewController(control.Opt{
SessionManager: opt.SessionManager,
WorkerController: wc,
Frontends: frontends,
CacheKeyStorage: cacheStorage,
// CacheExporter: ce,
// CacheImporter: ci,
CacheImporter: ci,
})
}

124
builder/builder-next/containerimage/pull.go
View file

@ -18,6 +18,7 @@ import (
"github.com/containerd/containerd/remotes"
"github.com/containerd/containerd/remotes/docker"
"github.com/containerd/containerd/remotes/docker/schema1"
distreference "github.com/docker/distribution/reference"
"github.com/docker/docker/distribution"
"github.com/docker/docker/distribution/metadata"
"github.com/docker/docker/distribution/xfer"
@ -40,6 +41,8 @@ import (
"golang.org/x/time/rate"
)
const preferLocal = true // FIXME: make this optional from the op
type SourceOpt struct {
SessionManager *session.Manager
ContentStore content.Store
@ -47,6 +50,7 @@ type SourceOpt struct {
ReferenceStore reference.Store
DownloadManager distribution.RootFSDownloadManager
MetadataStore metadata.V2MetadataService
ImageStore image.Store
}
type imageSource struct {
@ -91,7 +95,30 @@ func (is *imageSource) getCredentialsFromSession(ctx context.Context) func(strin
}
}
func (is *imageSource) resolveLocal(refStr string) ([]byte, error) {
ref, err := distreference.ParseNormalizedNamed(refStr)
if err != nil {
return nil, err
}
dgst, err := is.ReferenceStore.Get(ref)
if err != nil {
return nil, err
}
img, err := is.ImageStore.Get(image.ID(dgst))
if err != nil {
return nil, err
}
return img.RawJSON(), nil
}
func (is *imageSource) ResolveImageConfig(ctx context.Context, ref string) (digest.Digest, []byte, error) {
if preferLocal {
dt, err := is.resolveLocal(ref)
if err == nil {
return "", dt, nil
}
}
type t struct {
dgst digest.Digest
dt []byte
@ -132,41 +159,84 @@ type puller struct {
ref string
resolveErr error
resolver remotes.Resolver
imageID image.ID
cacheKey digest.Digest
}
func (p *puller) resolve(ctx context.Context) error {
p.resolveOnce.Do(func() {
resolveProgressDone := oneOffProgress(ctx, "resolve "+p.src.Reference.String())
dgst := p.src.Reference.Digest()
if dgst != "" {
info, err := p.is.ContentStore.Info(ctx, dgst)
// dgst := p.src.Reference.Digest()
// if dgst != "" {
// info, err := p.is.ContentStore.Info(ctx, dgst)
// if err == nil {
// p.ref = p.src.Reference.String()
// ra, err := p.is.ContentStore.ReaderAt(ctx, dgst)
// if err == nil {
// mt, err := imageutil.DetectManifestMediaType(ra)
// if err == nil {
// p.desc = ocispec.Descriptor{
// Size: info.Size,
// Digest: dgst,
// MediaType: mt,
// }
// resolveProgressDone(nil)
// return
// }
// }
// }
// }
// ref, desc, err := p.resolver.Resolve(ctx, p.src.Reference.String())
// if err != nil {
// p.resolveErr = err
// resolveProgressDone(err)
// return
// }
if preferLocal {
dt, err := p.is.resolveLocal(p.src.Reference.String())
if err == nil {
p.ref = p.src.Reference.String()
ra, err := p.is.ContentStore.ReaderAt(ctx, dgst)
if err == nil {
mt, err := imageutil.DetectManifestMediaType(ra)
if err == nil {
p.desc = ocispec.Descriptor{
Size: info.Size,
Digest: dgst,
MediaType: mt,
}
resolveProgressDone(nil)
return
}
}
dgst := digest.FromBytes(dt)
p.imageID = image.ID(dgst)
p.cacheKey = dgst
resolveProgressDone(nil)
return
}
}
ref, desc, err := p.resolver.Resolve(ctx, p.src.Reference.String())
ref, err := distreference.ParseNormalizedNamed(p.src.Reference.String())
if err != nil {
p.resolveErr = err
resolveProgressDone(err)
return
}
outRef, desc, err := p.resolver.Resolve(ctx, p.src.Reference.String())
if err != nil {
p.resolveErr = err
resolveProgressDone(err)
return
}
ref, err = distreference.WithDigest(ref, desc.Digest)
if err != nil {
p.resolveErr = err
resolveProgressDone(err)
return
}
_, dt, err := p.is.ResolveImageConfig(ctx, ref.String())
if err != nil {
p.resolveErr = err
resolveProgressDone(err)
return
}
p.desc = desc
p.ref = ref
p.cacheKey = digest.FromBytes(dt)
p.ref = outRef
resolveProgressDone(nil)
})
return p.resolveErr
@ -176,7 +246,7 @@ func (p *puller) CacheKey(ctx context.Context) (string, error) {
if err := p.resolve(ctx); err != nil {
return "", err
}
return p.desc.Digest.String(), nil
return p.cacheKey.String(), nil
}
func (p *puller) Snapshot(ctx context.Context) (cache.ImmutableRef, error) {
@ -184,6 +254,18 @@ func (p *puller) Snapshot(ctx context.Context) (cache.ImmutableRef, error) {
return nil, err
}
if p.imageID != "" {
img, err := p.is.ImageStore.Get(p.imageID)
if err != nil {
return nil, err
}
ref, err := p.is.CacheAccessor.Get(ctx, string(img.RootFS.ChainID()), cache.WithDescription(fmt.Sprintf("from local %s", p.ref)))
if err != nil {
return nil, err
}
return ref, nil
}
ongoing := newJobs(p.ref)
pctx, stopProgress := context.WithCancel(ctx)
@ -393,7 +475,7 @@ func (ld *layerDescriptor) Download(ctx netcontext.Context, progressOutput pkgpr
}
func (ld *layerDescriptor) Close() {
ld.is.ContentStore.Delete(context.TODO(), ld.desc.Digest)
// ld.is.ContentStore.Delete(context.TODO(), ld.desc.Digest))
}
func (ld *layerDescriptor) Registered(diffID layer.DiffID) {

125
builder/builder-next/worker/worker.go
View file

@ -2,11 +2,20 @@ package worker
import (
"context"
"fmt"
"io"
"io/ioutil"
"runtime"
"time"
"github.com/containerd/containerd/content"
"github.com/containerd/containerd/rootfs"
"github.com/docker/docker/distribution"
distmetadata "github.com/docker/docker/distribution/metadata"
"github.com/docker/docker/distribution/xfer"
"github.com/docker/docker/image"
"github.com/docker/docker/layer"
pkgprogress "github.com/docker/docker/pkg/progress"
"github.com/moby/buildkit/cache"
"github.com/moby/buildkit/cache/metadata"
"github.com/moby/buildkit/client"
@ -22,10 +31,12 @@ import (
"github.com/moby/buildkit/source/git"
"github.com/moby/buildkit/source/http"
"github.com/moby/buildkit/source/local"
"github.com/moby/buildkit/util/contentutil"
"github.com/moby/buildkit/util/progress"
digest "github.com/opencontainers/go-digest"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/pkg/errors"
netcontext "golang.org/x/net/context"
)
// TODO: this file should be removed. containerd defines ContainerdWorker, oci defines OCIWorker. There is no base worker.
@ -33,16 +44,18 @@ import (
// WorkerOpt is specific to a worker.
// See also CommonOpt.
type WorkerOpt struct {
ID string
Labels map[string]string
SessionManager *session.Manager
MetadataStore *metadata.Store
Executor executor.Executor
Snapshotter snapshot.Snapshotter
ContentStore content.Store
CacheManager cache.Manager
ImageSource source.Source
Exporters map[string]exporter.Exporter
ID string
Labels map[string]string
SessionManager *session.Manager
MetadataStore *metadata.Store
Executor executor.Executor
Snapshotter snapshot.Snapshotter
ContentStore content.Store
CacheManager cache.Manager
ImageSource source.Source
Exporters map[string]exporter.Exporter
DownloadManager distribution.RootFSDownloadManager
V2MetadataService distmetadata.V2MetadataService
// ImageStore images.Store // optional
}
@ -198,6 +211,43 @@ func (w *Worker) GetRemote(ctx context.Context, ref cache.ImmutableRef) (*solver
}
func (w *Worker) FromRemote(ctx context.Context, remote *solver.Remote) (cache.ImmutableRef, error) {
rootfs, err := getLayers(ctx, remote.Descriptors)
if err != nil {
return nil, err
}
layers := make([]xfer.DownloadDescriptor, 0, len(rootfs))
for _, l := range rootfs {
// ongoing.add(desc)
layers = append(layers, &layerDescriptor{
desc: l.Blob,
diffID: layer.DiffID(l.Diff.Digest),
provider: remote.Provider,
w: w,
pctx: ctx,
// ref: l.Blob.Digest.String(),
})
}
defer func() {
for _, l := range rootfs {
w.ContentStore.Delete(context.TODO(), l.Blob.Digest)
}
}()
r := image.NewRootFS()
rootFS, release, err := w.DownloadManager.Download(ctx, *r, runtime.GOOS, layers, &discardProgress{})
if err != nil {
return nil, err
}
defer release()
ref, err := w.CacheManager.Get(ctx, string(rootFS.ChainID()), cache.WithDescription(fmt.Sprintf("imported %s", remote.Descriptors[len(remote.Descriptors)-1].Digest)))
if err != nil {
return nil, err
}
// eg, gctx := errgroup.WithContext(ctx)
// for _, desc := range remote.Descriptors {
// func(desc ocispec.Descriptor) {
@ -223,7 +273,8 @@ func (w *Worker) FromRemote(ctx context.Context, remote *solver.Remote) (cache.I
// unpackProgressDone(nil)
//
// return w.CacheManager.Get(ctx, chainID, cache.WithDescription(fmt.Sprintf("imported %s", remote.Descriptors[len(remote.Descriptors)-1].Digest)))
return nil, errors.Errorf("fromremote not implemented")
// return nil, errors.Errorf("fromremote not implemented")
return ref, nil
}
// utility function. could be moved to the constructor logic?
@ -259,6 +310,58 @@ func (w *Worker) FromRemote(ctx context.Context, remote *solver.Remote) (cache.I
// return string(b), nil
// }
type discardProgress struct{}
func (_ *discardProgress) WriteProgress(_ pkgprogress.Progress) error {
return nil
}
// Fetch(ctx context.Context, desc ocispec.Descriptor) (io.ReadCloser, error)
type layerDescriptor struct {
provider content.Provider
desc ocispec.Descriptor
diffID layer.DiffID
// ref ctdreference.Spec
w *Worker
pctx context.Context
}
func (ld *layerDescriptor) Key() string {
return "v2:" + ld.desc.Digest.String()
}
func (ld *layerDescriptor) ID() string {
return ld.desc.Digest.String()
}
func (ld *layerDescriptor) DiffID() (layer.DiffID, error) {
return ld.diffID, nil
}
func (ld *layerDescriptor) Download(ctx netcontext.Context, progressOutput pkgprogress.Output) (io.ReadCloser, int64, error) {
done := oneOffProgress(ld.pctx, fmt.Sprintf("pulling %s", ld.desc.Digest))
if err := contentutil.Copy(ctx, ld.w.ContentStore, ld.provider, ld.desc); err != nil {
return nil, 0, done(err)
}
done(nil)
ra, err := ld.w.ContentStore.ReaderAt(ctx, ld.desc.Digest)
if err != nil {
return nil, 0, err
}
return ioutil.NopCloser(content.NewReader(ra)), ld.desc.Size, nil
}
func (ld *layerDescriptor) Close() {
// ld.is.ContentStore.Delete(context.TODO(), ld.desc.Digest)
}
func (ld *layerDescriptor) Registered(diffID layer.DiffID) {
// Cache mapping from this layer's DiffID to the blobsum
ld.w.V2MetadataService.Add(diffID, distmetadata.V2Metadata{Digest: ld.desc.Digest})
}
func getLayers(ctx context.Context, descs []ocispec.Descriptor) ([]rootfs.Layer, error) {
layers := make([]rootfs.Layer, len(descs))
for i, desc := range descs {