Merge pull request #44920 from tonistiigi/worker-getremotes

builder: define GetRemotes for the worker

.github/workflows/buildkit.yml

@@ -70,9 +70,7 @@ jobs:
         name: BuildKit ref
         run: |
           echo "BUILDKIT_REPO=moby/buildkit" >> $GITHUB_ENV
-          # FIXME(thaJeztah) remove when updating BuildKit to v0.11.3
-          # echo "BUILDKIT_REF=$(./hack/buildkit-ref)" >> $GITHUB_ENV
-          echo "BUILDKIT_REF=09223a2c58d06e0333b48107be5490397d11e65a" >> $GITHUB_ENV
+          echo "BUILDKIT_REF=$(./hack/buildkit-ref)" >> $GITHUB_ENV
         working-directory: moby
       -
         name: Checkout BuildKit ${{ env.BUILDKIT_REF }}

builder/builder-next/worker/worker.go

@@ -19,6 +19,7 @@ import (
 	"github.com/docker/docker/layer"
 	pkgprogress "github.com/docker/docker/pkg/progress"
 	"github.com/moby/buildkit/cache"
+	cacheconfig "github.com/moby/buildkit/cache/config"
 	"github.com/moby/buildkit/client"
 	"github.com/moby/buildkit/client/llb"
 	"github.com/moby/buildkit/executor"
@@ -37,7 +38,6 @@ import (
 	"github.com/moby/buildkit/source/http"
 	"github.com/moby/buildkit/source/local"
 	"github.com/moby/buildkit/util/archutil"
-	"github.com/moby/buildkit/util/compression"
 	"github.com/moby/buildkit/util/contentutil"
 	"github.com/moby/buildkit/util/progress"
 	"github.com/moby/buildkit/version"
@@ -86,6 +86,10 @@ type Worker struct {
 	SourceManager *source.Manager
 }
 
+var _ interface {
+	GetRemotes(context.Context, cache.ImmutableRef, bool, cacheconfig.RefConfig, bool, session.Group) ([]*solver.Remote, error)
+} = &Worker{}
+
 // NewWorker instantiates a local worker
 func NewWorker(opt Opt) (*Worker, error) {
 	sm, err := source.NewManager()
@@ -259,8 +263,11 @@ func (w *Worker) Exporter(name string, sm *session.Manager) (exporter.Exporter,
 	}
 }
 
-// GetRemote returns a remote snapshot reference for a local one
-func (w *Worker) GetRemote(ctx context.Context, ref cache.ImmutableRef, createIfNeeded bool, _ compression.Type, s session.Group) (*solver.Remote, error) {
+// GetRemotes returns the remote snapshot references given a local reference
+func (w *Worker) GetRemotes(ctx context.Context, ref cache.ImmutableRef, createIfNeeded bool, _ cacheconfig.RefConfig, all bool, s session.Group) ([]*solver.Remote, error) {
+	if ref == nil {
+		return nil, nil
+	}
 	var diffIDs []layer.DiffID
 	var err error
 	if !createIfNeeded {
@@ -290,10 +297,10 @@ func (w *Worker) GetRemote(ctx context.Context, ref cache.ImmutableRef, createIf
 		}
 	}
 
-	return &solver.Remote{
+	return []*solver.Remote{{
 		Descriptors: descriptors,
 		Provider:    &emptyProvider{},
-	}, nil
+	}}, nil
 }
 
 // PruneCacheMounts removes the current cache snapshots for specified IDs

vendor.mod

@@ -56,7 +56,7 @@ require (
 	github.com/klauspost/compress v1.15.12
 	github.com/miekg/dns v1.1.43
 	github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible
-	github.com/moby/buildkit v0.11.2 // FIXME(thaJeztah): remove override from .github/workflows/buildkit.yml when updating to v0.11.3
+	github.com/moby/buildkit v0.11.3-0.20230206194711-49992f513921
 	github.com/moby/ipvs v1.1.0
 	github.com/moby/locker v1.0.1
 	github.com/moby/patternmatcher v0.5.0

vendor.sum

@@ -754,8 +754,8 @@ github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0Qu
 github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
 github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
 github.com/mitchellh/osext v0.0.0-20151018003038-5e2d6d41470f/go.mod h1:OkQIRizQZAeMln+1tSwduZz7+Af5oFlKirV/MSYes2A=
-github.com/moby/buildkit v0.11.2 h1:hNNsYuRssvFnp/qJ8FifStEUzROl5riPAEwk7cRzMjg=
-github.com/moby/buildkit v0.11.2/go.mod h1:b5hR8j3BZaOj5+gf6yielP9YLT9mU92zy3zZtdoUTrw=
+github.com/moby/buildkit v0.11.3-0.20230206194711-49992f513921 h1:B6x+BCRpTTUhEsHtvBpi2IsRnrLCJ7xoJ3oZrO8oQEU=
+github.com/moby/buildkit v0.11.3-0.20230206194711-49992f513921/go.mod h1:P8MqGq7YrIDldCdZLhK8M/vPcrFYZ6GX1crX0j4hOmQ=
 github.com/moby/ipvs v1.1.0 h1:ONN4pGaZQgAx+1Scz5RvWV4Q7Gb+mvfRh3NsPS+1XQQ=
 github.com/moby/ipvs v1.1.0/go.mod h1:4VJMWuf098bsUMmZEiD4Tjk/O7mOn3l1PTD3s4OoYAs=
 github.com/moby/locker v1.0.1 h1:fOXqR41zeveg4fFODix+1Ch4mj/gT0NE1XJbp/epuBg=

vendor/github.com/moby/buildkit/cache/remotecache/inline/inline.go

@@ -56,16 +56,20 @@ func (ce *exporter) ExportForLayers(ctx context.Context, layers []digest.Digest)
 		return nil, err
 	}
 
+	layerBlobDigests := make([]digest.Digest, len(layers))
+
 	descs2 := map[digest.Digest]v1.DescriptorProviderPair{}
-	for _, k := range layers {
+	for i, k := range layers {
 		if v, ok := descs[k]; ok {
 			descs2[k] = v
+			layerBlobDigests[i] = k
 			continue
 		}
 		// fallback for uncompressed digests
 		for _, v := range descs {
 			if uc := v.Descriptor.Annotations["containerd.io/uncompressed"]; uc == string(k) {
 				descs2[v.Descriptor.Digest] = v
+				layerBlobDigests[i] = v.Descriptor.Digest
 			}
 		}
 	}
@@ -87,7 +91,7 @@ func (ce *exporter) ExportForLayers(ctx context.Context, layers []digest.Digest)
 
 	// reorder layers based on the order in the image
 	blobIndexes := make(map[digest.Digest]int, len(layers))
-	for i, blob := range layers {
+	for i, blob := range layerBlobDigests {
 		blobIndexes[blob] = i
 	}
 

vendor/modules.txt

@@ -540,7 +540,7 @@ github.com/mistifyio/go-zfs
 # github.com/mitchellh/hashstructure/v2 v2.0.2
 ## explicit; go 1.14
 github.com/mitchellh/hashstructure/v2
-# github.com/moby/buildkit v0.11.2
+# github.com/moby/buildkit v0.11.3-0.20230206194711-49992f513921
 ## explicit; go 1.18
 github.com/moby/buildkit/api/services/control
 github.com/moby/buildkit/api/types