Merge pull request #44028 from thaJeztah/bump_buildkit

vendor: github.com/moby/buildkit v0.10.4

.github/workflows/ci.yml

@@ -128,7 +128,9 @@ jobs:
         name: BuildKit ref
         run: |
           ./hack/go-mod-prepare.sh
-          echo "BUILDKIT_REF=$(./hack/buildkit-ref)" >> $GITHUB_ENV
+# FIXME(thaJeztah) temporarily overriding version to use for tests; see https://github.com/moby/moby/pull/44028#issuecomment-1225964929
+#          echo "BUILDKIT_REF=$(./hack/buildkit-ref)" >> $GITHUB_ENV
+          echo "BUILDKIT_REF=8e2d9b9006caadb74c1745608889a37ba139acc1" >> $GITHUB_ENV
         working-directory: moby
       -
         name: Checkout BuildKit ${{ env.BUILDKIT_REF }}

vendor.mod

@@ -50,7 +50,7 @@ require (
 	github.com/klauspost/compress v1.15.1
 	github.com/miekg/dns v1.1.27
 	github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible
-	github.com/moby/buildkit v0.10.4-0.20220719175648-8e2d9b9006ca // v0.10 branch
+	github.com/moby/buildkit v0.10.4
 	github.com/moby/ipvs v1.0.2
 	github.com/moby/locker v1.0.1
 	github.com/moby/swarmkit/v2 v2.0.0-20220721174824-48dd89375d0a

vendor.sum

@@ -753,8 +753,8 @@ github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0Qu
 github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
 github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
 github.com/mitchellh/osext v0.0.0-20151018003038-5e2d6d41470f/go.mod h1:OkQIRizQZAeMln+1tSwduZz7+Af5oFlKirV/MSYes2A=
-github.com/moby/buildkit v0.10.4-0.20220719175648-8e2d9b9006ca h1:w/07TXrn/GqvjmFoWv8ISHa+Z3YLCDrErIOmi9JeFbE=
-github.com/moby/buildkit v0.10.4-0.20220719175648-8e2d9b9006ca/go.mod h1:hSExepqMIdfcKis7f7V1YzweBB7zt9DnlK+PUDCKuSI=
+github.com/moby/buildkit v0.10.4 h1:FvC+buO8isGpUFZ1abdSLdGHZVqg9sqI4BbFL8tlzP4=
+github.com/moby/buildkit v0.10.4/go.mod h1:Yajz9vt1Zw5q9Pp4pdb3TCSUXJBIroIQGQ3TTs/sLug=
 github.com/moby/ipvs v1.0.2 h1:NSbzuRTvfneftLU3VwPU5QuA6NZ0IUmqq9+VHcQxqHw=
 github.com/moby/ipvs v1.0.2/go.mod h1:2pngiyseZbIKXNv7hsKj3O9UEz30c53MT9005gt2hxQ=
 github.com/moby/locker v1.0.1 h1:fOXqR41zeveg4fFODix+1Ch4mj/gT0NE1XJbp/epuBg=
@@ -1678,8 +1678,8 @@ gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.0 h1:hjy8E9ON/egN1tAYqKb61G10WtihqetD4sz2H+8nIeA=
 gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw=
 gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk=
 gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8=

vendor/github.com/moby/buildkit/executor/oci/resolvconf.go

@@ -7,7 +7,6 @@ import (
 	"path/filepath"
 
 	"github.com/docker/docker/libnetwork/resolvconf"
-	"github.com/docker/docker/libnetwork/types"
 	"github.com/docker/docker/pkg/idtools"
 	"github.com/moby/buildkit/util/flightcontrol"
 	"github.com/pkg/errors"
@@ -74,7 +73,7 @@ func GetResolvConf(ctx context.Context, stateDir string, idmap *idtools.Identity
 
 		if dns != nil {
 			var (
-				dnsNameservers   = resolvconf.GetNameservers(dt, types.IP)
+				dnsNameservers   = resolvconf.GetNameservers(dt, resolvconf.IP)
 				dnsSearchDomains = resolvconf.GetSearchDomains(dt)
 				dnsOptions       = resolvconf.GetOptions(dt)
 			)

vendor/github.com/moby/buildkit/frontend/dockerfile/builder/build.go

@@ -463,7 +463,7 @@ func Build(ctx context.Context, c client.Client) (*client.Result, error) {
 						}
 						c.Warn(ctx, defVtx, msg, warnOpts(sourceMap, location, detail, url))
 					},
-					ContextByName: contextByNameFunc(c, tp),
+					ContextByName: contextByNameFunc(c),
 				})
 
 				if err != nil {
@@ -812,8 +812,8 @@ func warnOpts(sm *llb.SourceMap, r *parser.Range, detail [][]byte, url string) c
 	return opts
 }
 
-func contextByNameFunc(c client.Client, p *ocispecs.Platform) func(context.Context, string, string) (*llb.State, *dockerfile2llb.Image, *binfotypes.BuildInfo, error) {
-	return func(ctx context.Context, name, resolveMode string) (*llb.State, *dockerfile2llb.Image, *binfotypes.BuildInfo, error) {
+func contextByNameFunc(c client.Client) func(context.Context, string, string, *ocispecs.Platform) (*llb.State, *dockerfile2llb.Image, *binfotypes.BuildInfo, error) {
+	return func(ctx context.Context, name, resolveMode string, p *ocispecs.Platform) (*llb.State, *dockerfile2llb.Image, *binfotypes.BuildInfo, error) {
 		named, err := reference.ParseNormalizedNamed(name)
 		if err != nil {
 			return nil, nil, nil, errors.Wrapf(err, "invalid context name %s", name)
@@ -879,6 +879,7 @@ func contextByName(ctx context.Context, c client.Client, name string, platform *
 		if err := json.Unmarshal(data, &img); err != nil {
 			return nil, nil, nil, err
 		}
+		img.Created = nil
 
 		st := llb.Image(ref, imgOpt...)
 		st, err = st.WithImageConfig(data)

vendor/github.com/moby/buildkit/frontend/dockerfile/dockerfile2llb/convert.go

@@ -70,16 +70,19 @@ type ConvertOpt struct {
 	SourceMap         *llb.SourceMap
 	Hostname          string
 	Warn              func(short, url string, detail [][]byte, location *parser.Range)
-	ContextByName     func(ctx context.Context, name, resolveMode string) (*llb.State, *Image, *binfotypes.BuildInfo, error)
+	ContextByName     func(ctx context.Context, name, resolveMode string, p *ocispecs.Platform) (*llb.State, *Image, *binfotypes.BuildInfo, error)
 }
 
 func Dockerfile2LLB(ctx context.Context, dt []byte, opt ConvertOpt) (*llb.State, *Image, *binfotypes.BuildInfo, error) {
 	buildInfo := &binfotypes.BuildInfo{}
 	contextByName := opt.ContextByName
-	opt.ContextByName = func(ctx context.Context, name, resolveMode string) (*llb.State, *Image, *binfotypes.BuildInfo, error) {
+	opt.ContextByName = func(ctx context.Context, name, resolveMode string, p *ocispecs.Platform) (*llb.State, *Image, *binfotypes.BuildInfo, error) {
 		if !strings.EqualFold(name, "scratch") && !strings.EqualFold(name, "context") {
 			if contextByName != nil {
-				st, img, bi, err := contextByName(ctx, name, resolveMode)
+				if p == nil {
+					p = opt.TargetPlatform
+				}
+				st, img, bi, err := contextByName(ctx, name, resolveMode, p)
 				if err != nil {
 					return nil, nil, nil, err
 				}
@@ -165,8 +168,21 @@ func Dockerfile2LLB(ctx context.Context, dt []byte, opt ConvertOpt) (*llb.State,
 			prefixPlatform: opt.PrefixPlatform,
 		}
 
+		if v := st.Platform; v != "" {
+			v, err := shlex.ProcessWordWithMap(v, metaArgsToMap(optMetaArgs))
+			if err != nil {
+				return nil, nil, nil, parser.WithLocation(errors.Wrapf(err, "failed to process arguments for platform %s", v), st.Location)
+			}
+
+			p, err := platforms.Parse(v)
+			if err != nil {
+				return nil, nil, nil, parser.WithLocation(errors.Wrapf(err, "failed to parse platform %s", v), st.Location)
+			}
+			ds.platform = &p
+		}
+
 		if st.Name != "" {
-			s, img, bi, err := opt.ContextByName(ctx, st.Name, opt.ImageResolveMode.String())
+			s, img, bi, err := opt.ContextByName(ctx, st.Name, opt.ImageResolveMode.String(), ds.platform)
 			if err != nil {
 				return nil, nil, nil, err
 			}
@@ -195,18 +211,6 @@ func Dockerfile2LLB(ctx context.Context, dt []byte, opt ConvertOpt) (*llb.State,
 			ds.stageName = fmt.Sprintf("stage-%d", i)
 		}
 
-		if v := st.Platform; v != "" {
-			v, err := shlex.ProcessWordWithMap(v, metaArgsToMap(optMetaArgs))
-			if err != nil {
-				return nil, nil, nil, parser.WithLocation(errors.Wrapf(err, "failed to process arguments for platform %s", v), st.Location)
-			}
-
-			p, err := platforms.Parse(v)
-			if err != nil {
-				return nil, nil, nil, parser.WithLocation(errors.Wrapf(err, "failed to parse platform %s", v), st.Location)
-			}
-			ds.platform = &p
-		}
 		allDispatchStates.addState(ds)
 
 		total := 0
@@ -313,7 +317,7 @@ func Dockerfile2LLB(ctx context.Context, dt []byte, opt ConvertOpt) (*llb.State,
 					d.stage.BaseName = reference.TagNameOnly(ref).String()
 
 					var isScratch bool
-					st, img, bi, err := opt.ContextByName(ctx, d.stage.BaseName, opt.ImageResolveMode.String())
+					st, img, bi, err := opt.ContextByName(ctx, d.stage.BaseName, opt.ImageResolveMode.String(), platform)
 					if err != nil {
 						return err
 					}

vendor/github.com/moby/buildkit/frontend/gateway/container.go

@@ -361,6 +361,8 @@ func (gwCtr *gatewayContainer) Start(ctx context.Context, req client.StartReques
 }
 
 func (gwCtr *gatewayContainer) Release(ctx context.Context) error {
+	gwCtr.mu.Lock()
+	defer gwCtr.mu.Unlock()
 	gwCtr.cancel()
 	err1 := gwCtr.errGroup.Wait()
 
@@ -371,6 +373,7 @@ func (gwCtr *gatewayContainer) Release(ctx context.Context) error {
 			err2 = err
 		}
 	}
+	gwCtr.cleanup = nil
 
 	if err1 != nil {
 		return stack.Enable(err1)

vendor/github.com/moby/buildkit/frontend/gateway/forwarder/forward.go

@@ -52,6 +52,7 @@ type bridgeClient struct {
 	workers       worker.Infos
 	workerRefByID map[string]*worker.WorkerRef
 	buildOpts     client.BuildOpts
+	ctrs          []client.Container
 }
 
 func (c *bridgeClient) Solve(ctx context.Context, req client.SolveRequest) (*client.Result, error) {
@@ -212,6 +213,10 @@ func (c *bridgeClient) toFrontendResult(r *client.Result) (*frontend.Result, err
 }
 
 func (c *bridgeClient) discard(err error) {
+	for _, ctr := range c.ctrs {
+		ctr.Release(context.TODO())
+	}
+
 	for id, workerRef := range c.workerRefByID {
 		workerRef.ImmutableRef.Release(context.TODO())
 		delete(c.workerRefByID, id)
@@ -300,6 +305,7 @@ func (c *bridgeClient) NewContainer(ctx context.Context, req client.NewContainer
 	if err != nil {
 		return nil, err
 	}
+	c.ctrs = append(c.ctrs, ctr)
 	return ctr, nil
 }
 

vendor/github.com/moby/buildkit/frontend/gateway/gateway.go

@@ -278,7 +278,7 @@ func (gf *gatewayFrontend) Solve(ctx context.Context, llbBridge frontend.Fronten
 	err = w.Executor().Run(ctx, "", mountWithSession(rootFS, session.NewGroup(sid)), mnts, executor.ProcessInfo{Meta: meta, Stdin: lbf.Stdin, Stdout: lbf.Stdout, Stderr: os.Stderr}, nil)
 
 	if err != nil {
-		if errdefs.IsCanceled(err) && lbf.isErrServerClosed {
+		if errdefs.IsCanceled(ctx, err) && lbf.isErrServerClosed {
 			err = errors.Errorf("frontend grpc server closed unexpectedly")
 		}
 		// An existing error (set via Return rpc) takes
@@ -345,6 +345,13 @@ func (b *bindMount) IdentityMapping() *idtools.IdentityMapping {
 func (lbf *llbBridgeForwarder) Discard() {
 	lbf.mu.Lock()
 	defer lbf.mu.Unlock()
+
+	for ctr := range lbf.ctrs {
+		lbf.ReleaseContainer(context.TODO(), &pb.ReleaseContainerRequest{
+			ContainerID: ctr,
+		})
+	}
+
 	for id, workerRef := range lbf.workerRefByID {
 		workerRef.ImmutableRef.Release(context.TODO())
 		delete(lbf.workerRefByID, id)

vendor/github.com/moby/buildkit/session/grpc.go

@@ -2,6 +2,7 @@ package session
 
 import (
 	"context"
+	"math"
 	"net"
 	"sync/atomic"
 	"time"
@@ -10,6 +11,7 @@ import (
 	"github.com/moby/buildkit/util/bklog"
 	"github.com/moby/buildkit/util/grpcerrors"
 	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
 	"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc"
 	"go.opentelemetry.io/otel/trace"
 	"golang.org/x/net/http2"
@@ -79,21 +81,55 @@ func monitorHealth(ctx context.Context, cc *grpc.ClientConn, cancelConn func())
 	defer cancelConn()
 	defer cc.Close()
 
-	ticker := time.NewTicker(1 * time.Second)
+	ticker := time.NewTicker(5 * time.Second)
 	defer ticker.Stop()
 	healthClient := grpc_health_v1.NewHealthClient(cc)
 
+	failedBefore := false
+	consecutiveSuccessful := 0
+	defaultHealthcheckDuration := 30 * time.Second
+	lastHealthcheckDuration := time.Duration(0)
+
 	for {
 		select {
 		case <-ctx.Done():
 			return
 		case <-ticker.C:
-			ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
+			// This healthcheck can erroneously fail in some instances, such as receiving lots of data in a low-bandwidth scenario or too many concurrent builds.
+			// So, this healthcheck is purposely long, and can tolerate some failures on purpose.
+
+			healthcheckStart := time.Now()
+
+			timeout := time.Duration(math.Max(float64(defaultHealthcheckDuration), float64(lastHealthcheckDuration)*1.5))
+			ctx, cancel := context.WithTimeout(ctx, timeout)
 			_, err := healthClient.Check(ctx, &grpc_health_v1.HealthCheckRequest{})
 			cancel()
+
+			lastHealthcheckDuration = time.Since(healthcheckStart)
+			logFields := logrus.Fields{
+				"timeout":        timeout,
+				"actualDuration": lastHealthcheckDuration,
+			}
+
 			if err != nil {
-				return
+				if failedBefore {
+					bklog.G(ctx).Error("healthcheck failed fatally")
+					return
+				}
+
+				failedBefore = true
+				consecutiveSuccessful = 0
+				bklog.G(ctx).WithFields(logFields).Warn("healthcheck failed")
+			} else {
+				consecutiveSuccessful++
+
+				if consecutiveSuccessful >= 5 && failedBefore {
+					failedBefore = false
+					bklog.G(ctx).WithFields(logFields).Debug("reset healthcheck failure")
+				}
 			}
+
+			bklog.G(ctx).WithFields(logFields).Debug("healthcheck completed")
 		}
 	}
 }

vendor/github.com/moby/buildkit/solver/errdefs/context.go

@@ -3,11 +3,25 @@ package errdefs
 import (
 	"context"
 	"errors"
+	"strings"
 
 	"github.com/moby/buildkit/util/grpcerrors"
 	"google.golang.org/grpc/codes"
 )
 
-func IsCanceled(err error) bool {
-	return errors.Is(err, context.Canceled) || grpcerrors.Code(err) == codes.Canceled
+func IsCanceled(ctx context.Context, err error) bool {
+	if errors.Is(err, context.Canceled) || grpcerrors.Code(err) == codes.Canceled {
+		return true
+	}
+	// grpc does not set cancel correctly when stream gets cancelled and then Recv is called
+	if err != nil && ctx.Err() == context.Canceled {
+		// when this error comes from containerd it is not typed at all, just concatenated string
+		if strings.Contains(err.Error(), "EOF") {
+			return true
+		}
+		if strings.Contains(err.Error(), context.Canceled.Error()) {
+			return true
+		}
+	}
+	return false
 }

vendor/github.com/moby/buildkit/solver/jobs.go

@@ -3,7 +3,6 @@ package solver
 import (
 	"context"
 	"fmt"
-	"strings"
 	"sync"
 	"time"
 
@@ -705,7 +704,7 @@ func (s *sharedOp) CalcSlowCache(ctx context.Context, index Index, p PreprocessF
 		if err != nil {
 			select {
 			case <-ctx.Done():
-				if strings.Contains(err.Error(), context.Canceled.Error()) {
+				if errdefs.IsCanceled(ctx, err) {
 					complete = false
 					releaseError(err)
 					err = errors.Wrap(ctx.Err(), err.Error())
@@ -771,7 +770,7 @@ func (s *sharedOp) CacheMap(ctx context.Context, index int) (resp *cacheMapResp,
 		if err != nil {
 			select {
 			case <-ctx.Done():
-				if strings.Contains(err.Error(), context.Canceled.Error()) {
+				if errdefs.IsCanceled(ctx, err) {
 					complete = false
 					releaseError(err)
 					err = errors.Wrap(ctx.Err(), err.Error())
@@ -810,8 +809,11 @@ func (s *sharedOp) Exec(ctx context.Context, inputs []Result) (outputs []Result,
 	}
 	flightControlKey := "exec"
 	res, err := s.g.Do(ctx, flightControlKey, func(ctx context.Context) (ret interface{}, retErr error) {
-		if s.execRes != nil || s.execErr != nil {
-			return s.execRes, s.execErr
+		if s.execErr != nil {
+			return nil, s.execErr
+		}
+		if s.execRes != nil {
+			return s.execRes, nil
 		}
 		release, err := op.Acquire(ctx)
 		if err != nil {
@@ -838,7 +840,7 @@ func (s *sharedOp) Exec(ctx context.Context, inputs []Result) (outputs []Result,
 		if err != nil {
 			select {
 			case <-ctx.Done():
-				if strings.Contains(err.Error(), context.Canceled.Error()) {
+				if errdefs.IsCanceled(ctx, err) {
 					complete = false
 					releaseError(err)
 					err = errors.Wrap(ctx.Err(), err.Error())
@@ -859,9 +861,12 @@ func (s *sharedOp) Exec(ctx context.Context, inputs []Result) (outputs []Result,
 			}
 			s.execErr = err
 		}
-		return s.execRes, err
+		if s.execRes == nil || err != nil {
+			return nil, err
+		}
+		return s.execRes, nil
 	})
-	if err != nil {
+	if res == nil || err != nil {
 		return nil, nil, err
 	}
 	r := res.(*execRes)

vendor/github.com/moby/buildkit/solver/llbsolver/bridge.go

@@ -3,7 +3,6 @@ package llbsolver
 import (
 	"context"
 	"fmt"
-	"strings"
 	"sync"
 	"time"
 
@@ -290,7 +289,7 @@ func (rp *resultProxy) Result(ctx context.Context) (res solver.CachedResult, err
 		if err != nil {
 			select {
 			case <-ctx.Done():
-				if strings.Contains(err.Error(), context.Canceled.Error()) {
+				if errdefs.IsCanceled(ctx, err) {
 					return v, err
 				}
 			default:

vendor/modules.txt

@@ -474,7 +474,7 @@ github.com/mistifyio/go-zfs
 # github.com/mitchellh/hashstructure/v2 v2.0.2
 ## explicit; go 1.14
 github.com/mitchellh/hashstructure/v2
-# github.com/moby/buildkit v0.10.4-0.20220719175648-8e2d9b9006ca
+# github.com/moby/buildkit v0.10.4
 ## explicit; go 1.17
 github.com/moby/buildkit/api/services/control
 github.com/moby/buildkit/api/types