vendor: github.com/moby/buildkit v0.12.5

full diff: https://github.com/moby/buildkit/compare/v0.12.4...v0.12.5

Signed-off-by: Paweł Gronowski <pawel.gronowski@docker.com>
This commit is contained in:
Paweł Gronowski 2024-01-31 17:18:03 +01:00
parent 71fa3ab079
commit 13ce91825f
No known key found for this signature in database
GPG key ID: B85EFCFE26DEF92A
32 changed files with 448 additions and 103 deletions

View file

@ -142,8 +142,8 @@ func newSnapshotterController(ctx context.Context, rt http.RoundTripper, opt Opt
return nil, err
}
frontends := map[string]frontend.Frontend{
"dockerfile.v0": forwarder.NewGatewayForwarder(wc, dockerfile.Build),
"gateway.v0": gateway.NewGatewayFrontend(wc),
"dockerfile.v0": forwarder.NewGatewayForwarder(wc.Infos(), dockerfile.Build),
"gateway.v0": gateway.NewGatewayFrontend(wc.Infos()),
}
return control.NewController(control.Opt{
@ -364,8 +364,8 @@ func newGraphDriverController(ctx context.Context, rt http.RoundTripper, opt Opt
wc.Add(w)
frontends := map[string]frontend.Frontend{
"dockerfile.v0": forwarder.NewGatewayForwarder(wc, dockerfile.Build),
"gateway.v0": gateway.NewGatewayFrontend(wc),
"dockerfile.v0": forwarder.NewGatewayForwarder(wc.Infos(), dockerfile.Build),
"gateway.v0": gateway.NewGatewayFrontend(wc.Infos()),
}
return control.NewController(control.Opt{

View file

@ -60,7 +60,7 @@ require (
github.com/miekg/dns v1.1.43
github.com/mistifyio/go-zfs/v3 v3.0.1
github.com/mitchellh/copystructure v1.2.0
github.com/moby/buildkit v0.12.5-0.20231208203051-3b6880d2a00f // v0.12 branch (v0.12.5-dev)
github.com/moby/buildkit v0.12.5
github.com/moby/ipvs v1.1.0
github.com/moby/locker v1.0.1
github.com/moby/patternmatcher v0.6.0

View file

@ -900,8 +900,8 @@ github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zx
github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw=
github.com/mndrix/tap-go v0.0.0-20171203230836-629fa407e90b/go.mod h1:pzzDgJWZ34fGzaAZGFW22KVZDfyrYW+QABMrWnJBnSs=
github.com/moby/buildkit v0.8.1/go.mod h1:/kyU1hKy/aYCuP39GZA9MaKioovHku57N6cqlKZIaiQ=
github.com/moby/buildkit v0.12.5-0.20231208203051-3b6880d2a00f h1:nYPkpfWrlQznHPLNrXxXIQMaTlmnsSBiiRTgnQ5hrZ0=
github.com/moby/buildkit v0.12.5-0.20231208203051-3b6880d2a00f/go.mod h1:XG74uz06nPWQpnxYwgCryrVidvor0+ElUxGosbZPQG4=
github.com/moby/buildkit v0.12.5 h1:RNHH1l3HDhYyZafr5EgstEu8aGNCwyfvMtrQDtjH9T0=
github.com/moby/buildkit v0.12.5/go.mod h1:YGwjA2loqyiYfZeEo8FtI7z4x5XponAaIWsWcSjWwso=
github.com/moby/ipvs v1.1.0 h1:ONN4pGaZQgAx+1Scz5RvWV4Q7Gb+mvfRh3NsPS+1XQQ=
github.com/moby/ipvs v1.1.0/go.mod h1:4VJMWuf098bsUMmZEiD4Tjk/O7mOn3l1PTD3s4OoYAs=
github.com/moby/locker v1.0.1 h1:fOXqR41zeveg4fFODix+1Ch4mj/gT0NE1XJbp/epuBg=

View file

@ -405,6 +405,9 @@ func (c *Controller) Solve(ctx context.Context, req *controlapi.SolveRequest) (*
var cacheImports []frontend.CacheOptionsEntry
for _, im := range req.Cache.Imports {
if im == nil {
continue
}
cacheImports = append(cacheImports, frontend.CacheOptionsEntry{
Type: im.Type,
Attrs: im.Attrs,

View file

@ -6,8 +6,9 @@ import (
"net"
"syscall"
"github.com/containerd/containerd/mount"
"github.com/docker/docker/pkg/idtools"
resourcestypes "github.com/moby/buildkit/executor/resources/types"
"github.com/moby/buildkit/snapshot"
"github.com/moby/buildkit/solver/pb"
)
@ -28,8 +29,13 @@ type Meta struct {
RemoveMountStubsRecursive bool
}
type MountableRef interface {
Mount() ([]mount.Mount, func() error, error)
IdentityMapping() *idtools.IdentityMapping
}
type Mountable interface {
Mount(ctx context.Context, readonly bool) (snapshot.Mountable, error)
Mount(ctx context.Context, readonly bool) (MountableRef, error)
}
type Mount struct {

View file

@ -12,7 +12,6 @@ import (
"github.com/containerd/containerd/namespaces"
"github.com/containerd/containerd/oci"
"github.com/containerd/containerd/pkg/userns"
"github.com/containerd/continuity/fs"
"github.com/docker/docker/pkg/idtools"
"github.com/mitchellh/hashstructure/v2"
"github.com/moby/buildkit/executor"
@ -215,6 +214,7 @@ func GenerateSpec(ctx context.Context, meta executor.Meta, mounts []executor.Mou
type mountRef struct {
mount mount.Mount
unmount func() error
subRefs map[string]mountRef
}
type submounts struct {
@ -230,12 +230,19 @@ func (s *submounts) subMount(m mount.Mount, subPath string) (mount.Mount, error)
}
h, err := hashstructure.Hash(m, hashstructure.FormatV2, nil)
if err != nil {
return mount.Mount{}, nil
return mount.Mount{}, err
}
if mr, ok := s.m[h]; ok {
sm, err := sub(mr.mount, subPath)
if sm, ok := mr.subRefs[subPath]; ok {
return sm.mount, nil
}
sm, unmount, err := sub(mr.mount, subPath)
if err != nil {
return mount.Mount{}, nil
return mount.Mount{}, err
}
mr.subRefs[subPath] = mountRef{
mount: sm,
unmount: unmount,
}
return sm, nil
}
@ -261,12 +268,17 @@ func (s *submounts) subMount(m mount.Mount, subPath string) (mount.Mount, error)
Options: opts,
},
unmount: lm.Unmount,
subRefs: map[string]mountRef{},
}
sm, err := sub(s.m[h].mount, subPath)
sm, unmount, err := sub(s.m[h].mount, subPath)
if err != nil {
return mount.Mount{}, err
}
s.m[h].subRefs[subPath] = mountRef{
mount: sm,
unmount: unmount,
}
return sm, nil
}
@ -276,6 +288,9 @@ func (s *submounts) cleanup() {
for _, m := range s.m {
func(m mountRef) {
go func() {
for _, sm := range m.subRefs {
sm.unmount()
}
m.unmount()
wg.Done()
}()
@ -284,15 +299,6 @@ func (s *submounts) cleanup() {
wg.Wait()
}
func sub(m mount.Mount, subPath string) (mount.Mount, error) {
src, err := fs.RootPath(m.Source, subPath)
if err != nil {
return mount.Mount{}, err
}
m.Source = src
return m, nil
}
func specMapping(s []idtools.IDMap) []specs.LinuxIDMapping {
var ids []specs.LinuxIDMapping
for _, item := range s {

View file

@ -0,0 +1,15 @@
package oci
import (
"github.com/containerd/containerd/mount"
"github.com/containerd/continuity/fs"
)
func sub(m mount.Mount, subPath string) (mount.Mount, func() error, error) {
src, err := fs.RootPath(m.Source, subPath)
if err != nil {
return mount.Mount{}, nil, err
}
m.Source = src
return m, func() error { return nil }, nil
}

View file

@ -0,0 +1,57 @@
//go:build linux
// +build linux
package oci
import (
"os"
"strconv"
"github.com/containerd/containerd/mount"
"github.com/containerd/continuity/fs"
"github.com/moby/buildkit/snapshot"
"github.com/pkg/errors"
"golang.org/x/sys/unix"
)
func sub(m mount.Mount, subPath string) (mount.Mount, func() error, error) {
var retries = 10
root := m.Source
for {
src, err := fs.RootPath(root, subPath)
if err != nil {
return mount.Mount{}, nil, err
}
// similar to runc.WithProcfd
fh, err := os.OpenFile(src, unix.O_PATH|unix.O_CLOEXEC, 0)
if err != nil {
return mount.Mount{}, nil, err
}
fdPath := "/proc/self/fd/" + strconv.Itoa(int(fh.Fd()))
if resolved, err := os.Readlink(fdPath); err != nil {
fh.Close()
return mount.Mount{}, nil, err
} else if resolved != src {
retries--
if retries <= 0 {
fh.Close()
return mount.Mount{}, nil, errors.Errorf("unable to safely resolve subpath %s", subPath)
}
fh.Close()
continue
}
m.Source = fdPath
lm := snapshot.LocalMounterWithMounts([]mount.Mount{m}, snapshot.ForceRemount())
mp, err := lm.Mount()
if err != nil {
fh.Close()
return mount.Mount{}, nil, err
}
m.Source = mp
fh.Close() // release the fd, we don't need it anymore
return m, lm.Unmount, nil
}
}

View file

@ -7,7 +7,9 @@ import (
"fmt"
"path/filepath"
"github.com/containerd/containerd/mount"
"github.com/containerd/containerd/oci"
"github.com/containerd/continuity/fs"
"github.com/docker/docker/pkg/idtools"
"github.com/moby/buildkit/solver/pb"
specs "github.com/opencontainers/runtime-spec/specs-go"
@ -67,3 +69,12 @@ func getTracingSocket() string {
func cgroupV2NamespaceSupported() bool {
return false
}
func sub(m mount.Mount, subPath string) (mount.Mount, func() error, error) {
src, err := fs.RootPath(m.Source, subPath)
if err != nil {
return mount.Mount{}, nil, err
}
m.Source = src
return m, func() error { return nil }, nil
}

View file

@ -5,6 +5,7 @@ import (
"errors"
"os"
"path/filepath"
"strings"
"syscall"
"github.com/containerd/continuity/fs"
@ -43,7 +44,7 @@ func MountStubsCleaner(ctx context.Context, dir string, mounts []Mount, recursiv
}
realPathNext := filepath.Dir(realPath)
if realPath == realPathNext {
if realPath == realPathNext || realPathNext == dir {
break
}
realPath = realPathNext
@ -52,6 +53,11 @@ func MountStubsCleaner(ctx context.Context, dir string, mounts []Mount, recursiv
return func() {
for _, p := range paths {
p, err := fs.RootPath(dir, strings.TrimPrefix(p, dir))
if err != nil {
continue
}
st, err := os.Lstat(p)
if err != nil {
continue
@ -70,8 +76,12 @@ func MountStubsCleaner(ctx context.Context, dir string, mounts []Mount, recursiv
// Back up the timestamps of the dir for reproducible builds
// https://github.com/moby/buildkit/issues/3148
dir := filepath.Dir(p)
dirSt, err := os.Stat(dir)
parent := filepath.Dir(p)
if realPath, err := fs.RootPath(dir, strings.TrimPrefix(parent, dir)); err != nil || realPath != parent {
continue
}
dirSt, err := os.Stat(parent)
if err != nil {
bklog.G(ctx).WithError(err).Warnf("Failed to stat %q (parent of mount stub %q)", dir, p)
continue
@ -88,7 +98,7 @@ func MountStubsCleaner(ctx context.Context, dir string, mounts []Mount, recursiv
}
// Restore the timestamps of the dir
if err := os.Chtimes(dir, atime, mtime); err != nil {
if err := os.Chtimes(parent, atime, mtime); err != nil {
bklog.G(ctx).WithError(err).Warnf("Failed to restore time time mount stub timestamp (os.Chtimes(%q, %v, %v))", dir, atime, mtime)
}
}

View file

@ -17,6 +17,18 @@ func ParsePlatforms(meta map[string][]byte) (Platforms, error) {
return Platforms{}, errors.Wrapf(err, "failed to parse platforms passed to provenance processor")
}
}
if len(ps.Platforms) == 0 {
return Platforms{}, errors.Errorf("invalid empty platforms index for exporter")
}
for i, p := range ps.Platforms {
if p.ID == "" {
return Platforms{}, errors.Errorf("invalid empty platform key for exporter")
}
if p.Platform.OS == "" || p.Platform.Architecture == "" {
return Platforms{}, errors.Errorf("invalid platform value %v for exporter", p.Platform)
}
ps.Platforms[i].Platform = platforms.Normalize(p.Platform)
}
return ps, nil
}
@ -36,6 +48,8 @@ func ParsePlatforms(meta map[string][]byte) (Platforms, error) {
OSFeatures: img.OSFeatures,
Variant: img.Variant,
}
} else if img.OS != "" || img.Architecture != "" {
return Platforms{}, errors.Errorf("invalid image config: os and architecture must be specified together")
}
}
p = platforms.Normalize(p)

View file

@ -569,11 +569,27 @@ func parseHistoryFromConfig(dt []byte) ([]ocispecs.History, error) {
}
func patchImageConfig(dt []byte, descs []ocispecs.Descriptor, history []ocispecs.History, cache []byte, epoch *time.Time) ([]byte, error) {
var img ocispecs.Image
if err := json.Unmarshal(dt, &img); err != nil {
return nil, errors.Wrap(err, "invalid image config for export")
}
m := map[string]json.RawMessage{}
if err := json.Unmarshal(dt, &m); err != nil {
return nil, errors.Wrap(err, "failed to parse image config for patch")
}
if m == nil {
return nil, errors.Errorf("invalid null image config for export")
}
if img.OS == "" {
return nil, errors.Errorf("invalid image config for export: missing os")
}
if img.Architecture == "" {
return nil, errors.Errorf("invalid image config for export: missing architecture")
}
var rootFS ocispecs.RootFS
rootFS.Type = "layers"
for _, desc := range descs {

View file

@ -4,6 +4,7 @@ import (
"context"
"github.com/moby/buildkit/client/llb"
"github.com/moby/buildkit/executor"
gw "github.com/moby/buildkit/frontend/gateway/client"
"github.com/moby/buildkit/session"
"github.com/moby/buildkit/solver"
@ -17,7 +18,7 @@ type Result = result.Result[solver.ResultProxy]
type Attestation = result.Attestation[solver.ResultProxy]
type Frontend interface {
Solve(ctx context.Context, llb FrontendLLBBridge, opt map[string]string, inputs map[string]*pb.Definition, sid string, sm *session.Manager) (*Result, error)
Solve(ctx context.Context, llb FrontendLLBBridge, exec executor.Executor, opt map[string]string, inputs map[string]*pb.Definition, sid string, sm *session.Manager) (*Result, error)
}
type FrontendLLBBridge interface {

View file

@ -30,8 +30,14 @@ func AttestationToPB[T any](a *result.Attestation[T]) (*pb.Attestation, error) {
}
func AttestationFromPB[T any](a *pb.Attestation) (*result.Attestation[T], error) {
if a == nil {
return nil, errors.Errorf("invalid nil attestation")
}
subjects := make([]result.InTotoSubject, len(a.InTotoSubjects))
for i, subject := range a.InTotoSubjects {
if subject == nil {
return nil, errors.Errorf("invalid nil attestation subject")
}
subjects[i] = result.InTotoSubject{
Kind: subject.Kind,
Name: subject.Name,

View file

@ -47,7 +47,7 @@ type Mount struct {
WorkerRef *worker.WorkerRef
}
func NewContainer(ctx context.Context, w worker.Worker, sm *session.Manager, g session.Group, req NewContainerRequest) (client.Container, error) {
func NewContainer(ctx context.Context, cm cache.Manager, exec executor.Executor, sm *session.Manager, g session.Group, req NewContainerRequest) (client.Container, error) {
ctx, cancel := context.WithCancel(ctx)
eg, ctx := errgroup.WithContext(ctx)
platform := opspb.Platform{
@ -63,7 +63,7 @@ func NewContainer(ctx context.Context, w worker.Worker, sm *session.Manager, g s
hostname: req.Hostname,
extraHosts: req.ExtraHosts,
platform: platform,
executor: w.Executor(),
executor: exec,
sm: sm,
group: g,
errGroup: eg,
@ -86,9 +86,8 @@ func NewContainer(ctx context.Context, w worker.Worker, sm *session.Manager, g s
}
name := fmt.Sprintf("container %s", req.ContainerID)
mm := mounts.NewMountManager(name, w.CacheManager(), sm)
p, err := PrepareMounts(ctx, mm, w.CacheManager(), g, "", mnts, refs, func(m *opspb.Mount, ref cache.ImmutableRef) (cache.MutableRef, error) {
cm := w.CacheManager()
mm := mounts.NewMountManager(name, cm, sm)
p, err := PrepareMounts(ctx, mm, cm, g, "", mnts, refs, func(m *opspb.Mount, ref cache.ImmutableRef) (cache.MutableRef, error) {
if m.Input != opspb.Empty {
cm = refs[m.Input].Worker.CacheManager()
}

View file

@ -6,6 +6,7 @@ import (
cacheutil "github.com/moby/buildkit/cache/util"
"github.com/moby/buildkit/client/llb"
"github.com/moby/buildkit/executor"
"github.com/moby/buildkit/frontend"
"github.com/moby/buildkit/frontend/gateway/client"
"github.com/moby/buildkit/frontend/gateway/container"
@ -26,7 +27,7 @@ import (
"golang.org/x/sync/errgroup"
)
func LLBBridgeToGatewayClient(ctx context.Context, llbBridge frontend.FrontendLLBBridge, opts map[string]string, inputs map[string]*opspb.Definition, w worker.Infos, sid string, sm *session.Manager) (*BridgeClient, error) {
func LLBBridgeToGatewayClient(ctx context.Context, llbBridge frontend.FrontendLLBBridge, exec executor.Executor, opts map[string]string, inputs map[string]*opspb.Definition, w worker.Infos, sid string, sm *session.Manager) (*BridgeClient, error) {
bc := &BridgeClient{
opts: opts,
inputs: inputs,
@ -35,6 +36,7 @@ func LLBBridgeToGatewayClient(ctx context.Context, llbBridge frontend.FrontendLL
sm: sm,
workers: w,
workerRefByID: make(map[string]*worker.WorkerRef),
executor: exec,
}
bc.buildOpts = bc.loadBuildOpts()
return bc, nil
@ -52,6 +54,7 @@ type BridgeClient struct {
workerRefByID map[string]*worker.WorkerRef
buildOpts client.BuildOpts
ctrs []client.Container
executor executor.Executor
}
func (c *BridgeClient) Solve(ctx context.Context, req client.SolveRequest) (*client.Result, error) {
@ -293,13 +296,13 @@ func (c *BridgeClient) NewContainer(ctx context.Context, req client.NewContainer
return nil, err
}
w, err := c.workers.GetDefault()
cm, err := c.workers.DefaultCacheManager()
if err != nil {
return nil, err
}
group := session.NewGroup(c.sid)
ctr, err := container.NewContainer(ctx, w, c.sm, group, ctrReq)
ctr, err := container.NewContainer(ctx, cm, c.executor, c.sm, group, ctrReq)
if err != nil {
return nil, err
}

View file

@ -3,6 +3,7 @@ package forwarder
import (
"context"
"github.com/moby/buildkit/executor"
"github.com/moby/buildkit/frontend"
"github.com/moby/buildkit/frontend/gateway/client"
"github.com/moby/buildkit/session"
@ -22,8 +23,8 @@ type GatewayForwarder struct {
f client.BuildFunc
}
func (gf *GatewayForwarder) Solve(ctx context.Context, llbBridge frontend.FrontendLLBBridge, opts map[string]string, inputs map[string]*pb.Definition, sid string, sm *session.Manager) (retRes *frontend.Result, retErr error) {
c, err := LLBBridgeToGatewayClient(ctx, llbBridge, opts, inputs, gf.workers, sid, sm)
func (gf *GatewayForwarder) Solve(ctx context.Context, llbBridge frontend.FrontendLLBBridge, exec executor.Executor, opts map[string]string, inputs map[string]*pb.Definition, sid string, sm *session.Manager) (retRes *frontend.Result, retErr error) {
c, err := LLBBridgeToGatewayClient(ctx, llbBridge, exec, opts, inputs, gf.workers, sid, sm)
if err != nil {
return nil, err
}

View file

@ -86,7 +86,7 @@ func filterPrefix(opts map[string]string, pfx string) map[string]string {
return m
}
func (gf *gatewayFrontend) Solve(ctx context.Context, llbBridge frontend.FrontendLLBBridge, opts map[string]string, inputs map[string]*opspb.Definition, sid string, sm *session.Manager) (*frontend.Result, error) {
func (gf *gatewayFrontend) Solve(ctx context.Context, llbBridge frontend.FrontendLLBBridge, exec executor.Executor, opts map[string]string, inputs map[string]*opspb.Definition, sid string, sm *session.Manager) (*frontend.Result, error) {
source, ok := opts[keySource]
if !ok {
return nil, errors.Errorf("no source specified for gateway")
@ -141,7 +141,7 @@ func (gf *gatewayFrontend) Solve(ctx context.Context, llbBridge frontend.Fronten
}
}
} else {
c, err := forwarder.LLBBridgeToGatewayClient(ctx, llbBridge, opts, inputs, gf.workers, sid, sm)
c, err := forwarder.LLBBridgeToGatewayClient(ctx, llbBridge, exec, opts, inputs, gf.workers, sid, sm)
if err != nil {
return nil, err
}
@ -281,18 +281,13 @@ func (gf *gatewayFrontend) Solve(ctx context.Context, llbBridge frontend.Fronten
}
}
lbf, ctx, err := serveLLBBridgeForwarder(ctx, llbBridge, gf.workers, inputs, sid, sm)
lbf, ctx, err := serveLLBBridgeForwarder(ctx, llbBridge, exec, gf.workers, inputs, sid, sm)
defer lbf.conn.Close() //nolint
if err != nil {
return nil, err
}
defer lbf.Discard()
w, err := gf.workers.GetDefault()
if err != nil {
return nil, err
}
mdmnt, release, err := metadataMount(frontendDef)
if err != nil {
return nil, err
@ -305,7 +300,7 @@ func (gf *gatewayFrontend) Solve(ctx context.Context, llbBridge frontend.Fronten
mnts = append(mnts, *mdmnt)
}
_, err = w.Executor().Run(ctx, "", container.MountWithSession(rootFS, session.NewGroup(sid)), mnts, executor.ProcessInfo{Meta: meta, Stdin: lbf.Stdin, Stdout: lbf.Stdout, Stderr: os.Stderr}, nil)
_, err = exec.Run(ctx, "", container.MountWithSession(rootFS, session.NewGroup(sid)), mnts, executor.ProcessInfo{Meta: meta, Stdin: lbf.Stdin, Stdout: lbf.Stdout, Stderr: os.Stderr}, nil)
if err != nil {
if errdefs.IsCanceled(ctx, err) && lbf.isErrServerClosed {
err = errors.Errorf("frontend grpc server closed unexpectedly")
@ -434,11 +429,11 @@ func (lbf *llbBridgeForwarder) Result() (*frontend.Result, error) {
return lbf.result, nil
}
func NewBridgeForwarder(ctx context.Context, llbBridge frontend.FrontendLLBBridge, workers worker.Infos, inputs map[string]*opspb.Definition, sid string, sm *session.Manager) LLBBridgeForwarder {
return newBridgeForwarder(ctx, llbBridge, workers, inputs, sid, sm)
func NewBridgeForwarder(ctx context.Context, llbBridge frontend.FrontendLLBBridge, exec executor.Executor, workers worker.Infos, inputs map[string]*opspb.Definition, sid string, sm *session.Manager) LLBBridgeForwarder {
return newBridgeForwarder(ctx, llbBridge, exec, workers, inputs, sid, sm)
}
func newBridgeForwarder(ctx context.Context, llbBridge frontend.FrontendLLBBridge, workers worker.Infos, inputs map[string]*opspb.Definition, sid string, sm *session.Manager) *llbBridgeForwarder {
func newBridgeForwarder(ctx context.Context, llbBridge frontend.FrontendLLBBridge, exec executor.Executor, workers worker.Infos, inputs map[string]*opspb.Definition, sid string, sm *session.Manager) *llbBridgeForwarder {
lbf := &llbBridgeForwarder{
callCtx: ctx,
llbBridge: llbBridge,
@ -451,13 +446,14 @@ func newBridgeForwarder(ctx context.Context, llbBridge frontend.FrontendLLBBridg
sid: sid,
sm: sm,
ctrs: map[string]gwclient.Container{},
executor: exec,
}
return lbf
}
func serveLLBBridgeForwarder(ctx context.Context, llbBridge frontend.FrontendLLBBridge, workers worker.Infos, inputs map[string]*opspb.Definition, sid string, sm *session.Manager) (*llbBridgeForwarder, context.Context, error) {
func serveLLBBridgeForwarder(ctx context.Context, llbBridge frontend.FrontendLLBBridge, exec executor.Executor, workers worker.Infos, inputs map[string]*opspb.Definition, sid string, sm *session.Manager) (*llbBridgeForwarder, context.Context, error) {
ctx, cancel := context.WithCancel(ctx)
lbf := newBridgeForwarder(ctx, llbBridge, workers, inputs, sid, sm)
lbf := newBridgeForwarder(ctx, llbBridge, exec, workers, inputs, sid, sm)
server := grpc.NewServer(grpc.UnaryInterceptor(grpcerrors.UnaryServerInterceptor), grpc.StreamInterceptor(grpcerrors.StreamServerInterceptor))
grpc_health_v1.RegisterHealthServer(server, health.NewServer())
pb.RegisterLLBBridgeServer(server, lbf)
@ -552,6 +548,7 @@ type llbBridgeForwarder struct {
isErrServerClosed bool
sid string
sm *session.Manager
executor executor.Executor
*pipe
ctrs map[string]gwclient.Container
ctrsMu sync.Mutex
@ -646,12 +643,21 @@ func (lbf *llbBridgeForwarder) registerResultIDs(results ...solver.Result) (ids
func (lbf *llbBridgeForwarder) Solve(ctx context.Context, req *pb.SolveRequest) (*pb.SolveResponse, error) {
var cacheImports []frontend.CacheOptionsEntry
for _, e := range req.CacheImports {
if e == nil {
return nil, errors.Errorf("invalid nil cache import")
}
cacheImports = append(cacheImports, frontend.CacheOptionsEntry{
Type: e.Type,
Attrs: e.Attrs,
})
}
for _, p := range req.SourcePolicies {
if p == nil {
return nil, errors.Errorf("invalid nil source policy")
}
}
ctx = tracing.ContextWithSpanFromContext(ctx, lbf.callCtx)
res, err := lbf.llbBridge.Solve(ctx, frontend.SolveRequest{
Evaluate: req.Evaluate,
@ -1033,7 +1039,7 @@ func (lbf *llbBridgeForwarder) NewContainer(ctx context.Context, in *pb.NewConta
// and we want the context to live for the duration of the container.
group := session.NewGroup(lbf.sid)
w, err := lbf.workers.GetDefault()
cm, err := lbf.workers.DefaultCacheManager()
if err != nil {
return nil, stack.Enable(err)
}
@ -1043,7 +1049,7 @@ func (lbf *llbBridgeForwarder) NewContainer(ctx context.Context, in *pb.NewConta
return nil, stack.Enable(err)
}
ctr, err := container.NewContainer(context.Background(), w, lbf.sm, group, ctrReq)
ctr, err := container.NewContainer(context.Background(), cm, lbf.executor, lbf.sm, group, ctrReq)
if err != nil {
return nil, stack.Enable(err)
}
@ -1077,6 +1083,12 @@ func (lbf *llbBridgeForwarder) ReleaseContainer(ctx context.Context, in *pb.Rele
}
func (lbf *llbBridgeForwarder) Warn(ctx context.Context, in *pb.WarnRequest) (*pb.WarnResponse, error) {
// validate ranges are valid
for _, r := range in.Ranges {
if r == nil {
return nil, status.Errorf(codes.InvalidArgument, "invalid source range")
}
}
err := lbf.llbBridge.Warn(ctx, in.Digest, string(in.Short), frontend.WarnOpts{
Level: int(in.Level),
SourceInfo: in.Info,

View file

@ -11,22 +11,39 @@ type Mounter interface {
Unmount() error
}
type LocalMounterOpt func(*localMounter)
// LocalMounter is a helper for mounting mountfactory to temporary path. In
// addition it can mount binds without privileges
func LocalMounter(mountable Mountable) Mounter {
return &localMounter{mountable: mountable}
func LocalMounter(mountable Mountable, opts ...LocalMounterOpt) Mounter {
lm := &localMounter{mountable: mountable}
for _, opt := range opts {
opt(lm)
}
return lm
}
// LocalMounterWithMounts is a helper for mounting to temporary path. In
// addition it can mount binds without privileges
func LocalMounterWithMounts(mounts []mount.Mount) Mounter {
return &localMounter{mounts: mounts}
func LocalMounterWithMounts(mounts []mount.Mount, opts ...LocalMounterOpt) Mounter {
lm := &localMounter{mounts: mounts}
for _, opt := range opts {
opt(lm)
}
return lm
}
type localMounter struct {
mu sync.Mutex
mounts []mount.Mount
mountable Mountable
target string
release func() error
mu sync.Mutex
mounts []mount.Mount
mountable Mountable
target string
release func() error
forceRemount bool
}
func ForceRemount() LocalMounterOpt {
return func(lm *localMounter) {
lm.forceRemount = true
}
}

View file

@ -5,6 +5,7 @@ package snapshot
import (
"os"
"path/filepath"
"syscall"
"github.com/containerd/containerd/mount"
@ -34,30 +35,48 @@ func (lm *localMounter) Mount() (string, error) {
}
}
var isFile bool
if len(lm.mounts) == 1 && (lm.mounts[0].Type == "bind" || lm.mounts[0].Type == "rbind") {
ro := false
for _, opt := range lm.mounts[0].Options {
if opt == "ro" {
ro = true
break
if !lm.forceRemount {
ro := false
for _, opt := range lm.mounts[0].Options {
if opt == "ro" {
ro = true
break
}
}
if !ro {
return lm.mounts[0].Source, nil
}
}
if !ro {
return lm.mounts[0].Source, nil
fi, err := os.Stat(lm.mounts[0].Source)
if err != nil {
return "", err
}
if !fi.IsDir() {
isFile = true
}
}
dir, err := os.MkdirTemp("", "buildkit-mount")
dest, err := os.MkdirTemp("", "buildkit-mount")
if err != nil {
return "", errors.Wrap(err, "failed to create temp dir")
}
if err := mount.All(lm.mounts, dir); err != nil {
os.RemoveAll(dir)
return "", errors.Wrapf(err, "failed to mount %s: %+v", dir, lm.mounts)
if isFile {
dest = filepath.Join(dest, "file")
if err := os.WriteFile(dest, []byte{}, 0644); err != nil {
os.RemoveAll(dest)
return "", errors.Wrap(err, "failed to create temp file")
}
}
lm.target = dir
return dir, nil
if err := mount.All(lm.mounts, dest); err != nil {
os.RemoveAll(dest)
return "", errors.Wrapf(err, "failed to mount %s: %+v", dest, lm.mounts)
}
lm.target = dest
return dest, nil
}
func (lm *localMounter) Unmount() error {

View file

@ -10,14 +10,11 @@ import (
"github.com/containerd/containerd/pkg/userns"
"github.com/containerd/containerd/snapshots"
"github.com/docker/docker/pkg/idtools"
"github.com/moby/buildkit/executor"
"github.com/pkg/errors"
)
type Mountable interface {
// ID() string
Mount() ([]mount.Mount, func() error, error)
IdentityMapping() *idtools.IdentityMapping
}
type Mountable = executor.MountableRef
// Snapshotter defines interface that any snapshot implementation should satisfy
type Snapshotter interface {

View file

@ -11,6 +11,8 @@ import (
"github.com/moby/buildkit/cache/remotecache"
"github.com/moby/buildkit/client"
"github.com/moby/buildkit/client/llb"
"github.com/moby/buildkit/executor"
resourcestypes "github.com/moby/buildkit/executor/resources/types"
"github.com/moby/buildkit/frontend"
gw "github.com/moby/buildkit/frontend/gateway/client"
"github.com/moby/buildkit/identity"
@ -23,6 +25,7 @@ import (
"github.com/moby/buildkit/sourcepolicy"
spb "github.com/moby/buildkit/sourcepolicy/pb"
"github.com/moby/buildkit/util/bklog"
"github.com/moby/buildkit/util/entitlements"
"github.com/moby/buildkit/util/flightcontrol"
"github.com/moby/buildkit/util/progress"
"github.com/moby/buildkit/worker"
@ -39,6 +42,10 @@ type llbBridge struct {
cms map[string]solver.CacheManager
cmsMu sync.Mutex
sm *session.Manager
executorOnce sync.Once
executorErr error
executor executor.Executor
}
func (b *llbBridge) Warn(ctx context.Context, dgst digest.Digest, msg string, opts frontend.WarnOpts) error {
@ -79,6 +86,14 @@ func (b *llbBridge) loadResult(ctx context.Context, def *pb.Definition, cacheImp
}
var polEngine SourcePolicyEvaluator
if srcPol != nil || len(pol) > 0 {
for _, p := range pol {
if p == nil {
return nil, errors.Errorf("invalid nil policy")
}
if err := validateSourcePolicy(*p); err != nil {
return nil, err
}
}
if srcPol != nil {
pol = append([]*spb.Policy{srcPol}, pol...)
}
@ -151,6 +166,52 @@ func (b *llbBridge) loadResult(ctx context.Context, def *pb.Definition, cacheImp
return res, nil
}
func (b *llbBridge) validateEntitlements(p executor.ProcessInfo) error {
ent, err := loadEntitlements(b.builder)
if err != nil {
return err
}
v := entitlements.Values{
NetworkHost: p.Meta.NetMode == pb.NetMode_HOST,
SecurityInsecure: p.Meta.SecurityMode == pb.SecurityMode_INSECURE,
}
return ent.Check(v)
}
func (b *llbBridge) Run(ctx context.Context, id string, rootfs executor.Mount, mounts []executor.Mount, process executor.ProcessInfo, started chan<- struct{}) (resourcestypes.Recorder, error) {
if err := b.validateEntitlements(process); err != nil {
return nil, err
}
if err := b.loadExecutor(); err != nil {
return nil, err
}
return b.executor.Run(ctx, id, rootfs, mounts, process, started)
}
func (b *llbBridge) Exec(ctx context.Context, id string, process executor.ProcessInfo) error {
if err := b.validateEntitlements(process); err != nil {
return err
}
if err := b.loadExecutor(); err != nil {
return err
}
return b.executor.Exec(ctx, id, process)
}
func (b *llbBridge) loadExecutor() error {
b.executorOnce.Do(func() {
w, err := b.resolveWorker()
if err != nil {
b.executorErr = err
return
}
b.executor = w.Executor()
})
return b.executorErr
}
type resultProxy struct {
id string
b *provenanceBridge

View file

@ -165,7 +165,7 @@ func (b *provenanceBridge) Solve(ctx context.Context, req frontend.SolveRequest,
return nil, errors.Errorf("invalid frontend: %s", req.Frontend)
}
wb := &provenanceBridge{llbBridge: b.llbBridge, req: &req}
res, err = f.Solve(ctx, wb, req.FrontendOpt, req.FrontendInputs, sid, b.llbBridge.sm)
res, err = f.Solve(ctx, wb, b.llbBridge, req.FrontendOpt, req.FrontendInputs, sid, b.llbBridge.sm)
if err != nil {
return nil, err
}

View file

@ -447,6 +447,9 @@ func (s *Solver) Solve(ctx context.Context, id string, sessionID string, req fro
j.SetValue(keyEntitlements, set)
if srcPol != nil {
if err := validateSourcePolicy(*srcPol); err != nil {
return nil, err
}
j.SetValue(keySourcePolicy, *srcPol)
}
@ -455,7 +458,7 @@ func (s *Solver) Solve(ctx context.Context, id string, sessionID string, req fro
br := s.bridge(j)
var fwd gateway.LLBBridgeForwarder
if s.gatewayForwarder != nil && req.Definition == nil && req.Frontend == "" {
fwd = gateway.NewBridgeForwarder(ctx, br, s.workerController, req.FrontendInputs, sessionID, s.sm)
fwd = gateway.NewBridgeForwarder(ctx, br, br, s.workerController.Infos(), req.FrontendInputs, sessionID, s.sm)
defer fwd.Discard()
// Register build before calling s.recordBuildHistory, because
// s.recordBuildHistory can block for several seconds on
@ -595,6 +598,23 @@ func (s *Solver) Solve(ctx context.Context, id string, sessionID string, req fro
}, nil
}
func validateSourcePolicy(pol spb.Policy) error {
for _, r := range pol.Rules {
if r == nil {
return errors.New("invalid nil rule in policy")
}
if r.Selector == nil {
return errors.New("invalid nil selector in policy")
}
for _, c := range r.Selector.Constraints {
if c == nil {
return errors.New("invalid nil constraint in policy")
}
}
}
return nil
}
func runCacheExporters(ctx context.Context, exporters []RemoteCacheExporter, j *solver.Job, cached *result.Result[solver.CachedResult], inp *result.Result[cache.ImmutableRef]) (map[string]string, error) {
eg, ctx := errgroup.WithContext(ctx)
g := session.NewGroup(j.SessionID)
@ -991,6 +1011,9 @@ func loadSourcePolicy(b solver.Builder) (*spb.Policy, error) {
return errors.Errorf("invalid source policy %T", v)
}
for _, f := range x.Rules {
if f == nil {
return errors.Errorf("invalid nil policy rule")
}
r := *f
srcPol.Rules = append(srcPol.Rules, &r)
}

View file

@ -101,16 +101,12 @@ func ValidateEntitlements(ent entitlements.Set) LoadOpt {
return func(op *pb.Op, _ *pb.OpMetadata, opt *solver.VertexOptions) error {
switch op := op.Op.(type) {
case *pb.Op_Exec:
if op.Exec.Network == pb.NetMode_HOST {
if !ent.Allowed(entitlements.EntitlementNetworkHost) {
return errors.Errorf("%s is not allowed", entitlements.EntitlementNetworkHost)
}
v := entitlements.Values{
NetworkHost: op.Exec.Network == pb.NetMode_HOST,
SecurityInsecure: op.Exec.Security == pb.SecurityMode_INSECURE,
}
if op.Exec.Security == pb.SecurityMode_INSECURE {
if !ent.Allowed(entitlements.EntitlementSecurityInsecure) {
return errors.Errorf("%s is not allowed", entitlements.EntitlementSecurityInsecure)
}
if err := ent.Check(v); err != nil {
return err
}
}
return nil

View file

@ -10,6 +10,9 @@ import (
func match(ctx context.Context, src *selectorCache, ref string, attrs map[string]string) (bool, error) {
for _, c := range src.Constraints {
if c == nil {
return false, errors.Errorf("invalid nil constraint for %v", src)
}
switch c.Condition {
case spb.AttrMatch_EQUAL:
if attrs[c.Key] != c.Value {

View file

@ -58,3 +58,23 @@ func (s Set) Allowed(e Entitlement) bool {
_, ok := s[e]
return ok
}
func (s Set) Check(v Values) error {
if v.NetworkHost {
if !s.Allowed(EntitlementNetworkHost) {
return errors.Errorf("%s is not allowed", EntitlementNetworkHost)
}
}
if v.SecurityInsecure {
if !s.Allowed(EntitlementSecurityInsecure) {
return errors.Errorf("%s is not allowed", EntitlementSecurityInsecure)
}
}
return nil
}
type Values struct {
NetworkHost bool
SecurityInsecure bool
}

View file

@ -13,6 +13,9 @@ func Attributes(attrs []*commonpb.KeyValue) []attribute.KeyValue {
out := make([]attribute.KeyValue, 0, len(attrs))
for _, a := range attrs {
if a == nil {
continue
}
kv := attribute.KeyValue{
Key: attribute.Key(a.Key),
Value: toValue(a.Value),
@ -42,7 +45,9 @@ func toValue(v *commonpb.AnyValue) attribute.Value {
func boolArray(kv []*commonpb.AnyValue) attribute.Value {
arr := make([]bool, len(kv))
for i, v := range kv {
arr[i] = v.GetBoolValue()
if v != nil {
arr[i] = v.GetBoolValue()
}
}
return attribute.BoolSliceValue(arr)
}
@ -50,7 +55,9 @@ func boolArray(kv []*commonpb.AnyValue) attribute.Value {
func intArray(kv []*commonpb.AnyValue) attribute.Value {
arr := make([]int64, len(kv))
for i, v := range kv {
arr[i] = v.GetIntValue()
if v != nil {
arr[i] = v.GetIntValue()
}
}
return attribute.Int64SliceValue(arr)
}
@ -58,7 +65,9 @@ func intArray(kv []*commonpb.AnyValue) attribute.Value {
func doubleArray(kv []*commonpb.AnyValue) attribute.Value {
arr := make([]float64, len(kv))
for i, v := range kv {
arr[i] = v.GetDoubleValue()
if v != nil {
arr[i] = v.GetDoubleValue()
}
}
return attribute.Float64SliceValue(arr)
}
@ -66,13 +75,15 @@ func doubleArray(kv []*commonpb.AnyValue) attribute.Value {
func stringArray(kv []*commonpb.AnyValue) attribute.Value {
arr := make([]string, len(kv))
for i, v := range kv {
arr[i] = v.GetStringValue()
if v != nil {
arr[i] = v.GetStringValue()
}
}
return attribute.StringSliceValue(arr)
}
func arrayValues(kv []*commonpb.AnyValue) attribute.Value {
if len(kv) == 0 {
if len(kv) == 0 || kv[0] == nil {
return attribute.StringSliceValue([]string{})
}

View file

@ -32,14 +32,20 @@ func Spans(sdl []*tracepb.ResourceSpans) []tracesdk.ReadOnlySpan {
}
for _, sdi := range sd.ScopeSpans {
sda := make([]tracesdk.ReadOnlySpan, len(sdi.Spans))
for i, s := range sdi.Spans {
sda[i] = &readOnlySpan{
if sdi == nil {
continue
}
sda := make([]tracesdk.ReadOnlySpan, 0, len(sdi.Spans))
for _, s := range sdi.Spans {
if s == nil {
continue
}
sda = append(sda, &readOnlySpan{
pb: s,
is: sdi.Scope,
resource: sd.Resource,
schemaURL: sd.SchemaUrl,
}
})
}
out = append(out, sda...)
}
@ -170,6 +176,9 @@ var _ tracesdk.ReadOnlySpan = &readOnlySpan{}
// status transform a OTLP span status into span code.
func statusCode(st *tracepb.Status) codes.Code {
if st == nil {
return codes.Unset
}
switch st.Code {
case tracepb.Status_STATUS_CODE_ERROR:
return codes.Error
@ -186,6 +195,9 @@ func links(links []*tracepb.Span_Link) []tracesdk.Link {
sl := make([]tracesdk.Link, 0, len(links))
for _, otLink := range links {
if otLink == nil {
continue
}
// This redefinition is necessary to prevent otLink.*ID[:] copies
// being reused -- in short we need a new otLink per iteration.
otLink := otLink
@ -226,6 +238,9 @@ func spanEvents(es []*tracepb.Span_Event) []tracesdk.Event {
if messageEvents >= maxMessageEventsPerSpan {
break
}
if e == nil {
continue
}
messageEvents++
events = append(events,
tracesdk.Event{

View file

@ -43,6 +43,6 @@ type Worker interface {
}
type Infos interface {
GetDefault() (Worker, error)
DefaultCacheManager() (cache.Manager, error)
WorkerInfos() []client.WorkerInfo
}

View file

@ -3,6 +3,7 @@ package worker
import (
"github.com/containerd/containerd/filters"
"github.com/hashicorp/go-multierror"
"github.com/moby/buildkit/cache"
"github.com/moby/buildkit/client"
"github.com/pkg/errors"
)
@ -81,3 +82,25 @@ func (c *Controller) WorkerInfos() []client.WorkerInfo {
}
return out
}
func (c *Controller) Infos() Infos {
return &infosController{c: c}
}
type infosController struct {
c *Controller
}
var _ Infos = &infosController{}
func (c *infosController) DefaultCacheManager() (cache.Manager, error) {
w, err := c.c.GetDefault()
if err != nil {
return nil, err
}
return w.CacheManager(), nil
}
func (c *infosController) WorkerInfos() []client.WorkerInfo {
return c.c.WorkerInfos()
}

2
vendor/modules.txt vendored
View file

@ -682,7 +682,7 @@ github.com/mitchellh/hashstructure/v2
# github.com/mitchellh/reflectwalk v1.0.2
## explicit
github.com/mitchellh/reflectwalk
# github.com/moby/buildkit v0.12.5-0.20231208203051-3b6880d2a00f
# github.com/moby/buildkit v0.12.5
## explicit; go 1.20
github.com/moby/buildkit/api/services/control
github.com/moby/buildkit/api/types