vendor: github.com/moby/buildkit v0.11.5

full diff: https://github.com/moby/buildkit/compare/v0.11.4...v0.11.5

Signed-off-by: Sebastiaan van Stijn <github@gone.nl>
This commit is contained in:
Sebastiaan van Stijn 2023-03-25 14:37:07 +01:00
parent 270d4ec23d
commit 988d26afe4
No known key found for this signature in database
GPG key ID: 76698F39D527CE8C
18 changed files with 291 additions and 88 deletions

View file

@ -82,9 +82,7 @@ jobs:
-
name: BuildKit ref
run: |
# FIXME(crazy-max) remove when updating BuildKit to v0.11.5
echo "BUILDKIT_REF=237fee9c00877d64a1c757fc5270c98b65a51abf" >> $GITHUB_ENV
#echo "BUILDKIT_REF=$(./hack/buildkit-ref)" >> $GITHUB_ENV
echo "BUILDKIT_REF=$(./hack/buildkit-ref)" >> $GITHUB_ENV
working-directory: moby
-
name: Checkout BuildKit ${{ env.BUILDKIT_REF }}

View file

@ -56,7 +56,7 @@ require (
github.com/klauspost/compress v1.16.3
github.com/miekg/dns v1.1.43
github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible
github.com/moby/buildkit v0.11.4 // FIXME(crazy-max): remove override from .github/workflows/buildkit.yml when updating to v0.11.5
github.com/moby/buildkit v0.11.5
github.com/moby/ipvs v1.1.0
github.com/moby/locker v1.0.1
github.com/moby/patternmatcher v0.5.0

View file

@ -1042,8 +1042,8 @@ github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh
github.com/mitchellh/mapstructure v1.3.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
github.com/mitchellh/osext v0.0.0-20151018003038-5e2d6d41470f/go.mod h1:OkQIRizQZAeMln+1tSwduZz7+Af5oFlKirV/MSYes2A=
github.com/moby/buildkit v0.8.1/go.mod h1:/kyU1hKy/aYCuP39GZA9MaKioovHku57N6cqlKZIaiQ=
github.com/moby/buildkit v0.11.4 h1:mleVHr+n7HUD65QNUkgkT3d8muTzhYUoHE9FM3Ej05s=
github.com/moby/buildkit v0.11.4/go.mod h1:P5Qi041LvCfhkfYBHry+Rwoo3Wi6H971J2ggE+PcIoo=
github.com/moby/buildkit v0.11.5 h1:S6YrFJ0bfBT2w9e8kOxqsDV8Bw+HtfqdB6eHL17BXRI=
github.com/moby/buildkit v0.11.5/go.mod h1:P5Qi041LvCfhkfYBHry+Rwoo3Wi6H971J2ggE+PcIoo=
github.com/moby/ipvs v1.1.0 h1:ONN4pGaZQgAx+1Scz5RvWV4Q7Gb+mvfRh3NsPS+1XQQ=
github.com/moby/ipvs v1.1.0/go.mod h1:4VJMWuf098bsUMmZEiD4Tjk/O7mOn3l1PTD3s4OoYAs=
github.com/moby/locker v1.0.1 h1:fOXqR41zeveg4fFODix+1Ch4mj/gT0NE1XJbp/epuBg=

View file

@ -14,6 +14,7 @@ import (
"github.com/containerd/containerd/images"
"github.com/containerd/containerd/leases"
"github.com/containerd/containerd/mount"
"github.com/containerd/containerd/pkg/userns"
"github.com/containerd/containerd/snapshots"
"github.com/docker/docker/pkg/idtools"
"github.com/hashicorp/go-multierror"
@ -27,6 +28,7 @@ import (
"github.com/moby/buildkit/util/flightcontrol"
"github.com/moby/buildkit/util/leaseutil"
"github.com/moby/buildkit/util/progress"
rootlessmountopts "github.com/moby/buildkit/util/rootless/mountopts"
"github.com/moby/buildkit/util/winlayers"
"github.com/moby/sys/mountinfo"
digest "github.com/opencontainers/go-digest"
@ -1640,6 +1642,12 @@ func (sm *sharableMountable) Mount() (_ []mount.Mount, _ func() error, retErr er
os.Remove(dir)
}
}()
if userns.RunningInUserNS() {
mounts, err = rootlessmountopts.FixUp(mounts)
if err != nil {
return nil, nil, err
}
}
if err := mount.All(mounts, dir); err != nil {
return nil, nil, err
}

View file

@ -209,6 +209,7 @@ func (d *DefinitionOp) Inputs() []Output {
dgst: input.Digest,
index: input.Index,
inputCache: d.inputCache,
sources: d.sources,
}
existingIndexes := d.inputCache[input.Digest]
indexDiff := int(input.Index) - len(existingIndexes)

View file

@ -11,12 +11,14 @@ import (
"github.com/containerd/containerd/mount"
"github.com/containerd/containerd/namespaces"
"github.com/containerd/containerd/oci"
"github.com/containerd/containerd/pkg/userns"
"github.com/containerd/continuity/fs"
"github.com/docker/docker/pkg/idtools"
"github.com/mitchellh/hashstructure/v2"
"github.com/moby/buildkit/executor"
"github.com/moby/buildkit/snapshot"
"github.com/moby/buildkit/util/network"
rootlessmountopts "github.com/moby/buildkit/util/rootless/mountopts"
traceexec "github.com/moby/buildkit/util/tracing/exec"
specs "github.com/opencontainers/runtime-spec/specs-go"
"github.com/opencontainers/selinux/go-selinux"
@ -192,6 +194,14 @@ func GenerateSpec(ctx context.Context, meta executor.Meta, mounts []executor.Mou
}
s.Mounts = dedupMounts(s.Mounts)
if userns.RunningInUserNS() {
s.Mounts, err = rootlessmountopts.FixUpOCI(s.Mounts)
if err != nil {
return nil, nil, err
}
}
return s, releaseAll, nil
}

View file

@ -92,7 +92,7 @@ func New(opt Opt, networkProviders map[pb.NetMode]network.Provider) (executor.Ex
root := opt.Root
if err := os.MkdirAll(root, 0711); err != nil {
if err := os.MkdirAll(root, 0o711); err != nil {
return nil, errors.Wrapf(err, "failed to create %s", root)
}
@ -205,7 +205,7 @@ func (w *runcExecutor) Run(ctx context.Context, id string, root executor.Mount,
}
bundle := filepath.Join(w.root, id)
if err := os.Mkdir(bundle, 0711); err != nil {
if err := os.Mkdir(bundle, 0o711); err != nil {
return err
}
defer os.RemoveAll(bundle)
@ -216,7 +216,7 @@ func (w *runcExecutor) Run(ctx context.Context, id string, root executor.Mount,
}
rootFSPath := filepath.Join(bundle, "rootfs")
if err := idtools.MkdirAllAndChown(rootFSPath, 0700, identity); err != nil {
if err := idtools.MkdirAllAndChown(rootFSPath, 0o700, identity); err != nil {
return err
}
if err := mount.All(rootMount, rootFSPath); err != nil {
@ -270,7 +270,7 @@ func (w *runcExecutor) Run(ctx context.Context, id string, root executor.Mount,
return errors.Wrapf(err, "working dir %s points to invalid target", newp)
}
if _, err := os.Stat(newp); err != nil {
if err := idtools.MkdirAllAndChown(newp, 0755, identity); err != nil {
if err := idtools.MkdirAllAndChown(newp, 0o755, identity); err != nil {
return errors.Wrapf(err, "failed to create working directory %s", newp)
}
}
@ -287,42 +287,10 @@ func (w *runcExecutor) Run(ctx context.Context, id string, root executor.Mount,
return err
}
// runCtx/killCtx is used for extra check in case the kill command blocks
runCtx, cancelRun := context.WithCancel(context.Background())
defer cancelRun()
ended := make(chan struct{})
go func() {
for {
select {
case <-ctx.Done():
killCtx, timeout := context.WithTimeout(context.Background(), 7*time.Second)
if err := w.runc.Kill(killCtx, id, int(syscall.SIGKILL), nil); err != nil {
bklog.G(ctx).Errorf("failed to kill runc %s: %+v", id, err)
select {
case <-killCtx.Done():
timeout()
cancelRun()
return
default:
}
}
timeout()
select {
case <-time.After(50 * time.Millisecond):
case <-ended:
return
}
case <-ended:
return
}
}
}()
bklog.G(ctx).Debugf("> creating %s %v", id, meta.Args)
trace.SpanFromContext(ctx).AddEvent("Container created")
err = w.run(runCtx, id, bundle, process, func() {
err = w.run(ctx, id, bundle, process, func() {
startedOnce.Do(func() {
trace.SpanFromContext(ctx).AddEvent("Container started")
if started != nil {
@ -330,7 +298,6 @@ func (w *runcExecutor) Run(ctx context.Context, id string, root executor.Mount,
}
})
})
close(ended)
return exitError(ctx, err)
}
@ -462,23 +429,87 @@ func (s *forwardIO) Stderr() io.ReadCloser {
return nil
}
// startingProcess is to track the os process so we can send signals to it.
type startingProcess struct {
Process *os.Process
ready chan struct{}
// procHandle is to track the os process so we can send signals to it.
type procHandle struct {
Process *os.Process
ready chan struct{}
ended chan struct{}
shutdown func()
}
// Release will free resources with a startingProcess.
func (p *startingProcess) Release() {
// runcProcessHandle will create a procHandle that will be monitored, where
// on ctx.Done the process will be killed. If the kill fails, then the cancel
// will be called. This is to allow for runc to go through its normal shutdown
// procedure if the ctx is canceled and to ensure there are no zombie processes
// left by runc.
func runcProcessHandle(ctx context.Context, id string) (*procHandle, context.Context) {
runcCtx, cancel := context.WithCancel(context.Background())
p := &procHandle{
ready: make(chan struct{}),
ended: make(chan struct{}),
shutdown: cancel,
}
// preserve the logger on the context used for the runc process handling
runcCtx = bklog.WithLogger(runcCtx, bklog.G(ctx))
go func() {
// Wait for pid
select {
case <-ctx.Done():
return // nothing to kill
case <-p.ready:
}
for {
select {
case <-ctx.Done():
killCtx, timeout := context.WithTimeout(context.Background(), 7*time.Second)
if err := p.Process.Kill(); err != nil {
bklog.G(ctx).Errorf("failed to kill runc %s: %+v", id, err)
select {
case <-killCtx.Done():
timeout()
cancel()
return
default:
}
}
timeout()
select {
case <-time.After(50 * time.Millisecond):
case <-p.ended:
return
}
case <-p.ended:
return
}
}
}()
return p, runcCtx
}
// Release will free resources with a procHandle.
func (p *procHandle) Release() {
close(p.ended)
if p.Process != nil {
p.Process.Release()
}
}
// Shutdown should be called after the runc process has exited. This will allow
// the signal handling and tty resize loops to exit, terminating the
// goroutines.
func (p *procHandle) Shutdown() {
if p.shutdown != nil {
p.shutdown()
}
}
// WaitForReady will wait until the Process has been populated or the
// provided context was cancelled. This should be called before using
// the Process field.
func (p *startingProcess) WaitForReady(ctx context.Context) error {
func (p *procHandle) WaitForReady(ctx context.Context) error {
select {
case <-ctx.Done():
return ctx.Err()
@ -490,7 +521,7 @@ func (p *startingProcess) WaitForReady(ctx context.Context) error {
// WaitForStart will record the pid reported by Runc via the channel.
// We wait for up to 10s for the runc process to start. If the started
// callback is non-nil it will be called after receiving the pid.
func (p *startingProcess) WaitForStart(ctx context.Context, startedCh <-chan int, started func()) error {
func (p *procHandle) WaitForStart(ctx context.Context, startedCh <-chan int, started func()) error {
startedCtx, timeout := context.WithTimeout(ctx, 10*time.Second)
defer timeout()
var err error
@ -515,7 +546,7 @@ func (p *startingProcess) WaitForStart(ctx context.Context, startedCh <-chan int
// handleSignals will wait until the runcProcess is ready then will
// send each signal received on the channel to the process.
func handleSignals(ctx context.Context, runcProcess *startingProcess, signals <-chan syscall.Signal) error {
func handleSignals(ctx context.Context, runcProcess *procHandle, signals <-chan syscall.Signal) error {
if signals == nil {
return nil
}

View file

@ -49,23 +49,20 @@ type runcCall func(ctx context.Context, started chan<- int, io runc.IO) error
// is only supported for linux, so this really just handles signal propagation
// to the started runc process.
func (w *runcExecutor) commonCall(ctx context.Context, id, bundle string, process executor.ProcessInfo, started func(), call runcCall) error {
runcProcess := &startingProcess{
ready: make(chan struct{}),
}
runcProcess, ctx := runcProcessHandle(ctx, id)
defer runcProcess.Release()
var eg errgroup.Group
egCtx, cancel := context.WithCancel(ctx)
eg, ctx := errgroup.WithContext(ctx)
defer eg.Wait()
defer cancel()
defer runcProcess.Shutdown()
startedCh := make(chan int, 1)
eg.Go(func() error {
return runcProcess.WaitForStart(egCtx, startedCh, started)
return runcProcess.WaitForStart(ctx, startedCh, started)
})
eg.Go(func() error {
return handleSignals(egCtx, runcProcess, process.Signal)
return handleSignals(ctx, runcProcess, process.Signal)
})
return call(ctx, startedCh, &forwardIO{stdin: process.Stdin, stdout: process.Stdout, stderr: process.Stderr})

View file

@ -44,23 +44,20 @@ func (w *runcExecutor) exec(ctx context.Context, id, bundle string, specsProcess
type runcCall func(ctx context.Context, started chan<- int, io runc.IO) error
func (w *runcExecutor) callWithIO(ctx context.Context, id, bundle string, process executor.ProcessInfo, started func(), call runcCall) error {
runcProcess := &startingProcess{
ready: make(chan struct{}),
}
runcProcess, ctx := runcProcessHandle(ctx, id)
defer runcProcess.Release()
var eg errgroup.Group
egCtx, cancel := context.WithCancel(ctx)
eg, ctx := errgroup.WithContext(ctx)
defer eg.Wait()
defer cancel()
defer runcProcess.Shutdown()
startedCh := make(chan int, 1)
eg.Go(func() error {
return runcProcess.WaitForStart(egCtx, startedCh, started)
return runcProcess.WaitForStart(ctx, startedCh, started)
})
eg.Go(func() error {
return handleSignals(egCtx, runcProcess, process.Signal)
return handleSignals(ctx, runcProcess, process.Signal)
})
if !process.Meta.Tty {
@ -84,7 +81,7 @@ func (w *runcExecutor) callWithIO(ctx context.Context, id, bundle string, proces
}
pts.Close()
ptm.Close()
cancel() // this will shutdown resize and signal loops
runcProcess.Shutdown()
err := eg.Wait()
if err != nil {
bklog.G(ctx).Warningf("error while shutting down tty io: %s", err)
@ -119,13 +116,13 @@ func (w *runcExecutor) callWithIO(ctx context.Context, id, bundle string, proces
}
eg.Go(func() error {
err := runcProcess.WaitForReady(egCtx)
err := runcProcess.WaitForReady(ctx)
if err != nil {
return err
}
for {
select {
case <-egCtx.Done():
case <-ctx.Done():
return nil
case resize := <-process.Resize:
err = ptm.Resize(console.WinSize{

View file

@ -927,11 +927,11 @@ func (ctr *container) Start(ctx context.Context, req client.StartRequest) (clien
if msg == nil {
// empty message from ctx cancel, so just start shutting down
// input, but continue processing more exit/done messages
// input
closeDoneOnce.Do(func() {
close(done)
})
continue
return ctx.Err()
}
if file := msg.GetFile(); file != nil {

View file

@ -8,6 +8,8 @@ import (
"syscall"
"github.com/containerd/containerd/mount"
"github.com/containerd/containerd/pkg/userns"
rootlessmountopts "github.com/moby/buildkit/util/rootless/mountopts"
"github.com/pkg/errors"
)
@ -24,6 +26,14 @@ func (lm *localMounter) Mount() (string, error) {
lm.release = release
}
if userns.RunningInUserNS() {
var err error
lm.mounts, err = rootlessmountopts.FixUp(lm.mounts)
if err != nil {
return "", err
}
}
if len(lm.mounts) == 1 && (lm.mounts[0].Type == "bind" || lm.mounts[0].Type == "rbind") {
ro := false
for _, opt := range lm.mounts[0].Options {

View file

@ -102,13 +102,13 @@ func (h *HistoryQueue) gc() error {
}
// in order for record to get deleted by gc it exceed both maxentries and maxage criteria
if len(records) < int(h.CleanConfig.MaxEntries) {
return nil
}
// sort array by newest records first
sort.Slice(records, func(i, j int) bool {
return records[i].CompletedAt.Before(*records[j].CompletedAt)
return records[i].CompletedAt.After(*records[j].CompletedAt)
})
h.mu.Lock()

View file

@ -423,15 +423,6 @@ func (s *Solver) Solve(ctx context.Context, id string, sessionID string, req fro
if internal {
defer j.CloseProgress()
} else {
rec, err1 := s.recordBuildHistory(ctx, id, req, exp, j)
if err != nil {
defer j.CloseProgress()
return nil, err1
}
defer func() {
err = rec(resProv, descref, err)
}()
}
set, err := entitlements.WhiteList(ent, supportedEntitlements(s.entitlements))
@ -447,14 +438,32 @@ func (s *Solver) Solve(ctx context.Context, id string, sessionID string, req fro
j.SessionID = sessionID
br := s.bridge(j)
var fwd gateway.LLBBridgeForwarder
if s.gatewayForwarder != nil && req.Definition == nil && req.Frontend == "" {
fwd := gateway.NewBridgeForwarder(ctx, br, s.workerController, req.FrontendInputs, sessionID, s.sm)
fwd = gateway.NewBridgeForwarder(ctx, br, s.workerController, req.FrontendInputs, sessionID, s.sm)
defer fwd.Discard()
// Register build before calling s.recordBuildHistory, because
// s.recordBuildHistory can block for several seconds on
// LeaseManager calls, and there is a fixed 3s timeout in
// GatewayForwarder on build registration.
if err := s.gatewayForwarder.RegisterBuild(ctx, id, fwd); err != nil {
return nil, err
}
defer s.gatewayForwarder.UnregisterBuild(ctx, id)
}
if !internal {
rec, err1 := s.recordBuildHistory(ctx, id, req, exp, j)
if err1 != nil {
defer j.CloseProgress()
return nil, err1
}
defer func() {
err = rec(resProv, descref, err)
}()
}
if fwd != nil {
var err error
select {
case <-fwd.Done():

View file

@ -210,6 +210,7 @@ func recomputeDigests(ctx context.Context, all map[digest.Digest]*pb.Op, visited
}
if !mutated {
visited[dgst] = dgst
return dgst, nil
}
@ -274,7 +275,7 @@ func loadLLB(ctx context.Context, def *pb.Definition, polEngine SourcePolicyEval
for {
newDgst, ok := mutatedDigests[lastDgst]
if !ok {
if !ok || newDgst == lastDgst {
break
}
lastDgst = newDgst

View file

@ -0,0 +1,88 @@
package mountopts
import (
"github.com/containerd/containerd/mount"
"github.com/moby/buildkit/util/strutil"
specs "github.com/opencontainers/runtime-spec/specs-go"
"github.com/pkg/errors"
"golang.org/x/sys/unix"
)
// UnprivilegedMountFlags gets the set of mount flags that are set on the mount that contains the given
// path and are locked by CL_UNPRIVILEGED. This is necessary to ensure that
// bind-mounting "with options" will not fail with user namespaces, due to
// kernel restrictions that require user namespace mounts to preserve
// CL_UNPRIVILEGED locked flags.
//
// From https://github.com/moby/moby/blob/v23.0.1/daemon/oci_linux.go#L430-L460
func UnprivilegedMountFlags(path string) ([]string, error) {
var statfs unix.Statfs_t
if err := unix.Statfs(path, &statfs); err != nil {
return nil, err
}
// The set of keys come from https://github.com/torvalds/linux/blob/v4.13/fs/namespace.c#L1034-L1048.
unprivilegedFlags := map[uint64]string{
unix.MS_RDONLY: "ro",
unix.MS_NODEV: "nodev",
unix.MS_NOEXEC: "noexec",
unix.MS_NOSUID: "nosuid",
unix.MS_NOATIME: "noatime",
unix.MS_RELATIME: "relatime",
unix.MS_NODIRATIME: "nodiratime",
}
var flags []string
for mask, flag := range unprivilegedFlags {
if uint64(statfs.Flags)&mask == mask {
flags = append(flags, flag)
}
}
return flags, nil
}
// FixUp is for https://github.com/moby/buildkit/issues/3098
func FixUp(mounts []mount.Mount) ([]mount.Mount, error) {
for i, m := range mounts {
var isBind bool
for _, o := range m.Options {
switch o {
case "bind", "rbind":
isBind = true
}
}
if !isBind {
continue
}
unpriv, err := UnprivilegedMountFlags(m.Source)
if err != nil {
return nil, errors.Wrapf(err, "failed to get unprivileged mount flags for %+v", m)
}
m.Options = strutil.DedupeSlice(append(m.Options, unpriv...))
mounts[i] = m
}
return mounts, nil
}
func FixUpOCI(mounts []specs.Mount) ([]specs.Mount, error) {
for i, m := range mounts {
var isBind bool
for _, o := range m.Options {
switch o {
case "bind", "rbind":
isBind = true
}
}
if !isBind {
continue
}
unpriv, err := UnprivilegedMountFlags(m.Source)
if err != nil {
return nil, errors.Wrapf(err, "failed to get unprivileged mount flags for %+v", m)
}
m.Options = strutil.DedupeSlice(append(m.Options, unpriv...))
mounts[i] = m
}
return mounts, nil
}

View file

@ -0,0 +1,21 @@
//go:build !linux
// +build !linux
package mountopts
import (
"github.com/containerd/containerd/mount"
specs "github.com/opencontainers/runtime-spec/specs-go"
)
func UnprivilegedMountFlags(path string) ([]string, error) {
return []string{}, nil
}
func FixUp(mounts []mount.Mount) ([]mount.Mount, error) {
return mounts, nil
}
func FixUpOCI(mounts []specs.Mount) ([]specs.Mount, error) {
return mounts, nil
}

View file

@ -0,0 +1,30 @@
/*
Copyright The containerd Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package strutil
// DedupeSlice is from https://github.com/containerd/nerdctl/blob/v1.2.1/pkg/strutil/strutil.go#L72-L82
func DedupeSlice(in []string) []string {
m := make(map[string]struct{})
var res []string
for _, s := range in {
if _, ok := m[s]; !ok {
res = append(res, s)
m[s] = struct{}{}
}
}
return res
}

4
vendor/modules.txt vendored
View file

@ -575,7 +575,7 @@ github.com/mistifyio/go-zfs
# github.com/mitchellh/hashstructure/v2 v2.0.2
## explicit; go 1.14
github.com/mitchellh/hashstructure/v2
# github.com/moby/buildkit v0.11.4
# github.com/moby/buildkit v0.11.5
## explicit; go 1.18
github.com/moby/buildkit/api/services/control
github.com/moby/buildkit/api/types
@ -700,10 +700,12 @@ github.com/moby/buildkit/util/resolver
github.com/moby/buildkit/util/resolver/config
github.com/moby/buildkit/util/resolver/limited
github.com/moby/buildkit/util/resolver/retryhandler
github.com/moby/buildkit/util/rootless/mountopts
github.com/moby/buildkit/util/rootless/specconv
github.com/moby/buildkit/util/sshutil
github.com/moby/buildkit/util/stack
github.com/moby/buildkit/util/staticfs
github.com/moby/buildkit/util/strutil
github.com/moby/buildkit/util/suggest
github.com/moby/buildkit/util/system
github.com/moby/buildkit/util/throttle