diff --git a/vendor.conf b/vendor.conf index 12419d3b79..181f7ded94 100644 --- a/vendor.conf +++ b/vendor.conf @@ -33,7 +33,8 @@ github.com/imdario/mergo 1afb36080aec31e0d1528973ebe6 golang.org/x/sync cd5d95a43a6e21273425c7ae415d3df9ea832eeb # buildkit -github.com/moby/buildkit c014937225cba29cfb1d5161fd134316c0e9bdaa # v0.8.3-31-gc0149372 +github.com/armon/circbuf 5111143e8da2e98b4ea6a8f32b9065ea1821c191 +github.com/moby/buildkit eeb7b65ab7d651770a5ec52a06ea7c96eb97a249 # v0.8.4-0.20221020190723-eeb7b65ab7d6 github.com/tonistiigi/fsutil 0834f99b7b85462efb69b4f571a4fa3ca7da5ac9 github.com/tonistiigi/units 6950e57a87eaf136bbe44ef2ec8e75b9e3569de2 github.com/grpc-ecosystem/grpc-opentracing 8e809c8a86450a29b90dcc9efbf062d0fe6d9746 diff --git a/vendor/github.com/armon/circbuf/LICENSE b/vendor/github.com/armon/circbuf/LICENSE new file mode 100755 index 0000000000..106569e542 --- /dev/null +++ b/vendor/github.com/armon/circbuf/LICENSE @@ -0,0 +1,20 @@ +The MIT License (MIT) + +Copyright (c) 2013 Armon Dadgar + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/vendor/github.com/armon/circbuf/README.md b/vendor/github.com/armon/circbuf/README.md new file mode 100644 index 0000000000..f2e356b8d7 --- /dev/null +++ b/vendor/github.com/armon/circbuf/README.md @@ -0,0 +1,28 @@ +circbuf +======= + +This repository provides the `circbuf` package. This provides a `Buffer` object +which is a circular (or ring) buffer. It has a fixed size, but can be written +to infinitely. Only the last `size` bytes are ever retained. The buffer implements +the `io.Writer` interface. + +Documentation +============= + +Full documentation can be found on [Godoc](http://godoc.org/github.com/armon/circbuf) + +Usage +===== + +The `circbuf` package is very easy to use: + +```go +buf, _ := NewBuffer(6) +buf.Write([]byte("hello world")) + +if string(buf.Bytes()) != " world" { + panic("should only have last 6 bytes!") +} + +``` + diff --git a/vendor/github.com/armon/circbuf/circbuf.go b/vendor/github.com/armon/circbuf/circbuf.go new file mode 100644 index 0000000000..de3cb94a39 --- /dev/null +++ b/vendor/github.com/armon/circbuf/circbuf.go @@ -0,0 +1,92 @@ +package circbuf + +import ( + "fmt" +) + +// Buffer implements a circular buffer. It is a fixed size, +// and new writes overwrite older data, such that for a buffer +// of size N, for any amount of writes, only the last N bytes +// are retained. +type Buffer struct { + data []byte + size int64 + writeCursor int64 + written int64 +} + +// NewBuffer creates a new buffer of a given size. The size +// must be greater than 0. +func NewBuffer(size int64) (*Buffer, error) { + if size <= 0 { + return nil, fmt.Errorf("Size must be positive") + } + + b := &Buffer{ + size: size, + data: make([]byte, size), + } + return b, nil +} + +// Write writes up to len(buf) bytes to the internal ring, +// overriding older data if necessary. +func (b *Buffer) Write(buf []byte) (int, error) { + // Account for total bytes written + n := len(buf) + b.written += int64(n) + + // If the buffer is larger than ours, then we only care + // about the last size bytes anyways + if int64(n) > b.size { + buf = buf[int64(n)-b.size:] + } + + // Copy in place + remain := b.size - b.writeCursor + copy(b.data[b.writeCursor:], buf) + if int64(len(buf)) > remain { + copy(b.data, buf[remain:]) + } + + // Update location of the cursor + b.writeCursor = ((b.writeCursor + int64(len(buf))) % b.size) + return n, nil +} + +// Size returns the size of the buffer +func (b *Buffer) Size() int64 { + return b.size +} + +// TotalWritten provides the total number of bytes written +func (b *Buffer) TotalWritten() int64 { + return b.written +} + +// Bytes provides a slice of the bytes written. This +// slice should not be written to. +func (b *Buffer) Bytes() []byte { + switch { + case b.written >= b.size && b.writeCursor == 0: + return b.data + case b.written > b.size: + out := make([]byte, b.size) + copy(out, b.data[b.writeCursor:]) + copy(out[b.size-b.writeCursor:], b.data[:b.writeCursor]) + return out + default: + return b.data[:b.writeCursor] + } +} + +// Reset resets the buffer so it has no content. +func (b *Buffer) Reset() { + b.writeCursor = 0 + b.written = 0 +} + +// String returns the contents of the buffer as a string +func (b *Buffer) String() string { + return string(b.Bytes()) +} diff --git a/vendor/github.com/armon/circbuf/go.mod b/vendor/github.com/armon/circbuf/go.mod new file mode 100644 index 0000000000..23362b19bc --- /dev/null +++ b/vendor/github.com/armon/circbuf/go.mod @@ -0,0 +1 @@ +module github.com/armon/circbuf diff --git a/vendor/github.com/moby/buildkit/go.mod b/vendor/github.com/moby/buildkit/go.mod index cf4c029732..39aff7ac32 100644 --- a/vendor/github.com/moby/buildkit/go.mod +++ b/vendor/github.com/moby/buildkit/go.mod @@ -7,6 +7,7 @@ require ( github.com/BurntSushi/toml v0.3.1 github.com/Microsoft/go-winio v0.4.15 github.com/Microsoft/hcsshim v0.8.10 + github.com/armon/circbuf v0.0.0-20190214190532-5111143e8da2 github.com/codahale/hdrhistogram v0.0.0-20160425231609-f8ad88b59a58 // indirect github.com/containerd/console v1.0.1 github.com/containerd/containerd v1.4.1-0.20201117152358-0edc412565dc // the actual version is replaced in replace() diff --git a/vendor/github.com/moby/buildkit/solver/llbsolver/ops/exec.go b/vendor/github.com/moby/buildkit/solver/llbsolver/ops/exec.go index bbf3d171c5..fe707f7060 100644 --- a/vendor/github.com/moby/buildkit/solver/llbsolver/ops/exec.go +++ b/vendor/github.com/moby/buildkit/solver/llbsolver/ops/exec.go @@ -327,9 +327,14 @@ func (e *execOp) Exec(ctx context.Context, g session.Group, inputs []solver.Resu } meta.Env = addDefaultEnvvar(meta.Env, "PATH", utilsystem.DefaultPathEnv(currentOS)) - stdout, stderr := logs.NewLogStreams(ctx, os.Getenv("BUILDKIT_DEBUG_EXEC_OUTPUT") == "1") + stdout, stderr, flush := logs.NewLogStreams(ctx, os.Getenv("BUILDKIT_DEBUG_EXEC_OUTPUT") == "1") defer stdout.Close() defer stderr.Close() + defer func() { + if err != nil { + flush() + } + }() execErr := e.exec.Run(ctx, "", p.Root, p.Mounts, executor.ProcessInfo{ Meta: meta, diff --git a/vendor/github.com/moby/buildkit/source/git/gitsource.go b/vendor/github.com/moby/buildkit/source/git/gitsource.go index 9245008aa8..15e0d56a21 100644 --- a/vendor/github.com/moby/buildkit/source/git/gitsource.go +++ b/vendor/github.com/moby/buildkit/source/git/gitsource.go @@ -593,11 +593,16 @@ func getGitSSHCommand(knownHosts string) string { return gitSSHCommand } -func git(ctx context.Context, dir, sshAuthSock, knownHosts string, args ...string) (*bytes.Buffer, error) { +func git(ctx context.Context, dir, sshAuthSock, knownHosts string, args ...string) (_ *bytes.Buffer, err error) { for { - stdout, stderr := logs.NewLogStreams(ctx, false) + stdout, stderr, flush := logs.NewLogStreams(ctx, false) defer stdout.Close() defer stderr.Close() + defer func() { + if err != nil { + flush() + } + }() args = append([]string{"-c", "protocol.file.allow=user"}, args...) // Block sneaky repositories from using repos from the filesystem as submodules. cmd := exec.Command("git", args...) cmd.Dir = dir // some commands like submodule require this diff --git a/vendor/github.com/moby/buildkit/util/progress/logs/logs.go b/vendor/github.com/moby/buildkit/util/progress/logs/logs.go index 43408b8a8c..bfecdda6b3 100644 --- a/vendor/github.com/moby/buildkit/util/progress/logs/logs.go +++ b/vendor/github.com/moby/buildkit/util/progress/logs/logs.go @@ -10,6 +10,7 @@ import ( "sync" "time" + "github.com/armon/circbuf" "github.com/moby/buildkit/client" "github.com/moby/buildkit/identity" "github.com/moby/buildkit/util/progress" @@ -17,8 +18,8 @@ import ( "github.com/tonistiigi/units" ) -var defaultMaxLogSize = 1024 * 1024 -var defaultMaxLogSpeed = 100 * 1024 // per second +var defaultMaxLogSize = 2 * 1024 * 1024 +var defaultMaxLogSpeed = 200 * 1024 // per second const ( stdout = 1 @@ -27,11 +28,16 @@ const ( var configCheckOnce sync.Once -func NewLogStreams(ctx context.Context, printOutput bool) (io.WriteCloser, io.WriteCloser) { - return newStreamWriter(ctx, stdout, printOutput), newStreamWriter(ctx, stderr, printOutput) +func NewLogStreams(ctx context.Context, printOutput bool) (io.WriteCloser, io.WriteCloser, func()) { + stdout := newStreamWriter(ctx, stdout, printOutput) + stderr := newStreamWriter(ctx, stderr, printOutput) + return stdout, stderr, func() { + stdout.flushBuffer() + stderr.flushBuffer() + } } -func newStreamWriter(ctx context.Context, stream int, printOutput bool) io.WriteCloser { +func newStreamWriter(ctx context.Context, stream int, printOutput bool) *streamWriter { pw, _, _ := progress.FromContext(ctx) return &streamWriter{ pw: pw, @@ -49,6 +55,7 @@ type streamWriter struct { size int clipping bool clipReasonSpeed bool + buf *circbuf.Buffer } func (sw *streamWriter) checkLimit(n int) int { @@ -96,7 +103,19 @@ func (sw *streamWriter) clipLimitMessage() string { func (sw *streamWriter) Write(dt []byte) (int, error) { oldSize := len(dt) - dt = append([]byte{}, dt[:sw.checkLimit(len(dt))]...) + limit := sw.checkLimit(len(dt)) + if sw.buf == nil && limit < len(dt) { + var err error + sw.buf, err = circbuf.NewBuffer(256 * 1024) + if err != nil { + return 0, err + } + } + if sw.buf != nil { + sw.buf.Write(dt) + } + + dt = append([]byte{}, dt[:limit]...) if sw.clipping && oldSize == len(dt) { sw.clipping = false @@ -106,25 +125,42 @@ func (sw *streamWriter) Write(dt []byte) (int, error) { sw.clipping = true } - if len(dt) != 0 { - sw.pw.Write(identity.NewID(), client.VertexLog{ - Stream: sw.stream, - Data: dt, - }) - if sw.printOutput { - switch sw.stream { - case 1: - return os.Stdout.Write(dt) - case 2: - return os.Stderr.Write(dt) - default: - return 0, errors.Errorf("invalid stream %d", sw.stream) - } - } + _, err := sw.write(dt) + if err != nil { + return 0, err } return oldSize, nil } +func (sw *streamWriter) write(dt []byte) (int, error) { + if len(dt) == 0 { + return 0, nil + } + sw.pw.Write(identity.NewID(), client.VertexLog{ + Stream: sw.stream, + Data: dt, + }) + if sw.printOutput { + switch sw.stream { + case 1: + return os.Stdout.Write(dt) + case 2: + return os.Stderr.Write(dt) + default: + return 0, errors.Errorf("invalid stream %d", sw.stream) + } + } + return len(dt), nil +} + +func (sw *streamWriter) flushBuffer() { + if sw.buf == nil { + return + } + _, _ = sw.write(sw.buf.Bytes()) + sw.buf = nil +} + func (sw *streamWriter) Close() error { return sw.pw.Close() }