beenull/moby
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Merge pull request #43294 from corhere/logfile-follow-without-fsnotify

Browse source 
LogFile follow without filenotify
This commit is contained in:
Sebastiaan van Stijn 2022-05-19 22:50:23 +02:00 committed by GitHub
commit 5996b32fe4
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
43 changed files with 1639 additions and 3855 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
daemon/logger/adapter_test.go
View file

@ -174,7 +174,7 @@ func TestAdapterReadLogs(t *testing.T) {
t.Fatal("timeout waiting for message channel to close")
}
lw.ProducerGone()
lw.ConsumerGone()
lw = lr.ReadLogs(ReadConfig{Follow: true})
for _, x := range testMsg {

14
daemon/logger/journald/journald.go
View file

@ -8,7 +8,6 @@ package journald // import "github.com/docker/docker/daemon/logger/journald"
import (
"fmt"
"strconv"
"sync"
"unicode"
"github.com/coreos/go-systemd/v22/journal"
@ -19,9 +18,9 @@ import (
const name = "journald"
type journald struct {
mu sync.Mutex //nolint:structcheck,unused
vars map[string]string // additional variables and values to send to the journal along with the log message
readers map[*logger.LogWatcher]struct{}
vars map[string]string // additional variables and values to send to the journal along with the log message
closed chan struct{}
}
func init() {
@ -81,7 +80,7 @@ func New(info logger.Info) (logger.Logger, error) {
for k, v := range extraAttrs {
vars[k] = v
}
return &journald{vars: vars, readers: make(map[*logger.LogWatcher]struct{})}, nil
return &journald{vars: vars, closed: make(chan struct{})}, nil
}
// We don't actually accept any options, but we have to supply a callback for
@ -128,3 +127,8 @@ func (s *journald) Log(msg *logger.Message) error {
func (s *journald) Name() string {
return name
}
func (s *journald) Close() error {
close(s.closed)
return nil
}

25
daemon/logger/journald/read.go
View file

@ -116,16 +116,6 @@ import (
"github.com/sirupsen/logrus"
)
func (s *journald) Close() error {
s.mu.Lock()
for r := range s.readers {
r.ProducerGone()
delete(s.readers, r)
}
s.mu.Unlock()
return nil
}
// CErr converts error code returned from a sd_journal_* function
// (which returns -errno) to a string
func CErr(ret C.int) string {
@ -233,9 +223,7 @@ drain:
}
func (s *journald) followJournal(logWatcher *logger.LogWatcher, j *C.sd_journal, cursor *C.char, untilUnixMicro uint64) *C.char {
s.mu.Lock()
s.readers[logWatcher] = struct{}{}
s.mu.Unlock()
defer close(logWatcher.Msg)
waitTimeout := C.uint64_t(250000) // 0.25s
@ -243,12 +231,12 @@ func (s *journald) followJournal(logWatcher *logger.LogWatcher, j *C.sd_journal,
status := C.sd_journal_wait(j, waitTimeout)
if status < 0 {
logWatcher.Err <- errors.New("error waiting for journal: " + CErr(status))
goto cleanup
break
}
select {
case <-logWatcher.WatchConsumerGone():
goto cleanup // won't be able to write anything anymore
case <-logWatcher.WatchProducerGone():
break // won't be able to write anything anymore
case <-s.closed:
// container is gone, drain journal
default:
// container is still alive
@ -264,11 +252,6 @@ func (s *journald) followJournal(logWatcher *logger.LogWatcher, j *C.sd_journal,
}
}
cleanup:
s.mu.Lock()
delete(s.readers, logWatcher)
s.mu.Unlock()
close(logWatcher.Msg)
return cursor
}

8
daemon/logger/journald/read_unsupported.go
View file

@ -1,8 +0,0 @@
//go:build !linux || !cgo || static_build || !journald
// +build !linux !cgo static_build !journald
package journald // import "github.com/docker/docker/daemon/logger/journald"
func (s *journald) Close() error {
return nil
}

28
daemon/logger/jsonfilelog/jsonfilelog.go
View file

@ -8,7 +8,6 @@ import (
"encoding/json"
"fmt"
"strconv"
"sync"
"github.com/docker/docker/daemon/logger"
"github.com/docker/docker/daemon/logger/jsonfilelog/jsonlog"
@ -22,11 +21,8 @@ const Name = "json-file"
// JSONFileLogger is Logger implementation for default Docker logging.
type JSONFileLogger struct {
mu sync.Mutex
closed bool
writer *loggerutils.LogFile
readers map[*logger.LogWatcher]struct{} // stores the active log followers
tag string // tag values requested by the user to log
writer *loggerutils.LogFile
tag string // tag values requested by the user to log
}
func init() {
@ -115,18 +111,14 @@ func New(info logger.Info) (logger.Logger, error) {
}
return &JSONFileLogger{
writer: writer,
readers: make(map[*logger.LogWatcher]struct{}),
tag: tag,
writer: writer,
tag: tag,
}, nil
}
// Log converts logger.Message to jsonlog.JSONLog and serializes it to file.
func (l *JSONFileLogger) Log(msg *logger.Message) error {
l.mu.Lock()
err := l.writer.WriteLogEntry(msg)
l.mu.Unlock()
return err
return l.writer.WriteLogEntry(msg)
}
func marshalMessage(msg *logger.Message, extra json.RawMessage, buf *bytes.Buffer) error {
@ -169,15 +161,7 @@ func ValidateLogOpt(cfg map[string]string) error {
// Close closes underlying file and signals all the readers
// that the logs producer is gone.
func (l *JSONFileLogger) Close() error {
l.mu.Lock()
l.closed = true
err := l.writer.Close()
for r := range l.readers {
r.ProducerGone()
delete(l.readers, r)
}
l.mu.Unlock()
return err
return l.writer.Close()
}
// Name returns name of this logger.

98
daemon/logger/jsonfilelog/read.go
View file

@ -10,32 +10,12 @@ import (
"github.com/docker/docker/daemon/logger/jsonfilelog/jsonlog"
"github.com/docker/docker/daemon/logger/loggerutils"
"github.com/docker/docker/pkg/tailfile"
"github.com/sirupsen/logrus"
)
const maxJSONDecodeRetry = 20000
// ReadLogs implements the logger's LogReader interface for the logs
// created by this driver.
func (l *JSONFileLogger) ReadLogs(config logger.ReadConfig) *logger.LogWatcher {
logWatcher := logger.NewLogWatcher()
go l.readLogs(logWatcher, config)
return logWatcher
}
func (l *JSONFileLogger) readLogs(watcher *logger.LogWatcher, config logger.ReadConfig) {
defer close(watcher.Msg)
l.mu.Lock()
l.readers[watcher] = struct{}{}
l.mu.Unlock()
l.writer.ReadLogs(config, watcher)
l.mu.Lock()
delete(l.readers, watcher)
l.mu.Unlock()
return l.writer.ReadLogs(config)
}
func decodeLogLine(dec *json.Decoder, l *jsonlog.JSONLog) (*logger.Message, error) {
@ -61,10 +41,9 @@ func decodeLogLine(dec *json.Decoder, l *jsonlog.JSONLog) (*logger.Message, erro
}
type decoder struct {
rdr io.Reader
dec *json.Decoder
jl *jsonlog.JSONLog
maxRetry int
rdr io.Reader
dec *json.Decoder
jl *jsonlog.JSONLog
}
func (d *decoder) Reset(rdr io.Reader) {
@ -88,74 +67,7 @@ func (d *decoder) Decode() (msg *logger.Message, err error) {
if d.jl == nil {
d.jl = &jsonlog.JSONLog{}
}
if d.maxRetry == 0 {
// We aren't using maxJSONDecodeRetry directly so we can give a custom value for testing.
d.maxRetry = maxJSONDecodeRetry
}
for retries := 0; retries < d.maxRetry; retries++ {
msg, err = decodeLogLine(d.dec, d.jl)
if err == nil || err == io.EOF {
break
}
logrus.WithError(err).WithField("retries", retries).Warn("got error while decoding json")
// try again, could be due to a an incomplete json object as we read
if _, ok := err.(*json.SyntaxError); ok {
d.dec = json.NewDecoder(d.rdr)
continue
}
// io.ErrUnexpectedEOF is returned from json.Decoder when there is
// remaining data in the parser's buffer while an io.EOF occurs.
// If the json logger writes a partial json log entry to the disk
// while at the same time the decoder tries to decode it, the race condition happens.
if err == io.ErrUnexpectedEOF {
d.rdr = combineReaders(d.dec.Buffered(), d.rdr)
d.dec = json.NewDecoder(d.rdr)
continue
}
}
return msg, err
}
func combineReaders(pre, rdr io.Reader) io.Reader {
return &combinedReader{pre: pre, rdr: rdr}
}
// combinedReader is a reader which is like `io.MultiReader` where except it does not cache a full EOF.
// Once `io.MultiReader` returns EOF, it is always EOF.
//
// For this usecase we have an underlying reader which is a file which may reach EOF but have more data written to it later.
// As such, io.MultiReader does not work for us.
type combinedReader struct {
pre io.Reader
rdr io.Reader
}
func (r *combinedReader) Read(p []byte) (int, error) {
var read int
if r.pre != nil {
n, err := r.pre.Read(p)
if err != nil {
if err != io.EOF {
return n, err
}
r.pre = nil
}
read = n
}
if read < len(p) {
n, err := r.rdr.Read(p[read:])
if n > 0 {
read += n
}
if err != nil {
return read, err
}
}
return read, nil
return decodeLogLine(d.dec, d.jl)
}
// decodeFunc is used to create a decoder for the log file reader

206
daemon/logger/jsonfilelog/read_test.go
View file

@ -1,24 +1,28 @@
package jsonfilelog // import "github.com/docker/docker/daemon/logger/jsonfilelog"
import (
"bufio"
"bytes"
"encoding/json"
"fmt"
"io"
"os"
"path/filepath"
"strconv"
"testing"
"text/tabwriter"
"time"
"github.com/docker/docker/daemon/logger"
"github.com/docker/docker/daemon/logger/loggertest"
"gotest.tools/v3/assert"
"gotest.tools/v3/fs"
)
func BenchmarkJSONFileLoggerReadLogs(b *testing.B) {
tmp := fs.NewDir(b, "bench-jsonfilelog")
defer tmp.Remove()
tmp := b.TempDir()
jsonlogger, err := New(logger.Info{
ContainerID: "a7317399f3f857173c6179d44823594f8294678dea9999662e5c625b5a1c7657",
LogPath: tmp.Join("container.log"),
LogPath: filepath.Join(tmp, "container.log"),
Config: map[string]string{
"labels": "first,second",
},
@ -30,36 +34,43 @@ func BenchmarkJSONFileLoggerReadLogs(b *testing.B) {
assert.NilError(b, err)
defer jsonlogger.Close()
msg := &logger.Message{
Line: []byte("Line that thinks that it is log line from docker\n"),
Source: "stderr",
Timestamp: time.Now().UTC(),
const line = "Line that thinks that it is log line from docker\n"
ts := time.Date(2007, 1, 2, 3, 4, 5, 6, time.UTC)
msg := func() *logger.Message {
m := logger.NewMessage()
m.Line = append(m.Line, line...)
m.Source = "stderr"
m.Timestamp = ts
return m
}
buf := bytes.NewBuffer(nil)
assert.NilError(b, marshalMessage(msg, nil, buf))
var buf bytes.Buffer
assert.NilError(b, marshalMessage(msg(), nil, &buf))
b.SetBytes(int64(buf.Len()))
b.ResetTimer()
chError := make(chan error, b.N+1)
chError := make(chan error)
go func() {
for i := 0; i < b.N; i++ {
chError <- jsonlogger.Log(msg)
if err := jsonlogger.Log(msg()); err != nil {
chError <- err
}
}
if err := jsonlogger.Close(); err != nil {
chError <- err
}
chError <- jsonlogger.Close()
}()
lw := jsonlogger.(*JSONFileLogger).ReadLogs(logger.ReadConfig{Follow: true})
for {
select {
case <-lw.Msg:
case <-lw.WatchProducerGone():
return
case err := <-chError:
if err != nil {
b.Fatal(err)
case _, ok := <-lw.Msg:
if !ok {
return
}
case err := <-chError:
b.Fatal(err)
}
}
}
@ -95,60 +106,111 @@ func TestEncodeDecode(t *testing.T) {
assert.Assert(t, err == io.EOF)
}
func TestUnexpectedEOF(t *testing.T) {
func TestReadLogs(t *testing.T) {
t.Parallel()
r := loggertest.Reader{
Factory: func(t *testing.T, info logger.Info) func(*testing.T) logger.Logger {
dir := t.TempDir()
info.LogPath = filepath.Join(dir, info.ContainerID+".log")
return func(t *testing.T) logger.Logger {
l, err := New(info)
assert.NilError(t, err)
return l
}
},
}
t.Run("Tail", r.TestTail)
t.Run("Follow", r.TestFollow)
}
func TestTailLogsWithRotation(t *testing.T) {
t.Parallel()
compress := func(cmprs bool) {
t.Run(fmt.Sprintf("compress=%v", cmprs), func(t *testing.T) {
t.Parallel()
(&loggertest.Reader{
Factory: func(t *testing.T, info logger.Info) func(*testing.T) logger.Logger {
info.Config = map[string]string{
"compress": strconv.FormatBool(cmprs),
"max-size": "1b",
"max-file": "10",
}
dir := t.TempDir()
t.Cleanup(func() {
t.Logf("%s:\n%s", t.Name(), dirStringer{dir})
})
info.LogPath = filepath.Join(dir, info.ContainerID+".log")
return func(t *testing.T) logger.Logger {
l, err := New(info)
assert.NilError(t, err)
return l
}
},
}).TestTail(t)
})
}
compress(true)
compress(false)
}
func TestFollowLogsWithRotation(t *testing.T) {
t.Parallel()
compress := func(cmprs bool) {
t.Run(fmt.Sprintf("compress=%v", cmprs), func(t *testing.T) {
t.Parallel()
(&loggertest.Reader{
Factory: func(t *testing.T, info logger.Info) func(*testing.T) logger.Logger {
// The log follower can fall behind and drop logs if there are too many
// rotations in a short time. If that was to happen, loggertest would fail the
// test. Configure the logger so that there will be only one rotation with the
// set of logs that loggertest writes.
info.Config = map[string]string{
"compress": strconv.FormatBool(cmprs),
"max-size": "4096b",
"max-file": "3",
}
dir := t.TempDir()
t.Cleanup(func() {
t.Logf("%s:\n%s", t.Name(), dirStringer{dir})
})
info.LogPath = filepath.Join(dir, info.ContainerID+".log")
return func(t *testing.T) logger.Logger {
l, err := New(info)
assert.NilError(t, err)
return l
}
},
}).TestFollow(t)
})
}
compress(true)
compress(false)
}
type dirStringer struct {
d string
}
func (d dirStringer) String() string {
ls, err := os.ReadDir(d.d)
if err != nil {
return ""
}
buf := bytes.NewBuffer(nil)
msg1 := &logger.Message{Timestamp: time.Now(), Line: []byte("hello1")}
msg2 := &logger.Message{Timestamp: time.Now(), Line: []byte("hello2")}
tw := tabwriter.NewWriter(buf, 1, 8, 1, '\t', 0)
buf.WriteString("\n")
err := marshalMessage(msg1, json.RawMessage{}, buf)
assert.NilError(t, err)
err = marshalMessage(msg2, json.RawMessage{}, buf)
assert.NilError(t, err)
btw := bufio.NewWriter(tw)
r := &readerWithErr{
err: io.EOF,
after: buf.Len() / 4,
r: buf,
for _, entry := range ls {
fi, err := entry.Info()
if err != nil {
return ""
}
btw.WriteString(fmt.Sprintf("%s\t%s\t%dB\t%s\n", fi.Name(), fi.Mode(), fi.Size(), fi.ModTime()))
}
dec := &decoder{rdr: r, maxRetry: 1}
_, err = dec.Decode()
assert.Error(t, err, io.ErrUnexpectedEOF.Error())
// again just to check
_, err = dec.Decode()
assert.Error(t, err, io.ErrUnexpectedEOF.Error())
// reset the error
// from here all reads should succeed until we get EOF on the underlying reader
r.err = nil
msg, err := dec.Decode()
assert.NilError(t, err)
assert.Equal(t, string(msg1.Line)+"\n", string(msg.Line))
msg, err = dec.Decode()
assert.NilError(t, err)
assert.Equal(t, string(msg2.Line)+"\n", string(msg.Line))
_, err = dec.Decode()
assert.Error(t, err, io.EOF.Error())
}
type readerWithErr struct {
err error
after int
r io.Reader
read int
}
func (r *readerWithErr) Read(p []byte) (int, error) {
if r.err != nil && r.read > r.after {
return 0, r.err
}
n, err := r.r.Read(p[:1])
if n > 0 {
r.read += n
}
return n, err
btw.Flush()
tw.Flush()
return buf.String()
}

20
daemon/logger/local/local.go
View file

@ -4,7 +4,6 @@ import (
"encoding/binary"
"io"
"strconv"
"sync"
"time"
"github.com/docker/docker/api/types/backend"
@ -56,10 +55,7 @@ func init() {
}
type driver struct {
mu sync.Mutex
closed bool
logfile *loggerutils.LogFile
readers map[*logger.LogWatcher]struct{} // stores the active log followers
}
// New creates a new local logger
@ -145,7 +141,6 @@ func newDriver(logPath string, cfg *CreateConfig) (logger.Logger, error) {
}
return &driver{
logfile: lf,
readers: make(map[*logger.LogWatcher]struct{}),
}, nil
}
@ -154,22 +149,11 @@ func (d *driver) Name() string {
}
func (d *driver) Log(msg *logger.Message) error {
d.mu.Lock()
err := d.logfile.WriteLogEntry(msg)
d.mu.Unlock()
return err
return d.logfile.WriteLogEntry(msg)
}
func (d *driver) Close() error {
d.mu.Lock()
d.closed = true
err := d.logfile.Close()
for r := range d.readers {
r.ProducerGone()
delete(d.readers, r)
}
d.mu.Unlock()
return err
return d.logfile.Close()
}
func messageToProto(msg *logger.Message, proto *logdriver.LogEntry, partial *logdriver.PartialLogEntryMetadata) {

98
daemon/logger/local/local_test.go
View file

@ -2,19 +2,18 @@ package local
import (
"bytes"
"context"
"encoding/binary"
"fmt"
"io"
"os"
"path/filepath"
"strings"
"testing"
"time"
"github.com/docker/docker/api/types/backend"
"github.com/docker/docker/api/types/plugins/logdriver"
"github.com/docker/docker/daemon/logger"
"github.com/docker/docker/daemon/logger/loggertest"
protoio "github.com/gogo/protobuf/io"
"gotest.tools/v3/assert"
is "gotest.tools/v3/assert/cmp"
@ -80,92 +79,19 @@ func TestWriteLog(t *testing.T) {
}
func TestReadLog(t *testing.T) {
t.Parallel()
dir, err := os.MkdirTemp("", t.Name())
assert.NilError(t, err)
defer os.RemoveAll(dir)
logPath := filepath.Join(dir, "test.log")
l, err := New(logger.Info{LogPath: logPath})
assert.NilError(t, err)
defer l.Close()
m1 := logger.Message{Source: "stdout", Timestamp: time.Now().Add(-1 * 30 * time.Minute), Line: []byte("a message")}
m2 := logger.Message{Source: "stdout", Timestamp: time.Now().Add(-1 * 20 * time.Minute), Line: []byte("another message"), PLogMetaData: &backend.PartialLogMetaData{Ordinal: 1, Last: true}}
longMessage := []byte("a really long message " + strings.Repeat("a", initialBufSize*2))
m3 := logger.Message{Source: "stderr", Timestamp: time.Now().Add(-1 * 10 * time.Minute), Line: longMessage}
m4 := logger.Message{Source: "stderr", Timestamp: time.Now().Add(-1 * 10 * time.Minute), Line: []byte("just one more message")}
// copy the log message because the underlying log writer resets the log message and returns it to a buffer pool
err = l.Log(copyLogMessage(&m1))
assert.NilError(t, err)
err = l.Log(copyLogMessage(&m2))
assert.NilError(t, err)
err = l.Log(copyLogMessage(&m3))
assert.NilError(t, err)
err = l.Log(copyLogMessage(&m4))
assert.NilError(t, err)
lr := l.(logger.LogReader)
testMessage := func(t *testing.T, lw *logger.LogWatcher, m *logger.Message) {
t.Helper()
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
select {
case <-ctx.Done():
assert.Assert(t, ctx.Err())
case err := <-lw.Err:
assert.NilError(t, err)
case msg, open := <-lw.Msg:
if !open {
select {
case err := <-lw.Err:
assert.NilError(t, err)
default:
assert.Assert(t, m == nil)
return
}
r := loggertest.Reader{
Factory: func(t *testing.T, info logger.Info) func(*testing.T) logger.Logger {
dir := t.TempDir()
info.LogPath = filepath.Join(dir, info.ContainerID+".log")
return func(t *testing.T) logger.Logger {
l, err := New(info)
assert.NilError(t, err)
return l
}
assert.Assert(t, m != nil)
if m.PLogMetaData == nil {
// a `\n` is appended on read to make this work with the existing API's when the message is not a partial.
// make sure it's the last entry in the line, and then truncate it for the deep equal below.
assert.Check(t, msg.Line[len(msg.Line)-1] == '\n')
msg.Line = msg.Line[:len(msg.Line)-1]
}
assert.Check(t, is.DeepEqual(m, msg), fmt.Sprintf("\n%+v\n%+v", m, msg))
}
},
}
t.Run("tail exact", func(t *testing.T) {
lw := lr.ReadLogs(logger.ReadConfig{Tail: 4})
testMessage(t, lw, &m1)
testMessage(t, lw, &m2)
testMessage(t, lw, &m3)
testMessage(t, lw, &m4)
testMessage(t, lw, nil) // no more messages
})
t.Run("tail less than available", func(t *testing.T) {
lw := lr.ReadLogs(logger.ReadConfig{Tail: 2})
testMessage(t, lw, &m3)
testMessage(t, lw, &m4)
testMessage(t, lw, nil) // no more messages
})
t.Run("tail more than available", func(t *testing.T) {
lw := lr.ReadLogs(logger.ReadConfig{Tail: 100})
testMessage(t, lw, &m1)
testMessage(t, lw, &m2)
testMessage(t, lw, &m3)
testMessage(t, lw, &m4)
testMessage(t, lw, nil) // no more messages
})
t.Run("Tail", r.TestTail)
t.Run("Follow", r.TestFollow)
}
func BenchmarkLogWrite(b *testing.B) {

21
daemon/logger/local/read.go
View file

@ -19,24 +19,7 @@ import (
const maxMsgLen int = 1e6 // 1MB.
func (d *driver) ReadLogs(config logger.ReadConfig) *logger.LogWatcher {
logWatcher := logger.NewLogWatcher()
go d.readLogs(logWatcher, config)
return logWatcher
}
func (d *driver) readLogs(watcher *logger.LogWatcher, config logger.ReadConfig) {
defer close(watcher.Msg)
d.mu.Lock()
d.readers[watcher] = struct{}{}
d.mu.Unlock()
d.logfile.ReadLogs(config, watcher)
d.mu.Lock()
delete(d.readers, watcher)
d.mu.Unlock()
return d.logfile.ReadLogs(config)
}
func getTailReader(ctx context.Context, r loggerutils.SizeReaderAt, req int) (io.Reader, int, error) {
@ -219,7 +202,7 @@ func (d *decoder) decodeLogEntry() (*logger.Message, error) {
}
msg := protoToMessage(d.proto)
if msg.PLogMetaData == nil {
if msg.PLogMetaData == nil || msg.PLogMetaData.Last {
msg.Line = append(msg.Line, '\n')
}

18
daemon/logger/logger.go
View file

@ -97,8 +97,6 @@ type LogWatcher struct {
Msg chan *Message
// For sending error messages that occur while reading logs.
Err chan error
producerOnce sync.Once
producerGone chan struct{}
consumerOnce sync.Once
consumerGone chan struct{}
}
@ -108,26 +106,10 @@ func NewLogWatcher() *LogWatcher {
return &LogWatcher{
Msg: make(chan *Message, logWatcherBufferSize),
Err: make(chan error, 1),
producerGone: make(chan struct{}),
consumerGone: make(chan struct{}),
}
}
// ProducerGone notifies the underlying log reader that
// the logs producer (a container) is gone.
func (w *LogWatcher) ProducerGone() {
// only close if not already closed
w.producerOnce.Do(func() {
close(w.producerGone)
})
}
// WatchProducerGone returns a channel receiver that receives notification
// once the logs producer (a container) is gone.
func (w *LogWatcher) WatchProducerGone() <-chan struct{} {
return w.producerGone
}
// ConsumerGone notifies that the logs consumer is gone.
func (w *LogWatcher) ConsumerGone() {
// only close if not already closed

461
daemon/logger/loggertest/logreader.go Normal file
View file

@ -0,0 +1,461 @@
package loggertest // import "github.com/docker/docker/daemon/logger/loggertest"
import (
"runtime"
"strings"
"testing"
"time"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
"gotest.tools/v3/assert"
"github.com/docker/docker/api/types/backend"
"github.com/docker/docker/daemon/logger"
)
// Reader tests that a logger.LogReader implementation behaves as it should.
type Reader struct {
// Factory returns a function which constructs loggers for the container
// specified in info. Each call to the returned function must yield a
// distinct logger instance which can read back logs written by earlier
// instances.
Factory func(*testing.T, logger.Info) func(*testing.T) logger.Logger
}
var compareLog cmp.Options = []cmp.Option{
// The json-log driver does not round-trip PLogMetaData and API users do
// not expect it.
cmpopts.IgnoreFields(logger.Message{}, "PLogMetaData"),
cmp.Transformer("string", func(b []byte) string { return string(b) }),
}
// TestTail tests the behavior of the LogReader's tail implementation.
func (tr Reader) TestTail(t *testing.T) {
t.Run("Live", func(t *testing.T) { tr.testTail(t, true) })
t.Run("LiveEmpty", func(t *testing.T) { tr.testTailEmptyLogs(t, true) })
t.Run("Stopped", func(t *testing.T) { tr.testTail(t, false) })
t.Run("StoppedEmpty", func(t *testing.T) { tr.testTailEmptyLogs(t, false) })
}
func makeTestMessages() []*logger.Message {
return []*logger.Message{
{Source: "stdout", Timestamp: time.Now().Add(-1 * 30 * time.Minute), Line: []byte("a message")},
{Source: "stdout", Timestamp: time.Now().Add(-1 * 20 * time.Minute), Line: []byte("another message"), PLogMetaData: &backend.PartialLogMetaData{ID: "aaaaaaaa", Ordinal: 1, Last: true}},
{Source: "stderr", Timestamp: time.Now().Add(-1 * 15 * time.Minute), Line: []byte("to be..."), PLogMetaData: &backend.PartialLogMetaData{ID: "bbbbbbbb", Ordinal: 1}},
{Source: "stderr", Timestamp: time.Now().Add(-1 * 15 * time.Minute), Line: []byte("continued"), PLogMetaData: &backend.PartialLogMetaData{ID: "bbbbbbbb", Ordinal: 2, Last: true}},
{Source: "stderr", Timestamp: time.Now().Add(-1 * 10 * time.Minute), Line: []byte("a really long message " + strings.Repeat("a", 4096))},
{Source: "stderr", Timestamp: time.Now().Add(-1 * 10 * time.Minute), Line: []byte("just one more message")},
}
}
func (tr Reader) testTail(t *testing.T, live bool) {
t.Parallel()
factory := tr.Factory(t, logger.Info{
ContainerID: "tailtest0000",
ContainerName: "logtail",
})
l := factory(t)
if live {
defer func() { assert.NilError(t, l.Close()) }()
}
mm := makeTestMessages()
expected := logMessages(t, l, mm)
if !live {
// Simulate reading from a stopped container.
assert.NilError(t, l.Close())
l = factory(t)
defer func() { assert.NilError(t, l.Close()) }()
}
lr := l.(logger.LogReader)
t.Run("Exact", func(t *testing.T) {
t.Parallel()
lw := lr.ReadLogs(logger.ReadConfig{Tail: len(mm)})
defer lw.ConsumerGone()
assert.DeepEqual(t, readAll(t, lw), expected, compareLog)
})
t.Run("LessThanAvailable", func(t *testing.T) {
t.Parallel()
lw := lr.ReadLogs(logger.ReadConfig{Tail: 2})
defer lw.ConsumerGone()
assert.DeepEqual(t, readAll(t, lw), expected[len(mm)-2:], compareLog)
})
t.Run("MoreThanAvailable", func(t *testing.T) {
t.Parallel()
lw := lr.ReadLogs(logger.ReadConfig{Tail: 100})
defer lw.ConsumerGone()
assert.DeepEqual(t, readAll(t, lw), expected, compareLog)
})
t.Run("All", func(t *testing.T) {
t.Parallel()
lw := lr.ReadLogs(logger.ReadConfig{Tail: -1})
defer lw.ConsumerGone()
assert.DeepEqual(t, readAll(t, lw), expected, compareLog)
})
t.Run("Since", func(t *testing.T) {
t.Parallel()
lw := lr.ReadLogs(logger.ReadConfig{Tail: -1, Since: mm[1].Timestamp.Truncate(time.Millisecond)})
defer lw.ConsumerGone()
assert.DeepEqual(t, readAll(t, lw), expected[1:], compareLog)
})
t.Run("MoreThanSince", func(t *testing.T) {
t.Parallel()
lw := lr.ReadLogs(logger.ReadConfig{Tail: len(mm), Since: mm[1].Timestamp.Truncate(time.Millisecond)})
defer lw.ConsumerGone()
assert.DeepEqual(t, readAll(t, lw), expected[1:], compareLog)
})
t.Run("LessThanSince", func(t *testing.T) {
t.Parallel()
lw := lr.ReadLogs(logger.ReadConfig{Tail: len(mm) - 2, Since: mm[1].Timestamp.Truncate(time.Millisecond)})
defer lw.ConsumerGone()
assert.DeepEqual(t, readAll(t, lw), expected[2:], compareLog)
})
t.Run("Until", func(t *testing.T) {
t.Parallel()
lw := lr.ReadLogs(logger.ReadConfig{Tail: -1, Until: mm[2].Timestamp.Add(-time.Millisecond)})
defer lw.ConsumerGone()
assert.DeepEqual(t, readAll(t, lw), expected[:2], compareLog)
})
t.Run("SinceAndUntil", func(t *testing.T) {
t.Parallel()
lw := lr.ReadLogs(logger.ReadConfig{Tail: -1, Since: mm[1].Timestamp.Truncate(time.Millisecond), Until: mm[1].Timestamp.Add(time.Millisecond)})
defer lw.ConsumerGone()
assert.DeepEqual(t, readAll(t, lw), expected[1:2], compareLog)
})
}
func (tr Reader) testTailEmptyLogs(t *testing.T, live bool) {
t.Parallel()
factory := tr.Factory(t, logger.Info{
ContainerID: "tailemptytest",
ContainerName: "logtail",
})
l := factory(t)
if !live {
assert.NilError(t, l.Close())
l = factory(t)
}
defer func() { assert.NilError(t, l.Close()) }()
for _, tt := range []struct {
name string
cfg logger.ReadConfig
}{
{name: "Zero", cfg: logger.ReadConfig{}},
{name: "All", cfg: logger.ReadConfig{Tail: -1}},
{name: "Tail", cfg: logger.ReadConfig{Tail: 42}},
{name: "Since", cfg: logger.ReadConfig{Since: time.Unix(1, 0)}},
{name: "Until", cfg: logger.ReadConfig{Until: time.Date(2100, time.January, 1, 1, 1, 1, 0, time.UTC)}},
{name: "SinceAndUntil", cfg: logger.ReadConfig{Since: time.Unix(1, 0), Until: time.Date(2100, time.January, 1, 1, 1, 1, 0, time.UTC)}},
} {
tt := tt
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
lw := l.(logger.LogReader).ReadLogs(logger.ReadConfig{})
defer lw.ConsumerGone()
assert.DeepEqual(t, readAll(t, lw), ([]*logger.Message)(nil), cmpopts.EquateEmpty())
})
}
}
// TestFollow tests the LogReader's follow implementation.
//
// The LogReader is expected to be able to follow an arbitrary number of
// messages at a high rate with no dropped messages.
func (tr Reader) TestFollow(t *testing.T) {
// Reader sends all logs and closes after logger is closed
// - Starting from empty log (like run)
t.Run("FromEmptyLog", func(t *testing.T) {
t.Parallel()
l := tr.Factory(t, logger.Info{
ContainerID: "followstart0",
ContainerName: "logloglog",
})(t)
lw := l.(logger.LogReader).ReadLogs(logger.ReadConfig{Tail: -1, Follow: true})
defer lw.ConsumerGone()
doneReading := make(chan struct{})
var logs []*logger.Message
go func() {
defer close(doneReading)
logs = readAll(t, lw)
}()
mm := makeTestMessages()
expected := logMessages(t, l, mm)
assert.NilError(t, l.Close())
<-doneReading
assert.DeepEqual(t, logs, expected, compareLog)
})
t.Run("AttachMidStream", func(t *testing.T) {
t.Parallel()
l := tr.Factory(t, logger.Info{
ContainerID: "followmiddle",
ContainerName: "logloglog",
})(t)
mm := makeTestMessages()
expected := logMessages(t, l, mm[0:1])
lw := l.(logger.LogReader).ReadLogs(logger.ReadConfig{Tail: -1, Follow: true})
defer lw.ConsumerGone()
doneReading := make(chan struct{})
var logs []*logger.Message
go func() {
defer close(doneReading)
logs = readAll(t, lw)
}()
expected = append(expected, logMessages(t, l, mm[1:])...)
assert.NilError(t, l.Close())
<-doneReading
assert.DeepEqual(t, logs, expected, compareLog)
})
t.Run("Since", func(t *testing.T) {
t.Parallel()
l := tr.Factory(t, logger.Info{
ContainerID: "followsince0",
ContainerName: "logloglog",
})(t)
mm := makeTestMessages()
lw := l.(logger.LogReader).ReadLogs(logger.ReadConfig{Tail: -1, Follow: true, Since: mm[2].Timestamp.Truncate(time.Millisecond)})
defer lw.ConsumerGone()
doneReading := make(chan struct{})
var logs []*logger.Message
go func() {
defer close(doneReading)
logs = readAll(t, lw)
}()
expected := logMessages(t, l, mm)[2:]
assert.NilError(t, l.Close())
<-doneReading
assert.DeepEqual(t, logs, expected, compareLog)
})
t.Run("Until", func(t *testing.T) {
t.Parallel()
l := tr.Factory(t, logger.Info{
ContainerID: "followuntil0",
ContainerName: "logloglog",
})(t)
mm := makeTestMessages()
lw := l.(logger.LogReader).ReadLogs(logger.ReadConfig{Tail: -1, Follow: true, Until: mm[2].Timestamp.Add(-time.Millisecond)})
defer lw.ConsumerGone()
doneReading := make(chan struct{})
var logs []*logger.Message
go func() {
defer close(doneReading)
logs = readAll(t, lw)
}()
expected := logMessages(t, l, mm)[:2]
defer assert.NilError(t, l.Close()) // Reading should end before the logger is closed.
<-doneReading
assert.DeepEqual(t, logs, expected, compareLog)
})
t.Run("SinceAndUntil", func(t *testing.T) {
t.Parallel()
l := tr.Factory(t, logger.Info{
ContainerID: "followbounded",
ContainerName: "logloglog",
})(t)
mm := makeTestMessages()
lw := l.(logger.LogReader).ReadLogs(logger.ReadConfig{Tail: -1, Follow: true, Since: mm[1].Timestamp.Add(-time.Millisecond), Until: mm[2].Timestamp.Add(-time.Millisecond)})
defer lw.ConsumerGone()
doneReading := make(chan struct{})
var logs []*logger.Message
go func() {
defer close(doneReading)
logs = readAll(t, lw)
}()
expected := logMessages(t, l, mm)[1:2]
defer assert.NilError(t, l.Close()) // Reading should end before the logger is closed.
<-doneReading
assert.DeepEqual(t, logs, expected, compareLog)
})
t.Run("Tail=0", func(t *testing.T) {
t.Parallel()
l := tr.Factory(t, logger.Info{
ContainerID: "followtail00",
ContainerName: "logloglog",
})(t)
mm := makeTestMessages()
logMessages(t, l, mm[0:2])
lw := l.(logger.LogReader).ReadLogs(logger.ReadConfig{Tail: 0, Follow: true})
defer lw.ConsumerGone()
doneReading := make(chan struct{})
var logs []*logger.Message
go func() {
defer close(doneReading)
logs = readAll(t, lw)
}()
expected := logMessages(t, l, mm[2:])
assert.NilError(t, l.Close())
<-doneReading
assert.DeepEqual(t, logs, expected, compareLog)
})
t.Run("Tail>0", func(t *testing.T) {
t.Parallel()
l := tr.Factory(t, logger.Info{
ContainerID: "followtail00",
ContainerName: "logloglog",
})(t)
mm := makeTestMessages()
expected := logMessages(t, l, mm[0:2])[1:]
lw := l.(logger.LogReader).ReadLogs(logger.ReadConfig{Tail: 1, Follow: true})
defer lw.ConsumerGone()
doneReading := make(chan struct{})
var logs []*logger.Message
go func() {
defer close(doneReading)
logs = readAll(t, lw)
}()
expected = append(expected, logMessages(t, l, mm[2:])...)
assert.NilError(t, l.Close())
<-doneReading
assert.DeepEqual(t, logs, expected, compareLog)
})
t.Run("MultipleStarts", func(t *testing.T) {
t.Parallel()
factory := tr.Factory(t, logger.Info{
ContainerID: "startrestart",
ContainerName: "startmeup",
})
mm := makeTestMessages()
l := factory(t)
expected := logMessages(t, l, mm[:3])
assert.NilError(t, l.Close())
l = factory(t)
lw := l.(logger.LogReader).ReadLogs(logger.ReadConfig{Tail: -1, Follow: true})
defer lw.ConsumerGone()
doneReading := make(chan struct{})
var logs []*logger.Message
go func() {
defer close(doneReading)
logs = readAll(t, lw)
}()
expected = append(expected, logMessages(t, l, mm[3:])...)
assert.NilError(t, l.Close())
<-doneReading
assert.DeepEqual(t, logs, expected, compareLog)
})
}
// logMessages logs messages to l and returns a slice of messages as would be
// expected to be read back. The message values are not modified and the
// returned slice of messages are deep-copied.
func logMessages(t *testing.T, l logger.Logger, messages []*logger.Message) []*logger.Message {
t.Helper()
var expected []*logger.Message
for _, m := range messages {
// Copy the log message because the underlying log writer resets
// the log message and returns it to a buffer pool.
assert.NilError(t, l.Log(copyLogMessage(m)))
runtime.Gosched()
// Copy the log message again so as not to mutate the input.
expect := copyLogMessage(m)
// Existing API consumers expect a newline to be appended to
// messages other than nonterminal partials as that matches the
// existing behavior of the json-file log driver.
if m.PLogMetaData == nil || m.PLogMetaData.Last {
expect.Line = append(expect.Line, '\n')
}
expected = append(expected, expect)
}
return expected
}
func copyLogMessage(src *logger.Message) *logger.Message {
dst := logger.NewMessage()
dst.Source = src.Source
dst.Timestamp = src.Timestamp
dst.Attrs = src.Attrs
dst.Err = src.Err
dst.Line = append(dst.Line, src.Line...)
if src.PLogMetaData != nil {
lmd := *src.PLogMetaData
dst.PLogMetaData = &lmd
}
return dst
}
func readMessage(t *testing.T, lw *logger.LogWatcher) *logger.Message {
t.Helper()
timeout := time.NewTimer(5 * time.Second)
defer timeout.Stop()
select {
case <-timeout.C:
t.Error("timed out waiting for message")
return nil
case err, open := <-lw.Err:
t.Errorf("unexpected receive on lw.Err: err=%v, open=%v", err, open)
return nil
case msg, open := <-lw.Msg:
if !open {
select {
case err, open := <-lw.Err:
t.Errorf("unexpected receive on lw.Err with closed lw.Msg: err=%v, open=%v", err, open)
return nil
default:
}
}
if msg != nil {
t.Logf("loggertest: ReadMessage [%v %v] %s", msg.Source, msg.Timestamp, msg.Line)
}
return msg
}
}
func readAll(t *testing.T, lw *logger.LogWatcher) []*logger.Message {
t.Helper()
var msgs []*logger.Message
for {
m := readMessage(t, lw)
if m == nil {
return msgs
}
msgs = append(msgs, m)
}
}

4
daemon/logger/loggerutils/file_unix.go
View file

@ -12,3 +12,7 @@ func openFile(name string, flag int, perm os.FileMode) (*os.File, error) {
func open(name string) (*os.File, error) {
return os.Open(name)
}
func unlink(name string) error {
return os.Remove(name)
}

26
daemon/logger/loggerutils/file_windows.go
View file

@ -2,6 +2,7 @@ package loggerutils
import (
"os"
"path/filepath"
"syscall"
"unsafe"
)
@ -224,3 +225,28 @@ func volumeName(path string) (v string) {
}
return ""
}
func unlink(name string) error {
// Rename the file before deleting it so that the original name is freed
// up to be reused, even while there are still open FILE_SHARE_DELETE
// file handles. Emulate POSIX unlink() semantics, essentially.
name, err := filepath.Abs(name)
if err != nil {
return err
}
dir, fname := filepath.Split(name)
f, err := os.CreateTemp(dir, fname+".*.deleted")
if err != nil {
return err
}
tmpname := f.Name()
if err := f.Close(); err != nil {
return err
}
err = os.Rename(name, tmpname)
rmErr := os.Remove(tmpname)
if err != nil {
return err
}
return rmErr
}

42
daemon/logger/loggerutils/file_windows_test.go
View file

@ -1,6 +1,7 @@
package loggerutils
import (
"io"
"os"
"path/filepath"
"testing"
@ -9,10 +10,7 @@ import (
)
func TestOpenFileDelete(t *testing.T) {
tmpDir, err := os.MkdirTemp("", t.Name())
assert.NilError(t, err)
defer os.RemoveAll(tmpDir)
tmpDir := t.TempDir()
f, err := openFile(filepath.Join(tmpDir, "test.txt"), os.O_CREATE|os.O_RDWR, 644)
assert.NilError(t, err)
defer f.Close()
@ -21,13 +19,41 @@ func TestOpenFileDelete(t *testing.T) {
}
func TestOpenFileRename(t *testing.T) {
tmpDir, err := os.MkdirTemp("", t.Name())
assert.NilError(t, err)
defer os.RemoveAll(tmpDir)
tmpDir := t.TempDir()
f, err := openFile(filepath.Join(tmpDir, "test.txt"), os.O_CREATE|os.O_RDWR, 0644)
assert.NilError(t, err)
defer f.Close()
assert.NilError(t, os.Rename(f.Name(), f.Name()+"renamed"))
}
func TestUnlinkOpenFile(t *testing.T) {
tmpDir := t.TempDir()
name := filepath.Join(tmpDir, "test.txt")
f, err := openFile(name, os.O_CREATE|os.O_RDWR, 0644)
assert.NilError(t, err)
defer func() { assert.NilError(t, f.Close()) }()
_, err = io.WriteString(f, "first")
assert.NilError(t, err)
assert.NilError(t, unlink(name))
f2, err := openFile(name, os.O_CREATE|os.O_RDWR, 0644)
assert.NilError(t, err)
defer func() { assert.NilError(t, f2.Close()) }()
_, err = io.WriteString(f2, "second")
assert.NilError(t, err)
_, err = f.Seek(0, io.SeekStart)
assert.NilError(t, err)
fdata, err := io.ReadAll(f)
assert.NilError(t, err)
assert.Check(t, "first" == string(fdata))
_, err = f2.Seek(0, io.SeekStart)
assert.NilError(t, err)
f2data, err := io.ReadAll(f2)
assert.NilError(t, err)
assert.Check(t, "second" == string(f2data))
}

310
daemon/logger/loggerutils/follow.go
View file

@ -1,211 +1,165 @@
package loggerutils // import "github.com/docker/docker/daemon/logger/loggerutils"
import (
"fmt"
"io"
"os"
"time"
"github.com/docker/docker/daemon/logger"
"github.com/docker/docker/pkg/filenotify"
"github.com/fsnotify/fsnotify"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
)
var errRetry = errors.New("retry")
var errDone = errors.New("done")
type follow struct {
file *os.File
dec Decoder
fileWatcher filenotify.FileWatcher
logWatcher *logger.LogWatcher
notifyRotate, notifyEvict chan interface{}
oldSize int64
retries int
LogFile *LogFile
Watcher *logger.LogWatcher
Decoder Decoder
Since, Until time.Time
log *logrus.Entry
c chan logPos
}
func (fl *follow) handleRotate() error {
name := fl.file.Name()
// Do follows the log file as it is written, starting from f at read.
func (fl *follow) Do(f *os.File, read logPos) {
fl.log = logrus.WithFields(logrus.Fields{
"module": "logger",
"file": f.Name(),
})
// Optimization: allocate the write-notifications channel only once and
// reuse it for multiple invocations of nextPos().
fl.c = make(chan logPos, 1)
fl.file.Close()
fl.fileWatcher.Remove(name)
// retry when the file doesn't exist
var err error
for retries := 0; retries <= 5; retries++ {
f, err := open(name)
if err == nil || !os.IsNotExist(err) {
fl.file = f
break
defer func() {
if err := f.Close(); err != nil && !errors.Is(err, os.ErrClosed) {
fl.log.WithError(err).Warn("error closing current log file")
}
}
if err != nil {
return err
}
if err := fl.fileWatcher.Add(name); err != nil {
return err
}
fl.dec.Reset(fl.file)
return nil
}
func (fl *follow) handleMustClose(evictErr error) {
fl.file.Close()
fl.dec.Close()
fl.logWatcher.Err <- errors.Wrap(evictErr, "log reader evicted due to errors")
logrus.WithField("file", fl.file.Name()).Error("Log reader notified that it must re-open log file, some log data may not be streamed to the client.")
}
func (fl *follow) waitRead() error {
select {
case e := <-fl.notifyEvict:
if e != nil {
err := e.(error)
fl.handleMustClose(err)
}
return errDone
case e := <-fl.fileWatcher.Events():
switch e.Op {
case fsnotify.Write:
fl.dec.Reset(fl.file)
return nil
case fsnotify.Rename, fsnotify.Remove:
select {
case <-fl.notifyRotate:
case <-fl.logWatcher.WatchProducerGone():
return errDone
case <-fl.logWatcher.WatchConsumerGone():
return errDone
}
if err := fl.handleRotate(); err != nil {
return err
}
return nil
}
return errRetry
case err := <-fl.fileWatcher.Errors():
logrus.Debugf("logger got error watching file: %v", err)
// Something happened, let's try and stay alive and create a new watcher
if fl.retries <= 5 {
fl.fileWatcher.Close()
fl.fileWatcher, err = watchFile(fl.file.Name())
if err != nil {
return err
}
fl.retries++
return errRetry
}
return err
case <-fl.logWatcher.WatchProducerGone():
return errDone
case <-fl.logWatcher.WatchConsumerGone():
return errDone
}
}
func (fl *follow) handleDecodeErr(err error) error {
if !errors.Is(err, io.EOF) {
return err
}
// Handle special case (#39235): max-file=1 and file was truncated
st, stErr := fl.file.Stat()
if stErr == nil {
size := st.Size()
defer func() { fl.oldSize = size }()
if size < fl.oldSize { // truncated
fl.file.Seek(0, 0)
fl.dec.Reset(fl.file)
return nil
}
} else {
logrus.WithError(stErr).Warn("logger: stat error")
}
}()
for {
err := fl.waitRead()
if err == nil {
break
}
if err == errRetry {
continue
}
return err
}
return nil
}
func (fl *follow) mainLoop(since, until time.Time) {
for {
select {
case err := <-fl.notifyEvict:
if err != nil {
fl.handleMustClose(err.(error))
}
wrote, ok := fl.nextPos(read)
if !ok {
return
default:
}
msg, err := fl.dec.Decode()
if err != nil {
if err := fl.handleDecodeErr(err); err != nil {
if err == errDone {
return
}
// we got an unrecoverable error, so return
fl.logWatcher.Err <- err
if wrote.rotation != read.rotation {
// Flush the current file before moving on to the next.
if _, err := f.Seek(read.size, io.SeekStart); err != nil {
fl.Watcher.Err <- err
return
}
// ready to try again
continue
if fl.decode(f) {
return
}
// Open the new file, which has the same name as the old
// file thanks to file rotation. Make no mistake: they
// are different files, with distinct identities.
// Atomically capture the wrote position to make
// absolutely sure that the position corresponds to the
// file we have opened; more rotations could have
// occurred since we previously received it.
if err := f.Close(); err != nil {
fl.log.WithError(err).Warn("error closing rotated log file")
}
var err error
func() {
fl.LogFile.fsopMu.RLock()
st := <-fl.LogFile.read
defer func() {
fl.LogFile.read <- st
fl.LogFile.fsopMu.RUnlock()
}()
f, err = open(f.Name())
wrote = st.pos
}()
// We tried to open the file inside a critical section
// so we shouldn't have been racing the rotation of the
// file. Any error, even fs.ErrNotFound, is exceptional.
if err != nil {
fl.Watcher.Err <- fmt.Errorf("logger: error opening log file for follow after rotation: %w", err)
return
}
if nrot := wrote.rotation - read.rotation; nrot > 1 {
fl.log.WithField("missed-rotations", nrot).
Warn("file rotations were missed while following logs; some log messages have been skipped over")
}
// Set up our read position to start from the top of the file.
read.size = 0
}
fl.retries = 0 // reset retries since we've succeeded
if !since.IsZero() && msg.Timestamp.Before(since) {
if fl.decode(io.NewSectionReader(f, read.size, wrote.size-read.size)) {
return
}
read = wrote
}
}
// nextPos waits until the write position of the LogFile being followed has
// advanced from current and returns the new position.
func (fl *follow) nextPos(current logPos) (next logPos, ok bool) {
var st logReadState
select {
case <-fl.Watcher.WatchConsumerGone():
return current, false
case st = <-fl.LogFile.read:
}
// Have any any logs been written since we last checked?
if st.pos == current { // Nope.
// Add ourself to the notify list.
st.wait = append(st.wait, fl.c)
} else { // Yes.
// "Notify" ourself immediately.
fl.c <- st.pos
}
fl.LogFile.read <- st
select {
case <-fl.LogFile.closed: // No more logs will be written.
select { // Have we followed to the end?
case next = <-fl.c: // No: received a new position.
default: // Yes.
return current, false
}
case <-fl.Watcher.WatchConsumerGone():
return current, false
case next = <-fl.c:
}
return next, true
}
// decode decodes log messages from r and sends messages with timestamps between
// Since and Until to the log watcher.
//
// The return value, done, signals whether following should end due to a
// condition encountered during decode.
func (fl *follow) decode(r io.Reader) (done bool) {
fl.Decoder.Reset(r)
for {
msg, err := fl.Decoder.Decode()
if err != nil {
if errors.Is(err, io.EOF) {
return false
}
fl.Watcher.Err <- err
return true
}
if !fl.Since.IsZero() && msg.Timestamp.Before(fl.Since) {
continue
}
if !until.IsZero() && msg.Timestamp.After(until) {
return
if !fl.Until.IsZero() && msg.Timestamp.After(fl.Until) {
return true
}
// send the message, unless the consumer is gone
select {
case e := <-fl.notifyEvict:
if e != nil {
err := e.(error)
logrus.WithError(err).Debug("Reader evicted while sending log message")
fl.logWatcher.Err <- err
}
return
case fl.logWatcher.Msg <- msg:
case <-fl.logWatcher.WatchConsumerGone():
return
case fl.Watcher.Msg <- msg:
case <-fl.Watcher.WatchConsumerGone():
return true
}
}
}
func followLogs(f *os.File, logWatcher *logger.LogWatcher, notifyRotate, notifyEvict chan interface{}, dec Decoder, since, until time.Time) {
dec.Reset(f)
name := f.Name()
fileWatcher, err := watchFile(name)
if err != nil {
logWatcher.Err <- err
return
}
defer func() {
f.Close()
dec.Close()
fileWatcher.Close()
}()
fl := &follow{
file: f,
oldSize: -1,
logWatcher: logWatcher,
fileWatcher: fileWatcher,
notifyRotate: notifyRotate,
notifyEvict: notifyEvict,
dec: dec,
}
fl.mainLoop(since, until)
}

37
daemon/logger/loggerutils/follow_test.go
View file

@ -1,37 +0,0 @@
package loggerutils // import "github.com/docker/docker/daemon/logger/loggerutils"
import (
"io"
"os"
"testing"
"gotest.tools/v3/assert"
)
func TestHandleDecoderErr(t *testing.T) {
f, err := os.CreateTemp("", t.Name())
assert.NilError(t, err)
defer os.Remove(f.Name())
_, err = f.Write([]byte("hello"))
assert.NilError(t, err)
pos, err := f.Seek(0, io.SeekCurrent)
assert.NilError(t, err)
assert.Assert(t, pos != 0)
dec := &testDecoder{}
// Simulate "turncate" case, where the file was bigger before.
fl := &follow{file: f, dec: dec, oldSize: 100}
err = fl.handleDecodeErr(io.EOF)
assert.NilError(t, err)
// handleDecodeErr seeks to zero.
pos, err = f.Seek(0, io.SeekCurrent)
assert.NilError(t, err)
assert.Equal(t, int64(0), pos)
// Reset is called.
assert.Equal(t, 1, dec.resetCount)
}

597
daemon/logger/loggerutils/logfile.go
View file

@ -6,91 +6,83 @@ import (
"encoding/json"
"fmt"
"io"
"io/fs"
"math"
"os"
"runtime"
"strconv"
"strings"
"sync"
"time"
"github.com/docker/docker/daemon/logger"
"github.com/docker/docker/pkg/filenotify"
"github.com/docker/docker/pkg/pools"
"github.com/docker/docker/pkg/pubsub"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
)
const tmpLogfileSuffix = ".tmp"
// rotateFileMetadata is a metadata of the gzip header of the compressed log file
type rotateFileMetadata struct {
LastTime time.Time `json:"lastTime,omitempty"`
}
// refCounter is a counter of logfile being referenced
type refCounter struct {
mu sync.Mutex
counter map[string]int
}
// Reference increase the reference counter for specified logfile
func (rc *refCounter) GetReference(fileName string, openRefFile func(fileName string, exists bool) (*os.File, error)) (*os.File, error) {
rc.mu.Lock()
defer rc.mu.Unlock()
var (
file *os.File
err error
)
_, ok := rc.counter[fileName]
file, err = openRefFile(fileName, ok)
if err != nil {
return nil, err
}
if ok {
rc.counter[fileName]++
} else if file != nil {
rc.counter[file.Name()] = 1
}
return file, nil
}
// Dereference reduce the reference counter for specified logfile
func (rc *refCounter) Dereference(fileName string) error {
rc.mu.Lock()
defer rc.mu.Unlock()
rc.counter[fileName]--
if rc.counter[fileName] <= 0 {
delete(rc.counter, fileName)
err := os.Remove(fileName)
if err != nil && !os.IsNotExist(err) {
return err
}
}
return nil
}
// LogFile is Logger implementation for default Docker logging.
type LogFile struct {
mu sync.RWMutex // protects the logfile access
f *os.File // store for closing
closed bool
rotateMu sync.Mutex // blocks the next rotation until the current rotation is completed
capacity int64 // maximum size of each file
currentSize int64 // current size of the latest file
maxFiles int // maximum number of files
compress bool // whether old versions of log files are compressed
lastTimestamp time.Time // timestamp of the last log
filesRefCounter refCounter // keep reference-counted of decompressed files
notifyReaders *pubsub.Publisher
marshal logger.MarshalFunc
createDecoder MakeDecoderFn
getTailReader GetTailReaderFunc
perms os.FileMode
mu sync.Mutex // protects the logfile access
closed chan struct{}
rotateMu sync.Mutex // blocks the next rotation until the current rotation is completed
// Lock out readers while performing a non-atomic sequence of filesystem
// operations (RLock: open, Lock: rename, delete).
//
// fsopMu should be locked for writing only while holding rotateMu.
fsopMu sync.RWMutex
// Logger configuration
capacity int64 // maximum size of each file
maxFiles int // maximum number of files
compress bool // whether old versions of log files are compressed
perms os.FileMode
// Log file codec
marshal logger.MarshalFunc
createDecoder MakeDecoderFn
getTailReader GetTailReaderFunc
// Log reader state in a 1-buffered channel.
//
// Share memory by communicating: receive to acquire, send to release.
// The state struct is passed around by value so that use-after-send
// bugs cannot escalate to data races.
//
// A method which receives the state value takes ownership of it. The
// owner is responsible for either passing ownership along or sending
// the state back to the channel. By convention, the semantics of
// passing along ownership is expressed with function argument types.
// Methods which take a pointer *logReadState argument borrow the state,
// analogous to functions which require a lock to be held when calling.
// The caller retains ownership. Calling a method which which takes a
// value logFileState argument gives ownership to the callee.
read chan logReadState
decompress *sharedTempFileConverter
pos logPos // Current log file write position.
f *os.File // Current log file for writing.
lastTimestamp time.Time // timestamp of the last log
}
type logPos struct {
// Size of the current file.
size int64
// File rotation sequence number (modulo 2**16).
rotation uint16
}
type logReadState struct {
// Current log file position.
pos logPos
// Wait list to be notified of the value of pos next time it changes.
wait []chan<- logPos
}
// MakeDecoderFn creates a decoder
@ -111,10 +103,16 @@ type Decoder interface {
// SizeReaderAt defines a ReaderAt that also reports its size.
// This is used for tailing log files.
type SizeReaderAt interface {
io.Reader
io.ReaderAt
Size() int64
}
type readAtCloser interface {
io.ReaderAt
io.Closer
}
// GetTailReaderFunc is used to truncate a reader to only read as much as is required
// in order to get the passed in number of log lines.
// It returns the sectioned reader, the number of lines that the section reader
@ -133,18 +131,28 @@ func NewLogFile(logPath string, capacity int64, maxFiles int, compress bool, mar
return nil, err
}
pos := logPos{
size: size,
// Force a wraparound on first rotation to shake out any
// modular-arithmetic bugs.
rotation: math.MaxUint16,
}
st := make(chan logReadState, 1)
st <- logReadState{pos: pos}
return &LogFile{
f: log,
capacity: capacity,
currentSize: size,
maxFiles: maxFiles,
compress: compress,
filesRefCounter: refCounter{counter: make(map[string]int)},
notifyReaders: pubsub.NewPublisher(0, 1),
marshal: marshaller,
createDecoder: decodeFunc,
perms: perms,
getTailReader: getTailReader,
f: log,
read: st,
pos: pos,
closed: make(chan struct{}),
capacity: capacity,
maxFiles: maxFiles,
compress: compress,
decompress: newSharedTempFileConverter(decompress),
marshal: marshaller,
createDecoder: decodeFunc,
perms: perms,
getTailReader: getTailReader,
}, nil
}
@ -160,35 +168,45 @@ func (w *LogFile) WriteLogEntry(msg *logger.Message) error {
logger.PutMessage(msg)
msg = nil // Turn use-after-put bugs into panics.
w.mu.Lock()
if w.closed {
w.mu.Unlock()
select {
case <-w.closed:
return errors.New("cannot write because the output file was closed")
default:
}
w.mu.Lock()
defer w.mu.Unlock()
if err := w.checkCapacityAndRotate(); err != nil {
w.mu.Unlock()
return errors.Wrap(err, "error rotating log file")
// Are we due for a rotation?
if w.capacity != -1 && w.pos.size >= w.capacity {
if err := w.rotate(); err != nil {
return errors.Wrap(err, "error rotating log file")
}
}
n, err := w.f.Write(b)
if err == nil {
w.currentSize += int64(n)
w.lastTimestamp = ts
if err != nil {
return errors.Wrap(err, "error writing log entry")
}
w.pos.size += int64(n)
w.lastTimestamp = ts
w.mu.Unlock()
return errors.Wrap(err, "error writing log entry")
// Notify any waiting readers that there is a new log entry to read.
st := <-w.read
defer func() { w.read <- st }()
st.pos = w.pos
for _, c := range st.wait {
c <- st.pos
}
// Optimization: retain the backing array to save a heap allocation next
// time a reader appends to the list.
if st.wait != nil {
st.wait = st.wait[:0]
}
return nil
}
func (w *LogFile) checkCapacityAndRotate() (retErr error) {
if w.capacity == -1 {
return nil
}
if w.currentSize < w.capacity {
return nil
}
func (w *LogFile) rotate() (retErr error) {
w.rotateMu.Lock()
noCompress := w.maxFiles <= 1 || !w.compress
defer func() {
@ -202,49 +220,61 @@ func (w *LogFile) checkCapacityAndRotate() (retErr error) {
fname := w.f.Name()
if err := w.f.Close(); err != nil {
// if there was an error during a prior rotate, the file could already be closed
if !errors.Is(err, os.ErrClosed) {
if !errors.Is(err, fs.ErrClosed) {
return errors.Wrap(err, "error closing file")
}
}
if err := rotate(fname, w.maxFiles, w.compress); err != nil {
logrus.WithError(err).Warn("Error rotating log file, log data may have been lost")
} else {
var renameErr error
for i := 0; i < 10; i++ {
if renameErr = os.Rename(fname, fname+".1"); renameErr != nil && !os.IsNotExist(renameErr) {
logrus.WithError(renameErr).WithField("file", fname).Debug("Error rotating current container log file, evicting readers and retrying")
w.notifyReaders.Publish(renameErr)
time.Sleep(100 * time.Millisecond)
continue
}
break
}
if renameErr != nil {
logrus.WithError(renameErr).Error("Error renaming current log file")
}
}
file, err := func() (*os.File, error) {
w.fsopMu.Lock()
defer w.fsopMu.Unlock()
file, err := openFile(fname, os.O_WRONLY|os.O_TRUNC|os.O_CREATE, w.perms)
if err := rotate(fname, w.maxFiles, w.compress); err != nil {
logrus.WithError(err).Warn("Error rotating log file, log data may have been lost")
} else {
// We may have readers working their way through the
// current log file so we can't truncate it. We need to
// start writing new logs to an empty file with the same
// name as the current one so we need to rotate the
// current file out of the way.
if w.maxFiles < 2 {
if err := unlink(fname); err != nil && !errors.Is(err, fs.ErrNotExist) {
logrus.WithError(err).Error("Error unlinking current log file")
}
} else {
if err := os.Rename(fname, fname+".1"); err != nil && !errors.Is(err, fs.ErrNotExist) {
logrus.WithError(err).Error("Error renaming current log file")
}
}
}
// Notwithstanding the above, open with the truncate flag anyway
// in case rotation didn't work out as planned.
return openFile(fname, os.O_WRONLY|os.O_TRUNC|os.O_CREATE, w.perms)
}()
if err != nil {
return err
}
w.f = file
w.currentSize = 0
w.notifyReaders.Publish(struct{}{})
w.pos = logPos{rotation: w.pos.rotation + 1}
if noCompress {
return nil
}
ts := w.lastTimestamp
go func() {
defer w.rotateMu.Unlock()
// No need to hold fsopMu as at no point will the filesystem be
// in a state which would cause problems for readers. Opening
// the uncompressed file is tried first, falling back to the
// compressed one. compressFile only deletes the uncompressed
// file once the compressed one is fully written out, so at no
// point during the compression process will a reader fail to
// open a complete copy of the file.
if err := compressFile(fname+".1", ts); err != nil {
logrus.WithError(err).Error("Error compressing log file after rotation")
}
w.rotateMu.Unlock()
}()
return nil
@ -261,16 +291,17 @@ func rotate(name string, maxFiles int, compress bool) error {
}
lastFile := fmt.Sprintf("%s.%d%s", name, maxFiles-1, extension)
err := os.Remove(lastFile)
if err != nil && !os.IsNotExist(err) {
err := unlink(lastFile)
if err != nil && !errors.Is(err, fs.ErrNotExist) {
return errors.Wrap(err, "error removing oldest log file")
}
for i := maxFiles - 1; i > 1; i-- {
toPath := name + "." + strconv.Itoa(i) + extension
fromPath := name + "." + strconv.Itoa(i-1) + extension
logrus.WithField("source", fromPath).WithField("target", toPath).Trace("Rotating log file")
if err := os.Rename(fromPath, toPath); err != nil && !os.IsNotExist(err) {
err := os.Rename(fromPath, toPath)
logrus.WithError(err).WithField("source", fromPath).WithField("target", toPath).Trace("Rotating log file")
if err != nil && !errors.Is(err, fs.ErrNotExist) {
return err
}
}
@ -281,7 +312,7 @@ func rotate(name string, maxFiles int, compress bool) error {
func compressFile(fileName string, lastTimestamp time.Time) (retErr error) {
file, err := open(fileName)
if err != nil {
if os.IsNotExist(err) {
if errors.Is(err, fs.ErrNotExist) {
logrus.WithField("file", fileName).WithError(err).Debug("Could not open log file to compress")
return nil
}
@ -290,8 +321,8 @@ func compressFile(fileName string, lastTimestamp time.Time) (retErr error) {
defer func() {
file.Close()
if retErr == nil {
err := os.Remove(fileName)
if err != nil && !os.IsNotExist(err) {
err := unlink(fileName)
if err != nil && !errors.Is(err, fs.ErrNotExist) {
retErr = errors.Wrap(err, "failed to remove source log file")
}
}
@ -304,7 +335,7 @@ func compressFile(fileName string, lastTimestamp time.Time) (retErr error) {
defer func() {
outFile.Close()
if retErr != nil {
if err := os.Remove(fileName + ".gz"); err != nil && !os.IsExist(err) {
if err := unlink(fileName + ".gz"); err != nil && !errors.Is(err, fs.ErrNotExist) {
logrus.WithError(err).Error("Error cleaning up after failed log compression")
}
}
@ -339,25 +370,49 @@ func (w *LogFile) MaxFiles() int {
func (w *LogFile) Close() error {
w.mu.Lock()
defer w.mu.Unlock()
if w.closed {
select {
case <-w.closed:
return nil
default:
}
if err := w.f.Close(); err != nil && !errors.Is(err, os.ErrClosed) {
if err := w.f.Close(); err != nil && !errors.Is(err, fs.ErrClosed) {
return err
}
w.closed = true
close(w.closed)
// Wait until any in-progress rotation is complete.
w.rotateMu.Lock()
w.rotateMu.Unlock() //nolint:staticcheck
return nil
}
// ReadLogs decodes entries from log files and sends them the passed in watcher
// ReadLogs decodes entries from log files.
//
// Note: Using the follow option can become inconsistent in cases with very frequent rotations and max log files is 1.
// TODO: Consider a different implementation which can effectively follow logs under frequent rotations.
func (w *LogFile) ReadLogs(config logger.ReadConfig, watcher *logger.LogWatcher) {
w.mu.RLock()
// It is the caller's responsibility to call ConsumerGone on the LogWatcher.
func (w *LogFile) ReadLogs(config logger.ReadConfig) *logger.LogWatcher {
watcher := logger.NewLogWatcher()
// Lock out filesystem operations so that we can capture the read
// position and atomically open the corresponding log file, without the
// file getting rotated out from under us.
w.fsopMu.RLock()
// Capture the read position synchronously to ensure that we start
// following from the last entry logged before ReadLogs was called,
// which is required for flake-free unit testing.
st := <-w.read
pos := st.pos
w.read <- st
go w.readLogsLocked(pos, config, watcher)
return watcher
}
// readLogsLocked is the bulk of the implementation of ReadLogs.
//
// w.fsopMu must be locked for reading when calling this method.
// w.fsopMu.RUnlock() is called before returning.
func (w *LogFile) readLogsLocked(currentPos logPos, config logger.ReadConfig, watcher *logger.LogWatcher) {
defer close(watcher.Msg)
currentFile, err := open(w.f.Name())
if err != nil {
w.mu.RUnlock()
watcher.Err <- err
return
}
@ -366,25 +421,13 @@ func (w *LogFile) ReadLogs(config logger.ReadConfig, watcher *logger.LogWatcher)
dec := w.createDecoder(nil)
defer dec.Close()
currentChunk, err := newSectionReader(currentFile)
if err != nil {
w.mu.RUnlock()
watcher.Err <- err
return
}
notifyEvict := w.notifyReaders.SubscribeTopicWithBuffer(func(i interface{}) bool {
_, ok := i.(error)
return ok
}, 1)
defer w.notifyReaders.Evict(notifyEvict)
currentChunk := io.NewSectionReader(currentFile, 0, currentPos.size)
if config.Tail != 0 {
// TODO(@cpuguy83): Instead of opening every file, only get the files which
// are needed to tail.
// This is especially costly when compression is enabled.
files, err := w.openRotatedFiles(config)
w.mu.RUnlock()
if err != nil {
watcher.Err <- err
return
@ -393,115 +436,123 @@ func (w *LogFile) ReadLogs(config logger.ReadConfig, watcher *logger.LogWatcher)
closeFiles := func() {
for _, f := range files {
f.Close()
fileName := f.Name()
if strings.HasSuffix(fileName, tmpLogfileSuffix) {
err := w.filesRefCounter.Dereference(fileName)
if err != nil {
logrus.WithError(err).WithField("file", fileName).Error("Failed to dereference the log file")
}
}
}
}
readers := make([]SizeReaderAt, 0, len(files)+1)
for _, f := range files {
stat, err := f.Stat()
if err != nil {
watcher.Err <- errors.Wrap(err, "error reading size of rotated file")
closeFiles()
return
switch ff := f.(type) {
case SizeReaderAt:
readers = append(readers, ff)
case interface{ Stat() (fs.FileInfo, error) }:
stat, err := ff.Stat()
if err != nil {
watcher.Err <- errors.Wrap(err, "error reading size of rotated file")
closeFiles()
return
}
readers = append(readers, io.NewSectionReader(f, 0, stat.Size()))
default:
panic(fmt.Errorf("rotated file value %#v (%[1]T) has neither Size() nor Stat() methods", f))
}
readers = append(readers, io.NewSectionReader(f, 0, stat.Size()))
}
if currentChunk.Size() > 0 {
readers = append(readers, currentChunk)
}
ok := tailFiles(readers, watcher, dec, w.getTailReader, config, notifyEvict)
ok := tailFiles(readers, watcher, dec, w.getTailReader, config)
closeFiles()
if !ok {
return
}
w.mu.RLock()
} else {
w.fsopMu.RUnlock()
}
if !config.Follow || w.closed {
w.mu.RUnlock()
if !config.Follow {
return
}
w.mu.RUnlock()
notifyRotate := w.notifyReaders.SubscribeTopic(func(i interface{}) bool {
_, ok := i.(struct{})
return ok
})
defer w.notifyReaders.Evict(notifyRotate)
followLogs(currentFile, watcher, notifyRotate, notifyEvict, dec, config.Since, config.Until)
(&follow{
LogFile: w,
Watcher: watcher,
Decoder: dec,
Since: config.Since,
Until: config.Until,
}).Do(currentFile, currentPos)
}
func (w *LogFile) openRotatedFiles(config logger.ReadConfig) (files []*os.File, err error) {
w.rotateMu.Lock()
defer w.rotateMu.Unlock()
// openRotatedFiles returns a slice of files open for reading, in order from
// oldest to newest, and calls w.fsopMu.RUnlock() before returning.
//
// This method must only be called with w.fsopMu locked for reading.
func (w *LogFile) openRotatedFiles(config logger.ReadConfig) (files []readAtCloser, err error) {
type rotatedFile struct {
f *os.File
compressed bool
}
var q []rotatedFile
defer func() {
if err == nil {
return
}
for _, f := range files {
f.Close()
if strings.HasSuffix(f.Name(), tmpLogfileSuffix) {
err := os.Remove(f.Name())
if err != nil && !os.IsNotExist(err) {
logrus.Warnf("Failed to remove logfile: %v", err)
}
if err != nil {
for _, qq := range q {
qq.f.Close()
}
for _, f := range files {
f.Close()
}
}
}()
for i := w.maxFiles; i > 1; i-- {
f, err := open(fmt.Sprintf("%s.%d", w.f.Name(), i-1))
if err != nil {
if !os.IsNotExist(err) {
return nil, errors.Wrap(err, "error opening rotated log file")
}
fileName := fmt.Sprintf("%s.%d.gz", w.f.Name(), i-1)
decompressedFileName := fileName + tmpLogfileSuffix
tmpFile, err := w.filesRefCounter.GetReference(decompressedFileName, func(refFileName string, exists bool) (*os.File, error) {
if exists {
return open(refFileName)
}
return decompressfile(fileName, refFileName, config.Since)
})
q, err = func() (q []rotatedFile, err error) {
defer w.fsopMu.RUnlock()
q = make([]rotatedFile, 0, w.maxFiles)
for i := w.maxFiles; i > 1; i-- {
var f rotatedFile
f.f, err = open(fmt.Sprintf("%s.%d", w.f.Name(), i-1))
if err != nil {
if !errors.Is(err, os.ErrNotExist) {
return nil, errors.Wrap(err, "error getting reference to decompressed log file")
if !errors.Is(err, fs.ErrNotExist) {
return nil, errors.Wrap(err, "error opening rotated log file")
}
f.compressed = true
f.f, err = open(fmt.Sprintf("%s.%d.gz", w.f.Name(), i-1))
if err != nil {
if !errors.Is(err, fs.ErrNotExist) {
return nil, errors.Wrap(err, "error opening file for decompression")
}
continue
}
continue
}
if tmpFile == nil {
// The log before `config.Since` does not need to read
break
}
files = append(files, tmpFile)
continue
q = append(q, f)
}
files = append(files, f)
return q, nil
}()
if err != nil {
return nil, err
}
for len(q) > 0 {
qq := q[0]
q = q[1:]
if qq.compressed {
defer qq.f.Close()
f, err := w.maybeDecompressFile(qq.f, config)
if err != nil {
return nil, err
}
if f != nil {
// The log before `config.Since` does not need to read
files = append(files, f)
}
} else {
files = append(files, qq.f)
}
}
return files, nil
}
func decompressfile(fileName, destFileName string, since time.Time) (*os.File, error) {
cf, err := open(fileName)
if err != nil {
return nil, errors.Wrap(err, "error opening file for decompression")
}
defer cf.Close()
func (w *LogFile) maybeDecompressFile(cf *os.File, config logger.ReadConfig) (readAtCloser, error) {
rc, err := gzip.NewReader(cf)
if err != nil {
return nil, errors.Wrap(err, "error making gzip reader for compressed log file")
@ -511,41 +562,29 @@ func decompressfile(fileName, destFileName string, since time.Time) (*os.File, e
// Extract the last log entry timestramp from the gzip header
extra := &rotateFileMetadata{}
err = json.Unmarshal(rc.Header.Extra, extra)
if err == nil && extra.LastTime.Before(since) {
if err == nil && !extra.LastTime.IsZero() && extra.LastTime.Before(config.Since) {
return nil, nil
}
rs, err := openFile(destFileName, os.O_CREATE|os.O_RDWR, 0640)
if err != nil {
return nil, errors.Wrap(err, "error creating file for copying decompressed log stream")
}
_, err = pools.Copy(rs, rc)
if err != nil {
rs.Close()
rErr := os.Remove(rs.Name())
if rErr != nil && !os.IsNotExist(rErr) {
logrus.Errorf("Failed to remove logfile: %v", rErr)
}
return nil, errors.Wrap(err, "error while copying decompressed log stream to file")
}
return rs, nil
tmpf, err := w.decompress.Do(cf)
return tmpf, errors.Wrap(err, "error decompressing log file")
}
func newSectionReader(f *os.File) (*io.SectionReader, error) {
// seek to the end to get the size
// we'll leave this at the end of the file since section reader does not advance the reader
size, err := f.Seek(0, io.SeekEnd)
if err != nil {
return nil, errors.Wrap(err, "error getting current file size")
func decompress(dst io.WriteSeeker, src io.ReadSeeker) error {
if _, err := src.Seek(0, io.SeekStart); err != nil {
return err
}
return io.NewSectionReader(f, 0, size), nil
rc, err := gzip.NewReader(src)
if err != nil {
return err
}
_, err = pools.Copy(dst, rc)
if err != nil {
return err
}
return rc.Close()
}
func tailFiles(files []SizeReaderAt, watcher *logger.LogWatcher, dec Decoder, getTailReader GetTailReaderFunc, config logger.ReadConfig, notifyEvict <-chan interface{}) (cont bool) {
nLines := config.Tail
func tailFiles(files []SizeReaderAt, watcher *logger.LogWatcher, dec Decoder, getTailReader GetTailReaderFunc, config logger.ReadConfig) (cont bool) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
@ -553,12 +592,6 @@ func tailFiles(files []SizeReaderAt, watcher *logger.LogWatcher, dec Decoder, ge
// TODO(@cpuguy83): we should plumb a context through instead of dealing with `WatchClose()` here.
go func() {
select {
case err := <-notifyEvict:
if err != nil {
watcher.Err <- err.(error)
cont = false
cancel()
}
case <-ctx.Done():
case <-watcher.WatchConsumerGone():
cont = false
@ -569,6 +602,7 @@ func tailFiles(files []SizeReaderAt, watcher *logger.LogWatcher, dec Decoder, ge
readers := make([]io.Reader, 0, len(files))
if config.Tail > 0 {
nLines := config.Tail
for i := len(files) - 1; i >= 0 && nLines > 0; i-- {
tail, n, err := getTailReader(ctx, files[i], nLines)
if err != nil {
@ -580,7 +614,7 @@ func tailFiles(files []SizeReaderAt, watcher *logger.LogWatcher, dec Decoder, ge
}
} else {
for _, r := range files {
readers = append(readers, &wrappedReaderAt{ReaderAt: r})
readers = append(readers, r)
}
}
@ -608,52 +642,3 @@ func tailFiles(files []SizeReaderAt, watcher *logger.LogWatcher, dec Decoder, ge
}
}
}
func watchFile(name string) (filenotify.FileWatcher, error) {
var fileWatcher filenotify.FileWatcher
if runtime.GOOS == "windows" {
// FileWatcher on Windows files is based on the syscall notifications which has an issue because of file caching.
// It is based on ReadDirectoryChangesW() which doesn't detect writes to the cache. It detects writes to disk only.
// Because of the OS lazy writing, we don't get notifications for file writes and thereby the watcher
// doesn't work. Hence for Windows we will use poll based notifier.
fileWatcher = filenotify.NewPollingWatcher()
} else {
var err error
fileWatcher, err = filenotify.New()
if err != nil {
return nil, err
}
}
logger := logrus.WithFields(logrus.Fields{
"module": "logger",
"file": name,
})
if err := fileWatcher.Add(name); err != nil {
// we will retry using file poller.
logger.WithError(err).Warnf("falling back to file poller")
fileWatcher.Close()
fileWatcher = filenotify.NewPollingWatcher()
if err := fileWatcher.Add(name); err != nil {
fileWatcher.Close()
logger.WithError(err).Debugf("error watching log file for modifications")
return nil, err
}
}
return fileWatcher, nil
}
type wrappedReaderAt struct {
io.ReaderAt
pos int64
}
func (r *wrappedReaderAt) Read(p []byte) (int, error) {
n, err := r.ReaderAt.ReadAt(p, r.pos)
r.pos += int64(n)
return n, err
}

204
daemon/logger/loggerutils/logfile_test.go
View file

@ -9,13 +9,11 @@ import (
"os"
"path/filepath"
"strings"
"sync/atomic"
"testing"
"text/tabwriter"
"time"
"github.com/docker/docker/daemon/logger"
"github.com/docker/docker/pkg/pubsub"
"github.com/docker/docker/pkg/tailfile"
"gotest.tools/v3/assert"
"gotest.tools/v3/poll"
@ -68,7 +66,7 @@ func TestTailFiles(t *testing.T) {
started := make(chan struct{})
go func() {
close(started)
tailFiles(files, watcher, dec, tailReader, config, make(chan interface{}))
tailFiles(files, watcher, dec, tailReader, config)
}()
<-started
})
@ -78,7 +76,7 @@ func TestTailFiles(t *testing.T) {
started := make(chan struct{})
go func() {
close(started)
tailFiles(files, watcher, dec, tailReader, config, make(chan interface{}))
tailFiles(files, watcher, dec, tailReader, config)
}()
<-started
@ -112,180 +110,44 @@ func (dummyDecoder) Decode() (*logger.Message, error) {
func (dummyDecoder) Close() {}
func (dummyDecoder) Reset(io.Reader) {}
func TestFollowLogsConsumerGone(t *testing.T) {
lw := logger.NewLogWatcher()
f, err := os.CreateTemp("", t.Name())
assert.NilError(t, err)
defer func() {
f.Close()
os.Remove(f.Name())
}()
dec := dummyDecoder{}
followLogsDone := make(chan struct{})
var since, until time.Time
go func() {
followLogs(f, lw, make(chan interface{}), make(chan interface{}), dec, since, until)
close(followLogsDone)
}()
select {
case <-lw.Msg:
case err := <-lw.Err:
assert.NilError(t, err)
case <-followLogsDone:
t.Fatal("follow logs finished unexpectedly")
case <-time.After(10 * time.Second):
t.Fatal("timeout waiting for log message")
}
lw.ConsumerGone()
select {
case <-followLogsDone:
case <-time.After(20 * time.Second):
t.Fatal("timeout waiting for followLogs() to finish")
}
}
type dummyWrapper struct {
dummyDecoder
fn func() error
}
func (d *dummyWrapper) Decode() (*logger.Message, error) {
if err := d.fn(); err != nil {
return nil, err
}
return d.dummyDecoder.Decode()
}
func TestFollowLogsProducerGone(t *testing.T) {
lw := logger.NewLogWatcher()
defer lw.ConsumerGone()
f, err := os.CreateTemp("", t.Name())
assert.NilError(t, err)
defer os.Remove(f.Name())
var sent, received, closed int32
dec := &dummyWrapper{fn: func() error {
switch atomic.LoadInt32(&closed) {
case 0:
atomic.AddInt32(&sent, 1)
return nil
case 1:
atomic.AddInt32(&closed, 1)
t.Logf("logDecode() closed after sending %d messages\n", sent)
return io.EOF
default:
t.Fatal("logDecode() called after closing!")
return io.EOF
}
}}
var since, until time.Time
followLogsDone := make(chan struct{})
go func() {
followLogs(f, lw, make(chan interface{}), make(chan interface{}), dec, since, until)
close(followLogsDone)
}()
// read 1 message
select {
case <-lw.Msg:
received++
case err := <-lw.Err:
assert.NilError(t, err)
case <-followLogsDone:
t.Fatal("followLogs() finished unexpectedly")
case <-time.After(10 * time.Second):
t.Fatal("timeout waiting for log message")
}
// "stop" the "container"
atomic.StoreInt32(&closed, 1)
lw.ProducerGone()
// should receive all the messages sent
readDone := make(chan struct{})
go func() {
defer close(readDone)
for {
select {
case <-lw.Msg:
received++
if received == atomic.LoadInt32(&sent) {
return
}
case err := <-lw.Err:
assert.NilError(t, err)
}
}
}()
select {
case <-readDone:
case <-time.After(30 * time.Second):
t.Fatalf("timeout waiting for log messages to be read (sent: %d, received: %d", sent, received)
}
t.Logf("messages sent: %d, received: %d", atomic.LoadInt32(&sent), received)
// followLogs() should be done by now
select {
case <-followLogsDone:
case <-time.After(30 * time.Second):
t.Fatal("timeout waiting for followLogs() to finish")
}
select {
case <-lw.WatchConsumerGone():
t.Fatal("consumer should not have exited")
default:
}
}
func TestCheckCapacityAndRotate(t *testing.T) {
dir, err := os.MkdirTemp("", t.Name())
assert.NilError(t, err)
defer os.RemoveAll(dir)
dir := t.TempDir()
f, err := os.CreateTemp(dir, "log")
assert.NilError(t, err)
l := &LogFile{
f: f,
capacity: 5,
maxFiles: 3,
compress: true,
notifyReaders: pubsub.NewPublisher(0, 1),
perms: 0600,
filesRefCounter: refCounter{counter: make(map[string]int)},
getTailReader: func(ctx context.Context, r SizeReaderAt, lines int) (io.Reader, int, error) {
return tailfile.NewTailReader(ctx, r, lines)
},
createDecoder: func(io.Reader) Decoder {
return dummyDecoder{}
},
marshal: func(msg *logger.Message) ([]byte, error) {
return msg.Line, nil
},
logPath := filepath.Join(dir, "log")
getTailReader := func(ctx context.Context, r SizeReaderAt, lines int) (io.Reader, int, error) {
return tailfile.NewTailReader(ctx, r, lines)
}
createDecoder := func(io.Reader) Decoder {
return dummyDecoder{}
}
marshal := func(msg *logger.Message) ([]byte, error) {
return msg.Line, nil
}
l, err := NewLogFile(
logPath,
5, // capacity
3, // maxFiles
true, // compress
marshal,
createDecoder,
0600, // perms
getTailReader,
)
assert.NilError(t, err)
defer l.Close()
ls := dirStringer{dir}
assert.NilError(t, l.WriteLogEntry(&logger.Message{Line: []byte("hello world!")}))
_, err = os.Stat(f.Name() + ".1")
_, err = os.Stat(logPath + ".1")
assert.Assert(t, os.IsNotExist(err), ls)
assert.NilError(t, l.WriteLogEntry(&logger.Message{Line: []byte("hello world!")}))
poll.WaitOn(t, checkFileExists(f.Name()+".1.gz"), poll.WithDelay(time.Millisecond), poll.WithTimeout(30*time.Second))
poll.WaitOn(t, checkFileExists(logPath+".1.gz"), poll.WithDelay(time.Millisecond), poll.WithTimeout(30*time.Second))
assert.NilError(t, l.WriteLogEntry(&logger.Message{Line: []byte("hello world!")}))
poll.WaitOn(t, checkFileExists(f.Name()+".1.gz"), poll.WithDelay(time.Millisecond), poll.WithTimeout(30*time.Second))
poll.WaitOn(t, checkFileExists(f.Name()+".2.gz"), poll.WithDelay(time.Millisecond), poll.WithTimeout(30*time.Second))
poll.WaitOn(t, checkFileExists(logPath+".1.gz"), poll.WithDelay(time.Millisecond), poll.WithTimeout(30*time.Second))
poll.WaitOn(t, checkFileExists(logPath+".2.gz"), poll.WithDelay(time.Millisecond), poll.WithTimeout(30*time.Second))
t.Run("closed log file", func(t *testing.T) {
// Now let's simulate a failed rotation where the file was able to be closed but something else happened elsewhere
@ -293,14 +155,13 @@ func TestCheckCapacityAndRotate(t *testing.T) {
// We want to make sure that we can recover in the case that `l.f` was closed while attempting a rotation.
l.f.Close()
assert.NilError(t, l.WriteLogEntry(&logger.Message{Line: []byte("hello world!")}))
assert.NilError(t, os.Remove(f.Name()+".2.gz"))
assert.NilError(t, os.Remove(logPath+".2.gz"))
})
t.Run("with log reader", func(t *testing.T) {
// Make sure rotate works with an active reader
lw := logger.NewLogWatcher()
lw := l.ReadLogs(logger.ReadConfig{Follow: true, Tail: 1000})
defer lw.ConsumerGone()
go l.ReadLogs(logger.ReadConfig{Follow: true, Tail: 1000}, lw)
assert.NilError(t, l.WriteLogEntry(&logger.Message{Line: []byte("hello world 0!")}), ls)
// make sure the log reader is primed
@ -310,7 +171,7 @@ func TestCheckCapacityAndRotate(t *testing.T) {
assert.NilError(t, l.WriteLogEntry(&logger.Message{Line: []byte("hello world 2!")}), ls)
assert.NilError(t, l.WriteLogEntry(&logger.Message{Line: []byte("hello world 3!")}), ls)
assert.NilError(t, l.WriteLogEntry(&logger.Message{Line: []byte("hello world 4!")}), ls)
poll.WaitOn(t, checkFileExists(f.Name()+".2.gz"), poll.WithDelay(time.Millisecond), poll.WithTimeout(30*time.Second))
poll.WaitOn(t, checkFileExists(logPath+".2.gz"), poll.WithDelay(time.Millisecond), poll.WithTimeout(30*time.Second))
})
}
@ -321,9 +182,8 @@ func waitForMsg(t *testing.T, lw *logger.LogWatcher, timeout time.Duration) {
defer timer.Stop()
select {
case <-lw.Msg:
case <-lw.WatchProducerGone():
t.Fatal("log producer gone before log message arrived")
case _, ok := <-lw.Msg:
assert.Assert(t, ok, "log producer gone before log message arrived")
case err := <-lw.Err:
assert.NilError(t, err)
case <-timer.C:

227
daemon/logger/loggerutils/sharedtemp.go Normal file
View file

@ -0,0 +1,227 @@
package loggerutils // import "github.com/docker/docker/daemon/logger/loggerutils"
import (
"io"
"io/fs"
"os"
"runtime"
)
type fileConvertFn func(dst io.WriteSeeker, src io.ReadSeeker) error
type stfID uint64
// sharedTempFileConverter converts files using a user-supplied function and
// writes the results to temporary files which are automatically cleaned up on
// close. If another request is made to convert the same file, the conversion
// result and temporary file are reused if they have not yet been cleaned up.
//
// A file is considered the same as another file using the os.SameFile function,
// which compares file identity (e.g. device and inode numbers on Linux) and is
// robust to file renames. Input files are assumed to be immutable; no attempt
// is made to ascertain whether the file contents have changed between requests.
//
// One file descriptor is used per source file, irrespective of the number of
// concurrent readers of the converted contents.
type sharedTempFileConverter struct {
// The directory where temporary converted files are to be written to.
// If set to the empty string, the default directory for temporary files
// is used.
TempDir string
conv fileConvertFn
st chan stfcState
}
type stfcState struct {
fl map[stfID]sharedTempFile
nextID stfID
}
type sharedTempFile struct {
src os.FileInfo // Info about the source file for path-independent identification with os.SameFile.
fd *os.File
size int64
ref int // Reference count of open readers on the temporary file.
wait []chan<- stfConvertResult // Wait list for the conversion to complete.
}
type stfConvertResult struct {
fr *sharedFileReader
err error
}
func newSharedTempFileConverter(conv fileConvertFn) *sharedTempFileConverter {
st := make(chan stfcState, 1)
st <- stfcState{fl: make(map[stfID]sharedTempFile)}
return &sharedTempFileConverter{conv: conv, st: st}
}
// Do returns a reader for the contents of f as converted by the c.C function.
// It is the caller's responsibility to close the returned reader.
//
// This function is safe for concurrent use by multiple goroutines.
func (c *sharedTempFileConverter) Do(f *os.File) (*sharedFileReader, error) {
stat, err := f.Stat()
if err != nil {
return nil, err
}
st := <-c.st
for id, tf := range st.fl {
// os.SameFile can have false positives if one of the files was
// deleted before the other file was created -- such as during
// log rotations... https://github.com/golang/go/issues/36895
// Weed out those false positives by also comparing the files'
// ModTime, which conveniently also handles the case of true
// positives where the file has also been modified since it was
// first converted.
if os.SameFile(tf.src, stat) && tf.src.ModTime() == stat.ModTime() {
return c.openExisting(st, id, tf)
}
}
return c.openNew(st, f, stat)
}
func (c *sharedTempFileConverter) openNew(st stfcState, f *os.File, stat os.FileInfo) (*sharedFileReader, error) {
// Record that we are starting to convert this file so that any other
// requests for the same source file while the conversion is in progress
// can join.
id := st.nextID
st.nextID++
st.fl[id] = sharedTempFile{src: stat}
c.st <- st
dst, size, convErr := c.convert(f)
st = <-c.st
flid := st.fl[id]
if convErr != nil {
// Conversion failed. Delete it from the state so that future
// requests to convert the same file can try again fresh.
delete(st.fl, id)
c.st <- st
for _, w := range flid.wait {
w <- stfConvertResult{err: convErr}
}
return nil, convErr
}
flid.fd = dst
flid.size = size
flid.ref = len(flid.wait) + 1
for _, w := range flid.wait {
// Each waiter needs its own reader with an independent read pointer.
w <- stfConvertResult{fr: flid.Reader(c, id)}
}
flid.wait = nil
st.fl[id] = flid
c.st <- st
return flid.Reader(c, id), nil
}
func (c *sharedTempFileConverter) openExisting(st stfcState, id stfID, v sharedTempFile) (*sharedFileReader, error) {
if v.fd != nil {
// Already converted.
v.ref++
st.fl[id] = v
c.st <- st
return v.Reader(c, id), nil
}
// The file has not finished being converted.
// Add ourselves to the wait list. "Don't call us; we'll call you."
wait := make(chan stfConvertResult, 1)
v.wait = append(v.wait, wait)
st.fl[id] = v
c.st <- st
res := <-wait
return res.fr, res.err
}
func (c *sharedTempFileConverter) convert(f *os.File) (converted *os.File, size int64, err error) {
dst, err := os.CreateTemp(c.TempDir, "dockerdtemp.*")
if err != nil {
return nil, 0, err
}
defer func() {
_ = dst.Close()
// Delete the temporary file immediately so that final cleanup
// of the file on disk is deferred to the OS once we close all
// our file descriptors (or the process dies). Assuming no early
// returns due to errors, the file will be open by this process
// with a read-only descriptor at this point. As we don't care
// about being able to reuse the file name -- it's randomly
// generated and unique -- we can safely use os.Remove on
// Windows.
_ = os.Remove(dst.Name())
}()
err = c.conv(dst, f)
if err != nil {
return nil, 0, err
}
// Close the exclusive read-write file descriptor, catching any delayed
// write errors (and on Windows, releasing the share-locks on the file)
if err := dst.Close(); err != nil {
_ = os.Remove(dst.Name())
return nil, 0, err
}
// Open the file again read-only (without locking the file against
// deletion on Windows).
converted, err = open(dst.Name())
if err != nil {
return nil, 0, err
}
// The position of the file's read pointer doesn't matter as all readers
// will be accessing the file through its io.ReaderAt interface.
size, err = converted.Seek(0, io.SeekEnd)
if err != nil {
_ = converted.Close()
return nil, 0, err
}
return converted, size, nil
}
type sharedFileReader struct {
*io.SectionReader
c *sharedTempFileConverter
id stfID
closed bool
}
func (stf sharedTempFile) Reader(c *sharedTempFileConverter, id stfID) *sharedFileReader {
rdr := &sharedFileReader{SectionReader: io.NewSectionReader(stf.fd, 0, stf.size), c: c, id: id}
runtime.SetFinalizer(rdr, (*sharedFileReader).Close)
return rdr
}
func (r *sharedFileReader) Close() error {
if r.closed {
return fs.ErrClosed
}
st := <-r.c.st
flid, ok := st.fl[r.id]
if !ok {
panic("invariant violation: temp file state missing from map")
}
flid.ref--
lastRef := flid.ref <= 0
if lastRef {
delete(st.fl, r.id)
} else {
st.fl[r.id] = flid
}
r.closed = true
r.c.st <- st
if lastRef {
return flid.fd.Close()
}
runtime.SetFinalizer(r, nil)
return nil
}

256
daemon/logger/loggerutils/sharedtemp_test.go Normal file
View file

@ -0,0 +1,256 @@
package loggerutils // import "github.com/docker/docker/daemon/logger/loggerutils"
import (
"io"
"io/fs"
"os"
"path/filepath"
"runtime"
"strings"
"sync"
"sync/atomic"
"testing"
"time"
"github.com/pkg/errors"
"gotest.tools/v3/assert"
"gotest.tools/v3/assert/cmp"
)
func TestSharedTempFileConverter(t *testing.T) {
t.Parallel()
t.Run("OneReaderAtATime", func(t *testing.T) {
t.Parallel()
dir := t.TempDir()
name := filepath.Join(dir, "test.txt")
createFile(t, name, "hello, world!")
uut := newSharedTempFileConverter(copyTransform(strings.ToUpper))
uut.TempDir = dir
for i := 0; i < 3; i++ {
t.Logf("Iteration %v", i)
rdr := convertPath(t, uut, name)
assert.Check(t, cmp.Equal("HELLO, WORLD!", readAll(t, rdr)))
assert.Check(t, rdr.Close())
assert.Check(t, cmp.Equal(fs.ErrClosed, rdr.Close()), "closing an already-closed reader should return an error")
}
assert.NilError(t, os.Remove(name))
checkDirEmpty(t, dir)
})
t.Run("RobustToRenames", func(t *testing.T) {
t.Parallel()
dir := t.TempDir()
apath := filepath.Join(dir, "test.txt")
createFile(t, apath, "file a")
var conversions int
uut := newSharedTempFileConverter(
func(dst io.WriteSeeker, src io.ReadSeeker) error {
conversions++
return copyTransform(strings.ToUpper)(dst, src)
},
)
uut.TempDir = dir
ra1 := convertPath(t, uut, apath)
// Rotate the file to a new name and write a new file in its place.
bpath := apath
apath = filepath.Join(dir, "test2.txt")
assert.NilError(t, os.Rename(bpath, apath))
createFile(t, bpath, "file b")
rb1 := convertPath(t, uut, bpath) // Same path, different file.
ra2 := convertPath(t, uut, apath) // New path, old file.
assert.Check(t, cmp.Equal(2, conversions), "expected only one conversion per unique file")
// Interleave reading and closing to shake out ref-counting bugs:
// closing one reader shouldn't affect any other open readers.
assert.Check(t, cmp.Equal("FILE A", readAll(t, ra1)))
assert.NilError(t, ra1.Close())
assert.Check(t, cmp.Equal("FILE A", readAll(t, ra2)))
assert.NilError(t, ra2.Close())
assert.Check(t, cmp.Equal("FILE B", readAll(t, rb1)))
assert.NilError(t, rb1.Close())
assert.NilError(t, os.Remove(apath))
assert.NilError(t, os.Remove(bpath))
checkDirEmpty(t, dir)
})
t.Run("ConcurrentRequests", func(t *testing.T) {
t.Parallel()
dir := t.TempDir()
name := filepath.Join(dir, "test.txt")
createFile(t, name, "hi there")
var conversions int32
notify := make(chan chan struct{}, 1)
firstConversionStarted := make(chan struct{})
notify <- firstConversionStarted
unblock := make(chan struct{})
uut := newSharedTempFileConverter(
func(dst io.WriteSeeker, src io.ReadSeeker) error {
t.Log("Convert: enter")
defer t.Log("Convert: exit")
select {
case c := <-notify:
close(c)
default:
}
<-unblock
atomic.AddInt32(&conversions, 1)
return copyTransform(strings.ToUpper)(dst, src)
},
)
uut.TempDir = dir
closers := make(chan io.Closer, 4)
var wg sync.WaitGroup
wg.Add(3)
for i := 0; i < 3; i++ {
i := i
go func() {
defer wg.Done()
t.Logf("goroutine %v: enter", i)
defer t.Logf("goroutine %v: exit", i)
f := convertPath(t, uut, name)
assert.Check(t, cmp.Equal("HI THERE", readAll(t, f)), "in goroutine %v", i)
closers <- f
}()
}
select {
case <-firstConversionStarted:
case <-time.After(2 * time.Second):
t.Fatal("the first conversion should have started by now")
}
close(unblock)
t.Log("starting wait")
wg.Wait()
t.Log("wait done")
f := convertPath(t, uut, name)
closers <- f
close(closers)
assert.Check(t, cmp.Equal("HI THERE", readAll(t, f)), "after all goroutines returned")
for c := range closers {
assert.Check(t, c.Close())
}
assert.Check(t, cmp.Equal(int32(1), conversions))
assert.NilError(t, os.Remove(name))
checkDirEmpty(t, dir)
})
t.Run("ConvertError", func(t *testing.T) {
t.Parallel()
dir := t.TempDir()
name := filepath.Join(dir, "test.txt")
createFile(t, name, "hi there")
src, err := open(name)
assert.NilError(t, err)
defer src.Close()
fakeErr := errors.New("fake error")
var start sync.WaitGroup
start.Add(3)
uut := newSharedTempFileConverter(
func(dst io.WriteSeeker, src io.ReadSeeker) error {
start.Wait()
runtime.Gosched()
if fakeErr != nil {
return fakeErr
}
return copyTransform(strings.ToUpper)(dst, src)
},
)
uut.TempDir = dir
var done sync.WaitGroup
done.Add(3)
for i := 0; i < 3; i++ {
i := i
go func() {
defer done.Done()
t.Logf("goroutine %v: enter", i)
defer t.Logf("goroutine %v: exit", i)
start.Done()
_, err := uut.Do(src)
assert.Check(t, errors.Is(err, fakeErr), "in goroutine %v", i)
}()
}
done.Wait()
// Conversion errors should not be "sticky". A subsequent
// request should retry from scratch.
fakeErr = errors.New("another fake error")
_, err = uut.Do(src)
assert.Check(t, errors.Is(err, fakeErr))
fakeErr = nil
f, err := uut.Do(src)
assert.Check(t, err)
assert.Check(t, cmp.Equal("HI THERE", readAll(t, f)))
assert.Check(t, f.Close())
// Files pending delete continue to show up in directory
// listings on Windows RS5. Close the remaining handle before
// deleting the file to prevent spurious failures with
// checkDirEmpty.
assert.Check(t, src.Close())
assert.NilError(t, os.Remove(name))
checkDirEmpty(t, dir)
})
}
func createFile(t *testing.T, path string, content string) {
t.Helper()
f, err := openFile(path, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0644)
assert.NilError(t, err)
_, err = io.WriteString(f, content)
assert.NilError(t, err)
assert.NilError(t, f.Close())
}
func convertPath(t *testing.T, uut *sharedTempFileConverter, path string) *sharedFileReader {
t.Helper()
f, err := open(path)
assert.NilError(t, err)
defer func() { assert.NilError(t, f.Close()) }()
r, err := uut.Do(f)
assert.NilError(t, err)
return r
}
func readAll(t *testing.T, r io.Reader) string {
t.Helper()
v, err := io.ReadAll(r)
assert.NilError(t, err)
return string(v)
}
func checkDirEmpty(t *testing.T, path string) {
t.Helper()
ls, err := os.ReadDir(path)
assert.NilError(t, err)
assert.Check(t, cmp.Len(ls, 0), "directory should be free of temp files")
}
func copyTransform(f func(string) string) func(dst io.WriteSeeker, src io.ReadSeeker) error {
return func(dst io.WriteSeeker, src io.ReadSeeker) error {
s, err := io.ReadAll(src)
if err != nil {
return err
}
_, err = io.WriteString(dst, f(string(s)))
return err
}
}

40
pkg/filenotify/filenotify.go
View file

@ -1,40 +0,0 @@
// Package filenotify provides a mechanism for watching file(s) for changes.
// Generally leans on fsnotify, but provides a poll-based notifier which fsnotify does not support.
// These are wrapped up in a common interface so that either can be used interchangeably in your code.
package filenotify // import "github.com/docker/docker/pkg/filenotify"
import "github.com/fsnotify/fsnotify"
// FileWatcher is an interface for implementing file notification watchers
type FileWatcher interface {
Events() <-chan fsnotify.Event
Errors() <-chan error
Add(name string) error
Remove(name string) error
Close() error
}
// New tries to use an fs-event watcher, and falls back to the poller if there is an error
func New() (FileWatcher, error) {
if watcher, err := NewEventWatcher(); err == nil {
return watcher, nil
}
return NewPollingWatcher(), nil
}
// NewPollingWatcher returns a poll-based file watcher
func NewPollingWatcher() FileWatcher {
return &filePoller{
events: make(chan fsnotify.Event),
errors: make(chan error),
}
}
// NewEventWatcher returns an fs-event based file watcher
func NewEventWatcher() (FileWatcher, error) {
watcher, err := fsnotify.NewWatcher()
if err != nil {
return nil, err
}
return &fsNotifyWatcher{watcher}, nil
}

18
pkg/filenotify/fsnotify.go
View file

@ -1,18 +0,0 @@
package filenotify // import "github.com/docker/docker/pkg/filenotify"
import "github.com/fsnotify/fsnotify"
// fsNotifyWatcher wraps the fsnotify package to satisfy the FileNotifier interface
type fsNotifyWatcher struct {
*fsnotify.Watcher
}
// Events returns the fsnotify event channel receiver
func (w *fsNotifyWatcher) Events() <-chan fsnotify.Event {
return w.Watcher.Events
}
// Errors returns the fsnotify error channel receiver
func (w *fsNotifyWatcher) Errors() <-chan error {
return w.Watcher.Errors
}

213
pkg/filenotify/poller.go
View file

@ -1,213 +0,0 @@
package filenotify // import "github.com/docker/docker/pkg/filenotify"
import (
"errors"
"fmt"
"os"
"sync"
"time"
"github.com/sirupsen/logrus"
"github.com/fsnotify/fsnotify"
)
var (
// errPollerClosed is returned when the poller is closed
errPollerClosed = errors.New("poller is closed")
// errNoSuchWatch is returned when trying to remove a watch that doesn't exist
errNoSuchWatch = errors.New("watch does not exist")
)
// watchWaitTime is the time to wait between file poll loops
const watchWaitTime = 200 * time.Millisecond
// filePoller is used to poll files for changes, especially in cases where fsnotify
// can't be run (e.g. when inotify handles are exhausted)
// filePoller satisfies the FileWatcher interface
type filePoller struct {
// watches is the list of files currently being polled, close the associated channel to stop the watch
watches map[string]chan struct{}
// events is the channel to listen to for watch events
events chan fsnotify.Event
// errors is the channel to listen to for watch errors
errors chan error
// mu locks the poller for modification
mu sync.Mutex
// closed is used to specify when the poller has already closed
closed bool
}
// Add adds a filename to the list of watches
// once added the file is polled for changes in a separate goroutine
func (w *filePoller) Add(name string) error {
w.mu.Lock()
defer w.mu.Unlock()
if w.closed {
return errPollerClosed
}
f, err := os.Open(name)
if err != nil {
return err
}
fi, err := os.Stat(name)
if err != nil {
f.Close()
return err
}
if w.watches == nil {
w.watches = make(map[string]chan struct{})
}
if _, exists := w.watches[name]; exists {
f.Close()
return fmt.Errorf("watch exists")
}
chClose := make(chan struct{})
w.watches[name] = chClose
go w.watch(f, fi, chClose)
return nil
}
// Remove stops and removes watch with the specified name
func (w *filePoller) Remove(name string) error {
w.mu.Lock()
defer w.mu.Unlock()
return w.remove(name)
}
func (w *filePoller) remove(name string) error {
if w.closed {
return errPollerClosed
}
chClose, exists := w.watches[name]
if !exists {
return errNoSuchWatch
}
close(chClose)
delete(w.watches, name)
return nil
}
// Events returns the event channel
// This is used for notifications on events about watched files
func (w *filePoller) Events() <-chan fsnotify.Event {
return w.events
}
// Errors returns the errors channel
// This is used for notifications about errors on watched files
func (w *filePoller) Errors() <-chan error {
return w.errors
}
// Close closes the poller
// All watches are stopped, removed, and the poller cannot be added to
func (w *filePoller) Close() error {
w.mu.Lock()
defer w.mu.Unlock()
if w.closed {
return nil
}
for name := range w.watches {
w.remove(name)
}
w.closed = true
return nil
}
// sendEvent publishes the specified event to the events channel
func (w *filePoller) sendEvent(e fsnotify.Event, chClose <-chan struct{}) error {
select {
case w.events <- e:
case <-chClose:
return fmt.Errorf("closed")
}
return nil
}
// sendErr publishes the specified error to the errors channel
func (w *filePoller) sendErr(e error, chClose <-chan struct{}) error {
select {
case w.errors <- e:
case <-chClose:
return fmt.Errorf("closed")
}
return nil
}
// watch is responsible for polling the specified file for changes
// upon finding changes to a file or errors, sendEvent/sendErr is called
func (w *filePoller) watch(f *os.File, lastFi os.FileInfo, chClose chan struct{}) {
defer f.Close()
timer := time.NewTimer(watchWaitTime)
if !timer.Stop() {
<-timer.C
}
defer timer.Stop()
for {
timer.Reset(watchWaitTime)
select {
case <-timer.C:
case <-chClose:
logrus.Debugf("watch for %s closed", f.Name())
return
}
fi, err := os.Stat(f.Name())
if err != nil {
// if we got an error here and lastFi is not set, we can presume that nothing has changed
// This should be safe since before `watch()` is called, a stat is performed, there is any error `watch` is not called
if lastFi == nil {
continue
}
// If it doesn't exist at this point, it must have been removed
// no need to send the error here since this is a valid operation
if os.IsNotExist(err) {
if err := w.sendEvent(fsnotify.Event{Op: fsnotify.Remove, Name: f.Name()}, chClose); err != nil {
return
}
lastFi = nil
continue
}
// at this point, send the error
if err := w.sendErr(err, chClose); err != nil {
return
}
continue
}
if lastFi == nil {
if err := w.sendEvent(fsnotify.Event{Op: fsnotify.Create, Name: fi.Name()}, chClose); err != nil {
return
}
lastFi = fi
continue
}
if fi.Mode() != lastFi.Mode() {
if err := w.sendEvent(fsnotify.Event{Op: fsnotify.Chmod, Name: fi.Name()}, chClose); err != nil {
return
}
lastFi = fi
continue
}
if fi.ModTime() != lastFi.ModTime() || fi.Size() != lastFi.Size() {
if err := w.sendEvent(fsnotify.Event{Op: fsnotify.Write, Name: fi.Name()}, chClose); err != nil {
return
}
lastFi = fi
continue
}
}
}

131
pkg/filenotify/poller_test.go
View file

@ -1,131 +0,0 @@
package filenotify // import "github.com/docker/docker/pkg/filenotify"
import (
"fmt"
"os"
"runtime"
"testing"
"time"
"github.com/fsnotify/fsnotify"
)
func TestPollerAddRemove(t *testing.T) {
w := NewPollingWatcher()
if err := w.Add("no-such-file"); err == nil {
t.Fatal("should have gotten error when adding a non-existent file")
}
if err := w.Remove("no-such-file"); err == nil {
t.Fatal("should have gotten error when removing non-existent watch")
}
f, err := os.CreateTemp("", "asdf")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(f.Name())
if err := w.Add(f.Name()); err != nil {
t.Fatal(err)
}
if err := w.Remove(f.Name()); err != nil {
t.Fatal(err)
}
}
func TestPollerEvent(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("No chmod on Windows")
}
w := NewPollingWatcher()
f, err := os.CreateTemp("", "test-poller")
if err != nil {
t.Fatal("error creating temp file")
}
defer os.RemoveAll(f.Name())
f.Close()
if err := w.Add(f.Name()); err != nil {
t.Fatal(err)
}
select {
case <-w.Events():
t.Fatal("got event before anything happened")
case <-w.Errors():
t.Fatal("got error before anything happened")
default:
}
if err := os.WriteFile(f.Name(), []byte("hello"), 0600); err != nil {
t.Fatal(err)
}
assertFileMode(t, f.Name(), 0600)
if err := assertEvent(w, fsnotify.Write); err != nil {
t.Fatal(err)
}
if err := os.Chmod(f.Name(), 0644); err != nil {
t.Fatal(err)
}
assertFileMode(t, f.Name(), 0644)
if err := assertEvent(w, fsnotify.Chmod); err != nil {
t.Fatal(err)
}
if err := os.Remove(f.Name()); err != nil {
t.Fatal(err)
}
if err := assertEvent(w, fsnotify.Remove); err != nil {
t.Fatal(err)
}
}
func TestPollerClose(t *testing.T) {
w := NewPollingWatcher()
if err := w.Close(); err != nil {
t.Fatal(err)
}
// test double-close
if err := w.Close(); err != nil {
t.Fatal(err)
}
f, err := os.CreateTemp("", "asdf")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(f.Name())
if err := w.Add(f.Name()); err == nil {
t.Fatal("should have gotten error adding watch for closed watcher")
}
}
func assertFileMode(t *testing.T, fileName string, mode uint32) {
t.Helper()
f, err := os.Stat(fileName)
if err != nil {
t.Fatal(err)
}
if f.Mode() != os.FileMode(mode) {
t.Fatalf("expected file %s to have mode %#o, but got %#o", fileName, mode, f.Mode())
}
}
func assertEvent(w FileWatcher, eType fsnotify.Op) error {
var err error
select {
case e := <-w.Events():
if e.Op != eType {
err = fmt.Errorf("got wrong event type, expected %q: %v", eType, e.Op)
}
case e := <-w.Errors():
err = fmt.Errorf("got unexpected error waiting for events %v: %v", eType, e)
case <-time.After(watchWaitTime * 3):
err = fmt.Errorf("timeout waiting for event %v", eType)
}
return err
}

2
vendor.mod
View file

@ -33,7 +33,6 @@ require (
github.com/docker/libkv v0.2.2-0.20211217103745-e480589147e3
github.com/docker/libtrust v0.0.0-20150526203908-9cbd2a1374f4
github.com/fluent/fluent-logger-golang v1.9.0
github.com/fsnotify/fsnotify v1.5.1
github.com/godbus/dbus/v5 v5.0.6
github.com/gogo/protobuf v1.3.2
github.com/golang/gddo v0.0.0-20190904175337-72a348e765d2
@ -106,6 +105,7 @@ require (
github.com/dustin/go-humanize v1.0.0 // indirect
github.com/felixge/httpsnoop v1.0.2 // indirect
github.com/fernet/fernet-go v0.0.0-20180830025343-9eac43b88a5e // indirect
github.com/fsnotify/fsnotify v1.5.1 // indirect
github.com/go-logr/logr v1.2.2 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/gofrs/flock v0.8.1 // indirect

1
vendor/github.com/fsnotify/fsnotify/.gitattributes generated vendored
View file

@ -1 +0,0 @@
go.sum linguist-generated

6
vendor/github.com/fsnotify/fsnotify/.gitignore generated vendored
View file

@ -1,6 +0,0 @@
# Setup a Global .gitignore for OS and editor generated files:
# https://help.github.com/articles/ignoring-files
# git config --global core.excludesfile ~/.gitignore_global
.vagrant
*.sublime-project

2
vendor/github.com/fsnotify/fsnotify/.mailmap generated vendored
View file

@ -1,2 +0,0 @@
Chris Howey <howeyc@gmail.com> <chris@howey.me>
Nathan Youngman <git@nathany.com> <4566+nathany@users.noreply.github.com>

62
vendor/github.com/fsnotify/fsnotify/AUTHORS generated vendored
View file

@ -1,62 +0,0 @@
# Names should be added to this file as
# Name or Organization <email address>
# The email address is not required for organizations.
# You can update this list using the following command:
#
# $ (head -n10 AUTHORS && git shortlog -se | sed -E 's/^\s+[0-9]+\t//') | tee AUTHORS
# Please keep the list sorted.
Aaron L <aaron@bettercoder.net>
Adrien Bustany <adrien@bustany.org>
Alexey Kazakov <alkazako@redhat.com>
Amit Krishnan <amit.krishnan@oracle.com>
Anmol Sethi <me@anmol.io>
Bjørn Erik Pedersen <bjorn.erik.pedersen@gmail.com>
Brian Goff <cpuguy83@gmail.com>
Bruno Bigras <bigras.bruno@gmail.com>
Caleb Spare <cespare@gmail.com>
Case Nelson <case@teammating.com>
Chris Howey <howeyc@gmail.com>
Christoffer Buchholz <christoffer.buchholz@gmail.com>
Daniel Wagner-Hall <dawagner@gmail.com>
Dave Cheney <dave@cheney.net>
Eric Lin <linxiulei@gmail.com>
Evan Phoenix <evan@fallingsnow.net>
Francisco Souza <f@souza.cc>
Gautam Dey <gautam.dey77@gmail.com>
Hari haran <hariharan.uno@gmail.com>
Ichinose Shogo <shogo82148@gmail.com>
Johannes Ebke <johannes@ebke.org>
John C Barstow <jbowtie@amathaine.com>
Kelvin Fo <vmirage@gmail.com>
Ken-ichirou MATSUZAWA <chamas@h4.dion.ne.jp>
Matt Layher <mdlayher@gmail.com>
Matthias Stone <matthias@bellstone.ca>
Nathan Youngman <git@nathany.com>
Nickolai Zeldovich <nickolai@csail.mit.edu>
Oliver Bristow <evilumbrella+github@gmail.com>
Patrick <patrick@dropbox.com>
Paul Hammond <paul@paulhammond.org>
Pawel Knap <pawelknap88@gmail.com>
Pieter Droogendijk <pieter@binky.org.uk>
Pratik Shinde <pratikshinde320@gmail.com>
Pursuit92 <JoshChase@techpursuit.net>
Riku Voipio <riku.voipio@linaro.org>
Rob Figueiredo <robfig@gmail.com>
Rodrigo Chiossi <rodrigochiossi@gmail.com>
Slawek Ligus <root@ooz.ie>
Soge Zhang <zhssoge@gmail.com>
Tiffany Jernigan <tiffany.jernigan@intel.com>
Tilak Sharma <tilaks@google.com>
Tobias Klauser <tobias.klauser@gmail.com>
Tom Payne <twpayne@gmail.com>
Travis Cline <travis.cline@gmail.com>
Tudor Golubenco <tudor.g@gmail.com>
Vahe Khachikyan <vahe@live.ca>
Yukang <moorekang@gmail.com>
bronze1man <bronze1man@gmail.com>
debrando <denis.brandolini@gmail.com>
henrikedwards <henrik.edwards@gmail.com>
铁哥 <guotie.9@gmail.com>

339
vendor/github.com/fsnotify/fsnotify/CHANGELOG.md generated vendored
View file

@ -1,339 +0,0 @@
# Changelog
All notable changes to this project will be documented in this file.
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
## [Unreleased]
## [1.5.1] - 2021-08-24
* Revert Add AddRaw to not follow symlinks
## [1.5.0] - 2021-08-20
* Go: Increase minimum required version to Go 1.12 [#381](https://github.com/fsnotify/fsnotify/pull/381)
* Feature: Add AddRaw method which does not follow symlinks when adding a watch [#289](https://github.com/fsnotify/fsnotify/pull/298)
* Windows: Follow symlinks by default like on all other systems [#289](https://github.com/fsnotify/fsnotify/pull/289)
* CI: Use GitHub Actions for CI and cover go 1.12-1.17
[#378](https://github.com/fsnotify/fsnotify/pull/378)
[#381](https://github.com/fsnotify/fsnotify/pull/381)
[#385](https://github.com/fsnotify/fsnotify/pull/385)
* Go 1.14+: Fix unsafe pointer conversion [#325](https://github.com/fsnotify/fsnotify/pull/325)
## [1.4.7] - 2018-01-09
* BSD/macOS: Fix possible deadlock on closing the watcher on kqueue (thanks @nhooyr and @glycerine)
* Tests: Fix missing verb on format string (thanks @rchiossi)
* Linux: Fix deadlock in Remove (thanks @aarondl)
* Linux: Watch.Add improvements (avoid race, fix consistency, reduce garbage) (thanks @twpayne)
* Docs: Moved FAQ into the README (thanks @vahe)
* Linux: Properly handle inotify's IN_Q_OVERFLOW event (thanks @zeldovich)
* Docs: replace references to OS X with macOS
## [1.4.2] - 2016-10-10
* Linux: use InotifyInit1 with IN_CLOEXEC to stop leaking a file descriptor to a child process when using fork/exec [#178](https://github.com/fsnotify/fsnotify/pull/178) (thanks @pattyshack)
## [1.4.1] - 2016-10-04
* Fix flaky inotify stress test on Linux [#177](https://github.com/fsnotify/fsnotify/pull/177) (thanks @pattyshack)
## [1.4.0] - 2016-10-01
* add a String() method to Event.Op [#165](https://github.com/fsnotify/fsnotify/pull/165) (thanks @oozie)
## [1.3.1] - 2016-06-28
* Windows: fix for double backslash when watching the root of a drive [#151](https://github.com/fsnotify/fsnotify/issues/151) (thanks @brunoqc)
## [1.3.0] - 2016-04-19
* Support linux/arm64 by [patching](https://go-review.googlesource.com/#/c/21971/) x/sys/unix and switching to to it from syscall (thanks @suihkulokki) [#135](https://github.com/fsnotify/fsnotify/pull/135)
## [1.2.10] - 2016-03-02
* Fix golint errors in windows.go [#121](https://github.com/fsnotify/fsnotify/pull/121) (thanks @tiffanyfj)
## [1.2.9] - 2016-01-13
kqueue: Fix logic for CREATE after REMOVE [#111](https://github.com/fsnotify/fsnotify/pull/111) (thanks @bep)
## [1.2.8] - 2015-12-17
* kqueue: fix race condition in Close [#105](https://github.com/fsnotify/fsnotify/pull/105) (thanks @djui for reporting the issue and @ppknap for writing a failing test)
* inotify: fix race in test
* enable race detection for continuous integration (Linux, Mac, Windows)
## [1.2.5] - 2015-10-17
* inotify: use epoll_create1 for arm64 support (requires Linux 2.6.27 or later) [#100](https://github.com/fsnotify/fsnotify/pull/100) (thanks @suihkulokki)
* inotify: fix path leaks [#73](https://github.com/fsnotify/fsnotify/pull/73) (thanks @chamaken)
* kqueue: watch for rename events on subdirectories [#83](https://github.com/fsnotify/fsnotify/pull/83) (thanks @guotie)
* kqueue: avoid infinite loops from symlinks cycles [#101](https://github.com/fsnotify/fsnotify/pull/101) (thanks @illicitonion)
## [1.2.1] - 2015-10-14
* kqueue: don't watch named pipes [#98](https://github.com/fsnotify/fsnotify/pull/98) (thanks @evanphx)
## [1.2.0] - 2015-02-08
* inotify: use epoll to wake up readEvents [#66](https://github.com/fsnotify/fsnotify/pull/66) (thanks @PieterD)
* inotify: closing watcher should now always shut down goroutine [#63](https://github.com/fsnotify/fsnotify/pull/63) (thanks @PieterD)
* kqueue: close kqueue after removing watches, fixes [#59](https://github.com/fsnotify/fsnotify/issues/59)
## [1.1.1] - 2015-02-05
* inotify: Retry read on EINTR [#61](https://github.com/fsnotify/fsnotify/issues/61) (thanks @PieterD)
## [1.1.0] - 2014-12-12
* kqueue: rework internals [#43](https://github.com/fsnotify/fsnotify/pull/43)
* add low-level functions
* only need to store flags on directories
* less mutexes [#13](https://github.com/fsnotify/fsnotify/issues/13)
* done can be an unbuffered channel
* remove calls to os.NewSyscallError
* More efficient string concatenation for Event.String() [#52](https://github.com/fsnotify/fsnotify/pull/52) (thanks @mdlayher)
* kqueue: fix regression in rework causing subdirectories to be watched [#48](https://github.com/fsnotify/fsnotify/issues/48)
* kqueue: cleanup internal watch before sending remove event [#51](https://github.com/fsnotify/fsnotify/issues/51)
## [1.0.4] - 2014-09-07
* kqueue: add dragonfly to the build tags.
* Rename source code files, rearrange code so exported APIs are at the top.
* Add done channel to example code. [#37](https://github.com/fsnotify/fsnotify/pull/37) (thanks @chenyukang)
## [1.0.3] - 2014-08-19
* [Fix] Windows MOVED_TO now translates to Create like on BSD and Linux. [#36](https://github.com/fsnotify/fsnotify/issues/36)
## [1.0.2] - 2014-08-17
* [Fix] Missing create events on macOS. [#14](https://github.com/fsnotify/fsnotify/issues/14) (thanks @zhsso)
* [Fix] Make ./path and path equivalent. (thanks @zhsso)
## [1.0.0] - 2014-08-15
* [API] Remove AddWatch on Windows, use Add.
* Improve documentation for exported identifiers. [#30](https://github.com/fsnotify/fsnotify/issues/30)
* Minor updates based on feedback from golint.
## dev / 2014-07-09
* Moved to [github.com/fsnotify/fsnotify](https://github.com/fsnotify/fsnotify).
* Use os.NewSyscallError instead of returning errno (thanks @hariharan-uno)
## dev / 2014-07-04
* kqueue: fix incorrect mutex used in Close()
* Update example to demonstrate usage of Op.
## dev / 2014-06-28
* [API] Don't set the Write Op for attribute notifications [#4](https://github.com/fsnotify/fsnotify/issues/4)
* Fix for String() method on Event (thanks Alex Brainman)
* Don't build on Plan 9 or Solaris (thanks @4ad)
## dev / 2014-06-21
* Events channel of type Event rather than *Event.
* [internal] use syscall constants directly for inotify and kqueue.
* [internal] kqueue: rename events to kevents and fileEvent to event.
## dev / 2014-06-19
* Go 1.3+ required on Windows (uses syscall.ERROR_MORE_DATA internally).
* [internal] remove cookie from Event struct (unused).
* [internal] Event struct has the same definition across every OS.
* [internal] remove internal watch and removeWatch methods.
## dev / 2014-06-12
* [API] Renamed Watch() to Add() and RemoveWatch() to Remove().
* [API] Pluralized channel names: Events and Errors.
* [API] Renamed FileEvent struct to Event.
* [API] Op constants replace methods like IsCreate().
## dev / 2014-06-12
* Fix data race on kevent buffer (thanks @tilaks) [#98](https://github.com/howeyc/fsnotify/pull/98)
## dev / 2014-05-23
* [API] Remove current implementation of WatchFlags.
* current implementation doesn't take advantage of OS for efficiency
* provides little benefit over filtering events as they are received, but has extra bookkeeping and mutexes
* no tests for the current implementation
* not fully implemented on Windows [#93](https://github.com/howeyc/fsnotify/issues/93#issuecomment-39285195)
## [0.9.3] - 2014-12-31
* kqueue: cleanup internal watch before sending remove event [#51](https://github.com/fsnotify/fsnotify/issues/51)
## [0.9.2] - 2014-08-17
* [Backport] Fix missing create events on macOS. [#14](https://github.com/fsnotify/fsnotify/issues/14) (thanks @zhsso)
## [0.9.1] - 2014-06-12
* Fix data race on kevent buffer (thanks @tilaks) [#98](https://github.com/howeyc/fsnotify/pull/98)
## [0.9.0] - 2014-01-17
* IsAttrib() for events that only concern a file's metadata [#79][] (thanks @abustany)
* [Fix] kqueue: fix deadlock [#77][] (thanks @cespare)
* [NOTICE] Development has moved to `code.google.com/p/go.exp/fsnotify` in preparation for inclusion in the Go standard library.
## [0.8.12] - 2013-11-13
* [API] Remove FD_SET and friends from Linux adapter
## [0.8.11] - 2013-11-02
* [Doc] Add Changelog [#72][] (thanks @nathany)
* [Doc] Spotlight and double modify events on macOS [#62][] (reported by @paulhammond)
## [0.8.10] - 2013-10-19
* [Fix] kqueue: remove file watches when parent directory is removed [#71][] (reported by @mdwhatcott)
* [Fix] kqueue: race between Close and readEvents [#70][] (reported by @bernerdschaefer)
* [Doc] specify OS-specific limits in README (thanks @debrando)
## [0.8.9] - 2013-09-08
* [Doc] Contributing (thanks @nathany)
* [Doc] update package path in example code [#63][] (thanks @paulhammond)
* [Doc] GoCI badge in README (Linux only) [#60][]
* [Doc] Cross-platform testing with Vagrant [#59][] (thanks @nathany)
## [0.8.8] - 2013-06-17
* [Fix] Windows: handle `ERROR_MORE_DATA` on Windows [#49][] (thanks @jbowtie)
## [0.8.7] - 2013-06-03
* [API] Make syscall flags internal
* [Fix] inotify: ignore event changes
* [Fix] race in symlink test [#45][] (reported by @srid)
* [Fix] tests on Windows
* lower case error messages
## [0.8.6] - 2013-05-23
* kqueue: Use EVT_ONLY flag on Darwin
* [Doc] Update README with full example
## [0.8.5] - 2013-05-09
* [Fix] inotify: allow monitoring of "broken" symlinks (thanks @tsg)
## [0.8.4] - 2013-04-07
* [Fix] kqueue: watch all file events [#40][] (thanks @ChrisBuchholz)
## [0.8.3] - 2013-03-13
* [Fix] inoitfy/kqueue memory leak [#36][] (reported by @nbkolchin)
* [Fix] kqueue: use fsnFlags for watching a directory [#33][] (reported by @nbkolchin)
## [0.8.2] - 2013-02-07
* [Doc] add Authors
* [Fix] fix data races for map access [#29][] (thanks @fsouza)
## [0.8.1] - 2013-01-09
* [Fix] Windows path separators
* [Doc] BSD License
## [0.8.0] - 2012-11-09
* kqueue: directory watching improvements (thanks @vmirage)
* inotify: add `IN_MOVED_TO` [#25][] (requested by @cpisto)
* [Fix] kqueue: deleting watched directory [#24][] (reported by @jakerr)
## [0.7.4] - 2012-10-09
* [Fix] inotify: fixes from https://codereview.appspot.com/5418045/ (ugorji)
* [Fix] kqueue: preserve watch flags when watching for delete [#21][] (reported by @robfig)
* [Fix] kqueue: watch the directory even if it isn't a new watch (thanks @robfig)
* [Fix] kqueue: modify after recreation of file
## [0.7.3] - 2012-09-27
* [Fix] kqueue: watch with an existing folder inside the watched folder (thanks @vmirage)
* [Fix] kqueue: no longer get duplicate CREATE events
## [0.7.2] - 2012-09-01
* kqueue: events for created directories
## [0.7.1] - 2012-07-14
* [Fix] for renaming files
## [0.7.0] - 2012-07-02
* [Feature] FSNotify flags
* [Fix] inotify: Added file name back to event path
## [0.6.0] - 2012-06-06
* kqueue: watch files after directory created (thanks @tmc)
## [0.5.1] - 2012-05-22
* [Fix] inotify: remove all watches before Close()
## [0.5.0] - 2012-05-03
* [API] kqueue: return errors during watch instead of sending over channel
* kqueue: match symlink behavior on Linux
* inotify: add `DELETE_SELF` (requested by @taralx)
* [Fix] kqueue: handle EINTR (reported by @robfig)
* [Doc] Godoc example [#1][] (thanks @davecheney)
## [0.4.0] - 2012-03-30
* Go 1 released: build with go tool
* [Feature] Windows support using winfsnotify
* Windows does not have attribute change notifications
* Roll attribute notifications into IsModify
## [0.3.0] - 2012-02-19
* kqueue: add files when watch directory
## [0.2.0] - 2011-12-30
* update to latest Go weekly code
## [0.1.0] - 2011-10-19
* kqueue: add watch on file creation to match inotify
* kqueue: create file event
* inotify: ignore `IN_IGNORED` events
* event String()
* linux: common FileEvent functions
* initial commit
[#79]: https://github.com/howeyc/fsnotify/pull/79
[#77]: https://github.com/howeyc/fsnotify/pull/77
[#72]: https://github.com/howeyc/fsnotify/issues/72
[#71]: https://github.com/howeyc/fsnotify/issues/71
[#70]: https://github.com/howeyc/fsnotify/issues/70
[#63]: https://github.com/howeyc/fsnotify/issues/63
[#62]: https://github.com/howeyc/fsnotify/issues/62
[#60]: https://github.com/howeyc/fsnotify/issues/60
[#59]: https://github.com/howeyc/fsnotify/issues/59
[#49]: https://github.com/howeyc/fsnotify/issues/49
[#45]: https://github.com/howeyc/fsnotify/issues/45
[#40]: https://github.com/howeyc/fsnotify/issues/40
[#36]: https://github.com/howeyc/fsnotify/issues/36
[#33]: https://github.com/howeyc/fsnotify/issues/33
[#29]: https://github.com/howeyc/fsnotify/issues/29
[#25]: https://github.com/howeyc/fsnotify/issues/25
[#24]: https://github.com/howeyc/fsnotify/issues/24
[#21]: https://github.com/howeyc/fsnotify/issues/21

77
vendor/github.com/fsnotify/fsnotify/CONTRIBUTING.md generated vendored
View file

@ -1,77 +0,0 @@
# Contributing
## Issues
* Request features and report bugs using the [GitHub Issue Tracker](https://github.com/fsnotify/fsnotify/issues).
* Please indicate the platform you are using fsnotify on.
* A code example to reproduce the problem is appreciated.
## Pull Requests
### Contributor License Agreement
fsnotify is derived from code in the [golang.org/x/exp](https://godoc.org/golang.org/x/exp) package and it may be included [in the standard library](https://github.com/fsnotify/fsnotify/issues/1) in the future. Therefore fsnotify carries the same [LICENSE](https://github.com/fsnotify/fsnotify/blob/master/LICENSE) as Go. Contributors retain their copyright, so you need to fill out a short form before we can accept your contribution: [Google Individual Contributor License Agreement](https://developers.google.com/open-source/cla/individual).
Please indicate that you have signed the CLA in your pull request.
### How fsnotify is Developed
* Development is done on feature branches.
* Tests are run on BSD, Linux, macOS and Windows.
* Pull requests are reviewed and [applied to master][am] using [hub][].
* Maintainers may modify or squash commits rather than asking contributors to.
* To issue a new release, the maintainers will:
* Update the CHANGELOG
* Tag a version, which will become available through gopkg.in.
### How to Fork
For smooth sailing, always use the original import path. Installing with `go get` makes this easy.
1. Install from GitHub (`go get -u github.com/fsnotify/fsnotify`)
2. Create your feature branch (`git checkout -b my-new-feature`)
3. Ensure everything works and the tests pass (see below)
4. Commit your changes (`git commit -am 'Add some feature'`)
Contribute upstream:
1. Fork fsnotify on GitHub
2. Add your remote (`git remote add fork git@github.com:mycompany/repo.git`)
3. Push to the branch (`git push fork my-new-feature`)
4. Create a new Pull Request on GitHub
This workflow is [thoroughly explained by Katrina Owen](https://splice.com/blog/contributing-open-source-git-repositories-go/).
### Testing
fsnotify uses build tags to compile different code on Linux, BSD, macOS, and Windows.
Before doing a pull request, please do your best to test your changes on multiple platforms, and list which platforms you were able/unable to test on.
To aid in cross-platform testing there is a Vagrantfile for Linux and BSD.
* Install [Vagrant](http://www.vagrantup.com/) and [VirtualBox](https://www.virtualbox.org/)
* Setup [Vagrant Gopher](https://github.com/nathany/vagrant-gopher) in your `src` folder.
* Run `vagrant up` from the project folder. You can also setup just one box with `vagrant up linux` or `vagrant up bsd` (note: the BSD box doesn't support Windows hosts at this time, and NFS may prompt for your host OS password)
* Once setup, you can run the test suite on a given OS with a single command `vagrant ssh linux -c 'cd fsnotify/fsnotify; go test'`.
* When you're done, you will want to halt or destroy the Vagrant boxes.
Notice: fsnotify file system events won't trigger in shared folders. The tests get around this limitation by using the /tmp directory.
Right now there is no equivalent solution for Windows and macOS, but there are Windows VMs [freely available from Microsoft](http://www.modern.ie/en-us/virtualization-tools#downloads).
### Maintainers
Help maintaining fsnotify is welcome. To be a maintainer:
* Submit a pull request and sign the CLA as above.
* You must be able to run the test suite on Mac, Windows, Linux and BSD.
To keep master clean, the fsnotify project uses the "apply mail" workflow outlined in Nathaniel Talbott's post ["Merge pull request" Considered Harmful][am]. This requires installing [hub][].
All code changes should be internal pull requests.
Releases are tagged using [Semantic Versioning](http://semver.org/).
[hub]: https://github.com/github/hub
[am]: http://blog.spreedly.com/2014/06/24/merge-pull-request-considered-harmful/#.VGa5yZPF_Zs

28
vendor/github.com/fsnotify/fsnotify/LICENSE generated vendored
View file

@ -1,28 +0,0 @@
Copyright (c) 2012 The Go Authors. All rights reserved.
Copyright (c) 2012-2019 fsnotify Authors. All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
* Neither the name of Google Inc. nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

130
vendor/github.com/fsnotify/fsnotify/README.md generated vendored
View file

@ -1,130 +0,0 @@
# File system notifications for Go
[![GoDoc](https://godoc.org/github.com/fsnotify/fsnotify?status.svg)](https://godoc.org/github.com/fsnotify/fsnotify) [![Go Report Card](https://goreportcard.com/badge/github.com/fsnotify/fsnotify)](https://goreportcard.com/report/github.com/fsnotify/fsnotify)
fsnotify utilizes [golang.org/x/sys](https://godoc.org/golang.org/x/sys) rather than `syscall` from the standard library. Ensure you have the latest version installed by running:
```console
go get -u golang.org/x/sys/...
```
Cross platform: Windows, Linux, BSD and macOS.
| Adapter | OS | Status |
| --------------------- | -------------------------------- | ------------------------------------------------------------------------------------------------------------------------------- |
| inotify | Linux 2.6.27 or later, Android\* | Supported |
| kqueue | BSD, macOS, iOS\* | Supported |
| ReadDirectoryChangesW | Windows | Supported |
| FSEvents | macOS | [Planned](https://github.com/fsnotify/fsnotify/issues/11) |
| FEN | Solaris 11 | [In Progress](https://github.com/fsnotify/fsnotify/issues/12) |
| fanotify | Linux 2.6.37+ | [Planned](https://github.com/fsnotify/fsnotify/issues/114) |
| USN Journals | Windows | [Maybe](https://github.com/fsnotify/fsnotify/issues/53) |
| Polling | *All* | [Maybe](https://github.com/fsnotify/fsnotify/issues/9) |
\* Android and iOS are untested.
Please see [the documentation](https://godoc.org/github.com/fsnotify/fsnotify) and consult the [FAQ](#faq) for usage information.
## API stability
fsnotify is a fork of [howeyc/fsnotify](https://godoc.org/github.com/howeyc/fsnotify) with a new API as of v1.0. The API is based on [this design document](http://goo.gl/MrYxyA).
All [releases](https://github.com/fsnotify/fsnotify/releases) are tagged based on [Semantic Versioning](http://semver.org/). Further API changes are [planned](https://github.com/fsnotify/fsnotify/milestones), and will be tagged with a new major revision number.
Go 1.6 supports dependencies located in the `vendor/` folder. Unless you are creating a library, it is recommended that you copy fsnotify into `vendor/github.com/fsnotify/fsnotify` within your project, and likewise for `golang.org/x/sys`.
## Usage
```go
package main
import (
"log"
"github.com/fsnotify/fsnotify"
)
func main() {
watcher, err := fsnotify.NewWatcher()
if err != nil {
log.Fatal(err)
}
defer watcher.Close()
done := make(chan bool)
go func() {
for {
select {
case event, ok := <-watcher.Events:
if !ok {
return
}
log.Println("event:", event)
if event.Op&fsnotify.Write == fsnotify.Write {
log.Println("modified file:", event.Name)
}
case err, ok := <-watcher.Errors:
if !ok {
return
}
log.Println("error:", err)
}
}
}()
err = watcher.Add("/tmp/foo")
if err != nil {
log.Fatal(err)
}
<-done
}
```
## Contributing
Please refer to [CONTRIBUTING][] before opening an issue or pull request.
## Example
See [example_test.go](https://github.com/fsnotify/fsnotify/blob/master/example_test.go).
## FAQ
**When a file is moved to another directory is it still being watched?**
No (it shouldn't be, unless you are watching where it was moved to).
**When I watch a directory, are all subdirectories watched as well?**
No, you must add watches for any directory you want to watch (a recursive watcher is on the roadmap [#18][]).
**Do I have to watch the Error and Event channels in a separate goroutine?**
As of now, yes. Looking into making this single-thread friendly (see [howeyc #7][#7])
**Why am I receiving multiple events for the same file on OS X?**
Spotlight indexing on OS X can result in multiple events (see [howeyc #62][#62]). A temporary workaround is to add your folder(s) to the *Spotlight Privacy settings* until we have a native FSEvents implementation (see [#11][]).
**How many files can be watched at once?**
There are OS-specific limits as to how many watches can be created:
* Linux: /proc/sys/fs/inotify/max_user_watches contains the limit, reaching this limit results in a "no space left on device" error.
* BSD / OSX: sysctl variables "kern.maxfiles" and "kern.maxfilesperproc", reaching these limits results in a "too many open files" error.
**Why don't notifications work with NFS filesystems or filesystem in userspace (FUSE)?**
fsnotify requires support from underlying OS to work. The current NFS protocol does not provide network level support for file notifications.
[#62]: https://github.com/howeyc/fsnotify/issues/62
[#18]: https://github.com/fsnotify/fsnotify/issues/18
[#11]: https://github.com/fsnotify/fsnotify/issues/11
[#7]: https://github.com/howeyc/fsnotify/issues/7
[contributing]: https://github.com/fsnotify/fsnotify/blob/master/CONTRIBUTING.md
## Related Projects
* [notify](https://github.com/rjeczalik/notify)
* [fsevents](https://github.com/fsnotify/fsevents)

38
vendor/github.com/fsnotify/fsnotify/fen.go generated vendored
View file

@ -1,38 +0,0 @@
// Copyright 2010 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//go:build solaris
// +build solaris
package fsnotify
import (
"errors"
)
// Watcher watches a set of files, delivering events to a channel.
type Watcher struct {
Events chan Event
Errors chan error
}
// NewWatcher establishes a new watcher with the underlying OS and begins waiting for events.
func NewWatcher() (*Watcher, error) {
return nil, errors.New("FEN based watcher not yet supported for fsnotify\n")
}
// Close removes all watches and closes the events channel.
func (w *Watcher) Close() error {
return nil
}
// Add starts watching the named file or directory (non-recursively).
func (w *Watcher) Add(name string) error {
return nil
}
// Remove stops watching the the named file or directory (non-recursively).
func (w *Watcher) Remove(name string) error {
return nil
}

69
vendor/github.com/fsnotify/fsnotify/fsnotify.go generated vendored
View file

@ -1,69 +0,0 @@
// Copyright 2012 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//go:build !plan9
// +build !plan9
// Package fsnotify provides a platform-independent interface for file system notifications.
package fsnotify
import (
"bytes"
"errors"
"fmt"
)
// Event represents a single file system notification.
type Event struct {
Name string // Relative path to the file or directory.
Op Op // File operation that triggered the event.
}
// Op describes a set of file operations.
type Op uint32
// These are the generalized file operations that can trigger a notification.
const (
Create Op = 1 << iota
Write
Remove
Rename
Chmod
)
func (op Op) String() string {
// Use a buffer for efficient string concatenation
var buffer bytes.Buffer
if op&Create == Create {
buffer.WriteString("|CREATE")
}
if op&Remove == Remove {
buffer.WriteString("|REMOVE")
}
if op&Write == Write {
buffer.WriteString("|WRITE")
}
if op&Rename == Rename {
buffer.WriteString("|RENAME")
}
if op&Chmod == Chmod {
buffer.WriteString("|CHMOD")
}
if buffer.Len() == 0 {
return ""
}
return buffer.String()[1:] // Strip leading pipe
}
// String returns a string representation of the event in the form
// "file: REMOVE|WRITE|..."
func (e Event) String() string {
return fmt.Sprintf("%q: %s", e.Name, e.Op.String())
}
// Common errors that can be reported by a watcher
var (
ErrEventOverflow = errors.New("fsnotify queue overflow")
)

338
vendor/github.com/fsnotify/fsnotify/inotify.go generated vendored
View file

@ -1,338 +0,0 @@
// Copyright 2010 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//go:build linux
// +build linux
package fsnotify
import (
"errors"
"fmt"
"io"
"os"
"path/filepath"
"strings"
"sync"
"unsafe"
"golang.org/x/sys/unix"
)
// Watcher watches a set of files, delivering events to a channel.
type Watcher struct {
Events chan Event
Errors chan error
mu sync.Mutex // Map access
fd int
poller *fdPoller
watches map[string]*watch // Map of inotify watches (key: path)
paths map[int]string // Map of watched paths (key: watch descriptor)
done chan struct{} // Channel for sending a "quit message" to the reader goroutine
doneResp chan struct{} // Channel to respond to Close
}
// NewWatcher establishes a new watcher with the underlying OS and begins waiting for events.
func NewWatcher() (*Watcher, error) {
// Create inotify fd
fd, errno := unix.InotifyInit1(unix.IN_CLOEXEC)
if fd == -1 {
return nil, errno
}
// Create epoll
poller, err := newFdPoller(fd)
if err != nil {
unix.Close(fd)
return nil, err
}
w := &Watcher{
fd: fd,
poller: poller,
watches: make(map[string]*watch),
paths: make(map[int]string),
Events: make(chan Event),
Errors: make(chan error),
done: make(chan struct{}),
doneResp: make(chan struct{}),
}
go w.readEvents()
return w, nil
}
func (w *Watcher) isClosed() bool {
select {
case <-w.done:
return true
default:
return false
}
}
// Close removes all watches and closes the events channel.
func (w *Watcher) Close() error {
if w.isClosed() {
return nil
}
// Send 'close' signal to goroutine, and set the Watcher to closed.
close(w.done)
// Wake up goroutine
w.poller.wake()
// Wait for goroutine to close
<-w.doneResp
return nil
}
// Add starts watching the named file or directory (non-recursively).
func (w *Watcher) Add(name string) error {
name = filepath.Clean(name)
if w.isClosed() {
return errors.New("inotify instance already closed")
}
const agnosticEvents = unix.IN_MOVED_TO | unix.IN_MOVED_FROM |
unix.IN_CREATE | unix.IN_ATTRIB | unix.IN_MODIFY |
unix.IN_MOVE_SELF | unix.IN_DELETE | unix.IN_DELETE_SELF
var flags uint32 = agnosticEvents
w.mu.Lock()
defer w.mu.Unlock()
watchEntry := w.watches[name]
if watchEntry != nil {
flags |= watchEntry.flags | unix.IN_MASK_ADD
}
wd, errno := unix.InotifyAddWatch(w.fd, name, flags)
if wd == -1 {
return errno
}
if watchEntry == nil {
w.watches[name] = &watch{wd: uint32(wd), flags: flags}
w.paths[wd] = name
} else {
watchEntry.wd = uint32(wd)
watchEntry.flags = flags
}
return nil
}
// Remove stops watching the named file or directory (non-recursively).
func (w *Watcher) Remove(name string) error {
name = filepath.Clean(name)
// Fetch the watch.
w.mu.Lock()
defer w.mu.Unlock()
watch, ok := w.watches[name]
// Remove it from inotify.
if !ok {
return fmt.Errorf("can't remove non-existent inotify watch for: %s", name)
}
// We successfully removed the watch if InotifyRmWatch doesn't return an
// error, we need to clean up our internal state to ensure it matches
// inotify's kernel state.
delete(w.paths, int(watch.wd))
delete(w.watches, name)
// inotify_rm_watch will return EINVAL if the file has been deleted;
// the inotify will already have been removed.
// watches and pathes are deleted in ignoreLinux() implicitly and asynchronously
// by calling inotify_rm_watch() below. e.g. readEvents() goroutine receives IN_IGNORE
// so that EINVAL means that the wd is being rm_watch()ed or its file removed
// by another thread and we have not received IN_IGNORE event.
success, errno := unix.InotifyRmWatch(w.fd, watch.wd)
if success == -1 {
// TODO: Perhaps it's not helpful to return an error here in every case.
// the only two possible errors are:
// EBADF, which happens when w.fd is not a valid file descriptor of any kind.
// EINVAL, which is when fd is not an inotify descriptor or wd is not a valid watch descriptor.
// Watch descriptors are invalidated when they are removed explicitly or implicitly;
// explicitly by inotify_rm_watch, implicitly when the file they are watching is deleted.
return errno
}
return nil
}
type watch struct {
wd uint32 // Watch descriptor (as returned by the inotify_add_watch() syscall)
flags uint32 // inotify flags of this watch (see inotify(7) for the list of valid flags)
}
// readEvents reads from the inotify file descriptor, converts the
// received events into Event objects and sends them via the Events channel
func (w *Watcher) readEvents() {
var (
buf [unix.SizeofInotifyEvent * 4096]byte // Buffer for a maximum of 4096 raw events
n int // Number of bytes read with read()
errno error // Syscall errno
ok bool // For poller.wait
)
defer close(w.doneResp)
defer close(w.Errors)
defer close(w.Events)
defer unix.Close(w.fd)
defer w.poller.close()
for {
// See if we have been closed.
if w.isClosed() {
return
}
ok, errno = w.poller.wait()
if errno != nil {
select {
case w.Errors <- errno:
case <-w.done:
return
}
continue
}
if !ok {
continue
}
n, errno = unix.Read(w.fd, buf[:])
// If a signal interrupted execution, see if we've been asked to close, and try again.
// http://man7.org/linux/man-pages/man7/signal.7.html :
// "Before Linux 3.8, reads from an inotify(7) file descriptor were not restartable"
if errno == unix.EINTR {
continue
}
// unix.Read might have been woken up by Close. If so, we're done.
if w.isClosed() {
return
}
if n < unix.SizeofInotifyEvent {
var err error
if n == 0 {
// If EOF is received. This should really never happen.
err = io.EOF
} else if n < 0 {
// If an error occurred while reading.
err = errno
} else {
// Read was too short.
err = errors.New("notify: short read in readEvents()")
}
select {
case w.Errors <- err:
case <-w.done:
return
}
continue
}
var offset uint32
// We don't know how many events we just read into the buffer
// While the offset points to at least one whole event...
for offset <= uint32(n-unix.SizeofInotifyEvent) {
// Point "raw" to the event in the buffer
raw := (*unix.InotifyEvent)(unsafe.Pointer(&buf[offset]))
mask := uint32(raw.Mask)
nameLen := uint32(raw.Len)
if mask&unix.IN_Q_OVERFLOW != 0 {
select {
case w.Errors <- ErrEventOverflow:
case <-w.done:
return
}
}
// If the event happened to the watched directory or the watched file, the kernel
// doesn't append the filename to the event, but we would like to always fill the
// the "Name" field with a valid filename. We retrieve the path of the watch from
// the "paths" map.
w.mu.Lock()
name, ok := w.paths[int(raw.Wd)]
// IN_DELETE_SELF occurs when the file/directory being watched is removed.
// This is a sign to clean up the maps, otherwise we are no longer in sync
// with the inotify kernel state which has already deleted the watch
// automatically.
if ok && mask&unix.IN_DELETE_SELF == unix.IN_DELETE_SELF {
delete(w.paths, int(raw.Wd))
delete(w.watches, name)
}
w.mu.Unlock()
if nameLen > 0 {
// Point "bytes" at the first byte of the filename
bytes := (*[unix.PathMax]byte)(unsafe.Pointer(&buf[offset+unix.SizeofInotifyEvent]))[:nameLen:nameLen]
// The filename is padded with NULL bytes. TrimRight() gets rid of those.
name += "/" + strings.TrimRight(string(bytes[0:nameLen]), "\000")
}
event := newEvent(name, mask)
// Send the events that are not ignored on the events channel
if !event.ignoreLinux(mask) {
select {
case w.Events <- event:
case <-w.done:
return
}
}
// Move to the next event in the buffer
offset += unix.SizeofInotifyEvent + nameLen
}
}
}
// Certain types of events can be "ignored" and not sent over the Events
// channel. Such as events marked ignore by the kernel, or MODIFY events
// against files that do not exist.
func (e *Event) ignoreLinux(mask uint32) bool {
// Ignore anything the inotify API says to ignore
if mask&unix.IN_IGNORED == unix.IN_IGNORED {
return true
}
// If the event is not a DELETE or RENAME, the file must exist.
// Otherwise the event is ignored.
// *Note*: this was put in place because it was seen that a MODIFY
// event was sent after the DELETE. This ignores that MODIFY and
// assumes a DELETE will come or has come if the file doesn't exist.
if !(e.Op&Remove == Remove || e.Op&Rename == Rename) {
_, statErr := os.Lstat(e.Name)
return os.IsNotExist(statErr)
}
return false
}
// newEvent returns an platform-independent Event based on an inotify mask.
func newEvent(name string, mask uint32) Event {
e := Event{Name: name}
if mask&unix.IN_CREATE == unix.IN_CREATE || mask&unix.IN_MOVED_TO == unix.IN_MOVED_TO {
e.Op |= Create
}
if mask&unix.IN_DELETE_SELF == unix.IN_DELETE_SELF || mask&unix.IN_DELETE == unix.IN_DELETE {
e.Op |= Remove
}
if mask&unix.IN_MODIFY == unix.IN_MODIFY {
e.Op |= Write
}
if mask&unix.IN_MOVE_SELF == unix.IN_MOVE_SELF || mask&unix.IN_MOVED_FROM == unix.IN_MOVED_FROM {
e.Op |= Rename
}
if mask&unix.IN_ATTRIB == unix.IN_ATTRIB {
e.Op |= Chmod
}
return e
}

188
vendor/github.com/fsnotify/fsnotify/inotify_poller.go generated vendored
View file

@ -1,188 +0,0 @@
// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//go:build linux
// +build linux
package fsnotify
import (
"errors"
"golang.org/x/sys/unix"
)
type fdPoller struct {
fd int // File descriptor (as returned by the inotify_init() syscall)
epfd int // Epoll file descriptor
pipe [2]int // Pipe for waking up
}
func emptyPoller(fd int) *fdPoller {
poller := new(fdPoller)
poller.fd = fd
poller.epfd = -1
poller.pipe[0] = -1
poller.pipe[1] = -1
return poller
}
// Create a new inotify poller.
// This creates an inotify handler, and an epoll handler.
func newFdPoller(fd int) (*fdPoller, error) {
var errno error
poller := emptyPoller(fd)
defer func() {
if errno != nil {
poller.close()
}
}()
poller.fd = fd
// Create epoll fd
poller.epfd, errno = unix.EpollCreate1(unix.EPOLL_CLOEXEC)
if poller.epfd == -1 {
return nil, errno
}
// Create pipe; pipe[0] is the read end, pipe[1] the write end.
errno = unix.Pipe2(poller.pipe[:], unix.O_NONBLOCK|unix.O_CLOEXEC)
if errno != nil {
return nil, errno
}
// Register inotify fd with epoll
event := unix.EpollEvent{
Fd: int32(poller.fd),
Events: unix.EPOLLIN,
}
errno = unix.EpollCtl(poller.epfd, unix.EPOLL_CTL_ADD, poller.fd, &event)
if errno != nil {
return nil, errno
}
// Register pipe fd with epoll
event = unix.EpollEvent{
Fd: int32(poller.pipe[0]),
Events: unix.EPOLLIN,
}
errno = unix.EpollCtl(poller.epfd, unix.EPOLL_CTL_ADD, poller.pipe[0], &event)
if errno != nil {
return nil, errno
}
return poller, nil
}
// Wait using epoll.
// Returns true if something is ready to be read,
// false if there is not.
func (poller *fdPoller) wait() (bool, error) {
// 3 possible events per fd, and 2 fds, makes a maximum of 6 events.
// I don't know whether epoll_wait returns the number of events returned,
// or the total number of events ready.
// I decided to catch both by making the buffer one larger than the maximum.
events := make([]unix.EpollEvent, 7)
for {
n, errno := unix.EpollWait(poller.epfd, events, -1)
if n == -1 {
if errno == unix.EINTR {
continue
}
return false, errno
}
if n == 0 {
// If there are no events, try again.
continue
}
if n > 6 {
// This should never happen. More events were returned than should be possible.
return false, errors.New("epoll_wait returned more events than I know what to do with")
}
ready := events[:n]
epollhup := false
epollerr := false
epollin := false
for _, event := range ready {
if event.Fd == int32(poller.fd) {
if event.Events&unix.EPOLLHUP != 0 {
// This should not happen, but if it does, treat it as a wakeup.
epollhup = true
}
if event.Events&unix.EPOLLERR != 0 {
// If an error is waiting on the file descriptor, we should pretend
// something is ready to read, and let unix.Read pick up the error.
epollerr = true
}
if event.Events&unix.EPOLLIN != 0 {
// There is data to read.
epollin = true
}
}
if event.Fd == int32(poller.pipe[0]) {
if event.Events&unix.EPOLLHUP != 0 {
// Write pipe descriptor was closed, by us. This means we're closing down the
// watcher, and we should wake up.
}
if event.Events&unix.EPOLLERR != 0 {
// If an error is waiting on the pipe file descriptor.
// This is an absolute mystery, and should never ever happen.
return false, errors.New("Error on the pipe descriptor.")
}
if event.Events&unix.EPOLLIN != 0 {
// This is a regular wakeup, so we have to clear the buffer.
err := poller.clearWake()
if err != nil {
return false, err
}
}
}
}
if epollhup || epollerr || epollin {
return true, nil
}
return false, nil
}
}
// Close the write end of the poller.
func (poller *fdPoller) wake() error {
buf := make([]byte, 1)
n, errno := unix.Write(poller.pipe[1], buf)
if n == -1 {
if errno == unix.EAGAIN {
// Buffer is full, poller will wake.
return nil
}
return errno
}
return nil
}
func (poller *fdPoller) clearWake() error {
// You have to be woken up a LOT in order to get to 100!
buf := make([]byte, 100)
n, errno := unix.Read(poller.pipe[0], buf)
if n == -1 {
if errno == unix.EAGAIN {
// Buffer is empty, someone else cleared our wake.
return nil
}
return errno
}
return nil
}
// Close all poller file descriptors, but not the one passed to it.
func (poller *fdPoller) close() {
if poller.pipe[1] != -1 {
unix.Close(poller.pipe[1])
}
if poller.pipe[0] != -1 {
unix.Close(poller.pipe[0])
}
if poller.epfd != -1 {
unix.Close(poller.epfd)
}
}

522
vendor/github.com/fsnotify/fsnotify/kqueue.go generated vendored
View file

@ -1,522 +0,0 @@
// Copyright 2010 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//go:build freebsd || openbsd || netbsd || dragonfly || darwin
// +build freebsd openbsd netbsd dragonfly darwin
package fsnotify
import (
"errors"
"fmt"
"io/ioutil"
"os"
"path/filepath"
"sync"
"time"
"golang.org/x/sys/unix"
)
// Watcher watches a set of files, delivering events to a channel.
type Watcher struct {
Events chan Event
Errors chan error
done chan struct{} // Channel for sending a "quit message" to the reader goroutine
kq int // File descriptor (as returned by the kqueue() syscall).
mu sync.Mutex // Protects access to watcher data
watches map[string]int // Map of watched file descriptors (key: path).
externalWatches map[string]bool // Map of watches added by user of the library.
dirFlags map[string]uint32 // Map of watched directories to fflags used in kqueue.
paths map[int]pathInfo // Map file descriptors to path names for processing kqueue events.
fileExists map[string]bool // Keep track of if we know this file exists (to stop duplicate create events).
isClosed bool // Set to true when Close() is first called
}
type pathInfo struct {
name string
isDir bool
}
// NewWatcher establishes a new watcher with the underlying OS and begins waiting for events.
func NewWatcher() (*Watcher, error) {
kq, err := kqueue()
if err != nil {
return nil, err
}
w := &Watcher{
kq: kq,
watches: make(map[string]int),
dirFlags: make(map[string]uint32),
paths: make(map[int]pathInfo),
fileExists: make(map[string]bool),
externalWatches: make(map[string]bool),
Events: make(chan Event),
Errors: make(chan error),
done: make(chan struct{}),
}
go w.readEvents()
return w, nil
}
// Close removes all watches and closes the events channel.
func (w *Watcher) Close() error {
w.mu.Lock()
if w.isClosed {
w.mu.Unlock()
return nil
}
w.isClosed = true
// copy paths to remove while locked
var pathsToRemove = make([]string, 0, len(w.watches))
for name := range w.watches {
pathsToRemove = append(pathsToRemove, name)
}
w.mu.Unlock()
// unlock before calling Remove, which also locks
for _, name := range pathsToRemove {
w.Remove(name)
}
// send a "quit" message to the reader goroutine
close(w.done)
return nil
}
// Add starts watching the named file or directory (non-recursively).
func (w *Watcher) Add(name string) error {
w.mu.Lock()
w.externalWatches[name] = true
w.mu.Unlock()
_, err := w.addWatch(name, noteAllEvents)
return err
}
// Remove stops watching the the named file or directory (non-recursively).
func (w *Watcher) Remove(name string) error {
name = filepath.Clean(name)
w.mu.Lock()
watchfd, ok := w.watches[name]
w.mu.Unlock()
if !ok {
return fmt.Errorf("can't remove non-existent kevent watch for: %s", name)
}
const registerRemove = unix.EV_DELETE
if err := register(w.kq, []int{watchfd}, registerRemove, 0); err != nil {
return err
}
unix.Close(watchfd)
w.mu.Lock()
isDir := w.paths[watchfd].isDir
delete(w.watches, name)
delete(w.paths, watchfd)
delete(w.dirFlags, name)
w.mu.Unlock()
// Find all watched paths that are in this directory that are not external.
if isDir {
var pathsToRemove []string
w.mu.Lock()
for _, path := range w.paths {
wdir, _ := filepath.Split(path.name)
if filepath.Clean(wdir) == name {
if !w.externalWatches[path.name] {
pathsToRemove = append(pathsToRemove, path.name)
}
}
}
w.mu.Unlock()
for _, name := range pathsToRemove {
// Since these are internal, not much sense in propagating error
// to the user, as that will just confuse them with an error about
// a path they did not explicitly watch themselves.
w.Remove(name)
}
}
return nil
}
// Watch all events (except NOTE_EXTEND, NOTE_LINK, NOTE_REVOKE)
const noteAllEvents = unix.NOTE_DELETE | unix.NOTE_WRITE | unix.NOTE_ATTRIB | unix.NOTE_RENAME
// keventWaitTime to block on each read from kevent
var keventWaitTime = durationToTimespec(100 * time.Millisecond)
// addWatch adds name to the watched file set.
// The flags are interpreted as described in kevent(2).
// Returns the real path to the file which was added, if any, which may be different from the one passed in the case of symlinks.
func (w *Watcher) addWatch(name string, flags uint32) (string, error) {
var isDir bool
// Make ./name and name equivalent
name = filepath.Clean(name)
w.mu.Lock()
if w.isClosed {
w.mu.Unlock()
return "", errors.New("kevent instance already closed")
}
watchfd, alreadyWatching := w.watches[name]
// We already have a watch, but we can still override flags.
if alreadyWatching {
isDir = w.paths[watchfd].isDir
}
w.mu.Unlock()
if !alreadyWatching {
fi, err := os.Lstat(name)
if err != nil {
return "", err
}
// Don't watch sockets.
if fi.Mode()&os.ModeSocket == os.ModeSocket {
return "", nil
}
// Don't watch named pipes.
if fi.Mode()&os.ModeNamedPipe == os.ModeNamedPipe {
return "", nil
}
// Follow Symlinks
// Unfortunately, Linux can add bogus symlinks to watch list without
// issue, and Windows can't do symlinks period (AFAIK). To maintain
// consistency, we will act like everything is fine. There will simply
// be no file events for broken symlinks.
// Hence the returns of nil on errors.
if fi.Mode()&os.ModeSymlink == os.ModeSymlink {
name, err = filepath.EvalSymlinks(name)
if err != nil {
return "", nil
}
w.mu.Lock()
_, alreadyWatching = w.watches[name]
w.mu.Unlock()
if alreadyWatching {
return name, nil
}
fi, err = os.Lstat(name)
if err != nil {
return "", nil
}
}
watchfd, err = unix.Open(name, openMode, 0700)
if watchfd == -1 {
return "", err
}
isDir = fi.IsDir()
}
const registerAdd = unix.EV_ADD | unix.EV_CLEAR | unix.EV_ENABLE
if err := register(w.kq, []int{watchfd}, registerAdd, flags); err != nil {
unix.Close(watchfd)
return "", err
}
if !alreadyWatching {
w.mu.Lock()
w.watches[name] = watchfd
w.paths[watchfd] = pathInfo{name: name, isDir: isDir}
w.mu.Unlock()
}
if isDir {
// Watch the directory if it has not been watched before,
// or if it was watched before, but perhaps only a NOTE_DELETE (watchDirectoryFiles)
w.mu.Lock()
watchDir := (flags&unix.NOTE_WRITE) == unix.NOTE_WRITE &&
(!alreadyWatching || (w.dirFlags[name]&unix.NOTE_WRITE) != unix.NOTE_WRITE)
// Store flags so this watch can be updated later
w.dirFlags[name] = flags
w.mu.Unlock()
if watchDir {
if err := w.watchDirectoryFiles(name); err != nil {
return "", err
}
}
}
return name, nil
}
// readEvents reads from kqueue and converts the received kevents into
// Event values that it sends down the Events channel.
func (w *Watcher) readEvents() {
eventBuffer := make([]unix.Kevent_t, 10)
loop:
for {
// See if there is a message on the "done" channel
select {
case <-w.done:
break loop
default:
}
// Get new events
kevents, err := read(w.kq, eventBuffer, &keventWaitTime)
// EINTR is okay, the syscall was interrupted before timeout expired.
if err != nil && err != unix.EINTR {
select {
case w.Errors <- err:
case <-w.done:
break loop
}
continue
}
// Flush the events we received to the Events channel
for len(kevents) > 0 {
kevent := &kevents[0]
watchfd := int(kevent.Ident)
mask := uint32(kevent.Fflags)
w.mu.Lock()
path := w.paths[watchfd]
w.mu.Unlock()
event := newEvent(path.name, mask)
if path.isDir && !(event.Op&Remove == Remove) {
// Double check to make sure the directory exists. This can happen when
// we do a rm -fr on a recursively watched folders and we receive a
// modification event first but the folder has been deleted and later
// receive the delete event
if _, err := os.Lstat(event.Name); os.IsNotExist(err) {
// mark is as delete event
event.Op |= Remove
}
}
if event.Op&Rename == Rename || event.Op&Remove == Remove {
w.Remove(event.Name)
w.mu.Lock()
delete(w.fileExists, event.Name)
w.mu.Unlock()
}
if path.isDir && event.Op&Write == Write && !(event.Op&Remove == Remove) {
w.sendDirectoryChangeEvents(event.Name)
} else {
// Send the event on the Events channel.
select {
case w.Events <- event:
case <-w.done:
break loop
}
}
if event.Op&Remove == Remove {
// Look for a file that may have overwritten this.
// For example, mv f1 f2 will delete f2, then create f2.
if path.isDir {
fileDir := filepath.Clean(event.Name)
w.mu.Lock()
_, found := w.watches[fileDir]
w.mu.Unlock()
if found {
// make sure the directory exists before we watch for changes. When we
// do a recursive watch and perform rm -fr, the parent directory might
// have gone missing, ignore the missing directory and let the
// upcoming delete event remove the watch from the parent directory.
if _, err := os.Lstat(fileDir); err == nil {
w.sendDirectoryChangeEvents(fileDir)
}
}
} else {
filePath := filepath.Clean(event.Name)
if fileInfo, err := os.Lstat(filePath); err == nil {
w.sendFileCreatedEventIfNew(filePath, fileInfo)
}
}
}
// Move to next event
kevents = kevents[1:]
}
}
// cleanup
err := unix.Close(w.kq)
if err != nil {
// only way the previous loop breaks is if w.done was closed so we need to async send to w.Errors.
select {
case w.Errors <- err:
default:
}
}
close(w.Events)
close(w.Errors)
}
// newEvent returns an platform-independent Event based on kqueue Fflags.
func newEvent(name string, mask uint32) Event {
e := Event{Name: name}
if mask&unix.NOTE_DELETE == unix.NOTE_DELETE {
e.Op |= Remove
}
if mask&unix.NOTE_WRITE == unix.NOTE_WRITE {
e.Op |= Write
}
if mask&unix.NOTE_RENAME == unix.NOTE_RENAME {
e.Op |= Rename
}
if mask&unix.NOTE_ATTRIB == unix.NOTE_ATTRIB {
e.Op |= Chmod
}
return e
}
func newCreateEvent(name string) Event {
return Event{Name: name, Op: Create}
}
// watchDirectoryFiles to mimic inotify when adding a watch on a directory
func (w *Watcher) watchDirectoryFiles(dirPath string) error {
// Get all files
files, err := ioutil.ReadDir(dirPath)
if err != nil {
return err
}
for _, fileInfo := range files {
filePath := filepath.Join(dirPath, fileInfo.Name())
filePath, err = w.internalWatch(filePath, fileInfo)
if err != nil {
return err
}
w.mu.Lock()
w.fileExists[filePath] = true
w.mu.Unlock()
}
return nil
}
// sendDirectoryEvents searches the directory for newly created files
// and sends them over the event channel. This functionality is to have
// the BSD version of fsnotify match Linux inotify which provides a
// create event for files created in a watched directory.
func (w *Watcher) sendDirectoryChangeEvents(dirPath string) {
// Get all files
files, err := ioutil.ReadDir(dirPath)
if err != nil {
select {
case w.Errors <- err:
case <-w.done:
return
}
}
// Search for new files
for _, fileInfo := range files {
filePath := filepath.Join(dirPath, fileInfo.Name())
err := w.sendFileCreatedEventIfNew(filePath, fileInfo)
if err != nil {
return
}
}
}
// sendFileCreatedEvent sends a create event if the file isn't already being tracked.
func (w *Watcher) sendFileCreatedEventIfNew(filePath string, fileInfo os.FileInfo) (err error) {
w.mu.Lock()
_, doesExist := w.fileExists[filePath]
w.mu.Unlock()
if !doesExist {
// Send create event
select {
case w.Events <- newCreateEvent(filePath):
case <-w.done:
return
}
}
// like watchDirectoryFiles (but without doing another ReadDir)
filePath, err = w.internalWatch(filePath, fileInfo)
if err != nil {
return err
}
w.mu.Lock()
w.fileExists[filePath] = true
w.mu.Unlock()
return nil
}
func (w *Watcher) internalWatch(name string, fileInfo os.FileInfo) (string, error) {
if fileInfo.IsDir() {
// mimic Linux providing delete events for subdirectories
// but preserve the flags used if currently watching subdirectory
w.mu.Lock()
flags := w.dirFlags[name]
w.mu.Unlock()
flags |= unix.NOTE_DELETE | unix.NOTE_RENAME
return w.addWatch(name, flags)
}
// watch file to mimic Linux inotify
return w.addWatch(name, noteAllEvents)
}
// kqueue creates a new kernel event queue and returns a descriptor.
func kqueue() (kq int, err error) {
kq, err = unix.Kqueue()
if kq == -1 {
return kq, err
}
return kq, nil
}
// register events with the queue
func register(kq int, fds []int, flags int, fflags uint32) error {
changes := make([]unix.Kevent_t, len(fds))
for i, fd := range fds {
// SetKevent converts int to the platform-specific types:
unix.SetKevent(&changes[i], fd, unix.EVFILT_VNODE, flags)
changes[i].Fflags = fflags
}
// register the events
success, err := unix.Kevent(kq, changes, nil, nil)
if success == -1 {
return err
}
return nil
}
// read retrieves pending events, or waits until an event occurs.
// A timeout of nil blocks indefinitely, while 0 polls the queue.
func read(kq int, events []unix.Kevent_t, timeout *unix.Timespec) ([]unix.Kevent_t, error) {
n, err := unix.Kevent(kq, nil, events, timeout)
if err != nil {
return nil, err
}
return events[0:n], nil
}
// durationToTimespec prepares a timeout value
func durationToTimespec(d time.Duration) unix.Timespec {
return unix.NsecToTimespec(d.Nanoseconds())
}

12
vendor/github.com/fsnotify/fsnotify/open_mode_bsd.go generated vendored
View file

@ -1,12 +0,0 @@
// Copyright 2013 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//go:build freebsd || openbsd || netbsd || dragonfly
// +build freebsd openbsd netbsd dragonfly
package fsnotify
import "golang.org/x/sys/unix"
const openMode = unix.O_NONBLOCK | unix.O_RDONLY | unix.O_CLOEXEC

13
vendor/github.com/fsnotify/fsnotify/open_mode_darwin.go generated vendored
View file

@ -1,13 +0,0 @@
// Copyright 2013 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//go:build darwin
// +build darwin
package fsnotify
import "golang.org/x/sys/unix"
// note: this constant is not defined on BSD
const openMode = unix.O_EVTONLY | unix.O_CLOEXEC

562
vendor/github.com/fsnotify/fsnotify/windows.go generated vendored
View file

@ -1,562 +0,0 @@
// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//go:build windows
// +build windows
package fsnotify
import (
"errors"
"fmt"
"os"
"path/filepath"
"runtime"
"sync"
"syscall"
"unsafe"
)
// Watcher watches a set of files, delivering events to a channel.
type Watcher struct {
Events chan Event
Errors chan error
isClosed bool // Set to true when Close() is first called
mu sync.Mutex // Map access
port syscall.Handle // Handle to completion port
watches watchMap // Map of watches (key: i-number)
input chan *input // Inputs to the reader are sent on this channel
quit chan chan<- error
}
// NewWatcher establishes a new watcher with the underlying OS and begins waiting for events.
func NewWatcher() (*Watcher, error) {
port, e := syscall.CreateIoCompletionPort(syscall.InvalidHandle, 0, 0, 0)
if e != nil {
return nil, os.NewSyscallError("CreateIoCompletionPort", e)
}
w := &Watcher{
port: port,
watches: make(watchMap),
input: make(chan *input, 1),
Events: make(chan Event, 50),
Errors: make(chan error),
quit: make(chan chan<- error, 1),
}
go w.readEvents()
return w, nil
}
// Close removes all watches and closes the events channel.
func (w *Watcher) Close() error {
if w.isClosed {
return nil
}
w.isClosed = true
// Send "quit" message to the reader goroutine
ch := make(chan error)
w.quit <- ch
if err := w.wakeupReader(); err != nil {
return err
}
return <-ch
}
// Add starts watching the named file or directory (non-recursively).
func (w *Watcher) Add(name string) error {
if w.isClosed {
return errors.New("watcher already closed")
}
in := &input{
op: opAddWatch,
path: filepath.Clean(name),
flags: sysFSALLEVENTS,
reply: make(chan error),
}
w.input <- in
if err := w.wakeupReader(); err != nil {
return err
}
return <-in.reply
}
// Remove stops watching the the named file or directory (non-recursively).
func (w *Watcher) Remove(name string) error {
in := &input{
op: opRemoveWatch,
path: filepath.Clean(name),
reply: make(chan error),
}
w.input <- in
if err := w.wakeupReader(); err != nil {
return err
}
return <-in.reply
}
const (
// Options for AddWatch
sysFSONESHOT = 0x80000000
sysFSONLYDIR = 0x1000000
// Events
sysFSACCESS = 0x1
sysFSALLEVENTS = 0xfff
sysFSATTRIB = 0x4
sysFSCLOSE = 0x18
sysFSCREATE = 0x100
sysFSDELETE = 0x200
sysFSDELETESELF = 0x400
sysFSMODIFY = 0x2
sysFSMOVE = 0xc0
sysFSMOVEDFROM = 0x40
sysFSMOVEDTO = 0x80
sysFSMOVESELF = 0x800
// Special events
sysFSIGNORED = 0x8000
sysFSQOVERFLOW = 0x4000
)
func newEvent(name string, mask uint32) Event {
e := Event{Name: name}
if mask&sysFSCREATE == sysFSCREATE || mask&sysFSMOVEDTO == sysFSMOVEDTO {
e.Op |= Create
}
if mask&sysFSDELETE == sysFSDELETE || mask&sysFSDELETESELF == sysFSDELETESELF {
e.Op |= Remove
}
if mask&sysFSMODIFY == sysFSMODIFY {
e.Op |= Write
}
if mask&sysFSMOVE == sysFSMOVE || mask&sysFSMOVESELF == sysFSMOVESELF || mask&sysFSMOVEDFROM == sysFSMOVEDFROM {
e.Op |= Rename
}
if mask&sysFSATTRIB == sysFSATTRIB {
e.Op |= Chmod
}
return e
}
const (
opAddWatch = iota
opRemoveWatch
)
const (
provisional uint64 = 1 << (32 + iota)
)
type input struct {
op int
path string
flags uint32
reply chan error
}
type inode struct {
handle syscall.Handle
volume uint32
index uint64
}
type watch struct {
ov syscall.Overlapped
ino *inode // i-number
path string // Directory path
mask uint64 // Directory itself is being watched with these notify flags
names map[string]uint64 // Map of names being watched and their notify flags
rename string // Remembers the old name while renaming a file
buf [4096]byte
}
type indexMap map[uint64]*watch
type watchMap map[uint32]indexMap
func (w *Watcher) wakeupReader() error {
e := syscall.PostQueuedCompletionStatus(w.port, 0, 0, nil)
if e != nil {
return os.NewSyscallError("PostQueuedCompletionStatus", e)
}
return nil
}
func getDir(pathname string) (dir string, err error) {
attr, e := syscall.GetFileAttributes(syscall.StringToUTF16Ptr(pathname))
if e != nil {
return "", os.NewSyscallError("GetFileAttributes", e)
}
if attr&syscall.FILE_ATTRIBUTE_DIRECTORY != 0 {
dir = pathname
} else {
dir, _ = filepath.Split(pathname)
dir = filepath.Clean(dir)
}
return
}
func getIno(path string) (ino *inode, err error) {
h, e := syscall.CreateFile(syscall.StringToUTF16Ptr(path),
syscall.FILE_LIST_DIRECTORY,
syscall.FILE_SHARE_READ|syscall.FILE_SHARE_WRITE|syscall.FILE_SHARE_DELETE,
nil, syscall.OPEN_EXISTING,
syscall.FILE_FLAG_BACKUP_SEMANTICS|syscall.FILE_FLAG_OVERLAPPED, 0)
if e != nil {
return nil, os.NewSyscallError("CreateFile", e)
}
var fi syscall.ByHandleFileInformation
if e = syscall.GetFileInformationByHandle(h, &fi); e != nil {
syscall.CloseHandle(h)
return nil, os.NewSyscallError("GetFileInformationByHandle", e)
}
ino = &inode{
handle: h,
volume: fi.VolumeSerialNumber,
index: uint64(fi.FileIndexHigh)<<32 | uint64(fi.FileIndexLow),
}
return ino, nil
}
// Must run within the I/O thread.
func (m watchMap) get(ino *inode) *watch {
if i := m[ino.volume]; i != nil {
return i[ino.index]
}
return nil
}
// Must run within the I/O thread.
func (m watchMap) set(ino *inode, watch *watch) {
i := m[ino.volume]
if i == nil {
i = make(indexMap)
m[ino.volume] = i
}
i[ino.index] = watch
}
// Must run within the I/O thread.
func (w *Watcher) addWatch(pathname string, flags uint64) error {
dir, err := getDir(pathname)
if err != nil {
return err
}
if flags&sysFSONLYDIR != 0 && pathname != dir {
return nil
}
ino, err := getIno(dir)
if err != nil {
return err
}
w.mu.Lock()
watchEntry := w.watches.get(ino)
w.mu.Unlock()
if watchEntry == nil {
if _, e := syscall.CreateIoCompletionPort(ino.handle, w.port, 0, 0); e != nil {
syscall.CloseHandle(ino.handle)
return os.NewSyscallError("CreateIoCompletionPort", e)
}
watchEntry = &watch{
ino: ino,
path: dir,
names: make(map[string]uint64),
}
w.mu.Lock()
w.watches.set(ino, watchEntry)
w.mu.Unlock()
flags |= provisional
} else {
syscall.CloseHandle(ino.handle)
}
if pathname == dir {
watchEntry.mask |= flags
} else {
watchEntry.names[filepath.Base(pathname)] |= flags
}
if err = w.startRead(watchEntry); err != nil {
return err
}
if pathname == dir {
watchEntry.mask &= ^provisional
} else {
watchEntry.names[filepath.Base(pathname)] &= ^provisional
}
return nil
}
// Must run within the I/O thread.
func (w *Watcher) remWatch(pathname string) error {
dir, err := getDir(pathname)
if err != nil {
return err
}
ino, err := getIno(dir)
if err != nil {
return err
}
w.mu.Lock()
watch := w.watches.get(ino)
w.mu.Unlock()
if watch == nil {
return fmt.Errorf("can't remove non-existent watch for: %s", pathname)
}
if pathname == dir {
w.sendEvent(watch.path, watch.mask&sysFSIGNORED)
watch.mask = 0
} else {
name := filepath.Base(pathname)
w.sendEvent(filepath.Join(watch.path, name), watch.names[name]&sysFSIGNORED)
delete(watch.names, name)
}
return w.startRead(watch)
}
// Must run within the I/O thread.
func (w *Watcher) deleteWatch(watch *watch) {
for name, mask := range watch.names {
if mask&provisional == 0 {
w.sendEvent(filepath.Join(watch.path, name), mask&sysFSIGNORED)
}
delete(watch.names, name)
}
if watch.mask != 0 {
if watch.mask&provisional == 0 {
w.sendEvent(watch.path, watch.mask&sysFSIGNORED)
}
watch.mask = 0
}
}
// Must run within the I/O thread.
func (w *Watcher) startRead(watch *watch) error {
if e := syscall.CancelIo(watch.ino.handle); e != nil {
w.Errors <- os.NewSyscallError("CancelIo", e)
w.deleteWatch(watch)
}
mask := toWindowsFlags(watch.mask)
for _, m := range watch.names {
mask |= toWindowsFlags(m)
}
if mask == 0 {
if e := syscall.CloseHandle(watch.ino.handle); e != nil {
w.Errors <- os.NewSyscallError("CloseHandle", e)
}
w.mu.Lock()
delete(w.watches[watch.ino.volume], watch.ino.index)
w.mu.Unlock()
return nil
}
e := syscall.ReadDirectoryChanges(watch.ino.handle, &watch.buf[0],
uint32(unsafe.Sizeof(watch.buf)), false, mask, nil, &watch.ov, 0)
if e != nil {
err := os.NewSyscallError("ReadDirectoryChanges", e)
if e == syscall.ERROR_ACCESS_DENIED && watch.mask&provisional == 0 {
// Watched directory was probably removed
if w.sendEvent(watch.path, watch.mask&sysFSDELETESELF) {
if watch.mask&sysFSONESHOT != 0 {
watch.mask = 0
}
}
err = nil
}
w.deleteWatch(watch)
w.startRead(watch)
return err
}
return nil
}
// readEvents reads from the I/O completion port, converts the
// received events into Event objects and sends them via the Events channel.
// Entry point to the I/O thread.
func (w *Watcher) readEvents() {
var (
n, key uint32
ov *syscall.Overlapped
)
runtime.LockOSThread()
for {
e := syscall.GetQueuedCompletionStatus(w.port, &n, &key, &ov, syscall.INFINITE)
watch := (*watch)(unsafe.Pointer(ov))
if watch == nil {
select {
case ch := <-w.quit:
w.mu.Lock()
var indexes []indexMap
for _, index := range w.watches {
indexes = append(indexes, index)
}
w.mu.Unlock()
for _, index := range indexes {
for _, watch := range index {
w.deleteWatch(watch)
w.startRead(watch)
}
}
var err error
if e := syscall.CloseHandle(w.port); e != nil {
err = os.NewSyscallError("CloseHandle", e)
}
close(w.Events)
close(w.Errors)
ch <- err
return
case in := <-w.input:
switch in.op {
case opAddWatch:
in.reply <- w.addWatch(in.path, uint64(in.flags))
case opRemoveWatch:
in.reply <- w.remWatch(in.path)
}
default:
}
continue
}
switch e {
case syscall.ERROR_MORE_DATA:
if watch == nil {
w.Errors <- errors.New("ERROR_MORE_DATA has unexpectedly null lpOverlapped buffer")
} else {
// The i/o succeeded but the buffer is full.
// In theory we should be building up a full packet.
// In practice we can get away with just carrying on.
n = uint32(unsafe.Sizeof(watch.buf))
}
case syscall.ERROR_ACCESS_DENIED:
// Watched directory was probably removed
w.sendEvent(watch.path, watch.mask&sysFSDELETESELF)
w.deleteWatch(watch)
w.startRead(watch)
continue
case syscall.ERROR_OPERATION_ABORTED:
// CancelIo was called on this handle
continue
default:
w.Errors <- os.NewSyscallError("GetQueuedCompletionPort", e)
continue
case nil:
}
var offset uint32
for {
if n == 0 {
w.Events <- newEvent("", sysFSQOVERFLOW)
w.Errors <- errors.New("short read in readEvents()")
break
}
// Point "raw" to the event in the buffer
raw := (*syscall.FileNotifyInformation)(unsafe.Pointer(&watch.buf[offset]))
buf := (*[syscall.MAX_PATH]uint16)(unsafe.Pointer(&raw.FileName))
name := syscall.UTF16ToString(buf[:raw.FileNameLength/2])
fullname := filepath.Join(watch.path, name)
var mask uint64
switch raw.Action {
case syscall.FILE_ACTION_REMOVED:
mask = sysFSDELETESELF
case syscall.FILE_ACTION_MODIFIED:
mask = sysFSMODIFY
case syscall.FILE_ACTION_RENAMED_OLD_NAME:
watch.rename = name
case syscall.FILE_ACTION_RENAMED_NEW_NAME:
if watch.names[watch.rename] != 0 {
watch.names[name] |= watch.names[watch.rename]
delete(watch.names, watch.rename)
mask = sysFSMOVESELF
}
}
sendNameEvent := func() {
if w.sendEvent(fullname, watch.names[name]&mask) {
if watch.names[name]&sysFSONESHOT != 0 {
delete(watch.names, name)
}
}
}
if raw.Action != syscall.FILE_ACTION_RENAMED_NEW_NAME {
sendNameEvent()
}
if raw.Action == syscall.FILE_ACTION_REMOVED {
w.sendEvent(fullname, watch.names[name]&sysFSIGNORED)
delete(watch.names, name)
}
if w.sendEvent(fullname, watch.mask&toFSnotifyFlags(raw.Action)) {
if watch.mask&sysFSONESHOT != 0 {
watch.mask = 0
}
}
if raw.Action == syscall.FILE_ACTION_RENAMED_NEW_NAME {
fullname = filepath.Join(watch.path, watch.rename)
sendNameEvent()
}
// Move to the next event in the buffer
if raw.NextEntryOffset == 0 {
break
}
offset += raw.NextEntryOffset
// Error!
if offset >= n {
w.Errors <- errors.New("Windows system assumed buffer larger than it is, events have likely been missed.")
break
}
}
if err := w.startRead(watch); err != nil {
w.Errors <- err
}
}
}
func (w *Watcher) sendEvent(name string, mask uint64) bool {
if mask == 0 {
return false
}
event := newEvent(name, uint32(mask))
select {
case ch := <-w.quit:
w.quit <- ch
case w.Events <- event:
}
return true
}
func toWindowsFlags(mask uint64) uint32 {
var m uint32
if mask&sysFSACCESS != 0 {
m |= syscall.FILE_NOTIFY_CHANGE_LAST_ACCESS
}
if mask&sysFSMODIFY != 0 {
m |= syscall.FILE_NOTIFY_CHANGE_LAST_WRITE
}
if mask&sysFSATTRIB != 0 {
m |= syscall.FILE_NOTIFY_CHANGE_ATTRIBUTES
}
if mask&(sysFSMOVE|sysFSCREATE|sysFSDELETE) != 0 {
m |= syscall.FILE_NOTIFY_CHANGE_FILE_NAME | syscall.FILE_NOTIFY_CHANGE_DIR_NAME
}
return m
}
func toFSnotifyFlags(action uint32) uint64 {
switch action {
case syscall.FILE_ACTION_ADDED:
return sysFSCREATE
case syscall.FILE_ACTION_REMOVED:
return sysFSDELETE
case syscall.FILE_ACTION_MODIFIED:
return sysFSMODIFY
case syscall.FILE_ACTION_RENAMED_OLD_NAME:
return sysFSMOVEDFROM
case syscall.FILE_ACTION_RENAMED_NEW_NAME:
return sysFSMOVEDTO
}
return 0
}

1
vendor/modules.txt vendored
View file

@ -319,7 +319,6 @@ github.com/fernet/fernet-go
github.com/fluent/fluent-logger-golang/fluent
# github.com/fsnotify/fsnotify v1.5.1
## explicit; go 1.13
github.com/fsnotify/fsnotify
# github.com/go-logr/logr v1.2.2
## explicit; go 1.16
github.com/go-logr/logr