diff --git a/daemon/logger/adapter_test.go b/daemon/logger/adapter_test.go index 1ca28de6d9..51fb475b1d 100644 --- a/daemon/logger/adapter_test.go +++ b/daemon/logger/adapter_test.go @@ -174,7 +174,7 @@ func TestAdapterReadLogs(t *testing.T) { t.Fatal("timeout waiting for message channel to close") } - lw.ProducerGone() + lw.ConsumerGone() lw = lr.ReadLogs(ReadConfig{Follow: true}) for _, x := range testMsg { diff --git a/daemon/logger/journald/journald.go b/daemon/logger/journald/journald.go index dd7414e2ee..3de3ca7e39 100644 --- a/daemon/logger/journald/journald.go +++ b/daemon/logger/journald/journald.go @@ -8,7 +8,6 @@ package journald // import "github.com/docker/docker/daemon/logger/journald" import ( "fmt" "strconv" - "sync" "unicode" "github.com/coreos/go-systemd/v22/journal" @@ -19,9 +18,9 @@ import ( const name = "journald" type journald struct { - mu sync.Mutex //nolint:structcheck,unused - vars map[string]string // additional variables and values to send to the journal along with the log message - readers map[*logger.LogWatcher]struct{} + vars map[string]string // additional variables and values to send to the journal along with the log message + + closed chan struct{} } func init() { @@ -81,7 +80,7 @@ func New(info logger.Info) (logger.Logger, error) { for k, v := range extraAttrs { vars[k] = v } - return &journald{vars: vars, readers: make(map[*logger.LogWatcher]struct{})}, nil + return &journald{vars: vars, closed: make(chan struct{})}, nil } // We don't actually accept any options, but we have to supply a callback for @@ -128,3 +127,8 @@ func (s *journald) Log(msg *logger.Message) error { func (s *journald) Name() string { return name } + +func (s *journald) Close() error { + close(s.closed) + return nil +} diff --git a/daemon/logger/journald/read.go b/daemon/logger/journald/read.go index 5a43ecf4ac..eb299e2c5e 100644 --- a/daemon/logger/journald/read.go +++ b/daemon/logger/journald/read.go @@ -116,16 +116,6 @@ import ( "github.com/sirupsen/logrus" ) -func (s *journald) Close() error { - s.mu.Lock() - for r := range s.readers { - r.ProducerGone() - delete(s.readers, r) - } - s.mu.Unlock() - return nil -} - // CErr converts error code returned from a sd_journal_* function // (which returns -errno) to a string func CErr(ret C.int) string { @@ -233,9 +223,7 @@ drain: } func (s *journald) followJournal(logWatcher *logger.LogWatcher, j *C.sd_journal, cursor *C.char, untilUnixMicro uint64) *C.char { - s.mu.Lock() - s.readers[logWatcher] = struct{}{} - s.mu.Unlock() + defer close(logWatcher.Msg) waitTimeout := C.uint64_t(250000) // 0.25s @@ -243,12 +231,12 @@ func (s *journald) followJournal(logWatcher *logger.LogWatcher, j *C.sd_journal, status := C.sd_journal_wait(j, waitTimeout) if status < 0 { logWatcher.Err <- errors.New("error waiting for journal: " + CErr(status)) - goto cleanup + break } select { case <-logWatcher.WatchConsumerGone(): - goto cleanup // won't be able to write anything anymore - case <-logWatcher.WatchProducerGone(): + break // won't be able to write anything anymore + case <-s.closed: // container is gone, drain journal default: // container is still alive @@ -264,11 +252,6 @@ func (s *journald) followJournal(logWatcher *logger.LogWatcher, j *C.sd_journal, } } -cleanup: - s.mu.Lock() - delete(s.readers, logWatcher) - s.mu.Unlock() - close(logWatcher.Msg) return cursor } diff --git a/daemon/logger/journald/read_unsupported.go b/daemon/logger/journald/read_unsupported.go deleted file mode 100644 index 199d7683a6..0000000000 --- a/daemon/logger/journald/read_unsupported.go +++ /dev/null @@ -1,8 +0,0 @@ -//go:build !linux || !cgo || static_build || !journald -// +build !linux !cgo static_build !journald - -package journald // import "github.com/docker/docker/daemon/logger/journald" - -func (s *journald) Close() error { - return nil -} diff --git a/daemon/logger/jsonfilelog/jsonfilelog.go b/daemon/logger/jsonfilelog/jsonfilelog.go index 385f63abea..8ef82bbc33 100644 --- a/daemon/logger/jsonfilelog/jsonfilelog.go +++ b/daemon/logger/jsonfilelog/jsonfilelog.go @@ -8,7 +8,6 @@ import ( "encoding/json" "fmt" "strconv" - "sync" "github.com/docker/docker/daemon/logger" "github.com/docker/docker/daemon/logger/jsonfilelog/jsonlog" @@ -22,11 +21,8 @@ const Name = "json-file" // JSONFileLogger is Logger implementation for default Docker logging. type JSONFileLogger struct { - mu sync.Mutex - closed bool - writer *loggerutils.LogFile - readers map[*logger.LogWatcher]struct{} // stores the active log followers - tag string // tag values requested by the user to log + writer *loggerutils.LogFile + tag string // tag values requested by the user to log } func init() { @@ -115,18 +111,14 @@ func New(info logger.Info) (logger.Logger, error) { } return &JSONFileLogger{ - writer: writer, - readers: make(map[*logger.LogWatcher]struct{}), - tag: tag, + writer: writer, + tag: tag, }, nil } // Log converts logger.Message to jsonlog.JSONLog and serializes it to file. func (l *JSONFileLogger) Log(msg *logger.Message) error { - l.mu.Lock() - err := l.writer.WriteLogEntry(msg) - l.mu.Unlock() - return err + return l.writer.WriteLogEntry(msg) } func marshalMessage(msg *logger.Message, extra json.RawMessage, buf *bytes.Buffer) error { @@ -169,15 +161,7 @@ func ValidateLogOpt(cfg map[string]string) error { // Close closes underlying file and signals all the readers // that the logs producer is gone. func (l *JSONFileLogger) Close() error { - l.mu.Lock() - l.closed = true - err := l.writer.Close() - for r := range l.readers { - r.ProducerGone() - delete(l.readers, r) - } - l.mu.Unlock() - return err + return l.writer.Close() } // Name returns name of this logger. diff --git a/daemon/logger/jsonfilelog/read.go b/daemon/logger/jsonfilelog/read.go index 5099a0b947..bea8ceedb3 100644 --- a/daemon/logger/jsonfilelog/read.go +++ b/daemon/logger/jsonfilelog/read.go @@ -10,32 +10,12 @@ import ( "github.com/docker/docker/daemon/logger/jsonfilelog/jsonlog" "github.com/docker/docker/daemon/logger/loggerutils" "github.com/docker/docker/pkg/tailfile" - "github.com/sirupsen/logrus" ) -const maxJSONDecodeRetry = 20000 - // ReadLogs implements the logger's LogReader interface for the logs // created by this driver. func (l *JSONFileLogger) ReadLogs(config logger.ReadConfig) *logger.LogWatcher { - logWatcher := logger.NewLogWatcher() - - go l.readLogs(logWatcher, config) - return logWatcher -} - -func (l *JSONFileLogger) readLogs(watcher *logger.LogWatcher, config logger.ReadConfig) { - defer close(watcher.Msg) - - l.mu.Lock() - l.readers[watcher] = struct{}{} - l.mu.Unlock() - - l.writer.ReadLogs(config, watcher) - - l.mu.Lock() - delete(l.readers, watcher) - l.mu.Unlock() + return l.writer.ReadLogs(config) } func decodeLogLine(dec *json.Decoder, l *jsonlog.JSONLog) (*logger.Message, error) { @@ -61,10 +41,9 @@ func decodeLogLine(dec *json.Decoder, l *jsonlog.JSONLog) (*logger.Message, erro } type decoder struct { - rdr io.Reader - dec *json.Decoder - jl *jsonlog.JSONLog - maxRetry int + rdr io.Reader + dec *json.Decoder + jl *jsonlog.JSONLog } func (d *decoder) Reset(rdr io.Reader) { @@ -88,74 +67,7 @@ func (d *decoder) Decode() (msg *logger.Message, err error) { if d.jl == nil { d.jl = &jsonlog.JSONLog{} } - if d.maxRetry == 0 { - // We aren't using maxJSONDecodeRetry directly so we can give a custom value for testing. - d.maxRetry = maxJSONDecodeRetry - } - for retries := 0; retries < d.maxRetry; retries++ { - msg, err = decodeLogLine(d.dec, d.jl) - if err == nil || err == io.EOF { - break - } - - logrus.WithError(err).WithField("retries", retries).Warn("got error while decoding json") - // try again, could be due to a an incomplete json object as we read - if _, ok := err.(*json.SyntaxError); ok { - d.dec = json.NewDecoder(d.rdr) - continue - } - - // io.ErrUnexpectedEOF is returned from json.Decoder when there is - // remaining data in the parser's buffer while an io.EOF occurs. - // If the json logger writes a partial json log entry to the disk - // while at the same time the decoder tries to decode it, the race condition happens. - if err == io.ErrUnexpectedEOF { - d.rdr = combineReaders(d.dec.Buffered(), d.rdr) - d.dec = json.NewDecoder(d.rdr) - continue - } - } - return msg, err -} - -func combineReaders(pre, rdr io.Reader) io.Reader { - return &combinedReader{pre: pre, rdr: rdr} -} - -// combinedReader is a reader which is like `io.MultiReader` where except it does not cache a full EOF. -// Once `io.MultiReader` returns EOF, it is always EOF. -// -// For this usecase we have an underlying reader which is a file which may reach EOF but have more data written to it later. -// As such, io.MultiReader does not work for us. -type combinedReader struct { - pre io.Reader - rdr io.Reader -} - -func (r *combinedReader) Read(p []byte) (int, error) { - var read int - if r.pre != nil { - n, err := r.pre.Read(p) - if err != nil { - if err != io.EOF { - return n, err - } - r.pre = nil - } - read = n - } - - if read < len(p) { - n, err := r.rdr.Read(p[read:]) - if n > 0 { - read += n - } - if err != nil { - return read, err - } - } - - return read, nil + return decodeLogLine(d.dec, d.jl) } // decodeFunc is used to create a decoder for the log file reader diff --git a/daemon/logger/jsonfilelog/read_test.go b/daemon/logger/jsonfilelog/read_test.go index 1f0a295f63..dd56be0ff3 100644 --- a/daemon/logger/jsonfilelog/read_test.go +++ b/daemon/logger/jsonfilelog/read_test.go @@ -1,24 +1,28 @@ package jsonfilelog // import "github.com/docker/docker/daemon/logger/jsonfilelog" import ( + "bufio" "bytes" - "encoding/json" + "fmt" "io" + "os" + "path/filepath" + "strconv" "testing" + "text/tabwriter" "time" "github.com/docker/docker/daemon/logger" + "github.com/docker/docker/daemon/logger/loggertest" "gotest.tools/v3/assert" - "gotest.tools/v3/fs" ) func BenchmarkJSONFileLoggerReadLogs(b *testing.B) { - tmp := fs.NewDir(b, "bench-jsonfilelog") - defer tmp.Remove() + tmp := b.TempDir() jsonlogger, err := New(logger.Info{ ContainerID: "a7317399f3f857173c6179d44823594f8294678dea9999662e5c625b5a1c7657", - LogPath: tmp.Join("container.log"), + LogPath: filepath.Join(tmp, "container.log"), Config: map[string]string{ "labels": "first,second", }, @@ -30,36 +34,43 @@ func BenchmarkJSONFileLoggerReadLogs(b *testing.B) { assert.NilError(b, err) defer jsonlogger.Close() - msg := &logger.Message{ - Line: []byte("Line that thinks that it is log line from docker\n"), - Source: "stderr", - Timestamp: time.Now().UTC(), + const line = "Line that thinks that it is log line from docker\n" + ts := time.Date(2007, 1, 2, 3, 4, 5, 6, time.UTC) + msg := func() *logger.Message { + m := logger.NewMessage() + m.Line = append(m.Line, line...) + m.Source = "stderr" + m.Timestamp = ts + return m } - buf := bytes.NewBuffer(nil) - assert.NilError(b, marshalMessage(msg, nil, buf)) + var buf bytes.Buffer + assert.NilError(b, marshalMessage(msg(), nil, &buf)) b.SetBytes(int64(buf.Len())) b.ResetTimer() - chError := make(chan error, b.N+1) + chError := make(chan error) go func() { for i := 0; i < b.N; i++ { - chError <- jsonlogger.Log(msg) + if err := jsonlogger.Log(msg()); err != nil { + chError <- err + } + } + if err := jsonlogger.Close(); err != nil { + chError <- err } - chError <- jsonlogger.Close() }() lw := jsonlogger.(*JSONFileLogger).ReadLogs(logger.ReadConfig{Follow: true}) for { select { - case <-lw.Msg: - case <-lw.WatchProducerGone(): - return - case err := <-chError: - if err != nil { - b.Fatal(err) + case _, ok := <-lw.Msg: + if !ok { + return } + case err := <-chError: + b.Fatal(err) } } } @@ -95,60 +106,111 @@ func TestEncodeDecode(t *testing.T) { assert.Assert(t, err == io.EOF) } -func TestUnexpectedEOF(t *testing.T) { +func TestReadLogs(t *testing.T) { + t.Parallel() + r := loggertest.Reader{ + Factory: func(t *testing.T, info logger.Info) func(*testing.T) logger.Logger { + dir := t.TempDir() + info.LogPath = filepath.Join(dir, info.ContainerID+".log") + return func(t *testing.T) logger.Logger { + l, err := New(info) + assert.NilError(t, err) + return l + } + }, + } + t.Run("Tail", r.TestTail) + t.Run("Follow", r.TestFollow) +} + +func TestTailLogsWithRotation(t *testing.T) { + t.Parallel() + compress := func(cmprs bool) { + t.Run(fmt.Sprintf("compress=%v", cmprs), func(t *testing.T) { + t.Parallel() + (&loggertest.Reader{ + Factory: func(t *testing.T, info logger.Info) func(*testing.T) logger.Logger { + info.Config = map[string]string{ + "compress": strconv.FormatBool(cmprs), + "max-size": "1b", + "max-file": "10", + } + dir := t.TempDir() + t.Cleanup(func() { + t.Logf("%s:\n%s", t.Name(), dirStringer{dir}) + }) + info.LogPath = filepath.Join(dir, info.ContainerID+".log") + return func(t *testing.T) logger.Logger { + l, err := New(info) + assert.NilError(t, err) + return l + } + }, + }).TestTail(t) + }) + } + compress(true) + compress(false) +} + +func TestFollowLogsWithRotation(t *testing.T) { + t.Parallel() + compress := func(cmprs bool) { + t.Run(fmt.Sprintf("compress=%v", cmprs), func(t *testing.T) { + t.Parallel() + (&loggertest.Reader{ + Factory: func(t *testing.T, info logger.Info) func(*testing.T) logger.Logger { + // The log follower can fall behind and drop logs if there are too many + // rotations in a short time. If that was to happen, loggertest would fail the + // test. Configure the logger so that there will be only one rotation with the + // set of logs that loggertest writes. + info.Config = map[string]string{ + "compress": strconv.FormatBool(cmprs), + "max-size": "4096b", + "max-file": "3", + } + dir := t.TempDir() + t.Cleanup(func() { + t.Logf("%s:\n%s", t.Name(), dirStringer{dir}) + }) + info.LogPath = filepath.Join(dir, info.ContainerID+".log") + return func(t *testing.T) logger.Logger { + l, err := New(info) + assert.NilError(t, err) + return l + } + }, + }).TestFollow(t) + }) + } + compress(true) + compress(false) +} + +type dirStringer struct { + d string +} + +func (d dirStringer) String() string { + ls, err := os.ReadDir(d.d) + if err != nil { + return "" + } buf := bytes.NewBuffer(nil) - msg1 := &logger.Message{Timestamp: time.Now(), Line: []byte("hello1")} - msg2 := &logger.Message{Timestamp: time.Now(), Line: []byte("hello2")} + tw := tabwriter.NewWriter(buf, 1, 8, 1, '\t', 0) + buf.WriteString("\n") - err := marshalMessage(msg1, json.RawMessage{}, buf) - assert.NilError(t, err) - err = marshalMessage(msg2, json.RawMessage{}, buf) - assert.NilError(t, err) + btw := bufio.NewWriter(tw) - r := &readerWithErr{ - err: io.EOF, - after: buf.Len() / 4, - r: buf, + for _, entry := range ls { + fi, err := entry.Info() + if err != nil { + return "" + } + + btw.WriteString(fmt.Sprintf("%s\t%s\t%dB\t%s\n", fi.Name(), fi.Mode(), fi.Size(), fi.ModTime())) } - dec := &decoder{rdr: r, maxRetry: 1} - - _, err = dec.Decode() - assert.Error(t, err, io.ErrUnexpectedEOF.Error()) - // again just to check - _, err = dec.Decode() - assert.Error(t, err, io.ErrUnexpectedEOF.Error()) - - // reset the error - // from here all reads should succeed until we get EOF on the underlying reader - r.err = nil - - msg, err := dec.Decode() - assert.NilError(t, err) - assert.Equal(t, string(msg1.Line)+"\n", string(msg.Line)) - - msg, err = dec.Decode() - assert.NilError(t, err) - assert.Equal(t, string(msg2.Line)+"\n", string(msg.Line)) - - _, err = dec.Decode() - assert.Error(t, err, io.EOF.Error()) -} - -type readerWithErr struct { - err error - after int - r io.Reader - read int -} - -func (r *readerWithErr) Read(p []byte) (int, error) { - if r.err != nil && r.read > r.after { - return 0, r.err - } - - n, err := r.r.Read(p[:1]) - if n > 0 { - r.read += n - } - return n, err + btw.Flush() + tw.Flush() + return buf.String() } diff --git a/daemon/logger/local/local.go b/daemon/logger/local/local.go index 169d2d38b8..fe1ef083f7 100644 --- a/daemon/logger/local/local.go +++ b/daemon/logger/local/local.go @@ -4,7 +4,6 @@ import ( "encoding/binary" "io" "strconv" - "sync" "time" "github.com/docker/docker/api/types/backend" @@ -56,10 +55,7 @@ func init() { } type driver struct { - mu sync.Mutex - closed bool logfile *loggerutils.LogFile - readers map[*logger.LogWatcher]struct{} // stores the active log followers } // New creates a new local logger @@ -145,7 +141,6 @@ func newDriver(logPath string, cfg *CreateConfig) (logger.Logger, error) { } return &driver{ logfile: lf, - readers: make(map[*logger.LogWatcher]struct{}), }, nil } @@ -154,22 +149,11 @@ func (d *driver) Name() string { } func (d *driver) Log(msg *logger.Message) error { - d.mu.Lock() - err := d.logfile.WriteLogEntry(msg) - d.mu.Unlock() - return err + return d.logfile.WriteLogEntry(msg) } func (d *driver) Close() error { - d.mu.Lock() - d.closed = true - err := d.logfile.Close() - for r := range d.readers { - r.ProducerGone() - delete(d.readers, r) - } - d.mu.Unlock() - return err + return d.logfile.Close() } func messageToProto(msg *logger.Message, proto *logdriver.LogEntry, partial *logdriver.PartialLogEntryMetadata) { diff --git a/daemon/logger/local/local_test.go b/daemon/logger/local/local_test.go index ca31318499..28de7d2485 100644 --- a/daemon/logger/local/local_test.go +++ b/daemon/logger/local/local_test.go @@ -2,19 +2,18 @@ package local import ( "bytes" - "context" "encoding/binary" "fmt" "io" "os" "path/filepath" - "strings" "testing" "time" "github.com/docker/docker/api/types/backend" "github.com/docker/docker/api/types/plugins/logdriver" "github.com/docker/docker/daemon/logger" + "github.com/docker/docker/daemon/logger/loggertest" protoio "github.com/gogo/protobuf/io" "gotest.tools/v3/assert" is "gotest.tools/v3/assert/cmp" @@ -80,92 +79,19 @@ func TestWriteLog(t *testing.T) { } func TestReadLog(t *testing.T) { - t.Parallel() - - dir, err := os.MkdirTemp("", t.Name()) - assert.NilError(t, err) - defer os.RemoveAll(dir) - - logPath := filepath.Join(dir, "test.log") - l, err := New(logger.Info{LogPath: logPath}) - assert.NilError(t, err) - defer l.Close() - - m1 := logger.Message{Source: "stdout", Timestamp: time.Now().Add(-1 * 30 * time.Minute), Line: []byte("a message")} - m2 := logger.Message{Source: "stdout", Timestamp: time.Now().Add(-1 * 20 * time.Minute), Line: []byte("another message"), PLogMetaData: &backend.PartialLogMetaData{Ordinal: 1, Last: true}} - longMessage := []byte("a really long message " + strings.Repeat("a", initialBufSize*2)) - m3 := logger.Message{Source: "stderr", Timestamp: time.Now().Add(-1 * 10 * time.Minute), Line: longMessage} - m4 := logger.Message{Source: "stderr", Timestamp: time.Now().Add(-1 * 10 * time.Minute), Line: []byte("just one more message")} - - // copy the log message because the underlying log writer resets the log message and returns it to a buffer pool - err = l.Log(copyLogMessage(&m1)) - assert.NilError(t, err) - err = l.Log(copyLogMessage(&m2)) - assert.NilError(t, err) - err = l.Log(copyLogMessage(&m3)) - assert.NilError(t, err) - err = l.Log(copyLogMessage(&m4)) - assert.NilError(t, err) - - lr := l.(logger.LogReader) - - testMessage := func(t *testing.T, lw *logger.LogWatcher, m *logger.Message) { - t.Helper() - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) - defer cancel() - select { - case <-ctx.Done(): - assert.Assert(t, ctx.Err()) - case err := <-lw.Err: - assert.NilError(t, err) - case msg, open := <-lw.Msg: - if !open { - select { - case err := <-lw.Err: - assert.NilError(t, err) - default: - assert.Assert(t, m == nil) - return - } + r := loggertest.Reader{ + Factory: func(t *testing.T, info logger.Info) func(*testing.T) logger.Logger { + dir := t.TempDir() + info.LogPath = filepath.Join(dir, info.ContainerID+".log") + return func(t *testing.T) logger.Logger { + l, err := New(info) + assert.NilError(t, err) + return l } - assert.Assert(t, m != nil) - if m.PLogMetaData == nil { - // a `\n` is appended on read to make this work with the existing API's when the message is not a partial. - // make sure it's the last entry in the line, and then truncate it for the deep equal below. - assert.Check(t, msg.Line[len(msg.Line)-1] == '\n') - msg.Line = msg.Line[:len(msg.Line)-1] - } - assert.Check(t, is.DeepEqual(m, msg), fmt.Sprintf("\n%+v\n%+v", m, msg)) - } + }, } - - t.Run("tail exact", func(t *testing.T) { - lw := lr.ReadLogs(logger.ReadConfig{Tail: 4}) - - testMessage(t, lw, &m1) - testMessage(t, lw, &m2) - testMessage(t, lw, &m3) - testMessage(t, lw, &m4) - testMessage(t, lw, nil) // no more messages - }) - - t.Run("tail less than available", func(t *testing.T) { - lw := lr.ReadLogs(logger.ReadConfig{Tail: 2}) - - testMessage(t, lw, &m3) - testMessage(t, lw, &m4) - testMessage(t, lw, nil) // no more messages - }) - - t.Run("tail more than available", func(t *testing.T) { - lw := lr.ReadLogs(logger.ReadConfig{Tail: 100}) - - testMessage(t, lw, &m1) - testMessage(t, lw, &m2) - testMessage(t, lw, &m3) - testMessage(t, lw, &m4) - testMessage(t, lw, nil) // no more messages - }) + t.Run("Tail", r.TestTail) + t.Run("Follow", r.TestFollow) } func BenchmarkLogWrite(b *testing.B) { diff --git a/daemon/logger/local/read.go b/daemon/logger/local/read.go index d517995cff..6a0b166103 100644 --- a/daemon/logger/local/read.go +++ b/daemon/logger/local/read.go @@ -19,24 +19,7 @@ import ( const maxMsgLen int = 1e6 // 1MB. func (d *driver) ReadLogs(config logger.ReadConfig) *logger.LogWatcher { - logWatcher := logger.NewLogWatcher() - - go d.readLogs(logWatcher, config) - return logWatcher -} - -func (d *driver) readLogs(watcher *logger.LogWatcher, config logger.ReadConfig) { - defer close(watcher.Msg) - - d.mu.Lock() - d.readers[watcher] = struct{}{} - d.mu.Unlock() - - d.logfile.ReadLogs(config, watcher) - - d.mu.Lock() - delete(d.readers, watcher) - d.mu.Unlock() + return d.logfile.ReadLogs(config) } func getTailReader(ctx context.Context, r loggerutils.SizeReaderAt, req int) (io.Reader, int, error) { @@ -219,7 +202,7 @@ func (d *decoder) decodeLogEntry() (*logger.Message, error) { } msg := protoToMessage(d.proto) - if msg.PLogMetaData == nil { + if msg.PLogMetaData == nil || msg.PLogMetaData.Last { msg.Line = append(msg.Line, '\n') } diff --git a/daemon/logger/logger.go b/daemon/logger/logger.go index 15899e07ed..05ea1b3af0 100644 --- a/daemon/logger/logger.go +++ b/daemon/logger/logger.go @@ -97,8 +97,6 @@ type LogWatcher struct { Msg chan *Message // For sending error messages that occur while reading logs. Err chan error - producerOnce sync.Once - producerGone chan struct{} consumerOnce sync.Once consumerGone chan struct{} } @@ -108,26 +106,10 @@ func NewLogWatcher() *LogWatcher { return &LogWatcher{ Msg: make(chan *Message, logWatcherBufferSize), Err: make(chan error, 1), - producerGone: make(chan struct{}), consumerGone: make(chan struct{}), } } -// ProducerGone notifies the underlying log reader that -// the logs producer (a container) is gone. -func (w *LogWatcher) ProducerGone() { - // only close if not already closed - w.producerOnce.Do(func() { - close(w.producerGone) - }) -} - -// WatchProducerGone returns a channel receiver that receives notification -// once the logs producer (a container) is gone. -func (w *LogWatcher) WatchProducerGone() <-chan struct{} { - return w.producerGone -} - // ConsumerGone notifies that the logs consumer is gone. func (w *LogWatcher) ConsumerGone() { // only close if not already closed diff --git a/daemon/logger/loggertest/logreader.go b/daemon/logger/loggertest/logreader.go new file mode 100644 index 0000000000..f494602537 --- /dev/null +++ b/daemon/logger/loggertest/logreader.go @@ -0,0 +1,461 @@ +package loggertest // import "github.com/docker/docker/daemon/logger/loggertest" + +import ( + "runtime" + "strings" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "gotest.tools/v3/assert" + + "github.com/docker/docker/api/types/backend" + "github.com/docker/docker/daemon/logger" +) + +// Reader tests that a logger.LogReader implementation behaves as it should. +type Reader struct { + // Factory returns a function which constructs loggers for the container + // specified in info. Each call to the returned function must yield a + // distinct logger instance which can read back logs written by earlier + // instances. + Factory func(*testing.T, logger.Info) func(*testing.T) logger.Logger +} + +var compareLog cmp.Options = []cmp.Option{ + // The json-log driver does not round-trip PLogMetaData and API users do + // not expect it. + cmpopts.IgnoreFields(logger.Message{}, "PLogMetaData"), + cmp.Transformer("string", func(b []byte) string { return string(b) }), +} + +// TestTail tests the behavior of the LogReader's tail implementation. +func (tr Reader) TestTail(t *testing.T) { + t.Run("Live", func(t *testing.T) { tr.testTail(t, true) }) + t.Run("LiveEmpty", func(t *testing.T) { tr.testTailEmptyLogs(t, true) }) + t.Run("Stopped", func(t *testing.T) { tr.testTail(t, false) }) + t.Run("StoppedEmpty", func(t *testing.T) { tr.testTailEmptyLogs(t, false) }) +} + +func makeTestMessages() []*logger.Message { + return []*logger.Message{ + {Source: "stdout", Timestamp: time.Now().Add(-1 * 30 * time.Minute), Line: []byte("a message")}, + {Source: "stdout", Timestamp: time.Now().Add(-1 * 20 * time.Minute), Line: []byte("another message"), PLogMetaData: &backend.PartialLogMetaData{ID: "aaaaaaaa", Ordinal: 1, Last: true}}, + {Source: "stderr", Timestamp: time.Now().Add(-1 * 15 * time.Minute), Line: []byte("to be..."), PLogMetaData: &backend.PartialLogMetaData{ID: "bbbbbbbb", Ordinal: 1}}, + {Source: "stderr", Timestamp: time.Now().Add(-1 * 15 * time.Minute), Line: []byte("continued"), PLogMetaData: &backend.PartialLogMetaData{ID: "bbbbbbbb", Ordinal: 2, Last: true}}, + {Source: "stderr", Timestamp: time.Now().Add(-1 * 10 * time.Minute), Line: []byte("a really long message " + strings.Repeat("a", 4096))}, + {Source: "stderr", Timestamp: time.Now().Add(-1 * 10 * time.Minute), Line: []byte("just one more message")}, + } + +} + +func (tr Reader) testTail(t *testing.T, live bool) { + t.Parallel() + factory := tr.Factory(t, logger.Info{ + ContainerID: "tailtest0000", + ContainerName: "logtail", + }) + l := factory(t) + if live { + defer func() { assert.NilError(t, l.Close()) }() + } + + mm := makeTestMessages() + expected := logMessages(t, l, mm) + + if !live { + // Simulate reading from a stopped container. + assert.NilError(t, l.Close()) + l = factory(t) + defer func() { assert.NilError(t, l.Close()) }() + } + lr := l.(logger.LogReader) + + t.Run("Exact", func(t *testing.T) { + t.Parallel() + lw := lr.ReadLogs(logger.ReadConfig{Tail: len(mm)}) + defer lw.ConsumerGone() + assert.DeepEqual(t, readAll(t, lw), expected, compareLog) + }) + + t.Run("LessThanAvailable", func(t *testing.T) { + t.Parallel() + lw := lr.ReadLogs(logger.ReadConfig{Tail: 2}) + defer lw.ConsumerGone() + assert.DeepEqual(t, readAll(t, lw), expected[len(mm)-2:], compareLog) + }) + + t.Run("MoreThanAvailable", func(t *testing.T) { + t.Parallel() + lw := lr.ReadLogs(logger.ReadConfig{Tail: 100}) + defer lw.ConsumerGone() + assert.DeepEqual(t, readAll(t, lw), expected, compareLog) + }) + + t.Run("All", func(t *testing.T) { + t.Parallel() + lw := lr.ReadLogs(logger.ReadConfig{Tail: -1}) + defer lw.ConsumerGone() + assert.DeepEqual(t, readAll(t, lw), expected, compareLog) + }) + + t.Run("Since", func(t *testing.T) { + t.Parallel() + lw := lr.ReadLogs(logger.ReadConfig{Tail: -1, Since: mm[1].Timestamp.Truncate(time.Millisecond)}) + defer lw.ConsumerGone() + assert.DeepEqual(t, readAll(t, lw), expected[1:], compareLog) + }) + + t.Run("MoreThanSince", func(t *testing.T) { + t.Parallel() + lw := lr.ReadLogs(logger.ReadConfig{Tail: len(mm), Since: mm[1].Timestamp.Truncate(time.Millisecond)}) + defer lw.ConsumerGone() + assert.DeepEqual(t, readAll(t, lw), expected[1:], compareLog) + }) + + t.Run("LessThanSince", func(t *testing.T) { + t.Parallel() + lw := lr.ReadLogs(logger.ReadConfig{Tail: len(mm) - 2, Since: mm[1].Timestamp.Truncate(time.Millisecond)}) + defer lw.ConsumerGone() + assert.DeepEqual(t, readAll(t, lw), expected[2:], compareLog) + }) + + t.Run("Until", func(t *testing.T) { + t.Parallel() + lw := lr.ReadLogs(logger.ReadConfig{Tail: -1, Until: mm[2].Timestamp.Add(-time.Millisecond)}) + defer lw.ConsumerGone() + assert.DeepEqual(t, readAll(t, lw), expected[:2], compareLog) + }) + + t.Run("SinceAndUntil", func(t *testing.T) { + t.Parallel() + lw := lr.ReadLogs(logger.ReadConfig{Tail: -1, Since: mm[1].Timestamp.Truncate(time.Millisecond), Until: mm[1].Timestamp.Add(time.Millisecond)}) + defer lw.ConsumerGone() + assert.DeepEqual(t, readAll(t, lw), expected[1:2], compareLog) + }) +} + +func (tr Reader) testTailEmptyLogs(t *testing.T, live bool) { + t.Parallel() + factory := tr.Factory(t, logger.Info{ + ContainerID: "tailemptytest", + ContainerName: "logtail", + }) + l := factory(t) + if !live { + assert.NilError(t, l.Close()) + l = factory(t) + } + defer func() { assert.NilError(t, l.Close()) }() + + for _, tt := range []struct { + name string + cfg logger.ReadConfig + }{ + {name: "Zero", cfg: logger.ReadConfig{}}, + {name: "All", cfg: logger.ReadConfig{Tail: -1}}, + {name: "Tail", cfg: logger.ReadConfig{Tail: 42}}, + {name: "Since", cfg: logger.ReadConfig{Since: time.Unix(1, 0)}}, + {name: "Until", cfg: logger.ReadConfig{Until: time.Date(2100, time.January, 1, 1, 1, 1, 0, time.UTC)}}, + {name: "SinceAndUntil", cfg: logger.ReadConfig{Since: time.Unix(1, 0), Until: time.Date(2100, time.January, 1, 1, 1, 1, 0, time.UTC)}}, + } { + tt := tt + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + lw := l.(logger.LogReader).ReadLogs(logger.ReadConfig{}) + defer lw.ConsumerGone() + assert.DeepEqual(t, readAll(t, lw), ([]*logger.Message)(nil), cmpopts.EquateEmpty()) + }) + } +} + +// TestFollow tests the LogReader's follow implementation. +// +// The LogReader is expected to be able to follow an arbitrary number of +// messages at a high rate with no dropped messages. +func (tr Reader) TestFollow(t *testing.T) { + // Reader sends all logs and closes after logger is closed + // - Starting from empty log (like run) + t.Run("FromEmptyLog", func(t *testing.T) { + t.Parallel() + l := tr.Factory(t, logger.Info{ + ContainerID: "followstart0", + ContainerName: "logloglog", + })(t) + lw := l.(logger.LogReader).ReadLogs(logger.ReadConfig{Tail: -1, Follow: true}) + defer lw.ConsumerGone() + + doneReading := make(chan struct{}) + var logs []*logger.Message + go func() { + defer close(doneReading) + logs = readAll(t, lw) + }() + + mm := makeTestMessages() + expected := logMessages(t, l, mm) + assert.NilError(t, l.Close()) + <-doneReading + assert.DeepEqual(t, logs, expected, compareLog) + }) + + t.Run("AttachMidStream", func(t *testing.T) { + t.Parallel() + l := tr.Factory(t, logger.Info{ + ContainerID: "followmiddle", + ContainerName: "logloglog", + })(t) + + mm := makeTestMessages() + expected := logMessages(t, l, mm[0:1]) + + lw := l.(logger.LogReader).ReadLogs(logger.ReadConfig{Tail: -1, Follow: true}) + defer lw.ConsumerGone() + + doneReading := make(chan struct{}) + var logs []*logger.Message + go func() { + defer close(doneReading) + logs = readAll(t, lw) + }() + + expected = append(expected, logMessages(t, l, mm[1:])...) + assert.NilError(t, l.Close()) + <-doneReading + assert.DeepEqual(t, logs, expected, compareLog) + }) + + t.Run("Since", func(t *testing.T) { + t.Parallel() + l := tr.Factory(t, logger.Info{ + ContainerID: "followsince0", + ContainerName: "logloglog", + })(t) + + mm := makeTestMessages() + + lw := l.(logger.LogReader).ReadLogs(logger.ReadConfig{Tail: -1, Follow: true, Since: mm[2].Timestamp.Truncate(time.Millisecond)}) + defer lw.ConsumerGone() + + doneReading := make(chan struct{}) + var logs []*logger.Message + go func() { + defer close(doneReading) + logs = readAll(t, lw) + }() + + expected := logMessages(t, l, mm)[2:] + assert.NilError(t, l.Close()) + <-doneReading + assert.DeepEqual(t, logs, expected, compareLog) + }) + + t.Run("Until", func(t *testing.T) { + t.Parallel() + l := tr.Factory(t, logger.Info{ + ContainerID: "followuntil0", + ContainerName: "logloglog", + })(t) + + mm := makeTestMessages() + + lw := l.(logger.LogReader).ReadLogs(logger.ReadConfig{Tail: -1, Follow: true, Until: mm[2].Timestamp.Add(-time.Millisecond)}) + defer lw.ConsumerGone() + + doneReading := make(chan struct{}) + var logs []*logger.Message + go func() { + defer close(doneReading) + logs = readAll(t, lw) + }() + + expected := logMessages(t, l, mm)[:2] + defer assert.NilError(t, l.Close()) // Reading should end before the logger is closed. + <-doneReading + assert.DeepEqual(t, logs, expected, compareLog) + }) + + t.Run("SinceAndUntil", func(t *testing.T) { + t.Parallel() + l := tr.Factory(t, logger.Info{ + ContainerID: "followbounded", + ContainerName: "logloglog", + })(t) + + mm := makeTestMessages() + + lw := l.(logger.LogReader).ReadLogs(logger.ReadConfig{Tail: -1, Follow: true, Since: mm[1].Timestamp.Add(-time.Millisecond), Until: mm[2].Timestamp.Add(-time.Millisecond)}) + defer lw.ConsumerGone() + + doneReading := make(chan struct{}) + var logs []*logger.Message + go func() { + defer close(doneReading) + logs = readAll(t, lw) + }() + + expected := logMessages(t, l, mm)[1:2] + defer assert.NilError(t, l.Close()) // Reading should end before the logger is closed. + <-doneReading + assert.DeepEqual(t, logs, expected, compareLog) + }) + + t.Run("Tail=0", func(t *testing.T) { + t.Parallel() + l := tr.Factory(t, logger.Info{ + ContainerID: "followtail00", + ContainerName: "logloglog", + })(t) + + mm := makeTestMessages() + logMessages(t, l, mm[0:2]) + + lw := l.(logger.LogReader).ReadLogs(logger.ReadConfig{Tail: 0, Follow: true}) + defer lw.ConsumerGone() + + doneReading := make(chan struct{}) + var logs []*logger.Message + go func() { + defer close(doneReading) + logs = readAll(t, lw) + }() + + expected := logMessages(t, l, mm[2:]) + assert.NilError(t, l.Close()) + <-doneReading + assert.DeepEqual(t, logs, expected, compareLog) + }) + + t.Run("Tail>0", func(t *testing.T) { + t.Parallel() + l := tr.Factory(t, logger.Info{ + ContainerID: "followtail00", + ContainerName: "logloglog", + })(t) + + mm := makeTestMessages() + expected := logMessages(t, l, mm[0:2])[1:] + + lw := l.(logger.LogReader).ReadLogs(logger.ReadConfig{Tail: 1, Follow: true}) + defer lw.ConsumerGone() + + doneReading := make(chan struct{}) + var logs []*logger.Message + go func() { + defer close(doneReading) + logs = readAll(t, lw) + }() + + expected = append(expected, logMessages(t, l, mm[2:])...) + assert.NilError(t, l.Close()) + <-doneReading + assert.DeepEqual(t, logs, expected, compareLog) + }) + + t.Run("MultipleStarts", func(t *testing.T) { + t.Parallel() + factory := tr.Factory(t, logger.Info{ + ContainerID: "startrestart", + ContainerName: "startmeup", + }) + + mm := makeTestMessages() + l := factory(t) + expected := logMessages(t, l, mm[:3]) + assert.NilError(t, l.Close()) + + l = factory(t) + lw := l.(logger.LogReader).ReadLogs(logger.ReadConfig{Tail: -1, Follow: true}) + defer lw.ConsumerGone() + + doneReading := make(chan struct{}) + var logs []*logger.Message + go func() { + defer close(doneReading) + logs = readAll(t, lw) + }() + + expected = append(expected, logMessages(t, l, mm[3:])...) + assert.NilError(t, l.Close()) + <-doneReading + assert.DeepEqual(t, logs, expected, compareLog) + }) +} + +// logMessages logs messages to l and returns a slice of messages as would be +// expected to be read back. The message values are not modified and the +// returned slice of messages are deep-copied. +func logMessages(t *testing.T, l logger.Logger, messages []*logger.Message) []*logger.Message { + t.Helper() + var expected []*logger.Message + for _, m := range messages { + // Copy the log message because the underlying log writer resets + // the log message and returns it to a buffer pool. + assert.NilError(t, l.Log(copyLogMessage(m))) + runtime.Gosched() + + // Copy the log message again so as not to mutate the input. + expect := copyLogMessage(m) + // Existing API consumers expect a newline to be appended to + // messages other than nonterminal partials as that matches the + // existing behavior of the json-file log driver. + if m.PLogMetaData == nil || m.PLogMetaData.Last { + expect.Line = append(expect.Line, '\n') + } + expected = append(expected, expect) + } + return expected +} + +func copyLogMessage(src *logger.Message) *logger.Message { + dst := logger.NewMessage() + dst.Source = src.Source + dst.Timestamp = src.Timestamp + dst.Attrs = src.Attrs + dst.Err = src.Err + dst.Line = append(dst.Line, src.Line...) + if src.PLogMetaData != nil { + lmd := *src.PLogMetaData + dst.PLogMetaData = &lmd + } + return dst +} +func readMessage(t *testing.T, lw *logger.LogWatcher) *logger.Message { + t.Helper() + timeout := time.NewTimer(5 * time.Second) + defer timeout.Stop() + select { + case <-timeout.C: + t.Error("timed out waiting for message") + return nil + case err, open := <-lw.Err: + t.Errorf("unexpected receive on lw.Err: err=%v, open=%v", err, open) + return nil + case msg, open := <-lw.Msg: + if !open { + select { + case err, open := <-lw.Err: + t.Errorf("unexpected receive on lw.Err with closed lw.Msg: err=%v, open=%v", err, open) + return nil + default: + } + } + if msg != nil { + t.Logf("loggertest: ReadMessage [%v %v] %s", msg.Source, msg.Timestamp, msg.Line) + } + return msg + } +} + +func readAll(t *testing.T, lw *logger.LogWatcher) []*logger.Message { + t.Helper() + var msgs []*logger.Message + for { + m := readMessage(t, lw) + if m == nil { + return msgs + } + msgs = append(msgs, m) + } +} diff --git a/daemon/logger/loggerutils/file_unix.go b/daemon/logger/loggerutils/file_unix.go index 1d2553c280..0deabefe1a 100644 --- a/daemon/logger/loggerutils/file_unix.go +++ b/daemon/logger/loggerutils/file_unix.go @@ -12,3 +12,7 @@ func openFile(name string, flag int, perm os.FileMode) (*os.File, error) { func open(name string) (*os.File, error) { return os.Open(name) } + +func unlink(name string) error { + return os.Remove(name) +} diff --git a/daemon/logger/loggerutils/file_windows.go b/daemon/logger/loggerutils/file_windows.go index b16bf01d70..d2989252aa 100644 --- a/daemon/logger/loggerutils/file_windows.go +++ b/daemon/logger/loggerutils/file_windows.go @@ -2,6 +2,7 @@ package loggerutils import ( "os" + "path/filepath" "syscall" "unsafe" ) @@ -224,3 +225,28 @@ func volumeName(path string) (v string) { } return "" } + +func unlink(name string) error { + // Rename the file before deleting it so that the original name is freed + // up to be reused, even while there are still open FILE_SHARE_DELETE + // file handles. Emulate POSIX unlink() semantics, essentially. + name, err := filepath.Abs(name) + if err != nil { + return err + } + dir, fname := filepath.Split(name) + f, err := os.CreateTemp(dir, fname+".*.deleted") + if err != nil { + return err + } + tmpname := f.Name() + if err := f.Close(); err != nil { + return err + } + err = os.Rename(name, tmpname) + rmErr := os.Remove(tmpname) + if err != nil { + return err + } + return rmErr +} diff --git a/daemon/logger/loggerutils/file_windows_test.go b/daemon/logger/loggerutils/file_windows_test.go index 9978c94219..da1a9b92e4 100644 --- a/daemon/logger/loggerutils/file_windows_test.go +++ b/daemon/logger/loggerutils/file_windows_test.go @@ -1,6 +1,7 @@ package loggerutils import ( + "io" "os" "path/filepath" "testing" @@ -9,10 +10,7 @@ import ( ) func TestOpenFileDelete(t *testing.T) { - tmpDir, err := os.MkdirTemp("", t.Name()) - assert.NilError(t, err) - defer os.RemoveAll(tmpDir) - + tmpDir := t.TempDir() f, err := openFile(filepath.Join(tmpDir, "test.txt"), os.O_CREATE|os.O_RDWR, 644) assert.NilError(t, err) defer f.Close() @@ -21,13 +19,41 @@ func TestOpenFileDelete(t *testing.T) { } func TestOpenFileRename(t *testing.T) { - tmpDir, err := os.MkdirTemp("", t.Name()) - assert.NilError(t, err) - defer os.RemoveAll(tmpDir) - + tmpDir := t.TempDir() f, err := openFile(filepath.Join(tmpDir, "test.txt"), os.O_CREATE|os.O_RDWR, 0644) assert.NilError(t, err) defer f.Close() assert.NilError(t, os.Rename(f.Name(), f.Name()+"renamed")) } + +func TestUnlinkOpenFile(t *testing.T) { + tmpDir := t.TempDir() + name := filepath.Join(tmpDir, "test.txt") + f, err := openFile(name, os.O_CREATE|os.O_RDWR, 0644) + assert.NilError(t, err) + defer func() { assert.NilError(t, f.Close()) }() + + _, err = io.WriteString(f, "first") + assert.NilError(t, err) + + assert.NilError(t, unlink(name)) + f2, err := openFile(name, os.O_CREATE|os.O_RDWR, 0644) + assert.NilError(t, err) + defer func() { assert.NilError(t, f2.Close()) }() + + _, err = io.WriteString(f2, "second") + assert.NilError(t, err) + + _, err = f.Seek(0, io.SeekStart) + assert.NilError(t, err) + fdata, err := io.ReadAll(f) + assert.NilError(t, err) + assert.Check(t, "first" == string(fdata)) + + _, err = f2.Seek(0, io.SeekStart) + assert.NilError(t, err) + f2data, err := io.ReadAll(f2) + assert.NilError(t, err) + assert.Check(t, "second" == string(f2data)) +} diff --git a/daemon/logger/loggerutils/follow.go b/daemon/logger/loggerutils/follow.go index 755a483d7a..b7f93a9aff 100644 --- a/daemon/logger/loggerutils/follow.go +++ b/daemon/logger/loggerutils/follow.go @@ -1,211 +1,165 @@ package loggerutils // import "github.com/docker/docker/daemon/logger/loggerutils" import ( + "fmt" "io" "os" "time" "github.com/docker/docker/daemon/logger" - "github.com/docker/docker/pkg/filenotify" - "github.com/fsnotify/fsnotify" "github.com/pkg/errors" "github.com/sirupsen/logrus" ) -var errRetry = errors.New("retry") -var errDone = errors.New("done") - type follow struct { - file *os.File - dec Decoder - fileWatcher filenotify.FileWatcher - logWatcher *logger.LogWatcher - notifyRotate, notifyEvict chan interface{} - oldSize int64 - retries int + LogFile *LogFile + Watcher *logger.LogWatcher + Decoder Decoder + Since, Until time.Time + + log *logrus.Entry + c chan logPos } -func (fl *follow) handleRotate() error { - name := fl.file.Name() +// Do follows the log file as it is written, starting from f at read. +func (fl *follow) Do(f *os.File, read logPos) { + fl.log = logrus.WithFields(logrus.Fields{ + "module": "logger", + "file": f.Name(), + }) + // Optimization: allocate the write-notifications channel only once and + // reuse it for multiple invocations of nextPos(). + fl.c = make(chan logPos, 1) - fl.file.Close() - fl.fileWatcher.Remove(name) - - // retry when the file doesn't exist - var err error - for retries := 0; retries <= 5; retries++ { - f, err := open(name) - if err == nil || !os.IsNotExist(err) { - fl.file = f - break + defer func() { + if err := f.Close(); err != nil && !errors.Is(err, os.ErrClosed) { + fl.log.WithError(err).Warn("error closing current log file") } - } - if err != nil { - return err - } - if err := fl.fileWatcher.Add(name); err != nil { - return err - } - fl.dec.Reset(fl.file) - return nil -} - -func (fl *follow) handleMustClose(evictErr error) { - fl.file.Close() - fl.dec.Close() - fl.logWatcher.Err <- errors.Wrap(evictErr, "log reader evicted due to errors") - logrus.WithField("file", fl.file.Name()).Error("Log reader notified that it must re-open log file, some log data may not be streamed to the client.") -} - -func (fl *follow) waitRead() error { - select { - case e := <-fl.notifyEvict: - if e != nil { - err := e.(error) - fl.handleMustClose(err) - } - return errDone - case e := <-fl.fileWatcher.Events(): - switch e.Op { - case fsnotify.Write: - fl.dec.Reset(fl.file) - return nil - case fsnotify.Rename, fsnotify.Remove: - select { - case <-fl.notifyRotate: - case <-fl.logWatcher.WatchProducerGone(): - return errDone - case <-fl.logWatcher.WatchConsumerGone(): - return errDone - } - if err := fl.handleRotate(); err != nil { - return err - } - return nil - } - return errRetry - case err := <-fl.fileWatcher.Errors(): - logrus.Debugf("logger got error watching file: %v", err) - // Something happened, let's try and stay alive and create a new watcher - if fl.retries <= 5 { - fl.fileWatcher.Close() - fl.fileWatcher, err = watchFile(fl.file.Name()) - if err != nil { - return err - } - fl.retries++ - return errRetry - } - return err - case <-fl.logWatcher.WatchProducerGone(): - return errDone - case <-fl.logWatcher.WatchConsumerGone(): - return errDone - } -} - -func (fl *follow) handleDecodeErr(err error) error { - if !errors.Is(err, io.EOF) { - return err - } - - // Handle special case (#39235): max-file=1 and file was truncated - st, stErr := fl.file.Stat() - if stErr == nil { - size := st.Size() - defer func() { fl.oldSize = size }() - if size < fl.oldSize { // truncated - fl.file.Seek(0, 0) - fl.dec.Reset(fl.file) - return nil - } - } else { - logrus.WithError(stErr).Warn("logger: stat error") - } + }() for { - err := fl.waitRead() - if err == nil { - break - } - if err == errRetry { - continue - } - return err - } - return nil -} - -func (fl *follow) mainLoop(since, until time.Time) { - for { - select { - case err := <-fl.notifyEvict: - if err != nil { - fl.handleMustClose(err.(error)) - } + wrote, ok := fl.nextPos(read) + if !ok { return - default: } - msg, err := fl.dec.Decode() - if err != nil { - if err := fl.handleDecodeErr(err); err != nil { - if err == errDone { - return - } - // we got an unrecoverable error, so return - fl.logWatcher.Err <- err + + if wrote.rotation != read.rotation { + // Flush the current file before moving on to the next. + if _, err := f.Seek(read.size, io.SeekStart); err != nil { + fl.Watcher.Err <- err return } - // ready to try again - continue + if fl.decode(f) { + return + } + + // Open the new file, which has the same name as the old + // file thanks to file rotation. Make no mistake: they + // are different files, with distinct identities. + // Atomically capture the wrote position to make + // absolutely sure that the position corresponds to the + // file we have opened; more rotations could have + // occurred since we previously received it. + if err := f.Close(); err != nil { + fl.log.WithError(err).Warn("error closing rotated log file") + } + var err error + func() { + fl.LogFile.fsopMu.RLock() + st := <-fl.LogFile.read + defer func() { + fl.LogFile.read <- st + fl.LogFile.fsopMu.RUnlock() + }() + f, err = open(f.Name()) + wrote = st.pos + }() + // We tried to open the file inside a critical section + // so we shouldn't have been racing the rotation of the + // file. Any error, even fs.ErrNotFound, is exceptional. + if err != nil { + fl.Watcher.Err <- fmt.Errorf("logger: error opening log file for follow after rotation: %w", err) + return + } + + if nrot := wrote.rotation - read.rotation; nrot > 1 { + fl.log.WithField("missed-rotations", nrot). + Warn("file rotations were missed while following logs; some log messages have been skipped over") + } + + // Set up our read position to start from the top of the file. + read.size = 0 } - fl.retries = 0 // reset retries since we've succeeded - if !since.IsZero() && msg.Timestamp.Before(since) { + if fl.decode(io.NewSectionReader(f, read.size, wrote.size-read.size)) { + return + } + read = wrote + } +} + +// nextPos waits until the write position of the LogFile being followed has +// advanced from current and returns the new position. +func (fl *follow) nextPos(current logPos) (next logPos, ok bool) { + var st logReadState + select { + case <-fl.Watcher.WatchConsumerGone(): + return current, false + case st = <-fl.LogFile.read: + } + + // Have any any logs been written since we last checked? + if st.pos == current { // Nope. + // Add ourself to the notify list. + st.wait = append(st.wait, fl.c) + } else { // Yes. + // "Notify" ourself immediately. + fl.c <- st.pos + } + fl.LogFile.read <- st + + select { + case <-fl.LogFile.closed: // No more logs will be written. + select { // Have we followed to the end? + case next = <-fl.c: // No: received a new position. + default: // Yes. + return current, false + } + case <-fl.Watcher.WatchConsumerGone(): + return current, false + case next = <-fl.c: + } + return next, true +} + +// decode decodes log messages from r and sends messages with timestamps between +// Since and Until to the log watcher. +// +// The return value, done, signals whether following should end due to a +// condition encountered during decode. +func (fl *follow) decode(r io.Reader) (done bool) { + fl.Decoder.Reset(r) + for { + msg, err := fl.Decoder.Decode() + if err != nil { + if errors.Is(err, io.EOF) { + return false + } + fl.Watcher.Err <- err + return true + } + + if !fl.Since.IsZero() && msg.Timestamp.Before(fl.Since) { continue } - if !until.IsZero() && msg.Timestamp.After(until) { - return + if !fl.Until.IsZero() && msg.Timestamp.After(fl.Until) { + return true } // send the message, unless the consumer is gone select { - case e := <-fl.notifyEvict: - if e != nil { - err := e.(error) - logrus.WithError(err).Debug("Reader evicted while sending log message") - fl.logWatcher.Err <- err - } - return - case fl.logWatcher.Msg <- msg: - case <-fl.logWatcher.WatchConsumerGone(): - return + case fl.Watcher.Msg <- msg: + case <-fl.Watcher.WatchConsumerGone(): + return true } } } - -func followLogs(f *os.File, logWatcher *logger.LogWatcher, notifyRotate, notifyEvict chan interface{}, dec Decoder, since, until time.Time) { - dec.Reset(f) - - name := f.Name() - fileWatcher, err := watchFile(name) - if err != nil { - logWatcher.Err <- err - return - } - defer func() { - f.Close() - dec.Close() - fileWatcher.Close() - }() - - fl := &follow{ - file: f, - oldSize: -1, - logWatcher: logWatcher, - fileWatcher: fileWatcher, - notifyRotate: notifyRotate, - notifyEvict: notifyEvict, - dec: dec, - } - fl.mainLoop(since, until) -} diff --git a/daemon/logger/loggerutils/follow_test.go b/daemon/logger/loggerutils/follow_test.go deleted file mode 100644 index 224cd192e9..0000000000 --- a/daemon/logger/loggerutils/follow_test.go +++ /dev/null @@ -1,37 +0,0 @@ -package loggerutils // import "github.com/docker/docker/daemon/logger/loggerutils" - -import ( - "io" - "os" - "testing" - - "gotest.tools/v3/assert" -) - -func TestHandleDecoderErr(t *testing.T) { - f, err := os.CreateTemp("", t.Name()) - assert.NilError(t, err) - defer os.Remove(f.Name()) - - _, err = f.Write([]byte("hello")) - assert.NilError(t, err) - - pos, err := f.Seek(0, io.SeekCurrent) - assert.NilError(t, err) - assert.Assert(t, pos != 0) - - dec := &testDecoder{} - - // Simulate "turncate" case, where the file was bigger before. - fl := &follow{file: f, dec: dec, oldSize: 100} - err = fl.handleDecodeErr(io.EOF) - assert.NilError(t, err) - - // handleDecodeErr seeks to zero. - pos, err = f.Seek(0, io.SeekCurrent) - assert.NilError(t, err) - assert.Equal(t, int64(0), pos) - - // Reset is called. - assert.Equal(t, 1, dec.resetCount) -} diff --git a/daemon/logger/loggerutils/logfile.go b/daemon/logger/loggerutils/logfile.go index 9319ce5818..516b1dacec 100644 --- a/daemon/logger/loggerutils/logfile.go +++ b/daemon/logger/loggerutils/logfile.go @@ -6,91 +6,83 @@ import ( "encoding/json" "fmt" "io" + "io/fs" + "math" "os" - "runtime" "strconv" - "strings" "sync" "time" "github.com/docker/docker/daemon/logger" - "github.com/docker/docker/pkg/filenotify" "github.com/docker/docker/pkg/pools" - "github.com/docker/docker/pkg/pubsub" "github.com/pkg/errors" "github.com/sirupsen/logrus" ) -const tmpLogfileSuffix = ".tmp" - // rotateFileMetadata is a metadata of the gzip header of the compressed log file type rotateFileMetadata struct { LastTime time.Time `json:"lastTime,omitempty"` } -// refCounter is a counter of logfile being referenced -type refCounter struct { - mu sync.Mutex - counter map[string]int -} - -// Reference increase the reference counter for specified logfile -func (rc *refCounter) GetReference(fileName string, openRefFile func(fileName string, exists bool) (*os.File, error)) (*os.File, error) { - rc.mu.Lock() - defer rc.mu.Unlock() - - var ( - file *os.File - err error - ) - _, ok := rc.counter[fileName] - file, err = openRefFile(fileName, ok) - if err != nil { - return nil, err - } - - if ok { - rc.counter[fileName]++ - } else if file != nil { - rc.counter[file.Name()] = 1 - } - - return file, nil -} - -// Dereference reduce the reference counter for specified logfile -func (rc *refCounter) Dereference(fileName string) error { - rc.mu.Lock() - defer rc.mu.Unlock() - - rc.counter[fileName]-- - if rc.counter[fileName] <= 0 { - delete(rc.counter, fileName) - err := os.Remove(fileName) - if err != nil && !os.IsNotExist(err) { - return err - } - } - return nil -} - // LogFile is Logger implementation for default Docker logging. type LogFile struct { - mu sync.RWMutex // protects the logfile access - f *os.File // store for closing - closed bool - rotateMu sync.Mutex // blocks the next rotation until the current rotation is completed - capacity int64 // maximum size of each file - currentSize int64 // current size of the latest file - maxFiles int // maximum number of files - compress bool // whether old versions of log files are compressed - lastTimestamp time.Time // timestamp of the last log - filesRefCounter refCounter // keep reference-counted of decompressed files - notifyReaders *pubsub.Publisher - marshal logger.MarshalFunc - createDecoder MakeDecoderFn - getTailReader GetTailReaderFunc - perms os.FileMode + mu sync.Mutex // protects the logfile access + closed chan struct{} + rotateMu sync.Mutex // blocks the next rotation until the current rotation is completed + // Lock out readers while performing a non-atomic sequence of filesystem + // operations (RLock: open, Lock: rename, delete). + // + // fsopMu should be locked for writing only while holding rotateMu. + fsopMu sync.RWMutex + + // Logger configuration + + capacity int64 // maximum size of each file + maxFiles int // maximum number of files + compress bool // whether old versions of log files are compressed + perms os.FileMode + + // Log file codec + + marshal logger.MarshalFunc + createDecoder MakeDecoderFn + getTailReader GetTailReaderFunc + + // Log reader state in a 1-buffered channel. + // + // Share memory by communicating: receive to acquire, send to release. + // The state struct is passed around by value so that use-after-send + // bugs cannot escalate to data races. + // + // A method which receives the state value takes ownership of it. The + // owner is responsible for either passing ownership along or sending + // the state back to the channel. By convention, the semantics of + // passing along ownership is expressed with function argument types. + // Methods which take a pointer *logReadState argument borrow the state, + // analogous to functions which require a lock to be held when calling. + // The caller retains ownership. Calling a method which which takes a + // value logFileState argument gives ownership to the callee. + read chan logReadState + + decompress *sharedTempFileConverter + + pos logPos // Current log file write position. + f *os.File // Current log file for writing. + lastTimestamp time.Time // timestamp of the last log +} + +type logPos struct { + // Size of the current file. + size int64 + // File rotation sequence number (modulo 2**16). + rotation uint16 +} + +type logReadState struct { + // Current log file position. + pos logPos + // Wait list to be notified of the value of pos next time it changes. + wait []chan<- logPos } // MakeDecoderFn creates a decoder @@ -111,10 +103,16 @@ type Decoder interface { // SizeReaderAt defines a ReaderAt that also reports its size. // This is used for tailing log files. type SizeReaderAt interface { + io.Reader io.ReaderAt Size() int64 } +type readAtCloser interface { + io.ReaderAt + io.Closer +} + // GetTailReaderFunc is used to truncate a reader to only read as much as is required // in order to get the passed in number of log lines. // It returns the sectioned reader, the number of lines that the section reader @@ -133,18 +131,28 @@ func NewLogFile(logPath string, capacity int64, maxFiles int, compress bool, mar return nil, err } + pos := logPos{ + size: size, + // Force a wraparound on first rotation to shake out any + // modular-arithmetic bugs. + rotation: math.MaxUint16, + } + st := make(chan logReadState, 1) + st <- logReadState{pos: pos} + return &LogFile{ - f: log, - capacity: capacity, - currentSize: size, - maxFiles: maxFiles, - compress: compress, - filesRefCounter: refCounter{counter: make(map[string]int)}, - notifyReaders: pubsub.NewPublisher(0, 1), - marshal: marshaller, - createDecoder: decodeFunc, - perms: perms, - getTailReader: getTailReader, + f: log, + read: st, + pos: pos, + closed: make(chan struct{}), + capacity: capacity, + maxFiles: maxFiles, + compress: compress, + decompress: newSharedTempFileConverter(decompress), + marshal: marshaller, + createDecoder: decodeFunc, + perms: perms, + getTailReader: getTailReader, }, nil } @@ -160,35 +168,45 @@ func (w *LogFile) WriteLogEntry(msg *logger.Message) error { logger.PutMessage(msg) msg = nil // Turn use-after-put bugs into panics. - w.mu.Lock() - if w.closed { - w.mu.Unlock() + select { + case <-w.closed: return errors.New("cannot write because the output file was closed") + default: } + w.mu.Lock() + defer w.mu.Unlock() - if err := w.checkCapacityAndRotate(); err != nil { - w.mu.Unlock() - return errors.Wrap(err, "error rotating log file") + // Are we due for a rotation? + if w.capacity != -1 && w.pos.size >= w.capacity { + if err := w.rotate(); err != nil { + return errors.Wrap(err, "error rotating log file") + } } n, err := w.f.Write(b) - if err == nil { - w.currentSize += int64(n) - w.lastTimestamp = ts + if err != nil { + return errors.Wrap(err, "error writing log entry") } + w.pos.size += int64(n) + w.lastTimestamp = ts - w.mu.Unlock() - return errors.Wrap(err, "error writing log entry") + // Notify any waiting readers that there is a new log entry to read. + st := <-w.read + defer func() { w.read <- st }() + st.pos = w.pos + + for _, c := range st.wait { + c <- st.pos + } + // Optimization: retain the backing array to save a heap allocation next + // time a reader appends to the list. + if st.wait != nil { + st.wait = st.wait[:0] + } + return nil } -func (w *LogFile) checkCapacityAndRotate() (retErr error) { - if w.capacity == -1 { - return nil - } - if w.currentSize < w.capacity { - return nil - } - +func (w *LogFile) rotate() (retErr error) { w.rotateMu.Lock() noCompress := w.maxFiles <= 1 || !w.compress defer func() { @@ -202,49 +220,61 @@ func (w *LogFile) checkCapacityAndRotate() (retErr error) { fname := w.f.Name() if err := w.f.Close(); err != nil { // if there was an error during a prior rotate, the file could already be closed - if !errors.Is(err, os.ErrClosed) { + if !errors.Is(err, fs.ErrClosed) { return errors.Wrap(err, "error closing file") } } - if err := rotate(fname, w.maxFiles, w.compress); err != nil { - logrus.WithError(err).Warn("Error rotating log file, log data may have been lost") - } else { - var renameErr error - for i := 0; i < 10; i++ { - if renameErr = os.Rename(fname, fname+".1"); renameErr != nil && !os.IsNotExist(renameErr) { - logrus.WithError(renameErr).WithField("file", fname).Debug("Error rotating current container log file, evicting readers and retrying") - w.notifyReaders.Publish(renameErr) - time.Sleep(100 * time.Millisecond) - continue - } - break - } - if renameErr != nil { - logrus.WithError(renameErr).Error("Error renaming current log file") - } - } + file, err := func() (*os.File, error) { + w.fsopMu.Lock() + defer w.fsopMu.Unlock() - file, err := openFile(fname, os.O_WRONLY|os.O_TRUNC|os.O_CREATE, w.perms) + if err := rotate(fname, w.maxFiles, w.compress); err != nil { + logrus.WithError(err).Warn("Error rotating log file, log data may have been lost") + } else { + // We may have readers working their way through the + // current log file so we can't truncate it. We need to + // start writing new logs to an empty file with the same + // name as the current one so we need to rotate the + // current file out of the way. + if w.maxFiles < 2 { + if err := unlink(fname); err != nil && !errors.Is(err, fs.ErrNotExist) { + logrus.WithError(err).Error("Error unlinking current log file") + } + } else { + if err := os.Rename(fname, fname+".1"); err != nil && !errors.Is(err, fs.ErrNotExist) { + logrus.WithError(err).Error("Error renaming current log file") + } + } + } + + // Notwithstanding the above, open with the truncate flag anyway + // in case rotation didn't work out as planned. + return openFile(fname, os.O_WRONLY|os.O_TRUNC|os.O_CREATE, w.perms) + }() if err != nil { return err } w.f = file - w.currentSize = 0 - - w.notifyReaders.Publish(struct{}{}) + w.pos = logPos{rotation: w.pos.rotation + 1} if noCompress { return nil } ts := w.lastTimestamp - go func() { + defer w.rotateMu.Unlock() + // No need to hold fsopMu as at no point will the filesystem be + // in a state which would cause problems for readers. Opening + // the uncompressed file is tried first, falling back to the + // compressed one. compressFile only deletes the uncompressed + // file once the compressed one is fully written out, so at no + // point during the compression process will a reader fail to + // open a complete copy of the file. if err := compressFile(fname+".1", ts); err != nil { logrus.WithError(err).Error("Error compressing log file after rotation") } - w.rotateMu.Unlock() }() return nil @@ -261,16 +291,17 @@ func rotate(name string, maxFiles int, compress bool) error { } lastFile := fmt.Sprintf("%s.%d%s", name, maxFiles-1, extension) - err := os.Remove(lastFile) - if err != nil && !os.IsNotExist(err) { + err := unlink(lastFile) + if err != nil && !errors.Is(err, fs.ErrNotExist) { return errors.Wrap(err, "error removing oldest log file") } for i := maxFiles - 1; i > 1; i-- { toPath := name + "." + strconv.Itoa(i) + extension fromPath := name + "." + strconv.Itoa(i-1) + extension - logrus.WithField("source", fromPath).WithField("target", toPath).Trace("Rotating log file") - if err := os.Rename(fromPath, toPath); err != nil && !os.IsNotExist(err) { + err := os.Rename(fromPath, toPath) + logrus.WithError(err).WithField("source", fromPath).WithField("target", toPath).Trace("Rotating log file") + if err != nil && !errors.Is(err, fs.ErrNotExist) { return err } } @@ -281,7 +312,7 @@ func rotate(name string, maxFiles int, compress bool) error { func compressFile(fileName string, lastTimestamp time.Time) (retErr error) { file, err := open(fileName) if err != nil { - if os.IsNotExist(err) { + if errors.Is(err, fs.ErrNotExist) { logrus.WithField("file", fileName).WithError(err).Debug("Could not open log file to compress") return nil } @@ -290,8 +321,8 @@ func compressFile(fileName string, lastTimestamp time.Time) (retErr error) { defer func() { file.Close() if retErr == nil { - err := os.Remove(fileName) - if err != nil && !os.IsNotExist(err) { + err := unlink(fileName) + if err != nil && !errors.Is(err, fs.ErrNotExist) { retErr = errors.Wrap(err, "failed to remove source log file") } } @@ -304,7 +335,7 @@ func compressFile(fileName string, lastTimestamp time.Time) (retErr error) { defer func() { outFile.Close() if retErr != nil { - if err := os.Remove(fileName + ".gz"); err != nil && !os.IsExist(err) { + if err := unlink(fileName + ".gz"); err != nil && !errors.Is(err, fs.ErrNotExist) { logrus.WithError(err).Error("Error cleaning up after failed log compression") } } @@ -339,25 +370,49 @@ func (w *LogFile) MaxFiles() int { func (w *LogFile) Close() error { w.mu.Lock() defer w.mu.Unlock() - if w.closed { + select { + case <-w.closed: return nil + default: } - if err := w.f.Close(); err != nil && !errors.Is(err, os.ErrClosed) { + if err := w.f.Close(); err != nil && !errors.Is(err, fs.ErrClosed) { return err } - w.closed = true + close(w.closed) + // Wait until any in-progress rotation is complete. + w.rotateMu.Lock() + w.rotateMu.Unlock() //nolint:staticcheck return nil } -// ReadLogs decodes entries from log files and sends them the passed in watcher +// ReadLogs decodes entries from log files. // -// Note: Using the follow option can become inconsistent in cases with very frequent rotations and max log files is 1. -// TODO: Consider a different implementation which can effectively follow logs under frequent rotations. -func (w *LogFile) ReadLogs(config logger.ReadConfig, watcher *logger.LogWatcher) { - w.mu.RLock() +// It is the caller's responsibility to call ConsumerGone on the LogWatcher. +func (w *LogFile) ReadLogs(config logger.ReadConfig) *logger.LogWatcher { + watcher := logger.NewLogWatcher() + // Lock out filesystem operations so that we can capture the read + // position and atomically open the corresponding log file, without the + // file getting rotated out from under us. + w.fsopMu.RLock() + // Capture the read position synchronously to ensure that we start + // following from the last entry logged before ReadLogs was called, + // which is required for flake-free unit testing. + st := <-w.read + pos := st.pos + w.read <- st + go w.readLogsLocked(pos, config, watcher) + return watcher +} + +// readLogsLocked is the bulk of the implementation of ReadLogs. +// +// w.fsopMu must be locked for reading when calling this method. +// w.fsopMu.RUnlock() is called before returning. +func (w *LogFile) readLogsLocked(currentPos logPos, config logger.ReadConfig, watcher *logger.LogWatcher) { + defer close(watcher.Msg) + currentFile, err := open(w.f.Name()) if err != nil { - w.mu.RUnlock() watcher.Err <- err return } @@ -366,25 +421,13 @@ func (w *LogFile) ReadLogs(config logger.ReadConfig, watcher *logger.LogWatcher) dec := w.createDecoder(nil) defer dec.Close() - currentChunk, err := newSectionReader(currentFile) - if err != nil { - w.mu.RUnlock() - watcher.Err <- err - return - } - - notifyEvict := w.notifyReaders.SubscribeTopicWithBuffer(func(i interface{}) bool { - _, ok := i.(error) - return ok - }, 1) - defer w.notifyReaders.Evict(notifyEvict) + currentChunk := io.NewSectionReader(currentFile, 0, currentPos.size) if config.Tail != 0 { // TODO(@cpuguy83): Instead of opening every file, only get the files which // are needed to tail. // This is especially costly when compression is enabled. files, err := w.openRotatedFiles(config) - w.mu.RUnlock() if err != nil { watcher.Err <- err return @@ -393,115 +436,123 @@ func (w *LogFile) ReadLogs(config logger.ReadConfig, watcher *logger.LogWatcher) closeFiles := func() { for _, f := range files { f.Close() - fileName := f.Name() - if strings.HasSuffix(fileName, tmpLogfileSuffix) { - err := w.filesRefCounter.Dereference(fileName) - if err != nil { - logrus.WithError(err).WithField("file", fileName).Error("Failed to dereference the log file") - } - } } } readers := make([]SizeReaderAt, 0, len(files)+1) for _, f := range files { - stat, err := f.Stat() - if err != nil { - watcher.Err <- errors.Wrap(err, "error reading size of rotated file") - closeFiles() - return + switch ff := f.(type) { + case SizeReaderAt: + readers = append(readers, ff) + case interface{ Stat() (fs.FileInfo, error) }: + stat, err := ff.Stat() + if err != nil { + watcher.Err <- errors.Wrap(err, "error reading size of rotated file") + closeFiles() + return + } + readers = append(readers, io.NewSectionReader(f, 0, stat.Size())) + default: + panic(fmt.Errorf("rotated file value %#v (%[1]T) has neither Size() nor Stat() methods", f)) } - readers = append(readers, io.NewSectionReader(f, 0, stat.Size())) } if currentChunk.Size() > 0 { readers = append(readers, currentChunk) } - ok := tailFiles(readers, watcher, dec, w.getTailReader, config, notifyEvict) + ok := tailFiles(readers, watcher, dec, w.getTailReader, config) closeFiles() if !ok { return } - w.mu.RLock() + } else { + w.fsopMu.RUnlock() } - if !config.Follow || w.closed { - w.mu.RUnlock() + if !config.Follow { return } - w.mu.RUnlock() - notifyRotate := w.notifyReaders.SubscribeTopic(func(i interface{}) bool { - _, ok := i.(struct{}) - return ok - }) - defer w.notifyReaders.Evict(notifyRotate) - - followLogs(currentFile, watcher, notifyRotate, notifyEvict, dec, config.Since, config.Until) + (&follow{ + LogFile: w, + Watcher: watcher, + Decoder: dec, + Since: config.Since, + Until: config.Until, + }).Do(currentFile, currentPos) } -func (w *LogFile) openRotatedFiles(config logger.ReadConfig) (files []*os.File, err error) { - w.rotateMu.Lock() - defer w.rotateMu.Unlock() +// openRotatedFiles returns a slice of files open for reading, in order from +// oldest to newest, and calls w.fsopMu.RUnlock() before returning. +// +// This method must only be called with w.fsopMu locked for reading. +func (w *LogFile) openRotatedFiles(config logger.ReadConfig) (files []readAtCloser, err error) { + type rotatedFile struct { + f *os.File + compressed bool + } + var q []rotatedFile defer func() { - if err == nil { - return - } - for _, f := range files { - f.Close() - if strings.HasSuffix(f.Name(), tmpLogfileSuffix) { - err := os.Remove(f.Name()) - if err != nil && !os.IsNotExist(err) { - logrus.Warnf("Failed to remove logfile: %v", err) - } + if err != nil { + for _, qq := range q { + qq.f.Close() + } + for _, f := range files { + f.Close() } } }() - for i := w.maxFiles; i > 1; i-- { - f, err := open(fmt.Sprintf("%s.%d", w.f.Name(), i-1)) - if err != nil { - if !os.IsNotExist(err) { - return nil, errors.Wrap(err, "error opening rotated log file") - } - - fileName := fmt.Sprintf("%s.%d.gz", w.f.Name(), i-1) - decompressedFileName := fileName + tmpLogfileSuffix - tmpFile, err := w.filesRefCounter.GetReference(decompressedFileName, func(refFileName string, exists bool) (*os.File, error) { - if exists { - return open(refFileName) - } - return decompressfile(fileName, refFileName, config.Since) - }) + q, err = func() (q []rotatedFile, err error) { + defer w.fsopMu.RUnlock() + q = make([]rotatedFile, 0, w.maxFiles) + for i := w.maxFiles; i > 1; i-- { + var f rotatedFile + f.f, err = open(fmt.Sprintf("%s.%d", w.f.Name(), i-1)) if err != nil { - if !errors.Is(err, os.ErrNotExist) { - return nil, errors.Wrap(err, "error getting reference to decompressed log file") + if !errors.Is(err, fs.ErrNotExist) { + return nil, errors.Wrap(err, "error opening rotated log file") + } + f.compressed = true + f.f, err = open(fmt.Sprintf("%s.%d.gz", w.f.Name(), i-1)) + if err != nil { + if !errors.Is(err, fs.ErrNotExist) { + return nil, errors.Wrap(err, "error opening file for decompression") + } + continue } - continue } - if tmpFile == nil { - // The log before `config.Since` does not need to read - break - } - - files = append(files, tmpFile) - continue + q = append(q, f) } - files = append(files, f) + return q, nil + }() + if err != nil { + return nil, err } + for len(q) > 0 { + qq := q[0] + q = q[1:] + if qq.compressed { + defer qq.f.Close() + f, err := w.maybeDecompressFile(qq.f, config) + if err != nil { + return nil, err + } + if f != nil { + // The log before `config.Since` does not need to read + files = append(files, f) + } + } else { + files = append(files, qq.f) + } + } return files, nil } -func decompressfile(fileName, destFileName string, since time.Time) (*os.File, error) { - cf, err := open(fileName) - if err != nil { - return nil, errors.Wrap(err, "error opening file for decompression") - } - defer cf.Close() - +func (w *LogFile) maybeDecompressFile(cf *os.File, config logger.ReadConfig) (readAtCloser, error) { rc, err := gzip.NewReader(cf) if err != nil { return nil, errors.Wrap(err, "error making gzip reader for compressed log file") @@ -511,41 +562,29 @@ func decompressfile(fileName, destFileName string, since time.Time) (*os.File, e // Extract the last log entry timestramp from the gzip header extra := &rotateFileMetadata{} err = json.Unmarshal(rc.Header.Extra, extra) - if err == nil && extra.LastTime.Before(since) { + if err == nil && !extra.LastTime.IsZero() && extra.LastTime.Before(config.Since) { return nil, nil } - - rs, err := openFile(destFileName, os.O_CREATE|os.O_RDWR, 0640) - if err != nil { - return nil, errors.Wrap(err, "error creating file for copying decompressed log stream") - } - - _, err = pools.Copy(rs, rc) - if err != nil { - rs.Close() - rErr := os.Remove(rs.Name()) - if rErr != nil && !os.IsNotExist(rErr) { - logrus.Errorf("Failed to remove logfile: %v", rErr) - } - return nil, errors.Wrap(err, "error while copying decompressed log stream to file") - } - - return rs, nil + tmpf, err := w.decompress.Do(cf) + return tmpf, errors.Wrap(err, "error decompressing log file") } -func newSectionReader(f *os.File) (*io.SectionReader, error) { - // seek to the end to get the size - // we'll leave this at the end of the file since section reader does not advance the reader - size, err := f.Seek(0, io.SeekEnd) - if err != nil { - return nil, errors.Wrap(err, "error getting current file size") +func decompress(dst io.WriteSeeker, src io.ReadSeeker) error { + if _, err := src.Seek(0, io.SeekStart); err != nil { + return err } - return io.NewSectionReader(f, 0, size), nil + rc, err := gzip.NewReader(src) + if err != nil { + return err + } + _, err = pools.Copy(dst, rc) + if err != nil { + return err + } + return rc.Close() } -func tailFiles(files []SizeReaderAt, watcher *logger.LogWatcher, dec Decoder, getTailReader GetTailReaderFunc, config logger.ReadConfig, notifyEvict <-chan interface{}) (cont bool) { - nLines := config.Tail - +func tailFiles(files []SizeReaderAt, watcher *logger.LogWatcher, dec Decoder, getTailReader GetTailReaderFunc, config logger.ReadConfig) (cont bool) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -553,12 +592,6 @@ func tailFiles(files []SizeReaderAt, watcher *logger.LogWatcher, dec Decoder, ge // TODO(@cpuguy83): we should plumb a context through instead of dealing with `WatchClose()` here. go func() { select { - case err := <-notifyEvict: - if err != nil { - watcher.Err <- err.(error) - cont = false - cancel() - } case <-ctx.Done(): case <-watcher.WatchConsumerGone(): cont = false @@ -569,6 +602,7 @@ func tailFiles(files []SizeReaderAt, watcher *logger.LogWatcher, dec Decoder, ge readers := make([]io.Reader, 0, len(files)) if config.Tail > 0 { + nLines := config.Tail for i := len(files) - 1; i >= 0 && nLines > 0; i-- { tail, n, err := getTailReader(ctx, files[i], nLines) if err != nil { @@ -580,7 +614,7 @@ func tailFiles(files []SizeReaderAt, watcher *logger.LogWatcher, dec Decoder, ge } } else { for _, r := range files { - readers = append(readers, &wrappedReaderAt{ReaderAt: r}) + readers = append(readers, r) } } @@ -608,52 +642,3 @@ func tailFiles(files []SizeReaderAt, watcher *logger.LogWatcher, dec Decoder, ge } } } - -func watchFile(name string) (filenotify.FileWatcher, error) { - var fileWatcher filenotify.FileWatcher - - if runtime.GOOS == "windows" { - // FileWatcher on Windows files is based on the syscall notifications which has an issue because of file caching. - // It is based on ReadDirectoryChangesW() which doesn't detect writes to the cache. It detects writes to disk only. - // Because of the OS lazy writing, we don't get notifications for file writes and thereby the watcher - // doesn't work. Hence for Windows we will use poll based notifier. - fileWatcher = filenotify.NewPollingWatcher() - } else { - var err error - fileWatcher, err = filenotify.New() - if err != nil { - return nil, err - } - } - - logger := logrus.WithFields(logrus.Fields{ - "module": "logger", - "file": name, - }) - - if err := fileWatcher.Add(name); err != nil { - // we will retry using file poller. - logger.WithError(err).Warnf("falling back to file poller") - fileWatcher.Close() - fileWatcher = filenotify.NewPollingWatcher() - - if err := fileWatcher.Add(name); err != nil { - fileWatcher.Close() - logger.WithError(err).Debugf("error watching log file for modifications") - return nil, err - } - } - - return fileWatcher, nil -} - -type wrappedReaderAt struct { - io.ReaderAt - pos int64 -} - -func (r *wrappedReaderAt) Read(p []byte) (int, error) { - n, err := r.ReaderAt.ReadAt(p, r.pos) - r.pos += int64(n) - return n, err -} diff --git a/daemon/logger/loggerutils/logfile_test.go b/daemon/logger/loggerutils/logfile_test.go index e816d782cf..df3d0a1226 100644 --- a/daemon/logger/loggerutils/logfile_test.go +++ b/daemon/logger/loggerutils/logfile_test.go @@ -9,13 +9,11 @@ import ( "os" "path/filepath" "strings" - "sync/atomic" "testing" "text/tabwriter" "time" "github.com/docker/docker/daemon/logger" - "github.com/docker/docker/pkg/pubsub" "github.com/docker/docker/pkg/tailfile" "gotest.tools/v3/assert" "gotest.tools/v3/poll" @@ -68,7 +66,7 @@ func TestTailFiles(t *testing.T) { started := make(chan struct{}) go func() { close(started) - tailFiles(files, watcher, dec, tailReader, config, make(chan interface{})) + tailFiles(files, watcher, dec, tailReader, config) }() <-started }) @@ -78,7 +76,7 @@ func TestTailFiles(t *testing.T) { started := make(chan struct{}) go func() { close(started) - tailFiles(files, watcher, dec, tailReader, config, make(chan interface{})) + tailFiles(files, watcher, dec, tailReader, config) }() <-started @@ -112,180 +110,44 @@ func (dummyDecoder) Decode() (*logger.Message, error) { func (dummyDecoder) Close() {} func (dummyDecoder) Reset(io.Reader) {} -func TestFollowLogsConsumerGone(t *testing.T) { - lw := logger.NewLogWatcher() - - f, err := os.CreateTemp("", t.Name()) - assert.NilError(t, err) - defer func() { - f.Close() - os.Remove(f.Name()) - }() - - dec := dummyDecoder{} - - followLogsDone := make(chan struct{}) - var since, until time.Time - go func() { - followLogs(f, lw, make(chan interface{}), make(chan interface{}), dec, since, until) - close(followLogsDone) - }() - - select { - case <-lw.Msg: - case err := <-lw.Err: - assert.NilError(t, err) - case <-followLogsDone: - t.Fatal("follow logs finished unexpectedly") - case <-time.After(10 * time.Second): - t.Fatal("timeout waiting for log message") - } - - lw.ConsumerGone() - select { - case <-followLogsDone: - case <-time.After(20 * time.Second): - t.Fatal("timeout waiting for followLogs() to finish") - } -} - -type dummyWrapper struct { - dummyDecoder - fn func() error -} - -func (d *dummyWrapper) Decode() (*logger.Message, error) { - if err := d.fn(); err != nil { - return nil, err - } - return d.dummyDecoder.Decode() -} - -func TestFollowLogsProducerGone(t *testing.T) { - lw := logger.NewLogWatcher() - defer lw.ConsumerGone() - - f, err := os.CreateTemp("", t.Name()) - assert.NilError(t, err) - defer os.Remove(f.Name()) - - var sent, received, closed int32 - dec := &dummyWrapper{fn: func() error { - switch atomic.LoadInt32(&closed) { - case 0: - atomic.AddInt32(&sent, 1) - return nil - case 1: - atomic.AddInt32(&closed, 1) - t.Logf("logDecode() closed after sending %d messages\n", sent) - return io.EOF - default: - t.Fatal("logDecode() called after closing!") - return io.EOF - } - }} - var since, until time.Time - - followLogsDone := make(chan struct{}) - go func() { - followLogs(f, lw, make(chan interface{}), make(chan interface{}), dec, since, until) - close(followLogsDone) - }() - - // read 1 message - select { - case <-lw.Msg: - received++ - case err := <-lw.Err: - assert.NilError(t, err) - case <-followLogsDone: - t.Fatal("followLogs() finished unexpectedly") - case <-time.After(10 * time.Second): - t.Fatal("timeout waiting for log message") - } - - // "stop" the "container" - atomic.StoreInt32(&closed, 1) - lw.ProducerGone() - - // should receive all the messages sent - readDone := make(chan struct{}) - go func() { - defer close(readDone) - for { - select { - case <-lw.Msg: - received++ - if received == atomic.LoadInt32(&sent) { - return - } - case err := <-lw.Err: - assert.NilError(t, err) - } - } - }() - select { - case <-readDone: - case <-time.After(30 * time.Second): - t.Fatalf("timeout waiting for log messages to be read (sent: %d, received: %d", sent, received) - } - - t.Logf("messages sent: %d, received: %d", atomic.LoadInt32(&sent), received) - - // followLogs() should be done by now - select { - case <-followLogsDone: - case <-time.After(30 * time.Second): - t.Fatal("timeout waiting for followLogs() to finish") - } - - select { - case <-lw.WatchConsumerGone(): - t.Fatal("consumer should not have exited") - default: - } -} - func TestCheckCapacityAndRotate(t *testing.T) { - dir, err := os.MkdirTemp("", t.Name()) - assert.NilError(t, err) - defer os.RemoveAll(dir) + dir := t.TempDir() - f, err := os.CreateTemp(dir, "log") - assert.NilError(t, err) - - l := &LogFile{ - f: f, - capacity: 5, - maxFiles: 3, - compress: true, - notifyReaders: pubsub.NewPublisher(0, 1), - perms: 0600, - filesRefCounter: refCounter{counter: make(map[string]int)}, - getTailReader: func(ctx context.Context, r SizeReaderAt, lines int) (io.Reader, int, error) { - return tailfile.NewTailReader(ctx, r, lines) - }, - createDecoder: func(io.Reader) Decoder { - return dummyDecoder{} - }, - marshal: func(msg *logger.Message) ([]byte, error) { - return msg.Line, nil - }, + logPath := filepath.Join(dir, "log") + getTailReader := func(ctx context.Context, r SizeReaderAt, lines int) (io.Reader, int, error) { + return tailfile.NewTailReader(ctx, r, lines) } + createDecoder := func(io.Reader) Decoder { + return dummyDecoder{} + } + marshal := func(msg *logger.Message) ([]byte, error) { + return msg.Line, nil + } + l, err := NewLogFile( + logPath, + 5, // capacity + 3, // maxFiles + true, // compress + marshal, + createDecoder, + 0600, // perms + getTailReader, + ) + assert.NilError(t, err) defer l.Close() ls := dirStringer{dir} assert.NilError(t, l.WriteLogEntry(&logger.Message{Line: []byte("hello world!")})) - _, err = os.Stat(f.Name() + ".1") + _, err = os.Stat(logPath + ".1") assert.Assert(t, os.IsNotExist(err), ls) assert.NilError(t, l.WriteLogEntry(&logger.Message{Line: []byte("hello world!")})) - poll.WaitOn(t, checkFileExists(f.Name()+".1.gz"), poll.WithDelay(time.Millisecond), poll.WithTimeout(30*time.Second)) + poll.WaitOn(t, checkFileExists(logPath+".1.gz"), poll.WithDelay(time.Millisecond), poll.WithTimeout(30*time.Second)) assert.NilError(t, l.WriteLogEntry(&logger.Message{Line: []byte("hello world!")})) - poll.WaitOn(t, checkFileExists(f.Name()+".1.gz"), poll.WithDelay(time.Millisecond), poll.WithTimeout(30*time.Second)) - poll.WaitOn(t, checkFileExists(f.Name()+".2.gz"), poll.WithDelay(time.Millisecond), poll.WithTimeout(30*time.Second)) + poll.WaitOn(t, checkFileExists(logPath+".1.gz"), poll.WithDelay(time.Millisecond), poll.WithTimeout(30*time.Second)) + poll.WaitOn(t, checkFileExists(logPath+".2.gz"), poll.WithDelay(time.Millisecond), poll.WithTimeout(30*time.Second)) t.Run("closed log file", func(t *testing.T) { // Now let's simulate a failed rotation where the file was able to be closed but something else happened elsewhere @@ -293,14 +155,13 @@ func TestCheckCapacityAndRotate(t *testing.T) { // We want to make sure that we can recover in the case that `l.f` was closed while attempting a rotation. l.f.Close() assert.NilError(t, l.WriteLogEntry(&logger.Message{Line: []byte("hello world!")})) - assert.NilError(t, os.Remove(f.Name()+".2.gz")) + assert.NilError(t, os.Remove(logPath+".2.gz")) }) t.Run("with log reader", func(t *testing.T) { // Make sure rotate works with an active reader - lw := logger.NewLogWatcher() + lw := l.ReadLogs(logger.ReadConfig{Follow: true, Tail: 1000}) defer lw.ConsumerGone() - go l.ReadLogs(logger.ReadConfig{Follow: true, Tail: 1000}, lw) assert.NilError(t, l.WriteLogEntry(&logger.Message{Line: []byte("hello world 0!")}), ls) // make sure the log reader is primed @@ -310,7 +171,7 @@ func TestCheckCapacityAndRotate(t *testing.T) { assert.NilError(t, l.WriteLogEntry(&logger.Message{Line: []byte("hello world 2!")}), ls) assert.NilError(t, l.WriteLogEntry(&logger.Message{Line: []byte("hello world 3!")}), ls) assert.NilError(t, l.WriteLogEntry(&logger.Message{Line: []byte("hello world 4!")}), ls) - poll.WaitOn(t, checkFileExists(f.Name()+".2.gz"), poll.WithDelay(time.Millisecond), poll.WithTimeout(30*time.Second)) + poll.WaitOn(t, checkFileExists(logPath+".2.gz"), poll.WithDelay(time.Millisecond), poll.WithTimeout(30*time.Second)) }) } @@ -321,9 +182,8 @@ func waitForMsg(t *testing.T, lw *logger.LogWatcher, timeout time.Duration) { defer timer.Stop() select { - case <-lw.Msg: - case <-lw.WatchProducerGone(): - t.Fatal("log producer gone before log message arrived") + case _, ok := <-lw.Msg: + assert.Assert(t, ok, "log producer gone before log message arrived") case err := <-lw.Err: assert.NilError(t, err) case <-timer.C: diff --git a/daemon/logger/loggerutils/sharedtemp.go b/daemon/logger/loggerutils/sharedtemp.go new file mode 100644 index 0000000000..8d0ad987da --- /dev/null +++ b/daemon/logger/loggerutils/sharedtemp.go @@ -0,0 +1,227 @@ +package loggerutils // import "github.com/docker/docker/daemon/logger/loggerutils" + +import ( + "io" + "io/fs" + "os" + "runtime" +) + +type fileConvertFn func(dst io.WriteSeeker, src io.ReadSeeker) error + +type stfID uint64 + +// sharedTempFileConverter converts files using a user-supplied function and +// writes the results to temporary files which are automatically cleaned up on +// close. If another request is made to convert the same file, the conversion +// result and temporary file are reused if they have not yet been cleaned up. +// +// A file is considered the same as another file using the os.SameFile function, +// which compares file identity (e.g. device and inode numbers on Linux) and is +// robust to file renames. Input files are assumed to be immutable; no attempt +// is made to ascertain whether the file contents have changed between requests. +// +// One file descriptor is used per source file, irrespective of the number of +// concurrent readers of the converted contents. +type sharedTempFileConverter struct { + // The directory where temporary converted files are to be written to. + // If set to the empty string, the default directory for temporary files + // is used. + TempDir string + + conv fileConvertFn + st chan stfcState +} + +type stfcState struct { + fl map[stfID]sharedTempFile + nextID stfID +} + +type sharedTempFile struct { + src os.FileInfo // Info about the source file for path-independent identification with os.SameFile. + fd *os.File + size int64 + ref int // Reference count of open readers on the temporary file. + wait []chan<- stfConvertResult // Wait list for the conversion to complete. +} + +type stfConvertResult struct { + fr *sharedFileReader + err error +} + +func newSharedTempFileConverter(conv fileConvertFn) *sharedTempFileConverter { + st := make(chan stfcState, 1) + st <- stfcState{fl: make(map[stfID]sharedTempFile)} + return &sharedTempFileConverter{conv: conv, st: st} +} + +// Do returns a reader for the contents of f as converted by the c.C function. +// It is the caller's responsibility to close the returned reader. +// +// This function is safe for concurrent use by multiple goroutines. +func (c *sharedTempFileConverter) Do(f *os.File) (*sharedFileReader, error) { + stat, err := f.Stat() + if err != nil { + return nil, err + } + + st := <-c.st + for id, tf := range st.fl { + // os.SameFile can have false positives if one of the files was + // deleted before the other file was created -- such as during + // log rotations... https://github.com/golang/go/issues/36895 + // Weed out those false positives by also comparing the files' + // ModTime, which conveniently also handles the case of true + // positives where the file has also been modified since it was + // first converted. + if os.SameFile(tf.src, stat) && tf.src.ModTime() == stat.ModTime() { + return c.openExisting(st, id, tf) + } + } + return c.openNew(st, f, stat) +} + +func (c *sharedTempFileConverter) openNew(st stfcState, f *os.File, stat os.FileInfo) (*sharedFileReader, error) { + // Record that we are starting to convert this file so that any other + // requests for the same source file while the conversion is in progress + // can join. + id := st.nextID + st.nextID++ + st.fl[id] = sharedTempFile{src: stat} + c.st <- st + + dst, size, convErr := c.convert(f) + + st = <-c.st + flid := st.fl[id] + + if convErr != nil { + // Conversion failed. Delete it from the state so that future + // requests to convert the same file can try again fresh. + delete(st.fl, id) + c.st <- st + for _, w := range flid.wait { + w <- stfConvertResult{err: convErr} + } + return nil, convErr + } + + flid.fd = dst + flid.size = size + flid.ref = len(flid.wait) + 1 + for _, w := range flid.wait { + // Each waiter needs its own reader with an independent read pointer. + w <- stfConvertResult{fr: flid.Reader(c, id)} + } + flid.wait = nil + st.fl[id] = flid + c.st <- st + return flid.Reader(c, id), nil +} + +func (c *sharedTempFileConverter) openExisting(st stfcState, id stfID, v sharedTempFile) (*sharedFileReader, error) { + if v.fd != nil { + // Already converted. + v.ref++ + st.fl[id] = v + c.st <- st + return v.Reader(c, id), nil + } + // The file has not finished being converted. + // Add ourselves to the wait list. "Don't call us; we'll call you." + wait := make(chan stfConvertResult, 1) + v.wait = append(v.wait, wait) + st.fl[id] = v + c.st <- st + + res := <-wait + return res.fr, res.err + +} + +func (c *sharedTempFileConverter) convert(f *os.File) (converted *os.File, size int64, err error) { + dst, err := os.CreateTemp(c.TempDir, "dockerdtemp.*") + if err != nil { + return nil, 0, err + } + defer func() { + _ = dst.Close() + // Delete the temporary file immediately so that final cleanup + // of the file on disk is deferred to the OS once we close all + // our file descriptors (or the process dies). Assuming no early + // returns due to errors, the file will be open by this process + // with a read-only descriptor at this point. As we don't care + // about being able to reuse the file name -- it's randomly + // generated and unique -- we can safely use os.Remove on + // Windows. + _ = os.Remove(dst.Name()) + }() + err = c.conv(dst, f) + if err != nil { + return nil, 0, err + } + // Close the exclusive read-write file descriptor, catching any delayed + // write errors (and on Windows, releasing the share-locks on the file) + if err := dst.Close(); err != nil { + _ = os.Remove(dst.Name()) + return nil, 0, err + } + // Open the file again read-only (without locking the file against + // deletion on Windows). + converted, err = open(dst.Name()) + if err != nil { + return nil, 0, err + } + + // The position of the file's read pointer doesn't matter as all readers + // will be accessing the file through its io.ReaderAt interface. + size, err = converted.Seek(0, io.SeekEnd) + if err != nil { + _ = converted.Close() + return nil, 0, err + } + return converted, size, nil +} + +type sharedFileReader struct { + *io.SectionReader + + c *sharedTempFileConverter + id stfID + closed bool +} + +func (stf sharedTempFile) Reader(c *sharedTempFileConverter, id stfID) *sharedFileReader { + rdr := &sharedFileReader{SectionReader: io.NewSectionReader(stf.fd, 0, stf.size), c: c, id: id} + runtime.SetFinalizer(rdr, (*sharedFileReader).Close) + return rdr +} + +func (r *sharedFileReader) Close() error { + if r.closed { + return fs.ErrClosed + } + + st := <-r.c.st + flid, ok := st.fl[r.id] + if !ok { + panic("invariant violation: temp file state missing from map") + } + flid.ref-- + lastRef := flid.ref <= 0 + if lastRef { + delete(st.fl, r.id) + } else { + st.fl[r.id] = flid + } + r.closed = true + r.c.st <- st + + if lastRef { + return flid.fd.Close() + } + runtime.SetFinalizer(r, nil) + return nil +} diff --git a/daemon/logger/loggerutils/sharedtemp_test.go b/daemon/logger/loggerutils/sharedtemp_test.go new file mode 100644 index 0000000000..ac2249e81f --- /dev/null +++ b/daemon/logger/loggerutils/sharedtemp_test.go @@ -0,0 +1,256 @@ +package loggerutils // import "github.com/docker/docker/daemon/logger/loggerutils" + +import ( + "io" + "io/fs" + "os" + "path/filepath" + "runtime" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/pkg/errors" + "gotest.tools/v3/assert" + "gotest.tools/v3/assert/cmp" +) + +func TestSharedTempFileConverter(t *testing.T) { + t.Parallel() + + t.Run("OneReaderAtATime", func(t *testing.T) { + t.Parallel() + dir := t.TempDir() + name := filepath.Join(dir, "test.txt") + createFile(t, name, "hello, world!") + + uut := newSharedTempFileConverter(copyTransform(strings.ToUpper)) + uut.TempDir = dir + + for i := 0; i < 3; i++ { + t.Logf("Iteration %v", i) + + rdr := convertPath(t, uut, name) + assert.Check(t, cmp.Equal("HELLO, WORLD!", readAll(t, rdr))) + assert.Check(t, rdr.Close()) + assert.Check(t, cmp.Equal(fs.ErrClosed, rdr.Close()), "closing an already-closed reader should return an error") + } + + assert.NilError(t, os.Remove(name)) + checkDirEmpty(t, dir) + }) + + t.Run("RobustToRenames", func(t *testing.T) { + t.Parallel() + dir := t.TempDir() + apath := filepath.Join(dir, "test.txt") + createFile(t, apath, "file a") + + var conversions int + uut := newSharedTempFileConverter( + func(dst io.WriteSeeker, src io.ReadSeeker) error { + conversions++ + return copyTransform(strings.ToUpper)(dst, src) + }, + ) + uut.TempDir = dir + + ra1 := convertPath(t, uut, apath) + + // Rotate the file to a new name and write a new file in its place. + bpath := apath + apath = filepath.Join(dir, "test2.txt") + assert.NilError(t, os.Rename(bpath, apath)) + createFile(t, bpath, "file b") + + rb1 := convertPath(t, uut, bpath) // Same path, different file. + ra2 := convertPath(t, uut, apath) // New path, old file. + assert.Check(t, cmp.Equal(2, conversions), "expected only one conversion per unique file") + + // Interleave reading and closing to shake out ref-counting bugs: + // closing one reader shouldn't affect any other open readers. + assert.Check(t, cmp.Equal("FILE A", readAll(t, ra1))) + assert.NilError(t, ra1.Close()) + assert.Check(t, cmp.Equal("FILE A", readAll(t, ra2))) + assert.NilError(t, ra2.Close()) + assert.Check(t, cmp.Equal("FILE B", readAll(t, rb1))) + assert.NilError(t, rb1.Close()) + + assert.NilError(t, os.Remove(apath)) + assert.NilError(t, os.Remove(bpath)) + checkDirEmpty(t, dir) + }) + + t.Run("ConcurrentRequests", func(t *testing.T) { + t.Parallel() + dir := t.TempDir() + name := filepath.Join(dir, "test.txt") + createFile(t, name, "hi there") + + var conversions int32 + notify := make(chan chan struct{}, 1) + firstConversionStarted := make(chan struct{}) + notify <- firstConversionStarted + unblock := make(chan struct{}) + uut := newSharedTempFileConverter( + func(dst io.WriteSeeker, src io.ReadSeeker) error { + t.Log("Convert: enter") + defer t.Log("Convert: exit") + select { + case c := <-notify: + close(c) + default: + } + <-unblock + atomic.AddInt32(&conversions, 1) + return copyTransform(strings.ToUpper)(dst, src) + }, + ) + uut.TempDir = dir + + closers := make(chan io.Closer, 4) + var wg sync.WaitGroup + wg.Add(3) + for i := 0; i < 3; i++ { + i := i + go func() { + defer wg.Done() + t.Logf("goroutine %v: enter", i) + defer t.Logf("goroutine %v: exit", i) + f := convertPath(t, uut, name) + assert.Check(t, cmp.Equal("HI THERE", readAll(t, f)), "in goroutine %v", i) + closers <- f + }() + } + + select { + case <-firstConversionStarted: + case <-time.After(2 * time.Second): + t.Fatal("the first conversion should have started by now") + } + close(unblock) + t.Log("starting wait") + wg.Wait() + t.Log("wait done") + + f := convertPath(t, uut, name) + closers <- f + close(closers) + assert.Check(t, cmp.Equal("HI THERE", readAll(t, f)), "after all goroutines returned") + for c := range closers { + assert.Check(t, c.Close()) + } + + assert.Check(t, cmp.Equal(int32(1), conversions)) + + assert.NilError(t, os.Remove(name)) + checkDirEmpty(t, dir) + }) + + t.Run("ConvertError", func(t *testing.T) { + t.Parallel() + dir := t.TempDir() + name := filepath.Join(dir, "test.txt") + createFile(t, name, "hi there") + src, err := open(name) + assert.NilError(t, err) + defer src.Close() + + fakeErr := errors.New("fake error") + var start sync.WaitGroup + start.Add(3) + uut := newSharedTempFileConverter( + func(dst io.WriteSeeker, src io.ReadSeeker) error { + start.Wait() + runtime.Gosched() + if fakeErr != nil { + return fakeErr + } + return copyTransform(strings.ToUpper)(dst, src) + }, + ) + uut.TempDir = dir + + var done sync.WaitGroup + done.Add(3) + for i := 0; i < 3; i++ { + i := i + go func() { + defer done.Done() + t.Logf("goroutine %v: enter", i) + defer t.Logf("goroutine %v: exit", i) + start.Done() + _, err := uut.Do(src) + assert.Check(t, errors.Is(err, fakeErr), "in goroutine %v", i) + }() + } + done.Wait() + + // Conversion errors should not be "sticky". A subsequent + // request should retry from scratch. + fakeErr = errors.New("another fake error") + _, err = uut.Do(src) + assert.Check(t, errors.Is(err, fakeErr)) + + fakeErr = nil + f, err := uut.Do(src) + assert.Check(t, err) + assert.Check(t, cmp.Equal("HI THERE", readAll(t, f))) + assert.Check(t, f.Close()) + + // Files pending delete continue to show up in directory + // listings on Windows RS5. Close the remaining handle before + // deleting the file to prevent spurious failures with + // checkDirEmpty. + assert.Check(t, src.Close()) + assert.NilError(t, os.Remove(name)) + checkDirEmpty(t, dir) + + }) +} + +func createFile(t *testing.T, path string, content string) { + t.Helper() + f, err := openFile(path, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0644) + assert.NilError(t, err) + _, err = io.WriteString(f, content) + assert.NilError(t, err) + assert.NilError(t, f.Close()) +} + +func convertPath(t *testing.T, uut *sharedTempFileConverter, path string) *sharedFileReader { + t.Helper() + f, err := open(path) + assert.NilError(t, err) + defer func() { assert.NilError(t, f.Close()) }() + r, err := uut.Do(f) + assert.NilError(t, err) + return r +} + +func readAll(t *testing.T, r io.Reader) string { + t.Helper() + v, err := io.ReadAll(r) + assert.NilError(t, err) + return string(v) +} + +func checkDirEmpty(t *testing.T, path string) { + t.Helper() + ls, err := os.ReadDir(path) + assert.NilError(t, err) + assert.Check(t, cmp.Len(ls, 0), "directory should be free of temp files") +} + +func copyTransform(f func(string) string) func(dst io.WriteSeeker, src io.ReadSeeker) error { + return func(dst io.WriteSeeker, src io.ReadSeeker) error { + s, err := io.ReadAll(src) + if err != nil { + return err + } + _, err = io.WriteString(dst, f(string(s))) + return err + } +} diff --git a/pkg/filenotify/filenotify.go b/pkg/filenotify/filenotify.go deleted file mode 100644 index 8b6cb56f17..0000000000 --- a/pkg/filenotify/filenotify.go +++ /dev/null @@ -1,40 +0,0 @@ -// Package filenotify provides a mechanism for watching file(s) for changes. -// Generally leans on fsnotify, but provides a poll-based notifier which fsnotify does not support. -// These are wrapped up in a common interface so that either can be used interchangeably in your code. -package filenotify // import "github.com/docker/docker/pkg/filenotify" - -import "github.com/fsnotify/fsnotify" - -// FileWatcher is an interface for implementing file notification watchers -type FileWatcher interface { - Events() <-chan fsnotify.Event - Errors() <-chan error - Add(name string) error - Remove(name string) error - Close() error -} - -// New tries to use an fs-event watcher, and falls back to the poller if there is an error -func New() (FileWatcher, error) { - if watcher, err := NewEventWatcher(); err == nil { - return watcher, nil - } - return NewPollingWatcher(), nil -} - -// NewPollingWatcher returns a poll-based file watcher -func NewPollingWatcher() FileWatcher { - return &filePoller{ - events: make(chan fsnotify.Event), - errors: make(chan error), - } -} - -// NewEventWatcher returns an fs-event based file watcher -func NewEventWatcher() (FileWatcher, error) { - watcher, err := fsnotify.NewWatcher() - if err != nil { - return nil, err - } - return &fsNotifyWatcher{watcher}, nil -} diff --git a/pkg/filenotify/fsnotify.go b/pkg/filenotify/fsnotify.go deleted file mode 100644 index 5a737d6530..0000000000 --- a/pkg/filenotify/fsnotify.go +++ /dev/null @@ -1,18 +0,0 @@ -package filenotify // import "github.com/docker/docker/pkg/filenotify" - -import "github.com/fsnotify/fsnotify" - -// fsNotifyWatcher wraps the fsnotify package to satisfy the FileNotifier interface -type fsNotifyWatcher struct { - *fsnotify.Watcher -} - -// Events returns the fsnotify event channel receiver -func (w *fsNotifyWatcher) Events() <-chan fsnotify.Event { - return w.Watcher.Events -} - -// Errors returns the fsnotify error channel receiver -func (w *fsNotifyWatcher) Errors() <-chan error { - return w.Watcher.Errors -} diff --git a/pkg/filenotify/poller.go b/pkg/filenotify/poller.go deleted file mode 100644 index 01ef057981..0000000000 --- a/pkg/filenotify/poller.go +++ /dev/null @@ -1,213 +0,0 @@ -package filenotify // import "github.com/docker/docker/pkg/filenotify" - -import ( - "errors" - "fmt" - "os" - "sync" - "time" - - "github.com/sirupsen/logrus" - - "github.com/fsnotify/fsnotify" -) - -var ( - // errPollerClosed is returned when the poller is closed - errPollerClosed = errors.New("poller is closed") - // errNoSuchWatch is returned when trying to remove a watch that doesn't exist - errNoSuchWatch = errors.New("watch does not exist") -) - -// watchWaitTime is the time to wait between file poll loops -const watchWaitTime = 200 * time.Millisecond - -// filePoller is used to poll files for changes, especially in cases where fsnotify -// can't be run (e.g. when inotify handles are exhausted) -// filePoller satisfies the FileWatcher interface -type filePoller struct { - // watches is the list of files currently being polled, close the associated channel to stop the watch - watches map[string]chan struct{} - // events is the channel to listen to for watch events - events chan fsnotify.Event - // errors is the channel to listen to for watch errors - errors chan error - // mu locks the poller for modification - mu sync.Mutex - // closed is used to specify when the poller has already closed - closed bool -} - -// Add adds a filename to the list of watches -// once added the file is polled for changes in a separate goroutine -func (w *filePoller) Add(name string) error { - w.mu.Lock() - defer w.mu.Unlock() - - if w.closed { - return errPollerClosed - } - - f, err := os.Open(name) - if err != nil { - return err - } - fi, err := os.Stat(name) - if err != nil { - f.Close() - return err - } - - if w.watches == nil { - w.watches = make(map[string]chan struct{}) - } - if _, exists := w.watches[name]; exists { - f.Close() - return fmt.Errorf("watch exists") - } - chClose := make(chan struct{}) - w.watches[name] = chClose - - go w.watch(f, fi, chClose) - return nil -} - -// Remove stops and removes watch with the specified name -func (w *filePoller) Remove(name string) error { - w.mu.Lock() - defer w.mu.Unlock() - return w.remove(name) -} - -func (w *filePoller) remove(name string) error { - if w.closed { - return errPollerClosed - } - - chClose, exists := w.watches[name] - if !exists { - return errNoSuchWatch - } - close(chClose) - delete(w.watches, name) - return nil -} - -// Events returns the event channel -// This is used for notifications on events about watched files -func (w *filePoller) Events() <-chan fsnotify.Event { - return w.events -} - -// Errors returns the errors channel -// This is used for notifications about errors on watched files -func (w *filePoller) Errors() <-chan error { - return w.errors -} - -// Close closes the poller -// All watches are stopped, removed, and the poller cannot be added to -func (w *filePoller) Close() error { - w.mu.Lock() - defer w.mu.Unlock() - - if w.closed { - return nil - } - - for name := range w.watches { - w.remove(name) - } - w.closed = true - return nil -} - -// sendEvent publishes the specified event to the events channel -func (w *filePoller) sendEvent(e fsnotify.Event, chClose <-chan struct{}) error { - select { - case w.events <- e: - case <-chClose: - return fmt.Errorf("closed") - } - return nil -} - -// sendErr publishes the specified error to the errors channel -func (w *filePoller) sendErr(e error, chClose <-chan struct{}) error { - select { - case w.errors <- e: - case <-chClose: - return fmt.Errorf("closed") - } - return nil -} - -// watch is responsible for polling the specified file for changes -// upon finding changes to a file or errors, sendEvent/sendErr is called -func (w *filePoller) watch(f *os.File, lastFi os.FileInfo, chClose chan struct{}) { - defer f.Close() - - timer := time.NewTimer(watchWaitTime) - if !timer.Stop() { - <-timer.C - } - defer timer.Stop() - - for { - timer.Reset(watchWaitTime) - - select { - case <-timer.C: - case <-chClose: - logrus.Debugf("watch for %s closed", f.Name()) - return - } - - fi, err := os.Stat(f.Name()) - if err != nil { - // if we got an error here and lastFi is not set, we can presume that nothing has changed - // This should be safe since before `watch()` is called, a stat is performed, there is any error `watch` is not called - if lastFi == nil { - continue - } - // If it doesn't exist at this point, it must have been removed - // no need to send the error here since this is a valid operation - if os.IsNotExist(err) { - if err := w.sendEvent(fsnotify.Event{Op: fsnotify.Remove, Name: f.Name()}, chClose); err != nil { - return - } - lastFi = nil - continue - } - // at this point, send the error - if err := w.sendErr(err, chClose); err != nil { - return - } - continue - } - - if lastFi == nil { - if err := w.sendEvent(fsnotify.Event{Op: fsnotify.Create, Name: fi.Name()}, chClose); err != nil { - return - } - lastFi = fi - continue - } - - if fi.Mode() != lastFi.Mode() { - if err := w.sendEvent(fsnotify.Event{Op: fsnotify.Chmod, Name: fi.Name()}, chClose); err != nil { - return - } - lastFi = fi - continue - } - - if fi.ModTime() != lastFi.ModTime() || fi.Size() != lastFi.Size() { - if err := w.sendEvent(fsnotify.Event{Op: fsnotify.Write, Name: fi.Name()}, chClose); err != nil { - return - } - lastFi = fi - continue - } - } -} diff --git a/pkg/filenotify/poller_test.go b/pkg/filenotify/poller_test.go deleted file mode 100644 index 53ec7b8508..0000000000 --- a/pkg/filenotify/poller_test.go +++ /dev/null @@ -1,131 +0,0 @@ -package filenotify // import "github.com/docker/docker/pkg/filenotify" - -import ( - "fmt" - "os" - "runtime" - "testing" - "time" - - "github.com/fsnotify/fsnotify" -) - -func TestPollerAddRemove(t *testing.T) { - w := NewPollingWatcher() - - if err := w.Add("no-such-file"); err == nil { - t.Fatal("should have gotten error when adding a non-existent file") - } - if err := w.Remove("no-such-file"); err == nil { - t.Fatal("should have gotten error when removing non-existent watch") - } - - f, err := os.CreateTemp("", "asdf") - if err != nil { - t.Fatal(err) - } - defer os.RemoveAll(f.Name()) - - if err := w.Add(f.Name()); err != nil { - t.Fatal(err) - } - - if err := w.Remove(f.Name()); err != nil { - t.Fatal(err) - } -} - -func TestPollerEvent(t *testing.T) { - if runtime.GOOS == "windows" { - t.Skip("No chmod on Windows") - } - w := NewPollingWatcher() - - f, err := os.CreateTemp("", "test-poller") - if err != nil { - t.Fatal("error creating temp file") - } - defer os.RemoveAll(f.Name()) - f.Close() - - if err := w.Add(f.Name()); err != nil { - t.Fatal(err) - } - - select { - case <-w.Events(): - t.Fatal("got event before anything happened") - case <-w.Errors(): - t.Fatal("got error before anything happened") - default: - } - - if err := os.WriteFile(f.Name(), []byte("hello"), 0600); err != nil { - t.Fatal(err) - } - assertFileMode(t, f.Name(), 0600) - if err := assertEvent(w, fsnotify.Write); err != nil { - t.Fatal(err) - } - - if err := os.Chmod(f.Name(), 0644); err != nil { - t.Fatal(err) - } - assertFileMode(t, f.Name(), 0644) - if err := assertEvent(w, fsnotify.Chmod); err != nil { - t.Fatal(err) - } - - if err := os.Remove(f.Name()); err != nil { - t.Fatal(err) - } - if err := assertEvent(w, fsnotify.Remove); err != nil { - t.Fatal(err) - } -} - -func TestPollerClose(t *testing.T) { - w := NewPollingWatcher() - if err := w.Close(); err != nil { - t.Fatal(err) - } - // test double-close - if err := w.Close(); err != nil { - t.Fatal(err) - } - - f, err := os.CreateTemp("", "asdf") - if err != nil { - t.Fatal(err) - } - defer os.RemoveAll(f.Name()) - if err := w.Add(f.Name()); err == nil { - t.Fatal("should have gotten error adding watch for closed watcher") - } -} - -func assertFileMode(t *testing.T, fileName string, mode uint32) { - t.Helper() - f, err := os.Stat(fileName) - if err != nil { - t.Fatal(err) - } - if f.Mode() != os.FileMode(mode) { - t.Fatalf("expected file %s to have mode %#o, but got %#o", fileName, mode, f.Mode()) - } -} - -func assertEvent(w FileWatcher, eType fsnotify.Op) error { - var err error - select { - case e := <-w.Events(): - if e.Op != eType { - err = fmt.Errorf("got wrong event type, expected %q: %v", eType, e.Op) - } - case e := <-w.Errors(): - err = fmt.Errorf("got unexpected error waiting for events %v: %v", eType, e) - case <-time.After(watchWaitTime * 3): - err = fmt.Errorf("timeout waiting for event %v", eType) - } - return err -} diff --git a/vendor.mod b/vendor.mod index dd6b00259a..e674f8a366 100644 --- a/vendor.mod +++ b/vendor.mod @@ -33,7 +33,6 @@ require ( github.com/docker/libkv v0.2.2-0.20211217103745-e480589147e3 github.com/docker/libtrust v0.0.0-20150526203908-9cbd2a1374f4 github.com/fluent/fluent-logger-golang v1.9.0 - github.com/fsnotify/fsnotify v1.5.1 github.com/godbus/dbus/v5 v5.0.6 github.com/gogo/protobuf v1.3.2 github.com/golang/gddo v0.0.0-20190904175337-72a348e765d2 @@ -106,6 +105,7 @@ require ( github.com/dustin/go-humanize v1.0.0 // indirect github.com/felixge/httpsnoop v1.0.2 // indirect github.com/fernet/fernet-go v0.0.0-20180830025343-9eac43b88a5e // indirect + github.com/fsnotify/fsnotify v1.5.1 // indirect github.com/go-logr/logr v1.2.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/gofrs/flock v0.8.1 // indirect diff --git a/vendor/github.com/fsnotify/fsnotify/.gitattributes b/vendor/github.com/fsnotify/fsnotify/.gitattributes deleted file mode 100644 index 32f1001be0..0000000000 --- a/vendor/github.com/fsnotify/fsnotify/.gitattributes +++ /dev/null @@ -1 +0,0 @@ -go.sum linguist-generated diff --git a/vendor/github.com/fsnotify/fsnotify/.gitignore b/vendor/github.com/fsnotify/fsnotify/.gitignore deleted file mode 100644 index 4cd0cbaf43..0000000000 --- a/vendor/github.com/fsnotify/fsnotify/.gitignore +++ /dev/null @@ -1,6 +0,0 @@ -# Setup a Global .gitignore for OS and editor generated files: -# https://help.github.com/articles/ignoring-files -# git config --global core.excludesfile ~/.gitignore_global - -.vagrant -*.sublime-project diff --git a/vendor/github.com/fsnotify/fsnotify/.mailmap b/vendor/github.com/fsnotify/fsnotify/.mailmap deleted file mode 100644 index a04f2907fe..0000000000 --- a/vendor/github.com/fsnotify/fsnotify/.mailmap +++ /dev/null @@ -1,2 +0,0 @@ -Chris Howey -Nathan Youngman <4566+nathany@users.noreply.github.com> diff --git a/vendor/github.com/fsnotify/fsnotify/AUTHORS b/vendor/github.com/fsnotify/fsnotify/AUTHORS deleted file mode 100644 index 6cbabe5ef5..0000000000 --- a/vendor/github.com/fsnotify/fsnotify/AUTHORS +++ /dev/null @@ -1,62 +0,0 @@ -# Names should be added to this file as -# Name or Organization -# The email address is not required for organizations. - -# You can update this list using the following command: -# -# $ (head -n10 AUTHORS && git shortlog -se | sed -E 's/^\s+[0-9]+\t//') | tee AUTHORS - -# Please keep the list sorted. - -Aaron L -Adrien Bustany -Alexey Kazakov -Amit Krishnan -Anmol Sethi -Bjørn Erik Pedersen -Brian Goff -Bruno Bigras -Caleb Spare -Case Nelson -Chris Howey -Christoffer Buchholz -Daniel Wagner-Hall -Dave Cheney -Eric Lin -Evan Phoenix -Francisco Souza -Gautam Dey -Hari haran -Ichinose Shogo -Johannes Ebke -John C Barstow -Kelvin Fo -Ken-ichirou MATSUZAWA -Matt Layher -Matthias Stone -Nathan Youngman -Nickolai Zeldovich -Oliver Bristow -Patrick -Paul Hammond -Pawel Knap -Pieter Droogendijk -Pratik Shinde -Pursuit92 -Riku Voipio -Rob Figueiredo -Rodrigo Chiossi -Slawek Ligus -Soge Zhang -Tiffany Jernigan -Tilak Sharma -Tobias Klauser -Tom Payne -Travis Cline -Tudor Golubenco -Vahe Khachikyan -Yukang -bronze1man -debrando -henrikedwards -铁哥 diff --git a/vendor/github.com/fsnotify/fsnotify/CHANGELOG.md b/vendor/github.com/fsnotify/fsnotify/CHANGELOG.md deleted file mode 100644 index a438fe4b4a..0000000000 --- a/vendor/github.com/fsnotify/fsnotify/CHANGELOG.md +++ /dev/null @@ -1,339 +0,0 @@ -# Changelog - -All notable changes to this project will be documented in this file. - -The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), -and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - -## [Unreleased] - -## [1.5.1] - 2021-08-24 - -* Revert Add AddRaw to not follow symlinks - -## [1.5.0] - 2021-08-20 - -* Go: Increase minimum required version to Go 1.12 [#381](https://github.com/fsnotify/fsnotify/pull/381) -* Feature: Add AddRaw method which does not follow symlinks when adding a watch [#289](https://github.com/fsnotify/fsnotify/pull/298) -* Windows: Follow symlinks by default like on all other systems [#289](https://github.com/fsnotify/fsnotify/pull/289) -* CI: Use GitHub Actions for CI and cover go 1.12-1.17 - [#378](https://github.com/fsnotify/fsnotify/pull/378) - [#381](https://github.com/fsnotify/fsnotify/pull/381) - [#385](https://github.com/fsnotify/fsnotify/pull/385) -* Go 1.14+: Fix unsafe pointer conversion [#325](https://github.com/fsnotify/fsnotify/pull/325) - -## [1.4.7] - 2018-01-09 - -* BSD/macOS: Fix possible deadlock on closing the watcher on kqueue (thanks @nhooyr and @glycerine) -* Tests: Fix missing verb on format string (thanks @rchiossi) -* Linux: Fix deadlock in Remove (thanks @aarondl) -* Linux: Watch.Add improvements (avoid race, fix consistency, reduce garbage) (thanks @twpayne) -* Docs: Moved FAQ into the README (thanks @vahe) -* Linux: Properly handle inotify's IN_Q_OVERFLOW event (thanks @zeldovich) -* Docs: replace references to OS X with macOS - -## [1.4.2] - 2016-10-10 - -* Linux: use InotifyInit1 with IN_CLOEXEC to stop leaking a file descriptor to a child process when using fork/exec [#178](https://github.com/fsnotify/fsnotify/pull/178) (thanks @pattyshack) - -## [1.4.1] - 2016-10-04 - -* Fix flaky inotify stress test on Linux [#177](https://github.com/fsnotify/fsnotify/pull/177) (thanks @pattyshack) - -## [1.4.0] - 2016-10-01 - -* add a String() method to Event.Op [#165](https://github.com/fsnotify/fsnotify/pull/165) (thanks @oozie) - -## [1.3.1] - 2016-06-28 - -* Windows: fix for double backslash when watching the root of a drive [#151](https://github.com/fsnotify/fsnotify/issues/151) (thanks @brunoqc) - -## [1.3.0] - 2016-04-19 - -* Support linux/arm64 by [patching](https://go-review.googlesource.com/#/c/21971/) x/sys/unix and switching to to it from syscall (thanks @suihkulokki) [#135](https://github.com/fsnotify/fsnotify/pull/135) - -## [1.2.10] - 2016-03-02 - -* Fix golint errors in windows.go [#121](https://github.com/fsnotify/fsnotify/pull/121) (thanks @tiffanyfj) - -## [1.2.9] - 2016-01-13 - -kqueue: Fix logic for CREATE after REMOVE [#111](https://github.com/fsnotify/fsnotify/pull/111) (thanks @bep) - -## [1.2.8] - 2015-12-17 - -* kqueue: fix race condition in Close [#105](https://github.com/fsnotify/fsnotify/pull/105) (thanks @djui for reporting the issue and @ppknap for writing a failing test) -* inotify: fix race in test -* enable race detection for continuous integration (Linux, Mac, Windows) - -## [1.2.5] - 2015-10-17 - -* inotify: use epoll_create1 for arm64 support (requires Linux 2.6.27 or later) [#100](https://github.com/fsnotify/fsnotify/pull/100) (thanks @suihkulokki) -* inotify: fix path leaks [#73](https://github.com/fsnotify/fsnotify/pull/73) (thanks @chamaken) -* kqueue: watch for rename events on subdirectories [#83](https://github.com/fsnotify/fsnotify/pull/83) (thanks @guotie) -* kqueue: avoid infinite loops from symlinks cycles [#101](https://github.com/fsnotify/fsnotify/pull/101) (thanks @illicitonion) - -## [1.2.1] - 2015-10-14 - -* kqueue: don't watch named pipes [#98](https://github.com/fsnotify/fsnotify/pull/98) (thanks @evanphx) - -## [1.2.0] - 2015-02-08 - -* inotify: use epoll to wake up readEvents [#66](https://github.com/fsnotify/fsnotify/pull/66) (thanks @PieterD) -* inotify: closing watcher should now always shut down goroutine [#63](https://github.com/fsnotify/fsnotify/pull/63) (thanks @PieterD) -* kqueue: close kqueue after removing watches, fixes [#59](https://github.com/fsnotify/fsnotify/issues/59) - -## [1.1.1] - 2015-02-05 - -* inotify: Retry read on EINTR [#61](https://github.com/fsnotify/fsnotify/issues/61) (thanks @PieterD) - -## [1.1.0] - 2014-12-12 - -* kqueue: rework internals [#43](https://github.com/fsnotify/fsnotify/pull/43) - * add low-level functions - * only need to store flags on directories - * less mutexes [#13](https://github.com/fsnotify/fsnotify/issues/13) - * done can be an unbuffered channel - * remove calls to os.NewSyscallError -* More efficient string concatenation for Event.String() [#52](https://github.com/fsnotify/fsnotify/pull/52) (thanks @mdlayher) -* kqueue: fix regression in rework causing subdirectories to be watched [#48](https://github.com/fsnotify/fsnotify/issues/48) -* kqueue: cleanup internal watch before sending remove event [#51](https://github.com/fsnotify/fsnotify/issues/51) - -## [1.0.4] - 2014-09-07 - -* kqueue: add dragonfly to the build tags. -* Rename source code files, rearrange code so exported APIs are at the top. -* Add done channel to example code. [#37](https://github.com/fsnotify/fsnotify/pull/37) (thanks @chenyukang) - -## [1.0.3] - 2014-08-19 - -* [Fix] Windows MOVED_TO now translates to Create like on BSD and Linux. [#36](https://github.com/fsnotify/fsnotify/issues/36) - -## [1.0.2] - 2014-08-17 - -* [Fix] Missing create events on macOS. [#14](https://github.com/fsnotify/fsnotify/issues/14) (thanks @zhsso) -* [Fix] Make ./path and path equivalent. (thanks @zhsso) - -## [1.0.0] - 2014-08-15 - -* [API] Remove AddWatch on Windows, use Add. -* Improve documentation for exported identifiers. [#30](https://github.com/fsnotify/fsnotify/issues/30) -* Minor updates based on feedback from golint. - -## dev / 2014-07-09 - -* Moved to [github.com/fsnotify/fsnotify](https://github.com/fsnotify/fsnotify). -* Use os.NewSyscallError instead of returning errno (thanks @hariharan-uno) - -## dev / 2014-07-04 - -* kqueue: fix incorrect mutex used in Close() -* Update example to demonstrate usage of Op. - -## dev / 2014-06-28 - -* [API] Don't set the Write Op for attribute notifications [#4](https://github.com/fsnotify/fsnotify/issues/4) -* Fix for String() method on Event (thanks Alex Brainman) -* Don't build on Plan 9 or Solaris (thanks @4ad) - -## dev / 2014-06-21 - -* Events channel of type Event rather than *Event. -* [internal] use syscall constants directly for inotify and kqueue. -* [internal] kqueue: rename events to kevents and fileEvent to event. - -## dev / 2014-06-19 - -* Go 1.3+ required on Windows (uses syscall.ERROR_MORE_DATA internally). -* [internal] remove cookie from Event struct (unused). -* [internal] Event struct has the same definition across every OS. -* [internal] remove internal watch and removeWatch methods. - -## dev / 2014-06-12 - -* [API] Renamed Watch() to Add() and RemoveWatch() to Remove(). -* [API] Pluralized channel names: Events and Errors. -* [API] Renamed FileEvent struct to Event. -* [API] Op constants replace methods like IsCreate(). - -## dev / 2014-06-12 - -* Fix data race on kevent buffer (thanks @tilaks) [#98](https://github.com/howeyc/fsnotify/pull/98) - -## dev / 2014-05-23 - -* [API] Remove current implementation of WatchFlags. - * current implementation doesn't take advantage of OS for efficiency - * provides little benefit over filtering events as they are received, but has extra bookkeeping and mutexes - * no tests for the current implementation - * not fully implemented on Windows [#93](https://github.com/howeyc/fsnotify/issues/93#issuecomment-39285195) - -## [0.9.3] - 2014-12-31 - -* kqueue: cleanup internal watch before sending remove event [#51](https://github.com/fsnotify/fsnotify/issues/51) - -## [0.9.2] - 2014-08-17 - -* [Backport] Fix missing create events on macOS. [#14](https://github.com/fsnotify/fsnotify/issues/14) (thanks @zhsso) - -## [0.9.1] - 2014-06-12 - -* Fix data race on kevent buffer (thanks @tilaks) [#98](https://github.com/howeyc/fsnotify/pull/98) - -## [0.9.0] - 2014-01-17 - -* IsAttrib() for events that only concern a file's metadata [#79][] (thanks @abustany) -* [Fix] kqueue: fix deadlock [#77][] (thanks @cespare) -* [NOTICE] Development has moved to `code.google.com/p/go.exp/fsnotify` in preparation for inclusion in the Go standard library. - -## [0.8.12] - 2013-11-13 - -* [API] Remove FD_SET and friends from Linux adapter - -## [0.8.11] - 2013-11-02 - -* [Doc] Add Changelog [#72][] (thanks @nathany) -* [Doc] Spotlight and double modify events on macOS [#62][] (reported by @paulhammond) - -## [0.8.10] - 2013-10-19 - -* [Fix] kqueue: remove file watches when parent directory is removed [#71][] (reported by @mdwhatcott) -* [Fix] kqueue: race between Close and readEvents [#70][] (reported by @bernerdschaefer) -* [Doc] specify OS-specific limits in README (thanks @debrando) - -## [0.8.9] - 2013-09-08 - -* [Doc] Contributing (thanks @nathany) -* [Doc] update package path in example code [#63][] (thanks @paulhammond) -* [Doc] GoCI badge in README (Linux only) [#60][] -* [Doc] Cross-platform testing with Vagrant [#59][] (thanks @nathany) - -## [0.8.8] - 2013-06-17 - -* [Fix] Windows: handle `ERROR_MORE_DATA` on Windows [#49][] (thanks @jbowtie) - -## [0.8.7] - 2013-06-03 - -* [API] Make syscall flags internal -* [Fix] inotify: ignore event changes -* [Fix] race in symlink test [#45][] (reported by @srid) -* [Fix] tests on Windows -* lower case error messages - -## [0.8.6] - 2013-05-23 - -* kqueue: Use EVT_ONLY flag on Darwin -* [Doc] Update README with full example - -## [0.8.5] - 2013-05-09 - -* [Fix] inotify: allow monitoring of "broken" symlinks (thanks @tsg) - -## [0.8.4] - 2013-04-07 - -* [Fix] kqueue: watch all file events [#40][] (thanks @ChrisBuchholz) - -## [0.8.3] - 2013-03-13 - -* [Fix] inoitfy/kqueue memory leak [#36][] (reported by @nbkolchin) -* [Fix] kqueue: use fsnFlags for watching a directory [#33][] (reported by @nbkolchin) - -## [0.8.2] - 2013-02-07 - -* [Doc] add Authors -* [Fix] fix data races for map access [#29][] (thanks @fsouza) - -## [0.8.1] - 2013-01-09 - -* [Fix] Windows path separators -* [Doc] BSD License - -## [0.8.0] - 2012-11-09 - -* kqueue: directory watching improvements (thanks @vmirage) -* inotify: add `IN_MOVED_TO` [#25][] (requested by @cpisto) -* [Fix] kqueue: deleting watched directory [#24][] (reported by @jakerr) - -## [0.7.4] - 2012-10-09 - -* [Fix] inotify: fixes from https://codereview.appspot.com/5418045/ (ugorji) -* [Fix] kqueue: preserve watch flags when watching for delete [#21][] (reported by @robfig) -* [Fix] kqueue: watch the directory even if it isn't a new watch (thanks @robfig) -* [Fix] kqueue: modify after recreation of file - -## [0.7.3] - 2012-09-27 - -* [Fix] kqueue: watch with an existing folder inside the watched folder (thanks @vmirage) -* [Fix] kqueue: no longer get duplicate CREATE events - -## [0.7.2] - 2012-09-01 - -* kqueue: events for created directories - -## [0.7.1] - 2012-07-14 - -* [Fix] for renaming files - -## [0.7.0] - 2012-07-02 - -* [Feature] FSNotify flags -* [Fix] inotify: Added file name back to event path - -## [0.6.0] - 2012-06-06 - -* kqueue: watch files after directory created (thanks @tmc) - -## [0.5.1] - 2012-05-22 - -* [Fix] inotify: remove all watches before Close() - -## [0.5.0] - 2012-05-03 - -* [API] kqueue: return errors during watch instead of sending over channel -* kqueue: match symlink behavior on Linux -* inotify: add `DELETE_SELF` (requested by @taralx) -* [Fix] kqueue: handle EINTR (reported by @robfig) -* [Doc] Godoc example [#1][] (thanks @davecheney) - -## [0.4.0] - 2012-03-30 - -* Go 1 released: build with go tool -* [Feature] Windows support using winfsnotify -* Windows does not have attribute change notifications -* Roll attribute notifications into IsModify - -## [0.3.0] - 2012-02-19 - -* kqueue: add files when watch directory - -## [0.2.0] - 2011-12-30 - -* update to latest Go weekly code - -## [0.1.0] - 2011-10-19 - -* kqueue: add watch on file creation to match inotify -* kqueue: create file event -* inotify: ignore `IN_IGNORED` events -* event String() -* linux: common FileEvent functions -* initial commit - -[#79]: https://github.com/howeyc/fsnotify/pull/79 -[#77]: https://github.com/howeyc/fsnotify/pull/77 -[#72]: https://github.com/howeyc/fsnotify/issues/72 -[#71]: https://github.com/howeyc/fsnotify/issues/71 -[#70]: https://github.com/howeyc/fsnotify/issues/70 -[#63]: https://github.com/howeyc/fsnotify/issues/63 -[#62]: https://github.com/howeyc/fsnotify/issues/62 -[#60]: https://github.com/howeyc/fsnotify/issues/60 -[#59]: https://github.com/howeyc/fsnotify/issues/59 -[#49]: https://github.com/howeyc/fsnotify/issues/49 -[#45]: https://github.com/howeyc/fsnotify/issues/45 -[#40]: https://github.com/howeyc/fsnotify/issues/40 -[#36]: https://github.com/howeyc/fsnotify/issues/36 -[#33]: https://github.com/howeyc/fsnotify/issues/33 -[#29]: https://github.com/howeyc/fsnotify/issues/29 -[#25]: https://github.com/howeyc/fsnotify/issues/25 -[#24]: https://github.com/howeyc/fsnotify/issues/24 -[#21]: https://github.com/howeyc/fsnotify/issues/21 diff --git a/vendor/github.com/fsnotify/fsnotify/CONTRIBUTING.md b/vendor/github.com/fsnotify/fsnotify/CONTRIBUTING.md deleted file mode 100644 index 828a60b24b..0000000000 --- a/vendor/github.com/fsnotify/fsnotify/CONTRIBUTING.md +++ /dev/null @@ -1,77 +0,0 @@ -# Contributing - -## Issues - -* Request features and report bugs using the [GitHub Issue Tracker](https://github.com/fsnotify/fsnotify/issues). -* Please indicate the platform you are using fsnotify on. -* A code example to reproduce the problem is appreciated. - -## Pull Requests - -### Contributor License Agreement - -fsnotify is derived from code in the [golang.org/x/exp](https://godoc.org/golang.org/x/exp) package and it may be included [in the standard library](https://github.com/fsnotify/fsnotify/issues/1) in the future. Therefore fsnotify carries the same [LICENSE](https://github.com/fsnotify/fsnotify/blob/master/LICENSE) as Go. Contributors retain their copyright, so you need to fill out a short form before we can accept your contribution: [Google Individual Contributor License Agreement](https://developers.google.com/open-source/cla/individual). - -Please indicate that you have signed the CLA in your pull request. - -### How fsnotify is Developed - -* Development is done on feature branches. -* Tests are run on BSD, Linux, macOS and Windows. -* Pull requests are reviewed and [applied to master][am] using [hub][]. - * Maintainers may modify or squash commits rather than asking contributors to. -* To issue a new release, the maintainers will: - * Update the CHANGELOG - * Tag a version, which will become available through gopkg.in. - -### How to Fork - -For smooth sailing, always use the original import path. Installing with `go get` makes this easy. - -1. Install from GitHub (`go get -u github.com/fsnotify/fsnotify`) -2. Create your feature branch (`git checkout -b my-new-feature`) -3. Ensure everything works and the tests pass (see below) -4. Commit your changes (`git commit -am 'Add some feature'`) - -Contribute upstream: - -1. Fork fsnotify on GitHub -2. Add your remote (`git remote add fork git@github.com:mycompany/repo.git`) -3. Push to the branch (`git push fork my-new-feature`) -4. Create a new Pull Request on GitHub - -This workflow is [thoroughly explained by Katrina Owen](https://splice.com/blog/contributing-open-source-git-repositories-go/). - -### Testing - -fsnotify uses build tags to compile different code on Linux, BSD, macOS, and Windows. - -Before doing a pull request, please do your best to test your changes on multiple platforms, and list which platforms you were able/unable to test on. - -To aid in cross-platform testing there is a Vagrantfile for Linux and BSD. - -* Install [Vagrant](http://www.vagrantup.com/) and [VirtualBox](https://www.virtualbox.org/) -* Setup [Vagrant Gopher](https://github.com/nathany/vagrant-gopher) in your `src` folder. -* Run `vagrant up` from the project folder. You can also setup just one box with `vagrant up linux` or `vagrant up bsd` (note: the BSD box doesn't support Windows hosts at this time, and NFS may prompt for your host OS password) -* Once setup, you can run the test suite on a given OS with a single command `vagrant ssh linux -c 'cd fsnotify/fsnotify; go test'`. -* When you're done, you will want to halt or destroy the Vagrant boxes. - -Notice: fsnotify file system events won't trigger in shared folders. The tests get around this limitation by using the /tmp directory. - -Right now there is no equivalent solution for Windows and macOS, but there are Windows VMs [freely available from Microsoft](http://www.modern.ie/en-us/virtualization-tools#downloads). - -### Maintainers - -Help maintaining fsnotify is welcome. To be a maintainer: - -* Submit a pull request and sign the CLA as above. -* You must be able to run the test suite on Mac, Windows, Linux and BSD. - -To keep master clean, the fsnotify project uses the "apply mail" workflow outlined in Nathaniel Talbott's post ["Merge pull request" Considered Harmful][am]. This requires installing [hub][]. - -All code changes should be internal pull requests. - -Releases are tagged using [Semantic Versioning](http://semver.org/). - -[hub]: https://github.com/github/hub -[am]: http://blog.spreedly.com/2014/06/24/merge-pull-request-considered-harmful/#.VGa5yZPF_Zs diff --git a/vendor/github.com/fsnotify/fsnotify/LICENSE b/vendor/github.com/fsnotify/fsnotify/LICENSE deleted file mode 100644 index e180c8fb05..0000000000 --- a/vendor/github.com/fsnotify/fsnotify/LICENSE +++ /dev/null @@ -1,28 +0,0 @@ -Copyright (c) 2012 The Go Authors. All rights reserved. -Copyright (c) 2012-2019 fsnotify Authors. All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/github.com/fsnotify/fsnotify/README.md b/vendor/github.com/fsnotify/fsnotify/README.md deleted file mode 100644 index df57b1b282..0000000000 --- a/vendor/github.com/fsnotify/fsnotify/README.md +++ /dev/null @@ -1,130 +0,0 @@ -# File system notifications for Go - -[![GoDoc](https://godoc.org/github.com/fsnotify/fsnotify?status.svg)](https://godoc.org/github.com/fsnotify/fsnotify) [![Go Report Card](https://goreportcard.com/badge/github.com/fsnotify/fsnotify)](https://goreportcard.com/report/github.com/fsnotify/fsnotify) - -fsnotify utilizes [golang.org/x/sys](https://godoc.org/golang.org/x/sys) rather than `syscall` from the standard library. Ensure you have the latest version installed by running: - -```console -go get -u golang.org/x/sys/... -``` - -Cross platform: Windows, Linux, BSD and macOS. - -| Adapter | OS | Status | -| --------------------- | -------------------------------- | ------------------------------------------------------------------------------------------------------------------------------- | -| inotify | Linux 2.6.27 or later, Android\* | Supported | -| kqueue | BSD, macOS, iOS\* | Supported | -| ReadDirectoryChangesW | Windows | Supported | -| FSEvents | macOS | [Planned](https://github.com/fsnotify/fsnotify/issues/11) | -| FEN | Solaris 11 | [In Progress](https://github.com/fsnotify/fsnotify/issues/12) | -| fanotify | Linux 2.6.37+ | [Planned](https://github.com/fsnotify/fsnotify/issues/114) | -| USN Journals | Windows | [Maybe](https://github.com/fsnotify/fsnotify/issues/53) | -| Polling | *All* | [Maybe](https://github.com/fsnotify/fsnotify/issues/9) | - -\* Android and iOS are untested. - -Please see [the documentation](https://godoc.org/github.com/fsnotify/fsnotify) and consult the [FAQ](#faq) for usage information. - -## API stability - -fsnotify is a fork of [howeyc/fsnotify](https://godoc.org/github.com/howeyc/fsnotify) with a new API as of v1.0. The API is based on [this design document](http://goo.gl/MrYxyA). - -All [releases](https://github.com/fsnotify/fsnotify/releases) are tagged based on [Semantic Versioning](http://semver.org/). Further API changes are [planned](https://github.com/fsnotify/fsnotify/milestones), and will be tagged with a new major revision number. - -Go 1.6 supports dependencies located in the `vendor/` folder. Unless you are creating a library, it is recommended that you copy fsnotify into `vendor/github.com/fsnotify/fsnotify` within your project, and likewise for `golang.org/x/sys`. - -## Usage - -```go -package main - -import ( - "log" - - "github.com/fsnotify/fsnotify" -) - -func main() { - watcher, err := fsnotify.NewWatcher() - if err != nil { - log.Fatal(err) - } - defer watcher.Close() - - done := make(chan bool) - go func() { - for { - select { - case event, ok := <-watcher.Events: - if !ok { - return - } - log.Println("event:", event) - if event.Op&fsnotify.Write == fsnotify.Write { - log.Println("modified file:", event.Name) - } - case err, ok := <-watcher.Errors: - if !ok { - return - } - log.Println("error:", err) - } - } - }() - - err = watcher.Add("/tmp/foo") - if err != nil { - log.Fatal(err) - } - <-done -} -``` - -## Contributing - -Please refer to [CONTRIBUTING][] before opening an issue or pull request. - -## Example - -See [example_test.go](https://github.com/fsnotify/fsnotify/blob/master/example_test.go). - -## FAQ - -**When a file is moved to another directory is it still being watched?** - -No (it shouldn't be, unless you are watching where it was moved to). - -**When I watch a directory, are all subdirectories watched as well?** - -No, you must add watches for any directory you want to watch (a recursive watcher is on the roadmap [#18][]). - -**Do I have to watch the Error and Event channels in a separate goroutine?** - -As of now, yes. Looking into making this single-thread friendly (see [howeyc #7][#7]) - -**Why am I receiving multiple events for the same file on OS X?** - -Spotlight indexing on OS X can result in multiple events (see [howeyc #62][#62]). A temporary workaround is to add your folder(s) to the *Spotlight Privacy settings* until we have a native FSEvents implementation (see [#11][]). - -**How many files can be watched at once?** - -There are OS-specific limits as to how many watches can be created: -* Linux: /proc/sys/fs/inotify/max_user_watches contains the limit, reaching this limit results in a "no space left on device" error. -* BSD / OSX: sysctl variables "kern.maxfiles" and "kern.maxfilesperproc", reaching these limits results in a "too many open files" error. - -**Why don't notifications work with NFS filesystems or filesystem in userspace (FUSE)?** - -fsnotify requires support from underlying OS to work. The current NFS protocol does not provide network level support for file notifications. - -[#62]: https://github.com/howeyc/fsnotify/issues/62 -[#18]: https://github.com/fsnotify/fsnotify/issues/18 -[#11]: https://github.com/fsnotify/fsnotify/issues/11 -[#7]: https://github.com/howeyc/fsnotify/issues/7 - -[contributing]: https://github.com/fsnotify/fsnotify/blob/master/CONTRIBUTING.md - -## Related Projects - -* [notify](https://github.com/rjeczalik/notify) -* [fsevents](https://github.com/fsnotify/fsevents) - diff --git a/vendor/github.com/fsnotify/fsnotify/fen.go b/vendor/github.com/fsnotify/fsnotify/fen.go deleted file mode 100644 index b3ac3d8f55..0000000000 --- a/vendor/github.com/fsnotify/fsnotify/fen.go +++ /dev/null @@ -1,38 +0,0 @@ -// Copyright 2010 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//go:build solaris -// +build solaris - -package fsnotify - -import ( - "errors" -) - -// Watcher watches a set of files, delivering events to a channel. -type Watcher struct { - Events chan Event - Errors chan error -} - -// NewWatcher establishes a new watcher with the underlying OS and begins waiting for events. -func NewWatcher() (*Watcher, error) { - return nil, errors.New("FEN based watcher not yet supported for fsnotify\n") -} - -// Close removes all watches and closes the events channel. -func (w *Watcher) Close() error { - return nil -} - -// Add starts watching the named file or directory (non-recursively). -func (w *Watcher) Add(name string) error { - return nil -} - -// Remove stops watching the the named file or directory (non-recursively). -func (w *Watcher) Remove(name string) error { - return nil -} diff --git a/vendor/github.com/fsnotify/fsnotify/fsnotify.go b/vendor/github.com/fsnotify/fsnotify/fsnotify.go deleted file mode 100644 index 0f4ee52e8a..0000000000 --- a/vendor/github.com/fsnotify/fsnotify/fsnotify.go +++ /dev/null @@ -1,69 +0,0 @@ -// Copyright 2012 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//go:build !plan9 -// +build !plan9 - -// Package fsnotify provides a platform-independent interface for file system notifications. -package fsnotify - -import ( - "bytes" - "errors" - "fmt" -) - -// Event represents a single file system notification. -type Event struct { - Name string // Relative path to the file or directory. - Op Op // File operation that triggered the event. -} - -// Op describes a set of file operations. -type Op uint32 - -// These are the generalized file operations that can trigger a notification. -const ( - Create Op = 1 << iota - Write - Remove - Rename - Chmod -) - -func (op Op) String() string { - // Use a buffer for efficient string concatenation - var buffer bytes.Buffer - - if op&Create == Create { - buffer.WriteString("|CREATE") - } - if op&Remove == Remove { - buffer.WriteString("|REMOVE") - } - if op&Write == Write { - buffer.WriteString("|WRITE") - } - if op&Rename == Rename { - buffer.WriteString("|RENAME") - } - if op&Chmod == Chmod { - buffer.WriteString("|CHMOD") - } - if buffer.Len() == 0 { - return "" - } - return buffer.String()[1:] // Strip leading pipe -} - -// String returns a string representation of the event in the form -// "file: REMOVE|WRITE|..." -func (e Event) String() string { - return fmt.Sprintf("%q: %s", e.Name, e.Op.String()) -} - -// Common errors that can be reported by a watcher -var ( - ErrEventOverflow = errors.New("fsnotify queue overflow") -) diff --git a/vendor/github.com/fsnotify/fsnotify/inotify.go b/vendor/github.com/fsnotify/fsnotify/inotify.go deleted file mode 100644 index eb87699b5b..0000000000 --- a/vendor/github.com/fsnotify/fsnotify/inotify.go +++ /dev/null @@ -1,338 +0,0 @@ -// Copyright 2010 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//go:build linux -// +build linux - -package fsnotify - -import ( - "errors" - "fmt" - "io" - "os" - "path/filepath" - "strings" - "sync" - "unsafe" - - "golang.org/x/sys/unix" -) - -// Watcher watches a set of files, delivering events to a channel. -type Watcher struct { - Events chan Event - Errors chan error - mu sync.Mutex // Map access - fd int - poller *fdPoller - watches map[string]*watch // Map of inotify watches (key: path) - paths map[int]string // Map of watched paths (key: watch descriptor) - done chan struct{} // Channel for sending a "quit message" to the reader goroutine - doneResp chan struct{} // Channel to respond to Close -} - -// NewWatcher establishes a new watcher with the underlying OS and begins waiting for events. -func NewWatcher() (*Watcher, error) { - // Create inotify fd - fd, errno := unix.InotifyInit1(unix.IN_CLOEXEC) - if fd == -1 { - return nil, errno - } - // Create epoll - poller, err := newFdPoller(fd) - if err != nil { - unix.Close(fd) - return nil, err - } - w := &Watcher{ - fd: fd, - poller: poller, - watches: make(map[string]*watch), - paths: make(map[int]string), - Events: make(chan Event), - Errors: make(chan error), - done: make(chan struct{}), - doneResp: make(chan struct{}), - } - - go w.readEvents() - return w, nil -} - -func (w *Watcher) isClosed() bool { - select { - case <-w.done: - return true - default: - return false - } -} - -// Close removes all watches and closes the events channel. -func (w *Watcher) Close() error { - if w.isClosed() { - return nil - } - - // Send 'close' signal to goroutine, and set the Watcher to closed. - close(w.done) - - // Wake up goroutine - w.poller.wake() - - // Wait for goroutine to close - <-w.doneResp - - return nil -} - -// Add starts watching the named file or directory (non-recursively). -func (w *Watcher) Add(name string) error { - name = filepath.Clean(name) - if w.isClosed() { - return errors.New("inotify instance already closed") - } - - const agnosticEvents = unix.IN_MOVED_TO | unix.IN_MOVED_FROM | - unix.IN_CREATE | unix.IN_ATTRIB | unix.IN_MODIFY | - unix.IN_MOVE_SELF | unix.IN_DELETE | unix.IN_DELETE_SELF - - var flags uint32 = agnosticEvents - - w.mu.Lock() - defer w.mu.Unlock() - watchEntry := w.watches[name] - if watchEntry != nil { - flags |= watchEntry.flags | unix.IN_MASK_ADD - } - wd, errno := unix.InotifyAddWatch(w.fd, name, flags) - if wd == -1 { - return errno - } - - if watchEntry == nil { - w.watches[name] = &watch{wd: uint32(wd), flags: flags} - w.paths[wd] = name - } else { - watchEntry.wd = uint32(wd) - watchEntry.flags = flags - } - - return nil -} - -// Remove stops watching the named file or directory (non-recursively). -func (w *Watcher) Remove(name string) error { - name = filepath.Clean(name) - - // Fetch the watch. - w.mu.Lock() - defer w.mu.Unlock() - watch, ok := w.watches[name] - - // Remove it from inotify. - if !ok { - return fmt.Errorf("can't remove non-existent inotify watch for: %s", name) - } - - // We successfully removed the watch if InotifyRmWatch doesn't return an - // error, we need to clean up our internal state to ensure it matches - // inotify's kernel state. - delete(w.paths, int(watch.wd)) - delete(w.watches, name) - - // inotify_rm_watch will return EINVAL if the file has been deleted; - // the inotify will already have been removed. - // watches and pathes are deleted in ignoreLinux() implicitly and asynchronously - // by calling inotify_rm_watch() below. e.g. readEvents() goroutine receives IN_IGNORE - // so that EINVAL means that the wd is being rm_watch()ed or its file removed - // by another thread and we have not received IN_IGNORE event. - success, errno := unix.InotifyRmWatch(w.fd, watch.wd) - if success == -1 { - // TODO: Perhaps it's not helpful to return an error here in every case. - // the only two possible errors are: - // EBADF, which happens when w.fd is not a valid file descriptor of any kind. - // EINVAL, which is when fd is not an inotify descriptor or wd is not a valid watch descriptor. - // Watch descriptors are invalidated when they are removed explicitly or implicitly; - // explicitly by inotify_rm_watch, implicitly when the file they are watching is deleted. - return errno - } - - return nil -} - -type watch struct { - wd uint32 // Watch descriptor (as returned by the inotify_add_watch() syscall) - flags uint32 // inotify flags of this watch (see inotify(7) for the list of valid flags) -} - -// readEvents reads from the inotify file descriptor, converts the -// received events into Event objects and sends them via the Events channel -func (w *Watcher) readEvents() { - var ( - buf [unix.SizeofInotifyEvent * 4096]byte // Buffer for a maximum of 4096 raw events - n int // Number of bytes read with read() - errno error // Syscall errno - ok bool // For poller.wait - ) - - defer close(w.doneResp) - defer close(w.Errors) - defer close(w.Events) - defer unix.Close(w.fd) - defer w.poller.close() - - for { - // See if we have been closed. - if w.isClosed() { - return - } - - ok, errno = w.poller.wait() - if errno != nil { - select { - case w.Errors <- errno: - case <-w.done: - return - } - continue - } - - if !ok { - continue - } - - n, errno = unix.Read(w.fd, buf[:]) - // If a signal interrupted execution, see if we've been asked to close, and try again. - // http://man7.org/linux/man-pages/man7/signal.7.html : - // "Before Linux 3.8, reads from an inotify(7) file descriptor were not restartable" - if errno == unix.EINTR { - continue - } - - // unix.Read might have been woken up by Close. If so, we're done. - if w.isClosed() { - return - } - - if n < unix.SizeofInotifyEvent { - var err error - if n == 0 { - // If EOF is received. This should really never happen. - err = io.EOF - } else if n < 0 { - // If an error occurred while reading. - err = errno - } else { - // Read was too short. - err = errors.New("notify: short read in readEvents()") - } - select { - case w.Errors <- err: - case <-w.done: - return - } - continue - } - - var offset uint32 - // We don't know how many events we just read into the buffer - // While the offset points to at least one whole event... - for offset <= uint32(n-unix.SizeofInotifyEvent) { - // Point "raw" to the event in the buffer - raw := (*unix.InotifyEvent)(unsafe.Pointer(&buf[offset])) - - mask := uint32(raw.Mask) - nameLen := uint32(raw.Len) - - if mask&unix.IN_Q_OVERFLOW != 0 { - select { - case w.Errors <- ErrEventOverflow: - case <-w.done: - return - } - } - - // If the event happened to the watched directory or the watched file, the kernel - // doesn't append the filename to the event, but we would like to always fill the - // the "Name" field with a valid filename. We retrieve the path of the watch from - // the "paths" map. - w.mu.Lock() - name, ok := w.paths[int(raw.Wd)] - // IN_DELETE_SELF occurs when the file/directory being watched is removed. - // This is a sign to clean up the maps, otherwise we are no longer in sync - // with the inotify kernel state which has already deleted the watch - // automatically. - if ok && mask&unix.IN_DELETE_SELF == unix.IN_DELETE_SELF { - delete(w.paths, int(raw.Wd)) - delete(w.watches, name) - } - w.mu.Unlock() - - if nameLen > 0 { - // Point "bytes" at the first byte of the filename - bytes := (*[unix.PathMax]byte)(unsafe.Pointer(&buf[offset+unix.SizeofInotifyEvent]))[:nameLen:nameLen] - // The filename is padded with NULL bytes. TrimRight() gets rid of those. - name += "/" + strings.TrimRight(string(bytes[0:nameLen]), "\000") - } - - event := newEvent(name, mask) - - // Send the events that are not ignored on the events channel - if !event.ignoreLinux(mask) { - select { - case w.Events <- event: - case <-w.done: - return - } - } - - // Move to the next event in the buffer - offset += unix.SizeofInotifyEvent + nameLen - } - } -} - -// Certain types of events can be "ignored" and not sent over the Events -// channel. Such as events marked ignore by the kernel, or MODIFY events -// against files that do not exist. -func (e *Event) ignoreLinux(mask uint32) bool { - // Ignore anything the inotify API says to ignore - if mask&unix.IN_IGNORED == unix.IN_IGNORED { - return true - } - - // If the event is not a DELETE or RENAME, the file must exist. - // Otherwise the event is ignored. - // *Note*: this was put in place because it was seen that a MODIFY - // event was sent after the DELETE. This ignores that MODIFY and - // assumes a DELETE will come or has come if the file doesn't exist. - if !(e.Op&Remove == Remove || e.Op&Rename == Rename) { - _, statErr := os.Lstat(e.Name) - return os.IsNotExist(statErr) - } - return false -} - -// newEvent returns an platform-independent Event based on an inotify mask. -func newEvent(name string, mask uint32) Event { - e := Event{Name: name} - if mask&unix.IN_CREATE == unix.IN_CREATE || mask&unix.IN_MOVED_TO == unix.IN_MOVED_TO { - e.Op |= Create - } - if mask&unix.IN_DELETE_SELF == unix.IN_DELETE_SELF || mask&unix.IN_DELETE == unix.IN_DELETE { - e.Op |= Remove - } - if mask&unix.IN_MODIFY == unix.IN_MODIFY { - e.Op |= Write - } - if mask&unix.IN_MOVE_SELF == unix.IN_MOVE_SELF || mask&unix.IN_MOVED_FROM == unix.IN_MOVED_FROM { - e.Op |= Rename - } - if mask&unix.IN_ATTRIB == unix.IN_ATTRIB { - e.Op |= Chmod - } - return e -} diff --git a/vendor/github.com/fsnotify/fsnotify/inotify_poller.go b/vendor/github.com/fsnotify/fsnotify/inotify_poller.go deleted file mode 100644 index e9ff9439f7..0000000000 --- a/vendor/github.com/fsnotify/fsnotify/inotify_poller.go +++ /dev/null @@ -1,188 +0,0 @@ -// Copyright 2015 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//go:build linux -// +build linux - -package fsnotify - -import ( - "errors" - - "golang.org/x/sys/unix" -) - -type fdPoller struct { - fd int // File descriptor (as returned by the inotify_init() syscall) - epfd int // Epoll file descriptor - pipe [2]int // Pipe for waking up -} - -func emptyPoller(fd int) *fdPoller { - poller := new(fdPoller) - poller.fd = fd - poller.epfd = -1 - poller.pipe[0] = -1 - poller.pipe[1] = -1 - return poller -} - -// Create a new inotify poller. -// This creates an inotify handler, and an epoll handler. -func newFdPoller(fd int) (*fdPoller, error) { - var errno error - poller := emptyPoller(fd) - defer func() { - if errno != nil { - poller.close() - } - }() - poller.fd = fd - - // Create epoll fd - poller.epfd, errno = unix.EpollCreate1(unix.EPOLL_CLOEXEC) - if poller.epfd == -1 { - return nil, errno - } - // Create pipe; pipe[0] is the read end, pipe[1] the write end. - errno = unix.Pipe2(poller.pipe[:], unix.O_NONBLOCK|unix.O_CLOEXEC) - if errno != nil { - return nil, errno - } - - // Register inotify fd with epoll - event := unix.EpollEvent{ - Fd: int32(poller.fd), - Events: unix.EPOLLIN, - } - errno = unix.EpollCtl(poller.epfd, unix.EPOLL_CTL_ADD, poller.fd, &event) - if errno != nil { - return nil, errno - } - - // Register pipe fd with epoll - event = unix.EpollEvent{ - Fd: int32(poller.pipe[0]), - Events: unix.EPOLLIN, - } - errno = unix.EpollCtl(poller.epfd, unix.EPOLL_CTL_ADD, poller.pipe[0], &event) - if errno != nil { - return nil, errno - } - - return poller, nil -} - -// Wait using epoll. -// Returns true if something is ready to be read, -// false if there is not. -func (poller *fdPoller) wait() (bool, error) { - // 3 possible events per fd, and 2 fds, makes a maximum of 6 events. - // I don't know whether epoll_wait returns the number of events returned, - // or the total number of events ready. - // I decided to catch both by making the buffer one larger than the maximum. - events := make([]unix.EpollEvent, 7) - for { - n, errno := unix.EpollWait(poller.epfd, events, -1) - if n == -1 { - if errno == unix.EINTR { - continue - } - return false, errno - } - if n == 0 { - // If there are no events, try again. - continue - } - if n > 6 { - // This should never happen. More events were returned than should be possible. - return false, errors.New("epoll_wait returned more events than I know what to do with") - } - ready := events[:n] - epollhup := false - epollerr := false - epollin := false - for _, event := range ready { - if event.Fd == int32(poller.fd) { - if event.Events&unix.EPOLLHUP != 0 { - // This should not happen, but if it does, treat it as a wakeup. - epollhup = true - } - if event.Events&unix.EPOLLERR != 0 { - // If an error is waiting on the file descriptor, we should pretend - // something is ready to read, and let unix.Read pick up the error. - epollerr = true - } - if event.Events&unix.EPOLLIN != 0 { - // There is data to read. - epollin = true - } - } - if event.Fd == int32(poller.pipe[0]) { - if event.Events&unix.EPOLLHUP != 0 { - // Write pipe descriptor was closed, by us. This means we're closing down the - // watcher, and we should wake up. - } - if event.Events&unix.EPOLLERR != 0 { - // If an error is waiting on the pipe file descriptor. - // This is an absolute mystery, and should never ever happen. - return false, errors.New("Error on the pipe descriptor.") - } - if event.Events&unix.EPOLLIN != 0 { - // This is a regular wakeup, so we have to clear the buffer. - err := poller.clearWake() - if err != nil { - return false, err - } - } - } - } - - if epollhup || epollerr || epollin { - return true, nil - } - return false, nil - } -} - -// Close the write end of the poller. -func (poller *fdPoller) wake() error { - buf := make([]byte, 1) - n, errno := unix.Write(poller.pipe[1], buf) - if n == -1 { - if errno == unix.EAGAIN { - // Buffer is full, poller will wake. - return nil - } - return errno - } - return nil -} - -func (poller *fdPoller) clearWake() error { - // You have to be woken up a LOT in order to get to 100! - buf := make([]byte, 100) - n, errno := unix.Read(poller.pipe[0], buf) - if n == -1 { - if errno == unix.EAGAIN { - // Buffer is empty, someone else cleared our wake. - return nil - } - return errno - } - return nil -} - -// Close all poller file descriptors, but not the one passed to it. -func (poller *fdPoller) close() { - if poller.pipe[1] != -1 { - unix.Close(poller.pipe[1]) - } - if poller.pipe[0] != -1 { - unix.Close(poller.pipe[0]) - } - if poller.epfd != -1 { - unix.Close(poller.epfd) - } -} diff --git a/vendor/github.com/fsnotify/fsnotify/kqueue.go b/vendor/github.com/fsnotify/fsnotify/kqueue.go deleted file mode 100644 index 368f5b790d..0000000000 --- a/vendor/github.com/fsnotify/fsnotify/kqueue.go +++ /dev/null @@ -1,522 +0,0 @@ -// Copyright 2010 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//go:build freebsd || openbsd || netbsd || dragonfly || darwin -// +build freebsd openbsd netbsd dragonfly darwin - -package fsnotify - -import ( - "errors" - "fmt" - "io/ioutil" - "os" - "path/filepath" - "sync" - "time" - - "golang.org/x/sys/unix" -) - -// Watcher watches a set of files, delivering events to a channel. -type Watcher struct { - Events chan Event - Errors chan error - done chan struct{} // Channel for sending a "quit message" to the reader goroutine - - kq int // File descriptor (as returned by the kqueue() syscall). - - mu sync.Mutex // Protects access to watcher data - watches map[string]int // Map of watched file descriptors (key: path). - externalWatches map[string]bool // Map of watches added by user of the library. - dirFlags map[string]uint32 // Map of watched directories to fflags used in kqueue. - paths map[int]pathInfo // Map file descriptors to path names for processing kqueue events. - fileExists map[string]bool // Keep track of if we know this file exists (to stop duplicate create events). - isClosed bool // Set to true when Close() is first called -} - -type pathInfo struct { - name string - isDir bool -} - -// NewWatcher establishes a new watcher with the underlying OS and begins waiting for events. -func NewWatcher() (*Watcher, error) { - kq, err := kqueue() - if err != nil { - return nil, err - } - - w := &Watcher{ - kq: kq, - watches: make(map[string]int), - dirFlags: make(map[string]uint32), - paths: make(map[int]pathInfo), - fileExists: make(map[string]bool), - externalWatches: make(map[string]bool), - Events: make(chan Event), - Errors: make(chan error), - done: make(chan struct{}), - } - - go w.readEvents() - return w, nil -} - -// Close removes all watches and closes the events channel. -func (w *Watcher) Close() error { - w.mu.Lock() - if w.isClosed { - w.mu.Unlock() - return nil - } - w.isClosed = true - - // copy paths to remove while locked - var pathsToRemove = make([]string, 0, len(w.watches)) - for name := range w.watches { - pathsToRemove = append(pathsToRemove, name) - } - w.mu.Unlock() - // unlock before calling Remove, which also locks - - for _, name := range pathsToRemove { - w.Remove(name) - } - - // send a "quit" message to the reader goroutine - close(w.done) - - return nil -} - -// Add starts watching the named file or directory (non-recursively). -func (w *Watcher) Add(name string) error { - w.mu.Lock() - w.externalWatches[name] = true - w.mu.Unlock() - _, err := w.addWatch(name, noteAllEvents) - return err -} - -// Remove stops watching the the named file or directory (non-recursively). -func (w *Watcher) Remove(name string) error { - name = filepath.Clean(name) - w.mu.Lock() - watchfd, ok := w.watches[name] - w.mu.Unlock() - if !ok { - return fmt.Errorf("can't remove non-existent kevent watch for: %s", name) - } - - const registerRemove = unix.EV_DELETE - if err := register(w.kq, []int{watchfd}, registerRemove, 0); err != nil { - return err - } - - unix.Close(watchfd) - - w.mu.Lock() - isDir := w.paths[watchfd].isDir - delete(w.watches, name) - delete(w.paths, watchfd) - delete(w.dirFlags, name) - w.mu.Unlock() - - // Find all watched paths that are in this directory that are not external. - if isDir { - var pathsToRemove []string - w.mu.Lock() - for _, path := range w.paths { - wdir, _ := filepath.Split(path.name) - if filepath.Clean(wdir) == name { - if !w.externalWatches[path.name] { - pathsToRemove = append(pathsToRemove, path.name) - } - } - } - w.mu.Unlock() - for _, name := range pathsToRemove { - // Since these are internal, not much sense in propagating error - // to the user, as that will just confuse them with an error about - // a path they did not explicitly watch themselves. - w.Remove(name) - } - } - - return nil -} - -// Watch all events (except NOTE_EXTEND, NOTE_LINK, NOTE_REVOKE) -const noteAllEvents = unix.NOTE_DELETE | unix.NOTE_WRITE | unix.NOTE_ATTRIB | unix.NOTE_RENAME - -// keventWaitTime to block on each read from kevent -var keventWaitTime = durationToTimespec(100 * time.Millisecond) - -// addWatch adds name to the watched file set. -// The flags are interpreted as described in kevent(2). -// Returns the real path to the file which was added, if any, which may be different from the one passed in the case of symlinks. -func (w *Watcher) addWatch(name string, flags uint32) (string, error) { - var isDir bool - // Make ./name and name equivalent - name = filepath.Clean(name) - - w.mu.Lock() - if w.isClosed { - w.mu.Unlock() - return "", errors.New("kevent instance already closed") - } - watchfd, alreadyWatching := w.watches[name] - // We already have a watch, but we can still override flags. - if alreadyWatching { - isDir = w.paths[watchfd].isDir - } - w.mu.Unlock() - - if !alreadyWatching { - fi, err := os.Lstat(name) - if err != nil { - return "", err - } - - // Don't watch sockets. - if fi.Mode()&os.ModeSocket == os.ModeSocket { - return "", nil - } - - // Don't watch named pipes. - if fi.Mode()&os.ModeNamedPipe == os.ModeNamedPipe { - return "", nil - } - - // Follow Symlinks - // Unfortunately, Linux can add bogus symlinks to watch list without - // issue, and Windows can't do symlinks period (AFAIK). To maintain - // consistency, we will act like everything is fine. There will simply - // be no file events for broken symlinks. - // Hence the returns of nil on errors. - if fi.Mode()&os.ModeSymlink == os.ModeSymlink { - name, err = filepath.EvalSymlinks(name) - if err != nil { - return "", nil - } - - w.mu.Lock() - _, alreadyWatching = w.watches[name] - w.mu.Unlock() - - if alreadyWatching { - return name, nil - } - - fi, err = os.Lstat(name) - if err != nil { - return "", nil - } - } - - watchfd, err = unix.Open(name, openMode, 0700) - if watchfd == -1 { - return "", err - } - - isDir = fi.IsDir() - } - - const registerAdd = unix.EV_ADD | unix.EV_CLEAR | unix.EV_ENABLE - if err := register(w.kq, []int{watchfd}, registerAdd, flags); err != nil { - unix.Close(watchfd) - return "", err - } - - if !alreadyWatching { - w.mu.Lock() - w.watches[name] = watchfd - w.paths[watchfd] = pathInfo{name: name, isDir: isDir} - w.mu.Unlock() - } - - if isDir { - // Watch the directory if it has not been watched before, - // or if it was watched before, but perhaps only a NOTE_DELETE (watchDirectoryFiles) - w.mu.Lock() - - watchDir := (flags&unix.NOTE_WRITE) == unix.NOTE_WRITE && - (!alreadyWatching || (w.dirFlags[name]&unix.NOTE_WRITE) != unix.NOTE_WRITE) - // Store flags so this watch can be updated later - w.dirFlags[name] = flags - w.mu.Unlock() - - if watchDir { - if err := w.watchDirectoryFiles(name); err != nil { - return "", err - } - } - } - return name, nil -} - -// readEvents reads from kqueue and converts the received kevents into -// Event values that it sends down the Events channel. -func (w *Watcher) readEvents() { - eventBuffer := make([]unix.Kevent_t, 10) - -loop: - for { - // See if there is a message on the "done" channel - select { - case <-w.done: - break loop - default: - } - - // Get new events - kevents, err := read(w.kq, eventBuffer, &keventWaitTime) - // EINTR is okay, the syscall was interrupted before timeout expired. - if err != nil && err != unix.EINTR { - select { - case w.Errors <- err: - case <-w.done: - break loop - } - continue - } - - // Flush the events we received to the Events channel - for len(kevents) > 0 { - kevent := &kevents[0] - watchfd := int(kevent.Ident) - mask := uint32(kevent.Fflags) - w.mu.Lock() - path := w.paths[watchfd] - w.mu.Unlock() - event := newEvent(path.name, mask) - - if path.isDir && !(event.Op&Remove == Remove) { - // Double check to make sure the directory exists. This can happen when - // we do a rm -fr on a recursively watched folders and we receive a - // modification event first but the folder has been deleted and later - // receive the delete event - if _, err := os.Lstat(event.Name); os.IsNotExist(err) { - // mark is as delete event - event.Op |= Remove - } - } - - if event.Op&Rename == Rename || event.Op&Remove == Remove { - w.Remove(event.Name) - w.mu.Lock() - delete(w.fileExists, event.Name) - w.mu.Unlock() - } - - if path.isDir && event.Op&Write == Write && !(event.Op&Remove == Remove) { - w.sendDirectoryChangeEvents(event.Name) - } else { - // Send the event on the Events channel. - select { - case w.Events <- event: - case <-w.done: - break loop - } - } - - if event.Op&Remove == Remove { - // Look for a file that may have overwritten this. - // For example, mv f1 f2 will delete f2, then create f2. - if path.isDir { - fileDir := filepath.Clean(event.Name) - w.mu.Lock() - _, found := w.watches[fileDir] - w.mu.Unlock() - if found { - // make sure the directory exists before we watch for changes. When we - // do a recursive watch and perform rm -fr, the parent directory might - // have gone missing, ignore the missing directory and let the - // upcoming delete event remove the watch from the parent directory. - if _, err := os.Lstat(fileDir); err == nil { - w.sendDirectoryChangeEvents(fileDir) - } - } - } else { - filePath := filepath.Clean(event.Name) - if fileInfo, err := os.Lstat(filePath); err == nil { - w.sendFileCreatedEventIfNew(filePath, fileInfo) - } - } - } - - // Move to next event - kevents = kevents[1:] - } - } - - // cleanup - err := unix.Close(w.kq) - if err != nil { - // only way the previous loop breaks is if w.done was closed so we need to async send to w.Errors. - select { - case w.Errors <- err: - default: - } - } - close(w.Events) - close(w.Errors) -} - -// newEvent returns an platform-independent Event based on kqueue Fflags. -func newEvent(name string, mask uint32) Event { - e := Event{Name: name} - if mask&unix.NOTE_DELETE == unix.NOTE_DELETE { - e.Op |= Remove - } - if mask&unix.NOTE_WRITE == unix.NOTE_WRITE { - e.Op |= Write - } - if mask&unix.NOTE_RENAME == unix.NOTE_RENAME { - e.Op |= Rename - } - if mask&unix.NOTE_ATTRIB == unix.NOTE_ATTRIB { - e.Op |= Chmod - } - return e -} - -func newCreateEvent(name string) Event { - return Event{Name: name, Op: Create} -} - -// watchDirectoryFiles to mimic inotify when adding a watch on a directory -func (w *Watcher) watchDirectoryFiles(dirPath string) error { - // Get all files - files, err := ioutil.ReadDir(dirPath) - if err != nil { - return err - } - - for _, fileInfo := range files { - filePath := filepath.Join(dirPath, fileInfo.Name()) - filePath, err = w.internalWatch(filePath, fileInfo) - if err != nil { - return err - } - - w.mu.Lock() - w.fileExists[filePath] = true - w.mu.Unlock() - } - - return nil -} - -// sendDirectoryEvents searches the directory for newly created files -// and sends them over the event channel. This functionality is to have -// the BSD version of fsnotify match Linux inotify which provides a -// create event for files created in a watched directory. -func (w *Watcher) sendDirectoryChangeEvents(dirPath string) { - // Get all files - files, err := ioutil.ReadDir(dirPath) - if err != nil { - select { - case w.Errors <- err: - case <-w.done: - return - } - } - - // Search for new files - for _, fileInfo := range files { - filePath := filepath.Join(dirPath, fileInfo.Name()) - err := w.sendFileCreatedEventIfNew(filePath, fileInfo) - - if err != nil { - return - } - } -} - -// sendFileCreatedEvent sends a create event if the file isn't already being tracked. -func (w *Watcher) sendFileCreatedEventIfNew(filePath string, fileInfo os.FileInfo) (err error) { - w.mu.Lock() - _, doesExist := w.fileExists[filePath] - w.mu.Unlock() - if !doesExist { - // Send create event - select { - case w.Events <- newCreateEvent(filePath): - case <-w.done: - return - } - } - - // like watchDirectoryFiles (but without doing another ReadDir) - filePath, err = w.internalWatch(filePath, fileInfo) - if err != nil { - return err - } - - w.mu.Lock() - w.fileExists[filePath] = true - w.mu.Unlock() - - return nil -} - -func (w *Watcher) internalWatch(name string, fileInfo os.FileInfo) (string, error) { - if fileInfo.IsDir() { - // mimic Linux providing delete events for subdirectories - // but preserve the flags used if currently watching subdirectory - w.mu.Lock() - flags := w.dirFlags[name] - w.mu.Unlock() - - flags |= unix.NOTE_DELETE | unix.NOTE_RENAME - return w.addWatch(name, flags) - } - - // watch file to mimic Linux inotify - return w.addWatch(name, noteAllEvents) -} - -// kqueue creates a new kernel event queue and returns a descriptor. -func kqueue() (kq int, err error) { - kq, err = unix.Kqueue() - if kq == -1 { - return kq, err - } - return kq, nil -} - -// register events with the queue -func register(kq int, fds []int, flags int, fflags uint32) error { - changes := make([]unix.Kevent_t, len(fds)) - - for i, fd := range fds { - // SetKevent converts int to the platform-specific types: - unix.SetKevent(&changes[i], fd, unix.EVFILT_VNODE, flags) - changes[i].Fflags = fflags - } - - // register the events - success, err := unix.Kevent(kq, changes, nil, nil) - if success == -1 { - return err - } - return nil -} - -// read retrieves pending events, or waits until an event occurs. -// A timeout of nil blocks indefinitely, while 0 polls the queue. -func read(kq int, events []unix.Kevent_t, timeout *unix.Timespec) ([]unix.Kevent_t, error) { - n, err := unix.Kevent(kq, nil, events, timeout) - if err != nil { - return nil, err - } - return events[0:n], nil -} - -// durationToTimespec prepares a timeout value -func durationToTimespec(d time.Duration) unix.Timespec { - return unix.NsecToTimespec(d.Nanoseconds()) -} diff --git a/vendor/github.com/fsnotify/fsnotify/open_mode_bsd.go b/vendor/github.com/fsnotify/fsnotify/open_mode_bsd.go deleted file mode 100644 index 36cc3845b6..0000000000 --- a/vendor/github.com/fsnotify/fsnotify/open_mode_bsd.go +++ /dev/null @@ -1,12 +0,0 @@ -// Copyright 2013 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//go:build freebsd || openbsd || netbsd || dragonfly -// +build freebsd openbsd netbsd dragonfly - -package fsnotify - -import "golang.org/x/sys/unix" - -const openMode = unix.O_NONBLOCK | unix.O_RDONLY | unix.O_CLOEXEC diff --git a/vendor/github.com/fsnotify/fsnotify/open_mode_darwin.go b/vendor/github.com/fsnotify/fsnotify/open_mode_darwin.go deleted file mode 100644 index 98cd8476ff..0000000000 --- a/vendor/github.com/fsnotify/fsnotify/open_mode_darwin.go +++ /dev/null @@ -1,13 +0,0 @@ -// Copyright 2013 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//go:build darwin -// +build darwin - -package fsnotify - -import "golang.org/x/sys/unix" - -// note: this constant is not defined on BSD -const openMode = unix.O_EVTONLY | unix.O_CLOEXEC diff --git a/vendor/github.com/fsnotify/fsnotify/windows.go b/vendor/github.com/fsnotify/fsnotify/windows.go deleted file mode 100644 index c02b75f7c3..0000000000 --- a/vendor/github.com/fsnotify/fsnotify/windows.go +++ /dev/null @@ -1,562 +0,0 @@ -// Copyright 2011 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//go:build windows -// +build windows - -package fsnotify - -import ( - "errors" - "fmt" - "os" - "path/filepath" - "runtime" - "sync" - "syscall" - "unsafe" -) - -// Watcher watches a set of files, delivering events to a channel. -type Watcher struct { - Events chan Event - Errors chan error - isClosed bool // Set to true when Close() is first called - mu sync.Mutex // Map access - port syscall.Handle // Handle to completion port - watches watchMap // Map of watches (key: i-number) - input chan *input // Inputs to the reader are sent on this channel - quit chan chan<- error -} - -// NewWatcher establishes a new watcher with the underlying OS and begins waiting for events. -func NewWatcher() (*Watcher, error) { - port, e := syscall.CreateIoCompletionPort(syscall.InvalidHandle, 0, 0, 0) - if e != nil { - return nil, os.NewSyscallError("CreateIoCompletionPort", e) - } - w := &Watcher{ - port: port, - watches: make(watchMap), - input: make(chan *input, 1), - Events: make(chan Event, 50), - Errors: make(chan error), - quit: make(chan chan<- error, 1), - } - go w.readEvents() - return w, nil -} - -// Close removes all watches and closes the events channel. -func (w *Watcher) Close() error { - if w.isClosed { - return nil - } - w.isClosed = true - - // Send "quit" message to the reader goroutine - ch := make(chan error) - w.quit <- ch - if err := w.wakeupReader(); err != nil { - return err - } - return <-ch -} - -// Add starts watching the named file or directory (non-recursively). -func (w *Watcher) Add(name string) error { - if w.isClosed { - return errors.New("watcher already closed") - } - in := &input{ - op: opAddWatch, - path: filepath.Clean(name), - flags: sysFSALLEVENTS, - reply: make(chan error), - } - w.input <- in - if err := w.wakeupReader(); err != nil { - return err - } - return <-in.reply -} - -// Remove stops watching the the named file or directory (non-recursively). -func (w *Watcher) Remove(name string) error { - in := &input{ - op: opRemoveWatch, - path: filepath.Clean(name), - reply: make(chan error), - } - w.input <- in - if err := w.wakeupReader(); err != nil { - return err - } - return <-in.reply -} - -const ( - // Options for AddWatch - sysFSONESHOT = 0x80000000 - sysFSONLYDIR = 0x1000000 - - // Events - sysFSACCESS = 0x1 - sysFSALLEVENTS = 0xfff - sysFSATTRIB = 0x4 - sysFSCLOSE = 0x18 - sysFSCREATE = 0x100 - sysFSDELETE = 0x200 - sysFSDELETESELF = 0x400 - sysFSMODIFY = 0x2 - sysFSMOVE = 0xc0 - sysFSMOVEDFROM = 0x40 - sysFSMOVEDTO = 0x80 - sysFSMOVESELF = 0x800 - - // Special events - sysFSIGNORED = 0x8000 - sysFSQOVERFLOW = 0x4000 -) - -func newEvent(name string, mask uint32) Event { - e := Event{Name: name} - if mask&sysFSCREATE == sysFSCREATE || mask&sysFSMOVEDTO == sysFSMOVEDTO { - e.Op |= Create - } - if mask&sysFSDELETE == sysFSDELETE || mask&sysFSDELETESELF == sysFSDELETESELF { - e.Op |= Remove - } - if mask&sysFSMODIFY == sysFSMODIFY { - e.Op |= Write - } - if mask&sysFSMOVE == sysFSMOVE || mask&sysFSMOVESELF == sysFSMOVESELF || mask&sysFSMOVEDFROM == sysFSMOVEDFROM { - e.Op |= Rename - } - if mask&sysFSATTRIB == sysFSATTRIB { - e.Op |= Chmod - } - return e -} - -const ( - opAddWatch = iota - opRemoveWatch -) - -const ( - provisional uint64 = 1 << (32 + iota) -) - -type input struct { - op int - path string - flags uint32 - reply chan error -} - -type inode struct { - handle syscall.Handle - volume uint32 - index uint64 -} - -type watch struct { - ov syscall.Overlapped - ino *inode // i-number - path string // Directory path - mask uint64 // Directory itself is being watched with these notify flags - names map[string]uint64 // Map of names being watched and their notify flags - rename string // Remembers the old name while renaming a file - buf [4096]byte -} - -type indexMap map[uint64]*watch -type watchMap map[uint32]indexMap - -func (w *Watcher) wakeupReader() error { - e := syscall.PostQueuedCompletionStatus(w.port, 0, 0, nil) - if e != nil { - return os.NewSyscallError("PostQueuedCompletionStatus", e) - } - return nil -} - -func getDir(pathname string) (dir string, err error) { - attr, e := syscall.GetFileAttributes(syscall.StringToUTF16Ptr(pathname)) - if e != nil { - return "", os.NewSyscallError("GetFileAttributes", e) - } - if attr&syscall.FILE_ATTRIBUTE_DIRECTORY != 0 { - dir = pathname - } else { - dir, _ = filepath.Split(pathname) - dir = filepath.Clean(dir) - } - return -} - -func getIno(path string) (ino *inode, err error) { - h, e := syscall.CreateFile(syscall.StringToUTF16Ptr(path), - syscall.FILE_LIST_DIRECTORY, - syscall.FILE_SHARE_READ|syscall.FILE_SHARE_WRITE|syscall.FILE_SHARE_DELETE, - nil, syscall.OPEN_EXISTING, - syscall.FILE_FLAG_BACKUP_SEMANTICS|syscall.FILE_FLAG_OVERLAPPED, 0) - if e != nil { - return nil, os.NewSyscallError("CreateFile", e) - } - var fi syscall.ByHandleFileInformation - if e = syscall.GetFileInformationByHandle(h, &fi); e != nil { - syscall.CloseHandle(h) - return nil, os.NewSyscallError("GetFileInformationByHandle", e) - } - ino = &inode{ - handle: h, - volume: fi.VolumeSerialNumber, - index: uint64(fi.FileIndexHigh)<<32 | uint64(fi.FileIndexLow), - } - return ino, nil -} - -// Must run within the I/O thread. -func (m watchMap) get(ino *inode) *watch { - if i := m[ino.volume]; i != nil { - return i[ino.index] - } - return nil -} - -// Must run within the I/O thread. -func (m watchMap) set(ino *inode, watch *watch) { - i := m[ino.volume] - if i == nil { - i = make(indexMap) - m[ino.volume] = i - } - i[ino.index] = watch -} - -// Must run within the I/O thread. -func (w *Watcher) addWatch(pathname string, flags uint64) error { - dir, err := getDir(pathname) - if err != nil { - return err - } - if flags&sysFSONLYDIR != 0 && pathname != dir { - return nil - } - ino, err := getIno(dir) - if err != nil { - return err - } - w.mu.Lock() - watchEntry := w.watches.get(ino) - w.mu.Unlock() - if watchEntry == nil { - if _, e := syscall.CreateIoCompletionPort(ino.handle, w.port, 0, 0); e != nil { - syscall.CloseHandle(ino.handle) - return os.NewSyscallError("CreateIoCompletionPort", e) - } - watchEntry = &watch{ - ino: ino, - path: dir, - names: make(map[string]uint64), - } - w.mu.Lock() - w.watches.set(ino, watchEntry) - w.mu.Unlock() - flags |= provisional - } else { - syscall.CloseHandle(ino.handle) - } - if pathname == dir { - watchEntry.mask |= flags - } else { - watchEntry.names[filepath.Base(pathname)] |= flags - } - if err = w.startRead(watchEntry); err != nil { - return err - } - if pathname == dir { - watchEntry.mask &= ^provisional - } else { - watchEntry.names[filepath.Base(pathname)] &= ^provisional - } - return nil -} - -// Must run within the I/O thread. -func (w *Watcher) remWatch(pathname string) error { - dir, err := getDir(pathname) - if err != nil { - return err - } - ino, err := getIno(dir) - if err != nil { - return err - } - w.mu.Lock() - watch := w.watches.get(ino) - w.mu.Unlock() - if watch == nil { - return fmt.Errorf("can't remove non-existent watch for: %s", pathname) - } - if pathname == dir { - w.sendEvent(watch.path, watch.mask&sysFSIGNORED) - watch.mask = 0 - } else { - name := filepath.Base(pathname) - w.sendEvent(filepath.Join(watch.path, name), watch.names[name]&sysFSIGNORED) - delete(watch.names, name) - } - return w.startRead(watch) -} - -// Must run within the I/O thread. -func (w *Watcher) deleteWatch(watch *watch) { - for name, mask := range watch.names { - if mask&provisional == 0 { - w.sendEvent(filepath.Join(watch.path, name), mask&sysFSIGNORED) - } - delete(watch.names, name) - } - if watch.mask != 0 { - if watch.mask&provisional == 0 { - w.sendEvent(watch.path, watch.mask&sysFSIGNORED) - } - watch.mask = 0 - } -} - -// Must run within the I/O thread. -func (w *Watcher) startRead(watch *watch) error { - if e := syscall.CancelIo(watch.ino.handle); e != nil { - w.Errors <- os.NewSyscallError("CancelIo", e) - w.deleteWatch(watch) - } - mask := toWindowsFlags(watch.mask) - for _, m := range watch.names { - mask |= toWindowsFlags(m) - } - if mask == 0 { - if e := syscall.CloseHandle(watch.ino.handle); e != nil { - w.Errors <- os.NewSyscallError("CloseHandle", e) - } - w.mu.Lock() - delete(w.watches[watch.ino.volume], watch.ino.index) - w.mu.Unlock() - return nil - } - e := syscall.ReadDirectoryChanges(watch.ino.handle, &watch.buf[0], - uint32(unsafe.Sizeof(watch.buf)), false, mask, nil, &watch.ov, 0) - if e != nil { - err := os.NewSyscallError("ReadDirectoryChanges", e) - if e == syscall.ERROR_ACCESS_DENIED && watch.mask&provisional == 0 { - // Watched directory was probably removed - if w.sendEvent(watch.path, watch.mask&sysFSDELETESELF) { - if watch.mask&sysFSONESHOT != 0 { - watch.mask = 0 - } - } - err = nil - } - w.deleteWatch(watch) - w.startRead(watch) - return err - } - return nil -} - -// readEvents reads from the I/O completion port, converts the -// received events into Event objects and sends them via the Events channel. -// Entry point to the I/O thread. -func (w *Watcher) readEvents() { - var ( - n, key uint32 - ov *syscall.Overlapped - ) - runtime.LockOSThread() - - for { - e := syscall.GetQueuedCompletionStatus(w.port, &n, &key, &ov, syscall.INFINITE) - watch := (*watch)(unsafe.Pointer(ov)) - - if watch == nil { - select { - case ch := <-w.quit: - w.mu.Lock() - var indexes []indexMap - for _, index := range w.watches { - indexes = append(indexes, index) - } - w.mu.Unlock() - for _, index := range indexes { - for _, watch := range index { - w.deleteWatch(watch) - w.startRead(watch) - } - } - var err error - if e := syscall.CloseHandle(w.port); e != nil { - err = os.NewSyscallError("CloseHandle", e) - } - close(w.Events) - close(w.Errors) - ch <- err - return - case in := <-w.input: - switch in.op { - case opAddWatch: - in.reply <- w.addWatch(in.path, uint64(in.flags)) - case opRemoveWatch: - in.reply <- w.remWatch(in.path) - } - default: - } - continue - } - - switch e { - case syscall.ERROR_MORE_DATA: - if watch == nil { - w.Errors <- errors.New("ERROR_MORE_DATA has unexpectedly null lpOverlapped buffer") - } else { - // The i/o succeeded but the buffer is full. - // In theory we should be building up a full packet. - // In practice we can get away with just carrying on. - n = uint32(unsafe.Sizeof(watch.buf)) - } - case syscall.ERROR_ACCESS_DENIED: - // Watched directory was probably removed - w.sendEvent(watch.path, watch.mask&sysFSDELETESELF) - w.deleteWatch(watch) - w.startRead(watch) - continue - case syscall.ERROR_OPERATION_ABORTED: - // CancelIo was called on this handle - continue - default: - w.Errors <- os.NewSyscallError("GetQueuedCompletionPort", e) - continue - case nil: - } - - var offset uint32 - for { - if n == 0 { - w.Events <- newEvent("", sysFSQOVERFLOW) - w.Errors <- errors.New("short read in readEvents()") - break - } - - // Point "raw" to the event in the buffer - raw := (*syscall.FileNotifyInformation)(unsafe.Pointer(&watch.buf[offset])) - buf := (*[syscall.MAX_PATH]uint16)(unsafe.Pointer(&raw.FileName)) - name := syscall.UTF16ToString(buf[:raw.FileNameLength/2]) - fullname := filepath.Join(watch.path, name) - - var mask uint64 - switch raw.Action { - case syscall.FILE_ACTION_REMOVED: - mask = sysFSDELETESELF - case syscall.FILE_ACTION_MODIFIED: - mask = sysFSMODIFY - case syscall.FILE_ACTION_RENAMED_OLD_NAME: - watch.rename = name - case syscall.FILE_ACTION_RENAMED_NEW_NAME: - if watch.names[watch.rename] != 0 { - watch.names[name] |= watch.names[watch.rename] - delete(watch.names, watch.rename) - mask = sysFSMOVESELF - } - } - - sendNameEvent := func() { - if w.sendEvent(fullname, watch.names[name]&mask) { - if watch.names[name]&sysFSONESHOT != 0 { - delete(watch.names, name) - } - } - } - if raw.Action != syscall.FILE_ACTION_RENAMED_NEW_NAME { - sendNameEvent() - } - if raw.Action == syscall.FILE_ACTION_REMOVED { - w.sendEvent(fullname, watch.names[name]&sysFSIGNORED) - delete(watch.names, name) - } - if w.sendEvent(fullname, watch.mask&toFSnotifyFlags(raw.Action)) { - if watch.mask&sysFSONESHOT != 0 { - watch.mask = 0 - } - } - if raw.Action == syscall.FILE_ACTION_RENAMED_NEW_NAME { - fullname = filepath.Join(watch.path, watch.rename) - sendNameEvent() - } - - // Move to the next event in the buffer - if raw.NextEntryOffset == 0 { - break - } - offset += raw.NextEntryOffset - - // Error! - if offset >= n { - w.Errors <- errors.New("Windows system assumed buffer larger than it is, events have likely been missed.") - break - } - } - - if err := w.startRead(watch); err != nil { - w.Errors <- err - } - } -} - -func (w *Watcher) sendEvent(name string, mask uint64) bool { - if mask == 0 { - return false - } - event := newEvent(name, uint32(mask)) - select { - case ch := <-w.quit: - w.quit <- ch - case w.Events <- event: - } - return true -} - -func toWindowsFlags(mask uint64) uint32 { - var m uint32 - if mask&sysFSACCESS != 0 { - m |= syscall.FILE_NOTIFY_CHANGE_LAST_ACCESS - } - if mask&sysFSMODIFY != 0 { - m |= syscall.FILE_NOTIFY_CHANGE_LAST_WRITE - } - if mask&sysFSATTRIB != 0 { - m |= syscall.FILE_NOTIFY_CHANGE_ATTRIBUTES - } - if mask&(sysFSMOVE|sysFSCREATE|sysFSDELETE) != 0 { - m |= syscall.FILE_NOTIFY_CHANGE_FILE_NAME | syscall.FILE_NOTIFY_CHANGE_DIR_NAME - } - return m -} - -func toFSnotifyFlags(action uint32) uint64 { - switch action { - case syscall.FILE_ACTION_ADDED: - return sysFSCREATE - case syscall.FILE_ACTION_REMOVED: - return sysFSDELETE - case syscall.FILE_ACTION_MODIFIED: - return sysFSMODIFY - case syscall.FILE_ACTION_RENAMED_OLD_NAME: - return sysFSMOVEDFROM - case syscall.FILE_ACTION_RENAMED_NEW_NAME: - return sysFSMOVEDTO - } - return 0 -} diff --git a/vendor/modules.txt b/vendor/modules.txt index cea9c80ddc..edb8b03d7c 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -319,7 +319,6 @@ github.com/fernet/fernet-go github.com/fluent/fluent-logger-golang/fluent # github.com/fsnotify/fsnotify v1.5.1 ## explicit; go 1.13 -github.com/fsnotify/fsnotify # github.com/go-logr/logr v1.2.2 ## explicit; go 1.16 github.com/go-logr/logr