0ct0pu5/moby
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Limit the rate at which logger errors are logged into daemon logs

Browse source 
Logging to daemon logs every time there's an error with a log driver can be
problematic since daemon logs can grow rapidly, potentially exhausting disk
space.

Instead, it's preferable to limit the rate at which log driver errors are allowed
to be written. By default, this limit is 333 entries per second max.

Signed-off-by: Angel Velazquez <angelcar@amazon.com>
This commit is contained in:
Angel Velazquez 2021-05-20 16:59:30 -07:00
parent e02bc91dcb
commit fb5a9ec741
No known key found for this signature in database
GPG key ID: 70D561A8F6E6CB0A
3 changed files with 28 additions and 14 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
daemon/logger/copier.go
View file

@ -126,8 +126,7 @@ func (c *Copier) copySrc(name string, src io.Reader) {
}
if logErr := c.dst.Log(msg); logErr != nil {
logWritesFailedCount.Inc(1)
logrus.Errorf("Failed to log msg %q for logger %s: %s", msg.Line, c.dst.Name(), logErr)
logDriverError(c.dst.Name(), string(msg.Line), logErr)
}
}
p += q + 1
@ -159,8 +158,7 @@ func (c *Copier) copySrc(name string, src io.Reader) {
hasMorePartial = true
if logErr := c.dst.Log(msg); logErr != nil {
logWritesFailedCount.Inc(1)
logrus.Errorf("Failed to log msg %q for logger %s: %s", msg.Line, c.dst.Name(), logErr)
logDriverError(c.dst.Name(), string(msg.Line), logErr)
}
p = 0
n = 0

24
daemon/logger/logger_error.go Normal file
View file

@ -0,0 +1,24 @@
package logger
import (
"github.com/sirupsen/logrus"
"golang.org/x/time/rate"
)
// Rates based on journald defaults of 10,000 messages in 30s.
// reference: https://www.freedesktop.org/software/systemd/man/journald.conf.html#RateLimitIntervalSec=
var logErrorLimiter = rate.NewLimiter(333, 333)
// logDriverError logs errors produced by log drivers to the daemon logs. It also increments the logWritesFailedCount
// metric.
// Logging to the daemon logs is limited to 333 operations per second at most. If this limit is exceeded, the
// logWritesFailedCount is still counted, but logging to the daemon logs is omitted in order to prevent disk saturation.
func logDriverError(loggerName, msgLine string, logErr error) {
logWritesFailedCount.Inc(1)
if logErrorLimiter.Allow() {
logrus.WithError(logErr).
WithField("driver", loggerName).
WithField("message", msgLine).
Errorf("Error writing log message")
}
}

12
daemon/logger/ring.go
View file

@ -4,8 +4,6 @@ import (
"errors"
"sync"
"sync/atomic"
"github.com/sirupsen/logrus"
)
const (
@ -104,10 +102,7 @@ func (r *RingLogger) Close() error {
}
if err := r.l.Log(msg); err != nil {
logrus.WithField("driver", r.l.Name()).
WithField("container", r.logInfo.ContainerID).
WithError(err).
Errorf("Error writing log message")
logDriverError(r.l.Name(), string(msg.Line), err)
logErr = true
}
}
@ -128,10 +123,7 @@ func (r *RingLogger) run() {
return
}
if err := r.l.Log(msg); err != nil {
logrus.WithField("driver", r.l.Name()).
WithField("container", r.logInfo.ContainerID).
WithError(err).
Errorf("Error writing log message")
logDriverError(r.l.Name(), string(msg.Line), err)
}
}
}