diff --git a/daemon/logger/copier.go b/daemon/logger/copier.go index e2ee36c098..30c68ea364 100644 --- a/daemon/logger/copier.go +++ b/daemon/logger/copier.go @@ -126,8 +126,7 @@ func (c *Copier) copySrc(name string, src io.Reader) { } if logErr := c.dst.Log(msg); logErr != nil { - logWritesFailedCount.Inc(1) - logrus.Errorf("Failed to log msg %q for logger %s: %s", msg.Line, c.dst.Name(), logErr) + logDriverError(c.dst.Name(), string(msg.Line), logErr) } } p += q + 1 @@ -159,8 +158,7 @@ func (c *Copier) copySrc(name string, src io.Reader) { hasMorePartial = true if logErr := c.dst.Log(msg); logErr != nil { - logWritesFailedCount.Inc(1) - logrus.Errorf("Failed to log msg %q for logger %s: %s", msg.Line, c.dst.Name(), logErr) + logDriverError(c.dst.Name(), string(msg.Line), logErr) } p = 0 n = 0 diff --git a/daemon/logger/logger_error.go b/daemon/logger/logger_error.go new file mode 100644 index 0000000000..70f4311979 --- /dev/null +++ b/daemon/logger/logger_error.go @@ -0,0 +1,24 @@ +package logger + +import ( + "github.com/sirupsen/logrus" + "golang.org/x/time/rate" +) + +// Rates based on journald defaults of 10,000 messages in 30s. +// reference: https://www.freedesktop.org/software/systemd/man/journald.conf.html#RateLimitIntervalSec= +var logErrorLimiter = rate.NewLimiter(333, 333) + +// logDriverError logs errors produced by log drivers to the daemon logs. It also increments the logWritesFailedCount +// metric. +// Logging to the daemon logs is limited to 333 operations per second at most. If this limit is exceeded, the +// logWritesFailedCount is still counted, but logging to the daemon logs is omitted in order to prevent disk saturation. +func logDriverError(loggerName, msgLine string, logErr error) { + logWritesFailedCount.Inc(1) + if logErrorLimiter.Allow() { + logrus.WithError(logErr). + WithField("driver", loggerName). + WithField("message", msgLine). + Errorf("Error writing log message") + } +} diff --git a/daemon/logger/ring.go b/daemon/logger/ring.go index 658f0fcfd0..ff43baac2f 100644 --- a/daemon/logger/ring.go +++ b/daemon/logger/ring.go @@ -4,8 +4,6 @@ import ( "errors" "sync" "sync/atomic" - - "github.com/sirupsen/logrus" ) const ( @@ -107,10 +105,7 @@ func (r *RingLogger) Close() error { } if err := r.l.Log(msg); err != nil { - logrus.WithField("driver", r.l.Name()). - WithField("container", r.logInfo.ContainerID). - WithError(err). - Errorf("Error writing log message") + logDriverError(r.l.Name(), string(msg.Line), err) logErr = true } } @@ -132,10 +127,7 @@ func (r *RingLogger) run() { return } if err := r.l.Log(msg); err != nil { - logrus.WithField("driver", r.l.Name()). - WithField("container", r.logInfo.ContainerID). - WithError(err). - Errorf("Error writing log message") + logDriverError(r.l.Name(), string(msg.Line), err) } } }