Merge pull request #42291 from angelcar/awslogs-dont-log-messge-discarded-errors

Limit the rate at which logger errors are logged into daemon logs

daemon/logger/copier.go

@@ -126,8 +126,7 @@ func (c *Copier) copySrc(name string, src io.Reader) {
 					}
 
 					if logErr := c.dst.Log(msg); logErr != nil {
-						logWritesFailedCount.Inc(1)
-						logrus.Errorf("Failed to log msg %q for logger %s: %s", msg.Line, c.dst.Name(), logErr)
+						logDriverError(c.dst.Name(), string(msg.Line), logErr)
 					}
 				}
 				p += q + 1
@@ -159,8 +158,7 @@ func (c *Copier) copySrc(name string, src io.Reader) {
 					hasMorePartial = true
 
 					if logErr := c.dst.Log(msg); logErr != nil {
-						logWritesFailedCount.Inc(1)
-						logrus.Errorf("Failed to log msg %q for logger %s: %s", msg.Line, c.dst.Name(), logErr)
+						logDriverError(c.dst.Name(), string(msg.Line), logErr)
 					}
 					p = 0
 					n = 0

daemon/logger/logger_error.go

@@ -0,0 +1,24 @@
+package logger
+
+import (
+	"github.com/sirupsen/logrus"
+	"golang.org/x/time/rate"
+)
+
+// Rates based on journald defaults of 10,000 messages in 30s.
+// reference: https://www.freedesktop.org/software/systemd/man/journald.conf.html#RateLimitIntervalSec=
+var logErrorLimiter = rate.NewLimiter(333, 333)
+
+// logDriverError logs errors produced by log drivers to the daemon logs. It also increments the logWritesFailedCount
+// metric.
+// Logging to the daemon logs is limited to 333 operations per second at most. If this limit is exceeded, the
+// logWritesFailedCount is still counted, but logging to the daemon logs is omitted in order to prevent disk saturation.
+func logDriverError(loggerName, msgLine string, logErr error) {
+	logWritesFailedCount.Inc(1)
+	if logErrorLimiter.Allow() {
+		logrus.WithError(logErr).
+			WithField("driver", loggerName).
+			WithField("message", msgLine).
+			Errorf("Error writing log message")
+	}
+}

daemon/logger/ring.go

@@ -4,8 +4,6 @@ import (
 	"errors"
 	"sync"
 	"sync/atomic"
-
-	"github.com/sirupsen/logrus"
 )
 
 const (
@@ -107,10 +105,7 @@ func (r *RingLogger) Close() error {
 		}
 
 		if err := r.l.Log(msg); err != nil {
-			logrus.WithField("driver", r.l.Name()).
-				WithField("container", r.logInfo.ContainerID).
-				WithError(err).
-				Errorf("Error writing log message")
+			logDriverError(r.l.Name(), string(msg.Line), err)
 			logErr = true
 		}
 	}
@@ -132,10 +127,7 @@ func (r *RingLogger) run() {
 			return
 		}
 		if err := r.l.Log(msg); err != nil {
-			logrus.WithField("driver", r.l.Name()).
-				WithField("container", r.logInfo.ContainerID).
-				WithError(err).
-				Errorf("Error writing log message")
+			logDriverError(r.l.Name(), string(msg.Line), err)
 		}
 	}
 }