From 0d03ecc2cb11330b5a3267725ef8f4a3ff744df6 Mon Sep 17 00:00:00 2001 From: Thibault bui Koechlin Date: Wed, 27 May 2020 11:51:24 +0200 Subject: [PATCH 1/8] move the setLogger config and ConfigureLogger to be part of types for reuse accross modules --- cmd/crowdsec/main.go | 31 +--------------------- pkg/types/utils.go | 62 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 63 insertions(+), 30 deletions(-) diff --git a/cmd/crowdsec/main.go b/cmd/crowdsec/main.go index c1d4b31e6..319869da5 100644 --- a/cmd/crowdsec/main.go +++ b/cmd/crowdsec/main.go @@ -1,7 +1,6 @@ package main import ( - "fmt" "strings" "io/ioutil" @@ -19,7 +18,6 @@ import ( log "github.com/sirupsen/logrus" - "gopkg.in/natefinch/lumberjack.v2" "gopkg.in/tomb.v2" "gopkg.in/yaml.v2" ) @@ -39,33 +37,6 @@ var ( lastProcessedItem time.Time /*keep track of last item timestamp in time-machine. it is used to GC buckets when we dump them.*/ ) -func configureLogger(logMode string, logFolder string, logLevel log.Level) error { - /*Configure logs*/ - if logMode == "file" { - log.SetOutput(&lumberjack.Logger{ - Filename: logFolder + "/crowdsec.log", - MaxSize: 500, //megabytes - MaxBackups: 3, - MaxAge: 28, //days - Compress: true, //disabled by default - }) - log.SetFormatter(&log.TextFormatter{TimestampFormat: "02-01-2006 15:04:05", FullTimestamp: true}) - } else if logMode != "stdout" { - return fmt.Errorf("log mode '%s' unknown", logMode) - } - - log.Printf("setting loglevel to %s", logLevel) - log.SetLevel(logLevel) - log.SetFormatter(&log.TextFormatter{FullTimestamp: true}) - if logLevel >= log.InfoLevel { - log.SetFormatter(&log.TextFormatter{TimestampFormat: "02-01-2006 15:04:05", FullTimestamp: true}) - } - if logLevel >= log.DebugLevel { - log.SetReportCaller(true) - } - return nil -} - func main() { var ( err error @@ -92,7 +63,7 @@ func main() { log.Fatalf(err.Error()) } - if err = configureLogger(cConfig.LogMode, cConfig.LogFolder, cConfig.LogLevel); err != nil { + if err = types.SetDefaultLoggerConfig(cConfig.LogMode, cConfig.LogFolder, cConfig.LogLevel); err != nil { log.Fatal(err.Error()) } diff --git a/pkg/types/utils.go b/pkg/types/utils.go index d71dced6e..e221e4a38 100644 --- a/pkg/types/utils.go +++ b/pkg/types/utils.go @@ -2,7 +2,11 @@ package types import ( "encoding/binary" + "fmt" "net" + + log "github.com/sirupsen/logrus" + "gopkg.in/natefinch/lumberjack.v2" ) func IP2Int(ip net.IP) uint32 { @@ -40,3 +44,61 @@ func LastAddress(n *net.IPNet) net.IP { ip[2]|^n.Mask[2], ip[3]|^n.Mask[3]) } + +var logMode string +var logFolder string +var logLevel log.Level + +func SetDefaultLoggerConfig(inlogMode string, inlogFolder string, inlogLevel log.Level) error { + logMode = inlogMode + logFolder = inlogFolder + logLevel = inlogLevel + + /*Configure logs*/ + if logMode == "file" { + log.SetOutput(&lumberjack.Logger{ + Filename: logFolder + "/crowdsec.log", + MaxSize: 500, //megabytes + MaxBackups: 3, + MaxAge: 28, //days + Compress: true, //disabled by default + }) + log.SetFormatter(&log.TextFormatter{TimestampFormat: "02-01-2006 15:04:05", FullTimestamp: true}) + } else if logMode != "stdout" { + return fmt.Errorf("log mode '%s' unknown", logMode) + } + log.SetLevel(logLevel) + log.SetFormatter(&log.TextFormatter{FullTimestamp: true}) + if logLevel >= log.InfoLevel { + log.SetFormatter(&log.TextFormatter{TimestampFormat: "02-01-2006 15:04:05", FullTimestamp: true}) + } + if logLevel >= log.DebugLevel { + log.SetReportCaller(true) + } + return nil +} + +func ConfigureLogger(clog *log.Logger) error { + /*Configure logs*/ + if logMode == "file" { + clog.SetOutput(&lumberjack.Logger{ + Filename: logFolder + "/crowdsec.log", + MaxSize: 500, //megabytes + MaxBackups: 3, + MaxAge: 28, //days + Compress: true, //disabled by default + }) + clog.SetFormatter(&log.TextFormatter{TimestampFormat: "02-01-2006 15:04:05", FullTimestamp: true}) + } else if logMode != "stdout" { + return fmt.Errorf("log mode '%s' unknown", logMode) + } + clog.SetLevel(logLevel) + clog.SetFormatter(&log.TextFormatter{FullTimestamp: true}) + if logLevel >= log.InfoLevel { + clog.SetFormatter(&log.TextFormatter{TimestampFormat: "02-01-2006 15:04:05", FullTimestamp: true}) + } + if logLevel >= log.DebugLevel { + clog.SetReportCaller(true) + } + return nil +} From 8b6332f950e5e51be4d50628872d5be044713074 Mon Sep 17 00:00:00 2001 From: Thibault bui Koechlin Date: Wed, 27 May 2020 11:51:49 +0200 Subject: [PATCH 2/8] unify loggers and improve the log message about groks when they're not called by name --- pkg/leakybucket/manager.go | 4 +++- pkg/parser/node.go | 17 ++++++++++++++--- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/pkg/leakybucket/manager.go b/pkg/leakybucket/manager.go index 377f1282a..e742eb835 100644 --- a/pkg/leakybucket/manager.go +++ b/pkg/leakybucket/manager.go @@ -191,7 +191,9 @@ func LoadBucket(g *BucketFactory) error { var err error if g.Debug { var clog = logrus.New() - clog.SetFormatter(&log.TextFormatter{FullTimestamp: true}) + if types.ConfigureLogger(clog) != err { + log.Fatalf("While creating bucket-specific logger : %s", err) + } clog.SetLevel(log.DebugLevel) g.logger = clog.WithFields(log.Fields{ "cfg": g.BucketName, diff --git a/pkg/parser/node.go b/pkg/parser/node.go index c96a40bd8..34e5fb480 100644 --- a/pkg/parser/node.go +++ b/pkg/parser/node.go @@ -252,10 +252,15 @@ func (n *Node) process(p *types.Event, ctx UnixParserCtx) (bool, error) { //return false, nil } } - + var groklabel string + if n.Grok.RegexpName == "" { + groklabel = fmt.Sprintf("%5.5s...", n.Grok.RegexpValue) + } else { + groklabel = n.Grok.RegexpName + } grok := n.Grok.RunTimeRegexp.Parse(gstr) if len(grok) > 0 { - clog.Debugf("+ Grok '%s' returned %d entries to merge in Parsed", n.Grok.RegexpName, len(grok)) + clog.Debugf("+ Grok '%s' returned %d entries to merge in Parsed", groklabel, len(grok)) //We managed to grok stuff, merged into parse for k, v := range grok { clog.Debugf("\t.Parsed['%s'] = '%s'", k, v) @@ -268,7 +273,7 @@ func (n *Node) process(p *types.Event, ctx UnixParserCtx) (bool, error) { } } else { //grok failed, node failed - clog.Debugf("+ Grok '%s' didn't return data on '%s'", n.Grok.RegexpName, gstr) + clog.Debugf("+ Grok '%s' didn't return data on '%s'", groklabel, gstr) //clog.Tracef("on '%s'", gstr) NodeState = false } @@ -336,6 +341,9 @@ func (n *Node) compile(pctx *UnixParserCtx) error { that will be used only for processing this node ;) */ if n.Debug { var clog = logrus.New() + if types.ConfigureLogger(clog) != err { + log.Fatalf("While creating bucket-specific logger : %s", err) + } clog.SetLevel(log.DebugLevel) n.logger = clog.WithFields(log.Fields{ "id": n.rn, @@ -411,6 +419,9 @@ func (n *Node) compile(pctx *UnixParserCtx) error { /* compile leafs if present */ if len(n.SuccessNodes) > 0 { for idx := range n.SuccessNodes { + if n.SuccessNodes[idx].Name == "" { + n.SuccessNodes[idx].Name = fmt.Sprintf("child-%s", n.Name) + } /*propagate debug/stats to child nodes*/ if !n.SuccessNodes[idx].Debug && n.Debug { n.SuccessNodes[idx].Debug = true From 225c9fe872fca06c565d85e81b1a0d28ef275251 Mon Sep 17 00:00:00 2001 From: Thibault bui Koechlin Date: Wed, 27 May 2020 12:08:38 +0200 Subject: [PATCH 3/8] lower verbosity for this, give context to that --- pkg/leakybucket/manager.go | 2 +- pkg/parser/stage.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/leakybucket/manager.go b/pkg/leakybucket/manager.go index e742eb835..f23642c87 100644 --- a/pkg/leakybucket/manager.go +++ b/pkg/leakybucket/manager.go @@ -146,7 +146,7 @@ func LoadBuckets(files []string) ([]BucketFactory, chan types.Event, error) { } //check compat if g.FormatVersion == "" { - log.Warningf("no version in %s : %s, assuming '1.0'", g.Name, f) + log.Debugf("no version in %s : %s, assuming '1.0'", g.Name, f) g.FormatVersion = "1.0" } ok, err := cwversion.Statisfies(g.FormatVersion, cwversion.Constraint_scenario) diff --git a/pkg/parser/stage.go b/pkg/parser/stage.go index 4d27da883..fca5cef3c 100644 --- a/pkg/parser/stage.go +++ b/pkg/parser/stage.go @@ -76,7 +76,7 @@ func LoadStages(stageFiles []Stagefile, pctx *UnixParserCtx) ([]Node, error) { //check for empty bucket if node.Name == "" && node.Description == "" && node.Author == "" { - log.Infof("Node has no name,author or description. Skipping.") + log.Infof("Node in %s has no name,author or description. Skipping.", stageFile.Filename) continue } //check compat From da4da67a3496acdf33a5d06dd0d09450706e4506 Mon Sep 17 00:00:00 2001 From: Thibault bui Koechlin Date: Wed, 27 May 2020 12:12:51 +0200 Subject: [PATCH 4/8] no prometheus, no api in cli mode --- config/user.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/user.yaml b/config/user.yaml index addbdf726..8d3f83c6c 100644 --- a/config/user.yaml +++ b/config/user.yaml @@ -8,9 +8,9 @@ log_mode: stdout log_level: info profiling: false sqlite_path: ${DATA}/crowdsec.db -apimode: true +apimode: false daemon: false -prometheus: true +prometheus: false #for prometheus agent / golang debugging http_listen: 127.0.0.1:6060 plugin: From 1591d1b8e3f611f2f6235703146aa72f3e862cc0 Mon Sep 17 00:00:00 2001 From: Thibault bui Koechlin Date: Wed, 27 May 2020 12:37:04 +0200 Subject: [PATCH 5/8] build plugins when we build --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index 39c72e6b8..353cc73d4 100644 --- a/Makefile +++ b/Makefile @@ -62,6 +62,7 @@ else @echo "Required golang version is $(REQUIRE_GOVERSION). The current one is $(CURRENT_GOVERSION). Exiting.." @exit 1; endif + @bash ./scripts/build_plugins.sh cscli_static: From 0529a256bbe1849c809632274a09bac780f4e353 Mon Sep 17 00:00:00 2001 From: Thibault bui Koechlin Date: Wed, 27 May 2020 12:37:18 +0200 Subject: [PATCH 6/8] simplify the code --- pkg/types/utils.go | 53 ++++++++++++++++++---------------------------- 1 file changed, 21 insertions(+), 32 deletions(-) diff --git a/pkg/types/utils.go b/pkg/types/utils.go index e221e4a38..51a92a679 100644 --- a/pkg/types/utils.go +++ b/pkg/types/utils.go @@ -3,6 +3,7 @@ package types import ( "encoding/binary" "fmt" + "io" "net" log "github.com/sirupsen/logrus" @@ -45,34 +46,34 @@ func LastAddress(n *net.IPNet) net.IP { ip[3]|^n.Mask[3]) } -var logMode string -var logFolder string +var logFormatter log.Formatter +var logOutput io.Writer var logLevel log.Level +var logReportCaller bool -func SetDefaultLoggerConfig(inlogMode string, inlogFolder string, inlogLevel log.Level) error { - logMode = inlogMode - logFolder = inlogFolder - logLevel = inlogLevel +func SetDefaultLoggerConfig(cfgMode string, cfgFolder string, cfgLevel log.Level) error { /*Configure logs*/ - if logMode == "file" { - log.SetOutput(&lumberjack.Logger{ - Filename: logFolder + "/crowdsec.log", + if cfgMode == "file" { + logOutput = &lumberjack.Logger{ + Filename: cfgFolder + "/crowdsec.log", MaxSize: 500, //megabytes MaxBackups: 3, MaxAge: 28, //days Compress: true, //disabled by default - }) - log.SetFormatter(&log.TextFormatter{TimestampFormat: "02-01-2006 15:04:05", FullTimestamp: true}) - } else if logMode != "stdout" { - return fmt.Errorf("log mode '%s' unknown", logMode) + } + log.SetOutput(logOutput) + } else if cfgMode != "stdout" { + return fmt.Errorf("log mode '%s' unknown", cfgMode) } + logLevel = cfgLevel log.SetLevel(logLevel) - log.SetFormatter(&log.TextFormatter{FullTimestamp: true}) if logLevel >= log.InfoLevel { - log.SetFormatter(&log.TextFormatter{TimestampFormat: "02-01-2006 15:04:05", FullTimestamp: true}) + logFormatter = &log.TextFormatter{TimestampFormat: "02-01-2006 15:04:05", FullTimestamp: true} + log.SetFormatter(logFormatter) } if logLevel >= log.DebugLevel { + logReportCaller = true log.SetReportCaller(true) } return nil @@ -80,25 +81,13 @@ func SetDefaultLoggerConfig(inlogMode string, inlogFolder string, inlogLevel log func ConfigureLogger(clog *log.Logger) error { /*Configure logs*/ - if logMode == "file" { - clog.SetOutput(&lumberjack.Logger{ - Filename: logFolder + "/crowdsec.log", - MaxSize: 500, //megabytes - MaxBackups: 3, - MaxAge: 28, //days - Compress: true, //disabled by default - }) - clog.SetFormatter(&log.TextFormatter{TimestampFormat: "02-01-2006 15:04:05", FullTimestamp: true}) - } else if logMode != "stdout" { - return fmt.Errorf("log mode '%s' unknown", logMode) + if logOutput != nil { + clog.SetOutput(logOutput) } - clog.SetLevel(logLevel) - clog.SetFormatter(&log.TextFormatter{FullTimestamp: true}) - if logLevel >= log.InfoLevel { - clog.SetFormatter(&log.TextFormatter{TimestampFormat: "02-01-2006 15:04:05", FullTimestamp: true}) - } - if logLevel >= log.DebugLevel { + if logReportCaller { clog.SetReportCaller(true) } + clog.SetFormatter(logFormatter) + clog.SetLevel(logLevel) return nil } From 890da37793b2c4afdd75232858aee1847e298610 Mon Sep 17 00:00:00 2001 From: Thibault bui Koechlin Date: Wed, 27 May 2020 12:45:16 +0200 Subject: [PATCH 7/8] only set if non-nil, avoid crash on unconfigured logger --- pkg/types/utils.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pkg/types/utils.go b/pkg/types/utils.go index 51a92a679..a5f8fded9 100644 --- a/pkg/types/utils.go +++ b/pkg/types/utils.go @@ -87,7 +87,9 @@ func ConfigureLogger(clog *log.Logger) error { if logReportCaller { clog.SetReportCaller(true) } - clog.SetFormatter(logFormatter) + if logFormatter != nil { + clog.SetFormatter(logFormatter) + } clog.SetLevel(logLevel) return nil } From 2209899a8f0890447d016c44af537a55f9f4cf6c Mon Sep 17 00:00:00 2001 From: Thibault bui Koechlin Date: Wed, 27 May 2020 14:35:26 +0200 Subject: [PATCH 8/8] type --- pkg/leakybucket/manager.go | 2 +- pkg/parser/node.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/leakybucket/manager.go b/pkg/leakybucket/manager.go index f23642c87..685269acf 100644 --- a/pkg/leakybucket/manager.go +++ b/pkg/leakybucket/manager.go @@ -191,7 +191,7 @@ func LoadBucket(g *BucketFactory) error { var err error if g.Debug { var clog = logrus.New() - if types.ConfigureLogger(clog) != err { + if err := types.ConfigureLogger(clog); err != nil { log.Fatalf("While creating bucket-specific logger : %s", err) } clog.SetLevel(log.DebugLevel) diff --git a/pkg/parser/node.go b/pkg/parser/node.go index 34e5fb480..d862b4c23 100644 --- a/pkg/parser/node.go +++ b/pkg/parser/node.go @@ -341,7 +341,7 @@ func (n *Node) compile(pctx *UnixParserCtx) error { that will be used only for processing this node ;) */ if n.Debug { var clog = logrus.New() - if types.ConfigureLogger(clog) != err { + if err := types.ConfigureLogger(clog); err != nil { log.Fatalf("While creating bucket-specific logger : %s", err) } clog.SetLevel(log.DebugLevel)