From 0d03ecc2cb11330b5a3267725ef8f4a3ff744df6 Mon Sep 17 00:00:00 2001 From: Thibault bui Koechlin Date: Wed, 27 May 2020 11:51:24 +0200 Subject: [PATCH 01/11] move the setLogger config and ConfigureLogger to be part of types for reuse accross modules --- cmd/crowdsec/main.go | 31 +--------------------- pkg/types/utils.go | 62 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 63 insertions(+), 30 deletions(-) diff --git a/cmd/crowdsec/main.go b/cmd/crowdsec/main.go index c1d4b31e6..319869da5 100644 --- a/cmd/crowdsec/main.go +++ b/cmd/crowdsec/main.go @@ -1,7 +1,6 @@ package main import ( - "fmt" "strings" "io/ioutil" @@ -19,7 +18,6 @@ import ( log "github.com/sirupsen/logrus" - "gopkg.in/natefinch/lumberjack.v2" "gopkg.in/tomb.v2" "gopkg.in/yaml.v2" ) @@ -39,33 +37,6 @@ var ( lastProcessedItem time.Time /*keep track of last item timestamp in time-machine. it is used to GC buckets when we dump them.*/ ) -func configureLogger(logMode string, logFolder string, logLevel log.Level) error { - /*Configure logs*/ - if logMode == "file" { - log.SetOutput(&lumberjack.Logger{ - Filename: logFolder + "/crowdsec.log", - MaxSize: 500, //megabytes - MaxBackups: 3, - MaxAge: 28, //days - Compress: true, //disabled by default - }) - log.SetFormatter(&log.TextFormatter{TimestampFormat: "02-01-2006 15:04:05", FullTimestamp: true}) - } else if logMode != "stdout" { - return fmt.Errorf("log mode '%s' unknown", logMode) - } - - log.Printf("setting loglevel to %s", logLevel) - log.SetLevel(logLevel) - log.SetFormatter(&log.TextFormatter{FullTimestamp: true}) - if logLevel >= log.InfoLevel { - log.SetFormatter(&log.TextFormatter{TimestampFormat: "02-01-2006 15:04:05", FullTimestamp: true}) - } - if logLevel >= log.DebugLevel { - log.SetReportCaller(true) - } - return nil -} - func main() { var ( err error @@ -92,7 +63,7 @@ func main() { log.Fatalf(err.Error()) } - if err = configureLogger(cConfig.LogMode, cConfig.LogFolder, cConfig.LogLevel); err != nil { + if err = types.SetDefaultLoggerConfig(cConfig.LogMode, cConfig.LogFolder, cConfig.LogLevel); err != nil { log.Fatal(err.Error()) } diff --git a/pkg/types/utils.go b/pkg/types/utils.go index d71dced6e..e221e4a38 100644 --- a/pkg/types/utils.go +++ b/pkg/types/utils.go @@ -2,7 +2,11 @@ package types import ( "encoding/binary" + "fmt" "net" + + log "github.com/sirupsen/logrus" + "gopkg.in/natefinch/lumberjack.v2" ) func IP2Int(ip net.IP) uint32 { @@ -40,3 +44,61 @@ func LastAddress(n *net.IPNet) net.IP { ip[2]|^n.Mask[2], ip[3]|^n.Mask[3]) } + +var logMode string +var logFolder string +var logLevel log.Level + +func SetDefaultLoggerConfig(inlogMode string, inlogFolder string, inlogLevel log.Level) error { + logMode = inlogMode + logFolder = inlogFolder + logLevel = inlogLevel + + /*Configure logs*/ + if logMode == "file" { + log.SetOutput(&lumberjack.Logger{ + Filename: logFolder + "/crowdsec.log", + MaxSize: 500, //megabytes + MaxBackups: 3, + MaxAge: 28, //days + Compress: true, //disabled by default + }) + log.SetFormatter(&log.TextFormatter{TimestampFormat: "02-01-2006 15:04:05", FullTimestamp: true}) + } else if logMode != "stdout" { + return fmt.Errorf("log mode '%s' unknown", logMode) + } + log.SetLevel(logLevel) + log.SetFormatter(&log.TextFormatter{FullTimestamp: true}) + if logLevel >= log.InfoLevel { + log.SetFormatter(&log.TextFormatter{TimestampFormat: "02-01-2006 15:04:05", FullTimestamp: true}) + } + if logLevel >= log.DebugLevel { + log.SetReportCaller(true) + } + return nil +} + +func ConfigureLogger(clog *log.Logger) error { + /*Configure logs*/ + if logMode == "file" { + clog.SetOutput(&lumberjack.Logger{ + Filename: logFolder + "/crowdsec.log", + MaxSize: 500, //megabytes + MaxBackups: 3, + MaxAge: 28, //days + Compress: true, //disabled by default + }) + clog.SetFormatter(&log.TextFormatter{TimestampFormat: "02-01-2006 15:04:05", FullTimestamp: true}) + } else if logMode != "stdout" { + return fmt.Errorf("log mode '%s' unknown", logMode) + } + clog.SetLevel(logLevel) + clog.SetFormatter(&log.TextFormatter{FullTimestamp: true}) + if logLevel >= log.InfoLevel { + clog.SetFormatter(&log.TextFormatter{TimestampFormat: "02-01-2006 15:04:05", FullTimestamp: true}) + } + if logLevel >= log.DebugLevel { + clog.SetReportCaller(true) + } + return nil +} From 8b6332f950e5e51be4d50628872d5be044713074 Mon Sep 17 00:00:00 2001 From: Thibault bui Koechlin Date: Wed, 27 May 2020 11:51:49 +0200 Subject: [PATCH 02/11] unify loggers and improve the log message about groks when they're not called by name --- pkg/leakybucket/manager.go | 4 +++- pkg/parser/node.go | 17 ++++++++++++++--- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/pkg/leakybucket/manager.go b/pkg/leakybucket/manager.go index 377f1282a..e742eb835 100644 --- a/pkg/leakybucket/manager.go +++ b/pkg/leakybucket/manager.go @@ -191,7 +191,9 @@ func LoadBucket(g *BucketFactory) error { var err error if g.Debug { var clog = logrus.New() - clog.SetFormatter(&log.TextFormatter{FullTimestamp: true}) + if types.ConfigureLogger(clog) != err { + log.Fatalf("While creating bucket-specific logger : %s", err) + } clog.SetLevel(log.DebugLevel) g.logger = clog.WithFields(log.Fields{ "cfg": g.BucketName, diff --git a/pkg/parser/node.go b/pkg/parser/node.go index c96a40bd8..34e5fb480 100644 --- a/pkg/parser/node.go +++ b/pkg/parser/node.go @@ -252,10 +252,15 @@ func (n *Node) process(p *types.Event, ctx UnixParserCtx) (bool, error) { //return false, nil } } - + var groklabel string + if n.Grok.RegexpName == "" { + groklabel = fmt.Sprintf("%5.5s...", n.Grok.RegexpValue) + } else { + groklabel = n.Grok.RegexpName + } grok := n.Grok.RunTimeRegexp.Parse(gstr) if len(grok) > 0 { - clog.Debugf("+ Grok '%s' returned %d entries to merge in Parsed", n.Grok.RegexpName, len(grok)) + clog.Debugf("+ Grok '%s' returned %d entries to merge in Parsed", groklabel, len(grok)) //We managed to grok stuff, merged into parse for k, v := range grok { clog.Debugf("\t.Parsed['%s'] = '%s'", k, v) @@ -268,7 +273,7 @@ func (n *Node) process(p *types.Event, ctx UnixParserCtx) (bool, error) { } } else { //grok failed, node failed - clog.Debugf("+ Grok '%s' didn't return data on '%s'", n.Grok.RegexpName, gstr) + clog.Debugf("+ Grok '%s' didn't return data on '%s'", groklabel, gstr) //clog.Tracef("on '%s'", gstr) NodeState = false } @@ -336,6 +341,9 @@ func (n *Node) compile(pctx *UnixParserCtx) error { that will be used only for processing this node ;) */ if n.Debug { var clog = logrus.New() + if types.ConfigureLogger(clog) != err { + log.Fatalf("While creating bucket-specific logger : %s", err) + } clog.SetLevel(log.DebugLevel) n.logger = clog.WithFields(log.Fields{ "id": n.rn, @@ -411,6 +419,9 @@ func (n *Node) compile(pctx *UnixParserCtx) error { /* compile leafs if present */ if len(n.SuccessNodes) > 0 { for idx := range n.SuccessNodes { + if n.SuccessNodes[idx].Name == "" { + n.SuccessNodes[idx].Name = fmt.Sprintf("child-%s", n.Name) + } /*propagate debug/stats to child nodes*/ if !n.SuccessNodes[idx].Debug && n.Debug { n.SuccessNodes[idx].Debug = true From 225c9fe872fca06c565d85e81b1a0d28ef275251 Mon Sep 17 00:00:00 2001 From: Thibault bui Koechlin Date: Wed, 27 May 2020 12:08:38 +0200 Subject: [PATCH 03/11] lower verbosity for this, give context to that --- pkg/leakybucket/manager.go | 2 +- pkg/parser/stage.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/leakybucket/manager.go b/pkg/leakybucket/manager.go index e742eb835..f23642c87 100644 --- a/pkg/leakybucket/manager.go +++ b/pkg/leakybucket/manager.go @@ -146,7 +146,7 @@ func LoadBuckets(files []string) ([]BucketFactory, chan types.Event, error) { } //check compat if g.FormatVersion == "" { - log.Warningf("no version in %s : %s, assuming '1.0'", g.Name, f) + log.Debugf("no version in %s : %s, assuming '1.0'", g.Name, f) g.FormatVersion = "1.0" } ok, err := cwversion.Statisfies(g.FormatVersion, cwversion.Constraint_scenario) diff --git a/pkg/parser/stage.go b/pkg/parser/stage.go index 4d27da883..fca5cef3c 100644 --- a/pkg/parser/stage.go +++ b/pkg/parser/stage.go @@ -76,7 +76,7 @@ func LoadStages(stageFiles []Stagefile, pctx *UnixParserCtx) ([]Node, error) { //check for empty bucket if node.Name == "" && node.Description == "" && node.Author == "" { - log.Infof("Node has no name,author or description. Skipping.") + log.Infof("Node in %s has no name,author or description. Skipping.", stageFile.Filename) continue } //check compat From da4da67a3496acdf33a5d06dd0d09450706e4506 Mon Sep 17 00:00:00 2001 From: Thibault bui Koechlin Date: Wed, 27 May 2020 12:12:51 +0200 Subject: [PATCH 04/11] no prometheus, no api in cli mode --- config/user.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/user.yaml b/config/user.yaml index addbdf726..8d3f83c6c 100644 --- a/config/user.yaml +++ b/config/user.yaml @@ -8,9 +8,9 @@ log_mode: stdout log_level: info profiling: false sqlite_path: ${DATA}/crowdsec.db -apimode: true +apimode: false daemon: false -prometheus: true +prometheus: false #for prometheus agent / golang debugging http_listen: 127.0.0.1:6060 plugin: From 1591d1b8e3f611f2f6235703146aa72f3e862cc0 Mon Sep 17 00:00:00 2001 From: Thibault bui Koechlin Date: Wed, 27 May 2020 12:37:04 +0200 Subject: [PATCH 05/11] build plugins when we build --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index 39c72e6b8..353cc73d4 100644 --- a/Makefile +++ b/Makefile @@ -62,6 +62,7 @@ else @echo "Required golang version is $(REQUIRE_GOVERSION). The current one is $(CURRENT_GOVERSION). Exiting.." @exit 1; endif + @bash ./scripts/build_plugins.sh cscli_static: From 0529a256bbe1849c809632274a09bac780f4e353 Mon Sep 17 00:00:00 2001 From: Thibault bui Koechlin Date: Wed, 27 May 2020 12:37:18 +0200 Subject: [PATCH 06/11] simplify the code --- pkg/types/utils.go | 53 ++++++++++++++++++---------------------------- 1 file changed, 21 insertions(+), 32 deletions(-) diff --git a/pkg/types/utils.go b/pkg/types/utils.go index e221e4a38..51a92a679 100644 --- a/pkg/types/utils.go +++ b/pkg/types/utils.go @@ -3,6 +3,7 @@ package types import ( "encoding/binary" "fmt" + "io" "net" log "github.com/sirupsen/logrus" @@ -45,34 +46,34 @@ func LastAddress(n *net.IPNet) net.IP { ip[3]|^n.Mask[3]) } -var logMode string -var logFolder string +var logFormatter log.Formatter +var logOutput io.Writer var logLevel log.Level +var logReportCaller bool -func SetDefaultLoggerConfig(inlogMode string, inlogFolder string, inlogLevel log.Level) error { - logMode = inlogMode - logFolder = inlogFolder - logLevel = inlogLevel +func SetDefaultLoggerConfig(cfgMode string, cfgFolder string, cfgLevel log.Level) error { /*Configure logs*/ - if logMode == "file" { - log.SetOutput(&lumberjack.Logger{ - Filename: logFolder + "/crowdsec.log", + if cfgMode == "file" { + logOutput = &lumberjack.Logger{ + Filename: cfgFolder + "/crowdsec.log", MaxSize: 500, //megabytes MaxBackups: 3, MaxAge: 28, //days Compress: true, //disabled by default - }) - log.SetFormatter(&log.TextFormatter{TimestampFormat: "02-01-2006 15:04:05", FullTimestamp: true}) - } else if logMode != "stdout" { - return fmt.Errorf("log mode '%s' unknown", logMode) + } + log.SetOutput(logOutput) + } else if cfgMode != "stdout" { + return fmt.Errorf("log mode '%s' unknown", cfgMode) } + logLevel = cfgLevel log.SetLevel(logLevel) - log.SetFormatter(&log.TextFormatter{FullTimestamp: true}) if logLevel >= log.InfoLevel { - log.SetFormatter(&log.TextFormatter{TimestampFormat: "02-01-2006 15:04:05", FullTimestamp: true}) + logFormatter = &log.TextFormatter{TimestampFormat: "02-01-2006 15:04:05", FullTimestamp: true} + log.SetFormatter(logFormatter) } if logLevel >= log.DebugLevel { + logReportCaller = true log.SetReportCaller(true) } return nil @@ -80,25 +81,13 @@ func SetDefaultLoggerConfig(inlogMode string, inlogFolder string, inlogLevel log func ConfigureLogger(clog *log.Logger) error { /*Configure logs*/ - if logMode == "file" { - clog.SetOutput(&lumberjack.Logger{ - Filename: logFolder + "/crowdsec.log", - MaxSize: 500, //megabytes - MaxBackups: 3, - MaxAge: 28, //days - Compress: true, //disabled by default - }) - clog.SetFormatter(&log.TextFormatter{TimestampFormat: "02-01-2006 15:04:05", FullTimestamp: true}) - } else if logMode != "stdout" { - return fmt.Errorf("log mode '%s' unknown", logMode) + if logOutput != nil { + clog.SetOutput(logOutput) } - clog.SetLevel(logLevel) - clog.SetFormatter(&log.TextFormatter{FullTimestamp: true}) - if logLevel >= log.InfoLevel { - clog.SetFormatter(&log.TextFormatter{TimestampFormat: "02-01-2006 15:04:05", FullTimestamp: true}) - } - if logLevel >= log.DebugLevel { + if logReportCaller { clog.SetReportCaller(true) } + clog.SetFormatter(logFormatter) + clog.SetLevel(logLevel) return nil } From 890da37793b2c4afdd75232858aee1847e298610 Mon Sep 17 00:00:00 2001 From: Thibault bui Koechlin Date: Wed, 27 May 2020 12:45:16 +0200 Subject: [PATCH 07/11] only set if non-nil, avoid crash on unconfigured logger --- pkg/types/utils.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pkg/types/utils.go b/pkg/types/utils.go index 51a92a679..a5f8fded9 100644 --- a/pkg/types/utils.go +++ b/pkg/types/utils.go @@ -87,7 +87,9 @@ func ConfigureLogger(clog *log.Logger) error { if logReportCaller { clog.SetReportCaller(true) } - clog.SetFormatter(logFormatter) + if logFormatter != nil { + clog.SetFormatter(logFormatter) + } clog.SetLevel(logLevel) return nil } From 2209899a8f0890447d016c44af537a55f9f4cf6c Mon Sep 17 00:00:00 2001 From: Thibault bui Koechlin Date: Wed, 27 May 2020 14:35:26 +0200 Subject: [PATCH 08/11] type --- pkg/leakybucket/manager.go | 2 +- pkg/parser/node.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/leakybucket/manager.go b/pkg/leakybucket/manager.go index f23642c87..685269acf 100644 --- a/pkg/leakybucket/manager.go +++ b/pkg/leakybucket/manager.go @@ -191,7 +191,7 @@ func LoadBucket(g *BucketFactory) error { var err error if g.Debug { var clog = logrus.New() - if types.ConfigureLogger(clog) != err { + if err := types.ConfigureLogger(clog); err != nil { log.Fatalf("While creating bucket-specific logger : %s", err) } clog.SetLevel(log.DebugLevel) diff --git a/pkg/parser/node.go b/pkg/parser/node.go index 34e5fb480..d862b4c23 100644 --- a/pkg/parser/node.go +++ b/pkg/parser/node.go @@ -341,7 +341,7 @@ func (n *Node) compile(pctx *UnixParserCtx) error { that will be used only for processing this node ;) */ if n.Debug { var clog = logrus.New() - if types.ConfigureLogger(clog) != err { + if err := types.ConfigureLogger(clog); err != nil { log.Fatalf("While creating bucket-specific logger : %s", err) } clog.SetLevel(log.DebugLevel) From 1b3f1dfee17afba1056542c7893f93eee49a5f6a Mon Sep 17 00:00:00 2001 From: erenJag Date: Wed, 27 May 2020 15:54:40 +0200 Subject: [PATCH 09/11] trying to be [FHS](https://en.wikipedia.org/wiki/Filesystem_Hierarchy_Standard) compliant --- config/plugins/backend/sqlite.yaml | 2 +- docs/getting_started/installation.md | 1 + docs/references/output.md | 2 +- mkdocs.yml | 2 +- wizard.sh | 12 +++++++----- 5 files changed, 11 insertions(+), 8 deletions(-) diff --git a/config/plugins/backend/sqlite.yaml b/config/plugins/backend/sqlite.yaml index c4250c15f..e2009270e 100644 --- a/config/plugins/backend/sqlite.yaml +++ b/config/plugins/backend/sqlite.yaml @@ -1,5 +1,5 @@ name: sqlite -path: /var/lib/crowdsec/plugins/backend/sqlite.so +path: /usr/local/crowdsec/plugins/backend/sqlite.so config: db_path: /var/lib/crowdsec/data/crowdsec.db flush: true diff --git a/docs/getting_started/installation.md b/docs/getting_started/installation.md index a7a5c1c01..627687638 100644 --- a/docs/getting_started/installation.md +++ b/docs/getting_started/installation.md @@ -65,6 +65,7 @@ You will need as well to configure your {{ref.acquis}} file to feed {{crowdsec.n * [Go](https://golang.org/doc/install) v1.13+ * `git clone {{crowdsec.url}}` + * [jq](https://stedolan.github.io/jq/download/) Go in {{crowdsec.name}} folder and build the binaries : diff --git a/docs/references/output.md b/docs/references/output.md index 04f1dc409..2a66f30d2 100644 --- a/docs/references/output.md +++ b/docs/references/output.md @@ -62,7 +62,7 @@ config: # in a form of key(string)/value(string) For the plugin sqlite, here is its configuration file: ```yaml name: sqlite -path: /var/lib/crowdsec/plugins/backend/sqlite.so +path: /usr/local/crowdsec/plugins/backend/sqlite.so config: db_path: /var/lib/crowdsec/data/crowdsec.db flush: true diff --git a/mkdocs.yml b/mkdocs.yml index b630cdd7e..b68709b7b 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -129,7 +129,7 @@ extra: plugins: name: blockers url: "https://hub.crowdsec.net/" - binpath: "/var/lib/crowdsec/plugins/" + binpath: "/usr/local/crowdsec/plugins/" configpath: "/etc/crowdsec/plugins/" metabase: name: metabase diff --git a/wizard.sh b/wizard.sh index 557e35fad..8197c6367 100755 --- a/wizard.sh +++ b/wizard.sh @@ -14,12 +14,13 @@ SILENT="false" CROWDSEC_RUN_DIR="/var/run" CROWDSEC_LIB_DIR="/var/lib/crowdsec" +CROWDSEC_USR_DIR="/usr/local/crowdsec" CROWDSEC_DATA_DIR="${CROWDSEC_LIB_DIR}/data" -CROWDSEC_PLUGIN_DIR="${CROWDSEC_LIB_DIR}/plugins" +CROWDSEC_PLUGIN_DIR="${CROWDSEC_USR_DIR}/plugins" CROWDSEC_PLUGIN_BACKEND_DIR="${CROWDSEC_PLUGIN_DIR}/backend" CROWDSEC_DB_PATH="${CROWDSEC_DATA_DIR}/crowdsec.db" -CROWDSEC_CONFIG_PATH="/etc/crowdsec" -CROWDSEC_CONFIG_PATH="${CROWDSEC_CONFIG_PATH}/config" +CROWDSEC_PATH="/etc/crowdsec" +CROWDSEC_CONFIG_PATH="${CROWDSEC_PATH}/config" CROWDSEC_LOG_FILE="/var/log/crowdsec.log" CROWDSEC_BACKEND_FOLDER="/etc/crowdsec/plugins/backend" CSCLI_FOLDER="/etc/crowdsec/config/cscli" @@ -39,7 +40,7 @@ setup_cron_pull() { } -PID_DIR="/var/run" +PID_DIR="${CROWDSEC_RUN_DIR}" SYSTEMD_PATH_FILE="/etc/systemd/system/crowdsec.service" PATTERNS_FOLDER="config/patterns" @@ -340,10 +341,11 @@ uninstall_crowdsec() { systemctl stop crowdsec.service ${CSCLI_BIN} dashboard stop --remove delete_bins - rm -rf ${CROWDSEC_CONFIG_PATH} || echo "" + rm -rf ${CROWDSEC_PATH} || echo "" rm -f ${CROWDSEC_LOG_FILE} || echo "" rm -f ${CROWDSEC_DB_PATH} || echo "" rm -rf ${CROWDSEC_LIB_DIR} || echo "" + rm -rf ${CROWDSEC_USR_DIR} || echo "" rm -f ${SYSTEMD_PATH_FILE} || echo "" log_info "crowdsec successfully uninstalled" } From 1fc83caf5d0518dc53f54bd7542de1ae4a04a1c9 Mon Sep 17 00:00:00 2001 From: erenJag Date: Wed, 27 May 2020 17:35:35 +0200 Subject: [PATCH 10/11] fix lib path --- config/plugins/backend/sqlite.yaml | 2 +- docs/references/output.md | 2 +- wizard.sh | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/config/plugins/backend/sqlite.yaml b/config/plugins/backend/sqlite.yaml index e2009270e..b177b22a5 100644 --- a/config/plugins/backend/sqlite.yaml +++ b/config/plugins/backend/sqlite.yaml @@ -1,5 +1,5 @@ name: sqlite -path: /usr/local/crowdsec/plugins/backend/sqlite.so +path: /usr/local/lib/crowdsec/plugins/backend/sqlite.so config: db_path: /var/lib/crowdsec/data/crowdsec.db flush: true diff --git a/docs/references/output.md b/docs/references/output.md index 2a66f30d2..0214237e7 100644 --- a/docs/references/output.md +++ b/docs/references/output.md @@ -62,7 +62,7 @@ config: # in a form of key(string)/value(string) For the plugin sqlite, here is its configuration file: ```yaml name: sqlite -path: /usr/local/crowdsec/plugins/backend/sqlite.so +path: /usr/local/lib/crowdsec/plugins/backend/sqlite.so config: db_path: /var/lib/crowdsec/data/crowdsec.db flush: true diff --git a/wizard.sh b/wizard.sh index 8197c6367..b97dce8ac 100755 --- a/wizard.sh +++ b/wizard.sh @@ -14,7 +14,7 @@ SILENT="false" CROWDSEC_RUN_DIR="/var/run" CROWDSEC_LIB_DIR="/var/lib/crowdsec" -CROWDSEC_USR_DIR="/usr/local/crowdsec" +CROWDSEC_USR_DIR="/usr/local/lib/crowdsec" CROWDSEC_DATA_DIR="${CROWDSEC_LIB_DIR}/data" CROWDSEC_PLUGIN_DIR="${CROWDSEC_USR_DIR}/plugins" CROWDSEC_PLUGIN_BACKEND_DIR="${CROWDSEC_PLUGIN_DIR}/backend" From b4258b332495f11a9e7115b89e0ca76bc3bedf82 Mon Sep 17 00:00:00 2001 From: "Thibault \"bui\" Koechlin" Date: Wed, 27 May 2020 18:09:23 +0200 Subject: [PATCH 11/11] add support in cscli to switch branches of hub (#43) --- cmd/crowdsec-cli/main.go | 5 ++++- pkg/cwhub/hubMgmt.go | 11 ++++++----- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/cmd/crowdsec-cli/main.go b/cmd/crowdsec-cli/main.go index 189848be6..8c9095f03 100644 --- a/cmd/crowdsec-cli/main.go +++ b/cmd/crowdsec-cli/main.go @@ -117,7 +117,10 @@ API interaction: rootCmd.PersistentFlags().BoolVar(&nfo_lvl, "info", false, "Set logging to info.") rootCmd.PersistentFlags().BoolVar(&wrn_lvl, "warning", false, "Set logging to warning.") rootCmd.PersistentFlags().BoolVar(&err_lvl, "error", false, "Set logging to error.") - + rootCmd.PersistentFlags().StringVar(&cwhub.HubBranch, "branch", "master", "Override hub branch on github") + if err := rootCmd.PersistentFlags().MarkHidden("branch"); err != nil { + log.Fatalf("failed to make branch hidden : %s", err) + } cobra.OnInitialize(initConfig) /*don't sort flags so we can enforce order*/ rootCmd.Flags().SortFlags = false diff --git a/pkg/cwhub/hubMgmt.go b/pkg/cwhub/hubMgmt.go index 29100a167..0a8fbf9e9 100644 --- a/pkg/cwhub/hubMgmt.go +++ b/pkg/cwhub/hubMgmt.go @@ -32,8 +32,9 @@ var Installdir = "/etc/crowdsec/" var Hubdir = "/etc/crowdsec/cscli/hub/" var Cfgdir = "/etc/crowdsec/cscli/" -var RawFileURLTemplate = "https://raw.githubusercontent.com/crowdsecurity/hub/master/%s" -var HUB_INDEX_FILE = ".index.json" +var RawFileURLTemplate = "https://raw.githubusercontent.com/crowdsecurity/hub/%s/%s" +var HubIndexFile = ".index.json" +var HubBranch = "master" type ItemVersion struct { Digest string @@ -406,7 +407,7 @@ func UpdateHubIdx() error { } func DownloadHubIdx() ([]byte, error) { - req, err := http.NewRequest("GET", fmt.Sprintf(RawFileURLTemplate, HUB_INDEX_FILE), nil) + req, err := http.NewRequest("GET", fmt.Sprintf(RawFileURLTemplate, HubBranch, HubIndexFile), nil) if err != nil { log.Errorf("failed request : %s", err) return nil, err @@ -418,7 +419,7 @@ func DownloadHubIdx() ([]byte, error) { } if resp.StatusCode != 200 { log.Errorf("got code %d while requesting %s, abort", resp.StatusCode, - fmt.Sprintf(RawFileURLTemplate, HUB_INDEX_FILE)) + fmt.Sprintf(RawFileURLTemplate, HubBranch, HubIndexFile)) return nil, fmt.Errorf("bad http code") } defer resp.Body.Close() @@ -678,7 +679,7 @@ func DownloadItem(target Item, tdir string, overwrite bool, dataFolder string) ( } //log.Infof("Downloading %s to %s", target.Name, tdir) - req, err := http.NewRequest("GET", fmt.Sprintf(RawFileURLTemplate, target.RemotePath), nil) + req, err := http.NewRequest("GET", fmt.Sprintf(RawFileURLTemplate, HubBranch, target.RemotePath), nil) if err != nil { log.Errorf("%s : request creation failed : %s", target.Name, err) return target, err