diff --git a/Makefile b/Makefile index 39c72e6b8..353cc73d4 100644 --- a/Makefile +++ b/Makefile @@ -62,6 +62,7 @@ else @echo "Required golang version is $(REQUIRE_GOVERSION). The current one is $(CURRENT_GOVERSION). Exiting.." @exit 1; endif + @bash ./scripts/build_plugins.sh cscli_static: diff --git a/cmd/crowdsec-cli/main.go b/cmd/crowdsec-cli/main.go index 189848be6..8c9095f03 100644 --- a/cmd/crowdsec-cli/main.go +++ b/cmd/crowdsec-cli/main.go @@ -117,7 +117,10 @@ API interaction: rootCmd.PersistentFlags().BoolVar(&nfo_lvl, "info", false, "Set logging to info.") rootCmd.PersistentFlags().BoolVar(&wrn_lvl, "warning", false, "Set logging to warning.") rootCmd.PersistentFlags().BoolVar(&err_lvl, "error", false, "Set logging to error.") - + rootCmd.PersistentFlags().StringVar(&cwhub.HubBranch, "branch", "master", "Override hub branch on github") + if err := rootCmd.PersistentFlags().MarkHidden("branch"); err != nil { + log.Fatalf("failed to make branch hidden : %s", err) + } cobra.OnInitialize(initConfig) /*don't sort flags so we can enforce order*/ rootCmd.Flags().SortFlags = false diff --git a/cmd/crowdsec/main.go b/cmd/crowdsec/main.go index bad79c543..cc9bbf85d 100644 --- a/cmd/crowdsec/main.go +++ b/cmd/crowdsec/main.go @@ -1,7 +1,6 @@ package main import ( - "fmt" "strings" "io/ioutil" @@ -19,7 +18,6 @@ import ( log "github.com/sirupsen/logrus" - "gopkg.in/natefinch/lumberjack.v2" "gopkg.in/tomb.v2" "gopkg.in/yaml.v2" ) @@ -39,33 +37,6 @@ var ( lastProcessedItem time.Time /*keep track of last item timestamp in time-machine. it is used to GC buckets when we dump them.*/ ) -func configureLogger(logMode string, logFolder string, logLevel log.Level) error { - /*Configure logs*/ - if logMode == "file" { - log.SetOutput(&lumberjack.Logger{ - Filename: logFolder + "/crowdsec.log", - MaxSize: 500, //megabytes - MaxBackups: 3, - MaxAge: 28, //days - Compress: true, //disabled by default - }) - log.SetFormatter(&log.TextFormatter{TimestampFormat: "02-01-2006 15:04:05", FullTimestamp: true}) - } else if logMode != "stdout" { - return fmt.Errorf("log mode '%s' unknown", logMode) - } - - log.Printf("setting loglevel to %s", logLevel) - log.SetLevel(logLevel) - log.SetFormatter(&log.TextFormatter{FullTimestamp: true}) - if logLevel >= log.InfoLevel { - log.SetFormatter(&log.TextFormatter{TimestampFormat: "02-01-2006 15:04:05", FullTimestamp: true}) - } - if logLevel >= log.DebugLevel { - log.SetReportCaller(true) - } - return nil -} - func main() { var ( err error @@ -92,7 +63,7 @@ func main() { log.Fatalf(err.Error()) } - if err = configureLogger(cConfig.LogMode, cConfig.LogFolder, cConfig.LogLevel); err != nil { + if err = types.SetDefaultLoggerConfig(cConfig.LogMode, cConfig.LogFolder, cConfig.LogLevel); err != nil { log.Fatal(err.Error()) } diff --git a/config/plugins/backend/sqlite.yaml b/config/plugins/backend/sqlite.yaml index c4250c15f..b177b22a5 100644 --- a/config/plugins/backend/sqlite.yaml +++ b/config/plugins/backend/sqlite.yaml @@ -1,5 +1,5 @@ name: sqlite -path: /var/lib/crowdsec/plugins/backend/sqlite.so +path: /usr/local/lib/crowdsec/plugins/backend/sqlite.so config: db_path: /var/lib/crowdsec/data/crowdsec.db flush: true diff --git a/config/user.yaml b/config/user.yaml index addbdf726..8d3f83c6c 100644 --- a/config/user.yaml +++ b/config/user.yaml @@ -8,9 +8,9 @@ log_mode: stdout log_level: info profiling: false sqlite_path: ${DATA}/crowdsec.db -apimode: true +apimode: false daemon: false -prometheus: true +prometheus: false #for prometheus agent / golang debugging http_listen: 127.0.0.1:6060 plugin: diff --git a/docs/getting_started/installation.md b/docs/getting_started/installation.md index a7a5c1c01..627687638 100644 --- a/docs/getting_started/installation.md +++ b/docs/getting_started/installation.md @@ -65,6 +65,7 @@ You will need as well to configure your {{ref.acquis}} file to feed {{crowdsec.n * [Go](https://golang.org/doc/install) v1.13+ * `git clone {{crowdsec.url}}` + * [jq](https://stedolan.github.io/jq/download/) Go in {{crowdsec.name}} folder and build the binaries : diff --git a/docs/references/output.md b/docs/references/output.md index 04f1dc409..0214237e7 100644 --- a/docs/references/output.md +++ b/docs/references/output.md @@ -62,7 +62,7 @@ config: # in a form of key(string)/value(string) For the plugin sqlite, here is its configuration file: ```yaml name: sqlite -path: /var/lib/crowdsec/plugins/backend/sqlite.so +path: /usr/local/lib/crowdsec/plugins/backend/sqlite.so config: db_path: /var/lib/crowdsec/data/crowdsec.db flush: true diff --git a/mkdocs.yml b/mkdocs.yml index b630cdd7e..b68709b7b 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -129,7 +129,7 @@ extra: plugins: name: blockers url: "https://hub.crowdsec.net/" - binpath: "/var/lib/crowdsec/plugins/" + binpath: "/usr/local/crowdsec/plugins/" configpath: "/etc/crowdsec/plugins/" metabase: name: metabase diff --git a/pkg/cwhub/hubMgmt.go b/pkg/cwhub/hubMgmt.go index daa9ea7b4..b9957d3bc 100644 --- a/pkg/cwhub/hubMgmt.go +++ b/pkg/cwhub/hubMgmt.go @@ -35,8 +35,9 @@ var Installdir = "/etc/crowdsec/" var Hubdir = "/etc/crowdsec/cscli/hub/" var Cfgdir = "/etc/crowdsec/cscli/" -var RawFileURLTemplate = "https://raw.githubusercontent.com/crowdsecurity/hub/master/%s" -var HUB_INDEX_FILE = ".index.json" +var RawFileURLTemplate = "https://raw.githubusercontent.com/crowdsecurity/hub/%s/%s" +var HubIndexFile = ".index.json" +var HubBranch = "master" type ItemVersion struct { Digest string @@ -409,7 +410,7 @@ func UpdateHubIdx() error { } func DownloadHubIdx() ([]byte, error) { - req, err := http.NewRequest("GET", fmt.Sprintf(RawFileURLTemplate, HUB_INDEX_FILE), nil) + req, err := http.NewRequest("GET", fmt.Sprintf(RawFileURLTemplate, HubBranch, HubIndexFile), nil) if err != nil { log.Errorf("failed request : %s", err) return nil, err @@ -421,7 +422,7 @@ func DownloadHubIdx() ([]byte, error) { } if resp.StatusCode != 200 { log.Errorf("got code %d while requesting %s, abort", resp.StatusCode, - fmt.Sprintf(RawFileURLTemplate, HUB_INDEX_FILE)) + fmt.Sprintf(RawFileURLTemplate, HubBranch, HubIndexFile)) return nil, fmt.Errorf("bad http code") } defer resp.Body.Close() @@ -687,7 +688,7 @@ func DownloadItem(target Item, tdir string, overwrite bool, dataFolder string) ( } //log.Infof("Downloading %s to %s", target.Name, tdir) - req, err := http.NewRequest("GET", fmt.Sprintf(RawFileURLTemplate, target.RemotePath), nil) + req, err := http.NewRequest("GET", fmt.Sprintf(RawFileURLTemplate, HubBranch, target.RemotePath), nil) if err != nil { log.Errorf("%s : request creation failed : %s", target.Name, err) return target, err diff --git a/pkg/leakybucket/manager.go b/pkg/leakybucket/manager.go index c73eab128..59f7874c1 100644 --- a/pkg/leakybucket/manager.go +++ b/pkg/leakybucket/manager.go @@ -151,7 +151,7 @@ func LoadBuckets(files []string, dataFolder string) ([]BucketFactory, chan types } //check compat if g.FormatVersion == "" { - log.Warningf("no version in %s : %s, assuming '1.0'", g.Name, f) + log.Debugf("no version in %s : %s, assuming '1.0'", g.Name, f) g.FormatVersion = "1.0" } ok, err := cwversion.Statisfies(g.FormatVersion, cwversion.Constraint_scenario) @@ -195,8 +195,10 @@ func LoadBucketDir(dir string, dataFolder string) ([]BucketFactory, chan types.E func LoadBucket(g *BucketFactory, dataFolder string) error { var err error if g.Debug { - var clog = log.New() - clog.SetFormatter(&log.TextFormatter{FullTimestamp: true}) + var clog = logrus.New() + if err := types.ConfigureLogger(clog); err != nil { + log.Fatalf("While creating bucket-specific logger : %s", err) + } clog.SetLevel(log.DebugLevel) g.logger = clog.WithFields(log.Fields{ "cfg": g.BucketName, diff --git a/pkg/parser/node.go b/pkg/parser/node.go index 2411b76f6..6f504a33f 100644 --- a/pkg/parser/node.go +++ b/pkg/parser/node.go @@ -253,10 +253,15 @@ func (n *Node) process(p *types.Event, ctx UnixParserCtx) (bool, error) { //return false, nil } } - + var groklabel string + if n.Grok.RegexpName == "" { + groklabel = fmt.Sprintf("%5.5s...", n.Grok.RegexpValue) + } else { + groklabel = n.Grok.RegexpName + } grok := n.Grok.RunTimeRegexp.Parse(gstr) if len(grok) > 0 { - clog.Debugf("+ Grok '%s' returned %d entries to merge in Parsed", n.Grok.RegexpName, len(grok)) + clog.Debugf("+ Grok '%s' returned %d entries to merge in Parsed", groklabel, len(grok)) //We managed to grok stuff, merged into parse for k, v := range grok { clog.Debugf("\t.Parsed['%s'] = '%s'", k, v) @@ -269,7 +274,7 @@ func (n *Node) process(p *types.Event, ctx UnixParserCtx) (bool, error) { } } else { //grok failed, node failed - clog.Debugf("+ Grok '%s' didn't return data on '%s'", n.Grok.RegexpName, gstr) + clog.Debugf("+ Grok '%s' didn't return data on '%s'", groklabel, gstr) //clog.Tracef("on '%s'", gstr) NodeState = false } @@ -337,6 +342,9 @@ func (n *Node) compile(pctx *UnixParserCtx) error { that will be used only for processing this node ;) */ if n.Debug { var clog = logrus.New() + if err := types.ConfigureLogger(clog); err != nil { + log.Fatalf("While creating bucket-specific logger : %s", err) + } clog.SetLevel(log.DebugLevel) n.logger = clog.WithFields(log.Fields{ "id": n.rn, @@ -412,6 +420,9 @@ func (n *Node) compile(pctx *UnixParserCtx) error { /* compile leafs if present */ if len(n.SuccessNodes) > 0 { for idx := range n.SuccessNodes { + if n.SuccessNodes[idx].Name == "" { + n.SuccessNodes[idx].Name = fmt.Sprintf("child-%s", n.Name) + } /*propagate debug/stats to child nodes*/ if !n.SuccessNodes[idx].Debug && n.Debug { n.SuccessNodes[idx].Debug = true diff --git a/pkg/parser/stage.go b/pkg/parser/stage.go index d5fb65295..37bdb9baf 100644 --- a/pkg/parser/stage.go +++ b/pkg/parser/stage.go @@ -81,7 +81,7 @@ func LoadStages(stageFiles []Stagefile, pctx *UnixParserCtx) ([]Node, error) { //check for empty bucket if node.Name == "" && node.Description == "" && node.Author == "" { - log.Infof("Node has no name,author or description. Skipping.") + log.Infof("Node in %s has no name,author or description. Skipping.", stageFile.Filename) continue } //check compat diff --git a/pkg/types/utils.go b/pkg/types/utils.go index d71dced6e..a5f8fded9 100644 --- a/pkg/types/utils.go +++ b/pkg/types/utils.go @@ -2,7 +2,12 @@ package types import ( "encoding/binary" + "fmt" + "io" "net" + + log "github.com/sirupsen/logrus" + "gopkg.in/natefinch/lumberjack.v2" ) func IP2Int(ip net.IP) uint32 { @@ -40,3 +45,51 @@ func LastAddress(n *net.IPNet) net.IP { ip[2]|^n.Mask[2], ip[3]|^n.Mask[3]) } + +var logFormatter log.Formatter +var logOutput io.Writer +var logLevel log.Level +var logReportCaller bool + +func SetDefaultLoggerConfig(cfgMode string, cfgFolder string, cfgLevel log.Level) error { + + /*Configure logs*/ + if cfgMode == "file" { + logOutput = &lumberjack.Logger{ + Filename: cfgFolder + "/crowdsec.log", + MaxSize: 500, //megabytes + MaxBackups: 3, + MaxAge: 28, //days + Compress: true, //disabled by default + } + log.SetOutput(logOutput) + } else if cfgMode != "stdout" { + return fmt.Errorf("log mode '%s' unknown", cfgMode) + } + logLevel = cfgLevel + log.SetLevel(logLevel) + if logLevel >= log.InfoLevel { + logFormatter = &log.TextFormatter{TimestampFormat: "02-01-2006 15:04:05", FullTimestamp: true} + log.SetFormatter(logFormatter) + } + if logLevel >= log.DebugLevel { + logReportCaller = true + log.SetReportCaller(true) + } + return nil +} + +func ConfigureLogger(clog *log.Logger) error { + /*Configure logs*/ + if logOutput != nil { + clog.SetOutput(logOutput) + } + if logReportCaller { + clog.SetReportCaller(true) + } + if logFormatter != nil { + clog.SetFormatter(logFormatter) + } + clog.SetLevel(logLevel) + return nil +} diff --git a/wizard.sh b/wizard.sh index 557e35fad..b97dce8ac 100755 --- a/wizard.sh +++ b/wizard.sh @@ -14,12 +14,13 @@ SILENT="false" CROWDSEC_RUN_DIR="/var/run" CROWDSEC_LIB_DIR="/var/lib/crowdsec" +CROWDSEC_USR_DIR="/usr/local/lib/crowdsec" CROWDSEC_DATA_DIR="${CROWDSEC_LIB_DIR}/data" -CROWDSEC_PLUGIN_DIR="${CROWDSEC_LIB_DIR}/plugins" +CROWDSEC_PLUGIN_DIR="${CROWDSEC_USR_DIR}/plugins" CROWDSEC_PLUGIN_BACKEND_DIR="${CROWDSEC_PLUGIN_DIR}/backend" CROWDSEC_DB_PATH="${CROWDSEC_DATA_DIR}/crowdsec.db" -CROWDSEC_CONFIG_PATH="/etc/crowdsec" -CROWDSEC_CONFIG_PATH="${CROWDSEC_CONFIG_PATH}/config" +CROWDSEC_PATH="/etc/crowdsec" +CROWDSEC_CONFIG_PATH="${CROWDSEC_PATH}/config" CROWDSEC_LOG_FILE="/var/log/crowdsec.log" CROWDSEC_BACKEND_FOLDER="/etc/crowdsec/plugins/backend" CSCLI_FOLDER="/etc/crowdsec/config/cscli" @@ -39,7 +40,7 @@ setup_cron_pull() { } -PID_DIR="/var/run" +PID_DIR="${CROWDSEC_RUN_DIR}" SYSTEMD_PATH_FILE="/etc/systemd/system/crowdsec.service" PATTERNS_FOLDER="config/patterns" @@ -340,10 +341,11 @@ uninstall_crowdsec() { systemctl stop crowdsec.service ${CSCLI_BIN} dashboard stop --remove delete_bins - rm -rf ${CROWDSEC_CONFIG_PATH} || echo "" + rm -rf ${CROWDSEC_PATH} || echo "" rm -f ${CROWDSEC_LOG_FILE} || echo "" rm -f ${CROWDSEC_DB_PATH} || echo "" rm -rf ${CROWDSEC_LIB_DIR} || echo "" + rm -rf ${CROWDSEC_USR_DIR} || echo "" rm -f ${SYSTEMD_PATH_FILE} || echo "" log_info "crowdsec successfully uninstalled" }