Merge branch 'master' into support_data_download

Makefile

@@ -62,6 +62,7 @@ else
 	@echo "Required golang version is $(REQUIRE_GOVERSION). The current one is $(CURRENT_GOVERSION). Exiting.."
 	@exit 1;
 endif
+	@bash ./scripts/build_plugins.sh
 
 
 cscli_static:

cmd/crowdsec-cli/main.go

@@ -117,7 +117,10 @@ API interaction:
 	rootCmd.PersistentFlags().BoolVar(&nfo_lvl, "info", false, "Set logging to info.")
 	rootCmd.PersistentFlags().BoolVar(&wrn_lvl, "warning", false, "Set logging to warning.")
 	rootCmd.PersistentFlags().BoolVar(&err_lvl, "error", false, "Set logging to error.")
-
+	rootCmd.PersistentFlags().StringVar(&cwhub.HubBranch, "branch", "master", "Override hub branch on github")
+	if err := rootCmd.PersistentFlags().MarkHidden("branch"); err != nil {
+		log.Fatalf("failed to make branch hidden : %s", err)
+	}
 	cobra.OnInitialize(initConfig)
 	/*don't sort flags so we can enforce order*/
 	rootCmd.Flags().SortFlags = false

cmd/crowdsec/main.go

@@ -1,7 +1,6 @@
 package main
 
 import (
-	"fmt"
 	"strings"
 
 	"io/ioutil"
@@ -19,7 +18,6 @@ import (
 
 	log "github.com/sirupsen/logrus"
 
-	"gopkg.in/natefinch/lumberjack.v2"
 	"gopkg.in/tomb.v2"
 	"gopkg.in/yaml.v2"
 )
@@ -39,33 +37,6 @@ var (
 	lastProcessedItem time.Time /*keep track of last item timestamp in time-machine. it is used to GC buckets when we dump them.*/
 )
 
-func configureLogger(logMode string, logFolder string, logLevel log.Level) error {
-	/*Configure logs*/
-	if logMode == "file" {
-		log.SetOutput(&lumberjack.Logger{
-			Filename:   logFolder + "/crowdsec.log",
-			MaxSize:    500, //megabytes
-			MaxBackups: 3,
-			MaxAge:     28,   //days
-			Compress:   true, //disabled by default
-		})
-		log.SetFormatter(&log.TextFormatter{TimestampFormat: "02-01-2006 15:04:05", FullTimestamp: true})
-	} else if logMode != "stdout" {
-		return fmt.Errorf("log mode '%s' unknown", logMode)
-	}
-
-	log.Printf("setting loglevel to %s", logLevel)
-	log.SetLevel(logLevel)
-	log.SetFormatter(&log.TextFormatter{FullTimestamp: true})
-	if logLevel >= log.InfoLevel {
-		log.SetFormatter(&log.TextFormatter{TimestampFormat: "02-01-2006 15:04:05", FullTimestamp: true})
-	}
-	if logLevel >= log.DebugLevel {
-		log.SetReportCaller(true)
-	}
-	return nil
-}
-
 func main() {
 	var (
 		err                 error
@@ -92,7 +63,7 @@ func main() {
 		log.Fatalf(err.Error())
 	}
 
-	if err = configureLogger(cConfig.LogMode, cConfig.LogFolder, cConfig.LogLevel); err != nil {
+	if err = types.SetDefaultLoggerConfig(cConfig.LogMode, cConfig.LogFolder, cConfig.LogLevel); err != nil {
 		log.Fatal(err.Error())
 	}
 

config/plugins/backend/sqlite.yaml

@@ -1,5 +1,5 @@
 name: sqlite
-path: /var/lib/crowdsec/plugins/backend/sqlite.so
+path: /usr/local/lib/crowdsec/plugins/backend/sqlite.so
 config:
   db_path: /var/lib/crowdsec/data/crowdsec.db
   flush: true

config/user.yaml

@@ -8,9 +8,9 @@ log_mode: stdout
 log_level: info
 profiling: false
 sqlite_path: ${DATA}/crowdsec.db
-apimode: true
+apimode: false
 daemon: false
-prometheus: true
+prometheus: false
 #for prometheus agent / golang debugging
 http_listen: 127.0.0.1:6060
 plugin:

docs/getting_started/installation.md

@@ -65,6 +65,7 @@ You will need as well to configure your {{ref.acquis}} file to feed {{crowdsec.n
     
     * [Go](https://golang.org/doc/install) v1.13+
     * `git clone {{crowdsec.url}}`
+    * [jq](https://stedolan.github.io/jq/download/)
 
 
 Go in {{crowdsec.name}} folder and build the binaries :

docs/references/output.md

@@ -62,7 +62,7 @@ config: <plugin_config> # in a form of key(string)/value(string)
 For the plugin sqlite, here is its configuration file:
 ```yaml
 name: sqlite
-path: /var/lib/crowdsec/plugins/backend/sqlite.so
+path: /usr/local/lib/crowdsec/plugins/backend/sqlite.so
 config:
   db_path: /var/lib/crowdsec/data/crowdsec.db
   flush: true

mkdocs.yml

@@ -129,7 +129,7 @@ extra:
     plugins:
         name: blockers
         url: "https://hub.crowdsec.net/"
-        binpath: "/var/lib/crowdsec/plugins/"
+        binpath: "/usr/local/crowdsec/plugins/"
         configpath: "/etc/crowdsec/plugins/"
     metabase:
         name: metabase

pkg/cwhub/hubMgmt.go

@@ -35,8 +35,9 @@ var Installdir = "/etc/crowdsec/"
 var Hubdir = "/etc/crowdsec/cscli/hub/"
 var Cfgdir = "/etc/crowdsec/cscli/"
 
-var RawFileURLTemplate = "https://raw.githubusercontent.com/crowdsecurity/hub/master/%s"
-var HUB_INDEX_FILE = ".index.json"
+var RawFileURLTemplate = "https://raw.githubusercontent.com/crowdsecurity/hub/%s/%s"
+var HubIndexFile = ".index.json"
+var HubBranch = "master"
 
 type ItemVersion struct {
 	Digest     string
@@ -409,7 +410,7 @@ func UpdateHubIdx() error {
 }
 
 func DownloadHubIdx() ([]byte, error) {
-	req, err := http.NewRequest("GET", fmt.Sprintf(RawFileURLTemplate, HUB_INDEX_FILE), nil)
+	req, err := http.NewRequest("GET", fmt.Sprintf(RawFileURLTemplate, HubBranch, HubIndexFile), nil)
 	if err != nil {
 		log.Errorf("failed request : %s", err)
 		return nil, err
@@ -421,7 +422,7 @@ func DownloadHubIdx() ([]byte, error) {
 	}
 	if resp.StatusCode != 200 {
 		log.Errorf("got code %d while requesting %s, abort", resp.StatusCode,
-			fmt.Sprintf(RawFileURLTemplate, HUB_INDEX_FILE))
+			fmt.Sprintf(RawFileURLTemplate, HubBranch, HubIndexFile))
 		return nil, fmt.Errorf("bad http code")
 	}
 	defer resp.Body.Close()
@@ -687,7 +688,7 @@ func DownloadItem(target Item, tdir string, overwrite bool, dataFolder string) (
 	}
 
 	//log.Infof("Downloading %s to %s", target.Name, tdir)
-	req, err := http.NewRequest("GET", fmt.Sprintf(RawFileURLTemplate, target.RemotePath), nil)
+	req, err := http.NewRequest("GET", fmt.Sprintf(RawFileURLTemplate, HubBranch, target.RemotePath), nil)
 	if err != nil {
 		log.Errorf("%s : request creation failed : %s", target.Name, err)
 		return target, err

pkg/leakybucket/manager.go

@@ -151,7 +151,7 @@ func LoadBuckets(files []string, dataFolder string) ([]BucketFactory, chan types
 			}
 			//check compat
 			if g.FormatVersion == "" {
-				log.Warningf("no version in %s : %s, assuming '1.0'", g.Name, f)
+				log.Debugf("no version in %s : %s, assuming '1.0'", g.Name, f)
 				g.FormatVersion = "1.0"
 			}
 			ok, err := cwversion.Statisfies(g.FormatVersion, cwversion.Constraint_scenario)
@@ -195,8 +195,10 @@ func LoadBucketDir(dir string, dataFolder string) ([]BucketFactory, chan types.E
 func LoadBucket(g *BucketFactory, dataFolder string) error {
 	var err error
 	if g.Debug {
-		var clog = log.New()
-		clog.SetFormatter(&log.TextFormatter{FullTimestamp: true})
+		var clog = logrus.New()
+		if err := types.ConfigureLogger(clog); err != nil {
+			log.Fatalf("While creating bucket-specific logger : %s", err)
+		}
 		clog.SetLevel(log.DebugLevel)
 		g.logger = clog.WithFields(log.Fields{
 			"cfg":  g.BucketName,

pkg/parser/node.go

@@ -253,10 +253,15 @@ func (n *Node) process(p *types.Event, ctx UnixParserCtx) (bool, error) {
 				//return false, nil
 			}
 		}
-
+		var groklabel string
+		if n.Grok.RegexpName == "" {
+			groklabel = fmt.Sprintf("%5.5s...", n.Grok.RegexpValue)
+		} else {
+			groklabel = n.Grok.RegexpName
+		}
 		grok := n.Grok.RunTimeRegexp.Parse(gstr)
 		if len(grok) > 0 {
-			clog.Debugf("+ Grok '%s' returned %d entries to merge in Parsed", n.Grok.RegexpName, len(grok))
+			clog.Debugf("+ Grok '%s' returned %d entries to merge in Parsed", groklabel, len(grok))
 			//We managed to grok stuff, merged into parse
 			for k, v := range grok {
 				clog.Debugf("\t.Parsed['%s'] = '%s'", k, v)
@@ -269,7 +274,7 @@ func (n *Node) process(p *types.Event, ctx UnixParserCtx) (bool, error) {
 			}
 		} else {
 			//grok failed, node failed
-			clog.Debugf("+ Grok '%s' didn't return data on '%s'", n.Grok.RegexpName, gstr)
+			clog.Debugf("+ Grok '%s' didn't return data on '%s'", groklabel, gstr)
 			//clog.Tracef("on '%s'", gstr)
 			NodeState = false
 		}
@@ -337,6 +342,9 @@ func (n *Node) compile(pctx *UnixParserCtx) error {
 	that will be used only for processing this node ;) */
 	if n.Debug {
 		var clog = logrus.New()
+		if err := types.ConfigureLogger(clog); err != nil {
+			log.Fatalf("While creating bucket-specific logger : %s", err)
+		}
 		clog.SetLevel(log.DebugLevel)
 		n.logger = clog.WithFields(log.Fields{
 			"id": n.rn,
@@ -412,6 +420,9 @@ func (n *Node) compile(pctx *UnixParserCtx) error {
 	/* compile leafs if present */
 	if len(n.SuccessNodes) > 0 {
 		for idx := range n.SuccessNodes {
+			if n.SuccessNodes[idx].Name == "" {
+				n.SuccessNodes[idx].Name = fmt.Sprintf("child-%s", n.Name)
+			}
 			/*propagate debug/stats to child nodes*/
 			if !n.SuccessNodes[idx].Debug && n.Debug {
 				n.SuccessNodes[idx].Debug = true

pkg/parser/stage.go

@@ -81,7 +81,7 @@ func LoadStages(stageFiles []Stagefile, pctx *UnixParserCtx) ([]Node, error) {
 
 			//check for empty bucket
 			if node.Name == "" && node.Description == "" && node.Author == "" {
-				log.Infof("Node has no name,author or description. Skipping.")
+				log.Infof("Node in %s has no name,author or description. Skipping.", stageFile.Filename)
 				continue
 			}
 			//check compat

pkg/types/utils.go

@@ -2,7 +2,12 @@ package types
 
 import (
 	"encoding/binary"
+	"fmt"
+	"io"
 	"net"
+
+	log "github.com/sirupsen/logrus"
+	"gopkg.in/natefinch/lumberjack.v2"
 )
 
 func IP2Int(ip net.IP) uint32 {
@@ -40,3 +45,51 @@ func LastAddress(n *net.IPNet) net.IP {
 		ip[2]|^n.Mask[2],
 		ip[3]|^n.Mask[3])
 }
+
+var logFormatter log.Formatter
+var logOutput io.Writer
+var logLevel log.Level
+var logReportCaller bool
+
+func SetDefaultLoggerConfig(cfgMode string, cfgFolder string, cfgLevel log.Level) error {
+
+	/*Configure logs*/
+	if cfgMode == "file" {
+		logOutput = &lumberjack.Logger{
+			Filename:   cfgFolder + "/crowdsec.log",
+			MaxSize:    500, //megabytes
+			MaxBackups: 3,
+			MaxAge:     28,   //days
+			Compress:   true, //disabled by default
+		}
+		log.SetOutput(logOutput)
+	} else if cfgMode != "stdout" {
+		return fmt.Errorf("log mode '%s' unknown", cfgMode)
+	}
+	logLevel = cfgLevel
+	log.SetLevel(logLevel)
+	if logLevel >= log.InfoLevel {
+		logFormatter = &log.TextFormatter{TimestampFormat: "02-01-2006 15:04:05", FullTimestamp: true}
+		log.SetFormatter(logFormatter)
+	}
+	if logLevel >= log.DebugLevel {
+		logReportCaller = true
+		log.SetReportCaller(true)
+	}
+	return nil
+}
+
+func ConfigureLogger(clog *log.Logger) error {
+	/*Configure logs*/
+	if logOutput != nil {
+		clog.SetOutput(logOutput)
+	}
+	if logReportCaller {
+		clog.SetReportCaller(true)
+	}
+	if logFormatter != nil {
+		clog.SetFormatter(logFormatter)
+	}
+	clog.SetLevel(logLevel)
+	return nil
+}

wizard.sh

@@ -14,12 +14,13 @@ SILENT="false"
 
 CROWDSEC_RUN_DIR="/var/run"
 CROWDSEC_LIB_DIR="/var/lib/crowdsec"
+CROWDSEC_USR_DIR="/usr/local/lib/crowdsec"
 CROWDSEC_DATA_DIR="${CROWDSEC_LIB_DIR}/data"
-CROWDSEC_PLUGIN_DIR="${CROWDSEC_LIB_DIR}/plugins"
+CROWDSEC_PLUGIN_DIR="${CROWDSEC_USR_DIR}/plugins"
 CROWDSEC_PLUGIN_BACKEND_DIR="${CROWDSEC_PLUGIN_DIR}/backend"
 CROWDSEC_DB_PATH="${CROWDSEC_DATA_DIR}/crowdsec.db"
-CROWDSEC_CONFIG_PATH="/etc/crowdsec"
-CROWDSEC_CONFIG_PATH="${CROWDSEC_CONFIG_PATH}/config"
+CROWDSEC_PATH="/etc/crowdsec"
+CROWDSEC_CONFIG_PATH="${CROWDSEC_PATH}/config"
 CROWDSEC_LOG_FILE="/var/log/crowdsec.log"
 CROWDSEC_BACKEND_FOLDER="/etc/crowdsec/plugins/backend"
 CSCLI_FOLDER="/etc/crowdsec/config/cscli"
@@ -39,7 +40,7 @@ setup_cron_pull() {
 }
 
 
-PID_DIR="/var/run"
+PID_DIR="${CROWDSEC_RUN_DIR}"
 SYSTEMD_PATH_FILE="/etc/systemd/system/crowdsec.service"
 
 PATTERNS_FOLDER="config/patterns"
@@ -340,10 +341,11 @@ uninstall_crowdsec() {
     systemctl stop crowdsec.service
     ${CSCLI_BIN} dashboard stop --remove
     delete_bins
-    rm -rf ${CROWDSEC_CONFIG_PATH} || echo ""
+    rm -rf ${CROWDSEC_PATH} || echo ""
     rm -f ${CROWDSEC_LOG_FILE} || echo ""
     rm -f ${CROWDSEC_DB_PATH} || echo ""
     rm -rf ${CROWDSEC_LIB_DIR} || echo ""
+    rm -rf ${CROWDSEC_USR_DIR} || echo ""
     rm -f ${SYSTEMD_PATH_FILE} || echo ""
     log_info "crowdsec successfully uninstalled"
 }