Merge branch 'master' into sqlite_flush

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,33 @@
+---
+name: Bug report
+about: Create a report to help us improve
+title: Bug/
+labels: bug
+assignees: ''
+
+---
+
+Please, start your issue name (after `Bug`) with the component name impacted by this feature request and a small description of the Bug. Example: `Bug/cscli: issue with ....` and remove this line :)
+
+**Describe the bug**
+A clear and concise description of what the bug is.
+
+**To Reproduce**
+Steps to reproduce the behavior:
+1. Go to '...'
+2. Click on '....'
+3. Scroll down to '....'
+4. See error
+
+**Expected behavior**
+A clear and concise description of what you expected to happen.
+
+**Screenshots**
+If applicable, add screenshots to help explain your problem.
+
+**Technical Information (please complete the following information):**
+ - OS: [e.g. Ubuntu, Redhat ..]
+ - Version [e.g. v0.2.0, v0.1.5 ..]
+
+**Additional context**
+Add any other context about the problem here.

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,25 @@
+---
+name: Feature request
+about: Suggest an idea for this project
+title: Improvment/
+labels: enhancement
+assignees: ''
+
+---
+
+Please, start your issue name (after `improvment`) with the component name impacted by this feature request and a small description of the FR. Example: `Improvment/cscli: add this feature ....` and remove this line :)
+
+**Is your feature request related to a problem? Please describe.**
+A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
+
+**Describe the solution you'd like**
+A clear and concise description of what you want to happen.
+
+**Describe alternatives you've considered** (Optional) 
+A clear and concise description of any alternative solutions or features you've considered.
+
+**Example of what you imagine**
+If applicable, add an example of what you would expect from this feature request.
+
+**Additional context**
+Add any other context or screenshots about the feature request here.

.github/workflows/hub-ci.yml

@@ -43,6 +43,6 @@ jobs:
       run: |
         for i in `find ./tests -mindepth 1 -maxdepth 1 -type d` ; do 
           echo "::group::Test-${i}" ; 
-          ./main $i || (echo "::error file=${i}::Failed test for ${i}" ; diff ${i}"/results.yaml" ${i}"/results.yaml.fail") ;
+          ./main -c dev.yaml $i || (echo "::error file=${i}::Failed test for ${i}" ; diff ${i}"/results.json" ${i}"/results.json.fail") ;
           echo "::endgroup::" ;
         done ;

cmd/crowdsec-cli/api.go

@@ -3,6 +3,7 @@ package main
 import (
 	"encoding/json"
 	"fmt"
+	"io/ioutil"
 	"math/rand"
 	"path"
 	"strings"
@@ -26,9 +27,13 @@ var (
 )
 
 var (
+	userID    string // for flag parsing
+	outputCTX *outputs.Output
+)
+
+const (
+	uuid          = "/proc/sys/kernel/random/uuid"
 	apiConfigFile = "api.yaml"
-	userID        string // for flag parsing
-	outputCTX     *outputs.Output
 )
 
 func dumpCredentials() error {
@@ -177,7 +182,15 @@ cscli api credentials   # Display your API credentials
 		Run: func(cmd *cobra.Command, args []string) {
 			id, err := machineid.ID()
 			if err != nil {
-				log.Fatalf("failed to get machine id: %s", err)
+				log.Debugf("failed to get machine-id with usual files : %s", err)
+			}
+			if id == "" || err != nil {
+				bID, err := ioutil.ReadFile(uuid)
+				if err != nil {
+					log.Fatalf("can'get a valid machine_id")
+				}
+				id = string(bID)
+				id = strings.ReplaceAll(id, "-", "")[:32]
 			}
 			password := generatePassword()
 
@@ -214,8 +227,17 @@ cscli api credentials   # Display your API credentials
 		Run: func(cmd *cobra.Command, args []string) {
 			id, err := machineid.ID()
 			if err != nil {
-				log.Fatalf("failed to get machine id: %s", err)
+				log.Debugf("failed to get machine-id with usual files : %s", err)
 			}
+			if id == "" || err != nil {
+				bID, err := ioutil.ReadFile(uuid)
+				if err != nil {
+					log.Fatalf("can'get a valid machine_id")
+				}
+				id = string(bID)
+				id = strings.ReplaceAll(id, "-", "")[:32]
+			}
+
 			password := generatePassword()
 			if err := outputCTX.API.ResetPassword(id, password); err != nil {
 				log.Fatalf(err.Error())

cmd/crowdsec-cli/backup-restore.go

@@ -416,7 +416,7 @@ cscli backup restore ./my-backup`,
 			}
 			outputCTX, err = outputs.NewOutput(&outputConfig)
 			if err != nil {
-				log.Fatalf("Failed to load output plugins")
+				log.Fatalf("Failed to load output plugins : %v", err)
 			}
 			if err := cwhub.GetHubIdx(); err != nil {
 				log.Fatalf("Failed to get Hub index : %v", err)
@@ -458,7 +458,7 @@ cscli backup restore ./my-backup`,
 			}
 			outputCTX, err = outputs.NewOutput(&outputConfig)
 			if err != nil {
-				log.Fatalf("Failed to load output plugins")
+				log.Fatalf("Failed to load output plugins : %v", err)
 			}
 
 			if err := cwhub.GetHubIdx(); err != nil {

cmd/crowdsec-cli/ban.go

@@ -96,7 +96,12 @@ func BanList() error {
 	if err != nil {
 		return fmt.Errorf("unable to get records from sqlite : %v", err)
 	}
-	if config.output == "json" {
+	if config.output == "raw" {
+		fmt.Printf("source,ip,reason,bans,action,country,as,events_count,expiration\n")
+		for _, rm := range ret {
+			fmt.Printf("%s,%s,%s,%s,%s,%s,%s,%s,%s\n", rm["source"], rm["iptext"], rm["reason"], rm["bancount"], rm["action"], rm["cn"], rm["as"], rm["events_count"], rm["until"])
+		}
+	} else if config.output == "json" {
 		x, _ := json.MarshalIndent(ret, "", " ")
 		fmt.Printf("%s", string(x))
 	} else if config.output == "human" {

cmd/crowdsec/main.go

@@ -214,27 +214,24 @@ func StartProcessingRoutines(cConfig *csconfig.CrowdSec) (chan types.Event, erro
 		})
 	}
 
-	for i := 0; i < cConfig.NbParsers; i++ {
-		bucketsTomb.Go(func() error {
-			err := runPour(inputEventChan, holders, buckets)
-			if err != nil {
-				log.Errorf("runPour error : %s", err)
-				return err
-			}
-			return nil
-		})
-	}
+	bucketsTomb.Go(func() error {
+		err := runPour(inputEventChan, holders, buckets)
+		if err != nil {
+			log.Errorf("runPour error : %s", err)
+			return err
+		}
+		return nil
+	})
+
+	outputsTomb.Go(func() error {
+		err := runOutput(inputEventChan, outputEventChan, holders, buckets, *postOverflowCTX, postOverflowNodes, outputProfiles, OutputRunner)
+		if err != nil {
+			log.Errorf("runPour error : %s", err)
+			return err
+		}
+		return nil
+	})
 
-	for i := 0; i < cConfig.NbParsers; i++ {
-		outputsTomb.Go(func() error {
-			err := runOutput(inputEventChan, outputEventChan, holders, buckets, *postOverflowCTX, postOverflowNodes, outputProfiles, OutputRunner)
-			if err != nil {
-				log.Errorf("runPour error : %s", err)
-				return err
-			}
-			return nil
-		})
-	}
 	return inputLineChan, nil
 }
 

config/acquis.yaml

@@ -8,7 +8,6 @@ labels:
 filenames:
  - /var/log/auth.log
  - /var/log/syslog
-#no need to set the prog_name, syslog format contains this info
 labels:
   type: syslog
 ---

docs/getting_started/glossary.md

@@ -41,7 +41,7 @@ filenames: #a list of file or regexp to read from (supports regular expressions)
   - /var/log/nginx/https_access.log
   - /var/log/nginx/error.log
 labels:
-  prog_name: nginx
+  type: nginx
 ---
 filenames:
   - /var/log/auth.log
@@ -49,7 +49,7 @@ labels:
   type: syslog
 ```
 
-The `labels` part is here to tag the incoming logs with a type. `labels.prog_name` and `labels.type` are used by the parsers to know which logs to process.
+The `labels` part is here to tag the incoming logs with a type. `labels.type` are used by the parsers to know which logs to process.
 
 ### **Parser**
 

docs/write_configurations/acquisition.md

@@ -6,14 +6,14 @@ There is two option:
  - Your logs are wrote from a syslog server, so you just have to install the [syslog parser](https://master.d3padiiorjhf1k.amplifyapp.com/author/crowdsecurity/configurations/syslog-logs)
  - You're log are read from a log file. Please add this kind of configuration in your `acquis.yaml` file:
 
-ⓘ the `prog_name` is the one that the parser in `s01-parse` filter will need to match.
+ⓘ the `type` is the one that the parser in `s01-parse` filter will need to match.
 
 
 ```
 ---
 filename: <PATH_TO_YOUR_LOG_FILE>
 labels:
-  prog_name: <PROGRAM_NAME>
+  type: <PROGRAM_NAME>
 
 ```
 Here an example:
@@ -25,7 +25,7 @@ Here an example:
 ---
 filename: /var/log/nginx/access.log
 labels:
-  prog_name: nginx
+  type: nginx
 ```
 
 </details>

docs/write_configurations/scenarios.md

@@ -260,6 +260,7 @@ INFO[12-05-2020 11:50:43] Processing Overflow with no decisions yy.yy.yy.yy perf
 WARN[12-05-2020 11:51:05] read 78215 lines                              file=./kern.log
 ...
 ```
+</details>
 
 It seems to work correctly !
 

pkg/cwapi/auth.go

@@ -31,14 +31,14 @@ type ApiCtx struct {
 	CfgUser      string   `yaml:"machine_id"`
 	CfgPassword  string   `yaml:"password"`
 	Creds        ApiCreds `yaml:"-"`
-	Muted        bool     `yaml:"muted"`
-	DebugDump    bool     `yaml:"debug_dump"`
-
+	/*mostly for mocking/faking api*/
+	Muted     bool `yaml:"-"`
+	DebugDump bool `yaml:"-"`
 	/*runtime*/
 	tokenExpired bool          `yaml:"-"`
 	toPush       []types.Event `yaml:"-"`
 	Http         *sling.Sling  `yaml:"-"`
-	PusherTomb   tomb.Tomb
+	PusherTomb   tomb.Tomb     `yaml:"-"`
 }
 
 type ApiCreds struct {

pkg/cwhub/hubMgmt.go

@@ -572,6 +572,7 @@ func DisableItem(target Item, tdir string, hdir string, purge bool) (Item, error
 }
 
 func EnableItem(target Item, tdir string, hdir string) (Item, error) {
+	var err error
 	parent_dir := filepath.Clean(tdir + "/" + target.Type + "/" + target.Stage + "/")
 	/*create directories if needed*/
 	if target.Installed {
@@ -581,8 +582,9 @@ func EnableItem(target Item, tdir string, hdir string) (Item, error) {
 		if target.Local {
 			return target, fmt.Errorf("%s is local, won't enable", target.Name)
 		}
-		if target.UpToDate {
-			log.Tracef("%s is installed and up-to-date, skip.", target.Name)
+		/* if it's a collection, check sub-items even if the collection file itself is up-to-date */
+		if target.UpToDate && target.Type != COLLECTIONS {
+			log.Debugf("%s is installed and up-to-date, skip.", target.Name)
 			return target, nil
 		}
 	}
@@ -592,26 +594,28 @@ func EnableItem(target Item, tdir string, hdir string) (Item, error) {
 			return target, fmt.Errorf("unable to create parent directories")
 		}
 	}
-	if _, err := os.Lstat(parent_dir + "/" + target.FileName); os.IsNotExist(err) {
-		/*install sub-items if it's a collection*/
-		if target.Type == COLLECTIONS {
-			var tmp = [][]string{target.Parsers, target.PostOverflows, target.Scenarios, target.Collections}
-			for idx, ptr := range tmp {
-				ptrtype := ItemTypes[idx]
-				for _, p := range ptr {
-					if val, ok := HubIdx[ptrtype][p]; ok {
-						HubIdx[ptrtype][p], err = EnableItem(val, Installdir, Hubdir)
-						if err != nil {
-							log.Errorf("Encountered error while installing sub-item %s %s : %s.", ptrtype, p, err)
-							return target, fmt.Errorf("encountered error while install %s for %s, abort.", val.Name, target.Name)
-						}
-					} else {
-						//log.Errorf("Referred %s %s in collection %s doesn't exist.", ptrtype, p, target.Name)
-						return target, fmt.Errorf("required %s %s of %s doesn't exist, abort.", ptrtype, p, target.Name)
+
+	/*install sub-items if it's a collection*/
+	if target.Type == COLLECTIONS {
+		var tmp = [][]string{target.Parsers, target.PostOverflows, target.Scenarios, target.Collections}
+		for idx, ptr := range tmp {
+			ptrtype := ItemTypes[idx]
+			for _, p := range ptr {
+				if val, ok := HubIdx[ptrtype][p]; ok {
+					HubIdx[ptrtype][p], err = EnableItem(val, Installdir, Hubdir)
+					if err != nil {
+						log.Errorf("Encountered error while installing sub-item %s %s : %s.", ptrtype, p, err)
+						return target, fmt.Errorf("encountered error while install %s for %s, abort.", val.Name, target.Name)
 					}
+				} else {
+					//log.Errorf("Referred %s %s in collection %s doesn't exist.", ptrtype, p, target.Name)
+					return target, fmt.Errorf("required %s %s of %s doesn't exist, abort.", ptrtype, p, target.Name)
 				}
 			}
 		}
+	}
+
+	if _, err := os.Lstat(parent_dir + "/" + target.FileName); os.IsNotExist(err) {
 		//tdir+target.RemotePath
 		srcPath, err := filepath.Abs(hdir + "/" + target.RemotePath)
 		if err != nil {

pkg/outputs/ouputs.go

@@ -55,6 +55,11 @@ func OvflwToOrder(sig types.SignalOccurence, prof types.Profile) (*types.BanOrde
 		/*if the profil has no remediation, no order */
 		return nil, nil, fmt.Errorf("no remediation")
 	}
+
+	if sig.Source == nil {
+		return nil, nil, fmt.Errorf("no 'source' in event (Meta.source_ip empty?)")
+	}
+
 	ordr.MeasureSource = "local"
 	ordr.Reason = sig.Scenario
 	//Identify scope

pkg/parser/enrich.go

@@ -19,6 +19,7 @@ type EnricherCtx struct {
 	Name       string
 	Path       string      //path to .so ?
 	RuntimeCtx interface{} //the internal context of plugin, given back over every call
+	initiated  bool
 }
 
 /* mimic plugin loading */
@@ -40,8 +41,10 @@ func Loadplugin(path string) (EnricherCtx, error) {
 
 	c.RuntimeCtx, err = c.Init(map[string]string{"datadir": path})
 	if err != nil {
-		log.Fatalf("load (fake) plugin load : %v", err)
+		log.Warningf("load (fake) plugin load : %v", err)
+		c.initiated = false
 	}
+	c.initiated = true
 	return c, nil
 }
 

pkg/parser/enrich_geoip.go

@@ -92,18 +92,18 @@ func GeoIpInit(cfg map[string]string) (interface{}, error) {
 	var err error
 	ctx.dbc, err = geoip2.Open(cfg["datadir"] + "/GeoLite2-City.mmdb")
 	if err != nil {
-		log.Errorf("couldn't open geoip : %v", err)
+		log.Debugf("couldn't open geoip : %v", err)
 		return nil, err
 	}
 	ctx.dba, err = geoip2.Open(cfg["datadir"] + "/GeoLite2-ASN.mmdb")
 	if err != nil {
-		log.Errorf("couldn't open geoip : %v", err)
+		log.Debugf("couldn't open geoip : %v", err)
 		return nil, err
 	}
 
 	ctx.dbraw, err = maxminddb.Open(cfg["datadir"] + "/GeoLite2-ASN.mmdb")
 	if err != nil {
-		log.Errorf("couldn't open geoip : %v", err)
+		log.Debugf("couldn't open geoip : %v", err)
 		return nil, err
 	}
 

pkg/parser/node.go

@@ -88,13 +88,13 @@ func (n *Node) validate(pctx *UnixParserCtx) error {
 			}
 			method_found := false
 			for _, enricherCtx := range ECTX {
-				if _, ok := enricherCtx.Funcs[static.Method]; ok {
+				if _, ok := enricherCtx.Funcs[static.Method]; ok && enricherCtx.initiated {
 					method_found = true
 					break
 				}
 			}
 			if !method_found {
-				return fmt.Errorf("the method '%s' doesn't exist", static.Method)
+				return fmt.Errorf("the method '%s' doesn't exist or the plugin has not been initialized", static.Method)
 			}
 		} else {
 			if static.Meta == "" && static.Parsed == "" && static.TargetByName == "" {

pkg/parser/runtime.go

@@ -148,7 +148,7 @@ func ProcessStatics(statics []types.ExtraField, p *types.Event, clog *logrus.Ent
 			processed := false
 			/*still way too hackish, but : inject all the results in enriched, and */
 			for _, x := range ECTX {
-				if fptr, ok := x.Funcs[static.Method]; ok {
+				if fptr, ok := x.Funcs[static.Method]; ok && x.initiated {
 					clog.Tracef("Found method '%s'", static.Method)
 					ret, err := fptr(value, p, x.RuntimeCtx)
 					if err != nil {
@@ -165,7 +165,7 @@ func ProcessStatics(statics []types.ExtraField, p *types.Event, clog *logrus.Ent
 					}
 					break
 				} else {
-					clog.Warningf("method '%s' doesn't exist", static.Method)
+					clog.Warningf("method '%s' doesn't exist or plugin not initialized", static.Method)
 				}
 			}
 			if !processed {

wizard.sh

@@ -314,14 +314,14 @@ update_full() {
     fi
 
     log_info "Backing up existing configuration"
-    ${CSCLI_BIN} backup save ${BACKUP_DIR}
+    ${CSCLI_BIN_INSTALLED} backup save ${BACKUP_DIR}
     log_info "Cleanup existing crowdsec configuration"
     uninstall_crowdsec
     log_info "Installing crowdsec"
     install_crowdsec
     log_info "Restoring configuration"
-    ${CSCLI_BIN} update
-    ${CSCLI_BIN} backup restore ${BACKUP_DIR}
+    ${CSCLI_BIN_INSTALLED} update
+    ${CSCLI_BIN_INSTALLED} backup restore ${BACKUP_DIR}
     log_info "Finished, restarting"
     systemctl restart crowdsec || log_err "Failed to restart crowdsec"
 }