initial import

Dockerfile

@@ -0,0 +1,73 @@
+############################
+# STEP 1 build executable binary
+############################
+
+FROM golang:alpine AS builder
+
+RUN apk update && apk add make gettext gcc g++
+
+
+WORKDIR $GOPATH/src/JohnDoeCrowdSec/granola
+
+# COPY the source
+COPY ./ .
+
+RUN make build
+
+RUN make install
+
+RUN cp ./docker/docker.yaml /etc/crowdsec/docker.yaml
+RUN cp ./docker/acquis.yaml /etc/crowdsec/crowdsec/
+
+############################
+# STEP 2
+############################
+
+FROM alpine:latest
+
+COPY --from=builder /usr/local/bin/crowdsec /usr/local/bin/crowdsec
+COPY --from=builder /usr/local/bin/cscli /usr/local/bin/cscli
+
+
+COPY --from=builder /etc/crowdsec /etc/crowdsec
+COPY --from=builder /var/run/crowdsec /var/run/crowdsec
+
+RUN apk add --update bash rsyslog && rm -rf /var/cache/apk/*
+
+###########################
+##### Prepare rsyslog #####
+###########################
+
+RUN mkdir -p /etc/rsyslog.d/
+RUN mkdir -p /var/spool/rsyslog/
+RUN mkdir -p /var/log/rsyslog
+RUN touch /var/log/syslog
+
+EXPOSE 514 514
+
+COPY ./docker/rsyslog.conf /etc/rsyslog.conf
+
+###########################################
+###### Configure crowdsec ###########
+###########################################
+
+RUN cscli config token "6ba94afde0fbf41310f7191934bc1d920245c9f1" 
+RUN cscli config installdir "/etc/crowdsec/crowdsec/"
+RUN cscli config dbpath "/var/run/crowdsec/crowdsec.db"
+
+RUN cscli update
+
+RUN cscli install collection crowdsec/base-http-scenarios
+RUN cscli install collection crowdsec/linux
+RUN cscli install collection crowdsec/nginx
+RUN cscli install collection crowdsec/sshd
+
+######################################
+## Wrapper to launch multi services ##
+######################################
+
+COPY ./docker/wrapper.sh .
+RUN chmod +x ./wrapper.sh
+
+ENTRYPOINT ["./wrapper.sh"]
+

Makefile

@@ -0,0 +1,118 @@
+PREFIX?="/tmp/crowdsec/"
+CFG_PREFIX = $(PREFIX)"/etc/crowdsec/"
+BIN_PREFIX = $(PREFIX)"/usr/local/bin/"
+DATA_PREFIX = $(PREFIX)"/var/run/crowdsec/"
+
+PLUGIN_FOLDER="./plugins"
+PID_DIR = $(PREFIX)"/var/run/"
+CROWDSEC_FOLDER = "./cmd/crowdsec"
+CSCLI_FOLDER = "./cmd/crowdsec-cli/"
+CROWDSEC_BIN = "crowdsec"
+CSCLI_BIN = "cscli"
+BUILD_CMD="build"
+
+GOARCH=amd64
+GOOS=linux
+REQUIRE_GOVERSION="1.13"
+
+
+#Current versioning information from env
+export BUILD_VERSION=$(shell cat RELEASE.json | jq -r .Version)
+export BUILD_GOVERSION="$(shell go version | cut -d " " -f3 | sed -r 's/[go]+//g')"
+export BUILD_CODENAME=$(shell cat RELEASE.json | jq -r .CodeName)
+export BUILD_TIMESTAMP=$(shell date +%F"_"%T)
+export BUILD_TAG="$(shell git rev-parse HEAD)"
+export LD_OPTS=-ldflags "-X github.com/crowdsecurity/crowdsec/pkg/cwversion.Version=$(BUILD_VERSION) \
+-X github.com/crowdsecurity/crowdsec/pkg/cwversion.BuildDate=$(BUILD_TIMESTAMP) \
+-X github.com/crowdsecurity/crowdsec/pkg/cwversion.Codename=$(BUILD_CODENAME)  \
+-X github.com/crowdsecurity/crowdsec/pkg/cwversion.Tag=$(BUILD_TAG) \
+-X github.com/crowdsecurity/crowdsec/pkg/cwversion.GoVersion=$(BUILD_GOVERSION)"
+RELDIR = crowdsec-$(BUILD_VERSION)
+
+all: clean test build
+
+build: clean goversion crowdsec cscli
+
+static: goversion crowdsec_static cscli_static
+
+goversion:
+	CURRENT_GOVERSION="$(shell go version | cut -d " " -f3 | sed -r 's/[go]+//g')"
+	RESPECT_VERSION="$(shell echo "$(CURRENT_GOVERSION),$(REQUIRE_GOVERSION)" | tr ',' '\n' | sort -V)"
+
+clean:
+	@make -C $(CROWDSEC_FOLDER) clean --no-print-directory
+	@make -C $(CSCLI_FOLDER) clean --no-print-directory
+	@rm -f $(CROWDSEC_BIN)
+	@rm -f $(CSCLI_BIN)
+	@rm -f *.log
+
+cscli:
+ifeq ($(lastword $(RESPECT_VERSION)), $(CURRENT_GOVERSION))
+	@make -C $(CSCLI_FOLDER) build --no-print-directory
+else
+	@echo "Required golang version is $(REQUIRE_GOVERSION). The current one is $(CURRENT_GOVERSION). Exiting.."
+	@exit 1;
+endif
+
+
+crowdsec:
+ifeq ($(lastword $(RESPECT_VERSION)), $(CURRENT_GOVERSION))
+	@make -C $(CROWDSEC_FOLDER) build --no-print-directory
+else
+	@echo "Required golang version is $(REQUIRE_GOVERSION). The current one is $(CURRENT_GOVERSION). Exiting.."
+	@exit 1;
+endif
+
+
+cscli_static:
+ifeq ($(lastword $(RESPECT_VERSION)), $(CURRENT_GOVERSION))
+	@make -C $(CSCLI_FOLDER) static --no-print-directory
+else
+	@echo "Required golang version is $(REQUIRE_GOVERSION). The current one is $(CURRENT_GOVERSION). Exiting.."
+	@exit 1;
+endif
+
+
+crowdsec_static:
+ifeq ($(lastword $(RESPECT_VERSION)), $(CURRENT_GOVERSION))
+	@make -C $(CROWDSEC_FOLDER) static --no-print-directory
+else
+	@echo "Required golang version is $(REQUIRE_GOVERSION). The current one is $(CURRENT_GOVERSION). Exiting.."
+	@exit 1;
+endif
+
+
+#.PHONY: test
+test:
+ifeq ($(lastword $(RESPECT_VERSION)), $(CURRENT_GOVERSION))
+	@make -C $(CROWDSEC_FOLDER) test --no-print-directory
+else
+	@echo "Required golang version is $(REQUIRE_GOVERSION). The current one is $(CURRENT_GOVERSION). Exiting.."
+	@exit 1;
+endif
+
+.PHONY: uninstall
+uninstall:
+	@rm -rf "$(CFG_PREFIX)" || exit
+	@rm -rf "$(DATA_PREFIX)" || exit
+	@rm -rf "$(SYSTEMD_PATH_FILE)" || exit
+
+.PHONY: check_release
+check_release:
+	@if [ -d $(RELDIR) ]; then echo "$(RELDIR) already exists, abort" ;  exit 1 ; fi
+
+.PHONY:
+release: check_release build
+	@echo Building Release to dir $(RELDIR)
+	@mkdir -p $(RELDIR)/cmd/crowdsec
+	@mkdir -p $(RELDIR)/cmd/crowdsec-cli
+	@cp $(CROWDSEC_FOLDER)/$(CROWDSEC_BIN) $(RELDIR)/cmd/crowdsec
+	@cp $(CSCLI_FOLDER)/$(CSCLI_BIN) $(RELDIR)/cmd/crowdsec-cli
+	@cp -R ./config/ $(RELDIR)
+	@cp -R ./data/ $(RELDIR)
+	@cp wizard.sh $(RELDIR)
+	@cp scripts/test_env.sh $(RELDIR)
+	@bash ./scripts/build_plugins.sh
+	@mkdir -p "$(RELDIR)/plugins/backend"
+	@find ./plugins -type f -name "*.so" -exec install -Dm 644 {} "$(RELDIR)/{}" \; || exiting 
+	@tar cvzf crowdsec-release.tgz $(RELDIR)	

README.md

@@ -0,0 +1,153 @@
+![Go](https://github.com/crowdsecurity/crowdsec/workflows/Go/badge.svg)
+![build-binary-package](https://github.com/crowdsecurity/crowdsec/workflows/build-binary-package/badge.svg)
+
+# CrowdSec project
+
+Please see [terminology](#terminology) if you're new to the projetct.
+
+## Foreword
+
+This repository contains the code for the two main components of crowdsec :
+ - `crowdsec` : the daemon a-la-fail2ban that can read, parse, enrich and apply heuristis to logs. This is the component in charge of "detecting" the attacks
+ - `cscli` : the cli tool mainly used to interact with crowdsec : ban/unban/view current bans, enable/disable parsers and scenarios.
+
+## Plugins
+
+The plugins are in charge of blocking that Ip/Ranges that have been tagged as malevolent.
+They do so by querying a sqlite database when they see a new IP. This SQLite database is being fed by crowdsec.
+The following plugins are available :
+ - `netfilter-plugin` : an iptables/ipset service that can be deployed by the wizard. it will allow to ban IP/Ranges as you would do with iptables.
+ - `nginx-plugin` : a LUA connector for nginx that can be deployed by the wizard. it will allow to ban ip/ranges at the applicative level (ie. more suitable than iptables if your website is behind a CDN).
+ - `wordpress-plugin` : a Wordpress/php module that can be deployed in Wordpress to block the requests at the applicative level. (it comes as a library for easy re-use).
+ 
+
+
+
+# Software architecture
+
+
+![global crowdsec architecture](./doc/img/crowdsec-global.png)
+
+
+**NOTE** the API part isn't enabled by default.
+
+# Build
+
+**To build crowdsec you need golang >= 1.13.**
+To build binaries :
+
+```
+$ make build
+```
+ 
+# Installation
+
+## With the wizard
+ 
+ The wizard is here is significatively improve the user experience, and aims at providing a _next-next-next-finish_ installer that should work out of the box on most linux flavored systems.
+
+ The wizard will help you in the following steps :
+  - detect running services
+  - detect their log files(by default in `/etc/crowdsec/`) 
+  - suggest collections/scenarios according to the detect services
+  - deploy crowdsec service
+  - deploy plugins
+ 
+ ```bash
+ $ make build
+ $ sudo ./wizard.sh -i
+ ```
+ 
+ and you're done !
+ 
+
+## Without the wizard
+
+> You man of little faith
+
+You can install crowdsec and its cli without the wizard :
+
+```bash
+$ make build
+$ make systemd
+```
+
+**NOTE** if you install without the wizard, it will be your responsability to configure the acquisition (which file to read for which service) and to deploy scenarios and parsers (how to parse logs, and which scenarios should be applied to which services).
+
+## After the installation
+
+Services are deployed as systemd units : 
+ - `crowdsec` is the detection component
+ - `crowdsec-netfilter` is the netfilter plugin
+ - the nginx, wordpress etc. plugins usually are ran inside said service (ie. nginx plugin is a LUA script, wordpress plugin is a php module)
+ - `cscli` is deployed in standard path.
+
+```bash
+$ sudo systemctl status crowdsec
+# stop the netfilter plugin. If you didn't install other plugins, decisions won't be 'applied' anymore unless you start it again.
+$ sudo systemctl stop crowdsec-netfilter
+```
+
+# Configuration
+
+crowdsec relies on the following configuration files (by default in `/etc/crowdsec/`) :
+
+ - default.yaml : The main configuration of crowdsec, you will find here informations about logging, path to sqlite DB etc.
+ - acquis.yaml : Describes the files that will be read (a-la `tail -f`) and which type of logs to expect from it
+ - api.yaml : url & token for api push and pulls (pushes **signal occurences** and fetchs **crowd-sourced reputation**)
+ - profiles.yaml : (you shouldn't modify this one) Describes in which condition crowdsec should insert a ban decision in database. It's usually because a scenario has a `remediation: true` flag in its tags.
+
+However, the "real" configuration of crowdsec relies on the collections of scenarios and parsers that you have deployed.
+Those are deployed / upgraded / removed (ideally) with `cscli`, see [its dedicated documentation](./cmd/crowdsec-cli/doc/cscli.md)
+
+If you used the wizard, chances are that you don't have anything specific to configure.
+
+# Usage / FAQ
+
+[See `cscli`dedicated documentation](./cmd/crowdsec-cli/doc/cscli.md)
+
+## stop the netfilter plugin
+
+**note** when netfilter plugin is disabled, no bans will be applied if no other plugins are enabled.
+
+```
+#view netfilter logs
+$ journalctl -u -f crowdsec-netfilter
+#stop service
+$ systemctl stop crowdsec-netfilter
+```
+
+## view/add/remove bans
+
+```
+# cscli  ban list
+INFO[0000] 38 non-expired ban applications              
++-----------------+---------------+--------------------+--------+---------+--------------------------------+--------------+--------------------+
+|     SOURCE      |   SCENARIO    | CURRENT BANS COUNT | ACTION | COUNTRY |               AS               | EVENTS COUNT |     EXPIRATION     |
++-----------------+---------------+--------------------+--------+---------+--------------------------------+--------------+--------------------+
+| 37.195.50.41    | ssh_user_enum |                  1 | ban    | RU      | 31200 Novotelecom Ltd          |            4 | 3h59m56.337435741s |
+| 159.203.143.58  | ssh_user_enum |                  1 | ban    | US      | 14061 DigitalOcean, LLC        |            4 | 3h59m55.585257629s |
+...
+# cscli  ban  add range 37.139.4.0/24 10m spam
+# cscli  ban  add ip 37.139.4.123 10m spam
+```
+
+# Terminology
+
+ - **crowdsec** : the daemon that reads log files, parses logs and triggers scenarios, alerts and bans.
+ - **crowdsec database** : a local file that contains at a given time the list of banned ip/ranges.
+ - **plugin** : a software component that can interact with crowdsec database to block/delay attackers.
+ - **parser** : a configuration file that allows crowdsec to 'understand' a specific log file format. Each service will generally require its own parser (nginx, apache, sshd, mysql etc.). parsers are usually found on the **hub** and downloaded via the **cli**.
+ - **scenario** : a leakybucket description that allows to detect a specific attack : _more that 5 failed ssh authentication attempts from the same IP within less than 20 seconds is a ssh bruteforce and should be punished_
+ - **signal** : the information resulting from a scenario being triggered, this information is shared amongst participants and will lead to consensus : _users A, B, C, D all reported that ip 1.2.3.4 targetted them with a ssh bruteforce_
+ - **bucket**, **bucket overflow** : a more technical term referring to a scenario being triggered.
+ - **hub** : the portal on which users can find, share and publish parsers and scenarios.
+ - **cli** : the `cscli` tool.
+
+# Making a release
+
+ - release-drafter maintains a draft release up-to-date with MRs
+ - when you publish the release with the "pre-release" flag, it's going to launch action to add the built release package to release.
+ - once extra manual steps are done, you can remove the "pre-release" flag from published release "and voila"
+ 
+

RELEASE.json

@@ -0,0 +1,4 @@
+{
+    "Version": "v0.0.21",
+    "CodeName": "road2beta"
+}

cmd/crowdsec-cli/Makefile

@@ -0,0 +1,31 @@
+# Go parameters
+GOCMD=go
+GOBUILD=$(GOCMD) build
+GOCLEAN=$(GOCMD) clean
+GOTEST=$(GOCMD) test
+GOGET=$(GOCMD) get
+BINARY_NAME=cscli
+PREFIX?="/"
+BIN_PREFIX = $(PREFIX)"/usr/local/bin/"
+
+all: clean build
+
+build: clean
+	@$(GOBUILD) $(LD_OPTS) -o $(BINARY_NAME) -v
+
+static: clean
+	@$(GOBUILD) -o $(BINARY_NAME) -v -a -tags netgo -ldflags '-w -extldflags "-static"'
+
+install: install-conf install-bin
+
+install-conf:
+
+install-bin:
+	@install -v -m 755 -D "$(BINARY_NAME)" "$(BIN_PREFIX)/$(BINARY_NAME)" || exit
+
+uninstall:
+	@rm -rf $(CSCLI_CONFIG)
+	@rm -rf $(BIN_PREFIX)$(BINARY_NAME)
+
+clean:
+	@rm -f $(BINARY_NAME)

cmd/crowdsec-cli/README.md

@@ -0,0 +1 @@
+see doc in `doc/`

cmd/crowdsec-cli/api.go

@@ -0,0 +1,271 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"math/rand"
+	"path"
+	"strings"
+	"time"
+
+	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
+	"github.com/crowdsecurity/crowdsec/pkg/outputs"
+	"github.com/crowdsecurity/crowdsec/pkg/sqlite"
+	"github.com/crowdsecurity/crowdsec/pkg/types"
+
+	"github.com/denisbrodbeck/machineid"
+	log "github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+	"gopkg.in/yaml.v2"
+)
+
+var (
+	passwordLength = 64
+	upper          = "ABCDEFGHIJKLMNOPQRSTUVWXY"
+	lower          = "abcdefghijklmnopqrstuvwxyz"
+	digits         = "0123456789"
+)
+
+var (
+	apiConfigFile = "api.yaml"
+)
+
+var userID string // for flag parsing
+var dbctx *sqlite.Context
+
+var outputCTX *outputs.Output
+
+func dumpCredentials() error {
+	if config.output == "json" {
+		credsYaml, err := json.Marshal(&outputCTX.API.Creds)
+		if err != nil {
+			log.Fatalf("Can't marshal credentials : %v", err)
+		}
+		fmt.Printf("%s\n", string(credsYaml))
+	} else {
+		credsYaml, err := yaml.Marshal(&outputCTX.API.Creds)
+		if err != nil {
+			log.Fatalf("Can't marshal credentials : %v", err)
+		}
+		fmt.Printf("%s\n", string(credsYaml))
+	}
+	return nil
+}
+
+func generatePassword() string {
+	rand.Seed(time.Now().UnixNano())
+	charset := upper + lower + digits
+
+	buf := make([]byte, passwordLength)
+	buf[0] = digits[rand.Intn(len(digits))]
+	buf[1] = upper[rand.Intn(len(upper))]
+	buf[2] = lower[rand.Intn(len(lower))]
+
+	for i := 3; i < passwordLength; i++ {
+		buf[i] = charset[rand.Intn(len(charset))]
+	}
+	rand.Shuffle(len(buf), func(i, j int) {
+		buf[i], buf[j] = buf[j], buf[i]
+	})
+
+	return string(buf)
+}
+
+func pullTOP() error {
+	/*profile from cwhub*/
+	var profiles []string
+	if _, ok := cwhub.HubIdx[cwhub.SCENARIOS]; !ok || len(cwhub.HubIdx[cwhub.SCENARIOS]) == 0 {
+		log.Errorf("no loaded scenarios, can't fill profiles")
+		return fmt.Errorf("no profiles")
+	}
+	for _, item := range cwhub.HubIdx[cwhub.SCENARIOS] {
+		if item.Tainted || !item.Installed {
+			continue
+		}
+		profiles = append(profiles, item.Name)
+	}
+	outputCTX.API.Creds.Profile = strings.Join(profiles[:], ",")
+	if err := outputCTX.API.Signin(); err != nil {
+		log.Fatalf(err.Error())
+	}
+
+	ret, err := outputCTX.API.PullTop()
+	if err != nil {
+		log.Fatalf(err.Error())
+	}
+	log.Warningf("api pull returned %d entries", len(ret))
+	for _, item := range ret {
+		if _, ok := item["range_ip"]; !ok {
+			continue
+		}
+		if _, ok := item["scenario"]; !ok {
+			continue
+		}
+		item["scenario"] = fmt.Sprintf("api: %s", item["scenario"])
+
+		if _, ok := item["action"]; !ok {
+			continue
+		}
+		if _, ok := item["expiration"]; !ok {
+			continue
+		}
+		if _, ok := item["country"]; !ok {
+			item["country"] = ""
+		}
+		if _, ok := item["as_org"]; !ok {
+			item["as_org"] = ""
+		}
+		if _, ok := item["as_num"]; !ok {
+			item["as_num"] = ""
+		}
+		var signalOcc types.SignalOccurence
+		signalOcc, err = simpleBanToSignal(item["range_ip"], item["scenario"], item["expiration"], item["action"], item["as_name"], item["as_num"], item["country"], "api")
+		if err := outputCTX.Insert(signalOcc); err != nil {
+			log.Fatalf("Unable to write pull to sqliteDB : %+s", err.Error())
+		}
+	}
+	outputCTX.Flush()
+	log.Infof("Wrote %d bans from api to database.", len(ret))
+	return nil
+}
+
+func NewAPICmd() *cobra.Command {
+
+	var cmdAPI = &cobra.Command{
+		Use:   "api [action]",
+		Short: "Crowdsec API interaction",
+		Long: `
+Allow to register your machine into crowdsec API to send and receive signal.
+		`,
+		Example: `
+cscli api register      # Register to Crowdsec API
+cscli api pull          # Pull malevolant IPs from Crowdsec API
+cscli api reset         # Reset your machines credentials
+cscli api enroll        # Enroll your machine to the user account you created on Crowdsec backend
+cscli api credentials   # Display your API credentials
+`,
+		Args: cobra.MinimumNArgs(1),
+		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
+			var err error
+			if !config.configured {
+				return fmt.Errorf("you must configure cli before interacting with hub")
+			}
+
+			outputConfig := outputs.OutputFactory{
+				BackendFolder: config.BackendPluginFolder,
+			}
+			outputCTX, err = outputs.NewOutput(&outputConfig, false)
+			if err != nil {
+				return err
+			}
+
+			err = outputCTX.LoadAPIConfig(path.Join(config.InstallFolder, apiConfigFile))
+			if err != nil {
+				return err
+			}
+			return nil
+		},
+	}
+
+	var cmdAPIRegister = &cobra.Command{
+		Use:   "register",
+		Short: "Register on Crowdsec API",
+		Long: `This command will register your machine to crowdsec API to allow you to receive list of malveolent IPs. 
+		The printed machine_id and password should be added to your api.yaml file.`,
+		Example: `cscli api register`,
+		Args:    cobra.MinimumNArgs(0),
+		Run: func(cmd *cobra.Command, args []string) {
+			id, err := machineid.ID()
+			if err != nil {
+				log.Fatalf("failed to get machine id: %s", err)
+			}
+			password := generatePassword()
+
+			if err := outputCTX.API.RegisterMachine(id, password); err != nil {
+				log.Fatalf(err.Error())
+			}
+			fmt.Printf("machine_id: %s\n", outputCTX.API.Creds.User)
+			fmt.Printf("password: %s\n", outputCTX.API.Creds.Password)
+			return
+		},
+	}
+
+	var cmdAPIEnroll = &cobra.Command{
+		Use:     "enroll",
+		Short:   "Associate your machine to an existing crowdsec user",
+		Long:    `Enrolling your machine into your user account will allow for more accurate lists and threat detection. See website to create user account.`,
+		Example: `cscli api enroll -u 1234567890ffff`,
+		Args:    cobra.MinimumNArgs(0),
+		Run: func(cmd *cobra.Command, args []string) {
+			if err := outputCTX.API.Signin(); err != nil {
+				log.Fatalf("unable to signin : %s", err)
+			}
+			if err := outputCTX.API.Enroll(userID); err != nil {
+				log.Fatalf(err.Error())
+			}
+			return
+		},
+	}
+
+	var cmdAPIResetPassword = &cobra.Command{
+		Use:     "reset",
+		Short:   "Reset password on CrowdSec API",
+		Long:    `Attempts to reset your credentials to the API.`,
+		Example: `cscli api reset`,
+		Args:    cobra.MinimumNArgs(0),
+		Run: func(cmd *cobra.Command, args []string) {
+			id, err := machineid.ID()
+			if err != nil {
+				log.Fatalf("failed to get machine id: %s", err)
+			}
+			password := generatePassword()
+			if err := outputCTX.API.ResetPassword(id, password); err != nil {
+				log.Fatalf(err.Error())
+			}
+			fmt.Printf("machine_id: %s\n", outputCTX.API.Creds.User)
+			fmt.Printf("password: %s\n", outputCTX.API.Creds.Password)
+			return
+		},
+	}
+
+	var cmdAPIPull = &cobra.Command{
+		Use:     "pull",
+		Short:   "Pull crowdsec API TopX",
+		Long:    `Pulls a list of malveolent IPs relevant to your situation and add them into the local ban database.`,
+		Example: `cscli api pull`,
+		Args:    cobra.MinimumNArgs(0),
+		Run: func(cmd *cobra.Command, args []string) {
+			if err := cwhub.GetHubIdx(); err != nil {
+				log.Fatalf(err.Error())
+			}
+			err := pullTOP()
+			if err != nil {
+				log.Fatalf(err.Error())
+			}
+			return
+		},
+	}
+
+	var cmdAPICreds = &cobra.Command{
+		Use:     "credentials",
+		Short:   "Display api credentials",
+		Long:    ``,
+		Example: `cscli api credentials`,
+		Args:    cobra.MinimumNArgs(0),
+		Run: func(cmd *cobra.Command, args []string) {
+			if err := dumpCredentials(); err != nil {
+				log.Fatalf(err.Error())
+			}
+			return
+		},
+	}
+
+	cmdAPI.AddCommand(cmdAPICreds)
+	cmdAPIEnroll.Flags().StringVarP(&userID, "user", "u", "", "User ID (required)")
+	cmdAPIEnroll.MarkFlagRequired("user")
+	cmdAPI.AddCommand(cmdAPIEnroll)
+	cmdAPI.AddCommand(cmdAPIResetPassword)
+	cmdAPI.AddCommand(cmdAPIRegister)
+	cmdAPI.AddCommand(cmdAPIPull)
+	return cmdAPI
+}

cmd/crowdsec-cli/backup-restore.go

@@ -0,0 +1,473 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
+
+	"github.com/crowdsecurity/crowdsec/pkg/cwapi"
+	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
+	"github.com/crowdsecurity/crowdsec/pkg/outputs"
+	log "github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+)
+
+//it's a rip of the cli version, but in silent-mode
+func silenceInstallItem(name string, obtype string) (string, error) {
+	for _, it := range cwhub.HubIdx[obtype] {
+		if it.Name == name {
+			if download_only && it.Downloaded && it.UpToDate {
+				return fmt.Sprintf("%s is already downloaded and up-to-date", it.Name), nil
+			}
+			it, err := cwhub.DownloadLatest(it, cwhub.Hubdir, force_install)
+			if err != nil {
+				return "", fmt.Errorf("error while downloading %s : %v", it.Name, err)
+			}
+			cwhub.HubIdx[obtype][it.Name] = it
+			if download_only {
+				return fmt.Sprintf("Downloaded %s to %s", it.Name, cwhub.Hubdir+"/"+it.RemotePath), nil
+			}
+			it, err = cwhub.EnableItem(it, cwhub.Installdir, cwhub.Hubdir)
+			if err != nil {
+				return "", fmt.Errorf("error while enabled %s : %v", it.Name, err)
+			}
+			cwhub.HubIdx[obtype][it.Name] = it
+
+			return fmt.Sprintf("Enabled %s", it.Name), nil
+		}
+	}
+	return "", fmt.Errorf("%s not found in hub index", name)
+}
+
+/*help to copy the file, ioutil doesn't offer the feature*/
+
+func copyFileContents(src, dst string) (err error) {
+	in, err := os.Open(src)
+	if err != nil {
+		return
+	}
+	defer in.Close()
+	out, err := os.Create(dst)
+	if err != nil {
+		return
+	}
+	defer func() {
+		cerr := out.Close()
+		if err == nil {
+			err = cerr
+		}
+	}()
+	if _, err = io.Copy(out, in); err != nil {
+		return
+	}
+	err = out.Sync()
+	return
+}
+
+/*copy the file, ioutile doesn't offer the feature*/
+func copyFile(sourceSymLink, destinationFile string) (err error) {
+
+	sourceFile, err := filepath.EvalSymlinks(sourceSymLink)
+	if err != nil {
+		log.Infof("Not a symlink : %s", err)
+		sourceFile = sourceSymLink
+	}
+
+	sourceFileStat, err := os.Stat(sourceFile)
+	if err != nil {
+		return
+	}
+	if !sourceFileStat.Mode().IsRegular() {
+		// cannot copy non-regular files (e.g., directories,
+		// symlinks, devices, etc.)
+		return fmt.Errorf("CopyFile: non-regular source file %s (%q)", sourceFileStat.Name(), sourceFileStat.Mode().String())
+	}
+	destinationFileStat, err := os.Stat(destinationFile)
+	if err != nil {
+		if !os.IsNotExist(err) {
+			return
+		}
+	} else {
+		if !(destinationFileStat.Mode().IsRegular()) {
+			return fmt.Errorf("CopyFile: non-regular destination file %s (%q)", destinationFileStat.Name(), destinationFileStat.Mode().String())
+		}
+		if os.SameFile(sourceFileStat, destinationFileStat) {
+			return
+		}
+	}
+	if err = os.Link(sourceFile, destinationFile); err == nil {
+		return
+	}
+	err = copyFileContents(sourceFile, destinationFile)
+	return
+}
+
+/*given a backup directory, restore configs (parser,collections..) both tainted and untainted.
+as well attempts to restore api credentials after verifying the existing ones aren't good
+finally restores the acquis.yaml file*/
+func restoreFromDirectory(source string) error {
+	var err error
+	/*backup scenarios etc.*/
+	for _, itype := range cwhub.ItemTypes {
+		itemDirectory := fmt.Sprintf("%s/%s/", source, itype)
+		if _, err = os.Stat(itemDirectory); err != nil {
+			log.Infof("no %s in backup", itype)
+			continue
+		}
+		/*restore the upstream items*/
+		upstreamListFN := fmt.Sprintf("%s/upstream-%s.json", itemDirectory, itype)
+		file, err := ioutil.ReadFile(upstreamListFN)
+		if err != nil {
+			return fmt.Errorf("error while opening %s : %s", upstreamListFN, err)
+		}
+		var upstreamList []string
+		err = json.Unmarshal([]byte(file), &upstreamList)
+		if err != nil {
+			return fmt.Errorf("error unmarshaling %s : %s", upstreamListFN, err)
+		}
+		for _, toinstall := range upstreamList {
+			label, err := silenceInstallItem(toinstall, itype)
+			if err != nil {
+				log.Errorf("Error while installing %s : %s", toinstall, err)
+			} else if label != "" {
+				log.Infof("Installed %s : %s", toinstall, label)
+			} else {
+				log.Printf("Installed %s : ok", toinstall)
+			}
+		}
+		/*restore the local and tainted items*/
+		files, err := ioutil.ReadDir(itemDirectory)
+		if err != nil {
+			return fmt.Errorf("Failed enumerating files of %s : %s", itemDirectory, err)
+		}
+		for _, file := range files {
+			//dir are stages, keep track
+			if !file.IsDir() {
+				continue
+			}
+			stage := file.Name()
+			stagedir := fmt.Sprintf("%s/%s/%s/", config.InstallFolder, itype, stage)
+			log.Debugf("Found stage %s in %s, target directory : %s", stage, itype, stagedir)
+			if err = os.MkdirAll(stagedir, os.ModePerm); err != nil {
+				return fmt.Errorf("Error while creating stage directory %s : %s", stagedir, err)
+			}
+			/*find items*/
+			ifiles, err := ioutil.ReadDir(itemDirectory + "/" + stage + "/")
+			if err != nil {
+				return fmt.Errorf("Failed enumerating files of %s : %s", itemDirectory+"/"+stage, err)
+			}
+			//finaly copy item
+			for _, tfile := range ifiles {
+				log.Infof("Going to restore local/tainted [%s]", tfile.Name())
+				sourceFile := fmt.Sprintf("%s/%s/%s", itemDirectory, stage, tfile.Name())
+				destinationFile := fmt.Sprintf("%s%s", stagedir, tfile.Name())
+				if err = copyFile(sourceFile, destinationFile); err != nil {
+					return fmt.Errorf("failed copy %s %s to %s : %s", itype, sourceFile, destinationFile, err)
+				} else {
+					log.Infof("restored %s to %s", sourceFile, destinationFile)
+				}
+
+			}
+		}
+	}
+	/*restore api credentials*/
+	//check if credentials exists :
+	// - if no, restore
+	// - if yes, try them :
+	//		- if it works, left untouched
+	//		- if not, restore
+	// -> try login
+	if err := restoreAPICreds(source); err != nil {
+		return fmt.Errorf("Failed to restore api credentials : %s", err)
+	}
+	/*
+		Restore acquis
+	*/
+	yamlAcquisFile := fmt.Sprintf("%s/acquis.yaml", config.InstallFolder)
+	bac := fmt.Sprintf("%s/acquis.yaml", source)
+	if err = copyFile(bac, yamlAcquisFile); err != nil {
+		return fmt.Errorf("failed copy %s to %s : %s", bac, yamlAcquisFile, err)
+	}
+	log.Infof("Restore acquis to %s", yamlAcquisFile)
+
+	return nil
+}
+
+func restoreAPICreds(source string) error {
+	var err error
+
+	/*check existing configuration*/
+	apiyaml := path.Join(config.InstallFolder, apiConfigFile)
+
+	api := &cwapi.ApiCtx{}
+	if err = api.LoadConfig(apiyaml); err != nil {
+		return fmt.Errorf("Unable to load api config %s : %s", apiyaml, err)
+	}
+	if api.Creds.User != "" {
+		log.Infof("Credentials present in existing configuration, try before override")
+		err := api.Signin()
+		if err == nil {
+			log.Infof("Credentials present allow authentication, don't override !")
+			return nil
+		} else {
+			log.Infof("Credentials aren't valid : %s", err)
+		}
+	}
+	/*existing config isn't good, override it !*/
+	ret, err := ioutil.ReadFile(path.Join(source, "api_creds.json"))
+	if err != nil {
+		return fmt.Errorf("Failed to read api creds from save : %s", err)
+	}
+	if err := json.Unmarshal(ret, &api.Creds); err != nil {
+		return fmt.Errorf("Failed unmarshaling saved credentials : %s", err)
+	}
+	api.CfgUser = api.Creds.User
+	api.CfgPassword = api.Creds.Password
+	/*override the existing yaml file*/
+	if err := api.WriteConfig(apiyaml); err != nil {
+		return fmt.Errorf("Failed writing to %s : %s", apiyaml, err)
+	} else {
+		log.Infof("Overwritting %s with backup info", apiyaml)
+	}
+
+	/*reload to check everything is safe*/
+	if err = api.LoadConfig(apiyaml); err != nil {
+		return fmt.Errorf("Unable to load api config %s : %s", apiyaml, err)
+	}
+
+	if err := api.Signin(); err != nil {
+		log.Errorf("Failed to authenticate after credentials restaurtion : %v", err)
+	} else {
+		log.Infof("Successfully auth to API after credentials restauration")
+	}
+
+	return nil
+}
+
+func backupToDirectory(target string) error {
+	var itemDirectory string
+	var upstreamParsers []string
+	var err error
+	if target == "" {
+		return fmt.Errorf("target directory can't be empty")
+	}
+	log.Warningf("Starting configuration backup")
+	_, err = os.Stat(target)
+	if err == nil {
+		return fmt.Errorf("%s already exists", target)
+	}
+	if err = os.MkdirAll(target, os.ModePerm); err != nil {
+		return fmt.Errorf("Error while creating %s : %s", target, err)
+	}
+	/*
+		backup configurations :
+			- parers, scenarios, collections, postoverflows
+	*/
+
+	for _, itemType := range cwhub.ItemTypes {
+		clog := log.WithFields(log.Fields{
+			"type": itemType,
+		})
+		if _, ok := cwhub.HubIdx[itemType]; ok {
+			itemDirectory = fmt.Sprintf("%s/%s/", target, itemType)
+			if err := os.MkdirAll(itemDirectory, os.ModePerm); err != nil {
+				return fmt.Errorf("Error while creating %s : %s", itemDirectory, err)
+			}
+			upstreamParsers = []string{}
+			stage := ""
+			for k, v := range cwhub.HubIdx[itemType] {
+				clog = clog.WithFields(log.Fields{
+					"file": v.Name,
+				})
+				if !v.Installed { //only backup installed ones
+					clog.Debugf("[%s] : not installed", k)
+					continue
+				}
+
+				//for the local/tainted ones, we backup the full file
+				if v.Tainted || v.Local || !v.UpToDate {
+					//we need to backup stages for parsers
+					if itemType == cwhub.PARSERS || itemType == cwhub.PARSERS_OVFLW {
+						tmp := strings.Split(v.LocalPath, "/")
+						stage = "/" + tmp[len(tmp)-2] + "/"
+						fstagedir := fmt.Sprintf("%s%s", itemDirectory, stage)
+						if err := os.MkdirAll(fstagedir, os.ModePerm); err != nil {
+							return fmt.Errorf("Error while creating stage dir %s : %s", fstagedir, err)
+						}
+					}
+					clog.Debugf("[%s] : backuping file (tainted:%t local:%t up-to-date:%t)", k, v.Tainted, v.Local, v.UpToDate)
+					tfile := fmt.Sprintf("%s%s%s", itemDirectory, stage, v.FileName)
+					//clog.Infof("item : %s", spew.Sdump(v))
+					if err = copyFile(v.LocalPath, tfile); err != nil {
+						return fmt.Errorf("failed copy %s %s to %s : %s", itemType, v.LocalPath, tfile, err)
+					}
+					clog.Infof("local/tainted saved %s to %s", v.LocalPath, tfile)
+					continue
+				}
+				clog.Debugf("[%s] : from hub, just backup name (up-to-date:%t)", k, v.UpToDate)
+				clog.Infof("saving, version:%s, up-to-date:%t", v.Version, v.UpToDate)
+				upstreamParsers = append(upstreamParsers, v.Name)
+			}
+			//write the upstream items
+			upstreamParsersFname := fmt.Sprintf("%s/upstream-%s.json", itemDirectory, itemType)
+			upstreamParsersContent, err := json.MarshalIndent(upstreamParsers, "", " ")
+			if err != nil {
+				return fmt.Errorf("failed marshaling upstream parsers : %s", err)
+			}
+			err = ioutil.WriteFile(upstreamParsersFname, upstreamParsersContent, 0644)
+			if err != nil {
+				return fmt.Errorf("unable to write to %s %s : %s", itemType, upstreamParsersFname, err)
+			}
+			clog.Infof("Wrote %d entries for %s to %s", len(upstreamParsers), itemType, upstreamParsersFname)
+
+		} else {
+			clog.Infof("No %s to backup.", itemType)
+		}
+	}
+	/*
+		Backup acquis
+	*/
+	yamlAcquisFile := fmt.Sprintf("%s/acquis.yaml", config.InstallFolder)
+	bac := fmt.Sprintf("%s/acquis.yaml", target)
+	if err = copyFile(yamlAcquisFile, bac); err != nil {
+		return fmt.Errorf("failed copy %s to %s : %s", yamlAcquisFile, bac, err)
+	}
+	log.Infof("Saved acquis to %s", bac)
+	/*
+		Backup default.yaml
+	*/
+	defyaml := fmt.Sprintf("%s/default.yaml", config.InstallFolder)
+	bac = fmt.Sprintf("%s/default.yaml", target)
+	if err = copyFile(defyaml, bac); err != nil {
+		return fmt.Errorf("failed copy %s to %s : %s", yamlAcquisFile, bac, err)
+	}
+	log.Infof("Saved default yaml to %s", bac)
+	/*
+		Backup API info
+	*/
+	if outputCTX == nil {
+		log.Fatalf("no API output context, won't save api credentials")
+	}
+	outputCTX.API = &cwapi.ApiCtx{}
+	if err = outputCTX.API.LoadConfig(path.Join(config.InstallFolder, apiConfigFile)); err != nil {
+		return fmt.Errorf("unable to load api config %s : %s", path.Join(config.InstallFolder, apiConfigFile), err)
+	}
+	credsYaml, err := json.Marshal(&outputCTX.API.Creds)
+	if err != nil {
+		log.Fatalf("can't marshal credentials : %v", err)
+	}
+	apiCredsDumped := fmt.Sprintf("%s/api_creds.json", target)
+	err = ioutil.WriteFile(apiCredsDumped, credsYaml, 0600)
+	if err != nil {
+		return fmt.Errorf("unable to write credentials to %s : %s", apiCredsDumped, err)
+	}
+	log.Infof("Saved configuration to %s", target)
+	return nil
+}
+
+func NewBackupCmd() *cobra.Command {
+	var cmdBackup = &cobra.Command{
+		Use:   "backup [save|restore] <directory>",
+		Short: "Backup or restore configuration (api, parsers, scenarios etc.) to/from directory",
+		Long:  `This command is here to help you save and/or restore crowdsec configurations to simple replication`,
+		Example: `cscli backup save ./my-backup
+cscli backup restore ./my-backup`,
+		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
+			if !config.configured {
+				return fmt.Errorf("you must configure cli before interacting with hub")
+			}
+			return nil
+		},
+	}
+
+	var cmdBackupSave = &cobra.Command{
+		Use:   "save <directory>",
+		Short: "Backup configuration (api, parsers, scenarios etc.) to directory",
+		Long: `backup command will try to save all relevant informations to crowdsec config, including :
+
+- List of scenarios, parsers, postoverflows and collections that are up-to-date
+
+- Actual backup of tainted/local/out-of-date scenarios, parsers, postoverflows and collections
+
+- Backup of API credentials
+
+- Backup of acqusition configuration
+		
+		`,
+		Example: `cscli backup save ./my-backup`,
+		Args:    cobra.ExactArgs(1),
+		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
+			if !config.configured {
+				return fmt.Errorf("you must configure cli before interacting with hub")
+			}
+			return nil
+		},
+		Run: func(cmd *cobra.Command, args []string) {
+			var err error
+
+			outputConfig := outputs.OutputFactory{
+				BackendFolder: config.BackendPluginFolder,
+			}
+			outputCTX, err = outputs.NewOutput(&outputConfig, false)
+			if err != nil {
+				log.Fatalf("Failed to load output plugins")
+			}
+			if err := cwhub.GetHubIdx(); err != nil {
+				log.Fatalf("Failed to get Hub index : %v", err)
+			}
+			if err := backupToDirectory(args[0]); err != nil {
+				log.Fatalf("Failed backuping to %s : %s", args[0], err)
+			}
+		},
+	}
+	cmdBackup.AddCommand(cmdBackupSave)
+
+	var cmdBackupRestore = &cobra.Command{
+		Use:   "restore <directory>",
+		Short: "Restore configuration (api, parsers, scenarios etc.) from directory",
+		Long: `restore command will try to restore all saved information from <directory> to yor local setup, including :
+
+- Installation of up-to-date scenarios/parsers/... via cscli
+
+- Restauration of tainted/local/out-of-date scenarios/parsers/... file
+
+- Restauration of API credentials (if the existing ones aren't working)
+
+- Restauration of acqusition configuration
+`,
+		Example: `cscli backup restore ./my-backup`,
+		Args:    cobra.ExactArgs(1),
+		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
+			if !config.configured {
+				return fmt.Errorf("you must configure cli before interacting with hub")
+			}
+			return nil
+		},
+		Run: func(cmd *cobra.Command, args []string) {
+			var err error
+
+			outputConfig := outputs.OutputFactory{
+				BackendFolder: config.BackendPluginFolder,
+			}
+			outputCTX, err = outputs.NewOutput(&outputConfig, false)
+			if err != nil {
+				log.Fatalf("Failed to load output plugins")
+			}
+
+			if err := cwhub.GetHubIdx(); err != nil {
+				log.Fatalf("failed to get Hub index : %v", err)
+			}
+			if err := restoreFromDirectory(args[0]); err != nil {
+				log.Fatalf("failed restoring from %s : %s", args[0], err)
+			}
+		},
+	}
+	cmdBackup.AddCommand(cmdBackupRestore)
+
+	return cmdBackup
+}

cmd/crowdsec-cli/ban.go

@@ -0,0 +1,326 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"net"
+	"os"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/crowdsecurity/crowdsec/pkg/outputs"
+	"github.com/crowdsecurity/crowdsec/pkg/parser"
+	"github.com/crowdsecurity/crowdsec/pkg/types"
+
+	"github.com/olekukonko/tablewriter"
+	log "github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+)
+
+var remediationType string
+var atTime string
+var all bool
+
+func simpleBanToSignal(targetIP string, reason string, expirationStr string, action string, asName string, asNum string, country string, banSource string) (types.SignalOccurence, error) {
+	var signalOcc types.SignalOccurence
+
+	expiration, err := time.ParseDuration(expirationStr)
+	if err != nil {
+		return signalOcc, err
+	}
+
+	asOrgInt := 0
+	if asNum != "" {
+		asOrgInt, err = strconv.Atoi(asNum)
+		if err != nil {
+			log.Infof("Invalid as value %s : %s", asNum, err)
+		}
+	}
+
+	banApp := types.BanApplication{
+		MeasureSource: banSource,
+		MeasureType:   action,
+		Until:         time.Now().Add(expiration),
+		IpText:        targetIP,
+		TargetCN:      country,
+		TargetAS:      asOrgInt,
+		TargetASName:  asName,
+		Reason:        reason,
+	}
+	var parsedIP net.IP
+	var parsedRange *net.IPNet
+	if strings.Contains(targetIP, "/") {
+		if parsedIP, parsedRange, err = net.ParseCIDR(targetIP); err != nil {
+			return signalOcc, fmt.Errorf("'%s' is not a valid CIDR", targetIP)
+		}
+		if parsedRange == nil {
+			return signalOcc, fmt.Errorf("Unable to parse network : %s", err)
+		}
+		banApp.StartIp = types.IP2Int(parsedRange.IP)
+		banApp.EndIp = types.IP2Int(types.LastAddress(parsedRange))
+	} else {
+		parsedIP = net.ParseIP(targetIP)
+		if parsedIP == nil {
+			return signalOcc, fmt.Errorf("'%s' is not a valid IP", targetIP)
+		}
+	}
+
+	var banApps = make([]types.BanApplication, 1)
+	banApps = append(banApps, banApp)
+	signalOcc = types.SignalOccurence{
+		Scenario:                            reason,
+		Events_count:                        1,
+		Start_at:                            time.Now(),
+		Stop_at:                             time.Now(),
+		BanApplications:                     banApps,
+		Source_ip:                           targetIP,
+		Source_AutonomousSystemNumber:       asNum,
+		Source_AutonomousSystemOrganization: asName,
+		Source_Country:                      country,
+	}
+	return signalOcc, nil
+}
+
+func BanList() error {
+	at := time.Now()
+	if atTime != "" {
+		_, at = parser.GenDateParse(atTime)
+		if at.IsZero() {
+			return fmt.Errorf("Unable to parse date '%s'", atTime)
+		}
+	}
+	ret, err := outputCTX.ReadAT(at)
+	if err != nil {
+		return fmt.Errorf("unable to get records from sqlite : %v", err)
+	}
+	if config.output == "json" {
+		x, _ := json.MarshalIndent(ret, "", " ")
+		fmt.Printf("%s", string(x))
+	} else if config.output == "human" {
+
+		uniqAS := map[string]bool{}
+		uniqCN := map[string]bool{}
+
+		table := tablewriter.NewWriter(os.Stdout)
+		table.SetHeader([]string{"Source", "Ip", "Reason", "Bans", "Action", "Country", "AS", "Events", "Expiration"})
+
+		dispcount := 0
+		totcount := 0
+		apicount := 0
+		for _, rm := range ret {
+			if !all && rm["source"] == "api" {
+				apicount++
+				if _, ok := uniqAS[rm["as"]]; !ok {
+					uniqAS[rm["as"]] = true
+				}
+				if _, ok := uniqCN[rm["cn"]]; !ok {
+					uniqCN[rm["cn"]] = true
+				}
+				continue
+			}
+			if dispcount < 20 {
+				table.Append([]string{rm["source"], rm["iptext"], rm["reason"], rm["bancount"], rm["action"], rm["cn"], rm["as"], rm["events_count"], rm["until"]})
+			}
+			totcount++
+			dispcount++
+
+		}
+		if dispcount > 0 {
+			if !all {
+				fmt.Printf("%d local decisions:\n", totcount)
+			}
+			table.Render() // Send output
+			if dispcount > 20 {
+				fmt.Printf("Additional records stripped.\n")
+			}
+		} else {
+			fmt.Printf("No local decisions.\n")
+		}
+		if !all {
+			fmt.Printf("And %d records from API, %d distinct AS, %d distinct countries\n", apicount, len(uniqAS), len(uniqCN))
+		}
+	}
+	return nil
+}
+
+func BanAdd(target string, duration string, reason string, action string) error {
+	var signalOcc types.SignalOccurence
+	var err error
+
+	signalOcc, err = simpleBanToSignal(target, reason, duration, action, "", "", "", "cli")
+	if err != nil {
+		return fmt.Errorf("Unable to insert ban : %v", err)
+	}
+	err = outputCTX.Insert(signalOcc)
+	if err != nil {
+		return err
+	}
+	err = outputCTX.Flush()
+	if err != nil {
+		return err
+	}
+	log.Infof("Wrote ban to database.")
+	return nil
+}
+
+func banFlush() error {
+	allBa := types.BanApplication{}
+	records := dbctx.Db.Delete(&allBa)
+	if records.Error != nil {
+		return records.Error
+	}
+	return nil
+}
+
+func NewBanCmds() *cobra.Command {
+	/*TODO : add a remediation type*/
+	var cmdBan = &cobra.Command{
+		Use:   "ban [command] <target> <duration> <reason>",
+		Short: "Manage bans/mitigations",
+		Long: `This is the main interaction point with local ban database for humans.
+
+You can add/delete/list or flush current bans in your local ban DB.`,
+		Args: cobra.MinimumNArgs(1),
+		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
+			var err error
+			if !config.configured {
+				return fmt.Errorf("you must configure cli before using bans")
+			}
+
+			outputConfig := outputs.OutputFactory{
+				BackendFolder: config.BackendPluginFolder,
+			}
+
+			outputCTX, err = outputs.NewOutput(&outputConfig, false)
+			if err != nil {
+				return fmt.Errorf(err.Error())
+			}
+			return nil
+		},
+	}
+	cmdBan.PersistentFlags().StringVar(&config.dbPath, "db", "", "Set path to SQLite DB.")
+	cmdBan.PersistentFlags().StringVar(&remediationType, "remediation", "ban", "Set specific remediation type : ban|slow|captcha")
+	cmdBan.Flags().SortFlags = false
+	cmdBan.PersistentFlags().SortFlags = false
+
+	var cmdBanAdd = &cobra.Command{
+		Use:   "add [ip|range] <target> <duration> <reason>",
+		Short: "Adds a ban against a given ip/range for the provided duration",
+		Long: `
+Allows to add a ban against a specific ip or range target for a specific duration.  
+
+The duration argument can be expressed in seconds(s), minutes(m) or hours (h).
+		
+See [time.ParseDuration](https://golang.org/pkg/time/#ParseDuration) for more informations.`,
+		Example: `cscli ban add ip 1.2.3.4 24h "scan"  
+cscli ban add range 1.2.3.0/24 24h "the whole range"`,
+		Args: cobra.MinimumNArgs(4),
+	}
+	cmdBan.AddCommand(cmdBanAdd)
+	var cmdBanAddIp = &cobra.Command{
+		Use:     "ip <target> <duration> <reason>",
+		Short:   "Adds the specific ip to the ban db",
+		Long:    `Duration must be [time.ParseDuration](https://golang.org/pkg/time/#ParseDuration), expressed in s/m/h.`,
+		Example: `cscli ban add ip 1.2.3.4 12h "the scan"`,
+		Args:    cobra.ExactArgs(3),
+		Run: func(cmd *cobra.Command, args []string) {
+			if err := BanAdd(args[0], args[1], args[2], remediationType); err != nil {
+				log.Fatalf("failed to add ban to sqlite : %v", err)
+			}
+		},
+	}
+	cmdBanAdd.AddCommand(cmdBanAddIp)
+	var cmdBanAddRange = &cobra.Command{
+		Use:     "range <target> <duration> <reason>",
+		Short:   "Adds the specific ip to the ban db",
+		Long:    `Duration must be [time.ParseDuration](https://golang.org/pkg/time/#ParseDuration) compatible, expressed in s/m/h.`,
+		Example: `cscli ban add range 1.2.3.0/24 12h "the whole range"`,
+		Args:    cobra.ExactArgs(3),
+		Run: func(cmd *cobra.Command, args []string) {
+			if err := BanAdd(args[0], args[1], args[2], remediationType); err != nil {
+				log.Fatalf("failed to add ban to sqlite : %v", err)
+			}
+		},
+	}
+	cmdBanAdd.AddCommand(cmdBanAddRange)
+	var cmdBanDel = &cobra.Command{
+		Use:   "del [command] <target>",
+		Short: "Delete bans from db",
+		Long:  "The removal of the bans can be applied on a single IP address or directly on a IP range.",
+		Example: `cscli ban del ip 1.2.3.4  
+cscli ban del range 1.2.3.0/24`,
+		Args: cobra.MinimumNArgs(2),
+	}
+	cmdBan.AddCommand(cmdBanDel)
+
+	var cmdBanFlush = &cobra.Command{
+		Use:     "flush",
+		Short:   "Fush ban DB",
+		Example: `cscli ban flush`,
+		Args:    cobra.NoArgs,
+		Run: func(cmd *cobra.Command, args []string) {
+			if err := outputCTX.DeleteAll(); err != nil {
+				log.Fatalf(err.Error())
+			}
+			log.Printf("Ban DB flushed")
+		},
+	}
+	cmdBan.AddCommand(cmdBanFlush)
+	var cmdBanDelIp = &cobra.Command{
+		Use:     "ip <target>",
+		Short:   "Delete bans for given ip from db",
+		Example: `cscli ban del ip 1.2.3.4`,
+		Args:    cobra.ExactArgs(1),
+		Run: func(cmd *cobra.Command, args []string) {
+			count, err := outputCTX.Delete(args[0])
+			if err != nil {
+				log.Fatalf("failed to delete %s : %v", args[0], err)
+			}
+			log.Infof("Deleted %d entries", count)
+		},
+	}
+	cmdBanDel.AddCommand(cmdBanDelIp)
+	var cmdBanDelRange = &cobra.Command{
+		Use:     "range <target>",
+		Short:   "Delete bans for given ip from db",
+		Example: `cscli ban del range 1.2.3.0/24`,
+		Args:    cobra.ExactArgs(1),
+		Run: func(cmd *cobra.Command, args []string) {
+			count, err := outputCTX.Delete(args[0])
+			if err != nil {
+				log.Fatalf("failed to delete %s : %v", args[0], err)
+			}
+			log.Infof("Deleted %d entries", count)
+		},
+	}
+	cmdBanDel.AddCommand(cmdBanDelRange)
+
+	var cmdBanList = &cobra.Command{
+		Use:   "list",
+		Short: "List local or api bans/remediations",
+		Long: `List the bans, by default only local decisions.
+
+If --all/-a is specified, api-provided bans will be displayed too.
+
+Time can be specified with --at and support a variety of date formats:  
+ - Jan  2 15:04:05  
+ - Mon Jan 02 15:04:05.000000 2006  
+ - 2006-01-02T15:04:05Z07:00  
+ - 2006/01/02  
+ - 2006/01/02 15:04  
+ - 2006-01-02  
+ - 2006-01-02 15:04
+`,
+		Args: cobra.ExactArgs(0),
+		Run: func(cmd *cobra.Command, args []string) {
+			if err := BanList(); err != nil {
+				log.Fatalf("failed to list bans : %v", err)
+			}
+		},
+	}
+	cmdBanList.PersistentFlags().StringVar(&atTime, "at", "", "List bans at given time")
+	cmdBanList.PersistentFlags().BoolVarP(&all, "all", "a", false, "List as well bans received from API")
+	cmdBan.AddCommand(cmdBanList)
+	return cmdBan
+}

cmd/crowdsec-cli/config.go

@@ -0,0 +1,160 @@
+package main
+
+import (
+	"bufio"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path"
+	"strings"
+
+	log "github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+	"gopkg.in/yaml.v2"
+)
+
+/*CliCfg is the cli configuration structure, might be unexported*/
+type cliConfig struct {
+	configured          bool
+	simulation          bool      /*are we in simulation mode*/
+	configFolder        string    `yaml:"cliconfig,omitempty"` /*overload ~/.cscli/*/
+	output              string    /*output is human, json*/
+	logLevel            log.Level /*debug,info,warning,error*/
+	hubFolder           string
+	InstallFolder       string `yaml:"installdir"` /*/etc/crowdsec/*/
+	BackendPluginFolder string `yaml:"backend"`
+	dbPath              string
+}
+
+func interactiveCfg() error {
+	var err error
+	reader := bufio.NewReader(os.Stdin)
+	fmt.Print("crowdsec installation directory (default: /etc/crowdsec/crowdsec/): ")
+	config.InstallFolder, err = reader.ReadString('\n')
+	config.InstallFolder = strings.Replace(config.InstallFolder, "\n", "", -1) //CRLF to LF (windows)
+	if config.InstallFolder == "" {
+		config.InstallFolder = "/etc/crowdsec/crowdsec/"
+	}
+	if err != nil {
+		log.Fatalf("failed to read input : %v", err.Error())
+	}
+
+	fmt.Print("crowdsec backend plugin directory (default: /etc/crowdsec/plugin/backend): ")
+	config.BackendPluginFolder, err = reader.ReadString('\n')
+	config.BackendPluginFolder = strings.Replace(config.BackendPluginFolder, "\n", "", -1) //CRLF to LF (windows)
+	if config.BackendPluginFolder == "" {
+		config.BackendPluginFolder = "/etc/crowdsec/plugin/backend"
+	}
+	if err != nil {
+		log.Fatalf("failed to read input : %v", err.Error())
+	}
+	if err := writeCfg(); err != nil {
+		log.Fatalf("failed writting configuration file : %s", err)
+	}
+	return nil
+}
+
+func writeCfg() error {
+
+	if config.configFolder == "" {
+		return fmt.Errorf("config dir is unset")
+	}
+
+	config.hubFolder = config.configFolder + "/hub/"
+	if _, err := os.Stat(config.hubFolder); os.IsNotExist(err) {
+
+		log.Warningf("creating skeleton!")
+		if err := os.MkdirAll(config.hubFolder, os.ModePerm); err != nil {
+			return fmt.Errorf("failed to create missing directory : '%s'", config.hubFolder)
+		}
+	}
+	out := path.Join(config.configFolder, "/config")
+	configYaml, err := yaml.Marshal(&config)
+	if err != nil {
+		return fmt.Errorf("failed marshaling config: %s", err)
+	}
+	err = ioutil.WriteFile(out, configYaml, 0644)
+	if err != nil {
+		return fmt.Errorf("failed to write to %s : %s", out, err)
+	}
+	log.Infof("wrote config to %s ", out)
+	return nil
+}
+
+func NewConfigCmd() *cobra.Command {
+
+	var cmdConfig = &cobra.Command{
+		Use:   "config [command] <value>",
+		Short: "Allows to view/edit cscli config",
+		Long: `Allow to configure sqlite path and installation directory.
+If no commands are specified, config is in interactive mode.`,
+		Example: ` - cscli config show
+- cscli config prompt`,
+		Args: cobra.ExactArgs(1),
+	}
+	var cmdConfigShow = &cobra.Command{
+		Use:   "show",
+		Short: "Displays current config",
+		Long:  `Displays the current cli configuration.`,
+		Args:  cobra.ExactArgs(0),
+		Run: func(cmd *cobra.Command, args []string) {
+			if config.output == "json" {
+				log.WithFields(log.Fields{
+					"installdir": config.InstallFolder,
+					"cliconfig":  path.Join(config.configFolder, "/config"),
+				}).Warning("Current config")
+			} else {
+				x, err := yaml.Marshal(config)
+				if err != nil {
+					log.Fatalf("failed to marshal current configuration : %v", err)
+				}
+				fmt.Printf("%s", x)
+				fmt.Printf("#cliconfig: %s", path.Join(config.configFolder, "/config"))
+			}
+		},
+	}
+	cmdConfig.AddCommand(cmdConfigShow)
+	var cmdConfigInterctive = &cobra.Command{
+		Use:   "prompt",
+		Short: "Prompt for configuration values in an interactive fashion",
+		Long:  `Start interactive configuration of cli. It will successively ask for install dir, db path.`,
+		Args:  cobra.ExactArgs(0),
+		Run: func(cmd *cobra.Command, args []string) {
+			err := interactiveCfg()
+			if err != nil {
+				log.Fatalf("Failed to run interactive config : %s", err)
+			}
+			log.Warningf("Configured, please run update.")
+		},
+	}
+	cmdConfig.AddCommand(cmdConfigInterctive)
+	var cmdConfigInstalldir = &cobra.Command{
+		Use:   "installdir [value]",
+		Short: `Configure installation directory`,
+		Long:  `Configure the installation directory of crowdsec, such as /etc/crowdsec/crowdsec/`,
+		Args:  cobra.ExactArgs(1),
+		Run: func(cmd *cobra.Command, args []string) {
+			config.InstallFolder = args[0]
+			if err := writeCfg(); err != nil {
+				log.Fatalf("failed writting configuration: %s", err)
+			}
+		},
+	}
+	cmdConfig.AddCommand(cmdConfigInstalldir)
+
+	var cmdConfigBackendFolder = &cobra.Command{
+		Use:   "backend [value]",
+		Short: `Configure installation directory`,
+		Long:  `Configure the backend plugin directory of crowdsec, such as /etc/crowdsec/plugins/backend`,
+		Args:  cobra.ExactArgs(1),
+		Run: func(cmd *cobra.Command, args []string) {
+			config.BackendPluginFolder = args[0]
+			if err := writeCfg(); err != nil {
+				log.Fatalf("failed writting configuration: %s", err)
+			}
+		},
+	}
+	cmdConfig.AddCommand(cmdConfigBackendFolder)
+
+	return cmdConfig
+}

cmd/crowdsec-cli/dashboard.go

@@ -0,0 +1,371 @@
+package main
+
+import (
+	"archive/zip"
+	"bufio"
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"os"
+	"path"
+	"time"
+
+	"github.com/crowdsecurity/crowdsec/pkg/cwversion"
+	"github.com/dghubble/sling"
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/container"
+	"github.com/docker/docker/api/types/mount"
+	"github.com/docker/docker/client"
+	"github.com/docker/go-connections/nat"
+	log "github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+)
+
+var (
+	metabaseImage  = "metabase/metabase"
+	metabaseDbURI  = "https://crowdsec-statics-assets.s3-eu-west-1.amazonaws.com/metabase.db.zip"
+	metabaseDbPath = "/var/lib/crowdsec/data"
+	/**/
+	metabaseListenAddress = "127.0.0.1"
+	metabaseListenPort    = "3000"
+	metabaseContainerID   = "/crowdsec-metabase"
+	/*informations needed to setup a random password on user's behalf*/
+	metabaseURI          = "http://localhost:3000/api/"
+	metabaseURISession   = "session"
+	metabaseURIRescan    = "database/2/rescan_values"
+	metabaseURIUpdatepwd = "user/1/password"
+	defaultPassword      = "c6cmetabase"
+	defaultEmail         = "metabase@crowdsec.net"
+)
+
+func NewDashboardCmd() *cobra.Command {
+	/* ---- UPDATE COMMAND */
+	var cmdDashboard = &cobra.Command{
+		Use:   "dashboard",
+		Short: "Start a dashboard (metabase) container.",
+		Long:  `Start a metabase container exposing dashboards and metrics.`,
+		Args:  cobra.ExactArgs(1),
+		Example: `cscli dashboard setup
+cscli dashboard start
+cscli dashboard stop
+cscli dashboard setup --force`,
+	}
+
+	var force bool
+	var cmdDashSetup = &cobra.Command{
+		Use:   "setup",
+		Short: "Setup a metabase container.",
+		Long:  `Perform a metabase docker setup, download standard dashboards, create a fresh user and start the container`,
+		Args:  cobra.ExactArgs(0),
+		Example: `cscli dashboard setup
+cscli dashboard setup --force
+cscli dashboard setup -l 0.0.0.0 -p 443
+ `,
+		Run: func(cmd *cobra.Command, args []string) {
+			if err := downloadMetabaseDB(force); err != nil {
+				log.Fatalf("Failed to download metabase DB : %s", err)
+			}
+			log.Infof("Downloaded metabase DB")
+			if err := createMetabase(); err != nil {
+				log.Fatalf("Failed to start metabase container : %s", err)
+			}
+			log.Infof("Started metabase")
+			newpassword := generatePassword()
+			if err := resetMetabasePassword(newpassword); err != nil {
+				log.Fatalf("Failed to reset password : %s", err)
+			}
+			log.Infof("Setup finished")
+			log.Infof("url : http://%s:%s", metabaseListenAddress, metabaseListenPort)
+			log.Infof("username: %s", defaultEmail)
+			log.Infof("password: %s", newpassword)
+		},
+	}
+	cmdDashSetup.Flags().BoolVarP(&force, "force", "f", false, "Force setup : override existing files.")
+	cmdDashSetup.Flags().StringVarP(&metabaseDbPath, "dir", "d", metabaseDbPath, "Shared directory with metabase container.")
+	cmdDashSetup.Flags().StringVarP(&metabaseListenAddress, "listen", "l", metabaseListenAddress, "Listen address of container")
+	cmdDashSetup.Flags().StringVarP(&metabaseListenPort, "port", "p", metabaseListenPort, "Listen port of container")
+	cmdDashboard.AddCommand(cmdDashSetup)
+
+	var cmdDashStart = &cobra.Command{
+		Use:   "start",
+		Short: "Start the metabase container.",
+		Long:  `Stats the metabase container using docker.`,
+		Args:  cobra.ExactArgs(0),
+		Run: func(cmd *cobra.Command, args []string) {
+			if err := startMetabase(); err != nil {
+				log.Fatalf("Failed to start metabase container : %s", err)
+			}
+			log.Infof("Started metabase")
+			log.Infof("url : http://%s:%s", metabaseListenAddress, metabaseListenPort)
+		},
+	}
+	cmdDashboard.AddCommand(cmdDashStart)
+
+	var remove bool
+	var cmdDashStop = &cobra.Command{
+		Use:   "stop",
+		Short: "Stops the metabase container.",
+		Long:  `Stops the metabase container using docker.`,
+		Args:  cobra.ExactArgs(0),
+		Run: func(cmd *cobra.Command, args []string) {
+			if err := stopMetabase(remove); err != nil {
+				log.Fatalf("Failed to stop metabase container : %s", err)
+			}
+		},
+	}
+	cmdDashStop.Flags().BoolVarP(&remove, "remove", "r", false, "remove (docker rm) container as well.")
+	cmdDashboard.AddCommand(cmdDashStop)
+	return cmdDashboard
+}
+
+func downloadMetabaseDB(force bool) error {
+
+	metabaseDBSubpath := path.Join(metabaseDbPath, "metabase.db")
+
+	_, err := os.Stat(metabaseDBSubpath)
+	if err == nil && force == false {
+		log.Printf("%s exists, skip.", metabaseDBSubpath)
+		return nil
+	}
+
+	if err := os.MkdirAll(metabaseDBSubpath, 0755); err != nil {
+		return fmt.Errorf("failed to create %s : %s", metabaseDBSubpath, err)
+	}
+
+	req, err := http.NewRequest("GET", metabaseDbURI, nil)
+	if err != nil {
+		return fmt.Errorf("failed to build request to fetch metabase db : %s", err)
+	}
+	//This needs to be removed once we move the zip out of github
+	req.Header.Add("Accept", `application/vnd.github.v3.raw`)
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return fmt.Errorf("failed request to fetch metabase db : %s", err)
+	}
+	if resp.StatusCode != 200 {
+		return fmt.Errorf("got http %d while requesting metabase db %s, stop", resp.StatusCode, metabaseDbURI)
+	}
+	defer resp.Body.Close()
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return fmt.Errorf("failed request read while fetching metabase db : %s", err)
+	}
+
+	log.Printf("Got %d bytes archive", len(body))
+	if err := extractMetabaseDB(bytes.NewReader(body)); err != nil {
+		return fmt.Errorf("while extracting zip : %s", err)
+	}
+	return nil
+}
+
+func extractMetabaseDB(buf *bytes.Reader) error {
+	r, err := zip.NewReader(buf, int64(buf.Len()))
+	if err != nil {
+		log.Fatal(err)
+	}
+	for _, f := range r.File {
+		tfname := fmt.Sprintf("%s/%s", metabaseDbPath, f.Name)
+		log.Debugf("%s -> %d", f.Name, f.UncompressedSize64)
+		if f.UncompressedSize64 == 0 {
+			continue
+		}
+		tfd, err := os.OpenFile(tfname, os.O_RDWR|os.O_TRUNC|os.O_CREATE, 0644)
+		if err != nil {
+			return fmt.Errorf("Failed opening target file '%s' : %s", tfname, err)
+		}
+		rc, err := f.Open()
+		if err != nil {
+			return fmt.Errorf("While opening zip content %s : %s", f.Name, err)
+		}
+		written, err := io.Copy(tfd, rc)
+		if err == io.EOF {
+			log.Printf("files finished ok")
+		} else if err != nil {
+			return fmt.Errorf("While copying content to %s : %s", tfname, err)
+		}
+		log.Infof("written %d bytes to %s", written, tfname)
+		rc.Close()
+	}
+	return nil
+}
+
+func resetMetabasePassword(newpassword string) error {
+
+	httpctx := sling.New().Base(metabaseURI).Set("User-Agent", fmt.Sprintf("CrowdWatch/%s", cwversion.VersionStr()))
+
+	log.Printf("Waiting for metabase API to be up (can take up to a minute)")
+	for {
+		sessionreq, err := httpctx.New().Post(metabaseURISession).BodyJSON(map[string]string{"username": defaultEmail, "password": defaultPassword}).Request()
+		if err != nil {
+			return fmt.Errorf("api signin: HTTP request creation failed: %s", err)
+		}
+		httpClient := http.Client{Timeout: 20 * time.Second}
+		resp, err := httpClient.Do(sessionreq)
+		if err != nil {
+			fmt.Printf(".")
+			log.Debugf("While waiting for metabase to be up : %s", err)
+			time.Sleep(1 * time.Second)
+			continue
+		}
+		defer resp.Body.Close()
+		fmt.Printf("\n")
+		log.Printf("Metabase API is up")
+		body, err := ioutil.ReadAll(resp.Body)
+		if err != nil {
+			return fmt.Errorf("metabase session unable to read API response body: '%s'", err)
+		}
+		if resp.StatusCode != 200 {
+			return fmt.Errorf("metabase session http error (%d): %s", resp.StatusCode, string(body))
+		}
+		log.Printf("Successfully authenticated")
+		jsonResp := make(map[string]string)
+		err = json.Unmarshal(body, &jsonResp)
+		if err != nil {
+			return fmt.Errorf("failed to unmarshal metabase api response '%s': %s", string(body), err.Error())
+		}
+		log.Debugf("unmarshaled response : %v", jsonResp)
+		httpctx = httpctx.Set("Cookie", fmt.Sprintf("metabase.SESSION=%s", jsonResp["id"]))
+		break
+	}
+
+	/*rescan values*/
+	sessionreq, err := httpctx.New().Post(metabaseURIRescan).Request()
+	if err != nil {
+		return fmt.Errorf("metabase rescan_values http error : %s", err)
+	}
+	httpClient := http.Client{Timeout: 20 * time.Second}
+	resp, err := httpClient.Do(sessionreq)
+	if err != nil {
+		return fmt.Errorf("While trying to do rescan api call to metabase : %s", err)
+	}
+	defer resp.Body.Close()
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return fmt.Errorf("While reading rescan api call response : %s", err)
+	}
+	if resp.StatusCode != 200 {
+		return fmt.Errorf("Got '%s' (http:%d) while trying to rescan metabase", string(body), resp.StatusCode)
+	}
+	/*update password*/
+	sessionreq, err = httpctx.New().Put(metabaseURIUpdatepwd).BodyJSON(map[string]string{
+		"id":           "1",
+		"password":     newpassword,
+		"old_password": defaultPassword}).Request()
+	if err != nil {
+		return fmt.Errorf("metabase password change http error : %s", err)
+	}
+	httpClient = http.Client{Timeout: 20 * time.Second}
+	resp, err = httpClient.Do(sessionreq)
+	if err != nil {
+		return fmt.Errorf("While trying to reset metabase password : %s", err)
+	}
+	defer resp.Body.Close()
+	body, err = ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return fmt.Errorf("while reading from %s: '%s'", metabaseURIUpdatepwd, err)
+	}
+	if resp.StatusCode != 200 {
+		log.Printf("Got %s (http:%d) while trying to reset password.", string(body), resp.StatusCode)
+		log.Printf("Password has probably already been changed.")
+		log.Printf("Use the dashboard install command to reset existing setup.")
+		return fmt.Errorf("got http error %d on %s : %s", resp.StatusCode, metabaseURIUpdatepwd, string(body))
+	}
+	log.Printf("Changed password !")
+	return nil
+}
+
+func startMetabase() error {
+	ctx := context.Background()
+	cli, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation())
+	if err != nil {
+		return fmt.Errorf("Failed to create docker client : %s", err)
+	}
+
+	if err := cli.ContainerStart(ctx, metabaseContainerID, types.ContainerStartOptions{}); err != nil {
+		return fmt.Errorf("Failed while starting %s : %s", metabaseContainerID, err)
+	}
+
+	return nil
+}
+
+func stopMetabase(remove bool) error {
+	log.Printf("Stop docker metabase %s", metabaseContainerID)
+	ctx := context.Background()
+	cli, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation())
+	if err != nil {
+		return fmt.Errorf("Failed to create docker client : %s", err)
+	}
+	var to time.Duration = 20 * time.Second
+	if err := cli.ContainerStop(ctx, metabaseContainerID, &to); err != nil {
+		return fmt.Errorf("Failed while stopping %s : %s", metabaseContainerID, err)
+	}
+
+	if remove {
+		log.Printf("Removing docker metabase %s", metabaseContainerID)
+		if err := cli.ContainerRemove(ctx, metabaseContainerID, types.ContainerRemoveOptions{}); err != nil {
+			return fmt.Errorf("Failed remove container %s : %s", metabaseContainerID, err)
+		}
+	}
+	return nil
+}
+
+func createMetabase() error {
+	ctx := context.Background()
+	cli, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation())
+	if err != nil {
+		return fmt.Errorf("Failed to start docker client : %s", err)
+	}
+
+	log.Printf("Pulling docker image %s", metabaseImage)
+	reader, err := cli.ImagePull(ctx, metabaseImage, types.ImagePullOptions{})
+	if err != nil {
+		return fmt.Errorf("Failed to pull docker image : %s", err)
+	}
+	defer reader.Close()
+	scanner := bufio.NewScanner(reader)
+	for scanner.Scan() {
+		fmt.Print(".")
+	}
+	if err := scanner.Err(); err != nil {
+		return fmt.Errorf("failed to read imagepull reader: %s", err)
+	}
+	fmt.Print("\n")
+
+	hostConfig := &container.HostConfig{
+		PortBindings: nat.PortMap{
+			"3000/tcp": []nat.PortBinding{
+				{
+					HostIP:   metabaseListenAddress,
+					HostPort: metabaseListenPort,
+				},
+			},
+		},
+		Mounts: []mount.Mount{
+			{
+				Type:   mount.TypeBind,
+				Source: metabaseDbPath,
+				Target: "/metabase-data",
+			},
+		},
+	}
+	dockerConfig := &container.Config{
+		Image: metabaseImage,
+		Tty:   true,
+		Env:   []string{"MB_DB_FILE=/metabase-data/metabase.db"},
+	}
+
+	log.Printf("Creating container")
+	resp, err := cli.ContainerCreate(ctx, dockerConfig, hostConfig, nil, metabaseContainerID)
+	if err != nil {
+		return fmt.Errorf("Failed to create container : %s", err)
+	}
+	log.Printf("Starting container")
+	if err := cli.ContainerStart(ctx, resp.ID, types.ContainerStartOptions{}); err != nil {
+		return fmt.Errorf("Failed to start docker container : %s", err)
+	}
+	return nil
+}

cmd/crowdsec-cli/doc/cwcli.md

@@ -0,0 +1,57 @@
+## cscli
+
+cscli allows you to manage crowdsec
+
+### Synopsis
+
+cscli is the main command to interact with your crowdsec service, scenarios & db.
+It is meant to allow you to manage bans, parsers/scenarios/etc, api and generally manage you crowdsec setup.
+
+### Examples
+
+```
+View/Add/Remove bans:  
+ - cscli ban list  
+ - cscli ban add ip 1.2.3.4 24h 'go away'  
+ - cscli ban del 1.2.3.4  
+		
+View/Add/Upgrade/Remove scenarios and parsers:  
+ - cscli list  
+ - cscli install collection crowdsec/linux-web  
+ - cscli remove scenario crowdsec/ssh_enum  
+ - cscli upgrade --all  
+
+API interaction:
+ - cscli api pull
+ - cscli api register
+ 
+```
+
+### Options
+
+```
+  -c, --config-dir string   Configuration directory to use. (default "/etc/crowdsec/cscli/")
+  -o, --output string       Output format : human, json, raw. (default "human")
+      --debug               Set logging to debug.
+      --info                Set logging to info.
+      --warning             Set logging to warning.
+      --error               Set logging to error.
+  -h, --help                help for cscli
+```
+
+### SEE ALSO
+
+* [cscli api](cscli_api.md)	 - Crowdsec API interaction
+* [cscli backup](cscli_backup.md)	 - Backup or restore configuration (api, parsers, scenarios etc.) to/from directory
+* [cscli ban](cscli_ban.md)	 - Manage bans/mitigations
+* [cscli config](cscli_config.md)	 - Allows to view/edit cscli config
+* [cscli dashboard](cscli_dashboard.md)	 - Start a dashboard (metabase) container.
+* [cscli inspect](cscli_inspect.md)	 - Inspect configuration(s)
+* [cscli install](cscli_install.md)	 - Install configuration(s) from hub
+* [cscli list](cscli_list.md)	 - List enabled configs
+* [cscli metrics](cscli_metrics.md)	 - Display crowdsec prometheus metrics.
+* [cscli remove](cscli_remove.md)	 - Remove/disable configuration(s)
+* [cscli update](cscli_update.md)	 - Fetch available configs from hub
+* [cscli upgrade](cscli_upgrade.md)	 - Upgrade configuration(s)
+
+###### Auto generated by spf13/cobra on 14-May-2020

cmd/crowdsec-cli/doc/cwcli_api.md

@@ -0,0 +1,49 @@
+## cscli api
+
+Crowdsec API interaction
+
+### Synopsis
+
+
+Allow to register your machine into crowdsec API to send and receive signal.
+		
+
+### Examples
+
+```
+
+cscli api register      # Register to Crowdsec API
+cscli api pull          # Pull malevolant IPs from Crowdsec API
+cscli api reset         # Reset your machines credentials
+cscli api enroll        # Enroll your machine to the user account you created on Crowdsec backend
+cscli api credentials   # Display your API credentials
+
+```
+
+### Options
+
+```
+  -h, --help   help for api
+```
+
+### Options inherited from parent commands
+
+```
+  -c, --config-dir string   Configuration directory to use. (default "/etc/crowdsec/cscli/")
+      --debug               Set logging to debug.
+      --error               Set logging to error.
+      --info                Set logging to info.
+  -o, --output string       Output format : human, json, raw. (default "human")
+      --warning             Set logging to warning.
+```
+
+### SEE ALSO
+
+* [cscli](cscli.md)	 - cscli allows you to manage crowdsec
+* [cscli api credentials](cscli_api_credentials.md)	 - Display api credentials
+* [cscli api enroll](cscli_api_enroll.md)	 - Associate your machine to an existing crowdsec user
+* [cscli api pull](cscli_api_pull.md)	 - Pull crowdsec API TopX
+* [cscli api register](cscli_api_register.md)	 - Register on Crowdsec API
+* [cscli api reset](cscli_api_reset.md)	 - Reset password on CrowdSec API
+
+###### Auto generated by spf13/cobra on 14-May-2020

cmd/crowdsec-cli/doc/cwcli_api_credentials.md

@@ -0,0 +1,40 @@
+## cscli api credentials
+
+Display api credentials
+
+### Synopsis
+
+Display api credentials
+
+```
+cscli api credentials [flags]
+```
+
+### Examples
+
+```
+cscli api credentials
+```
+
+### Options
+
+```
+  -h, --help   help for credentials
+```
+
+### Options inherited from parent commands
+
+```
+  -c, --config-dir string   Configuration directory to use. (default "/etc/crowdsec/cscli/")
+      --debug               Set logging to debug.
+      --error               Set logging to error.
+      --info                Set logging to info.
+  -o, --output string       Output format : human, json, raw. (default "human")
+      --warning             Set logging to warning.
+```
+
+### SEE ALSO
+
+* [cscli api](cscli_api.md)	 - Crowdsec API interaction
+
+###### Auto generated by spf13/cobra on 14-May-2020

cmd/crowdsec-cli/doc/cwcli_api_enroll.md

@@ -0,0 +1,41 @@
+## cscli api enroll
+
+Associate your machine to an existing crowdsec user
+
+### Synopsis
+
+Enrolling your machine into your user account will allow for more accurate lists and threat detection. See website to create user account.
+
+```
+cscli api enroll [flags]
+```
+
+### Examples
+
+```
+cscli api enroll -u 1234567890ffff
+```
+
+### Options
+
+```
+  -h, --help          help for enroll
+  -u, --user string   User ID (required)
+```
+
+### Options inherited from parent commands
+
+```
+  -c, --config-dir string   Configuration directory to use. (default "/etc/crowdsec/cscli/")
+      --debug               Set logging to debug.
+      --error               Set logging to error.
+      --info                Set logging to info.
+  -o, --output string       Output format : human, json, raw. (default "human")
+      --warning             Set logging to warning.
+```
+
+### SEE ALSO
+
+* [cscli api](cscli_api.md)	 - Crowdsec API interaction
+
+###### Auto generated by spf13/cobra on 14-May-2020

cmd/crowdsec-cli/doc/cwcli_api_pull.md

@@ -0,0 +1,40 @@
+## cscli api pull
+
+Pull crowdsec API TopX
+
+### Synopsis
+
+Pulls a list of malveolent IPs relevant to your situation and add them into the local ban database.
+
+```
+cscli api pull [flags]
+```
+
+### Examples
+
+```
+cscli api pull
+```
+
+### Options
+
+```
+  -h, --help   help for pull
+```
+
+### Options inherited from parent commands
+
+```
+  -c, --config-dir string   Configuration directory to use. (default "/etc/crowdsec/cscli/")
+      --debug               Set logging to debug.
+      --error               Set logging to error.
+      --info                Set logging to info.
+  -o, --output string       Output format : human, json, raw. (default "human")
+      --warning             Set logging to warning.
+```
+
+### SEE ALSO
+
+* [cscli api](cscli_api.md)	 - Crowdsec API interaction
+
+###### Auto generated by spf13/cobra on 14-May-2020

cmd/crowdsec-cli/doc/cwcli_api_register.md

@@ -0,0 +1,41 @@
+## cscli api register
+
+Register on Crowdsec API
+
+### Synopsis
+
+This command will register your machine to crowdsec API to allow you to receive list of malveolent IPs. 
+		The printed machine_id and password should be added to your api.yaml file.
+
+```
+cscli api register [flags]
+```
+
+### Examples
+
+```
+cscli api register
+```
+
+### Options
+
+```
+  -h, --help   help for register
+```
+
+### Options inherited from parent commands
+
+```
+  -c, --config-dir string   Configuration directory to use. (default "/etc/crowdsec/cscli/")
+      --debug               Set logging to debug.
+      --error               Set logging to error.
+      --info                Set logging to info.
+  -o, --output string       Output format : human, json, raw. (default "human")
+      --warning             Set logging to warning.
+```
+
+### SEE ALSO
+
+* [cscli api](cscli_api.md)	 - Crowdsec API interaction
+
+###### Auto generated by spf13/cobra on 14-May-2020

cmd/crowdsec-cli/doc/cwcli_api_reset.md

@@ -0,0 +1,40 @@
+## cscli api reset
+
+Reset password on CrowdSec API
+
+### Synopsis
+
+Attempts to reset your credentials to the API.
+
+```
+cscli api reset [flags]
+```
+
+### Examples
+
+```
+cscli api reset
+```
+
+### Options
+
+```
+  -h, --help   help for reset
+```
+
+### Options inherited from parent commands
+
+```
+  -c, --config-dir string   Configuration directory to use. (default "/etc/crowdsec/cscli/")
+      --debug               Set logging to debug.
+      --error               Set logging to error.
+      --info                Set logging to info.
+  -o, --output string       Output format : human, json, raw. (default "human")
+      --warning             Set logging to warning.
+```
+
+### SEE ALSO
+
+* [cscli api](cscli_api.md)	 - Crowdsec API interaction
+
+###### Auto generated by spf13/cobra on 14-May-2020

cmd/crowdsec-cli/doc/cwcli_backup.md

@@ -0,0 +1,40 @@
+## cscli backup
+
+Backup or restore configuration (api, parsers, scenarios etc.) to/from directory
+
+### Synopsis
+
+This command is here to help you save and/or restore crowdsec configurations to simple replication
+
+### Examples
+
+```
+cscli backup save ./my-backup
+cscli backup restore ./my-backup
+```
+
+### Options
+
+```
+      --cfgdir string   Configuration directory (default "/etc/crowdsec/")
+  -h, --help            help for backup
+```
+
+### Options inherited from parent commands
+
+```
+  -c, --config-dir string   Configuration directory to use. (default "/etc/crowdsec/cscli/")
+      --debug               Set logging to debug.
+      --error               Set logging to error.
+      --info                Set logging to info.
+  -o, --output string       Output format : human, json, raw. (default "human")
+      --warning             Set logging to warning.
+```
+
+### SEE ALSO
+
+* [cscli](cscli.md)	 - cscli allows you to manage crowdsec
+* [cscli backup restore](cscli_backup_restore.md)	 - Restore configuration (api, parsers, scenarios etc.) from directory
+* [cscli backup save](cscli_backup_save.md)	 - Backup configuration (api, parsers, scenarios etc.) to directory
+
+###### Auto generated by spf13/cobra on 14-May-2020

cmd/crowdsec-cli/doc/cwcli_backup_restore.md

@@ -0,0 +1,50 @@
+## cscli backup restore
+
+Restore configuration (api, parsers, scenarios etc.) from directory
+
+### Synopsis
+
+restore command will try to restore all saved information from <directory> to yor local setup, including :
+
+- Installation of up-to-date scenarios/parsers/... via cscli
+
+- Restauration of tainted/local/out-of-date scenarios/parsers/... file
+
+- Restauration of API credentials (if the existing ones aren't working)
+
+- Restauration of acqusition configuration
+
+
+```
+cscli backup restore <directory> [flags]
+```
+
+### Examples
+
+```
+cscli backup restore ./my-backup
+```
+
+### Options
+
+```
+  -h, --help   help for restore
+```
+
+### Options inherited from parent commands
+
+```
+      --cfgdir string       Configuration directory (default "/etc/crowdsec/")
+  -c, --config-dir string   Configuration directory to use. (default "/etc/crowdsec/cscli/")
+      --debug               Set logging to debug.
+      --error               Set logging to error.
+      --info                Set logging to info.
+  -o, --output string       Output format : human, json, raw. (default "human")
+      --warning             Set logging to warning.
+```
+
+### SEE ALSO
+
+* [cscli backup](cscli_backup.md)	 - Backup or restore configuration (api, parsers, scenarios etc.) to/from directory
+
+###### Auto generated by spf13/cobra on 14-May-2020

cmd/crowdsec-cli/doc/cwcli_backup_save.md

@@ -0,0 +1,51 @@
+## cscli backup save
+
+Backup configuration (api, parsers, scenarios etc.) to directory
+
+### Synopsis
+
+backup command will try to save all relevant informations to crowdsec config, including :
+
+- List of scenarios, parsers, postoverflows and collections that are up-to-date
+
+- Actual backup of tainted/local/out-of-date scenarios, parsers, postoverflows and collections
+
+- Backup of API credentials
+
+- Backup of acqusition configuration
+		
+		
+
+```
+cscli backup save <directory> [flags]
+```
+
+### Examples
+
+```
+cscli backup save ./my-backup
+```
+
+### Options
+
+```
+  -h, --help   help for save
+```
+
+### Options inherited from parent commands
+
+```
+      --cfgdir string       Configuration directory (default "/etc/crowdsec/")
+  -c, --config-dir string   Configuration directory to use. (default "/etc/crowdsec/cscli/")
+      --debug               Set logging to debug.
+      --error               Set logging to error.
+      --info                Set logging to info.
+  -o, --output string       Output format : human, json, raw. (default "human")
+      --warning             Set logging to warning.
+```
+
+### SEE ALSO
+
+* [cscli backup](cscli_backup.md)	 - Backup or restore configuration (api, parsers, scenarios etc.) to/from directory
+
+###### Auto generated by spf13/cobra on 14-May-2020

cmd/crowdsec-cli/doc/cwcli_ban.md

@@ -0,0 +1,38 @@
+## cscli ban
+
+Manage bans/mitigations
+
+### Synopsis
+
+This is the main interaction point with local ban database for humans.
+
+You can add/delete/list or flush current bans in your local ban DB.
+
+### Options
+
+```
+      --db string            Set path to SQLite DB.
+      --remediation string   Set specific remediation type : ban|slow|captcha (default "ban")
+  -h, --help                 help for ban
+```
+
+### Options inherited from parent commands
+
+```
+  -c, --config-dir string   Configuration directory to use. (default "/etc/crowdsec/cscli/")
+      --debug               Set logging to debug.
+      --error               Set logging to error.
+      --info                Set logging to info.
+  -o, --output string       Output format : human, json, raw. (default "human")
+      --warning             Set logging to warning.
+```
+
+### SEE ALSO
+
+* [cscli](cscli.md)	 - cscli allows you to manage crowdsec
+* [cscli ban add](cscli_ban_add.md)	 - Adds a ban against a given ip/range for the provided duration
+* [cscli ban del](cscli_ban_del.md)	 - Delete bans from db
+* [cscli ban flush](cscli_ban_flush.md)	 - Fush ban DB
+* [cscli ban list](cscli_ban_list.md)	 - List local or api bans/remediations
+
+###### Auto generated by spf13/cobra on 14-May-2020

cmd/crowdsec-cli/doc/cwcli_ban_add.md

@@ -0,0 +1,46 @@
+## cscli ban add
+
+Adds a ban against a given ip/range for the provided duration
+
+### Synopsis
+
+
+Allows to add a ban against a specific ip or range target for a specific duration.  
+
+The duration argument can be expressed in seconds(s), minutes(m) or hours (h).
+		
+See [time.ParseDuration](https://golang.org/pkg/time/#ParseDuration) for more informations.
+
+### Examples
+
+```
+cscli ban add ip 1.2.3.4 24h "scan"  
+cscli ban add range 1.2.3.0/24 24h "the whole range"
+```
+
+### Options
+
+```
+  -h, --help   help for add
+```
+
+### Options inherited from parent commands
+
+```
+  -c, --config-dir string    Configuration directory to use. (default "/etc/crowdsec/cscli/")
+      --db string            Set path to SQLite DB.
+      --debug                Set logging to debug.
+      --error                Set logging to error.
+      --info                 Set logging to info.
+  -o, --output string        Output format : human, json, raw. (default "human")
+      --remediation string   Set specific remediation type : ban|slow|captcha (default "ban")
+      --warning              Set logging to warning.
+```
+
+### SEE ALSO
+
+* [cscli ban](cscli_ban.md)	 - Manage bans/mitigations
+* [cscli ban add ip](cscli_ban_add_ip.md)	 - Adds the specific ip to the ban db
+* [cscli ban add range](cscli_ban_add_range.md)	 - Adds the specific ip to the ban db
+
+###### Auto generated by spf13/cobra on 14-May-2020

cmd/crowdsec-cli/doc/cwcli_ban_add_ip.md

@@ -0,0 +1,42 @@
+## cscli ban add ip
+
+Adds the specific ip to the ban db
+
+### Synopsis
+
+Duration must be [time.ParseDuration](https://golang.org/pkg/time/#ParseDuration), expressed in s/m/h.
+
+```
+cscli ban add ip <target> <duration> <reason> [flags]
+```
+
+### Examples
+
+```
+cscli ban add ip 1.2.3.4 12h "the scan"
+```
+
+### Options
+
+```
+  -h, --help   help for ip
+```
+
+### Options inherited from parent commands
+
+```
+  -c, --config-dir string    Configuration directory to use. (default "/etc/crowdsec/cscli/")
+      --db string            Set path to SQLite DB.
+      --debug                Set logging to debug.
+      --error                Set logging to error.
+      --info                 Set logging to info.
+  -o, --output string        Output format : human, json, raw. (default "human")
+      --remediation string   Set specific remediation type : ban|slow|captcha (default "ban")
+      --warning              Set logging to warning.
+```
+
+### SEE ALSO
+
+* [cscli ban add](cscli_ban_add.md)	 - Adds a ban against a given ip/range for the provided duration
+
+###### Auto generated by spf13/cobra on 14-May-2020

cmd/crowdsec-cli/doc/cwcli_ban_add_range.md

@@ -0,0 +1,42 @@
+## cscli ban add range
+
+Adds the specific ip to the ban db
+
+### Synopsis
+
+Duration must be [time.ParseDuration](https://golang.org/pkg/time/#ParseDuration) compatible, expressed in s/m/h.
+
+```
+cscli ban add range <target> <duration> <reason> [flags]
+```
+
+### Examples
+
+```
+cscli ban add range 1.2.3.0/24 12h "the whole range"
+```
+
+### Options
+
+```
+  -h, --help   help for range
+```
+
+### Options inherited from parent commands
+
+```
+  -c, --config-dir string    Configuration directory to use. (default "/etc/crowdsec/cscli/")
+      --db string            Set path to SQLite DB.
+      --debug                Set logging to debug.
+      --error                Set logging to error.
+      --info                 Set logging to info.
+  -o, --output string        Output format : human, json, raw. (default "human")
+      --remediation string   Set specific remediation type : ban|slow|captcha (default "ban")
+      --warning              Set logging to warning.
+```
+
+### SEE ALSO
+
+* [cscli ban add](cscli_ban_add.md)	 - Adds a ban against a given ip/range for the provided duration
+
+###### Auto generated by spf13/cobra on 14-May-2020

cmd/crowdsec-cli/doc/cwcli_ban_del.md

@@ -0,0 +1,41 @@
+## cscli ban del
+
+Delete bans from db
+
+### Synopsis
+
+The removal of the bans can be applied on a single IP address or directly on a IP range.
+
+### Examples
+
+```
+cscli ban del ip 1.2.3.4  
+cscli ban del range 1.2.3.0/24
+```
+
+### Options
+
+```
+  -h, --help   help for del
+```
+
+### Options inherited from parent commands
+
+```
+  -c, --config-dir string    Configuration directory to use. (default "/etc/crowdsec/cscli/")
+      --db string            Set path to SQLite DB.
+      --debug                Set logging to debug.
+      --error                Set logging to error.
+      --info                 Set logging to info.
+  -o, --output string        Output format : human, json, raw. (default "human")
+      --remediation string   Set specific remediation type : ban|slow|captcha (default "ban")
+      --warning              Set logging to warning.
+```
+
+### SEE ALSO
+
+* [cscli ban](cscli_ban.md)	 - Manage bans/mitigations
+* [cscli ban del ip](cscli_ban_del_ip.md)	 - Delete bans for given ip from db
+* [cscli ban del range](cscli_ban_del_range.md)	 - Delete bans for given ip from db
+
+###### Auto generated by spf13/cobra on 14-May-2020

cmd/crowdsec-cli/doc/cwcli_ban_del_ip.md

@@ -0,0 +1,42 @@
+## cscli ban del ip
+
+Delete bans for given ip from db
+
+### Synopsis
+
+Delete bans for given ip from db
+
+```
+cscli ban del ip <target> [flags]
+```
+
+### Examples
+
+```
+cscli ban del ip 1.2.3.4
+```
+
+### Options
+
+```
+  -h, --help   help for ip
+```
+
+### Options inherited from parent commands
+
+```
+  -c, --config-dir string    Configuration directory to use. (default "/etc/crowdsec/cscli/")
+      --db string            Set path to SQLite DB.
+      --debug                Set logging to debug.
+      --error                Set logging to error.
+      --info                 Set logging to info.
+  -o, --output string        Output format : human, json, raw. (default "human")
+      --remediation string   Set specific remediation type : ban|slow|captcha (default "ban")
+      --warning              Set logging to warning.
+```
+
+### SEE ALSO
+
+* [cscli ban del](cscli_ban_del.md)	 - Delete bans from db
+
+###### Auto generated by spf13/cobra on 14-May-2020

cmd/crowdsec-cli/doc/cwcli_ban_del_range.md

@@ -0,0 +1,42 @@
+## cscli ban del range
+
+Delete bans for given ip from db
+
+### Synopsis
+
+Delete bans for given ip from db
+
+```
+cscli ban del range <target> [flags]
+```
+
+### Examples
+
+```
+cscli ban del range 1.2.3.0/24
+```
+
+### Options
+
+```
+  -h, --help   help for range
+```
+
+### Options inherited from parent commands
+
+```
+  -c, --config-dir string    Configuration directory to use. (default "/etc/crowdsec/cscli/")
+      --db string            Set path to SQLite DB.
+      --debug                Set logging to debug.
+      --error                Set logging to error.
+      --info                 Set logging to info.
+  -o, --output string        Output format : human, json, raw. (default "human")
+      --remediation string   Set specific remediation type : ban|slow|captcha (default "ban")
+      --warning              Set logging to warning.
+```
+
+### SEE ALSO
+
+* [cscli ban del](cscli_ban_del.md)	 - Delete bans from db
+
+###### Auto generated by spf13/cobra on 14-May-2020

cmd/crowdsec-cli/doc/cwcli_ban_flush.md

@@ -0,0 +1,42 @@
+## cscli ban flush
+
+Fush ban DB
+
+### Synopsis
+
+Fush ban DB
+
+```
+cscli ban flush [flags]
+```
+
+### Examples
+
+```
+cscli ban flush
+```
+
+### Options
+
+```
+  -h, --help   help for flush
+```
+
+### Options inherited from parent commands
+
+```
+  -c, --config-dir string    Configuration directory to use. (default "/etc/crowdsec/cscli/")
+      --db string            Set path to SQLite DB.
+      --debug                Set logging to debug.
+      --error                Set logging to error.
+      --info                 Set logging to info.
+  -o, --output string        Output format : human, json, raw. (default "human")
+      --remediation string   Set specific remediation type : ban|slow|captcha (default "ban")
+      --warning              Set logging to warning.
+```
+
+### SEE ALSO
+
+* [cscli ban](cscli_ban.md)	 - Manage bans/mitigations
+
+###### Auto generated by spf13/cobra on 14-May-2020

cmd/crowdsec-cli/doc/cwcli_ban_list.md

@@ -0,0 +1,50 @@
+## cscli ban list
+
+List local or api bans/remediations
+
+### Synopsis
+
+List the bans, by default only local decisions.
+
+If --all/-a is specified, api-provided bans will be displayed too.
+
+Time can be specified with --at and support a variety of date formats:  
+ - Jan  2 15:04:05  
+ - Mon Jan 02 15:04:05.000000 2006  
+ - 2006-01-02T15:04:05Z07:00  
+ - 2006/01/02  
+ - 2006/01/02 15:04  
+ - 2006-01-02  
+ - 2006-01-02 15:04
+
+
+```
+cscli ban list [flags]
+```
+
+### Options
+
+```
+  -a, --all         List as well bans received from API
+      --at string   List bans at given time
+  -h, --help        help for list
+```
+
+### Options inherited from parent commands
+
+```
+  -c, --config-dir string    Configuration directory to use. (default "/etc/crowdsec/cscli/")
+      --db string            Set path to SQLite DB.
+      --debug                Set logging to debug.
+      --error                Set logging to error.
+      --info                 Set logging to info.
+  -o, --output string        Output format : human, json, raw. (default "human")
+      --remediation string   Set specific remediation type : ban|slow|captcha (default "ban")
+      --warning              Set logging to warning.
+```
+
+### SEE ALSO
+
+* [cscli ban](cscli_ban.md)	 - Manage bans/mitigations
+
+###### Auto generated by spf13/cobra on 14-May-2020

cmd/crowdsec-cli/doc/cwcli_config.md

@@ -0,0 +1,42 @@
+## cscli config
+
+Allows to view/edit cscli config
+
+### Synopsis
+
+Allow to configure sqlite path and installation directory.
+If no commands are specified, config is in interactive mode.
+
+### Examples
+
+```
+ - cscli config show
+- cscli config prompt
+```
+
+### Options
+
+```
+  -h, --help   help for config
+```
+
+### Options inherited from parent commands
+
+```
+  -c, --config-dir string   Configuration directory to use. (default "/etc/crowdsec/cscli/")
+      --debug               Set logging to debug.
+      --error               Set logging to error.
+      --info                Set logging to info.
+  -o, --output string       Output format : human, json, raw. (default "human")
+      --warning             Set logging to warning.
+```
+
+### SEE ALSO
+
+* [cscli](cscli.md)	 - cscli allows you to manage crowdsec
+* [cscli config backend](cscli_config_backend.md)	 - Configure installation directory
+* [cscli config installdir](cscli_config_installdir.md)	 - Configure installation directory
+* [cscli config prompt](cscli_config_prompt.md)	 - Prompt for configuration values in an interactive fashion
+* [cscli config show](cscli_config_show.md)	 - Displays current config
+
+###### Auto generated by spf13/cobra on 14-May-2020

cmd/crowdsec-cli/doc/cwcli_config_backend.md

@@ -0,0 +1,34 @@
+## cscli config backend
+
+Configure installation directory
+
+### Synopsis
+
+Configure the backend plugin directory of crowdsec, such as /etc/crowdsec/plugins/backend
+
+```
+cscli config backend [value] [flags]
+```
+
+### Options
+
+```
+  -h, --help   help for backend
+```
+
+### Options inherited from parent commands
+
+```
+  -c, --config-dir string   Configuration directory to use. (default "/etc/crowdsec/cscli/")
+      --debug               Set logging to debug.
+      --error               Set logging to error.
+      --info                Set logging to info.
+  -o, --output string       Output format : human, json, raw. (default "human")
+      --warning             Set logging to warning.
+```
+
+### SEE ALSO
+
+* [cscli config](cscli_config.md)	 - Allows to view/edit cscli config
+
+###### Auto generated by spf13/cobra on 14-May-2020

cmd/crowdsec-cli/doc/cwcli_config_installdir.md

@@ -0,0 +1,34 @@
+## cscli config installdir
+
+Configure installation directory
+
+### Synopsis
+
+Configure the installation directory of crowdsec, such as /etc/crowdsec/crowdsec/
+
+```
+cscli config installdir [value] [flags]
+```
+
+### Options
+
+```
+  -h, --help   help for installdir
+```
+
+### Options inherited from parent commands
+
+```
+  -c, --config-dir string   Configuration directory to use. (default "/etc/crowdsec/cscli/")
+      --debug               Set logging to debug.
+      --error               Set logging to error.
+      --info                Set logging to info.
+  -o, --output string       Output format : human, json, raw. (default "human")
+      --warning             Set logging to warning.
+```
+
+### SEE ALSO
+
+* [cscli config](cscli_config.md)	 - Allows to view/edit cscli config
+
+###### Auto generated by spf13/cobra on 14-May-2020

cmd/crowdsec-cli/doc/cwcli_config_prompt.md

@@ -0,0 +1,34 @@
+## cscli config prompt
+
+Prompt for configuration values in an interactive fashion
+
+### Synopsis
+
+Start interactive configuration of cli. It will successively ask for install dir, db path.
+
+```
+cscli config prompt [flags]
+```
+
+### Options
+
+```
+  -h, --help   help for prompt
+```
+
+### Options inherited from parent commands
+
+```
+  -c, --config-dir string   Configuration directory to use. (default "/etc/crowdsec/cscli/")
+      --debug               Set logging to debug.
+      --error               Set logging to error.
+      --info                Set logging to info.
+  -o, --output string       Output format : human, json, raw. (default "human")
+      --warning             Set logging to warning.
+```
+
+### SEE ALSO
+
+* [cscli config](cscli_config.md)	 - Allows to view/edit cscli config
+
+###### Auto generated by spf13/cobra on 14-May-2020

cmd/crowdsec-cli/doc/cwcli_config_show.md

@@ -0,0 +1,34 @@
+## cscli config show
+
+Displays current config
+
+### Synopsis
+
+Displays the current cli configuration.
+
+```
+cscli config show [flags]
+```
+
+### Options
+
+```
+  -h, --help   help for show
+```
+
+### Options inherited from parent commands
+
+```
+  -c, --config-dir string   Configuration directory to use. (default "/etc/crowdsec/cscli/")
+      --debug               Set logging to debug.
+      --error               Set logging to error.
+      --info                Set logging to info.
+  -o, --output string       Output format : human, json, raw. (default "human")
+      --warning             Set logging to warning.
+```
+
+### SEE ALSO
+
+* [cscli config](cscli_config.md)	 - Allows to view/edit cscli config
+
+###### Auto generated by spf13/cobra on 14-May-2020

cmd/crowdsec-cli/doc/cwcli_dashboard.md

@@ -0,0 +1,42 @@
+## cscli dashboard
+
+Start a dashboard (metabase) container.
+
+### Synopsis
+
+Start a metabase container exposing dashboards and metrics.
+
+### Examples
+
+```
+cscli dashboard setup
+cscli dashboard start
+cscli dashboard stop
+cscli dashboard setup --force
+```
+
+### Options
+
+```
+  -h, --help   help for dashboard
+```
+
+### Options inherited from parent commands
+
+```
+  -c, --config-dir string   Configuration directory to use. (default "/etc/crowdsec/cscli/")
+      --debug               Set logging to debug.
+      --error               Set logging to error.
+      --info                Set logging to info.
+  -o, --output string       Output format : human, json, raw. (default "human")
+      --warning             Set logging to warning.
+```
+
+### SEE ALSO
+
+* [cscli](cscli.md)	 - cscli allows you to manage crowdsec
+* [cscli dashboard setup](cscli_dashboard_setup.md)	 - Setup a metabase container.
+* [cscli dashboard start](cscli_dashboard_start.md)	 - Start the metabase container.
+* [cscli dashboard stop](cscli_dashboard_stop.md)	 - Stops the metabase container.
+
+###### Auto generated by spf13/cobra on 14-May-2020

cmd/crowdsec-cli/doc/cwcli_dashboard_setup.md

@@ -0,0 +1,47 @@
+## cscli dashboard setup
+
+Setup a metabase container.
+
+### Synopsis
+
+Perform a metabase docker setup, download standard dashboards, create a fresh user and start the container
+
+```
+cscli dashboard setup [flags]
+```
+
+### Examples
+
+```
+cscli dashboard setup
+cscli dashboard setup --force
+cscli dashboard setup -l 0.0.0.0 -p 443
+ 
+```
+
+### Options
+
+```
+  -d, --dir string      Shared directory with metabase container. (default "/var/lib/crowdsec/data")
+  -f, --force           Force setup : override existing files.
+  -h, --help            help for setup
+  -l, --listen string   Listen address of container (default "127.0.0.1")
+  -p, --port string     Listen port of container (default "3000")
+```
+
+### Options inherited from parent commands
+
+```
+  -c, --config-dir string   Configuration directory to use. (default "/etc/crowdsec/cscli/")
+      --debug               Set logging to debug.
+      --error               Set logging to error.
+      --info                Set logging to info.
+  -o, --output string       Output format : human, json, raw. (default "human")
+      --warning             Set logging to warning.
+```
+
+### SEE ALSO
+
+* [cscli dashboard](cscli_dashboard.md)	 - Start a dashboard (metabase) container.
+
+###### Auto generated by spf13/cobra on 14-May-2020

cmd/crowdsec-cli/doc/cwcli_dashboard_start.md

@@ -0,0 +1,34 @@
+## cscli dashboard start
+
+Start the metabase container.
+
+### Synopsis
+
+Stats the metabase container using docker.
+
+```
+cscli dashboard start [flags]
+```
+
+### Options
+
+```
+  -h, --help   help for start
+```
+
+### Options inherited from parent commands
+
+```
+  -c, --config-dir string   Configuration directory to use. (default "/etc/crowdsec/cscli/")
+      --debug               Set logging to debug.
+      --error               Set logging to error.
+      --info                Set logging to info.
+  -o, --output string       Output format : human, json, raw. (default "human")
+      --warning             Set logging to warning.
+```
+
+### SEE ALSO
+
+* [cscli dashboard](cscli_dashboard.md)	 - Start a dashboard (metabase) container.
+
+###### Auto generated by spf13/cobra on 14-May-2020

cmd/crowdsec-cli/doc/cwcli_dashboard_stop.md

@@ -0,0 +1,35 @@
+## cscli dashboard stop
+
+Stops the metabase container.
+
+### Synopsis
+
+Stops the metabase container using docker.
+
+```
+cscli dashboard stop [flags]
+```
+
+### Options
+
+```
+  -h, --help     help for stop
+  -r, --remove   remove (docker rm) container as well.
+```
+
+### Options inherited from parent commands
+
+```
+  -c, --config-dir string   Configuration directory to use. (default "/etc/crowdsec/cscli/")
+      --debug               Set logging to debug.
+      --error               Set logging to error.
+      --info                Set logging to info.
+  -o, --output string       Output format : human, json, raw. (default "human")
+      --warning             Set logging to warning.
+```
+
+### SEE ALSO
+
+* [cscli dashboard](cscli_dashboard.md)	 - Start a dashboard (metabase) container.
+
+###### Auto generated by spf13/cobra on 14-May-2020

cmd/crowdsec-cli/doc/cwcli_inspect.md

@@ -0,0 +1,47 @@
+## cscli inspect
+
+Inspect configuration(s)
+
+### Synopsis
+
+
+Inspect give you full detail about local installed configuration.
+
+[type] must be parser, scenario, postoverflow, collection.
+
+[config_name] must be a valid config name from [Crowdsec Hub](https://hub.crowdsec.net) or locally installed.
+
+
+### Examples
+
+```
+cscli inspect parser crowdsec/xxx  
+cscli inspect collection crowdsec/xxx
+```
+
+### Options
+
+```
+  -h, --help   help for inspect
+```
+
+### Options inherited from parent commands
+
+```
+  -c, --config-dir string   Configuration directory to use. (default "/etc/crowdsec/cscli/")
+      --debug               Set logging to debug.
+      --error               Set logging to error.
+      --info                Set logging to info.
+  -o, --output string       Output format : human, json, raw. (default "human")
+      --warning             Set logging to warning.
+```
+
+### SEE ALSO
+
+* [cscli](cscli.md)	 - cscli allows you to manage crowdsec
+* [cscli inspect collection](cscli_inspect_collection.md)	 - Inspect given collection
+* [cscli inspect parser](cscli_inspect_parser.md)	 - Inspect given log parser
+* [cscli inspect postoverflow](cscli_inspect_postoverflow.md)	 - Inspect given postoverflow parser
+* [cscli inspect scenario](cscli_inspect_scenario.md)	 - Inspect given scenario
+
+###### Auto generated by spf13/cobra on 14-May-2020

cmd/crowdsec-cli/doc/cwcli_inspect_collection.md

@@ -0,0 +1,40 @@
+## cscli inspect collection
+
+Inspect given collection
+
+### Synopsis
+
+Inspect given collection from hub
+
+```
+cscli inspect collection [config] [flags]
+```
+
+### Examples
+
+```
+cscli inspect collection crowdsec/xxx
+```
+
+### Options
+
+```
+  -h, --help   help for collection
+```
+
+### Options inherited from parent commands
+
+```
+  -c, --config-dir string   Configuration directory to use. (default "/etc/crowdsec/cscli/")
+      --debug               Set logging to debug.
+      --error               Set logging to error.
+      --info                Set logging to info.
+  -o, --output string       Output format : human, json, raw. (default "human")
+      --warning             Set logging to warning.
+```
+
+### SEE ALSO
+
+* [cscli inspect](cscli_inspect.md)	 - Inspect configuration(s)
+
+###### Auto generated by spf13/cobra on 14-May-2020

cmd/crowdsec-cli/doc/cwcli_inspect_parser.md

@@ -0,0 +1,40 @@
+## cscli inspect parser
+
+Inspect given log parser
+
+### Synopsis
+
+Inspect given parser from hub
+
+```
+cscli inspect parser [config] [flags]
+```
+
+### Examples
+
+```
+cscli inspect parser crowdsec/xxx
+```
+
+### Options
+
+```
+  -h, --help   help for parser
+```
+
+### Options inherited from parent commands
+
+```
+  -c, --config-dir string   Configuration directory to use. (default "/etc/crowdsec/cscli/")
+      --debug               Set logging to debug.
+      --error               Set logging to error.
+      --info                Set logging to info.
+  -o, --output string       Output format : human, json, raw. (default "human")
+      --warning             Set logging to warning.
+```
+
+### SEE ALSO
+
+* [cscli inspect](cscli_inspect.md)	 - Inspect configuration(s)
+
+###### Auto generated by spf13/cobra on 14-May-2020

cmd/crowdsec-cli/doc/cwcli_inspect_postoverflow.md

@@ -0,0 +1,40 @@
+## cscli inspect postoverflow
+
+Inspect given postoverflow parser
+
+### Synopsis
+
+Inspect given postoverflow from hub.
+
+```
+cscli inspect postoverflow [config] [flags]
+```
+
+### Examples
+
+```
+cscli inspect postoverflow crowdsec/xxx
+```
+
+### Options
+
+```
+  -h, --help   help for postoverflow
+```
+
+### Options inherited from parent commands
+
+```
+  -c, --config-dir string   Configuration directory to use. (default "/etc/crowdsec/cscli/")
+      --debug               Set logging to debug.
+      --error               Set logging to error.
+      --info                Set logging to info.
+  -o, --output string       Output format : human, json, raw. (default "human")
+      --warning             Set logging to warning.
+```
+
+### SEE ALSO
+
+* [cscli inspect](cscli_inspect.md)	 - Inspect configuration(s)
+
+###### Auto generated by spf13/cobra on 14-May-2020

cmd/crowdsec-cli/doc/cwcli_inspect_scenario.md

@@ -0,0 +1,40 @@
+## cscli inspect scenario
+
+Inspect given scenario
+
+### Synopsis
+
+Inspect given scenario from hub
+
+```
+cscli inspect scenario [config] [flags]
+```
+
+### Examples
+
+```
+cscli inspect scenario crowdsec/xxx
+```
+
+### Options
+
+```
+  -h, --help   help for scenario
+```
+
+### Options inherited from parent commands
+
+```
+  -c, --config-dir string   Configuration directory to use. (default "/etc/crowdsec/cscli/")
+      --debug               Set logging to debug.
+      --error               Set logging to error.
+      --info                Set logging to info.
+  -o, --output string       Output format : human, json, raw. (default "human")
+      --warning             Set logging to warning.
+```
+
+### SEE ALSO
+
+* [cscli inspect](cscli_inspect.md)	 - Inspect configuration(s)
+
+###### Auto generated by spf13/cobra on 14-May-2020

cmd/crowdsec-cli/doc/cwcli_install.md

@@ -0,0 +1,51 @@
+## cscli install
+
+Install configuration(s) from hub
+
+### Synopsis
+
+
+Install configuration from the CrowdSec Hub.
+
+In order to download latest versions of configuration, 
+you should [update cscli](./cscli_update.md).
+
+[type] must be parser, scenario, postoverflow, collection.
+
+[config_name] must be a valid config name from [Crowdsec Hub](https://hub.crowdsec.net).
+
+
+### Examples
+
+```
+cscli install [type] [config_name]
+```
+
+### Options
+
+```
+  -d, --download-only   Only download packages, don't enable
+      --force           Force install : Overwrite tainted and outdated files
+  -h, --help            help for install
+```
+
+### Options inherited from parent commands
+
+```
+  -c, --config-dir string   Configuration directory to use. (default "/etc/crowdsec/cscli/")
+      --debug               Set logging to debug.
+      --error               Set logging to error.
+      --info                Set logging to info.
+  -o, --output string       Output format : human, json, raw. (default "human")
+      --warning             Set logging to warning.
+```
+
+### SEE ALSO
+
+* [cscli](cscli.md)	 - cscli allows you to manage crowdsec
+* [cscli install collection](cscli_install_collection.md)	 - Install given collection
+* [cscli install parser](cscli_install_parser.md)	 - Install given log parser
+* [cscli install postoverflow](cscli_install_postoverflow.md)	 - Install given postoverflow parser
+* [cscli install scenario](cscli_install_scenario.md)	 - Install given scenario
+
+###### Auto generated by spf13/cobra on 14-May-2020

cmd/crowdsec-cli/doc/cwcli_install_collection.md

@@ -0,0 +1,42 @@
+## cscli install collection
+
+Install given collection
+
+### Synopsis
+
+Fetch and install given collection from hub
+
+```
+cscli install collection [config] [flags]
+```
+
+### Examples
+
+```
+cscli install collection crowdsec/xxx
+```
+
+### Options
+
+```
+  -h, --help   help for collection
+```
+
+### Options inherited from parent commands
+
+```
+  -c, --config-dir string   Configuration directory to use. (default "/etc/crowdsec/cscli/")
+      --debug               Set logging to debug.
+  -d, --download-only       Only download packages, don't enable
+      --error               Set logging to error.
+      --force               Force install : Overwrite tainted and outdated files
+      --info                Set logging to info.
+  -o, --output string       Output format : human, json, raw. (default "human")
+      --warning             Set logging to warning.
+```
+
+### SEE ALSO
+
+* [cscli install](cscli_install.md)	 - Install configuration(s) from hub
+
+###### Auto generated by spf13/cobra on 14-May-2020

cmd/crowdsec-cli/doc/cwcli_install_parser.md

@@ -0,0 +1,42 @@
+## cscli install parser
+
+Install given log parser
+
+### Synopsis
+
+Fetch and install given parser from hub
+
+```
+cscli install parser [config] [flags]
+```
+
+### Examples
+
+```
+cscli install parser crowdsec/xxx
+```
+
+### Options
+
+```
+  -h, --help   help for parser
+```
+
+### Options inherited from parent commands
+
+```
+  -c, --config-dir string   Configuration directory to use. (default "/etc/crowdsec/cscli/")
+      --debug               Set logging to debug.
+  -d, --download-only       Only download packages, don't enable
+      --error               Set logging to error.
+      --force               Force install : Overwrite tainted and outdated files
+      --info                Set logging to info.
+  -o, --output string       Output format : human, json, raw. (default "human")
+      --warning             Set logging to warning.
+```
+
+### SEE ALSO
+
+* [cscli install](cscli_install.md)	 - Install configuration(s) from hub
+
+###### Auto generated by spf13/cobra on 14-May-2020

cmd/crowdsec-cli/doc/cwcli_install_postoverflow.md

@@ -0,0 +1,43 @@
+## cscli install postoverflow
+
+Install given postoverflow parser
+
+### Synopsis
+
+Fetch and install given postoverflow from hub.
+As a reminder, postoverflows are parsing configuration that will occur after the overflow (before a decision is applied).
+
+```
+cscli install postoverflow [config] [flags]
+```
+
+### Examples
+
+```
+cscli install collection crowdsec/xxx
+```
+
+### Options
+
+```
+  -h, --help   help for postoverflow
+```
+
+### Options inherited from parent commands
+
+```
+  -c, --config-dir string   Configuration directory to use. (default "/etc/crowdsec/cscli/")
+      --debug               Set logging to debug.
+  -d, --download-only       Only download packages, don't enable
+      --error               Set logging to error.
+      --force               Force install : Overwrite tainted and outdated files
+      --info                Set logging to info.
+  -o, --output string       Output format : human, json, raw. (default "human")
+      --warning             Set logging to warning.
+```
+
+### SEE ALSO
+
+* [cscli install](cscli_install.md)	 - Install configuration(s) from hub
+
+###### Auto generated by spf13/cobra on 14-May-2020

cmd/crowdsec-cli/doc/cwcli_install_scenario.md

@@ -0,0 +1,42 @@
+## cscli install scenario
+
+Install given scenario
+
+### Synopsis
+
+Fetch and install given scenario from hub
+
+```
+cscli install scenario [config] [flags]
+```
+
+### Examples
+
+```
+cscli install scenario crowdsec/xxx
+```
+
+### Options
+
+```
+  -h, --help   help for scenario
+```
+
+### Options inherited from parent commands
+
+```
+  -c, --config-dir string   Configuration directory to use. (default "/etc/crowdsec/cscli/")
+      --debug               Set logging to debug.
+  -d, --download-only       Only download packages, don't enable
+      --error               Set logging to error.
+      --force               Force install : Overwrite tainted and outdated files
+      --info                Set logging to info.
+  -o, --output string       Output format : human, json, raw. (default "human")
+      --warning             Set logging to warning.
+```
+
+### SEE ALSO
+
+* [cscli install](cscli_install.md)	 - Install configuration(s) from hub
+
+###### Auto generated by spf13/cobra on 14-May-2020

cmd/crowdsec-cli/doc/cwcli_list.md

@@ -0,0 +1,54 @@
+## cscli list
+
+List enabled configs
+
+### Synopsis
+
+
+List enabled configurations (parser/scenarios/collections) on your host.
+
+It is possible to list also configuration from [Crowdsec Hub](https://hub.crowdsec.net) with the '-a' options.
+
+[type] must be parsers, scenarios, postoverflows, collections
+		
+
+```
+cscli list [-a] [flags]
+```
+
+### Examples
+
+```
+cscli list  # List all local configurations
+cscli list [type] # List all local configuration of type [type]
+cscli list -a # List all local and remote configurations
+		
+```
+
+### Options
+
+```
+  -a, --all    List as well disabled items
+  -h, --help   help for list
+```
+
+### Options inherited from parent commands
+
+```
+  -c, --config-dir string   Configuration directory to use. (default "/etc/crowdsec/cscli/")
+      --debug               Set logging to debug.
+      --error               Set logging to error.
+      --info                Set logging to info.
+  -o, --output string       Output format : human, json, raw. (default "human")
+      --warning             Set logging to warning.
+```
+
+### SEE ALSO
+
+* [cscli](cscli.md)	 - cscli allows you to manage crowdsec
+* [cscli list collections](cscli_list_collections.md)	 - List enabled collections
+* [cscli list parsers](cscli_list_parsers.md)	 - List enabled parsers
+* [cscli list postoverflows](cscli_list_postoverflows.md)	 - List enabled postoverflow parsers
+* [cscli list scenarios](cscli_list_scenarios.md)	 - List enabled scenarios
+
+###### Auto generated by spf13/cobra on 14-May-2020

cmd/crowdsec-cli/doc/cwcli_list_collections.md

@@ -0,0 +1,35 @@
+## cscli list collections
+
+List enabled collections
+
+### Synopsis
+
+List enabled collections
+
+```
+cscli list collections [-a] [flags]
+```
+
+### Options
+
+```
+  -h, --help   help for collections
+```
+
+### Options inherited from parent commands
+
+```
+  -a, --all                 List as well disabled items
+  -c, --config-dir string   Configuration directory to use. (default "/etc/crowdsec/cscli/")
+      --debug               Set logging to debug.
+      --error               Set logging to error.
+      --info                Set logging to info.
+  -o, --output string       Output format : human, json, raw. (default "human")
+      --warning             Set logging to warning.
+```
+
+### SEE ALSO
+
+* [cscli list](cscli_list.md)	 - List enabled configs
+
+###### Auto generated by spf13/cobra on 14-May-2020

cmd/crowdsec-cli/doc/cwcli_list_parsers.md

@@ -0,0 +1,35 @@
+## cscli list parsers
+
+List enabled parsers
+
+### Synopsis
+
+List enabled parsers
+
+```
+cscli list parsers [-a] [flags]
+```
+
+### Options
+
+```
+  -h, --help   help for parsers
+```
+
+### Options inherited from parent commands
+
+```
+  -a, --all                 List as well disabled items
+  -c, --config-dir string   Configuration directory to use. (default "/etc/crowdsec/cscli/")
+      --debug               Set logging to debug.
+      --error               Set logging to error.
+      --info                Set logging to info.
+  -o, --output string       Output format : human, json, raw. (default "human")
+      --warning             Set logging to warning.
+```
+
+### SEE ALSO
+
+* [cscli list](cscli_list.md)	 - List enabled configs
+
+###### Auto generated by spf13/cobra on 14-May-2020

cmd/crowdsec-cli/doc/cwcli_list_postoverflows.md

@@ -0,0 +1,35 @@
+## cscli list postoverflows
+
+List enabled postoverflow parsers
+
+### Synopsis
+
+List enabled postoverflow parsers
+
+```
+cscli list postoverflows [-a] [flags]
+```
+
+### Options
+
+```
+  -h, --help   help for postoverflows
+```
+
+### Options inherited from parent commands
+
+```
+  -a, --all                 List as well disabled items
+  -c, --config-dir string   Configuration directory to use. (default "/etc/crowdsec/cscli/")
+      --debug               Set logging to debug.
+      --error               Set logging to error.
+      --info                Set logging to info.
+  -o, --output string       Output format : human, json, raw. (default "human")
+      --warning             Set logging to warning.
+```
+
+### SEE ALSO
+
+* [cscli list](cscli_list.md)	 - List enabled configs
+
+###### Auto generated by spf13/cobra on 14-May-2020

cmd/crowdsec-cli/doc/cwcli_list_scenarios.md

@@ -0,0 +1,35 @@
+## cscli list scenarios
+
+List enabled scenarios
+
+### Synopsis
+
+List enabled scenarios
+
+```
+cscli list scenarios [-a] [flags]
+```
+
+### Options
+
+```
+  -h, --help   help for scenarios
+```
+
+### Options inherited from parent commands
+
+```
+  -a, --all                 List as well disabled items
+  -c, --config-dir string   Configuration directory to use. (default "/etc/crowdsec/cscli/")
+      --debug               Set logging to debug.
+      --error               Set logging to error.
+      --info                Set logging to info.
+  -o, --output string       Output format : human, json, raw. (default "human")
+      --warning             Set logging to warning.
+```
+
+### SEE ALSO
+
+* [cscli list](cscli_list.md)	 - List enabled configs
+
+###### Auto generated by spf13/cobra on 14-May-2020

cmd/crowdsec-cli/doc/cwcli_metrics.md

@@ -0,0 +1,35 @@
+## cscli metrics
+
+Display crowdsec prometheus metrics.
+
+### Synopsis
+
+Fetch metrics from the prometheus server and display them in a human-friendly way
+
+```
+cscli metrics [flags]
+```
+
+### Options
+
+```
+  -h, --help         help for metrics
+  -u, --url string   Prometheus url (default "http://127.0.0.1:6060/metrics")
+```
+
+### Options inherited from parent commands
+
+```
+  -c, --config-dir string   Configuration directory to use. (default "/etc/crowdsec/cscli/")
+      --debug               Set logging to debug.
+      --error               Set logging to error.
+      --info                Set logging to info.
+  -o, --output string       Output format : human, json, raw. (default "human")
+      --warning             Set logging to warning.
+```
+
+### SEE ALSO
+
+* [cscli](cscli.md)	 - cscli allows you to manage crowdsec
+
+###### Auto generated by spf13/cobra on 14-May-2020

cmd/crowdsec-cli/doc/cwcli_remove.md

@@ -0,0 +1,48 @@
+## cscli remove
+
+Remove/disable configuration(s)
+
+### Synopsis
+
+
+ Remove local configuration. 
+ 
+[type] must be parser, scenario, postoverflow, collection
+
+[config_name] must be a valid config name from [Crowdsec Hub](https://hub.crowdsec.net) or locally installed.
+ 
+
+### Examples
+
+```
+cscli remove [type] [config_name]
+```
+
+### Options
+
+```
+      --all     Delete all the files in selected scope
+  -h, --help    help for remove
+      --purge   Delete source file in ~/.cscli/hub/ too
+```
+
+### Options inherited from parent commands
+
+```
+  -c, --config-dir string   Configuration directory to use. (default "/etc/crowdsec/cscli/")
+      --debug               Set logging to debug.
+      --error               Set logging to error.
+      --info                Set logging to info.
+  -o, --output string       Output format : human, json, raw. (default "human")
+      --warning             Set logging to warning.
+```
+
+### SEE ALSO
+
+* [cscli](cscli.md)	 - cscli allows you to manage crowdsec
+* [cscli remove collection](cscli_remove_collection.md)	 - Remove/disable collection
+* [cscli remove parser](cscli_remove_parser.md)	 - Remove/disable parser
+* [cscli remove postoverflow](cscli_remove_postoverflow.md)	 - Remove/disable postoverflow parser
+* [cscli remove scenario](cscli_remove_scenario.md)	 - Remove/disable scenario
+
+###### Auto generated by spf13/cobra on 14-May-2020

cmd/crowdsec-cli/doc/cwcli_remove_collection.md

@@ -0,0 +1,36 @@
+## cscli remove collection
+
+Remove/disable collection
+
+### Synopsis
+
+<config> must be a valid collection.
+
+```
+cscli remove collection [config] [flags]
+```
+
+### Options
+
+```
+  -h, --help   help for collection
+```
+
+### Options inherited from parent commands
+
+```
+      --all                 Delete all the files in selected scope
+  -c, --config-dir string   Configuration directory to use. (default "/etc/crowdsec/cscli/")
+      --debug               Set logging to debug.
+      --error               Set logging to error.
+      --info                Set logging to info.
+  -o, --output string       Output format : human, json, raw. (default "human")
+      --purge               Delete source file in ~/.cscli/hub/ too
+      --warning             Set logging to warning.
+```
+
+### SEE ALSO
+
+* [cscli remove](cscli_remove.md)	 - Remove/disable configuration(s)
+
+###### Auto generated by spf13/cobra on 14-May-2020

cmd/crowdsec-cli/doc/cwcli_remove_parser.md

@@ -0,0 +1,36 @@
+## cscli remove parser
+
+Remove/disable parser
+
+### Synopsis
+
+<config> must be a valid parser.
+
+```
+cscli remove parser <config> [flags]
+```
+
+### Options
+
+```
+  -h, --help   help for parser
+```
+
+### Options inherited from parent commands
+
+```
+      --all                 Delete all the files in selected scope
+  -c, --config-dir string   Configuration directory to use. (default "/etc/crowdsec/cscli/")
+      --debug               Set logging to debug.
+      --error               Set logging to error.
+      --info                Set logging to info.
+  -o, --output string       Output format : human, json, raw. (default "human")
+      --purge               Delete source file in ~/.cscli/hub/ too
+      --warning             Set logging to warning.
+```
+
+### SEE ALSO
+
+* [cscli remove](cscli_remove.md)	 - Remove/disable configuration(s)
+
+###### Auto generated by spf13/cobra on 14-May-2020

cmd/crowdsec-cli/doc/cwcli_remove_postoverflow.md

@@ -0,0 +1,36 @@
+## cscli remove postoverflow
+
+Remove/disable postoverflow parser
+
+### Synopsis
+
+<config> must be a valid collection.
+
+```
+cscli remove postoverflow [config] [flags]
+```
+
+### Options
+
+```
+  -h, --help   help for postoverflow
+```
+
+### Options inherited from parent commands
+
+```
+      --all                 Delete all the files in selected scope
+  -c, --config-dir string   Configuration directory to use. (default "/etc/crowdsec/cscli/")
+      --debug               Set logging to debug.
+      --error               Set logging to error.
+      --info                Set logging to info.
+  -o, --output string       Output format : human, json, raw. (default "human")
+      --purge               Delete source file in ~/.cscli/hub/ too
+      --warning             Set logging to warning.
+```
+
+### SEE ALSO
+
+* [cscli remove](cscli_remove.md)	 - Remove/disable configuration(s)
+
+###### Auto generated by spf13/cobra on 14-May-2020

cmd/crowdsec-cli/doc/cwcli_remove_scenario.md

@@ -0,0 +1,36 @@
+## cscli remove scenario
+
+Remove/disable scenario
+
+### Synopsis
+
+<config> must be a valid scenario.
+
+```
+cscli remove scenario [config] [flags]
+```
+
+### Options
+
+```
+  -h, --help   help for scenario
+```
+
+### Options inherited from parent commands
+
+```
+      --all                 Delete all the files in selected scope
+  -c, --config-dir string   Configuration directory to use. (default "/etc/crowdsec/cscli/")
+      --debug               Set logging to debug.
+      --error               Set logging to error.
+      --info                Set logging to info.
+  -o, --output string       Output format : human, json, raw. (default "human")
+      --purge               Delete source file in ~/.cscli/hub/ too
+      --warning             Set logging to warning.
+```
+
+### SEE ALSO
+
+* [cscli remove](cscli_remove.md)	 - Remove/disable configuration(s)
+
+###### Auto generated by spf13/cobra on 14-May-2020

cmd/crowdsec-cli/doc/cwcli_update.md

@@ -0,0 +1,36 @@
+## cscli update
+
+Fetch available configs from hub
+
+### Synopsis
+
+
+Fetches the [.index.json](https://github.com/crowdsecurity/hub/blob/master/.index.json) file from hub, containing the list of available configs.
+
+
+```
+cscli update [flags]
+```
+
+### Options
+
+```
+  -h, --help   help for update
+```
+
+### Options inherited from parent commands
+
+```
+  -c, --config-dir string   Configuration directory to use. (default "/etc/crowdsec/cscli/")
+      --debug               Set logging to debug.
+      --error               Set logging to error.
+      --info                Set logging to info.
+  -o, --output string       Output format : human, json, raw. (default "human")
+      --warning             Set logging to warning.
+```
+
+### SEE ALSO
+
+* [cscli](cscli.md)	 - cscli allows you to manage crowdsec
+
+###### Auto generated by spf13/cobra on 14-May-2020

cmd/crowdsec-cli/doc/cwcli_upgrade.md

@@ -0,0 +1,62 @@
+## cscli upgrade
+
+Upgrade configuration(s)
+
+### Synopsis
+
+
+Upgrade configuration from the CrowdSec Hub.
+
+In order to upgrade latest versions of configuration, 
+the Hub cache should be [updated](./cscli_update.md).
+ 
+Tainted configuration will not be updated (use --force to update them).
+
+[type] must be parser, scenario, postoverflow, collection.
+
+[config_name] must be a valid config name from [Crowdsec Hub](https://hub.crowdsec.net).
+ 
+
+ 
+
+```
+cscli upgrade [type] [config] [flags]
+```
+
+### Examples
+
+```
+cscli upgrade [type] [config_name]
+cscli upgrade --all   # Upgrade all configurations types
+cscli upgrade --force # Overwrite tainted configuration
+		
+```
+
+### Options
+
+```
+      --all     Upgrade all configuration in scope
+      --force   Overwrite existing files, even if tainted
+  -h, --help    help for upgrade
+```
+
+### Options inherited from parent commands
+
+```
+  -c, --config-dir string   Configuration directory to use. (default "/etc/crowdsec/cscli/")
+      --debug               Set logging to debug.
+      --error               Set logging to error.
+      --info                Set logging to info.
+  -o, --output string       Output format : human, json, raw. (default "human")
+      --warning             Set logging to warning.
+```
+
+### SEE ALSO
+
+* [cscli](cscli.md)	 - cscli allows you to manage crowdsec
+* [cscli upgrade collection](cscli_upgrade_collection.md)	 - Upgrade collection configuration(s)
+* [cscli upgrade parser](cscli_upgrade_parser.md)	 - Upgrade parser configuration(s)
+* [cscli upgrade postoverflow](cscli_upgrade_postoverflow.md)	 - Upgrade postoverflow parser configuration(s)
+* [cscli upgrade scenario](cscli_upgrade_scenario.md)	 - Upgrade scenario configuration(s)
+
+###### Auto generated by spf13/cobra on 14-May-2020

cmd/crowdsec-cli/doc/cwcli_upgrade_collection.md

@@ -0,0 +1,44 @@
+## cscli upgrade collection
+
+Upgrade collection configuration(s)
+
+### Synopsis
+
+Upgrade one or more collection configurations
+
+```
+cscli upgrade collection [config] [flags]
+```
+
+### Examples
+
+```
+ - cscli upgrade collection crowdsec/apache-lamp  
+ - cscli upgrade collection -all  
+ - cscli upgrade collection crowdsec/apache-lamp --force
+```
+
+### Options
+
+```
+  -h, --help   help for collection
+```
+
+### Options inherited from parent commands
+
+```
+      --all                 Upgrade all configuration in scope
+  -c, --config-dir string   Configuration directory to use. (default "/etc/crowdsec/cscli/")
+      --debug               Set logging to debug.
+      --error               Set logging to error.
+      --force               Overwrite existing files, even if tainted
+      --info                Set logging to info.
+  -o, --output string       Output format : human, json, raw. (default "human")
+      --warning             Set logging to warning.
+```
+
+### SEE ALSO
+
+* [cscli upgrade](cscli_upgrade.md)	 - Upgrade configuration(s)
+
+###### Auto generated by spf13/cobra on 14-May-2020

cmd/crowdsec-cli/doc/cwcli_upgrade_parser.md

@@ -0,0 +1,44 @@
+## cscli upgrade parser
+
+Upgrade parser configuration(s)
+
+### Synopsis
+
+Upgrade one or more parser configurations
+
+```
+cscli upgrade parser [config] [flags]
+```
+
+### Examples
+
+```
+ - cscli upgrade parser crowdsec/apache-logs  
+ - cscli upgrade parser -all  
+ - cscli upgrade parser crowdsec/apache-logs --force
+```
+
+### Options
+
+```
+  -h, --help   help for parser
+```
+
+### Options inherited from parent commands
+
+```
+      --all                 Upgrade all configuration in scope
+  -c, --config-dir string   Configuration directory to use. (default "/etc/crowdsec/cscli/")
+      --debug               Set logging to debug.
+      --error               Set logging to error.
+      --force               Overwrite existing files, even if tainted
+      --info                Set logging to info.
+  -o, --output string       Output format : human, json, raw. (default "human")
+      --warning             Set logging to warning.
+```
+
+### SEE ALSO
+
+* [cscli upgrade](cscli_upgrade.md)	 - Upgrade configuration(s)
+
+###### Auto generated by spf13/cobra on 14-May-2020

cmd/crowdsec-cli/doc/cwcli_upgrade_postoverflow.md

@@ -0,0 +1,44 @@
+## cscli upgrade postoverflow
+
+Upgrade postoverflow parser configuration(s)
+
+### Synopsis
+
+Upgrade one or more postoverflow parser configurations
+
+```
+cscli upgrade postoverflow [config] [flags]
+```
+
+### Examples
+
+```
+ - cscli upgrade postoverflow crowdsec/enrich-rdns  
+ - cscli upgrade postoverflow -all  
+ - cscli upgrade postoverflow crowdsec/enrich-rdns --force
+```
+
+### Options
+
+```
+  -h, --help   help for postoverflow
+```
+
+### Options inherited from parent commands
+
+```
+      --all                 Upgrade all configuration in scope
+  -c, --config-dir string   Configuration directory to use. (default "/etc/crowdsec/cscli/")
+      --debug               Set logging to debug.
+      --error               Set logging to error.
+      --force               Overwrite existing files, even if tainted
+      --info                Set logging to info.
+  -o, --output string       Output format : human, json, raw. (default "human")
+      --warning             Set logging to warning.
+```
+
+### SEE ALSO
+
+* [cscli upgrade](cscli_upgrade.md)	 - Upgrade configuration(s)
+
+###### Auto generated by spf13/cobra on 14-May-2020

cmd/crowdsec-cli/doc/cwcli_upgrade_scenario.md

@@ -0,0 +1,43 @@
+## cscli upgrade scenario
+
+Upgrade scenario configuration(s)
+
+### Synopsis
+
+Upgrade one or more scenario configurations
+
+```
+cscli upgrade scenario [config] [flags]
+```
+
+### Examples
+
+```
+ - cscli	upgrade scenario -all  
+ - cscli upgrade scenario crowdsec/http-404 --force  
+```
+
+### Options
+
+```
+  -h, --help   help for scenario
+```
+
+### Options inherited from parent commands
+
+```
+      --all                 Upgrade all configuration in scope
+  -c, --config-dir string   Configuration directory to use. (default "/etc/crowdsec/cscli/")
+      --debug               Set logging to debug.
+      --error               Set logging to error.
+      --force               Overwrite existing files, even if tainted
+      --info                Set logging to info.
+  -o, --output string       Output format : human, json, raw. (default "human")
+      --warning             Set logging to warning.
+```
+
+### SEE ALSO
+
+* [cscli upgrade](cscli_upgrade.md)	 - Upgrade configuration(s)
+
+###### Auto generated by spf13/cobra on 14-May-2020

cmd/crowdsec-cli/inspect.go

@@ -0,0 +1,110 @@
+package main
+
+import (
+	"fmt"
+
+	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
+	"gopkg.in/yaml.v2"
+
+	log "github.com/sirupsen/logrus"
+
+	"github.com/spf13/cobra"
+)
+
+func InspectItem(name string, objectType string) {
+
+	for _, hubItem := range cwhub.HubIdx[objectType] {
+		if hubItem.Name != name {
+			continue
+		}
+		buff, err := yaml.Marshal(hubItem)
+		if err != nil {
+			log.Fatalf("unable to marshal item : %s", err)
+		}
+		fmt.Printf("%s", string(buff))
+	}
+}
+
+func NewInspectCmd() *cobra.Command {
+	var cmdInspect = &cobra.Command{
+		Use:   "inspect [type] [config]",
+		Short: "Inspect configuration(s)",
+		Long: `
+Inspect give you full detail about local installed configuration.
+
+[type] must be parser, scenario, postoverflow, collection.
+
+[config_name] must be a valid config name from [Crowdsec Hub](https://hub.crowdsec.net) or locally installed.
+`,
+		Example: `cscli inspect parser crowdsec/xxx  
+cscli inspect collection crowdsec/xxx`,
+		Args: cobra.MinimumNArgs(1),
+		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
+			if !config.configured {
+				return fmt.Errorf("you must configure cli before interacting with hub")
+			}
+			return nil
+		},
+	}
+
+	var cmdInspectParser = &cobra.Command{
+		Use:     "parser [config]",
+		Short:   "Inspect given log parser",
+		Long:    `Inspect given parser from hub`,
+		Example: `cscli inspect parser crowdsec/xxx`,
+		Args:    cobra.MinimumNArgs(1),
+		Run: func(cmd *cobra.Command, args []string) {
+			if err := cwhub.GetHubIdx(); err != nil {
+				log.Fatalf("failed to get Hub index : %v", err)
+			}
+			InspectItem(args[0], cwhub.PARSERS)
+		},
+	}
+	cmdInspect.AddCommand(cmdInspectParser)
+	var cmdInspectScenario = &cobra.Command{
+		Use:     "scenario [config]",
+		Short:   "Inspect given scenario",
+		Long:    `Inspect given scenario from hub`,
+		Example: `cscli inspect scenario crowdsec/xxx`,
+		Args:    cobra.MinimumNArgs(1),
+		Run: func(cmd *cobra.Command, args []string) {
+			if err := cwhub.GetHubIdx(); err != nil {
+				log.Fatalf("failed to get Hub index : %v", err)
+			}
+			InspectItem(args[0], cwhub.SCENARIOS)
+		},
+	}
+	cmdInspect.AddCommand(cmdInspectScenario)
+
+	var cmdInspectCollection = &cobra.Command{
+		Use:     "collection [config]",
+		Short:   "Inspect given collection",
+		Long:    `Inspect given collection from hub`,
+		Example: `cscli inspect collection crowdsec/xxx`,
+		Args:    cobra.MinimumNArgs(1),
+		Run: func(cmd *cobra.Command, args []string) {
+			if err := cwhub.GetHubIdx(); err != nil {
+				log.Fatalf("failed to get Hub index : %v", err)
+			}
+			InspectItem(args[0], cwhub.COLLECTIONS)
+		},
+	}
+	cmdInspect.AddCommand(cmdInspectCollection)
+
+	var cmdInspectPostoverflow = &cobra.Command{
+		Use:     "postoverflow [config]",
+		Short:   "Inspect given postoverflow parser",
+		Long:    `Inspect given postoverflow from hub.`,
+		Example: `cscli inspect postoverflow crowdsec/xxx`,
+		Args:    cobra.MinimumNArgs(1),
+		Run: func(cmd *cobra.Command, args []string) {
+			if err := cwhub.GetHubIdx(); err != nil {
+				log.Fatalf("failed to get Hub index : %v", err)
+			}
+			InspectItem(args[0], cwhub.PARSERS_OVFLW)
+		},
+	}
+	cmdInspect.AddCommand(cmdInspectPostoverflow)
+
+	return cmdInspect
+}

cmd/crowdsec-cli/install.go

@@ -0,0 +1,150 @@
+package main
+
+import (
+	"fmt"
+
+	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
+
+	log "github.com/sirupsen/logrus"
+
+	"github.com/spf13/cobra"
+)
+
+var download_only, force_install bool
+
+func InstallItem(name string, obtype string) {
+	for _, it := range cwhub.HubIdx[obtype] {
+		if it.Name == name {
+			if download_only && it.Downloaded && it.UpToDate {
+				log.Warningf("%s is already downloaded and up-to-date", it.Name)
+				return
+			}
+			it, err := cwhub.DownloadLatest(it, cwhub.Hubdir, force_install)
+			if err != nil {
+				log.Fatalf("error while downloading %s : %v", it.Name, err)
+			}
+			cwhub.HubIdx[obtype][it.Name] = it
+			if download_only {
+				log.Infof("Downloaded %s to %s", it.Name, cwhub.Hubdir+"/"+it.RemotePath)
+				return
+			}
+			it, err = cwhub.EnableItem(it, cwhub.Installdir, cwhub.Hubdir)
+			if err != nil {
+				log.Fatalf("error while enabled %s : %v.", it.Name, err)
+			}
+			cwhub.HubIdx[obtype][it.Name] = it
+			log.Infof("Enabled %s", it.Name)
+			return
+		}
+	}
+	log.Warningf("%s not found in hub index", name)
+	/*iterate of pkg index data*/
+}
+
+func InstallScenario(name string) {
+	InstallItem(name, cwhub.SCENARIOS)
+}
+
+func InstallCollection(name string) {
+	InstallItem(name, cwhub.COLLECTIONS)
+
+}
+
+func InstallParser(name string) {
+	InstallItem(name, cwhub.PARSERS)
+}
+
+func InstallPostoverflow(name string) {
+	InstallItem(name, cwhub.PARSERS_OVFLW)
+}
+
+func NewInstallCmd() *cobra.Command {
+	/* ---- INSTALL COMMAND */
+
+	var cmdInstall = &cobra.Command{
+		Use:   "install [type] [config]",
+		Short: "Install configuration(s) from hub",
+		Long: `
+Install configuration from the CrowdSec Hub.
+
+In order to download latest versions of configuration, 
+you should [update cscli](./cscli_update.md).
+
+[type] must be parser, scenario, postoverflow, collection.
+
+[config_name] must be a valid config name from [Crowdsec Hub](https://hub.crowdsec.net).
+`,
+		Example: `cscli install [type] [config_name]`,
+		Args:    cobra.MinimumNArgs(1),
+		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
+			if !config.configured {
+				return fmt.Errorf("you must configure cli before interacting with hub")
+			}
+			return nil
+		},
+	}
+	cmdInstall.PersistentFlags().BoolVarP(&download_only, "download-only", "d", false, "Only download packages, don't enable")
+	cmdInstall.PersistentFlags().BoolVar(&force_install, "force", false, "Force install : Overwrite tainted and outdated files")
+
+	var cmdInstallParser = &cobra.Command{
+		Use:     "parser [config]",
+		Short:   "Install given log parser",
+		Long:    `Fetch and install given parser from hub`,
+		Example: `cscli install parser crowdsec/xxx`,
+		Args:    cobra.MinimumNArgs(1),
+		Run: func(cmd *cobra.Command, args []string) {
+			if err := cwhub.GetHubIdx(); err != nil {
+				log.Fatalf("failed to get Hub index : %v", err)
+			}
+			InstallItem(args[0], cwhub.PARSERS)
+		},
+	}
+	cmdInstall.AddCommand(cmdInstallParser)
+	var cmdInstallScenario = &cobra.Command{
+		Use:     "scenario [config]",
+		Short:   "Install given scenario",
+		Long:    `Fetch and install given scenario from hub`,
+		Example: `cscli install scenario crowdsec/xxx`,
+		Args:    cobra.MinimumNArgs(1),
+		Run: func(cmd *cobra.Command, args []string) {
+			if err := cwhub.GetHubIdx(); err != nil {
+				log.Fatalf("failed to get Hub index : %v", err)
+			}
+			InstallItem(args[0], cwhub.SCENARIOS)
+		},
+	}
+	cmdInstall.AddCommand(cmdInstallScenario)
+
+	var cmdInstallCollection = &cobra.Command{
+		Use:     "collection [config]",
+		Short:   "Install given collection",
+		Long:    `Fetch and install given collection from hub`,
+		Example: `cscli install collection crowdsec/xxx`,
+		Args:    cobra.MinimumNArgs(1),
+		Run: func(cmd *cobra.Command, args []string) {
+			if err := cwhub.GetHubIdx(); err != nil {
+				log.Fatalf("failed to get Hub index : %v", err)
+			}
+			InstallItem(args[0], cwhub.COLLECTIONS)
+		},
+	}
+	cmdInstall.AddCommand(cmdInstallCollection)
+
+	var cmdInstallPostoverflow = &cobra.Command{
+		Use:   "postoverflow [config]",
+		Short: "Install given postoverflow parser",
+		Long: `Fetch and install given postoverflow from hub.
+As a reminder, postoverflows are parsing configuration that will occur after the overflow (before a decision is applied).`,
+		Example: `cscli install collection crowdsec/xxx`,
+		Args:    cobra.MinimumNArgs(1),
+		Run: func(cmd *cobra.Command, args []string) {
+			if err := cwhub.GetHubIdx(); err != nil {
+				log.Fatalf("failed to get Hub index : %v", err)
+			}
+			InstallItem(args[0], cwhub.PARSERS_OVFLW)
+		},
+	}
+	cmdInstall.AddCommand(cmdInstallPostoverflow)
+
+	return cmdInstall
+}

cmd/crowdsec-cli/list.go

@@ -0,0 +1,152 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+
+	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
+
+	"github.com/enescakir/emoji"
+	"github.com/olekukonko/tablewriter"
+	log "github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+)
+
+var listAll bool
+
+func doListing(ttype string, args []string) {
+
+	var pkgst []map[string]string
+
+	if len(args) == 1 {
+		pkgst = cwhub.HubStatus(ttype, args[0], listAll)
+	} else {
+		pkgst = cwhub.HubStatus(ttype, "", listAll)
+	}
+
+	if config.output == "human" {
+
+		table := tablewriter.NewWriter(os.Stdout)
+		table.SetCenterSeparator("")
+		table.SetColumnSeparator("")
+
+		table.SetHeaderAlignment(tablewriter.ALIGN_LEFT)
+		table.SetAlignment(tablewriter.ALIGN_LEFT)
+		table.SetHeader([]string{"Name", fmt.Sprintf("%v Status", emoji.Package), "Version", "Local Path"})
+		for _, v := range pkgst {
+			table.Append([]string{v["name"], v["utf8_status"], v["local_version"], v["local_path"]})
+		}
+		table.Render()
+	} else if config.output == "json" {
+		x, err := json.MarshalIndent(pkgst, "", " ")
+		if err != nil {
+			log.Fatalf("failed to unmarshal")
+		}
+		fmt.Printf("%s", string(x))
+	} else if config.output == "raw" {
+		for _, v := range pkgst {
+			fmt.Printf("%s %s\n", v["name"], v["description"])
+		}
+	}
+}
+
+func NewListCmd() *cobra.Command {
+	/* ---- LIST COMMAND */
+	var cmdList = &cobra.Command{
+		Use:   "list [-a]",
+		Short: "List enabled configs",
+		Long: `
+List enabled configurations (parser/scenarios/collections) on your host.
+
+It is possible to list also configuration from [Crowdsec Hub](https://hub.crowdsec.net) with the '-a' options.
+
+[type] must be parsers, scenarios, postoverflows, collections
+		`,
+		Example: `cscli list  # List all local configurations
+cscli list [type] # List all local configuration of type [type]
+cscli list -a # List all local and remote configurations
+		`,
+		Args: cobra.ExactArgs(0),
+		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
+			if !config.configured {
+				return fmt.Errorf("you must configure cli before interacting with hub")
+			}
+			return nil
+		},
+		Run: func(cmd *cobra.Command, args []string) {
+			if err := cwhub.GetHubIdx(); err != nil {
+				log.Fatalf("Failed to get Hub index : %v", err)
+			}
+
+			cwhub.DisplaySummary()
+			log.Printf("PARSERS:")
+			doListing(cwhub.PARSERS, args)
+			log.Printf("SCENARIOS:")
+			doListing(cwhub.SCENARIOS, args)
+			log.Printf("COLLECTIONS:")
+			doListing(cwhub.COLLECTIONS, args)
+			log.Printf("POSTOVERFLOWS:")
+			doListing(cwhub.PARSERS_OVFLW, args)
+		},
+	}
+	cmdList.PersistentFlags().BoolVarP(&listAll, "all", "a", false, "List as well disabled items")
+
+	var cmdListParsers = &cobra.Command{
+		Use:   "parsers [-a]",
+		Short: "List enabled parsers",
+		Long:  ``,
+		Args:  cobra.ExactArgs(0),
+		Run: func(cmd *cobra.Command, args []string) {
+			if err := cwhub.GetHubIdx(); err != nil {
+				log.Fatalf("Failed to get Hub index : %v", err)
+			}
+			doListing(cwhub.PARSERS, args)
+		},
+	}
+	cmdList.AddCommand(cmdListParsers)
+
+	var cmdListScenarios = &cobra.Command{
+		Use:   "scenarios [-a]",
+		Short: "List enabled scenarios",
+		Long:  ``,
+		Args:  cobra.ExactArgs(0),
+		Run: func(cmd *cobra.Command, args []string) {
+			if err := cwhub.GetHubIdx(); err != nil {
+				log.Fatalf("Failed to get Hub index : %v", err)
+			}
+			doListing(cwhub.SCENARIOS, args)
+		},
+	}
+	cmdList.AddCommand(cmdListScenarios)
+
+	var cmdListCollections = &cobra.Command{
+		Use:   "collections [-a]",
+		Short: "List enabled collections",
+		Long:  ``,
+		Args:  cobra.ExactArgs(0),
+		Run: func(cmd *cobra.Command, args []string) {
+			if err := cwhub.GetHubIdx(); err != nil {
+				log.Fatalf("Failed to get Hub index : %v", err)
+			}
+			doListing(cwhub.COLLECTIONS, args)
+		},
+	}
+	cmdList.AddCommand(cmdListCollections)
+
+	var cmdListPostoverflows = &cobra.Command{
+		Use:   "postoverflows [-a]",
+		Short: "List enabled postoverflow parsers",
+		Long:  ``,
+		Args:  cobra.ExactArgs(0),
+		Run: func(cmd *cobra.Command, args []string) {
+			if err := cwhub.GetHubIdx(); err != nil {
+				log.Fatalf("Failed to get Hub index : %v", err)
+			}
+			doListing(cwhub.PARSERS_OVFLW, args)
+		},
+	}
+	cmdList.AddCommand(cmdListPostoverflows)
+
+	return cmdList
+}

cmd/crowdsec-cli/main.go

@@ -0,0 +1,139 @@
+package main
+
+import (
+	"io/ioutil"
+	"os/user"
+	"path/filepath"
+	"strings"
+
+	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
+	"github.com/crowdsecurity/crowdsec/pkg/cwversion"
+
+	log "github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+	"github.com/spf13/cobra/doc"
+	"gopkg.in/yaml.v2"
+)
+
+var dbg_lvl, nfo_lvl, wrn_lvl, err_lvl bool
+
+var config cliConfig
+
+func initConfig() {
+
+	if dbg_lvl {
+		log.SetLevel(log.DebugLevel)
+	} else if nfo_lvl {
+		log.SetLevel(log.InfoLevel)
+	} else if wrn_lvl {
+		log.SetLevel(log.WarnLevel)
+	} else if err_lvl {
+		log.SetLevel(log.ErrorLevel)
+	}
+	if config.output == "json" {
+		log.SetLevel(log.WarnLevel)
+		log.SetFormatter(&log.JSONFormatter{})
+	} else if config.output == "raw" {
+		log.SetLevel(log.ErrorLevel)
+	}
+
+	if strings.HasPrefix(config.configFolder, "~/") {
+		usr, err := user.Current()
+		if err != nil {
+			log.Fatalf("failed to resolve path ~/ : %s", err)
+		}
+		config.configFolder = usr.HomeDir + "/" + config.configFolder[2:]
+	}
+	/*read config*/
+	buf, err := ioutil.ReadFile(filepath.Clean(config.configFolder + "/config"))
+	if err != nil {
+		log.Infof("Failed to open config %s : %s", filepath.Clean(config.configFolder+"/config"), err)
+	} else {
+		err = yaml.UnmarshalStrict(buf, &config)
+		if err != nil {
+			log.Fatalf("Failed to parse config %s : %s, please configure", filepath.Clean(config.configFolder+"/config"), err)
+		}
+		config.InstallFolder = filepath.Clean(config.InstallFolder)
+		config.hubFolder = filepath.Clean(config.configFolder + "/hub/")
+		config.BackendPluginFolder = filepath.Clean(config.BackendPluginFolder)
+		//
+		cwhub.Installdir = config.InstallFolder
+		cwhub.Cfgdir = config.configFolder
+		cwhub.Hubdir = config.hubFolder
+		config.configured = true
+	}
+}
+
+func main() {
+
+	var rootCmd = &cobra.Command{
+		Use:   "cscli",
+		Short: "cscli allows you to manage crowdsec",
+		Long: `cscli is the main command to interact with your crowdsec service, scenarios & db.
+It is meant to allow you to manage bans, parsers/scenarios/etc, api and generally manage you crowdsec setup.`,
+		Example: `View/Add/Remove bans:  
+ - cscli ban list  
+ - cscli ban add ip 1.2.3.4 24h 'go away'  
+ - cscli ban del 1.2.3.4  
+		
+View/Add/Upgrade/Remove scenarios and parsers:  
+ - cscli list  
+ - cscli install collection crowdsec/linux-web  
+ - cscli remove scenario crowdsec/ssh_enum  
+ - cscli upgrade --all  
+
+API interaction:
+ - cscli api pull
+ - cscli api register
+ `}
+	/*TODO : add a remediation type*/
+	var cmdDocGen = &cobra.Command{
+		Use:    "doc",
+		Short:  "Generate the documentation in `./doc/`. Directory must exist.",
+		Args:   cobra.ExactArgs(0),
+		Hidden: true,
+		Run: func(cmd *cobra.Command, args []string) {
+			doc.GenMarkdownTree(rootCmd, "./doc/")
+		},
+	}
+	rootCmd.AddCommand(cmdDocGen)
+	/*usage*/
+	var cmdVersion = &cobra.Command{
+		Use:    "version",
+		Short:  "Display version and exit.",
+		Args:   cobra.ExactArgs(0),
+		Hidden: true,
+		Run: func(cmd *cobra.Command, args []string) {
+			cwversion.Show()
+		},
+	}
+	rootCmd.AddCommand(cmdVersion)
+
+	//rootCmd.PersistentFlags().BoolVarP(&config.simulation, "simulate", "s", false, "No action; perform a simulation of events that would occur based on the current arguments.")
+	rootCmd.PersistentFlags().StringVarP(&config.configFolder, "config-dir", "c", "/etc/crowdsec/cscli/", "Configuration directory to use.")
+	rootCmd.PersistentFlags().StringVarP(&config.output, "output", "o", "human", "Output format : human, json, raw.")
+	rootCmd.PersistentFlags().BoolVar(&dbg_lvl, "debug", false, "Set logging to debug.")
+	rootCmd.PersistentFlags().BoolVar(&nfo_lvl, "info", false, "Set logging to info.")
+	rootCmd.PersistentFlags().BoolVar(&wrn_lvl, "warning", false, "Set logging to warning.")
+	rootCmd.PersistentFlags().BoolVar(&err_lvl, "error", false, "Set logging to error.")
+
+	cobra.OnInitialize(initConfig)
+	/*don't sort flags so we can enforce order*/
+	rootCmd.Flags().SortFlags = false
+	rootCmd.PersistentFlags().SortFlags = false
+
+	rootCmd.AddCommand(NewBanCmds())
+	rootCmd.AddCommand(NewConfigCmd())
+	rootCmd.AddCommand(NewInstallCmd())
+	rootCmd.AddCommand(NewListCmd())
+	rootCmd.AddCommand(NewRemoveCmd())
+	rootCmd.AddCommand(NewUpdateCmd())
+	rootCmd.AddCommand(NewUpgradeCmd())
+	rootCmd.AddCommand(NewAPICmd())
+	rootCmd.AddCommand(NewMetricsCmd())
+	rootCmd.AddCommand(NewBackupCmd())
+	rootCmd.AddCommand(NewDashboardCmd())
+	rootCmd.AddCommand(NewInspectCmd())
+
+	rootCmd.Execute()
+}

cmd/crowdsec-cli/metrics.go

@@ -0,0 +1,229 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"os"
+	"strconv"
+	"strings"
+	"time"
+
+	log "github.com/sirupsen/logrus"
+	"gopkg.in/yaml.v2"
+
+	"github.com/olekukonko/tablewriter"
+	dto "github.com/prometheus/client_model/go"
+	"github.com/prometheus/prom2json"
+	"github.com/spf13/cobra"
+)
+
+/*This is a complete rip from prom2json*/
+func ShowPrometheus(url string) {
+	mfChan := make(chan *dto.MetricFamily, 1024)
+
+	// Start with the DefaultTransport for sane defaults.
+	transport := http.DefaultTransport.(*http.Transport).Clone()
+	// Conservatively disable HTTP keep-alives as this program will only
+	// ever need a single HTTP request.
+	transport.DisableKeepAlives = true
+	// Timeout early if the server doesn't even return the headers.
+	transport.ResponseHeaderTimeout = time.Minute
+
+	go func() {
+		err := prom2json.FetchMetricFamilies(url, mfChan, transport)
+		if err != nil {
+			log.Fatalf("failed to fetch prometheus metrics : %v", err)
+		}
+	}()
+
+	result := []*prom2json.Family{}
+	for mf := range mfChan {
+		result = append(result, prom2json.NewFamily(mf))
+	}
+	log.Debugf("Finished reading prometheus output, %d entries", len(result))
+	/*walk*/
+	acquis_stats := map[string]map[string]int{}
+	parsers_stats := map[string]map[string]int{}
+	buckets_stats := map[string]map[string]int{}
+	for idx, fam := range result {
+		if !strings.HasPrefix(fam.Name, "cs_") {
+			continue
+		}
+		log.Debugf("round %d", idx)
+		for _, m := range fam.Metrics {
+			metric := m.(prom2json.Metric)
+			name, ok := metric.Labels["name"]
+			if !ok {
+				log.Debugf("no name in Metric")
+			}
+			source, ok := metric.Labels["source"]
+			if !ok {
+				log.Debugf("no source in Metric")
+			}
+			value := m.(prom2json.Metric).Value
+			ival, err := strconv.Atoi(value)
+			if err != nil {
+				log.Errorf("Unexpected int value %s : %s", value, err)
+			}
+			switch fam.Name {
+			/*buckets*/
+			case "cs_bucket_create":
+				if _, ok := buckets_stats[name]; !ok {
+					buckets_stats[name] = make(map[string]int)
+				}
+				buckets_stats[name]["instanciation"] += ival
+			case "cs_bucket_overflow":
+				if _, ok := buckets_stats[name]; !ok {
+					buckets_stats[name] = make(map[string]int)
+				}
+				buckets_stats[name]["overflow"] += ival
+			case "cs_bucket_pour":
+				if _, ok := buckets_stats[name]; !ok {
+					buckets_stats[name] = make(map[string]int)
+				}
+				if _, ok := acquis_stats[source]; !ok {
+					acquis_stats[source] = make(map[string]int)
+				}
+				buckets_stats[name]["pour"] += ival
+				acquis_stats[source]["pour"] += ival
+			case "cs_bucket_underflow":
+				if _, ok := buckets_stats[name]; !ok {
+					buckets_stats[name] = make(map[string]int)
+				}
+				buckets_stats[name]["underflow"] += ival
+				/*acquis*/
+			case "cs_reader_hits":
+				if _, ok := acquis_stats[source]; !ok {
+					acquis_stats[source] = make(map[string]int)
+				}
+				acquis_stats[source]["reads"] += ival
+			case "cs_parser_hits_ok":
+				if _, ok := acquis_stats[source]; !ok {
+					acquis_stats[source] = make(map[string]int)
+				}
+				acquis_stats[source]["parsed"] += ival
+			case "cs_parser_hits_ko":
+				if _, ok := acquis_stats[source]; !ok {
+					acquis_stats[source] = make(map[string]int)
+				}
+				acquis_stats[source]["unparsed"] += ival
+			case "cs_node_hits":
+				if _, ok := parsers_stats[name]; !ok {
+					parsers_stats[name] = make(map[string]int)
+				}
+				parsers_stats[name]["hits"] += ival
+			case "cs_node_hits_ok":
+				if _, ok := parsers_stats[name]; !ok {
+					parsers_stats[name] = make(map[string]int)
+				}
+				parsers_stats[name]["parsed"] += ival
+			default:
+				continue
+			}
+
+		}
+	}
+	if config.output == "human" {
+		atable := tablewriter.NewWriter(os.Stdout)
+		atable.SetHeader([]string{"Source", "Lines read", "Lines parsed", "Lines unparsed", "Lines poured to bucket"})
+		for alabel, astats := range acquis_stats {
+
+			if alabel == "" {
+				continue
+			}
+			row := []string{}
+			row = append(row, alabel) //name
+			for _, sl := range []string{"reads", "parsed", "unparsed", "pour"} {
+				if v, ok := astats[sl]; ok {
+					row = append(row, fmt.Sprintf("%d", v))
+				} else {
+					row = append(row, "-")
+				}
+			}
+			atable.Append(row)
+		}
+		btable := tablewriter.NewWriter(os.Stdout)
+		btable.SetHeader([]string{"Bucket", "Overflows", "Instanciated", "Poured", "Expired"})
+		for blabel, bstats := range buckets_stats {
+			if blabel == "" {
+				continue
+			}
+			row := []string{}
+			row = append(row, blabel) //name
+			for _, sl := range []string{"overflow", "instanciation", "pour", "underflow"} {
+				if v, ok := bstats[sl]; ok {
+					row = append(row, fmt.Sprintf("%d", v))
+				} else {
+					row = append(row, "-")
+				}
+			}
+			btable.Append(row)
+		}
+		ptable := tablewriter.NewWriter(os.Stdout)
+		ptable.SetHeader([]string{"Parsers", "Hits", "Parsed", "Unparsed"})
+		for plabel, pstats := range parsers_stats {
+			if plabel == "" {
+				continue
+			}
+			row := []string{}
+			row = append(row, plabel) //name
+			hits := 0
+			parsed := 0
+			for _, sl := range []string{"hits", "parsed"} {
+				if v, ok := pstats[sl]; ok {
+					row = append(row, fmt.Sprintf("%d", v))
+					if sl == "hits" {
+						hits = v
+					} else if sl == "parsed" {
+						parsed = v
+					}
+				} else {
+					row = append(row, "-")
+				}
+			}
+			row = append(row, fmt.Sprintf("%d", hits-parsed))
+			ptable.Append(row)
+		}
+		log.Printf("Buckets Metrics:")
+		btable.Render() // Send output
+		log.Printf("Acquisition Metrics:")
+		atable.Render() // Send output
+		log.Printf("Parser Metrics:")
+		ptable.Render() // Send output
+	} else if config.output == "json" {
+		for _, val := range []map[string]map[string]int{acquis_stats, parsers_stats, buckets_stats} {
+			x, err := json.MarshalIndent(val, "", " ")
+			if err != nil {
+				log.Fatalf("failed to unmarshal metrics : %v", err)
+			}
+			fmt.Printf("%s\n", string(x))
+		}
+	} else if config.output == "raw" {
+		for _, val := range []map[string]map[string]int{acquis_stats, parsers_stats, buckets_stats} {
+			x, err := yaml.Marshal(val)
+			if err != nil {
+				log.Fatalf("failed to unmarshal metrics : %v", err)
+			}
+			fmt.Printf("%s\n", string(x))
+		}
+	}
+}
+
+var purl string
+
+func NewMetricsCmd() *cobra.Command {
+	/* ---- UPDATE COMMAND */
+	var cmdMetrics = &cobra.Command{
+		Use:   "metrics",
+		Short: "Display crowdsec prometheus metrics.",
+		Long:  `Fetch metrics from the prometheus server and display them in a human-friendly way`,
+		Args:  cobra.ExactArgs(0),
+		Run: func(cmd *cobra.Command, args []string) {
+			ShowPrometheus(purl)
+		},
+	}
+	cmdMetrics.PersistentFlags().StringVarP(&purl, "url", "u", "http://127.0.0.1:6060/metrics", "Prometheus url")
+
+	return cmdMetrics
+}

cmd/crowdsec-cli/remove.go

@@ -0,0 +1,150 @@
+package main
+
+import (
+	"fmt"
+
+	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
+
+	log "github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+)
+
+var purge_remove, remove_all bool
+
+func RemoveMany(ttype string, name string) {
+	var err error
+	var disabled int
+	for _, v := range cwhub.HubIdx[ttype] {
+		if name != "" && v.Name == name {
+			v, err = cwhub.DisableItem(v, cwhub.Installdir, cwhub.Hubdir, purge_remove)
+			if err != nil {
+				log.Fatalf("unable to disable %s : %v", v.Name, err)
+			}
+			disabled += 1
+			cwhub.HubIdx[ttype][v.Name] = v
+			return
+		} else if name == "" && remove_all {
+			v, err = cwhub.DisableItem(v, cwhub.Installdir, cwhub.Hubdir, purge_remove)
+			if err != nil {
+				log.Fatalf("unable to disable %s : %v", v.Name, err)
+			}
+			cwhub.HubIdx[ttype][v.Name] = v
+			disabled += 1
+		}
+	}
+	if name != "" && !remove_all {
+		log.Errorf("%s not found", name)
+		return
+	}
+	log.Infof("Disabled %d items", disabled)
+}
+
+func NewRemoveCmd() *cobra.Command {
+
+	var cmdRemove = &cobra.Command{
+		Use:   "remove [type] <config>",
+		Short: "Remove/disable configuration(s)",
+		Long: `
+ Remove local configuration. 
+ 
+[type] must be parser, scenario, postoverflow, collection
+
+[config_name] must be a valid config name from [Crowdsec Hub](https://hub.crowdsec.net) or locally installed.
+ `,
+		Example: `cscli remove [type] [config_name]`,
+		Args:    cobra.MinimumNArgs(1),
+		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
+			if !config.configured {
+				return fmt.Errorf("you must configure cli before interacting with hub")
+			}
+			return nil
+		},
+	}
+	cmdRemove.PersistentFlags().BoolVar(&purge_remove, "purge", false, "Delete source file in ~/.cscli/hub/ too")
+	cmdRemove.PersistentFlags().BoolVar(&remove_all, "all", false, "Delete all the files in selected scope")
+	var cmdRemoveParser = &cobra.Command{
+		Use:   "parser <config>",
+		Short: "Remove/disable parser",
+		Long:  `<config> must be a valid parser.`,
+		Args:  cobra.MinimumNArgs(0),
+		Run: func(cmd *cobra.Command, args []string) {
+			if err := cwhub.GetHubIdx(); err != nil {
+				log.Fatalf("Failed to get Hub index : %v", err)
+			}
+
+			if remove_all && len(args) == 0 {
+				RemoveMany(cwhub.PARSERS, "")
+			} else if len(args) == 1 {
+				RemoveMany(cwhub.PARSERS, args[0])
+			} else {
+				_ = cmd.Help()
+				return
+			}
+			//fmt.Println("remove/disable parser: " + strings.Join(args, " "))
+		},
+	}
+	cmdRemove.AddCommand(cmdRemoveParser)
+	var cmdRemoveScenario = &cobra.Command{
+		Use:   "scenario [config]",
+		Short: "Remove/disable scenario",
+		Long:  `<config> must be a valid scenario.`,
+		Args:  cobra.MinimumNArgs(0),
+		Run: func(cmd *cobra.Command, args []string) {
+			if err := cwhub.GetHubIdx(); err != nil {
+				log.Fatalf("Failed to get Hub index : %v", err)
+			}
+			if remove_all && len(args) == 0 {
+				RemoveMany(cwhub.SCENARIOS, "")
+			} else if len(args) == 1 {
+				RemoveMany(cwhub.SCENARIOS, args[0])
+			} else {
+				_ = cmd.Help()
+				return
+			}
+		},
+	}
+	cmdRemove.AddCommand(cmdRemoveScenario)
+	var cmdRemoveCollection = &cobra.Command{
+		Use:   "collection [config]",
+		Short: "Remove/disable collection",
+		Long:  `<config> must be a valid collection.`,
+		Args:  cobra.MinimumNArgs(0),
+		Run: func(cmd *cobra.Command, args []string) {
+			if err := cwhub.GetHubIdx(); err != nil {
+				log.Fatalf("Failed to get Hub index : %v", err)
+			}
+			if remove_all && len(args) == 0 {
+				RemoveMany(cwhub.COLLECTIONS, "")
+			} else if len(args) == 1 {
+				RemoveMany(cwhub.COLLECTIONS, args[0])
+			} else {
+				_ = cmd.Help()
+				return
+			}
+		},
+	}
+	cmdRemove.AddCommand(cmdRemoveCollection)
+
+	var cmdRemovePostoverflow = &cobra.Command{
+		Use:   "postoverflow [config]",
+		Short: "Remove/disable postoverflow parser",
+		Long:  `<config> must be a valid collection.`,
+		Args:  cobra.MinimumNArgs(0),
+		Run: func(cmd *cobra.Command, args []string) {
+			if err := cwhub.GetHubIdx(); err != nil {
+				log.Fatalf("Failed to get Hub index : %v", err)
+			}
+			if remove_all && len(args) == 0 {
+				RemoveMany(cwhub.PARSERS_OVFLW, "")
+			} else if len(args) == 1 {
+				RemoveMany(cwhub.PARSERS_OVFLW, args[0])
+			} else {
+				_ = cmd.Help()
+				return
+			}
+		},
+	}
+	cmdRemove.AddCommand(cmdRemovePostoverflow)
+
+	return cmdRemove
+}

cmd/crowdsec-cli/update.go

@@ -0,0 +1,34 @@
+package main
+
+import (
+	"fmt"
+
+	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
+
+	log "github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+)
+
+func NewUpdateCmd() *cobra.Command {
+	/* ---- UPDATE COMMAND */
+	var cmdUpdate = &cobra.Command{
+		Use:   "update",
+		Short: "Fetch available configs from hub",
+		Long: `
+Fetches the [.index.json](https://github.com/crowdsecurity/hub/blob/master/.index.json) file from hub, containing the list of available configs.
+`,
+		Args: cobra.ExactArgs(0),
+		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
+			if !config.configured {
+				return fmt.Errorf("You must configure cli before interacting with hub.")
+			}
+			return nil
+		},
+		Run: func(cmd *cobra.Command, args []string) {
+			if err := cwhub.UpdateHubIdx(); err != nil {
+				log.Fatalf("Failed to get Hub index : %v", err)
+			}
+		},
+	}
+	return cmdUpdate
+}

cmd/crowdsec-cli/upgrade.go

@@ -0,0 +1,205 @@
+package main
+
+import (
+	"fmt"
+
+	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
+
+	"github.com/enescakir/emoji"
+	log "github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+)
+
+var upgrade_all, force_upgrade bool
+
+func UpgradeConfig(ttype string, name string) {
+	var err error
+	var updated int
+	var found bool
+
+	for _, v := range cwhub.HubIdx[ttype] {
+		//name mismatch
+		if name != "" && name != v.Name {
+			continue
+		}
+		if !v.Installed {
+			log.Debugf("skip %s, not installed", v.Name)
+			continue
+		}
+		if !v.Downloaded {
+			log.Warningf("%s : not downloaded, please install.", v.Name)
+			continue
+		}
+		found = true
+		if v.UpToDate {
+			log.Infof("%s : up-to-date", v.Name)
+			continue
+		}
+		v, err = cwhub.DownloadLatest(v, cwhub.Hubdir, force_upgrade)
+		if err != nil {
+			log.Fatalf("%s : download failed : %v", v.Name, err)
+		}
+		if !v.UpToDate {
+			if v.Tainted {
+				log.Infof("%v %s is tainted, --force to overwrite", emoji.Warning, v.Name)
+			} else if v.Local {
+				log.Infof("%v %s is local", emoji.Prohibited, v.Name)
+			}
+		} else {
+			log.Infof("%v %s : updated", emoji.Package, v.Name)
+			updated += 1
+		}
+		cwhub.HubIdx[ttype][v.Name] = v
+	}
+	if found == false {
+		log.Errorf("Didn't find %s", name)
+	} else if updated == 0 && found == true {
+		log.Errorf("Nothing to update")
+	} else if updated != 0 {
+		log.Infof("Upgraded %d items", updated)
+	}
+
+}
+
+func NewUpgradeCmd() *cobra.Command {
+
+	var cmdUpgrade = &cobra.Command{
+		Use:   "upgrade [type] [config]",
+		Short: "Upgrade configuration(s)",
+		Long: `
+Upgrade configuration from the CrowdSec Hub.
+
+In order to upgrade latest versions of configuration, 
+the Hub cache should be [updated](./cscli_update.md).
+ 
+Tainted configuration will not be updated (use --force to update them).
+
+[type] must be parser, scenario, postoverflow, collection.
+
+[config_name] must be a valid config name from [Crowdsec Hub](https://hub.crowdsec.net).
+ 
+
+ `,
+		Example: `cscli upgrade [type] [config_name]
+cscli upgrade --all   # Upgrade all configurations types
+cscli upgrade --force # Overwrite tainted configuration
+		`,
+
+		Args: cobra.MinimumNArgs(0),
+		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
+			if !config.configured {
+				return fmt.Errorf("you must configure cli before interacting with hub")
+			}
+			return nil
+		},
+		Run: func(cmd *cobra.Command, args []string) {
+			if upgrade_all == false && len(args) < 2 {
+				_ = cmd.Help()
+				return
+			}
+			if err := cwhub.GetHubIdx(); err != nil {
+				log.Fatalf("Failed to get Hub index : %v", err)
+			}
+			if upgrade_all == true && len(args) == 0 {
+				log.Warningf("Upgrade all : parsers, scenarios, collections.")
+				UpgradeConfig(cwhub.PARSERS, "")
+				UpgradeConfig(cwhub.PARSERS_OVFLW, "")
+				UpgradeConfig(cwhub.SCENARIOS, "")
+				UpgradeConfig(cwhub.COLLECTIONS, "")
+			}
+			//fmt.Println("upgrade all ?!: " + strings.Join(args, " "))
+		},
+	}
+	cmdUpgrade.PersistentFlags().BoolVar(&upgrade_all, "all", false, "Upgrade all configuration in scope")
+	cmdUpgrade.PersistentFlags().BoolVar(&force_upgrade, "force", false, "Overwrite existing files, even if tainted")
+	var cmdUpgradeParser = &cobra.Command{
+		Use:   "parser [config]",
+		Short: "Upgrade parser configuration(s)",
+		Long:  `Upgrade one or more parser configurations`,
+		Example: ` - cscli upgrade parser crowdsec/apache-logs  
+ - cscli upgrade parser -all  
+ - cscli upgrade parser crowdsec/apache-logs --force`,
+		Args: cobra.MinimumNArgs(0),
+		Run: func(cmd *cobra.Command, args []string) {
+			if err := cwhub.GetHubIdx(); err != nil {
+				log.Fatalf("Failed to get Hub index : %v", err)
+			}
+			if len(args) == 1 {
+				UpgradeConfig(cwhub.PARSERS, args[0])
+				//UpgradeConfig(cwhub.PARSERS_OVFLW, "")
+			} else if upgrade_all == true {
+				UpgradeConfig(cwhub.PARSERS, "")
+			} else {
+				_ = cmd.Help()
+			}
+		},
+	}
+	cmdUpgrade.AddCommand(cmdUpgradeParser)
+	var cmdUpgradeScenario = &cobra.Command{
+		Use:   "scenario [config]",
+		Short: "Upgrade scenario configuration(s)",
+		Long:  `Upgrade one or more scenario configurations`,
+		Example: ` - cscli	upgrade scenario -all  
+ - cscli upgrade scenario crowdsec/http-404 --force  `,
+		Args: cobra.MinimumNArgs(0),
+		Run: func(cmd *cobra.Command, args []string) {
+			if err := cwhub.GetHubIdx(); err != nil {
+				log.Fatalf("Failed to get Hub index : %v", err)
+			}
+			if len(args) == 1 {
+				UpgradeConfig(cwhub.SCENARIOS, args[0])
+			} else if upgrade_all == true {
+				UpgradeConfig(cwhub.SCENARIOS, "")
+			} else {
+				_ = cmd.Help()
+			}
+		},
+	}
+	cmdUpgrade.AddCommand(cmdUpgradeScenario)
+	var cmdUpgradeCollection = &cobra.Command{
+		Use:   "collection [config]",
+		Short: "Upgrade collection configuration(s)",
+		Long:  `Upgrade one or more collection configurations`,
+		Example: ` - cscli upgrade collection crowdsec/apache-lamp  
+ - cscli upgrade collection -all  
+ - cscli upgrade collection crowdsec/apache-lamp --force`,
+		Args: cobra.MinimumNArgs(0),
+		Run: func(cmd *cobra.Command, args []string) {
+			if err := cwhub.GetHubIdx(); err != nil {
+				log.Fatalf("Failed to get Hub index : %v", err)
+			}
+			if len(args) == 1 {
+				UpgradeConfig(cwhub.COLLECTIONS, args[0])
+			} else if upgrade_all == true {
+				UpgradeConfig(cwhub.COLLECTIONS, "")
+			} else {
+				_ = cmd.Help()
+			}
+		},
+	}
+	cmdUpgrade.AddCommand(cmdUpgradeCollection)
+
+	var cmdUpgradePostoverflow = &cobra.Command{
+		Use:   "postoverflow [config]",
+		Short: "Upgrade postoverflow parser configuration(s)",
+		Long:  `Upgrade one or more postoverflow parser configurations`,
+		Example: ` - cscli upgrade postoverflow crowdsec/enrich-rdns  
+ - cscli upgrade postoverflow -all  
+ - cscli upgrade postoverflow crowdsec/enrich-rdns --force`,
+		Args: cobra.MinimumNArgs(0),
+		Run: func(cmd *cobra.Command, args []string) {
+			if err := cwhub.GetHubIdx(); err != nil {
+				log.Fatalf("Failed to get Hub index : %v", err)
+			}
+			if len(args) == 1 {
+				UpgradeConfig(cwhub.PARSERS_OVFLW, args[0])
+			} else if upgrade_all == true {
+				UpgradeConfig(cwhub.PARSERS_OVFLW, "")
+			} else {
+				_ = cmd.Help()
+			}
+		},
+	}
+	cmdUpgrade.AddCommand(cmdUpgradePostoverflow)
+	return cmdUpgrade
+}

cmd/crowdsec/Makefile

@@ -0,0 +1,68 @@
+# Go parameters
+GOCMD=go
+GOBUILD=$(GOCMD) build
+GOCLEAN=$(GOCMD) clean
+GOTEST=$(GOCMD) test
+GOGET=$(GOCMD) get
+
+CROWDSEC_BIN=crowdsec
+PREFIX?="/"
+CFG_PREFIX = $(PREFIX)"/etc/crowdsec/crowdsec/"
+BIN_PREFIX = $(PREFIX)"/usr/local/bin/"
+DATA_PREFIX = $(PREFIX)"/var/run/crowdsec/"
+PID_DIR = $(PREFIX)"/var/run/"
+
+SYSTEMD_PATH_FILE="/etc/systemd/system/crowdsec.service"
+
+
+all: clean test build
+
+build: clean
+	$(GOBUILD) $(LD_OPTS) -o $(CROWDSEC_BIN) -v
+
+static: clean
+	$(GOBUILD) -o $(CROWDSEC_BIN) -v -a -tags netgo -ldflags '-w -extldflags "-static"'
+
+test:
+	$(GOTEST) -v ./...
+
+clean:
+	rm -f $(CROWDSEC_BIN)
+
+.PHONY: install
+install: install-conf install-bin
+
+.PHONY: install-conf
+install-conf:
+	mkdir -p $(DATA_PREFIX) || exit
+	(cd ../.. / && find ./data -type f -exec install -Dm 755 "{}" "$(DATA_PREFIX){}" \; && cd ./cmd/crowdsec) || exit
+	(cd ../../config && find ./patterns -type f -exec install -Dm 755 "{}" "$(CFG_PREFIX){}" \; && cd ../cmd/crowdsec) || exit
+	mkdir -p "$(CFG_PREFIX)" || exit
+	mkdir -p "$(CFG_PREFIX)/parsers" || exit
+	mkdir -p "$(CFG_PREFIX)/scenarios" || exit
+	mkdir -p "$(CFG_PREFIX)/postoverflows" || exit
+	mkdir -p "$(CFG_PREFIX)/collections" || exit
+	mkdir -p "$(CFG_PREFIX)/patterns" || exit
+	install -v -m 755 -D ../../config/prod.yaml "$(CFG_PREFIX)" || exit
+	install -v -m 755 -D ../../config/dev.yaml "$(CFG_PREFIX)" || exit
+	install -v -m 755 -D ../../config/acquis.yaml "$(CFG_PREFIX)" || exit
+	install -v -m 755 -D ../../config/profiles.yaml "$(CFG_PREFIX)" || exit
+	install -v -m 755 -D ../../config/api.yaml "$(CFG_PREFIX)" || exit
+	mkdir -p $(PID_DIR) || exit
+	PID=$(PID_DIR) DATA=$(DATA_PREFIX)"/data/" CFG=$(CFG_PREFIX) envsubst < ../../config/prod.yaml > $(CFG_PREFIX)"/default.yaml"	
+
+.PHONY: install-bin
+install-bin:
+	install -v -m 755 -D "$(CROWDSEC_BIN)" "$(BIN_PREFIX)/$(CROWDSEC_BIN)" || exit
+
+.PHONY: systemd"$(BIN_PREFI"$(BIN_PREFIX)/$(CROWDSEC_BIN)""$(BIN_PREFIX)/$(CROWDSEC_BIN)"X)/$(CROWDSEC_BIN)"
+systemd: install
+	CFG=$(CFG_PREFIX) PID=$(PID_DIR) BIN=$(BIN_PREFIX)"/"$(CROWDSEC_BIN) envsubst < ../../config/crowdsec.service > "$(SYSTEMD_PATH_FILE)"
+	systemctl daemon-reload
+
+.PHONY: uninstall
+uninstall:
+	rm -rf $(CFG_PREFIX)
+	rm -rf $(DATA_PREFIX)
+	rm -f "$(BIN_PREFIX)/$(CROWDSEC_BIN)"
+	rm -f "$(SYSTEMD_PATH_FILE)"

cmd/crowdsec/acquisition.go

@@ -0,0 +1,33 @@
+package main
+
+import (
+	"fmt"
+	"github.com/crowdsecurity/crowdsec/pkg/acquisition"
+)
+
+func loadAcquisition() (*acquisition.FileAcquisCtx, error) {
+	var acquisitionCTX *acquisition.FileAcquisCtx
+	var err error
+	/*Init the acqusition : from cli or from acquis.yaml file*/
+	if cConfig.SingleFile != "" {
+		var input acquisition.FileCtx
+		input.Filename = cConfig.SingleFile
+		input.Mode = acquisition.CATMODE
+		input.Labels = make(map[string]string)
+		input.Labels["type"] = cConfig.SingleFileLabel
+		acquisitionCTX, err = acquisition.InitReaderFromFileCtx([]acquisition.FileCtx{input})
+	} else { /* Init file reader if we tail */
+		acquisitionCTX, err = acquisition.InitReader(cConfig.AcquisitionFile)
+	}
+	if err != nil {
+		return nil, fmt.Errorf("unable to start file acquisition, bailout %v", err)
+	}
+	if acquisitionCTX == nil {
+		return nil, fmt.Errorf("no inputs to process")
+	}
+	if cConfig.Profiling == true {
+		acquisitionCTX.Profiling = true
+	}
+
+	return acquisitionCTX, nil
+}

cmd/crowdsec/main.go

@@ -0,0 +1,320 @@
+package main
+
+import (
+	"fmt"
+	"strings"
+
+	"io/ioutil"
+
+	_ "net/http/pprof"
+	"time"
+
+	"github.com/crowdsecurity/crowdsec/pkg/acquisition"
+	config "github.com/crowdsecurity/crowdsec/pkg/config/crowdsec"
+	"github.com/crowdsecurity/crowdsec/pkg/cwversion"
+	leaky "github.com/crowdsecurity/crowdsec/pkg/leakybucket"
+	"github.com/crowdsecurity/crowdsec/pkg/outputs"
+	"github.com/crowdsecurity/crowdsec/pkg/parser"
+	"github.com/crowdsecurity/crowdsec/pkg/types"
+
+	log "github.com/sirupsen/logrus"
+
+	"gopkg.in/natefinch/lumberjack.v2"
+	"gopkg.in/tomb.v2"
+	"gopkg.in/yaml.v2"
+)
+
+var (
+	/*tombs for the parser, buckets and outputs.*/
+	acquisTomb  tomb.Tomb
+	parsersTomb tomb.Tomb
+	bucketsTomb tomb.Tomb
+	outputsTomb tomb.Tomb
+
+	holders []leaky.BucketFactory
+	buckets *leaky.Buckets
+	cConfig *config.Crowdwatch
+
+	/*settings*/
+	lastProcessedItem time.Time /*keep track of last item timestamp in time-machine. it is used to GC buckets when we dump them.*/
+)
+
+func configureLogger(logMode string, logFolder string, logLevel log.Level) error {
+	/*Configure logs*/
+	if logMode == "file" {
+		log.SetOutput(&lumberjack.Logger{
+			Filename:   logFolder + "/crowdsec.log",
+			MaxSize:    500, //megabytes
+			MaxBackups: 3,
+			MaxAge:     28,   //days
+			Compress:   true, //disabled by default
+		})
+		log.SetFormatter(&log.TextFormatter{TimestampFormat: "02-01-2006 15:04:05", FullTimestamp: true})
+	} else if logMode != "stdout" {
+		return fmt.Errorf("log mode '%s' unknown", logMode)
+	}
+
+	log.Printf("setting loglevel to %s", logLevel)
+	log.SetLevel(logLevel)
+	log.SetFormatter(&log.TextFormatter{FullTimestamp: true})
+	if logLevel >= log.InfoLevel {
+		log.SetFormatter(&log.TextFormatter{TimestampFormat: "02-01-2006 15:04:05", FullTimestamp: true})
+	}
+	if logLevel >= log.DebugLevel {
+		log.SetReportCaller(true)
+	}
+	return nil
+}
+
+func main() {
+	var (
+		err                 error
+		p                   parser.UnixParser
+		parserNodes         []parser.Node = make([]parser.Node, 0)
+		postOverflowNodes   []parser.Node = make([]parser.Node, 0)
+		nbParser            int           = 1
+		parserCTX           *parser.UnixParserCtx
+		postOverflowCTX     *parser.UnixParserCtx
+		acquisitionCTX      *acquisition.FileAcquisCtx
+		CustomParsers       []parser.Stagefile
+		CustomPostoverflows []parser.Stagefile
+		CustomScenarios     []parser.Stagefile
+		outputEventChan     chan types.Event
+	)
+
+	inputLineChan := make(chan types.Event)
+	inputEventChan := make(chan types.Event)
+
+	cConfig = config.NewCrowdwatchConfig()
+
+	// Handle command line arguments
+	if err := cConfig.GetOPT(); err != nil {
+		log.Fatalf(err.Error())
+	}
+
+	if err = configureLogger(cConfig.LogMode, cConfig.LogFolder, cConfig.LogLevel); err != nil {
+		log.Fatal(err.Error())
+	}
+
+	log.Infof("Crowdwatch %s", cwversion.VersionStr())
+
+	if cConfig.Prometheus == true {
+		registerPrometheus()
+		cConfig.Profiling = true
+	}
+
+	log.Infof("Loading grok library")
+	/* load base regexps for two grok parsers */
+	parserCTX, err = p.Init(map[string]interface{}{"patterns": cConfig.ConfigFolder + string("/patterns/")})
+	if err != nil {
+		log.Errorf("failed to initialize parser : %v", err)
+		return
+	}
+	postOverflowCTX, err = p.Init(map[string]interface{}{"patterns": cConfig.ConfigFolder + string("/patterns/")})
+	if err != nil {
+		log.Errorf("failed to initialize postoverflow : %v", err)
+		return
+	}
+
+	/*enable profiling*/
+	if cConfig.Profiling == true {
+		go runTachymeter(cConfig.HTTPListen)
+		parserCTX.Profiling = true
+		postOverflowCTX.Profiling = true
+	}
+
+	/*
+		Load enrichers
+	*/
+	log.Infof("Loading enrich plugins")
+	parserPlugins, err := parser.Loadplugin(cConfig.DataFolder)
+	if err != nil {
+		log.Errorf("Failed to load plugin geoip : %v", err)
+	}
+	parser.ECTX = append(parser.ECTX, parserPlugins)
+
+	/*parser the validatormode option if present. mostly used for testing purposes*/
+	if cConfig.ValidatorMode != "" {
+		//beurk : provided 'parser:file.yaml,postoverflow:file.yaml,scenario:file.yaml load only those
+		validators := strings.Split(cConfig.ValidatorMode, ",")
+		for _, val := range validators {
+			splittedValidator := strings.Split(val, ":")
+			if len(splittedValidator) != 2 {
+				log.Fatalf("parser:file,scenario:file,postoverflow:file")
+			}
+
+			configType := splittedValidator[0]
+			configFile := splittedValidator[1]
+
+			var parsedFile []parser.Stagefile
+			dataFile, err := ioutil.ReadFile(configFile)
+
+			if err != nil {
+				log.Fatalf("failed opening %s : %s", configFile, err)
+			}
+			if err := yaml.UnmarshalStrict(dataFile, &parsedFile); err != nil {
+				log.Fatalf("failed unmarshalling %s : %s", configFile, err)
+			}
+			switch configType {
+			case "parser":
+				CustomParsers = parsedFile
+			case "scenario":
+				CustomScenarios = parsedFile
+			case "postoverflow":
+				CustomPostoverflows = parsedFile
+			default:
+				log.Fatalf("wrong type, format is parser:file,scenario:file,postoverflow:file")
+			}
+
+		}
+	}
+
+	/* load the parser nodes */
+	if cConfig.ValidatorMode != "" && len(CustomParsers) > 0 {
+		log.Infof("Loading (validatormode) parsers")
+		parserNodes, err = parser.LoadStages(CustomParsers, parserCTX)
+	} else {
+		log.Infof("Loading parsers")
+		parserNodes, err = parser.LoadStageDir(cConfig.ConfigFolder+"/parsers/", parserCTX)
+	}
+	if err != nil {
+		log.Fatalf("failed to load parser config : %v", err)
+	}
+	/* parsers loaded */
+
+	/* load the post-overflow stages*/
+	if cConfig.ValidatorMode != "" && len(CustomPostoverflows) > 0 {
+		log.Infof("Loading (validatormode) postoverflow parsers")
+		postOverflowNodes, err = parser.LoadStages(CustomPostoverflows, postOverflowCTX)
+	} else {
+		log.Infof("Loading postoverflow parsers")
+		postOverflowNodes, err = parser.LoadStageDir(cConfig.ConfigFolder+"/postoverflows/", postOverflowCTX)
+	}
+	if err != nil {
+		log.Fatalf("failed to load postoverflow config : %v", err)
+	}
+
+	log.Infof("Loaded Nodes : %d parser, %d postoverflow", len(parserNodes), len(postOverflowNodes))
+	/* post overflow loaded */
+
+	/* Loading buckets / scenarios */
+	if cConfig.ValidatorMode != "" && len(CustomScenarios) > 0 {
+		log.Infof("Loading (validatormode) scenarios")
+		bucketFiles := []string{}
+		for _, scenarios := range CustomScenarios {
+			bucketFiles = append(bucketFiles, scenarios.Filename)
+		}
+		holders, outputEventChan, err = leaky.LoadBuckets(bucketFiles)
+
+	} else {
+		log.Infof("Loading scenarios")
+		holders, outputEventChan, err = leaky.Init(map[string]string{"patterns": cConfig.ConfigFolder + "/scenarios/"})
+	}
+	if err != nil {
+		log.Fatalf("Scenario loading failed : %v", err)
+	}
+	/* buckets/scenarios loaded */
+
+	/*keep track of scenarios name for consensus profiling*/
+	var scenariosEnabled string
+	for _, x := range holders {
+		if scenariosEnabled != "" {
+			scenariosEnabled += ","
+		}
+		scenariosEnabled += x.Name
+	}
+
+	buckets = leaky.NewBuckets()
+
+	/*restore as well previous state if present*/
+	if cConfig.RestoreMode != "" {
+		log.Warningf("Restoring buckets state from %s", cConfig.RestoreMode)
+		if err := leaky.LoadBucketsState(cConfig.RestoreMode, buckets, holders); err != nil {
+			log.Fatalf("unable to restore buckets : %s", err)
+		}
+	}
+	if cConfig.Profiling == true {
+		//force the profiling in all buckets
+		for holderIndex := range holders {
+			holders[holderIndex].Profiling = true
+		}
+	}
+
+	/*
+		Load output profiles
+	*/
+	log.Infof("Loading output profiles")
+	outputProfiles, err := outputs.LoadOutputProfiles(cConfig.ConfigFolder + "/profiles.yaml")
+	if err != nil || len(outputProfiles) == 0 {
+		log.Fatalf("Failed to load output profiles : %v", err)
+	}
+	/* Linting is done */
+	if cConfig.Linter {
+		return
+	}
+
+	outputRunner, err := outputs.NewOutput(cConfig.OutputConfig, cConfig.Daemonize)
+	if err != nil {
+		log.Fatalf("output plugins initialization error : %s", err.Error())
+	}
+
+	/* Init the API connector */
+	if cConfig.APIMode {
+		log.Infof("Loading API client")
+		var apiConfig = map[string]string{
+			"path":    cConfig.ConfigFolder + "/api.yaml",
+			"profile": scenariosEnabled,
+		}
+		if err := outputRunner.InitAPI(apiConfig); err != nil {
+			log.Fatalf(err.Error())
+		}
+	}
+
+	/*if the user is in "single file mode" (might be writting scenario or parsers), allow loading **without** parsers or scenarios */
+	if cConfig.SingleFile == "" {
+		if len(parserNodes) == 0 {
+			log.Fatalf("no parser(s) loaded, abort.")
+		}
+
+		if len(holders) == 0 {
+			log.Fatalf("no bucket(s) loaded, abort.")
+		}
+
+		if len(outputProfiles) == 0 {
+			log.Fatalf("no output profile(s) loaded, abort.")
+		}
+	}
+
+	log.Infof("Starting processing routines")
+	//start go-routines for parsing, buckets pour and ouputs.
+	for i := 0; i < nbParser; i++ {
+		parsersTomb.Go(func() error {
+			return runParse(inputLineChan, inputEventChan, *parserCTX, parserNodes)
+		})
+	}
+
+	for i := 0; i < nbParser; i++ {
+		bucketsTomb.Go(func() error {
+			return runPour(inputEventChan, holders, buckets)
+		})
+	}
+
+	for i := 0; i < nbParser; i++ {
+		outputsTomb.Go(func() error {
+			return runOutput(inputEventChan, outputEventChan, holders, buckets, *postOverflowCTX, postOverflowNodes, outputProfiles, outputRunner)
+		})
+	}
+
+	log.Warningf("Starting processing data")
+
+	//Init the acqusition : from cli or from acquis.yaml file
+	acquisitionCTX, err = loadAcquisition()
+
+	//start reading in the background
+	acquisition.AcquisStartReading(acquisitionCTX, inputLineChan, &acquisTomb)
+
+	if err = serve(*outputRunner); err != nil {
+		log.Fatalf(err.Error())
+	}
+
+}

cmd/crowdsec/metrics.go

@@ -0,0 +1,121 @@
+package main
+
+import (
+	"time"
+
+	"github.com/crowdsecurity/crowdsec/pkg/acquisition"
+	leaky "github.com/crowdsecurity/crowdsec/pkg/leakybucket"
+	"github.com/crowdsecurity/crowdsec/pkg/parser"
+	"github.com/jamiealquiza/tachymeter"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+
+	log "github.com/sirupsen/logrus"
+	"net/http"
+
+	"runtime"
+)
+
+var (
+	parseStat     *tachymeter.Tachymeter
+	bucketStat    *tachymeter.Tachymeter
+	outputStat    *tachymeter.Tachymeter
+	linesReadOK   uint64
+	linesReadKO   uint64
+	linesParsedOK uint64
+	linesParsedKO uint64
+	linesPouredOK uint64
+	linesPouredKO uint64
+)
+
+/*prometheus*/
+var globalParserHits = prometheus.NewCounterVec(
+	prometheus.CounterOpts{
+		Name: "cs_parser_hits",
+		Help: "How many time an event entered the parser.",
+	},
+	[]string{"source"},
+)
+var globalParserHitsOk = prometheus.NewCounterVec(
+	prometheus.CounterOpts{
+		Name: "cs_parser_hits_ok",
+		Help: "How many time an event was successfully parsed.",
+	},
+	[]string{"source"},
+)
+var globalParserHitsKo = prometheus.NewCounterVec(
+	prometheus.CounterOpts{
+		Name: "cs_parser_hits_ko",
+		Help: "How many time an event was unsuccessfully parsed.",
+	},
+	[]string{"source"},
+)
+
+var globalBucketPourKo = prometheus.NewCounter(
+	prometheus.CounterOpts{
+		Name: "cs_bucket_pour_ko",
+		Help: "How many time an event was poured in no bucket.",
+	},
+)
+
+var globalBucketPourOk = prometheus.NewCounter(
+	prometheus.CounterOpts{
+		Name: "cs_bucket_pour_ok",
+		Help: "How many time an event was poured in at least one bucket.",
+	},
+)
+
+func dumpMetrics() {
+
+	if cConfig.DumpBuckets == true {
+		log.Infof("!! Dumping buckets state")
+		if err := leaky.DumpBucketsStateAt("buckets_state.json", time.Now(), buckets); err != nil {
+			log.Fatalf("Failed dumping bucket state : %s", err)
+		}
+	}
+
+	if cConfig.Profiling {
+		var memoryStats runtime.MemStats
+		runtime.ReadMemStats(&memoryStats)
+
+		log.Infof("parser evt/s : %s", parseStat.Calc())
+		log.Infof("bucket pour evt/s : %s", bucketStat.Calc())
+		log.Infof("outputs evt/s : %s", outputStat.Calc())
+		log.Infof("Alloc = %v MiB", bToMb(memoryStats.Alloc))
+		log.Infof("TotalAlloc = %v MiB", bToMb(memoryStats.TotalAlloc))
+		log.Infof("Sys = %v MiB", bToMb(memoryStats.Sys))
+		log.Infof("NumGC = %v", memoryStats.NumGC)
+		log.Infof("Lines read ok : %d", linesReadOK)
+		if linesReadKO > 0 {
+			log.Infof("Lines discarded : %d (%.2f%%)", linesReadKO, float64(linesReadKO)/float64(linesReadOK)*100.0)
+		}
+		log.Infof("Lines parsed ok : %d", linesParsedOK)
+		if linesParsedKO > 0 {
+			log.Infof("Lines unparsed : %d (%.2f%%)", linesParsedKO, float64(linesParsedKO)/float64(linesParsedOK)*100.0)
+		}
+		log.Infof("Lines poured ok : %d", linesPouredOK)
+		if linesPouredKO > 0 {
+			log.Infof("Lines never poured : %d (%.2f%%)", linesPouredKO, float64(linesPouredKO)/float64(linesPouredOK)*100.0)
+		}
+		log.Infof("Writting metrics dump to %s", cConfig.WorkingFolder+"/crowdsec.profile")
+		prometheus.WriteToTextfile(cConfig.WorkingFolder+"/crowdsec.profile", prometheus.DefaultGatherer)
+	}
+}
+
+func runTachymeter(HTTPListen string) {
+	log.Warningf("Starting profiling and http server")
+	/*Tachymeter for global perfs */
+	parseStat = tachymeter.New(&tachymeter.Config{Size: 100})
+	bucketStat = tachymeter.New(&tachymeter.Config{Size: 100})
+	outputStat = tachymeter.New(&tachymeter.Config{Size: 100})
+	log.Fatal(http.ListenAndServe(HTTPListen, nil))
+}
+
+func registerPrometheus() {
+	/*Registering prometheus*/
+	log.Warningf("Loading prometheus collectors")
+	prometheus.MustRegister(globalParserHits, globalParserHitsOk, globalParserHitsKo, parser.NodesHits, parser.NodesHitsOk,
+		parser.NodesHitsKo, acquisition.ReaderHits, leaky.BucketsPour, leaky.BucketsUnderflow, leaky.BucketsInstanciation,
+		leaky.BucketsOverflow)
+	http.Handle("/metrics", promhttp.Handler())
+}

cmd/crowdsec/output.go

@@ -0,0 +1,54 @@
+package main
+
+import (
+	log "github.com/sirupsen/logrus"
+
+	"time"
+
+	leaky "github.com/crowdsecurity/crowdsec/pkg/leakybucket"
+	"github.com/crowdsecurity/crowdsec/pkg/outputs"
+	"github.com/crowdsecurity/crowdsec/pkg/parser"
+	"github.com/crowdsecurity/crowdsec/pkg/types"
+)
+
+func runOutput(input chan types.Event, overflow chan types.Event, holders []leaky.BucketFactory, buckets *leaky.Buckets,
+	poctx parser.UnixParserCtx, ponodes []parser.Node, outputProfiles []types.Profile, output *outputs.Output) error {
+	var (
+		//action string
+		start time.Time
+	)
+
+LOOP:
+	for {
+		select {
+		case <-bucketsTomb.Dying():
+			log.Infof("Exiting output processing")
+			output.FlushAll()
+			break LOOP
+		case event := <-overflow:
+			if cConfig.Profiling {
+				start = time.Now()
+			}
+
+			if event.Overflow.Reprocess {
+				log.Debugf("Overflow being reprocessed.")
+				input <- event
+			}
+
+			if event.Overflow.Scenario == "" && event.Overflow.MapKey != "" {
+				//log.Infof("Deleting expired entry %s", event.Overflow.MapKey)
+				buckets.Bucket_map.Delete(event.Overflow.MapKey)
+			} else {
+				/*let's handle output profiles */
+				if err := output.ProcessOutput(event.Overflow, outputProfiles); err != nil {
+					log.Warningf("Error while processing overflow/output : %s", err)
+				}
+			}
+		}
+		if cConfig.Profiling {
+			outputStat.AddTime(time.Since(start))
+		}
+	}
+	return nil
+
+}

cmd/crowdsec/parse.go

@@ -0,0 +1,74 @@
+package main
+
+import (
+	"errors"
+	"sync/atomic"
+	"time"
+
+	"github.com/prometheus/client_golang/prometheus"
+	log "github.com/sirupsen/logrus"
+
+	"github.com/crowdsecurity/crowdsec/pkg/parser"
+	"github.com/crowdsecurity/crowdsec/pkg/types"
+)
+
+func runParse(input chan types.Event, output chan types.Event, parserCTX parser.UnixParserCtx, nodes []parser.Node) error {
+	var start time.Time
+	var discardCPT, processCPT int
+
+LOOP:
+	for {
+		select {
+		case <-parsersTomb.Dying():
+			log.Infof("Killing parser routines")
+			break LOOP
+		case event := <-input:
+			if cConfig.Profiling {
+				start = time.Now()
+			}
+			if event.Process == false {
+				if cConfig.Profiling {
+					atomic.AddUint64(&linesReadKO, 1)
+				}
+				continue
+			}
+			if cConfig.Profiling {
+				atomic.AddUint64(&linesReadOK, 1)
+				globalParserHits.With(prometheus.Labels{"source": event.Line.Src}).Inc()
+
+			}
+			/* parse the log using magic */
+			parsed, error := parser.Parse(parserCTX, event, nodes)
+			if error != nil {
+				log.Errorf("failed parsing : %v\n", error)
+				return errors.New("parsing failed :/")
+			}
+			if parsed.Process == false {
+				if cConfig.Profiling {
+					globalParserHitsKo.With(prometheus.Labels{"source": event.Line.Src}).Inc()
+					atomic.AddUint64(&linesParsedKO, 1)
+				}
+				log.Debugf("Discarding line %+v", parsed)
+				discardCPT++
+				continue
+			}
+			if cConfig.Profiling {
+				globalParserHitsOk.With(prometheus.Labels{"source": event.Line.Src}).Inc()
+				atomic.AddUint64(&linesParsedOK, 1)
+			}
+			processCPT++
+			if parsed.Whitelisted == true {
+				log.Debugf("event whitelisted, discard")
+				continue
+			}
+			if processCPT%1000 == 0 {
+				log.Debugf("%d lines processed, %d lines discarded (unparsed)", processCPT, discardCPT)
+			}
+			output <- parsed
+			if cConfig.Profiling {
+				parseStat.AddTime(time.Since(start))
+			}
+		}
+	}
+	return nil
+}

cmd/crowdsec/pour.go

@@ -0,0 +1,65 @@
+package main
+
+import (
+	"fmt"
+	"sync/atomic"
+	"time"
+
+	leaky "github.com/crowdsecurity/crowdsec/pkg/leakybucket"
+	"github.com/crowdsecurity/crowdsec/pkg/types"
+	log "github.com/sirupsen/logrus"
+)
+
+func runPour(input chan types.Event, holders []leaky.BucketFactory, buckets *leaky.Buckets) error {
+	var (
+		start time.Time
+		count int
+	)
+LOOP:
+	for {
+		//bucket is now ready
+		select {
+		case <-bucketsTomb.Dying():
+			log.Infof("Exiting Bucketify")
+			break LOOP
+		case parsed := <-input:
+			count++
+			if cConfig.Profiling {
+				start = time.Now()
+			}
+
+			if count%5000 == 0 {
+				log.Warningf("%d existing LeakyRoutine", leaky.LeakyRoutineCount)
+				//when in forensics mode, garbage collect buckets
+				if parsed.MarshaledTime != "" && cConfig.SingleFile != "" {
+					var z *time.Time = &time.Time{}
+					if err := z.UnmarshalText([]byte(parsed.MarshaledTime)); err != nil {
+						log.Warningf("Failed to unmarshal time from event '%s' : %s", parsed.MarshaledTime, err)
+					} else {
+						log.Warningf("Starting buckets garbage collection ...")
+						leaky.GarbageCollectBuckets(*z, buckets)
+					}
+				}
+			}
+			//here we can bucketify with parsed
+			poured, err := leaky.PourItemToHolders(parsed, holders, buckets)
+			if err != nil {
+				log.Fatalf("bucketify failed for: %v", parsed)
+				return fmt.Errorf("process of event failed : %v", err)
+			}
+			if poured {
+				globalBucketPourOk.Inc()
+				atomic.AddUint64(&linesPouredOK, 1)
+			} else {
+				globalBucketPourKo.Inc()
+				atomic.AddUint64(&linesPouredKO, 1)
+			}
+			if cConfig.Profiling {
+				bucketStat.AddTime(time.Since(start))
+			}
+			lastProcessedItem.UnmarshalText([]byte(parsed.MarshaledTime))
+		}
+	}
+	log.Infof("Sending signal Bucketify")
+	return nil
+}

cmd/crowdsec/serve.go

@@ -0,0 +1,130 @@
+package main
+
+import (
+	"fmt"
+	"os"
+	"syscall"
+	"time"
+
+	"github.com/crowdsecurity/crowdsec/pkg/outputs"
+	log "github.com/sirupsen/logrus"
+
+	"github.com/sevlyar/go-daemon"
+)
+
+func reloadHandler(sig os.Signal) error {
+	dumpMetrics()
+	return nil
+}
+
+func termHandler(sig os.Signal) error {
+	log.Warningf("Shutting down routines")
+
+	acquisTomb.Kill(nil)
+	log.Infof("waiting for acquisition to finish")
+	if err := acquisTomb.Wait(); err != nil {
+		log.Warningf("Acquisition returned error : %s", err)
+	}
+	log.Infof("acquisition is finished, wait for parser/bucket/ouputs.")
+	//let's wait more than enough for in-flight events to be parsed.
+	time.Sleep(5 * time.Second)
+	parsersTomb.Kill(nil)
+	if err := parsersTomb.Wait(); err != nil {
+		log.Warningf("Parsers returned error : %s", err)
+	}
+	log.Infof("parsers is done")
+	bucketsTomb.Kill(nil)
+	if err := bucketsTomb.Wait(); err != nil {
+		log.Warningf("Buckets returned error : %s", err)
+	}
+	log.Infof("buckets is done")
+	outputsTomb.Kill(nil)
+	if err := outputsTomb.Wait(); err != nil {
+		log.Warningf("Ouputs returned error : %s", err)
+
+	}
+	log.Infof("ouputs is done")
+	dumpMetrics()
+	log.Warningf("all routines are done, bye.")
+	return daemon.ErrStop
+}
+
+func serveDaemon() error {
+	var daemonCTX *daemon.Context
+
+	daemon.SetSigHandler(termHandler, syscall.SIGTERM)
+	daemon.SetSigHandler(reloadHandler, syscall.SIGHUP)
+
+	daemonCTX = &daemon.Context{
+		PidFileName: cConfig.PIDFolder + "/crowdsec.pid",
+		PidFilePerm: 0644,
+		WorkDir:     "./",
+		Umask:       027,
+	}
+
+	d, err := daemonCTX.Reborn()
+	if err != nil {
+		return fmt.Errorf("unable to run daemon: %s ", err.Error())
+	}
+	if d != nil {
+		return nil
+	}
+	defer daemonCTX.Release()
+	err = daemon.ServeSignals()
+	if err != nil {
+		return fmt.Errorf("serveDaemon error : %s", err.Error())
+	}
+	return nil
+}
+
+func serveOneTimeRun(outputRunner outputs.Output) error {
+	log.Infof("waiting for acquisition to finish")
+
+	if err := acquisTomb.Wait(); err != nil {
+		log.Warningf("acquisition returned error : %s", err)
+	}
+	log.Infof("acquisition is finished, wait for parser/bucket/ouputs.")
+
+	//let's wait more than enough for in-flight events to be parsed.
+	time.Sleep(5 * time.Second)
+
+	// wait for the parser to parse all events
+	parsersTomb.Kill(nil)
+	if err := parsersTomb.Wait(); err != nil {
+		log.Warningf("parsers returned error : %s", err)
+	}
+	log.Infof("parsers is done")
+
+	// wait for the bucket to pour all events
+	bucketsTomb.Kill(nil)
+	if err := bucketsTomb.Wait(); err != nil {
+		log.Warningf("buckets returned error : %s", err)
+	}
+	log.Infof("buckets is done")
+
+	// wait for output to output all event
+	outputsTomb.Kill(nil)
+	if err := outputsTomb.Wait(); err != nil {
+		log.Warningf("ouputs returned error : %s", err)
+
+	}
+	log.Infof("ouputs is done")
+	dumpMetrics()
+	outputRunner.Flush()
+	log.Warningf("all routines are done, bye.")
+	return nil
+}
+
+func serve(outputRunner outputs.Output) error {
+	var err error
+	if cConfig.Daemonize == true {
+		if err = serveDaemon(); err != nil {
+			return fmt.Errorf(err.Error())
+		}
+	} else {
+		if err = serveOneTimeRun(outputRunner); err != nil {
+			return fmt.Errorf(err.Error())
+		}
+	}
+	return nil
+}

cmd/crowdsec/utils.go

@@ -0,0 +1,5 @@
+package main
+
+func bToMb(b uint64) uint64 {
+	return b / 1024 / 1024
+}

config/acquis.yaml

@@ -0,0 +1,52 @@
+filenames:
+  - /var/log/nginx/*.log
+  - ./tests/nginx/nginx.log
+#this is not a syslog log, indicate which kind of logs it is
+labels:
+  prog_name: nginx
+  type: nginx_raw_log
+---
+filenames:
+ - /var/log/auth.log
+ - /var/log/syslog
+#no need to set the prog_name, syslog format contains this info
+labels:
+  type: syslog
+---
+filename: /var/log/apache2/*.log
+labels:
+  prog_name: apache2
+  type: nginx_raw_log
+---
+filenames:
+  - ./tests/tcpdump.out
+  - /root/granola/tcpdump.out
+labels:
+  prog_name: tcpdump
+  type: tcpdump_raw_log
+---
+filename: ./tests/apache.log
+labels:
+  prog_name: apache2
+---
+filename: ./tests/nginx.log
+labels:
+  prog_name: nginx
+  
+
+
+  
+  # #list of files to be tailed
+  # #it's ok to add files that don't exist, they will juste be skipped :)
+  # - /var/log/nginx/*.log
+  # - /root/granola/tcpdump.out
+  # - /var/log/auth.log
+  # - tests/*.log
+  # - tests/tcpdump.out
+  # - tests/nginx/nginx.log
+  
+  # # for honeypots
+  # - /data/logs/*.log
+  # - /var/log/tcpdump.out
+  # - /var/log/auth.log
+  # - /var/log/syslog

config/api.yaml

@@ -0,0 +1,8 @@
+version: v1
+url: https://tmsov6x2n9.execute-api.eu-west-1.amazonaws.com
+signin_path: signin
+push_path: signals
+pull_path: pull
+enroll_path: enroll
+reset_pwd_path: resetpassword
+register_path: register

config/crowdsec.service

@@ -0,0 +1,13 @@
+[Unit]
+Description=Crowdwatch agent
+After=syslog.target network.target remote-fs.target nss-lookup.target
+
+[Service]
+Type=forking
+#PIDFile=${PID}/crowdsec.pid
+ExecStartPre=${BIN} -c ${CFG}/default.yaml -t
+ExecStart=${BIN} -c ${CFG}/default.yaml
+ExecStartPost=/bin/sleep 0.1
+
+[Install]
+WantedBy=multi-user.target

config/crowdsec_pull

@@ -0,0 +1,6 @@
+# /etc/cron.d/crowdsec_pull: crontab to pull crowdsec API 
+# bad IP in ban DB.
+
+# Run everyday at 08:00 A.M
+
+0 8 * * * root /usr/local/bin/cscli api pull >> /var/log/cscli.log 2>&1

config/dev.yaml

@@ -0,0 +1,12 @@
+working_dir: "."
+data_dir: "./data"
+config_dir: "./config"
+pid_dir: "./"
+log_dir: "./logs"
+log_mode: "stdout"
+log_level: info
+profiling: false
+sqlite_path: "./test.db"
+apimode: false
+plugin:
+  backend: "./config/plugins/backend"

config/patterns/aws

@@ -0,0 +1,11 @@
+S3_REQUEST_LINE (?:%{WORD:verb} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})
+
+S3_ACCESS_LOG %{WORD:owner} %{NOTSPACE:bucket} \[%{HTTPDATE:timestamp}\] %{IP:clientip} %{NOTSPACE:requester} %{NOTSPACE:request_id} %{NOTSPACE:operation} %{NOTSPACE:key} (?:"%{S3_REQUEST_LINE}"|-) (?:%{INT:response:int}|-) (?:-|%{NOTSPACE:error_code}) (?:%{INT:bytes:int}|-) (?:%{INT:object_size:int}|-) (?:%{INT:request_time_ms:int}|-) (?:%{INT:turnaround_time_ms:int}|-) (?:%{QS:referrer}|-) (?:"?%{QS:agent}"?|-) (?:-|%{NOTSPACE:version_id})
+
+ELB_URIPATHPARAM %{URIPATH:path}(?:%{URIPARAM:params})?
+
+ELB_URI %{URIPROTO:proto}://(?:%{USER}(?::[^@]*)?@)?(?:%{URIHOST:urihost})?(?:%{ELB_URIPATHPARAM})?
+
+ELB_REQUEST_LINE (?:%{WORD:verb} %{ELB_URI:request}(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})
+
+ELB_ACCESS_LOG %{TIMESTAMP_ISO8601:timestamp} %{NOTSPACE:elb} %{IP:clientip}:%{INT:clientport:int} (?:(%{IP:backendip}:?:%{INT:backendport:int})|-) %{NUMBER:request_processing_time:float} %{NUMBER:backend_processing_time:float} %{NUMBER:response_processing_time:float} %{INT:response:int} %{INT:backend_response:int} %{INT:received_bytes:int} %{INT:bytes:int} "%{ELB_REQUEST_LINE}"

config/patterns/bacula

@@ -0,0 +1,50 @@
+BACULA_TIMESTAMP %{MONTHDAY}-%{MONTH} %{HOUR}:%{MINUTE}
+BACULA_HOST [a-zA-Z0-9-]+
+BACULA_VOLUME %{USER}
+BACULA_DEVICE %{USER}
+BACULA_DEVICEPATH %{UNIXPATH}
+BACULA_CAPACITY %{INT}{1,3}(,%{INT}{3})*
+BACULA_VERSION %{USER}
+BACULA_JOB %{USER}
+
+BACULA_LOG_MAX_CAPACITY User defined maximum volume capacity %{BACULA_CAPACITY} exceeded on device \"%{BACULA_DEVICE:device}\" \(%{BACULA_DEVICEPATH}\)
+BACULA_LOG_END_VOLUME End of medium on Volume \"%{BACULA_VOLUME:volume}\" Bytes=%{BACULA_CAPACITY} Blocks=%{BACULA_CAPACITY} at %{MONTHDAY}-%{MONTH}-%{YEAR} %{HOUR}:%{MINUTE}.
+BACULA_LOG_NEW_VOLUME Created new Volume \"%{BACULA_VOLUME:volume}\" in catalog.
+BACULA_LOG_NEW_LABEL Labeled new Volume \"%{BACULA_VOLUME:volume}\" on device \"%{BACULA_DEVICE:device}\" \(%{BACULA_DEVICEPATH}\).
+BACULA_LOG_WROTE_LABEL Wrote label to prelabeled Volume \"%{BACULA_VOLUME:volume}\" on device \"%{BACULA_DEVICE}\" \(%{BACULA_DEVICEPATH}\)
+BACULA_LOG_NEW_MOUNT New volume \"%{BACULA_VOLUME:volume}\" mounted on device \"%{BACULA_DEVICE:device}\" \(%{BACULA_DEVICEPATH}\) at %{MONTHDAY}-%{MONTH}-%{YEAR} %{HOUR}:%{MINUTE}.
+BACULA_LOG_NOOPEN \s+Cannot open %{DATA}: ERR=%{GREEDYDATA:berror}
+BACULA_LOG_NOOPENDIR \s+Could not open directory %{DATA}: ERR=%{GREEDYDATA:berror}
+BACULA_LOG_NOSTAT \s+Could not stat %{DATA}: ERR=%{GREEDYDATA:berror}
+BACULA_LOG_NOJOBS There are no more Jobs associated with Volume \"%{BACULA_VOLUME:volume}\". Marking it purged.
+BACULA_LOG_ALL_RECORDS_PRUNED All records pruned from Volume \"%{BACULA_VOLUME:volume}\"; marking it \"Purged\"
+BACULA_LOG_BEGIN_PRUNE_JOBS Begin pruning Jobs older than %{INT} month %{INT} days .
+BACULA_LOG_BEGIN_PRUNE_FILES Begin pruning Files.
+BACULA_LOG_PRUNED_JOBS Pruned %{INT} Jobs* for client %{BACULA_HOST:client} from catalog.
+BACULA_LOG_PRUNED_FILES Pruned Files from %{INT} Jobs* for client %{BACULA_HOST:client} from catalog.
+BACULA_LOG_ENDPRUNE End auto prune.
+BACULA_LOG_STARTJOB Start Backup JobId %{INT}, Job=%{BACULA_JOB:job}
+BACULA_LOG_STARTRESTORE Start Restore Job %{BACULA_JOB:job}
+BACULA_LOG_USEDEVICE Using Device \"%{BACULA_DEVICE:device}\"
+BACULA_LOG_DIFF_FS \s+%{UNIXPATH} is a different filesystem. Will not descend from %{UNIXPATH} into it.
+BACULA_LOG_JOBEND Job write elapsed time = %{DATA:elapsed}, Transfer rate = %{NUMBER} (K|M|G)? Bytes/second
+BACULA_LOG_NOPRUNE_JOBS No Jobs found to prune.
+BACULA_LOG_NOPRUNE_FILES No Files found to prune.
+BACULA_LOG_VOLUME_PREVWRITTEN Volume \"%{BACULA_VOLUME:volume}\" previously written, moving to end of data.
+BACULA_LOG_READYAPPEND Ready to append to end of Volume \"%{BACULA_VOLUME:volume}\" size=%{INT}
+BACULA_LOG_CANCELLING Cancelling duplicate JobId=%{INT}.
+BACULA_LOG_MARKCANCEL JobId %{INT}, Job %{BACULA_JOB:job} marked to be canceled.
+BACULA_LOG_CLIENT_RBJ shell command: run ClientRunBeforeJob \"%{GREEDYDATA:runjob}\"
+BACULA_LOG_VSS (Generate )?VSS (Writer)?
+BACULA_LOG_MAXSTART Fatal error: Job canceled because max start delay time exceeded.
+BACULA_LOG_DUPLICATE Fatal error: JobId %{INT:duplicate} already running. Duplicate job not allowed.
+BACULA_LOG_NOJOBSTAT Fatal error: No Job status returned from FD.
+BACULA_LOG_FATAL_CONN Fatal error: bsock.c:133 Unable to connect to (Client: %{BACULA_HOST:client}|Storage daemon) on %{HOSTNAME}:%{POSINT}. ERR=%{GREEDYDATA:berror}
+BACULA_LOG_NO_CONNECT Warning: bsock.c:127 Could not connect to (Client: %{BACULA_HOST:client}|Storage daemon) on %{HOSTNAME}:%{POSINT}. ERR=%{GREEDYDATA:berror}
+BACULA_LOG_NO_AUTH Fatal error: Unable to authenticate with File daemon at %{HOSTNAME}. Possible causes:
+BACULA_LOG_NOSUIT No prior or suitable Full backup found in catalog. Doing FULL backup.
+BACULA_LOG_NOPRIOR No prior Full backup Job record found.
+
+BACULA_LOG_JOB (Error: )?Bacula %{BACULA_HOST} %{BACULA_VERSION} \(%{BACULA_VERSION}\):
+
+BACULA_LOGLINE %{BACULA_TIMESTAMP:bts} %{BACULA_HOST:hostname} JobId %{INT:jobid}: (%{BACULA_LOG_MAX_CAPACITY}|%{BACULA_LOG_END_VOLUME}|%{BACULA_LOG_NEW_VOLUME}|%{BACULA_LOG_NEW_LABEL}|%{BACULA_LOG_WROTE_LABEL}|%{BACULA_LOG_NEW_MOUNT}|%{BACULA_LOG_NOOPEN}|%{BACULA_LOG_NOOPENDIR}|%{BACULA_LOG_NOSTAT}|%{BACULA_LOG_NOJOBS}|%{BACULA_LOG_ALL_RECORDS_PRUNED}|%{BACULA_LOG_BEGIN_PRUNE_JOBS}|%{BACULA_LOG_BEGIN_PRUNE_FILES}|%{BACULA_LOG_PRUNED_JOBS}|%{BACULA_LOG_PRUNED_FILES}|%{BACULA_LOG_ENDPRUNE}|%{BACULA_LOG_STARTJOB}|%{BACULA_LOG_STARTRESTORE}|%{BACULA_LOG_USEDEVICE}|%{BACULA_LOG_DIFF_FS}|%{BACULA_LOG_JOBEND}|%{BACULA_LOG_NOPRUNE_JOBS}|%{BACULA_LOG_NOPRUNE_FILES}|%{BACULA_LOG_VOLUME_PREVWRITTEN}|%{BACULA_LOG_READYAPPEND}|%{BACULA_LOG_CANCELLING}|%{BACULA_LOG_MARKCANCEL}|%{BACULA_LOG_CLIENT_RBJ}|%{BACULA_LOG_VSS}|%{BACULA_LOG_MAXSTART}|%{BACULA_LOG_DUPLICATE}|%{BACULA_LOG_NOJOBSTAT}|%{BACULA_LOG_FATAL_CONN}|%{BACULA_LOG_NO_CONNECT}|%{BACULA_LOG_NO_AUTH}|%{BACULA_LOG_NOSUIT}|%{BACULA_LOG_JOB}|%{BACULA_LOG_NOPRIOR})

config/patterns/bro

@@ -0,0 +1,13 @@
+# https://www.bro.org/sphinx/script-reference/log-files.html
+
+# http.log
+BRO_HTTP %{NUMBER:ts}\t%{NOTSPACE:uid}\t%{IP:orig_h}\t%{INT:orig_p}\t%{IP:resp_h}\t%{INT:resp_p}\t%{INT:trans_depth}\t%{GREEDYDATA:method}\t%{GREEDYDATA:domain}\t%{GREEDYDATA:uri}\t%{GREEDYDATA:referrer}\t%{GREEDYDATA:user_agent}\t%{NUMBER:request_body_len}\t%{NUMBER:response_body_len}\t%{GREEDYDATA:status_code}\t%{GREEDYDATA:status_msg}\t%{GREEDYDATA:info_code}\t%{GREEDYDATA:info_msg}\t%{GREEDYDATA:filename}\t%{GREEDYDATA:bro_tags}\t%{GREEDYDATA:username}\t%{GREEDYDATA:password}\t%{GREEDYDATA:proxied}\t%{GREEDYDATA:orig_fuids}\t%{GREEDYDATA:orig_mime_types}\t%{GREEDYDATA:resp_fuids}\t%{GREEDYDATA:resp_mime_types}
+
+# dns.log
+BRO_DNS %{NUMBER:ts}\t%{NOTSPACE:uid}\t%{IP:orig_h}\t%{INT:orig_p}\t%{IP:resp_h}\t%{INT:resp_p}\t%{WORD:proto}\t%{INT:trans_id}\t%{GREEDYDATA:query}\t%{GREEDYDATA:qclass}\t%{GREEDYDATA:qclass_name}\t%{GREEDYDATA:qtype}\t%{GREEDYDATA:qtype_name}\t%{GREEDYDATA:rcode}\t%{GREEDYDATA:rcode_name}\t%{GREEDYDATA:AA}\t%{GREEDYDATA:TC}\t%{GREEDYDATA:RD}\t%{GREEDYDATA:RA}\t%{GREEDYDATA:Z}\t%{GREEDYDATA:answers}\t%{GREEDYDATA:TTLs}\t%{GREEDYDATA:rejected}
+
+# conn.log
+BRO_CONN %{NUMBER:ts}\t%{NOTSPACE:uid}\t%{IP:orig_h}\t%{INT:orig_p}\t%{IP:resp_h}\t%{INT:resp_p}\t%{WORD:proto}\t%{GREEDYDATA:service}\t%{NUMBER:duration}\t%{NUMBER:orig_bytes}\t%{NUMBER:resp_bytes}\t%{GREEDYDATA:conn_state}\t%{GREEDYDATA:local_orig}\t%{GREEDYDATA:missed_bytes}\t%{GREEDYDATA:history}\t%{GREEDYDATA:orig_pkts}\t%{GREEDYDATA:orig_ip_bytes}\t%{GREEDYDATA:resp_pkts}\t%{GREEDYDATA:resp_ip_bytes}\t%{GREEDYDATA:tunnel_parents}
+
+# files.log
+BRO_FILES %{NUMBER:ts}\t%{NOTSPACE:fuid}\t%{IP:tx_hosts}\t%{IP:rx_hosts}\t%{NOTSPACE:conn_uids}\t%{GREEDYDATA:source}\t%{GREEDYDATA:depth}\t%{GREEDYDATA:analyzers}\t%{GREEDYDATA:mime_type}\t%{GREEDYDATA:filename}\t%{GREEDYDATA:duration}\t%{GREEDYDATA:local_orig}\t%{GREEDYDATA:is_orig}\t%{GREEDYDATA:seen_bytes}\t%{GREEDYDATA:total_bytes}\t%{GREEDYDATA:missing_bytes}\t%{GREEDYDATA:overflow_bytes}\t%{GREEDYDATA:timedout}\t%{GREEDYDATA:parent_fuid}\t%{GREEDYDATA:md5}\t%{GREEDYDATA:sha1}\t%{GREEDYDATA:sha256}\t%{GREEDYDATA:extracted}

config/patterns/cowrie_honeypot

@@ -0,0 +1 @@
+COWRIE_NEW_CO New connection: %{IPV4:source_ip}:[0-9]+ \(%{IPV4:dest_ip}:%{INT:dest_port}\) \[session: %{DATA:telnet_session}\]$

config/patterns/exim

@@ -0,0 +1,12 @@
+EXIM_MSGID [0-9A-Za-z]{6}-[0-9A-Za-z]{6}-[0-9A-Za-z]{2}
+EXIM_FLAGS (<=|[-=>*]>|[*]{2}|==)
+EXIM_DATE %{YEAR:exim_year}-%{MONTHNUM:exim_month}-%{MONTHDAY:exim_day} %{TIME:exim_time}
+EXIM_PID \[%{POSINT}\]
+EXIM_QT ((\d+y)?(\d+w)?(\d+d)?(\d+h)?(\d+m)?(\d+s)?)
+EXIM_EXCLUDE_TERMS (Message is frozen|(Start|End) queue run| Warning: | retry time not reached | no (IP address|host name) found for (IP address|host) | unexpected disconnection while reading SMTP command | no immediate delivery: |another process is handling this message)
+EXIM_REMOTE_HOST (H=(%{NOTSPACE:remote_hostname} )?(\(%{NOTSPACE:remote_heloname}\) )?\[%{IP:remote_host}\])
+EXIM_INTERFACE (I=\[%{IP:exim_interface}\](:%{NUMBER:exim_interface_port}))
+EXIM_PROTOCOL (P=%{NOTSPACE:protocol})
+EXIM_MSG_SIZE (S=%{NUMBER:exim_msg_size})
+EXIM_HEADER_ID (id=%{NOTSPACE:exim_header_id})
+EXIM_SUBJECT (T=%{QS:exim_subject})

config/patterns/firewalls

@@ -0,0 +1,86 @@
+# NetScreen firewall logs
+NETSCREENSESSIONLOG %{SYSLOGTIMESTAMP:date} %{IPORHOST:device} %{IPORHOST}: NetScreen device_id=%{WORD:device_id}%{DATA}: start_time=%{QUOTEDSTRING:start_time} duration=%{INT:duration} policy_id=%{INT:policy_id} service=%{DATA:service} proto=%{INT:proto} src zone=%{WORD:src_zone} dst zone=%{WORD:dst_zone} action=%{WORD:action} sent=%{INT:sent} rcvd=%{INT:rcvd} src=%{IPORHOST:src_ip} dst=%{IPORHOST:dst_ip} src_port=%{INT:src_port} dst_port=%{INT:dst_port} src-xlated ip=%{IPORHOST:src_xlated_ip} port=%{INT:src_xlated_port} dst-xlated ip=%{IPORHOST:dst_xlated_ip} port=%{INT:dst_xlated_port} session_id=%{INT:session_id} reason=%{GREEDYDATA:reason}
+
+#== Cisco ASA ==
+CISCOTAG [A-Z0-9]+-%{INT}-(?:[A-Z0-9_]+)
+CISCOTIMESTAMP %{MONTH} +%{MONTHDAY}(?: %{YEAR})? %{TIME}
+CISCO_TAGGED_SYSLOG ^<%{POSINT:syslog_pri}>%{CISCOTIMESTAMP:timestamp}( %{SYSLOGHOST:sysloghost})? ?: %%{CISCOTAG:ciscotag}:
+# Common Particles
+CISCO_ACTION Built|Teardown|Deny|Denied|denied|requested|permitted|denied by ACL|discarded|est-allowed|Dropping|created|deleted
+CISCO_REASON Duplicate TCP SYN|Failed to locate egress interface|Invalid transport field|No matching connection|DNS Response|DNS Query|(?:%{WORD}\s*)*
+CISCO_DIRECTION Inbound|inbound|Outbound|outbound
+CISCO_INTERVAL first hit|%{INT}-second interval
+CISCO_XLATE_TYPE static|dynamic
+# ASA-1-104001
+CISCOFW104001 \((?:Primary|Secondary)\) Switching to ACTIVE - %{GREEDYDATA:switch_reason}
+# ASA-1-104002
+CISCOFW104002 \((?:Primary|Secondary)\) Switching to STANDBY - %{GREEDYDATA:switch_reason}
+# ASA-1-104003
+CISCOFW104003 \((?:Primary|Secondary)\) Switching to FAILED\.
+# ASA-1-104004
+CISCOFW104004 \((?:Primary|Secondary)\) Switching to OK\.
+# ASA-1-105003
+CISCOFW105003 \((?:Primary|Secondary)\) Monitoring on [Ii]nterface %{GREEDYDATA:interface_name} waiting
+# ASA-1-105004
+CISCOFW105004 \((?:Primary|Secondary)\) Monitoring on [Ii]nterface %{GREEDYDATA:interface_name} normal
+# ASA-1-105005
+CISCOFW105005 \((?:Primary|Secondary)\) Lost Failover communications with mate on [Ii]nterface %{GREEDYDATA:interface_name}
+# ASA-1-105008
+CISCOFW105008 \((?:Primary|Secondary)\) Testing [Ii]nterface %{GREEDYDATA:interface_name}
+# ASA-1-105009
+CISCOFW105009 \((?:Primary|Secondary)\) Testing on [Ii]nterface %{GREEDYDATA:interface_name} (?:Passed|Failed)
+# ASA-2-106001
+CISCOFW106001 %{CISCO_DIRECTION:direction} %{WORD:protocol} connection %{CISCO_ACTION:action} from %{IP:src_ip}/%{INT:src_port} to %{IP:dst_ip}/%{INT:dst_port} flags %{GREEDYDATA:tcp_flags} on interface %{GREEDYDATA:interface}
+# ASA-2-106006, ASA-2-106007, ASA-2-106010
+CISCOFW106006_106007_106010 %{CISCO_ACTION:action} %{CISCO_DIRECTION:direction} %{WORD:protocol} (?:from|src) %{IP:src_ip}/%{INT:src_port}(\(%{DATA:src_fwuser}\))? (?:to|dst) %{IP:dst_ip}/%{INT:dst_port}(\(%{DATA:dst_fwuser}\))? (?:on interface %{DATA:interface}|due to %{CISCO_REASON:reason})
+# ASA-3-106014
+CISCOFW106014 %{CISCO_ACTION:action} %{CISCO_DIRECTION:direction} %{WORD:protocol} src %{DATA:src_interface}:%{IP:src_ip}(\(%{DATA:src_fwuser}\))? dst %{DATA:dst_interface}:%{IP:dst_ip}(\(%{DATA:dst_fwuser}\))? \(type %{INT:icmp_type}, code %{INT:icmp_code}\)
+# ASA-6-106015
+CISCOFW106015 %{CISCO_ACTION:action} %{WORD:protocol} \(%{DATA:policy_id}\) from %{IP:src_ip}/%{INT:src_port} to %{IP:dst_ip}/%{INT:dst_port} flags %{DATA:tcp_flags}  on interface %{GREEDYDATA:interface}
+# ASA-1-106021
+CISCOFW106021 %{CISCO_ACTION:action} %{WORD:protocol} reverse path check from %{IP:src_ip} to %{IP:dst_ip} on interface %{GREEDYDATA:interface}
+# ASA-4-106023
+CISCOFW106023 %{CISCO_ACTION:action}( protocol)? %{WORD:protocol} src %{DATA:src_interface}:%{DATA:src_ip}(/%{INT:src_port})?(\(%{DATA:src_fwuser}\))? dst %{DATA:dst_interface}:%{DATA:dst_ip}(/%{INT:dst_port})?(\(%{DATA:dst_fwuser}\))?( \(type %{INT:icmp_type}, code %{INT:icmp_code}\))? by access-group "?%{DATA:policy_id}"? \[%{DATA:hashcode1}, %{DATA:hashcode2}\]
+# ASA-4-106100, ASA-4-106102, ASA-4-106103
+CISCOFW106100_2_3 access-list %{NOTSPACE:policy_id} %{CISCO_ACTION:action} %{WORD:protocol} for user '%{DATA:src_fwuser}' %{DATA:src_interface}/%{IP:src_ip}\(%{INT:src_port}\) -> %{DATA:dst_interface}/%{IP:dst_ip}\(%{INT:dst_port}\) hit-cnt %{INT:hit_count} %{CISCO_INTERVAL:interval} \[%{DATA:hashcode1}, %{DATA:hashcode2}\]
+# ASA-5-106100
+CISCOFW106100 access-list %{NOTSPACE:policy_id} %{CISCO_ACTION:action} %{WORD:protocol} %{DATA:src_interface}/%{IP:src_ip}\(%{INT:src_port}\)(\(%{DATA:src_fwuser}\))? -> %{DATA:dst_interface}/%{IP:dst_ip}\(%{INT:dst_port}\)(\(%{DATA:src_fwuser}\))? hit-cnt %{INT:hit_count} %{CISCO_INTERVAL:interval} \[%{DATA:hashcode1}, %{DATA:hashcode2}\]
+# ASA-6-110002
+CISCOFW110002 %{CISCO_REASON:reason} for %{WORD:protocol} from %{DATA:src_interface}:%{IP:src_ip}/%{INT:src_port} to %{IP:dst_ip}/%{INT:dst_port}
+# ASA-6-302010
+CISCOFW302010 %{INT:connection_count} in use, %{INT:connection_count_max} most used
+# ASA-6-302013, ASA-6-302014, ASA-6-302015, ASA-6-302016
+CISCOFW302013_302014_302015_302016 %{CISCO_ACTION:action}(?: %{CISCO_DIRECTION:direction})? %{WORD:protocol} connection %{INT:connection_id} for %{DATA:src_interface}:%{IP:src_ip}/%{INT:src_port}( \(%{IP:src_mapped_ip}/%{INT:src_mapped_port}\))?(\(%{DATA:src_fwuser}\))? to %{DATA:dst_interface}:%{IP:dst_ip}/%{INT:dst_port}( \(%{IP:dst_mapped_ip}/%{INT:dst_mapped_port}\))?(\(%{DATA:dst_fwuser}\))?( duration %{TIME:duration} bytes %{INT:bytes})?(?: %{CISCO_REASON:reason})?( \(%{DATA:user}\))?
+# ASA-6-302020, ASA-6-302021
+CISCOFW302020_302021 %{CISCO_ACTION:action}(?: %{CISCO_DIRECTION:direction})? %{WORD:protocol} connection for faddr %{IP:dst_ip}/%{INT:icmp_seq_num}(?:\(%{DATA:fwuser}\))? gaddr %{IP:src_xlated_ip}/%{INT:icmp_code_xlated} laddr %{IP:src_ip}/%{INT:icmp_code}( \(%{DATA:user}\))?
+# ASA-6-305011
+CISCOFW305011 %{CISCO_ACTION:action} %{CISCO_XLATE_TYPE:xlate_type} %{WORD:protocol} translation from %{DATA:src_interface}:%{IP:src_ip}(/%{INT:src_port})?(\(%{DATA:src_fwuser}\))? to %{DATA:src_xlated_interface}:%{IP:src_xlated_ip}/%{DATA:src_xlated_port}
+# ASA-3-313001, ASA-3-313004, ASA-3-313008
+CISCOFW313001_313004_313008 %{CISCO_ACTION:action} %{WORD:protocol} type=%{INT:icmp_type}, code=%{INT:icmp_code} from %{IP:src_ip} on interface %{DATA:interface}( to %{IP:dst_ip})?
+# ASA-4-313005
+CISCOFW313005 %{CISCO_REASON:reason} for %{WORD:protocol} error message: %{WORD:err_protocol} src %{DATA:err_src_interface}:%{IP:err_src_ip}(\(%{DATA:err_src_fwuser}\))? dst %{DATA:err_dst_interface}:%{IP:err_dst_ip}(\(%{DATA:err_dst_fwuser}\))? \(type %{INT:err_icmp_type}, code %{INT:err_icmp_code}\) on %{DATA:interface} interface\.  Original IP payload: %{WORD:protocol} src %{IP:orig_src_ip}/%{INT:orig_src_port}(\(%{DATA:orig_src_fwuser}\))? dst %{IP:orig_dst_ip}/%{INT:orig_dst_port}(\(%{DATA:orig_dst_fwuser}\))?
+# ASA-5-321001
+CISCOFW321001 Resource '%{WORD:resource_name}' limit of %{POSINT:resource_limit} reached for system
+# ASA-4-402117
+CISCOFW402117 %{WORD:protocol}: Received a non-IPSec packet \(protocol= %{WORD:orig_protocol}\) from %{IP:src_ip} to %{IP:dst_ip}
+# ASA-4-402119
+CISCOFW402119 %{WORD:protocol}: Received an %{WORD:orig_protocol} packet \(SPI= %{DATA:spi}, sequence number= %{DATA:seq_num}\) from %{IP:src_ip} \(user= %{DATA:user}\) to %{IP:dst_ip} that failed anti-replay checking
+# ASA-4-419001
+CISCOFW419001 %{CISCO_ACTION:action} %{WORD:protocol} packet from %{DATA:src_interface}:%{IP:src_ip}/%{INT:src_port} to %{DATA:dst_interface}:%{IP:dst_ip}/%{INT:dst_port}, reason: %{GREEDYDATA:reason}
+# ASA-4-419002
+CISCOFW419002 %{CISCO_REASON:reason} from %{DATA:src_interface}:%{IP:src_ip}/%{INT:src_port} to %{DATA:dst_interface}:%{IP:dst_ip}/%{INT:dst_port} with different initial sequence number
+# ASA-4-500004
+CISCOFW500004 %{CISCO_REASON:reason} for protocol=%{WORD:protocol}, from %{IP:src_ip}/%{INT:src_port} to %{IP:dst_ip}/%{INT:dst_port}
+# ASA-6-602303, ASA-6-602304
+CISCOFW602303_602304 %{WORD:protocol}: An %{CISCO_DIRECTION:direction} %{GREEDYDATA:tunnel_type} SA \(SPI= %{DATA:spi}\) between %{IP:src_ip} and %{IP:dst_ip} \(user= %{DATA:user}\) has been %{CISCO_ACTION:action}
+# ASA-7-710001, ASA-7-710002, ASA-7-710003, ASA-7-710005, ASA-7-710006
+CISCOFW710001_710002_710003_710005_710006 %{WORD:protocol} (?:request|access) %{CISCO_ACTION:action} from %{IP:src_ip}/%{INT:src_port} to %{DATA:dst_interface}:%{IP:dst_ip}/%{INT:dst_port}
+# ASA-6-713172
+CISCOFW713172 Group = %{GREEDYDATA:group}, IP = %{IP:src_ip}, Automatic NAT Detection Status:\s+Remote end\s*%{DATA:is_remote_natted}\s*behind a NAT device\s+This\s+end\s*%{DATA:is_local_natted}\s*behind a NAT device
+# ASA-4-733100
+CISCOFW733100 \[\s*%{DATA:drop_type}\s*\] drop %{DATA:drop_rate_id} exceeded. Current burst rate is %{INT:drop_rate_current_burst} per second, max configured rate is %{INT:drop_rate_max_burst}; Current average rate is %{INT:drop_rate_current_avg} per second, max configured rate is %{INT:drop_rate_max_avg}; Cumulative total count is %{INT:drop_total_count}
+#== End Cisco ASA ==
+
+# Shorewall firewall logs
+SHOREWALL (%{SYSLOGTIMESTAMP:timestamp}) (%{WORD:nf_host}) kernel:.*Shorewall:(%{WORD:nf_action1})?:(%{WORD:nf_action2})?.*IN=(%{USERNAME:nf_in_interface})?.*(OUT= *MAC=(%{COMMONMAC:nf_dst_mac}):(%{COMMONMAC:nf_src_mac})?|OUT=%{USERNAME:nf_out_interface}).*SRC=(%{IPV4:nf_src_ip}).*DST=(%{IPV4:nf_dst_ip}).*LEN=(%{WORD:nf_len}).*?TOS=(%{WORD:nf_tos}).*?PREC=(%{WORD:nf_prec}).*?TTL=(%{INT:nf_ttl}).*?ID=(%{INT:nf_id}).*?PROTO=(%{WORD:nf_protocol}).*?SPT=(%{INT:nf_src_port}?.*DPT=%{INT:nf_dst_port}?.*)
+#== End Shorewall

config/patterns/haproxy

@@ -0,0 +1,39 @@
+## These patterns were tested w/ haproxy-1.4.15
+
+## Documentation of the haproxy log formats can be found at the following links:
+## http://code.google.com/p/haproxy-docs/wiki/HTTPLogFormat
+## http://code.google.com/p/haproxy-docs/wiki/TCPLogFormat
+
+HAPROXYTIME %{HOUR:haproxy_hour}:%{MINUTE:haproxy_minute}(?::%{SECOND:haproxy_second})
+HAPROXYDATE %{MONTHDAY:haproxy_monthday}/%{MONTH:haproxy_month}/%{YEAR:haproxy_year}:%{HAPROXYTIME:haproxy_time}.%{INT:haproxy_milliseconds}
+
+# Override these default patterns to parse out what is captured in your haproxy.cfg
+HAPROXYCAPTUREDREQUESTHEADERS %{DATA:captured_request_headers}
+HAPROXYCAPTUREDRESPONSEHEADERS %{DATA:captured_response_headers}
+
+# Example:
+#  These haproxy config lines will add data to the logs that are captured
+#  by the patterns below. Place them in your custom patterns directory to
+#  override the defaults.
+#
+#  capture request header Host len 40
+#  capture request header X-Forwarded-For len 50
+#  capture request header Accept-Language len 50
+#  capture request header Referer len 200
+#  capture request header User-Agent len 200
+#
+#  capture response header Content-Type len 30
+#  capture response header Content-Encoding len 10
+#  capture response header Cache-Control len 200
+#  capture response header Last-Modified len 200
+#
+# HAPROXYCAPTUREDREQUESTHEADERS %{DATA:request_header_host}\|%{DATA:request_header_x_forwarded_for}\|%{DATA:request_header_accept_language}\|%{DATA:request_header_referer}\|%{DATA:request_header_user_agent}
+# HAPROXYCAPTUREDRESPONSEHEADERS %{DATA:response_header_content_type}\|%{DATA:response_header_content_encoding}\|%{DATA:response_header_cache_control}\|%{DATA:response_header_last_modified}
+
+# parse a haproxy 'httplog' line
+HAPROXYHTTPBASE %{IP:client_ip}:%{INT:client_port} \[%{HAPROXYDATE:accept_date}\] %{NOTSPACE:frontend_name} %{NOTSPACE:backend_name}/%{NOTSPACE:server_name} %{INT:time_request}/%{INT:time_queue}/%{INT:time_backend_connect}/%{INT:time_backend_response}/%{NOTSPACE:time_duration} %{INT:http_status_code} %{NOTSPACE:bytes_read} %{DATA:captured_request_cookie} %{DATA:captured_response_cookie} %{NOTSPACE:termination_state} %{INT:actconn}/%{INT:feconn}/%{INT:beconn}/%{INT:srvconn}/%{NOTSPACE:retries} %{INT:srv_queue}/%{INT:backend_queue} (\{%{HAPROXYCAPTUREDREQUESTHEADERS}\})?( )?(\{%{HAPROXYCAPTUREDRESPONSEHEADERS}\})?( )?"(<BADREQ>|(%{WORD:http_verb} (%{URIPROTO:http_proto}://)?(?:%{USER:http_user}(?::[^@]*)?@)?(?:%{URIHOST:http_host})?(?:%{URIPATHPARAM:http_request})?( HTTP/%{NUMBER:http_version})?))?"
+
+HAPROXYHTTP (?:%{SYSLOGTIMESTAMP:syslog_timestamp}|%{TIMESTAMP_ISO8601:timestamp8601}) %{IPORHOST:syslog_server} %{SYSLOGPROG}: %{HAPROXYHTTPBASE}
+
+# parse a haproxy 'tcplog' line
+HAPROXYTCP (?:%{SYSLOGTIMESTAMP:syslog_timestamp}|%{TIMESTAMP_ISO8601:timestamp8601}) %{IPORHOST:syslog_server} %{SYSLOGPROG}: %{IP:client_ip}:%{INT:client_port} \[%{HAPROXYDATE:accept_date}\] %{NOTSPACE:frontend_name} %{NOTSPACE:backend_name}/%{NOTSPACE:server_name} %{INT:time_queue}/%{INT:time_backend_connect}/%{NOTSPACE:time_duration} %{NOTSPACE:bytes_read} %{NOTSPACE:termination_state} %{INT:actconn}/%{INT:feconn}/%{INT:beconn}/%{INT:srvconn}/%{NOTSPACE:retries} %{INT:srv_queue}/%{INT:backend_queue}

config/patterns/java

@@ -0,0 +1,20 @@
+JAVACLASS (?:[a-zA-Z$_][a-zA-Z$_0-9]*\.)*[a-zA-Z$_][a-zA-Z$_0-9]*
+#Space is an allowed character to match special cases like 'Native Method' or 'Unknown Source'
+JAVAFILE (?:[A-Za-z0-9_. -]+)
+#Allow special <init> method
+JAVAMETHOD (?:(<init>)|[a-zA-Z$_][a-zA-Z$_0-9]*)
+#Line number is optional in special cases 'Native method' or 'Unknown source'
+JAVASTACKTRACEPART %{SPACE}at %{JAVACLASS:class}\.%{JAVAMETHOD:method}\(%{JAVAFILE:file}(?::%{NUMBER:line})?\)
+# Java Logs
+JAVATHREAD (?:[A-Z]{2}-Processor[\d]+)
+##JAVACLASS (?:[a-zA-Z0-9-]+\.)+[A-Za-z0-9$]+
+##JAVAFILE (?:[A-Za-z0-9_.-]+)
+##JAVASTACKTRACEPART at %{JAVACLASS:class}\.%{WORD:method}\(%{JAVAFILE:file}:%{NUMBER:line}\)
+JAVALOGMESSAGE (.*)
+# MMM dd, yyyy HH:mm:ss eg: Jan 9, 2014 7:13:13 AM
+CATALINA_DATESTAMP %{MONTH} %{MONTHDAY}, 20%{YEAR} %{HOUR}:?%{MINUTE}(?::?%{SECOND}) (?:AM|PM)
+# yyyy-MM-dd HH:mm:ss,SSS ZZZ eg: 2014-01-09 17:32:25,527 -0800
+TOMCAT_DATESTAMP 20%{YEAR}-%{MONTHNUM}-%{MONTHDAY} %{HOUR}:?%{MINUTE}(?::?%{SECOND}) %{ISO8601_TIMEZONE}
+CATALINALOG %{CATALINA_DATESTAMP:timestamp} %{JAVACLASS:class} %{JAVALOGMESSAGE:logmessage}
+# 2014-01-09 20:03:28,269 -0800 | ERROR | com.example.service.ExampleService - something compeletely unexpected happened...
+TOMCATLOG %{TOMCAT_DATESTAMP:timestamp} \| %{LOGLEVEL:level} \| %{JAVACLASS:class} - %{JAVALOGMESSAGE:logmessage}

config/patterns/junos

@@ -0,0 +1,8 @@
+# JUNOS 11.4 RT_FLOW patterns
+RT_FLOW_EVENT (RT_FLOW_SESSION_CREATE|RT_FLOW_SESSION_CLOSE|RT_FLOW_SESSION_DENY)
+
+RT_FLOW1 %{RT_FLOW_EVENT:event}: %{GREEDYDATA:close-reason}: %{IP:src-ip}/%{INT:src-port}->%{IP:dst-ip}/%{INT:dst-port} %{DATA:service} %{IP:nat-src-ip}/%{INT:nat-src-port}->%{IP:nat-dst-ip}/%{INT:nat-dst-port} %{DATA:src-nat-rule-name} %{DATA:dst-nat-rule-name} %{INT:protocol-id} %{DATA:policy-name} %{DATA:from-zone} %{DATA:to-zone} %{INT:session-id} \d+\(%{DATA:sent}\) \d+\(%{DATA:received}\) %{INT:elapsed-time} .*
+
+RT_FLOW2 %{RT_FLOW_EVENT:event}: session created %{IP:src-ip}/%{INT:src-port}->%{IP:dst-ip}/%{INT:dst-port} %{DATA:service} %{IP:nat-src-ip}/%{INT:nat-src-port}->%{IP:nat-dst-ip}/%{INT:nat-dst-port} %{DATA:src-nat-rule-name} %{DATA:dst-nat-rule-name} %{INT:protocol-id} %{DATA:policy-name} %{DATA:from-zone} %{DATA:to-zone} %{INT:session-id} .*
+
+RT_FLOW3 %{RT_FLOW_EVENT:event}: session denied %{IP:src-ip}/%{INT:src-port}->%{IP:dst-ip}/%{INT:dst-port} %{DATA:service} %{INT:protocol-id}\(\d\) %{DATA:policy-name} %{DATA:from-zone} %{DATA:to-zone} .*

config/patterns/linux-syslog

@@ -0,0 +1,16 @@
+SYSLOG5424PRINTASCII [!-~]+
+
+SYSLOGBASE2 (?:%{SYSLOGTIMESTAMP:timestamp}|%{TIMESTAMP_ISO8601:timestamp8601}) (?:%{SYSLOGFACILITY} )?%{SYSLOGHOST:logsource}+(?: %{SYSLOGPROG}:|)
+SYSLOGPAMSESSION %{SYSLOGBASE} %{GREEDYDATA:message}%{WORD:pam_module}\(%{DATA:pam_caller}\): session %{WORD:pam_session_state} for user %{USERNAME:username}(?: by %{GREEDYDATA:pam_by})?
+
+CRON_ACTION [A-Z ]+
+CRONLOG %{SYSLOGBASE} \(%{USER:user}\) %{CRON_ACTION:action} \(%{DATA:message}\)
+
+SYSLOGLINE %{SYSLOGBASE2} %{GREEDYDATA:message}
+
+# IETF 5424 syslog(8) format (see http://www.rfc-editor.org/info/rfc5424)
+SYSLOG5424PRI <%{NONNEGINT:syslog5424_pri}>
+SYSLOG5424SD \[%{DATA}\]+
+SYSLOG5424BASE %{SYSLOG5424PRI}%{NONNEGINT:syslog5424_ver} +(?:%{TIMESTAMP_ISO8601:syslog5424_ts}|-) +(?:%{HOSTNAME:syslog5424_host}|-) +(-|%{SYSLOG5424PRINTASCII:syslog5424_app}) +(-|%{SYSLOG5424PRINTASCII:syslog5424_proc}) +(-|%{SYSLOG5424PRINTASCII:syslog5424_msgid}) +(?:%{SYSLOG5424SD:syslog5424_sd}|-|)
+
+SYSLOG5424LINE %{SYSLOG5424BASE} +%{GREEDYDATA:syslog5424_msg}

config/patterns/mcollective

@@ -0,0 +1,4 @@
+# Remember, these can be multi-line events.
+MCOLLECTIVE ., \[%{TIMESTAMP_ISO8601:timestamp} #%{POSINT:pid}\]%{SPACE}%{LOGLEVEL:event_level}
+
+MCOLLECTIVEAUDIT %{TIMESTAMP_ISO8601:timestamp}:

config/patterns/modsecurity

@@ -0,0 +1,18 @@
+APACHEERRORTIME %{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{YEAR}
+APACHEERRORPREFIX \[%{APACHEERRORTIME:timestamp}\] \[%{NOTSPACE:apacheseverity}\] (\[pid %{INT}:tid %{INT}\] )?\[client %{IPORHOST:sourcehost}(:%{INT:source_port})?\] (\[client %{IPORHOST}\])?
+GENERICAPACHEERROR %{APACHEERRORPREFIX} %{GREEDYDATA:message}
+MODSECPREFIX %{APACHEERRORPREFIX} ModSecurity: %{NOTSPACE:modsecseverity}\. %{GREEDYDATA:modsecmessage}
+MODSECRULEFILE \[file %{QUOTEDSTRING:rulefile}\]
+MODSECRULELINE \[line %{QUOTEDSTRING:ruleline}\]
+MODSECMATCHOFFSET \[offset %{QUOTEDSTRING:matchoffset}\]
+MODSECRULEID \[id %{QUOTEDSTRING:ruleid}\]
+MODSECRULEREV \[rev %{QUOTEDSTRING:rulerev}\]
+MODSECRULEMSG \[msg %{QUOTEDSTRING:rulemessage}\]
+MODSECRULEDATA \[data %{QUOTEDSTRING:ruledata}\]
+MODSECRULESEVERITY \[severity ["']%{WORD:ruleseverity}["']\]
+MODSECRULEVERS \[ver "[^"]+"\]
+MODSECRULETAGS (?:\[tag %{QUOTEDSTRING:ruletag0}\] )?(?:\[tag %{QUOTEDSTRING:ruletag1}\] )?(?:\[tag %{QUOTEDSTRING:ruletag2}\] )?(?:\[tag %{QUOTEDSTRING:ruletag3}\] )?(?:\[tag %{QUOTEDSTRING:ruletag4}\] )?(?:\[tag %{QUOTEDSTRING:ruletag5}\] )?(?:\[tag %{QUOTEDSTRING:ruletag6}\] )?(?:\[tag %{QUOTEDSTRING:ruletag7}\] )?(?:\[tag %{QUOTEDSTRING:ruletag8}\] )?(?:\[tag %{QUOTEDSTRING:ruletag9}\] )?(?:\[tag %{QUOTEDSTRING}\] )*
+MODSECHOSTNAME \[hostname ['"]%{DATA:targethost}["']\]
+MODSECURI \[uri ["']%{DATA:targeturi}["']\]
+MODSECUID \[unique_id %{QUOTEDSTRING:uniqueid}\]
+MODSECAPACHEERROR %{MODSECPREFIX} %{MODSECRULEFILE} %{MODSECRULELINE} (?:%{MODSECMATCHOFFSET} )?(?:%{MODSECRULEID} )?(?:%{MODSECRULEREV} )?(?:%{MODSECRULEMSG} )?(?:%{MODSECRULEDATA} )?(?:%{MODSECRULESEVERITY} )?(?:%{MODSECRULEVERS} )?%{MODSECRULETAGS}%{MODSECHOSTNAME} %{MODSECURI} %{MODSECUID}